From fc417db3f162d5269c0d22f8e467da4afa67c20a Mon Sep 17 00:00:00 2001 From: Kostya Shishkov Date: Mon, 14 May 2012 19:30:54 +0200 Subject: indeo3: ensure that decoded cell data is in 7-bit range as presumed by decoder --- libavcodec/indeo3.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) (limited to 'libavcodec/indeo3.c') diff --git a/libavcodec/indeo3.c b/libavcodec/indeo3.c index 80839406c0..0856d0607c 100644 --- a/libavcodec/indeo3.c +++ b/libavcodec/indeo3.c @@ -344,8 +344,10 @@ if (*data_ptr >= last_ptr) \ fill_64(dst, pix64, num_lines << 1, row_offset) #define APPLY_DELTA_4 \ - AV_WN16A(dst + line_offset , AV_RN16A(ref ) + delta_tab->deltas[dyad1]);\ - AV_WN16A(dst + line_offset + 2, AV_RN16A(ref + 2) + delta_tab->deltas[dyad2]);\ + AV_WN16A(dst + line_offset ,\ + (AV_RN16A(ref ) + delta_tab->deltas[dyad1]) & 0x7F7F);\ + AV_WN16A(dst + line_offset + 2,\ + (AV_RN16A(ref + 2) + delta_tab->deltas[dyad2]) & 0x7F7F);\ if (mode >= 3) {\ if (is_top_of_cell && !cell->ypos) {\ AV_COPY32(dst, dst + row_offset);\ @@ -358,14 +360,14 @@ if (*data_ptr >= last_ptr) \ /* apply two 32-bit VQ deltas to next even line */\ if (is_top_of_cell) { \ AV_WN32A(dst + row_offset , \ - replicate32(AV_RN32A(ref )) + delta_tab->deltas_m10[dyad1]);\ + (replicate32(AV_RN32A(ref )) + delta_tab->deltas_m10[dyad1]) & 0x7F7F7F7F);\ AV_WN32A(dst + row_offset + 4, \ - replicate32(AV_RN32A(ref + 4)) + delta_tab->deltas_m10[dyad2]);\ + (replicate32(AV_RN32A(ref + 4)) + delta_tab->deltas_m10[dyad2]) & 0x7F7F7F7F);\ } else { \ AV_WN32A(dst + row_offset , \ - AV_RN32A(ref ) + delta_tab->deltas_m10[dyad1]);\ + (AV_RN32A(ref ) + delta_tab->deltas_m10[dyad1]) & 0x7F7F7F7F);\ AV_WN32A(dst + row_offset + 4, \ - AV_RN32A(ref + 4) + delta_tab->deltas_m10[dyad2]);\ + (AV_RN32A(ref + 4) + delta_tab->deltas_m10[dyad2]) & 0x7F7F7F7F);\ } \ /* odd lines are not coded but rather interpolated/replicated */\ /* first line of the cell on the top of image? - replicate */\ @@ -379,22 +381,22 @@ if (*data_ptr >= last_ptr) \ #define APPLY_DELTA_1011_INTER \ if (mode == 10) { \ AV_WN32A(dst , \ - AV_RN32A(dst ) + delta_tab->deltas_m10[dyad1]);\ + (AV_RN32A(dst ) + delta_tab->deltas_m10[dyad1]) & 0x7F7F7F7F);\ AV_WN32A(dst + 4 , \ - AV_RN32A(dst + 4 ) + delta_tab->deltas_m10[dyad2]);\ + (AV_RN32A(dst + 4 ) + delta_tab->deltas_m10[dyad2]) & 0x7F7F7F7F);\ AV_WN32A(dst + row_offset , \ - AV_RN32A(dst + row_offset ) + delta_tab->deltas_m10[dyad1]);\ + (AV_RN32A(dst + row_offset ) + delta_tab->deltas_m10[dyad1]) & 0x7F7F7F7F);\ AV_WN32A(dst + row_offset + 4, \ - AV_RN32A(dst + row_offset + 4) + delta_tab->deltas_m10[dyad2]);\ + (AV_RN32A(dst + row_offset + 4) + delta_tab->deltas_m10[dyad2]) & 0x7F7F7F7F);\ } else { \ AV_WN16A(dst , \ - AV_RN16A(dst ) + delta_tab->deltas[dyad1]);\ + (AV_RN16A(dst ) + delta_tab->deltas[dyad1]) & 0x7F7F);\ AV_WN16A(dst + 2 , \ - AV_RN16A(dst + 2 ) + delta_tab->deltas[dyad2]);\ + (AV_RN16A(dst + 2 ) + delta_tab->deltas[dyad2]) & 0x7F7F);\ AV_WN16A(dst + row_offset , \ - AV_RN16A(dst + row_offset ) + delta_tab->deltas[dyad1]);\ + (AV_RN16A(dst + row_offset ) + delta_tab->deltas[dyad1]) & 0x7F7F);\ AV_WN16A(dst + row_offset + 2, \ - AV_RN16A(dst + row_offset + 2) + delta_tab->deltas[dyad2]);\ + (AV_RN16A(dst + row_offset + 2) + delta_tab->deltas[dyad2]) & 0x7F7F);\ } -- cgit v1.2.3 From 81064a8045028838fd32d18490034c207c8ecc06 Mon Sep 17 00:00:00 2001 From: Kostya Shishkov Date: Mon, 14 May 2012 19:33:03 +0200 Subject: indeo3: initialise pixel planes on allocation This prevents decoder from reading garbage from it in case of errors later. --- libavcodec/indeo3.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'libavcodec/indeo3.c') diff --git a/libavcodec/indeo3.c b/libavcodec/indeo3.c index 0856d0607c..1221ffab18 100644 --- a/libavcodec/indeo3.c +++ b/libavcodec/indeo3.c @@ -194,6 +194,8 @@ static av_cold int allocate_frame_buffers(Indeo3DecodeContext *ctx, /* set buffer pointers = buf_ptr + pitch and thus skip the INTRA prediction line */ ctx->planes[p].pixels[0] = ctx->planes[p].buffers[0] + ctx->planes[p].pitch; ctx->planes[p].pixels[1] = ctx->planes[p].buffers[1] + ctx->planes[p].pitch; + memset(ctx->planes[p].pixels[0], 0, ctx->planes[p].pitch * ctx->planes[p].height); + memset(ctx->planes[p].pixels[1], 0, ctx->planes[p].pitch * ctx->planes[p].height); } return 0; -- cgit v1.2.3 From bc00da27010ed9e5dbe47e5b6fae3dcddb999d78 Mon Sep 17 00:00:00 2001 From: Kostya Shishkov Date: Mon, 14 May 2012 19:45:41 +0200 Subject: indeo3: when freeing buffers, set pointers referencing them to NULL as well --- libavcodec/indeo3.c | 1 + 1 file changed, 1 insertion(+) (limited to 'libavcodec/indeo3.c') diff --git a/libavcodec/indeo3.c b/libavcodec/indeo3.c index 1221ffab18..e5b2ea7ef0 100644 --- a/libavcodec/indeo3.c +++ b/libavcodec/indeo3.c @@ -209,6 +209,7 @@ static av_cold void free_frame_buffers(Indeo3DecodeContext *ctx) for (p = 0; p < 3; p++) { av_freep(&ctx->planes[p].buffers[0]); av_freep(&ctx->planes[p].buffers[1]); + ctx->planes[p].pixels[0] = ctx->planes[p].pixels[1] = 0; } } -- cgit v1.2.3 From 6de226a2b8b703abc823f18c3fd7f39a0787aeb5 Mon Sep 17 00:00:00 2001 From: Kostya Shishkov Date: Mon, 14 May 2012 19:46:54 +0200 Subject: indeo3: validate new frame size before resetting decoder --- libavcodec/indeo3.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'libavcodec/indeo3.c') diff --git a/libavcodec/indeo3.c b/libavcodec/indeo3.c index e5b2ea7ef0..d526d910da 100644 --- a/libavcodec/indeo3.c +++ b/libavcodec/indeo3.c @@ -900,6 +900,14 @@ static int decode_frame_headers(Indeo3DecodeContext *ctx, AVCodecContext *avctx, av_dlog(avctx, "Frame dimensions changed!\n"); + if (width < 16 || width > 640 || + height < 16 || height > 480 || + width & 3 || height & 3) { + av_log(avctx, AV_LOG_ERROR, + "Invalid picture dimensions: %d x %d!\n", width, height); + return AVERROR_INVALIDDATA; + } + ctx->width = width; ctx->height = height; -- cgit v1.2.3