From 8b94df0f2047e9728cb872adc9e64557b7a5152f Mon Sep 17 00:00:00 2001 From: Reinhard Tartler Date: Sun, 4 Dec 2011 10:10:33 +0100 Subject: vp3dec: Check coefficient index in vp3_dequant() Based on a patch by Michael Niedermayer Fixes NGS00145, CVE-2011-4352 Found-by: Phillip Langlois Signed-off-by: Reinhard Tartler --- libavcodec/vp3.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'libavcodec/vp3.c') diff --git a/libavcodec/vp3.c b/libavcodec/vp3.c index 51ab048ac0..f44d084dbd 100644 --- a/libavcodec/vp3.c +++ b/libavcodec/vp3.c @@ -1363,6 +1363,10 @@ static inline int vp3_dequant(Vp3DecodeContext *s, Vp3Fragment *frag, case 1: // zero run s->dct_tokens[plane][i]++; i += (token >> 2) & 0x7f; + if (i > 63) { + av_log(s->avctx, AV_LOG_ERROR, "Coefficient index overflow\n"); + return i; + } block[perm[i]] = (token >> 9) * dequantizer[perm[i]]; i++; break; @@ -1566,7 +1570,10 @@ static void render_slice(Vp3DecodeContext *s, int slice) /* invert DCT and place (or add) in final output */ if (s->all_fragments[i].coding_method == MODE_INTRA) { - vp3_dequant(s, s->all_fragments + i, plane, 0, block); + int index; + index = vp3_dequant(s, s->all_fragments + i, plane, 0, block); + if (index > 63) + continue; if(s->avctx->idct_algo!=FF_IDCT_VP3) block[0] += 128<<3; s->dsp.idct_put( @@ -1574,7 +1581,10 @@ static void render_slice(Vp3DecodeContext *s, int slice) stride, block); } else { - if (vp3_dequant(s, s->all_fragments + i, plane, 1, block)) { + int index = vp3_dequant(s, s->all_fragments + i, plane, 1, block); + if (index > 63) + continue; + if (index > 0) { s->dsp.idct_add( output_plane + first_pixel, stride, -- cgit v1.2.3