From c23acbaed40101c677dfcfbbfe0d2c230a8e8f44 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Mon, 5 Mar 2012 16:01:19 -0800 Subject: Don't use ff_cropTbl[] for IDCT. Results of IDCT can by far outreach the range of ff_cropTbl[], leading to overreads and potentially crashes. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org --- libavcodec/h264idct_template.c | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) (limited to 'libavcodec/h264idct_template.c') diff --git a/libavcodec/h264idct_template.c b/libavcodec/h264idct_template.c index eba850ac6f..e476f89a6f 100644 --- a/libavcodec/h264idct_template.c +++ b/libavcodec/h264idct_template.c @@ -49,7 +49,6 @@ static const uint8_t scan8[16*3]={ void FUNCC(ff_h264_idct_add)(uint8_t *_dst, DCTELEM *_block, int stride) { int i; - INIT_CLIP pixel *dst = (pixel*)_dst; dctcoef *block = (dctcoef*)_block; stride /= sizeof(pixel); @@ -74,16 +73,15 @@ void FUNCC(ff_h264_idct_add)(uint8_t *_dst, DCTELEM *_block, int stride) const int z2= (block[1 + 4*i]>>1) - block[3 + 4*i]; const int z3= block[1 + 4*i] + (block[3 + 4*i]>>1); - dst[i + 0*stride]= CLIP(dst[i + 0*stride] + ((z0 + z3) >> 6)); - dst[i + 1*stride]= CLIP(dst[i + 1*stride] + ((z1 + z2) >> 6)); - dst[i + 2*stride]= CLIP(dst[i + 2*stride] + ((z1 - z2) >> 6)); - dst[i + 3*stride]= CLIP(dst[i + 3*stride] + ((z0 - z3) >> 6)); + dst[i + 0*stride]= av_clip_pixel(dst[i + 0*stride] + ((z0 + z3) >> 6)); + dst[i + 1*stride]= av_clip_pixel(dst[i + 1*stride] + ((z1 + z2) >> 6)); + dst[i + 2*stride]= av_clip_pixel(dst[i + 2*stride] + ((z1 - z2) >> 6)); + dst[i + 3*stride]= av_clip_pixel(dst[i + 3*stride] + ((z0 - z3) >> 6)); } } void FUNCC(ff_h264_idct8_add)(uint8_t *_dst, DCTELEM *_block, int stride){ int i; - INIT_CLIP pixel *dst = (pixel*)_dst; dctcoef *block = (dctcoef*)_block; stride /= sizeof(pixel); @@ -143,14 +141,14 @@ void FUNCC(ff_h264_idct8_add)(uint8_t *_dst, DCTELEM *_block, int stride){ const int b5 = (a3>>2) - a5; const int b7 = a7 - (a1>>2); - dst[i + 0*stride] = CLIP( dst[i + 0*stride] + ((b0 + b7) >> 6) ); - dst[i + 1*stride] = CLIP( dst[i + 1*stride] + ((b2 + b5) >> 6) ); - dst[i + 2*stride] = CLIP( dst[i + 2*stride] + ((b4 + b3) >> 6) ); - dst[i + 3*stride] = CLIP( dst[i + 3*stride] + ((b6 + b1) >> 6) ); - dst[i + 4*stride] = CLIP( dst[i + 4*stride] + ((b6 - b1) >> 6) ); - dst[i + 5*stride] = CLIP( dst[i + 5*stride] + ((b4 - b3) >> 6) ); - dst[i + 6*stride] = CLIP( dst[i + 6*stride] + ((b2 - b5) >> 6) ); - dst[i + 7*stride] = CLIP( dst[i + 7*stride] + ((b0 - b7) >> 6) ); + dst[i + 0*stride] = av_clip_pixel( dst[i + 0*stride] + ((b0 + b7) >> 6) ); + dst[i + 1*stride] = av_clip_pixel( dst[i + 1*stride] + ((b2 + b5) >> 6) ); + dst[i + 2*stride] = av_clip_pixel( dst[i + 2*stride] + ((b4 + b3) >> 6) ); + dst[i + 3*stride] = av_clip_pixel( dst[i + 3*stride] + ((b6 + b1) >> 6) ); + dst[i + 4*stride] = av_clip_pixel( dst[i + 4*stride] + ((b6 - b1) >> 6) ); + dst[i + 5*stride] = av_clip_pixel( dst[i + 5*stride] + ((b4 - b3) >> 6) ); + dst[i + 6*stride] = av_clip_pixel( dst[i + 6*stride] + ((b2 - b5) >> 6) ); + dst[i + 7*stride] = av_clip_pixel( dst[i + 7*stride] + ((b0 - b7) >> 6) ); } } @@ -158,13 +156,12 @@ void FUNCC(ff_h264_idct8_add)(uint8_t *_dst, DCTELEM *_block, int stride){ void FUNCC(ff_h264_idct_dc_add)(uint8_t *_dst, DCTELEM *block, int stride){ int i, j; int dc = (((dctcoef*)block)[0] + 32) >> 6; - INIT_CLIP pixel *dst = (pixel*)_dst; stride /= sizeof(pixel); for( j = 0; j < 4; j++ ) { for( i = 0; i < 4; i++ ) - dst[i] = CLIP( dst[i] + dc ); + dst[i] = av_clip_pixel( dst[i] + dc ); dst += stride; } } @@ -172,13 +169,12 @@ void FUNCC(ff_h264_idct_dc_add)(uint8_t *_dst, DCTELEM *block, int stride){ void FUNCC(ff_h264_idct8_dc_add)(uint8_t *_dst, DCTELEM *block, int stride){ int i, j; int dc = (((dctcoef*)block)[0] + 32) >> 6; - INIT_CLIP pixel *dst = (pixel*)_dst; stride /= sizeof(pixel); for( j = 0; j < 8; j++ ) { for( i = 0; i < 8; i++ ) - dst[i] = CLIP( dst[i] + dc ); + dst[i] = av_clip_pixel( dst[i] + dc ); dst += stride; } } -- cgit v1.2.3