diff options
| author | Ronald S. Bultje <rsbultje@gmail.com> | 2012-03-05 16:01:19 -0800 |
|---|---|---|
| committer | Ronald S. Bultje <rsbultje@gmail.com> | 2012-03-06 10:47:42 -0800 |
| commit | c23acbaed40101c677dfcfbbfe0d2c230a8e8f44 (patch) | |
| tree | 3fd50a826bbd6fb032f374c1eafd811ca3d8a642 /libavcodec/simple_idct.c | |
| parent | 2254b559cbcfc0418135f09add37c0a5866b1981 (diff) | |
Don't use ff_cropTbl[] for IDCT.
Results of IDCT can by far outreach the range of ff_cropTbl[], leading
to overreads and potentially crashes.
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
Diffstat (limited to 'libavcodec/simple_idct.c')
| -rw-r--r-- | libavcodec/simple_idct.c | 19 |
1 files changed, 8 insertions, 11 deletions
diff --git a/libavcodec/simple_idct.c b/libavcodec/simple_idct.c index 0c75261079..5812a87705 100644 --- a/libavcodec/simple_idct.c +++ b/libavcodec/simple_idct.c @@ -53,7 +53,6 @@ static inline void idct4col_put(uint8_t *dest, int line_size, const DCTELEM *col) { int c0, c1, c2, c3, a0, a1, a2, a3; - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; a0 = col[8*0]; a1 = col[8*2]; @@ -63,13 +62,13 @@ static inline void idct4col_put(uint8_t *dest, int line_size, const DCTELEM *col c2 = ((a0 - a2) << (CN_SHIFT - 1)) + (1 << (C_SHIFT - 1)); c1 = a1 * C1 + a3 * C2; c3 = a1 * C2 - a3 * C1; - dest[0] = cm[(c0 + c1) >> C_SHIFT]; + dest[0] = av_clip_uint8((c0 + c1) >> C_SHIFT); dest += line_size; - dest[0] = cm[(c2 + c3) >> C_SHIFT]; + dest[0] = av_clip_uint8((c2 + c3) >> C_SHIFT); dest += line_size; - dest[0] = cm[(c2 - c3) >> C_SHIFT]; + dest[0] = av_clip_uint8((c2 - c3) >> C_SHIFT); dest += line_size; - dest[0] = cm[(c0 - c1) >> C_SHIFT]; + dest[0] = av_clip_uint8((c0 - c1) >> C_SHIFT); } #define BF(k) \ @@ -133,7 +132,6 @@ void ff_simple_idct248_put(uint8_t *dest, int line_size, DCTELEM *block) static inline void idct4col_add(uint8_t *dest, int line_size, const DCTELEM *col) { int c0, c1, c2, c3, a0, a1, a2, a3; - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; a0 = col[8*0]; a1 = col[8*1]; @@ -143,13 +141,13 @@ static inline void idct4col_add(uint8_t *dest, int line_size, const DCTELEM *col c2 = (a0 - a2)*C3 + (1 << (C_SHIFT - 1)); c1 = a1 * C1 + a3 * C2; c3 = a1 * C2 - a3 * C1; - dest[0] = cm[dest[0] + ((c0 + c1) >> C_SHIFT)]; + dest[0] = av_clip_uint8(dest[0] + ((c0 + c1) >> C_SHIFT)); dest += line_size; - dest[0] = cm[dest[0] + ((c2 + c3) >> C_SHIFT)]; + dest[0] = av_clip_uint8(dest[0] + ((c2 + c3) >> C_SHIFT)); dest += line_size; - dest[0] = cm[dest[0] + ((c2 - c3) >> C_SHIFT)]; + dest[0] = av_clip_uint8(dest[0] + ((c2 - c3) >> C_SHIFT)); dest += line_size; - dest[0] = cm[dest[0] + ((c0 - c1) >> C_SHIFT)]; + dest[0] = av_clip_uint8(dest[0] + ((c0 - c1) >> C_SHIFT)); } #define RN_SHIFT 15 @@ -161,7 +159,6 @@ static inline void idct4col_add(uint8_t *dest, int line_size, const DCTELEM *col static inline void idct4row(DCTELEM *row) { int c0, c1, c2, c3, a0, a1, a2, a3; - //const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; a0 = row[0]; a1 = row[1]; |