diff options
| author | Michael Niedermayer <michael@niedermayer.cc> | 2017-05-10 14:50:40 +0200 |
|---|---|---|
| committer | Michael Niedermayer <michael@niedermayer.cc> | 2017-05-12 14:22:50 +0200 |
| commit | ccce2248bf56692fc7bd436ca2c9acca772d486a (patch) | |
| tree | e2943865506bf29a9a2bd7ea78da4cda349023fd /libavcodec/vp8dsp.c | |
| parent | d5711cb89121268e8d78ebe8563a68e67a236cbb (diff) | |
avcodec/vp8dsp: vp7_luma_dc_wht_c: Fix multiple runtime error: signed integer overflow: -1366381240 + -1262413604 cannot be represented in type 'int'
Fixes: 1440/clusterfuzz-testcase-minimized-5785716111966208
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
Diffstat (limited to 'libavcodec/vp8dsp.c')
| -rw-r--r-- | libavcodec/vp8dsp.c | 19 |
1 files changed, 10 insertions, 9 deletions
diff --git a/libavcodec/vp8dsp.c b/libavcodec/vp8dsp.c index 7d9cfa8278..fed5c67a90 100644 --- a/libavcodec/vp8dsp.c +++ b/libavcodec/vp8dsp.c @@ -53,7 +53,8 @@ static void name ## _idct_dc_add4y_c(uint8_t *dst, int16_t block[4][16], \ #if CONFIG_VP7_DECODER static void vp7_luma_dc_wht_c(int16_t block[4][4][16], int16_t dc[16]) { - int i, a1, b1, c1, d1; + int i; + unsigned a1, b1, c1, d1; int16_t tmp[16]; for (i = 0; i < 4; i++) { @@ -61,10 +62,10 @@ static void vp7_luma_dc_wht_c(int16_t block[4][4][16], int16_t dc[16]) b1 = (dc[i * 4 + 0] - dc[i * 4 + 2]) * 23170; c1 = dc[i * 4 + 1] * 12540 - dc[i * 4 + 3] * 30274; d1 = dc[i * 4 + 1] * 30274 + dc[i * 4 + 3] * 12540; - tmp[i * 4 + 0] = (a1 + d1) >> 14; - tmp[i * 4 + 3] = (a1 - d1) >> 14; - tmp[i * 4 + 1] = (b1 + c1) >> 14; - tmp[i * 4 + 2] = (b1 - c1) >> 14; + tmp[i * 4 + 0] = (int)(a1 + d1) >> 14; + tmp[i * 4 + 3] = (int)(a1 - d1) >> 14; + tmp[i * 4 + 1] = (int)(b1 + c1) >> 14; + tmp[i * 4 + 2] = (int)(b1 - c1) >> 14; } for (i = 0; i < 4; i++) { @@ -73,10 +74,10 @@ static void vp7_luma_dc_wht_c(int16_t block[4][4][16], int16_t dc[16]) c1 = tmp[i + 4] * 12540 - tmp[i + 12] * 30274; d1 = tmp[i + 4] * 30274 + tmp[i + 12] * 12540; AV_ZERO64(dc + i * 4); - block[0][i][0] = (a1 + d1 + 0x20000) >> 18; - block[3][i][0] = (a1 - d1 + 0x20000) >> 18; - block[1][i][0] = (b1 + c1 + 0x20000) >> 18; - block[2][i][0] = (b1 - c1 + 0x20000) >> 18; + block[0][i][0] = (int)(a1 + d1 + 0x20000) >> 18; + block[3][i][0] = (int)(a1 - d1 + 0x20000) >> 18; + block[1][i][0] = (int)(b1 + c1 + 0x20000) >> 18; + block[2][i][0] = (int)(b1 - c1 + 0x20000) >> 18; } } |