From a4895b75a3030d624fd15793e830543b94a0f29c Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Fri, 16 Oct 2020 17:55:26 +0200 Subject: avcodec/mobiclip: Avoid signed integer overflows in idct() Fixes: signed integer overflow: 536870912 + 1610612736 cannot be represented in type 'int' Fixes: 26288/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MOBICLIP_fuzzer-6194364759670784 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer --- libavcodec/mobiclip.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'libavcodec') diff --git a/libavcodec/mobiclip.c b/libavcodec/mobiclip.c index 82ff39e958..48467614ab 100644 --- a/libavcodec/mobiclip.c +++ b/libavcodec/mobiclip.c @@ -422,7 +422,8 @@ static void inverse4(unsigned *rs) static void idct(int *arr, int size) { - int e, f, g, h, x3, x2, x1, x0; + int e, f, g, h; + unsigned x3, x2, x1, x0; int tmp[4]; if (size == 4) { @@ -437,14 +438,14 @@ static void idct(int *arr, int size) inverse4(tmp); - e = arr[7] + arr[1] - arr[3] - (arr[3] >> 1); - f = arr[7] - arr[1] + arr[5] + (arr[5] >> 1); - g = arr[5] - arr[3] - arr[7] - (arr[7] >> 1); - h = arr[5] + arr[3] + arr[1] + (arr[1] >> 1); - x3 = g + (h >> 2); - x2 = e + (f >> 2); - x1 = (e >> 2) - f; - x0 = h - (g >> 2); + e = (unsigned)arr[7] + arr[1] - arr[3] - (arr[3] >> 1); + f = (unsigned)arr[7] - arr[1] + arr[5] + (arr[5] >> 1); + g = (unsigned)arr[5] - arr[3] - arr[7] - (arr[7] >> 1); + h = (unsigned)arr[5] + arr[3] + arr[1] + (arr[1] >> 1); + x3 = (unsigned)g + (h >> 2); + x2 = (unsigned)e + (f >> 2); + x1 = (e >> 2) - (unsigned)f; + x0 = (unsigned)h - (g >> 2); arr[0] = tmp[0] + x0; arr[1] = tmp[1] + x1; -- cgit v1.2.3