From 0764fe1d09833ae4dcf9e427df09378d0d6a3386 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sat, 22 Jul 2017 00:44:14 +0200 Subject: avcodec/aacps: Fix multiple integer overflow in map_val_34_to_20() Fixes: avcodec/aacps.c:511:40: runtime error: signed integer overflow: 1509077651 + 758068176 cannot be represented in type 'int' Fixes: 2678/clusterfuzz-testcase-minimized-4702787684270080 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer --- libavcodec/aacps.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'libavcodec/aacps.c') diff --git a/libavcodec/aacps.c b/libavcodec/aacps.c index 5758b919a1..b16c3393d1 100644 --- a/libavcodec/aacps.c +++ b/libavcodec/aacps.c @@ -504,13 +504,13 @@ static void map_idx_34_to_20(int8_t *par_mapped, const int8_t *par, int full) static void map_val_34_to_20(INTFLOAT par[PS_MAX_NR_IIDICC]) { #if USE_FIXED - par[ 0] = (int)(((int64_t)(par[ 0] + (par[ 1]>>1)) * 1431655765 + \ + par[ 0] = (int)(((int64_t)(par[ 0] + (unsigned)(par[ 1]>>1)) * 1431655765 + \ 0x40000000) >> 31); - par[ 1] = (int)(((int64_t)((par[ 1]>>1) + par[ 2]) * 1431655765 + \ + par[ 1] = (int)(((int64_t)((par[ 1]>>1) + (unsigned)par[ 2]) * 1431655765 + \ 0x40000000) >> 31); - par[ 2] = (int)(((int64_t)(par[ 3] + (par[ 4]>>1)) * 1431655765 + \ + par[ 2] = (int)(((int64_t)(par[ 3] + (unsigned)(par[ 4]>>1)) * 1431655765 + \ 0x40000000) >> 31); - par[ 3] = (int)(((int64_t)((par[ 4]>>1) + par[ 5]) * 1431655765 + \ + par[ 3] = (int)(((int64_t)((par[ 4]>>1) + (unsigned)par[ 5]) * 1431655765 + \ 0x40000000) >> 31); #else par[ 0] = (2*par[ 0] + par[ 1]) * 0.33333333f; -- cgit v1.2.3