avcodec/aacdec_fixed: Fix various integer overflows

Fixes: 1377/clusterfuzz-testcase-minimized-5487049807233024

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

1 files changed, 15 insertions, 13 deletions

diff --git a/libavcodec/sbrdsp_fixed.c b/libavcodec/sbrdsp_fixed.c
index f4e3de0c71..fb9aba4e8d 100644
--- a/libavcodec/sbrdsp_fixed.c
+++ b/libavcodec/sbrdsp_fixed.c
@@ -34,8 +34,9 @@
 static SoftFloat sbr_sum_square_c(int (*x)[2], int n)
 {
     SoftFloat ret;
-    int64_t accu = 0;
-    int i, nz, round;
+    uint64_t accu = 0, round;
+    int i, nz;
+    unsigned u;
 
     for (i = 0; i < n; i += 2) {
         // Larger values are inavlid and could cause overflows of accu.
@@ -49,22 +50,22 @@ static SoftFloat sbr_sum_square_c(int (*x)[2], int n)
         accu += (int64_t)x[i + 1][1] * x[i + 1][1];
     }
 
-    i = (int)(accu >> 32);
-    if (i == 0) {
+    u = accu >> 32;
+    if (u == 0) {
         nz = 1;
     } else {
-        nz = 0;
-        while (FFABS(i) < 0x40000000) {
-            i <<= 1;
+        nz = -1;
+        while (u < 0x80000000U) {
+            u <<= 1;
             nz++;
         }
         nz = 32 - nz;
     }
 
-    round = 1 << (nz-1);
-    i = (int)((accu + round) >> nz);
-    i >>= 1;
-    ret = av_int2sf(i, 15 - nz);
+    round = 1ULL << (nz-1);
+    u = ((accu + round) >> nz);
+    u >>= 1;
+    ret = av_int2sf(u, 15 - nz);
 
     return ret;
 }
@@ -107,7 +108,8 @@ static void sbr_qmf_deint_neg_c(int *v, const int *src)
 
 static av_always_inline SoftFloat autocorr_calc(int64_t accu)
 {
-        int nz, mant, expo, round;
+        int nz, mant, expo;
+        unsigned round;
         int i = (int)(accu >> 32);
         if (i == 0) {
             nz = 1;
@@ -120,7 +122,7 @@ static av_always_inline SoftFloat autocorr_calc(int64_t accu)
             nz = 32-nz;
         }
 
-        round = 1 << (nz-1);
+        round = 1U << (nz-1);
         mant = (int)((accu + round) >> nz);
         mant = (mant + 0x40)>>7;
         mant <<= 6;