avcodec/simple_idct_template: Fix several integer overflows

Fixes: simple_idct_template.c:184:30: runtime error: signed integer overflow: -1065517056 - 1392182838 cannot be represented in type 'int'
Fixes: simple_idct_template.c:269:21: runtime error: signed integer overflow: 16384 * 259254 cannot be represented in type 'int'
Fixes: simple_idct_template.c:164:17: runtime error: signed integer overflow: 21407 * 210162 cannot be represented in type 'int'
Fixes: simple_idct_template.c:167:17: runtime error: signed integer overflow: 21407 * 210162 cannot be represented in type 'int'
Fixes: simple_idct_template.c:169:19: runtime error: signed integer overflow: 22725 * 259190 cannot be represented in type 'int'
Fixes: simple_idct_template.c:171:19: runtime error: signed integer overflow: 19265 * 259190 cannot be represented in type 'int'
Fixes: simple_idct_template.c:173:19: runtime error: signed integer overflow: 12873 * 259190 cannot be represented in type 'int'
Fixes: simple_idct_template.c:183:28: runtime error: signed integer overflow: 1860878336 + 585177665 cannot be represented in type 'int'
Fixes: simple_idct_template.c:159:17: runtime error: signed integer overflow: 16384 * 189520 cannot be represented in type 'int'
Fixes: simple_idct_template.c:170:22: runtime error: signed integer overflow: 19265 * 130147 cannot be represented in type 'int'
Fixes: simple_idct_template.c:174:23: runtime error: signed integer overflow: -22725 * 130147 cannot be represented in type 'int'
Fixes: simple_idct_template.c:183:20: runtime error: signed integer overflow: 16384 * -175206 cannot be represented in type 'int'
Fixes: simple_idct_template.c:184:22: runtime error: signed integer overflow: -16384 * -175206 cannot be represented in type 'int'
Fixes: simple_idct_template.c:185:22: runtime error: signed integer overflow: -16384 * -175206 cannot be represented in type 'int'
Fixes: simple_idct_template.c:186:20: runtime error: signed integer overflow: 16384 * -175206 cannot be represented in type 'int'
Fixes: simple_idct_template.c:195:26: runtime error: signed integer overflow: 19265 * 150747 cannot be represented in type 'int'
Fixes: simple_idct_template.c:198:27: runtime error: signed integer overflow: -22725 * 150747 cannot be represented in type 'int'
Fixes: simple_idct_template.c:184:37: runtime error: signed integer overflow: 21407 * -171941 cannot be represented in type 'int'
Fixes: simple_idct_template.c:185:37: runtime error: signed integer overflow: 21407 * -171941 cannot be represented in type 'int'
Fixes: simple_idct_template.c:192:27: runtime error: signed integer overflow: -12873 * 206341 cannot be represented in type 'int'
Fixes: simple_idct_template.c:185:30: runtime error: signed integer overflow: 1196441600 + 1703756981 cannot be represented in type 'int'
Fixes: simple_idct_template.c:176:23: runtime error: signed integer overflow: -12873 * 168461 cannot be represented in type 'int'
Fixes: simple_idct_template.c:191:27: runtime error: signed integer overflow: -22725 * -140062 cannot be represented in type 'int'
Fixes: simple_idct_template.c:197:26: runtime error: signed integer overflow: 19265 * -140062 cannot be represented in type 'int'
Fixes: simple_idct_template.c:183:34: runtime error: signed integer overflow: 8867 * -243046 cannot be represented in type 'int'
Fixes: simple_idct_template.c:186:34: runtime error: signed integer overflow: 8867 * -243046 cannot be represented in type 'int'
Fixes: simple_idct_template.c:186:28: runtime error: signed integer overflow: -816234496 - 2139878414 cannot be represented in type 'int'
Fixes: simple_idct_template.c:188:26: runtime error: signed integer overflow: 12873 * -239872 cannot be represented in type 'int'
Fixes: simple_idct_template.c:165:16: runtime error: signed integer overflow: 8867 * -260084 cannot be represented in type 'int'
Fixes: simple_idct_template.c:166:16: runtime error: signed integer overflow: 8867 * -260084 cannot be represented in type 'int'
Fixes: 9135/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MPEG4_fuzzer-6324422955761664

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

1 files changed, 24 insertions, 24 deletions

diff --git a/libavcodec/simple_idct_template.c b/libavcodec/simple_idct_template.c
index 904263fc71..35c31321c6 100644
--- a/libavcodec/simple_idct_template.c
+++ b/libavcodec/simple_idct_template.c
@@ -101,8 +101,8 @@
 #define DC_SHIFT -1
 # endif
 
-#define MUL(a, b)    ((a) * (b))
-#define MAC(a, b, c) ((a) += (b) * (c))
+#define MUL(a, b)    ((int)((SUINT)(a) * (b)))
+#define MAC(a, b, c) ((a) += (SUINT)(b) * (c))
 
 #else
 
@@ -156,15 +156,15 @@ static inline void FUNC6(idctRowCondDC)(idctin *row, int extra_shift)
 #endif
 #endif
 
-    a0 = (W4 * row[0]) + (1 << (ROW_SHIFT + extra_shift - 1));
+    a0 = ((SUINT)W4 * row[0]) + (1 << (ROW_SHIFT + extra_shift - 1));
     a1 = a0;
     a2 = a0;
     a3 = a0;
 
-    a0 += W2 * row[2];
-    a1 += W6 * row[2];
-    a2 -= W6 * row[2];
-    a3 -= W2 * row[2];
+    a0 += (SUINT)W2 * row[2];
+    a1 += (SUINT)W6 * row[2];
+    a2 -= (SUINT)W6 * row[2];
+    a3 -= (SUINT)W2 * row[2];
 
     b0 = MUL(W1, row[1]);
     MAC(b0, W3, row[3]);
@@ -180,10 +180,10 @@ static inline void FUNC6(idctRowCondDC)(idctin *row, int extra_shift)
 #else
     if (AV_RN64A(row + 4)) {
 #endif
-        a0 +=   W4*row[4] + W6*row[6];
-        a1 += - W4*row[4] - W2*row[6];
-        a2 += - W4*row[4] + W2*row[6];
-        a3 +=   W4*row[4] - W6*row[6];
+        a0 += (SUINT)  W4*row[4] + (SUINT)W6*row[6];
+        a1 += (SUINT)- W4*row[4] - (SUINT)W2*row[6];
+        a2 += (SUINT)- W4*row[4] + (SUINT)W2*row[6];
+        a3 += (SUINT)  W4*row[4] - (SUINT)W6*row[6];
 
         MAC(b0,  W5, row[5]);
         MAC(b0,  W7, row[7]);
@@ -209,15 +209,15 @@ static inline void FUNC6(idctRowCondDC)(idctin *row, int extra_shift)
 }
 
 #define IDCT_COLS do {                                  \
-        a0 = W4 * (col[8*0] + ((1<<(COL_SHIFT-1))/W4)); \
+        a0 = (SUINT)W4 * (col[8*0] + ((1<<(COL_SHIFT-1))/W4)); \
         a1 = a0;                                        \
         a2 = a0;                                        \
         a3 = a0;                                        \
                                                         \
-        a0 +=  W2*col[8*2];                             \
-        a1 +=  W6*col[8*2];                             \
-        a2 += -W6*col[8*2];                             \
-        a3 += -W2*col[8*2];                             \
+        a0 += (SUINT) W2*col[8*2];                             \
+        a1 += (SUINT) W6*col[8*2];                             \
+        a2 += (SUINT)-W6*col[8*2];                             \
+        a3 += (SUINT)-W2*col[8*2];                             \
                                                         \
         b0 = MUL(W1, col[8*1]);                         \
         b1 = MUL(W3, col[8*1]);                         \
@@ -230,10 +230,10 @@ static inline void FUNC6(idctRowCondDC)(idctin *row, int extra_shift)
         MAC(b3, -W5, col[8*3]);                         \
                                                         \
         if (col[8*4]) {                                 \
-            a0 +=  W4*col[8*4];                         \
-            a1 += -W4*col[8*4];                         \
-            a2 += -W4*col[8*4];                         \
-            a3 +=  W4*col[8*4];                         \
+            a0 += (SUINT) W4*col[8*4];                         \
+            a1 += (SUINT)-W4*col[8*4];                         \
+            a2 += (SUINT)-W4*col[8*4];                         \
+            a3 += (SUINT) W4*col[8*4];                         \
         }                                               \
                                                         \
         if (col[8*5]) {                                 \
@@ -244,10 +244,10 @@ static inline void FUNC6(idctRowCondDC)(idctin *row, int extra_shift)
         }                                               \
                                                         \
         if (col[8*6]) {                                 \
-            a0 +=  W6*col[8*6];                         \
-            a1 += -W2*col[8*6];                         \
-            a2 +=  W2*col[8*6];                         \
-            a3 += -W6*col[8*6];                         \
+            a0 += (SUINT) W6*col[8*6];                         \
+            a1 += (SUINT)-W2*col[8*6];                         \
+            a2 += (SUINT) W2*col[8*6];                         \
+            a3 += (SUINT)-W6*col[8*6];                         \
         }                                               \
                                                         \
         if (col[8*7]) {                                 \