From 770c934fa1635f4fadf5db4fc5cc5ad15d82455a Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 5 Nov 2017 21:20:06 +0100 Subject: avcodec/mdct_*: Fix integer overflow in addition in RESCALE() Fixes: runtime error: signed integer overflow: 1219998458 - -1469874012 cannot be represented in type 'int' Fixes: 3443/clusterfuzz-testcase-minimized-5369987105554432 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer --- libavcodec/mdct_template.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'libavcodec/mdct_template.c') diff --git a/libavcodec/mdct_template.c b/libavcodec/mdct_template.c index 04396b46d6..e0ad9f1e53 100644 --- a/libavcodec/mdct_template.c +++ b/libavcodec/mdct_template.c @@ -33,12 +33,12 @@ */ #if FFT_FLOAT -# define RSCALE(x) (x) +# define RSCALE(x, y) ((x) + (y)) #else #if FFT_FIXED_32 -# define RSCALE(x) (((x) + 32) >> 6) +# define RSCALE(x, y) ((int)((x) + (unsigned)(y) + 32) >> 6) #else /* FFT_FIXED_32 */ -# define RSCALE(x) ((x) >> 1) +# define RSCALE(x, y) ((int)((x) + (unsigned)(y)) >> 1) #endif /* FFT_FIXED_32 */ #endif @@ -181,13 +181,13 @@ void ff_mdct_calc_c(FFTContext *s, FFTSample *out, const FFTSample *input) /* pre rotation */ for(i=0;i