diff options
| author | Michael Niedermayer <michael@niedermayer.cc> | 2018-10-11 01:04:34 +0200 |
|---|---|---|
| committer | Michael Niedermayer <michael@niedermayer.cc> | 2018-10-18 02:28:54 +0200 |
| commit | d7dbad12f820a72997772ff5e535bee3eb7c5b4b (patch) | |
| tree | bea76a7d7bb4402fb1ae95280489b5a46ccdba58 /libavcodec | |
| parent | d01788665767720441e2d506a270ee6ba85e9fe2 (diff) | |
avcodec/ilbcdec: Fix multiple integer overflows
Fixes: 10651/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_ILBC_fuzzer-5202341540659200
Fixes: signed integer overflow: -1707705920 - 1703592888 cannot be represented in type 'int'
This tries to follow the webrtc code. For example using cliping and 64 bit as in WebRtcSpl_DotProductWithScale()
and not doing so in other places.
I could not find anything in rfc3951 and the reference code inside which would
explain what to do in these corner cases.
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
Diffstat (limited to 'libavcodec')
| -rw-r--r-- | libavcodec/ilbcdec.c | 18 |
1 files changed, 9 insertions, 9 deletions
diff --git a/libavcodec/ilbcdec.c b/libavcodec/ilbcdec.c index 58044f4ba8..8f234b98e1 100644 --- a/libavcodec/ilbcdec.c +++ b/libavcodec/ilbcdec.c @@ -381,7 +381,7 @@ static void get_lsp_poly(int16_t *lsp, int32_t *f) tmp = ((high * lsp[k]) * 4) + (((low * lsp[k]) >> 15) * 4); f[l] += f[l - 2]; - f[l] -= tmp; + f[l] -= (unsigned)tmp; } f[l] -= lsp[k] * (1 << 10); @@ -402,16 +402,16 @@ static void lsf2poly(int16_t *a, int16_t *lsf) get_lsp_poly(&lsp[1], f[1]); for (i = 5; i > 0; i--) { - f[0][i] += f[0][i - 1]; - f[1][i] -= f[1][i - 1]; + f[0][i] += (unsigned)f[0][i - 1]; + f[1][i] -= (unsigned)f[1][i - 1]; } a[0] = 4096; for (i = 5; i > 0; i--) { - tmp = f[0][6 - i] + f[1][6 - i]; + tmp = f[0][6 - i] + (unsigned)f[1][6 - i]; a[6 - i] = (tmp + 4096) >> 13; - tmp = f[0][6 - i] - f[1][6 - i]; + tmp = f[0][6 - i] - (unsigned)f[1][6 - i]; a[5 + i] = (tmp + 4096) >> 13; } } @@ -508,10 +508,10 @@ static void filter_arfq12(const int16_t *data_in, int output = 0, sum = 0; for (j = coefficients_length - 1; j > 0; j--) { - sum += coefficients[j] * data_out[i - j]; + sum += (unsigned)(coefficients[j] * data_out[i - j]); } - output = coefficients[0] * data_in[i] - sum; + output = coefficients[0] * data_in[i] - (unsigned)sum; output = av_clip(output, -134217728, 134215679); data_out[i] = (output + 2048) >> 12; @@ -901,12 +901,12 @@ static int16_t get_size_in_bits(uint32_t n) static int32_t scale_dot_product(const int16_t *v1, const int16_t *v2, int length, int scaling) { - int32_t sum = 0; + int64_t sum = 0; for (int i = 0; i < length; i++) sum += (v1[i] * v2[i]) >> scaling; - return sum; + return av_clipl_int32(sum); } static void correlation(int32_t *corr, int32_t *ener, int16_t *buffer, |