diff options
| author | Ronald Bultje <rsbultje@gmail.com> | 2011-10-18 01:53:04 +0000 |
|---|---|---|
| committer | Janne Grunau <janne-libav@jannau.net> | 2011-10-18 10:29:49 +0200 |
| commit | d1d421cbc0d13b08535f7fc08d179572ee352072 (patch) | |
| tree | 3fd7fb588c9545a64ec965001c20f826c4c24f0c /libswscale/utils.c | |
| parent | 6e6003a4d244bb7b6d7ef24a98af9c6f3d269990 (diff) | |
swscale: prevent overflow during initialization
Signed-off-by: Janne Grunau <janne-libav@jannau.net>
Diffstat (limited to 'libswscale/utils.c')
| -rw-r--r-- | libswscale/utils.c | 6 |
1 files changed, 4 insertions, 2 deletions
diff --git a/libswscale/utils.c b/libswscale/utils.c index cfe4be9e20..8e5daf99dc 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -271,10 +271,12 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi floatd= d * (1.0/(1<<30)); if (flags & SWS_BICUBIC) { +#define SQRT_INT64_MAX 0xb504f333 int64_t B= (param[0] != SWS_PARAM_DEFAULT ? param[0] : 0) * (1<<24); int64_t C= (param[1] != SWS_PARAM_DEFAULT ? param[1] : 0.6) * (1<<24); - int64_t dd = ( d*d)>>30; - int64_t ddd= (dd*d)>>30; + int64_t dd = d > SQRT_INT64_MAX ? ((d >> 1) * d) >> 29 : (d * d) >> 30; + int64_t ddd = d > SQRT_INT64_MAX || dd > SQRT_INT64_MAX ? + ((dd >> 2) * d) >> 28 : (dd * d) >> 30; if (d < 1LL<<30) coeff = (12*(1<<24)-9*B-6*C)*ddd + (-18*(1<<24)+12*B+6*C)*dd + (6*(1<<24)-2*B)*(1<<30); |