diff options
| author | Ronald S. Bultje <rsbultje@gmail.com> | 2011-10-19 19:56:56 -0700 |
|---|---|---|
| committer | Ronald S. Bultje <rsbultje@gmail.com> | 2011-10-21 00:14:11 -0700 |
| commit | 8305041e137f4f2a49669dd588bf6ccfbbac2b58 (patch) | |
| tree | 9a2e5e519ca1e9206f70f7bbd4d2c80573a37adf /libswscale/utils.c | |
| parent | 65d3176aaf8f876e0769f200abe95cac42151c89 (diff) | |
swscale: prevent overflow in coefficient calculation.
Diffstat (limited to 'libswscale/utils.c')
| -rw-r--r-- | libswscale/utils.c | 23 |
1 files changed, 12 insertions, 11 deletions
diff --git a/libswscale/utils.c b/libswscale/utils.c index 8e5daf99dc..ea5a1ab468 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -271,19 +271,20 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi floatd= d * (1.0/(1<<30)); if (flags & SWS_BICUBIC) { -#define SQRT_INT64_MAX 0xb504f333 int64_t B= (param[0] != SWS_PARAM_DEFAULT ? param[0] : 0) * (1<<24); int64_t C= (param[1] != SWS_PARAM_DEFAULT ? param[1] : 0.6) * (1<<24); - int64_t dd = d > SQRT_INT64_MAX ? ((d >> 1) * d) >> 29 : (d * d) >> 30; - int64_t ddd = d > SQRT_INT64_MAX || dd > SQRT_INT64_MAX ? - ((dd >> 2) * d) >> 28 : (dd * d) >> 30; - - if (d < 1LL<<30) - coeff = (12*(1<<24)-9*B-6*C)*ddd + (-18*(1<<24)+12*B+6*C)*dd + (6*(1<<24)-2*B)*(1<<30); - else if (d < 1LL<<31) - coeff = (-B-6*C)*ddd + (6*B+30*C)*dd + (-12*B-48*C)*d + (8*B+24*C)*(1<<30); - else - coeff=0.0; + + if (d >= 1LL<<31) { + coeff = 0.0; + } else { + int64_t dd = (d * d) >> 30; + int64_t ddd = (dd * d) >> 30; + + if (d < 1LL<<30) + coeff = (12*(1<<24)-9*B-6*C)*ddd + (-18*(1<<24)+12*B+6*C)*dd + (6*(1<<24)-2*B)*(1<<30); + else + coeff = (-B-6*C)*ddd + (6*B+30*C)*dd + (-12*B-48*C)*d + (8*B+24*C)*(1<<30); + } coeff *= fone>>(30+24); } /* else if (flags & SWS_X) { |