From 7d7bacf0f1f43f5cc112977fc3b8438e3e4a4a9b Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 8 Jul 2011 14:57:07 -0700 Subject: swscale: fix overflow in 16-bit vertical scaling. We operated on 31-bits, but with e.g. lanczos scaling, values can add up to beyond 0x80000000, thus leading to output of zeroes. Drop one bit of precision fixes this. --- libswscale/swscale.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'libswscale') diff --git a/libswscale/swscale.c b/libswscale/swscale.c index 527dd80e0c..db4d231e13 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -195,7 +195,7 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc, int i; uint16_t *yDest = dest[0], *uDest = dest[1], *vDest = dest[2], *aDest = CONFIG_SWSCALE_ALPHA ? dest[3] : NULL; - int shift = 15 + 16 - output_bits; + int shift = 15 + 16 - output_bits - 1; #define output_pixel(pos, val) \ if (big_endian) { \ @@ -212,24 +212,24 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc, } \ } for (i = 0; i < dstW; i++) { - int val = 1 << (30-output_bits); + int val = 1 << (30-output_bits - 1); int j; for (j = 0; j < lumFilterSize; j++) - val += lumSrc[j][i] * lumFilter[j]; + val += (lumSrc[j][i] * lumFilter[j]) >> 1; output_pixel(&yDest[i], val); } if (uDest) { for (i = 0; i < chrDstW; i++) { - int u = 1 << (30-output_bits); - int v = 1 << (30-output_bits); + int u = 1 << (30-output_bits - 1); + int v = 1 << (30-output_bits - 1); int j; for (j = 0; j < chrFilterSize; j++) { - u += chrUSrc[j][i] * chrFilter[j]; - v += chrVSrc[j][i] * chrFilter[j]; + u += (chrUSrc[j][i] * chrFilter[j]) >> 1; + v += (chrVSrc[j][i] * chrFilter[j]) >> 1; } output_pixel(&uDest[i], u); @@ -239,11 +239,11 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc, if (CONFIG_SWSCALE_ALPHA && aDest) { for (i = 0; i < dstW; i++) { - int val = 1 << (30-output_bits); + int val = 1 << (30-output_bits - 1); int j; for (j = 0; j < lumFilterSize; j++) - val += alpSrc[j][i] * lumFilter[j]; + val += (alpSrc[j][i] * lumFilter[j]) >> 1; output_pixel(&aDest[i], val); } -- cgit v1.2.3