From 72dafea0fc0eb7230d7ebb0a7bc803e13b72aaad Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Sat, 17 Dec 2011 19:56:40 +0000 Subject: swscale: fix overflow in gray16 vertical scaling. This fixes the same overflow as in the RGB48/16-bit YUV scaling; some filters can overflow both negatively and positively (e.g. spline/lanczos), so we bias a signed integer so it's "half signed" and "half unsigned", and can cover overflows in both directions while maintaining full 31-bit depth. Signed-off-by: Mans Rullgard --- libswscale/swscale.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'libswscale/swscale.c') diff --git a/libswscale/swscale.c b/libswscale/swscale.c index 3ea37104e8..c9dfc8df2f 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -386,8 +386,8 @@ yuv2gray16_X_c_template(SwsContext *c, const int16_t *lumFilter, for (i = 0; i < (dstW >> 1); i++) { int j; - int Y1 = 1 << 14; - int Y2 = 1 << 14; + int Y1 = (1 << 14) - 0x40000000; + int Y2 = (1 << 14) - 0x40000000; for (j = 0; j < lumFilterSize; j++) { Y1 += lumSrc[j][i * 2] * lumFilter[j]; @@ -395,12 +395,10 @@ yuv2gray16_X_c_template(SwsContext *c, const int16_t *lumFilter, } Y1 >>= 15; Y2 >>= 15; - if ((Y1 | Y2) & 0x10000) { - Y1 = av_clip_uint16(Y1); - Y2 = av_clip_uint16(Y2); - } - output_pixel(&dest[i * 2 + 0], Y1); - output_pixel(&dest[i * 2 + 1], Y2); + Y1 = av_clip_int16(Y1); + Y2 = av_clip_int16(Y2); + output_pixel(&dest[i * 2 + 0], 0x8000 + Y1); + output_pixel(&dest[i * 2 + 1], 0x8000 + Y2); } } -- cgit v1.2.3