From be1bafc303a77fa9ca5a99e36a65d5765012d3f4 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 16 Dec 2011 00:03:54 +0000 Subject: swscale: fix overflows in output of RGB48 pixels. For certain types of filters where the intermediate sum of coefficients can go above the fixed-point equivalent of 1.0 in the middle of a filter, the sum of a 31-bit calculation can overflow in both directions and can thus not be represented in a 32-bit signed or unsigned integer. To work around this, we subtract 0x40000000 from a signed integer base, so that we're halfway signed/unsigned, which makes it fit even if it overflows. After the filter finishes, we add the scaled bias back after a shift. We use the same trick for 16-bit bpc YUV output routines. Signed-off-by: Mans Rullgard --- libswscale/swscale.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'libswscale/swscale.c') diff --git a/libswscale/swscale.c b/libswscale/swscale.c index f072378001..818c049f35 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -754,8 +754,8 @@ yuv2rgb48_X_c_template(SwsContext *c, const int16_t *lumFilter, for (i = 0; i < (dstW >> 1); i++) { int j; - int Y1 = 0; - int Y2 = 0; + int Y1 = -0x40000000; + int Y2 = -0x40000000; int U = -128 << 23; // 19 int V = -128 << 23; int R, G, B; @@ -771,7 +771,9 @@ yuv2rgb48_X_c_template(SwsContext *c, const int16_t *lumFilter, // 8bit: 12+15=27; 16-bit: 12+19=31 Y1 >>= 14; // 10 + Y1 += 0x10000; Y2 >>= 14; + Y2 += 0x10000; U >>= 14; V >>= 14; -- cgit v1.2.3