From fc1eda543c6ef043300612db90da5cfd972af650 Mon Sep 17 00:00:00 2001 From: Vittorio Giovara Date: Fri, 17 Oct 2014 14:31:35 +0100 Subject: swscale: fix sign extensions in yuv planar conversion Casting the left-most byte to unsigned avoids an undefined result of the shift by 24 if bit 7 is set. yuvPlanartouyvy_c and yuvPlanartoyuy2_c are affected. CC: libav-stable@libav.org Bug-Id: CID 732281 / CID 732282 --- libswscale/rgb2rgb_template.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'libswscale') diff --git a/libswscale/rgb2rgb_template.c b/libswscale/rgb2rgb_template.c index 65ea5dda8c..693c7f2d0e 100644 --- a/libswscale/rgb2rgb_template.c +++ b/libswscale/rgb2rgb_template.c @@ -376,9 +376,9 @@ static inline void yuvPlanartoyuy2_c(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc; for (i = 0; i < chromWidth; i += 2) { uint64_t k = yc[0] + (uc[0] << 8) + - (yc[1] << 16) + (vc[0] << 24); + (yc[1] << 16) + ((unsigned) vc[0] << 24); uint64_t l = yc[2] + (uc[1] << 8) + - (yc[3] << 16) + (vc[1] << 24); + (yc[3] << 16) + ((unsigned) vc[1] << 24); *ldst++ = k + (l << 32); yc += 4; uc += 2; @@ -440,9 +440,9 @@ static inline void yuvPlanartouyvy_c(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc; for (i = 0; i < chromWidth; i += 2) { uint64_t k = uc[0] + (yc[0] << 8) + - (vc[0] << 16) + (yc[1] << 24); + (vc[0] << 16) + ((unsigned) yc[1] << 24); uint64_t l = uc[1] + (yc[2] << 8) + - (vc[1] << 16) + (yc[3] << 24); + (vc[1] << 16) + ((unsigned) yc[3] << 24); *ldst++ = k + (l << 32); yc += 4; uc += 2; -- cgit v1.2.3