From 8d745281a489f0f69b00a99345c4a63026082eff Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Thu, 8 Aug 2013 02:56:26 +0200 Subject: swscale/utils: Fix potential overflow of srcPos*C before converting to 64bit Fixes CID1061054 Signed-off-by: Michael Niedermayer --- libswscale/utils.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'libswscale') diff --git a/libswscale/utils.c b/libswscale/utils.c index 3f8796f2dd..5aacd817b8 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -299,7 +299,7 @@ static av_cold int initFilter(int16_t **outFilter, int32_t **filterPos, FF_ALLOC_OR_GOTO(NULL, filter, dstW * sizeof(*filter) * filterSize, fail); - xDstInSrc = ((dstPos*(int64_t)xInc)>>8) - ((srcPos*0x8000)>>7); + xDstInSrc = ((dstPos*(int64_t)xInc)>>8) - ((srcPos*0x8000LL)>>7); for (i = 0; i < dstW; i++) { int xx = (xDstInSrc - ((filterSize - 1) << 15) + (1 << 15)) >> 16; @@ -315,7 +315,7 @@ static av_cold int initFilter(int16_t **outFilter, int32_t **filterPos, FF_ALLOC_OR_GOTO(NULL, filter, dstW * sizeof(*filter) * filterSize, fail); - xDstInSrc = ((dstPos*(int64_t)xInc)>>8) - ((srcPos*0x8000)>>7); + xDstInSrc = ((dstPos*(int64_t)xInc)>>8) - ((srcPos*0x8000LL)>>7); for (i = 0; i < dstW; i++) { int xx = (xDstInSrc - ((filterSize - 1) << 15) + (1 << 15)) >> 16; int j; @@ -366,7 +366,7 @@ static av_cold int initFilter(int16_t **outFilter, int32_t **filterPos, FF_ALLOC_OR_GOTO(NULL, filter, dstW * sizeof(*filter) * filterSize, fail); - xDstInSrc = ((dstPos*(int64_t)xInc)>>7) - ((srcPos*0x10000)>>7); + xDstInSrc = ((dstPos*(int64_t)xInc)>>7) - ((srcPos*0x10000LL)>>7); for (i = 0; i < dstW; i++) { int xx = (xDstInSrc - ((filterSize - 2) << 16)) / (1 << 17); int j; -- cgit v1.2.3