From 2254b559cbcfc0418135f09add37c0a5866b1981 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Mon, 5 Mar 2012 12:26:42 -0800 Subject: swscale: make filterPos 32bit. Fixes overflows for large image sizes. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org --- libswscale/utils.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libswscale/utils.c') diff --git a/libswscale/utils.c b/libswscale/utils.c index 7c89a70953..51dd331117 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -180,7 +180,7 @@ static double getSplineCoeff(double a, double b, double c, double d, double dist dist-1.0); } -static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSize, int xInc, +static int initFilter(int16_t **outFilter, int32_t **filterPos, int *outFilterSize, int xInc, int srcW, int dstW, int filterAlign, int one, int flags, int cpu_flags, SwsVector *srcFilter, SwsVector *dstFilter, double param[2], int is_horizontal) { @@ -196,7 +196,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi emms_c(); //FIXME this should not be required but it IS (even for non-MMX versions) // NOTE: the +3 is for the MMX(+1)/SSE(+3) scaler which reads over the end - FF_ALLOC_OR_GOTO(NULL, *filterPos, (dstW+3)*sizeof(int16_t), fail); + FF_ALLOC_OR_GOTO(NULL, *filterPos, (dstW+3)*sizeof(**filterPos), fail); if (FFABS(xInc - 0x10000) <10) { // unscaled int i; -- cgit v1.2.3