From 7248797c03e8fcddc1f1ab5887d1e53cc691a2c2 Mon Sep 17 00:00:00 2001 From: Ramiro Polla Date: Sat, 29 Aug 2009 23:02:01 +0000 Subject: swscale: Check for return values of malloc. Originally committed as revision 29584 to svn://svn.mplayerhq.hu/mplayer/trunk/libswscale --- libswscale/colorspace-test.c | 3 +++ libswscale/swscale-example.c | 8 ++++-- libswscale/swscale.c | 60 ++++++++++++++++++++++++++++---------------- 3 files changed, 47 insertions(+), 24 deletions(-) (limited to 'libswscale') diff --git a/libswscale/colorspace-test.c b/libswscale/colorspace-test.c index dbdf6bef90..5a48065d1f 100644 --- a/libswscale/colorspace-test.c +++ b/libswscale/colorspace-test.c @@ -66,6 +66,9 @@ int main(int argc, char **argv) int failedNum=0; int passedNum=0; + if (!srcBuffer || !dstBuffer) + return -1; + av_log(NULL, AV_LOG_INFO, "memory corruption test ...\n"); args_parse(argc, argv); av_log(NULL, AV_LOG_INFO, "CPU capabilities forced to %x\n", cpu_caps); diff --git a/libswscale/swscale-example.c b/libswscale/swscale-example.c index 44aef6a961..1402d44a17 100644 --- a/libswscale/swscale-example.c +++ b/libswscale/swscale-example.c @@ -183,8 +183,9 @@ end: static void selfTest(uint8_t *ref[4], int refStride[4], int w, int h) { const int flags[] = { SWS_FAST_BILINEAR, - SWS_BILINEAR, SWS_BICUBIC, - SWS_X , SWS_POINT , SWS_AREA, 0 }; +// SWS_BILINEAR, SWS_BICUBIC, +// SWS_X , SWS_POINT , SWS_AREA, 0 }; + 0 }; const int srcW = w; const int srcH = h; const int dstW[] = { srcW - srcW/3, srcW, srcW + srcW/3, 0 }; @@ -225,6 +226,9 @@ int main(int argc, char **argv) struct SwsContext *sws; AVLFG rand; + if (!rgb_data || !data) + return -1; + sws= sws_getContext(W/12, H/12, PIX_FMT_RGB32, W, H, PIX_FMT_YUVA420P, SWS_BILINEAR, NULL, NULL, NULL); av_lfg_init(&rand, 1); diff --git a/libswscale/swscale.c b/libswscale/swscale.c index 9266effc13..b8f3f35909 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -76,6 +76,7 @@ untested special converters #include "rgb2rgb.h" #include "libavutil/intreadwrite.h" #include "libavutil/x86_cpu.h" +#include "libavutil/avutil.h" #include "libavutil/bswap.h" unsigned swscale_version(void) @@ -1450,12 +1451,12 @@ static inline int initFilter(int16_t **outFilter, int16_t **filterPos, int *outF #endif // NOTE: the +1 is for the MMX scaler which reads over the end - *filterPos = av_malloc((dstW+1)*sizeof(int16_t)); + CHECKED_ALLOC(*filterPos, (dstW+1)*sizeof(int16_t)); if (FFABS(xInc - 0x10000) <10) { // unscaled int i; filterSize= 1; - filter= av_mallocz(dstW*sizeof(*filter)*filterSize); + CHECKED_ALLOCZ(filter, dstW*sizeof(*filter)*filterSize); for (i=0; i srcW-2) filterSize=srcW-2; - filter= av_malloc(dstW*sizeof(*filter)*filterSize); + CHECKED_ALLOC(filter, dstW*sizeof(*filter)*filterSize); xDstInSrc= xInc - 0x10000; for (i=0; ilength - 1; if (dstFilter) filter2Size+= dstFilter->length - 1; assert(filter2Size>0); - filter2= av_mallocz(filter2Size*dstW*sizeof(*filter2)); + CHECKED_ALLOCZ(filter2, filter2Size*dstW*sizeof(*filter2)); for (i=0; i 0); filter= av_malloc(filterSize*dstW*sizeof(*filter)); if (filterSize >= MAX_FILTER_SIZE*16/((flags&SWS_ACCURATE_RND) ? APCK_SIZE : 16) || !filter) - goto error; + goto fail; *outFilterSize= filterSize; if (flags&SWS_PRINT_INFO) @@ -1737,7 +1738,7 @@ static inline int initFilter(int16_t **outFilter, int16_t **filterPos, int *outF // Note the +1 is for the MMX scaler which reads over the end /* align at 16 for AltiVec (needed by hScale_altivec_real) */ - *outFilter= av_mallocz(*outFilterSize*(dstW+1)*sizeof(int16_t)); + CHECKED_ALLOCZ(*outFilter, *outFilterSize*(dstW+1)*sizeof(int16_t)); /* normalize & store in outFilter */ for (i=0; iav_class = &sws_context_class; c->srcW= srcW; @@ -2837,10 +2838,10 @@ SwsContext *sws_getContext(int srcW, int srcH, enum PixelFormat srcFormat, int d c->chrMmx2FilterCode = av_malloc(c->chrMmx2FilterCodeSize); #endif - c->lumMmx2Filter = av_malloc((dstW /8+8)*sizeof(int16_t)); - c->chrMmx2Filter = av_malloc((c->chrDstW /4+8)*sizeof(int16_t)); - c->lumMmx2FilterPos= av_malloc((dstW /2/8+8)*sizeof(int32_t)); - c->chrMmx2FilterPos= av_malloc((c->chrDstW/2/4+8)*sizeof(int32_t)); + CHECKED_ALLOCZ(c->lumMmx2Filter , (dstW /8+8)*sizeof(int16_t)); + CHECKED_ALLOCZ(c->chrMmx2Filter , (c->chrDstW /4+8)*sizeof(int16_t)); + CHECKED_ALLOCZ(c->lumMmx2FilterPos, (dstW /2/8+8)*sizeof(int32_t)); + CHECKED_ALLOCZ(c->chrMmx2FilterPos, (c->chrDstW/2/4+8)*sizeof(int32_t)); initMMX2HScaler( dstW, c->lumXInc, c->lumMmx2FilterCode, c->lumMmx2Filter, c->lumMmx2FilterPos, 8); initMMX2HScaler(c->chrDstW, c->chrXInc, c->chrMmx2FilterCode, c->chrMmx2Filter, c->chrMmx2FilterPos, 4); @@ -2872,8 +2873,8 @@ SwsContext *sws_getContext(int srcW, int srcH, enum PixelFormat srcFormat, int d srcFilter->chrV, dstFilter->chrV, c->param); #ifdef COMPILE_ALTIVEC - c->vYCoeffsBank = av_malloc(sizeof (vector signed short)*c->vLumFilterSize*c->dstH); - c->vCCoeffsBank = av_malloc(sizeof (vector signed short)*c->vChrFilterSize*c->chrDstH); + CHECKED_ALLOC(c->vYCoeffsBank, sizeof (vector signed short)*c->vLumFilterSize*c->dstH); + CHECKED_ALLOC(c->vCCoeffsBank, sizeof (vector signed short)*c->vChrFilterSize*c->chrDstH); for (i=0;ivLumFilterSize*c->dstH;i++) { int j; @@ -2909,19 +2910,28 @@ SwsContext *sws_getContext(int srcW, int srcH, enum PixelFormat srcFormat, int d // allocate pixbufs (we use dynamic allocation because otherwise we would need to // allocate several megabytes to handle all possible cases) - c->lumPixBuf= av_malloc(c->vLumBufSize*2*sizeof(int16_t*)); - c->chrPixBuf= av_malloc(c->vChrBufSize*2*sizeof(int16_t*)); + CHECKED_ALLOC(c->lumPixBuf, c->vLumBufSize*2*sizeof(int16_t*)); + CHECKED_ALLOC(c->chrPixBuf, c->vChrBufSize*2*sizeof(int16_t*)); if (CONFIG_SWSCALE_ALPHA && isALPHA(c->srcFormat) && isALPHA(c->dstFormat)) - c->alpPixBuf= av_malloc(c->vLumBufSize*2*sizeof(int16_t*)); + CHECKED_ALLOCZ(c->alpPixBuf, c->vLumBufSize*2*sizeof(int16_t*)); //Note we need at least one pixel more at the end because of the MMX code (just in case someone wanna replace the 4000/8000) /* align at 16 bytes for AltiVec */ for (i=0; ivLumBufSize; i++) - c->lumPixBuf[i]= c->lumPixBuf[i+c->vLumBufSize]= av_mallocz(VOF+1); + { + CHECKED_ALLOCZ(c->lumPixBuf[i+c->vLumBufSize], VOF+1); + c->lumPixBuf[i] = c->lumPixBuf[i+c->vLumBufSize]; + } for (i=0; ivChrBufSize; i++) - c->chrPixBuf[i]= c->chrPixBuf[i+c->vChrBufSize]= av_malloc((VOF+1)*2); + { + CHECKED_ALLOC(c->chrPixBuf[i+c->vChrBufSize], (VOF+1)*2); + c->chrPixBuf[i] = c->chrPixBuf[i+c->vChrBufSize]; + } if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) for (i=0; ivLumBufSize; i++) - c->alpPixBuf[i]= c->alpPixBuf[i+c->vLumBufSize]= av_mallocz(VOF+1); + { + CHECKED_ALLOCZ(c->alpPixBuf[i+c->vLumBufSize], VOF+1); + c->alpPixBuf[i] = c->alpPixBuf[i+c->vLumBufSize]; + } //try to avoid drawing green stuff between the right end and the stride end for (i=0; ivChrBufSize; i++) memset(c->chrPixBuf[i], 64, (VOF+1)*2); @@ -3045,6 +3055,10 @@ SwsContext *sws_getContext(int srcW, int srcH, enum PixelFormat srcFormat, int d c->swScale= getSwsFunc(c); return c; + +fail: + sws_freeContext(c); + return NULL; } static void reset_ptr(uint8_t* src[], int format) @@ -3189,6 +3203,8 @@ SwsFilter *sws_getDefaultFilter(float lumaGBlur, float chromaGBlur, int verbose) { SwsFilter *filter= av_malloc(sizeof(SwsFilter)); + if (!filter) + return NULL; if (lumaGBlur!=0.0) { filter->lumH= sws_getGaussianVec(lumaGBlur, 3.0); -- cgit v1.2.3