From e7843db3df0224cafcc1af9da103a3a7286ae2ba Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 12 Jan 2012 15:37:58 -0800 Subject: swscale: fix invalid conversions and memory problems. Fixes problems where rgbToRgbWrapper() is called even though it doesn't support this particular conversion (e.g. converting from RGB444 to anything). Thirdly, fixes issues where rgbToRgbWrapper() is called for non-native endiannness conversions (e.g. RGB555BE on a LE system). Fourthly, fixes crashes when converting from e.g. monowhite to monowhite, which calls planarCopyWrapper() and overwrites/reads because n_bytes != n_pixels. --- libswscale/swscale_unscaled.c | 49 +++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 20 deletions(-) (limited to 'libswscale/swscale_unscaled.c') diff --git a/libswscale/swscale_unscaled.c b/libswscale/swscale_unscaled.c index a1b7199a23..e497fef84f 100644 --- a/libswscale/swscale_unscaled.c +++ b/libswscale/swscale_unscaled.c @@ -352,17 +352,22 @@ static int palToRgbWrapper(SwsContext *c, const uint8_t *src[], int srcStride[], ) /* {RGB,BGR}{15,16,24,32,32_1} -> {RGB,BGR}{15,16,24,32} */ -static int rgbToRgbWrapper(SwsContext *c, const uint8_t *src[], int srcStride[], - int srcSliceY, int srcSliceH, uint8_t *dst[], - int dstStride[]) +typedef void (* rgbConvFn) (const uint8_t *, uint8_t *, int); +static rgbConvFn findRgbConvFn(SwsContext *c) { const enum PixelFormat srcFormat = c->srcFormat; const enum PixelFormat dstFormat = c->dstFormat; - const int srcBpp = (c->srcFormatBpp + 7) >> 3; - const int dstBpp = (c->dstFormatBpp + 7) >> 3; const int srcId = c->srcFormatBpp; const int dstId = c->dstFormatBpp; - void (*conv)(const uint8_t *src, uint8_t *dst, int src_size) = NULL; + rgbConvFn conv = NULL; + +#define IS_NOT_NE(bpp, fmt) \ + (((bpp + 7) >> 3) == 2 && \ + (!(av_pix_fmt_descriptors[fmt].flags & PIX_FMT_BE) != !HAVE_BIGENDIAN)) + + /* if this is non-native rgb444/555/565, don't handle it here. */ + if (IS_NOT_NE(srcId, srcFormat) || IS_NOT_NE(dstId, dstFormat)) + return NULL; #define CONV_IS(src, dst) (srcFormat == PIX_FMT_##src && dstFormat == PIX_FMT_##dst) @@ -419,6 +424,21 @@ static int rgbToRgbWrapper(SwsContext *c, const uint8_t *src[], int srcStride[], } } + return conv; +} + +/* {RGB,BGR}{15,16,24,32,32_1} -> {RGB,BGR}{15,16,24,32} */ +static int rgbToRgbWrapper(SwsContext *c, const uint8_t *src[], int srcStride[], + int srcSliceY, int srcSliceH, uint8_t *dst[], + int dstStride[]) + +{ + const enum PixelFormat srcFormat = c->srcFormat; + const enum PixelFormat dstFormat = c->dstFormat; + const int srcBpp = (c->srcFormatBpp + 7) >> 3; + const int dstBpp = (c->dstFormatBpp + 7) >> 3; + rgbConvFn conv = findRgbConvFn(c); + if (!conv) { av_log(c, AV_LOG_ERROR, "internal error %s -> %s converter\n", sws_format_name(srcFormat), sws_format_name(dstFormat)); @@ -716,6 +736,8 @@ static int planarCopyWrapper(SwsContext *c, const uint8_t *src[], } else { if (is16BPS(c->srcFormat) && is16BPS(c->dstFormat)) length *= 2; + else if (!av_pix_fmt_descriptors[c->srcFormat].comp[0].depth_minus1) + length >>= 3; // monowhite/black for (i = 0; i < height; i++) { memcpy(dstPtr, srcPtr, length); srcPtr += srcStride[plane]; @@ -770,20 +792,7 @@ void ff_get_unscaled_swscale(SwsContext *c) c->swScale = bgr24ToYv12Wrapper; /* RGB/BGR -> RGB/BGR (no dither needed forms) */ - if ( isAnyRGB(srcFormat) - && isAnyRGB(dstFormat) - && srcFormat != PIX_FMT_BGR8 && dstFormat != PIX_FMT_BGR8 - && srcFormat != PIX_FMT_RGB8 && dstFormat != PIX_FMT_RGB8 - && srcFormat != PIX_FMT_BGR4 && dstFormat != PIX_FMT_BGR4 - && srcFormat != PIX_FMT_RGB4 && dstFormat != PIX_FMT_RGB4 - && srcFormat != PIX_FMT_BGR4_BYTE && dstFormat != PIX_FMT_BGR4_BYTE - && srcFormat != PIX_FMT_RGB4_BYTE && dstFormat != PIX_FMT_RGB4_BYTE - && srcFormat != PIX_FMT_MONOBLACK && dstFormat != PIX_FMT_MONOBLACK - && srcFormat != PIX_FMT_MONOWHITE && dstFormat != PIX_FMT_MONOWHITE - && srcFormat != PIX_FMT_RGB48LE && dstFormat != PIX_FMT_RGB48LE - && srcFormat != PIX_FMT_RGB48BE && dstFormat != PIX_FMT_RGB48BE - && srcFormat != PIX_FMT_BGR48LE && dstFormat != PIX_FMT_BGR48LE - && srcFormat != PIX_FMT_BGR48BE && dstFormat != PIX_FMT_BGR48BE + if (isAnyRGB(srcFormat) && isAnyRGB(dstFormat) && findRgbConvFn(c) && (!needsDither || (c->flags&(SWS_FAST_BILINEAR|SWS_POINT)))) c->swScale= rgbToRgbWrapper; -- cgit v1.2.3