From 830f70442a87a31f7c75565e9380e3caf8333b8a Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 24 Feb 2012 14:11:04 -0800 Subject: fraps: release reference buffer on pix_fmt change. Prevents crash when trying to copy from a non-existing plane in e.g. a RGB32 reference image to a YUV420P target image Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org --- libavcodec/fraps.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'libavcodec/fraps.c') diff --git a/libavcodec/fraps.c b/libavcodec/fraps.c index 3643325328..e2355d8f2f 100644 --- a/libavcodec/fraps.c +++ b/libavcodec/fraps.c @@ -139,7 +139,7 @@ static int decode_frame(AVCodecContext *avctx, uint32_t *luma1,*luma2,*cb,*cr; uint32_t offs[4]; int i, j, is_chroma, planes; - + enum PixelFormat pix_fmt; header = AV_RL32(buf); version = header & 0xff; @@ -156,12 +156,16 @@ static int decode_frame(AVCodecContext *avctx, if (header_size == 8) buf+=4; + pix_fmt = version & 1 ? PIX_FMT_BGR24 : PIX_FMT_YUVJ420P; + if (avctx->pix_fmt != pix_fmt && f->data[0]) { + avctx->release_buffer(avctx, f); + } + avctx->pix_fmt = pix_fmt; + switch(version) { case 0: default: /* Fraps v0 is a reordered YUV420 */ - avctx->pix_fmt = PIX_FMT_YUVJ420P; - if ( (buf_size != avctx->width*avctx->height*3/2+header_size) && (buf_size != header_size) ) { av_log(avctx, AV_LOG_ERROR, @@ -209,8 +213,6 @@ static int decode_frame(AVCodecContext *avctx, case 1: /* Fraps v1 is an upside-down BGR24 */ - avctx->pix_fmt = PIX_FMT_BGR24; - if ( (buf_size != avctx->width*avctx->height*3+header_size) && (buf_size != header_size) ) { av_log(avctx, AV_LOG_ERROR, @@ -245,7 +247,6 @@ static int decode_frame(AVCodecContext *avctx, * Fraps v2 is Huffman-coded YUV420 planes * Fraps v4 is virtually the same */ - avctx->pix_fmt = PIX_FMT_YUVJ420P; planes = 3; f->reference = 1; f->buffer_hints = FF_BUFFER_HINTS_VALID | @@ -291,7 +292,6 @@ static int decode_frame(AVCodecContext *avctx, case 3: case 5: /* Virtually the same as version 4, but is for RGB24 */ - avctx->pix_fmt = PIX_FMT_BGR24; planes = 3; f->reference = 1; f->buffer_hints = FF_BUFFER_HINTS_VALID | -- cgit v1.2.3