From d9cf5f516974c64e01846ca685301014b38cf224 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 13 Jan 2013 19:52:45 +0100 Subject: lavc: check for overflow in init_get_bits Fix an undefined behaviour and make the function return a proper error in case of overflow. CC: libav-stable@libav.org --- libavcodec/get_bits.h | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) (limited to 'libavcodec') diff --git a/libavcodec/get_bits.h b/libavcodec/get_bits.h index c56a2c2d10..16cfd5e0fd 100644 --- a/libavcodec/get_bits.h +++ b/libavcodec/get_bits.h @@ -362,20 +362,27 @@ static inline int check_marker(GetBitContext *s, const char *msg) } /** - * Inititalize GetBitContext. - * @param buffer bitstream buffer, must be FF_INPUT_BUFFER_PADDING_SIZE bytes larger than the actual read bits - * because some optimized bitstream readers read 32 or 64 bit at once and could read over the end + * Initialize GetBitContext. + * @param buffer bitstream buffer, must be FF_INPUT_BUFFER_PADDING_SIZE bytes + * larger than the actual read bits because some optimized bitstream + * readers read 32 or 64 bit at once and could read over the end * @param bit_size the size of the buffer in bits + * @return 0 on success, AVERROR_INVALIDDATA if the buffer_size would overflow. */ -static inline void init_get_bits(GetBitContext *s, const uint8_t *buffer, - int bit_size) +static inline int init_get_bits(GetBitContext *s, const uint8_t *buffer, + int bit_size) { - int buffer_size = (bit_size+7)>>3; - if (buffer_size < 0 || bit_size < 0) { + int buffer_size; + int ret = 0; + + if (bit_size > INT_MAX - 7 || bit_size <= 0) { buffer_size = bit_size = 0; buffer = NULL; + ret = AVERROR_INVALIDDATA; } + buffer_size = (bit_size + 7) >> 3; + s->buffer = buffer; s->size_in_bits = bit_size; #if !UNCHECKED_BITSTREAM_READER @@ -383,6 +390,7 @@ static inline void init_get_bits(GetBitContext *s, const uint8_t *buffer, #endif s->buffer_end = buffer + buffer_size; s->index = 0; + return ret; } static inline void align_get_bits(GetBitContext *s) -- cgit v1.2.3 From e28ac6e5e27e64a206e399e958481c1e6f992189 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 13 Jan 2013 23:37:11 +0100 Subject: lavc: introduce the convenience function init_get_bits8 Accept the buffer size in bytes and check for overflow before passing the value in bits to init_get_bits. --- libavcodec/get_bits.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'libavcodec') diff --git a/libavcodec/get_bits.h b/libavcodec/get_bits.h index 16cfd5e0fd..12770a29a0 100644 --- a/libavcodec/get_bits.h +++ b/libavcodec/get_bits.h @@ -393,6 +393,22 @@ static inline int init_get_bits(GetBitContext *s, const uint8_t *buffer, return ret; } +/** + * Initialize GetBitContext. + * @param buffer bitstream buffer, must be FF_INPUT_BUFFER_PADDING_SIZE bytes + * larger than the actual read bits because some optimized bitstream + * readers read 32 or 64 bit at once and could read over the end + * @param byte_size the size of the buffer in bytes + * @return 0 on success, AVERROR_INVALIDDATA if the buffer_size would overflow. + */ +static inline int init_get_bits8(GetBitContext *s, const uint8_t *buffer, + int byte_size) +{ + if (byte_size > INT_MAX / 8) + return AVERROR_INVALIDDATA; + return init_get_bits(s, buffer, byte_size * 8); +} + static inline void align_get_bits(GetBitContext *s) { int n = -get_bits_count(s) & 7; -- cgit v1.2.3 From 8fd4d1f9b925f3b978756926b40315440c448696 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sat, 17 Nov 2012 15:45:41 +0100 Subject: cscd: return meaningful error codes. --- libavcodec/cscd.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'libavcodec') diff --git a/libavcodec/cscd.c b/libavcodec/cscd.c index 2c1b2e7858..b982f5029c 100644 --- a/libavcodec/cscd.c +++ b/libavcodec/cscd.c @@ -143,10 +143,11 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, int buf_size = avpkt->size; CamStudioContext *c = avctx->priv_data; AVFrame *picture = data; + int ret; if (buf_size < 2) { av_log(avctx, AV_LOG_ERROR, "coded frame too small\n"); - return -1; + return AVERROR_INVALIDDATA; } if (c->pic.data[0]) @@ -154,9 +155,9 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, c->pic.reference = 1; c->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_READABLE | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; - if (ff_get_buffer(avctx, &c->pic) < 0) { + if ((ret = ff_get_buffer(avctx, &c->pic)) < 0) { av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n"); - return -1; + return ret; } // decompress data @@ -175,12 +176,12 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, break; #else av_log(avctx, AV_LOG_ERROR, "compiled without zlib support\n"); - return -1; + return AVERROR(ENOSYS); #endif } default: av_log(avctx, AV_LOG_ERROR, "unknown compression\n"); - return -1; + return AVERROR_INVALIDDATA; } // flip upside down, add difference frame -- cgit v1.2.3 From 9e6764900a0732a4719ca219ceb59a4ae9d9be3b Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sat, 17 Nov 2012 17:48:38 +0100 Subject: msrle: return meaningful error codes. --- libavcodec/msrle.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'libavcodec') diff --git a/libavcodec/msrle.c b/libavcodec/msrle.c index 7cd8823682..7bca67f42e 100644 --- a/libavcodec/msrle.c +++ b/libavcodec/msrle.c @@ -63,7 +63,7 @@ static av_cold int msrle_decode_init(AVCodecContext *avctx) break; default: av_log(avctx, AV_LOG_ERROR, "unsupported bits per sample\n"); - return -1; + return AVERROR_INVALIDDATA; } s->frame.data[0] = NULL; @@ -79,15 +79,16 @@ static int msrle_decode_frame(AVCodecContext *avctx, int buf_size = avpkt->size; MsrleContext *s = avctx->priv_data; int istride = FFALIGN(avctx->width*avctx->bits_per_coded_sample, 32) / 8; + int ret; s->buf = buf; s->size = buf_size; s->frame.reference = 1; s->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; - if (avctx->reget_buffer(avctx, &s->frame)) { + if ((ret = avctx->reget_buffer(avctx, &s->frame)) < 0) { av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n"); - return -1; + return ret; } if (avctx->bits_per_coded_sample <= 8) { -- cgit v1.2.3 From 6bf70127d1d49f2a8c46803eda7b893b2b479282 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sat, 17 Nov 2012 17:52:52 +0100 Subject: tmv: return meaningful error codes. --- libavcodec/tmv.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'libavcodec') diff --git a/libavcodec/tmv.c b/libavcodec/tmv.c index d4e5aa1398..29d8e15fd9 100644 --- a/libavcodec/tmv.c +++ b/libavcodec/tmv.c @@ -47,20 +47,21 @@ static int tmv_decode_frame(AVCodecContext *avctx, void *data, unsigned char_cols = avctx->width >> 3; unsigned char_rows = avctx->height >> 3; unsigned x, y, fg, bg, c; + int ret; if (tmv->pic.data[0]) avctx->release_buffer(avctx, &tmv->pic); - if (ff_get_buffer(avctx, &tmv->pic) < 0) { + if ((ret = ff_get_buffer(avctx, &tmv->pic)) < 0) { av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n"); - return -1; + return ret; } if (avpkt->size < 2*char_rows*char_cols) { av_log(avctx, AV_LOG_ERROR, "Input buffer too small, truncated sample?\n"); *got_frame = 0; - return -1; + return AVERROR_INVALIDDATA; } tmv->pic.pict_type = AV_PICTURE_TYPE_I; -- cgit v1.2.3 From a1c525f7eb0783d31ba7a653865b6cbd3dc880de Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sat, 17 Nov 2012 17:57:06 +0100 Subject: pcx: return meaningful error codes. --- libavcodec/pcx.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'libavcodec') diff --git a/libavcodec/pcx.c b/libavcodec/pcx.c index 1bd861215e..04770a9ad2 100644 --- a/libavcodec/pcx.c +++ b/libavcodec/pcx.c @@ -94,7 +94,7 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, if (buf[0] != 0x0a || buf[1] > 5) { av_log(avctx, AV_LOG_ERROR, "this is not PCX encoded data\n"); - return -1; + return AVERROR_INVALIDDATA; } compressed = buf[2]; @@ -105,7 +105,7 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, if (xmax < xmin || ymax < ymin) { av_log(avctx, AV_LOG_ERROR, "invalid image dimensions\n"); - return -1; + return AVERROR_INVALIDDATA; } w = xmax - xmin + 1; @@ -118,7 +118,7 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, if (bytes_per_scanline < w * bits_per_pixel * nplanes / 8) { av_log(avctx, AV_LOG_ERROR, "PCX data is corrupted\n"); - return -1; + return AVERROR_INVALIDDATA; } switch ((nplanes<<8) + bits_per_pixel) { @@ -136,7 +136,7 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, break; default: av_log(avctx, AV_LOG_ERROR, "invalid PCX file\n"); - return -1; + return AVERROR_INVALIDDATA; } buf += 128; @@ -144,13 +144,13 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, if (p->data[0]) avctx->release_buffer(avctx, p); - if (av_image_check_size(w, h, 0, avctx)) - return -1; + if ((ret = av_image_check_size(w, h, 0, avctx)) < 0) + return ret; if (w != avctx->width || h != avctx->height) avcodec_set_dimensions(avctx, w, h); - if (ff_get_buffer(avctx, p) < 0) { + if ((ret = ff_get_buffer(avctx, p)) < 0) { av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n"); - return -1; + return ret; } p->pict_type = AV_PICTURE_TYPE_I; -- cgit v1.2.3