From d0393d79bc3d61c9f2ff832c0e273b7774ff0269 Mon Sep 17 00:00:00 2001 From: Diego Biurrun Date: Sun, 3 Aug 2014 12:19:10 -0700 Subject: huffyuv: Check and propagate function return values Bug-Id: CVE-2013-0868 inspired by a patch from Michael Niedermayer Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Diego Biurrun CC: libav-stable@libav.org --- libavcodec/huffyuvdec.c | 105 ++++++++++++++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 44 deletions(-) (limited to 'libavcodec/huffyuvdec.c') diff --git a/libavcodec/huffyuvdec.c b/libavcodec/huffyuvdec.c index 4d62ab3ef4..dc99d19a62 100644 --- a/libavcodec/huffyuvdec.c +++ b/libavcodec/huffyuvdec.c @@ -106,11 +106,13 @@ static int read_len_table(uint8_t *dst, GetBitContext *gb) return 0; } -static void generate_joint_tables(HYuvContext *s) +static int generate_joint_tables(HYuvContext *s) { uint16_t symbols[1 << VLC_BITS]; uint16_t bits[1 << VLC_BITS]; uint8_t len[1 << VLC_BITS]; + int ret; + if (s->bitstream_bpp < 24) { int p, i, y, u; for (p = 0; p < 3; p++) { @@ -131,8 +133,9 @@ static void generate_joint_tables(HYuvContext *s) } } ff_free_vlc(&s->vlc[3 + p]); - ff_init_vlc_sparse(&s->vlc[3 + p], VLC_BITS, i, len, 1, 1, - bits, 2, 2, symbols, 2, 2, 0); + if ((ret = ff_init_vlc_sparse(&s->vlc[3 + p], VLC_BITS, i, len, 1, 1, + bits, 2, 2, symbols, 2, 2, 0)) < 0) + return ret; } } else { uint8_t (*map)[4] = (uint8_t(*)[4]) s->pix_bgr_map; @@ -173,28 +176,34 @@ static void generate_joint_tables(HYuvContext *s) } } ff_free_vlc(&s->vlc[3]); - init_vlc(&s->vlc[3], VLC_BITS, i, len, 1, 1, bits, 2, 2, 0); + if ((ret = init_vlc(&s->vlc[3], VLC_BITS, i, len, 1, 1, + bits, 2, 2, 0)) < 0) + return ret; } + return 0; } static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length) { GetBitContext gb; - int i; + int i, ret; - init_get_bits(&gb, src, length * 8); + if ((ret = init_get_bits(&gb, src, length * 8)) < 0) + return ret; for (i = 0; i < 3; i++) { - if (read_len_table(s->len[i], &gb) < 0) - return -1; - if (ff_huffyuv_generate_bits_table(s->bits[i], s->len[i]) < 0) - return -1; + if ((ret = read_len_table(s->len[i], &gb)) < 0) + return ret; + if ((ret = ff_huffyuv_generate_bits_table(s->bits[i], s->len[i])) < 0) + return ret; ff_free_vlc(&s->vlc[i]); - init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, - s->bits[i], 4, 4, 0); + if ((ret = init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, + s->bits[i], 4, 4, 0)) < 0) + return ret; } - generate_joint_tables(s); + if ((ret = generate_joint_tables(s)) < 0) + return ret; return (get_bits_count(&gb) + 7) / 8; } @@ -202,17 +211,19 @@ static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length) static int read_old_huffman_tables(HYuvContext *s) { GetBitContext gb; - int i; + int i, ret; - init_get_bits(&gb, classic_shift_luma, - classic_shift_luma_table_size * 8); - if (read_len_table(s->len[0], &gb) < 0) - return -1; + if ((ret = init_get_bits(&gb, classic_shift_luma, + classic_shift_luma_table_size * 8)) < 0) + return ret; + if ((ret = read_len_table(s->len[0], &gb)) < 0) + return ret; - init_get_bits(&gb, classic_shift_chroma, - classic_shift_chroma_table_size * 8); - if (read_len_table(s->len[1], &gb) < 0) - return -1; + if ((ret = init_get_bits(&gb, classic_shift_chroma, + classic_shift_chroma_table_size * 8)) < 0) + return ret; + if ((ret = read_len_table(s->len[1], &gb)) < 0) + return ret; for (i = 0; i < 256; i++) s->bits[0][i] = classic_add_luma[i]; @@ -228,11 +239,13 @@ static int read_old_huffman_tables(HYuvContext *s) for (i = 0; i < 3; i++) { ff_free_vlc(&s->vlc[i]); - init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, - s->bits[i], 4, 4, 0); + if ((ret = init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, + s->bits[i], 4, 4, 0)) < 0) + return ret; } - generate_joint_tables(s); + if ((ret = generate_joint_tables(s)) < 0) + return ret; return 0; } @@ -240,6 +253,7 @@ static int read_old_huffman_tables(HYuvContext *s) static av_cold int decode_init(AVCodecContext *avctx) { HYuvContext *s = avctx->priv_data; + int ret; ff_huffyuv_common_init(avctx); ff_huffyuvdsp_init(&s->hdsp); @@ -273,9 +287,9 @@ static av_cold int decode_init(AVCodecContext *avctx) s->interlaced = (interlace == 1) ? 1 : (interlace == 2) ? 0 : s->interlaced; s->context = avctx->extradata[2] & 0x40 ? 1 : 0; - if (read_huffman_tables(s, avctx->extradata + 4, - avctx->extradata_size - 4) < 0) - return -1; + if ((ret = read_huffman_tables(s, avctx->extradata + 4, + avctx->extradata_size - 4)) < 0) + return ret; } else { switch (avctx->bits_per_coded_sample & 7) { case 1: @@ -302,8 +316,8 @@ static av_cold int decode_init(AVCodecContext *avctx) s->bitstream_bpp = avctx->bits_per_coded_sample & ~7; s->context = 0; - if (read_old_huffman_tables(s) < 0) - return -1; + if ((ret = read_old_huffman_tables(s)) < 0) + return ret; } switch (s->bitstream_bpp) { @@ -327,7 +341,8 @@ static av_cold int decode_init(AVCodecContext *avctx) return AVERROR_INVALIDDATA; } - ff_huffyuv_alloc_temp(s); + if ((ret = ff_huffyuv_alloc_temp(s)) < 0) + return ret; return 0; } @@ -335,20 +350,21 @@ static av_cold int decode_init(AVCodecContext *avctx) static av_cold int decode_init_thread_copy(AVCodecContext *avctx) { HYuvContext *s = avctx->priv_data; - int i; + int i, ret; - ff_huffyuv_alloc_temp(s); + if ((ret = ff_huffyuv_alloc_temp(s)) < 0) + return ret; for (i = 0; i < 6; i++) s->vlc[i].table = NULL; if (s->version == 2) { - if (read_huffman_tables(s, avctx->extradata + 4, - avctx->extradata_size) < 0) - return -1; + if ((ret = read_huffman_tables(s, avctx->extradata + 4, + avctx->extradata_size)) < 0) + return ret; } else { - if (read_old_huffman_tables(s) < 0) - return -1; + if ((ret = read_old_huffman_tables(s)) < 0) + return ret; } return 0; @@ -484,7 +500,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, int fake_ystride, fake_ustride, fake_vstride; ThreadFrame frame = { .f = data }; AVFrame *const p = data; - int table_size = 0; + int table_size = 0, ret; av_fast_malloc(&s->bitstream_buffer, &s->bitstream_buffer_size, @@ -496,22 +512,23 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, s->bdsp.bswap_buf((uint32_t *) s->bitstream_buffer, (const uint32_t *) buf, buf_size / 4); - if (ff_thread_get_buffer(avctx, &frame, 0) < 0) { + if ((ret = ff_thread_get_buffer(avctx, &frame, 0)) < 0) { av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n"); - return -1; + return ret; } if (s->context) { table_size = read_huffman_tables(s, s->bitstream_buffer, buf_size); if (table_size < 0) - return -1; + return table_size; } if ((unsigned) (buf_size - table_size) >= INT_MAX / 8) return AVERROR_INVALIDDATA; - init_get_bits(&s->gb, s->bitstream_buffer + table_size, - (buf_size - table_size) * 8); + if ((ret = init_get_bits(&s->gb, s->bitstream_buffer + table_size, + (buf_size - table_size) * 8)) < 0) + return ret; fake_ystride = s->interlaced ? p->linesize[0] * 2 : p->linesize[0]; fake_ustride = s->interlaced ? p->linesize[1] * 2 : p->linesize[1]; -- cgit v1.2.3