From 08b098169be079c4f124a351fda6764fbcd10e79 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Wed, 1 Feb 2017 17:19:18 +0100 Subject: speedhq: fix out-of-bounds write Certain alpha run lengths (for SHQ1/SHQ3/SHQ5) could be stored in both long and short versions, and we would only accept the short version, returning -1 (invalid code) for the others. This could cause an out-of-bounds write on malicious input, as discovered by Andreas Cadhalpun during fuzzing. Fix by simply allowing both versions, leaving no invalid codes in the alpha VLC. Signed-off-by: Andreas Cadhalpun --- libavcodec/speedhq.c | 128 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 78 insertions(+), 50 deletions(-) (limited to 'libavcodec/speedhq.c') diff --git a/libavcodec/speedhq.c b/libavcodec/speedhq.c index 385f779f83..45ee37a4e6 100644 --- a/libavcodec/speedhq.c +++ b/libavcodec/speedhq.c @@ -196,7 +196,7 @@ static inline int decode_alpha_block(const SHQContext *s, GetBitContext *gb, uin UPDATE_CACHE_LE(re, gb); GET_VLC(run, re, gb, ff_dc_alpha_run_vlc_le.table, ALPHA_VLC_BITS, 2); - if (run == 128) break; + if (run < 0) break; i += run; if (i >= 128) return AVERROR_INVALIDDATA; @@ -474,61 +474,89 @@ static int speedhq_decode_frame(AVCodecContext *avctx, */ static av_cold void compute_alpha_vlcs(void) { - uint16_t run_code[129], level_code[256]; - uint8_t run_bits[129], level_bits[256]; - int run, level; - - for (run = 0; run < 128; run++) { - if (!run) { - /* 0 -> 0. */ - run_code[run] = 0; - run_bits[run] = 1; - } else if (run <= 4) { - /* 10xx -> xx plus 1. */ - run_code[run] = ((run - 1) << 2) | 1; - run_bits[run] = 4; - } else { - /* 111xxxxxxx -> xxxxxxxx. */ - run_code[run] = (run << 3) | 7; - run_bits[run] = 10; - } + uint16_t run_code[134], level_code[266]; + uint8_t run_bits[134], level_bits[266]; + int16_t run_symbols[134], level_symbols[266]; + int entry, i, sign; + + /* Initialize VLC for alpha run. */ + entry = 0; + + /* 0 -> 0. */ + run_code[entry] = 0; + run_bits[entry] = 1; + run_symbols[entry] = 0; + ++entry; + + /* 10xx -> xx plus 1. */ + for (i = 0; i < 4; ++i) { + run_code[entry] = (i << 2) | 1; + run_bits[entry] = 4; + run_symbols[entry] = i + 1; + ++entry; + } + + /* 111xxxxxxx -> xxxxxxx. */ + for (i = 0; i < 128; ++i) { + run_code[entry] = (i << 3) | 7; + run_bits[entry] = 10; + run_symbols[entry] = i; + ++entry; } /* 110 -> EOB. */ - run_code[128] = 3; - run_bits[128] = 3; - - INIT_LE_VLC_STATIC(&ff_dc_alpha_run_vlc_le, ALPHA_VLC_BITS, 129, - run_bits, 1, 1, - run_code, 2, 2, 160); - - for (level = 0; level < 256; level++) { - int8_t signed_level = (int8_t)level; - int abs_signed_level = abs(signed_level); - int sign = (signed_level < 0) ? 1 : 0; - - if (abs_signed_level == 1) { - /* 1s -> -1 or +1 (depending on sign bit). */ - level_code[level] = (sign << 1) | 1; - level_bits[level] = 2; - } else if (abs_signed_level >= 2 && abs_signed_level <= 5) { - /* 01sxx -> xx plus 2 (2..5 or -2..-5, depending on sign bit). */ - level_code[level] = ((abs_signed_level - 2) << 3) | (sign << 2) | 2; - level_bits[level] = 5; - } else { - /* - * 00xxxxxxxx -> xxxxxxxx, in two's complement. 0 is technically an - * illegal code (that would be encoded by increasing run), but it - * doesn't hurt and simplifies indexing. - */ - level_code[level] = level << 2; - level_bits[level] = 10; + run_code[entry] = 3; + run_bits[entry] = 3; + run_symbols[entry] = -1; + ++entry; + + av_assert0(entry == FF_ARRAY_ELEMS(run_code)); + + INIT_LE_VLC_SPARSE_STATIC(&ff_dc_alpha_run_vlc_le, ALPHA_VLC_BITS, + FF_ARRAY_ELEMS(run_code), + run_bits, 1, 1, + run_code, 2, 2, + run_symbols, 2, 2, 160); + + /* Initialize VLC for alpha level. */ + entry = 0; + + for (sign = 0; sign <= 1; ++sign) { + /* 1s -> -1 or +1 (depending on sign bit). */ + level_code[entry] = (sign << 1) | 1; + level_bits[entry] = 2; + level_symbols[entry] = sign ? -1 : 1; + ++entry; + + /* 01sxx -> xx plus 2 (2..5 or -2..-5, depending on sign bit). */ + for (i = 0; i < 4; ++i) { + level_code[entry] = (i << 3) | (sign << 2) | 2; + level_bits[entry] = 5; + level_symbols[entry] = sign ? -(i + 2) : (i + 2); + ++entry; } } - INIT_LE_VLC_STATIC(&ff_dc_alpha_level_vlc_le, ALPHA_VLC_BITS, 256, - level_bits, 1, 1, - level_code, 2, 2, 288); + /* + * 00xxxxxxxx -> xxxxxxxx, in two's complement. There are many codes + * here that would better be encoded in other ways (e.g. 0 would be + * encoded by increasing run, and +/- 1 would be encoded with a + * shorter code), but it doesn't hurt to allow everything. + */ + for (i = 0; i < 256; ++i) { + level_code[entry] = i << 2; + level_bits[entry] = 10; + level_symbols[entry] = i; + ++entry; + } + + av_assert0(entry == FF_ARRAY_ELEMS(level_code)); + + INIT_LE_VLC_SPARSE_STATIC(&ff_dc_alpha_level_vlc_le, ALPHA_VLC_BITS, + FF_ARRAY_ELEMS(level_code), + level_bits, 1, 1, + level_code, 2, 2, + level_symbols, 2, 2, 288); } static uint32_t reverse(uint32_t num, int bits) -- cgit v1.2.3