diff options
| author | Michael Niedermayer <michael@niedermayer.cc> | 2021-11-14 18:09:17 +0100 |
|---|---|---|
| committer | Michael Niedermayer <michael@niedermayer.cc> | 2021-11-26 21:31:06 +0100 |
| commit | ff23cee200239a8368515c5d8ba94fc769c3fea2 (patch) | |
| tree | f62663e6fe070cfbbfa7a23259910b16f6edbc0c | |
| parent | 7c2b9067d065a1ed503aa76ea82e5dd1763452dd (diff) | |
avcodec/txd: Move input size checks before allocation
Fixes: Timeout
Fixes: 40882/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_TXD_fuzzer-4893364584054784
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
| -rw-r--r-- | libavcodec/txd.c | 30 |
1 files changed, 19 insertions, 11 deletions
diff --git a/libavcodec/txd.c b/libavcodec/txd.c index 86d60f3377..93e4fa9605 100644 --- a/libavcodec/txd.c +++ b/libavcodec/txd.c @@ -65,8 +65,26 @@ static int txd_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, if (depth == 8) { avctx->pix_fmt = AV_PIX_FMT_PAL8; - } else if (depth == 16 || depth == 32) { + if (bytestream2_get_bytes_left(&gb) < w * h + 4 * 256) + return AVERROR_INVALIDDATA; + } else if (depth == 16) { avctx->pix_fmt = AV_PIX_FMT_RGBA; + switch (d3d_format) { + case 0: + if (!(flags & 1)) + goto unsupported; + case TXD_DXT1: + if (bytestream2_get_bytes_left(&gb) < AV_CEIL_RSHIFT(w, 2) * AV_CEIL_RSHIFT(h, 2) * 8 + 4) + return AVERROR_INVALIDDATA; + break; + case TXD_DXT3: + if (bytestream2_get_bytes_left(&gb) < AV_CEIL_RSHIFT(w, 2) * AV_CEIL_RSHIFT(h, 2) * 16 + 4) + return AVERROR_INVALIDDATA; + } + } else if (depth == 32) { + avctx->pix_fmt = AV_PIX_FMT_RGBA; + if (bytestream2_get_bytes_left(&gb) < h * w * 4) + return AVERROR_INVALIDDATA; } else { avpriv_report_missing_feature(avctx, "Color depth of %u", depth); return AVERROR_PATCHWELCOME; @@ -92,8 +110,6 @@ static int txd_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, v = bytestream2_get_be32(&gb); pal[y] = (v >> 8) + (v << 24); } - if (bytestream2_get_bytes_left(&gb) < w * h) - return AVERROR_INVALIDDATA; bytestream2_skip(&gb, 4); for (y=0; y<h; y++) { bytestream2_get_buffer(&gb, ptr, w); @@ -103,11 +119,7 @@ static int txd_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, bytestream2_skip(&gb, 4); switch (d3d_format) { case 0: - if (!(flags & 1)) - goto unsupported; case TXD_DXT1: - if (bytestream2_get_bytes_left(&gb) < AV_CEIL_RSHIFT(w, 2) * AV_CEIL_RSHIFT(h, 2) * 8) - return AVERROR_INVALIDDATA; for (j = 0; j < avctx->height; j += 4) { for (i = 0; i < avctx->width; i += 4) { uint8_t *p = ptr + i * 4 + j * stride; @@ -117,8 +129,6 @@ static int txd_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, } break; case TXD_DXT3: - if (bytestream2_get_bytes_left(&gb) < AV_CEIL_RSHIFT(w, 2) * AV_CEIL_RSHIFT(h, 2) * 16) - return AVERROR_INVALIDDATA; for (j = 0; j < avctx->height; j += 4) { for (i = 0; i < avctx->width; i += 4) { uint8_t *p = ptr + i * 4 + j * stride; @@ -134,8 +144,6 @@ static int txd_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, switch (d3d_format) { case 0x15: case 0x16: - if (bytestream2_get_bytes_left(&gb) < h * w * 4) - return AVERROR_INVALIDDATA; for (y=0; y<h; y++) { bytestream2_get_buffer(&gb, ptr, w * 4); ptr += stride; |