From ec2b76aab44f55be22eb12d86eb0dfd2eff68581 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 21 May 2017 13:22:16 +0200 Subject: avcodec/vmnc: Check location before use Fixes: runtime error: signed integer overflow: 65535 * 64256 cannot be represented in type 'int' Fixes: 1717/clusterfuzz-testcase-minimized-5491696676634624 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer --- libavcodec/vmnc.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) (limited to 'libavcodec/vmnc.c') diff --git a/libavcodec/vmnc.c b/libavcodec/vmnc.c index a756dab211..30b1414e49 100644 --- a/libavcodec/vmnc.c +++ b/libavcodec/vmnc.c @@ -381,6 +381,12 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, w = bytestream2_get_be16(gb); h = bytestream2_get_be16(gb); enc = bytestream2_get_be32(gb); + if ((dx + w > c->width) || (dy + h > c->height)) { + av_log(avctx, AV_LOG_ERROR, + "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", + w, h, dx, dy, c->width, c->height); + return AVERROR_INVALIDDATA; + } outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0]; size_left = bytestream2_get_bytes_left(gb); switch (enc) { @@ -458,12 +464,6 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, bytestream2_skip(gb, 2); break; case 0x00000000: // raw rectangle data - if ((dx + w > c->width) || (dy + h > c->height)) { - av_log(avctx, AV_LOG_ERROR, - "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", - w, h, dx, dy, c->width, c->height); - return AVERROR_INVALIDDATA; - } if (size_left < w * h * c->bpp2) { av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", @@ -474,12 +474,6 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, c->pic->linesize[0]); break; case 0x00000005: // HexTile encoded rectangle - if ((dx + w > c->width) || (dy + h > c->height)) { - av_log(avctx, AV_LOG_ERROR, - "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", - w, h, dx, dy, c->width, c->height); - return AVERROR_INVALIDDATA; - } res = decode_hextile(c, outptr, gb, w, h, c->pic->linesize[0]); if (res < 0) return res; -- cgit v1.2.3