From 5e992a4682d2c09eed3839c6cacf70db3b65c2f4 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Wed, 9 Oct 2013 05:51:20 +0200 Subject: vmnc: Check the cursor dimensions And manage the reallocation failure path. Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org --- libavcodec/vmnc.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) (limited to 'libavcodec/vmnc.c') diff --git a/libavcodec/vmnc.c b/libavcodec/vmnc.c index 5b14877061..855bfac69c 100644 --- a/libavcodec/vmnc.c +++ b/libavcodec/vmnc.c @@ -301,6 +301,14 @@ static int decode_hextile(VmncContext *c, uint8_t* dst, GetByteContext *gb, return 0; } +static void reset_buffers(VmncContext *c) +{ + av_freep(&c->curbits); + av_freep(&c->curmask); + av_freep(&c->screendta); + c->cur_w = c->cur_h = 0; +} + static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, AVPacket *avpkt) { @@ -379,9 +387,18 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, c->cur_hx, c->cur_hy, c->cur_w, c->cur_h); c->cur_hx = c->cur_hy = 0; } - c->curbits = av_realloc(c->curbits, c->cur_w * c->cur_h * c->bpp2); - c->curmask = av_realloc(c->curmask, c->cur_w * c->cur_h * c->bpp2); - c->screendta = av_realloc(c->screendta, c->cur_w * c->cur_h * c->bpp2); + if (c->cur_w * c->cur_h >= INT_MAX / c->bpp2) { + reset_buffers(c); + return AVERROR(EINVAL); + } else { + int screen_size = c->cur_w * c->cur_h * c->bpp2; + if ((ret = av_reallocp(&c->curbits, screen_size)) < 0 || + (ret = av_reallocp(&c->curmask, screen_size)) < 0 || + (ret = av_reallocp(&c->screendta, screen_size)) < 0) { + reset_buffers(c); + return ret; + } + } load_cursor(c); break; case MAGIC_WMVe: // unknown -- cgit v1.2.3