diff options
Diffstat (limited to 'libavcodec/vmdav.c')
-rw-r--r-- | libavcodec/vmdav.c | 107 |
1 files changed, 56 insertions, 51 deletions
diff --git a/libavcodec/vmdav.c b/libavcodec/vmdav.c index e6c75868e5..3e376e301c 100644 --- a/libavcodec/vmdav.c +++ b/libavcodec/vmdav.c @@ -2,20 +2,20 @@ * Sierra VMD Audio & Video Decoders * Copyright (C) 2004 the ffmpeg project * - * This file is part of Libav. + * This file is part of FFmpeg. * - * Libav is free software; you can redistribute it and/or + * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * - * Libav is distributed in the hope that it will be useful, + * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public - * License along with Libav; if not, write to the Free Software + * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ @@ -98,7 +98,7 @@ static void lz_unpack(const unsigned char *src, int src_len, if (bytestream2_get_bytes_left(&gb) < 4) return; if (bytestream2_peek_le32(&gb) == 0x56781234) { - bytestream2_get_le32(&gb); + bytestream2_skipu(&gb, 4); qpos = 0x111; speclen = 0xF + 3; } else { @@ -109,7 +109,7 @@ static void lz_unpack(const unsigned char *src, int src_len, while (dataleft > 0 && bytestream2_get_bytes_left(&gb) > 0) { tag = bytestream2_get_byteu(&gb); if ((tag == 0xFF) && (dataleft > 8)) { - if (d + 8 > d_end || bytestream2_get_bytes_left(&gb) < 8) + if (d_end - d < 8 || bytestream2_get_bytes_left(&gb) < 8) return; for (i = 0; i < 8; i++) { queue[qpos++] = *d++ = bytestream2_get_byteu(&gb); @@ -121,9 +121,9 @@ static void lz_unpack(const unsigned char *src, int src_len, if (dataleft == 0) break; if (tag & 0x01) { - if (d + 1 > d_end || bytestream2_get_bytes_left(&gb) < 1) + if (d_end - d < 1 || bytestream2_get_bytes_left(&gb) < 1) return; - queue[qpos++] = *d++ = bytestream2_get_byte(&gb); + queue[qpos++] = *d++ = bytestream2_get_byteu(&gb); qpos &= QUEUE_MASK; dataleft--; } else { @@ -133,7 +133,7 @@ static void lz_unpack(const unsigned char *src, int src_len, if (chainlen == speclen) { chainlen = bytestream2_get_byte(&gb) + 0xF + 3; } - if (d + chainlen > d_end) + if (d_end - d < chainlen) return; for (j = 0; j < chainlen; j++) { *d = queue[chainofs++ & QUEUE_MASK]; @@ -147,12 +147,11 @@ static void lz_unpack(const unsigned char *src, int src_len, } } } - static int rle_unpack(const unsigned char *src, unsigned char *dest, - int src_count, int src_size, int dest_len) + int src_count, int src_size, int dest_len) { unsigned char *pd; - int i, l; + int i, j, l; unsigned char *dest_end = dest + dest_len; GetByteContext gb; @@ -172,18 +171,20 @@ static int rle_unpack(const unsigned char *src, unsigned char *dest, l = bytestream2_get_byteu(&gb); if (l & 0x80) { l = (l & 0x7F) * 2; - if (pd + l > dest_end || bytestream2_get_bytes_left(&gb) < l) + if (dest_end - pd < l || bytestream2_get_bytes_left(&gb) < l) return bytestream2_tell(&gb); - bytestream2_get_buffer(&gb, pd, l); + bytestream2_get_bufferu(&gb, pd, l); pd += l; } else { - if (pd + i > dest_end || bytestream2_get_bytes_left(&gb) < 2) + int ps[2]; + if (dest_end - pd < 2*l || bytestream2_get_bytes_left(&gb) < 2) return bytestream2_tell(&gb); - for (i = 0; i < l; i++) { - *pd++ = bytestream2_get_byteu(&gb); - *pd++ = bytestream2_get_byteu(&gb); + ps[0] = bytestream2_get_byteu(&gb); + ps[1] = bytestream2_get_byteu(&gb); + for (j = 0; j < l; j++) { + *pd++ = ps[0]; + *pd++ = ps[1]; } - bytestream2_skip(&gb, 2); } i += l; } while (i < src_count); @@ -191,7 +192,7 @@ static int rle_unpack(const unsigned char *src, unsigned char *dest, return bytestream2_tell(&gb); } -static void vmd_decode(VmdVideoContext *s, AVFrame *frame) +static int vmd_decode(VmdVideoContext *s, AVFrame *frame) { int i; unsigned int *palette32; @@ -212,16 +213,6 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame) frame_y = AV_RL16(&s->buf[8]); frame_width = AV_RL16(&s->buf[10]) - frame_x + 1; frame_height = AV_RL16(&s->buf[12]) - frame_y + 1; - if (frame_x < 0 || frame_width < 0 || - frame_x >= s->avctx->width || - frame_width > s->avctx->width || - frame_x + frame_width > s->avctx->width) - return; - if (frame_y < 0 || frame_height < 0 || - frame_y >= s->avctx->height || - frame_height > s->avctx->height || - frame_y + frame_height > s->avctx->height) - return; if ((frame_width == s->avctx->width && frame_height == s->avctx->height) && (frame_x || frame_y)) { @@ -232,6 +223,19 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame) frame_x -= s->x_off; frame_y -= s->y_off; + if (frame_x < 0 || frame_width < 0 || + frame_x >= s->avctx->width || + frame_width > s->avctx->width || + frame_x + frame_width > s->avctx->width) { + return AVERROR_INVALIDDATA; + } + if (frame_y < 0 || frame_height < 0 || + frame_y >= s->avctx->height || + frame_height > s->avctx->height || + frame_y + frame_height > s->avctx->height) { + return AVERROR_INVALIDDATA; + } + /* if only a certain region will be updated, copy the entire previous * frame before the decode */ if (s->prev_frame.data[0] && @@ -252,16 +256,16 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame) r = bytestream2_get_byteu(&gb) * 4; g = bytestream2_get_byteu(&gb) * 4; b = bytestream2_get_byteu(&gb) * 4; - palette32[i] = (r << 16) | (g << 8) | (b); + palette32[i] = 0xFFU << 24 | (r << 16) | (g << 8) | (b); + palette32[i] |= palette32[i] >> 6 & 0x30303; } } - s->size -= (256 * 3 + 2); } if (s->size > 0) { /* originally UnpackFrame in VAG's code */ bytestream2_init(&gb, gb.buffer, s->buf + s->size - gb.buffer); if (bytestream2_get_bytes_left(&gb) < 1) - return; + return AVERROR_INVALIDDATA; meth = bytestream2_get_byteu(&gb); if (meth & 0x80) { lz_unpack(gb.buffer, bytestream2_get_bytes_left(&gb), @@ -281,19 +285,19 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame) if (len & 0x80) { len = (len & 0x7F) + 1; if (ofs + len > frame_width || bytestream2_get_bytes_left(&gb) < len) - return; - bytestream2_get_buffer(&gb, &dp[ofs], len); + return AVERROR_INVALIDDATA; + bytestream2_get_bufferu(&gb, &dp[ofs], len); ofs += len; } else { /* interframe pixel copy */ if (ofs + len + 1 > frame_width || !s->prev_frame.data[0]) - return; + return AVERROR_INVALIDDATA; memcpy(&dp[ofs], &pp[ofs], len + 1); ofs += len + 1; } } while (ofs < frame_width); if (ofs > frame_width) { - av_log(s->avctx, AV_LOG_ERROR, "VMD video: offset > width (%d > %d)\n", + av_log(s->avctx, AV_LOG_ERROR, "offset > width (%d > %d)\n", ofs, frame_width); break; } @@ -327,13 +331,13 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame) } else { /* interframe pixel copy */ if (ofs + len + 1 > frame_width || !s->prev_frame.data[0]) - return; + return AVERROR_INVALIDDATA; memcpy(&dp[ofs], &pp[ofs], len + 1); ofs += len + 1; } } while (ofs < frame_width); if (ofs > frame_width) { - av_log(s->avctx, AV_LOG_ERROR, "VMD video: offset > width (%d > %d)\n", + av_log(s->avctx, AV_LOG_ERROR, "offset > width (%d > %d)\n", ofs, frame_width); } dp += frame->linesize[0]; @@ -342,6 +346,8 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame) break; } } + + return 0; } static av_cold int vmdvideo_decode_init(AVCodecContext *avctx) @@ -359,16 +365,16 @@ static av_cold int vmdvideo_decode_init(AVCodecContext *avctx) /* make sure the VMD header made it */ if (s->avctx->extradata_size != VMD_HEADER_SIZE) { - av_log(s->avctx, AV_LOG_ERROR, "VMD video: expected extradata size of %d\n", + av_log(s->avctx, AV_LOG_ERROR, "expected extradata size of %d\n", VMD_HEADER_SIZE); - return -1; + return AVERROR_INVALIDDATA; } vmd_header = (unsigned char *)avctx->extradata; s->unpack_buffer_size = AV_RL32(&vmd_header[800]); s->unpack_buffer = av_malloc(s->unpack_buffer_size); if (!s->unpack_buffer) - return -1; + return AVERROR(ENOMEM); /* load up the initial palette */ raw_palette = &vmd_header[28]; @@ -380,6 +386,8 @@ static av_cold int vmdvideo_decode_init(AVCodecContext *avctx) palette32[i] = (r << 16) | (g << 8) | (b); } + avcodec_get_frame_defaults(&s->prev_frame); + return 0; } @@ -399,12 +407,11 @@ static int vmdvideo_decode_frame(AVCodecContext *avctx, if (buf_size < 16) return buf_size; - if ((ret = ff_get_buffer(avctx, frame, AV_GET_BUFFER_FLAG_REF)) < 0) { - av_log(s->avctx, AV_LOG_ERROR, "VMD Video: get_buffer() failed\n"); + if ((ret = ff_get_buffer(avctx, frame, AV_GET_BUFFER_FLAG_REF)) < 0) return ret; - } - vmd_decode(s, frame); + if (vmd_decode(s, frame) < 0) + av_log(avctx, AV_LOG_WARNING, "decode error\n"); /* make the palette available on the way out */ memcpy(frame->data[1], s->palette, PALETTE_COUNT * 4); @@ -468,7 +475,7 @@ static av_cold int vmdaudio_decode_init(AVCodecContext *avctx) av_log(avctx, AV_LOG_ERROR, "invalid number of channels\n"); return AVERROR(EINVAL); } - if (avctx->block_align < 1) { + if (avctx->block_align < 1 || avctx->block_align % avctx->channels) { av_log(avctx, AV_LOG_ERROR, "invalid block align\n"); return AVERROR(EINVAL); } @@ -571,10 +578,8 @@ static int vmdaudio_decode_frame(AVCodecContext *avctx, void *data, /* get output buffer */ frame->nb_samples = ((silent_chunks + audio_chunks) * avctx->block_align) / avctx->channels; - if ((ret = ff_get_buffer(avctx, frame, 0)) < 0) { - av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n"); + if ((ret = ff_get_buffer(avctx, frame, 0)) < 0) return ret; - } output_samples_u8 = frame->data[0]; output_samples_s16 = (int16_t *)frame->data[0]; @@ -593,7 +598,7 @@ static int vmdaudio_decode_frame(AVCodecContext *avctx, void *data, /* decode audio chunks */ if (audio_chunks > 0) { buf_end = buf + buf_size; - while (buf + s->chunk_size <= buf_end) { + while (buf_end - buf >= s->chunk_size) { if (s->out_bps == 2) { decode_audio_s16(output_samples_s16, buf, s->chunk_size, avctx->channels); |