From f1901180e02a766edbc74b8fb8cfbb88b79ef347 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Thu, 29 Sep 2011 16:44:50 -0400 Subject: pcmdec: fix output buffer size check by calculating the actual output size prior to decoding. --- libavcodec/pcm.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'libavcodec/pcm.c') diff --git a/libavcodec/pcm.c b/libavcodec/pcm.c index 8d9399265e..c9eb543e92 100644 --- a/libavcodec/pcm.c +++ b/libavcodec/pcm.c @@ -250,7 +250,7 @@ static int pcm_decode_frame(AVCodecContext *avctx, const uint8_t *src = avpkt->data; int buf_size = avpkt->size; PCMDecode *s = avctx->priv_data; - int sample_size, c, n; + int sample_size, c, n, out_size; uint8_t *samples; int32_t *dst_int32_t; @@ -286,10 +286,17 @@ static int pcm_decode_frame(AVCodecContext *avctx, buf_size -= buf_size % n; } - buf_size= FFMIN(buf_size, *data_size/2); - n = buf_size/sample_size; + out_size = n * av_get_bytes_per_sample(avctx->sample_fmt); + if (avctx->codec_id == CODEC_ID_PCM_DVD || + avctx->codec_id == CODEC_ID_PCM_LXF) + out_size *= 2; + if (*data_size < out_size) { + av_log(avctx, AV_LOG_ERROR, "output buffer too small\n"); + return AVERROR(EINVAL); + } + switch(avctx->codec->id) { case CODEC_ID_PCM_U32LE: DECODE(32, le32, src, samples, n, 0, 0x80000000) @@ -450,7 +457,7 @@ static int pcm_decode_frame(AVCodecContext *avctx, default: return -1; } - *data_size = samples - (uint8_t *)data; + *data_size = out_size; return buf_size; } -- cgit v1.2.3