From 749fbfd0817528aa4f43339ce1a3d92f0a258a45 Mon Sep 17 00:00:00 2001 From: Donny Yang Date: Tue, 31 Mar 2015 12:37:22 +0000 Subject: png: Clear up the calculation of max packet size Signed-off-by: Michael Niedermayer --- libavcodec/pngenc.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'libavcodec/pngenc.c') diff --git a/libavcodec/pngenc.c b/libavcodec/pngenc.c index 8fff0f3356..c913cce221 100644 --- a/libavcodec/pngenc.c +++ b/libavcodec/pngenc.c @@ -307,12 +307,16 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt, row_size = (avctx->width * s->bits_per_pixel + 7) >> 3; enc_row_size = deflateBound(&s->zstream, row_size); - max_packet_size = avctx->height * (int64_t)(enc_row_size + - ((enc_row_size + IOBUF_SIZE - 1) / IOBUF_SIZE) * 12) - + FF_MIN_BUFFER_SIZE; + max_packet_size = + FF_MIN_BUFFER_SIZE + // headers + avctx->height * ( + enc_row_size + + 12 * (((int64_t)enc_row_size + IOBUF_SIZE - 1) / IOBUF_SIZE) // IDAT * ceil(enc_row_size / IOBUF_SIZE) + ); if (max_packet_size > INT_MAX) return AVERROR(ENOMEM); - if ((ret = ff_alloc_packet2(avctx, pkt, max_packet_size)) < 0) + ret = ff_alloc_packet2(avctx, pkt, max_packet_size); + if (ret < 0) return ret; s->bytestream_start = -- cgit v1.2.3