From ee26abf2a4884bb56959bac8215758195776c553 Mon Sep 17 00:00:00 2001 From: Vitor Sessak Date: Sun, 20 Mar 2011 18:30:29 +0100 Subject: Fix an infinite loop when RoQ encoded generated a frame with a size greater than the maximum valid size. Signed-off-by: Ronald S. Bultje --- libavcodec/roqvideoenc.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'libavcodec/roqvideoenc.c') diff --git a/libavcodec/roqvideoenc.c b/libavcodec/roqvideoenc.c index f4b5de0adf..052dcef3de 100644 --- a/libavcodec/roqvideoenc.c +++ b/libavcodec/roqvideoenc.c @@ -898,9 +898,20 @@ static void roq_encode_video(RoqContext *enc) for (i=0; iwidth*enc->height/64; i++) gather_data_for_cel(tempData->cel_evals + i, enc, tempData); - /* Quake 3 can't handle chunks bigger than 65536 bytes */ - if (tempData->mainChunkSize/8 > 65536) { - enc->lambda *= .8; + /* Quake 3 can't handle chunks bigger than 65535 bytes */ + if (tempData->mainChunkSize/8 > 65535) { + av_log(enc->avctx, AV_LOG_ERROR, + "Warning, generated a frame too big (%d > 65535), " + "try using a smaller qscale value.\n", + tempData->mainChunkSize/8); + enc->lambda *= 1.5; + tempData->mainChunkSize = 0; + memset(tempData->used_option, 0, sizeof(tempData->used_option)); + memset(tempData->codebooks.usedCB4, 0, + sizeof(tempData->codebooks.usedCB4)); + memset(tempData->codebooks.usedCB2, 0, + sizeof(tempData->codebooks.usedCB2)); + goto retry_encode; } -- cgit v1.2.3