From 3a2731cbd31d0c5681ddbc7c78edd5c53c4d0032 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Wed, 7 Nov 2012 14:48:28 -0500 Subject: flacenc: ensure the order is within the min/max range in LPC order search This fixes use of uninitialized values when the FLAC encoder uses the 2-level, 4-level, and 8-level search methods. Fixes failure of the fate-flac-24-comp-8 test when run using valgrind. --- libavcodec/flacenc.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'libavcodec/flacenc.c') diff --git a/libavcodec/flacenc.c b/libavcodec/flacenc.c index 93d4646604..54bc64c9de 100644 --- a/libavcodec/flacenc.c +++ b/libavcodec/flacenc.c @@ -799,14 +799,16 @@ static int encode_residual_ch(FlacEncodeContext *s, int ch) omethod == ORDER_METHOD_8LEVEL) { int levels = 1 << omethod; uint64_t bits[1 << ORDER_METHOD_8LEVEL]; - int order; + int order = -1; int opt_index = levels-1; opt_order = max_order-1; bits[opt_index] = UINT32_MAX; for (i = levels-1; i >= 0; i--) { + int last_order = order; order = min_order + (((max_order-min_order+1) * (i+1)) / levels)-1; - if (order < 0) - order = 0; + order = av_clip(order, min_order - 1, max_order - 1); + if (order == last_order) + continue; s->flac_dsp.lpc_encode(res, smp, n, order+1, coefs[order], shift[order]); bits[i] = find_subframe_rice_params(s, sub, order+1); -- cgit v1.2.3