From fd41cb43702498948ff14ba8c284fd5c15fc729d Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Sat, 25 Aug 2012 16:32:12 -0400 Subject: avconv: improve sample format negotiation for decoder request This takes into account planar vs. packed when comparing to find the closest sample format match. --- avconv.c | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) (limited to 'avconv.c') diff --git a/avconv.c b/avconv.c index 0b703edd3c..2f30ebc8c1 100644 --- a/avconv.c +++ b/avconv.c @@ -269,22 +269,41 @@ static void update_sample_fmt(AVCodecContext *dec, AVCodec *dec_codec, dec_codec->sample_fmts[0] != AV_SAMPLE_FMT_NONE && dec_codec->sample_fmts[1] != AV_SAMPLE_FMT_NONE) { const enum AVSampleFormat *p; - int min_dec = -1, min_inc = -1; + int min_dec = INT_MAX, min_inc = INT_MAX; + enum AVSampleFormat dec_fmt = AV_SAMPLE_FMT_NONE; + enum AVSampleFormat inc_fmt = AV_SAMPLE_FMT_NONE; /* find a matching sample format in the encoder */ for (p = dec_codec->sample_fmts; *p != AV_SAMPLE_FMT_NONE; p++) { if (*p == enc->sample_fmt) { dec->request_sample_fmt = *p; return; - } else if (*p > enc->sample_fmt) { - min_inc = FFMIN(min_inc, *p - enc->sample_fmt); - } else - min_dec = FFMIN(min_dec, enc->sample_fmt - *p); + } else { + enum AVSampleFormat dfmt = av_get_packed_sample_fmt(*p); + enum AVSampleFormat efmt = av_get_packed_sample_fmt(enc->sample_fmt); + int fmt_diff = 32 * abs(dfmt - efmt); + if (av_sample_fmt_is_planar(*p) != + av_sample_fmt_is_planar(enc->sample_fmt)) + fmt_diff++; + if (dfmt == efmt) { + min_inc = fmt_diff; + inc_fmt = *p; + } else if (dfmt > efmt) { + if (fmt_diff < min_inc) { + min_inc = fmt_diff; + inc_fmt = *p; + } + } else { + if (fmt_diff < min_dec) { + min_dec = fmt_diff; + dec_fmt = *p; + } + } + } } /* if none match, provide the one that matches quality closest */ - dec->request_sample_fmt = min_inc > 0 ? enc->sample_fmt + min_inc : - enc->sample_fmt - min_dec; + dec->request_sample_fmt = min_inc != INT_MAX ? inc_fmt : dec_fmt; } } -- cgit v1.2.3