From 8a442d7a8a687a469ca502a18a0c68f5302b15e0 Mon Sep 17 00:00:00 2001 From: Mark Thompson Date: Thu, 6 Jul 2017 22:50:35 +0100 Subject: pixdesc: Improve scoring for opaque/unknown pixel formats Hardware pixel formats do not tell you anything about their actual contents, but should still score higher than formats with completely unknown properties, which in turn should score higher than invalid formats. Do not return an AVERROR code as a score. Fixes a hang in libavfilter where format negotiation gets stuck in a loop because AV_PIX_FMT_NONE scores more highly than all other possibilities. --- libavutil/pixdesc.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'libavutil/pixdesc.c') diff --git a/libavutil/pixdesc.c b/libavutil/pixdesc.c index 46a7eff06d..1983ce9ef5 100644 --- a/libavutil/pixdesc.c +++ b/libavutil/pixdesc.c @@ -2511,8 +2511,16 @@ static int get_pix_fmt_score(enum AVPixelFormat dst_pix_fmt, int ret, loss, i, nb_components; int score = INT_MAX - 1; - if (dst_pix_fmt >= AV_PIX_FMT_NB || dst_pix_fmt <= AV_PIX_FMT_NONE) - return ~0; + if (!src_desc || !dst_desc) + return -4; + + if ((src_desc->flags & AV_PIX_FMT_FLAG_HWACCEL) || + (dst_desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) { + if (dst_pix_fmt == src_pix_fmt) + return -1; + else + return -2; + } /* compute loss */ *lossp = loss = 0; @@ -2521,9 +2529,9 @@ static int get_pix_fmt_score(enum AVPixelFormat dst_pix_fmt, return INT_MAX; if ((ret = get_pix_fmt_depth(&src_min_depth, &src_max_depth, src_pix_fmt)) < 0) - return ret; + return -3; if ((ret = get_pix_fmt_depth(&dst_min_depth, &dst_max_depth, dst_pix_fmt)) < 0) - return ret; + return -3; src_color = get_color_type(src_desc); dst_color = get_color_type(dst_desc); -- cgit v1.2.3