From b97d61f924eef1a8930f80cd43d0733c12f67f35 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Tue, 19 Feb 2013 15:00:01 +0100 Subject: avfilter/ff_merge_formats: only merge if doing so does not loose chroma or alpha Fixes Ticket1280 Signed-off-by: Michael Niedermayer --- libavfilter/formats.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'libavfilter/formats.c') diff --git a/libavfilter/formats.c b/libavfilter/formats.c index 0284db2119..eb9d94022c 100644 --- a/libavfilter/formats.c +++ b/libavfilter/formats.c @@ -92,10 +92,29 @@ do { AVFilterFormats *ff_merge_formats(AVFilterFormats *a, AVFilterFormats *b) { AVFilterFormats *ret = NULL; + int i, j; + int alpha1=0, alpha2=0; + int chroma1=0, chroma2=0; if (a == b) return a; + for (i = 0; i < a->format_count; i++) + for (j = 0; j < b->format_count; j++) { + const AVPixFmtDescriptor *adesc = av_pix_fmt_desc_get(a->formats[i]); + const AVPixFmtDescriptor *bdesc = av_pix_fmt_desc_get(b->formats[j]); + alpha2 |= adesc->flags & bdesc->flags & PIX_FMT_ALPHA; + chroma2|= adesc->nb_components > 1 && bdesc->nb_components > 1; + if (a->formats[i] == b->formats[j]) { + alpha1 |= adesc->flags & PIX_FMT_ALPHA; + chroma1|= adesc->nb_components > 1; + } + } + + // If chroma or alpha can be lost through merging then do not merge + if (alpha2 > alpha1 || chroma2 > chroma1) + return NULL; + MERGE_FORMATS(ret, a, b, formats, format_count, AVFilterFormats, fail); return ret; -- cgit v1.2.3