avfilter/ff_merge_formats: only merge if doing so does not loose chroma or alpha

Fixes Ticket1280

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 19 insertions, 0 deletions

diff --git a/libavfilter/formats.c b/libavfilter/formats.c
index 0284db2119..eb9d94022c 100644
--- a/libavfilter/formats.c
+++ b/libavfilter/formats.c
@@ -92,10 +92,29 @@ do {
 AVFilterFormats *ff_merge_formats(AVFilterFormats *a, AVFilterFormats *b)
 {
     AVFilterFormats *ret = NULL;
+    int i, j;
+    int alpha1=0, alpha2=0;
+    int chroma1=0, chroma2=0;
 
     if (a == b)
         return a;
 
+    for (i = 0; i < a->format_count; i++)
+        for (j = 0; j < b->format_count; j++) {
+            const AVPixFmtDescriptor *adesc = av_pix_fmt_desc_get(a->formats[i]);
+            const AVPixFmtDescriptor *bdesc = av_pix_fmt_desc_get(b->formats[j]);
+            alpha2 |= adesc->flags & bdesc->flags & PIX_FMT_ALPHA;
+            chroma2|= adesc->nb_components > 1 && bdesc->nb_components > 1;
+            if (a->formats[i] == b->formats[j]) {
+                alpha1 |= adesc->flags & PIX_FMT_ALPHA;
+                chroma1|= adesc->nb_components > 1;
+            }
+        }
+
+    // If chroma or alpha can be lost through merging then do not merge
+    if (alpha2 > alpha1 || chroma2 > chroma1)
+        return NULL;
+
     MERGE_FORMATS(ret, a, b, formats, format_count, AVFilterFormats, fail);
 
     return ret;