avfilter/vf_vpp_qsv: set outlink to EOF correctly

1. Return error if filter frame fail before set outlink to EOF in none
pass through mode.
2. Set outlink to EOF before return success in pass through mode.

Fix endless cmd:

ffmpeg -hwaccel qsv -qsv_device /dev/dri/renderD128 -hwaccel_output_format     \
qsv -v debug -c:v hevc_qsv -i 4k.h265                                          \
-filter_complex "vpp_qsv=w=3840:h=2160:async_depth=4[o1];[o1]split=2[s1][s2];
[s2]vpp_qsv=w=1920:h=1080:async_depth=4[o2];[o2]split=2[s3][s4];
[s4]vpp_qsv=w=1920:h=1080:async_depth=4[o3]" \
-map [s1] -c:v hevc_qsv -async 3 -async_depth 3 -b:v 9000k -preset 7 -g 33 -y -f null - \
-map [s3] -c:v hevc_qsv -async 3 -async_depth 3 -b:v 4000k -preset 7 -g 33 -y -f null - \
-map [o3] -c:v hevc_qsv -async 3 -async_depth 3 -b:v 3100k -preset 7 -g 33 -y -f null -

1 files changed, 23 insertions, 12 deletions

diff --git a/libavfilter/vf_vpp_qsv.c b/libavfilter/vf_vpp_qsv.c
index 6e5056d133..4a053f9145 100644
--- a/libavfilter/vf_vpp_qsv.c
+++ b/libavfilter/vf_vpp_qsv.c
@@ -547,15 +547,17 @@ static int activate(AVFilterContext *ctx)
             qsv->eof = s->eof;
             ret = ff_qsvvpp_filter_frame(qsv, inlink, in);
             av_frame_free(&in);
+            if (ret == AVERROR(EAGAIN))
+                goto not_ready;
+            else if (ret < 0)
+                return ret;
 
-            if (s->eof) {
-                ff_outlink_set_status(outlink, status, pts);
-                return 0;
-            }
+            if (s->eof)
+                goto eof;
 
             if (qsv->got_frame) {
                 qsv->got_frame = 0;
-                return ret;
+                return 0;
             }
         }
     } else {
@@ -564,18 +566,27 @@ static int activate(AVFilterContext *ctx)
                 in->pts = av_rescale_q(in->pts, inlink->time_base, outlink->time_base);
 
             ret = ff_filter_frame(outlink, in);
-            return ret;
+            if (ret < 0)
+                return ret;
+
+            if (s->eof)
+                goto eof;
+
+            return 0;
         }
     }
 
-    if (s->eof) {
-        ff_outlink_set_status(outlink, status, pts);
-        return 0;
-    } else {
-        FF_FILTER_FORWARD_WANTED(outlink, inlink);
-    }
+not_ready:
+    if (s->eof)
+        goto eof;
+
+    FF_FILTER_FORWARD_WANTED(outlink, inlink);
 
     return FFERROR_NOT_READY;
+
+eof:
+    ff_outlink_set_status(outlink, status, pts);
+    return 0;
 }
 
 static int query_formats(AVFilterContext *ctx)