nvenc: fix encoding with B-frames

When B-frames are enabled and the encoder returns success, all currently
pending buffers immediately become valid and can be returned to the
caller. We can only return one packet at a time, so all the other
pending buffers should be transferred to a new 'ready' fifo, from where
they can be returned in subsequent calls (in which the encoder does not
produce any new output). This bug was hidden by the incorrect testing of
the encoder return value (the return value was overwritten before it was
tested).

1 files changed, 15 insertions, 8 deletions

diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c
index c151ad5055..e23ec8868e 100644
--- a/libavcodec/nvenc.c
+++ b/libavcodec/nvenc.c
@@ -1067,7 +1067,7 @@ static int nvenc_get_frame(AVCodecContext *avctx, AVPacket *pkt)
     NVENCOutputSurface *out         = NULL;
     int ret;
 
-    ret = nvenc_dequeue_surface(ctx->pending, &out);
+    ret = nvenc_dequeue_surface(ctx->ready, &out);
     if (ret)
         return ret;
 
@@ -1127,7 +1127,7 @@ int ff_nvenc_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     NV_ENC_PIC_PARAMS params        = { 0 };
     NVENCInputSurface *in           = NULL;
     NVENCOutputSurface *out         = NULL;
-    int ret;
+    int enc_ret, ret;
 
     params.version = NV_ENC_PIC_PARAMS_VER;
 
@@ -1166,10 +1166,10 @@ int ff_nvenc_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
         params.encodePicFlags = NV_ENC_PIC_FLAG_EOS;
     }
 
-    ret = nv->nvEncEncodePicture(ctx->nvenc_ctx, &params);
-    if (ret != NV_ENC_SUCCESS &&
-        ret != NV_ENC_ERR_NEED_MORE_INPUT)
-        return nvenc_print_error(avctx, ret, "Error encoding the frame");
+    enc_ret = nv->nvEncEncodePicture(ctx->nvenc_ctx, &params);
+    if (enc_ret != NV_ENC_SUCCESS &&
+        enc_ret != NV_ENC_ERR_NEED_MORE_INPUT)
+        return nvenc_print_error(avctx, enc_ret, "Error encoding the frame");
 
     if (out) {
         ret = nvenc_enqueue_surface(ctx->pending, out);
@@ -1177,8 +1177,15 @@ int ff_nvenc_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
             return ret;
     }
 
-    if (ret != NV_ENC_ERR_NEED_MORE_INPUT &&
-        av_fifo_size(ctx->pending)) {
+    /* all the pending buffers are now ready for output */
+    if (enc_ret == NV_ENC_SUCCESS) {
+        while (av_fifo_size(ctx->pending) > 0) {
+            av_fifo_generic_read(ctx->pending, &out, sizeof(out), NULL);
+            av_fifo_generic_write(ctx->ready, &out, sizeof(out), NULL);
+        }
+    }
+
+    if (av_fifo_size(ctx->ready) > 0) {
         ret = nvenc_get_frame(avctx, pkt);
         if (ret < 0)
             return ret;