nvdec: attach real hw_frames to post-processed frames

1 files changed, 75 insertions, 17 deletions

diff --git a/libavcodec/nvdec.c b/libavcodec/nvdec.c
index b60da24301..6168a7e3f1 100644
--- a/libavcodec/nvdec.c
+++ b/libavcodec/nvdec.c
@@ -44,6 +44,7 @@ typedef struct NVDECDecoder {
     CUvideodecoder decoder;
 
     AVBufferRef *hw_device_ref;
+    AVBufferRef *real_hw_frames_ref;
     CUcontext    cuda_ctx;
     CUstream     stream;
 
@@ -163,6 +164,7 @@ static void nvdec_decoder_free(void *opaque, uint8_t *data)
         CHECK_CU(decoder->cudl->cuCtxPopCurrent(&dummy));
     }
 
+    av_buffer_unref(&decoder->real_hw_frames_ref);
     av_buffer_unref(&decoder->hw_device_ref);
 
     cuvid_free_functions(&decoder->cvdl);
@@ -269,10 +271,61 @@ int ff_nvdec_decode_uninit(AVCodecContext *avctx)
     return 0;
 }
 
+static void nvdec_free_dummy(struct AVHWFramesContext *ctx)
+{
+    av_buffer_pool_uninit(&ctx->pool);
+}
+
+static AVBufferRef *nvdec_alloc_dummy(int size)
+{
+    return av_buffer_create(NULL, 0, NULL, NULL, 0);
+}
+
+static int nvdec_init_hwframes(AVCodecContext *avctx, AVBufferRef **out_frames_ref, int dummy)
+{
+    AVHWFramesContext *frames_ctx;
+    int ret;
+
+    ret = avcodec_get_hw_frames_parameters(avctx,
+                                           avctx->hw_device_ctx,
+                                           avctx->hwaccel->pix_fmt,
+                                           out_frames_ref);
+    if (ret < 0)
+        return ret;
+
+    frames_ctx = (AVHWFramesContext*)(*out_frames_ref)->data;
+
+    if (dummy) {
+        // Copied from ff_decode_get_hw_frames_ctx for compatibility
+        frames_ctx->initial_pool_size += 3;
+
+        frames_ctx->free = nvdec_free_dummy;
+        frames_ctx->pool = av_buffer_pool_init(0, nvdec_alloc_dummy);
+
+        if (!frames_ctx->pool) {
+            av_buffer_unref(out_frames_ref);
+            return AVERROR(ENOMEM);
+        }
+    } else {
+        // This is normally not used to actually allocate frames from
+        frames_ctx->initial_pool_size = 0;
+    }
+
+    ret = av_hwframe_ctx_init(*out_frames_ref);
+    if (ret < 0) {
+        av_buffer_unref(out_frames_ref);
+        return ret;
+    }
+
+    return 0;
+}
+
 int ff_nvdec_decode_init(AVCodecContext *avctx)
 {
     NVDECContext *ctx = avctx->internal->hwaccel_priv_data;
 
+    NVDECDecoder        *decoder;
+    AVBufferRef         *real_hw_frames_ref;
     NVDECFramePool      *pool;
     AVHWFramesContext   *frames_ctx;
     const AVPixFmtDescriptor *sw_desc;
@@ -301,9 +354,17 @@ int ff_nvdec_decode_init(AVCodecContext *avctx)
     chroma_444 = ctx->supports_444 && cuvid_chroma_format == cudaVideoChromaFormat_444;
 
     if (!avctx->hw_frames_ctx) {
-        ret = ff_decode_get_hw_frames_ctx(avctx, AV_HWDEVICE_TYPE_CUDA);
+        ret = nvdec_init_hwframes(avctx, &avctx->hw_frames_ctx, 1);
+        if (ret < 0)
+            return ret;
+
+        ret = nvdec_init_hwframes(avctx, &real_hw_frames_ref, 0);
         if (ret < 0)
             return ret;
+    } else {
+        real_hw_frames_ref = av_buffer_ref(avctx->hw_frames_ctx);
+        if (!real_hw_frames_ref)
+            return AVERROR(ENOMEM);
     }
 
     switch (sw_desc->comp[0].depth) {
@@ -318,6 +379,7 @@ int ff_nvdec_decode_init(AVCodecContext *avctx)
         break;
     default:
         av_log(avctx, AV_LOG_ERROR, "Unsupported bit depth\n");
+        av_buffer_unref(&real_hw_frames_ref);
         return AVERROR(ENOSYS);
     }
 
@@ -342,9 +404,14 @@ int ff_nvdec_decode_init(AVCodecContext *avctx)
             av_log(avctx, AV_LOG_WARNING, "Try lowering the amount of threads. Using %d right now.\n",
                    avctx->thread_count);
         }
+        av_buffer_unref(&real_hw_frames_ref);
         return ret;
     }
 
+    decoder = (NVDECDecoder*)ctx->decoder_ref->data;
+    decoder->real_hw_frames_ref = real_hw_frames_ref;
+    real_hw_frames_ref = NULL;
+
     pool = av_mallocz(sizeof(*pool));
     if (!pool) {
         ret = AVERROR(ENOMEM);
@@ -447,6 +514,13 @@ static int nvdec_retrieve_data(void *logctx, AVFrame *frame)
         goto copy_fail;
     }
 
+    av_buffer_unref(&frame->hw_frames_ctx);
+    frame->hw_frames_ctx = av_buffer_ref(decoder->real_hw_frames_ref);
+    if (!frame->hw_frames_ctx) {
+        ret = AVERROR(ENOMEM);
+        goto copy_fail;
+    }
+
     unmap_data->idx = cf->idx;
     unmap_data->idx_ref = av_buffer_ref(cf->idx_ref);
     unmap_data->decoder_ref = av_buffer_ref(cf->decoder_ref);
@@ -575,16 +649,6 @@ int ff_nvdec_simple_decode_slice(AVCodecContext *avctx, const uint8_t *buffer,
     return 0;
 }
 
-static void nvdec_free_dummy(struct AVHWFramesContext *ctx)
-{
-    av_buffer_pool_uninit(&ctx->pool);
-}
-
-static AVBufferRef *nvdec_alloc_dummy(int size)
-{
-    return av_buffer_create(NULL, 0, NULL, NULL, 0);
-}
-
 int ff_nvdec_frame_params(AVCodecContext *avctx,
                           AVBufferRef *hw_frames_ctx,
                           int dpb_size,
@@ -620,12 +684,6 @@ int ff_nvdec_frame_params(AVCodecContext *avctx,
      */
     frames_ctx->initial_pool_size = dpb_size + 2;
 
-    frames_ctx->free = nvdec_free_dummy;
-    frames_ctx->pool = av_buffer_pool_init(0, nvdec_alloc_dummy);
-
-    if (!frames_ctx->pool)
-        return AVERROR(ENOMEM);
-
     switch (sw_desc->comp[0].depth) {
     case 8:
         frames_ctx->sw_format = chroma_444 ? AV_PIX_FMT_YUV444P : AV_PIX_FMT_NV12;