lavfi/vf_hwupload: Add support for HW -> HW transfers

As we find ourselves wanting a way to transfer frames between
HW devices (or more realistically, between APIs on the same device),
it's desirable to have a way to describe the relationship. While
we could imagine introducing a `hwtransfer` filter, there is
almost no difference from `hwupload`. The main new feature we need
is a way to specify the target device. Having a single device
for the filter chain is obviously insufficient if we're dealing
with two devices.

So let's add a way to specify the upload target device, and if none
is specified, continue with the existing behaviour.

We must also correctly preserve the sw_format on such a transfer.

3 files changed, 59 insertions, 15 deletions

diff --git a/doc/filters.texi b/doc/filters.texi
index 9b5f4543b8..96dccc45be 100644
--- a/doc/filters.texi
+++ b/doc/filters.texi
@@ -11981,7 +11981,18 @@ Upload system memory frames to hardware surfaces.
 
 The device to upload to must be supplied when the filter is initialised.  If
 using ffmpeg, select the appropriate device with the @option{-filter_hw_device}
-option.
+option or with the @option{derive_device} option.  The input and output devices
+must be of different types and compatible - the exact meaning of this is
+system-dependent, but typically it means that they must refer to the same
+underlying hardware context (for example, refer to the same graphics card).
+
+The following additional parameters are accepted:
+
+@table @option
+@item derive_device @var{type}
+Rather than using the device supplied at initialisation, instead derive a new
+device of type @var{type} from the device the input frames exist on.
+@end table
 
 @anchor{hwupload_cuda}
 @section hwupload_cuda
diff --git a/libavfilter/vf_hwupload.c b/libavfilter/vf_hwupload.c
index 50bc7e10f6..7c5dd497b0 100644
--- a/libavfilter/vf_hwupload.c
+++ b/libavfilter/vf_hwupload.c
@@ -32,10 +32,11 @@ typedef struct HWUploadContext {
     const AVClass *class;
 
     AVBufferRef       *hwdevice_ref;
-    AVHWDeviceContext *hwdevice;
 
     AVBufferRef       *hwframes_ref;
     AVHWFramesContext *hwframes;
+
+    char *device_type;
 } HWUploadContext;
 
 static int hwupload_query_formats(AVFilterContext *avctx)
@@ -46,17 +47,27 @@ static int hwupload_query_formats(AVFilterContext *avctx)
     AVFilterFormats *input_formats = NULL;
     int err, i;
 
-    if (!avctx->hw_device_ctx) {
+    if (ctx->hwdevice_ref) {
+        /* We already have a specified device. */
+    } else if (avctx->hw_device_ctx) {
+        if (ctx->device_type) {
+            err = av_hwdevice_ctx_create_derived(
+                &ctx->hwdevice_ref,
+                av_hwdevice_find_type_by_name(ctx->device_type),
+                avctx->hw_device_ctx, 0);
+            if (err < 0)
+                return err;
+        } else {
+            ctx->hwdevice_ref = av_buffer_ref(avctx->hw_device_ctx);
+            if (!ctx->hwdevice_ref)
+                return AVERROR(ENOMEM);
+        }
+    } else {
         av_log(ctx, AV_LOG_ERROR, "A hardware device reference is required "
                "to upload frames to.\n");
         return AVERROR(EINVAL);
     }
 
-    ctx->hwdevice_ref = av_buffer_ref(avctx->hw_device_ctx);
-    if (!ctx->hwdevice_ref)
-        return AVERROR(ENOMEM);
-    ctx->hwdevice = (AVHWDeviceContext*)ctx->hwdevice_ref->data;
-
     constraints = av_hwdevice_get_hwframe_constraints(ctx->hwdevice_ref, NULL);
     if (!constraints) {
         err = AVERROR(EINVAL);
@@ -127,7 +138,13 @@ static int hwupload_config_output(AVFilterLink *outlink)
            av_get_pix_fmt_name(inlink->format));
 
     ctx->hwframes->format    = outlink->format;
-    ctx->hwframes->sw_format = inlink->format;
+    if (inlink->hw_frames_ctx) {
+        AVHWFramesContext *in_hwframe_ctx =
+            (AVHWFramesContext*)inlink->hw_frames_ctx->data;
+        ctx->hwframes->sw_format = in_hwframe_ctx->sw_format;
+    } else {
+        ctx->hwframes->sw_format = inlink->format;
+    }
     ctx->hwframes->width     = inlink->w;
     ctx->hwframes->height    = inlink->h;
 
@@ -200,13 +217,21 @@ static av_cold void hwupload_uninit(AVFilterContext *avctx)
     av_buffer_unref(&ctx->hwdevice_ref);
 }
 
-static const AVClass hwupload_class = {
-    .class_name = "hwupload",
-    .item_name  = av_default_item_name,
-    .option     = NULL,
-    .version    = LIBAVUTIL_VERSION_INT,
+#define OFFSET(x) offsetof(HWUploadContext, x)
+#define FLAGS (AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM)
+static const AVOption hwupload_options[] = {
+    {
+        "derive_device", "Derive a new device of this type",
+        OFFSET(device_type), AV_OPT_TYPE_STRING,
+        { .str = NULL }, 0, 0, FLAGS
+    },
+    {
+        NULL
+    }
 };
 
+AVFILTER_DEFINE_CLASS(hwupload);
+
 static const AVFilterPad hwupload_inputs[] = {
     {
         .name         = "default",
diff --git a/libavfilter/vf_hwupload_cuda.c b/libavfilter/vf_hwupload_cuda.c
index 4d83e6c8f2..8ee0825859 100644
--- a/libavfilter/vf_hwupload_cuda.c
+++ b/libavfilter/vf_hwupload_cuda.c
@@ -60,6 +60,9 @@ static int cudaupload_query_formats(AVFilterContext *ctx)
         AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV444P,
         AV_PIX_FMT_P010, AV_PIX_FMT_P016, AV_PIX_FMT_YUV444P16,
         AV_PIX_FMT_0RGB32, AV_PIX_FMT_0BGR32,
+#if CONFIG_VULKAN
+        AV_PIX_FMT_VULKAN,
+#endif
         AV_PIX_FMT_NONE,
     };
     static const enum AVPixelFormat output_pix_fmts[] = {
@@ -97,7 +100,12 @@ static int cudaupload_config_output(AVFilterLink *outlink)
 
     hwframe_ctx            = (AVHWFramesContext*)s->hwframe->data;
     hwframe_ctx->format    = AV_PIX_FMT_CUDA;
-    hwframe_ctx->sw_format = inlink->format;
+    if (inlink->hw_frames_ctx) {
+        AVHWFramesContext *in_hwframe_ctx = (AVHWFramesContext*)inlink->hw_frames_ctx->data;
+        hwframe_ctx->sw_format = in_hwframe_ctx->sw_format;
+    } else {
+        hwframe_ctx->sw_format = inlink->format;
+    }
     hwframe_ctx->width     = inlink->w;
     hwframe_ctx->height    = inlink->h;