hwcontext: Hardware frame mapping

Adds the new av_hwframe_map() function, which allows mapping between
hardware frames and normal memory, along with internal support for
implementing it.

Also adds av_hwframe_ctx_create_derived(), for creating a hardware
frames context associated with one device using frames mapped from
another by some hardware-specific means.

4 files changed, 361 insertions, 7 deletions

diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c
index b47ef44fa7..83f733e9f9 100644
--- a/libavutil/hwcontext.c
+++ b/libavutil/hwcontext.c
@@ -157,14 +157,19 @@ static void hwframe_ctx_free(void *opaque, uint8_t *data)
 {
     AVHWFramesContext *ctx = (AVHWFramesContext*)data;
 
-    if (ctx->internal->pool_internal)
-        av_buffer_pool_uninit(&ctx->internal->pool_internal);
+    if (ctx->internal->source_frames) {
+        av_buffer_unref(&ctx->internal->source_frames);
 
-    if (ctx->internal->hw_type->frames_uninit)
-        ctx->internal->hw_type->frames_uninit(ctx);
+    } else {
+        if (ctx->internal->pool_internal)
+            av_buffer_pool_uninit(&ctx->internal->pool_internal);
 
-    if (ctx->free)
-        ctx->free(ctx);
+        if (ctx->internal->hw_type->frames_uninit)
+            ctx->internal->hw_type->frames_uninit(ctx);
+
+        if (ctx->free)
+            ctx->free(ctx);
+    }
 
     av_buffer_unref(&ctx->device_ref);
 
@@ -266,6 +271,11 @@ int av_hwframe_ctx_init(AVBufferRef *ref)
     const enum AVPixelFormat *pix_fmt;
     int ret;
 
+    if (ctx->internal->source_frames) {
+        /* A derived frame context is already initialised. */
+        return 0;
+    }
+
     /* validate the pixel format */
     for (pix_fmt = ctx->internal->hw_type->pix_fmts; *pix_fmt != AV_PIX_FMT_NONE; pix_fmt++) {
         if (*pix_fmt == ctx->format)
@@ -396,6 +406,35 @@ int av_hwframe_get_buffer(AVBufferRef *hwframe_ref, AVFrame *frame, int flags)
     AVHWFramesContext *ctx = (AVHWFramesContext*)hwframe_ref->data;
     int ret;
 
+    if (ctx->internal->source_frames) {
+        // This is a derived frame context, so we allocate in the source
+        // and map the frame immediately.
+        AVFrame *src_frame;
+
+        src_frame = av_frame_alloc();
+        if (!src_frame)
+            return AVERROR(ENOMEM);
+
+        ret = av_hwframe_get_buffer(ctx->internal->source_frames,
+                                    src_frame, 0);
+        if (ret < 0)
+            return ret;
+
+        ret = av_hwframe_map(frame, src_frame, 0);
+        if (ret) {
+            av_log(ctx, AV_LOG_ERROR, "Failed to map frame into derived "
+                   "frame context: %d.\n", ret);
+            av_frame_free(&src_frame);
+            return ret;
+        }
+
+        // Free the source frame immediately - the mapped frame still
+        // contains a reference to it.
+        av_frame_free(&src_frame);
+
+        return 0;
+    }
+
     if (!ctx->internal->hw_type->frames_get_buffer)
         return AVERROR(ENOSYS);
 
@@ -495,3 +534,191 @@ fail:
     *pdevice_ref = NULL;
     return ret;
 }
+
+static void ff_hwframe_unmap(void *opaque, uint8_t *data)
+{
+    HWMapDescriptor *hwmap = (HWMapDescriptor*)data;
+    AVHWFramesContext *ctx = opaque;
+
+    if (hwmap->unmap)
+        hwmap->unmap(ctx, hwmap);
+
+    av_frame_free(&hwmap->source);
+
+    av_buffer_unref(&hwmap->hw_frames_ctx);
+
+    av_free(hwmap);
+}
+
+int ff_hwframe_map_create(AVBufferRef *hwframe_ref,
+                          AVFrame *dst, const AVFrame *src,
+                          void (*unmap)(AVHWFramesContext *ctx,
+                                        HWMapDescriptor *hwmap),
+                          void *priv)
+{
+    AVHWFramesContext *ctx = (AVHWFramesContext*)hwframe_ref->data;
+    HWMapDescriptor *hwmap;
+    int ret;
+
+    hwmap = av_mallocz(sizeof(*hwmap));
+    if (!hwmap) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+
+    hwmap->source = av_frame_alloc();
+    if (!hwmap->source) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+    ret = av_frame_ref(hwmap->source, src);
+    if (ret < 0)
+        goto fail;
+
+    hwmap->hw_frames_ctx = av_buffer_ref(hwframe_ref);
+    if (!hwmap->hw_frames_ctx) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+
+    hwmap->unmap = unmap;
+    hwmap->priv  = priv;
+
+    dst->buf[0] = av_buffer_create((uint8_t*)hwmap, sizeof(*hwmap),
+                                   &ff_hwframe_unmap, ctx, 0);
+    if (!dst->buf[0]) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+
+    return 0;
+
+fail:
+    if (hwmap) {
+        av_buffer_unref(&hwmap->hw_frames_ctx);
+        av_frame_free(&hwmap->source);
+    }
+    av_free(hwmap);
+    return ret;
+}
+
+int av_hwframe_map(AVFrame *dst, const AVFrame *src, int flags)
+{
+    AVHWFramesContext *src_frames, *dst_frames;
+    HWMapDescriptor *hwmap;
+    int ret;
+
+    if (src->hw_frames_ctx && dst->hw_frames_ctx) {
+        src_frames = (AVHWFramesContext*)src->hw_frames_ctx->data;
+        dst_frames = (AVHWFramesContext*)dst->hw_frames_ctx->data;
+
+        if ((src_frames == dst_frames &&
+             src->format == dst_frames->sw_format &&
+             dst->format == dst_frames->format) ||
+            (src_frames->internal->source_frames &&
+             src_frames->internal->source_frames->data ==
+             (uint8_t*)dst_frames)) {
+            // This is an unmap operation.  We don't need to directly
+            // do anything here other than fill in the original frame,
+            // because the real unmap will be invoked when the last
+            // reference to the mapped frame disappears.
+            if (!src->buf[0]) {
+                av_log(src_frames, AV_LOG_ERROR, "Invalid mapping "
+                       "found when attempting unmap.\n");
+                return AVERROR(EINVAL);
+            }
+            hwmap = (HWMapDescriptor*)src->buf[0]->data;
+            av_frame_unref(dst);
+            return av_frame_ref(dst, hwmap->source);
+        }
+    }
+
+    if (src->hw_frames_ctx) {
+        src_frames = (AVHWFramesContext*)src->hw_frames_ctx->data;
+
+        if (src_frames->format == src->format &&
+            src_frames->internal->hw_type->map_from) {
+            ret = src_frames->internal->hw_type->map_from(src_frames,
+                                                          dst, src, flags);
+            if (ret != AVERROR(ENOSYS))
+                return ret;
+        }
+    }
+
+    if (dst->hw_frames_ctx) {
+        dst_frames = (AVHWFramesContext*)dst->hw_frames_ctx->data;
+
+        if (dst_frames->format == dst->format &&
+            dst_frames->internal->hw_type->map_to) {
+            ret = dst_frames->internal->hw_type->map_to(dst_frames,
+                                                        dst, src, flags);
+            if (ret != AVERROR(ENOSYS))
+                return ret;
+        }
+    }
+
+    return AVERROR(ENOSYS);
+}
+
+int av_hwframe_ctx_create_derived(AVBufferRef **derived_frame_ctx,
+                                  enum AVPixelFormat format,
+                                  AVBufferRef *derived_device_ctx,
+                                  AVBufferRef *source_frame_ctx,
+                                  int flags)
+{
+    AVBufferRef   *dst_ref = NULL;
+    AVHWFramesContext *dst = NULL;
+    AVHWFramesContext *src = (AVHWFramesContext*)source_frame_ctx->data;
+    int ret;
+
+    if (src->internal->source_frames) {
+        AVHWFramesContext *src_src =
+            (AVHWFramesContext*)src->internal->source_frames->data;
+        AVHWDeviceContext *dst_dev =
+            (AVHWDeviceContext*)derived_device_ctx->data;
+
+        if (src_src->device_ctx == dst_dev) {
+            // This is actually an unmapping, so we just return a
+            // reference to the source frame context.
+            *derived_frame_ctx =
+                av_buffer_ref(src->internal->source_frames);
+            if (!*derived_frame_ctx) {
+                ret = AVERROR(ENOMEM);
+                goto fail;
+            }
+            return 0;
+        }
+    }
+
+    dst_ref = av_hwframe_ctx_alloc(derived_device_ctx);
+    if (!dst_ref) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+
+    dst = (AVHWFramesContext*)dst_ref->data;
+
+    dst->format    = format;
+    dst->sw_format = src->sw_format;
+    dst->width     = src->width;
+    dst->height    = src->height;
+
+    dst->internal->source_frames = av_buffer_ref(source_frame_ctx);
+    if (!dst->internal->source_frames) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+
+    ret = av_hwframe_ctx_init(dst_ref);
+    if (ret)
+        goto fail;
+
+    *derived_frame_ctx = dst_ref;
+    return 0;
+
+fail:
+    if (dst)
+        av_buffer_unref(&dst->internal->source_frames);
+    av_buffer_unref(&dst_ref);
+    return ret;
+}
diff --git a/libavutil/hwcontext.h b/libavutil/hwcontext.h
index 0b12fe24bc..4067ff1db2 100644
--- a/libavutil/hwcontext.h
+++ b/libavutil/hwcontext.h
@@ -434,4 +434,91 @@ AVHWFramesConstraints *av_hwdevice_get_hwframe_constraints(AVBufferRef *ref,
  */
 void av_hwframe_constraints_free(AVHWFramesConstraints **constraints);
 
+
+/**
+ * Flags to apply to frame mappings.
+ */
+enum {
+    /**
+     * The mapping must be readable.
+     */
+    AV_HWFRAME_MAP_READ      = 1 << 0,
+    /**
+     * The mapping must be writeable.
+     */
+    AV_HWFRAME_MAP_WRITE     = 1 << 1,
+    /**
+     * The mapped frame will be overwritten completely in subsequent
+     * operations, so the current frame data need not be loaded.  Any values
+     * which are not overwritten are unspecified.
+     */
+    AV_HWFRAME_MAP_OVERWRITE = 1 << 2,
+    /**
+     * The mapping must be direct.  That is, there must not be any copying in
+     * the map or unmap steps.  Note that performance of direct mappings may
+     * be much lower than normal memory.
+     */
+    AV_HWFRAME_MAP_DIRECT    = 1 << 3,
+};
+
+/**
+ * Map a hardware frame.
+ *
+ * This has a number of different possible effects, depending on the format
+ * and origin of the src and dst frames.  On input, src should be a usable
+ * frame with valid buffers and dst should be blank (typically as just created
+ * by av_frame_alloc()).  src should have an associated hwframe context, and
+ * dst may optionally have a format and associated hwframe context.
+ *
+ * If src was created by mapping a frame from the hwframe context of dst,
+ * then this function undoes the mapping - dst is replaced by a reference to
+ * the frame that src was originally mapped from.
+ *
+ * If both src and dst have an associated hwframe context, then this function
+ * attempts to map the src frame from its hardware context to that of dst and
+ * then fill dst with appropriate data to be usable there.  This will only be
+ * possible if the hwframe contexts and associated devices are compatible -
+ * given compatible devices, av_hwframe_ctx_create_derived() can be used to
+ * create a hwframe context for dst in which mapping should be possible.
+ *
+ * If src has a hwframe context but dst does not, then the src frame is
+ * mapped to normal memory and should thereafter be usable as a normal frame.
+ * If the format is set on dst, then the mapping will attempt to create dst
+ * with that format and fail if it is not possible.  If format is unset (is
+ * AV_PIX_FMT_NONE) then dst will be mapped with whatever the most appropriate
+ * format to use is (probably the sw_format of the src hwframe context).
+ *
+ * A return value of AVERROR(ENOSYS) indicates that the mapping is not
+ * possible with the given arguments and hwframe setup, while other return
+ * values indicate that it failed somehow.
+ *
+ * @param dst Destination frame, to contain the mapping.
+ * @param src Source frame, to be mapped.
+ * @param flags Some combination of AV_HWFRAME_MAP_* flags.
+ * @return Zero on success, negative AVERROR code on failure.
+ */
+int av_hwframe_map(AVFrame *dst, const AVFrame *src, int flags);
+
+
+/**
+ * Create and initialise an AVHWFramesContext as a mapping of another existing
+ * AVHWFramesContext on a different device.
+ *
+ * av_hwframe_ctx_init() should not be called after this.
+ *
+ * @param derived_frame_ctx  On success, a reference to the newly created
+ *                           AVHWFramesContext.
+ * @param derived_device_ctx A reference to the device to create the new
+ *                           AVHWFramesContext on.
+ * @param source_frame_ctx   A reference to an existing AVHWFramesContext
+ *                           which will be mapped to the derived context.
+ * @param flags  Currently unused; should be set to zero.
+ * @return       Zero on success, negative AVERROR code on failure.
+ */
+int av_hwframe_ctx_create_derived(AVBufferRef **derived_frame_ctx,
+                                  enum AVPixelFormat format,
+                                  AVBufferRef *derived_device_ctx,
+                                  AVBufferRef *source_frame_ctx,
+                                  int flags);
+
 #endif /* AVUTIL_HWCONTEXT_H */
diff --git a/libavutil/hwcontext_internal.h b/libavutil/hwcontext_internal.h
index a391e2599e..11e3a68e4c 100644
--- a/libavutil/hwcontext_internal.h
+++ b/libavutil/hwcontext_internal.h
@@ -85,6 +85,11 @@ typedef struct HWContextType {
                                          const AVFrame *src);
     int              (*transfer_data_from)(AVHWFramesContext *ctx, AVFrame *dst,
                                            const AVFrame *src);
+
+    int              (*map_to)(AVHWFramesContext *ctx, AVFrame *dst,
+                               const AVFrame *src, int flags);
+    int              (*map_from)(AVHWFramesContext *ctx, AVFrame *dst,
+                                 const AVFrame *src, int flags);
 } HWContextType;
 
 struct AVHWDeviceInternal {
@@ -97,8 +102,43 @@ struct AVHWFramesInternal {
     void                *priv;
 
     AVBufferPool *pool_internal;
+
+    /**
+     * For a derived context, a reference to the original frames
+     * context it was derived from.
+     */
+    AVBufferRef *source_frames;
 };
 
+typedef struct HWMapDescriptor {
+    /**
+     * A reference to the original source of the mapping.
+     */
+    AVFrame *source;
+    /**
+     * A reference to the hardware frames context in which this
+     * mapping was made.  May be the same as source->hw_frames_ctx,
+     * but need not be.
+     */
+    AVBufferRef *hw_frames_ctx;
+    /**
+     * Unmap function.
+     */
+    void (*unmap)(AVHWFramesContext *ctx,
+                  struct HWMapDescriptor *hwmap);
+    /**
+     * Hardware-specific private data associated with the mapping.
+     */
+    void          *priv;
+} HWMapDescriptor;
+
+int ff_hwframe_map_create(AVBufferRef *hwframe_ref,
+                          AVFrame *dst, const AVFrame *src,
+                          void (*unmap)(AVHWFramesContext *ctx,
+                                        HWMapDescriptor *hwmap),
+                          void *priv);
+
+
 extern const HWContextType ff_hwcontext_type_cuda;
 extern const HWContextType ff_hwcontext_type_dxva2;
 extern const HWContextType ff_hwcontext_type_qsv;
diff --git a/libavutil/version.h b/libavutil/version.h
index c1cfcbba2f..9287a0b5bd 100644
--- a/libavutil/version.h
+++ b/libavutil/version.h
@@ -54,7 +54,7 @@
  */
 
 #define LIBAVUTIL_VERSION_MAJOR 55
-#define LIBAVUTIL_VERSION_MINOR 26
+#define LIBAVUTIL_VERSION_MINOR 27
 #define LIBAVUTIL_VERSION_MICRO  0
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \