lavc: add a codec flag for propagating opaque from frames to packets

This is intended to be a more convenient replacement for
reordered_opaque.

Add support for it in the two encoders that offer
AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE: libx264 and libx265. Other
encoders will be supported in future commits.

7 files changed, 81 insertions, 12 deletions

diff --git a/doc/APIchanges b/doc/APIchanges
index ab7ce15fae..a3c9819e32 100644
--- a/doc/APIchanges
+++ b/doc/APIchanges
@@ -14,6 +14,9 @@ libavutil:     2021-04-27
 
 API changes, most recent first:
 
+2022-12-xx - xxxxxxxxxx - lavc 59.55.100 - avcodec.h
+  Add AV_CODEC_FLAG_COPY_OPAQUE.
+
 2022-11-xx - xxxxxxxxxx - lavu 57.43.100 - tx.h
   Add AV_TX_FLOAT_DCT, AV_TX_DOUBLE_DCT and AV_TX_INT32_DCT.
 
diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
index 3edd8e2636..79558786ee 100644
--- a/libavcodec/avcodec.h
+++ b/libavcodec/avcodec.h
@@ -242,6 +242,14 @@ typedef struct RcOverride{
  */
 #define AV_CODEC_FLAG_RECON_FRAME     (1 <<  6)
 /**
+ * Request the encoder to propagate each frame's AVFrame.opaque and
+ * AVFrame.opaque_ref values to its corresponding output AVPacket.
+ *
+ * May only be set on encoders that have the
+ * @ref AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE capability flag.
+ */
+#define AV_CODEC_FLAG_COPY_OPAQUE     (1 <<  7)
+/**
  * Use internal 2pass ratecontrol in first pass mode.
  */
 #define AV_CODEC_FLAG_PASS1           (1 <<  9)
diff --git a/libavcodec/encode.c b/libavcodec/encode.c
index fbe2c97cd6..7e2d54ae9b 100644
--- a/libavcodec/encode.c
+++ b/libavcodec/encode.c
@@ -634,6 +634,13 @@ int ff_encode_preinit(AVCodecContext *avctx)
         return AVERROR(EINVAL);
     }
 
+    if (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE &&
+        !(avctx->codec->capabilities & AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE)) {
+        av_log(avctx, AV_LOG_ERROR, "The copy_opaque flag is set, but the "
+               "encoder does not support it.\n");
+        return AVERROR(EINVAL);
+    }
+
     switch (avctx->codec_type) {
     case AVMEDIA_TYPE_VIDEO: ret = encode_preinit_video(avctx); break;
     case AVMEDIA_TYPE_AUDIO: ret = encode_preinit_audio(avctx); break;
diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c
index 2bbd9044b6..8944a7df36 100644
--- a/libavcodec/libx264.c
+++ b/libavcodec/libx264.c
@@ -21,6 +21,7 @@
 
 #include "config_components.h"
 
+#include "libavutil/buffer.h"
 #include "libavutil/eval.h"
 #include "libavutil/internal.h"
 #include "libavutil/opt.h"
@@ -51,6 +52,9 @@
 typedef struct X264Opaque {
     int64_t reordered_opaque;
     int64_t wallclock;
+
+    void        *frame_opaque;
+    AVBufferRef *frame_opaque_ref;
 } X264Opaque;
 
 typedef struct X264Context {
@@ -133,6 +137,11 @@ static void X264_log(void *p, int level, const char *fmt, va_list args)
     av_vlog(p, level_map[level], fmt, args);
 }
 
+static void opaque_uninit(X264Opaque *o)
+{
+    av_buffer_unref(&o->frame_opaque_ref);
+    memset(o, 0, sizeof(*o));
+}
 
 static int encode_nals(AVCodecContext *ctx, AVPacket *pkt,
                        const x264_nal_t *nals, int nnal)
@@ -440,6 +449,17 @@ static int setup_frame(AVCodecContext *ctx, const AVFrame *frame,
 
     pic->i_pts  = frame->pts;
 
+    opaque_uninit(opaque);
+
+    opaque->frame_opaque = frame->opaque;
+    if (frame->opaque_ref && ctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) {
+        opaque->frame_opaque_ref = av_buffer_ref(frame->opaque_ref);
+        if (!opaque->frame_opaque_ref) {
+            ret = AVERROR(ENOMEM);
+            goto fail;
+        }
+    }
+
     opaque->reordered_opaque = frame->reordered_opaque;
     opaque->wallclock = wallclock;
     if (ctx->export_side_data & AV_CODEC_EXPORT_DATA_PRFT)
@@ -594,6 +614,14 @@ static int X264_frame(AVCodecContext *ctx, AVPacket *pkt, const AVFrame *frame,
         out_opaque < &x4->reordered_opaque[x4->nb_reordered_opaque]) {
         ctx->reordered_opaque = out_opaque->reordered_opaque;
         wallclock = out_opaque->wallclock;
+
+        if (ctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) {
+            pkt->opaque                  = out_opaque->frame_opaque;
+            pkt->opaque_ref              = out_opaque->frame_opaque_ref;
+            out_opaque->frame_opaque_ref = NULL;
+        }
+
+        opaque_uninit(out_opaque);
     } else {
         // Unexpected opaque pointer on picture output
         av_log(ctx, AV_LOG_ERROR, "Unexpected opaque pointer; "
@@ -634,6 +662,9 @@ static av_cold int X264_close(AVCodecContext *avctx)
     X264Context *x4 = avctx->priv_data;
 
     av_freep(&x4->sei);
+
+    for (int i = 0; i < x4->nb_reordered_opaque; i++)
+        opaque_uninit(&x4->reordered_opaque[i]);
     av_freep(&x4->reordered_opaque);
 
 #if X264_BUILD >= 161
diff --git a/libavcodec/libx265.c b/libavcodec/libx265.c
index 25de3c669b..7ba547a7e7 100644
--- a/libavcodec/libx265.c
+++ b/libavcodec/libx265.c
@@ -28,6 +28,7 @@
 #include <float.h>
 
 #include "libavutil/avassert.h"
+#include "libavutil/buffer.h"
 #include "libavutil/internal.h"
 #include "libavutil/common.h"
 #include "libavutil/opt.h"
@@ -43,6 +44,9 @@
 typedef struct ReorderedData {
     int64_t reordered_opaque;
 
+    void        *frame_opaque;
+    AVBufferRef *frame_opaque_ref;
+
     int in_use;
 } ReorderedData;
 
@@ -121,7 +125,7 @@ static int rd_get(libx265Context *ctx)
 static void rd_release(libx265Context *ctx, int idx)
 {
     av_assert0(idx >= 0 && idx < ctx->nb_rd);
-
+    av_buffer_unref(&ctx->rd[idx].frame_opaque_ref);
     memset(&ctx->rd[idx], 0, sizeof(ctx->rd[idx]));
 }
 
@@ -132,6 +136,8 @@ static av_cold int libx265_encode_close(AVCodecContext *avctx)
     ctx->api->param_free(ctx->params);
     av_freep(&ctx->sei_data);
 
+    for (int i = 0; i < ctx->nb_rd; i++)
+        rd_release(ctx, i);
     av_freep(&ctx->rd);
 
     if (ctx->encoder)
@@ -582,6 +588,9 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     sei->numPayloads = 0;
 
     if (pic) {
+        ReorderedData *rd;
+        int rd_idx;
+
         for (i = 0; i < 3; i++) {
            x265pic.planes[i] = pic->data[i];
            x265pic.stride[i] = pic->linesize[i];
@@ -600,21 +609,25 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
         if (ret < 0)
             return ret;
 
-        if (pic->reordered_opaque) {
-            ReorderedData *rd;
-            int rd_idx = rd_get(ctx);
+        rd_idx = rd_get(ctx);
+        if (rd_idx < 0) {
+            free_picture(ctx, &x265pic);
+            return rd_idx;
+        }
+        rd = &ctx->rd[rd_idx];
 
-            if (rd_idx < 0) {
+        rd->reordered_opaque = pic->reordered_opaque;
+        rd->frame_opaque     = pic->opaque;
+        if (pic->opaque_ref && avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) {
+            rd->frame_opaque_ref = av_buffer_ref(pic->opaque_ref);
+            if (!rd->frame_opaque_ref) {
                 free_picture(ctx, &x265pic);
-                return rd_idx;
+                return AVERROR(ENOMEM);
             }
-
-            x265pic.userData = (void*)(intptr_t)(rd_idx + 1);
-
-            rd = &ctx->rd[rd_idx];
-            rd->reordered_opaque = pic->reordered_opaque;
         }
 
+        x265pic.userData = (void*)(intptr_t)(rd_idx + 1);
+
         if (ctx->a53_cc) {
             void *sei_data;
             size_t sei_size;
@@ -742,6 +755,12 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
 
         avctx->reordered_opaque = rd->reordered_opaque;
 
+        if (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) {
+            pkt->opaque          = rd->frame_opaque;
+            pkt->opaque_ref      = rd->frame_opaque_ref;
+            rd->frame_opaque_ref = NULL;
+        }
+
         rd_release(ctx, idx);
     } else
         avctx->reordered_opaque = 0;
diff --git a/libavcodec/options_table.h b/libavcodec/options_table.h
index cd02f5096f..ad41dc7bed 100644
--- a/libavcodec/options_table.h
+++ b/libavcodec/options_table.h
@@ -58,6 +58,7 @@ static const AVOption avcodec_options[] = {
 {"loop", "use loop filter", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_LOOP_FILTER }, INT_MIN, INT_MAX, V|E, "flags"},
 {"qscale", "use fixed qscale", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_QSCALE }, INT_MIN, INT_MAX, 0, "flags"},
 {"recon_frame", "export reconstructed frames", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_RECON_FRAME}, .unit = "flags"},
+{"copy_opaque", "propagate opaque values", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_COPY_OPAQUE}, .unit = "flags"},
 {"pass1", "use internal 2-pass ratecontrol in first  pass mode", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_PASS1 }, INT_MIN, INT_MAX, 0, "flags"},
 {"pass2", "use internal 2-pass ratecontrol in second pass mode", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_PASS2 }, INT_MIN, INT_MAX, 0, "flags"},
 {"gray", "only decode/encode grayscale", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_GRAY }, INT_MIN, INT_MAX, V|E|D, "flags"},
diff --git a/libavcodec/version.h b/libavcodec/version.h
index d149bc6c46..9f42f09f4e 100644
--- a/libavcodec/version.h
+++ b/libavcodec/version.h
@@ -29,7 +29,7 @@
 
 #include "version_major.h"
 
-#define LIBAVCODEC_VERSION_MINOR  54
+#define LIBAVCODEC_VERSION_MINOR  55
 #define LIBAVCODEC_VERSION_MICRO 100
 
 #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \