lavc/videotoolbox: fix threaded decoding

AVHWAccel.end_frame can run on a worker thread. The assumption of the
frame threading code is that the worker thread will change the AVFrame
image data, not the AVFrame fields. So the AVFrame fields are not synced
back to the main thread. But this breaks videotoolbox due to its special
requirements (everything else is fine). It actually wants to update
AVFrame fields.

The actual videotoolbox frame is now stored in the dummy AVBufferRef, so
it mimics what happens in non-videotoolbox cases. (Changing the
AVBufferRef contents is a bit like changing the image data.) The
post_process callback copies that reference to the proper AVFrame field.

Based on a patch by wm4.

Signed-off-by: Aman Gupta <aman@tmm1.net>

1 files changed, 51 insertions, 17 deletions

diff --git a/libavcodec/videotoolbox.c b/libavcodec/videotoolbox.c
index afec1edf3f..f82c31c5df 100644
--- a/libavcodec/videotoolbox.c
+++ b/libavcodec/videotoolbox.c
@@ -45,8 +45,10 @@ enum { kCMVideoCodecType_HEVC = 'hvc1' };
 
 static void videotoolbox_buffer_release(void *opaque, uint8_t *data)
 {
-    CVPixelBufferRef cv_buffer = (CVImageBufferRef)data;
+    CVPixelBufferRef cv_buffer = *(CVPixelBufferRef *)data;
     CVPixelBufferRelease(cv_buffer);
+
+    av_free(data);
 }
 
 static int videotoolbox_buffer_copy(VTContext *vtctx,
@@ -69,19 +71,47 @@ static int videotoolbox_buffer_copy(VTContext *vtctx,
     return 0;
 }
 
+static int videotoolbox_postproc_frame(void *avctx, AVFrame *frame)
+{
+    CVPixelBufferRef ref = *(CVPixelBufferRef *)frame->buf[0]->data;
+
+    if (!ref) {
+        av_log(avctx, AV_LOG_ERROR, "No frame decoded?\n");
+        av_frame_unref(frame);
+        return AVERROR_EXTERNAL;
+    }
+
+    frame->data[3] = (uint8_t*)ref;
+
+    return 0;
+}
+
 int ff_videotoolbox_alloc_frame(AVCodecContext *avctx, AVFrame *frame)
 {
+    size_t      size = sizeof(CVPixelBufferRef);
+    uint8_t    *data = NULL;
+    AVBufferRef *buf = NULL;
     int ret = ff_attach_decode_data(frame);
+    FrameDecodeData *fdd;
     if (ret < 0)
         return ret;
 
+    data = av_mallocz(size);
+    if (!data)
+        return AVERROR(ENOMEM);
+    buf = av_buffer_create(data, size, videotoolbox_buffer_release, NULL, 0);
+    if (!buf) {
+        av_freep(&data);
+        return AVERROR(ENOMEM);
+    }
+    frame->buf[0] = buf;
+
+    fdd = (FrameDecodeData*)frame->private_ref->data;
+    fdd->post_process = videotoolbox_postproc_frame;
+
     frame->width  = avctx->width;
     frame->height = avctx->height;
     frame->format = avctx->pix_fmt;
-    frame->buf[0] = av_buffer_alloc(1);
-
-    if (!frame->buf[0])
-        return AVERROR(ENOMEM);
 
     return 0;
 }
@@ -285,20 +315,24 @@ CFDataRef ff_videotoolbox_hvcc_extradata_create(AVCodecContext *avctx)
     return data;
 }
 
-int ff_videotoolbox_buffer_create(VTContext *vtctx, AVFrame *frame)
+static int videotoolbox_set_frame(AVCodecContext *avctx, AVFrame *frame)
 {
-    av_buffer_unref(&frame->buf[0]);
-
-    frame->buf[0] = av_buffer_create((uint8_t*)vtctx->frame,
-                                     sizeof(vtctx->frame),
-                                     videotoolbox_buffer_release,
-                                     NULL,
-                                     AV_BUFFER_FLAG_READONLY);
-    if (!frame->buf[0]) {
-        return AVERROR(ENOMEM);
+    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
+    if (!frame->buf[0] || frame->data[3]) {
+        av_log(avctx, AV_LOG_ERROR, "videotoolbox: invalid state\n");
+        av_frame_unref(frame);
+        return AVERROR_EXTERNAL;
+    }
+
+    CVPixelBufferRef *ref = (CVPixelBufferRef *)frame->buf[0]->data;
+
+    if (*ref) {
+        av_log(avctx, AV_LOG_ERROR, "videotoolbox: frame already set?\n");
+        av_frame_unref(frame);
+        return AVERROR_EXTERNAL;
     }
 
-    frame->data[3] = (uint8_t*)vtctx->frame;
+    *ref = vtctx->frame;
     vtctx->frame = NULL;
 
     return 0;
@@ -406,7 +440,7 @@ static int videotoolbox_buffer_create(AVCodecContext *avctx, AVFrame *frame)
     AVHWFramesContext *cached_frames;
     int ret;
 
-    ret = ff_videotoolbox_buffer_create(vtctx, frame);
+    ret = videotoolbox_set_frame(avctx, frame);
     if (ret < 0)
         return ret;