lavc/videotoolbox: fix H.264 hwaccel init issue

Fixes VTDecompressionSessionCreate() error.

Signed-off-by: Rick Kern <kernrj@gmail.com>

1 files changed, 45 insertions, 14 deletions

diff --git a/libavcodec/videotoolbox.c b/libavcodec/videotoolbox.c
index 6bd850a182..384bb8a598 100644
--- a/libavcodec/videotoolbox.c
+++ b/libavcodec/videotoolbox.c
@@ -487,23 +487,53 @@ static CFDictionaryRef videotoolbox_buffer_attributes_create(int width,
     return buffer_attributes;
 }
 
-static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType codec_type,
+static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(AVCodecContext *avctx,
+                                                                   CMVideoCodecType codec_type,
                                                                    CFDictionaryRef decoder_spec,
                                                                    int width,
                                                                    int height)
 {
-    CMFormatDescriptionRef cm_fmt_desc;
-    OSStatus status;
-
-    status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
-                                            codec_type,
-                                            width,
-                                            height,
-                                            decoder_spec, // Dictionary of extension
-                                            &cm_fmt_desc);
-
-    if (status)
-        return NULL;
+    CMFormatDescriptionRef cm_fmt_desc = NULL;
+    int status;
+    H264Context *h = codec_type == kCMVideoCodecType_H264 ? avctx->priv_data : NULL;
+
+    if (h && h->ps.sps->data_size && h->ps.pps->data_size) {
+        int ps_count = 2;
+        const uint8_t **ps_data = av_malloc(sizeof(uint8_t*) * ps_count);
+        size_t *ps_sizes = av_malloc(sizeof(size_t)  * ps_count);
+
+        ps_data[0]  = h->ps.sps->data;
+        ps_sizes[0] = h->ps.sps->data_size;
+
+        ps_data[1]  = h->ps.pps->data;
+        ps_sizes[1] = h->ps.pps->data_size;
+
+        status = CMVideoFormatDescriptionCreateFromH264ParameterSets(NULL,
+                                                                     ps_count,
+                                                                     ps_data,
+                                                                     ps_sizes,
+                                                                     4,
+                                                                     &cm_fmt_desc);
+        av_freep(&ps_sizes);
+        av_freep(&ps_data);
+
+        if (status) {
+            av_log(avctx, AV_LOG_ERROR, "Error creating H.264 format description: %d\n", status);
+            return NULL;
+        }
+    } else {
+        status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
+                                                codec_type,
+                                                width,
+                                                height,
+                                                decoder_spec, // Dictionary of extension
+                                                &cm_fmt_desc);
+
+        if (status) {
+            av_log(avctx, AV_LOG_ERROR, "Error creating format description: %d\n", status);
+            return NULL;
+        }
+    }
 
     return cm_fmt_desc;
 }
@@ -543,7 +573,8 @@ static int videotoolbox_default_init(AVCodecContext *avctx)
 
     decoder_spec = videotoolbox_decoder_config_create(videotoolbox->cm_codec_type, avctx);
 
-    videotoolbox->cm_fmt_desc = videotoolbox_format_desc_create(videotoolbox->cm_codec_type,
+    videotoolbox->cm_fmt_desc = videotoolbox_format_desc_create(avctx,
+                                                                videotoolbox->cm_codec_type,
                                                                 decoder_spec,
                                                                 avctx->width,
                                                                 avctx->height);