avcodec/decode: Extend ff_hwaccel_frame_priv_alloc()'s task

All usages of ff_hwaccel_frame_priv_alloc() have the same pattern:
Check for whether a hwaccel is in use; check whether it needs
private frame-specific data; allocate the AVBuffer and set
it.

This commit modifies ff_hwaccel_frame_priv_alloc() to perform
this task on its own.

(It also seems that the H.264 decoder did not perform proper
cleanup in case the buffer could not be allocated. This has been
changed.)

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>

8 files changed, 62 insertions, 77 deletions

diff --git a/libavcodec/av1dec.c b/libavcodec/av1dec.c
index cc178464b9..09339529f7 100644
--- a/libavcodec/av1dec.c
+++ b/libavcodec/av1dec.c
@@ -907,17 +907,11 @@ static int av1_frame_alloc(AVCodecContext *avctx, AV1Frame *f)
         break;
     }
 
-    if (avctx->hwaccel) {
-        const AVHWAccel *hwaccel = avctx->hwaccel;
-        if (hwaccel->frame_priv_data_size) {
-            f->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(avctx, hwaccel);
-            if (!f->hwaccel_priv_buf) {
-                ret = AVERROR(ENOMEM);
-                goto fail;
-            }
-            f->hwaccel_picture_private = f->hwaccel_priv_buf->data;
-        }
-    }
+    ret = ff_hwaccel_frame_priv_alloc(avctx, &f->hwaccel_picture_private,
+                                      &f->hwaccel_priv_buf);
+    if (ret < 0)
+        goto fail;
+
     return 0;
 
 fail:
diff --git a/libavcodec/decode.c b/libavcodec/decode.c
index 53c90cddb7..be761873a3 100644
--- a/libavcodec/decode.c
+++ b/libavcodec/decode.c
@@ -1788,24 +1788,36 @@ int ff_copy_palette(void *dst, const AVPacket *src, void *logctx)
     return 0;
 }
 
-AVBufferRef *ff_hwaccel_frame_priv_alloc(AVCodecContext *avctx,
-                                         const AVHWAccel *hwaccel)
+int ff_hwaccel_frame_priv_alloc(AVCodecContext *avctx, void **hwaccel_picture_private,
+                                AVBufferRef **hwaccel_priv_buf)
 {
+    const AVHWAccel *hwaccel = avctx->hwaccel;
     AVBufferRef *ref;
-    AVHWFramesContext *frames_ctx = (AVHWFramesContext *)avctx->hw_frames_ctx->data;
-    uint8_t *data = av_mallocz(hwaccel->frame_priv_data_size);
+    AVHWFramesContext *frames_ctx;
+    uint8_t *data;
+
+    if (!hwaccel || !hwaccel->frame_priv_data_size)
+        return 0;
+
+    av_assert0(!*hwaccel_picture_private);
+    data       = av_mallocz(hwaccel->frame_priv_data_size);
     if (!data)
-        return NULL;
+        return AVERROR(ENOMEM);
+
+    frames_ctx = (AVHWFramesContext *)avctx->hw_frames_ctx->data;
 
     ref = av_buffer_create(data, hwaccel->frame_priv_data_size,
                            hwaccel->free_frame_priv,
                            frames_ctx->device_ctx, 0);
     if (!ref) {
         av_free(data);
-        return NULL;
+        return AVERROR(ENOMEM);
     }
 
-    return ref;
+    *hwaccel_priv_buf        = ref;
+    *hwaccel_picture_private = ref->data;
+
+    return 0;
 }
 
 void ff_decode_flush_buffers(AVCodecContext *avctx)
diff --git a/libavcodec/decode.h b/libavcodec/decode.h
index 2b9fe59907..a52152e4a7 100644
--- a/libavcodec/decode.h
+++ b/libavcodec/decode.h
@@ -140,14 +140,17 @@ int ff_side_data_update_matrix_encoding(AVFrame *frame,
                                         enum AVMatrixEncoding matrix_encoding);
 
 /**
- * Allocate a hwaccel frame private data and create an AVBufferRef
- * from it.
+ * Allocate a hwaccel frame private data if the provided avctx
+ * uses a hwaccel method that needs it. The private data will
+ * be refcounted via the AVBuffer API (if allocated).
  *
- * @param     avctx   The codec context which to attach as an opaque value
- * @param     hwaccel The hwaccel for which to allocate
- * @return            The allocated buffer
+ * @param  avctx                   The codec context
+ * @param  hwaccel_picture_private Pointer to return hwaccel_picture_private
+ * @param  hwaccel_priv_buf        Pointer to return the AVBufferRef owning
+ *                                 hwaccel_picture_private
+ * @return 0 on success, < 0 on error
  */
-AVBufferRef *ff_hwaccel_frame_priv_alloc(AVCodecContext *avctx,
-                                         const AVHWAccel *hwaccel);
+int ff_hwaccel_frame_priv_alloc(AVCodecContext *avctx, void **hwaccel_picture_private,
+                                AVBufferRef **hwaccel_priv_buf);
 
 #endif /* AVCODEC_DECODE_H */
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index 8baf9944c5..6cd7bb8fe7 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -205,16 +205,11 @@ static int alloc_picture(H264Context *h, H264Picture *pic)
             goto fail;
     }
 
-    if (h->avctx->hwaccel) {
-        const AVHWAccel *hwaccel = h->avctx->hwaccel;
-        av_assert0(!pic->hwaccel_picture_private);
-        if (hwaccel->frame_priv_data_size) {
-            pic->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(h->avctx, hwaccel);
-            if (!pic->hwaccel_priv_buf)
-                return AVERROR(ENOMEM);
-            pic->hwaccel_picture_private = pic->hwaccel_priv_buf->data;
-        }
-    }
+    ret = ff_hwaccel_frame_priv_alloc(h->avctx, &pic->hwaccel_picture_private,
+                                      &pic->hwaccel_priv_buf);
+    if (ret < 0)
+        goto fail;
+
     if (CONFIG_GRAY && !h->avctx->hwaccel && h->flags & AV_CODEC_FLAG_GRAY && pic->f->data[2]) {
         int h_chroma_shift, v_chroma_shift;
         av_pix_fmt_get_chroma_sub_sample(pic->f->format,
diff --git a/libavcodec/hevc_refs.c b/libavcodec/hevc_refs.c
index 3aa8e19950..c5c1203ef8 100644
--- a/libavcodec/hevc_refs.c
+++ b/libavcodec/hevc_refs.c
@@ -118,16 +118,10 @@ static HEVCFrame *alloc_frame(HEVCContext *s)
             (s->sei.picture_timing.picture_struct == AV_PICTURE_STRUCTURE_BOTTOM_FIELD))
             frame->frame->flags |= AV_FRAME_FLAG_INTERLACED;
 
-        if (s->avctx->hwaccel) {
-            const AVHWAccel *hwaccel = s->avctx->hwaccel;
-            av_assert0(!frame->hwaccel_picture_private);
-            if (hwaccel->frame_priv_data_size) {
-                frame->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(s->avctx, hwaccel);
-                if (!frame->hwaccel_priv_buf)
-                    goto fail;
-                frame->hwaccel_picture_private = frame->hwaccel_priv_buf->data;
-            }
-        }
+        ret = ff_hwaccel_frame_priv_alloc(s->avctx, &frame->hwaccel_picture_private,
+                                          &frame->hwaccel_priv_buf);
+        if (ret < 0)
+            goto fail;
 
         return frame;
 fail:
diff --git a/libavcodec/mpegpicture.c b/libavcodec/mpegpicture.c
index 71c7a3fd70..b7c804c8ec 100644
--- a/libavcodec/mpegpicture.c
+++ b/libavcodec/mpegpicture.c
@@ -171,17 +171,10 @@ static int alloc_frame_buffer(AVCodecContext *avctx,  Picture *pic,
         pic->f->height = avctx->height;
     }
 
-    if (avctx->hwaccel) {
-        assert(!pic->hwaccel_picture_private);
-        if (avctx->hwaccel->frame_priv_data_size) {
-            pic->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(avctx, avctx->hwaccel);
-            if (!pic->hwaccel_priv_buf) {
-                av_log(avctx, AV_LOG_ERROR, "alloc_frame_buffer() failed (hwaccel private data allocation)\n");
-                return -1;
-            }
-            pic->hwaccel_picture_private = pic->hwaccel_priv_buf->data;
-        }
-    }
+    ret = ff_hwaccel_frame_priv_alloc(avctx, &pic->hwaccel_picture_private,
+                                      &pic->hwaccel_priv_buf);
+    if (ret < 0)
+        return ret;
 
     if ((linesize   &&   linesize != pic->f->linesize[0]) ||
         (uvlinesize && uvlinesize != pic->f->linesize[1])) {
diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c
index 50afe19b7a..15a2961f56 100644
--- a/libavcodec/vp8.c
+++ b/libavcodec/vp8.c
@@ -104,23 +104,21 @@ static int vp8_alloc_frame(VP8Context *s, VP8Frame *f, int ref)
     if ((ret = ff_thread_get_ext_buffer(s->avctx, &f->tf,
                                         ref ? AV_GET_BUFFER_FLAG_REF : 0)) < 0)
         return ret;
-    if (!(f->seg_map = av_buffer_allocz(s->mb_width * s->mb_height)))
+    if (!(f->seg_map = av_buffer_allocz(s->mb_width * s->mb_height))) {
+        ret = AVERROR(ENOMEM);
         goto fail;
-    if (s->avctx->hwaccel) {
-        const AVHWAccel *hwaccel = s->avctx->hwaccel;
-        if (hwaccel->frame_priv_data_size) {
-            f->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(s->avctx, hwaccel);
-            if (!f->hwaccel_priv_buf)
-                goto fail;
-            f->hwaccel_picture_private = f->hwaccel_priv_buf->data;
-        }
     }
+    ret = ff_hwaccel_frame_priv_alloc(s->avctx, &f->hwaccel_picture_private,
+                                      &f->hwaccel_priv_buf);
+    if (ret < 0)
+        goto fail;
+
     return 0;
 
 fail:
     av_buffer_unref(&f->seg_map);
     ff_thread_release_ext_buffer(s->avctx, &f->tf);
-    return AVERROR(ENOMEM);
+    return ret;
 }
 
 static void vp8_release_frame(VP8Context *s, VP8Frame *f)
diff --git a/libavcodec/vp9.c b/libavcodec/vp9.c
index 4f704ec0dd..751dc7271f 100644
--- a/libavcodec/vp9.c
+++ b/libavcodec/vp9.c
@@ -119,12 +119,14 @@ static int vp9_frame_alloc(AVCodecContext *avctx, VP9Frame *f)
         s->frame_extradata_pool = av_buffer_pool_init(sz * (1 + sizeof(VP9mvrefPair)), NULL);
         if (!s->frame_extradata_pool) {
             s->frame_extradata_pool_size = 0;
+            ret = AVERROR(ENOMEM);
             goto fail;
         }
         s->frame_extradata_pool_size = sz;
     }
     f->extradata = av_buffer_pool_get(s->frame_extradata_pool);
     if (!f->extradata) {
+        ret = AVERROR(ENOMEM);
         goto fail;
     }
     memset(f->extradata->data, 0, f->extradata->size);
@@ -132,22 +134,16 @@ static int vp9_frame_alloc(AVCodecContext *avctx, VP9Frame *f)
     f->segmentation_map = f->extradata->data;
     f->mv = (VP9mvrefPair *) (f->extradata->data + sz);
 
-    if (avctx->hwaccel) {
-        const AVHWAccel *hwaccel = avctx->hwaccel;
-        av_assert0(!f->hwaccel_picture_private);
-        if (hwaccel->frame_priv_data_size) {
-            f->hwaccel_priv_buf = ff_hwaccel_frame_priv_alloc(avctx, hwaccel);
-            if (!f->hwaccel_priv_buf)
-                goto fail;
-            f->hwaccel_picture_private = f->hwaccel_priv_buf->data;
-        }
-    }
+    ret = ff_hwaccel_frame_priv_alloc(avctx, &f->hwaccel_picture_private,
+                                      &f->hwaccel_priv_buf);
+    if (ret < 0)
+        goto fail;
 
     return 0;
 
 fail:
     vp9_frame_unref(avctx, f);
-    return AVERROR(ENOMEM);
+    return ret;
 }
 
 static int vp9_frame_ref(AVCodecContext *avctx, VP9Frame *dst, VP9Frame *src)