vaapi_encode: Refactor initialisation

This allows better checking of capabilities and will make it easier
to add more functionality later.

It also commonises some duplicated code around rate control setup
and adds more comments explaining the internals.

(cherry picked from commit 80a5d05108cb218e8cd2e25c6621a3bfef0a832e)

1 files changed, 71 insertions, 139 deletions

diff --git a/libavcodec/vaapi_encode_h265.c b/libavcodec/vaapi_encode_h265.c
index 45f6f6de29..f3f0b4d2c0 100644
--- a/libavcodec/vaapi_encode_h265.c
+++ b/libavcodec/vaapi_encode_h265.c
@@ -798,8 +798,8 @@ static int vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
         vseq->intra_idr_period = 0;
         vseq->ip_period = 0;
 
-        vseq->pic_width_in_luma_samples  = ctx->aligned_width;
-        vseq->pic_height_in_luma_samples = ctx->aligned_height;
+        vseq->pic_width_in_luma_samples  = ctx->surface_width;
+        vseq->pic_height_in_luma_samples = ctx->surface_height;
 
         vseq->seq_fields.bits.chroma_format_idc = 1; // 4:2:0.
         vseq->seq_fields.bits.separate_colour_plane_flag = 0;
@@ -911,15 +911,15 @@ static int vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
         mseq->vps_poc_proportional_to_timing_flag = 1;
         mseq->vps_num_ticks_poc_diff_minus1 = 0;
 
-        if (ctx->input_width  != ctx->aligned_width ||
-            ctx->input_height != ctx->aligned_height) {
+        if (avctx->width  != ctx->surface_width ||
+            avctx->height != ctx->surface_height) {
             mseq->conformance_window_flag = 1;
             mseq->conf_win_left_offset   = 0;
             mseq->conf_win_right_offset  =
-                (ctx->aligned_width - ctx->input_width) / 2;
+                (ctx->surface_width - avctx->width) / 2;
             mseq->conf_win_top_offset    = 0;
             mseq->conf_win_bottom_offset =
-                (ctx->aligned_height - ctx->input_height) / 2;
+                (ctx->surface_height - avctx->height) / 2;
         } else {
             mseq->conformance_window_flag = 0;
         }
@@ -1154,99 +1154,78 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
     return 0;
 }
 
-static av_cold int vaapi_encode_h265_init_constant_bitrate(AVCodecContext *avctx)
+static av_cold int vaapi_encode_h265_configure(AVCodecContext *avctx)
 {
     VAAPIEncodeContext      *ctx = avctx->priv_data;
     VAAPIEncodeH265Context *priv = ctx->priv_data;
-    int hrd_buffer_size;
-    int hrd_initial_buffer_fullness;
+    VAAPIEncodeH265Options  *opt = ctx->codec_options;
 
-    if (avctx->bit_rate > INT32_MAX) {
-        av_log(avctx, AV_LOG_ERROR, "Target bitrate of 2^31 bps or "
-               "higher is not supported.\n");
-        return AVERROR(EINVAL);
+    priv->ctu_width     = FFALIGN(ctx->surface_width,  32) / 32;
+    priv->ctu_height    = FFALIGN(ctx->surface_height, 32) / 32;
+
+    av_log(avctx, AV_LOG_VERBOSE, "Input %ux%u -> Surface %ux%u -> CTU %ux%u.\n",
+           avctx->width, avctx->height, ctx->surface_width,
+           ctx->surface_height, priv->ctu_width, priv->ctu_height);
+
+    if (ctx->va_rc_mode == VA_RC_CQP) {
+        priv->fixed_qp_p = opt->qp;
+        if (avctx->i_quant_factor > 0.0)
+            priv->fixed_qp_idr = (int)((priv->fixed_qp_p * avctx->i_quant_factor +
+                                        avctx->i_quant_offset) + 0.5);
+        else
+            priv->fixed_qp_idr = priv->fixed_qp_p;
+        if (avctx->b_quant_factor > 0.0)
+            priv->fixed_qp_b = (int)((priv->fixed_qp_p * avctx->b_quant_factor +
+                                      avctx->b_quant_offset) + 0.5);
+        else
+            priv->fixed_qp_b = priv->fixed_qp_p;
+
+        av_log(avctx, AV_LOG_DEBUG, "Using fixed QP = "
+               "%d / %d / %d for IDR- / P- / B-frames.\n",
+               priv->fixed_qp_idr, priv->fixed_qp_p, priv->fixed_qp_b);
+
+    } else if (ctx->va_rc_mode == VA_RC_CBR) {
+        // These still need to be  set for pic_init_qp/slice_qp_delta.
+        priv->fixed_qp_idr = 30;
+        priv->fixed_qp_p   = 30;
+        priv->fixed_qp_b   = 30;
+
+        av_log(avctx, AV_LOG_DEBUG, "Using constant-bitrate = %d bps.\n",
+               avctx->bit_rate);
+
+    } else {
+        av_assert0(0 && "Invalid RC mode.");
     }
 
-    if (avctx->rc_buffer_size)
-        hrd_buffer_size = avctx->rc_buffer_size;
-    else
-        hrd_buffer_size = avctx->bit_rate;
-    if (avctx->rc_initial_buffer_occupancy)
-        hrd_initial_buffer_fullness = avctx->rc_initial_buffer_occupancy;
-    else
-        hrd_initial_buffer_fullness = hrd_buffer_size * 3 / 4;
-
-    priv->rc_params.misc.type = VAEncMiscParameterTypeRateControl;
-    priv->rc_params.rc = (VAEncMiscParameterRateControl) {
-        .bits_per_second   = avctx->bit_rate,
-        .target_percentage = 66,
-        .window_size       = 1000,
-        .initial_qp        = (avctx->qmax >= 0 ? avctx->qmax : 40),
-        .min_qp            = (avctx->qmin >= 0 ? avctx->qmin : 20),
-        .basic_unit_size   = 0,
-    };
-    ctx->global_params[ctx->nb_global_params] =
-        &priv->rc_params.misc;
-    ctx->global_params_size[ctx->nb_global_params++] =
-        sizeof(priv->rc_params);
-
-    priv->hrd_params.misc.type = VAEncMiscParameterTypeHRD;
-    priv->hrd_params.hrd = (VAEncMiscParameterHRD) {
-        .initial_buffer_fullness = hrd_initial_buffer_fullness,
-        .buffer_size             = hrd_buffer_size,
-    };
-    ctx->global_params[ctx->nb_global_params] =
-        &priv->hrd_params.misc;
-    ctx->global_params_size[ctx->nb_global_params++] =
-        sizeof(priv->hrd_params);
-
-    // These still need to be  set for pic_init_qp/slice_qp_delta.
-    priv->fixed_qp_idr = 30;
-    priv->fixed_qp_p   = 30;
-    priv->fixed_qp_b   = 30;
-
-    av_log(avctx, AV_LOG_DEBUG, "Using constant-bitrate = %"PRId64" bps.\n",
-           avctx->bit_rate);
     return 0;
 }
 
-static av_cold int vaapi_encode_h265_init_fixed_qp(AVCodecContext *avctx)
-{
-    VAAPIEncodeContext      *ctx = avctx->priv_data;
-    VAAPIEncodeH265Context *priv = ctx->priv_data;
-    VAAPIEncodeH265Options  *opt = ctx->codec_options;
+static const VAAPIEncodeType vaapi_encode_type_h265 = {
+    .priv_data_size        = sizeof(VAAPIEncodeH265Context),
 
-    priv->fixed_qp_p = opt->qp;
-    if (avctx->i_quant_factor > 0.0)
-        priv->fixed_qp_idr = (int)((priv->fixed_qp_p * avctx->i_quant_factor +
-                                    avctx->i_quant_offset) + 0.5);
-    else
-        priv->fixed_qp_idr = priv->fixed_qp_p;
-    if (avctx->b_quant_factor > 0.0)
-        priv->fixed_qp_b = (int)((priv->fixed_qp_p * avctx->b_quant_factor +
-                                  avctx->b_quant_offset) + 0.5);
-    else
-        priv->fixed_qp_b = priv->fixed_qp_p;
+    .configure             = &vaapi_encode_h265_configure,
 
-    av_log(avctx, AV_LOG_DEBUG, "Using fixed QP = "
-           "%d / %d / %d for IDR- / P- / B-frames.\n",
-           priv->fixed_qp_idr, priv->fixed_qp_p, priv->fixed_qp_b);
-    return 0;
-}
+    .sequence_params_size  = sizeof(VAEncSequenceParameterBufferHEVC),
+    .init_sequence_params  = &vaapi_encode_h265_init_sequence_params,
 
-static av_cold int vaapi_encode_h265_init_internal(AVCodecContext *avctx)
+    .picture_params_size   = sizeof(VAEncPictureParameterBufferHEVC),
+    .init_picture_params   = &vaapi_encode_h265_init_picture_params,
+
+    .slice_params_size     = sizeof(VAEncSliceParameterBufferHEVC),
+    .init_slice_params     = &vaapi_encode_h265_init_slice_params,
+
+    .sequence_header_type  = VAEncPackedHeaderSequence,
+    .write_sequence_header = &vaapi_encode_h265_write_sequence_header,
+
+    .slice_header_type     = VAEncPackedHeaderHEVC_Slice,
+    .write_slice_header    = &vaapi_encode_h265_write_slice_header,
+};
+
+static av_cold int vaapi_encode_h265_init(AVCodecContext *avctx)
 {
-    static const VAConfigAttrib default_config_attributes[] = {
-        { .type  = VAConfigAttribRTFormat,
-          .value = VA_RT_FORMAT_YUV420 },
-        { .type  = VAConfigAttribEncPackedHeaders,
-          .value = (VA_ENC_PACKED_HEADER_SEQUENCE |
-                    VA_ENC_PACKED_HEADER_SLICE) },
-    };
+    VAAPIEncodeContext *ctx = avctx->priv_data;
 
-    VAAPIEncodeContext      *ctx = avctx->priv_data;
-    VAAPIEncodeH265Context *priv = ctx->priv_data;
-    int i, err;
+    ctx->codec = &vaapi_encode_type_h265;
 
     switch (avctx->profile) {
     case FF_PROFILE_HEVC_MAIN:
@@ -1264,66 +1243,19 @@ static av_cold int vaapi_encode_h265_init_internal(AVCodecContext *avctx)
     }
     ctx->va_entrypoint = VAEntrypointEncSlice;
 
-    ctx->input_width    = avctx->width;
-    ctx->input_height   = avctx->height;
-    ctx->aligned_width  = FFALIGN(ctx->input_width,  16);
-    ctx->aligned_height = FFALIGN(ctx->input_height, 16);
-    priv->ctu_width     = FFALIGN(ctx->aligned_width,  32) / 32;
-    priv->ctu_height    = FFALIGN(ctx->aligned_height, 32) / 32;
-
-    av_log(avctx, AV_LOG_VERBOSE, "Input %ux%u -> Aligned %ux%u -> CTU %ux%u.\n",
-           ctx->input_width, ctx->input_height, ctx->aligned_width,
-           ctx->aligned_height, priv->ctu_width, priv->ctu_height);
+    // This will be dependent on profile when 10-bit is supported.
+    ctx->va_rt_format = VA_RT_FORMAT_YUV420;
 
-    for (i = 0; i < FF_ARRAY_ELEMS(default_config_attributes); i++) {
-        ctx->config_attributes[ctx->nb_config_attributes++] =
-            default_config_attributes[i];
-    }
-
-    if (avctx->bit_rate > 0) {
+    if (avctx->bit_rate > 0)
         ctx->va_rc_mode = VA_RC_CBR;
-        err = vaapi_encode_h265_init_constant_bitrate(avctx);
-    } else {
+    else
         ctx->va_rc_mode = VA_RC_CQP;
-        err = vaapi_encode_h265_init_fixed_qp(avctx);
-    }
-    if (err < 0)
-        return err;
 
-    ctx->config_attributes[ctx->nb_config_attributes++] = (VAConfigAttrib) {
-        .type  = VAConfigAttribRateControl,
-        .value = ctx->va_rc_mode,
-    };
 
-    ctx->nb_recon_frames = 20;
+    ctx->surface_width  = FFALIGN(avctx->width,  16);
+    ctx->surface_height = FFALIGN(avctx->height, 16);
 
-    return 0;
-}
-
-static VAAPIEncodeType vaapi_encode_type_h265 = {
-    .priv_data_size        = sizeof(VAAPIEncodeH265Context),
-
-    .init                  = &vaapi_encode_h265_init_internal,
-
-    .sequence_params_size  = sizeof(VAEncSequenceParameterBufferHEVC),
-    .init_sequence_params  = &vaapi_encode_h265_init_sequence_params,
-
-    .picture_params_size   = sizeof(VAEncPictureParameterBufferHEVC),
-    .init_picture_params   = &vaapi_encode_h265_init_picture_params,
-
-    .slice_params_size     = sizeof(VAEncSliceParameterBufferHEVC),
-    .init_slice_params     = &vaapi_encode_h265_init_slice_params,
-
-    .sequence_header_type  = VAEncPackedHeaderSequence,
-    .write_sequence_header = &vaapi_encode_h265_write_sequence_header,
-
-    .slice_header_type     = VAEncPackedHeaderHEVC_Slice,
-    .write_slice_header    = &vaapi_encode_h265_write_slice_header,
-};
-
-static av_cold int vaapi_encode_h265_init(AVCodecContext *avctx)
-{
-    return ff_vaapi_encode_init(avctx, &vaapi_encode_type_h265);
+    return ff_vaapi_encode_init(avctx);
 }
 
 #define OFFSET(x) (offsetof(VAAPIEncodeContext, codec_options_data) + \