avcodec/nvenc: Optimize nvenc parameters

Add 3 more presets: fast, medium, slow.
Improve min/max QP calculation.

Signed-off-by: Timo Rothenpieler <timo@rothenpieler.org>

1 files changed, 73 insertions, 23 deletions

diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c
index 3174b01c2a..15cf7937d7 100644
--- a/libavcodec/nvenc.c
+++ b/libavcodec/nvenc.c
@@ -408,7 +408,7 @@ static av_cold int nvenc_check_cuda(AVCodecContext *avctx)
         target_smver = 0x52;
         break;
     default:
-        av_log(avctx, AV_LOG_FATAL, "nvenc: Unknown codec name\n");
+        av_log(avctx, AV_LOG_FATAL, "Unknown codec name\n");
         goto error;
     }
 
@@ -556,6 +556,7 @@ static av_cold int nvenc_encode_init(AVCodecContext *avctx)
     int lossless = 0;
     int res = 0;
     int dw, dh;
+    int qp_inter_p;
 
     NvencContext *ctx = avctx->priv_data;
     NvencDynLoadFunctions *dl_fn = &ctx->nvenc_dload_funcs;
@@ -602,16 +603,25 @@ static av_cold int nvenc_encode_init(AVCodecContext *avctx)
     nv_status = p_nvenc->nvEncOpenEncodeSessionEx(&encode_session_params, &ctx->nvencoder);
     if (nv_status != NV_ENC_SUCCESS) {
         ctx->nvencoder = NULL;
-        av_log(avctx, AV_LOG_FATAL, "OpenEncodeSessionEx failed: 0x%x - invalid license key?\n", (int)nv_status);
+        av_log(avctx, AV_LOG_FATAL, "OpenEncodeSessionEx failed: 0x%x\n", (int)nv_status);
         res = AVERROR_EXTERNAL;
         goto error;
     }
 
     if (ctx->preset) {
-        if (!strcmp(ctx->preset, "hp")) {
+        if (!strcmp(ctx->preset, "slow")) {
+            encoder_preset = NV_ENC_PRESET_HQ_GUID;
+            ctx->twopass = 1;
+        } else if (!strcmp(ctx->preset, "medium")) {
+            encoder_preset = NV_ENC_PRESET_HQ_GUID;
+            ctx->twopass = 0;
+        } else if (!strcmp(ctx->preset, "fast")) {
             encoder_preset = NV_ENC_PRESET_HP_GUID;
+            ctx->twopass = 0;
         } else if (!strcmp(ctx->preset, "hq")) {
             encoder_preset = NV_ENC_PRESET_HQ_GUID;
+        } else if (!strcmp(ctx->preset, "hp")) {
+            encoder_preset = NV_ENC_PRESET_HP_GUID;
         } else if (!strcmp(ctx->preset, "bd")) {
             encoder_preset = NV_ENC_PRESET_BD_GUID;
         } else if (!strcmp(ctx->preset, "ll")) {
@@ -632,12 +642,16 @@ static av_cold int nvenc_encode_init(AVCodecContext *avctx)
         } else if (!strcmp(ctx->preset, "default")) {
             encoder_preset = NV_ENC_PRESET_DEFAULT_GUID;
         } else {
-            av_log(avctx, AV_LOG_FATAL, "Preset \"%s\" is unknown! Supported presets: hp, hq, bd, ll, llhp, llhq, lossless, losslesshp, default\n", ctx->preset);
+            av_log(avctx, AV_LOG_FATAL, "Preset \"%s\" is unknown! Supported presets: slow, medium, high, hp, hq, bd, ll, llhp, llhq, lossless, losslesshp, default\n", ctx->preset);
             res = AVERROR(EINVAL);
             goto error;
         }
     }
 
+    if (ctx->twopass < 0) {
+        ctx->twopass = isLL;
+    }
+
     switch (avctx->codec->id) {
     case AV_CODEC_ID_H264:
         codec = NV_ENC_CODEC_H264_GUID;
@@ -646,7 +660,7 @@ static av_cold int nvenc_encode_init(AVCodecContext *avctx)
         codec = NV_ENC_CODEC_HEVC_GUID;
         break;
     default:
-        av_log(avctx, AV_LOG_ERROR, "nvenc: Unknown codec name\n");
+        av_log(avctx, AV_LOG_ERROR, "Unknown codec name\n");
         res = AVERROR(EINVAL);
         goto error;
     }
@@ -728,6 +742,8 @@ static av_cold int nvenc_encode_init(AVCodecContext *avctx)
         switch (avctx->codec->id) {
         case AV_CODEC_ID_H264:
             ctx->encode_config.encodeCodecConfig.h264Config.idrPeriod = avctx->gop_size;
+            ctx->encode_config.encodeCodecConfig.h264Config.hierarchicalPFrames = 1;
+            ctx->encode_config.encodeCodecConfig.h264Config.hierarchicalBFrames = 1;
             break;
         case AV_CODEC_ID_H265:
             ctx->encode_config.encodeCodecConfig.hevcConfig.idrPeriod = avctx->gop_size;
@@ -772,15 +788,13 @@ static av_cold int nvenc_encode_init(AVCodecContext *avctx)
     } else if (ctx->cbr) {
         if (!ctx->twopass) {
             ctx->encode_config.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR;
-        } else if (ctx->twopass == 1 || isLL) {
+        } else {
             ctx->encode_config.rcParams.rateControlMode = NV_ENC_PARAMS_RC_2_PASS_QUALITY;
 
             if (avctx->codec->id == AV_CODEC_ID_H264) {
                 ctx->encode_config.encodeCodecConfig.h264Config.adaptiveTransformMode = NV_ENC_H264_ADAPTIVE_TRANSFORM_ENABLE;
                 ctx->encode_config.encodeCodecConfig.h264Config.fmoMode = NV_ENC_H264_FMO_DISABLE;
             }
-        } else {
-            ctx->encode_config.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR;
         }
     } else if (avctx->global_quality > 0) {
         ctx->encode_config.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CONSTQP;
@@ -791,7 +805,7 @@ static av_cold int nvenc_encode_init(AVCodecContext *avctx)
         avctx->qmin = -1;
         avctx->qmax = -1;
     } else if (avctx->qmin >= 0 && avctx->qmax >= 0) {
-        if (ctx->twopass == 1) {
+        if (ctx->twopass) {
             ctx->encode_config.rcParams.rateControlMode = NV_ENC_PARAMS_RC_2_PASS_VBR;
 
             if (avctx->codec->id == AV_CODEC_ID_H264) {
@@ -799,19 +813,55 @@ static av_cold int nvenc_encode_init(AVCodecContext *avctx)
                 ctx->encode_config.encodeCodecConfig.h264Config.fmoMode = NV_ENC_H264_FMO_DISABLE;
             }
         } else {
-            ctx->encode_config.rcParams.rateControlMode = NV_ENC_PARAMS_RC_VBR;
+            ctx->encode_config.rcParams.rateControlMode = NV_ENC_PARAMS_RC_VBR_MINQP;
         }
 
         ctx->encode_config.rcParams.enableMinQP = 1;
         ctx->encode_config.rcParams.enableMaxQP = 1;
 
-        ctx->encode_config.rcParams.minQP.qpInterB = avctx->qmin;
-        ctx->encode_config.rcParams.minQP.qpInterP = avctx->qmin;
+        ctx->encode_config.rcParams.minQP.qpInterB = avctx->qmin; // * fabs(avctx->b_quant_factor);
+        ctx->encode_config.rcParams.minQP.qpInterP = avctx->qmin; // * fabs(avctx->i_quant_factor);
         ctx->encode_config.rcParams.minQP.qpIntra = avctx->qmin;
 
-        ctx->encode_config.rcParams.maxQP.qpInterB = avctx->qmax;
-        ctx->encode_config.rcParams.maxQP.qpInterP = avctx->qmax;
+        ctx->encode_config.rcParams.maxQP.qpInterB = avctx->qmax; // / fabs(avctx->b_quant_factor);
+        ctx->encode_config.rcParams.maxQP.qpInterP = avctx->qmax; // / fabs(avctx->i_quant_factor);
         ctx->encode_config.rcParams.maxQP.qpIntra = avctx->qmax;
+
+        qp_inter_p = (avctx->qmax + 3 * avctx->qmin) / 4; // biased towards Qmin
+        ctx->encode_config.rcParams.initialRCQP.qpInterP  = qp_inter_p;
+
+        if (avctx->i_quant_factor != 0.0 && avctx->b_quant_factor != 0.0) {
+            ctx->encode_config.rcParams.initialRCQP.qpIntra = qp_inter_p * fabs(avctx->i_quant_factor);
+            ctx->encode_config.rcParams.initialRCQP.qpIntra += qp_inter_p * avctx->i_quant_offset;
+            ctx->encode_config.rcParams.initialRCQP.qpInterB = qp_inter_p * fabs(avctx->b_quant_factor);
+            ctx->encode_config.rcParams.initialRCQP.qpInterB += qp_inter_p * avctx->b_quant_offset;
+        } else {
+            ctx->encode_config.rcParams.initialRCQP.qpIntra = qp_inter_p;
+            ctx->encode_config.rcParams.initialRCQP.qpInterB = qp_inter_p;
+        }
+
+        ctx->encode_config.rcParams.enableInitialRCQP = 1;
+    } else {
+        if (!ctx->twopass) {
+            ctx->encode_config.rcParams.rateControlMode = NV_ENC_PARAMS_RC_VBR;
+        } else {
+            ctx->encode_config.rcParams.rateControlMode = NV_ENC_PARAMS_RC_2_PASS_VBR;
+        }
+
+        qp_inter_p = 26; // default to 26
+        ctx->encode_config.rcParams.initialRCQP.qpInterP = qp_inter_p;
+
+        if(avctx->i_quant_factor != 0.0 && avctx->b_quant_factor != 0.0) {
+            ctx->encode_config.rcParams.initialRCQP.qpIntra = qp_inter_p * fabs(avctx->i_quant_factor);
+            ctx->encode_config.rcParams.initialRCQP.qpIntra += qp_inter_p * avctx->i_quant_offset;
+            ctx->encode_config.rcParams.initialRCQP.qpInterB = qp_inter_p * fabs(avctx->b_quant_factor);
+            ctx->encode_config.rcParams.initialRCQP.qpInterB += qp_inter_p * avctx->b_quant_offset;
+        } else {
+            ctx->encode_config.rcParams.initialRCQP.qpIntra = qp_inter_p;
+            ctx->encode_config.rcParams.initialRCQP.qpInterB = qp_inter_p;
+        }
+
+        ctx->encode_config.rcParams.enableInitialRCQP = 1;
     }
 
     if (avctx->rc_buffer_size > 0)
@@ -1113,7 +1163,7 @@ static int process_output_surface(AVCodecContext *avctx, AVPacket *pkt, NvencOut
       slice_mode_data = ctx->encode_config.encodeCodecConfig.hevcConfig.sliceModeData;
       break;
     default:
-      av_log(avctx, AV_LOG_ERROR, "nvenc: Unknown codec name\n");
+      av_log(avctx, AV_LOG_ERROR, "Unknown codec name\n");
       res = AVERROR(EINVAL);
       goto error;
     }
@@ -1135,7 +1185,7 @@ static int process_output_surface(AVCodecContext *avctx, AVPacket *pkt, NvencOut
         goto error;
     }
 
-    if (res = ff_alloc_packet2(avctx, pkt, lock_params.bitstreamSizeInBytes, 0)) {
+    if (res = ff_alloc_packet2(avctx, pkt, lock_params.bitstreamSizeInBytes,0)) {
         p_nvenc->nvEncUnlockBitstream(ctx->nvencoder, tmpoutsurf->output_surface);
         goto error;
     }
@@ -1338,7 +1388,7 @@ static int nvenc_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
           pic_params.codecPicParams.hevcPicParams.sliceModeData = ctx->encode_config.encodeCodecConfig.hevcConfig.sliceModeData;
           break;
         default:
-          av_log(avctx, AV_LOG_ERROR, "nvenc: Unknown codec name\n");
+          av_log(avctx, AV_LOG_ERROR, "Unknown codec name\n");
           return AVERROR(EINVAL);
         }
 
@@ -1415,12 +1465,12 @@ static const enum AVPixelFormat pix_fmts_nvenc[] = {
 #define OFFSET(x) offsetof(NvencContext, x)
 #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
 static const AVOption options[] = {
-    { "preset", "Set the encoding preset (one of hq, hp, bd, ll, llhq, llhp, default)", OFFSET(preset), AV_OPT_TYPE_STRING, { .str = "hq" }, 0, 0, VE },
+    { "preset", "Set the encoding preset (one of slow = hq 2pass, medium = hq, fast = hp, hq, hp, bd, ll, llhq, llhp, default)", OFFSET(preset), AV_OPT_TYPE_STRING, { .str = "hq" }, 0, 0, VE },
     { "profile", "Set the encoding profile (high, main or baseline)", OFFSET(profile), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VE },
     { "level", "Set the encoding level restriction (auto, 1.0, 1.0b, 1.1, 1.2, ..., 4.2, 5.0, 5.1)", OFFSET(level), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VE },
     { "tier", "Set the encoding tier (main or high)", OFFSET(tier), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VE },
-    { "cbr", "Use cbr encoding mode", OFFSET(cbr), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE },
-    { "2pass", "Use 2pass cbr encoding mode", OFFSET(twopass), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, 1, VE },
+    { "cbr", "Use cbr encoding mode", OFFSET(cbr), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
+    { "2pass", "Use 2pass encoding mode", OFFSET(twopass), AV_OPT_TYPE_BOOL, { .i64 = -1 }, -1, 1, VE },
     { "gpu", "Selects which NVENC capable GPU to use. First GPU is 0, second is 1, and so on.", OFFSET(gpu), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, VE },
     { "delay", "Delays frame output by the given amount of frames.", OFFSET(buffer_delay), AV_OPT_TYPE_INT, { .i64 = INT_MAX }, 0, INT_MAX, VE },
     { NULL }
@@ -1446,7 +1496,7 @@ static const AVClass nvenc_class = {
 
 AVCodec ff_nvenc_encoder = {
     .name = "nvenc",
-    .long_name = NULL_IF_CONFIG_SMALL("Nvidia NVENC h264 encoder"),
+    .long_name = NULL_IF_CONFIG_SMALL("NVIDIA NVENC h264 encoder"),
     .type = AVMEDIA_TYPE_VIDEO,
     .id = AV_CODEC_ID_H264,
     .priv_data_size = sizeof(NvencContext),
@@ -1471,7 +1521,7 @@ static const AVClass nvenc_h264_class = {
 
 AVCodec ff_nvenc_h264_encoder = {
     .name = "nvenc_h264",
-    .long_name = NULL_IF_CONFIG_SMALL("Nvidia NVENC h264 encoder"),
+    .long_name = NULL_IF_CONFIG_SMALL("NVIDIA NVENC h264 encoder"),
     .type = AVMEDIA_TYPE_VIDEO,
     .id = AV_CODEC_ID_H264,
     .priv_data_size = sizeof(NvencContext),
@@ -1495,7 +1545,7 @@ static const AVClass nvenc_hevc_class = {
 
 AVCodec ff_nvenc_hevc_encoder = {
     .name = "nvenc_hevc",
-    .long_name = NULL_IF_CONFIG_SMALL("Nvidia NVENC hevc encoder"),
+    .long_name = NULL_IF_CONFIG_SMALL("NVIDIA NVENC hevc encoder"),
     .type = AVMEDIA_TYPE_VIDEO,
     .id = AV_CODEC_ID_H265,
     .priv_data_size = sizeof(NvencContext),