avcodec/nvenc: Add support for 2pass rc in vbr mode

Thanks to WereCatf for pointing out this now exists.

Github: Closes #143

1 files changed, 10 insertions, 1 deletions

diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c
index d9dcf964f5..54906526eb 100644
--- a/libavcodec/nvenc.c
+++ b/libavcodec/nvenc.c
@@ -791,7 +791,16 @@ static av_cold int nvenc_encode_init(AVCodecContext *avctx)
         avctx->qmin = -1;
         avctx->qmax = -1;
     } else if (avctx->qmin >= 0 && avctx->qmax >= 0) {
-        ctx->encode_config.rcParams.rateControlMode = NV_ENC_PARAMS_RC_VBR;
+        if (ctx->twopass == 1) {
+            ctx->encode_config.rcParams.rateControlMode = NV_ENC_PARAMS_RC_2_PASS_VBR;
+
+            if (avctx->codec->id == AV_CODEC_ID_H264) {
+                ctx->encode_config.encodeCodecConfig.h264Config.adaptiveTransformMode = NV_ENC_H264_ADAPTIVE_TRANSFORM_ENABLE;
+                ctx->encode_config.encodeCodecConfig.h264Config.fmoMode = NV_ENC_H264_FMO_DISABLE;
+            }
+        } else {
+            ctx->encode_config.rcParams.rateControlMode = NV_ENC_PARAMS_RC_VBR;
+        }
 
         ctx->encode_config.rcParams.enableMinQP = 1;
         ctx->encode_config.rcParams.enableMaxQP = 1;