From 4cc367bf9dc9d25cd7f96b4ceb5f6e648466cc43 Mon Sep 17 00:00:00 2001 From: Mark Thompson Date: Tue, 18 Sep 2018 23:30:49 +0100 Subject: vaapi_encode_h264: Properly set constraint flags constraint_set1_flag should be set for constrained baseline and main profiles, because the stream conforms to main profile. constraint_set3_flag should be set for high profile when the stream is intra-only. constraint_set4_flag should always be set for main and high profiles because interlaced encoding is not supported. constraint_set5_flag should be set for main and high profiles when B-frames are not used. Also fix the setting of max_num_ref_frames - use the gop_size value to check for intra-only rather than the constraint flag (which is not necessarily set). --- libavcodec/vaapi_encode_h264.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) (limited to 'libavcodec/vaapi_encode_h264.c') diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c index b233a91d04..bba6848f03 100644 --- a/libavcodec/vaapi_encode_h264.c +++ b/libavcodec/vaapi_encode_h264.c @@ -305,10 +305,19 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx) sps->nal_unit_header.nal_unit_type = H264_NAL_SPS; sps->profile_idc = avctx->profile & 0xff; - sps->constraint_set1_flag = - !!(avctx->profile & FF_PROFILE_H264_CONSTRAINED); - sps->constraint_set3_flag = - !!(avctx->profile & FF_PROFILE_H264_INTRA); + + if (avctx->profile == FF_PROFILE_H264_CONSTRAINED_BASELINE || + avctx->profile == FF_PROFILE_H264_MAIN) + sps->constraint_set1_flag = 1; + + if (avctx->profile == FF_PROFILE_H264_HIGH) + sps->constraint_set3_flag = ctx->gop_size == 1; + + if (avctx->profile == FF_PROFILE_H264_MAIN || + avctx->profile == FF_PROFILE_H264_HIGH) { + sps->constraint_set4_flag = 1; + sps->constraint_set5_flag = ctx->b_per_p == 0; + } sps->level_idc = avctx->level; @@ -321,8 +330,7 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx) av_clip(av_log2(ctx->b_per_p + 1) - 2, 0, 12); sps->max_num_ref_frames = - (avctx->profile & FF_PROFILE_H264_INTRA) ? 0 : - 1 + (ctx->b_per_p > 0); + ctx->gop_size == 1 ? 0 : 1 + (ctx->b_per_p > 0); sps->pic_width_in_mbs_minus1 = priv->mb_width - 1; sps->pic_height_in_map_units_minus1 = priv->mb_height - 1; -- cgit v1.2.3