From a830056b32feb2db463e062d4ec2b6fb47ec70ee Mon Sep 17 00:00:00 2001 From: Mark Thompson Date: Sun, 28 Oct 2018 17:13:53 +0000 Subject: vaapi_encode_h264: Reduce SAR to valid range The SAR of the input could have a numerator or denominator greater than 2^16 which would then be truncated to a 16-bit integer when written to the VUI parameters, giving a random result. Instead, reduce the SAR to the nearest representable fraction. Fixes #7502. --- libavcodec/vaapi_encode_h264.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'libavcodec/vaapi_encode_h264.c') diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c index 7bb77cfba2..f9402992b8 100644 --- a/libavcodec/vaapi_encode_h264.c +++ b/libavcodec/vaapi_encode_h264.c @@ -389,18 +389,20 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx) { 80, 33 }, { 18, 11 }, { 15, 11 }, { 64, 33 }, { 160, 99 }, { 4, 3 }, { 3, 2 }, { 2, 1 }, }; - int i; + int num, den, i; + av_reduce(&num, &den, avctx->sample_aspect_ratio.num, + avctx->sample_aspect_ratio.den, 65535); for (i = 0; i < FF_ARRAY_ELEMS(sar_idc); i++) { - if (avctx->sample_aspect_ratio.num == sar_idc[i].num && - avctx->sample_aspect_ratio.den == sar_idc[i].den) { + if (num == sar_idc[i].num && + den == sar_idc[i].den) { sps->vui.aspect_ratio_idc = i; break; } } if (i >= FF_ARRAY_ELEMS(sar_idc)) { sps->vui.aspect_ratio_idc = 255; - sps->vui.sar_width = avctx->sample_aspect_ratio.num; - sps->vui.sar_height = avctx->sample_aspect_ratio.den; + sps->vui.sar_width = num; + sps->vui.sar_height = den; } sps->vui.aspect_ratio_info_present_flag = 1; } -- cgit v1.2.3