From e22ebd04bcab7f86548794556c28ecca46d9c2ac Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sat, 11 Jan 2014 20:13:20 +0100 Subject: hevc: Bound check cu_qp_delta The T-REC-H.265-2013044 page 91 states it has to be in the range [-(26 + s->sps->qp_bd_offset / 2), (25 + s->sps->qp_bd_offset / 2)]. --- libavcodec/hevc.c | 77 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 23 deletions(-) (limited to 'libavcodec') diff --git a/libavcodec/hevc.c b/libavcodec/hevc.c index 9c47a73b4b..859b2abade 100644 --- a/libavcodec/hevc.c +++ b/libavcodec/hevc.c @@ -1224,10 +1224,10 @@ static void hls_residual_coding(HEVCContext *s, int x0, int y0, } } -static void hls_transform_unit(HEVCContext *s, int x0, int y0, - int xBase, int yBase, int cb_xBase, int cb_yBase, - int log2_cb_size, int log2_trafo_size, - int trafo_depth, int blk_idx) +static int hls_transform_unit(HEVCContext *s, int x0, int y0, + int xBase, int yBase, int cb_xBase, int cb_yBase, + int log2_cb_size, int log2_trafo_size, + int trafo_depth, int blk_idx) { HEVCLocalContext *lc = &s->HEVClc; @@ -1262,6 +1262,18 @@ static void hls_transform_unit(HEVCContext *s, int x0, int y0, if (ff_hevc_cu_qp_delta_sign_flag(s) == 1) lc->tu.cu_qp_delta = -lc->tu.cu_qp_delta; lc->tu.is_cu_qp_delta_coded = 1; + + if (lc->tu.cu_qp_delta < -(26 + s->sps->qp_bd_offset / 2) || + lc->tu.cu_qp_delta > (25 + s->sps->qp_bd_offset / 2)) { + av_log(s->avctx, AV_LOG_ERROR, + "The cu_qp_delta %d is outside the valid range " + "[%d, %d].\n", + lc->tu.cu_qp_delta, + -(26 + s->sps->qp_bd_offset / 2), + (25 + s->sps->qp_bd_offset / 2)); + return AVERROR_INVALIDDATA; + } + ff_hevc_set_qPy(s, x0, y0, cb_xBase, cb_yBase, log2_cb_size); } @@ -1297,6 +1309,7 @@ static void hls_transform_unit(HEVCContext *s, int x0, int y0, hls_residual_coding(s, xBase, yBase, log2_trafo_size, scan_idx_c, 2); } } + return 0; } static void set_deblocking_bypass(HEVCContext *s, int x0, int y0, int log2_cb_size) @@ -1314,13 +1327,14 @@ static void set_deblocking_bypass(HEVCContext *s, int x0, int y0, int log2_cb_si s->is_pcm[i + j * min_pu_width] = 2; } -static void hls_transform_tree(HEVCContext *s, int x0, int y0, - int xBase, int yBase, int cb_xBase, int cb_yBase, - int log2_cb_size, int log2_trafo_size, - int trafo_depth, int blk_idx) +static int hls_transform_tree(HEVCContext *s, int x0, int y0, + int xBase, int yBase, int cb_xBase, int cb_yBase, + int log2_cb_size, int log2_trafo_size, + int trafo_depth, int blk_idx) { HEVCLocalContext *lc = &s->HEVClc; uint8_t split_transform_flag; + int ret; if (trafo_depth > 0 && log2_trafo_size == 2) { SAMPLE_CBF(lc->tt.cbf_cb[trafo_depth], x0, y0) = @@ -1375,14 +1389,26 @@ static void hls_transform_tree(HEVCContext *s, int x0, int y0, int x1 = x0 + ((1 << log2_trafo_size) >> 1); int y1 = y0 + ((1 << log2_trafo_size) >> 1); - hls_transform_tree(s, x0, y0, x0, y0, cb_xBase, cb_yBase, log2_cb_size, - log2_trafo_size - 1, trafo_depth + 1, 0); - hls_transform_tree(s, x1, y0, x0, y0, cb_xBase, cb_yBase, log2_cb_size, - log2_trafo_size - 1, trafo_depth + 1, 1); - hls_transform_tree(s, x0, y1, x0, y0, cb_xBase, cb_yBase, log2_cb_size, - log2_trafo_size - 1, trafo_depth + 1, 2); - hls_transform_tree(s, x1, y1, x0, y0, cb_xBase, cb_yBase, log2_cb_size, - log2_trafo_size - 1, trafo_depth + 1, 3); + ret = hls_transform_tree(s, x0, y0, x0, y0, cb_xBase, cb_yBase, + log2_cb_size, log2_trafo_size - 1, + trafo_depth + 1, 0); + if (ret < 0) + return ret; + ret = hls_transform_tree(s, x1, y0, x0, y0, cb_xBase, cb_yBase, + log2_cb_size, log2_trafo_size - 1, + trafo_depth + 1, 1); + if (ret < 0) + return ret; + ret = hls_transform_tree(s, x0, y1, x0, y0, cb_xBase, cb_yBase, + log2_cb_size, log2_trafo_size - 1, + trafo_depth + 1, 2); + if (ret < 0) + return ret; + ret = hls_transform_tree(s, x1, y1, x0, y0, cb_xBase, cb_yBase, + log2_cb_size, log2_trafo_size - 1, + trafo_depth + 1, 3); + if (ret < 0) + return ret; } else { int min_tu_size = 1 << s->sps->log2_min_tb_size; int log2_min_tu_size = s->sps->log2_min_tb_size; @@ -1394,9 +1420,11 @@ static void hls_transform_tree(HEVCContext *s, int x0, int y0, lc->tt.cbf_luma = ff_hevc_cbf_luma_decode(s, trafo_depth); } - hls_transform_unit(s, x0, y0, xBase, yBase, cb_xBase, cb_yBase, - log2_cb_size, log2_trafo_size, trafo_depth, blk_idx); - + ret = hls_transform_unit(s, x0, y0, xBase, yBase, cb_xBase, cb_yBase, + log2_cb_size, log2_trafo_size, trafo_depth, + blk_idx); + if (ret < 0) + return ret; // TODO: store cbf_luma somewhere else if (lc->tt.cbf_luma) { int i, j; @@ -1416,6 +1444,7 @@ static void hls_transform_tree(HEVCContext *s, int x0, int y0, set_deblocking_bypass(s, x0, y0, log2_trafo_size); } } + return 0; } static int hls_pcm_sample(HEVCContext *s, int x0, int y0, int log2_cb_size) @@ -2048,7 +2077,7 @@ static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size) int min_cb_width = s->sps->min_cb_width; int x_cb = x0 >> log2_min_cb_size; int y_cb = y0 >> log2_min_cb_size; - int x, y; + int x, y, ret; lc->cu.x = x0; lc->cu.y = y0; @@ -2105,7 +2134,6 @@ static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size) lc->cu.pcm_flag = ff_hevc_pcm_flag_decode(s); } if (lc->cu.pcm_flag) { - int ret; intra_prediction_unit_default_value(s, x0, y0, log2_cb_size); ret = hls_pcm_sample(s, x0, y0, log2_cb_size); if (s->sps->pcm.loop_filter_disable_flag) @@ -2164,8 +2192,11 @@ static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size) lc->cu.max_trafo_depth = lc->cu.pred_mode == MODE_INTRA ? s->sps->max_transform_hierarchy_depth_intra + lc->cu.intra_split_flag : s->sps->max_transform_hierarchy_depth_inter; - hls_transform_tree(s, x0, y0, x0, y0, x0, y0, log2_cb_size, - log2_cb_size, 0, 0); + ret = hls_transform_tree(s, x0, y0, x0, y0, x0, y0, + log2_cb_size, + log2_cb_size, 0, 0); + if (ret < 0) + return ret; } else { if (!s->sh.disable_deblocking_filter_flag) ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_cb_size, -- cgit v1.2.3