From 6770057ac97e78c799f06eb3769fecdb0833d44f Mon Sep 17 00:00:00 2001 From: Andreas Rheinhardt Date: Tue, 29 Sep 2020 14:58:24 +0200 Subject: avcodec/dvenc: Fix undefined left shift of negative numbers The earlier code was based on the assumption that AVFrame.linesize can not be negative. Fixes ticket #8280. Signed-off-by: Andreas Rheinhardt --- libavcodec/dvenc.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'libavcodec') diff --git a/libavcodec/dvenc.c b/libavcodec/dvenc.c index 3e98d1b38c..21a8b841f6 100644 --- a/libavcodec/dvenc.c +++ b/libavcodec/dvenc.c @@ -215,8 +215,8 @@ static av_always_inline int dv_guess_dct_mode(DVVideoContext *s, uint8_t *data, if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) { int ps = s->ildct_cmp(NULL, data, NULL, linesize, 8) - 400; if (ps > 0) { - int is = s->ildct_cmp(NULL, data, NULL, linesize << 1, 4) + - s->ildct_cmp(NULL, data + linesize, NULL, linesize << 1, 4); + int is = s->ildct_cmp(NULL, data, NULL, linesize * 2, 4) + + s->ildct_cmp(NULL, data + linesize, NULL, linesize * 2, 4); return ps > is; } } @@ -511,7 +511,7 @@ static av_always_inline int dv_init_enc_block(EncBlockInfo* bi, uint8_t *data, i if (data) { if (DV_PROFILE_IS_HD(s->sys)) { - s->get_pixels(blk, data, linesize << bi->dct_mode); + s->get_pixels(blk, data, linesize * (1 << bi->dct_mode)); s->fdct[0](blk); } else { bi->dct_mode = dv_guess_dct_mode(s, data, linesize); @@ -860,7 +860,7 @@ static int dv_encode_video_segment(AVCodecContext *avctx, void *arg) qnos[mb_index] = DV_PROFILE_IS_HD(s->sys) ? 1 : 15; - y_ptr = s->frame->data[0] + ((mb_y * s->frame->linesize[0] + mb_x) << 3); + y_ptr = s->frame->data[0] + (mb_y * s->frame->linesize[0] + mb_x) * 8; linesize = s->frame->linesize[0]; if (s->sys->height == 1080 && mb_y < 134) @@ -874,12 +874,12 @@ static int dv_encode_video_segment(AVCodecContext *avctx, void *arg) if ((s->sys->pix_fmt == AV_PIX_FMT_YUV420P) || (s->sys->pix_fmt == AV_PIX_FMT_YUV411P && mb_x >= (704 / 8)) || (s->sys->height >= 720 && mb_y != 134)) { - y_stride = s->frame->linesize[0] << (3*!enc_blk->dct_mode); + y_stride = s->frame->linesize[0] * (1 << (3*!enc_blk->dct_mode)); } else { y_stride = 16; } y_ptr = s->frame->data[0] + - ((mb_y * s->frame->linesize[0] + mb_x) << 3); + (mb_y * s->frame->linesize[0] + mb_x) * 8; linesize = s->frame->linesize[0]; if (s->sys->video_stype == 4) { /* SD 422 */ @@ -898,17 +898,17 @@ static int dv_encode_video_segment(AVCodecContext *avctx, void *arg) enc_blk += 4; /* initializing chrominance blocks */ - c_offset = (((mb_y >> (s->sys->pix_fmt == AV_PIX_FMT_YUV420P)) * s->frame->linesize[1] + - (mb_x >> ((s->sys->pix_fmt == AV_PIX_FMT_YUV411P) ? 2 : 1))) << 3); + c_offset = ((mb_y >> (s->sys->pix_fmt == AV_PIX_FMT_YUV420P)) * s->frame->linesize[1] + + (mb_x >> ((s->sys->pix_fmt == AV_PIX_FMT_YUV411P) ? 2 : 1))) * 8; for (j = 2; j; j--) { uint8_t *c_ptr = s->frame->data[j] + c_offset; linesize = s->frame->linesize[j]; - y_stride = (mb_y == 134) ? 8 : (s->frame->linesize[j] << (3*!enc_blk->dct_mode)); + y_stride = (mb_y == 134) ? 8 : (s->frame->linesize[j] * (1 << (3*!enc_blk->dct_mode))); if (s->sys->pix_fmt == AV_PIX_FMT_YUV411P && mb_x >= (704 / 8)) { uint8_t *d; uint8_t *b = scratch; for (i = 0; i < 8; i++) { - d = c_ptr + (linesize << 3); + d = c_ptr + linesize * 8; b[0] = c_ptr[0]; b[1] = c_ptr[1]; b[2] = c_ptr[2]; -- cgit v1.2.3