From 1389b4c18d1042c196603ba66c25113bcee1738b Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Wed, 2 Mar 2016 18:52:25 -0500 Subject: idct8x8: Fix undefined negative shifts The original code left-shifts negative values, which is undefined in the C99 specification (the one used during normal Libav compilation). This change multiplies by (1 << shift), which is functionally equivalent, but has defined behavior. With this change, fate-idct8x8 compiled with --fsanitize=undefined works. Bug-Id: 686 --- libavcodec/jfdctint_template.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libavcodec/jfdctint_template.c') diff --git a/libavcodec/jfdctint_template.c b/libavcodec/jfdctint_template.c index c6a1638107..3ea2f5dadc 100644 --- a/libavcodec/jfdctint_template.c +++ b/libavcodec/jfdctint_template.c @@ -216,8 +216,8 @@ static av_always_inline void FUNC(row_fdct)(int16_t *data) tmp11 = tmp1 + tmp2; tmp12 = tmp1 - tmp2; - dataptr[0] = (int16_t) ((tmp10 + tmp11) << PASS1_BITS); - dataptr[4] = (int16_t) ((tmp10 - tmp11) << PASS1_BITS); + dataptr[0] = (int16_t) ((tmp10 + tmp11) * (1 << PASS1_BITS)); + dataptr[4] = (int16_t) ((tmp10 - tmp11) * (1 << PASS1_BITS)); z1 = MULTIPLY(tmp12 + tmp13, FIX_0_541196100); dataptr[2] = (int16_t) DESCALE(z1 + MULTIPLY(tmp13, FIX_0_765366865), -- cgit v1.2.3