diff options
| author | Katerina Barone-Adesi <katerinab@gmail.com> | 2016-03-02 18:52:25 -0500 |
|---|---|---|
| committer | Vittorio Giovara <vittorio.giovara@gmail.com> | 2016-03-05 08:26:36 -0500 |
| commit | 1389b4c18d1042c196603ba66c25113bcee1738b (patch) | |
| tree | e7a6f5258a868a4e6b854f986773431debe6eb38 /libavcodec/simple_idct_template.c | |
| parent | e10b7ef2fe56603fb1baac6b20fd6bd0a3fdd0d0 (diff) | |
idct8x8: Fix undefined negative shifts
The original code left-shifts negative values, which is undefined
in the C99 specification (the one used during normal Libav compilation).
This change multiplies by (1 << shift), which is functionally equivalent,
but has defined behavior.
With this change, fate-idct8x8 compiled with --fsanitize=undefined works.
Bug-Id: 686
Diffstat (limited to 'libavcodec/simple_idct_template.c')
| -rw-r--r-- | libavcodec/simple_idct_template.c | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/libavcodec/simple_idct_template.c b/libavcodec/simple_idct_template.c index b287c4f6bf..171ae08254 100644 --- a/libavcodec/simple_idct_template.c +++ b/libavcodec/simple_idct_template.c @@ -96,12 +96,12 @@ static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift) if (((((uint64_t *)row)[0] & ~ROW0_MASK) | ((uint64_t *)row)[1]) == 0) { uint64_t temp; if (DC_SHIFT - extra_shift > 0) { - temp = (row[0] << (DC_SHIFT - extra_shift)) & 0xffff; + temp = (row[0] * (1 << (DC_SHIFT - extra_shift))) & 0xffff; } else { temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff; } - temp += temp << 16; - temp += temp << 32; + temp += temp * (1 << 16); + temp += temp * ((uint64_t) 1 << 32); ((uint64_t *)row)[0] = temp; ((uint64_t *)row)[1] = temp; return; @@ -113,11 +113,11 @@ static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift) row[1])) { uint32_t temp; if (DC_SHIFT - extra_shift > 0) { - temp = (row[0] << (DC_SHIFT - extra_shift)) & 0xffff; + temp = (row[0] * (1 << (DC_SHIFT - extra_shift))) & 0xffff; } else { temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff; } - temp += temp << 16; + temp += temp * (1 << 16); ((uint32_t*)row)[0]=((uint32_t*)row)[1] = ((uint32_t*)row)[2]=((uint32_t*)row)[3] = temp; return; |