diff options
| author | Michael Niedermayer <michaelni@gmx.at> | 2014-01-12 02:27:00 +0100 |
|---|---|---|
| committer | Michael Niedermayer <michaelni@gmx.at> | 2014-01-14 15:58:59 +0100 |
| commit | 4b0cad6596bcbce5833251c9ba2a60617186a39c (patch) | |
| tree | 073328bd46a141a9d2ac0f3827e3ae2255b99173 | |
| parent | a7ea733b721e034447e4e5bb013e2dd25614f530 (diff) | |
avcodec/simple_idct_template: fix rounding of the special DC case for 10bit
MSE doesnt change for the test as the code only triggers
for very sparse matrixes
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
| -rw-r--r-- | libavcodec/simple_idct_template.c | 8 | ||||
| -rw-r--r-- | tests/ref/fate/prores-422_lt | 4 | ||||
| -rw-r--r-- | tests/ref/vsynth/vsynth2-prores | 2 |
3 files changed, 7 insertions, 7 deletions
diff --git a/libavcodec/simple_idct_template.c b/libavcodec/simple_idct_template.c index 838cea4951..ac04923d96 100644 --- a/libavcodec/simple_idct_template.c +++ b/libavcodec/simple_idct_template.c @@ -107,10 +107,10 @@ static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift) #define ROW0_MASK (0xffffLL << 48 * HAVE_BIGENDIAN) if (((((uint64_t *)row)[0] & ~ROW0_MASK) | ((uint64_t *)row)[1]) == 0) { uint64_t temp; - if (DC_SHIFT - extra_shift > 0) { + if (DC_SHIFT - extra_shift >= 0) { temp = (row[0] << (DC_SHIFT - extra_shift)) & 0xffff; } else { - temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff; + temp = ((row[0] + (1<<(extra_shift - DC_SHIFT-1))) >> (extra_shift - DC_SHIFT)) & 0xffff; } temp += temp << 16; temp += temp << 32; @@ -124,10 +124,10 @@ static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift) ((uint32_t*)row)[3] | row[1])) { uint32_t temp; - if (DC_SHIFT - extra_shift > 0) { + if (DC_SHIFT - extra_shift >= 0) { temp = (row[0] << (DC_SHIFT - extra_shift)) & 0xffff; } else { - temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff; + temp = ((row[0] + (1<<(extra_shift - DC_SHIFT-1))) >> (extra_shift - DC_SHIFT)) & 0xffff; } temp += temp << 16; ((uint32_t*)row)[0]=((uint32_t*)row)[1] = diff --git a/tests/ref/fate/prores-422_lt b/tests/ref/fate/prores-422_lt index 33976d0a44..3113c8a522 100644 --- a/tests/ref/fate/prores-422_lt +++ b/tests/ref/fate/prores-422_lt @@ -1,3 +1,3 @@ #tb 0: 100/2997 -0, 0, 0, 1, 8294400, 0xdaff4019 -0, 1, 1, 1, 8294400, 0xdaff4019 +0, 0, 0, 1, 8294400, 0x4ca110c7 +0, 1, 1, 1, 8294400, 0x4ca110c7 diff --git a/tests/ref/vsynth/vsynth2-prores b/tests/ref/vsynth/vsynth2-prores index b2169d827a..82311b4dfc 100644 --- a/tests/ref/vsynth/vsynth2-prores +++ b/tests/ref/vsynth/vsynth2-prores @@ -1,4 +1,4 @@ af885572d77d28e5ede27932bd68cdde *tests/data/fate/vsynth2-prores.mov 2844076 tests/data/fate/vsynth2-prores.mov -387a3c23586c679b411c3eff2c9d9d06 *tests/data/fate/vsynth2-prores.out.rawvideo +03fd29e3963716a09d232b6f817ecb57 *tests/data/fate/vsynth2-prores.out.rawvideo stddev: 1.31 PSNR: 45.77 MAXDIFF: 11 bytes: 7603200/ 7603200 |