diff options
| -rw-r--r-- | libavcodec/cfhd.c | 4 | ||||
| -rw-r--r-- | tests/ref/fate/cfhd-3 | 20 |
2 files changed, 12 insertions, 12 deletions
diff --git a/libavcodec/cfhd.c b/libavcodec/cfhd.c index aa6e2ebee7..0327b456ac 100644 --- a/libavcodec/cfhd.c +++ b/libavcodec/cfhd.c @@ -117,8 +117,8 @@ static inline int dequant_and_decompand(int level, int quantisation, int codeboo { if (codebook == 0 || codebook == 1) { int64_t abslevel = abs(level); - if (level < 264) - return (abslevel + ((768 * abslevel * abslevel * abslevel) / (255 * 255 * 255))) * + if (abslevel < 256) + return (abslevel + ((768 * abslevel * abslevel * abslevel) / (256 * 256 * 256))) * FFSIGN(level) * quantisation; else return level * quantisation; diff --git a/tests/ref/fate/cfhd-3 b/tests/ref/fate/cfhd-3 index 59fdc92260..79133a14b0 100644 --- a/tests/ref/fate/cfhd-3 +++ b/tests/ref/fate/cfhd-3 @@ -3,13 +3,13 @@ #codec_id 0: rawvideo #dimensions 0: 496x241 #sar 0: 0/1 -0, 0, 0, 1, 478144, 0x48a01dbb -0, 1, 1, 1, 478144, 0x48a01dbb -0, 2, 2, 1, 478144, 0x48a01dbb -0, 3, 3, 1, 478144, 0xb978a72f -0, 4, 4, 1, 478144, 0x7bbb4679 -0, 5, 5, 1, 478144, 0xc3fd3f59 -0, 6, 6, 1, 478144, 0xfd2a4816 -0, 7, 7, 1, 478144, 0x207f65d3 -0, 8, 8, 1, 478144, 0x207f65d3 -0, 9, 9, 1, 478144, 0x207f65d3 +0, 0, 0, 1, 478144, 0x1e5a0d6c +0, 1, 1, 1, 478144, 0x1e5a0d6c +0, 2, 2, 1, 478144, 0x1e5a0d6c +0, 3, 3, 1, 478144, 0x88788c7d +0, 4, 4, 1, 478144, 0x78643db8 +0, 5, 5, 1, 478144, 0x84303909 +0, 6, 6, 1, 478144, 0x8ddd4828 +0, 7, 7, 1, 478144, 0xc0845d58 +0, 8, 8, 1, 478144, 0xc0845d58 +0, 9, 9, 1, 478144, 0xc0845d58 |