dnxhddec: use dequantization formula from specs

The current one, while correct, does not yield the best possible
results. The specificiations suggest another formula, which results
in quality gains in the decoded output from fate tests. This
justifies changing said formula.

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

16 files changed, 31 insertions, 31 deletions

diff --git a/tests/ref/vsynth/vsynth1-dnxhd-1080i b/tests/ref/vsynth/vsynth1-dnxhd-1080i
index 28d55b6b9d..02f989f307 100644
--- a/tests/ref/vsynth/vsynth1-dnxhd-1080i
+++ b/tests/ref/vsynth/vsynth1-dnxhd-1080i
@@ -1,4 +1,4 @@
 a0234e0a8516d958f423b119aa9e35c4 *tests/data/fate/vsynth1-dnxhd-1080i.mov
 3031911 tests/data/fate/vsynth1-dnxhd-1080i.mov
-a09132c6db44f415e831dcaa630a351b *tests/data/fate/vsynth1-dnxhd-1080i.out.rawvideo
-stddev:    6.29 PSNR: 32.15 MAXDIFF:   64 bytes:  7603200/   760320
+fed9ed2a5179c9df0ef58772b025e303 *tests/data/fate/vsynth1-dnxhd-1080i.out.rawvideo
+stddev:    6.18 PSNR: 32.31 MAXDIFF:   64 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth1-dnxhd-1080i-colr b/tests/ref/vsynth/vsynth1-dnxhd-1080i-colr
index 16d895353c..ac429667d6 100644
--- a/tests/ref/vsynth/vsynth1-dnxhd-1080i-colr
+++ b/tests/ref/vsynth/vsynth1-dnxhd-1080i-colr
@@ -1,4 +1,4 @@
 5fccdb16c0f14dea1b6b603bac90b97e *tests/data/fate/vsynth1-dnxhd-1080i-colr.mov
 3031929 tests/data/fate/vsynth1-dnxhd-1080i-colr.mov
-5835dff88cb84e83bbe70b5ed5edd5ab *tests/data/fate/vsynth1-dnxhd-1080i-colr.out.rawvideo
-stddev:    5.79 PSNR: 32.87 MAXDIFF:   56 bytes:  7603200/   760320
+6f2d5429ffc4529a76acfeb28b560542 *tests/data/fate/vsynth1-dnxhd-1080i-colr.out.rawvideo
+stddev:    5.65 PSNR: 33.09 MAXDIFF:   55 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth1-dnxhd-720p b/tests/ref/vsynth/vsynth1-dnxhd-720p
index fd77e86360..16cf20c3dd 100644
--- a/tests/ref/vsynth/vsynth1-dnxhd-720p
+++ b/tests/ref/vsynth/vsynth1-dnxhd-720p
@@ -1,4 +1,4 @@
 af03d57b8320568027162132643f7814 *tests/data/fate/vsynth1-dnxhd-720p.dnxhd
 2293760 tests/data/fate/vsynth1-dnxhd-720p.dnxhd
-f074f1b5ed394871b3c73184ad55b895 *tests/data/fate/vsynth1-dnxhd-720p.out.rawvideo
-stddev:    6.26 PSNR: 32.19 MAXDIFF:   65 bytes:  7603200/   760320
+5f9fba5bacda81e77a72d8a816612564 *tests/data/fate/vsynth1-dnxhd-720p.out.rawvideo
+stddev:    6.22 PSNR: 32.24 MAXDIFF:   64 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth1-dnxhd-720p-10bit b/tests/ref/vsynth/vsynth1-dnxhd-720p-10bit
index 8a8f639d5b..ab588070e0 100644
--- a/tests/ref/vsynth/vsynth1-dnxhd-720p-10bit
+++ b/tests/ref/vsynth/vsynth1-dnxhd-720p-10bit
@@ -1,4 +1,4 @@
 f8c4b7aa165a80df2485d526161290a3 *tests/data/fate/vsynth1-dnxhd-720p-10bit.dnxhd
 2293760 tests/data/fate/vsynth1-dnxhd-720p-10bit.dnxhd
-ec26a6cbf53e38ffb9d5c51cbfbf4f7c *tests/data/fate/vsynth1-dnxhd-720p-10bit.out.rawvideo
-stddev:    6.27 PSNR: 32.18 MAXDIFF:   64 bytes:  7603200/   760320
+3cc84f9e8d2e704475b410de27dd9951 *tests/data/fate/vsynth1-dnxhd-720p-10bit.out.rawvideo
+stddev:    6.23 PSNR: 32.23 MAXDIFF:   64 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth1-dnxhd-720p-rd b/tests/ref/vsynth/vsynth1-dnxhd-720p-rd
index 0422776943..f030e92ae1 100644
--- a/tests/ref/vsynth/vsynth1-dnxhd-720p-rd
+++ b/tests/ref/vsynth/vsynth1-dnxhd-720p-rd
@@ -1,4 +1,4 @@
 276e5175376051218b0e3eb36f9e9a63 *tests/data/fate/vsynth1-dnxhd-720p-rd.dnxhd
 2293760 tests/data/fate/vsynth1-dnxhd-720p-rd.dnxhd
-28662df973b289798bf6069fbbee8071 *tests/data/fate/vsynth1-dnxhd-720p-rd.out.rawvideo
-stddev:    6.26 PSNR: 32.19 MAXDIFF:   65 bytes:  7603200/   760320
+1c75ad86ccd5cc2112da7d43ace50dbf *tests/data/fate/vsynth1-dnxhd-720p-rd.out.rawvideo
+stddev:    6.23 PSNR: 32.23 MAXDIFF:   64 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth2-dnxhd-1080i b/tests/ref/vsynth/vsynth2-dnxhd-1080i
index c3a5073df3..eabb6a2b28 100644
--- a/tests/ref/vsynth/vsynth2-dnxhd-1080i
+++ b/tests/ref/vsynth/vsynth2-dnxhd-1080i
@@ -1,4 +1,4 @@
 2b75889122f8d918e1b068d128b618ca *tests/data/fate/vsynth2-dnxhd-1080i.mov
 3031911 tests/data/fate/vsynth2-dnxhd-1080i.mov
-099001db73036eeb9545c463cf90f0ba *tests/data/fate/vsynth2-dnxhd-1080i.out.rawvideo
-stddev:    1.53 PSNR: 44.43 MAXDIFF:   31 bytes:  7603200/   760320
+e941d2587cfeccddc450da7f41f7f911 *tests/data/fate/vsynth2-dnxhd-1080i.out.rawvideo
+stddev:    1.50 PSNR: 44.56 MAXDIFF:   31 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth2-dnxhd-1080i-colr b/tests/ref/vsynth/vsynth2-dnxhd-1080i-colr
index ba5e6a1b76..06731a885c 100644
--- a/tests/ref/vsynth/vsynth2-dnxhd-1080i-colr
+++ b/tests/ref/vsynth/vsynth2-dnxhd-1080i-colr
@@ -1,4 +1,4 @@
 f9827e9867b0ea4f7585d8e362a58413 *tests/data/fate/vsynth2-dnxhd-1080i-colr.mov
 3031929 tests/data/fate/vsynth2-dnxhd-1080i-colr.mov
-e4cf5528c993b5e7d57a9d0a4d2cd0c6 *tests/data/fate/vsynth2-dnxhd-1080i-colr.out.rawvideo
-stddev:    1.58 PSNR: 44.15 MAXDIFF:   33 bytes:  7603200/   760320
+ec40a8014b819d02951b2f06bee7b514 *tests/data/fate/vsynth2-dnxhd-1080i-colr.out.rawvideo
+stddev:    1.54 PSNR: 44.33 MAXDIFF:   33 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth2-dnxhd-720p b/tests/ref/vsynth/vsynth2-dnxhd-720p
index f40da38346..fa26c2c058 100644
--- a/tests/ref/vsynth/vsynth2-dnxhd-720p
+++ b/tests/ref/vsynth/vsynth2-dnxhd-720p
@@ -1,4 +1,4 @@
 3bb2d4fe12b49eae830918d68bde0675 *tests/data/fate/vsynth2-dnxhd-720p.dnxhd
 2293760 tests/data/fate/vsynth2-dnxhd-720p.dnxhd
-903e5a7f2b84c0cd362a0f3a69549989 *tests/data/fate/vsynth2-dnxhd-720p.out.rawvideo
-stddev:    1.53 PSNR: 44.41 MAXDIFF:   31 bytes:  7603200/   760320
+bf3c208086609fea7cb40a9ad97b1fd4 *tests/data/fate/vsynth2-dnxhd-720p.out.rawvideo
+stddev:    1.52 PSNR: 44.49 MAXDIFF:   31 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth2-dnxhd-720p-10bit b/tests/ref/vsynth/vsynth2-dnxhd-720p-10bit
index c57bf7debe..5c21985bdd 100644
--- a/tests/ref/vsynth/vsynth2-dnxhd-720p-10bit
+++ b/tests/ref/vsynth/vsynth2-dnxhd-720p-10bit
@@ -1,4 +1,4 @@
 e49cb87f69acc809aee55d64990c84a9 *tests/data/fate/vsynth2-dnxhd-720p-10bit.dnxhd
 2293760 tests/data/fate/vsynth2-dnxhd-720p-10bit.dnxhd
-3eb47758e42db9fc704e1254b7abbeb0 *tests/data/fate/vsynth2-dnxhd-720p-10bit.out.rawvideo
-stddev:    1.56 PSNR: 44.25 MAXDIFF:   31 bytes:  7603200/   760320
+a98c4b69d4d036089a455e147d6922a7 *tests/data/fate/vsynth2-dnxhd-720p-10bit.out.rawvideo
+stddev:    1.54 PSNR: 44.36 MAXDIFF:   31 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth2-dnxhd-720p-rd b/tests/ref/vsynth/vsynth2-dnxhd-720p-rd
index 3d975570d1..f21299812d 100644
--- a/tests/ref/vsynth/vsynth2-dnxhd-720p-rd
+++ b/tests/ref/vsynth/vsynth2-dnxhd-720p-rd
@@ -1,4 +1,4 @@
 b723c7412a4c93f500b917ad721f6d21 *tests/data/fate/vsynth2-dnxhd-720p-rd.dnxhd
 2293760 tests/data/fate/vsynth2-dnxhd-720p-rd.dnxhd
-f7d437ea7024700cfd61c40197f44852 *tests/data/fate/vsynth2-dnxhd-720p-rd.out.rawvideo
-stddev:    1.53 PSNR: 44.40 MAXDIFF:   31 bytes:  7603200/   760320
+b7c5de5578317654792c6ea31bfeeb7d *tests/data/fate/vsynth2-dnxhd-720p-rd.out.rawvideo
+stddev:    1.52 PSNR: 44.48 MAXDIFF:   31 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth3-dnxhd-1080i-colr b/tests/ref/vsynth/vsynth3-dnxhd-1080i-colr
index 7cc2298189..8d7d3b6533 100644
--- a/tests/ref/vsynth/vsynth3-dnxhd-1080i-colr
+++ b/tests/ref/vsynth/vsynth3-dnxhd-1080i-colr
@@ -1,4 +1,4 @@
 ee7a70832f37793b62642f770d988bdb *tests/data/fate/vsynth3-dnxhd-1080i-colr.mov
 3031929 tests/data/fate/vsynth3-dnxhd-1080i-colr.mov
-7dd6b261e439cda21df4f01b45336b41 *tests/data/fate/vsynth3-dnxhd-1080i-colr.out.rawvideo
+f907fd2d48bedbc5283fbfc3fb9f61a0 *tests/data/fate/vsynth3-dnxhd-1080i-colr.out.rawvideo
 stddev:    6.92 PSNR: 31.32 MAXDIFF:   50 bytes:    86700/     8670
diff --git a/tests/ref/vsynth/vsynth_lena-dnxhd-1080i b/tests/ref/vsynth/vsynth_lena-dnxhd-1080i
index 3693d5797b..16702db712 100644
--- a/tests/ref/vsynth/vsynth_lena-dnxhd-1080i
+++ b/tests/ref/vsynth/vsynth_lena-dnxhd-1080i
@@ -1,4 +1,4 @@
 f7412afbcb4454692f7492f6710189e3 *tests/data/fate/vsynth_lena-dnxhd-1080i.mov
 3031911 tests/data/fate/vsynth_lena-dnxhd-1080i.mov
-744ba46da5d4c19a28562ea31061d170 *tests/data/fate/vsynth_lena-dnxhd-1080i.out.rawvideo
-stddev:    1.31 PSNR: 45.77 MAXDIFF:   23 bytes:  7603200/   760320
+7d0ca92f12711535d57eff3609462b31 *tests/data/fate/vsynth_lena-dnxhd-1080i.out.rawvideo
+stddev:    1.29 PSNR: 45.87 MAXDIFF:   22 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth_lena-dnxhd-1080i-colr b/tests/ref/vsynth/vsynth_lena-dnxhd-1080i-colr
index 1889786ea7..8e43a3f9de 100644
--- a/tests/ref/vsynth/vsynth_lena-dnxhd-1080i-colr
+++ b/tests/ref/vsynth/vsynth_lena-dnxhd-1080i-colr
@@ -1,4 +1,4 @@
 5ba3ddb58b10e5f0069cb4f82d594695 *tests/data/fate/vsynth_lena-dnxhd-1080i-colr.mov
 3031929 tests/data/fate/vsynth_lena-dnxhd-1080i-colr.mov
-864c3d5f49d9edf66ce8f82a2a6725f6 *tests/data/fate/vsynth_lena-dnxhd-1080i-colr.out.rawvideo
-stddev:    1.36 PSNR: 45.45 MAXDIFF:   22 bytes:  7603200/   760320
+ce4993a69ef55c8c4b18138716f17b6f *tests/data/fate/vsynth_lena-dnxhd-1080i-colr.out.rawvideo
+stddev:    1.33 PSNR: 45.59 MAXDIFF:   22 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth_lena-dnxhd-720p b/tests/ref/vsynth/vsynth_lena-dnxhd-720p
index 686be54808..eaa1af8812 100644
--- a/tests/ref/vsynth/vsynth_lena-dnxhd-720p
+++ b/tests/ref/vsynth/vsynth_lena-dnxhd-720p
@@ -1,4 +1,4 @@
 4ca9473a8d106bdfe36e9bf7c516b648 *tests/data/fate/vsynth_lena-dnxhd-720p.dnxhd
 2293760 tests/data/fate/vsynth_lena-dnxhd-720p.dnxhd
-d44c4b08cda8a8042ae345124fdfffcc *tests/data/fate/vsynth_lena-dnxhd-720p.out.rawvideo
-stddev:    1.32 PSNR: 45.68 MAXDIFF:   22 bytes:  7603200/   760320
+9ccd48d24b4f1af2323b65abb6d65d7f *tests/data/fate/vsynth_lena-dnxhd-720p.out.rawvideo
+stddev:    1.31 PSNR: 45.77 MAXDIFF:   22 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth_lena-dnxhd-720p-10bit b/tests/ref/vsynth/vsynth_lena-dnxhd-720p-10bit
index 794e9c1985..1dcadd80cf 100644
--- a/tests/ref/vsynth/vsynth_lena-dnxhd-720p-10bit
+++ b/tests/ref/vsynth/vsynth_lena-dnxhd-720p-10bit
@@ -1,4 +1,4 @@
 e96fc4a7d994b9369c50da32fd325822 *tests/data/fate/vsynth_lena-dnxhd-720p-10bit.dnxhd
 2293760 tests/data/fate/vsynth_lena-dnxhd-720p-10bit.dnxhd
-0449440eb3e8416840a27deb1a8f80b0 *tests/data/fate/vsynth_lena-dnxhd-720p-10bit.out.rawvideo
-stddev:    1.35 PSNR: 45.47 MAXDIFF:   22 bytes:  7603200/   760320
+2b497215c57558910a605ff8c78430d9 *tests/data/fate/vsynth_lena-dnxhd-720p-10bit.out.rawvideo
+stddev:    1.33 PSNR: 45.61 MAXDIFF:   22 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth_lena-dnxhd-720p-rd b/tests/ref/vsynth/vsynth_lena-dnxhd-720p-rd
index 453f68f018..b9aa690858 100644
--- a/tests/ref/vsynth/vsynth_lena-dnxhd-720p-rd
+++ b/tests/ref/vsynth/vsynth_lena-dnxhd-720p-rd
@@ -1,4 +1,4 @@
 b305b03708e905717b42fc0b304367d4 *tests/data/fate/vsynth_lena-dnxhd-720p-rd.dnxhd
 2293760 tests/data/fate/vsynth_lena-dnxhd-720p-rd.dnxhd
-13de1c5ed025abb5120450e134aa623d *tests/data/fate/vsynth_lena-dnxhd-720p-rd.out.rawvideo
-stddev:    1.32 PSNR: 45.66 MAXDIFF:   22 bytes:  7603200/   760320
+3793391df8448937b17d876b6df0d26d *tests/data/fate/vsynth_lena-dnxhd-720p-rd.out.rawvideo
+stddev:    1.31 PSNR: 45.75 MAXDIFF:   22 bytes:  7603200/   760320