libswcale/input: use more accurate rgbf32 yuv conversions

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

2 files changed, 65 insertions, 65 deletions

diff --git a/tests/ref/fate/filter-pixfmts-scale b/tests/ref/fate/filter-pixfmts-scale
index d7020ad2c3..30e7cd5b06 100644
--- a/tests/ref/fate/filter-pixfmts-scale
+++ b/tests/ref/fate/filter-pixfmts-scale
@@ -25,8 +25,8 @@ gbrap12be           1d9b57766ba9c2192403f43967cb9af0
 gbrap12le           bb1ba1c157717db3dd612a76d38a018e
 gbrap16be           c72b935a6e57a8e1c37bff08c2db55b1
 gbrap16le           13eb0e62b1ac9c1c86c81521eaefab5f
-gbrapf32be          42e53d9edccbd9e09c4cd78780ba92f3
-gbrapf32le          eebf3973ef94c841f0a1ceb1ed61621d
+gbrapf32be          366b804d5697276e8c481c4bdf05a00b
+gbrapf32le          558a268e6d6b907449d1056afab78f29
 gbrp                dc3387f925f972c61aae7eb23cdc19f0
 gbrp10be            0277d4c3a8498d75e2783fb81379e481
 gbrp10le            f3d70f8ab845c3c9b8f7452e4a6e285a
@@ -38,8 +38,8 @@ gbrp16be            5fc826cfabebfc1442cb793c4b6303e2
 gbrp16le            1b3e0b63d47a3e1b6b20931316883bf2
 gbrp9be             d9c88968001e1452ff31fbc8d16b18a0
 gbrp9le             2ccfed0816bf6bd4bb3a5b7591d9603a
-gbrpf32be           4614d32e4417f80e0adcc1bdcf6cde42
-gbrpf32le           1366ee77e5559672260bbe51040e28b2
+gbrpf32be           f3d0cefdf11c861001880772d817aac8
+gbrpf32le           290468205c1c18a0667edfca45061aee
 gray                221201cc7cfc4964eacd8b3e426fd276
 gray10be            9452756d0b37f4f5c7cae7635e22d747
 gray10le            37fd2e1ec6b66410212d39a342e864df
diff --git a/tests/ref/fate/sws-floatimg-cmp b/tests/ref/fate/sws-floatimg-cmp
index 24204254c4..cf6788fc23 100644
--- a/tests/ref/fate/sws-floatimg-cmp
+++ b/tests/ref/fate/sws-floatimg-cmp
@@ -1,120 +1,120 @@
 gbrpf32le -> yuv444p16le -> gbrpf32le
-avg diff: 0.003852
+avg diff: 0.000125
 min diff: 0.000000
-max diff: 0.006638
+max diff: 0.000501
 gbrpf32le -> yuv444p -> gbrpf32le
-avg diff: 0.004316
+avg diff: 0.001804
 min diff: 0.000000
-max diff: 0.012704
+max diff: 0.006399
 gbrpf32le -> yuv444p9le -> gbrpf32le
-avg diff: 0.004053
-min diff: 0.000001
-max diff: 0.009402
+avg diff: 0.000906
+min diff: 0.000000
+max diff: 0.003313
 gbrpf32le -> yuv444p10le -> gbrpf32le
-avg diff: 0.003960
+avg diff: 0.000467
 min diff: 0.000000
-max diff: 0.008123
+max diff: 0.001912
 gbrpf32le -> yuv444p12le -> gbrpf32le
-avg diff: 0.003878
+avg diff: 0.000166
 min diff: 0.000000
-max diff: 0.007011
+max diff: 0.000802
 gbrpf32le -> yuv444p14le -> gbrpf32le
-avg diff: 0.003868
+avg diff: 0.000127
 min diff: 0.000000
-max diff: 0.006729
+max diff: 0.000524
 gbrpf32le -> rgb24 -> gbrpf32le
-avg diff: 0.004122
+avg diff: 0.001011
 min diff: 0.000000
-max diff: 0.008975
+max diff: 0.004229
 gbrpf32le -> bgr24 -> gbrpf32le
-avg diff: 0.004122
+avg diff: 0.001011
 min diff: 0.000000
-max diff: 0.008975
+max diff: 0.004229
 gbrpf32le -> rgba -> gbrpf32le
-avg diff: 0.004122
+avg diff: 0.001011
 min diff: 0.000000
-max diff: 0.008975
+max diff: 0.004229
 gbrpf32le -> bgra -> gbrpf32le
-avg diff: 0.004122
+avg diff: 0.001011
 min diff: 0.000000
-max diff: 0.008975
+max diff: 0.004229
 gbrpf32le -> argb -> gbrpf32le
-avg diff: 0.004122
+avg diff: 0.001011
 min diff: 0.000000
-max diff: 0.008975
+max diff: 0.004229
 gbrpf32le -> abgr -> gbrpf32le
-avg diff: 0.004122
+avg diff: 0.001011
 min diff: 0.000000
-max diff: 0.008975
+max diff: 0.004229
 gbrpf32le -> 0rgb -> gbrpf32le
-avg diff: 0.004122
+avg diff: 0.001011
 min diff: 0.000000
-max diff: 0.008975
+max diff: 0.004229
 gbrpf32le -> 0bgr -> gbrpf32le
-avg diff: 0.004122
+avg diff: 0.001011
 min diff: 0.000000
-max diff: 0.008975
+max diff: 0.004229
 gbrpf32le -> rgb0 -> gbrpf32le
-avg diff: 0.004122
+avg diff: 0.001011
 min diff: 0.000000
-max diff: 0.008975
+max diff: 0.004229
 gbrpf32le -> bgr0 -> gbrpf32le
-avg diff: 0.004122
+avg diff: 0.001011
 min diff: 0.000000
-max diff: 0.008975
+max diff: 0.004229
 gbrpf32le -> rgb48le -> gbrpf32le
-avg diff: 0.003851
+avg diff: 0.000249
 min diff: 0.000000
-max diff: 0.007076
+max diff: 0.000990
 gbrpf32le -> bgr48le -> gbrpf32le
-avg diff: 0.003851
+avg diff: 0.000249
 min diff: 0.000000
-max diff: 0.007076
+max diff: 0.000990
 gbrpf32le -> rgba64le -> gbrpf32le
-avg diff: 0.003851
+avg diff: 0.000249
 min diff: 0.000000
-max diff: 0.007076
+max diff: 0.000990
 gbrpf32le -> bgra64le -> gbrpf32le
-avg diff: 0.003851
+avg diff: 0.000249
 min diff: 0.000000
-max diff: 0.007076
+max diff: 0.000990
 gbrpf32le -> gbrp -> gbrpf32le
-avg diff: 0.004122
+avg diff: 0.001011
 min diff: 0.000000
-max diff: 0.008975
+max diff: 0.004229
 gbrpf32le -> gbrap -> gbrpf32le
-avg diff: 0.004122
+avg diff: 0.001011
 min diff: 0.000000
-max diff: 0.008975
+max diff: 0.004229
 gbrpf32le -> gbrp9le -> gbrpf32le
-avg diff: 0.007737
+avg diff: 0.003917
 min diff: 0.000000
-max diff: 0.014009
+max diff: 0.007870
 gbrpf32le -> gbrp10le -> gbrpf32le
-avg diff: 0.007662
+avg diff: 0.003841
 min diff: 0.000000
-max diff: 0.013605
+max diff: 0.007456
 gbrpf32le -> gbrap10le -> gbrpf32le
-avg diff: 0.007662
+avg diff: 0.003841
 min diff: 0.000000
-max diff: 0.013605
+max diff: 0.007456
 gbrpf32le -> gbrp12le -> gbrpf32le
-avg diff: 0.007622
+avg diff: 0.003796
 min diff: 0.000000
-max diff: 0.013335
+max diff: 0.007140
 gbrpf32le -> gbrap12le -> gbrpf32le
-avg diff: 0.007622
+avg diff: 0.003796
 min diff: 0.000000
-max diff: 0.013335
+max diff: 0.007140
 gbrpf32le -> gbrp14le -> gbrpf32le
-avg diff: 0.007620
+avg diff: 0.003792
 min diff: 0.000000
-max diff: 0.013232
+max diff: 0.007034
 gbrpf32le -> gbrp16le -> gbrpf32le
-avg diff: 0.007680
+avg diff: 0.003853
 min diff: 0.000000
-max diff: 0.013275
+max diff: 0.007098
 gbrpf32le -> gbrap16le -> gbrpf32le
-avg diff: 0.007680
+avg diff: 0.003853
 min diff: 0.000000
-max diff: 0.013275
+max diff: 0.007098