swscale: reintroduce full precision in 16-bit output.

8 files changed, 39 insertions, 33 deletions

diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 1d9b965ac9..8ed68c5980 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -202,20 +202,26 @@ yuv2yuvX16_c_template(const int16_t *filter, int filterSize,
 {
 #define output_pixel(pos, val) \
     if (big_endian) { \
-        AV_WB16(pos, av_clip_uint16(val >> shift)); \
+        AV_WB16(pos, 0x8000 + av_clip_int16(val >> shift)); \
     } else { \
-        AV_WL16(pos, av_clip_uint16(val >> shift)); \
+        AV_WL16(pos, 0x8000 + av_clip_int16(val >> shift)); \
     }
 
     int i;
-    int shift = 15 + 16 - output_bits - 1;
+    int shift = 15 + 16 - output_bits;
 
     for (i = 0; i < dstW; i++) {
-        int val = 1 << (30-output_bits - 1);
+        int val = 1 << (30-output_bits);
         int j;
 
+        /* range of val is [0,0x7FFFFFFF], so 31 bits, but with lanczos/spline
+         * filters (or anything with negative coeffs, the range can be slightly
+         * wider in both directions. To account for this overflow, we subtract
+         * a constant so it always fits in the signed range (assuming a
+         * reasonable filterSize), and re-add that at the end. */
+        val -= 0x40000000;
         for (j = 0; j < filterSize; j++)
-            val += (src[j][i] * filter[j]) >> 1;
+            val += src[j][i] * filter[j];
 
         output_pixel(&dest[i], val);
     }
diff --git a/tests/ref/lavfi/pixdesc b/tests/ref/lavfi/pixdesc
index c4d789095c..90dcbf56bc 100644
--- a/tests/ref/lavfi/pixdesc
+++ b/tests/ref/lavfi/pixdesc
@@ -40,16 +40,16 @@ yuv420p9le          16543fda8f87d94a6cf857d2e8d4461a
 yuv422p             c9bba4529821d796a6ab09f6a5fd355a
 yuv422p10be         bdc13b630fd668b34c6fe1aae28dfc71
 yuv422p10le         d0607c260a45c973e6639f4e449730ad
-yuv422p16be         4e9b3b3467aeebb6a528cee5966800ed
-yuv422p16le         f87c81bf16916b64d201359be0b4b6f4
+yuv422p16be         5499502e1c29534a158a1fe60e889f60
+yuv422p16le         e3d61fde6978591596bc36b914386623
 yuv422p9be          29b71579946940a8c00fa844c9dff507
 yuv422p9le          062b7f9cbb972bf36b5bdb1a7623701a
 yuv440p             5a064afe2b453bb52cdb3f176b1aa1cf
 yuv444p             0a98447b78fd476aa39686da6a74fa2e
 yuv444p10be         e65cbae7e4f1892c23defbc8e8052cf6
 yuv444p10le         767179dd82846cf00ee4c340c9c1ab74
-yuv444p16be         3ad639fff73e56f3b09dd20c335478d6
-yuv444p16le         8a7e66dc91ab7971fd24a9105ff2699b
+yuv444p16be         1c6ea2c2f5e539006112ceec3d4e7d90
+yuv444p16le         20f86bc2f68d2b3f1f2b48b97b2189f4
 yuv444p9be          6ab31f4c12b533ce318ecdff83cdd054
 yuv444p9le          f0606604a5c08becab6ba500124c4b7c
 yuva420p            a29884f3f3dfe1e00b961bc17bef3d47
diff --git a/tests/ref/lavfi/pixfmts_copy b/tests/ref/lavfi/pixfmts_copy
index c4d789095c..90dcbf56bc 100644
--- a/tests/ref/lavfi/pixfmts_copy
+++ b/tests/ref/lavfi/pixfmts_copy
@@ -40,16 +40,16 @@ yuv420p9le          16543fda8f87d94a6cf857d2e8d4461a
 yuv422p             c9bba4529821d796a6ab09f6a5fd355a
 yuv422p10be         bdc13b630fd668b34c6fe1aae28dfc71
 yuv422p10le         d0607c260a45c973e6639f4e449730ad
-yuv422p16be         4e9b3b3467aeebb6a528cee5966800ed
-yuv422p16le         f87c81bf16916b64d201359be0b4b6f4
+yuv422p16be         5499502e1c29534a158a1fe60e889f60
+yuv422p16le         e3d61fde6978591596bc36b914386623
 yuv422p9be          29b71579946940a8c00fa844c9dff507
 yuv422p9le          062b7f9cbb972bf36b5bdb1a7623701a
 yuv440p             5a064afe2b453bb52cdb3f176b1aa1cf
 yuv444p             0a98447b78fd476aa39686da6a74fa2e
 yuv444p10be         e65cbae7e4f1892c23defbc8e8052cf6
 yuv444p10le         767179dd82846cf00ee4c340c9c1ab74
-yuv444p16be         3ad639fff73e56f3b09dd20c335478d6
-yuv444p16le         8a7e66dc91ab7971fd24a9105ff2699b
+yuv444p16be         1c6ea2c2f5e539006112ceec3d4e7d90
+yuv444p16le         20f86bc2f68d2b3f1f2b48b97b2189f4
 yuv444p9be          6ab31f4c12b533ce318ecdff83cdd054
 yuv444p9le          f0606604a5c08becab6ba500124c4b7c
 yuva420p            a29884f3f3dfe1e00b961bc17bef3d47
diff --git a/tests/ref/lavfi/pixfmts_crop b/tests/ref/lavfi/pixfmts_crop
index d11720f83c..cbbcf90bbf 100644
--- a/tests/ref/lavfi/pixfmts_crop
+++ b/tests/ref/lavfi/pixfmts_crop
@@ -33,8 +33,8 @@ yuv422p16be         167e4338811a7d272925a4c6417d60da
 yuv422p16le         3359395d5875d581fa1e975013d30114
 yuv440p             2472417d980e395ad6843cbb8b633b29
 yuv444p             1f151980486848c96bc5585ced99003e
-yuv444p16be         5d0c0ea66ab43c0c590d8c2a9256e43f
-yuv444p16le         3c0a747c1b64feb0ab8dfba92f92579a
+yuv444p16be         1ce8fcd4712d525af983e6179d6a4f9e
+yuv444p16le         5f1441e18345aadb3f881dac99c6c08a
 yuva420p            7536753dfbc7932560fb50c921369a0e
 yuvj420p            21f891093006d42d7683b0e1d773a657
 yuvj422p            9a43d474c407590ad8f213880586b45e
diff --git a/tests/ref/lavfi/pixfmts_hflip b/tests/ref/lavfi/pixfmts_hflip
index 5eb1b31140..3741713ee2 100644
--- a/tests/ref/lavfi/pixfmts_hflip
+++ b/tests/ref/lavfi/pixfmts_hflip
@@ -29,12 +29,12 @@ yuv420p             2d5c80f9ba2ddd85b2aeda3564cc7d64
 yuv420p16be         758b0c1e2113b15e7afde48da4e4d024
 yuv420p16le         480ccd951dcb806bc875d307e02e50a0
 yuv422p             6e728f4eb9eae287c224f396d84be6ea
-yuv422p16be         69cf0605496c321546899a8442ee64fb
-yuv422p16le         f0b443fea72f4b6f462859a73b159664
+yuv422p16be         a05d43cd62b790087bd37083174557de
+yuv422p16le         6954abebcbc62d81068d58d0c62bdd5b
 yuv440p             a99e2b57ed601f39852715c9d675d0d3
 yuv444p             947e47f7bb5fdccc659d19b7df2b6fc3
-yuv444p16be         bc7d53923cff1d7e98d24540845fb64b
-yuv444p16le         5df206a93f85ef8b77f5bdc81d9b0a0b
+yuv444p16be         58c012e5ab73b066ef3c2b6411a395f1
+yuv444p16le         32c12794e184042a59738ab2de608c8d
 yuva420p            d83ec0c01498189f179ec574918185f1
 yuvj420p            df3aaaec3bb157c3bde5f0365af30f4f
 yuvj422p            d113871528d510a192797af59df9c05c
diff --git a/tests/ref/lavfi/pixfmts_null b/tests/ref/lavfi/pixfmts_null
index c4d789095c..90dcbf56bc 100644
--- a/tests/ref/lavfi/pixfmts_null
+++ b/tests/ref/lavfi/pixfmts_null
@@ -40,16 +40,16 @@ yuv420p9le          16543fda8f87d94a6cf857d2e8d4461a
 yuv422p             c9bba4529821d796a6ab09f6a5fd355a
 yuv422p10be         bdc13b630fd668b34c6fe1aae28dfc71
 yuv422p10le         d0607c260a45c973e6639f4e449730ad
-yuv422p16be         4e9b3b3467aeebb6a528cee5966800ed
-yuv422p16le         f87c81bf16916b64d201359be0b4b6f4
+yuv422p16be         5499502e1c29534a158a1fe60e889f60
+yuv422p16le         e3d61fde6978591596bc36b914386623
 yuv422p9be          29b71579946940a8c00fa844c9dff507
 yuv422p9le          062b7f9cbb972bf36b5bdb1a7623701a
 yuv440p             5a064afe2b453bb52cdb3f176b1aa1cf
 yuv444p             0a98447b78fd476aa39686da6a74fa2e
 yuv444p10be         e65cbae7e4f1892c23defbc8e8052cf6
 yuv444p10le         767179dd82846cf00ee4c340c9c1ab74
-yuv444p16be         3ad639fff73e56f3b09dd20c335478d6
-yuv444p16le         8a7e66dc91ab7971fd24a9105ff2699b
+yuv444p16be         1c6ea2c2f5e539006112ceec3d4e7d90
+yuv444p16le         20f86bc2f68d2b3f1f2b48b97b2189f4
 yuv444p9be          6ab31f4c12b533ce318ecdff83cdd054
 yuv444p9le          f0606604a5c08becab6ba500124c4b7c
 yuva420p            a29884f3f3dfe1e00b961bc17bef3d47
diff --git a/tests/ref/lavfi/pixfmts_scale b/tests/ref/lavfi/pixfmts_scale
index 83b523e8a3..8ca9cbf78d 100644
--- a/tests/ref/lavfi/pixfmts_scale
+++ b/tests/ref/lavfi/pixfmts_scale
@@ -33,23 +33,23 @@ yuv411p             1143e7c5cc28fe0922b051b17733bc4c
 yuv420p             fdad2d8df8985e3d17e73c71f713cb14
 yuv420p10be         c143e77e97d2f7d62c3b518857ba9f9b
 yuv420p10le         72d90eccf5c34691ff057dafb7447aa2
-yuv420p16be         9688e33e03b8c8275ab2fb1df0f06bee
-yuv420p16le         cba8b390ad5e7b8678e419b8ce79c008
+yuv420p16be         d7270efce54eb59c7b01c14157a1b890
+yuv420p16le         e85abf00bad940a922b623c91c9026d7
 yuv420p9be          bb87fddca65d1742412c8d2b1caf96c6
 yuv420p9le          828eec50014a41258a5423c1fe56ac97
 yuv422p             918e37701ee7377d16a8a6c119c56a40
 yuv422p10be         cea7ca6b0e66d6f29539885896c88603
 yuv422p10le         a10c4a5837547716f13cd61918b145f9
-yuv422p16be         285993ee0c0f4f8e511ee46f93c5f38c
-yuv422p16le         61bfcee8e54465f760164f5a75d40b5e
+yuv422p16be         e7e34fe9264784763ab6cb406524c0f3
+yuv422p16le         c435b76b08204dda6908640fb5fd4621
 yuv422p9be          82494823944912f73cebc58ad2979bbd
 yuv422p9le          fc69c8a21f473916a4b4225636b97e06
 yuv440p             461503fdb9b90451020aa3b25ddf041c
 yuv444p             81b2eba962d12e8d64f003ac56f6faf2
 yuv444p10be         e9d3c8e744b8b0d8187ca092fa203fc9
 yuv444p10le         02f0a336e9da062a64df1ba487e102c5
-yuv444p16be         2677f3074d255f9dab625e9e2e092ca5
-yuv444p16le         65fa92521ef97088599ea83f9508cd5b
+yuv444p16be         0da9bed80f5542682ab286f3261cf24c
+yuv444p16le         a0c5d3c7bf3f181db503cf8e450d1335
 yuv444p9be          9ac2643ce7f7e5c4e17c8c9fd8494d4a
 yuv444p9le          896a1cc9cccca1ba410dd53942d33cc4
 yuva420p            8673a9131fb47de69788863f93a50eb7
diff --git a/tests/ref/lavfi/pixfmts_vflip b/tests/ref/lavfi/pixfmts_vflip
index 66d803cbdf..4aaef16b5a 100644
--- a/tests/ref/lavfi/pixfmts_vflip
+++ b/tests/ref/lavfi/pixfmts_vflip
@@ -40,16 +40,16 @@ yuv420p9le          7e606310d3f5ff12badf911e8f333471
 yuv422p             d7f5cb44d9b0210d66d6a8762640ab34
 yuv422p10be         588fe319b96513c32e21d3e32b45447f
 yuv422p10le         11b57f2bd9661024153f3973b9090cdb
-yuv422p16be         c092d083548c2a144c372a98c46875c7
-yuv422p16le         c071b9397a416d51cbe339345cbcba84
+yuv422p16be         9bd8f8c961822b586fa4cf992be54acc
+yuv422p16le         9c4a1239605c7952b736ac3130163f14
 yuv422p9be          7c6f1e140b3999ee7d923854e507752a
 yuv422p9le          51f10d79c07989060dd06e767e6d7d60
 yuv440p             876385e96165acf51271b20e5d85a416
 yuv444p             9c3c667d1613b72d15bc6d851c5eb8f7
 yuv444p10be         944a4997c4edb3a8dd0f0493cfd5a1fd
 yuv444p10le         2d0947ae89ecc6a501eee6832cb27e06
-yuv444p16be         6a954614fd2a8ae0df53e4fd76937af8
-yuv444p16le         65613965fb58cc4c3cd480a68b6540ea
+yuv444p16be         de2dedfc6f12073ffead113f86e07ecf
+yuv444p16le         8e83323cf102d6c823a03ae8a7b7e033
 yuv444p9be          6ac92b7dc9ab2fc59bee99204886899a
 yuv444p9le          85aef13a654953d3455d89770b0d74bd
 yuva420p            c705d1cf061d8c6580ac690b55f92276