swscale: fix overflow in 16-bit vertical scaling.

We operated on 31-bits, but with e.g. lanczos scaling, values can
add up to beyond 0x80000000, thus leading to output of zeroes. Drop
one bit of precision fixes this.

8 files changed, 41 insertions, 41 deletions

diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 527dd80e0c..db4d231e13 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -195,7 +195,7 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc,
     int i;
     uint16_t *yDest = dest[0], *uDest = dest[1], *vDest = dest[2],
              *aDest = CONFIG_SWSCALE_ALPHA ? dest[3] : NULL;
-    int shift = 15 + 16 - output_bits;
+    int shift = 15 + 16 - output_bits - 1;
 
 #define output_pixel(pos, val) \
     if (big_endian) { \
@@ -212,24 +212,24 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc,
         } \
     }
     for (i = 0; i < dstW; i++) {
-        int val = 1 << (30-output_bits);
+        int val = 1 << (30-output_bits - 1);
         int j;
 
         for (j = 0; j < lumFilterSize; j++)
-            val += lumSrc[j][i] * lumFilter[j];
+            val += (lumSrc[j][i] * lumFilter[j]) >> 1;
 
         output_pixel(&yDest[i], val);
     }
 
     if (uDest) {
         for (i = 0; i < chrDstW; i++) {
-            int u = 1 << (30-output_bits);
-            int v = 1 << (30-output_bits);
+            int u = 1 << (30-output_bits - 1);
+            int v = 1 << (30-output_bits - 1);
             int j;
 
             for (j = 0; j < chrFilterSize; j++) {
-                u += chrUSrc[j][i] * chrFilter[j];
-                v += chrVSrc[j][i] * chrFilter[j];
+                u += (chrUSrc[j][i] * chrFilter[j]) >> 1;
+                v += (chrVSrc[j][i] * chrFilter[j]) >> 1;
             }
 
             output_pixel(&uDest[i], u);
@@ -239,11 +239,11 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc,
 
     if (CONFIG_SWSCALE_ALPHA && aDest) {
         for (i = 0; i < dstW; i++) {
-            int val = 1 << (30-output_bits);
+            int val = 1 << (30-output_bits - 1);
             int j;
 
             for (j = 0; j < lumFilterSize; j++)
-                val += alpSrc[j][i] * lumFilter[j];
+                val += (alpSrc[j][i] * lumFilter[j]) >> 1;
 
             output_pixel(&aDest[i], val);
         }
diff --git a/tests/ref/lavfi/pixdesc b/tests/ref/lavfi/pixdesc
index 76d845abb8..a82f8ed9b6 100644
--- a/tests/ref/lavfi/pixdesc
+++ b/tests/ref/lavfi/pixdesc
@@ -38,12 +38,12 @@ yuv420p16le         2d59c4f1d0314a5a957a7cfc4b6fabcc
 yuv420p9be          ce880fa07830e5297c22acf6e20555ce
 yuv420p9le          16543fda8f87d94a6cf857d2e8d4461a
 yuv422p             c9bba4529821d796a6ab09f6a5fd355a
-yuv422p16be         86ad3447f97969ce095afeef81fa9abf
-yuv422p16le         a53a9b451f4a81eeae33362c1bbd07f8
+yuv422p16be         dc9886f2fccf87cc54b27e071a2c251e
+yuv422p16le         f181c8d8436f1233ba566d9bc88005ec
 yuv440p             5a064afe2b453bb52cdb3f176b1aa1cf
 yuv444p             0a98447b78fd476aa39686da6a74fa2e
-yuv444p16be         2a8e2bddfe4c208df4119aaa7dc5db28
-yuv444p16le         f2117fc9cf66d3a832183e159ff71803
+yuv444p16be         af555dbaa401b142a995566864f47545
+yuv444p16le         a803e8016997dad95c5b2a72f54c34d6
 yuva420p            a29884f3f3dfe1e00b961bc17bef3d47
 yuvj420p            32eec78ba51857b16ce9b813a49b7189
 yuvj422p            0dfa0ed434f73be51428758c69e082cb
diff --git a/tests/ref/lavfi/pixfmts_copy b/tests/ref/lavfi/pixfmts_copy
index 76d845abb8..a82f8ed9b6 100644
--- a/tests/ref/lavfi/pixfmts_copy
+++ b/tests/ref/lavfi/pixfmts_copy
@@ -38,12 +38,12 @@ yuv420p16le         2d59c4f1d0314a5a957a7cfc4b6fabcc
 yuv420p9be          ce880fa07830e5297c22acf6e20555ce
 yuv420p9le          16543fda8f87d94a6cf857d2e8d4461a
 yuv422p             c9bba4529821d796a6ab09f6a5fd355a
-yuv422p16be         86ad3447f97969ce095afeef81fa9abf
-yuv422p16le         a53a9b451f4a81eeae33362c1bbd07f8
+yuv422p16be         dc9886f2fccf87cc54b27e071a2c251e
+yuv422p16le         f181c8d8436f1233ba566d9bc88005ec
 yuv440p             5a064afe2b453bb52cdb3f176b1aa1cf
 yuv444p             0a98447b78fd476aa39686da6a74fa2e
-yuv444p16be         2a8e2bddfe4c208df4119aaa7dc5db28
-yuv444p16le         f2117fc9cf66d3a832183e159ff71803
+yuv444p16be         af555dbaa401b142a995566864f47545
+yuv444p16le         a803e8016997dad95c5b2a72f54c34d6
 yuva420p            a29884f3f3dfe1e00b961bc17bef3d47
 yuvj420p            32eec78ba51857b16ce9b813a49b7189
 yuvj422p            0dfa0ed434f73be51428758c69e082cb
diff --git a/tests/ref/lavfi/pixfmts_crop b/tests/ref/lavfi/pixfmts_crop
index fb5c838a4e..e3bb88c101 100644
--- a/tests/ref/lavfi/pixfmts_crop
+++ b/tests/ref/lavfi/pixfmts_crop
@@ -29,12 +29,12 @@ yuv420p             bfea0188ddd4889787c403caae119cc7
 yuv420p16be         8365eff38b8c329aeb95fc605fa229bb
 yuv420p16le         5e8dd38d973d5854abe1ad4efad20cc1
 yuv422p             f2f930a91fe00d4252c4720b5ecd8961
-yuv422p16be         6647fe1c381c148f8207c988c0e22bf0
-yuv422p16le         e1548c9dc51202db38a9625c8954203f
+yuv422p16be         93f9b6f33f9529db6de6a9f0ddd70eb5
+yuv422p16le         2e66dcfec54ca6b57aa4bbd9ac234639
 yuv440p             2472417d980e395ad6843cbb8b633b29
 yuv444p             1f151980486848c96bc5585ced99003e
-yuv444p16be         02d78b564a23df2f68cf6895d3bfe6bf
-yuv444p16le         cbea9591b954ea31d6a0cb25a9aed599
+yuv444p16be         e7d1ecf0c11a41b5db192f761f55bd3c
+yuv444p16le         3298a0043d982e7cf1a33a1292fa11f0
 yuva420p            7536753dfbc7932560fb50c921369a0e
 yuvj420p            21f891093006d42d7683b0e1d773a657
 yuvj422p            9a43d474c407590ad8f213880586b45e
diff --git a/tests/ref/lavfi/pixfmts_hflip b/tests/ref/lavfi/pixfmts_hflip
index 4a32e35a1c..2084d581e1 100644
--- a/tests/ref/lavfi/pixfmts_hflip
+++ b/tests/ref/lavfi/pixfmts_hflip
@@ -29,12 +29,12 @@ yuv420p             2d5c80f9ba2ddd85b2aeda3564cc7d64
 yuv420p16be         758b0c1e2113b15e7afde48da4e4d024
 yuv420p16le         480ccd951dcb806bc875d307e02e50a0
 yuv422p             6e728f4eb9eae287c224f396d84be6ea
-yuv422p16be         9dbe0af0eb877987611cf04bfa577202
-yuv422p16le         2d8f37231110177cc5e1b61c8cb4b163
+yuv422p16be         8657d2c8d443940300fdb4028d555631
+yuv422p16le         4ab27609981e50de5b1150125718ae76
 yuv440p             a99e2b57ed601f39852715c9d675d0d3
 yuv444p             947e47f7bb5fdccc659d19b7df2b6fc3
-yuv444p16be         38cdb28061ebb8e7aa3797238615e77f
-yuv444p16le         80d654986d6f3754e924310a045fdb24
+yuv444p16be         a5154ce329db0d2caf0bd43f1347dba3
+yuv444p16le         1f703308b90feb048191b3bccc695671
 yuva420p            d83ec0c01498189f179ec574918185f1
 yuvj420p            df3aaaec3bb157c3bde5f0365af30f4f
 yuvj422p            d113871528d510a192797af59df9c05c
diff --git a/tests/ref/lavfi/pixfmts_null b/tests/ref/lavfi/pixfmts_null
index 76d845abb8..a82f8ed9b6 100644
--- a/tests/ref/lavfi/pixfmts_null
+++ b/tests/ref/lavfi/pixfmts_null
@@ -38,12 +38,12 @@ yuv420p16le         2d59c4f1d0314a5a957a7cfc4b6fabcc
 yuv420p9be          ce880fa07830e5297c22acf6e20555ce
 yuv420p9le          16543fda8f87d94a6cf857d2e8d4461a
 yuv422p             c9bba4529821d796a6ab09f6a5fd355a
-yuv422p16be         86ad3447f97969ce095afeef81fa9abf
-yuv422p16le         a53a9b451f4a81eeae33362c1bbd07f8
+yuv422p16be         dc9886f2fccf87cc54b27e071a2c251e
+yuv422p16le         f181c8d8436f1233ba566d9bc88005ec
 yuv440p             5a064afe2b453bb52cdb3f176b1aa1cf
 yuv444p             0a98447b78fd476aa39686da6a74fa2e
-yuv444p16be         2a8e2bddfe4c208df4119aaa7dc5db28
-yuv444p16le         f2117fc9cf66d3a832183e159ff71803
+yuv444p16be         af555dbaa401b142a995566864f47545
+yuv444p16le         a803e8016997dad95c5b2a72f54c34d6
 yuva420p            a29884f3f3dfe1e00b961bc17bef3d47
 yuvj420p            32eec78ba51857b16ce9b813a49b7189
 yuvj422p            0dfa0ed434f73be51428758c69e082cb
diff --git a/tests/ref/lavfi/pixfmts_scale b/tests/ref/lavfi/pixfmts_scale
index 14c748ff10..094e52462f 100644
--- a/tests/ref/lavfi/pixfmts_scale
+++ b/tests/ref/lavfi/pixfmts_scale
@@ -31,19 +31,19 @@ uyvy422             314bd486277111a95d9369b944fa0400
 yuv410p             7df8f6d69b56a8dcb6c7ee908e5018b5
 yuv411p             1143e7c5cc28fe0922b051b17733bc4c
 yuv420p             fdad2d8df8985e3d17e73c71f713cb14
-yuv420p10be         aad747a7634ba4ed48f149cdfc78792e
-yuv420p10le         46547f19a7f58638dff73657477b11b9
-yuv420p16be         d7270efce54eb59c7b01c14157a1b890
-yuv420p16le         e85abf00bad940a922b623c91c9026d7
+yuv420p10be         af5429f27b9f95bf955e795921c65cdc
+yuv420p10le         d0b47e6a8a44e6b5ca0fe4349a4e393b
+yuv420p16be         9688e33e03b8c8275ab2fb1df0f06bee
+yuv420p16le         cba8b390ad5e7b8678e419b8ce79c008
 yuv420p9be          a073b2d93b2a7dce2069ba252bc43175
 yuv420p9le          b67233c3c7d93763d07d88f697c145e1
 yuv422p             918e37701ee7377d16a8a6c119c56a40
-yuv422p16be         5cd8fe1da161d21b65bf75bf4cb50c75
-yuv422p16le         3b545317b1f5e627751525fb2958d88e
+yuv422p16be         2cf502d7d386db1f1b3b946679d897b1
+yuv422p16le         3002a4e47520731dcee5929aff49eb74
 yuv440p             461503fdb9b90451020aa3b25ddf041c
 yuv444p             81b2eba962d12e8d64f003ac56f6faf2
-yuv444p16be         2bd3e992d1533f8e0978a8e0c7008243
-yuv444p16le         daee5a461ae3bc53295c392e423aa1d7
+yuv444p16be         b9f051ce7335923fe33efd162e48da1d
+yuv444p16le         fa47e317efac988b4a7fa55141c89126
 yuva420p            8673a9131fb47de69788863f93a50eb7
 yuvj420p            30427bd6caf5bda93a173dbebe759e09
 yuvj422p            fc8288f64fd149573f73cf8da05d8e6d
diff --git a/tests/ref/lavfi/pixfmts_vflip b/tests/ref/lavfi/pixfmts_vflip
index b440efa26b..8702eca796 100644
--- a/tests/ref/lavfi/pixfmts_vflip
+++ b/tests/ref/lavfi/pixfmts_vflip
@@ -38,12 +38,12 @@ yuv420p16le         0f609e588e5a258644ef85170d70e030
 yuv420p9be          be40ec975fb2873891643cbbbddbc3b0
 yuv420p9le          7e606310d3f5ff12badf911e8f333471
 yuv422p             d7f5cb44d9b0210d66d6a8762640ab34
-yuv422p16be         8cdfbddf2dd4c44c3efef4ee00170eba
-yuv422p16le         a2f421f6a1af950544081c1797de01ae
+yuv422p16be         51d9aa4e78d121c226d919ce97976fe4
+yuv422p16le         12965c54bda8932ca72da194419a9908
 yuv440p             876385e96165acf51271b20e5d85a416
 yuv444p             9c3c667d1613b72d15bc6d851c5eb8f7
-yuv444p16be         bb86de32c67dd49469989ac184b89592
-yuv444p16le         64b899f52d820b14d05ff95954c15790
+yuv444p16be         6502abd75030d462c58d99a8673ec517
+yuv444p16le         cd7e88b6d08425450a57555bc86ab210
 yuva420p            c705d1cf061d8c6580ac690b55f92276
 yuvj420p            41fd02b204da0ab62452cd14b595e2e4
 yuvj422p            7f6ca9bc1812cde02036d7d29a7cce43