swscale: Properly scale YUV

Only shift limited range luma, and always only shift chroma
for upconversion.

Based off a patch by Michael Niedermayer.

Signed-off-by: Derek Buitenhuis <derek.buitenhuis@gmail.com>
Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

14 files changed, 76 insertions, 53 deletions

diff --git a/libswscale/swscale_unscaled.c b/libswscale/swscale_unscaled.c
index ffc813e1ec..ddf177ae65 100644
--- a/libswscale/swscale_unscaled.c
+++ b/libswscale/swscale_unscaled.c
@@ -786,6 +786,7 @@ static int planarCopyWrapper(SwsContext *c, const uint8_t *src[],
         int height = (plane == 0 || plane == 3) ? srcSliceH: -((-srcSliceH) >> c->chrDstVSubSample);
         const uint8_t *srcPtr = src[plane];
         uint8_t *dstPtr = dst[plane] + dstStride[plane] * y;
+        int shiftonly = plane == 1 || plane == 2 || (!c->srcRange && plane == 0);
 
         if (!dst[plane])
             continue;
@@ -812,13 +813,24 @@ static int planarCopyWrapper(SwsContext *c, const uint8_t *src[],
                 if (is16BPS(c->dstFormat)) {
                     uint16_t *dstPtr2 = (uint16_t *) dstPtr;
 #define COPY9_OR_10TO16(rfunc, wfunc) \
-                    for (i = 0; i < height; i++) { \
-                        for (j = 0; j < length; j++) { \
-                            int srcpx = rfunc(&srcPtr2[j]); \
-                            wfunc(&dstPtr2[j], (srcpx << (16 - src_depth)) | (srcpx >> (2 * src_depth - 16))); \
+                    if (shiftonly) { \
+                        for (i = 0; i < height; i++) { \
+                            for (j = 0; j < length; j++) { \
+                                int srcpx = rfunc(&srcPtr2[j]); \
+                                wfunc(&dstPtr2[j], srcpx << (16 - src_depth)); \
+                            } \
+                            dstPtr2 += dstStride[plane] / 2; \
+                            srcPtr2 += srcStride[plane] / 2; \
+                        } \
+                    } else { \
+                        for (i = 0; i < height; i++) { \
+                            for (j = 0; j < length; j++) { \
+                                int srcpx = rfunc(&srcPtr2[j]); \
+                                wfunc(&dstPtr2[j], (srcpx << (16 - src_depth)) | (srcpx >> (2 * src_depth - 16))); \
+                            } \
+                            dstPtr2 += dstStride[plane] / 2; \
+                            srcPtr2 += srcStride[plane] / 2; \
                         } \
-                        dstPtr2 += dstStride[plane] / 2; \
-                        srcPtr2 += srcStride[plane] / 2; \
                     }
                     if (isBE(c->dstFormat)) {
                         if (isBE(c->srcFormat)) {
@@ -916,13 +928,24 @@ static int planarCopyWrapper(SwsContext *c, const uint8_t *src[],
                     }
                 } else /* 8bit */ {
 #define COPY8TO9_OR_10(wfunc) \
-                    for (i = 0; i < height; i++) { \
-                        for (j = 0; j < length; j++) { \
-                            const int srcpx = srcPtr[j]; \
-                            wfunc(&dstPtr2[j], (srcpx << (dst_depth - 8)) | (srcpx >> (16 - dst_depth))); \
+                    if (shiftonly) { \
+                        for (i = 0; i < height; i++) { \
+                            for (j = 0; j < length; j++) { \
+                                const int srcpx = srcPtr[j]; \
+                                wfunc(&dstPtr2[j], srcpx << (dst_depth - 8)); \
+                            } \
+                            dstPtr2 += dstStride[plane] / 2; \
+                            srcPtr  += srcStride[plane]; \
+                        } \
+                    } else { \
+                        for (i = 0; i < height; i++) { \
+                            for (j = 0; j < length; j++) { \
+                                const int srcpx = srcPtr[j]; \
+                                wfunc(&dstPtr2[j], (srcpx << (dst_depth - 8)) | (srcpx >> (16 - dst_depth))); \
+                            } \
+                            dstPtr2 += dstStride[plane] / 2; \
+                            srcPtr  += srcStride[plane]; \
                         } \
-                        dstPtr2 += dstStride[plane] / 2; \
-                        srcPtr  += srcStride[plane]; \
                     }
                     if (isBE(c->dstFormat)) {
                         COPY8TO9_OR_10(AV_WB16);
diff --git a/tests/ref/fate/filter-pixdesc-yuv420p10be b/tests/ref/fate/filter-pixdesc-yuv420p10be
index 4fe037144c..54db2287b3 100644
--- a/tests/ref/fate/filter-pixdesc-yuv420p10be
+++ b/tests/ref/fate/filter-pixdesc-yuv420p10be
@@ -1 +1 @@
-pixdesc-yuv420p10be 299fe1d785a3d3dd5e70778700d7fb06
+pixdesc-yuv420p10be 2f88c301feeaccd2a5fb55f54fc30be9
diff --git a/tests/ref/fate/filter-pixdesc-yuv420p10le b/tests/ref/fate/filter-pixdesc-yuv420p10le
index f1aaedfe36..433113c36e 100644
--- a/tests/ref/fate/filter-pixdesc-yuv420p10le
+++ b/tests/ref/fate/filter-pixdesc-yuv420p10le
@@ -1 +1 @@
-pixdesc-yuv420p10le 8aee004e765a5383be0954f5e916b72f
+pixdesc-yuv420p10le 93f175084af4e78f97c7710e505f3057
diff --git a/tests/ref/fate/filter-pixdesc-yuv420p9be b/tests/ref/fate/filter-pixdesc-yuv420p9be
index cd2e847d59..7548154b6c 100644
--- a/tests/ref/fate/filter-pixdesc-yuv420p9be
+++ b/tests/ref/fate/filter-pixdesc-yuv420p9be
@@ -1 +1 @@
-pixdesc-yuv420p9be  ce880fa07830e5297c22acf6e20555ce
+pixdesc-yuv420p9be  64e36fd90573f67ac2006d103972a79b
diff --git a/tests/ref/fate/filter-pixdesc-yuv420p9le b/tests/ref/fate/filter-pixdesc-yuv420p9le
index c887495d1c..535c63b622 100644
--- a/tests/ref/fate/filter-pixdesc-yuv420p9le
+++ b/tests/ref/fate/filter-pixdesc-yuv420p9le
@@ -1 +1 @@
-pixdesc-yuv420p9le  16543fda8f87d94a6cf857d2e8d4461a
+pixdesc-yuv420p9le  9ed4b1dfabc53fd9e586ff6c4c43af80
diff --git a/tests/ref/fate/filter-pixdesc-yuva420p10be b/tests/ref/fate/filter-pixdesc-yuva420p10be
index 6d3723fd93..1139323e4b 100644
--- a/tests/ref/fate/filter-pixdesc-yuva420p10be
+++ b/tests/ref/fate/filter-pixdesc-yuva420p10be
@@ -1 +1 @@
-pixdesc-yuva420p10be145366ff1632de3e300d947f49844284
+pixdesc-yuva420p10be80d27a23837b18f7e268d33532b89362
diff --git a/tests/ref/fate/filter-pixdesc-yuva420p10le b/tests/ref/fate/filter-pixdesc-yuva420p10le
index 5b7f692cd8..255154d2dc 100644
--- a/tests/ref/fate/filter-pixdesc-yuva420p10le
+++ b/tests/ref/fate/filter-pixdesc-yuva420p10le
@@ -1 +1 @@
-pixdesc-yuva420p10led797038552d7f698e4d1db4dfa18ceb0
+pixdesc-yuva420p10le62772ae74d4bacbd462607d98ffe556f
diff --git a/tests/ref/fate/filter-pixdesc-yuva420p9be b/tests/ref/fate/filter-pixdesc-yuva420p9be
index 3a621bdf12..dfafa34df7 100644
--- a/tests/ref/fate/filter-pixdesc-yuva420p9be
+++ b/tests/ref/fate/filter-pixdesc-yuva420p9be
@@ -1 +1 @@
-pixdesc-yuva420p9be 06b764d85bd3c22e9b7ca4babed84d4f
+pixdesc-yuva420p9be ee0e8e3ffeb198e6e20c27de366309ed
diff --git a/tests/ref/fate/filter-pixdesc-yuva420p9le b/tests/ref/fate/filter-pixdesc-yuva420p9le
index 5edc34df4e..5e4ef4b162 100644
--- a/tests/ref/fate/filter-pixdesc-yuva420p9le
+++ b/tests/ref/fate/filter-pixdesc-yuva420p9le
@@ -1 +1 @@
-pixdesc-yuva420p9le 1f01cdd4fc46f98d4c11b2947307a0e3
+pixdesc-yuva420p9le 8b2ba3275e5a90dc10b98af9001e3800
diff --git a/tests/ref/fate/filter-pixfmts-copy b/tests/ref/fate/filter-pixfmts-copy
index f99e8ead46..5f550250de 100644
--- a/tests/ref/fate/filter-pixfmts-copy
+++ b/tests/ref/fate/filter-pixfmts-copy
@@ -40,12 +40,12 @@ uyvy422             adcf64516a19fce44df77082bdb16291
 yuv410p             2d9225153c83ee1132397d619d94d1b3
 yuv411p             8b298af3e43348ca1b11eb8a3252ac6c
 yuv420p             eba2f135a08829387e2f698ff72a2939
-yuv420p10be         299fe1d785a3d3dd5e70778700d7fb06
-yuv420p10le         8aee004e765a5383be0954f5e916b72f
+yuv420p10be         2f88c301feeaccd2a5fb55f54fc30be9
+yuv420p10le         93f175084af4e78f97c7710e505f3057
 yuv420p16be         16c009a235cd52b74791a895423152a3
 yuv420p16le         2d59c4f1d0314a5a957a7cfc4b6fabcc
-yuv420p9be          ce880fa07830e5297c22acf6e20555ce
-yuv420p9le          16543fda8f87d94a6cf857d2e8d4461a
+yuv420p9be          64e36fd90573f67ac2006d103972a79b
+yuv420p9le          9ed4b1dfabc53fd9e586ff6c4c43af80
 yuv422p             c9bba4529821d796a6ab09f6a5fd355a
 yuv422p10be         11af7dfafe8bc025c7e3bd82b830fe8a
 yuv422p10le         ec04efb76efa79bf0d02b21572371a56
@@ -62,12 +62,12 @@ yuv444p16le         20f86bc2f68d2b3f1f2b48b97b2189f4
 yuv444p9be          6ab31f4c12b533ce318ecdff83cdd054
 yuv444p9le          f0606604a5c08becab6ba500124c4b7c
 yuva420p            a29884f3f3dfe1e00b961bc17bef3d47
-yuva420p10be        145366ff1632de3e300d947f49844284
-yuva420p10le        d797038552d7f698e4d1db4dfa18ceb0
+yuva420p10be        80d27a23837b18f7e268d33532b89362
+yuva420p10le        62772ae74d4bacbd462607d98ffe556f
 yuva420p16be        25a335f66a0670911ced818aa42fb670
 yuva420p16le        97bf252e6c030f0f0412d3826c2ea259
-yuva420p9be         06b764d85bd3c22e9b7ca4babed84d4f
-yuva420p9le         1f01cdd4fc46f98d4c11b2947307a0e3
+yuva420p9be         ee0e8e3ffeb198e6e20c27de366309ed
+yuva420p9le         8b2ba3275e5a90dc10b98af9001e3800
 yuva422p            92b6815f465297284cdb843711682cee
 yuva422p10be        fb240ff9ac49b45b1b3d40df2c89e39d
 yuva422p10le        f767ede9ba1d427faadc963cf41d2412
diff --git a/tests/ref/fate/filter-pixfmts-null b/tests/ref/fate/filter-pixfmts-null
index f99e8ead46..5f550250de 100644
--- a/tests/ref/fate/filter-pixfmts-null
+++ b/tests/ref/fate/filter-pixfmts-null
@@ -40,12 +40,12 @@ uyvy422             adcf64516a19fce44df77082bdb16291
 yuv410p             2d9225153c83ee1132397d619d94d1b3
 yuv411p             8b298af3e43348ca1b11eb8a3252ac6c
 yuv420p             eba2f135a08829387e2f698ff72a2939
-yuv420p10be         299fe1d785a3d3dd5e70778700d7fb06
-yuv420p10le         8aee004e765a5383be0954f5e916b72f
+yuv420p10be         2f88c301feeaccd2a5fb55f54fc30be9
+yuv420p10le         93f175084af4e78f97c7710e505f3057
 yuv420p16be         16c009a235cd52b74791a895423152a3
 yuv420p16le         2d59c4f1d0314a5a957a7cfc4b6fabcc
-yuv420p9be          ce880fa07830e5297c22acf6e20555ce
-yuv420p9le          16543fda8f87d94a6cf857d2e8d4461a
+yuv420p9be          64e36fd90573f67ac2006d103972a79b
+yuv420p9le          9ed4b1dfabc53fd9e586ff6c4c43af80
 yuv422p             c9bba4529821d796a6ab09f6a5fd355a
 yuv422p10be         11af7dfafe8bc025c7e3bd82b830fe8a
 yuv422p10le         ec04efb76efa79bf0d02b21572371a56
@@ -62,12 +62,12 @@ yuv444p16le         20f86bc2f68d2b3f1f2b48b97b2189f4
 yuv444p9be          6ab31f4c12b533ce318ecdff83cdd054
 yuv444p9le          f0606604a5c08becab6ba500124c4b7c
 yuva420p            a29884f3f3dfe1e00b961bc17bef3d47
-yuva420p10be        145366ff1632de3e300d947f49844284
-yuva420p10le        d797038552d7f698e4d1db4dfa18ceb0
+yuva420p10be        80d27a23837b18f7e268d33532b89362
+yuva420p10le        62772ae74d4bacbd462607d98ffe556f
 yuva420p16be        25a335f66a0670911ced818aa42fb670
 yuva420p16le        97bf252e6c030f0f0412d3826c2ea259
-yuva420p9be         06b764d85bd3c22e9b7ca4babed84d4f
-yuva420p9le         1f01cdd4fc46f98d4c11b2947307a0e3
+yuva420p9be         ee0e8e3ffeb198e6e20c27de366309ed
+yuva420p9le         8b2ba3275e5a90dc10b98af9001e3800
 yuva422p            92b6815f465297284cdb843711682cee
 yuva422p10be        fb240ff9ac49b45b1b3d40df2c89e39d
 yuva422p10le        f767ede9ba1d427faadc963cf41d2412
diff --git a/tests/ref/fate/filter-pixfmts-scale b/tests/ref/fate/filter-pixfmts-scale
index dcd73bd557..a06aa490aa 100644
--- a/tests/ref/fate/filter-pixfmts-scale
+++ b/tests/ref/fate/filter-pixfmts-scale
@@ -40,12 +40,12 @@ uyvy422             314bd486277111a95d9369b944fa0400
 yuv410p             7df8f6d69b56a8dcb6c7ee908e5018b5
 yuv411p             1143e7c5cc28fe0922b051b17733bc4c
 yuv420p             fdad2d8df8985e3d17e73c71f713cb14
-yuv420p10be         27f28a6e09b1c04d0f755035a5db1f43
-yuv420p10le         a5a1692e026590ba2eddb46b9b827529
+yuv420p10be         418039dbd82cf612db88417276aa0d1a
+yuv420p10le         ff7e5321208ab995b4f95634ebdf192b
 yuv420p16be         d7270efce54eb59c7b01c14157a1b890
 yuv420p16le         e85abf00bad940a922b623c91c9026d7
-yuv420p9be          bb87fddca65d1742412c8d2b1caf96c6
-yuv420p9le          828eec50014a41258a5423c1fe56ac97
+yuv420p9be          ec4983b7a949c0472110a7a2c58e278a
+yuv420p9le          c136dce5913a722eee44ab72cff664b2
 yuv422p             918e37701ee7377d16a8a6c119c56a40
 yuv422p10be         315654908d50718e175aae018c484732
 yuv422p10le         91bbc78a9a56f659b55abc17722dcc09
@@ -62,12 +62,12 @@ yuv444p16le         a0c5d3c7bf3f181db503cf8e450d1335
 yuv444p9be          9ac2643ce7f7e5c4e17c8c9fd8494d4a
 yuv444p9le          896a1cc9cccca1ba410dd53942d33cc4
 yuva420p            8673a9131fb47de69788863f93a50eb7
-yuva420p10be        d92a95061809f251175f5d5e3074930e
-yuva420p10le        bad90ba2d4c260e379a7aa6dc7760853
+yuva420p10be        5ed78d252f3bb41224b9c69508e1d4a7
+yuva420p10le        ca9b865a5dc4f2658c440b15dfe3fa56
 yuva420p16be        a61d8ddb646e2d26020fc7ed2a48c1a9
 yuva420p16le        90ef774f86ad3177ec57eca8744b4e09
-yuva420p9be         f7655546446bfdc875243d7cdeb13b30
-yuva420p9le         ada2b719827059d70ebc57e2a3f9da92
+yuva420p9be         aefa147c4a8cc6e758a0a76ef99644bd
+yuva420p9le         83fb9caae435a587966620ba23fc62ce
 yuva422p            3c76ebeca0a7d3aa5f8e31ef80a86ffe
 yuva422p10be        01dd539e4a62762a3c97e965c76bb6f7
 yuva422p10le        76355d9d8fdcd085a24d48832b72e40b
diff --git a/tests/ref/fate/filter-pixfmts-vflip b/tests/ref/fate/filter-pixfmts-vflip
index 073322df2f..819c507df6 100644
--- a/tests/ref/fate/filter-pixfmts-vflip
+++ b/tests/ref/fate/filter-pixfmts-vflip
@@ -40,12 +40,12 @@ uyvy422             ffbd36720c77398d9a0d03ce2625928f
 yuv410p             7bfb39d7afb49d6a6173e6b23ae321eb
 yuv411p             4a90048cc3a65fac150e53289700efe1
 yuv420p             2e6d6062e8cad37fb3ab2c433b55f382
-yuv420p10be         fb0772f5e2b9da20ff826e64c3893137
-yuv420p10le         e95879e14c4a6805f39643964baf41f7
+yuv420p10be         7ce787a888381dd46b0212c099ecaad9
+yuv420p10le         bf22a1c543a7b3dbc556a0eb9592e179
 yuv420p16be         539076782902664a8acf381bf4f713e8
 yuv420p16le         0f609e588e5a258644ef85170d70e030
-yuv420p9be          be40ec975fb2873891643cbbbddbc3b0
-yuv420p9le          7e606310d3f5ff12badf911e8f333471
+yuv420p9be          9865bf5c4392b56b1c4eb4f5a3fd32f9
+yuv420p9le          0f1e371a1374d3cba2205b70cc7cac90
 yuv422p             d7f5cb44d9b0210d66d6a8762640ab34
 yuv422p10be         0be8378c3773e1c0b394315ef4994351
 yuv422p10le         6518094fe8de6bee95af21af1e5dc1e1
@@ -62,12 +62,12 @@ yuv444p16le         8e83323cf102d6c823a03ae8a7b7e033
 yuv444p9be          6ac92b7dc9ab2fc59bee99204886899a
 yuv444p9le          85aef13a654953d3455d89770b0d74bd
 yuva420p            c705d1cf061d8c6580ac690b55f92276
-yuva420p10be        baa5e3b0ff6d0ebbb0958560cd763c6e
-yuva420p10le        a36dc59ad55b406e5fee475236e9753c
+yuva420p10be        8785f414c4876a8e8acbce25e4b4408b
+yuva420p10le        9a6fa75022a9ab5a1653bfc2e0ffbf7f
 yuva420p16be        bf3b134eb70878df9afba61d03e930b8
 yuva420p16le        105d375154329a381aa58379a0a6ec46
-yuva420p9be         8273d591e055f48990c29dd905a6cdfd
-yuva420p9le         95ced0bb07e422d98db61a35cdb3fb8f
+yuva420p9be         8c96b6fddf6a775ad25d8a572e79319b
+yuva420p9le         4d06884fff790d125cb352ed92256ac6
 yuva422p            6aed0ea657ed51cc047a4fbdd981aec8
 yuva422p10be        d69a3404984c5fd30c0fc548532bcb6b
 yuva422p10le        a40c8e6f50e12d94bf7484107ec98559
diff --git a/tests/ref/fate/v210 b/tests/ref/fate/v210
index 2847564b6b..7ad9dcf9e7 100644
--- a/tests/ref/fate/v210
+++ b/tests/ref/fate/v210
@@ -1,2 +1,2 @@
 #tb 0: 1/50
-0,          0,          0,        1,  3686400, 0x75ee1dde
+0,          0,          0,        1,  3686400, 0x8d5c3847