swscale: fix 12 and 14 bit YUV planar output with scaling.

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

7 files changed, 65 insertions, 54 deletions

diff --git a/libswscale/output.c b/libswscale/output.c
index 8306298d64..8c200685a9 100644
--- a/libswscale/output.c
+++ b/libswscale/output.c
@@ -244,6 +244,10 @@ yuv2NBPS( 9, BE, 1, 10, int16_t)
 yuv2NBPS( 9, LE, 0, 10, int16_t)
 yuv2NBPS(10, BE, 1, 10, int16_t)
 yuv2NBPS(10, LE, 0, 10, int16_t)
+yuv2NBPS(12, BE, 1, 10, int16_t)
+yuv2NBPS(12, LE, 0, 10, int16_t)
+yuv2NBPS(14, BE, 1, 10, int16_t)
+yuv2NBPS(14, LE, 0, 10, int16_t)
 yuv2NBPS(16, BE, 1, 16, int32_t)
 yuv2NBPS(16, LE, 0, 16, int32_t)
 
@@ -1279,10 +1283,17 @@ av_cold void ff_sws_init_output_funcs(SwsContext *c,
         if (desc->comp[0].depth_minus1 == 8) {
             *yuv2planeX = isBE(dstFormat) ? yuv2planeX_9BE_c  : yuv2planeX_9LE_c;
             *yuv2plane1 = isBE(dstFormat) ? yuv2plane1_9BE_c  : yuv2plane1_9LE_c;
-        } else {
+        } else if (desc->comp[0].depth_minus1 == 9) {
             *yuv2planeX = isBE(dstFormat) ? yuv2planeX_10BE_c  : yuv2planeX_10LE_c;
             *yuv2plane1 = isBE(dstFormat) ? yuv2plane1_10BE_c  : yuv2plane1_10LE_c;
-        }
+        } else if (desc->comp[0].depth_minus1 == 11) {
+            *yuv2planeX = isBE(dstFormat) ? yuv2planeX_12BE_c  : yuv2planeX_12LE_c;
+            *yuv2plane1 = isBE(dstFormat) ? yuv2plane1_12BE_c  : yuv2plane1_12LE_c;
+        } else if (desc->comp[0].depth_minus1 == 13) {
+            *yuv2planeX = isBE(dstFormat) ? yuv2planeX_14BE_c  : yuv2planeX_14LE_c;
+            *yuv2plane1 = isBE(dstFormat) ? yuv2plane1_14BE_c  : yuv2plane1_14LE_c;
+        } else
+            av_assert0(0);
     } else {
         *yuv2plane1 = yuv2plane1_8_c;
         *yuv2planeX = yuv2planeX_8_c;
diff --git a/tests/ref/lavfi/field b/tests/ref/lavfi/field
index 3ca797a0e4..ec50dc11a9 100644
--- a/tests/ref/lavfi/field
+++ b/tests/ref/lavfi/field
@@ -53,10 +53,10 @@ yuv420p9le          bbf80e57389578be66d4a7a12335a613
 yuv422p             e461a21995da361b88202339a2ebb879
 yuv422p10be         a3e13070215f5a016ac9bae7e7115417
 yuv422p10le         8e9e3d9adc8fdb8a0a03d79bdc31eefe
-yuv422p12be         4c339f71d79d2dac1dabc6121e1cf021
-yuv422p12le         69d336fccbe1ffa88106ea5bde0c8743
-yuv422p14be         f20c21dbfda632d26816fce27c1cb6e9
-yuv422p14le         70dddb8bdad188079a05113059d139f8
+yuv422p12be         6a2aca2ea007534e729203641a82c3a3
+yuv422p12le         aef072b55c6e9df8fd9f398d79f864df
+yuv422p14be         34e5f31372c0fdb4d77ccb7fd7bddb2f
+yuv422p14le         c2c50fbea0cff7edad5c5e907380cbad
 yuv422p16be         55cfed8fa610f82b6625e16871dab235
 yuv422p16le         e2488df0f22987fe7ed12a5ef2adf835
 yuv422p9be          80fcdd7fd9cdd79632104dcc32f78b4b
@@ -65,10 +65,10 @@ yuv440p             f8e80596babcdb94378ec8bebf2dd46d
 yuv444p             572bad9e12ed53e242658fa613412279
 yuv444p10be         c5304f086afc4624d4fffb66a3cf3cb8
 yuv444p10le         d1754974b936f74028752d49413d30aa
-yuv444p12be         206d6b0fbd84d4e013b9b074cbd65135
-yuv444p12le         dc2b1bfbecba71eba50e7e4da470a8d0
-yuv444p14be         19cd2ef75ed5698898c55040e51def88
-yuv444p14le         8d47c9575d1355572ee9bfc873d46753
+yuv444p12be         7ec8fd2d7986d6e923dfed572589e643
+yuv444p12le         79d0e287e2171a53cf4f285c0d1e83ef
+yuv444p14be         be021957cd819ca3146f3247138ccae9
+yuv444p14le         bac0b0c70bcf72272d724f0cd1ac411e
 yuv444p16be         3a67c28325978db734ba03b1828c15da
 yuv444p16le         c6dc275a4277fd3c65535253bb298263
 yuv444p9be          f47357cdd775fc399aeab3ae58712fb9
diff --git a/tests/ref/lavfi/pixfmts_copy b/tests/ref/lavfi/pixfmts_copy
index 53640cd33b..1171546538 100644
--- a/tests/ref/lavfi/pixfmts_copy
+++ b/tests/ref/lavfi/pixfmts_copy
@@ -53,10 +53,10 @@ yuv420p9le          3720ba1786b56c8f4913e67dddebbb07
 yuv422p             eee223e92f1f2e5ad4186b411ebf5816
 yuv422p10be         80fa87750491dcc3e93d175b64835654
 yuv422p10le         e1640cd315bd96c6b59f7753825a7306
-yuv422p12be         8a17ae3a5f565479ec4aa230ef78a6fa
-yuv422p12le         04215506e2eca5b79f47e322a5949a58
-yuv422p14be         0fec4a6615ff610e3ea5e85acbd1b273
-yuv422p14le         a54c6121c1cb713a33961ea0faa35956
+yuv422p12be         5c6d86e919515d1af71f8004a6252f9e
+yuv422p12le         7f331cadb0ab75d07656d29f47bd8bd9
+yuv422p14be         4f4e186ba683b90fe990cccc4db7e521
+yuv422p14le         d78016c9119b1f1d1c922bd5aaf47732
 yuv422p16be         1f3c216927d17966a009a42def124bba
 yuv422p16le         90adbdffaeea9b09398339c8a1322b02
 yuv422p9be          e46446534dd0821029d3b831c9c1d1b0
@@ -65,10 +65,10 @@ yuv440p             456d83524b299f2fa1bb988e421554d9
 yuv444p             068bc09d282d18c9fa35c2787943fc2a
 yuv444p10be         9f6e1490080a454243daee9229425e1e
 yuv444p10le         cc183e51041351d9ac06c4df6e8ad5f7
-yuv444p12be         df420773bb3cca2ffb585db3bbe01195
-yuv444p12le         25c0c73d98c3c1e281044d79097608f5
-yuv444p14be         516b18e8331e7d404c7541bafc5501e6
-yuv444p14le         7c88d9a233979d73762cffef2fe08306
+yuv444p12be         34f63ac6a47ac681016105d3bfb3e118
+yuv444p12le         7964701bfe8d718edf97f6e0002935bd
+yuv444p14be         4ad3a9d52fe9c050150c0f7d2f3cf63d
+yuv444p14le         69575e5a94b978f159c4fdb1a61587b0
 yuv444p16be         4000b12fa88ed0feef182da31c7e6b96
 yuv444p16le         96a857dba8dc6792c58daec872825b32
 yuv444p9be          07727e5c9040b7f0a17d591288ac330d
diff --git a/tests/ref/lavfi/pixfmts_null b/tests/ref/lavfi/pixfmts_null
index 53640cd33b..1171546538 100644
--- a/tests/ref/lavfi/pixfmts_null
+++ b/tests/ref/lavfi/pixfmts_null
@@ -53,10 +53,10 @@ yuv420p9le          3720ba1786b56c8f4913e67dddebbb07
 yuv422p             eee223e92f1f2e5ad4186b411ebf5816
 yuv422p10be         80fa87750491dcc3e93d175b64835654
 yuv422p10le         e1640cd315bd96c6b59f7753825a7306
-yuv422p12be         8a17ae3a5f565479ec4aa230ef78a6fa
-yuv422p12le         04215506e2eca5b79f47e322a5949a58
-yuv422p14be         0fec4a6615ff610e3ea5e85acbd1b273
-yuv422p14le         a54c6121c1cb713a33961ea0faa35956
+yuv422p12be         5c6d86e919515d1af71f8004a6252f9e
+yuv422p12le         7f331cadb0ab75d07656d29f47bd8bd9
+yuv422p14be         4f4e186ba683b90fe990cccc4db7e521
+yuv422p14le         d78016c9119b1f1d1c922bd5aaf47732
 yuv422p16be         1f3c216927d17966a009a42def124bba
 yuv422p16le         90adbdffaeea9b09398339c8a1322b02
 yuv422p9be          e46446534dd0821029d3b831c9c1d1b0
@@ -65,10 +65,10 @@ yuv440p             456d83524b299f2fa1bb988e421554d9
 yuv444p             068bc09d282d18c9fa35c2787943fc2a
 yuv444p10be         9f6e1490080a454243daee9229425e1e
 yuv444p10le         cc183e51041351d9ac06c4df6e8ad5f7
-yuv444p12be         df420773bb3cca2ffb585db3bbe01195
-yuv444p12le         25c0c73d98c3c1e281044d79097608f5
-yuv444p14be         516b18e8331e7d404c7541bafc5501e6
-yuv444p14le         7c88d9a233979d73762cffef2fe08306
+yuv444p12be         34f63ac6a47ac681016105d3bfb3e118
+yuv444p12le         7964701bfe8d718edf97f6e0002935bd
+yuv444p14be         4ad3a9d52fe9c050150c0f7d2f3cf63d
+yuv444p14le         69575e5a94b978f159c4fdb1a61587b0
 yuv444p16be         4000b12fa88ed0feef182da31c7e6b96
 yuv444p16le         96a857dba8dc6792c58daec872825b32
 yuv444p9be          07727e5c9040b7f0a17d591288ac330d
diff --git a/tests/ref/lavfi/pixfmts_pixdesctest b/tests/ref/lavfi/pixfmts_pixdesctest
index 8844a1ce71..1057c418a5 100644
--- a/tests/ref/lavfi/pixfmts_pixdesctest
+++ b/tests/ref/lavfi/pixfmts_pixdesctest
@@ -53,10 +53,10 @@ yuv420p9le          3720ba1786b56c8f4913e67dddebbb07
 yuv422p             eee223e92f1f2e5ad4186b411ebf5816
 yuv422p10be         80fa87750491dcc3e93d175b64835654
 yuv422p10le         e1640cd315bd96c6b59f7753825a7306
-yuv422p12be         8a17ae3a5f565479ec4aa230ef78a6fa
-yuv422p12le         04215506e2eca5b79f47e322a5949a58
-yuv422p14be         0fec4a6615ff610e3ea5e85acbd1b273
-yuv422p14le         a54c6121c1cb713a33961ea0faa35956
+yuv422p12be         5c6d86e919515d1af71f8004a6252f9e
+yuv422p12le         7f331cadb0ab75d07656d29f47bd8bd9
+yuv422p14be         4f4e186ba683b90fe990cccc4db7e521
+yuv422p14le         d78016c9119b1f1d1c922bd5aaf47732
 yuv422p16be         1f3c216927d17966a009a42def124bba
 yuv422p16le         90adbdffaeea9b09398339c8a1322b02
 yuv422p9be          e46446534dd0821029d3b831c9c1d1b0
@@ -65,10 +65,10 @@ yuv440p             456d83524b299f2fa1bb988e421554d9
 yuv444p             068bc09d282d18c9fa35c2787943fc2a
 yuv444p10be         9f6e1490080a454243daee9229425e1e
 yuv444p10le         cc183e51041351d9ac06c4df6e8ad5f7
-yuv444p12be         df420773bb3cca2ffb585db3bbe01195
-yuv444p12le         25c0c73d98c3c1e281044d79097608f5
-yuv444p14be         516b18e8331e7d404c7541bafc5501e6
-yuv444p14le         7c88d9a233979d73762cffef2fe08306
+yuv444p12be         34f63ac6a47ac681016105d3bfb3e118
+yuv444p12le         7964701bfe8d718edf97f6e0002935bd
+yuv444p14be         4ad3a9d52fe9c050150c0f7d2f3cf63d
+yuv444p14le         69575e5a94b978f159c4fdb1a61587b0
 yuv444p16be         4000b12fa88ed0feef182da31c7e6b96
 yuv444p16le         96a857dba8dc6792c58daec872825b32
 yuv444p9be          07727e5c9040b7f0a17d591288ac330d
diff --git a/tests/ref/lavfi/pixfmts_scale b/tests/ref/lavfi/pixfmts_scale
index 9d0d92c08f..825dc3a11c 100644
--- a/tests/ref/lavfi/pixfmts_scale
+++ b/tests/ref/lavfi/pixfmts_scale
@@ -42,10 +42,10 @@ yuv411p             e5c8f3ca024a88dd07e0a92db3e2133d
 yuv420p             485d9af8608f926ffffbf42230b4150d
 yuv420p10be         e4f12002b46549779886f27fd3c6c645
 yuv420p10le         698b7a346ae6d1315280d3e28c86b973
-yuv420p12be         3d2f7485d277fb66503f6af88bc81b62
-yuv420p12le         a19c3fd126c7a11d9632922e713ce555
-yuv420p14be         a1fcbe583e2e687e89ced29260a51fe2
-yuv420p14le         c689144ea25dd2f760331a6d525bebc2
+yuv420p12be         10ccb1da4f69f5578e017ffdaa19adf6
+yuv420p12le         4f68fcfeee4fb70ea941979cc6a39d1f
+yuv420p14be         d7b1183a6ca2f5d80d1bddf609c35995
+yuv420p14le         934dbea61d804e7c66e805a45f1e3914
 yuv420p16be         6ebcf48a9317c6265430cd3ba5f569a2
 yuv420p16le         604a7215f384488a38aa0cffdbcbe17e
 yuv420p9be          7b293aec92897bf0a32ec3a02a20acf9
@@ -53,10 +53,10 @@ yuv420p9le          f37fdff7d6c62ebe2ab0cf791cd049db
 yuv422p             b1724528079199908c667b3428a7c01c
 yuv422p10be         1c7281254fb980c123148dfd145c4a05
 yuv422p10le         959caa253354e897f01848db21b8006e
-yuv422p12be         15f63e4cc4a175db4599af1c2aad4047
-yuv422p12le         08d08b2d2ab7482c41ebdf50fcc3ee3c
-yuv422p14be         ca008297abbb10b931a9e142747291ec
-yuv422p14le         0fd39837ecd739f9da916dbf81f0b45c
+yuv422p12be         9e1db2326fb3dd3a8f8d32f3a6f61305
+yuv422p12le         5a4aecf4baf74358551e75fd7f1549cd
+yuv422p14be         e75088b934f86ddb6a7ff1749b77687c
+yuv422p14le         2cdae3e0447b35ddee6161639107205b
 yuv422p16be         4ccc3e0bc94eaf3a3350e4d694c3a56c
 yuv422p16le         dde889032f13fa5f8d76b8484dd73116
 yuv422p9be          ba5eae7669d02a13f54a592344612098
@@ -65,10 +65,10 @@ yuv440p             19de0ba1fd354b42189e96cdf73ef7cc
 yuv444p             fba8a00cd0b182af82e9de3b2f4d2c8a
 yuv444p10be         db89397ac4d33e66b5eab3678661f73a
 yuv444p10le         28208677f07d4e4191ef228c096f3297
-yuv444p12be         e41f8e27049c52da647e3d6927b591b3
-yuv444p12le         9428605417583e26041cb2b8d166df62
-yuv444p14be         a71cad94426e92d77e2f548d4d21a7c0
-yuv444p14le         adfe8463839fe67394529299620c078d
+yuv444p12be         55d291347d13f98b520c7d0cf7e0bdfe
+yuv444p12le         fd12d6054262cebda2a9b1a2ac4dac98
+yuv444p14be         a078f34e001828d576e748ca0850af56
+yuv444p14le         d02ba5c47b0bbb359df8ef478aa4c395
 yuv444p16be         07e27200d2f4382090ca260874f02e92
 yuv444p16le         f8bac16bf1f2afbd3626e07bcc815a9f
 yuv444p9be          db739906e3ae3b8792cdc5a0c3387565
diff --git a/tests/ref/lavfi/pixfmts_vflip b/tests/ref/lavfi/pixfmts_vflip
index 66d796aa76..1d5d1bd464 100644
--- a/tests/ref/lavfi/pixfmts_vflip
+++ b/tests/ref/lavfi/pixfmts_vflip
@@ -53,10 +53,10 @@ yuv420p9le          c6fed3016f03c49aa01bdcd63f846d3a
 yuv422p             5e48541d3e659046ac0e1eacb038485f
 yuv422p10be         9097b4d06f1fc598375b7c6480a2ba50
 yuv422p10le         da7bd36f65cac06bd54bc9857493d835
-yuv422p12be         1203cc6c2294234ddc5435f48b192d7c
-yuv422p12le         5cc76614210c026c3f1a5398f8654774
-yuv422p14be         d52e4b435329e806c03860421b94f9fd
-yuv422p14le         143937ee2e47fa35b71972a2aa74b772
+yuv422p12be         9c0fd650ec9872fbe06e82dc41a7399f
+yuv422p12le         3bdb982268c06b8876e206d8b89b4583
+yuv422p14be         bb0fc25212c2aa9f273aad2d562c3384
+yuv422p14le         1f89e82da25ce459db456ed27631054f
 yuv422p16be         95ed65e718929dbeb5092bf0d852026d
 yuv422p16le         c62e554dde4a170177bec2efc3f4f1a2
 yuv422p9be          b249cd58eaab5ccac6bae3e184774362
@@ -65,10 +65,10 @@ yuv440p             fca8395b7e43e2d0f38dd6ba822ef747
 yuv444p             1ef971a434c53e81c578e3c7cb9cbcad
 yuv444p10be         3114a7469c2b9c440a9a9ccbd6e167d6
 yuv444p10le         34cb8774a1eb627b5f32825ac7d0ae9a
-yuv444p12be         d770e81c67d6fe707c239b422021dd29
-yuv444p12le         9aa076fb7ac058b34ae4e8f09d7895e6
-yuv444p14be         40a9bef6e22c4c3dfd1d9dbeecd701e3
-yuv444p14le         123539bdc6946cfb61bd0d82cf8c6f17
+yuv444p12be         2fd83655c3223d176243d7ff871df3cc
+yuv444p12le         cfec541fde2894656f6b3401177b7604
+yuv444p14be         11671a128afd177e5eda13bd2e93ac26
+yuv444p14le         5aefe5d3f78e4f3b86ed7041a20f9b8e
 yuv444p16be         35340098dc6956c34afce17bc6ffcedd
 yuv444p16le         201e3acfa6f7628dfe2a1663de1128b1
 yuv444p9be          6143c321929ade9e0bc93ddea926e936