swscale/utils: Fix color range of gray16

Improves rgb -> gray16 conversion

Fixes Ticket3422

The pam and png output files look visually similar, in both cases the
dynamics increase to 0x0 -> 0xfffb.

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 37c07d45290cf9cdad061cd5c2af94a2783ba847)

13 files changed, 27 insertions, 25 deletions

diff --git a/libswscale/utils.c b/libswscale/utils.c
index 4e73853ebb..f2e41671a7 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -1055,6 +1055,8 @@ static int handle_jpeg(enum AVPixelFormat *format)
         *format = AV_PIX_FMT_YUV440P;
         return 1;
     case AV_PIX_FMT_GRAY8:
+    case AV_PIX_FMT_GRAY16LE:
+    case AV_PIX_FMT_GRAY16BE:
         return 1;
     default:
         return 0;
diff --git a/tests/ref/fate/filter-pixdesc b/tests/ref/fate/filter-pixdesc
index 5c7d203394..e7a5bc102f 100644
--- a/tests/ref/fate/filter-pixdesc
+++ b/tests/ref/fate/filter-pixdesc
@@ -26,8 +26,8 @@ gbrp14le            937ff1dd9f498b39f9e882316e371fbf
 gbrp9be             c76ab5850c9bc72bbbf36caa6d1c5ac7
 gbrp9le             5ad363dc9570187ad3e3f2344fbb30cf
 gray                2ee2ea2340d0ecf2dfa6f90f87384799
-gray16be            389f4e5a8ab413b3af32767b59ed7f9e
-gray16le            a1f912941247e45b394b9cf4f0e81130
+gray16be            a61507aec1088f5692036e1aabdb4f41
+gray16le            171fbdd46e3737bc865d0185a0006e1c
 monob               309b5785a36bd988d17e15d88f4ffad1
 monow               8809a02bc69b58d1114b09ca79ebffad
 nv12                75e90c54d858b993e99f4ee6d2a2a38f
diff --git a/tests/ref/fate/filter-pixfmts-copy b/tests/ref/fate/filter-pixfmts-copy
index 9040007963..ac04857813 100644
--- a/tests/ref/fate/filter-pixfmts-copy
+++ b/tests/ref/fate/filter-pixfmts-copy
@@ -26,8 +26,8 @@ gbrp14le            937ff1dd9f498b39f9e882316e371fbf
 gbrp9be             c76ab5850c9bc72bbbf36caa6d1c5ac7
 gbrp9le             5ad363dc9570187ad3e3f2344fbb30cf
 gray                2ee2ea2340d0ecf2dfa6f90f87384799
-gray16be            389f4e5a8ab413b3af32767b59ed7f9e
-gray16le            a1f912941247e45b394b9cf4f0e81130
+gray16be            a61507aec1088f5692036e1aabdb4f41
+gray16le            171fbdd46e3737bc865d0185a0006e1c
 monob               309b5785a36bd988d17e15d88f4ffad1
 monow               8809a02bc69b58d1114b09ca79ebffad
 nv12                75e90c54d858b993e99f4ee6d2a2a38f
diff --git a/tests/ref/fate/filter-pixfmts-crop b/tests/ref/fate/filter-pixfmts-crop
index 2efc8e9000..91875a5a9b 100644
--- a/tests/ref/fate/filter-pixfmts-crop
+++ b/tests/ref/fate/filter-pixfmts-crop
@@ -26,8 +26,8 @@ gbrp14le            d1465f7280f35aa0a70709e5a7bee1a4
 gbrp9be             f17b7ba66ba35ed0fcbbb5c32c7e0f56
 gbrp9le             fc11219debfbe8dd8c3d6f0ef92c4d50
 gray                6d34024704f862c75db3ba6989a4a039
-gray16be            02ac848ad4e28c06938599563ba81ff7
-gray16le            672aebfeb8a0f4067b3c6064340056e4
+gray16be            27cfdb4b211cad34f66a664cabd754b4
+gray16le            91cb081d457100a60c2d0f54110f064f
 nv12                923a313a7013fb0e87608155ef6aa9a4
 nv21                21e6b9273bb74203beabeb9edb9cf95c
 pal8                e1fd50b8a8a67fb5abd8b44abc778bbb
diff --git a/tests/ref/fate/filter-pixfmts-field b/tests/ref/fate/filter-pixfmts-field
index 5d6ff46caf..ea3c954bba 100644
--- a/tests/ref/fate/filter-pixfmts-field
+++ b/tests/ref/fate/filter-pixfmts-field
@@ -26,8 +26,8 @@ gbrp14le            775b50257b848007c4ef3441ba772db1
 gbrp9be             c293422f1395bfddc788282eef139ed6
 gbrp9le             0d2bb77c25d84611ec6222f3dffe11c0
 gray                42a0ad7625a0481183e375e38679d8d3
-gray16be            a447af6482b922c9997ac02e5d3535f1
-gray16le            c1dd0db327295898ff282d07f48c105d
+gray16be            e10bc0a8b015fdb0776eca402ffe5eff
+gray16le            2eb159fb4af25c3b4f033e6414fef63e
 monob               1b7fb7e69a913a0a1c0dffc54e5899ea
 monow               b5d3778a054fc73d515d36f8a6bc693b
 nv12                b3829e9ae2a15349432b7efac4236068
diff --git a/tests/ref/fate/filter-pixfmts-fieldorder b/tests/ref/fate/filter-pixfmts-fieldorder
index d5adddc169..33bcd8eb7e 100644
--- a/tests/ref/fate/filter-pixfmts-fieldorder
+++ b/tests/ref/fate/filter-pixfmts-fieldorder
@@ -26,8 +26,8 @@ gbrp14le            7baa94cd296e6ec8e41446bca95151e4
 gbrp9be             a6eb7cde03f19a25bf13627d731a2c9a
 gbrp9le             1b6d228b97a370ec76707ed2dcc15a66
 gray                3258910ef8c6f0c4a331368e7af87507
-gray16be            50cc8a29e8e81e174676542347804a4f
-gray16le            fb93e8aa2deed734dfd1ca6a5c48cf18
+gray16be            9a4c7e731ce572ad86985b56a653b5c3
+gray16le            d10df57cf0de14206c8e354e8f6b732b
 rgb0                5f8f8076e2b9a2422cac25a4f0459d79
 rgb24               66ca89ced3ade4c239ad6c5a79a6b8cd
 rgb444be            7197a09137ab5630a26226396bb7e313
diff --git a/tests/ref/fate/filter-pixfmts-hflip b/tests/ref/fate/filter-pixfmts-hflip
index 7fc2bb62ac..f5ee81fc49 100644
--- a/tests/ref/fate/filter-pixfmts-hflip
+++ b/tests/ref/fate/filter-pixfmts-hflip
@@ -26,8 +26,8 @@ gbrp14le            297e71281660b905711330a86eca8a71
 gbrp9be             8268b9a1e9f4d6a42e57db9c81d82fa5
 gbrp9le             b3a09bba825e16e6d160328706a9f62f
 gray                aaa9c2fe3c2a2a43a4b35226ea689b3c
-gray16be            d206a080739d89cb7dc0009ad4082ed4
-gray16le            7ebcfd9401ba85e584230de8fc02986d
+gray16be            5be35a44f0ba85eb8c55b4f062fdb80f
+gray16le            d9e73f7c4f53d795192ab87649270241
 nv12                719adbc47fa74e92f83150921917483f
 nv21                9c833b3ce53539d270e1f21e4319797b
 pal8                19c8735b23feeed18ec2d37913a5f3f8
diff --git a/tests/ref/fate/filter-pixfmts-il b/tests/ref/fate/filter-pixfmts-il
index 707c6f2a82..cdbfca1bba 100644
--- a/tests/ref/fate/filter-pixfmts-il
+++ b/tests/ref/fate/filter-pixfmts-il
@@ -26,8 +26,8 @@ gbrp14le            778c97b5ed06b9f1a230840a15771bac
 gbrp9be             b9fc10ab1ddad0e7945d6b047725d078
 gbrp9le             ef3d6bc8069b95cae31100908a7fa967
 gray                2cadbaed81ee12181bda9f4aa87ddbc0
-gray16be            cd9c1367dabd2f1858ae4f31693e622f
-gray16le            4ef774c282280d7ed4780690df6e5cb4
+gray16be            34cd1af8bbdd4266d5f7985ef22cfc9f
+gray16le            0eec98b32e4a2a57d9f51aac9ac3cf8d
 monob               07cffe7f5f25f39c3aa38866303791c6
 monow               f2d1bdb939813a49abd6348ecfbb2703
 nv12                6847b3f7141ca1e3c40d3a494f0e13cb
diff --git a/tests/ref/fate/filter-pixfmts-null b/tests/ref/fate/filter-pixfmts-null
index 9040007963..ac04857813 100644
--- a/tests/ref/fate/filter-pixfmts-null
+++ b/tests/ref/fate/filter-pixfmts-null
@@ -26,8 +26,8 @@ gbrp14le            937ff1dd9f498b39f9e882316e371fbf
 gbrp9be             c76ab5850c9bc72bbbf36caa6d1c5ac7
 gbrp9le             5ad363dc9570187ad3e3f2344fbb30cf
 gray                2ee2ea2340d0ecf2dfa6f90f87384799
-gray16be            389f4e5a8ab413b3af32767b59ed7f9e
-gray16le            a1f912941247e45b394b9cf4f0e81130
+gray16be            a61507aec1088f5692036e1aabdb4f41
+gray16le            171fbdd46e3737bc865d0185a0006e1c
 monob               309b5785a36bd988d17e15d88f4ffad1
 monow               8809a02bc69b58d1114b09ca79ebffad
 nv12                75e90c54d858b993e99f4ee6d2a2a38f
diff --git a/tests/ref/fate/filter-pixfmts-scale b/tests/ref/fate/filter-pixfmts-scale
index 150d8994d7..c772b242f8 100644
--- a/tests/ref/fate/filter-pixfmts-scale
+++ b/tests/ref/fate/filter-pixfmts-scale
@@ -26,8 +26,8 @@ gbrp14le            eb6cb4555edb175d807fe1b5382d2fc7
 gbrp9be             2c9adb80abc16546cac69b4872aaf557
 gbrp9le             fcfa1684553e3e185179462bca347649
 gray                c45dcee08887f43dc463f79d7ecd7d68
-gray16be            70064f9acdc5e3935ccda67e765bf2fb
-gray16le            578241fb43029e5ae841a3c94d940dce
+gray16be            52a6ff8ba7fe11032f370c0888d87fc4
+gray16le            e78f07e83698651639743f2f8501c51c
 monob               91ec2a25b13f6ca34d42da778b217de0
 monow               a991455fda8f60f373aeb744456996b9
 nv12                0617f1e13ae4a43d4cb49282b9c02f71
diff --git a/tests/ref/fate/filter-pixfmts-vflip b/tests/ref/fate/filter-pixfmts-vflip
index b513fde23b..07d2bb765e 100644
--- a/tests/ref/fate/filter-pixfmts-vflip
+++ b/tests/ref/fate/filter-pixfmts-vflip
@@ -26,8 +26,8 @@ gbrp14le            48b4273ff29b6b68a05a6027254ff75e
 gbrp9be             b4361a1ad66cdff0d32d4af769a8a960
 gbrp9le             5bc148ca18ff1bf7095e78a4e65ed8ab
 gray                800813149a825964025e75cf14ec528b
-gray16be            9b23f3e79c54a6ccb62e0135a32e3045
-gray16le            93cfa8fbb2a86ead275ce1817444e6d5
+gray16be            03fdfc40a4fd25252f1b03d9358f00ef
+gray16le            ebcd797559fef441e5baeff4e7a02472
 monob               c395a8efb9477b4ec53a77326e41ccd7
 monow               efaee1c763ccd5ce1a8519d2ed5aa5a9
 nv12                77373304a9c732b65dab0a33afba9295
diff --git a/tests/ref/lavf/pam b/tests/ref/lavf/pam
index 50048edf10..abb29743d0 100644
--- a/tests/ref/lavf/pam
+++ b/tests/ref/lavf/pam
@@ -7,8 +7,8 @@
 35cb9e42b2d3181be494f8693af1ddea *./tests/data/images/pam/02.pam
 ./tests/data/images/pam/%02d.pam CRC=0x0ff205be
 101445 ./tests/data/images/pam/02.pam
-ebd43e97839b2538a79f35757e84ffb0 *./tests/data/images/pam/02.pam
-./tests/data/images/pam/%02d.pam CRC=0x831a2963
+740eb42157af9e9eed46b70ba6a6cf4d *./tests/data/images/pam/02.pam
+./tests/data/images/pam/%02d.pam CRC=0x893f10ef
 202823 ./tests/data/images/pam/02.pam
 032538f0313b4f240b44a5bef115f5bf *./tests/data/images/pam/02.pam
 ./tests/data/images/pam/%02d.pam CRC=0x5984c023
diff --git a/tests/ref/lavf/png b/tests/ref/lavf/png
index 40e7cca014..9cf677b21b 100644
--- a/tests/ref/lavf/png
+++ b/tests/ref/lavf/png
@@ -1,9 +1,9 @@
 2af72da4468e61a37c220b25cb28618a *./tests/data/images/png/02.png
 ./tests/data/images/png/%02d.png CRC=0x6da01946
 248633 ./tests/data/images/png/02.png
-62b26c9582ba37dd37b76191338f3770 *./tests/data/images/png/02.png
-./tests/data/images/png/%02d.png CRC=0x831a2963
-41687 ./tests/data/images/png/02.png
+6cf54c13aa407b77547cf6dfe23ecba3 *./tests/data/images/png/02.png
+./tests/data/images/png/%02d.png CRC=0x893f10ef
+47365 ./tests/data/images/png/02.png
 b4e38244c97debe3f528e7d1adb283ef *./tests/data/images/png/02.png
 ./tests/data/images/png/%02d.png CRC=0x5984c023
 511900 ./tests/data/images/png/02.png