swscale: more accurate DITHER_COPY macro for full and limited range

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

11 files changed, 58 insertions, 55 deletions

diff --git a/libswscale/swscale_unscaled.c b/libswscale/swscale_unscaled.c
index ef36aec500..5d81cd5af9 100644
--- a/libswscale/swscale_unscaled.c
+++ b/libswscale/swscale_unscaled.c
@@ -110,24 +110,6 @@ DECLARE_ALIGNED(8, static const uint8_t, dithers)[8][8][8]={
   { 112, 16,104,  8,118, 22,110, 14,},
 }};
 
-static const uint16_t dither_scale[15][16]={
-{    2,    3,    3,    5,    5,    5,    5,    5,    5,    5,    5,    5,    5,    5,    5,    5,},
-{    2,    3,    7,    7,   13,   13,   25,   25,   25,   25,   25,   25,   25,   25,   25,   25,},
-{    3,    3,    4,   15,   15,   29,   57,   57,   57,  113,  113,  113,  113,  113,  113,  113,},
-{    3,    4,    4,    5,   31,   31,   61,  121,  241,  241,  241,  241,  481,  481,  481,  481,},
-{    3,    4,    5,    5,    6,   63,   63,  125,  249,  497,  993,  993,  993,  993,  993, 1985,},
-{    3,    5,    6,    6,    6,    7,  127,  127,  253,  505, 1009, 2017, 4033, 4033, 4033, 4033,},
-{    3,    5,    6,    7,    7,    7,    8,  255,  255,  509, 1017, 2033, 4065, 8129,16257,16257,},
-{    3,    5,    6,    8,    8,    8,    8,    9,  511,  511, 1021, 2041, 4081, 8161,16321,32641,},
-{    3,    5,    7,    8,    9,    9,    9,    9,   10, 1023, 1023, 2045, 4089, 8177,16353,32705,},
-{    3,    5,    7,    8,   10,   10,   10,   10,   10,   11, 2047, 2047, 4093, 8185,16369,32737,},
-{    3,    5,    7,    8,   10,   11,   11,   11,   11,   11,   12, 4095, 4095, 8189,16377,32753,},
-{    3,    5,    7,    9,   10,   12,   12,   12,   12,   12,   12,   13, 8191, 8191,16381,32761,},
-{    3,    5,    7,    9,   10,   12,   13,   13,   13,   13,   13,   13,   14,16383,16383,32765,},
-{    3,    5,    7,    9,   10,   12,   14,   14,   14,   14,   14,   14,   14,   15,32767,32767,},
-{    3,    5,    7,    9,   11,   12,   14,   15,   15,   15,   15,   15,   15,   15,   16,65535,},
-};
-
 
 static void fillPlane(uint8_t *plane, int stride, int width, int height, int y,
                       uint8_t val)
@@ -1502,24 +1484,45 @@ static int packedCopyWrapper(SwsContext *c, const uint8_t *src[],
 }
 
 #define DITHER_COPY(dst, dstStride, src, srcStride, bswap, dbswap)\
-    uint16_t scale= dither_scale[dst_depth-1][src_depth-1];\
-    int shift= src_depth-dst_depth + dither_scale[src_depth-2][dst_depth-1];\
-    for (i = 0; i < height; i++) {\
-        const uint8_t *dither= dithers[src_depth-9][i&7];\
-        for (j = 0; j < length-7; j+=8){\
-            dst[j+0] = dbswap((bswap(src[j+0]) + dither[0])*scale>>shift);\
-            dst[j+1] = dbswap((bswap(src[j+1]) + dither[1])*scale>>shift);\
-            dst[j+2] = dbswap((bswap(src[j+2]) + dither[2])*scale>>shift);\
-            dst[j+3] = dbswap((bswap(src[j+3]) + dither[3])*scale>>shift);\
-            dst[j+4] = dbswap((bswap(src[j+4]) + dither[4])*scale>>shift);\
-            dst[j+5] = dbswap((bswap(src[j+5]) + dither[5])*scale>>shift);\
-            dst[j+6] = dbswap((bswap(src[j+6]) + dither[6])*scale>>shift);\
-            dst[j+7] = dbswap((bswap(src[j+7]) + dither[7])*scale>>shift);\
+    unsigned shift= src_depth-dst_depth, tmp;\
+    if (shiftonly) {\
+        for (i = 0; i < height; i++) {\
+            const uint8_t *dither= dithers[shift-1][i&7];\
+            for (j = 0; j < length-7; j+=8) {\
+                tmp = (bswap(src[j+0]) + dither[0])>>shift; dst[j+0] = dbswap(tmp - (tmp>>dst_depth));\
+                tmp = (bswap(src[j+1]) + dither[1])>>shift; dst[j+1] = dbswap(tmp - (tmp>>dst_depth));\
+                tmp = (bswap(src[j+2]) + dither[2])>>shift; dst[j+2] = dbswap(tmp - (tmp>>dst_depth));\
+                tmp = (bswap(src[j+3]) + dither[3])>>shift; dst[j+3] = dbswap(tmp - (tmp>>dst_depth));\
+                tmp = (bswap(src[j+4]) + dither[4])>>shift; dst[j+4] = dbswap(tmp - (tmp>>dst_depth));\
+                tmp = (bswap(src[j+5]) + dither[5])>>shift; dst[j+5] = dbswap(tmp - (tmp>>dst_depth));\
+                tmp = (bswap(src[j+6]) + dither[6])>>shift; dst[j+6] = dbswap(tmp - (tmp>>dst_depth));\
+                tmp = (bswap(src[j+7]) + dither[7])>>shift; dst[j+7] = dbswap(tmp - (tmp>>dst_depth));\
+            }\
+            for (; j < length; j++) {\
+                tmp = (bswap(src[j]) + dither[j&7])>>shift; dst[j] = dbswap(tmp - (tmp>>dst_depth));\
+            }\
+            dst += dstStride;\
+            src += srcStride;\
+        }\
+    } else {\
+        for (i = 0; i < height; i++) {\
+            const uint8_t *dither= dithers[shift-1][i&7];\
+            for (j = 0; j < length-7; j+=8) {\
+                tmp = bswap(src[j+0]); dst[j+0] = dbswap((tmp - (tmp>>dst_depth) + dither[0])>>shift);\
+                tmp = bswap(src[j+1]); dst[j+1] = dbswap((tmp - (tmp>>dst_depth) + dither[1])>>shift);\
+                tmp = bswap(src[j+2]); dst[j+2] = dbswap((tmp - (tmp>>dst_depth) + dither[2])>>shift);\
+                tmp = bswap(src[j+3]); dst[j+3] = dbswap((tmp - (tmp>>dst_depth) + dither[3])>>shift);\
+                tmp = bswap(src[j+4]); dst[j+4] = dbswap((tmp - (tmp>>dst_depth) + dither[4])>>shift);\
+                tmp = bswap(src[j+5]); dst[j+5] = dbswap((tmp - (tmp>>dst_depth) + dither[5])>>shift);\
+                tmp = bswap(src[j+6]); dst[j+6] = dbswap((tmp - (tmp>>dst_depth) + dither[6])>>shift);\
+                tmp = bswap(src[j+7]); dst[j+7] = dbswap((tmp - (tmp>>dst_depth) + dither[7])>>shift);\
+            }\
+            for (; j < length; j++) {\
+                tmp = bswap(src[j]); dst[j] = dbswap((tmp - (tmp>>dst_depth) + dither[j&7])>>shift);\
+            }\
+            dst += dstStride;\
+            src += srcStride;\
         }\
-        for (; j < length; j++)\
-            dst[j] = dbswap((bswap(src[j]) + dither[j&7])*scale>>shift);\
-        dst += dstStride;\
-        src += srcStride;\
     }
 
 static int planarCopyWrapper(SwsContext *c, const uint8_t *src[],
diff --git a/tests/ref/vsynth/vsynth1-ffvhuff420p12 b/tests/ref/vsynth/vsynth1-ffvhuff420p12
index 0d80bd6d68..d4b22f3b4b 100644
--- a/tests/ref/vsynth/vsynth1-ffvhuff420p12
+++ b/tests/ref/vsynth/vsynth1-ffvhuff420p12
@@ -1,4 +1,4 @@
 866485c954242232878e40f0389790dd *tests/data/fate/vsynth1-ffvhuff420p12.avi
 14205356 tests/data/fate/vsynth1-ffvhuff420p12.avi
-b48f32c140712e8c7bf81cfdd66ae312 *tests/data/fate/vsynth1-ffvhuff420p12.out.rawvideo
-stddev:    0.68 PSNR: 51.47 MAXDIFF:    1 bytes:  7603200/  7603200
+c5ccac874dbf808e9088bc3107860042 *tests/data/fate/vsynth1-ffvhuff420p12.out.rawvideo
+stddev:    0.00 PSNR:999.99 MAXDIFF:    0 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth1-vc2-420p10 b/tests/ref/vsynth/vsynth1-vc2-420p10
index 037c77cacf..025a1cc779 100644
--- a/tests/ref/vsynth/vsynth1-vc2-420p10
+++ b/tests/ref/vsynth/vsynth1-vc2-420p10
@@ -1,4 +1,4 @@
 1365742985b6315f6796c765aa17f39e *tests/data/fate/vsynth1-vc2-420p10.mov
 1417047 tests/data/fate/vsynth1-vc2-420p10.mov
-d3deedfa461a2696f82910890412fa2d *tests/data/fate/vsynth1-vc2-420p10.out.rawvideo
-stddev:    0.60 PSNR: 52.47 MAXDIFF:    1 bytes:  7603200/   760320
+387696707c79cf1a6c9aeff4024226b9 *tests/data/fate/vsynth1-vc2-420p10.out.rawvideo
+stddev:    0.00 PSNR:999.99 MAXDIFF:    0 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth1-vc2-420p12 b/tests/ref/vsynth/vsynth1-vc2-420p12
index b0c56afcd5..719f0d5f37 100644
--- a/tests/ref/vsynth/vsynth1-vc2-420p12
+++ b/tests/ref/vsynth/vsynth1-vc2-420p12
@@ -1,4 +1,4 @@
 08a844d17940cd612da269fb08430628 *tests/data/fate/vsynth1-vc2-420p12.mov
 1746007 tests/data/fate/vsynth1-vc2-420p12.mov
-5a78509638a96b0fa17c1b7e9159fd24 *tests/data/fate/vsynth1-vc2-420p12.out.rawvideo
-stddev:    0.67 PSNR: 51.48 MAXDIFF:    1 bytes:  7603200/   760320
+387696707c79cf1a6c9aeff4024226b9 *tests/data/fate/vsynth1-vc2-420p12.out.rawvideo
+stddev:    0.00 PSNR:999.99 MAXDIFF:    0 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth2-ffvhuff420p12 b/tests/ref/vsynth/vsynth2-ffvhuff420p12
index 82c467ac3e..f97edfbf4e 100644
--- a/tests/ref/vsynth/vsynth2-ffvhuff420p12
+++ b/tests/ref/vsynth/vsynth2-ffvhuff420p12
@@ -1,4 +1,4 @@
 3ab9567895bf1ec31a82aadf16a5da0e *tests/data/fate/vsynth2-ffvhuff420p12.avi
 10562808 tests/data/fate/vsynth2-ffvhuff420p12.avi
-542327cb5ca7708085513ffc3d7c693c *tests/data/fate/vsynth2-ffvhuff420p12.out.rawvideo
-stddev:    0.72 PSNR: 50.87 MAXDIFF:    1 bytes:  7603200/  7603200
+36d7ca943916e1743cefa609eba0205c *tests/data/fate/vsynth2-ffvhuff420p12.out.rawvideo
+stddev:    0.00 PSNR:999.99 MAXDIFF:    0 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth2-vc2-420p10 b/tests/ref/vsynth/vsynth2-vc2-420p10
index 9de40d2275..48d97d60e4 100644
--- a/tests/ref/vsynth/vsynth2-vc2-420p10
+++ b/tests/ref/vsynth/vsynth2-vc2-420p10
@@ -1,4 +1,4 @@
 1197f8108683b9eb6b0777adb2db1aa8 *tests/data/fate/vsynth2-vc2-420p10.mov
 1181271 tests/data/fate/vsynth2-vc2-420p10.mov
-75174cb90e76c433f6d769531d573ac2 *tests/data/fate/vsynth2-vc2-420p10.out.rawvideo
-stddev:    0.63 PSNR: 52.11 MAXDIFF:    1 bytes:  7603200/   760320
+01389f7ae4f2a3dc0d7b8384d435fd83 *tests/data/fate/vsynth2-vc2-420p10.out.rawvideo
+stddev:    0.00 PSNR:999.99 MAXDIFF:    0 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth2-vc2-420p12 b/tests/ref/vsynth/vsynth2-vc2-420p12
index 660a1eeb5c..9b9ccb8ec1 100644
--- a/tests/ref/vsynth/vsynth2-vc2-420p12
+++ b/tests/ref/vsynth/vsynth2-vc2-420p12
@@ -1,4 +1,4 @@
 0e6b3aefd70fca45e67dc8cbc99640e8 *tests/data/fate/vsynth2-vc2-420p12.mov
 1525079 tests/data/fate/vsynth2-vc2-420p12.mov
-b4d45651e20faa7a0bb84a0738638c48 *tests/data/fate/vsynth2-vc2-420p12.out.rawvideo
-stddev:    0.73 PSNR: 50.84 MAXDIFF:    1 bytes:  7603200/   760320
+01389f7ae4f2a3dc0d7b8384d435fd83 *tests/data/fate/vsynth2-vc2-420p12.out.rawvideo
+stddev:    0.00 PSNR:999.99 MAXDIFF:    0 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth3-ffvhuff420p12 b/tests/ref/vsynth/vsynth3-ffvhuff420p12
index 72d412d644..201ec7658b 100644
--- a/tests/ref/vsynth/vsynth3-ffvhuff420p12
+++ b/tests/ref/vsynth/vsynth3-ffvhuff420p12
@@ -1,4 +1,4 @@
 e5a178d75afeda6df1d4eb6f7cdfa3a0 *tests/data/fate/vsynth3-ffvhuff420p12.avi
 175260 tests/data/fate/vsynth3-ffvhuff420p12.avi
-ee95a44ccd612b5057860b43fe9775d6 *tests/data/fate/vsynth3-ffvhuff420p12.out.rawvideo
-stddev:    0.69 PSNR: 51.35 MAXDIFF:    1 bytes:    86700/    86700
+a038ad7c3c09f776304ef7accdea9c74 *tests/data/fate/vsynth3-ffvhuff420p12.out.rawvideo
+stddev:    0.00 PSNR:999.99 MAXDIFF:    0 bytes:    86700/    86700
diff --git a/tests/ref/vsynth/vsynth_lena-ffvhuff420p12 b/tests/ref/vsynth/vsynth_lena-ffvhuff420p12
index e8ea4bcc85..e77698ba0c 100644
--- a/tests/ref/vsynth/vsynth_lena-ffvhuff420p12
+++ b/tests/ref/vsynth/vsynth_lena-ffvhuff420p12
@@ -1,4 +1,4 @@
 b2f3d04ca30c113b79877bb5518dd6ea *tests/data/fate/vsynth_lena-ffvhuff420p12.avi
 10925580 tests/data/fate/vsynth_lena-ffvhuff420p12.avi
-08b3c6c70eba608bae926608ff253f2a *tests/data/fate/vsynth_lena-ffvhuff420p12.out.rawvideo
-stddev:    0.68 PSNR: 51.38 MAXDIFF:    1 bytes:  7603200/  7603200
+dde5895817ad9d219f79a52d0bdfb001 *tests/data/fate/vsynth_lena-ffvhuff420p12.out.rawvideo
+stddev:    0.00 PSNR:999.99 MAXDIFF:    0 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth_lena-vc2-420p10 b/tests/ref/vsynth/vsynth_lena-vc2-420p10
index 6428dd1788..2558ef8e9b 100644
--- a/tests/ref/vsynth/vsynth_lena-vc2-420p10
+++ b/tests/ref/vsynth/vsynth_lena-vc2-420p10
@@ -1,4 +1,4 @@
 5bccec653c330f03b90065a84fad9b4b *tests/data/fate/vsynth_lena-vc2-420p10.mov
 1154775 tests/data/fate/vsynth_lena-vc2-420p10.mov
-32265ec286c54104b3be8f11c519da1b *tests/data/fate/vsynth_lena-vc2-420p10.out.rawvideo
-stddev:    0.61 PSNR: 52.34 MAXDIFF:    1 bytes:  7603200/   760320
+b1c660113acab8eb4075f3d9fbb9cee9 *tests/data/fate/vsynth_lena-vc2-420p10.out.rawvideo
+stddev:    0.00 PSNR:999.99 MAXDIFF:    0 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth_lena-vc2-420p12 b/tests/ref/vsynth/vsynth_lena-vc2-420p12
index e62b2b6d38..bc60116ffe 100644
--- a/tests/ref/vsynth/vsynth_lena-vc2-420p12
+++ b/tests/ref/vsynth/vsynth_lena-vc2-420p12
@@ -1,4 +1,4 @@
 d27a6d3517cc9a6d22e338f4b206545c *tests/data/fate/vsynth_lena-vc2-420p12.mov
 1516759 tests/data/fate/vsynth_lena-vc2-420p12.mov
-bc62f1c3bade7224c55219dba8a3c6af *tests/data/fate/vsynth_lena-vc2-420p12.out.rawvideo
-stddev:    0.68 PSNR: 51.39 MAXDIFF:    1 bytes:  7603200/   760320
+b1c660113acab8eb4075f3d9fbb9cee9 *tests/data/fate/vsynth_lena-vc2-420p12.out.rawvideo
+stddev:    0.00 PSNR:999.99 MAXDIFF:    0 bytes:  7603200/   760320