avcodec/jpeg2000: Improve FF_DWT97_INT to pass ISO/IEC 15444-4 conformance tests

Fixes https://trac.ffmpeg.org/ticket/10123

Signed-off-by: Pierre-Anthony Lemieux <pal@palemieux.com>

9 files changed, 41 insertions, 41 deletions

diff --git a/tests/ref/fate/j2k-dwt b/tests/ref/fate/j2k-dwt
index 42415f00f9..9fbaaefd02 100644
--- a/tests/ref/fate/j2k-dwt
+++ b/tests/ref/fate/j2k-dwt
@@ -1,60 +1,60 @@
 5/3i, decomp:15 border 151 170 140 183 milli-err2:        0
-9/7i, decomp:15 border 151 170 140 183 milli-err2:      544
+9/7i, decomp:15 border 151 170 140 183 milli-err2:      112
 9/7f, decomp:15 border 151 170 140 183 err2:               0.000
 5/3i, decomp:21 border 173 201  81 189 milli-err2:        0
-9/7i, decomp:21 border 173 201  81 189 milli-err2:      592
+9/7i, decomp:21 border 173 201  81 189 milli-err2:      109
 9/7f, decomp:21 border 173 201  81 189 err2:               0.000
 5/3i, decomp:22 border 213 227  76 245 milli-err2:        0
-9/7i, decomp:22 border 213 227  76 245 milli-err2:      533
+9/7i, decomp:22 border 213 227  76 245 milli-err2:      115
 9/7f, decomp:22 border 213 227  76 245 err2:               0.000
 5/3i, decomp:13 border 134 157 184 203 milli-err2:        0
-9/7i, decomp:13 border 134 157 184 203 milli-err2:      535
+9/7i, decomp:13 border 134 157 184 203 milli-err2:      109
 9/7f, decomp:13 border 134 157 184 203 err2:               0.000
 5/3i, decomp: 1 border 204 237   6 106 milli-err2:        0
-9/7i, decomp: 1 border 204 237   6 106 milli-err2:      219
+9/7i, decomp: 1 border 204 237   6 106 milli-err2:       95
 9/7f, decomp: 1 border 204 237   6 106 err2:               0.000
 5/3i, decomp:28 border  76 211  13 210 milli-err2:        0
-9/7i, decomp:28 border  76 211  13 210 milli-err2:      791
+9/7i, decomp:28 border  76 211  13 210 milli-err2:      117
 9/7f, decomp:28 border  76 211  13 210 err2:               0.000
 5/3i, decomp:21 border  76  99  43 123 milli-err2:        0
-9/7i, decomp:21 border  76  99  43 123 milli-err2:      686
+9/7i, decomp:21 border  76  99  43 123 milli-err2:      104
 9/7f, decomp:21 border  76  99  43 123 err2:               0.000
 5/3i, decomp:15 border 192 243 174 204 milli-err2:        0
-9/7i, decomp:15 border 192 243 174 204 milli-err2:      476
+9/7i, decomp:15 border 192 243 174 204 milli-err2:      122
 9/7f, decomp:15 border 192 243 174 204 err2:               0.000
 5/3i, decomp:21 border  17  68  93 204 milli-err2:        0
-9/7i, decomp:21 border  17  68  93 204 milli-err2:      633
+9/7i, decomp:21 border  17  68  93 204 milli-err2:      124
 9/7f, decomp:21 border  17  68  93 204 err2:               0.000
 5/3i, decomp:11 border 142 168  82 174 milli-err2:        0
-9/7i, decomp:11 border 142 168  82 174 milli-err2:      696
+9/7i, decomp:11 border 142 168  82 174 milli-err2:      115
 9/7f, decomp:11 border 142 168  82 174 err2:               0.000
 5/3i, decomp:23 border 142 209 171 235 milli-err2:        0
-9/7i, decomp:23 border 142 209 171 235 milli-err2:      626
+9/7i, decomp:23 border 142 209 171 235 milli-err2:      120
 9/7f, decomp:23 border 142 209 171 235 err2:               0.000
 5/3i, decomp:30 border  37 185  79 245 milli-err2:        0
-9/7i, decomp:30 border  37 185  79 245 milli-err2:      953
+9/7i, decomp:30 border  37 185  79 245 milli-err2:      116
 9/7f, decomp:30 border  37 185  79 245 err2:               0.000
 5/3i, decomp: 5 border 129 236  30 243 milli-err2:        0
-9/7i, decomp: 5 border 129 236  30 243 milli-err2:      620
+9/7i, decomp: 5 border 129 236  30 243 milli-err2:      117
 9/7f, decomp: 5 border 129 236  30 243 err2:               0.000
 5/3i, decomp:10 border   5 160 146 247 milli-err2:        0
-9/7i, decomp:10 border   5 160 146 247 milli-err2:      797
+9/7i, decomp:10 border   5 160 146 247 milli-err2:      117
 9/7f, decomp:10 border   5 160 146 247 err2:               0.000
 5/3i, decomp: 5 border 104 162   6  47 milli-err2:        0
-9/7i, decomp: 5 border 104 162   6  47 milli-err2:      603
+9/7i, decomp: 5 border 104 162   6  47 milli-err2:      120
 9/7f, decomp: 5 border 104 162   6  47 err2:               0.000
 5/3i, decomp:24 border  78 250 102 218 milli-err2:        0
-9/7i, decomp:24 border  78 250 102 218 milli-err2:      836
+9/7i, decomp:24 border  78 250 102 218 milli-err2:      113
 9/7f, decomp:24 border  78 250 102 218 err2:               0.000
 5/3i, decomp:28 border  86  98  56  79 milli-err2:        0
-9/7i, decomp:28 border  86  98  56  79 milli-err2:      597
+9/7i, decomp:28 border  86  98  56  79 milli-err2:      115
 9/7f, decomp:28 border  86  98  56  79 err2:               0.000
 5/3i, decomp: 6 border  95 238 197 214 milli-err2:        0
-9/7i, decomp: 6 border  95 238 197 214 milli-err2:      478
+9/7i, decomp: 6 border  95 238 197 214 milli-err2:      114
 9/7f, decomp: 6 border  95 238 197 214 err2:               0.000
 5/3i, decomp:17 border  77 169  93 165 milli-err2:        0
-9/7i, decomp:17 border  77 169  93 165 milli-err2:      616
+9/7i, decomp:17 border  77 169  93 165 milli-err2:      124
 9/7f, decomp:17 border  77 169  93 165 err2:               0.000
 5/3i, decomp:22 border 178 187   7 119 milli-err2:        0
-9/7i, decomp:22 border 178 187   7 119 milli-err2:      392
+9/7i, decomp:22 border 178 187   7 119 milli-err2:       96
 9/7f, decomp:22 border 178 187   7 119 err2:               0.000
diff --git a/tests/ref/fate/jpeg2000-dcinema b/tests/ref/fate/jpeg2000-dcinema
index cdf8cd4fc6..223d753580 100644
--- a/tests/ref/fate/jpeg2000-dcinema
+++ b/tests/ref/fate/jpeg2000-dcinema
@@ -3,5 +3,5 @@
 #codec_id 0: rawvideo
 #dimensions 0: 1920x1080
 #sar 0: 1/1
-0,          0,          0,        1, 12441600, 0xfcf6a127
-0,          1,          1,        1, 12441600, 0x577b6a64
+0,          0,          0,        1, 12441600, 0x0cf44be1
+0,          1,          1,        1, 12441600, 0xefe54482
diff --git a/tests/ref/fate/jpeg2000dec-p0_04 b/tests/ref/fate/jpeg2000dec-p0_04
index 5de7880c44..4c3ec322e4 100644
--- a/tests/ref/fate/jpeg2000dec-p0_04
+++ b/tests/ref/fate/jpeg2000dec-p0_04
@@ -3,4 +3,4 @@
 #codec_id 0: rawvideo
 #dimensions 0: 640x480
 #sar 0: 0/1
-0,          0,          0,        1,   921600, 0x097d9665
+0,          0,          0,        1,   921600, 0x38311bba
diff --git a/tests/ref/fate/jpeg2000dec-p0_05 b/tests/ref/fate/jpeg2000dec-p0_05
index bb215043a1..410a08a62c 100644
--- a/tests/ref/fate/jpeg2000dec-p0_05
+++ b/tests/ref/fate/jpeg2000dec-p0_05
@@ -3,4 +3,4 @@
 #codec_id 0: rawvideo
 #dimensions 0: 1024x1024
 #sar 0: 0/1
-0,          0,          0,        1,  2621440, 0x081f5048
+0,          0,          0,        1,  2621440, 0x9608ad8b
diff --git a/tests/ref/fate/jpeg2000dec-p0_09 b/tests/ref/fate/jpeg2000dec-p0_09
index 1755e7cc7d..ff78bf9dc7 100644
--- a/tests/ref/fate/jpeg2000dec-p0_09
+++ b/tests/ref/fate/jpeg2000dec-p0_09
@@ -3,4 +3,4 @@
 #codec_id 0: rawvideo
 #dimensions 0: 17x37
 #sar 0: 0/1
-0,          0,          0,        1,      629, 0x5c9c389d
+0,          0,          0,        1,      629, 0xf35d38d6
diff --git a/tests/ref/vsynth/vsynth1-jpeg2000-97 b/tests/ref/vsynth/vsynth1-jpeg2000-97
index c979ab5c36..820249c6c0 100644
--- a/tests/ref/vsynth/vsynth1-jpeg2000-97
+++ b/tests/ref/vsynth/vsynth1-jpeg2000-97
@@ -1,4 +1,4 @@
-5e6d32b7205d31245b0d1f015d08b515 *tests/data/fate/vsynth1-jpeg2000-97.avi
-3643886 tests/data/fate/vsynth1-jpeg2000-97.avi
-a2262f1da2f49bc196b780a6b47ec4e8 *tests/data/fate/vsynth1-jpeg2000-97.out.rawvideo
-stddev:    4.23 PSNR: 35.59 MAXDIFF:   53 bytes:  7603200/  7603200
+803c2e8a4d054c5d603eed4c77abe492 *tests/data/fate/vsynth1-jpeg2000-97.avi
+4466514 tests/data/fate/vsynth1-jpeg2000-97.avi
+c9cf5a4580f10b00056c8d8731d21395 *tests/data/fate/vsynth1-jpeg2000-97.out.rawvideo
+stddev:    3.82 PSNR: 36.49 MAXDIFF:   49 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth2-jpeg2000-97 b/tests/ref/vsynth/vsynth2-jpeg2000-97
index 591f8b6bb3..e40dca5b9a 100644
--- a/tests/ref/vsynth/vsynth2-jpeg2000-97
+++ b/tests/ref/vsynth/vsynth2-jpeg2000-97
@@ -1,4 +1,4 @@
-aa5573136c54b1855d8d00efe2a149bd *tests/data/fate/vsynth2-jpeg2000-97.avi
-2464134 tests/data/fate/vsynth2-jpeg2000-97.avi
-1f63c8b065e847e4c63d57ce23442ea8 *tests/data/fate/vsynth2-jpeg2000-97.out.rawvideo
-stddev:    3.21 PSNR: 37.99 MAXDIFF:   26 bytes:  7603200/  7603200
+c189c8b89c7aee3ab4f4a5aafdf7568f *tests/data/fate/vsynth2-jpeg2000-97.avi
+3225460 tests/data/fate/vsynth2-jpeg2000-97.avi
+4c0fbd7af969085d19dfabeb9634cddb *tests/data/fate/vsynth2-jpeg2000-97.out.rawvideo
+stddev:    2.55 PSNR: 39.98 MAXDIFF:   22 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth3-jpeg2000-97 b/tests/ref/vsynth/vsynth3-jpeg2000-97
index 5d9d083791..92376e64cf 100644
--- a/tests/ref/vsynth/vsynth3-jpeg2000-97
+++ b/tests/ref/vsynth/vsynth3-jpeg2000-97
@@ -1,4 +1,4 @@
-522e12684aca4262a9d613cb2db7006c *tests/data/fate/vsynth3-jpeg2000-97.avi
-85526 tests/data/fate/vsynth3-jpeg2000-97.avi
-8def36ad1413ab3a5c2af2e1af4603f9 *tests/data/fate/vsynth3-jpeg2000-97.out.rawvideo
-stddev:    4.51 PSNR: 35.04 MAXDIFF:   47 bytes:    86700/    86700
+943cbdefa18b4a83175943f4e81e037c *tests/data/fate/vsynth3-jpeg2000-97.avi
+95642 tests/data/fate/vsynth3-jpeg2000-97.avi
+c4d58f0da2e8be602f54f032b58a581b *tests/data/fate/vsynth3-jpeg2000-97.out.rawvideo
+stddev:    4.11 PSNR: 35.84 MAXDIFF:   46 bytes:    86700/    86700
diff --git a/tests/ref/vsynth/vsynth_lena-jpeg2000-97 b/tests/ref/vsynth/vsynth_lena-jpeg2000-97
index 0539300185..dd2819b8cb 100644
--- a/tests/ref/vsynth/vsynth_lena-jpeg2000-97
+++ b/tests/ref/vsynth/vsynth_lena-jpeg2000-97
@@ -1,4 +1,4 @@
-80fe872c8afaad914da6ef037957d93b *tests/data/fate/vsynth_lena-jpeg2000-97.avi
-1937216 tests/data/fate/vsynth_lena-jpeg2000-97.avi
-1b97333a8dc115a5ba609b0070d89d4d *tests/data/fate/vsynth_lena-jpeg2000-97.out.rawvideo
-stddev:    2.82 PSNR: 39.10 MAXDIFF:   24 bytes:  7603200/  7603200
+9e2f5705be9d08494530724b625e17a4 *tests/data/fate/vsynth_lena-jpeg2000-97.avi
+2599714 tests/data/fate/vsynth_lena-jpeg2000-97.avi
+ab207505ec9c8a16bb45621404199e5c *tests/data/fate/vsynth_lena-jpeg2000-97.out.rawvideo
+stddev:    2.23 PSNR: 41.16 MAXDIFF:   20 bytes:  7603200/  7603200