avcodec/jpeg2000: Move gainb handling into the quantization code

thats how the specification defines it, this also improves numerical
accuracy of the integer wavelet implementation. It otherwise should
be equivalent, in case of overflows this can be reverted.

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

6 files changed, 38 insertions, 38 deletions

diff --git a/tests/ref/fate/j2k-dwt b/tests/ref/fate/j2k-dwt
index 74b4d04458..aba9458dfd 100644
--- a/tests/ref/fate/j2k-dwt
+++ b/tests/ref/fate/j2k-dwt
@@ -1,60 +1,60 @@
 5/3i, decomp:15 border 151 170 140 183 milli-err2:        0
-9/7i, decomp:15 border 151 170 140 183 milli-err2:    13160
+9/7i, decomp:15 border 151 170 140 183 milli-err2:     5407
 9/7f, decomp:15 border 151 170 140 183 err2:              0.0001
 5/3i, decomp:21 border 173 201  81 189 milli-err2:        0
-9/7i, decomp:21 border 173 201  81 189 milli-err2:    24635
+9/7i, decomp:21 border 173 201  81 189 milli-err2:     8289
 9/7f, decomp:21 border 173 201  81 189 err2:              0.0001
 5/3i, decomp:22 border 213 227  76 245 milli-err2:        0
-9/7i, decomp:22 border 213 227  76 245 milli-err2:    15253
+9/7i, decomp:22 border 213 227  76 245 milli-err2:     7210
 9/7f, decomp:22 border 213 227  76 245 err2:              0.0001
 5/3i, decomp:13 border 134 157 184 203 milli-err2:        0
-9/7i, decomp:13 border 134 157 184 203 milli-err2:    21846
+9/7i, decomp:13 border 134 157 184 203 milli-err2:    17748
 9/7f, decomp:13 border 134 157 184 203 err2:              0.0001
 5/3i, decomp: 1 border 204 237   6 106 milli-err2:        0
-9/7i, decomp: 1 border 204 237   6 106 milli-err2:     2317
+9/7i, decomp: 1 border 204 237   6 106 milli-err2:      824
 9/7f, decomp: 1 border 204 237   6 106 err2:              0.0000
 5/3i, decomp:28 border  76 211  13 210 milli-err2:        0
-9/7i, decomp:28 border  76 211  13 210 milli-err2:    21360
+9/7i, decomp:28 border  76 211  13 210 milli-err2:    11710
 9/7f, decomp:28 border  76 211  13 210 err2:              0.0002
 5/3i, decomp:21 border  76  99  43 123 milli-err2:        0
-9/7i, decomp:21 border  76  99  43 123 milli-err2:    13513
+9/7i, decomp:21 border  76  99  43 123 milli-err2:     8555
 9/7f, decomp:21 border  76  99  43 123 err2:              0.0001
 5/3i, decomp:15 border 192 243 174 204 milli-err2:        0
-9/7i, decomp:15 border 192 243 174 204 milli-err2:    13302
+9/7i, decomp:15 border 192 243 174 204 milli-err2:     7769
 9/7f, decomp:15 border 192 243 174 204 err2:              0.0001
 5/3i, decomp:21 border  17  68  93 204 milli-err2:        0
-9/7i, decomp:21 border  17  68  93 204 milli-err2:    22975
+9/7i, decomp:21 border  17  68  93 204 milli-err2:     7727
 9/7f, decomp:21 border  17  68  93 204 err2:              0.0001
 5/3i, decomp:11 border 142 168  82 174 milli-err2:        0
-9/7i, decomp:11 border 142 168  82 174 milli-err2:    11936
+9/7i, decomp:11 border 142 168  82 174 milli-err2:    12277
 9/7f, decomp:11 border 142 168  82 174 err2:              0.0001
 5/3i, decomp:23 border 142 209 171 235 milli-err2:        0
-9/7i, decomp:23 border 142 209 171 235 milli-err2:    19428
+9/7i, decomp:23 border 142 209 171 235 milli-err2:     6570
 9/7f, decomp:23 border 142 209 171 235 err2:              0.0001
 5/3i, decomp:30 border  37 185  79 245 milli-err2:        0
-9/7i, decomp:30 border  37 185  79 245 milli-err2:    22981
+9/7i, decomp:30 border  37 185  79 245 milli-err2:    12266
 9/7f, decomp:30 border  37 185  79 245 err2:              0.0002
 5/3i, decomp: 5 border 129 236  30 243 milli-err2:        0
-9/7i, decomp: 5 border 129 236  30 243 milli-err2:    16632
+9/7i, decomp: 5 border 129 236  30 243 milli-err2:     8686
 9/7f, decomp: 5 border 129 236  30 243 err2:              0.0001
 5/3i, decomp:10 border   5 160 146 247 milli-err2:        0
-9/7i, decomp:10 border   5 160 146 247 milli-err2:    39669
+9/7i, decomp:10 border   5 160 146 247 milli-err2:    12231
 9/7f, decomp:10 border   5 160 146 247 err2:              0.0002
 5/3i, decomp: 5 border 104 162   6  47 milli-err2:        0
-9/7i, decomp: 5 border 104 162   6  47 milli-err2:    18290
+9/7i, decomp: 5 border 104 162   6  47 milli-err2:     7794
 9/7f, decomp: 5 border 104 162   6  47 err2:              0.0001
 5/3i, decomp:24 border  78 250 102 218 milli-err2:        0
-9/7i, decomp:24 border  78 250 102 218 milli-err2:    36623
+9/7i, decomp:24 border  78 250 102 218 milli-err2:    11632
 9/7f, decomp:24 border  78 250 102 218 err2:              0.0002
 5/3i, decomp:28 border  86  98  56  79 milli-err2:        0
-9/7i, decomp:28 border  86  98  56  79 milli-err2:     9891
+9/7i, decomp:28 border  86  98  56  79 milli-err2:    11105
 9/7f, decomp:28 border  86  98  56  79 err2:              0.0001
 5/3i, decomp: 6 border  95 238 197 214 milli-err2:        0
-9/7i, decomp: 6 border  95 238 197 214 milli-err2:    17172
+9/7i, decomp: 6 border  95 238 197 214 milli-err2:     8308
 9/7f, decomp: 6 border  95 238 197 214 err2:              0.0001
 5/3i, decomp:17 border  77 169  93 165 milli-err2:        0
-9/7i, decomp:17 border  77 169  93 165 milli-err2:    23573
+9/7i, decomp:17 border  77 169  93 165 milli-err2:     8820
 9/7f, decomp:17 border  77 169  93 165 err2:              0.0001
 5/3i, decomp:22 border 178 187   7 119 milli-err2:        0
-9/7i, decomp:22 border 178 187   7 119 milli-err2:    11305
+9/7i, decomp:22 border 178 187   7 119 milli-err2:     5554
 9/7f, decomp:22 border 178 187   7 119 err2:              0.0000
diff --git a/tests/ref/fate/jpeg2000-dcinema b/tests/ref/fate/jpeg2000-dcinema
index 343f438b86..00a0cae1ae 100644
--- a/tests/ref/fate/jpeg2000-dcinema
+++ b/tests/ref/fate/jpeg2000-dcinema
@@ -1,3 +1,3 @@
 #tb 0: 1/24
-0,          0,          0,        1, 12441600, 0x35b017fd
-0,          1,          1,        1, 12441600, 0xe94551b9
+0,          0,          0,        1, 12441600, 0x330aac66
+0,          1,          1,        1, 12441600, 0x112db395
diff --git a/tests/ref/vsynth/vsynth1-jpeg2000-97 b/tests/ref/vsynth/vsynth1-jpeg2000-97
index 15ca696a07..20ff0733cf 100644
--- a/tests/ref/vsynth/vsynth1-jpeg2000-97
+++ b/tests/ref/vsynth/vsynth1-jpeg2000-97
@@ -1,4 +1,4 @@
-4cac63aaf880ab0ee199ac44e520640a *tests/data/fate/vsynth1-jpeg2000-97.avi
-2188956 tests/data/fate/vsynth1-jpeg2000-97.avi
-d9227e7a7413cb48362654d879df985a *tests/data/fate/vsynth1-jpeg2000-97.out.rawvideo
-stddev:    6.18 PSNR: 32.30 MAXDIFF:   57 bytes:  7603200/  7603200
+0c704936bf54fad2b072dc0c8c289bb7 *tests/data/fate/vsynth1-jpeg2000-97.avi
+4336566 tests/data/fate/vsynth1-jpeg2000-97.avi
+1207ed65ca04d2c91936c4bc73c3a562 *tests/data/fate/vsynth1-jpeg2000-97.out.rawvideo
+stddev:    4.36 PSNR: 35.34 MAXDIFF:   51 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth2-jpeg2000-97 b/tests/ref/vsynth/vsynth2-jpeg2000-97
index f6f4505bc5..caeb9adaab 100644
--- a/tests/ref/vsynth/vsynth2-jpeg2000-97
+++ b/tests/ref/vsynth/vsynth2-jpeg2000-97
@@ -1,4 +1,4 @@
-352d3dc81b5c78638a2e7b575ce468c2 *tests/data/fate/vsynth2-jpeg2000-97.avi
-1448342 tests/data/fate/vsynth2-jpeg2000-97.avi
-d7b18b6dcac0139858e64e00073db1c6 *tests/data/fate/vsynth2-jpeg2000-97.out.rawvideo
-stddev:    5.35 PSNR: 33.56 MAXDIFF:   56 bytes:  7603200/  7603200
+7bb7ccf4e9743888a5bda48f431630cf *tests/data/fate/vsynth2-jpeg2000-97.avi
+3124708 tests/data/fate/vsynth2-jpeg2000-97.avi
+fe0f3a97aa8c0b2f02dd11ccd1b77ca6 *tests/data/fate/vsynth2-jpeg2000-97.out.rawvideo
+stddev:    3.26 PSNR: 37.86 MAXDIFF:   26 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth3-jpeg2000-97 b/tests/ref/vsynth/vsynth3-jpeg2000-97
index 236519863f..eff6945989 100644
--- a/tests/ref/vsynth/vsynth3-jpeg2000-97
+++ b/tests/ref/vsynth/vsynth3-jpeg2000-97
@@ -1,4 +1,4 @@
-36dddf487c4919d890db3484ab5d6486 *tests/data/fate/vsynth3-jpeg2000-97.avi
-66522 tests/data/fate/vsynth3-jpeg2000-97.avi
-332c0924bc1477a41eb413455c5b9d90 *tests/data/fate/vsynth3-jpeg2000-97.out.rawvideo
-stddev:    6.17 PSNR: 32.32 MAXDIFF:   51 bytes:    86700/    86700
+d551b1b1bfc691356e6f45f4386aff1d *tests/data/fate/vsynth3-jpeg2000-97.avi
+92144 tests/data/fate/vsynth3-jpeg2000-97.avi
+533535dad2ccc7aef3eb0e7ccb239bf2 *tests/data/fate/vsynth3-jpeg2000-97.out.rawvideo
+stddev:    4.60 PSNR: 34.87 MAXDIFF:   46 bytes:    86700/    86700
diff --git a/tests/ref/vsynth/vsynth_lena-jpeg2000-97 b/tests/ref/vsynth/vsynth_lena-jpeg2000-97
index 21160ef061..426787c0cd 100644
--- a/tests/ref/vsynth/vsynth_lena-jpeg2000-97
+++ b/tests/ref/vsynth/vsynth_lena-jpeg2000-97
@@ -1,4 +1,4 @@
-a39e052e7d109e33714329800c9df90f *tests/data/fate/vsynth_lena-jpeg2000-97.avi
-1105046 tests/data/fate/vsynth_lena-jpeg2000-97.avi
-c151476d2ed807575122064ada37d5ef *tests/data/fate/vsynth_lena-jpeg2000-97.out.rawvideo
-stddev:    4.85 PSNR: 34.41 MAXDIFF:   59 bytes:  7603200/  7603200
+6600b19f6c2e84c599f37d33a295ee35 *tests/data/fate/vsynth_lena-jpeg2000-97.avi
+2459730 tests/data/fate/vsynth_lena-jpeg2000-97.avi
+9d9c07b0d1936810032820a08d0b0b5d *tests/data/fate/vsynth_lena-jpeg2000-97.out.rawvideo
+stddev:    3.01 PSNR: 38.55 MAXDIFF:   27 bytes:  7603200/  7603200