avcodec/jpeg2000: Move H band scaling from wavelet into quantization code

This reduces the number of operations done and is equivalent except for
rounding

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

6 files changed, 38 insertions, 38 deletions

diff --git a/tests/ref/fate/j2k-dwt b/tests/ref/fate/j2k-dwt
index aba9458dfd..5ede1ce2f5 100644
--- a/tests/ref/fate/j2k-dwt
+++ b/tests/ref/fate/j2k-dwt
@@ -1,60 +1,60 @@
 5/3i, decomp:15 border 151 170 140 183 milli-err2:        0
-9/7i, decomp:15 border 151 170 140 183 milli-err2:     5407
+9/7i, decomp:15 border 151 170 140 183 milli-err2:     5188
 9/7f, decomp:15 border 151 170 140 183 err2:              0.0001
 5/3i, decomp:21 border 173 201  81 189 milli-err2:        0
-9/7i, decomp:21 border 173 201  81 189 milli-err2:     8289
+9/7i, decomp:21 border 173 201  81 189 milli-err2:     6478
 9/7f, decomp:21 border 173 201  81 189 err2:              0.0001
 5/3i, decomp:22 border 213 227  76 245 milli-err2:        0
-9/7i, decomp:22 border 213 227  76 245 milli-err2:     7210
+9/7i, decomp:22 border 213 227  76 245 milli-err2:     6539
 9/7f, decomp:22 border 213 227  76 245 err2:              0.0001
 5/3i, decomp:13 border 134 157 184 203 milli-err2:        0
-9/7i, decomp:13 border 134 157 184 203 milli-err2:    17748
+9/7i, decomp:13 border 134 157 184 203 milli-err2:    19203
 9/7f, decomp:13 border 134 157 184 203 err2:              0.0001
 5/3i, decomp: 1 border 204 237   6 106 milli-err2:        0
-9/7i, decomp: 1 border 204 237   6 106 milli-err2:      824
+9/7i, decomp: 1 border 204 237   6 106 milli-err2:      924
 9/7f, decomp: 1 border 204 237   6 106 err2:              0.0000
 5/3i, decomp:28 border  76 211  13 210 milli-err2:        0
-9/7i, decomp:28 border  76 211  13 210 milli-err2:    11710
+9/7i, decomp:28 border  76 211  13 210 milli-err2:    17297
 9/7f, decomp:28 border  76 211  13 210 err2:              0.0002
 5/3i, decomp:21 border  76  99  43 123 milli-err2:        0
-9/7i, decomp:21 border  76  99  43 123 milli-err2:     8555
+9/7i, decomp:21 border  76  99  43 123 milli-err2:     9039
 9/7f, decomp:21 border  76  99  43 123 err2:              0.0001
 5/3i, decomp:15 border 192 243 174 204 milli-err2:        0
-9/7i, decomp:15 border 192 243 174 204 milli-err2:     7769
+9/7i, decomp:15 border 192 243 174 204 milli-err2:     7693
 9/7f, decomp:15 border 192 243 174 204 err2:              0.0001
 5/3i, decomp:21 border  17  68  93 204 milli-err2:        0
-9/7i, decomp:21 border  17  68  93 204 milli-err2:     7727
+9/7i, decomp:21 border  17  68  93 204 milli-err2:     7810
 9/7f, decomp:21 border  17  68  93 204 err2:              0.0001
 5/3i, decomp:11 border 142 168  82 174 milli-err2:        0
-9/7i, decomp:11 border 142 168  82 174 milli-err2:    12277
+9/7i, decomp:11 border 142 168  82 174 milli-err2:    18168
 9/7f, decomp:11 border 142 168  82 174 err2:              0.0001
 5/3i, decomp:23 border 142 209 171 235 milli-err2:        0
-9/7i, decomp:23 border 142 209 171 235 milli-err2:     6570
+9/7i, decomp:23 border 142 209 171 235 milli-err2:     7313
 9/7f, decomp:23 border 142 209 171 235 err2:              0.0001
 5/3i, decomp:30 border  37 185  79 245 milli-err2:        0
-9/7i, decomp:30 border  37 185  79 245 milli-err2:    12266
+9/7i, decomp:30 border  37 185  79 245 milli-err2:    13498
 9/7f, decomp:30 border  37 185  79 245 err2:              0.0002
 5/3i, decomp: 5 border 129 236  30 243 milli-err2:        0
-9/7i, decomp: 5 border 129 236  30 243 milli-err2:     8686
+9/7i, decomp: 5 border 129 236  30 243 milli-err2:     8775
 9/7f, decomp: 5 border 129 236  30 243 err2:              0.0001
 5/3i, decomp:10 border   5 160 146 247 milli-err2:        0
-9/7i, decomp:10 border   5 160 146 247 milli-err2:    12231
+9/7i, decomp:10 border   5 160 146 247 milli-err2:    13478
 9/7f, decomp:10 border   5 160 146 247 err2:              0.0002
 5/3i, decomp: 5 border 104 162   6  47 milli-err2:        0
-9/7i, decomp: 5 border 104 162   6  47 milli-err2:     7794
+9/7i, decomp: 5 border 104 162   6  47 milli-err2:     7808
 9/7f, decomp: 5 border 104 162   6  47 err2:              0.0001
 5/3i, decomp:24 border  78 250 102 218 milli-err2:        0
-9/7i, decomp:24 border  78 250 102 218 milli-err2:    11632
+9/7i, decomp:24 border  78 250 102 218 milli-err2:    12570
 9/7f, decomp:24 border  78 250 102 218 err2:              0.0002
 5/3i, decomp:28 border  86  98  56  79 milli-err2:        0
-9/7i, decomp:28 border  86  98  56  79 milli-err2:    11105
+9/7i, decomp:28 border  86  98  56  79 milli-err2:     4148
 9/7f, decomp:28 border  86  98  56  79 err2:              0.0001
 5/3i, decomp: 6 border  95 238 197 214 milli-err2:        0
-9/7i, decomp: 6 border  95 238 197 214 milli-err2:     8308
+9/7i, decomp: 6 border  95 238 197 214 milli-err2:     7686
 9/7f, decomp: 6 border  95 238 197 214 err2:              0.0001
 5/3i, decomp:17 border  77 169  93 165 milli-err2:        0
-9/7i, decomp:17 border  77 169  93 165 milli-err2:     8820
+9/7i, decomp:17 border  77 169  93 165 milli-err2:    12026
 9/7f, decomp:17 border  77 169  93 165 err2:              0.0001
 5/3i, decomp:22 border 178 187   7 119 milli-err2:        0
-9/7i, decomp:22 border 178 187   7 119 milli-err2:     5554
+9/7i, decomp:22 border 178 187   7 119 milli-err2:     4971
 9/7f, decomp:22 border 178 187   7 119 err2:              0.0000
diff --git a/tests/ref/fate/jpeg2000-dcinema b/tests/ref/fate/jpeg2000-dcinema
index 00a0cae1ae..8040cb1d46 100644
--- a/tests/ref/fate/jpeg2000-dcinema
+++ b/tests/ref/fate/jpeg2000-dcinema
@@ -1,3 +1,3 @@
 #tb 0: 1/24
-0,          0,          0,        1, 12441600, 0x330aac66
-0,          1,          1,        1, 12441600, 0x112db395
+0,          0,          0,        1, 12441600, 0xbf142791
+0,          1,          1,        1, 12441600, 0x6b7a2ab5
diff --git a/tests/ref/vsynth/vsynth1-jpeg2000-97 b/tests/ref/vsynth/vsynth1-jpeg2000-97
index 20ff0733cf..b0fc029bfa 100644
--- a/tests/ref/vsynth/vsynth1-jpeg2000-97
+++ b/tests/ref/vsynth/vsynth1-jpeg2000-97
@@ -1,4 +1,4 @@
-0c704936bf54fad2b072dc0c8c289bb7 *tests/data/fate/vsynth1-jpeg2000-97.avi
-4336566 tests/data/fate/vsynth1-jpeg2000-97.avi
-1207ed65ca04d2c91936c4bc73c3a562 *tests/data/fate/vsynth1-jpeg2000-97.out.rawvideo
-stddev:    4.36 PSNR: 35.34 MAXDIFF:   51 bytes:  7603200/  7603200
+4c7dbe2451f56a49c29b0b5d7808d74d *tests/data/fate/vsynth1-jpeg2000-97.avi
+3661616 tests/data/fate/vsynth1-jpeg2000-97.avi
+d079e946a2fb75ad5ce6cb2760d1cc62 *tests/data/fate/vsynth1-jpeg2000-97.out.rawvideo
+stddev:    4.63 PSNR: 34.81 MAXDIFF:   54 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth2-jpeg2000-97 b/tests/ref/vsynth/vsynth2-jpeg2000-97
index caeb9adaab..a0c24d8ab1 100644
--- a/tests/ref/vsynth/vsynth2-jpeg2000-97
+++ b/tests/ref/vsynth/vsynth2-jpeg2000-97
@@ -1,4 +1,4 @@
-7bb7ccf4e9743888a5bda48f431630cf *tests/data/fate/vsynth2-jpeg2000-97.avi
-3124708 tests/data/fate/vsynth2-jpeg2000-97.avi
-fe0f3a97aa8c0b2f02dd11ccd1b77ca6 *tests/data/fate/vsynth2-jpeg2000-97.out.rawvideo
-stddev:    3.26 PSNR: 37.86 MAXDIFF:   26 bytes:  7603200/  7603200
+c3582d23a1fca31a6218346b82167f88 *tests/data/fate/vsynth2-jpeg2000-97.avi
+2451092 tests/data/fate/vsynth2-jpeg2000-97.avi
+d1329b49bcfcf74279eb07f7e20ddcec *tests/data/fate/vsynth2-jpeg2000-97.out.rawvideo
+stddev:    3.73 PSNR: 36.69 MAXDIFF:   30 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth3-jpeg2000-97 b/tests/ref/vsynth/vsynth3-jpeg2000-97
index eff6945989..1d8d148a78 100644
--- a/tests/ref/vsynth/vsynth3-jpeg2000-97
+++ b/tests/ref/vsynth/vsynth3-jpeg2000-97
@@ -1,4 +1,4 @@
-d551b1b1bfc691356e6f45f4386aff1d *tests/data/fate/vsynth3-jpeg2000-97.avi
-92144 tests/data/fate/vsynth3-jpeg2000-97.avi
-533535dad2ccc7aef3eb0e7ccb239bf2 *tests/data/fate/vsynth3-jpeg2000-97.out.rawvideo
-stddev:    4.60 PSNR: 34.87 MAXDIFF:   46 bytes:    86700/    86700
+2f8a9b514fbf1cb034076459463a7b76 *tests/data/fate/vsynth3-jpeg2000-97.avi
+83866 tests/data/fate/vsynth3-jpeg2000-97.avi
+febc7ef2ae9ec3f34b74d456922ae858 *tests/data/fate/vsynth3-jpeg2000-97.out.rawvideo
+stddev:    4.87 PSNR: 34.37 MAXDIFF:   51 bytes:    86700/    86700
diff --git a/tests/ref/vsynth/vsynth_lena-jpeg2000-97 b/tests/ref/vsynth/vsynth_lena-jpeg2000-97
index 426787c0cd..089479b04f 100644
--- a/tests/ref/vsynth/vsynth_lena-jpeg2000-97
+++ b/tests/ref/vsynth/vsynth_lena-jpeg2000-97
@@ -1,4 +1,4 @@
-6600b19f6c2e84c599f37d33a295ee35 *tests/data/fate/vsynth_lena-jpeg2000-97.avi
-2459730 tests/data/fate/vsynth_lena-jpeg2000-97.avi
-9d9c07b0d1936810032820a08d0b0b5d *tests/data/fate/vsynth_lena-jpeg2000-97.out.rawvideo
-stddev:    3.01 PSNR: 38.55 MAXDIFF:   27 bytes:  7603200/  7603200
+60808e880f1fd410b010feeca9105f4e *tests/data/fate/vsynth_lena-jpeg2000-97.avi
+1931500 tests/data/fate/vsynth_lena-jpeg2000-97.avi
+6d775a823d4b96cc6c121665bc7eb359 *tests/data/fate/vsynth_lena-jpeg2000-97.out.rawvideo
+stddev:    3.38 PSNR: 37.53 MAXDIFF:   28 bytes:  7603200/  7603200