avcodec/snowenc: Fix visual weight calculation

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 5b5fcadea059ab458a886261a5b7a1cc134b517a)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

8 files changed, 44 insertions, 40 deletions

diff --git a/libavcodec/snowenc.c b/libavcodec/snowenc.c
index b1cf1426ee..8e16fbc9a9 100644
--- a/libavcodec/snowenc.c
+++ b/libavcodec/snowenc.c
@@ -1544,10 +1544,10 @@ static void calculate_visual_weight(SnowContext *s, Plane *p){
     int level, orientation, x, y;
 
     for(level=0; level<s->spatial_decomposition_count; level++){
+        int64_t error=0;
         for(orientation=level ? 1 : 0; orientation<4; orientation++){
             SubBand *b= &p->band[level][orientation];
             IDWTELEM *ibuf= b->ibuf;
-            int64_t error=0;
 
             memset(s->spatial_idwt_buffer, 0, sizeof(*s->spatial_idwt_buffer)*width*height);
             ibuf[b->width/2 + b->height/2*b->stride]= 256*16;
@@ -1558,9 +1558,13 @@ static void calculate_visual_weight(SnowContext *s, Plane *p){
                     error += d*d;
                 }
             }
-
+            if (orientation == 2)
+                error /= 2;
             b->qlog= (int)(QROOT * log2(352256.0/sqrt(error)) + 0.5);
+            if (orientation != 1)
+                error = 0;
         }
+        p->band[level][1].qlog = p->band[level][2].qlog;
     }
 }
 
diff --git a/tests/ref/seek/vsynth_lena-snow b/tests/ref/seek/vsynth_lena-snow
index 33d6c27463..b2d2d22cda 100644
--- a/tests/ref/seek/vsynth_lena-snow
+++ b/tests/ref/seek/vsynth_lena-snow
@@ -2,45 +2,45 @@ ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5652 size:  3035
 ret: 0         st:-1 flags:0  ts:-1.000000
 ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5652 size:  3035
 ret: 0         st:-1 flags:1  ts: 1.894167
-ret: 0         st: 0 flags:1 dts: 1.440000 pts: 1.440000 pos:  39806 size:  3640
+ret: 0         st: 0 flags:1 dts: 1.440000 pts: 1.440000 pos:  39690 size:  3640
 ret: 0         st: 0 flags:0  ts: 0.800000
-ret: 0         st: 0 flags:1 dts: 0.960000 pts: 0.960000 pos:  27442 size:  3494
+ret: 0         st: 0 flags:1 dts: 0.960000 pts: 0.960000 pos:  27382 size:  3493
 ret:-1         st: 0 flags:1  ts:-0.320000
 ret:-1         st:-1 flags:0  ts: 2.576668
 ret: 0         st:-1 flags:1  ts: 1.470835
-ret: 0         st: 0 flags:1 dts: 1.440000 pts: 1.440000 pos:  39806 size:  3640
+ret: 0         st: 0 flags:1 dts: 1.440000 pts: 1.440000 pos:  39690 size:  3640
 ret: 0         st: 0 flags:0  ts: 0.360000
-ret: 0         st: 0 flags:1 dts: 0.480000 pts: 0.480000 pos:  16134 size:  3244
+ret: 0         st: 0 flags:1 dts: 0.480000 pts: 0.480000 pos:  16074 size:  3245
 ret:-1         st: 0 flags:1  ts:-0.760000
 ret:-1         st:-1 flags:0  ts: 2.153336
 ret: 0         st:-1 flags:1  ts: 1.047503
-ret: 0         st: 0 flags:1 dts: 0.960000 pts: 0.960000 pos:  27442 size:  3494
+ret: 0         st: 0 flags:1 dts: 0.960000 pts: 0.960000 pos:  27382 size:  3493
 ret: 0         st: 0 flags:0  ts:-0.040000
 ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5652 size:  3035
 ret: 0         st: 0 flags:1  ts: 2.840000
-ret: 0         st: 0 flags:1 dts: 1.920000 pts: 1.920000 pos:  52608 size:  3582
+ret: 0         st: 0 flags:1 dts: 1.920000 pts: 1.920000 pos:  52538 size:  3582
 ret: 0         st:-1 flags:0  ts: 1.730004
-ret: 0         st: 0 flags:1 dts: 1.920000 pts: 1.920000 pos:  52608 size:  3582
+ret: 0         st: 0 flags:1 dts: 1.920000 pts: 1.920000 pos:  52538 size:  3582
 ret: 0         st:-1 flags:1  ts: 0.624171
-ret: 0         st: 0 flags:1 dts: 0.480000 pts: 0.480000 pos:  16134 size:  3244
+ret: 0         st: 0 flags:1 dts: 0.480000 pts: 0.480000 pos:  16074 size:  3245
 ret: 0         st: 0 flags:0  ts:-0.480000
 ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5652 size:  3035
 ret: 0         st: 0 flags:1  ts: 2.400000
-ret: 0         st: 0 flags:1 dts: 1.920000 pts: 1.920000 pos:  52608 size:  3582
+ret: 0         st: 0 flags:1 dts: 1.920000 pts: 1.920000 pos:  52538 size:  3582
 ret: 0         st:-1 flags:0  ts: 1.306672
-ret: 0         st: 0 flags:1 dts: 1.440000 pts: 1.440000 pos:  39806 size:  3640
+ret: 0         st: 0 flags:1 dts: 1.440000 pts: 1.440000 pos:  39690 size:  3640
 ret: 0         st:-1 flags:1  ts: 0.200839
 ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5652 size:  3035
 ret: 0         st: 0 flags:0  ts:-0.920000
 ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5652 size:  3035
 ret: 0         st: 0 flags:1  ts: 2.000000
-ret: 0         st: 0 flags:1 dts: 1.920000 pts: 1.920000 pos:  52608 size:  3582
+ret: 0         st: 0 flags:1 dts: 1.920000 pts: 1.920000 pos:  52538 size:  3582
 ret: 0         st:-1 flags:0  ts: 0.883340
-ret: 0         st: 0 flags:1 dts: 0.960000 pts: 0.960000 pos:  27442 size:  3494
+ret: 0         st: 0 flags:1 dts: 0.960000 pts: 0.960000 pos:  27382 size:  3493
 ret:-1         st:-1 flags:1  ts:-0.222493
 ret:-1         st: 0 flags:0  ts: 2.680000
 ret: 0         st: 0 flags:1  ts: 1.560000
-ret: 0         st: 0 flags:1 dts: 1.440000 pts: 1.440000 pos:  39806 size:  3640
+ret: 0         st: 0 flags:1 dts: 1.440000 pts: 1.440000 pos:  39690 size:  3640
 ret: 0         st:-1 flags:0  ts: 0.460008
-ret: 0         st: 0 flags:1 dts: 0.480000 pts: 0.480000 pos:  16134 size:  3244
+ret: 0         st: 0 flags:1 dts: 0.480000 pts: 0.480000 pos:  16074 size:  3245
 ret:-1         st:-1 flags:1  ts:-0.645825
diff --git a/tests/ref/vsynth/vsynth1-snow b/tests/ref/vsynth/vsynth1-snow
index f20abd2ee4..b0e3a0bfd7 100644
--- a/tests/ref/vsynth/vsynth1-snow
+++ b/tests/ref/vsynth/vsynth1-snow
@@ -1,4 +1,4 @@
-67c10f8d52fcd1103caa675a1408bf6e *tests/data/fate/vsynth1-snow.avi
-136088 tests/data/fate/vsynth1-snow.avi
-bfc0bcc4bc7b956933aa58acc587018d *tests/data/fate/vsynth1-snow.out.rawvideo
-stddev:   22.77 PSNR: 20.98 MAXDIFF:  175 bytes:  7603200/  7603200
+c4c77a6fb926b89fe6591c398f5cd4db *tests/data/fate/vsynth1-snow.avi
+136160 tests/data/fate/vsynth1-snow.avi
+dcf8b3f62d9c3ae2b2d0fbbacbf83e4e *tests/data/fate/vsynth1-snow.out.rawvideo
+stddev:   22.74 PSNR: 20.99 MAXDIFF:  173 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth1-snow-hpel b/tests/ref/vsynth/vsynth1-snow-hpel
index 39780ad8a2..72b082b2ce 100644
--- a/tests/ref/vsynth/vsynth1-snow-hpel
+++ b/tests/ref/vsynth/vsynth1-snow-hpel
@@ -1,4 +1,4 @@
-e62ae25d5040d04622a965bcb27fdb1e *tests/data/fate/vsynth1-snow-hpel.avi
-138446 tests/data/fate/vsynth1-snow-hpel.avi
-57c914cd150f8fc260b5989ce3e5884c *tests/data/fate/vsynth1-snow-hpel.out.rawvideo
-stddev:   22.74 PSNR: 20.99 MAXDIFF:  172 bytes:  7603200/  7603200
+5c9eb93646eb0e5570d37e9adc9625e4 *tests/data/fate/vsynth1-snow-hpel.avi
+138580 tests/data/fate/vsynth1-snow-hpel.avi
+3382bdde624d8bb4af206a5ac6614605 *tests/data/fate/vsynth1-snow-hpel.out.rawvideo
+stddev:   22.71 PSNR: 21.00 MAXDIFF:  171 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth2-snow b/tests/ref/vsynth/vsynth2-snow
index e9607bb7d0..355f89d5f4 100644
--- a/tests/ref/vsynth/vsynth2-snow
+++ b/tests/ref/vsynth/vsynth2-snow
@@ -1,4 +1,4 @@
-0a41e73ddd2f54936490655b46dad4a3 *tests/data/fate/vsynth2-snow.avi
-72868 tests/data/fate/vsynth2-snow.avi
-34a75f5cf8a71159f1a572d9cedcfef9 *tests/data/fate/vsynth2-snow.out.rawvideo
-stddev:   13.73 PSNR: 25.37 MAXDIFF:  162 bytes:  7603200/  7603200
+5e130d6a48b69348eee7f7c76c5869a3 *tests/data/fate/vsynth2-snow.avi
+72942 tests/data/fate/vsynth2-snow.avi
+9b6cee60e3ec0d1f312a8a25a7878fcc *tests/data/fate/vsynth2-snow.out.rawvideo
+stddev:   13.39 PSNR: 25.59 MAXDIFF:  154 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth2-snow-hpel b/tests/ref/vsynth/vsynth2-snow-hpel
index 66839fd6f6..ec3b5dfad2 100644
--- a/tests/ref/vsynth/vsynth2-snow-hpel
+++ b/tests/ref/vsynth/vsynth2-snow-hpel
@@ -1,4 +1,4 @@
-9bc409e4794ee50691a26c9c836d31a7 *tests/data/fate/vsynth2-snow-hpel.avi
-79728 tests/data/fate/vsynth2-snow-hpel.avi
-2cc64d8171175a1532fd7d3ed3011fbf *tests/data/fate/vsynth2-snow-hpel.out.rawvideo
-stddev:   13.70 PSNR: 25.39 MAXDIFF:  162 bytes:  7603200/  7603200
+8edcf0fd7f066972ff77d5b891ed6dde *tests/data/fate/vsynth2-snow-hpel.avi
+79798 tests/data/fate/vsynth2-snow-hpel.avi
+7e0f2a24feda6fb3e54b85511a28c45f *tests/data/fate/vsynth2-snow-hpel.out.rawvideo
+stddev:   13.35 PSNR: 25.62 MAXDIFF:  157 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth_lena-snow b/tests/ref/vsynth/vsynth_lena-snow
index ec29a78483..582c294531 100644
--- a/tests/ref/vsynth/vsynth_lena-snow
+++ b/tests/ref/vsynth/vsynth_lena-snow
@@ -1,4 +1,4 @@
-8e96f337e8f4ccac7d72ef517e1d2208 *tests/data/fate/vsynth_lena-snow.avi
-57680 tests/data/fate/vsynth_lena-snow.avi
-90963cfd2359d460001c94d94256dc2b *tests/data/fate/vsynth_lena-snow.out.rawvideo
-stddev:   10.48 PSNR: 27.72 MAXDIFF:  119 bytes:  7603200/  7603200
+bf2cf9cacc1d98388798be98872049ee *tests/data/fate/vsynth_lena-snow.avi
+57604 tests/data/fate/vsynth_lena-snow.avi
+707a42eb20195913be55ba8dfadf72fb *tests/data/fate/vsynth_lena-snow.out.rawvideo
+stddev:   10.37 PSNR: 27.81 MAXDIFF:  120 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth_lena-snow-hpel b/tests/ref/vsynth/vsynth_lena-snow-hpel
index 2d6edd8a79..67effebc8a 100644
--- a/tests/ref/vsynth/vsynth_lena-snow-hpel
+++ b/tests/ref/vsynth/vsynth_lena-snow-hpel
@@ -1,4 +1,4 @@
-56b14cb1cbb637536233982e87f7ac3e *tests/data/fate/vsynth_lena-snow-hpel.avi
-61764 tests/data/fate/vsynth_lena-snow-hpel.avi
-244b0266127fa354d8485234b2c388e4 *tests/data/fate/vsynth_lena-snow-hpel.out.rawvideo
-stddev:   10.45 PSNR: 27.74 MAXDIFF:  119 bytes:  7603200/  7603200
+c6ec87a11415a99b1a781f9f5bacb722 *tests/data/fate/vsynth_lena-snow-hpel.avi
+61814 tests/data/fate/vsynth_lena-snow-hpel.avi
+40f330397b7acf6bdbb3ec6d908be451 *tests/data/fate/vsynth_lena-snow-hpel.out.rawvideo
+stddev:   10.34 PSNR: 27.83 MAXDIFF:  118 bytes:  7603200/  7603200