avcodec/asvenc: fix AAN scaling

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit b0f7de3e7c735cf6e222006ecbbd3e11b5f90a21)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

9 files changed, 73 insertions, 66 deletions

diff --git a/libavcodec/asvenc.c b/libavcodec/asvenc.c
index bbf4494866..3ad2c31591 100644
--- a/libavcodec/asvenc.c
+++ b/libavcodec/asvenc.c
@@ -26,8 +26,10 @@
 #include "libavutil/attributes.h"
 #include "libavutil/mem.h"
 
+#include "aandcttab.h"
 #include "asv.h"
 #include "avcodec.h"
+#include "dct.h"
 #include "fdctdsp.h"
 #include "internal.h"
 #include "mathops.h"
@@ -331,8 +333,13 @@ static av_cold int encode_init(AVCodecContext *avctx)
     ((uint32_t *) avctx->extradata)[1] = av_le2ne32(AV_RL32("ASUS"));
 
     for (i = 0; i < 64; i++) {
-        int q = 32 * scale * ff_mpeg1_default_intra_matrix[i];
-        a->q_intra_matrix[i] = ((a->inv_qscale << 16) + q / 2) / q;
+        if (a->fdsp.fdct == ff_fdct_ifast) {
+            int q = 32LL * scale * ff_mpeg1_default_intra_matrix[i] * ff_aanscales[i];
+            a->q_intra_matrix[i] = (((int64_t)a->inv_qscale << 30) + q / 2) / q;
+        } else {
+            int q = 32 * scale * ff_mpeg1_default_intra_matrix[i];
+            a->q_intra_matrix[i] = ((a->inv_qscale << 16) + q / 2) / q;
+        }
     }
 
     return 0;
diff --git a/tests/ref/seek/vsynth2-asv1 b/tests/ref/seek/vsynth2-asv1
index 5873bb17b7..e2556a7459 100644
--- a/tests/ref/seek/vsynth2-asv1
+++ b/tests/ref/seek/vsynth2-asv1
@@ -1,46 +1,46 @@
-ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 14316
+ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 12152
 ret: 0         st:-1 flags:0  ts:-1.000000
-ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 14316
+ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 12152
 ret: 0         st:-1 flags:1  ts: 1.894167
-ret: 0         st: 0 flags:1 dts: 1.880000 pts: 1.880000 pos: 776840 size: 18256
+ret: 0         st: 0 flags:1 dts: 1.880000 pts: 1.880000 pos: 643344 size: 15064
 ret: 0         st: 0 flags:0  ts: 0.800000
-ret: 0         st: 0 flags:1 dts: 0.800000 pts: 0.800000 pos: 305352 size: 16180
+ret: 0         st: 0 flags:1 dts: 0.800000 pts: 0.800000 pos: 255232 size: 13312
 ret:-1         st: 0 flags:1  ts:-0.320000
 ret:-1         st:-1 flags:0  ts: 2.576668
 ret: 0         st:-1 flags:1  ts: 1.470835
-ret: 0         st: 0 flags:1 dts: 1.480000 pts: 1.480000 pos: 595448 size: 17980
+ret: 0         st: 0 flags:1 dts: 1.480000 pts: 1.480000 pos: 493584 size: 14796
 ret: 0         st: 0 flags:0  ts: 0.360000
-ret: 0         st: 0 flags:1 dts: 0.360000 pts: 0.360000 pos: 135516 size: 14868
+ret: 0         st: 0 flags:1 dts: 0.360000 pts: 0.360000 pos: 114852 size: 12356
 ret:-1         st: 0 flags:1  ts:-0.760000
 ret:-1         st:-1 flags:0  ts: 2.153336
 ret: 0         st:-1 flags:1  ts: 1.047503
-ret: 0         st: 0 flags:1 dts: 1.040000 pts: 1.040000 pos: 404100 size: 16856
+ret: 0         st: 0 flags:1 dts: 1.040000 pts: 1.040000 pos: 336180 size: 13824
 ret: 0         st: 0 flags:0  ts:-0.040000
-ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 14316
+ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 12152
 ret: 0         st: 0 flags:1  ts: 2.840000
-ret: 0         st: 0 flags:1 dts: 1.960000 pts: 1.960000 pos: 813396 size: 18296
+ret: 0         st: 0 flags:1 dts: 1.960000 pts: 1.960000 pos: 673504 size: 15104
 ret: 0         st:-1 flags:0  ts: 1.730004
-ret: 0         st: 0 flags:1 dts: 1.720000 pts: 1.720000 pos: 704136 size: 18140
+ret: 0         st: 0 flags:1 dts: 1.720000 pts: 1.720000 pos: 583312 size: 14956
 ret: 0         st:-1 flags:1  ts: 0.624171
-ret: 0         st: 0 flags:1 dts: 0.640000 pts: 0.640000 pos: 241764 size: 15736
+ret: 0         st: 0 flags:1 dts: 0.640000 pts: 0.640000 pos: 202872 size: 12944
 ret: 0         st: 0 flags:0  ts:-0.480000
-ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 14316
+ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 12152
 ret: 0         st: 0 flags:1  ts: 2.400000
-ret: 0         st: 0 flags:1 dts: 1.960000 pts: 1.960000 pos: 813396 size: 18296
+ret: 0         st: 0 flags:1 dts: 1.960000 pts: 1.960000 pos: 673504 size: 15104
 ret: 0         st:-1 flags:0  ts: 1.306672
-ret: 0         st: 0 flags:1 dts: 1.320000 pts: 1.320000 pos: 524488 size: 17548
+ret: 0         st: 0 flags:1 dts: 1.320000 pts: 1.320000 pos: 435128 size: 14488
 ret: 0         st:-1 flags:1  ts: 0.200839
-ret: 0         st: 0 flags:1 dts: 0.200000 pts: 0.200000 pos:  77020 size: 14496
+ret: 0         st: 0 flags:1 dts: 0.200000 pts: 0.200000 pos:  65864 size: 12144
 ret: 0         st: 0 flags:0  ts:-0.920000
-ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 14316
+ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 12152
 ret: 0         st: 0 flags:1  ts: 2.000000
-ret: 0         st: 0 flags:1 dts: 1.960000 pts: 1.960000 pos: 813396 size: 18296
+ret: 0         st: 0 flags:1 dts: 1.960000 pts: 1.960000 pos: 673504 size: 15104
 ret: 0         st:-1 flags:0  ts: 0.883340
-ret: 0         st: 0 flags:1 dts: 0.880000 pts: 0.880000 pos: 337808 size: 16388
+ret: 0         st: 0 flags:1 dts: 0.880000 pts: 0.880000 pos: 281884 size: 13412
 ret:-1         st:-1 flags:1  ts:-0.222493
 ret:-1         st: 0 flags:0  ts: 2.680000
 ret: 0         st: 0 flags:1  ts: 1.560000
-ret: 0         st: 0 flags:1 dts: 1.560000 pts: 1.560000 pos: 631584 size: 18188
+ret: 0         st: 0 flags:1 dts: 1.560000 pts: 1.560000 pos: 523324 size: 14972
 ret: 0         st:-1 flags:0  ts: 0.460008
-ret: 0         st: 0 flags:1 dts: 0.480000 pts: 0.480000 pos: 180212 size: 15168
+ret: 0         st: 0 flags:1 dts: 0.480000 pts: 0.480000 pos: 152056 size: 12540
 ret:-1         st:-1 flags:1  ts:-0.645825
diff --git a/tests/ref/seek/vsynth2-asv2 b/tests/ref/seek/vsynth2-asv2
index 7d37b7ac06..cd99c45ac7 100644
--- a/tests/ref/seek/vsynth2-asv2
+++ b/tests/ref/seek/vsynth2-asv2
@@ -1,46 +1,46 @@
-ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 13732
+ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 12072
 ret: 0         st:-1 flags:0  ts:-1.000000
-ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 13732
+ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 12072
 ret: 0         st:-1 flags:1  ts: 1.894167
-ret: 0         st: 0 flags:1 dts: 1.880000 pts: 1.880000 pos: 736152 size: 17340
+ret: 0         st: 0 flags:1 dts: 1.880000 pts: 1.880000 pos: 630028 size: 14892
 ret: 0         st: 0 flags:0  ts: 0.800000
-ret: 0         st: 0 flags:1 dts: 0.800000 pts: 0.800000 pos: 289708 size: 15300
+ret: 0         st: 0 flags:1 dts: 0.800000 pts: 0.800000 pos: 248836 size: 13024
 ret:-1         st: 0 flags:1  ts:-0.320000
 ret:-1         st:-1 flags:0  ts: 2.576668
 ret: 0         st:-1 flags:1  ts: 1.470835
-ret: 0         st: 0 flags:1 dts: 1.480000 pts: 1.480000 pos: 564140 size: 17016
+ret: 0         st: 0 flags:1 dts: 1.480000 pts: 1.480000 pos: 482700 size: 14532
 ret: 0         st: 0 flags:0  ts: 0.360000
-ret: 0         st: 0 flags:1 dts: 0.360000 pts: 0.360000 pos: 128564 size: 14052
+ret: 0         st: 0 flags:1 dts: 0.360000 pts: 0.360000 pos: 111600 size: 12016
 ret:-1         st: 0 flags:1  ts:-0.760000
 ret:-1         st:-1 flags:0  ts: 2.153336
 ret: 0         st:-1 flags:1  ts: 1.047503
-ret: 0         st: 0 flags:1 dts: 1.040000 pts: 1.040000 pos: 383244 size: 15896
+ret: 0         st: 0 flags:1 dts: 1.040000 pts: 1.040000 pos: 328252 size: 13532
 ret: 0         st: 0 flags:0  ts:-0.040000
-ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 13732
+ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 12072
 ret: 0         st: 0 flags:1  ts: 2.840000
-ret: 0         st: 0 flags:1 dts: 1.960000 pts: 1.960000 pos: 770852 size: 17400
+ret: 0         st: 0 flags:1 dts: 1.960000 pts: 1.960000 pos: 659848 size: 14928
 ret: 0         st:-1 flags:0  ts: 1.730004
-ret: 0         st: 0 flags:1 dts: 1.720000 pts: 1.720000 pos: 667016 size: 17172
+ret: 0         st: 0 flags:1 dts: 1.720000 pts: 1.720000 pos: 570784 size: 14712
 ret: 0         st:-1 flags:1  ts: 0.624171
-ret: 0         st: 0 flags:1 dts: 0.640000 pts: 0.640000 pos: 229388 size: 14956
+ret: 0         st: 0 flags:1 dts: 0.640000 pts: 0.640000 pos: 197536 size: 12724
 ret: 0         st: 0 flags:0  ts:-0.480000
-ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 13732
+ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 12072
 ret: 0         st: 0 flags:1  ts: 2.400000
-ret: 0         st: 0 flags:1 dts: 1.960000 pts: 1.960000 pos: 770852 size: 17400
+ret: 0         st: 0 flags:1 dts: 1.960000 pts: 1.960000 pos: 659848 size: 14928
 ret: 0         st:-1 flags:0  ts: 1.306672
-ret: 0         st: 0 flags:1 dts: 1.320000 pts: 1.320000 pos: 496932 size: 16564
+ret: 0         st: 0 flags:1 dts: 1.320000 pts: 1.320000 pos: 425316 size: 14116
 ret: 0         st:-1 flags:1  ts: 0.200839
-ret: 0         st: 0 flags:1 dts: 0.200000 pts: 0.200000 pos:  73176 size: 13664
+ret: 0         st: 0 flags:1 dts: 0.200000 pts: 0.200000 pos:  64104 size: 11780
 ret: 0         st: 0 flags:0  ts:-0.920000
-ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 13732
+ret: 0         st: 0 flags:1 dts: 0.000000 pts: 0.000000 pos:   5656 size: 12072
 ret: 0         st: 0 flags:1  ts: 2.000000
-ret: 0         st: 0 flags:1 dts: 1.960000 pts: 1.960000 pos: 770852 size: 17400
+ret: 0         st: 0 flags:1 dts: 1.960000 pts: 1.960000 pos: 659848 size: 14928
 ret: 0         st:-1 flags:0  ts: 0.883340
-ret: 0         st: 0 flags:1 dts: 0.880000 pts: 0.880000 pos: 320444 size: 15592
+ret: 0         st: 0 flags:1 dts: 0.880000 pts: 0.880000 pos: 274964 size: 13176
 ret:-1         st:-1 flags:1  ts:-0.222493
 ret:-1         st: 0 flags:0  ts: 2.680000
 ret: 0         st: 0 flags:1  ts: 1.560000
-ret: 0         st: 0 flags:1 dts: 1.560000 pts: 1.560000 pos: 598288 size: 17180
+ret: 0         st: 0 flags:1 dts: 1.560000 pts: 1.560000 pos: 511944 size: 14668
 ret: 0         st:-1 flags:0  ts: 0.460008
-ret: 0         st: 0 flags:1 dts: 0.480000 pts: 0.480000 pos: 171012 size: 14392
+ret: 0         st: 0 flags:1 dts: 0.480000 pts: 0.480000 pos: 147820 size: 12240
 ret:-1         st:-1 flags:1  ts:-0.645825
diff --git a/tests/ref/vsynth/vsynth1-asv1 b/tests/ref/vsynth/vsynth1-asv1
index b450f0eacf..445197ac7e 100644
--- a/tests/ref/vsynth/vsynth1-asv1
+++ b/tests/ref/vsynth/vsynth1-asv1
@@ -1,4 +1,4 @@
-992195272b94d8779b6216030b85ae18 *tests/data/fate/vsynth1-asv1.avi
-1489644 tests/data/fate/vsynth1-asv1.avi
-2dfc5dfc2c1cbbc2543257cd3d2df6af *tests/data/fate/vsynth1-asv1.out.rawvideo
-stddev:   20.00 PSNR: 22.11 MAXDIFF:  158 bytes:  7603200/  7603200
+f446db5dfd6bcc14378bca9406e39adb *tests/data/fate/vsynth1-asv1.avi
+1298616 tests/data/fate/vsynth1-asv1.avi
+a4f95c58e8b9258da52b9d09153b1078 *tests/data/fate/vsynth1-asv1.out.rawvideo
+stddev:   11.89 PSNR: 26.62 MAXDIFF:  132 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth1-asv2 b/tests/ref/vsynth/vsynth1-asv2
index 7bc1a0d530..225712e4ed 100644
--- a/tests/ref/vsynth/vsynth1-asv2
+++ b/tests/ref/vsynth/vsynth1-asv2
@@ -1,4 +1,4 @@
-1e589b4dd70c4a8a7dbf4a466bab31ae *tests/data/fate/vsynth1-asv2.avi
-1456044 tests/data/fate/vsynth1-asv2.avi
-d451be09793cd0f35b6d91fc36e2571a *tests/data/fate/vsynth1-asv2.out.rawvideo
-stddev:   18.82 PSNR: 22.63 MAXDIFF:  131 bytes:  7603200/  7603200
+06b7b65686a1eb03d43f25e179c03872 *tests/data/fate/vsynth1-asv2.avi
+1434408 tests/data/fate/vsynth1-asv2.avi
+cb1f5560005800e889bfbb36cdc9fc40 *tests/data/fate/vsynth1-asv2.out.rawvideo
+stddev:    8.36 PSNR: 29.68 MAXDIFF:   58 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth2-asv1 b/tests/ref/vsynth/vsynth2-asv1
index 01fd2bf05e..78118887a7 100644
--- a/tests/ref/vsynth/vsynth2-asv1
+++ b/tests/ref/vsynth/vsynth2-asv1
@@ -1,4 +1,4 @@
-9218843465c3396e9b078e63ce7ba17e *tests/data/fate/vsynth2-asv1.avi
-832500 tests/data/fate/vsynth2-asv1.avi
-c96ff7fd17c52f99ddb7922a4cb9168f *tests/data/fate/vsynth2-asv1.out.rawvideo
-stddev:   10.47 PSNR: 27.73 MAXDIFF:   98 bytes:  7603200/  7603200
+bffe7188b4b5c3ff76c75561d0bebd77 *tests/data/fate/vsynth2-asv1.avi
+689416 tests/data/fate/vsynth2-asv1.avi
+a7cdefad200f48ab308c746461a8792e *tests/data/fate/vsynth2-asv1.out.rawvideo
+stddev:    5.07 PSNR: 34.03 MAXDIFF:   70 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth2-asv2 b/tests/ref/vsynth/vsynth2-asv2
index c66f056107..617a77fb8d 100644
--- a/tests/ref/vsynth/vsynth2-asv2
+++ b/tests/ref/vsynth/vsynth2-asv2
@@ -1,4 +1,4 @@
-065323f9873c367dd1e371fa47657ae4 *tests/data/fate/vsynth2-asv2.avi
-789060 tests/data/fate/vsynth2-asv2.avi
-74a78015b64b2cf8cb9da2e44f508a69 *tests/data/fate/vsynth2-asv2.out.rawvideo
-stddev:   10.28 PSNR: 27.89 MAXDIFF:   95 bytes:  7603200/  7603200
+f8c3b9899bbd9545757fac0c7ecf4e34 *tests/data/fate/vsynth2-asv2.avi
+675584 tests/data/fate/vsynth2-asv2.avi
+5990db66c7ac0bbe2f98ec2770c1bf3b *tests/data/fate/vsynth2-asv2.out.rawvideo
+stddev:    4.57 PSNR: 34.93 MAXDIFF:   47 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth3-asv1 b/tests/ref/vsynth/vsynth3-asv1
index fd5f82dc69..8486b040f8 100644
--- a/tests/ref/vsynth/vsynth3-asv1
+++ b/tests/ref/vsynth/vsynth3-asv1
@@ -1,4 +1,4 @@
-43c51aa52666de02dfa49fd1c10a558f *tests/data/fate/vsynth3-asv1.avi
-36664 tests/data/fate/vsynth3-asv1.avi
-fd156079afc2753dde7d6a8418b2a25a *tests/data/fate/vsynth3-asv1.out.rawvideo
-stddev:   21.25 PSNR: 21.58 MAXDIFF:  122 bytes:    86700/    86700
+642c88813798d857d236f21bb36d8783 *tests/data/fate/vsynth3-asv1.avi
+34700 tests/data/fate/vsynth3-asv1.avi
+3c8636e22a96267451684f42d7a6f608 *tests/data/fate/vsynth3-asv1.out.rawvideo
+stddev:   13.16 PSNR: 25.74 MAXDIFF:  112 bytes:    86700/    86700
diff --git a/tests/ref/vsynth/vsynth3-asv2 b/tests/ref/vsynth/vsynth3-asv2
index 360bb07203..2a5d23bb96 100644
--- a/tests/ref/vsynth/vsynth3-asv2
+++ b/tests/ref/vsynth/vsynth3-asv2
@@ -1,4 +1,4 @@
-8958facfc4853267ece3b7868df80229 *tests/data/fate/vsynth3-asv2.avi
-35620 tests/data/fate/vsynth3-asv2.avi
-ff711d6341966f03ee32b73ae22493ed *tests/data/fate/vsynth3-asv2.out.rawvideo
-stddev:   19.98 PSNR: 22.12 MAXDIFF:  113 bytes:    86700/    86700
+45f865e3d1cac1ef59ae4689f41dead5 *tests/data/fate/vsynth3-asv2.avi
+36204 tests/data/fate/vsynth3-asv2.avi
+5469c0735b7c9279e5e8e3439fc6acab *tests/data/fate/vsynth3-asv2.out.rawvideo
+stddev:    9.07 PSNR: 28.97 MAXDIFF:   51 bytes:    86700/    86700