avcodec/j2kenc: use uint64_t type for the lambda value

Should fix integer overflows, and improve encoding results.

Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: James Almer <jamrial@gmail.com>

5 files changed, 18 insertions, 18 deletions

diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c
index 2b0c58cff4..5f95b772d1 100644
--- a/libavcodec/j2kenc.c
+++ b/libavcodec/j2kenc.c
@@ -127,7 +127,7 @@ typedef struct {
     uint8_t *buf_end;
     int bit_index;
 
-    int64_t lambda;
+    uint64_t lambda;
 
     Jpeg2000CodingStyle codsty;
     Jpeg2000QuantStyle  qntsty;
@@ -1348,7 +1348,7 @@ static void makelayers(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile)
     }
 }
 
-static int getcut(Jpeg2000Cblk *cblk, int64_t lambda, int dwt_norm)
+static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm)
 {
     int passno, res = 0;
     for (passno = 0; passno < cblk->npasses; passno++){
diff --git a/tests/ref/vsynth/vsynth1-jpeg2000-yuva444p16 b/tests/ref/vsynth/vsynth1-jpeg2000-yuva444p16
index 363ddc4ffa..f3d3ba8a6c 100644
--- a/tests/ref/vsynth/vsynth1-jpeg2000-yuva444p16
+++ b/tests/ref/vsynth/vsynth1-jpeg2000-yuva444p16
@@ -1,4 +1,4 @@
-9261706cfd4dfd8db38fc9bdafe6bef7 *tests/data/fate/vsynth1-jpeg2000-yuva444p16.avi
-12433462 tests/data/fate/vsynth1-jpeg2000-yuva444p16.avi
-6bc7131cd19dbb59339bb13a6d53b94d *tests/data/fate/vsynth1-jpeg2000-yuva444p16.out.rawvideo
-stddev:   12.97 PSNR: 25.87 MAXDIFF:  169 bytes:  7603200/  7603200
+4c051562f5ac7c9871e034917326787c *tests/data/fate/vsynth1-jpeg2000-yuva444p16.avi
+12502390 tests/data/fate/vsynth1-jpeg2000-yuva444p16.avi
+59d53e883f76d41d17c926236ca9d7d2 *tests/data/fate/vsynth1-jpeg2000-yuva444p16.out.rawvideo
+stddev:    2.66 PSNR: 39.62 MAXDIFF:   44 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth2-jpeg2000-yuva444p16 b/tests/ref/vsynth/vsynth2-jpeg2000-yuva444p16
index bec08da307..3cbe5f4ad2 100644
--- a/tests/ref/vsynth/vsynth2-jpeg2000-yuva444p16
+++ b/tests/ref/vsynth/vsynth2-jpeg2000-yuva444p16
@@ -1,4 +1,4 @@
-54063ab98e48b2a079b63643fe26e082 *tests/data/fate/vsynth2-jpeg2000-yuva444p16.avi
-11489842 tests/data/fate/vsynth2-jpeg2000-yuva444p16.avi
-a4232b2959472f88b5310572456fa900 *tests/data/fate/vsynth2-jpeg2000-yuva444p16.out.rawvideo
-stddev:   11.69 PSNR: 26.77 MAXDIFF:  188 bytes:  7603200/  7603200
+37555b5b00bba2773d5a4212f6c7debd *tests/data/fate/vsynth2-jpeg2000-yuva444p16.avi
+11501630 tests/data/fate/vsynth2-jpeg2000-yuva444p16.avi
+61be7920a21190d946c22cb2c945ab6f *tests/data/fate/vsynth2-jpeg2000-yuva444p16.out.rawvideo
+stddev:    0.53 PSNR: 53.49 MAXDIFF:   13 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth3-jpeg2000-yuva444p16 b/tests/ref/vsynth/vsynth3-jpeg2000-yuva444p16
index dae44f40bd..ce92bf7e43 100644
--- a/tests/ref/vsynth/vsynth3-jpeg2000-yuva444p16
+++ b/tests/ref/vsynth/vsynth3-jpeg2000-yuva444p16
@@ -1,4 +1,4 @@
-afdff47e9c71d912153e59a56afdf2a9 *tests/data/fate/vsynth3-jpeg2000-yuva444p16.avi
-194250 tests/data/fate/vsynth3-jpeg2000-yuva444p16.avi
-9e0ef11f8c85206a950a919cb679f129 *tests/data/fate/vsynth3-jpeg2000-yuva444p16.out.rawvideo
-stddev:   10.72 PSNR: 27.53 MAXDIFF:   82 bytes:    86700/    86700
+ae5951cee3db63c7c6f81d23b601c0fb *tests/data/fate/vsynth3-jpeg2000-yuva444p16.avi
+194668 tests/data/fate/vsynth3-jpeg2000-yuva444p16.avi
+d959793c1f0bb86b0fc509d5747e03c7 *tests/data/fate/vsynth3-jpeg2000-yuva444p16.out.rawvideo
+stddev:    3.06 PSNR: 38.39 MAXDIFF:   40 bytes:    86700/    86700
diff --git a/tests/ref/vsynth/vsynth_lena-jpeg2000-yuva444p16 b/tests/ref/vsynth/vsynth_lena-jpeg2000-yuva444p16
index d9e856735b..6dc522bb01 100644
--- a/tests/ref/vsynth/vsynth_lena-jpeg2000-yuva444p16
+++ b/tests/ref/vsynth/vsynth_lena-jpeg2000-yuva444p16
@@ -1,4 +1,4 @@
-e9c56e128cf980db7c8e965d2afc0ea6 *tests/data/fate/vsynth_lena-jpeg2000-yuva444p16.avi
-11109882 tests/data/fate/vsynth_lena-jpeg2000-yuva444p16.avi
-3e1a4f9ca46c2dd3fd997c01a0dfb130 *tests/data/fate/vsynth_lena-jpeg2000-yuva444p16.out.rawvideo
-stddev:   11.67 PSNR: 26.79 MAXDIFF:  150 bytes:  7603200/  7603200
+fcfb2e4738ad6afd7f9d4e9f92084415 *tests/data/fate/vsynth_lena-jpeg2000-yuva444p16.avi
+11123818 tests/data/fate/vsynth_lena-jpeg2000-yuva444p16.avi
+62ac96a84eb370e5543582512dbe5d7c *tests/data/fate/vsynth_lena-jpeg2000-yuva444p16.out.rawvideo
+stddev:    0.45 PSNR: 54.88 MAXDIFF:    7 bytes:  7603200/  7603200