diff options
| author | Michael Niedermayer <michael@niedermayer.cc> | 2024-06-19 20:58:05 +0200 |
|---|---|---|
| committer | Michael Niedermayer <michael@niedermayer.cc> | 2024-07-16 19:03:50 +0200 |
| commit | a84fbd747119841942c67d2f55cc796ab25cd245 (patch) | |
| tree | 6172fa6b69468118f466e17a6ff528e27ab503b1 /libavcodec/j2kenc.c | |
| parent | 8a4e5e0e15fcc950edc2f4eba928b299811a212f (diff) | |
| download | ffmpeg-a84fbd747119841942c67d2f55cc796ab25cd245.tar.gz | |
avcodec/j2kenc: Merge dwt_norm into lambda
This moves computations out of a loop
This may help with UB in vsynth*-jpeg2000-yuva444p16
Fixes: signed integer overflow: 31665934879948800 * 9998 cannot be represented in type 'long'
Fixes: 69024/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_JPEG2000_fuzzer-5949662967169024
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
Diffstat (limited to 'libavcodec/j2kenc.c')
| -rw-r--r-- | libavcodec/j2kenc.c | 9 |
1 files changed, 5 insertions, 4 deletions
diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c index 8cf82f7216..91e66d8104 100644 --- a/libavcodec/j2kenc.c +++ b/libavcodec/j2kenc.c @@ -1349,7 +1349,7 @@ static void makelayers(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) } } -static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm) +static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda) { int passno, res = 0; for (passno = 0; passno < cblk->npasses; passno++){ @@ -1361,7 +1361,7 @@ static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm) dd = cblk->passes[passno].disto - (res ? cblk->passes[res-1].disto : 0); - if (((dd * dwt_norm) >> WMSEDEC_SHIFT) * dwt_norm >= dr * lambda) + if (dd >= dr * lambda) res = passno+1; } return res; @@ -1384,11 +1384,12 @@ static void truncpasses(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) Jpeg2000Band *band = reslevel->band + bandno; Jpeg2000Prec *prec = band->prec + precno; + int64_t dwt_norm = dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15; + int64_t lambda_prime = av_rescale(s->lambda, 1 << WMSEDEC_SHIFT, dwt_norm * dwt_norm); for (cblkno = 0; cblkno < prec->nb_codeblocks_height * prec->nb_codeblocks_width; cblkno++){ Jpeg2000Cblk *cblk = prec->cblk + cblkno; - cblk->ninclpasses = getcut(cblk, s->lambda, - (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15); + cblk->ninclpasses = getcut(cblk, lambda_prime); cblk->layers[0].data_start = cblk->data; cblk->layers[0].cum_passes = cblk->ninclpasses; cblk->layers[0].npasses = cblk->ninclpasses; |