diff options
| author | Michael Niedermayer <michaelni@gmx.at> | 2013-07-13 02:24:56 +0200 |
|---|---|---|
| committer | Michael Niedermayer <michaelni@gmx.at> | 2013-07-13 12:44:37 +0200 |
| commit | f57119b8e58cb5437c3ab40d797293ecb9b4a894 (patch) | |
| tree | 1c621704c0e7755bc05170655841595012f978f1 | |
| parent | dc072c98677fde8d716e20e70b147989aac232e1 (diff) | |
| download | ffmpeg-f57119b8e58cb5437c3ab40d797293ecb9b4a894.tar.gz | |
jpeg2000: fix overflow in dequantization
Fixes decoding of file generated with:
ffmpeg -f lavfi -i smptehdbars=hd720 -pix_fmt rgb48 /tmp/o.jp2
Reviewed-by: Nicolas BERTRAND <nicoinattendu@gmail.com>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
| -rw-r--r-- | libavcodec/j2kenc.c | 4 | ||||
| -rw-r--r-- | libavcodec/jpeg2000.c | 2 | ||||
| -rw-r--r-- | libavcodec/jpeg2000dec.c | 2 |
3 files changed, 4 insertions, 4 deletions
diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c index 48c1db8071..fa15aad120 100644 --- a/libavcodec/j2kenc.c +++ b/libavcodec/j2kenc.c @@ -802,7 +802,7 @@ static void truncpasses(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) Jpeg2000Cblk *cblk = prec->cblk + cblkno; cblk->ninclpasses = getcut(cblk, s->lambda, - (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 16); + (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15); } } } @@ -863,7 +863,7 @@ static int encode_tile(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile, int tileno int *ptr = t1.data[y-yy0]; for (x = xx0; x < xx1; x++){ *ptr = (comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x]); - *ptr = (int64_t)*ptr * (int64_t)(16384 * 65536 / band->i_stepsize) >> 14 - NMSEDEC_FRACBITS; + *ptr = (int64_t)*ptr * (int64_t)(16384 * 65536 / band->i_stepsize) >> 15 - NMSEDEC_FRACBITS; ptr++; } } diff --git a/libavcodec/jpeg2000.c b/libavcodec/jpeg2000.c index f044164ed6..cb1a64d143 100644 --- a/libavcodec/jpeg2000.c +++ b/libavcodec/jpeg2000.c @@ -320,7 +320,7 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp, if (!av_codec_is_encoder(avctx->codec)) band->f_stepsize *= 0.5; - band->i_stepsize = band->f_stepsize * (1 << 16); + band->i_stepsize = band->f_stepsize * (1 << 15); /* computation of tbx_0, tbx_1, tby_0, tby_1 * see ISO/IEC 15444-1:2002 B.5 eq. B-15 and tbl B.1 diff --git a/libavcodec/jpeg2000dec.c b/libavcodec/jpeg2000dec.c index e2a9a95f3e..d9b9888382 100644 --- a/libavcodec/jpeg2000dec.c +++ b/libavcodec/jpeg2000dec.c @@ -1024,7 +1024,7 @@ static void dequantization_int(int x, int y, Jpeg2000Cblk *cblk, int32_t *datap = &comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * (y + j) + x]; int *src = t1->data[j]; for (i = 0; i < w; ++i) - datap[i] = (src[i] * band->i_stepsize + (1 << 15)) >> 16; + datap[i] = (src[i] * band->i_stepsize + (1 << 14)) >> 15; } } |