diff options
| author | Andreas Rheinhardt <andreas.rheinhardt@gmail.com> | 2020-01-21 01:23:47 +0100 |
|---|---|---|
| committer | Michael Niedermayer <michael@niedermayer.cc> | 2020-01-21 15:56:39 +0100 |
| commit | 39b60359dbf7936eb1ef6a05b76ea7d821d115f5 (patch) | |
| tree | 485b016929c4153327210c216f162dfe1bde3644 /libavcodec | |
| parent | c07565627b5fe5f27f0fb13a2f3eed03cebb85ec (diff) | |
| download | ffmpeg-39b60359dbf7936eb1ef6a05b76ea7d821d115f5.tar.gz | |
avcodec/j2kenc: Fix undefined shifts of negative numbers
Also add parentheses to some lines to make the operator precedence
clearer.
This affected the FATE-tests vsynth*-jpeg2000 and vsynth*-jpeg2000-97
(where * ranges over { 1, 2, 3, _lena }) as well as ticket #7983.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
Diffstat (limited to 'libavcodec')
| -rw-r--r-- | libavcodec/j2kenc.c | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c index e91d932bb7..38643c9a28 100644 --- a/libavcodec/j2kenc.c +++ b/libavcodec/j2kenc.c @@ -521,13 +521,13 @@ static void init_luts(void) mask = ~((1<<NMSEDEC_FRACBITS)-1); for (i = 0; i < (1 << NMSEDEC_BITS); i++){ - lut_nmsedec_sig[i] = FFMAX(6*i - (9<<NMSEDEC_FRACBITS-1) << 12-NMSEDEC_FRACBITS, 0); + lut_nmsedec_sig[i] = FFMAX((3 * i << (13 - NMSEDEC_FRACBITS)) - (9 << 11), 0); lut_nmsedec_sig0[i] = FFMAX((i*i + (1<<NMSEDEC_FRACBITS-1) & mask) << 1, 0); a = (i >> (NMSEDEC_BITS-2)&2) + 1; - lut_nmsedec_ref[i] = FFMAX((-2*i + (1<<NMSEDEC_FRACBITS) + a*i - (a*a<<NMSEDEC_FRACBITS-2)) - << 13-NMSEDEC_FRACBITS, 0); - lut_nmsedec_ref0[i] = FFMAX(((i*i + (1-4*i << NMSEDEC_FRACBITS-1) + (1<<2*NMSEDEC_FRACBITS)) & mask) + lut_nmsedec_ref[i] = FFMAX((a - 2) * (i << (13 - NMSEDEC_FRACBITS)) + + (1 << 13) - (a * a << 11), 0); + lut_nmsedec_ref0[i] = FFMAX(((i * i - (i << NMSEDEC_BITS) + (1 << 2 * NMSEDEC_FRACBITS) + (1 << (NMSEDEC_FRACBITS - 1))) & mask) << 1, 0); } } @@ -927,7 +927,7 @@ static int encode_tile(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile, int tileno for (y = yy0; y < yy1; y++){ int *ptr = t1.data + (y-yy0)*t1.stride; for (x = xx0; x < xx1; x++){ - *ptr++ = comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x] << NMSEDEC_FRACBITS; + *ptr++ = comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x] * (1 << NMSEDEC_FRACBITS); } } } else{ |