diff options
| author | Michael Niedermayer <michael@niedermayer.cc> | 2017-05-18 17:13:18 +0200 |
|---|---|---|
| committer | Michael Niedermayer <michael@niedermayer.cc> | 2017-05-20 03:41:34 +0200 |
| commit | 1397c7f6160b0e6fbca7c1949fb5007dc9d846f9 (patch) | |
| tree | 27c2d03b3b74b49e0f19dec5e3b6a1b897a5742d | |
| parent | 346e2d16639c229ca9601b47a4ee25e3dd2e6c75 (diff) | |
| download | ffmpeg-1397c7f6160b0e6fbca7c1949fb5007dc9d846f9.tar.gz | |
avcodec/fic: Fix multiple runtime error: signed integer overflow: 5793 * 419752 cannot be represented in type 'int'
Fixes: 1669/clusterfuzz-testcase-minimized-5287529198649344
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit a173f484b52ed63292439de5347e49bd78cad0ed)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
| -rw-r--r-- | libavcodec/fic.c | 32 |
1 files changed, 16 insertions, 16 deletions
diff --git a/libavcodec/fic.c b/libavcodec/fic.c index 120188ccd0..2fa5c48b00 100644 --- a/libavcodec/fic.c +++ b/libavcodec/fic.c @@ -88,22 +88,22 @@ static av_always_inline void fic_idct(int16_t *blk, int step, int shift, int rnd const int t1 = 27246 * blk[5 * step] - 18405 * blk[3 * step]; const int t2 = 6393 * blk[7 * step] + 32139 * blk[1 * step]; const int t3 = 6393 * blk[1 * step] - 32139 * blk[7 * step]; - const int t4 = 5793 * (t2 + t0 + 0x800 >> 12); - const int t5 = 5793 * (t3 + t1 + 0x800 >> 12); - const int t6 = t2 - t0; - const int t7 = t3 - t1; - const int t8 = 17734 * blk[2 * step] - 42813 * blk[6 * step]; - const int t9 = 17734 * blk[6 * step] + 42814 * blk[2 * step]; - const int tA = (blk[0 * step] - blk[4 * step]) * 32768 + rnd; - const int tB = (blk[0 * step] + blk[4 * step]) * 32768 + rnd; - blk[0 * step] = ( t4 + t9 + tB) >> shift; - blk[1 * step] = ( t6 + t7 + t8 + tA) >> shift; - blk[2 * step] = ( t6 - t7 - t8 + tA) >> shift; - blk[3 * step] = ( t5 - t9 + tB) >> shift; - blk[4 * step] = ( -t5 - t9 + tB) >> shift; - blk[5 * step] = (-(t6 - t7) - t8 + tA) >> shift; - blk[6 * step] = (-(t6 + t7) + t8 + tA) >> shift; - blk[7 * step] = ( -t4 + t9 + tB) >> shift; + const unsigned t4 = 5793U * (t2 + t0 + 0x800 >> 12); + const unsigned t5 = 5793U * (t3 + t1 + 0x800 >> 12); + const unsigned t6 = t2 - t0; + const unsigned t7 = t3 - t1; + const unsigned t8 = 17734 * blk[2 * step] - 42813 * blk[6 * step]; + const unsigned t9 = 17734 * blk[6 * step] + 42814 * blk[2 * step]; + const unsigned tA = (blk[0 * step] - blk[4 * step]) * 32768 + rnd; + const unsigned tB = (blk[0 * step] + blk[4 * step]) * 32768 + rnd; + blk[0 * step] = (int)( t4 + t9 + tB) >> shift; + blk[1 * step] = (int)( t6 + t7 + t8 + tA) >> shift; + blk[2 * step] = (int)( t6 - t7 - t8 + tA) >> shift; + blk[3 * step] = (int)( t5 - t9 + tB) >> shift; + blk[4 * step] = (int)( -t5 - t9 + tB) >> shift; + blk[5 * step] = (int)(-(t6 - t7) - t8 + tA) >> shift; + blk[6 * step] = (int)(-(t6 + t7) + t8 + tA) >> shift; + blk[7 * step] = (int)( -t4 + t9 + tB) >> shift; } static void fic_idct_put(uint8_t *dst, int stride, int16_t *block) |