diff options
| author | Michael Niedermayer <michael@niedermayer.cc> | 2017-05-07 14:12:04 +0200 |
|---|---|---|
| committer | Michael Niedermayer <michael@niedermayer.cc> | 2017-05-14 00:21:49 +0200 |
| commit | 3f7a9eef516538db825497da198bd41d05e8cbe6 (patch) | |
| tree | 0856806ceda1db1251031632cc12c3396318d30d | |
| parent | eec20b665a9e86d3d6824f0a01fb49378243829c (diff) | |
| download | ffmpeg-3f7a9eef516538db825497da198bd41d05e8cbe6.tar.gz | |
avcodec/mss34dsp: Fix multiple signed integer overflow
Fixes: 1387/clusterfuzz-testcase-minimized-4802757766676480
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 464c4b86ee43b7912e6f23fd3e5ba40381b4c371)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
| -rw-r--r-- | libavcodec/mss34dsp.c | 44 |
1 files changed, 22 insertions, 22 deletions
diff --git a/libavcodec/mss34dsp.c b/libavcodec/mss34dsp.c index 36e69db8cc..f3405658f7 100644 --- a/libavcodec/mss34dsp.c +++ b/libavcodec/mss34dsp.c @@ -62,30 +62,30 @@ void ff_mss34_gen_quant_mat(uint16_t *qmat, int quality, int luma) } #define DCT_TEMPLATE(blk, step, SOP, shift) \ - const int t0 = -39409 * blk[7 * step] - 58980 * blk[1 * step]; \ - const int t1 = 39410 * blk[1 * step] - 58980 * blk[7 * step]; \ - const int t2 = -33410 * blk[5 * step] - 167963 * blk[3 * step]; \ - const int t3 = 33410 * blk[3 * step] - 167963 * blk[5 * step]; \ - const int t4 = blk[3 * step] + blk[7 * step]; \ - const int t5 = blk[1 * step] + blk[5 * step]; \ - const int t6 = 77062 * t4 + 51491 * t5; \ - const int t7 = 77062 * t5 - 51491 * t4; \ - const int t8 = 35470 * blk[2 * step] - 85623 * blk[6 * step]; \ - const int t9 = 35470 * blk[6 * step] + 85623 * blk[2 * step]; \ - const int tA = SOP(blk[0 * step] - blk[4 * step]); \ - const int tB = SOP(blk[0 * step] + blk[4 * step]); \ + const unsigned t0 =-39409U * blk[7 * step] - 58980U * blk[1 * step]; \ + const unsigned t1 = 39410U * blk[1 * step] - 58980U * blk[7 * step]; \ + const unsigned t2 =-33410U * blk[5 * step] -167963U * blk[3 * step]; \ + const unsigned t3 = 33410U * blk[3 * step] -167963U * blk[5 * step]; \ + const unsigned t4 = blk[3 * step] + blk[7 * step]; \ + const unsigned t5 = blk[1 * step] + blk[5 * step]; \ + const unsigned t6 = 77062U * t4 + 51491U * t5; \ + const unsigned t7 = 77062U * t5 - 51491U * t4; \ + const unsigned t8 = 35470U * blk[2 * step] - 85623U * blk[6 * step]; \ + const unsigned t9 = 35470U * blk[6 * step] + 85623U * blk[2 * step]; \ + const unsigned tA = SOP(blk[0 * step] - blk[4 * step]); \ + const unsigned tB = SOP(blk[0 * step] + blk[4 * step]); \ \ - blk[0 * step] = ( t1 + t6 + t9 + tB) >> shift; \ - blk[1 * step] = ( t3 + t7 + t8 + tA) >> shift; \ - blk[2 * step] = ( t2 + t6 - t8 + tA) >> shift; \ - blk[3 * step] = ( t0 + t7 - t9 + tB) >> shift; \ - blk[4 * step] = (-(t0 + t7) - t9 + tB) >> shift; \ - blk[5 * step] = (-(t2 + t6) - t8 + tA) >> shift; \ - blk[6 * step] = (-(t3 + t7) + t8 + tA) >> shift; \ - blk[7 * step] = (-(t1 + t6) + t9 + tB) >> shift; \ + blk[0 * step] = (int)( t1 + t6 + t9 + tB) >> shift; \ + blk[1 * step] = (int)( t3 + t7 + t8 + tA) >> shift; \ + blk[2 * step] = (int)( t2 + t6 - t8 + tA) >> shift; \ + blk[3 * step] = (int)( t0 + t7 - t9 + tB) >> shift; \ + blk[4 * step] = (int)(-(t0 + t7) - t9 + tB) >> shift; \ + blk[5 * step] = (int)(-(t2 + t6) - t8 + tA) >> shift; \ + blk[6 * step] = (int)(-(t3 + t7) + t8 + tA) >> shift; \ + blk[7 * step] = (int)(-(t1 + t6) + t9 + tB) >> shift; \ -#define SOP_ROW(a) (((a) << 16) + 0x2000) -#define SOP_COL(a) (((a) + 32) << 16) +#define SOP_ROW(a) (((a) * (1U << 16)) + 0x2000) +#define SOP_COL(a) (((a) + 32) * (1U << 16)) void ff_mss34_dct_put(uint8_t *dst, ptrdiff_t stride, int *block) { |