aboutsummaryrefslogtreecommitdiffstats
path: root/libavcodec
diff options
context:
space:
mode:
authorMichael Niedermayer <michael@niedermayer.cc>2017-05-07 14:12:04 +0200
committerMichael Niedermayer <michael@niedermayer.cc>2017-05-07 15:31:00 +0200
commit464c4b86ee43b7912e6f23fd3e5ba40381b4c371 (patch)
treef854dd4b08646f5860c325308fab29c3b74da93a /libavcodec
parentf89a89c5500c57b58b1fb154bf6cf412e92677d6 (diff)
downloadffmpeg-464c4b86ee43b7912e6f23fd3e5ba40381b4c371.tar.gz
avcodec/mss34dsp: Fix multiple signed integer overflow
Fixes: 1387/clusterfuzz-testcase-minimized-4802757766676480 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
Diffstat (limited to 'libavcodec')
-rw-r--r--libavcodec/mss34dsp.c44
1 files changed, 22 insertions, 22 deletions
diff --git a/libavcodec/mss34dsp.c b/libavcodec/mss34dsp.c
index 36e69db8cc..f3405658f7 100644
--- a/libavcodec/mss34dsp.c
+++ b/libavcodec/mss34dsp.c
@@ -62,30 +62,30 @@ void ff_mss34_gen_quant_mat(uint16_t *qmat, int quality, int luma)
}
#define DCT_TEMPLATE(blk, step, SOP, shift) \
- const int t0 = -39409 * blk[7 * step] - 58980 * blk[1 * step]; \
- const int t1 = 39410 * blk[1 * step] - 58980 * blk[7 * step]; \
- const int t2 = -33410 * blk[5 * step] - 167963 * blk[3 * step]; \
- const int t3 = 33410 * blk[3 * step] - 167963 * blk[5 * step]; \
- const int t4 = blk[3 * step] + blk[7 * step]; \
- const int t5 = blk[1 * step] + blk[5 * step]; \
- const int t6 = 77062 * t4 + 51491 * t5; \
- const int t7 = 77062 * t5 - 51491 * t4; \
- const int t8 = 35470 * blk[2 * step] - 85623 * blk[6 * step]; \
- const int t9 = 35470 * blk[6 * step] + 85623 * blk[2 * step]; \
- const int tA = SOP(blk[0 * step] - blk[4 * step]); \
- const int tB = SOP(blk[0 * step] + blk[4 * step]); \
+ const unsigned t0 =-39409U * blk[7 * step] - 58980U * blk[1 * step]; \
+ const unsigned t1 = 39410U * blk[1 * step] - 58980U * blk[7 * step]; \
+ const unsigned t2 =-33410U * blk[5 * step] -167963U * blk[3 * step]; \
+ const unsigned t3 = 33410U * blk[3 * step] -167963U * blk[5 * step]; \
+ const unsigned t4 = blk[3 * step] + blk[7 * step]; \
+ const unsigned t5 = blk[1 * step] + blk[5 * step]; \
+ const unsigned t6 = 77062U * t4 + 51491U * t5; \
+ const unsigned t7 = 77062U * t5 - 51491U * t4; \
+ const unsigned t8 = 35470U * blk[2 * step] - 85623U * blk[6 * step]; \
+ const unsigned t9 = 35470U * blk[6 * step] + 85623U * blk[2 * step]; \
+ const unsigned tA = SOP(blk[0 * step] - blk[4 * step]); \
+ const unsigned tB = SOP(blk[0 * step] + blk[4 * step]); \
\
- blk[0 * step] = ( t1 + t6 + t9 + tB) >> shift; \
- blk[1 * step] = ( t3 + t7 + t8 + tA) >> shift; \
- blk[2 * step] = ( t2 + t6 - t8 + tA) >> shift; \
- blk[3 * step] = ( t0 + t7 - t9 + tB) >> shift; \
- blk[4 * step] = (-(t0 + t7) - t9 + tB) >> shift; \
- blk[5 * step] = (-(t2 + t6) - t8 + tA) >> shift; \
- blk[6 * step] = (-(t3 + t7) + t8 + tA) >> shift; \
- blk[7 * step] = (-(t1 + t6) + t9 + tB) >> shift; \
+ blk[0 * step] = (int)( t1 + t6 + t9 + tB) >> shift; \
+ blk[1 * step] = (int)( t3 + t7 + t8 + tA) >> shift; \
+ blk[2 * step] = (int)( t2 + t6 - t8 + tA) >> shift; \
+ blk[3 * step] = (int)( t0 + t7 - t9 + tB) >> shift; \
+ blk[4 * step] = (int)(-(t0 + t7) - t9 + tB) >> shift; \
+ blk[5 * step] = (int)(-(t2 + t6) - t8 + tA) >> shift; \
+ blk[6 * step] = (int)(-(t3 + t7) + t8 + tA) >> shift; \
+ blk[7 * step] = (int)(-(t1 + t6) + t9 + tB) >> shift; \
-#define SOP_ROW(a) (((a) << 16) + 0x2000)
-#define SOP_COL(a) (((a) + 32) << 16)
+#define SOP_ROW(a) (((a) * (1U << 16)) + 0x2000)
+#define SOP_COL(a) (((a) + 32) * (1U << 16))
void ff_mss34_dct_put(uint8_t *dst, ptrdiff_t stride, int *block)
{