diff options
| author | Michael Niedermayer <michael@niedermayer.cc> | 2017-11-05 21:20:06 +0100 |
|---|---|---|
| committer | Michael Niedermayer <michael@niedermayer.cc> | 2017-12-02 00:28:59 +0100 |
| commit | f2763b8ba80cb258663807a5e61f028564b02ac3 (patch) | |
| tree | cd5218ca4c2bc09c9214c63c76c2e8cb74be34aa /libavcodec | |
| parent | f33f13172cfe34b3bedb59964957d8d784f9cea0 (diff) | |
| download | ffmpeg-f2763b8ba80cb258663807a5e61f028564b02ac3.tar.gz | |
avcodec/mdct_*: Fix integer overflow in addition in RESCALE()
Fixes: runtime error: signed integer overflow: 1219998458 - -1469874012 cannot be represented in type 'int'
Fixes: 3443/clusterfuzz-testcase-minimized-5369987105554432
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 770c934fa1635f4fadf5db4fc5cc5ad15d82455a)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
Diffstat (limited to 'libavcodec')
| -rw-r--r-- | libavcodec/mdct_fixed.c | 8 | ||||
| -rw-r--r-- | libavcodec/mdct_template.c | 14 |
2 files changed, 11 insertions, 11 deletions
diff --git a/libavcodec/mdct_fixed.c b/libavcodec/mdct_fixed.c index a32cb00ca0..aabf0c88f8 100644 --- a/libavcodec/mdct_fixed.c +++ b/libavcodec/mdct_fixed.c @@ -39,13 +39,13 @@ void ff_mdct_calcw_c(FFTContext *s, FFTDouble *out, const FFTSample *input) /* pre rotation */ for(i=0;i<n8;i++) { - re = RSCALE(-input[2*i+n3] - input[n3-1-2*i]); - im = RSCALE(-input[n4+2*i] + input[n4-1-2*i]); + re = RSCALE(-input[2*i+n3], - input[n3-1-2*i]); + im = RSCALE(-input[n4+2*i], + input[n4-1-2*i]); j = revtab[i]; CMUL(x[j].re, x[j].im, re, im, -tcos[i], tsin[i]); - re = RSCALE( input[2*i] - input[n2-1-2*i]); - im = RSCALE(-input[n2+2*i] - input[ n-1-2*i]); + re = RSCALE( input[2*i] , - input[n2-1-2*i]); + im = RSCALE(-input[n2+2*i], - input[ n-1-2*i]); j = revtab[n8 + i]; CMUL(x[j].re, x[j].im, re, im, -tcos[n8 + i], tsin[n8 + i]); } diff --git a/libavcodec/mdct_template.c b/libavcodec/mdct_template.c index 04396b46d6..e0ad9f1e53 100644 --- a/libavcodec/mdct_template.c +++ b/libavcodec/mdct_template.c @@ -33,12 +33,12 @@ */ #if FFT_FLOAT -# define RSCALE(x) (x) +# define RSCALE(x, y) ((x) + (y)) #else #if FFT_FIXED_32 -# define RSCALE(x) (((x) + 32) >> 6) +# define RSCALE(x, y) ((int)((x) + (unsigned)(y) + 32) >> 6) #else /* FFT_FIXED_32 */ -# define RSCALE(x) ((x) >> 1) +# define RSCALE(x, y) ((int)((x) + (unsigned)(y)) >> 1) #endif /* FFT_FIXED_32 */ #endif @@ -181,13 +181,13 @@ void ff_mdct_calc_c(FFTContext *s, FFTSample *out, const FFTSample *input) /* pre rotation */ for(i=0;i<n8;i++) { - re = RSCALE(-input[2*i+n3] - input[n3-1-2*i]); - im = RSCALE(-input[n4+2*i] + input[n4-1-2*i]); + re = RSCALE(-input[2*i+n3], - input[n3-1-2*i]); + im = RSCALE(-input[n4+2*i], + input[n4-1-2*i]); j = revtab[i]; CMUL(x[j].re, x[j].im, re, im, -tcos[i], tsin[i]); - re = RSCALE( input[2*i] - input[n2-1-2*i]); - im = RSCALE(-input[n2+2*i] - input[ n-1-2*i]); + re = RSCALE( input[2*i] , - input[n2-1-2*i]); + im = RSCALE(-input[n2+2*i], - input[ n-1-2*i]); j = revtab[n8 + i]; CMUL(x[j].re, x[j].im, re, im, -tcos[n8 + i], tsin[n8 + i]); } |