aboutsummaryrefslogtreecommitdiffstats
path: root/libavcodec
diff options
context:
space:
mode:
authorMichael Niedermayer <michael@niedermayer.cc>2019-07-02 12:30:32 +0200
committerMichael Niedermayer <michael@niedermayer.cc>2019-07-31 20:35:07 +0200
commit1aad8937f73f9c8cc337e3173402f47175d077e7 (patch)
tree467bb84164be5d538e3ede9e49ffca5a02cfc55f /libavcodec
parentbf778af1493b0814696307432763246fb53c75e7 (diff)
downloadffmpeg-1aad8937f73f9c8cc337e3173402f47175d077e7.tar.gz
avcodec/apedec: Make coeffsA/B uint32_t, this avoids several cases of undefined behavior
Changing the type to an unsigned one to avoid many casts was suggested This may be inadequate for fixing the UB on ILP64 Fixes: signed integer overflow: -1418162611 * 383 cannot be represented in type 'int' Fixes: 15547/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_APE_fuzzer-5691384901664768 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
Diffstat (limited to 'libavcodec')
-rw-r--r--libavcodec/apedec.c6
1 files changed, 3 insertions, 3 deletions
diff --git a/libavcodec/apedec.c b/libavcodec/apedec.c
index 7a7097e7a4..e9ffdfdcdf 100644
--- a/libavcodec/apedec.c
+++ b/libavcodec/apedec.c
@@ -125,8 +125,8 @@ typedef struct APEPredictor {
int32_t filterA[2];
int32_t filterB[2];
- int32_t coeffsA[2][4]; ///< adaption coefficients
- int32_t coeffsB[2][5]; ///< adaption coefficients
+ uint32_t coeffsA[2][4]; ///< adaption coefficients
+ uint32_t coeffsB[2][5]; ///< adaption coefficients
int32_t historybuffer[HISTORY_SIZE + PREDICTOR_SIZE];
unsigned int sample_pos;
@@ -829,7 +829,7 @@ static av_always_inline int filter_fast_3320(APEPredictor *p,
}
predictionA = p->buf[delayA] * 2 - p->buf[delayA - 1];
- p->lastA[filter] = decoded + (predictionA * p->coeffsA[filter][0] >> 9);
+ p->lastA[filter] = decoded + ((int32_t)(predictionA * p->coeffsA[filter][0]) >> 9);
if ((decoded ^ predictionA) > 0)
p->coeffsA[filter][0]++;