diff options
| author | Michael Niedermayer <michael@niedermayer.cc> | 2015-12-01 13:32:31 +0100 |
|---|---|---|
| committer | Michael Niedermayer <michael@niedermayer.cc> | 2015-12-14 16:51:00 +0100 |
| commit | 86a52988bd50b16d7f4c52e610de00d5354c5174 (patch) | |
| tree | f3f437c539cbaf8b0316f128cf967cc5bfb493a9 | |
| parent | d259a0534ee2f89b103cfc52b988b452c6411662 (diff) | |
| download | ffmpeg-86a52988bd50b16d7f4c52e610de00d5354c5174.tar.gz | |
avutil/mathematics: return INT64_MIN (=AV_NOPTS_VALUE) from av_rescale_rnd() for overflows
Fixes integer overflow
Fixes: mozilla bug 1229167
Found-by: Tyson Smith
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit f03c2ceec174877e03bb302f5971fbe9ffbe4856)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
| -rw-r--r-- | libavutil/mathematics.c | 13 |
1 files changed, 10 insertions, 3 deletions
diff --git a/libavutil/mathematics.c b/libavutil/mathematics.c index b1ffd652de..4d8467b8c8 100644 --- a/libavutil/mathematics.c +++ b/libavutil/mathematics.c @@ -77,7 +77,7 @@ int64_t av_rescale_rnd(int64_t a, int64_t b, int64_t c, enum AVRounding rnd) } if (a < 0) - return -av_rescale_rnd(-FFMAX(a, -INT64_MAX), b, c, rnd ^ ((rnd >> 1) & 1)); + return -(uint64_t)av_rescale_rnd(-FFMAX(a, -INT64_MAX), b, c, rnd ^ ((rnd >> 1) & 1)); if (rnd == AV_ROUND_NEAR_INF) r = c / 2; @@ -87,8 +87,13 @@ int64_t av_rescale_rnd(int64_t a, int64_t b, int64_t c, enum AVRounding rnd) if (b <= INT_MAX && c <= INT_MAX) { if (a <= INT_MAX) return (a * b + r) / c; - else - return a / c * b + (a % c * b + r) / c; + else { + int64_t ad = a / c; + int64_t a2 = (a % c * b + r) / c; + if (ad >= INT32_MAX && ad > (INT64_MAX - a2) / b) + return INT64_MIN; + return ad * b + a2; + } } else { #if 1 uint64_t a0 = a & 0xFFFFFFFF; @@ -112,6 +117,8 @@ int64_t av_rescale_rnd(int64_t a, int64_t b, int64_t c, enum AVRounding rnd) t1++; } } + if (t1 > INT64_MAX) + return INT64_MIN; return t1; } #else |