diff options
| author | Carl Eugen Hoyos <ceffmpeg@gmail.com> | 2022-09-03 22:50:19 +0200 |
|---|---|---|
| committer | Carl Eugen Hoyos <ceffmpeg@gmail.com> | 2022-10-10 02:58:39 +0200 |
| commit | 82479ef6bd107312aa086bab12c29ba4551d544a (patch) | |
| tree | c6140e2fa134a8849b0e9630bcb240afce567073 /libavfilter/vf_rotate.c | |
| parent | 60e87faf7f32a84a0aecf9a2969c7ce89b5e5f29 (diff) | |
| download | ffmpeg-82479ef6bd107312aa086bab12c29ba4551d544a.tar.gz | |
lavfi/rotate: Avoid undefined behaviour.
Fixes the following integer overflows:
libavfilter/vf_rotate.c:273:13: runtime error: signed integer overflow: 92951468 + 2058533568 cannot be represented in type 'int'
libavfilter/vf_rotate.c:273:37: runtime error: signed integer overflow: 39684 * 54149 cannot be represented in type 'int'
libavfilter/vf_rotate.c:272:13: runtime error: signed integer overflow: 247587320 + 1900985032 cannot be represented in type 'int'
libavfilter/vf_rotate.c:272:37: runtime error: signed integer overflow: 42584 * 50430 cannot be represented in type 'int'
libavfilter/vf_rotate.c:272:50: runtime error: signed integer overflow: 65083 * 52912 cannot be represented in type 'int'
libavfilter/vf_rotate.c:273:50: runtime error: signed integer overflow: 65286 * 38044 cannot be represented in type 'int'
Fixes ticket #9799, different output with different compilers.
Diffstat (limited to 'libavfilter/vf_rotate.c')
| -rw-r--r-- | libavfilter/vf_rotate.c | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/libavfilter/vf_rotate.c b/libavfilter/vf_rotate.c index 4429e3d543..96c250a459 100644 --- a/libavfilter/vf_rotate.c +++ b/libavfilter/vf_rotate.c @@ -258,8 +258,8 @@ static uint8_t *interpolate_bilinear16(uint8_t *dst_color, { int int_x = av_clip(x>>16, 0, max_x); int int_y = av_clip(y>>16, 0, max_y); - int frac_x = x&0xFFFF; - int frac_y = y&0xFFFF; + int64_t frac_x = x&0xFFFF; + int64_t frac_y = y&0xFFFF; int i; int int_x1 = FFMIN(int_x+1, max_x); int int_y1 = FFMIN(int_y+1, max_y); @@ -269,10 +269,10 @@ static uint8_t *interpolate_bilinear16(uint8_t *dst_color, int s01 = AV_RL16(&src[src_linestep * int_x1 + i + src_linesize * int_y ]); int s10 = AV_RL16(&src[src_linestep * int_x + i + src_linesize * int_y1]); int s11 = AV_RL16(&src[src_linestep * int_x1 + i + src_linesize * int_y1]); - int s0 = (((1<<16) - frac_x)*s00 + frac_x*s01); - int s1 = (((1<<16) - frac_x)*s10 + frac_x*s11); + int64_t s0 = (((1<<16) - frac_x)*s00 + frac_x*s01); + int64_t s1 = (((1<<16) - frac_x)*s10 + frac_x*s11); - AV_WL16(&dst_color[i], ((int64_t)((1<<16) - frac_y)*s0 + (int64_t)frac_y*s1) >> 32); + AV_WL16(&dst_color[i], (((1<<16) - frac_y)*s0 + frac_y*s1) >> 32); } return dst_color; |