diff options
| author | John Brooks <john.brooks@bluecherry.net> | 2011-12-11 02:37:21 -0700 |
|---|---|---|
| committer | Michael Niedermayer <michaelni@gmx.at> | 2012-01-05 02:23:08 +0100 |
| commit | e540446f6c5ca1613eea4f9e8c5db12f1ec3925c (patch) | |
| tree | 8be072d9e5de3639f935b763afe6d868561c2479 /libavcodec | |
| parent | fc9c5ad9ea70479d09a5d37c800665746ff4710d (diff) | |
| download | ffmpeg-e540446f6c5ca1613eea4f9e8c5db12f1ec3925c.tar.gz | |
vc1dec: fix invalid memory access for small video dimensions
For small video dimensions, these calculations of the upper bound
for pixel access may have a negative result. Using an unsigned
comparison to bound a potentially negative value only works if
the greater operand is non-negative. Fixed by doing edge emulation
when the upper bound is probably negative, everywhere that this
pattern appears.
Signed-off-by: Kostya Shishkov <kostya.shishkov@gmail.com>
(cherry picked from commit d209c27b09234cc40bbdbd680aa502b493edf595)
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
Diffstat (limited to 'libavcodec')
| -rw-r--r-- | libavcodec/vc1dec.c | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/libavcodec/vc1dec.c b/libavcodec/vc1dec.c index 20b7f72291..219f8dd468 100644 --- a/libavcodec/vc1dec.c +++ b/libavcodec/vc1dec.c @@ -568,6 +568,7 @@ static void vc1_mc_1mv(VC1Context *v, int dir) } if (v->rangeredfrm || (v->mv_mode == MV_PMODE_INTENSITY_COMP) + || s->h_edge_pos < 22 || v_edge_pos < 22 || (unsigned)(src_x - s->mspel) > s->h_edge_pos - (mx&3) - 16 - s->mspel * 3 || (unsigned)(src_y - s->mspel) > v_edge_pos - (my&3) - 16 - s->mspel * 3) { uint8_t *uvbuf = s->edge_emu_buffer + 19 * s->linesize; @@ -799,6 +800,7 @@ static void vc1_mc_4mv_luma(VC1Context *v, int n, int dir) if (fieldmv && (src_y & 1) && src_y < 4) src_y--; if (v->rangeredfrm || (v->mv_mode == MV_PMODE_INTENSITY_COMP) + || s->h_edge_pos < 13 || v_edge_pos < 23 || (unsigned)(src_x - s->mspel) > s->h_edge_pos - (mx & 3) - 8 - s->mspel * 2 || (unsigned)(src_y - (s->mspel << fieldmv)) > v_edge_pos - (my & 3) - ((8 + s->mspel * 2) << fieldmv)) { srcY -= s->mspel * (1 + (s->linesize << fieldmv)); @@ -998,6 +1000,7 @@ static void vc1_mc_4mv_chroma(VC1Context *v, int dir) } if (v->rangeredfrm || (v->mv_mode == MV_PMODE_INTENSITY_COMP) + || s->h_edge_pos < 18 || v_edge_pos < 18 || (unsigned)uvsrc_x > (s->h_edge_pos >> 1) - 9 || (unsigned)uvsrc_y > (v_edge_pos >> 1) - 9) { s->dsp.emulated_edge_mc(s->edge_emu_buffer , srcU, s->uvlinesize, @@ -1102,6 +1105,7 @@ static void vc1_mc_4mv_chroma4(VC1Context *v) if (fieldmv && (uvsrc_y & 1) && uvsrc_y < 2) uvsrc_y--; if ((v->mv_mode == MV_PMODE_INTENSITY_COMP) + || s->h_edge_pos < 10 || v_edge_pos < (5 << fieldmv) || (unsigned)uvsrc_x > (s->h_edge_pos >> 1) - 5 || (unsigned)uvsrc_y > v_edge_pos - (5 << fieldmv)) { s->dsp.emulated_edge_mc(s->edge_emu_buffer, srcU, s->uvlinesize, @@ -2006,7 +2010,7 @@ static void vc1_interp_mc(VC1Context *v) srcV = s->edge_emu_buffer + 18 * s->linesize; } - if (v->rangeredfrm + if (v->rangeredfrm || s->h_edge_pos < 22 || v_edge_pos < 22 || (unsigned)(src_x - s->mspel) > s->h_edge_pos - (mx & 3) - 16 - s->mspel * 3 || (unsigned)(src_y - s->mspel) > v_edge_pos - (my & 3) - 16 - s->mspel * 3) { uint8_t *uvbuf = s->edge_emu_buffer + 19 * s->linesize; |