aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRonald S. Bultje <rsbultje@gmail.com>2015-10-23 11:11:53 -0400
committerCarl Eugen Hoyos <cehoyos@ag.or.at>2015-10-25 01:36:56 +0200
commitbeaf2728cea1d832a50c5b91cc02aff1d79cf153 (patch)
treed44cb1379baf52bf21eba7f47eb25f81fbe38676
parent3ae3708246ec9ca46580b8ea40fd01bca31c97b3 (diff)
downloadffmpeg-beaf2728cea1d832a50c5b91cc02aff1d79cf153.tar.gz
videodsp: don't overread edges in vfix3 emu_edge.
Fixes trac ticket 3226. Also see Andreas' analysis in https://bugs.debian.org/801745, which was very helpful. (cherry picked from commit 52f84d82bdf1851ecfcc412c1719e5f6f3396209)
-rw-r--r--libavcodec/x86/videodsp.asm15
1 files changed, 11 insertions, 4 deletions
diff --git a/libavcodec/x86/videodsp.asm b/libavcodec/x86/videodsp.asm
index 1ac02574d6..77189fa6ef 100644
--- a/libavcodec/x86/videodsp.asm
+++ b/libavcodec/x86/videodsp.asm
@@ -185,8 +185,12 @@ hvar_fn
%elif (%2-%%off) == 2
mov valw, [srcq+%2-2]
%elifidn %1, body
- mov vald, [srcq+%2-3]
-%else
+ mov valb, [srcq+%2-1]
+ sal vald, 16
+ mov valw, [srcq+%2-3]
+%elifidn %1, bottom
+ movd mm %+ %%mmx_idx, [srcq+%2-4]
+%else ; top
movd mm %+ %%mmx_idx, [srcq+%2-3]
%endif
%endif ; (%2-%%off) >= 1
@@ -242,12 +246,15 @@ hvar_fn
mov [dstq+%2-2], valw
%elifidn %1, body
mov [dstq+%2-3], valw
- shr vald, 16
+ sar vald, 16
mov [dstq+%2-1], valb
%else
movd vald, mm %+ %%mmx_idx
+%ifidn %1, bottom
+ sar vald, 8
+%endif
mov [dstq+%2-3], valw
- shr vald, 16
+ sar vald, 16
mov [dstq+%2-1], valb
%endif
%endif ; (%2-%%off) >= 1