aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJohn Adcock <dscaler.johnad@googlemail.com>2009-08-04 07:42:55 +0000
committerJason Garrett-Glaser <darkshikari@gmail.com>2009-08-04 07:42:55 +0000
commit3f87f39cb898932a5dd2847e48d31017ab220b59 (patch)
tree3f7adcd9ec990302c257575af8086338c497041b
parentd8c2f8f71f054d77f51564c216177338137c05dc (diff)
downloadffmpeg-3f87f39cb898932a5dd2847e48d31017ab220b59.tar.gz
Update x264 asm code to latest to add support for 64-bit Windows.
Use the new x86inc features to support 64-bit Windows on all non-x264 nasm assembly code as well. Patch by John Adcock, dscaler.johnad AT googlemail DOT com. Win64 changes originally by Anton Mitrofanov. x86util changes mostly by Holger Lubitz. Originally committed as revision 19580 to svn://svn.ffmpeg.org/ffmpeg/trunk
-rw-r--r--libavcodec/x86/fft_mmx.asm2
-rw-r--r--libavcodec/x86/h264_deblock_sse2.asm86
-rw-r--r--libavcodec/x86/h264_idct_sse2.asm9
-rw-r--r--libavcodec/x86/x86inc.asm245
-rw-r--r--libavcodec/x86/x86util.asm307
5 files changed, 507 insertions, 142 deletions
diff --git a/libavcodec/x86/fft_mmx.asm b/libavcodec/x86/fft_mmx.asm
index 1a9d1894cd..6c1b26d401 100644
--- a/libavcodec/x86/fft_mmx.asm
+++ b/libavcodec/x86/fft_mmx.asm
@@ -457,7 +457,7 @@ section .text
; On x86_32, this function does the register saving and restoring for all of fft.
; The others pass args in registers and don't spill anything.
-cglobal fft_dispatch%3%2, 2,5,0, z, nbits
+cglobal fft_dispatch%3%2, 2,5,8, z, nbits
lea r2, [dispatch_tab%3%2 GLOBAL]
mov r2, [r2 + (nbitsq-2)*gprsize]
call r2
diff --git a/libavcodec/x86/h264_deblock_sse2.asm b/libavcodec/x86/h264_deblock_sse2.asm
index 0c98190ca0..bf45c7ea69 100644
--- a/libavcodec/x86/h264_deblock_sse2.asm
+++ b/libavcodec/x86/h264_deblock_sse2.asm
@@ -278,7 +278,7 @@ SECTION .text
; void x264_deblock_v_luma_sse2( uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0 )
;-----------------------------------------------------------------------------
INIT_XMM
-cglobal x264_deblock_v_luma_sse2
+cglobal x264_deblock_v_luma_sse2, 5,5,10
movd m8, [r4] ; tc0
lea r4, [r1*3]
dec r2d ; alpha-1
@@ -318,54 +318,66 @@ cglobal x264_deblock_v_luma_sse2
DEBLOCK_P0_Q0
mova [r4+2*r1], m1
mova [r0], m2
- ret
+ RET
;-----------------------------------------------------------------------------
; void x264_deblock_h_luma_sse2( uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0 )
;-----------------------------------------------------------------------------
INIT_MMX
-cglobal x264_deblock_h_luma_sse2
- movsxd r10, esi
+cglobal x264_deblock_h_luma_sse2, 5,7
+ movsxd r10, r1d
lea r11, [r10+r10*2]
- lea rax, [r0-4]
- lea r9, [r0-4+r11]
+ lea r6, [r0-4]
+ lea r5, [r0-4+r11]
+%ifdef WIN64
+ sub rsp, 0x98
+ %define pix_tmp rsp+0x30
+%else
sub rsp, 0x68
%define pix_tmp rsp
+%endif
; transpose 6x16 -> tmp space
- TRANSPOSE6x8_MEM PASS8ROWS(rax, r9, r10, r11), pix_tmp
- lea rax, [rax+r10*8]
- lea r9, [r9 +r10*8]
- TRANSPOSE6x8_MEM PASS8ROWS(rax, r9, r10, r11), pix_tmp+8
+ TRANSPOSE6x8_MEM PASS8ROWS(r6, r5, r10, r11), pix_tmp
+ lea r6, [r6+r10*8]
+ lea r5, [r5+r10*8]
+ TRANSPOSE6x8_MEM PASS8ROWS(r6, r5, r10, r11), pix_tmp+8
; vertical filter
; alpha, beta, tc0 are still in r2d, r3d, r4
- ; don't backup rax, r9, r10, r11 because x264_deblock_v_luma_sse2 doesn't use them
+ ; don't backup r6, r5, r10, r11 because x264_deblock_v_luma_sse2 doesn't use them
lea r0, [pix_tmp+0x30]
- mov esi, 0x10
+ mov r1d, 0x10
+%ifdef WIN64
+ mov [rsp+0x20], r4
+%endif
call x264_deblock_v_luma_sse2
; transpose 16x4 -> original space (only the middle 4 rows were changed by the filter)
- add rax, 2
- add r9, 2
+ add r6, 2
+ add r5, 2
movq m0, [pix_tmp+0x18]
movq m1, [pix_tmp+0x28]
movq m2, [pix_tmp+0x38]
movq m3, [pix_tmp+0x48]
- TRANSPOSE8x4_STORE PASS8ROWS(rax, r9, r10, r11)
+ TRANSPOSE8x4_STORE PASS8ROWS(r6, r5, r10, r11)
shl r10, 3
- sub rax, r10
- sub r9, r10
+ sub r6, r10
+ sub r5, r10
shr r10, 3
movq m0, [pix_tmp+0x10]
movq m1, [pix_tmp+0x20]
movq m2, [pix_tmp+0x30]
movq m3, [pix_tmp+0x40]
- TRANSPOSE8x4_STORE PASS8ROWS(rax, r9, r10, r11)
+ TRANSPOSE8x4_STORE PASS8ROWS(r6, r5, r10, r11)
+%ifdef WIN64
+ add rsp, 0x98
+%else
add rsp, 0x68
- ret
+%endif
+ RET
%else
@@ -388,7 +400,7 @@ cglobal x264_deblock_%2_luma_%1, 5,5
mova m3, [r0+r1] ; q1
LOAD_MASK r2, r3
- mov r3, r4m
+ mov r3, r4mp
movd m4, [r3] ; tc0
punpcklbw m4, m4
punpcklbw m4, m4 ; tc = 4x tc0[3], 4x tc0[2], 4x tc0[1], 4x tc0[0]
@@ -428,7 +440,7 @@ cglobal x264_deblock_%2_luma_%1, 5,5
;-----------------------------------------------------------------------------
INIT_MMX
cglobal x264_deblock_h_luma_%1, 0,5
- mov r0, r0m
+ mov r0, r0mp
mov r3, r1m
lea r4, [r3*3]
sub r0, 4
@@ -459,7 +471,7 @@ cglobal x264_deblock_h_luma_%1, 0,5
ADD esp, 20
; transpose 16x4 -> original space (only the middle 4 rows were changed by the filter)
- mov r0, r0m
+ mov r0, r0mp
sub r0, 2
lea r1, [r0+r4]
@@ -607,7 +619,7 @@ DEBLOCK_LUMA sse2, v, 16
;-----------------------------------------------------------------------------
; void x264_deblock_v_luma_intra_sse2( uint8_t *pix, int stride, int alpha, int beta )
;-----------------------------------------------------------------------------
-cglobal x264_deblock_%2_luma_intra_%1, 4,6
+cglobal x264_deblock_%2_luma_intra_%1, 4,6,16
%ifndef ARCH_X86_64
sub esp, 0x60
%endif
@@ -669,34 +681,34 @@ INIT_MMX
;-----------------------------------------------------------------------------
; void x264_deblock_h_luma_intra_sse2( uint8_t *pix, int stride, int alpha, int beta )
;-----------------------------------------------------------------------------
-cglobal x264_deblock_h_luma_intra_%1
+cglobal x264_deblock_h_luma_intra_%1, 4,7
movsxd r10, r1d
lea r11, [r10*3]
- lea rax, [r0-4]
- lea r9, [r0-4+r11]
+ lea r6, [r0-4]
+ lea r5, [r0-4+r11]
sub rsp, 0x88
%define pix_tmp rsp
; transpose 8x16 -> tmp space
- TRANSPOSE8x8_MEM PASS8ROWS(rax, r9, r10, r11), PASS8ROWS(pix_tmp, pix_tmp+0x30, 0x10, 0x30)
- lea rax, [rax+r10*8]
- lea r9, [r9+r10*8]
- TRANSPOSE8x8_MEM PASS8ROWS(rax, r9, r10, r11), PASS8ROWS(pix_tmp+8, pix_tmp+0x38, 0x10, 0x30)
+ TRANSPOSE8x8_MEM PASS8ROWS(r6, r5, r10, r11), PASS8ROWS(pix_tmp, pix_tmp+0x30, 0x10, 0x30)
+ lea r6, [r6+r10*8]
+ lea r5, [r5+r10*8]
+ TRANSPOSE8x8_MEM PASS8ROWS(r6, r5, r10, r11), PASS8ROWS(pix_tmp+8, pix_tmp+0x38, 0x10, 0x30)
lea r0, [pix_tmp+0x40]
mov r1, 0x10
call x264_deblock_v_luma_intra_%1
; transpose 16x6 -> original space (but we can't write only 6 pixels, so really 16x8)
- lea r9, [rax+r11]
- TRANSPOSE8x8_MEM PASS8ROWS(pix_tmp+8, pix_tmp+0x38, 0x10, 0x30), PASS8ROWS(rax, r9, r10, r11)
+ lea r5, [r6+r11]
+ TRANSPOSE8x8_MEM PASS8ROWS(pix_tmp+8, pix_tmp+0x38, 0x10, 0x30), PASS8ROWS(r6, r5, r10, r11)
shl r10, 3
- sub rax, r10
- sub r9, r10
+ sub r6, r10
+ sub r5, r10
shr r10, 3
- TRANSPOSE8x8_MEM PASS8ROWS(pix_tmp, pix_tmp+0x30, 0x10, 0x30), PASS8ROWS(rax, r9, r10, r11)
+ TRANSPOSE8x8_MEM PASS8ROWS(pix_tmp, pix_tmp+0x30, 0x10, 0x30), PASS8ROWS(r6, r5, r10, r11)
add rsp, 0x88
- ret
+ RET
%else
cglobal x264_deblock_h_luma_intra_%1, 2,4
lea r3, [r1*3]
@@ -725,7 +737,7 @@ cglobal x264_deblock_h_luma_intra_%1, 2,4
ADD esp, 16
mov r1, r1m
- mov r0, r0m
+ mov r0, r0mp
lea r3, [r1*3]
sub r0, 4
lea r2, [r0+r3]
diff --git a/libavcodec/x86/h264_idct_sse2.asm b/libavcodec/x86/h264_idct_sse2.asm
index aee8beb36c..f8ee2b6288 100644
--- a/libavcodec/x86/h264_idct_sse2.asm
+++ b/libavcodec/x86/h264_idct_sse2.asm
@@ -31,15 +31,8 @@ pw_32: times 8 dw 32
SECTION .text
-%macro IDCT4_1D 6
- SUMSUB_BA m%3, m%1
- SUMSUBD2_AB m%2, m%4, m%6, m%5
- SUMSUB_BADC m%2, m%3, m%5, m%1
- SWAP %1, %2, %5, %4, %3
-%endmacro
-
INIT_XMM
-cglobal x264_add8x4_idct_sse2, 3,3
+cglobal x264_add8x4_idct_sse2, 3,3,8
movq m0, [r1+ 0]
movq m1, [r1+ 8]
movq m2, [r1+16]
diff --git a/libavcodec/x86/x86inc.asm b/libavcodec/x86/x86inc.asm
index 3729b5b101..52624c3aca 100644
--- a/libavcodec/x86/x86inc.asm
+++ b/libavcodec/x86/x86inc.asm
@@ -20,6 +20,14 @@
;* 51, Inc., Foundation Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;*****************************************************************************
+%ifdef ARCH_X86_64
+ %ifidn __OUTPUT_FORMAT__,win32
+ %define WIN64
+ %else
+ %define UNIX64
+ %endif
+%endif
+
; FIXME: All of the 64bit asm functions that take a stride as an argument
; via register, assume that the high dword of that register is filled with 0.
; This is true in practice (since we never do any 64bit arithmetic on strides,
@@ -28,68 +36,39 @@
; Name of the .rodata section.
; Kludge: Something on OS X fails to align .rodata even given an align attribute,
; so use a different read-only section.
-%macro SECTION_RODATA 0
+%macro SECTION_RODATA 0-1 16
%ifidn __OUTPUT_FORMAT__,macho64
- SECTION .text align=16
+ SECTION .text align=%1
%elifidn __OUTPUT_FORMAT__,macho
- SECTION .text align=16
+ SECTION .text align=%1
fakegot:
%else
- SECTION .rodata align=16
+ SECTION .rodata align=%1
%endif
%endmacro
-; PIC support macros. All these macros are totally harmless when PIC is
-; not defined but can ruin everything if misused in PIC mode. On x86_32, shared
-; objects cannot directly access global variables by address, they need to
-; go through the GOT (global offset table). Most OSes do not care about it
-; and let you load non-shared .so objects (Linux, Win32...). However, OS X
-; requires PIC code in its .dylib objects.
-;
-; - GLOBAL should be used as a suffix for global addressing, eg.
-; picgetgot ebx
+; PIC support macros.
+; x86_64 can't fit 64bit address literals in most instruction types,
+; so shared objects (under the assumption that they might be anywhere
+; in memory) must use an address mode that does fit.
+; So all accesses to global variables must use this macro, e.g.
; mov eax, [foo GLOBAL]
; instead of
; mov eax, [foo]
;
-; - picgetgot computes the GOT address into the given register in PIC
-; mode, otherwise does nothing. You need to do this before using GLOBAL.
-; Before in both execution order and compiled code order (so GLOBAL knows
-; which register the GOT is in).
-
-%ifndef PIC
- %define GLOBAL
- %macro picgetgot 1
- %endmacro
-%elifdef ARCH_X86_64
- %define PIC64
+; x86_32 doesn't require PIC.
+; Some distros prefer shared objects to be PIC, but nothing breaks if
+; the code contains a few textrels, so we'll skip that complexity.
+
+%ifdef WIN64
+ %define PIC
+%elifndef ARCH_X86_64
+ %undef PIC
+%endif
+%ifdef PIC
%define GLOBAL wrt rip
- %macro picgetgot 1
- %endmacro
%else
- %define PIC32
- %ifidn __OUTPUT_FORMAT__,macho
- ; There is no real global offset table on OS X, but we still
- ; need to reference our variables by offset.
- %macro picgetgot 1
- call %%getgot
- %%getgot:
- pop %1
- add %1, $$ - %%getgot
- %undef GLOBAL
- %define GLOBAL + %1 - fakegot
- %endmacro
- %else ; elf
- extern _GLOBAL_OFFSET_TABLE_
- %macro picgetgot 1
- call %%getgot
- %%getgot:
- pop %1
- add %1, _GLOBAL_OFFSET_TABLE_ + $$ - %%getgot wrt ..gotpc
- %undef GLOBAL
- %define GLOBAL + %1 wrt ..gotoff
- %endmacro
- %endif
+ %define GLOBAL
%endif
; Macros to eliminate most code duplication between x86_32 and x86_64:
@@ -99,14 +78,14 @@
; PROLOGUE:
; %1 = number of arguments. loads them from stack if needed.
-; %2 = number of registers used, not including PIC. pushes callee-saved regs if needed.
-; %3 = whether global constants are used in this function. inits x86_32 PIC if needed.
+; %2 = number of registers used. pushes callee-saved regs if needed.
+; %3 = number of xmm registers used. pushes callee-saved xmm regs if needed.
; %4 = list of names to define to registers
; PROLOGUE can also be invoked by adding the same options to cglobal
; e.g.
-; cglobal foo, 2,3,0, dst, src, tmp
-; declares a function (foo), taking two args (dst and src), one local variable (tmp), and not using globals
+; cglobal foo, 2,3, dst, src, tmp
+; declares a function (foo), taking two args (dst and src) and one local variable (tmp)
; TODO Some functions can use some args directly from the stack. If they're the
; last args then you can just not declare them, but if they're in the middle
@@ -119,12 +98,25 @@
; Same, but if it doesn't pop anything it becomes a 2-byte ret, for athlons
; which are slow when a normal ret follows a branch.
+; registers:
+; rN and rNq are the native-size register holding function argument N
+; rNd, rNw, rNb are dword, word, and byte size
+; rNm is the original location of arg N (a register or on the stack), dword
+; rNmp is native size
+
%macro DECLARE_REG 6
%define r%1q %2
%define r%1d %3
%define r%1w %4
%define r%1b %5
%define r%1m %6
+ %ifid %6 ; i.e. it's a register
+ %define r%1mp %2
+ %elifdef ARCH_X86_64 ; memory
+ %define r%1mp qword %6
+ %else
+ %define r%1mp dword %6
+ %endif
%define r%1 %2
%endmacro
@@ -150,6 +142,29 @@ DECLARE_REG_SIZE si, sil
DECLARE_REG_SIZE di, dil
DECLARE_REG_SIZE bp, bpl
+; t# defines for when per-arch register allocation is more complex than just function arguments
+
+%macro DECLARE_REG_TMP 1-*
+ %assign %%i 0
+ %rep %0
+ CAT_XDEFINE t, %%i, r%1
+ %assign %%i %%i+1
+ %rotate 1
+ %endrep
+%endmacro
+
+%macro DECLARE_REG_TMP_SIZE 0-*
+ %rep %0
+ %define t%1q t%1 %+ q
+ %define t%1d t%1 %+ d
+ %define t%1w t%1 %+ w
+ %define t%1b t%1 %+ b
+ %rotate 1
+ %endrep
+%endmacro
+
+DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7
+
%ifdef ARCH_X86_64
%define gprsize 8
%else
@@ -224,8 +239,7 @@ DECLARE_REG_SIZE bp, bpl
%assign n_arg_names %%i
%endmacro
-%ifdef ARCH_X86_64 ;==========================================================
-%ifidn __OUTPUT_FORMAT__,win32
+%ifdef WIN64 ; Windows x64 ;=================================================
DECLARE_REG 0, rcx, ecx, cx, cl, ecx
DECLARE_REG 1, rdx, edx, dx, dl, edx
@@ -239,11 +253,75 @@ DECLARE_REG 6, rax, eax, ax, al, [rsp + stack_offset + 56]
%macro LOAD_IF_USED 2 ; reg_id, number_of_args
%if %1 < %2
- mov r%1, [rsp + 8 + %1*8]
+ mov r%1, [rsp + stack_offset + 8 + %1*8]
+ %endif
+%endmacro
+
+%macro PROLOGUE 2-4+ ; #args, #regs, #xmm_regs, arg_names...
+ ASSERT %2 >= %1
+ %assign regs_used %2
+ ASSERT regs_used <= 7
+ %if %0 > 2
+ %assign xmm_regs_used %3
+ %else
+ %assign xmm_regs_used 0
+ %endif
+ ASSERT xmm_regs_used <= 16
+ %if regs_used > 4
+ push r4
+ push r5
+ %assign stack_offset stack_offset+16
+ %endif
+ %if xmm_regs_used > 6
+ sub rsp, (xmm_regs_used-6)*16+16
+ %assign stack_offset stack_offset+(xmm_regs_used-6)*16+16
+ %assign %%i xmm_regs_used
+ %rep (xmm_regs_used-6)
+ %assign %%i %%i-1
+ movdqa [rsp + (%%i-6)*16+8], xmm %+ %%i
+ %endrep
+ %endif
+ LOAD_IF_USED 4, %1
+ LOAD_IF_USED 5, %1
+ LOAD_IF_USED 6, %1
+ DEFINE_ARGS %4
+%endmacro
+
+%macro RESTORE_XMM_INTERNAL 1
+ %if xmm_regs_used > 6
+ %assign %%i xmm_regs_used
+ %rep (xmm_regs_used-6)
+ %assign %%i %%i-1
+ movdqa xmm %+ %%i, [%1 + (%%i-6)*16+8]
+ %endrep
+ add %1, (xmm_regs_used-6)*16+16
+ %endif
+%endmacro
+
+%macro RESTORE_XMM 1
+ RESTORE_XMM_INTERNAL %1
+ %assign stack_offset stack_offset-(xmm_regs_used-6)*16+16
+ %assign xmm_regs_used 0
+%endmacro
+
+%macro RET 0
+ RESTORE_XMM_INTERNAL rsp
+ %if regs_used > 4
+ pop r5
+ pop r4
%endif
+ ret
%endmacro
-%else ;=======================================================================
+%macro REP_RET 0
+ %if regs_used > 4 || xmm_regs_used > 6
+ RET
+ %else
+ rep ret
+ %endif
+%endmacro
+
+%elifdef ARCH_X86_64 ; *nix x64 ;=============================================
DECLARE_REG 0, rdi, edi, di, dil, edi
DECLARE_REG 1, rsi, esi, si, sil, esi
@@ -261,16 +339,9 @@ DECLARE_REG 6, rax, eax, ax, al, [rsp + stack_offset + 8]
%endif
%endmacro
-%endif ; !WIN64
-
-%macro PROLOGUE 2-4+ 0 ; #args, #regs, pic, arg_names...
+%macro PROLOGUE 2-4+ ; #args, #regs, #xmm_regs, arg_names...
ASSERT %2 >= %1
ASSERT %2 <= 7
- %assign stack_offset 0
-%ifidn __OUTPUT_FORMAT__,win32
- LOAD_IF_USED 4, %1
- LOAD_IF_USED 5, %1
-%endif
LOAD_IF_USED 6, %1
DEFINE_ARGS %4
%endmacro
@@ -315,15 +386,9 @@ DECLARE_REG 6, ebp, ebp, bp, null, [esp + stack_offset + 28]
%endif
%endmacro
-%macro PROLOGUE 2-4+ 0 ; #args, #regs, pic, arg_names...
+%macro PROLOGUE 2-4+ ; #args, #regs, arg_names...
ASSERT %2 >= %1
- %assign stack_offset 0
%assign regs_used %2
- %ifdef PIC
- %if %3
- %assign regs_used regs_used+1
- %endif
- %endif
ASSERT regs_used <= 7
PUSH_IF_USED 3
PUSH_IF_USED 4
@@ -336,9 +401,6 @@ DECLARE_REG 6, ebp, ebp, bp, null, [esp + stack_offset + 28]
LOAD_IF_USED 4, %1
LOAD_IF_USED 5, %1
LOAD_IF_USED 6, %1
- %if %3
- picgetgot r%2
- %endif
DEFINE_ARGS %4
%endmacro
@@ -382,6 +444,7 @@ DECLARE_REG 6, ebp, ebp, bp, null, [esp + stack_offset + 28]
align function_align
%1:
RESET_MM_PERMUTATION ; not really needed, but makes disassembly somewhat nicer
+ %assign stack_offset 0
%if %0 > 1
PROLOGUE %2
%endif
@@ -389,11 +452,9 @@ DECLARE_REG 6, ebp, ebp, bp, null, [esp + stack_offset + 28]
%macro cextern 1
%ifdef PREFIX
- extern _%1
- %define %1 _%1
- %else
- extern %1
+ %xdefine %1 _%1
%endif
+ extern %1
%endmacro
; This is needed for ELF, otherwise the GNU linker assumes the stack is
@@ -523,6 +584,7 @@ INIT_MMX
%assign %%i 0
%rep num_mmregs
CAT_XDEFINE m, %%i, %1_m %+ %%i
+ CAT_XDEFINE n, m %+ %%i, %%i
%assign %%i %%i+1
%endrep
%endmacro
@@ -534,7 +596,30 @@ INIT_MMX
%endif
%endmacro
-; substitutions which are functionally identical but reduce code size
+;Substitutions that reduce instruction size but are functionally equivalent
%define movdqa movaps
%define movdqu movups
+%macro add 2
+ %ifnum %2
+ %if %2==128
+ sub %1, -128
+ %else
+ add %1, %2
+ %endif
+ %else
+ add %1, %2
+ %endif
+%endmacro
+
+%macro sub 2
+ %ifnum %2
+ %if %2==128
+ add %1, -128
+ %else
+ sub %1, %2
+ %endif
+ %else
+ sub %1, %2
+ %endif
+%endmacro
diff --git a/libavcodec/x86/x86util.asm b/libavcodec/x86/x86util.asm
index 51eee5c264..f3e0e2dbe4 100644
--- a/libavcodec/x86/x86util.asm
+++ b/libavcodec/x86/x86util.asm
@@ -93,7 +93,7 @@
SBUTTERFLY qdq, %4, %8, %2
SWAP %2, %5
SWAP %4, %7
-%if 0<11
+%if %0<11
movdqa m%5, %10
%endif
%endif
@@ -165,28 +165,203 @@
palignr %1, %2, %3
%endmacro
-%macro SUMSUB_BA 2
+%macro DEINTB 5 ; mask, reg1, mask, reg2, optional src to fill masks from
+%ifnum %5
+ mova m%1, m%5
+ mova m%3, m%5
+%else
+ mova m%1, %5
+ mova m%3, m%1
+%endif
+ pand m%1, m%2 ; dst .. y6 .. y4
+ pand m%3, m%4 ; src .. y6 .. y4
+ psrlw m%2, 8 ; dst .. y7 .. y5
+ psrlw m%4, 8 ; src .. y7 .. y5
+%endmacro
+
+%macro SUMSUB_BA 2-3
+%if %0==2
paddw %1, %2
paddw %2, %2
psubw %2, %1
+%else
+ mova %3, %1
+ paddw %1, %2
+ psubw %2, %3
+%endif
%endmacro
-%macro SUMSUB_BADC 4
+%macro SUMSUB_BADC 4-5
+%if %0==5
+ SUMSUB_BA %1, %2, %5
+ SUMSUB_BA %3, %4, %5
+%else
paddw %1, %2
paddw %3, %4
paddw %2, %2
paddw %4, %4
psubw %2, %1
psubw %4, %3
+%endif
%endmacro
-%macro HADAMARD8_1D 8
- SUMSUB_BADC %1, %5, %2, %6
- SUMSUB_BADC %3, %7, %4, %8
+%macro HADAMARD4_V 4+
+ SUMSUB_BADC %1, %2, %3, %4
SUMSUB_BADC %1, %3, %2, %4
- SUMSUB_BADC %5, %7, %6, %8
+%endmacro
+
+%macro HADAMARD8_V 8+
SUMSUB_BADC %1, %2, %3, %4
SUMSUB_BADC %5, %6, %7, %8
+ SUMSUB_BADC %1, %3, %2, %4
+ SUMSUB_BADC %5, %7, %6, %8
+ SUMSUB_BADC %1, %5, %2, %6
+ SUMSUB_BADC %3, %7, %4, %8
+%endmacro
+
+%macro TRANS_SSE2 5-6
+; TRANSPOSE2x2
+; %1: transpose width (d/q) - use SBUTTERFLY qdq for dq
+; %2: ord/unord (for compat with sse4, unused)
+; %3/%4: source regs
+; %5/%6: tmp regs
+%ifidn %1, d
+%define mask [mask_10 GLOBAL]
+%define shift 16
+%elifidn %1, q
+%define mask [mask_1100 GLOBAL]
+%define shift 32
+%endif
+%if %0==6 ; less dependency if we have two tmp
+ mova m%5, mask ; ff00
+ mova m%6, m%4 ; x5x4
+ psll%1 m%4, shift ; x4..
+ pand m%6, m%5 ; x5..
+ pandn m%5, m%3 ; ..x0
+ psrl%1 m%3, shift ; ..x1
+ por m%4, m%5 ; x4x0
+ por m%3, m%6 ; x5x1
+%else ; more dependency, one insn less. sometimes faster, sometimes not
+ mova m%5, m%4 ; x5x4
+ psll%1 m%4, shift ; x4..
+ pxor m%4, m%3 ; (x4^x1)x0
+ pand m%4, mask ; (x4^x1)..
+ pxor m%3, m%4 ; x4x0
+ psrl%1 m%4, shift ; ..(x1^x4)
+ pxor m%5, m%4 ; x5x1
+ SWAP %4, %3, %5
+%endif
+%endmacro
+
+%macro TRANS_SSE4 5-6 ; see above
+%ifidn %1, d
+ mova m%5, m%3
+%ifidn %2, ord
+ psrl%1 m%3, 16
+%endif
+ pblendw m%3, m%4, 10101010b
+ psll%1 m%4, 16
+%ifidn %2, ord
+ pblendw m%4, m%5, 01010101b
+%else
+ psrl%1 m%5, 16
+ por m%4, m%5
+%endif
+%elifidn %1, q
+ mova m%5, m%3
+ shufps m%3, m%4, 10001000b
+ shufps m%5, m%4, 11011101b
+ SWAP %4, %5
+%endif
+%endmacro
+
+%macro HADAMARD 5-6
+; %1=distance in words (0 for vertical pass, 1/2/4 for horizontal passes)
+; %2=sumsub/max/amax (sum and diff / maximum / maximum of absolutes)
+; %3/%4: regs
+; %5(%6): tmpregs
+%if %1!=0 ; have to reorder stuff for horizontal op
+ %ifidn %2, sumsub
+ %define ORDER ord
+ ; sumsub needs order because a-b != b-a unless a=b
+ %else
+ %define ORDER unord
+ ; if we just max, order doesn't matter (allows pblendw+or in sse4)
+ %endif
+ %if %1==1
+ TRANS d, ORDER, %3, %4, %5, %6
+ %elif %1==2
+ %if mmsize==8
+ SBUTTERFLY dq, %3, %4, %5
+ %else
+ TRANS q, ORDER, %3, %4, %5, %6
+ %endif
+ %elif %1==4
+ SBUTTERFLY qdq, %3, %4, %5
+ %endif
+%endif
+%ifidn %2, sumsub
+ SUMSUB_BA m%3, m%4, m%5
+%else
+ %ifidn %2, amax
+ %if %0==6
+ ABS2 m%3, m%4, m%5, m%6
+ %else
+ ABS1 m%3, m%5
+ ABS1 m%4, m%5
+ %endif
+ %endif
+ pmaxsw m%3, m%4
+%endif
+%endmacro
+
+
+%macro HADAMARD2_2D 6-7 sumsub
+ HADAMARD 0, sumsub, %1, %2, %5
+ HADAMARD 0, sumsub, %3, %4, %5
+ SBUTTERFLY %6, %1, %2, %5
+%ifnum %7
+ HADAMARD 0, amax, %1, %2, %5, %7
+%else
+ HADAMARD 0, %7, %1, %2, %5
+%endif
+ SBUTTERFLY %6, %3, %4, %5
+%ifnum %7
+ HADAMARD 0, amax, %3, %4, %5, %7
+%else
+ HADAMARD 0, %7, %3, %4, %5
+%endif
+%endmacro
+
+%macro HADAMARD4_2D 5-6 sumsub
+ HADAMARD2_2D %1, %2, %3, %4, %5, wd
+ HADAMARD2_2D %1, %3, %2, %4, %5, dq, %6
+ SWAP %2, %3
+%endmacro
+
+%macro HADAMARD4_2D_SSE 5-6 sumsub
+ HADAMARD 0, sumsub, %1, %2, %5 ; 1st V row 0 + 1
+ HADAMARD 0, sumsub, %3, %4, %5 ; 1st V row 2 + 3
+ SBUTTERFLY wd, %1, %2, %5 ; %1: m0 1+0 %2: m1 1+0
+ SBUTTERFLY wd, %3, %4, %5 ; %3: m0 3+2 %4: m1 3+2
+ HADAMARD2_2D %1, %3, %2, %4, %5, dq
+ SBUTTERFLY qdq, %1, %2, %5
+ HADAMARD 0, %6, %1, %2, %5 ; 2nd H m1/m0 row 0+1
+ SBUTTERFLY qdq, %3, %4, %5
+ HADAMARD 0, %6, %3, %4, %5 ; 2nd H m1/m0 row 2+3
+%endmacro
+
+%macro HADAMARD8_2D 9-10 sumsub
+ HADAMARD2_2D %1, %2, %3, %4, %9, wd
+ HADAMARD2_2D %5, %6, %7, %8, %9, wd
+ HADAMARD2_2D %1, %3, %2, %4, %9, dq
+ HADAMARD2_2D %5, %7, %6, %8, %9, dq
+ HADAMARD2_2D %1, %5, %3, %7, %9, qdq, %10
+ HADAMARD2_2D %2, %6, %4, %8, %9, qdq, %10
+%ifnidn %10, amax
+ SWAP %2, %5
+ SWAP %4, %7
+%endif
%endmacro
%macro SUMSUB2_AB 3
@@ -197,13 +372,49 @@
psubw %3, %2
%endmacro
+%macro SUMSUB2_BA 3
+ mova m%3, m%1
+ paddw m%1, m%2
+ paddw m%1, m%2
+ psubw m%2, m%3
+ psubw m%2, m%3
+%endmacro
+
%macro SUMSUBD2_AB 4
mova %4, %1
mova %3, %2
psraw %2, 1
- psraw %4, 1
- paddw %1, %2
- psubw %4, %3
+ psraw %1, 1
+ paddw %2, %4
+ psubw %1, %3
+%endmacro
+
+%macro DCT4_1D 5
+%ifnum %5
+ SUMSUB_BADC m%4, m%1, m%3, m%2; m%5
+ SUMSUB_BA m%3, m%4, m%5
+ SUMSUB2_AB m%1, m%2, m%5
+ SWAP %1, %3, %4, %5, %2
+%else
+ SUMSUB_BADC m%4, m%1, m%3, m%2
+ SUMSUB_BA m%3, m%4
+ mova [%5], m%2
+ SUMSUB2_AB m%1, [%5], m%2
+ SWAP %1, %3, %4, %2
+%endif
+%endmacro
+
+%macro IDCT4_1D 5-6
+%ifnum %5
+ SUMSUBD2_AB m%2, m%4, m%6, m%5
+ SUMSUB_BA m%3, m%1, m%6
+ SUMSUB_BADC m%4, m%3, m%2, m%1, m%6
+%else
+ SUMSUBD2_AB m%2, m%4, [%5], [%5+16]
+ SUMSUB_BA m%3, m%1
+ SUMSUB_BADC m%4, m%3, m%2, m%1
+%endif
+ SWAP %1, %4, %3
%endmacro
%macro LOAD_DIFF 5
@@ -222,17 +433,81 @@
%endif
%endmacro
-%macro LOAD_DIFF_8x4P 6-8 r0,r2 ; 4x dest, 2x temp, 2x pointer
- LOAD_DIFF %1, %5, none, [%7], [%8]
- LOAD_DIFF %2, %6, none, [%7+r1], [%8+r3]
- LOAD_DIFF %3, %5, none, [%7+2*r1], [%8+2*r3]
- LOAD_DIFF %4, %6, none, [%7+r4], [%8+r5]
+%macro LOAD_DIFF8x4_SSE2 8
+ LOAD_DIFF m%1, m%5, m%6, [%7+%1*FENC_STRIDE], [%8+%1*FDEC_STRIDE]
+ LOAD_DIFF m%2, m%5, m%6, [%7+%2*FENC_STRIDE], [%8+%2*FDEC_STRIDE]
+ LOAD_DIFF m%3, m%5, m%6, [%7+%3*FENC_STRIDE], [%8+%3*FDEC_STRIDE]
+ LOAD_DIFF m%4, m%5, m%6, [%7+%4*FENC_STRIDE], [%8+%4*FDEC_STRIDE]
%endmacro
-%macro STORE_DIFF 4
+%macro LOAD_DIFF8x4_SSSE3 8 ; 4x dst, 1x tmp, 1x mul, 2x ptr
+ movh m%2, [%8+%1*FDEC_STRIDE]
+ movh m%1, [%7+%1*FENC_STRIDE]
+ punpcklbw m%1, m%2
+ movh m%3, [%8+%2*FDEC_STRIDE]
+ movh m%2, [%7+%2*FENC_STRIDE]
+ punpcklbw m%2, m%3
+ movh m%4, [%8+%3*FDEC_STRIDE]
+ movh m%3, [%7+%3*FENC_STRIDE]
+ punpcklbw m%3, m%4
+ movh m%5, [%8+%4*FDEC_STRIDE]
+ movh m%4, [%7+%4*FENC_STRIDE]
+ punpcklbw m%4, m%5
+ pmaddubsw m%1, m%6
+ pmaddubsw m%2, m%6
+ pmaddubsw m%3, m%6
+ pmaddubsw m%4, m%6
+%endmacro
+
+%macro STORE_DCT 6
+ movq [%5+%6+ 0], m%1
+ movq [%5+%6+ 8], m%2
+ movq [%5+%6+16], m%3
+ movq [%5+%6+24], m%4
+ movhps [%5+%6+32], m%1
+ movhps [%5+%6+40], m%2
+ movhps [%5+%6+48], m%3
+ movhps [%5+%6+56], m%4
+%endmacro
+
+%macro STORE_IDCT 4
+ movhps [r0-4*FDEC_STRIDE], %1
+ movh [r0-3*FDEC_STRIDE], %1
+ movhps [r0-2*FDEC_STRIDE], %2
+ movh [r0-1*FDEC_STRIDE], %2
+ movhps [r0+0*FDEC_STRIDE], %3
+ movh [r0+1*FDEC_STRIDE], %3
+ movhps [r0+2*FDEC_STRIDE], %4
+ movh [r0+3*FDEC_STRIDE], %4
+%endmacro
+
+%macro LOAD_DIFF_8x4P 7-10 r0,r2,0 ; 4x dest, 2x temp, 2x pointer, increment?
+ LOAD_DIFF m%1, m%5, m%7, [%8], [%9]
+ LOAD_DIFF m%2, m%6, m%7, [%8+r1], [%9+r3]
+ LOAD_DIFF m%3, m%5, m%7, [%8+2*r1], [%9+2*r3]
+ LOAD_DIFF m%4, m%6, m%7, [%8+r4], [%9+r5]
+%if %10
+ lea %8, [%8+4*r1]
+ lea %9, [%9+4*r3]
+%endif
+%endmacro
+
+%macro DIFFx2 6-7
+ movh %3, %5
+ punpcklbw %3, %4
psraw %1, 6
+ paddsw %1, %3
+ movh %3, %6
+ punpcklbw %3, %4
+ psraw %2, 6
+ paddsw %2, %3
+ packuswb %2, %1
+%endmacro
+
+%macro STORE_DIFF 4
movh %2, %4
punpcklbw %2, %3
+ psraw %1, 6
paddsw %1, %2
packuswb %1, %1
movh %4, %1