yadif: Port inline assembly to yasm

Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

4 files changed, 264 insertions, 302 deletions

diff --git a/libavfilter/x86/Makefile b/libavfilter/x86/Makefile
index 0f08e39691..47569cf642 100644
--- a/libavfilter/x86/Makefile
+++ b/libavfilter/x86/Makefile
@@ -1,6 +1,7 @@
 OBJS-$(CONFIG_GRADFUN_FILTER)                += x86/gradfun.o
 OBJS-$(CONFIG_VOLUME_FILTER)                 += x86/af_volume_init.o
-OBJS-$(CONFIG_YADIF_FILTER)                  += x86/yadif.o
+OBJS-$(CONFIG_YADIF_FILTER)                  += x86/yadif_init.o
 
 YASM-OBJS-$(CONFIG_HQDN3D_FILTER)            += x86/hqdn3d.o
 YASM-OBJS-$(CONFIG_VOLUME_FILTER)            += x86/af_volume.o
+YASM-OBJS-$(CONFIG_YADIF_FILTER)             += x86/yadif.o
diff --git a/libavfilter/x86/yadif.asm b/libavfilter/x86/yadif.asm
new file mode 100644
index 0000000000..5e406a496a
--- /dev/null
+++ b/libavfilter/x86/yadif.asm
@@ -0,0 +1,242 @@
+;*****************************************************************************
+;* x86-optimized functions for yadif filter
+;*
+;* Copyright (C) 2006 Michael Niedermayer <michaelni@gmx.at>
+;* Copyright (c) 2013 Daniel Kang <daniel.d.kang@gmail.com>
+;*
+;* This file is part of Libav.
+;*
+;* Libav is free software; you can redistribute it and/or modify
+;* it under the terms of the GNU General Public License as published by
+;* the Free Software Foundation; either version 2 of the License, or
+;* (at your option) any later version.
+;*
+;* Libav is distributed in the hope that it will be useful,
+;* but WITHOUT ANY WARRANTY; without even the implied warranty of
+;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;* GNU General Public License for more details.
+;*
+;* You should have received a copy of the GNU General Public License along
+;* with Libav; if not, write to the Free Software Foundation, Inc.,
+;* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+;******************************************************************************
+
+%include "libavutil/x86/x86util.asm"
+
+SECTION_RODATA
+
+pb_1: times 16 db 1
+pw_1: times  8 dw 1
+
+SECTION .text
+
+%macro CHECK 2
+    movu      m2, [curq+mrefsq+%1]
+    movu      m3, [curq+prefsq+%2]
+    mova      m4, m2
+    mova      m5, m2
+    pxor      m4, m3
+    pavgb     m5, m3
+    pand      m4, [pb_1]
+    psubusb   m5, m4
+%if mmsize == 16
+    psrldq    m5, 1
+%else
+    psrlq     m5, 8
+%endif
+    punpcklbw m5, m7
+    mova      m4, m2
+    psubusb   m2, m3
+    psubusb   m3, m4
+    pmaxub    m2, m3
+    mova      m3, m2
+    mova      m4, m2
+%if mmsize == 16
+    psrldq    m3, 1
+    psrldq    m4, 2
+%else
+    psrlq     m3, 8
+    psrlq     m4, 16
+%endif
+    punpcklbw m2, m7
+    punpcklbw m3, m7
+    punpcklbw m4, m7
+    paddw     m2, m3
+    paddw     m2, m4
+%endmacro
+
+%macro CHECK1 0
+    mova    m3, m0
+    pcmpgtw m3, m2
+    pminsw  m0, m2
+    mova    m6, m3
+    pand    m5, m3
+    pandn   m3, m1
+    por     m3, m5
+    mova    m1, m3
+%endmacro
+
+%macro CHECK2 0
+    paddw   m6, [pw_1]
+    psllw   m6, 14
+    paddsw  m2, m6
+    mova    m3, m0
+    pcmpgtw m3, m2
+    pminsw  m0, m2
+    pand    m5, m3
+    pandn   m3, m1
+    por     m3, m5
+    mova    m1, m3
+%endmacro
+
+%macro LOAD 2
+    movh      m%1, %2
+    punpcklbw m%1, m7
+%endmacro
+
+%macro FILTER 3
+.loop%1:
+    pxor         m7, m7
+    LOAD          0, [curq+mrefsq]
+    LOAD          1, [curq+prefsq]
+    LOAD          2, [%2]
+    LOAD          3, [%3]
+    mova         m4, m3
+    paddw        m3, m2
+    psraw        m3, 1
+    mova   [rsp+ 0], m0
+    mova   [rsp+16], m3
+    mova   [rsp+32], m1
+    psubw        m2, m4
+    ABS1         m2, m4
+    LOAD          3, [prevq+mrefsq]
+    LOAD          4, [prevq+prefsq]
+    psubw        m3, m0
+    psubw        m4, m1
+    ABS1         m3, m5
+    ABS1         m4, m5
+    paddw        m3, m4
+    psrlw        m2, 1
+    psrlw        m3, 1
+    pmaxsw       m2, m3
+    LOAD          3, [nextq+mrefsq]
+    LOAD          4, [nextq+prefsq]
+    psubw        m3, m0
+    psubw        m4, m1
+    ABS1         m3, m5
+    ABS1         m4, m5
+    paddw        m3, m4
+    psrlw        m3, 1
+    pmaxsw       m2, m3
+    mova   [rsp+48], m2
+
+    paddw        m1, m0
+    paddw        m0, m0
+    psubw        m0, m1
+    psrlw        m1, 1
+    ABS1         m0, m2
+
+    movu         m2, [curq+mrefsq-1]
+    movu         m3, [curq+prefsq-1]
+    mova         m4, m2
+    psubusb      m2, m3
+    psubusb      m3, m4
+    pmaxub       m2, m3
+%if mmsize == 16
+    mova         m3, m2
+    psrldq       m3, 2
+%else
+    pshufw       m3, m2, q0021
+%endif
+    punpcklbw    m2, m7
+    punpcklbw    m3, m7
+    paddw        m0, m2
+    paddw        m0, m3
+    psubw        m0, [pw_1]
+
+    CHECK -2, 0
+    CHECK1
+    CHECK -3, 1
+    CHECK2
+    CHECK 0, -2
+    CHECK1
+    CHECK 1, -3
+    CHECK2
+
+    mova         m6, [rsp+48]
+    cmp DWORD modem, 2
+    jge .end%1
+    LOAD          2, [%2+mrefsq*2]
+    LOAD          4, [%3+mrefsq*2]
+    LOAD          3, [%2+prefsq*2]
+    LOAD          5, [%3+prefsq*2]
+    paddw        m2, m4
+    paddw        m3, m5
+    psrlw        m2, 1
+    psrlw        m3, 1
+    mova         m4, [rsp+ 0]
+    mova         m5, [rsp+16]
+    mova         m7, [rsp+32]
+    psubw        m2, m4
+    psubw        m3, m7
+    mova         m0, m5
+    psubw        m5, m4
+    psubw        m0, m7
+    mova         m4, m2
+    pminsw       m2, m3
+    pmaxsw       m3, m4
+    pmaxsw       m2, m5
+    pminsw       m3, m5
+    pmaxsw       m2, m0
+    pminsw       m3, m0
+    pxor         m4, m4
+    pmaxsw       m6, m3
+    psubw        m4, m2
+    pmaxsw       m6, m4
+
+.end%1:
+    mova         m2, [rsp+16]
+    mova         m3, m2
+    psubw        m2, m6
+    paddw        m3, m6
+    pmaxsw       m1, m2
+    pminsw       m1, m3
+    packuswb     m1, m1
+
+    movh     [dstq], m1
+    add        dstq, mmsize/2
+    add       prevq, mmsize/2
+    add        curq, mmsize/2
+    add       nextq, mmsize/2
+    sub          wd, mmsize/2
+    jg .loop%1
+%endmacro
+
+%macro YADIF 0
+cglobal yadif_filter_line, 7, 7, 8, 16*5, dst, prev, cur, next, w, prefs, \
+                                          mrefs, parity, mode
+    test             wq, wq
+    jle .ret
+    movsxdifnidn prefsq, prefsd
+    movsxdifnidn mrefsq, mrefsd
+
+    cmp   DWORD paritym, 0
+    je .parity0
+    FILTER 1, prevq, curq
+    jmp .ret
+
+.parity0:
+    FILTER 0, curq, nextq
+
+.ret:
+    RET
+%endmacro
+
+INIT_XMM ssse3
+YADIF
+INIT_XMM sse2
+YADIF
+%if ARCH_X86_32
+INIT_MMX mmxext
+YADIF
+%endif
diff --git a/libavfilter/x86/yadif.c b/libavfilter/x86/yadif_init.c
index ab1d282f9d..0cee8e56b4 100644
--- a/libavfilter/x86/yadif.c
+++ b/libavfilter/x86/yadif_init.c
@@ -23,52 +23,32 @@
 #include "libavutil/internal.h"
 #include "libavutil/mem.h"
 #include "libavutil/x86/asm.h"
+#include "libavutil/x86/cpu.h"
 #include "libavcodec/x86/dsputil_mmx.h"
 #include "libavfilter/yadif.h"
 
-#if HAVE_INLINE_ASM
-
-DECLARE_ASM_CONST(16, const xmm_reg, pb_1) = {0x0101010101010101ULL, 0x0101010101010101ULL};
-DECLARE_ASM_CONST(16, const xmm_reg, pw_1) = {0x0001000100010001ULL, 0x0001000100010001ULL};
-
-#if HAVE_SSSE3_INLINE
-#define COMPILE_TEMPLATE_SSE2 1
-#define COMPILE_TEMPLATE_SSSE3 1
-#undef RENAME
-#define RENAME(a) a ## _ssse3
-#include "yadif_template.c"
-#undef COMPILE_TEMPLATE_SSSE3
-#endif
-
-#if HAVE_SSE2_INLINE
-#undef RENAME
-#define RENAME(a) a ## _sse2
-#include "yadif_template.c"
-#undef COMPILE_TEMPLATE_SSE2
-#endif
-
-#if HAVE_MMXEXT_INLINE
-#undef RENAME
-#define RENAME(a) a ## _mmxext
-#include "yadif_template.c"
-#endif
-
-#endif /* HAVE_INLINE_ASM */
+void ff_yadif_filter_line_mmxext(uint8_t *dst, uint8_t *prev, uint8_t *cur,
+                                 uint8_t *next, int w, int prefs,
+                                 int mrefs, int parity, int mode);
+void ff_yadif_filter_line_sse2(uint8_t *dst, uint8_t *prev, uint8_t *cur,
+                               uint8_t *next, int w, int prefs,
+                               int mrefs, int parity, int mode);
+void ff_yadif_filter_line_ssse3(uint8_t *dst, uint8_t *prev, uint8_t *cur,
+                                uint8_t *next, int w, int prefs,
+                                int mrefs, int parity, int mode);
 
 av_cold void ff_yadif_init_x86(YADIFContext *yadif)
 {
     int cpu_flags = av_get_cpu_flags();
 
-#if HAVE_MMXEXT_INLINE
-    if (cpu_flags & AV_CPU_FLAG_MMXEXT)
-        yadif->filter_line = yadif_filter_line_mmxext;
-#endif
-#if HAVE_SSE2_INLINE
-    if (cpu_flags & AV_CPU_FLAG_SSE2)
-        yadif->filter_line = yadif_filter_line_sse2;
-#endif
-#if HAVE_SSSE3_INLINE
-    if (cpu_flags & AV_CPU_FLAG_SSSE3)
-        yadif->filter_line = yadif_filter_line_ssse3;
-#endif
+#if HAVE_YASM
+#if ARCH_X86_32
+    if (EXTERNAL_MMXEXT(cpu_flags))
+        yadif->filter_line = ff_yadif_filter_line_mmxext;
+#endif /* ARCH_X86_32 */
+    if (EXTERNAL_SSE2(cpu_flags))
+        yadif->filter_line = ff_yadif_filter_line_sse2;
+    if (EXTERNAL_SSSE3(cpu_flags))
+        yadif->filter_line = ff_yadif_filter_line_ssse3;
+#endif /* HAVE_YASM */
 }
diff --git a/libavfilter/x86/yadif_template.c b/libavfilter/x86/yadif_template.c
deleted file mode 100644
index 02b0c9f909..0000000000
--- a/libavfilter/x86/yadif_template.c
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Copyright (C) 2006 Michael Niedermayer <michaelni@gmx.at>
- *
- * This file is part of Libav.
- *
- * Libav is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * Libav is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with Libav; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- */
-
-#ifdef COMPILE_TEMPLATE_SSE2
-#define MM "%%xmm"
-#define MOV  "movq"
-#define MOVQ "movdqa"
-#define MOVQU "movdqu"
-#define STEP 8
-#define LOAD(mem,dst) \
-            MOV"       "mem", "dst" \n\t"\
-            "punpcklbw "MM"7, "dst" \n\t"
-#define PSRL1(reg) "psrldq $1, "reg" \n\t"
-#define PSRL2(reg) "psrldq $2, "reg" \n\t"
-#define PSHUF(src,dst) "movdqa "dst", "src" \n\t"\
-                       "psrldq $2, "src"     \n\t"
-#else
-#define MM "%%mm"
-#define MOV  "movd"
-#define MOVQ "movq"
-#define MOVQU "movq"
-#define STEP 4
-#define LOAD(mem,dst) \
-            MOV"       "mem", "dst" \n\t"\
-            "punpcklbw "MM"7, "dst" \n\t"
-#define PSRL1(reg) "psrlq $8, "reg" \n\t"
-#define PSRL2(reg) "psrlq $16, "reg" \n\t"
-#define PSHUF(src,dst) "pshufw $9, "dst", "src" \n\t"
-#endif
-
-#ifdef COMPILE_TEMPLATE_SSSE3
-#define PABS(tmp,dst) \
-            "pabsw     "dst", "dst" \n\t"
-#else
-#define PABS(tmp,dst) \
-            "pxor     "tmp", "tmp" \n\t"\
-            "psubw    "dst", "tmp" \n\t"\
-            "pmaxsw   "tmp", "dst" \n\t"
-#endif
-
-#define CHECK(pj,mj) \
-            MOVQU" "#pj"(%[cur],%[mrefs]), "MM"2 \n\t" /* cur[x-refs-1+j] */\
-            MOVQU" "#mj"(%[cur],%[prefs]), "MM"3 \n\t" /* cur[x+refs-1-j] */\
-            MOVQ"      "MM"2, "MM"4 \n\t"\
-            MOVQ"      "MM"2, "MM"5 \n\t"\
-            "pxor      "MM"3, "MM"4 \n\t"\
-            "pavgb     "MM"3, "MM"5 \n\t"\
-            "pand     "MANGLE(pb_1)", "MM"4 \n\t"\
-            "psubusb   "MM"4, "MM"5 \n\t"\
-            PSRL1(MM"5")                 \
-            "punpcklbw "MM"7, "MM"5 \n\t" /* (cur[x-refs+j] + cur[x+refs-j])>>1 */\
-            MOVQ"      "MM"2, "MM"4 \n\t"\
-            "psubusb   "MM"3, "MM"2 \n\t"\
-            "psubusb   "MM"4, "MM"3 \n\t"\
-            "pmaxub    "MM"3, "MM"2 \n\t"\
-            MOVQ"      "MM"2, "MM"3 \n\t"\
-            MOVQ"      "MM"2, "MM"4 \n\t" /* ABS(cur[x-refs-1+j] - cur[x+refs-1-j]) */\
-            PSRL1(MM"3")                  /* ABS(cur[x-refs  +j] - cur[x+refs  -j]) */\
-            PSRL2(MM"4")                  /* ABS(cur[x-refs+1+j] - cur[x+refs+1-j]) */\
-            "punpcklbw "MM"7, "MM"2 \n\t"\
-            "punpcklbw "MM"7, "MM"3 \n\t"\
-            "punpcklbw "MM"7, "MM"4 \n\t"\
-            "paddw     "MM"3, "MM"2 \n\t"\
-            "paddw     "MM"4, "MM"2 \n\t" /* score */
-
-#define CHECK1 \
-            MOVQ"      "MM"0, "MM"3 \n\t"\
-            "pcmpgtw   "MM"2, "MM"3 \n\t" /* if(score < spatial_score) */\
-            "pminsw    "MM"2, "MM"0 \n\t" /* spatial_score= score; */\
-            MOVQ"      "MM"3, "MM"6 \n\t"\
-            "pand      "MM"3, "MM"5 \n\t"\
-            "pandn     "MM"1, "MM"3 \n\t"\
-            "por       "MM"5, "MM"3 \n\t"\
-            MOVQ"      "MM"3, "MM"1 \n\t" /* spatial_pred= (cur[x-refs+j] + cur[x+refs-j])>>1; */
-
-#define CHECK2 /* pretend not to have checked dir=2 if dir=1 was bad.\
-                  hurts both quality and speed, but matches the C version. */\
-            "paddw    "MANGLE(pw_1)", "MM"6 \n\t"\
-            "psllw     $14,   "MM"6 \n\t"\
-            "paddsw    "MM"6, "MM"2 \n\t"\
-            MOVQ"      "MM"0, "MM"3 \n\t"\
-            "pcmpgtw   "MM"2, "MM"3 \n\t"\
-            "pminsw    "MM"2, "MM"0 \n\t"\
-            "pand      "MM"3, "MM"5 \n\t"\
-            "pandn     "MM"1, "MM"3 \n\t"\
-            "por       "MM"5, "MM"3 \n\t"\
-            MOVQ"      "MM"3, "MM"1 \n\t"
-
-static void RENAME(yadif_filter_line)(uint8_t *dst, uint8_t *prev, uint8_t *cur,
-                                      uint8_t *next, int w, int prefs,
-                                      int mrefs, int parity, int mode)
-{
-    DECLARE_ALIGNED(16, uint8_t, tmp)[16*4];
-    int x;
-
-#define FILTER\
-    for(x=0; x<w; x+=STEP){\
-        __asm__ volatile(\
-            "pxor      "MM"7, "MM"7 \n\t"\
-            LOAD("(%[cur],%[mrefs])", MM"0") /* c = cur[x-refs] */\
-            LOAD("(%[cur],%[prefs])", MM"1") /* e = cur[x+refs] */\
-            LOAD("(%["prev2"])", MM"2") /* prev2[x] */\
-            LOAD("(%["next2"])", MM"3") /* next2[x] */\
-            MOVQ"      "MM"3, "MM"4 \n\t"\
-            "paddw     "MM"2, "MM"3 \n\t"\
-            "psraw     $1,    "MM"3 \n\t" /* d = (prev2[x] + next2[x])>>1 */\
-            MOVQ"      "MM"0,   (%[tmp]) \n\t" /* c */\
-            MOVQ"      "MM"3, 16(%[tmp]) \n\t" /* d */\
-            MOVQ"      "MM"1, 32(%[tmp]) \n\t" /* e */\
-            "psubw     "MM"4, "MM"2 \n\t"\
-            PABS(      MM"4", MM"2") /* temporal_diff0 */\
-            LOAD("(%[prev],%[mrefs])", MM"3") /* prev[x-refs] */\
-            LOAD("(%[prev],%[prefs])", MM"4") /* prev[x+refs] */\
-            "psubw     "MM"0, "MM"3 \n\t"\
-            "psubw     "MM"1, "MM"4 \n\t"\
-            PABS(      MM"5", MM"3")\
-            PABS(      MM"5", MM"4")\
-            "paddw     "MM"4, "MM"3 \n\t" /* temporal_diff1 */\
-            "psrlw     $1,    "MM"2 \n\t"\
-            "psrlw     $1,    "MM"3 \n\t"\
-            "pmaxsw    "MM"3, "MM"2 \n\t"\
-            LOAD("(%[next],%[mrefs])", MM"3") /* next[x-refs] */\
-            LOAD("(%[next],%[prefs])", MM"4") /* next[x+refs] */\
-            "psubw     "MM"0, "MM"3 \n\t"\
-            "psubw     "MM"1, "MM"4 \n\t"\
-            PABS(      MM"5", MM"3")\
-            PABS(      MM"5", MM"4")\
-            "paddw     "MM"4, "MM"3 \n\t" /* temporal_diff2 */\
-            "psrlw     $1,    "MM"3 \n\t"\
-            "pmaxsw    "MM"3, "MM"2 \n\t"\
-            MOVQ"      "MM"2, 48(%[tmp]) \n\t" /* diff */\
-\
-            "paddw     "MM"0, "MM"1 \n\t"\
-            "paddw     "MM"0, "MM"0 \n\t"\
-            "psubw     "MM"1, "MM"0 \n\t"\
-            "psrlw     $1,    "MM"1 \n\t" /* spatial_pred */\
-            PABS(      MM"2", MM"0")      /* ABS(c-e) */\
-\
-            MOVQU" -1(%[cur],%[mrefs]), "MM"2 \n\t" /* cur[x-refs-1] */\
-            MOVQU" -1(%[cur],%[prefs]), "MM"3 \n\t" /* cur[x+refs-1] */\
-            MOVQ"      "MM"2, "MM"4 \n\t"\
-            "psubusb   "MM"3, "MM"2 \n\t"\
-            "psubusb   "MM"4, "MM"3 \n\t"\
-            "pmaxub    "MM"3, "MM"2 \n\t"\
-            PSHUF(MM"3", MM"2") \
-            "punpcklbw "MM"7, "MM"2 \n\t" /* ABS(cur[x-refs-1] - cur[x+refs-1]) */\
-            "punpcklbw "MM"7, "MM"3 \n\t" /* ABS(cur[x-refs+1] - cur[x+refs+1]) */\
-            "paddw     "MM"2, "MM"0 \n\t"\
-            "paddw     "MM"3, "MM"0 \n\t"\
-            "psubw    "MANGLE(pw_1)", "MM"0 \n\t" /* spatial_score */\
-\
-            CHECK(-2,0)\
-            CHECK1\
-            CHECK(-3,1)\
-            CHECK2\
-            CHECK(0,-2)\
-            CHECK1\
-            CHECK(1,-3)\
-            CHECK2\
-\
-            /* if(p->mode<2) ... */\
-            MOVQ" 48(%[tmp]), "MM"6 \n\t" /* diff */\
-            "cmpl      $2, %[mode] \n\t"\
-            "jge       1f \n\t"\
-            LOAD("(%["prev2"],%[mrefs],2)", MM"2") /* prev2[x-2*refs] */\
-            LOAD("(%["next2"],%[mrefs],2)", MM"4") /* next2[x-2*refs] */\
-            LOAD("(%["prev2"],%[prefs],2)", MM"3") /* prev2[x+2*refs] */\
-            LOAD("(%["next2"],%[prefs],2)", MM"5") /* next2[x+2*refs] */\
-            "paddw     "MM"4, "MM"2 \n\t"\
-            "paddw     "MM"5, "MM"3 \n\t"\
-            "psrlw     $1,    "MM"2 \n\t" /* b */\
-            "psrlw     $1,    "MM"3 \n\t" /* f */\
-            MOVQ"   (%[tmp]), "MM"4 \n\t" /* c */\
-            MOVQ" 16(%[tmp]), "MM"5 \n\t" /* d */\
-            MOVQ" 32(%[tmp]), "MM"7 \n\t" /* e */\
-            "psubw     "MM"4, "MM"2 \n\t" /* b-c */\
-            "psubw     "MM"7, "MM"3 \n\t" /* f-e */\
-            MOVQ"      "MM"5, "MM"0 \n\t"\
-            "psubw     "MM"4, "MM"5 \n\t" /* d-c */\
-            "psubw     "MM"7, "MM"0 \n\t" /* d-e */\
-            MOVQ"      "MM"2, "MM"4 \n\t"\
-            "pminsw    "MM"3, "MM"2 \n\t"\
-            "pmaxsw    "MM"4, "MM"3 \n\t"\
-            "pmaxsw    "MM"5, "MM"2 \n\t"\
-            "pminsw    "MM"5, "MM"3 \n\t"\
-            "pmaxsw    "MM"0, "MM"2 \n\t" /* max */\
-            "pminsw    "MM"0, "MM"3 \n\t" /* min */\
-            "pxor      "MM"4, "MM"4 \n\t"\
-            "pmaxsw    "MM"3, "MM"6 \n\t"\
-            "psubw     "MM"2, "MM"4 \n\t" /* -max */\
-            "pmaxsw    "MM"4, "MM"6 \n\t" /* diff= MAX3(diff, min, -max); */\
-            "1: \n\t"\
-\
-            MOVQ" 16(%[tmp]), "MM"2 \n\t" /* d */\
-            MOVQ"      "MM"2, "MM"3 \n\t"\
-            "psubw     "MM"6, "MM"2 \n\t" /* d-diff */\
-            "paddw     "MM"6, "MM"3 \n\t" /* d+diff */\
-            "pmaxsw    "MM"2, "MM"1 \n\t"\
-            "pminsw    "MM"3, "MM"1 \n\t" /* d = clip(spatial_pred, d-diff, d+diff); */\
-            "packuswb  "MM"1, "MM"1 \n\t"\
-\
-            ::[prev] "r"(prev),\
-             [cur]  "r"(cur),\
-             [next] "r"(next),\
-             [prefs]"r"((x86_reg)prefs),\
-             [mrefs]"r"((x86_reg)mrefs),\
-             [mode] "g"(mode),\
-             [tmp]  "r"(tmp)\
-        );\
-        __asm__ volatile(MOV" "MM"1, %0" :"=m"(*dst));\
-        dst += STEP;\
-        prev+= STEP;\
-        cur += STEP;\
-        next+= STEP;\
-    }
-
-    if (parity) {
-#define prev2 "prev"
-#define next2 "cur"
-        FILTER
-#undef prev2
-#undef next2
-    } else {
-#define prev2 "cur"
-#define next2 "next"
-        FILTER
-#undef prev2
-#undef next2
-    }
-}
-#undef STEP
-#undef MM
-#undef MOV
-#undef MOVQ
-#undef MOVQU
-#undef PSHUF
-#undef PSRL1
-#undef PSRL2
-#undef LOAD
-#undef PABS
-#undef CHECK
-#undef CHECK1
-#undef CHECK2
-#undef FILTER