Translate libmpeg2 MMX IDCT to plain asm

Originally committed as revision 24580 to svn://svn.ffmpeg.org/ffmpeg/trunk

1 files changed, 237 insertions, 208 deletions

diff --git a/libavcodec/x86/idct_mmx.c b/libavcodec/x86/idct_mmx.c
index d6ab04e8d2..eaefc246d5 100644
--- a/libavcodec/x86/idct_mmx.c
+++ b/libavcodec/x86/idct_mmx.c
@@ -23,8 +23,8 @@
 #include "libavutil/common.h"
 #include "libavcodec/dsputil.h"
 
+#include "libavutil/x86_cpu.h"
 #include "dsputil_mmx.h"
-#include "mmx.h"
 
 #define ROW_SHIFT 11
 #define COL_SHIFT 6
@@ -87,104 +87,115 @@ static inline void idct_row (int16_t * row, int offset,
 static inline void mmxext_row_head (int16_t * const row, const int offset,
                                     const int16_t * const table)
 {
-    movq_m2r (*(row+offset), mm2);      /* mm2 = x6 x4 x2 x0 */
+    __asm__ volatile(
+        "movq     (%0), %%mm2        \n\t"  /* mm2 = x6 x4 x2 x0 */
 
-    movq_m2r (*(row+offset+4), mm5);    /* mm5 = x7 x5 x3 x1 */
-    movq_r2r (mm2, mm0);                /* mm0 = x6 x4 x2 x0 */
+        "movq    8(%0), %%mm5        \n\t"  /* mm5 = x7 x5 x3 x1 */
+        "movq    %%mm2, %%mm0        \n\t"  /* mm0 = x6 x4 x2 x0 */
 
-    movq_m2r (*table, mm3);             /* mm3 = -C2 -C4 C2 C4 */
-    movq_r2r (mm5, mm6);                /* mm6 = x7 x5 x3 x1 */
+        "movq     (%1), %%mm3        \n\t"  /* mm3 = -C2 -C4 C2 C4 */
+        "movq    %%mm5, %%mm6        \n\t"  /* mm6 = x7 x5 x3 x1 */
 
-    movq_m2r (*(table+4), mm4);         /* mm4 = C6 C4 C6 C4 */
-    pmaddwd_r2r (mm0, mm3);             /* mm3 = -C4*x4-C2*x6 C4*x0+C2*x2 */
+        "movq    8(%1), %%mm4        \n\t"  /* mm4 = C6 C4 C6 C4 */
+        "pmaddwd %%mm0, %%mm3        \n\t"  /* mm3 = -C4*x4-C2*x6 C4*x0+C2*x2 */
 
-    pshufw_r2r (mm2, mm2, 0x4e);        /* mm2 = x2 x0 x6 x4 */
+        "pshufw  $0x4e, %%mm2, %%mm2 \n\t"  /* mm2 = x2 x0 x6 x4 */
+        :: "r" ((row+offset)), "r" (table)
+    );
 }
 
 static inline void mmxext_row (const int16_t * const table,
                                const int32_t * const rounder)
 {
-    movq_m2r (*(table+8), mm1);         /* mm1 = -C5 -C1 C3 C1 */
-    pmaddwd_r2r (mm2, mm4);             /* mm4 = C4*x0+C6*x2 C4*x4+C6*x6 */
+    __asm__ volatile (
+        "movq    16(%0), %%mm1         \n\t" /* mm1 = -C5 -C1 C3 C1 */
+        "pmaddwd  %%mm2, %%mm4         \n\t" /* mm4 = C4*x0+C6*x2 C4*x4+C6*x6 */
 
-    pmaddwd_m2r (*(table+16), mm0);     /* mm0 = C4*x4-C6*x6 C4*x0-C6*x2 */
-    pshufw_r2r (mm6, mm6, 0x4e);        /* mm6 = x3 x1 x7 x5 */
+        "pmaddwd 32(%0), %%mm0         \n\t" /* mm0 = C4*x4-C6*x6 C4*x0-C6*x2 */
+        "pshufw   $0x4e, %%mm6, %%mm6  \n\t" /* mm6 = x3 x1 x7 x5 */
 
-    movq_m2r (*(table+12), mm7);        /* mm7 = -C7 C3 C7 C5 */
-    pmaddwd_r2r (mm5, mm1);             /* mm1 = -C1*x5-C5*x7 C1*x1+C3*x3 */
+        "movq    24(%0), %%mm7         \n\t" /* mm7 = -C7 C3 C7 C5 */
+        "pmaddwd  %%mm5, %%mm1         \n\t" /* mm1= -C1*x5-C5*x7 C1*x1+C3*x3 */
 
-    paddd_m2r (*rounder, mm3);          /* mm3 += rounder */
-    pmaddwd_r2r (mm6, mm7);             /* mm7 = C3*x1-C7*x3 C5*x5+C7*x7 */
+        "paddd     (%1), %%mm3         \n\t" /* mm3 += rounder */
+        "pmaddwd  %%mm6, %%mm7         \n\t" /* mm7 = C3*x1-C7*x3 C5*x5+C7*x7 */
 
-    pmaddwd_m2r (*(table+20), mm2);     /* mm2 = C4*x0-C2*x2 -C4*x4+C2*x6 */
-    paddd_r2r (mm4, mm3);               /* mm3 = a1 a0 + rounder */
+        "pmaddwd 40(%0), %%mm2         \n\t" /* mm2= C4*x0-C2*x2 -C4*x4+C2*x6 */
+        "paddd    %%mm4, %%mm3         \n\t" /* mm3 = a1 a0 + rounder */
 
-    pmaddwd_m2r (*(table+24), mm5);     /* mm5 = C3*x5-C1*x7 C5*x1-C1*x3 */
-    movq_r2r (mm3, mm4);                /* mm4 = a1 a0 + rounder */
+        "pmaddwd 48(%0), %%mm5         \n\t" /* mm5 = C3*x5-C1*x7 C5*x1-C1*x3 */
+        "movq     %%mm3, %%mm4         \n\t" /* mm4 = a1 a0 + rounder */
 
-    pmaddwd_m2r (*(table+28), mm6);     /* mm6 = C7*x1-C5*x3 C7*x5+C3*x7 */
-    paddd_r2r (mm7, mm1);               /* mm1 = b1 b0 */
+        "pmaddwd 56(%0), %%mm6         \n\t" /* mm6 = C7*x1-C5*x3 C7*x5+C3*x7 */
+        "paddd    %%mm7, %%mm1         \n\t" /* mm1 = b1 b0 */
 
-    paddd_m2r (*rounder, mm0);          /* mm0 += rounder */
-    psubd_r2r (mm1, mm3);               /* mm3 = a1-b1 a0-b0 + rounder */
+        "paddd     (%1), %%mm0         \n\t" /* mm0 += rounder */
+        "psubd    %%mm1, %%mm3         \n\t" /* mm3 = a1-b1 a0-b0 + rounder */
 
-    psrad_i2r (ROW_SHIFT, mm3);         /* mm3 = y6 y7 */
-    paddd_r2r (mm4, mm1);               /* mm1 = a1+b1 a0+b0 + rounder */
+        "psrad $" AV_STRINGIFY(ROW_SHIFT) ", %%mm3         \n\t" /* mm3 = y6 y7 */
+        "paddd    %%mm4, %%mm1         \n\t" /* mm1 = a1+b1 a0+b0 + rounder */
 
-    paddd_r2r (mm2, mm0);               /* mm0 = a3 a2 + rounder */
-    psrad_i2r (ROW_SHIFT, mm1);         /* mm1 = y1 y0 */
+        "paddd    %%mm2, %%mm0         \n\t" /* mm0 = a3 a2 + rounder */
+        "psrad $" AV_STRINGIFY(ROW_SHIFT) ", %%mm1         \n\t" /* mm1 = y1 y0 */
 
-    paddd_r2r (mm6, mm5);               /* mm5 = b3 b2 */
-    movq_r2r (mm0, mm4);                /* mm4 = a3 a2 + rounder */
+        "paddd    %%mm6, %%mm5         \n\t" /* mm5 = b3 b2 */
+        "movq     %%mm0, %%mm4         \n\t" /* mm4 = a3 a2 + rounder */
 
-    paddd_r2r (mm5, mm0);               /* mm0 = a3+b3 a2+b2 + rounder */
-    psubd_r2r (mm5, mm4);               /* mm4 = a3-b3 a2-b2 + rounder */
+        "paddd    %%mm5, %%mm0         \n\t" /* mm0 = a3+b3 a2+b2 + rounder */
+        "psubd    %%mm5, %%mm4         \n\t" /* mm4 = a3-b3 a2-b2 + rounder */
+        : : "r" (table), "r" (rounder));
 }
 
 static inline void mmxext_row_tail (int16_t * const row, const int store)
 {
-    psrad_i2r (ROW_SHIFT, mm0);         /* mm0 = y3 y2 */
+    __asm__ volatile (
+        "psrad $" AV_STRINGIFY(ROW_SHIFT) ", %%mm0        \n\t"  /* mm0 = y3 y2 */
 
-    psrad_i2r (ROW_SHIFT, mm4);         /* mm4 = y4 y5 */
+        "psrad $" AV_STRINGIFY(ROW_SHIFT) ", %%mm4  \n\t"  /* mm4 = y4 y5 */
 
-    packssdw_r2r (mm0, mm1);            /* mm1 = y3 y2 y1 y0 */
+        "packssdw %%mm0, %%mm1        \n\t"  /* mm1 = y3 y2 y1 y0 */
 
-    packssdw_r2r (mm3, mm4);            /* mm4 = y6 y7 y4 y5 */
+        "packssdw %%mm3, %%mm4        \n\t"  /* mm4 = y6 y7 y4 y5 */
 
-    movq_r2m (mm1, *(row+store));       /* save y3 y2 y1 y0 */
-    pshufw_r2r (mm4, mm4, 0xb1);        /* mm4 = y7 y6 y5 y4 */
+        "movq     %%mm1, (%0)         \n\t"  /* save y3 y2 y1 y0 */
+        "pshufw   $0xb1, %%mm4, %%mm4 \n\t"  /* mm4 = y7 y6 y5 y4 */
 
-    /* slot */
+        /* slot */
 
-    movq_r2m (mm4, *(row+store+4));     /* save y7 y6 y5 y4 */
+        "movq     %%mm4, 8(%0)        \n\t"  /* save y7 y6 y5 y4 */
+        :: "r" (row+store)
+        );
 }
 
 static inline void mmxext_row_mid (int16_t * const row, const int store,
                                    const int offset,
                                    const int16_t * const table)
 {
-    movq_m2r (*(row+offset), mm2);      /* mm2 = x6 x4 x2 x0 */
-    psrad_i2r (ROW_SHIFT, mm0);         /* mm0 = y3 y2 */
+    __asm__ volatile (
+        "movq     (%0,%1), %%mm2       \n\t" /* mm2 = x6 x4 x2 x0 */
+        "psrad $" AV_STRINGIFY(ROW_SHIFT) ", %%mm0 \n\t"   /* mm0 = y3 y2 */
 
-    movq_m2r (*(row+offset+4), mm5);    /* mm5 = x7 x5 x3 x1 */
-    psrad_i2r (ROW_SHIFT, mm4);         /* mm4 = y4 y5 */
+        "movq    8(%0,%1), %%mm5       \n\t" /* mm5 = x7 x5 x3 x1 */
+        "psrad $" AV_STRINGIFY(ROW_SHIFT) ", %%mm4 \n\t" /* mm4 = y4 y5 */
 
-    packssdw_r2r (mm0, mm1);            /* mm1 = y3 y2 y1 y0 */
-    movq_r2r (mm5, mm6);                /* mm6 = x7 x5 x3 x1 */
+        "packssdw   %%mm0, %%mm1       \n\t" /* mm1 = y3 y2 y1 y0 */
+        "movq       %%mm5, %%mm6       \n\t" /* mm6 = x7 x5 x3 x1 */
 
-    packssdw_r2r (mm3, mm4);            /* mm4 = y6 y7 y4 y5 */
-    movq_r2r (mm2, mm0);                /* mm0 = x6 x4 x2 x0 */
+        "packssdw   %%mm3, %%mm4       \n\t" /* mm4 = y6 y7 y4 y5 */
+        "movq       %%mm2, %%mm0       \n\t" /* mm0 = x6 x4 x2 x0 */
 
-    movq_r2m (mm1, *(row+store));       /* save y3 y2 y1 y0 */
-    pshufw_r2r (mm4, mm4, 0xb1);        /* mm4 = y7 y6 y5 y4 */
+        "movq       %%mm1, (%0,%2)     \n\t" /* save y3 y2 y1 y0 */
+        "pshufw     $0xb1, %%mm4, %%mm4\n\t" /* mm4 = y7 y6 y5 y4 */
 
-    movq_m2r (*table, mm3);             /* mm3 = -C2 -C4 C2 C4 */
-    movq_r2m (mm4, *(row+store+4));     /* save y7 y6 y5 y4 */
+        "movq        (%3), %%mm3       \n\t" /* mm3 = -C2 -C4 C2 C4 */
+        "movq       %%mm4, 8(%0,%2)    \n\t" /* save y7 y6 y5 y4 */
 
-    pmaddwd_r2r (mm0, mm3);             /* mm3 = -C4*x4-C2*x6 C4*x0+C2*x2 */
+        "pmaddwd    %%mm0, %%mm3       \n\t" /* mm3= -C4*x4-C2*x6 C4*x0+C2*x2 */
 
-    movq_m2r (*(table+4), mm4);         /* mm4 = C6 C4 C6 C4 */
-    pshufw_r2r (mm2, mm2, 0x4e);        /* mm2 = x2 x0 x6 x4 */
+        "movq       8(%3), %%mm4       \n\t" /* mm4 = C6 C4 C6 C4 */
+        "pshufw     $0x4e, %%mm2, %%mm2\n\t" /* mm2 = x2 x0 x6 x4 */
+        :: "r" (row), "r" (2*offset), "r" (2*store), "r" (table)
+        );
 }
 
 
@@ -202,119 +213,132 @@ static inline void mmxext_row_mid (int16_t * const row, const int store,
 static inline void mmx_row_head (int16_t * const row, const int offset,
                                  const int16_t * const table)
 {
-    movq_m2r (*(row+offset), mm2);      /* mm2 = x6 x4 x2 x0 */
+    __asm__ volatile (
+        "movq (%0), %%mm2       \n\t"    /* mm2 = x6 x4 x2 x0 */
 
-    movq_m2r (*(row+offset+4), mm5);    /* mm5 = x7 x5 x3 x1 */
-    movq_r2r (mm2, mm0);                /* mm0 = x6 x4 x2 x0 */
+        "movq 8(%0), %%mm5      \n\t"    /* mm5 = x7 x5 x3 x1 */
+        "movq %%mm2, %%mm0      \n\t"    /* mm0 = x6 x4 x2 x0 */
 
-    movq_m2r (*table, mm3);             /* mm3 = C6 C4 C2 C4 */
-    movq_r2r (mm5, mm6);                /* mm6 = x7 x5 x3 x1 */
+        "movq (%1), %%mm3       \n\t"    /* mm3 = C6 C4 C2 C4 */
+        "movq %%mm5, %%mm6      \n\t"    /* mm6 = x7 x5 x3 x1 */
 
-    punpckldq_r2r (mm0, mm0);           /* mm0 = x2 x0 x2 x0 */
+        "punpckldq %%mm0, %%mm0 \n\t"    /* mm0 = x2 x0 x2 x0 */
 
-    movq_m2r (*(table+4), mm4);         /* mm4 = -C2 -C4 C6 C4 */
-    pmaddwd_r2r (mm0, mm3);             /* mm3 = C4*x0+C6*x2 C4*x0+C2*x2 */
+        "movq 8(%1), %%mm4      \n\t"    /* mm4 = -C2 -C4 C6 C4 */
+        "pmaddwd %%mm0, %%mm3   \n\t"    /* mm3 = C4*x0+C6*x2 C4*x0+C2*x2 */
 
-    movq_m2r (*(table+8), mm1);         /* mm1 = -C7 C3 C3 C1 */
-    punpckhdq_r2r (mm2, mm2);           /* mm2 = x6 x4 x6 x4 */
+        "movq 16(%1), %%mm1     \n\t"    /* mm1 = -C7 C3 C3 C1 */
+        "punpckhdq %%mm2, %%mm2 \n\t"    /* mm2 = x6 x4 x6 x4 */
+        :: "r" ((row+offset)), "r" (table)
+        );
 }
 
 static inline void mmx_row (const int16_t * const table,
                             const int32_t * const rounder)
 {
-    pmaddwd_r2r (mm2, mm4);             /* mm4 = -C4*x4-C2*x6 C4*x4+C6*x6 */
-    punpckldq_r2r (mm5, mm5);           /* mm5 = x3 x1 x3 x1 */
+    __asm__ volatile (
+        "pmaddwd   %%mm2, %%mm4    \n\t"  /* mm4 = -C4*x4-C2*x6 C4*x4+C6*x6 */
+        "punpckldq %%mm5, %%mm5    \n\t"  /* mm5 = x3 x1 x3 x1 */
 
-    pmaddwd_m2r (*(table+16), mm0);     /* mm0 = C4*x0-C2*x2 C4*x0-C6*x2 */
-    punpckhdq_r2r (mm6, mm6);           /* mm6 = x7 x5 x7 x5 */
+        "pmaddwd  32(%0), %%mm0    \n\t"  /* mm0 = C4*x0-C2*x2 C4*x0-C6*x2 */
+        "punpckhdq %%mm6, %%mm6    \n\t"  /* mm6 = x7 x5 x7 x5 */
 
-    movq_m2r (*(table+12), mm7);        /* mm7 = -C5 -C1 C7 C5 */
-    pmaddwd_r2r (mm5, mm1);             /* mm1 = C3*x1-C7*x3 C1*x1+C3*x3 */
+        "movq     24(%0), %%mm7    \n\t"  /* mm7 = -C5 -C1 C7 C5 */
+        "pmaddwd   %%mm5, %%mm1    \n\t"  /* mm1 = C3*x1-C7*x3 C1*x1+C3*x3 */
 
-    paddd_m2r (*rounder, mm3);          /* mm3 += rounder */
-    pmaddwd_r2r (mm6, mm7);             /* mm7 = -C1*x5-C5*x7 C5*x5+C7*x7 */
+        "paddd      (%1), %%mm3    \n\t"  /* mm3 += rounder */
+        "pmaddwd   %%mm6, %%mm7    \n\t"  /* mm7 = -C1*x5-C5*x7 C5*x5+C7*x7 */
 
-    pmaddwd_m2r (*(table+20), mm2);     /* mm2 = C4*x4-C6*x6 -C4*x4+C2*x6 */
-    paddd_r2r (mm4, mm3);               /* mm3 = a1 a0 + rounder */
+        "pmaddwd  40(%0), %%mm2    \n\t"  /* mm2 = C4*x4-C6*x6 -C4*x4+C2*x6 */
+        "paddd     %%mm4, %%mm3    \n\t"  /* mm3 = a1 a0 + rounder */
 
-    pmaddwd_m2r (*(table+24), mm5);     /* mm5 = C7*x1-C5*x3 C5*x1-C1*x3 */
-    movq_r2r (mm3, mm4);                /* mm4 = a1 a0 + rounder */
+        "pmaddwd  48(%0), %%mm5    \n\t"  /* mm5 = C7*x1-C5*x3 C5*x1-C1*x3 */
+        "movq      %%mm3, %%mm4    \n\t"  /* mm4 = a1 a0 + rounder */
 
-    pmaddwd_m2r (*(table+28), mm6);     /* mm6 = C3*x5-C1*x7 C7*x5+C3*x7 */
-    paddd_r2r (mm7, mm1);               /* mm1 = b1 b0 */
+        "pmaddwd  56(%0), %%mm6    \n\t"  /* mm6 = C3*x5-C1*x7 C7*x5+C3*x7 */
+        "paddd     %%mm7, %%mm1    \n\t"  /* mm1 = b1 b0 */
 
-    paddd_m2r (*rounder, mm0);          /* mm0 += rounder */
-    psubd_r2r (mm1, mm3);               /* mm3 = a1-b1 a0-b0 + rounder */
+        "paddd      (%1), %%mm0    \n\t"  /* mm0 += rounder */
+        "psubd     %%mm1, %%mm3    \n\t"  /* mm3 = a1-b1 a0-b0 + rounder */
 
-    psrad_i2r (ROW_SHIFT, mm3);         /* mm3 = y6 y7 */
-    paddd_r2r (mm4, mm1);               /* mm1 = a1+b1 a0+b0 + rounder */
+        "psrad $" AV_STRINGIFY(ROW_SHIFT) ", %%mm3    \n\t"  /* mm3 = y6 y7 */
+        "paddd     %%mm4, %%mm1    \n\t"  /* mm1 = a1+b1 a0+b0 + rounder */
 
-    paddd_r2r (mm2, mm0);               /* mm0 = a3 a2 + rounder */
-    psrad_i2r (ROW_SHIFT, mm1);         /* mm1 = y1 y0 */
+        "paddd     %%mm2, %%mm0    \n\t"  /* mm0 = a3 a2 + rounder */
+        "psrad $" AV_STRINGIFY(ROW_SHIFT) ", %%mm1    \n\t"  /* mm1 = y1 y0 */
 
-    paddd_r2r (mm6, mm5);               /* mm5 = b3 b2 */
-    movq_r2r (mm0, mm7);                /* mm7 = a3 a2 + rounder */
+        "paddd     %%mm6, %%mm5    \n\t"  /* mm5 = b3 b2 */
+        "movq      %%mm0, %%mm7    \n\t"  /* mm7 = a3 a2 + rounder */
 
-    paddd_r2r (mm5, mm0);               /* mm0 = a3+b3 a2+b2 + rounder */
-    psubd_r2r (mm5, mm7);               /* mm7 = a3-b3 a2-b2 + rounder */
+        "paddd     %%mm5, %%mm0    \n\t"  /* mm0 = a3+b3 a2+b2 + rounder */
+        "psubd     %%mm5, %%mm7    \n\t"  /* mm7 = a3-b3 a2-b2 + rounder */
+        :: "r" (table), "r" (rounder)
+        );
 }
 
 static inline void mmx_row_tail (int16_t * const row, const int store)
 {
-    psrad_i2r (ROW_SHIFT, mm0);         /* mm0 = y3 y2 */
+    __asm__ volatile (
+        "psrad $" AV_STRINGIFY(ROW_SHIFT) ", %%mm0      \n\t" /* mm0 = y3 y2 */
 
-    psrad_i2r (ROW_SHIFT, mm7);         /* mm7 = y4 y5 */
+        "psrad $" AV_STRINGIFY(ROW_SHIFT) ", %%mm7      \n\t" /* mm7 = y4 y5 */
 
-    packssdw_r2r (mm0, mm1);            /* mm1 = y3 y2 y1 y0 */
+        "packssdw %%mm0, %%mm1 \n\t" /* mm1 = y3 y2 y1 y0 */
 
-    packssdw_r2r (mm3, mm7);            /* mm7 = y6 y7 y4 y5 */
+        "packssdw %%mm3, %%mm7 \n\t" /* mm7 = y6 y7 y4 y5 */
 
-    movq_r2m (mm1, *(row+store));       /* save y3 y2 y1 y0 */
-    movq_r2r (mm7, mm4);                /* mm4 = y6 y7 y4 y5 */
+        "movq %%mm1, (%0)      \n\t" /* save y3 y2 y1 y0 */
+        "movq %%mm7, %%mm4     \n\t" /* mm4 = y6 y7 y4 y5 */
 
-    pslld_i2r (16, mm7);                /* mm7 = y7 0 y5 0 */
+        "pslld $16, %%mm7      \n\t" /* mm7 = y7 0 y5 0 */
 
-    psrld_i2r (16, mm4);                /* mm4 = 0 y6 0 y4 */
+        "psrld $16, %%mm4      \n\t" /* mm4 = 0 y6 0 y4 */
 
-    por_r2r (mm4, mm7);                 /* mm7 = y7 y6 y5 y4 */
+        "por %%mm4, %%mm7      \n\t" /* mm7 = y7 y6 y5 y4 */
 
-    /* slot */
+        /* slot */
 
-    movq_r2m (mm7, *(row+store+4));     /* save y7 y6 y5 y4 */
+        "movq %%mm7, 8(%0)     \n\t" /* save y7 y6 y5 y4 */
+        :: "r" (row+store)
+        );
 }
 
 static inline void mmx_row_mid (int16_t * const row, const int store,
                                 const int offset, const int16_t * const table)
 {
-    movq_m2r (*(row+offset), mm2);      /* mm2 = x6 x4 x2 x0 */
-    psrad_i2r (ROW_SHIFT, mm0);         /* mm0 = y3 y2 */
 
-    movq_m2r (*(row+offset+4), mm5);    /* mm5 = x7 x5 x3 x1 */
-    psrad_i2r (ROW_SHIFT, mm7);         /* mm7 = y4 y5 */
+    __asm__ volatile (
+        "movq    (%0,%1), %%mm2    \n\t" /* mm2 = x6 x4 x2 x0 */
+        "psrad $" AV_STRINGIFY(ROW_SHIFT) ", %%mm0 \n\t" /* mm0 = y3 y2 */
 
-    packssdw_r2r (mm0, mm1);            /* mm1 = y3 y2 y1 y0 */
-    movq_r2r (mm5, mm6);                /* mm6 = x7 x5 x3 x1 */
+        "movq   8(%0,%1), %%mm5    \n\t" /* mm5 = x7 x5 x3 x1 */
+        "psrad $" AV_STRINGIFY(ROW_SHIFT) ", %%mm7 \n\t" /* mm7 = y4 y5 */
 
-    packssdw_r2r (mm3, mm7);            /* mm7 = y6 y7 y4 y5 */
-    movq_r2r (mm2, mm0);                /* mm0 = x6 x4 x2 x0 */
+        "packssdw  %%mm0, %%mm1    \n\t" /* mm1 = y3 y2 y1 y0 */
+        "movq      %%mm5, %%mm6    \n\t" /* mm6 = x7 x5 x3 x1 */
 
-    movq_r2m (mm1, *(row+store));       /* save y3 y2 y1 y0 */
-    movq_r2r (mm7, mm1);                /* mm1 = y6 y7 y4 y5 */
+        "packssdw  %%mm3, %%mm7    \n\t" /* mm7 = y6 y7 y4 y5 */
+        "movq      %%mm2, %%mm0    \n\t" /* mm0 = x6 x4 x2 x0 */
 
-    punpckldq_r2r (mm0, mm0);           /* mm0 = x2 x0 x2 x0 */
-    psrld_i2r (16, mm7);                /* mm7 = 0 y6 0 y4 */
+        "movq      %%mm1, (%0,%2)  \n\t" /* save y3 y2 y1 y0 */
+        "movq      %%mm7, %%mm1    \n\t" /* mm1 = y6 y7 y4 y5 */
 
-    movq_m2r (*table, mm3);             /* mm3 = C6 C4 C2 C4 */
-    pslld_i2r (16, mm1);                /* mm1 = y7 0 y5 0 */
+        "punpckldq %%mm0, %%mm0    \n\t" /* mm0 = x2 x0 x2 x0 */
+        "psrld       $16, %%mm7    \n\t" /* mm7 = 0 y6 0 y4 */
 
-    movq_m2r (*(table+4), mm4);         /* mm4 = -C2 -C4 C6 C4 */
-    por_r2r (mm1, mm7);                 /* mm7 = y7 y6 y5 y4 */
+        "movq       (%3), %%mm3    \n\t" /* mm3 = C6 C4 C2 C4 */
+        "pslld       $16, %%mm1    \n\t" /* mm1 = y7 0 y5 0 */
 
-    movq_m2r (*(table+8), mm1);         /* mm1 = -C7 C3 C3 C1 */
-    punpckhdq_r2r (mm2, mm2);           /* mm2 = x6 x4 x6 x4 */
+        "movq      8(%3), %%mm4    \n\t" /* mm4 = -C2 -C4 C6 C4 */
+        "por       %%mm1, %%mm7    \n\t" /* mm7 = y7 y6 y5 y4 */
 
-    movq_r2m (mm7, *(row+store+4));     /* save y7 y6 y5 y4 */
-    pmaddwd_r2r (mm0, mm3);             /* mm3 = C4*x0+C6*x2 C4*x0+C2*x2 */
+        "movq     16(%3), %%mm1    \n\t" /* mm1 = -C7 C3 C3 C1 */
+        "punpckhdq %%mm2, %%mm2    \n\t" /* mm2 = x6 x4 x6 x4 */
+
+        "movq      %%mm7, 8(%0,%2) \n\t" /* save y7 y6 y5 y4 */
+        "pmaddwd   %%mm0, %%mm3    \n\t" /* mm3 = C4*x0+C6*x2 C4*x0+C2*x2 */
+        : : "r" (row), "r"(2*offset), "r" (2*store), "r" (table)
+        );
 }
 
 
@@ -398,140 +422,145 @@ static inline void idct_col (int16_t * const col, const int offset)
 #define T3 43790
 #define C4 23170
 
-    DECLARE_ALIGNED(8, static const short, t1_vector)[] = {T1,T1,T1,T1};
-    DECLARE_ALIGNED(8, static const short, t2_vector)[] = {T2,T2,T2,T2};
-    DECLARE_ALIGNED(8, static const short, t3_vector)[] = {T3,T3,T3,T3};
-    DECLARE_ALIGNED(8, static const short, c4_vector)[] = {C4,C4,C4,C4};
+    DECLARE_ALIGNED(8, static const short, t1_vector)[] = {
+        T1,T1,T1,T1,
+        T2,T2,T2,T2,
+        T3,T3,T3,T3,
+        C4,C4,C4,C4
+    };
 
     /* column code adapted from Peter Gubanov */
     /* http://www.elecard.com/peter/idct.shtml */
 
-    movq_m2r (*t1_vector, mm0);         /* mm0 = T1 */
+    __asm__ volatile (
+        "movq      (%0), %%mm0    \n\t" /* mm0 = T1 */
 
-    movq_m2r (*(col+offset+1*8), mm1);  /* mm1 = x1 */
-    movq_r2r (mm0, mm2);                /* mm2 = T1 */
+        "movq   2*8(%1), %%mm1    \n\t" /* mm1 = x1 */
+        "movq     %%mm0, %%mm2    \n\t" /* mm2 = T1 */
 
-    movq_m2r (*(col+offset+7*8), mm4);  /* mm4 = x7 */
-    pmulhw_r2r (mm1, mm0);              /* mm0 = T1*x1 */
+        "movq 7*2*8(%1), %%mm4    \n\t" /* mm4 = x7 */
+        "pmulhw   %%mm1, %%mm0    \n\t" /* mm0 = T1*x1 */
 
-    movq_m2r (*t3_vector, mm5);         /* mm5 = T3 */
-    pmulhw_r2r (mm4, mm2);              /* mm2 = T1*x7 */
+        "movq    16(%0), %%mm5    \n\t" /* mm5 = T3 */
+        "pmulhw   %%mm4, %%mm2    \n\t" /* mm2 = T1*x7 */
 
-    movq_m2r (*(col+offset+5*8), mm6);  /* mm6 = x5 */
-    movq_r2r (mm5, mm7);                /* mm7 = T3-1 */
+        "movq 2*5*8(%1), %%mm6    \n\t" /* mm6 = x5 */
+        "movq     %%mm5, %%mm7    \n\t" /* mm7 = T3-1 */
 
-    movq_m2r (*(col+offset+3*8), mm3);  /* mm3 = x3 */
-    psubsw_r2r (mm4, mm0);              /* mm0 = v17 */
+        "movq 3*8*2(%1), %%mm3    \n\t" /* mm3 = x3 */
+        "psubsw   %%mm4, %%mm0    \n\t" /* mm0 = v17 */
 
-    movq_m2r (*t2_vector, mm4);         /* mm4 = T2 */
-    pmulhw_r2r (mm3, mm5);              /* mm5 = (T3-1)*x3 */
+        "movq     8(%0), %%mm4    \n\t" /* mm4 = T2 */
+        "pmulhw   %%mm3, %%mm5    \n\t" /* mm5 = (T3-1)*x3 */
 
-    paddsw_r2r (mm2, mm1);              /* mm1 = u17 */
-    pmulhw_r2r (mm6, mm7);              /* mm7 = (T3-1)*x5 */
+        "paddsw   %%mm2, %%mm1    \n\t" /* mm1 = u17 */
+        "pmulhw   %%mm6, %%mm7    \n\t" /* mm7 = (T3-1)*x5 */
 
-    /* slot */
+        /* slot */
 
-    movq_r2r (mm4, mm2);                /* mm2 = T2 */
-    paddsw_r2r (mm3, mm5);              /* mm5 = T3*x3 */
+        "movq     %%mm4, %%mm2    \n\t" /* mm2 = T2 */
+        "paddsw   %%mm3, %%mm5    \n\t" /* mm5 = T3*x3 */
 
-    pmulhw_m2r (*(col+offset+2*8), mm4);/* mm4 = T2*x2 */
-    paddsw_r2r (mm6, mm7);              /* mm7 = T3*x5 */
+        "pmulhw 2*8*2(%1), %%mm4  \n\t" /* mm4 = T2*x2 */
+        "paddsw   %%mm6, %%mm7    \n\t" /* mm7 = T3*x5 */
 
-    psubsw_r2r (mm6, mm5);              /* mm5 = v35 */
-    paddsw_r2r (mm3, mm7);              /* mm7 = u35 */
+        "psubsw   %%mm6, %%mm5    \n\t" /* mm5 = v35 */
+        "paddsw   %%mm3, %%mm7    \n\t" /* mm7 = u35 */
 
-    movq_m2r (*(col+offset+6*8), mm3);  /* mm3 = x6 */
-    movq_r2r (mm0, mm6);                /* mm6 = v17 */
+        "movq 6*8*2(%1), %%mm3    \n\t" /* mm3 = x6 */
+        "movq     %%mm0, %%mm6    \n\t" /* mm6 = v17 */
 
-    pmulhw_r2r (mm3, mm2);              /* mm2 = T2*x6 */
-    psubsw_r2r (mm5, mm0);              /* mm0 = b3 */
+        "pmulhw   %%mm3, %%mm2    \n\t" /* mm2 = T2*x6 */
+        "psubsw   %%mm5, %%mm0    \n\t" /* mm0 = b3 */
 
-    psubsw_r2r (mm3, mm4);              /* mm4 = v26 */
-    paddsw_r2r (mm6, mm5);              /* mm5 = v12 */
+        "psubsw   %%mm3, %%mm4    \n\t" /* mm4 = v26 */
+        "paddsw   %%mm6, %%mm5    \n\t" /* mm5 = v12 */
 
-    movq_r2m (mm0, *(col+offset+3*8));  /* save b3 in scratch0 */
-    movq_r2r (mm1, mm6);                /* mm6 = u17 */
+        "movq     %%mm0, 3*8*2(%1)\n\t" /* save b3 in scratch0 */
+        "movq     %%mm1, %%mm6    \n\t" /* mm6 = u17 */
 
-    paddsw_m2r (*(col+offset+2*8), mm2);/* mm2 = u26 */
-    paddsw_r2r (mm7, mm6);              /* mm6 = b0 */
+        "paddsw 2*8*2(%1), %%mm2  \n\t" /* mm2 = u26 */
+        "paddsw   %%mm7, %%mm6    \n\t" /* mm6 = b0 */
 
-    psubsw_r2r (mm7, mm1);              /* mm1 = u12 */
-    movq_r2r (mm1, mm7);                /* mm7 = u12 */
+        "psubsw   %%mm7, %%mm1    \n\t" /* mm1 = u12 */
+        "movq     %%mm1, %%mm7    \n\t" /* mm7 = u12 */
 
-    movq_m2r (*(col+offset+0*8), mm3);  /* mm3 = x0 */
-    paddsw_r2r (mm5, mm1);              /* mm1 = u12+v12 */
+        "movq   0*8(%1), %%mm3    \n\t" /* mm3 = x0 */
+        "paddsw   %%mm5, %%mm1    \n\t" /* mm1 = u12+v12 */
 
-    movq_m2r (*c4_vector, mm0);         /* mm0 = C4/2 */
-    psubsw_r2r (mm5, mm7);              /* mm7 = u12-v12 */
+        "movq    24(%0), %%mm0    \n\t" /* mm0 = C4/2 */
+        "psubsw   %%mm5, %%mm7    \n\t" /* mm7 = u12-v12 */
 
-    movq_r2m (mm6, *(col+offset+5*8));  /* save b0 in scratch1 */
-    pmulhw_r2r (mm0, mm1);              /* mm1 = b1/2 */
+        "movq     %%mm6, 5*8*2(%1)\n\t" /* save b0 in scratch1 */
+        "pmulhw   %%mm0, %%mm1    \n\t" /* mm1 = b1/2 */
 
-    movq_r2r (mm4, mm6);                /* mm6 = v26 */
-    pmulhw_r2r (mm0, mm7);              /* mm7 = b2/2 */
+        "movq     %%mm4, %%mm6    \n\t" /* mm6 = v26 */
+        "pmulhw   %%mm0, %%mm7    \n\t" /* mm7 = b2/2 */
 
-    movq_m2r (*(col+offset+4*8), mm5);  /* mm5 = x4 */
-    movq_r2r (mm3, mm0);                /* mm0 = x0 */
+        "movq 4*8*2(%1), %%mm5    \n\t" /* mm5 = x4 */
+        "movq     %%mm3, %%mm0    \n\t" /* mm0 = x0 */
 
-    psubsw_r2r (mm5, mm3);              /* mm3 = v04 */
-    paddsw_r2r (mm5, mm0);              /* mm0 = u04 */
+        "psubsw   %%mm5, %%mm3    \n\t" /* mm3 = v04 */
+        "paddsw   %%mm5, %%mm0    \n\t" /* mm0 = u04 */
 
-    paddsw_r2r (mm3, mm4);              /* mm4 = a1 */
-    movq_r2r (mm0, mm5);                /* mm5 = u04 */
+        "paddsw   %%mm3, %%mm4    \n\t" /* mm4 = a1 */
+        "movq     %%mm0, %%mm5    \n\t" /* mm5 = u04 */
 
-    psubsw_r2r (mm6, mm3);              /* mm3 = a2 */
-    paddsw_r2r (mm2, mm5);              /* mm5 = a0 */
+        "psubsw   %%mm6, %%mm3    \n\t" /* mm3 = a2 */
+        "paddsw   %%mm2, %%mm5    \n\t" /* mm5 = a0 */
 
-    paddsw_r2r (mm1, mm1);              /* mm1 = b1 */
-    psubsw_r2r (mm2, mm0);              /* mm0 = a3 */
+        "paddsw   %%mm1, %%mm1    \n\t" /* mm1 = b1 */
+        "psubsw   %%mm2, %%mm0    \n\t" /* mm0 = a3 */
 
-    paddsw_r2r (mm7, mm7);              /* mm7 = b2 */
-    movq_r2r (mm3, mm2);                /* mm2 = a2 */
+        "paddsw   %%mm7, %%mm7    \n\t" /* mm7 = b2 */
+        "movq     %%mm3, %%mm2    \n\t" /* mm2 = a2 */
 
-    movq_r2r (mm4, mm6);                /* mm6 = a1 */
-    paddsw_r2r (mm7, mm3);              /* mm3 = a2+b2 */
+        "movq     %%mm4, %%mm6    \n\t" /* mm6 = a1 */
+        "paddsw   %%mm7, %%mm3    \n\t" /* mm3 = a2+b2 */
 
-    psraw_i2r (COL_SHIFT, mm3);         /* mm3 = y2 */
-    paddsw_r2r (mm1, mm4);              /* mm4 = a1+b1 */
+        "psraw $" AV_STRINGIFY(COL_SHIFT) ", %%mm3\n\t" /* mm3 = y2 */
+        "paddsw   %%mm1, %%mm4\n\t" /* mm4 = a1+b1 */
 
-    psraw_i2r (COL_SHIFT, mm4);         /* mm4 = y1 */
-    psubsw_r2r (mm1, mm6);              /* mm6 = a1-b1 */
+        "psraw $" AV_STRINGIFY(COL_SHIFT) ", %%mm4\n\t" /* mm4 = y1 */
+        "psubsw   %%mm1, %%mm6    \n\t" /* mm6 = a1-b1 */
 
-    movq_m2r (*(col+offset+5*8), mm1);  /* mm1 = b0 */
-    psubsw_r2r (mm7, mm2);              /* mm2 = a2-b2 */
+        "movq 5*8*2(%1), %%mm1    \n\t" /* mm1 = b0 */
+        "psubsw   %%mm7, %%mm2    \n\t" /* mm2 = a2-b2 */
 
-    psraw_i2r (COL_SHIFT, mm6);         /* mm6 = y6 */
-    movq_r2r (mm5, mm7);                /* mm7 = a0 */
+        "psraw $" AV_STRINGIFY(COL_SHIFT) ", %%mm6\n\t" /* mm6 = y6 */
+        "movq     %%mm5, %%mm7    \n\t" /* mm7 = a0 */
 
-    movq_r2m (mm4, *(col+offset+1*8));  /* save y1 */
-    psraw_i2r (COL_SHIFT, mm2);         /* mm2 = y5 */
+        "movq     %%mm4, 1*8*2(%1)\n\t" /* save y1 */
+        "psraw $" AV_STRINGIFY(COL_SHIFT) ", %%mm2\n\t" /* mm2 = y5 */
 
-    movq_r2m (mm3, *(col+offset+2*8));  /* save y2 */
-    paddsw_r2r (mm1, mm5);              /* mm5 = a0+b0 */
+        "movq     %%mm3, 2*8*2(%1)\n\t" /* save y2 */
+        "paddsw   %%mm1, %%mm5    \n\t" /* mm5 = a0+b0 */
 
-    movq_m2r (*(col+offset+3*8), mm4);  /* mm4 = b3 */
-    psubsw_r2r (mm1, mm7);              /* mm7 = a0-b0 */
+        "movq 3*8*2(%1), %%mm4    \n\t" /* mm4 = b3 */
+        "psubsw   %%mm1, %%mm7    \n\t" /* mm7 = a0-b0 */
 
-    psraw_i2r (COL_SHIFT, mm5);         /* mm5 = y0 */
-    movq_r2r (mm0, mm3);                /* mm3 = a3 */
+        "psraw $" AV_STRINGIFY(COL_SHIFT) ", %%mm5\n\t" /* mm5 = y0 */
+        "movq     %%mm0, %%mm3    \n\t" /* mm3 = a3 */
 
-    movq_r2m (mm2, *(col+offset+5*8));  /* save y5 */
-    psubsw_r2r (mm4, mm3);              /* mm3 = a3-b3 */
+        "movq     %%mm2, 5*8*2(%1)\n\t" /* save y5 */
+        "psubsw   %%mm4, %%mm3    \n\t" /* mm3 = a3-b3 */
 
-    psraw_i2r (COL_SHIFT, mm7);         /* mm7 = y7 */
-    paddsw_r2r (mm0, mm4);              /* mm4 = a3+b3 */
+        "psraw $" AV_STRINGIFY(COL_SHIFT) ", %%mm7\n\t" /* mm7 = y7 */
+        "paddsw   %%mm0, %%mm4    \n\t" /* mm4 = a3+b3 */
 
-    movq_r2m (mm5, *(col+offset+0*8));  /* save y0 */
-    psraw_i2r (COL_SHIFT, mm3);         /* mm3 = y4 */
+        "movq     %%mm5, 0*8*2(%1)\n\t" /* save y0 */
+        "psraw $" AV_STRINGIFY(COL_SHIFT) ", %%mm3\n\t" /* mm3 = y4 */
 
-    movq_r2m (mm6, *(col+offset+6*8));  /* save y6 */
-    psraw_i2r (COL_SHIFT, mm4);         /* mm4 = y3 */
+        "movq     %%mm6, 6*8*2(%1)\n\t" /* save y6 */
+        "psraw $" AV_STRINGIFY(COL_SHIFT) ", %%mm4\n\t" /* mm4 = y3 */
 
-    movq_r2m (mm7, *(col+offset+7*8));  /* save y7 */
+        "movq     %%mm7, 7*8*2(%1)\n\t" /* save y7 */
 
-    movq_r2m (mm3, *(col+offset+4*8));  /* save y4 */
+        "movq     %%mm3, 4*8*2(%1)\n\t" /* save y4 */
 
-    movq_r2m (mm4, *(col+offset+3*8));  /* save y3 */
+        "movq     %%mm4, 3*8*2(%1)\n\t" /* save y3 */
+        :: "r" (t1_vector), "r" (col+offset)
+        );
 
 #undef T1
 #undef T2