path: root/libswscale/x86/ops_int.asm

;******************************************************************************
;* Copyright (c) 2025 Niklas Haas
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************

%include "ops_common.asm"

SECTION_RODATA

align 16
expand16_shuf:  db   0,  0,  2,  2,  4,  4,  6,  6,  8,  8, 10, 10, 12, 12, 14, 14
expand32_shuf:  db   0,  0,  0,  0,  4,  4,  4,  4,  8,  8,  8,  8, 12, 12, 12, 12

read8_unpack2:  db   0,  2,  4,  6,  8, 10, 12, 14,  1,  3,  5,  7,  9, 11, 13, 15
read8_unpack3:  db   0,  3,  6,  9,  1,  4,  7, 10,  2,  5,  8, 11, -1, -1, -1, -1
read8_unpack4:  db   0,  4,  8, 12,  1,  5,  9, 13,  2,  6, 10, 14,  3,  7, 11, 15
read16_unpack2: db   0,  1,  4,  5,  8,  9, 12, 13,  2,  3,  6,  7, 10, 11, 14, 15
read16_unpack3: db   0,  1,  6,  7,  2,  3,  8,  9,  4,  5, 10, 11, -1, -1, -1, -1
read16_unpack4: db   0,  1,  8,  9,  2,  3, 10, 11,  4,  5, 12, 13,  6,  7, 14, 15
write8_pack2:   db   0,  8,  1,  9,  2, 10,  3, 11,  4, 12,  5, 13,  6, 14,  7, 15
write8_pack3:   db   0,  4,  8,  1,  5,  9,  2,  6, 10,  3,  7, 11, -1, -1, -1, -1
write16_pack3:  db   0,  1,  4,  5,  8,  9,  2,  3,  6,  7, 10, 11, -1, -1, -1, -1

%define write8_pack4  read8_unpack4
%define write16_pack4 read16_unpack2
%define write16_pack2 read16_unpack4

align 32
bits_shuf:      db   0,  0,  0,  0,  0,  0,  0,  0,  1,  1,  1,  1,  1,  1,  1,  1, \
                     2,  2,  2,  2,  2,  2,  2,  2,  3,  3,  3,  3,  3,  3,  3,  3
bits_mask:      db 128, 64, 32, 16,  8,  4,  2,  1,128, 64, 32, 16,  8,  4,  2,  1
bits_reverse:   db   7,  6,  5,  4,  3,  2,  1,  0, 15, 14, 13, 12, 11, 10,  9,  8,

align 32
mask1: times 32 db 0x01
mask2: times 32 db 0x03
mask3: times 32 db 0x07
mask4: times 32 db 0x0F

SECTION .text

;---------------------------------------------------------
; Global entry point. See `ops_common.asm` for info.

%macro process_fn 1 ; number of planes
cglobal sws_process%1_x86, 6, 6 + 2 * %1, 16
            ; Args:
            ;   execq, implq, bxd, yd as defined in ops_common.int
            ;   bx_end and y_end are initially in tmp0d / tmp1d
            ;   (see SwsOpFunc signature)
            ;
            ; Stack layout:
            ;   [rsp +  0] = [qword] impl->cont (address of first kernel)
            ;   [rsp +  8] = [qword] &impl[1]   (restore implq after chain)
            ;   [rsp + 16] = [dword] bx start   (restore after line finish)
            ;   [rsp + 20] = [dword] bx end     (loop counter limit)
            ;   [rsp + 24] = [dword] y end      (loop counter limit)
            sub rsp, 32
            mov [rsp + 16], bxd
            mov [rsp + 20], tmp0d ; bx_end
            mov [rsp + 24], tmp1d ; y_end
            mov tmp0q, [implq + SwsOpImpl.cont]
            add implq, SwsOpImpl.next
            mov [rsp +  0], tmp0q
            mov [rsp +  8], implq

            ; load plane pointers
            mov in0q,  [execq + SwsOpExec.in0]
IF %1 > 1,  mov in1q,  [execq + SwsOpExec.in1]
IF %1 > 2,  mov in2q,  [execq + SwsOpExec.in2]
IF %1 > 3,  mov in3q,  [execq + SwsOpExec.in3]
            mov out0q, [execq + SwsOpExec.out0]
IF %1 > 1,  mov out1q, [execq + SwsOpExec.out1]
IF %1 > 2,  mov out2q, [execq + SwsOpExec.out2]
IF %1 > 3,  mov out3q, [execq + SwsOpExec.out3]
            jmp [rsp] ; call into op chain

; Declare a separate global label for the return point, so that we can append
; it to the list of op function pointers from the C code, effectively ensuring
; that we end up here again after the op chain finishes processing a line.
; (See also: cglobal_label in x86inc.asm)
%if FORMAT_ELF
    global current_function %+ _return:function hidden
%elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN
    global current_function %+ _return:private_extern
%else
    global current_function %+ _return
%endif
align function_align
current_function %+ _return:

            ; op chain always returns back here
            mov implq, [rsp + 8]
            inc bxd
            cmp bxd, [rsp + 20]
            jne .continue
            ; end of line
            inc yd
            cmp yd, [rsp + 24]
            je .end
            ; bump addresses to point to start of next line
            add in0q,  [execq + SwsOpExec.in_bump0]
IF %1 > 1,  add in1q,  [execq + SwsOpExec.in_bump1]
IF %1 > 2,  add in2q,  [execq + SwsOpExec.in_bump2]
IF %1 > 3,  add in3q,  [execq + SwsOpExec.in_bump3]
            add out0q, [execq + SwsOpExec.out_bump0]
IF %1 > 1,  add out1q, [execq + SwsOpExec.out_bump1]
IF %1 > 2,  add out2q, [execq + SwsOpExec.out_bump2]
IF %1 > 3,  add out3q, [execq + SwsOpExec.out_bump3]
            mov bxd, [rsp + 16]
.continue:
            jmp [rsp]
.end:
            add rsp, 32
            RET
%endmacro

process_fn 1
process_fn 2
process_fn 3
process_fn 4

;---------------------------------------------------------
; Packed shuffle fast-path

; This is a special entry point for handling a subset of operation chains
; that can be reduced down to a single `pshufb` shuffle mask. For more details
; about when this works, refer to the documentation of `ff_sws_solve_shuffle`.
;
; We specialize this function for every possible combination of pixel strides.
; For example, gray -> gray16 is classified as an "8, 16" operation because it
; takes 8 bytes and expands them out to 16 bytes in each application of the
; 128-bit shuffle mask.
;
; Since pshufb can't shuffle across lanes, we only instantiate SSE4 versions for
; all shuffles that are not a clean multiple of 128 bits (e.g. rgb24 -> rgb0).
; For the clean multiples (e.g. rgba -> argb), we also define AVX2 and AVX512
; versions that can handle a larger number of bytes at once.

%macro packed_shuffle 2 ; size_in, size_out
cglobal packed_shuffle%1_%2, 6, 10, 2, \
    exec, shuffle, bx, y, bxend, yend, src, dst, src_stride, dst_stride
            mov srcq, [execq + SwsOpExec.in0]
            mov dstq, [execq + SwsOpExec.out0]
            mov src_strideq, [execq + SwsOpExec.in_stride0]
            mov dst_strideq, [execq + SwsOpExec.out_stride0]
            VBROADCASTI128 m1, [shuffleq]
            sub bxendd, bxd
            sub yendd, yd
            ; reuse now-unneeded regs
    %define srcidxq execq
            imul srcidxq, bxendq, -%1
%if %1 = %2
    %define dstidxq srcidxq
%else
    %define dstidxq shuffleq ; no longer needed reg
            imul dstidxq, bxendq, -%2
%endif
            sub srcq, srcidxq
            sub dstq, dstidxq
.loop:
    %if %1 <= 4
            movd m0, [srcq + srcidxq]
    %elif %1 <= 8
            movq m0, [srcq + srcidxq]
    %else
            movu m0, [srcq + srcidxq]
    %endif
            pshufb m0, m1
            movu [dstq + dstidxq], m0
            add srcidxq, %1
IF %1 != %2,add dstidxq, %2
            jnz .loop
            add srcq, src_strideq
            add dstq, dst_strideq
            imul srcidxq, bxendq, -%1
IF %1 != %2,imul dstidxq, bxendq, -%2
            dec yendd
            jnz .loop
            RET
%endmacro

INIT_XMM sse4
packed_shuffle  5, 15 ;  8 -> 24
packed_shuffle  4, 16 ;  8 -> 32, 16 -> 64
packed_shuffle  2, 12 ;  8 -> 48
packed_shuffle 16,  8 ; 16 -> 8
packed_shuffle 10, 15 ; 16 -> 24
packed_shuffle  8, 16 ; 16 -> 32, 32 -> 64
packed_shuffle  4, 12 ; 16 -> 48
packed_shuffle 15, 15 ; 24 -> 24
packed_shuffle 12, 16 ; 24 -> 32
packed_shuffle  6, 12 ; 24 -> 48
packed_shuffle 16, 12 ; 32 -> 24, 64 -> 48
packed_shuffle 16, 16 ; 32 -> 32, 64 -> 64
packed_shuffle  8, 12 ; 32 -> 48
packed_shuffle 12, 12 ; 48 -> 48

INIT_YMM avx2
packed_shuffle 32, 32

INIT_ZMM avx512
packed_shuffle 64, 64

;---------------------------------------------------------
; Planar reads / writes

%macro read_planar 1 ; elems
op read_planar%1
            movu mx, [in0q]
IF %1 > 1,  movu my, [in1q]
IF %1 > 2,  movu mz, [in2q]
IF %1 > 3,  movu mw, [in3q]
%if V2
            movu mx2, [in0q + mmsize]
IF %1 > 1,  movu my2, [in1q + mmsize]
IF %1 > 2,  movu mz2, [in2q + mmsize]
IF %1 > 3,  movu mw2, [in3q + mmsize]
%endif
            LOAD_CONT tmp0q
            add in0q, mmsize * (1 + V2)
IF %1 > 1,  add in1q, mmsize * (1 + V2)
IF %1 > 2,  add in2q, mmsize * (1 + V2)
IF %1 > 3,  add in3q, mmsize * (1 + V2)
            CONTINUE tmp0q
%endmacro

%macro write_planar 1 ; elems
op write_planar%1
            LOAD_CONT tmp0q
            movu [out0q], mx
IF %1 > 1,  movu [out1q], my
IF %1 > 2,  movu [out2q], mz
IF %1 > 3,  movu [out3q], mw
%if V2
            movu [out0q + mmsize], mx2
IF %1 > 1,  movu [out1q + mmsize], my2
IF %1 > 2,  movu [out2q + mmsize], mz2
IF %1 > 3,  movu [out3q + mmsize], mw2
%endif
            add out0q, mmsize * (1 + V2)
IF %1 > 1,  add out1q, mmsize * (1 + V2)
IF %1 > 2,  add out2q, mmsize * (1 + V2)
IF %1 > 3,  add out3q, mmsize * (1 + V2)
            FINISH tmp0q
%endmacro

%macro read_packed2 1 ; depth
op read%1_packed2
            movu m8,  [in0q + 0*mmsize]
            movu m9,  [in0q + 1*mmsize]
    IF V2,  movu m10, [in0q + 2*mmsize]
    IF V2,  movu m11, [in0q + 3*mmsize]
IF %1 < 32, VBROADCASTI128 m12, [read%1_unpack2]
            LOAD_CONT tmp0q
            add in0q, mmsize * (2 + V2 * 2)
%if %1 == 32
            shufps m8, m8, q3120
            shufps m9, m9, q3120
    IF V2,  shufps m10, m10, q3120
    IF V2,  shufps m11, m11, q3120
%else
            pshufb m8, m12              ; { X0 Y0 | X1 Y1 }
            pshufb m9, m12              ; { X2 Y2 | X3 Y3 }
    IF V2,  pshufb m10, m12
    IF V2,  pshufb m11, m12
%endif
            unpcklpd mx, m8, m9         ; { X0 X2 | X1 X3 }
            unpckhpd my, m8, m9         ; { Y0 Y2 | Y1 Y3 }
    IF V2,  unpcklpd mx2, m10, m11
    IF V2,  unpckhpd my2, m10, m11
%if avx_enabled
            vpermq mx, mx, q3120       ; { X0 X1 | X2 X3 }
            vpermq my, my, q3120       ; { Y0 Y1 | Y2 Y3 }
    IF V2,  vpermq mx2, mx2, q3120
    IF V2,  vpermq my2, my2, q3120
%endif
            CONTINUE tmp0q
%endmacro

%macro write_packed2 1 ; depth
op write%1_packed2
IF %1 < 32, VBROADCASTI128 m12, [write%1_pack2]
            LOAD_CONT tmp0q
%if avx_enabled
            vpermq mx, mx, q3120       ; { X0 X2 | X1 X3 }
            vpermq my, my, q3120       ; { Y0 Y2 | Y1 Y3 }
    IF V2,  vpermq mx2, mx2, q3120
    IF V2,  vpermq my2, my2, q3120
%endif
            unpcklpd m8, mx, my        ; { X0 Y0 | X1 Y1 }
            unpckhpd m9, mx, my        ; { X2 Y2 | X3 Y3 }
    IF V2,  unpcklpd m10, mx2, my2
    IF V2,  unpckhpd m11, mx2, my2
%if %1 == 32
            shufps m8, m8, q3120
            shufps m9, m9, q3120
    IF V2,  shufps m10, m10, q3120
    IF V2,  shufps m11, m11, q3120
%else
            pshufb m8, m12
            pshufb m9, m12
    IF V2,  pshufb m10, m12
    IF V2,  pshufb m11, m12
%endif
            movu [out0q + 0*mmsize], m8
            movu [out0q + 1*mmsize], m9
IF V2,      movu [out0q + 2*mmsize], m10
IF V2,      movu [out0q + 3*mmsize], m11
            add out0q, mmsize * (2 + V2 * 2)
            FINISH tmp0q
%endmacro

; helper macro reused for both 3 and 4 component packed reads
%macro read_packed_inner 7 ; x, y, z, w, addr, num, depth
            movu xm8,  [%5 + 0  * %6]
            movu xm9,  [%5 + 4  * %6]
            movu xm10, [%5 + 8  * %6]
            movu xm11, [%5 + 12 * %6]
    %if avx_enabled
            vinserti128 m8,  m8,  [%5 + 16 * %6], 1
            vinserti128 m9,  m9,  [%5 + 20 * %6], 1
            vinserti128 m10, m10, [%5 + 24 * %6], 1
            vinserti128 m11, m11, [%5 + 28 * %6], 1
    %endif
    %if %7 == 32
            mova %1, m8
            mova %2, m9
            mova %3, m10
            mova %4, m11
    %else
            pshufb %1, m8,  m12         ; { X0 Y0 Z0 W0 | X4 Y4 Z4 W4 }
            pshufb %2, m9,  m12         ; { X1 Y1 Z1 W1 | X5 Y5 Z5 W5 }
            pshufb %3, m10, m12         ; { X2 Y2 Z2 W2 | X6 Y6 Z6 W6 }
            pshufb %4, m11, m12         ; { X3 Y3 Z3 W3 | X7 Y7 Z7 W7 }
    %endif
            punpckldq m8,  %1, %2       ; { X0 X1 Y0 Y1 | X4 X5 Y4 Y5 }
            punpckldq m9,  %3, %4       ; { X2 X3 Y2 Y3 | X6 X7 Y6 Y7 }
            punpckhdq m10, %1, %2       ; { Z0 Z1 W0 W1 | Z4 Z5 W4 W5 }
            punpckhdq m11, %3, %4       ; { Z2 Z3 W2 W3 | Z6 Z7 W6 W7 }
            punpcklqdq %1, m8, m9       ; { X0 X1 X2 X3 | X4 X5 X6 X7 }
            punpckhqdq %2, m8, m9       ; { Y0 Y1 Y2 Y3 | Y4 Y5 Y6 Y7 }
            punpcklqdq %3, m10, m11     ; { Z0 Z1 Z2 Z3 | Z4 Z5 Z6 Z7 }
IF %6 > 3,  punpckhqdq %4, m10, m11     ; { W0 W1 W2 W3 | W4 W5 W6 W7 }
%endmacro

%macro read_packed 2 ; num, depth
op read%2_packed%1
IF %2 < 32, VBROADCASTI128 m12, [read%2_unpack%1]
            LOAD_CONT tmp0q
            read_packed_inner mx, my, mz, mw, in0q, %1, %2
IF1 V2,     read_packed_inner mx2, my2, mz2, mw2, in0q + %1 * mmsize, %1, %2
            add in0q, %1 * mmsize * (1 + V2)
            CONTINUE tmp0q
%endmacro

%macro write_packed_inner 7 ; x, y, z, w, addr, num, depth
        punpckldq m8,  %1, %2       ; { X0 Y0 X1 Y1 | X4 Y4 X5 Y5 }
        punpckldq m9,  %3, %4       ; { Z0 W0 Z1 W1 | Z4 W4 Z5 W5 }
        punpckhdq m10, %1, %2       ; { X2 Y2 X3 Y3 | X6 Y6 X7 Y7 }
        punpckhdq m11, %3, %4       ; { Z2 W2 Z3 W3 | Z6 W6 Z7 W7 }
        punpcklqdq %1, m8, m9       ; { X0 Y0 Z0 W0 | X4 Y4 Z4 W4 }
        punpckhqdq %2, m8, m9       ; { X1 Y1 Z1 W1 | X5 Y5 Z5 W5 }
        punpcklqdq %3, m10, m11     ; { X2 Y2 Z2 W2 | X6 Y6 Z6 W6 }
        punpckhqdq %4, m10, m11     ; { X3 Y3 Z3 W3 | X7 Y7 Z7 W7 }
    %if %7 == 32
        mova m8,  %1
        mova m9,  %2
        mova m10, %3
        mova m11, %4
    %else
        pshufb m8,  %1, m12
        pshufb m9,  %2, m12
        pshufb m10, %3, m12
        pshufb m11, %4, m12
    %endif
        movu [%5 +  0*%6], xm8
        movu [%5 +  4*%6], xm9
        movu [%5 +  8*%6], xm10
        movu [%5 + 12*%6], xm11
    %if avx_enabled
        vextracti128 [%5 + 16*%6], m8, 1
        vextracti128 [%5 + 20*%6], m9, 1
        vextracti128 [%5 + 24*%6], m10, 1
        vextracti128 [%5 + 28*%6], m11, 1
    %endif
%endmacro

%macro write_packed 2 ; num, depth
op write%2_packed%1
IF %2 < 32, VBROADCASTI128 m12, [write%2_pack%1]
            LOAD_CONT tmp0q
            write_packed_inner mx, my, mz, mw, out0q, %1, %2
IF1 V2,     write_packed_inner mx2, my2, mz2, mw2, out0q + %1 * mmsize, %1, %2
            add out0q, %1 * mmsize * (1 + V2)
            FINISH tmp0q
%endmacro

%macro rw_packed 1 ; depth
        read_packed2 %1
        read_packed 3, %1
        read_packed 4, %1
        write_packed2 %1
        write_packed 3, %1
        write_packed 4, %1
%endmacro

%macro read_nibbles 0
op read_nibbles1
%if avx_enabled
        movu xmx,  [in0q]
IF V2,  movu xmx2, [in0q + 16]
%else
        movq xmx,  [in0q]
IF V2,  movq xmx2, [in0q + 8]
%endif
        VBROADCASTI128 m8, [mask4]
        LOAD_CONT tmp0q
        add in0q, (mmsize >> 1) * (1 + V2)
        pmovzxbw mx, xmx
IF V2,  pmovzxbw mx2, xmx2
        psllw my, mx, 8
IF V2,  psllw my2, mx2, 8
        psrlw mx, 4
IF V2,  psrlw mx2, 4
        pand my, m8
IF V2,  pand my2, m8
        por mx, my
IF V2,  por mx2, my2
        CONTINUE tmp0q
%endmacro

%macro read_bits 0
op read_bits1
%if avx_enabled
        vpbroadcastd mx,  [in0q]
IF V2,  vpbroadcastd mx2, [in0q + 4]
%else
        movd mx, [in0q]
IF V2,  movd mx2, [in0q + 2]
%endif
        mova m8, [bits_shuf]
        VBROADCASTI128 m9,  [bits_mask]
        VBROADCASTI128 m10, [mask1]
        LOAD_CONT tmp0q
        add in0q, (mmsize >> 3) * (1 + V2)
        pshufb mx,  m8
IF V2,  pshufb mx2, m8
        pand mx,  m9
IF V2,  pand mx2, m9
        pcmpeqb mx,  m9
IF V2,  pcmpeqb mx2, m9
        pand mx,  m10
IF V2,  pand mx2, m10
        CONTINUE tmp0q
%endmacro

; TODO: write_nibbles

%macro write_bits 0
op write_bits1
        VBROADCASTI128 m8, [bits_reverse]
        psllw mx,  7
IF V2,  psllw mx2, 7
        pshufb mx,  m8
IF V2,  pshufb mx2, m8
        pmovmskb tmp0d, mx
IF V2,  pmovmskb tmp1d, mx2
%if avx_enabled
        mov [out0q],     tmp0d
IF V2,  mov [out0q + 4], tmp1d
%else
        mov [out0q],     tmp0d
IF V2,  mov [out0q + 2], tmp1d
%endif
        LOAD_CONT tmp0q
        add out0q, (mmsize >> 3) * (1 + V2)
        FINISH tmp0q
%endmacro

;--------------------------
; Pixel packing / unpacking

%macro pack_generic 3-4 0 ; x, y, z, w
op pack_%1%2%3%4
        ; pslld works for all sizes because the input should not overflow
IF %2,  pslld mx, %4+%3+%2
IF %3,  pslld my, %4+%3
IF %4,  pslld mz, %4
IF %2,  por mx, my
IF %3,  por mx, mz
IF %4,  por mx, mw
    %if V2
IF %2,  pslld mx2, %4+%3+%2
IF %3,  pslld my2, %4+%3
IF %4,  pslld mz2, %4
IF %2,  por mx2, my2
IF %3,  por mx2, mz2
IF %4,  por mx2, mw2
    %endif
        CONTINUE
%endmacro

%macro unpack 5-6 0 ; type, bits, x, y, z, w
op unpack_%3%4%5%6
        ; clear high bits by shifting left
IF %6,  vpsll%1 mw, mx, %2 - (%6)
IF %5,  vpsll%1 mz, mx, %2 - (%6+%5)
IF %4,  vpsll%1 my, mx, %2 - (%6+%5+%4)
        psrl%1 mx, %4+%5+%6
IF %4,  psrl%1 my, %2 - %4
IF %5,  psrl%1 mz, %2 - %5
IF %6,  psrl%1 mw, %2 - %6
    %if V2
IF %6,  vpsll%1 mw2, mx2, %2 - (%6)
IF %5,  vpsll%1 mz2, mx2, %2 - (%6+%5)
IF %4,  vpsll%1 my2, mx2, %2 - (%6+%5+%4)
        psrl%1 mx2, %4+%5+%6
IF %4,  psrl%1 my2, %2 - %4
IF %5,  psrl%1 mz2, %2 - %5
IF %6,  psrl%1 mw2, %2 - %6
    %endif
        CONTINUE
%endmacro

%macro unpack8 3 ; x, y, z
op unpack_%1%2%3 %+ 0
        pand mz, mx, [mask%3]
        psrld my, mx, %3
        psrld mx, %3+%2
        pand my, [mask%2]
        pand mx, [mask%1]
    %if V2
        pand mz2, mx2, [mask%3]
        psrld my2, mx2, %3
        psrld mx2, %3+%2
        pand my2, [mask%2]
        pand mx2, [mask%1]
    %endif
        CONTINUE
%endmacro

;---------------------------------------------------------
; Generic byte order shuffle (packed swizzle, endian, etc)

%macro shuffle 0
op shuffle
        VBROADCASTI128 m8, [implq + SwsOpImpl.priv]
        LOAD_CONT tmp0q
IF X,   pshufb mx, m8
IF Y,   pshufb my, m8
IF Z,   pshufb mz, m8
IF W,   pshufb mw, m8
%if V2
IF X,   pshufb mx2, m8
IF Y,   pshufb my2, m8
IF Z,   pshufb mz2, m8
IF W,   pshufb mw2, m8
%endif
        CONTINUE tmp0q
%endmacro

;---------------------------------------------------------
; Clearing

%macro clear_alpha 3 ; idx, vreg, vreg2
op clear_alpha%1
        LOAD_CONT tmp0q
        pcmpeqb %2, %2
IF V2,  mova %3, %2
        CONTINUE tmp0q
%endmacro

%macro clear_zero 3 ; idx, vreg, vreg2
op clear_zero%1
        LOAD_CONT tmp0q
        pxor %2, %2
IF V2,  mova %3, %2
        CONTINUE tmp0q
%endmacro

; note: the pattern is inverted for these functions; i.e. X=1 implies that we
; *keep* the X component, not that we clear it
%macro clear_generic 0
op clear
            LOAD_CONT tmp0q
%if avx_enabled
    IF !X,  vpbroadcastd mx, [implq + SwsOpImpl.priv + 0]
    IF !Y,  vpbroadcastd my, [implq + SwsOpImpl.priv + 4]
    IF !Z,  vpbroadcastd mz, [implq + SwsOpImpl.priv + 8]
    IF !W,  vpbroadcastd mw, [implq + SwsOpImpl.priv + 12]
%else ; !avx_enabled
    IF !X,  movd mx, [implq + SwsOpImpl.priv + 0]
    IF !Y,  movd my, [implq + SwsOpImpl.priv + 4]
    IF !Z,  movd mz, [implq + SwsOpImpl.priv + 8]
    IF !W,  movd mw, [implq + SwsOpImpl.priv + 12]
    IF !X,  pshufd mx, mx, 0
    IF !Y,  pshufd my, my, 0
    IF !Z,  pshufd mz, mz, 0
    IF !W,  pshufd mw, mw, 0
%endif
%if V2
    IF !X,  mova mx2, mx
    IF !Y,  mova my2, my
    IF !Z,  mova mz2, mz
    IF !W,  mova mw2, mw
%endif
            CONTINUE tmp0q
%endmacro

%macro clear_funcs 0
        decl_pattern 1, 1, 1, 0, clear_generic
        decl_pattern 0, 1, 1, 1, clear_generic
        decl_pattern 0, 0, 1, 1, clear_generic
        decl_pattern 1, 0, 0, 1, clear_generic
        decl_pattern 1, 1, 0, 0, clear_generic
        decl_pattern 0, 1, 0, 1, clear_generic
        decl_pattern 1, 0, 1, 0, clear_generic
        decl_pattern 1, 0, 0, 0, clear_generic
        decl_pattern 0, 1, 0, 0, clear_generic
        decl_pattern 0, 0, 1, 0, clear_generic
%endmacro

;---------------------------------------------------------
; Swizzling and duplicating

; mA := mB, mB := mC, ... mX := mA across both halves
%macro vrotate 2-* ; A, B, C, ...
    %rep %0
        %assign rot_a %1 + 4
        %assign rot_b %2 + 4
        mova m%1, m%2
        IF V2, mova m%[rot_a], m%[rot_b]
    %rotate 1
    %endrep
    %undef rot_a
    %undef rot_b
%endmacro

%macro swizzle_funcs 0
op swizzle_3012
    LOAD_CONT tmp0q
    vrotate 8, 0, 3, 2, 1
    CONTINUE tmp0q

op swizzle_3021
    LOAD_CONT tmp0q
    vrotate 8, 0, 3, 1
    CONTINUE tmp0q

op swizzle_2103
    LOAD_CONT tmp0q
    vrotate 8, 0, 2
    CONTINUE tmp0q

op swizzle_3210
    LOAD_CONT tmp0q
    vrotate 8, 0, 3
    vrotate 8, 1, 2
    CONTINUE tmp0q

op swizzle_3102
    LOAD_CONT tmp0q
    vrotate 8, 0, 3, 2
    CONTINUE tmp0q

op swizzle_3201
    LOAD_CONT tmp0q
    vrotate 8, 0, 3, 1, 2
    CONTINUE tmp0q

op swizzle_1203
    LOAD_CONT tmp0q
    vrotate 8, 0, 1, 2
    CONTINUE tmp0q

op swizzle_1023
    LOAD_CONT tmp0q
    vrotate 8, 0, 1
    CONTINUE tmp0q

op swizzle_2013
    LOAD_CONT tmp0q
    vrotate 8, 0, 2, 1
    CONTINUE tmp0q

op swizzle_2310
    LOAD_CONT tmp0q
    vrotate 8, 0, 2, 1, 3
    CONTINUE tmp0q

op swizzle_2130
    LOAD_CONT tmp0q
    vrotate 8, 0, 2, 3
    CONTINUE tmp0q

op swizzle_1230
    LOAD_CONT tmp0q
    vrotate 8, 0, 1, 2, 3
    CONTINUE tmp0q

op swizzle_1320
    LOAD_CONT tmp0q
    vrotate 8, 0, 1, 3
    CONTINUE tmp0q

op swizzle_0213
    LOAD_CONT tmp0q
    vrotate 8, 1, 2
    CONTINUE tmp0q

op swizzle_0231
    LOAD_CONT tmp0q
    vrotate 8, 1, 2, 3
    CONTINUE tmp0q

op swizzle_0312
    LOAD_CONT tmp0q
    vrotate 8, 1, 3, 2
    CONTINUE tmp0q

op swizzle_3120
    LOAD_CONT tmp0q
    vrotate 8, 0, 3
    CONTINUE tmp0q

op swizzle_0321
    LOAD_CONT tmp0q
    vrotate 8, 1, 3
    CONTINUE tmp0q

op swizzle_0003
    LOAD_CONT tmp0q
    mova my, mx
    mova mz, mx
%if V2
    mova my2, mx2
    mova mz2, mx2
%endif
    CONTINUE tmp0q

op swizzle_0001
    LOAD_CONT tmp0q
    mova mw, my
    mova mz, mx
    mova my, mx
%if V2
    mova mw2, my2
    mova mz2, mx2
    mova my2, mx2
%endif
    CONTINUE tmp0q

op swizzle_3000
    LOAD_CONT tmp0q
    mova my, mx
    mova mz, mx
    mova mx, mw
    mova mw, my
%if V2
    mova my2, mx2
    mova mz2, mx2
    mova mx2, mw2
    mova mw2, my2
%endif
    CONTINUE tmp0q

op swizzle_1000
    LOAD_CONT tmp0q
    mova mz, mx
    mova mw, mx
    mova mx, my
    mova my, mz
%if V2
    mova mz2, mx2
    mova mw2, mx2
    mova mx2, my2
    mova my2, mz2
%endif
    CONTINUE tmp0q
%endmacro

;---------------------------------------------------------
; Pixel type conversions

%macro conv8to16 1 ; type
op %1_U8_U16
            LOAD_CONT tmp0q
%if V2
    %if avx_enabled
    IF X,   vextracti128 xmx2, mx, 1
    IF Y,   vextracti128 xmy2, my, 1
    IF Z,   vextracti128 xmz2, mz, 1
    IF W,   vextracti128 xmw2, mw, 1
    %else
    IF X,   psrldq xmx2, mx, 8
    IF Y,   psrldq xmy2, my, 8
    IF Z,   psrldq xmz2, mz, 8
    IF W,   psrldq xmw2, mw, 8
    %endif
    IF X,   pmovzxbw mx2, xmx2
    IF Y,   pmovzxbw my2, xmy2
    IF Z,   pmovzxbw mz2, xmz2
    IF W,   pmovzxbw mw2, xmw2
%endif ; V2
    IF X,   pmovzxbw mx, xmx
    IF Y,   pmovzxbw my, xmy
    IF Z,   pmovzxbw mz, xmz
    IF W,   pmovzxbw mw, xmw

%ifidn %1, expand
            VBROADCASTI128 m8, [expand16_shuf]
    %if V2
    IF X,   pshufb mx2, m8
    IF Y,   pshufb my2, m8
    IF Z,   pshufb mz2, m8
    IF W,   pshufb mw2, m8
    %endif
    IF X,   pshufb mx, m8
    IF Y,   pshufb my, m8
    IF Z,   pshufb mz, m8
    IF W,   pshufb mw, m8
%endif ; expand
            CONTINUE tmp0q
%endmacro

%macro conv16to8 0
op convert_U16_U8
        LOAD_CONT tmp0q
%if V2
        ; this code technically works for the !V2 case as well, but slower
IF X,   packuswb mx, mx2
IF Y,   packuswb my, my2
IF Z,   packuswb mz, mz2
IF W,   packuswb mw, mw2
IF X,   vpermq mx, mx, q3120
IF Y,   vpermq my, my, q3120
IF Z,   vpermq mz, mz, q3120
IF W,   vpermq mw, mw, q3120
%else
IF X,   vextracti128  xm8, mx, 1
IF Y,   vextracti128  xm9, my, 1
IF Z,   vextracti128 xm10, mz, 1
IF W,   vextracti128 xm11, mw, 1
        vzeroupper
IF X,   packuswb xmx, xm8
IF Y,   packuswb xmy, xm9
IF Z,   packuswb xmz, xm10
IF W,   packuswb xmw, xm11
%endif
        CONTINUE tmp0q
%endmacro

%macro conv8to32 1 ; type
op %1_U8_U32
        LOAD_CONT tmp0q
IF X,   psrldq xmx2, xmx, 8
IF Y,   psrldq xmy2, xmy, 8
IF Z,   psrldq xmz2, xmz, 8
IF W,   psrldq xmw2, xmw, 8
IF X,   pmovzxbd mx, xmx
IF Y,   pmovzxbd my, xmy
IF Z,   pmovzxbd mz, xmz
IF W,   pmovzxbd mw, xmw
IF X,   pmovzxbd mx2, xmx2
IF Y,   pmovzxbd my2, xmy2
IF Z,   pmovzxbd mz2, xmz2
IF W,   pmovzxbd mw2, xmw2
%ifidn %1, expand
        VBROADCASTI128 m8, [expand32_shuf]
IF X,   pshufb mx, m8
IF Y,   pshufb my, m8
IF Z,   pshufb mz, m8
IF W,   pshufb mw, m8
IF X,   pshufb mx2, m8
IF Y,   pshufb my2, m8
IF Z,   pshufb mz2, m8
IF W,   pshufb mw2, m8
%endif ; expand
        CONTINUE tmp0q
%endmacro

%macro conv32to8 0
op convert_U32_U8
        LOAD_CONT tmp0q
IF X,   packusdw mx, mx2
IF Y,   packusdw my, my2
IF Z,   packusdw mz, mz2
IF W,   packusdw mw, mw2
IF X,   vextracti128 xmx2, mx, 1
IF Y,   vextracti128 xmy2, my, 1
IF Z,   vextracti128 xmz2, mz, 1
IF W,   vextracti128 xmw2, mw, 1
        vzeroupper
IF X,   packuswb xmx, xmx2
IF Y,   packuswb xmy, xmy2
IF Z,   packuswb xmz, xmz2
IF W,   packuswb xmw, xmw2
IF X,   vpshufd xmx, xmx, q3120
IF Y,   vpshufd xmy, xmy, q3120
IF Z,   vpshufd xmz, xmz, q3120
IF W,   vpshufd xmw, xmw, q3120
        CONTINUE tmp0q
%endmacro

%macro conv16to32 0
op convert_U16_U32
        LOAD_CONT tmp0q
IF X,   vextracti128 xmx2, mx, 1
IF Y,   vextracti128 xmy2, my, 1
IF Z,   vextracti128 xmz2, mz, 1
IF W,   vextracti128 xmw2, mw, 1
IF X,   pmovzxwd mx, xmx
IF Y,   pmovzxwd my, xmy
IF Z,   pmovzxwd mz, xmz
IF W,   pmovzxwd mw, xmw
IF X,   pmovzxwd mx2, xmx2
IF Y,   pmovzxwd my2, xmy2
IF Z,   pmovzxwd mz2, xmz2
IF W,   pmovzxwd mw2, xmw2
        CONTINUE tmp0q
%endmacro

%macro conv32to16 0
op convert_U32_U16
        LOAD_CONT tmp0q
IF X,   packusdw mx, mx2
IF Y,   packusdw my, my2
IF Z,   packusdw mz, mz2
IF W,   packusdw mw, mw2
IF X,   vpermq mx, mx, q3120
IF Y,   vpermq my, my, q3120
IF Z,   vpermq mz, mz, q3120
IF W,   vpermq mw, mw, q3120
        CONTINUE tmp0q
%endmacro

;---------------------------------------------------------
; Shifting

%macro lshift16 0
op lshift16
        vmovq xm8, [implq + SwsOpImpl.priv]
        LOAD_CONT tmp0q
IF X,   psllw mx, xm8
IF Y,   psllw my, xm8
IF Z,   psllw mz, xm8
IF W,   psllw mw, xm8
%if V2
IF X,   psllw mx2, xm8
IF Y,   psllw my2, xm8
IF Z,   psllw mz2, xm8
IF W,   psllw mw2, xm8
%endif
        CONTINUE tmp0q
%endmacro

%macro rshift16 0
op rshift16
        vmovq xm8, [implq + SwsOpImpl.priv]
        LOAD_CONT tmp0q
IF X,   psrlw mx, xm8
IF Y,   psrlw my, xm8
IF Z,   psrlw mz, xm8
IF W,   psrlw mw, xm8
%if V2
IF X,   psrlw mx2, xm8
IF Y,   psrlw my2, xm8
IF Z,   psrlw mz2, xm8
IF W,   psrlw mw2, xm8
%endif
        CONTINUE tmp0q
%endmacro

;---------------------------------------------------------
; Macro instantiations for kernel functions

%macro funcs_u8 0
    read_planar 1
    read_planar 2
    read_planar 3
    read_planar 4
    write_planar 1
    write_planar 2
    write_planar 3
    write_planar 4

    rw_packed 8
    read_nibbles
    read_bits
    write_bits

    pack_generic 1, 2, 1
    pack_generic 3, 3, 2
    pack_generic 2, 3, 3
    unpack8 1, 2, 1
    unpack8 3, 3, 2
    unpack8 2, 3, 3

    clear_alpha 0, mx, mx2
    clear_alpha 1, my, my2
    clear_alpha 3, mw, mw2
    clear_zero  0, mx, mx2
    clear_zero  1, my, my2
    clear_zero  3, mw, mw2
    clear_funcs
    swizzle_funcs

    decl_common_patterns shuffle
%endmacro

%macro funcs_u16 0
    rw_packed 16
    pack_generic  4, 4, 4
    pack_generic  5, 5, 5
    pack_generic  5, 6, 5
    unpack w, 16, 4, 4, 4
    unpack w, 16, 5, 5, 5
    unpack w, 16, 5, 6, 5
    decl_common_patterns conv8to16 convert
    decl_common_patterns conv8to16 expand
    decl_common_patterns conv16to8
    decl_common_patterns lshift16
    decl_common_patterns rshift16
%endmacro

INIT_XMM sse4
decl_v2 0, funcs_u8
decl_v2 1, funcs_u8

INIT_YMM avx2
decl_v2 0, funcs_u8
decl_v2 1, funcs_u8
decl_v2 0, funcs_u16
decl_v2 1, funcs_u16

INIT_YMM avx2
decl_v2 1, rw_packed 32
decl_v2 1, pack_generic  10, 10, 10,  2
decl_v2 1, pack_generic   2, 10, 10, 10
decl_v2 1, unpack d, 32, 10, 10, 10,  2
decl_v2 1, unpack d, 32,  2, 10, 10, 10
decl_common_patterns conv8to32 convert
decl_common_patterns conv8to32 expand
decl_common_patterns conv32to8
decl_common_patterns conv16to32
decl_common_patterns conv32to16