1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
/*
* This file is part of Libav.
*
* Libav is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* Libav is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with Libav; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "config.h"
#include "libavutil/cpu.h"
#include "libavutil/float_dsp.h"
#include "cpu.h"
#include "asm.h"
extern void ff_vector_fmul_sse(float *dst, const float *src0, const float *src1,
int len);
extern void ff_vector_fmul_avx(float *dst, const float *src0, const float *src1,
int len);
extern void ff_vector_fmac_scalar_sse(float *dst, const float *src, float mul,
int len);
extern void ff_vector_fmac_scalar_avx(float *dst, const float *src, float mul,
int len);
extern void ff_vector_fmul_scalar_sse(float *dst, const float *src, float mul,
int len);
extern void ff_vector_dmul_scalar_sse2(double *dst, const double *src,
double mul, int len);
extern void ff_vector_dmul_scalar_avx(double *dst, const double *src,
double mul, int len);
void ff_vector_fmul_add_sse(float *dst, const float *src0, const float *src1,
const float *src2, int len);
void ff_vector_fmul_add_avx(float *dst, const float *src0, const float *src1,
const float *src2, int len);
#if HAVE_6REGS && HAVE_INLINE_ASM
static void vector_fmul_window_3dnowext(float *dst, const float *src0,
const float *src1, const float *win,
int len)
{
x86_reg i = -len * 4;
x86_reg j = len * 4 - 8;
__asm__ volatile (
"1: \n"
"pswapd (%5, %1), %%mm1 \n"
"movq (%5, %0), %%mm0 \n"
"pswapd (%4, %1), %%mm5 \n"
"movq (%3, %0), %%mm4 \n"
"movq %%mm0, %%mm2 \n"
"movq %%mm1, %%mm3 \n"
"pfmul %%mm4, %%mm2 \n" // src0[len + i] * win[len + i]
"pfmul %%mm5, %%mm3 \n" // src1[j] * win[len + j]
"pfmul %%mm4, %%mm1 \n" // src0[len + i] * win[len + j]
"pfmul %%mm5, %%mm0 \n" // src1[j] * win[len + i]
"pfadd %%mm3, %%mm2 \n"
"pfsub %%mm0, %%mm1 \n"
"pswapd %%mm2, %%mm2 \n"
"movq %%mm1, (%2, %0) \n"
"movq %%mm2, (%2, %1) \n"
"sub $8, %1 \n"
"add $8, %0 \n"
"jl 1b \n"
"femms \n"
: "+r"(i), "+r"(j)
: "r"(dst + len), "r"(src0 + len), "r"(src1), "r"(win + len)
);
}
static void vector_fmul_window_sse(float *dst, const float *src0,
const float *src1, const float *win, int len)
{
x86_reg i = -len * 4;
x86_reg j = len * 4 - 16;
__asm__ volatile (
"1: \n"
"movaps (%5, %1), %%xmm1 \n"
"movaps (%5, %0), %%xmm0 \n"
"movaps (%4, %1), %%xmm5 \n"
"movaps (%3, %0), %%xmm4 \n"
"shufps $0x1b, %%xmm1, %%xmm1 \n"
"shufps $0x1b, %%xmm5, %%xmm5 \n"
"movaps %%xmm0, %%xmm2 \n"
"movaps %%xmm1, %%xmm3 \n"
"mulps %%xmm4, %%xmm2 \n" // src0[len + i] * win[len + i]
"mulps %%xmm5, %%xmm3 \n" // src1[j] * win[len + j]
"mulps %%xmm4, %%xmm1 \n" // src0[len + i] * win[len + j]
"mulps %%xmm5, %%xmm0 \n" // src1[j] * win[len + i]
"addps %%xmm3, %%xmm2 \n"
"subps %%xmm0, %%xmm1 \n"
"shufps $0x1b, %%xmm2, %%xmm2 \n"
"movaps %%xmm1, (%2, %0) \n"
"movaps %%xmm2, (%2, %1) \n"
"sub $16, %1 \n"
"add $16, %0 \n"
"jl 1b \n"
: "+r"(i), "+r"(j)
: "r"(dst + len), "r"(src0 + len), "r"(src1), "r"(win + len)
);
}
#endif /* HAVE_6REGS && HAVE_INLINE_ASM */
void ff_float_dsp_init_x86(AVFloatDSPContext *fdsp)
{
int mm_flags = av_get_cpu_flags();
#if HAVE_6REGS && HAVE_INLINE_ASM
if (INLINE_AMD3DNOWEXT(mm_flags)) {
fdsp->vector_fmul_window = vector_fmul_window_3dnowext;
}
if (INLINE_SSE(mm_flags)) {
fdsp->vector_fmul_window = vector_fmul_window_sse;
}
#endif
if (EXTERNAL_SSE(mm_flags)) {
fdsp->vector_fmul = ff_vector_fmul_sse;
fdsp->vector_fmac_scalar = ff_vector_fmac_scalar_sse;
fdsp->vector_fmul_scalar = ff_vector_fmul_scalar_sse;
fdsp->vector_fmul_add = ff_vector_fmul_add_sse;
}
if (EXTERNAL_SSE2(mm_flags)) {
fdsp->vector_dmul_scalar = ff_vector_dmul_scalar_sse2;
}
if (EXTERNAL_AVX(mm_flags)) {
fdsp->vector_fmul = ff_vector_fmul_avx;
fdsp->vector_fmac_scalar = ff_vector_fmac_scalar_avx;
fdsp->vector_dmul_scalar = ff_vector_dmul_scalar_avx;
fdsp->vector_fmul_add = ff_vector_fmul_add_avx;
}
}
|
