aboutsummaryrefslogtreecommitdiffstats
path: root/libavcodec/arm/fft_vfp.S
blob: ac601325f25ddc374d2c181ee6470d20a9659371 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
/*
 * Copyright (c) 2013 RISC OS Open Ltd
 * Author: Ben Avison <bavison@riscosopen.org>
 *
 * This file is part of FFmpeg.
 *
 * FFmpeg is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * FFmpeg is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with FFmpeg; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include "libavutil/arm/asm.S"

@ The fftx_internal_vfp versions of the functions obey a modified AAPCS:
@ VFP is in RunFast mode, vector length 4, stride 1 thoroughout, and
@ all single-precision VFP registers may be corrupted on exit. The a2
@ register may not be clobbered in these functions, as it holds the
@ stored original FPSCR.

function ff_fft_calc_vfp, export=1
        ldr     ip, [a1, #0]    @ nbits
        mov     a1, a2
        movrel  a2, (fft_tab_vfp - 8)
        ldr     pc, [a2, ip, lsl #2]
endfunc
const   fft_tab_vfp, relocate=1
        .word   fft4_vfp
        .word   fft8_vfp
        .word   X(ff_fft16_vfp)     @ this one alone is exported
        .word   fft32_vfp
        .word   fft64_vfp
        .word   fft128_vfp
        .word   fft256_vfp
        .word   fft512_vfp
        .word   fft1024_vfp
        .word   fft2048_vfp
        .word   fft4096_vfp
        .word   fft8192_vfp
        .word   fft16384_vfp
        .word   fft32768_vfp
        .word   fft65536_vfp
endconst

function fft4_vfp
        vldr    d0, [a1, #0*2*4]   @ s0,s1   = z[0]
        vldr    d4, [a1, #1*2*4]   @ s8,s9   = z[1]
        vldr    d1, [a1, #2*2*4]   @ s2,s3   = z[2]
        vldr    d5, [a1, #3*2*4]   @ s10,s11 = z[3]
        @ stall
        vadd.f  s12, s0, s8        @ i0
        vadd.f  s13, s1, s9        @ i1
        vadd.f  s14, s2, s10       @ i2
        vadd.f  s15, s3, s11       @ i3
        vsub.f  s8, s0, s8         @ i4
        vsub.f  s9, s1, s9         @ i5
        vsub.f  s10, s2, s10       @ i6
        vsub.f  s11, s3, s11       @ i7
        @ stall
        @ stall
        vadd.f  s0, s12, s14       @ z[0].re
        vsub.f  s4, s12, s14       @ z[2].re
        vadd.f  s1, s13, s15       @ z[0].im
        vsub.f  s5, s13, s15       @ z[2].im
        vadd.f  s7, s9, s10        @ z[3].im
        vsub.f  s3, s9, s10        @ z[1].im
        vadd.f  s2, s8, s11        @ z[1].re
        vsub.f  s6, s8, s11        @ z[3].re
        @ stall
        @ stall
        vstr    d0, [a1, #0*2*4]
        vstr    d2, [a1, #2*2*4]
        @ stall
        @ stall
        vstr    d1, [a1, #1*2*4]
        vstr    d3, [a1, #3*2*4]

        bx      lr
endfunc

.macro macro_fft8_head
        @ FFT4
        vldr    d4, [a1, #0 * 2*4]
        vldr    d6, [a1, #1 * 2*4]
        vldr    d5, [a1, #2 * 2*4]
        vldr    d7, [a1, #3 * 2*4]
            @ BF
            vldr    d12, [a1, #4 * 2*4]
        vadd.f  s16, s8, s12    @ vector op
            vldr    d14, [a1, #5 * 2*4]
            vldr    d13, [a1, #6 * 2*4]
            vldr    d15, [a1, #7 * 2*4]
        vsub.f  s20, s8, s12    @ vector op
        vadd.f  s0, s16, s18
        vsub.f  s2, s16, s18
        vadd.f  s1, s17, s19
        vsub.f  s3, s17, s19
        vadd.f  s7, s21, s22
        vsub.f  s5, s21, s22
        vadd.f  s4, s20, s23
        vsub.f  s6, s20, s23
            vsub.f  s20, s24, s28   @ vector op
        vstr    d0, [a1, #0 * 2*4]  @ transfer s0-s7 to s24-s31 via memory
        vstr    d1, [a1, #1 * 2*4]
        vldr    s0, cos1pi4
            vadd.f  s16, s24, s28   @ vector op
        vstr    d2, [a1, #2 * 2*4]
        vstr    d3, [a1, #3 * 2*4]
        vldr    d12, [a1, #0 * 2*4]
            @ TRANSFORM
            vmul.f  s20, s20, s0    @ vector x scalar op
        vldr    d13, [a1, #1 * 2*4]
        vldr    d14, [a1, #2 * 2*4]
        vldr    d15, [a1, #3 * 2*4]
        @ BUTTERFLIES
        vadd.f  s0, s18, s16
        vadd.f  s1, s17, s19
        vsub.f  s2, s17, s19
        vsub.f  s3, s18, s16
            vadd.f  s4, s21, s20
            vsub.f  s5, s21, s20
            vadd.f  s6, s22, s23
            vsub.f  s7, s22, s23
        vadd.f  s8, s0, s24         @ vector op
        vstr    d0, [a1, #0 * 2*4]  @ transfer s0-s3 to s12-s15 via memory
        vstr    d1, [a1, #1 * 2*4]
        vldr    d6, [a1, #0 * 2*4]
        vldr    d7, [a1, #1 * 2*4]
            vadd.f  s1, s5, s6
            vadd.f  s0, s7, s4
            vsub.f  s2, s5, s6
            vsub.f  s3, s7, s4
        vsub.f  s12, s24, s12       @ vector op
            vsub.f  s5, s29, s1
            vsub.f  s4, s28, s0
            vsub.f  s6, s30, s2
            vsub.f  s7, s31, s3
            vadd.f  s16, s0, s28    @ vector op
        vstr    d6, [a1, #4 * 2*4]
        vstr    d7, [a1, #6 * 2*4]
        vstr    d4, [a1, #0 * 2*4]
        vstr    d5, [a1, #2 * 2*4]
             vstr    d2, [a1, #5 * 2*4]
             vstr    d3, [a1, #7 * 2*4]
.endm

.macro macro_fft8_tail
             vstr    d8, [a1, #1 * 2*4]
             vstr    d9, [a1, #3 * 2*4]
.endm

function .Lfft8_internal_vfp
        macro_fft8_head
        macro_fft8_tail
        bx      lr
endfunc

function fft8_vfp
        ldr     a3, =0x03030000     @ RunFast mode, vector length 4, stride 1
        fmrx    a2, FPSCR
        fmxr    FPSCR, a3
        vpush   {s16-s31}
        mov     ip, lr
        bl      .Lfft8_internal_vfp
        vpop    {s16-s31}
        fmxr    FPSCR, a2
        bx      ip
endfunc

.align 3
cos1pi4:    @ cos(1*pi/4) = sqrt(2)
        .float  0.707106769084930419921875
cos1pi8:    @ cos(1*pi/8) = sqrt(2+sqrt(2))/2
        .float  0.92387950420379638671875
cos3pi8:    @ cos(2*pi/8) = sqrt(2-sqrt(2))/2
        .float  0.3826834261417388916015625

function .Lfft16_internal_vfp
        macro_fft8_head
        @ FFT4(z+8)
        vldr    d10, [a1, #8 * 2*4]
        vldr    d12, [a1, #9 * 2*4]
        vldr    d11, [a1, #10 * 2*4]
        vldr    d13, [a1, #11 * 2*4]
        macro_fft8_tail
        vadd.f  s16, s20, s24   @ vector op
            @ FFT4(z+12)
            vldr    d4, [a1, #12 * 2*4]
            vldr    d6, [a1, #13 * 2*4]
            vldr    d5, [a1, #14 * 2*4]
        vsub.f  s20, s20, s24   @ vector op
            vldr    d7, [a1, #15 * 2*4]
        vadd.f  s0, s16, s18
        vsub.f  s4, s16, s18
        vadd.f  s1, s17, s19
        vsub.f  s5, s17, s19
        vadd.f  s7, s21, s22
        vsub.f  s3, s21, s22
        vadd.f  s2, s20, s23
        vsub.f  s6, s20, s23
            vadd.f  s16, s8, s12    @ vector op
        vstr    d0, [a1, #8 * 2*4]
        vstr    d2, [a1, #10 * 2*4]
        vstr    d1, [a1, #9 * 2*4]
            vsub.f  s20, s8, s12
        vstr    d3, [a1, #11 * 2*4]
        @ TRANSFORM(z[2],z[6],z[10],z[14],cos1pi4,cos1pi4)
        vldr    d12, [a1, #10 * 2*4]
            vadd.f  s0, s16, s18
            vadd.f  s1, s17, s19
            vsub.f  s6, s16, s18
            vsub.f  s7, s17, s19
            vsub.f  s3, s21, s22
            vadd.f  s2, s20, s23
            vadd.f  s5, s21, s22
            vsub.f  s4, s20, s23
            vstr    d0, [a1, #12 * 2*4]
        vmov    s0, s6
          @ TRANSFORM(z[1],z[5],z[9],z[13],cos1pi8,cos3pi8)
          vldr    d6, [a1, #9 * 2*4]
            vstr    d1, [a1, #13 * 2*4]
        vldr    d1, cos1pi4 @ s2 = cos1pi4, s3 = cos1pi8
            vstr    d2, [a1, #15 * 2*4]
          vldr    d7, [a1, #13 * 2*4]
        vadd.f  s4, s25, s24
        vsub.f  s5, s25, s24
        vsub.f  s6, s0, s7
        vadd.f  s7, s0, s7
          vmul.f  s20, s12, s3  @ vector op
            @ TRANSFORM(z[3],z[7],z[11],z[15],cos3pi8,cos1pi8)
            vldr    d4, [a1, #11 * 2*4]
            vldr    d5, [a1, #15 * 2*4]
            vldr    s1, cos3pi8
        vmul.f  s24, s4, s2     @ vector * scalar op
          vmul.f  s28, s12, s1  @ vector * scalar op
            vmul.f  s12, s8, s1 @ vector * scalar op
          vadd.f  s4, s20, s29
          vsub.f  s5, s21, s28
          vsub.f  s6, s22, s31
          vadd.f  s7, s23, s30
            vmul.f  s8, s8, s3  @ vector * scalar op
          vldr    d8, [a1, #1 * 2*4]
          vldr    d9, [a1, #5 * 2*4]
            vldr    d10, [a1, #3 * 2*4]
            vldr    d11, [a1, #7 * 2*4]
        vldr    d14, [a1, #2 * 2*4]
          vadd.f  s0, s6, s4
          vadd.f  s1, s5, s7
          vsub.f  s2, s5, s7
          vsub.f  s3, s6, s4
            vadd.f  s4, s12, s9
            vsub.f  s5, s13, s8
            vsub.f  s6, s14, s11
            vadd.f  s7, s15, s10
          vadd.f  s12, s0, s16  @ vector op
          vstr    d0, [a1, #1 * 2*4]
          vstr    d1, [a1, #5 * 2*4]
          vldr    d4, [a1, #1 * 2*4]
          vldr    d5, [a1, #5 * 2*4]
            vadd.f  s0, s6, s4
            vadd.f  s1, s5, s7
            vsub.f  s2, s5, s7
            vsub.f  s3, s6, s4
          vsub.f  s8, s16, s8   @ vector op
          vstr    d6, [a1, #1 * 2*4]
          vstr    d7, [a1, #5 * 2*4]
        vldr    d15, [a1, #6 * 2*4]
            vsub.f  s4, s20, s0
            vsub.f  s5, s21, s1
            vsub.f  s6, s22, s2
            vsub.f  s7, s23, s3
            vadd.f  s20, s0, s20    @ vector op
          vstr    d4, [a1, #9 * 2*4]
              @ TRANSFORM_ZERO(z[0],z[4],z[8],z[12])
              vldr    d6, [a1, #8 * 2*4]
          vstr    d5, [a1, #13 * 2*4]
              vldr    d7, [a1, #12 * 2*4]
          vstr    d2, [a1, #11 * 2*4]
              vldr    d8, [a1, #0 * 2*4]
          vstr    d3, [a1, #15 * 2*4]
              vldr    d9, [a1, #4 * 2*4]
        vadd.f  s0, s26, s24
        vadd.f  s1, s25, s27
        vsub.f  s2, s25, s27
        vsub.f  s3, s26, s24
              vadd.f  s4, s14, s12
              vadd.f  s5, s13, s15
              vsub.f  s6, s13, s15
              vsub.f  s7, s14, s12
        vadd.f  s8, s0, s28 @ vector op
        vstr    d0, [a1, #3 * 2*4]
        vstr    d1, [a1, #7 * 2*4]
        vldr    d6, [a1, #3 * 2*4]
        vldr    d7, [a1, #7 * 2*4]
              vsub.f  s0, s16, s4
              vsub.f  s1, s17, s5
              vsub.f  s2, s18, s6
              vsub.f  s3, s19, s7
        vsub.f  s12, s28, s12       @ vector op
              vadd.f  s16, s4, s16  @ vector op
            vstr    d10, [a1, #3 * 2*4]
            vstr    d11, [a1, #7 * 2*4]
        vstr    d4, [a1, #2 * 2*4]
        vstr    d5, [a1, #6 * 2*4]
              vstr    d0, [a1, #8 * 2*4]
              vstr    d1, [a1, #12 * 2*4]
        vstr    d6, [a1, #10 * 2*4]
        vstr    d7, [a1, #14 * 2*4]
              vstr    d8, [a1, #0 * 2*4]
              vstr    d9, [a1, #4 * 2*4]

        bx      lr
endfunc

function ff_fft16_vfp, export=1
        ldr     a3, =0x03030000     @ RunFast mode, vector length 4, stride 1
        fmrx    a2, FPSCR
        fmxr    FPSCR, a3
        vpush   {s16-s31}
        mov     ip, lr
        bl      .Lfft16_internal_vfp
        vpop    {s16-s31}
        fmxr    FPSCR, a2
        bx      ip
endfunc

.macro pass n, z0, z1, z2, z3
        add     v6, v5, #4*2*\n
        @ TRANSFORM_ZERO(z[0],z[o1],z[o2],z[o3])
            @ TRANSFORM(z[1],z[o1+1],z[o2+1],z[o3+1],wre[1],wim[-1])
                @ TRANSFORM(z[0],z[o1],z[o2],z[o3],wre[0],wim[0])
                    @ TRANSFORM(z[1],z[o1+1],z[o2+1],z[o3+1],wre[1],wim[-1])
            vldr    d8, [\z2, #8*(o2+1)]        @ s16,s17
            vldmdb  v6!, {s2}
            vldr    d9, [\z3, #8*(o3+1)]        @ s18,s19
            vldmia  v5!, {s0,s1}                @ s0 is unused
        vldr    s7, [\z2, #8*o2]            @ t1
            vmul.f  s20, s16, s2                @ vector * scalar
        vldr    s0, [\z3, #8*o3]            @ t5
        vldr    s6, [\z2, #8*o2+4]          @ t2
        vldr    s3, [\z3, #8*o3+4]          @ t6
            vmul.f  s16, s16, s1                @ vector * scalar
        ldr     a4, =\n-1
1:      add     \z0, \z0, #8*2
 .if \n*4*2 >= 512
        add     \z1, \z1, #8*2
 .endif
 .if \n*4*2 >= 256
        add     \z2, \z2, #8*2
 .endif
 .if \n*4*2 >= 512
        add     \z3, \z3, #8*2
 .endif
        @ up to 2 stalls (VFP vector issuing / waiting for s0)
        @ depending upon whether this is the first iteration and
        @ how many add instructions are inserted above
        vadd.f  s4, s0, s7                  @ t5
        vadd.f  s5, s6, s3                  @ t6
        vsub.f  s6, s6, s3                  @ t4
        vsub.f  s7, s0, s7                  @ t3
        vldr    d6, [\z0, #8*0-8*2]         @ s12,s13
            vadd.f  s0, s16, s21                @ t1
        vldr    d7, [\z1, #8*o1-8*2]        @ s14,s15
            vsub.f  s1, s18, s23                @ t5
        vadd.f  s8, s4, s12                 @ vector + vector
        @ stall (VFP vector issuing)
        @ stall (VFP vector issuing)
        @ stall (VFP vector issuing)
        vsub.f  s4, s12, s4
        vsub.f  s5, s13, s5
        vsub.f  s6, s14, s6
        vsub.f  s7, s15, s7
            vsub.f  s2, s17, s20                @ t2
            vadd.f  s3, s19, s22                @ t6
        vstr    d4, [\z0, #8*0-8*2]         @ s8,s9
        vstr    d5, [\z1, #8*o1-8*2]        @ s10,s11
        @ stall (waiting for s5)
        vstr    d2, [\z2, #8*o2-8*2]        @ s4,s5
            vadd.f  s4, s1, s0                  @ t5
        vstr    d3, [\z3, #8*o3-8*2]        @ s6,s7
            vsub.f  s7, s1, s0                  @ t3
            vadd.f  s5, s2, s3                  @ t6
            vsub.f  s6, s2, s3                  @ t4
            vldr    d6, [\z0, #8*1-8*2]         @ s12,s13
            vldr    d7, [\z1, #8*(o1+1)-8*2]    @ s14,s15
                vldr    d4, [\z2, #8*o2]            @ s8,s9
                vldmdb  v6!, {s2,s3}
                vldr    d5, [\z3, #8*o3]            @ s10,s11
            vadd.f  s20, s4, s12                @ vector + vector
                vldmia  v5!, {s0,s1}
                    vldr    d8, [\z2, #8*(o2+1)]        @ s16,s17
            @ stall (VFP vector issuing)
            vsub.f  s4, s12, s4
            vsub.f  s5, s13, s5
            vsub.f  s6, s14, s6
            vsub.f  s7, s15, s7
                vmul.f  s12, s8, s3                 @ vector * scalar
            vstr    d10, [\z0, #8*1-8*2]        @ s20,s21
                    vldr    d9, [\z3, #8*(o3+1)]        @ s18,s19
            vstr    d11, [\z1, #8*(o1+1)-8*2]   @ s22,s23
                vmul.f  s8, s8, s0                  @ vector * scalar
            vstr    d2, [\z2, #8*(o2+1)-8*2]    @ s4,s5
            @ stall (waiting for s7)
            vstr    d3, [\z3, #8*(o3+1)-8*2]    @ s6,s7
                    vmul.f  s20, s16, s2                @ vector * scalar
                @ stall (VFP vector issuing)
                @ stall (VFP vector issuing)
                @ stall (VFP vector issuing)
                vadd.f  s7, s8, s13                 @ t1
                vsub.f  s6, s9, s12                 @ t2
                vsub.f  s0, s10, s15                @ t5
                vadd.f  s3, s11, s14                @ t6
                    vmul.f  s16, s16, s1                @ vector * scalar
        subs    a4, a4, #1
        bne     1b
        @ What remains is identical to the first two indentations of
        @ the above, but without the increment of z
        vadd.f  s4, s0, s7                  @ t5
        vadd.f  s5, s6, s3                  @ t6
        vsub.f  s6, s6, s3                  @ t4
        vsub.f  s7, s0, s7                  @ t3
        vldr    d6, [\z0, #8*0]             @ s12,s13
            vadd.f  s0, s16, s21                @ t1
        vldr    d7, [\z1, #8*o1]            @ s14,s15
            vsub.f  s1, s18, s23                @ t5
        vadd.f  s8, s4, s12                 @ vector + vector
        vsub.f  s4, s12, s4
        vsub.f  s5, s13, s5
        vsub.f  s6, s14, s6
        vsub.f  s7, s15, s7
            vsub.f  s2, s17, s20                @ t2
            vadd.f  s3, s19, s22                @ t6
        vstr    d4, [\z0, #8*0]             @ s8,s9
        vstr    d5, [\z1, #8*o1]            @ s10,s11
        vstr    d2, [\z2, #8*o2]            @ s4,s5
            vadd.f  s4, s1, s0                  @ t5
        vstr    d3, [\z3, #8*o3]            @ s6,s7
            vsub.f  s7, s1, s0                  @ t3
            vadd.f  s5, s2, s3                  @ t6
            vsub.f  s6, s2, s3                  @ t4
            vldr    d6, [\z0, #8*1]             @ s12,s13
            vldr    d7, [\z1, #8*(o1+1)]        @ s14,s15
            vadd.f  s20, s4, s12                @ vector + vector
            vsub.f  s4, s12, s4
            vsub.f  s5, s13, s5
            vsub.f  s6, s14, s6
            vsub.f  s7, s15, s7
            vstr    d10, [\z0, #8*1]            @ s20,s21
            vstr    d11, [\z1, #8*(o1+1)]       @ s22,s23
            vstr    d2, [\z2, #8*(o2+1)]        @ s4,s5
            vstr    d3, [\z3, #8*(o3+1)]        @ s6,s7
.endm

.macro  def_fft n, n2, n4
function .Lfft\n\()_internal_vfp
 .if \n >= 512
        push    {v1-v6,lr}
 .elseif \n >= 256
        push    {v1-v2,v5-v6,lr}
 .else
        push    {v1,v5-v6,lr}
 .endif
        mov     v1, a1
        bl      .Lfft\n2\()_internal_vfp
        add     a1, v1, #8*(\n/4)*2
        bl      .Lfft\n4\()_internal_vfp
        movrelx v5, X(ff_cos_\n), a1
        add     a1, v1, #8*(\n/4)*3
        bl      .Lfft\n4\()_internal_vfp
 .if \n >= 512
  .set o1, 0*(\n/4/2)
  .set o2, 0*(\n/4/2)
  .set o3, 0*(\n/4/2)
        add     v2, v1, #8*2*(\n/4/2)
        add     v3, v1, #8*4*(\n/4/2)
        add     v4, v1, #8*6*(\n/4/2)
        pass    (\n/4/2), v1, v2, v3, v4
        pop     {v1-v6,pc}
 .elseif \n >= 256
  .set o1, 2*(\n/4/2)
  .set o2, 0*(\n/4/2)
  .set o3, 2*(\n/4/2)
        add     v2, v1, #8*4*(\n/4/2)
        pass    (\n/4/2), v1, v1, v2, v2
        pop     {v1-v2,v5-v6,pc}
 .else
  .set o1, 2*(\n/4/2)
  .set o2, 4*(\n/4/2)
  .set o3, 6*(\n/4/2)
        pass    (\n/4/2), v1, v1, v1, v1
        pop     {v1,v5-v6,pc}
 .endif
endfunc

function fft\n\()_vfp
        ldr     a3, =0x03030000 /* RunFast mode, vector length 4, stride 1 */
        fmrx    a2, FPSCR
        fmxr    FPSCR, a3
        vpush   {s16-s31}
        mov     ip, lr
        bl      .Lfft\n\()_internal_vfp
        vpop    {s16-s31}
        fmxr    FPSCR, a2
        bx      ip
endfunc

.ltorg
.endm

        def_fft    32,    16,     8
        def_fft    64,    32,    16
        def_fft   128,    64,    32
        def_fft   256,   128,    64
        def_fft   512,   256,   128
        def_fft  1024,   512,   256
        def_fft  2048,  1024,   512
        def_fft  4096,  2048,  1024
        def_fft  8192,  4096,  2048
        def_fft 16384,  8192,  4096
        def_fft 32768, 16384,  8192
        def_fft 65536, 32768, 16384