Source code
Revision control
Copy as Markdown
Other Tools
/*
* Copyright (c) Lynne
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
/* Open `doc/transforms.md` to see the code upon which the transforms here were
* based upon.
*
* File conventions:
* GPRs: x0-x3 - arguments, untouched
* x4 - Lookup table base pointer
* x5-x6 - macro ld1 temps/function scratch
* x7-x9 - FFT table state
* x10-x17 - lookup table/macro scratch
* w19-w20 - current/target length when needed
* x21-x22 - len*2, len*6
*
* Vectors: v0-v7 - coefficients
* v8-v15 - coefficients when needed, otherwise untouched
* v16-v30 - used as needed
* v31 - -1.0, +1.0, -1.0, +1.0. Never touched after loading.
*
* Stack: backup for v8-v15 and x19-x22 when needed, and transform lengths
*/
#define M_SQRT1_2 0.707106781186547524401
#define COS16_1 0.92387950420379638671875
#define COS16_3 0.3826834261417388916015625
/* We only ever load this once at the start, and then live with losing an
* entire register as we need to lug this all the time everywhere.
* Clearly should be integrated into an fsadd and fmlsa, but "muh RISC!". */
const subadd, align=4
.float -1.0, 1.0, -1.0, 1.0
endconst
.macro LOAD_SUBADD
movrel x5, subadd
ld1 { v31.4s }, [x5]
.endm
.macro SETUP_LUT no_lut=0
.if \no_lut == 0
ldr x4, [x0, #8]
.endif
.endm
.macro LOAD_INPUT dst1, dst2, dst3, dst4, src, no_lut=0, discont=0
.if \no_lut == 1
.if \discont == 1
ldp q\dst1\(), q\dst2\(), [\src\()]
ldp q\dst3\(), q\dst4\(), [\src\(), #32]
add \src\(), \src\(), #64
.else
ld1 { v\dst1\().4s, v\dst2\().4s, v\dst3\().4s, v\dst4\().4s }, [\src], #64
.endif
.else
ldp w10, w11, [x4, #0 ]
ldp w12, w13, [x4, #8 ]
ldp w14, w15, [x4, #16]
ldp w16, w17, [x4, #24]
add x4, x4, #32
ldr d\dst1, [\src, x10, lsl #3]
add x11, \src, x11, lsl #3
ldr d\dst2, [\src, x12, lsl #3]
add x13, \src, x13, lsl #3
ldr d\dst3, [\src, x14, lsl #3]
add x15, \src, x15, lsl #3
ldr d\dst4, [\src, x16, lsl #3]
add x17, \src, x17, lsl #3
ld1 { v\dst1\().d }[1], [x11]
ld1 { v\dst2\().d }[1], [x13]
ld1 { v\dst3\().d }[1], [x15]
ld1 { v\dst4\().d }[1], [x17]
.endif
.endm
.macro FFT4 e0, o0, standalone
fadd v16.4s, \e0\().4s, \o0\().4s // r1..4
fsub \e0\().4s, \e0\().4s, \o0\().4s // t1..4
rev64 v18.4s, \e0\().4s
zip2 \o0\().2d, v16.2d, \e0\().2d
zip1 v17.2d, v16.2d, \e0\().2d
mov \o0\().d[1], v18.d[1]
fadd \e0\().4s, v17.4s, \o0\().4s // a1,2 b1,4
fsub v16.4s, v17.4s, \o0\().4s // a3,4 b3,2
mov \o0\().16b, v16.16b // Swap once again...
mov \o0\().s[3], \e0\().s[3]
mov \e0\().s[3], v16.s[3]
.if \standalone == 0
uzp2 \o0\().2d, \e0\().2d, \o0\().2d
uzp1 \e0\().2d, \e0\().2d, v16.2d
.endif
.endm
const shuf_4pt_x2, align=4
.byte 24, 25, 26, 27 // reg2, 3
.byte 12, 13, 14, 15 // reg1, 4
.byte 8, 9, 10, 11 // reg1, 3
.byte 28, 29, 30, 31 // reg2, 4
endconst
// Identical to FFT4, but does 2 transforms in parallel, with no deinterleaving
.macro FFT4_X2 e0, o0, e1, o1, \
t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22
fadd \t0\().4s, \e0\().4s, \o0\().4s // r1234
fadd \t2\().4s, \e1\().4s, \o1\().4s // r1234
fsub \e0\().4s, \e0\().4s, \o0\().4s // t1234
fsub \e1\().4s, \e1\().4s, \o1\().4s // t1234
movrel x5, shuf_4pt_x2
rev64 \t4\().4s, \e0\().4s
rev64 \t5\().4s, \e1\().4s
zip2 \o0\().2d, \t0\().2d, \e0\().2d // t3,4 r3,4
zip2 \o1\().2d, \t2\().2d, \e1\().2d // t3,4 r3,4
ld1 { \t6\().16b }, [x5]
mov \o0\().d[1], \t4\().d[1]
mov \o1\().d[1], \t5\().d[1]
zip1 \t1\().2d, \t0\().2d, \e0\().2d // t1,2 r1,2
zip1 \t3\().2d, \t2\().2d, \e1\().2d // t1,2 r1,2
fsub \t4\().4s, \t1\().4s, \o0\().4s // a34 b32
fadd \t5\().4s, \t1\().4s, \o0\().4s // a12 b14
fsub \t2\().4s, \t3\().4s, \o1\().4s // a34 b32
fadd \t3\().4s, \t3\().4s, \o1\().4s // a12 b14
// TODO: experiment with movs instead of tables here
tbl \o0\().16b, { \t4\().16b, \t5\().16b }, \t6\().16b // b1234
tbl \o1\().16b, { \t2\().16b, \t3\().16b }, \t6\().16b // b1234
zip1 \e0\().2d, \t5\().2d, \t4\().2d // a1234
// zip2 \o0\().2d, \t5\().2d, \t4\().2d // b1432
zip1 \e1\().2d, \t3\().2d, \t2\().2d // a1234
// zip2 \o1\().2d, \t3\().2d, \t2\().2d // b1432
// rev64 \o0\().4s, \o0\().4s // b4123
// rev64 \o1\().4s, \o1\().4s // b4123
// ext \o0\().16b, \o0\().16b, \o0\().16b, #4 // b1234
// ext \o1\().16b, \o1\().16b, \o1\().16b, #4 // b1234
.endm
const tab_8pt, align=4
.float M_SQRT1_2, -M_SQRT1_2, -M_SQRT1_2, M_SQRT1_2
endconst
.macro FFT8 e0, e1, o0, o1, \
t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22
movrel x5, tab_8pt
fsub \t1\().4s, \e1\().4s, \o1\().4s // j1234
fadd \o1\().4s, \e1\().4s, \o1\().4s // k1234
fsub \t0\().4s, \e0\().4s, \o0\().4s // r1234
fadd \o0\().4s, \e0\().4s, \o0\().4s // q1234
ld1 { \t5\().4s }, [x5]
ext \t4\().16b, \o1\().16b, \o1\().16b, #12
rev64 \t4\().4s, \t4\().4s
ext \t2\().16b, \o0\().16b, \t4\().16b, #8 // o0[0,1], o1[3,2]
mov \o0\().d[1], \t4\().d[1] // o0[3, 4]; o1[1, 4]
fsub \e1\().4s, \o0\().4s, \t2\().4s // s34, g43
fadd \t2\().4s, \o0\().4s, \t2\().4s // s12, g12
rev64 \t6\().4s, v31.4s // 1, -1, 1, -1
dup \o0\().2d, \t0\().d[0] // r1212
dup \o1\().2d, \t0\().d[1] // r3434
rev64 \t4\().4s, \e1\().4s // xxg34
rev64 \o1\().4s, \o1\().4s // r4343
ext \t6\().16b, v31.16b, \t6\().16b, #8 // -1, 1, 1, -1
zip1 \t3\().2d, \t2\().2d, \e1\().2d // s1234
zip2 \t2\().2d, \t2\().2d, \t4\().2d // g1234
fadd \e0\().4s, \t3\().4s, \t2\().4s // out_e1
fsub \e1\().4s, \t3\().4s, \t2\().4s // out_e2
fmul \t1\().4s, \t1\().4s, \t5\().4s // j * +--+M_SQRT1_2
fmls \o0\().4s, \o1\().4s, \t6\().4s // z1234
rev64 \t4\().4s, \t1\().4s // j2143
fmla \t1\().4s, \t4\().4s, v31.4s // l2143
rev64 \t4\().4s, \t1\().4s // l1234
ext \t4\().16b, \t4\().16b, \t4\().16b, #8 // l3412
fmla \t4\().4s, \t1\().4s, v31.4s // t1234
fadd \o1\().4s, \o0\().4s, \t4\().4s // out_o2
fsub \o0\().4s, \o0\().4s, \t4\().4s // out_o1
.endm
// Identical as FFT8, but does 2 transforms in parallel
.macro FFT8_X2 e0, e1, o0, o1, e2, e3, o2, o3
movrel x5, tab_8pt
fadd v19.4s, \e3\().4s, \o3\().4s // k1234
fadd v17.4s, \e1\().4s, \o1\().4s // k1234
fadd v18.4s, \e2\().4s, \o2\().4s // q1234
fadd v16.4s, \e0\().4s, \o0\().4s // q1234
ld1 { v23.4s }, [x5]
ext v22.16b, v19.16b, v19.16b, #12
ext v21.16b, v17.16b, v17.16b, #12
rev64 v22.4s, v22.4s
rev64 v21.4s, v21.4s
ext v19.16b, v18.16b, v22.16b, #8
ext v17.16b, v16.16b, v21.16b, #8
mov v18.d[1], v22.d[1]
mov v21.d[0], v16.d[0]
fadd v22.4s, v18.4s, v19.4s // s12, g12
fsub v19.4s, v18.4s, v19.4s // s34, g43
fsub v18.4s, v21.4s, v17.4s // s34, g43
fadd v16.4s, v21.4s, v17.4s // s12, g12
fsub \e0\().4s, \e0\().4s, \o0\().4s // r1234
fsub v20.4s, \e1\().4s, \o1\().4s // j1234
fsub \e2\().4s, \e2\().4s, \o2\().4s // r1234
fsub v21.4s, \e3\().4s, \o3\().4s // j1234
rev64 v24.4s, v31.4s // 1, -1, 1, -1
zip1 v17.2d, v16.2d, v18.2d // s1234
zip1 \e1\().2d, v22.2d, v19.2d // s1234
rev64 v18.4s, v18.4s // xxg34
rev64 v19.4s, v19.4s // xxg34
zip2 v16.2d, v16.2d, v18.2d // g1234
zip2 \e3\().2d, v22.2d, v19.2d // g1234
dup \o0\().2d, \e0\().d[0] // r1212
dup \o1\().2d, \e0\().d[1] // r3434
dup \o2\().2d, \e2\().d[0] // r1212
dup \o3\().2d, \e2\().d[1] // r3434
fadd \e2\().4s, \e1\().4s, \e3\().4s // out_e1
fsub \e3\().4s, \e1\().4s, \e3\().4s // out_e2
fadd \e0\().4s, v17.4s, v16.4s // out_e1
fsub \e1\().4s, v17.4s, v16.4s // out_e2
ext v24.16b, v31.16b, v24.16b, #8 // -1, 1, 1, -1
rev64 \o1\().4s, \o1\().4s // r4343
rev64 \o3\().4s, \o3\().4s // r4343
fmul v19.4s, v20.4s, v23.4s // j * +--+M_SQRT1_2
fmul v21.4s, v21.4s, v23.4s // j * +--+M_SQRT1_2
rev64 v20.4s, v19.4s // j2143
rev64 v18.4s, v21.4s // j2143
fmls \o0\().4s, \o1\().4s, v24.4s // z1234
fmls \o2\().4s, \o3\().4s, v24.4s // z1234
fmla v19.4s, v20.4s, v31.4s // l2143
fmla v21.4s, v18.4s, v31.4s // l2143
rev64 v20.4s, v19.4s // l1234
rev64 v18.4s, v21.4s // l1234
ext v20.16b, v20.16b, v20.16b, #8 // l3412
ext v18.16b, v18.16b, v18.16b, #8 // l3412
fmla v20.4s, v19.4s, v31.4s // t1234
fmla v18.4s, v21.4s, v31.4s // t1234
fadd \o1\().4s, \o0\().4s, v20.4s // out_o2
fadd \o3\().4s, \o2\().4s, v18.4s // out_o2
fsub \o0\().4s, \o0\().4s, v20.4s // out_o1
fsub \o2\().4s, \o2\().4s, v18.4s // out_o1
.endm
const tab_16pt, align=4
.float -COS16_1, COS16_1, -COS16_3, COS16_3 // Could be +-+- too
.float COS16_3, COS16_3, COS16_1, COS16_1
.float 1.0, 1.0, M_SQRT1_2, M_SQRT1_2
endconst
// 16-point FFT
// t3, t4, t5, t6 must be sequential
.macro FFT16 e0, e1, e2, e3, o0, o1, o2, o3, \
t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22
FFT8 \e0, \e1, \e2, \e3, \t0, \t1, \t2, \t3, \t4, \t5, \t6
FFT4_X2 \o0, \o1, \o2, \o3, \t0, \t1, \t2, \t3, \t4, \t5, \t6
movrel x5, tab_16pt
rev64 \t0\().4s, \o0\().4s // z[ 8, 9].imre
rev64 \t1\().4s, \o2\().4s // z[10,11].imre
ins \t0\().d[0], xzr
ins \t1\().d[0], xzr
ld1 { \t4\().4s, \t5\().4s, \t6\().4s }, [x5]
// TODO: We could derive \t4\() or \t5\() from either, but it seems cheaper to load
fmla \o2\().4s, \t1\().4s, v31.4s // s[4567]
fmls \o0\().4s, \t0\().4s, v31.4s // s[0123]
fmul \t2\().4s, \o1\().4s, \t4\().4s
fmul \t3\().4s, \o3\().4s, \t4\().4s
rev64 \o3\().4s, \o3\().4s
rev64 \o1\().4s, \o1\().4s
fmla \t3\().4s, \o3\().4s, \t5\().4s // s[12, 13, 14, 15]
fmls \t2\().4s, \o1\().4s, \t5\().4s // s[ 8, 9, 10, 11]
fmul \t1\().4s, \o2\().4s, \t6\().4s // s[4567] * mult
fmul \t0\().4s, \o0\().4s, \t6\().4s // s[0123] * mult
mov \o1\().16b, \t3\().16b
mov \o2\().16b, \t1\().16b
fsub \t3\().4s, \t3\().4s, \t2\().4s // y34, u34
fsub \t1\().4s, \t1\().4s, \t0\().4s // w34, x34
fadd \t2\().4s, \t2\().4s, \o1\().4s // y56, u56
rev64 \t3\().4s, \t3\().4s
fadd \t0\().4s, \t0\().4s, \o2\().4s // w56, x56
rev64 \t1\().4s, \t1\().4s
fmul \t2\().4s, \t2\().4s, v31.4s
fmul \t1\().4s, \t1\().4s, v31.4s
fadd \o3\().4s, \e3\().4s, \t3\().4s
fsub \o2\().4s, \e3\().4s, \t3\().4s
fsub \o1\().4s, \e2\().4s, \t2\().4s
fadd \o0\().4s, \e2\().4s, \t2\().4s
fsub \e2\().4s, \e0\().4s, \t0\().4s
fadd \e0\().4s, \e0\().4s, \t0\().4s
fsub \e3\().4s, \e1\().4s, \t1\().4s
fadd \e1\().4s, \e1\().4s, \t1\().4s
.endm
function ff_tx_fft2_float_neon, export=1
ld2r { v0.2d, v1.2d }, [x2]
fneg v2.2s, v1.2s
mov v2.d[1], v1.d[0]
fsub v2.4s, v0.4s, v2.4s
st1 { v2.4s }, [x1]
ret
endfunc
.macro FFT4_FN name, inv
function ff_tx_fft4_\name\()_float_neon, export=1
ld1 {v0.4s, v1.4s}, [x2]
.if \inv == 1
mov v2.d[0], v0.d[1]
mov v0.d[1], v1.d[1]
mov v1.d[1], v2.d[0]
.endif
FFT4 v0, v1, 1
st1 { v0.4s, v1.4s }, [x1]
ret
endfunc
.endm
FFT4_FN fwd, 0
FFT4_FN inv, 1
.macro FFT8_FN name, no_perm
function ff_tx_fft8_\name\()_neon, export=1
SETUP_LUT \no_perm
LOAD_INPUT 0, 1, 2, 3, x2, \no_perm
LOAD_SUBADD
FFT8 v0, v1, v2, v3
zip1 v16.2d, v0.2d, v2.2d
zip2 v17.2d, v0.2d, v2.2d
zip1 v18.2d, v1.2d, v3.2d
zip2 v19.2d, v1.2d, v3.2d
st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x1]
ret
endfunc
.endm
FFT8_FN float, 0
FFT8_FN ns_float, 1
.macro FFT16_FN name, no_perm
function ff_tx_fft16_\name\()_neon, export=1
SETUP_LUT \no_perm
LOAD_INPUT 0, 1, 2, 3, x2, \no_perm
LOAD_INPUT 4, 5, 6, 7, x2, \no_perm
LOAD_SUBADD
FFT16 v0, v1, v2, v3, v4, v5, v6, v7
zip1 v20.2d, v0.2d, v4.2d
zip2 v21.2d, v0.2d, v4.2d
zip1 v22.2d, v1.2d, v6.2d
zip2 v23.2d, v1.2d, v6.2d
st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x1], #64
zip1 v24.2d, v2.2d, v5.2d
zip2 v25.2d, v2.2d, v5.2d
zip1 v26.2d, v3.2d, v7.2d
zip2 v27.2d, v3.2d, v7.2d
st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x1]
ret
endfunc
.endm
FFT16_FN float, 0
FFT16_FN ns_float, 1
.macro SETUP_SR_RECOMB len, re, im, dec
ldr w5, =(\len - 4*7)
movrel \re, X(ff_tx_tab_\len\()_float)
add \im, \re, x5
mov \dec, #-32
.if \len > 32
mov x21, #2*\len
add x22, x21, x21, lsl #1
.endif
.endm
.macro SR_COMBINE e0, e1, e2, e3, e4, e5, e6, e7, \
o0, o1, o2, o3, o4, o5, o6, o7, \
re, im, dec, swap_im, \
t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, \
t6=v22, t7=v23, t8=v24, t9=v25, ta=v26, tb=v27
ld1 { \t8\().4s, \t9\().4s }, [\im], \dec
ld1 { \t0\().4s, \t1\().4s }, [\re], #32
.if \swap_im == 1
ext \t2\().16b, \t9\().16b, \t9\().16b, #8
ext \t3\().16b, \t8\().16b, \t8\().16b, #8
.else
ext \t2\().16b, \t8\().16b, \t8\().16b, #8
ext \t3\().16b, \t9\().16b, \t9\().16b, #8
.endif
trn1 \t4\().4s, \t0\().4s, \t0\().4s // cos0022
trn2 \t0\().4s, \t0\().4s, \t0\().4s // cos4466
trn1 \t5\().4s, \t1\().4s, \t1\().4s // cos1133
trn2 \t1\().4s, \t1\().4s, \t1\().4s // cos5577
rev64 \t6\().4s, \o0\().4s // E m2[0,1].imre
rev64 \t7\().4s, \o2\().4s // O m2[0,1].imre
rev64 \t8\().4s, \o4\().4s // E m2[2,3].imre
rev64 \t9\().4s, \o6\().4s // O m2[2,3].imre
fmul \t6\().4s, \t6\().4s, \t4\().4s // E m2[0,1].imre*t1[0,2]
fmul \t7\().4s, \t7\().4s, \t0\().4s // O m2[0,1].imre*t1[0,2]
fmul \t8\().4s, \t8\().4s, \t4\().4s // E m2[2,3].imre*t1[0,2]
fmul \t9\().4s, \t9\().4s, \t0\().4s // O m2[2,3].imre*t1[0,2]
rev64 \ta\().4s, \o1\().4s // E m3[0,1].imre
rev64 \tb\().4s, \o3\().4s // O m3[0,1].imre
rev64 \t4\().4s, \o5\().4s // E m3[2,3].imre
rev64 \t0\().4s, \o7\().4s // O m3[2,3].imre
fmul \ta\().4s, \ta\().4s, \t5\().4s // E m3[0,1].imre*t1[4,6]
fmul \tb\().4s, \tb\().4s, \t1\().4s // O m3[0,1].imre*t1[4,6]
fmul \t4\().4s, \t4\().4s, \t5\().4s // E m3[2,3].imre*t1[4,6]
fmul \t0\().4s, \t0\().4s, \t1\().4s // O m3[2,3].imre*t1[4,6]
trn1 \t5\().4s, \t3\().4s, \t3\().4s // wim2200
trn2 \t3\().4s, \t3\().4s, \t3\().4s // wim3311
trn1 \t1\().4s, \t2\().4s, \t2\().4s // wim6644
trn2 \t2\().4s, \t2\().4s, \t2\().4s // wim7755
fmul \t5\().4s, \t5\().4s, v31.4s
fmul \t3\().4s, \t3\().4s, v31.4s
fmul \t1\().4s, \t1\().4s, v31.4s
fmul \t2\().4s, \t2\().4s, v31.4s
fmla \t7\().4s, \o2\().4s, \t5\().4s // O w0123
fmls \t9\().4s, \o6\().4s, \t5\().4s // O j0123
fmla \t6\().4s, \o0\().4s, \t3\().4s // E w0123
fmls \t8\().4s, \o4\().4s, \t3\().4s // E j0123
fmla \ta\().4s, \o1\().4s, \t2\().4s // E w4567
fmla \tb\().4s, \o3\().4s, \t1\().4s // O w4567
fmls \t4\().4s, \o5\().4s, \t2\().4s // E j4567
fmls \t0\().4s, \o7\().4s, \t1\().4s // O j4567
fsub \t2\().4s, \t7\().4s, \t9\().4s
fsub \t1\().4s, \t8\().4s, \t6\().4s
fsub \t3\().4s, \t4\().4s, \ta\().4s
fsub \t5\().4s, \t0\().4s, \tb\().4s
fadd \t6\().4s, \t8\().4s, \t6\().4s
fadd \t7\().4s, \t9\().4s, \t7\().4s
fadd \t8\().4s, \t4\().4s, \ta\().4s
fadd \t9\().4s, \t0\().4s, \tb\().4s
fmul \t1\().4s, \t1\().4s, v31.4s
fmul \t2\().4s, \t2\().4s, v31.4s
fmul \t3\().4s, \t3\().4s, v31.4s
fmul \t5\().4s, \t5\().4s, v31.4s
rev64 \t6\().4s, \t6\().4s
rev64 \t8\().4s, \t8\().4s
rev64 \t7\().4s, \t7\().4s
rev64 \t9\().4s, \t9\().4s
fsub \o0\().4s, \e0\().4s, \t6\().4s
fsub \o1\().4s, \e1\().4s, \t8\().4s
fsub \o2\().4s, \e2\().4s, \t1\().4s
fsub \o3\().4s, \e3\().4s, \t3\().4s
fsub \o4\().4s, \e4\().4s, \t7\().4s
fsub \o5\().4s, \e6\().4s, \t9\().4s
fadd \o6\().4s, \e5\().4s, \t2\().4s
fsub \o7\().4s, \e7\().4s, \t5\().4s
fadd \e0\().4s, \e0\().4s, \t6\().4s
fadd \e1\().4s, \e1\().4s, \t8\().4s
fadd \e2\().4s, \e2\().4s, \t1\().4s
fadd \e3\().4s, \e3\().4s, \t3\().4s
fadd \e4\().4s, \e4\().4s, \t7\().4s
fsub \e5\().4s, \e5\().4s, \t2\().4s // swapped
fadd \e6\().4s, \e6\().4s, \t9\().4s // swapped
fadd \e7\().4s, \e7\().4s, \t5\().4s
.endm
.macro SR_COMBINE_HALF e0, e1, e2, e3, \
o0, o1, o2, o3, \
c0, c1, c2, c3, \
t0, t1, t2, t3, t4, t5, part
.if \part == 0
trn1 \t4\().4s, \c0\().4s, \c0\().4s // cos0022
trn1 \c1\().4s, \c1\().4s, \c1\().4s // cos1133
.else
trn2 \t4\().4s, \c0\().4s, \c0\().4s // cos0022
trn2 \c1\().4s, \c1\().4s, \c1\().4s // cos1133
.endif
.if \part == 0
trn2 \t5\().4s, \c2\().4s, \c2\().4s // wim7755
trn2 \c3\().4s, \c3\().4s, \c3\().4s // wim3311
.else
trn1 \t5\().4s, \c2\().4s, \c2\().4s // wim7755
trn1 \c3\().4s, \c3\().4s, \c3\().4s // wim3311
.endif
fmul \t5\().4s, \t5\().4s, v31.4s
fmul \c3\().4s, \c3\().4s, v31.4s
rev64 \t0\().4s, \o0\().4s // E m2[0,1].imre
rev64 \t1\().4s, \o2\().4s // E m2[2,3].imre
rev64 \t2\().4s, \o1\().4s // E m3[0,1].imre
rev64 \t3\().4s, \o3\().4s // E m3[2,3].imre
fmul \o0\().4s, \o0\().4s, \c3\().4s // E m2[0,1].imre*t1[0,2]
fmul \o1\().4s, \o1\().4s, \t5\().4s // E m3[0,1].imre*t1[4,6]
fmla \o0\().4s, \t0\().4s, \t4\().4s // E w0123
fmla \o1\().4s, \t2\().4s, \c1\().4s // E w4567
fmul \t1\().4s, \t1\().4s, \t4\().4s // E m2[2,3].imre*t1[0,2]
fmul \t3\().4s, \t3\().4s, \c1\().4s // E m3[2,3].imre*t1[4,6]
fmls \t1\().4s, \o2\().4s, \c3\().4s // E j0123
fmls \t3\().4s, \o3\().4s, \t5\().4s // E j4567
fsub \t0\().4s, \t1\().4s, \o0\().4s
fadd \t1\().4s, \t1\().4s, \o0\().4s
fadd \t2\().4s, \t3\().4s, \o1\().4s
fsub \t3\().4s, \t3\().4s, \o1\().4s
fmul \t0\().4s, \t0\().4s, v31.4s
fmul \t3\().4s, \t3\().4s, v31.4s
rev64 \t1\().4s, \t1\().4s
rev64 \t2\().4s, \t2\().4s
.if \part == 0
fsub \o0\().4s, \e0\().4s, \t1\().4s
fsub \o1\().4s, \e1\().4s, \t2\().4s
fsub \o2\().4s, \e2\().4s, \t0\().4s
fsub \o3\().4s, \e3\().4s, \t3\().4s
.else
fsub \o0\().4s, \e0\().4s, \t1\().4s
fadd \o2\().4s, \e1\().4s, \t2\().4s
fsub \o1\().4s, \e2\().4s, \t0\().4s
fadd \o3\().4s, \e3\().4s, \t3\().4s
.endif
.if \part == 0
fadd \e0\().4s, \e0\().4s, \t1\().4s
fadd \e1\().4s, \e1\().4s, \t2\().4s
fadd \e2\().4s, \e2\().4s, \t0\().4s
fadd \e3\().4s, \e3\().4s, \t3\().4s
.else
fadd \e0\().4s, \e0\().4s, \t1\().4s
fsub \e1\().4s, \e1\().4s, \t2\().4s // swapped
fadd \e2\().4s, \e2\().4s, \t0\().4s // swapped
fsub \e3\().4s, \e3\().4s, \t3\().4s
.endif
.endm
/* Same as SR_COMBINE_HALF, but heroically tries to use 3 temporary registers
* without touching the tables. */
.macro SR_COMBINE_LITE e0, e1, e2, e3, \
o0, o1, o2, o3, \
c0, c1, c2, c3, \
t0, t1, t2, part
rev64 \t0\().4s, \o0\().4s // E m2[0,1].imre
rev64 \t1\().4s, \o2\().4s // E m2[2,3].imre
.if \part == 0
trn2 \t2\().4s, \c3\().4s, \c3\().4s // wim3311
.else
trn1 \t2\().4s, \c3\().4s, \c3\().4s // wim3311
.endif
fmul \t2\().4s, \t2\().4s, v31.4s
fmul \o2\().4s, \o2\().4s, \t2\().4s
fmul \o0\().4s, \o0\().4s, \t2\().4s // E m2[0,1].imre*t1[0,2]
.if \part == 0
trn1 \t2\().4s, \c0\().4s, \c0\().4s // cos0022
.else
trn2 \t2\().4s, \c0\().4s, \c0\().4s // cos0022
.endif
fmul \t1\().4s, \t1\().4s, \t2\().4s // E m2[2,3].imre*t1[0,2]
fmla \o0\().4s, \t0\().4s, \t2\().4s // E w0123
fsub \t1\().4s, \t1\().4s, \o2\().4s // E j0123
rev64 \t2\().4s, \o1\().4s // E m3[0,1].imre
rev64 \o2\().4s, \o3\().4s // E m3[2,3].imre
.if \part == 0
trn2 \t0\().4s, \c2\().4s, \c2\().4s // wim7755
.else
trn1 \t0\().4s, \c2\().4s, \c2\().4s // wim7755
.endif
fmul \t0\().4s, \t0\().4s, v31.4s
fmul \o1\().4s, \o1\().4s, \t0\().4s // E m3[0,1].imre*t1[4,6]
fmul \o3\().4s, \o3\().4s, \t0\().4s
.if \part == 0
trn1 \t0\().4s, \c1\().4s, \c1\().4s // cos1133
.else
trn2 \t0\().4s, \c1\().4s, \c1\().4s // cos1133
.endif
fmul \o2\().4s, \o2\().4s, \t0\().4s // E m3[2,3].imre*t1[4,6]
fmla \o1\().4s, \t2\().4s, \t0\().4s // E w4567
fsub \o2\().4s, \o2\().4s, \o3\().4s // E j4567
fsub \t0\().4s, \t1\().4s, \o0\().4s
fadd \o0\().4s, \t1\().4s, \o0\().4s
fadd \t2\().4s, \o2\().4s, \o1\().4s
fsub \t1\().4s, \o2\().4s, \o1\().4s
fmul \t0\().4s, \t0\().4s, v31.4s
fmul \t1\().4s, \t1\().4s, v31.4s
rev64 \t2\().4s, \t2\().4s
rev64 \o0\().4s, \o0\().4s
.if \part == 0
fsub \o1\().4s, \e1\().4s, \t2\().4s
fsub \o2\().4s, \e2\().4s, \t0\().4s
fsub \o3\().4s, \e3\().4s, \t1\().4s
.else
fadd \o2\().4s, \e1\().4s, \t0\().4s
fsub \o1\().4s, \e2\().4s, \t2\().4s
fadd \o3\().4s, \e3\().4s, \t1\().4s
.endif
.if \part == 0
fadd \e1\().4s, \e1\().4s, \t2\().4s
fadd \e2\().4s, \e2\().4s, \t0\().4s
fadd \e3\().4s, \e3\().4s, \t1\().4s
.else
fsub \e1\().4s, \e1\().4s, \t0\().4s // swapped
fadd \e2\().4s, \e2\().4s, \t2\().4s // swapped
fsub \e3\().4s, \e3\().4s, \t1\().4s
.endif
mov \t1\().16b, \o0\().16b
fsub \o0\().4s, \e0\().4s, \t1\().4s
fadd \e0\().4s, \e0\().4s, \t1\().4s
.endm
.macro SR_COMBINE_4 len, part, off
add x10, x1, x21
add x11, x1, x21, lsl #1
add x12, x1, x22
ldp q0, q1, [x1, #((0 + \part)*32 + \off)]
ldp q4, q5, [x1, #((2 + \part)*32 + \off)]
ldp q2, q3, [x10, #((0 + \part)*32 + \off)]
ldp q6, q7, [x10, #((2 + \part)*32 + \off)]
ldp q8, q9, [x11, #((0 + \part)*32 + \off)]
ldp q10, q11, [x11, #((2 + \part)*32 + \off)]
ldp q12, q13, [x12, #((0 + \part)*32 + \off)]
ldp q14, q15, [x12, #((2 + \part)*32 + \off)]
SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \
v8, v9, v10, v11, v12, v13, v14, v15, \
x7, x8, x9, 0
stp q0, q1, [x1, #((0 + \part)*32 + \off)]
stp q4, q5, [x1, #((2 + \part)*32 + \off)]
stp q2, q3, [x10, #((0 + \part)*32 + \off)]
stp q6, q7, [x10, #((2 + \part)*32 + \off)]
stp q8, q9, [x11, #((0 + \part)*32 + \off)]
stp q12, q13, [x11, #((2 + \part)*32 + \off)]
stp q10, q11, [x12, #((0 + \part)*32 + \off)]
stp q14, q15, [x12, #((2 + \part)*32 + \off)]
.endm
.macro SR_COMBINE_FULL len, off=0
add x10, x1, x21
add x11, x1, x21, lsl #1
add x12, x1, x22
SR_COMBINE_4 \len, 0, \off
SR_COMBINE_4 \len, 1, \off
SR_COMBINE_4 \len, 4, \off
SR_COMBINE_4 \len, 5, \off
.endm
.macro SR_COMBINE_D2 part, off
add x10, x1, #((\part)*32 + \off)
add x11, x14, #((\part)*32 + \off)
add x12, x15, #((\part)*32 + \off)
add x13, x16, #((\part)*32 + \off)
ldp q0, q1, [x10]
ldp q4, q5, [x10, #(2*32)]
ldp q2, q3, [x11]
ldp q6, q7, [x11, #(2*32)]
ldp q8, q9, [x12]
ldp q10, q11, [x12, #(2*32)]
ldp q12, q13, [x13]
ldp q14, q15, [x13, #(2*32)]
SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \
v8, v9, v10, v11, v12, v13, v14, v15, \
x7, x8, x9, 0, \
v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27
zip1 v16.2d, v0.2d, v4.2d
zip2 v17.2d, v0.2d, v4.2d
zip1 v18.2d, v1.2d, v5.2d
zip2 v19.2d, v1.2d, v5.2d
zip1 v20.2d, v2.2d, v6.2d
zip2 v21.2d, v2.2d, v6.2d
zip1 v22.2d, v3.2d, v7.2d
zip2 v23.2d, v3.2d, v7.2d
ldp q0, q1, [x10, #(1*32)]
ldp q4, q5, [x10, #(3*32)]
ldp q2, q3, [x11, #(1*32)]
ldp q6, q7, [x11, #(3*32)]
st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x10], #64
st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x11], #64
zip1 v20.2d, v8.2d, v12.2d
zip2 v21.2d, v8.2d, v12.2d
zip1 v22.2d, v9.2d, v13.2d
zip2 v23.2d, v9.2d, v13.2d
zip1 v24.2d, v10.2d, v14.2d
zip2 v25.2d, v10.2d, v14.2d
zip1 v26.2d, v11.2d, v15.2d
zip2 v27.2d, v11.2d, v15.2d
ldp q8, q9, [x12, #(1*32)]
ldp q10, q11, [x12, #(3*32)]
ldp q12, q13, [x13, #(1*32)]
ldp q14, q15, [x13, #(3*32)]
st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x12], #64
st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x13], #64
SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \
v8, v9, v10, v11, v12, v13, v14, v15, \
x7, x8, x9, 0, \
v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27
zip1 v16.2d, v0.2d, v4.2d
zip2 v17.2d, v0.2d, v4.2d
zip1 v18.2d, v1.2d, v5.2d
zip2 v19.2d, v1.2d, v5.2d
st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x10]
zip1 v16.2d, v2.2d, v6.2d
zip2 v17.2d, v2.2d, v6.2d
zip1 v18.2d, v3.2d, v7.2d
zip2 v19.2d, v3.2d, v7.2d
st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x11]
zip1 v20.2d, v8.2d, v12.2d
zip2 v21.2d, v8.2d, v12.2d
zip1 v22.2d, v9.2d, v13.2d
zip2 v23.2d, v9.2d, v13.2d
st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x12]
zip1 v24.2d, v10.2d, v14.2d
zip2 v25.2d, v10.2d, v14.2d
zip1 v26.2d, v11.2d, v15.2d
zip2 v27.2d, v11.2d, v15.2d
st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x13]
.endm
.macro SR_COMBINE_DINT off=0
add x14, x1, x21
add x15, x1, x21, lsl #1
add x16, x1, x22
SR_COMBINE_D2 0, \off
SR_COMBINE_D2 4, \off
.endm
.macro FFT32_FN name, no_perm
function ff_tx_fft32_\name\()_neon, export=1
stp d14, d15, [sp, #-16*4]!
stp d8, d9, [sp, #16*3]
stp d10, d11, [sp, #16*2]
stp d12, d13, [sp, #16]
LOAD_SUBADD
SETUP_SR_RECOMB 32, x7, x8, x9
SETUP_LUT \no_perm
LOAD_INPUT 0, 1, 2, 3, x2, \no_perm
LOAD_INPUT 4, 5, 6, 7, x2, \no_perm
LOAD_INPUT 8, 9, 10, 11, x2, \no_perm
LOAD_INPUT 12, 13, 14, 15, x2, \no_perm
FFT8_X2 v8, v9, v10, v11, v12, v13, v14, v15
FFT16 v0, v1, v2, v3, v4, v5, v6, v7
SR_COMBINE v0, v1, v2, v3, v4, v5, v6, v7, \
v8, v9, v10, v11, v12, v13, v14, v15, \
x7, x8, x9, 0
zip1 v16.2d, v0.2d, v4.2d
zip2 v17.2d, v0.2d, v4.2d
zip1 v18.2d, v1.2d, v6.2d
zip2 v19.2d, v1.2d, v6.2d
st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x1], #64
zip1 v20.2d, v2.2d, v5.2d
zip2 v21.2d, v2.2d, v5.2d
zip1 v22.2d, v3.2d, v7.2d
zip2 v23.2d, v3.2d, v7.2d
st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x1], #64
zip1 v24.2d, v8.2d, v12.2d
zip2 v25.2d, v8.2d, v12.2d
zip1 v26.2d, v9.2d, v13.2d
zip2 v27.2d, v9.2d, v13.2d
st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x1], #64
zip1 v28.2d, v10.2d, v14.2d
zip2 v29.2d, v10.2d, v14.2d
zip1 v30.2d, v11.2d, v15.2d
zip2 v31.2d, v11.2d, v15.2d
st1 { v28.4s, v29.4s, v30.4s, v31.4s }, [x1]
ldp d12, d13, [sp, #16]
ldp d10, d11, [sp, #16*2]
ldp d8, d9, [sp, #16*3]
ldp d14, d15, [sp], #16*4
ret
endfunc
.endm
FFT32_FN float, 0
FFT32_FN ns_float, 1
.macro cmp_imm reg, imm
.if \imm >= 4096
cmp \reg, #((\imm)/4096), lsl #12
.else
cmp \reg, #(\imm)
.endif
.endm
.macro SR_TRANSFORM_DEF len, next=0
\len:
stp x20, x30, [sp, #-16]!
mov w20, #(\len/4)
mov x5, #((\len*4) - (\len/1))
add x1, x1, x5
bl 32b
mov x5, #((\len*2) - (\len/2))
add x1, x1, x5
bl 32b
ldp x20, x30, [sp], #16
ldr w5, =(\len*6 + \len/2)
sub x1, x1, x5
SETUP_SR_RECOMB \len, x7, x8, x9
.if \next\() != 0
cmp_imm w19, \len
b.eq 0f
mov w5, #(\len/128)
\len\()5:
SR_COMBINE_FULL \len
add x1, x1, 8*32
subs w5, w5, 1
b.gt \len\()5b
cmp_imm w20, \len
b.gt \next\()f
ret
.endif
.endm
.macro FFT_SPLIT_RADIX_FN name, no_perm
function ff_tx_fft_sr_\name\()_neon, export=1
stp x21, x22, [sp, #-16*6]!
stp d8, d9, [sp, #16*5]
stp d10, d11, [sp, #16*4]
stp d12, d13, [sp, #16*3]
stp d14, d15, [sp, #16*2]
stp x19, x20, [sp, #16]
ldr w19, [x0, #0] // global target
mov w20, w19 // local length
LOAD_SUBADD
SETUP_LUT \no_perm
32:
SETUP_SR_RECOMB 32, x7, x8, x9
LOAD_INPUT 0, 1, 2, 3, x2, \no_perm
LOAD_INPUT 4, 6, 5, 7, x2, \no_perm, 1
LOAD_INPUT 8, 9, 10, 11, x2, \no_perm
LOAD_INPUT 12, 13, 14, 15, x2, \no_perm
FFT8_X2 v8, v9, v10, v11, v12, v13, v14, v15
FFT16 v0, v1, v2, v3, v4, v6, v5, v7
SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \
v8, v9, v10, v11, v12, v13, v14, v15, \
x7, x8, x9, 0
stp q2, q3, [x1, #32*1]
stp q6, q7, [x1, #32*3]
stp q10, q11, [x1, #32*5]
stp q14, q15, [x1, #32*7]
cmp w20, #32
b.gt 64f
stp q0, q1, [x1, #32*0]
stp q4, q5, [x1, #32*2]
stp q8, q9, [x1, #32*4]
stp q12, q13, [x1, #32*6]
ret
64:
SETUP_SR_RECOMB 64, x7, x8, x9
LOAD_INPUT 2, 3, 10, 11, x2, \no_perm, 1
LOAD_INPUT 6, 14, 7, 15, x2, \no_perm, 1
FFT16 v2, v3, v10, v11, v6, v14, v7, v15
LOAD_INPUT 16, 17, 18, 19, x2, \no_perm
LOAD_INPUT 20, 22, 21, 23, x2, \no_perm, 1
FFT16 v16, v17, v18, v19, v20, v22, v21, v23, \
v24, v25, v26, v27, v28, v29, v30
ld1 { v26.4s, v27.4s }, [x8], x9
ldp q24, q25, [x7], #32
ext v26.16b, v26.16b, v26.16b, #8
ext v27.16b, v27.16b, v27.16b, #8
cmp w19, #64
b.eq 2f // custom deinterleave
// TODO: investigate doing the 2 combines like in deinterleave
// TODO: experiment with spilling to gprs and converting to HALF or full
SR_COMBINE_LITE v0, v1, v8, v9, \
v2, v3, v16, v17, \
v24, v25, v26, v27, \
v28, v29, v30, 0
stp q0, q1, [x1, #32* 0]
stp q8, q9, [x1, #32* 4]
stp q2, q3, [x1, #32* 8]
stp q16, q17, [x1, #32*12]
SR_COMBINE_HALF v4, v5, v12, v13, \
v6, v7, v20, v21, \
v24, v25, v26, v27, \
v28, v29, v30, v0, v1, v8, 1
stp q4, q20, [x1, #32* 2]
stp q12, q21, [x1, #32* 6]
stp q6, q5, [x1, #32*10]
stp q7, q13, [x1, #32*14]
ldp q2, q3, [x1, #32*1]
ldp q6, q7, [x1, #32*3]
ldp q12, q13, [x1, #32*5]
ldp q16, q17, [x1, #32*7]
SR_COMBINE v2, v3, v12, v13, v6, v16, v7, v17, \
v10, v11, v14, v15, v18, v19, v22, v23, \
x7, x8, x9, 0, \
v24, v25, v26, v27, v28, v29, v30, v8, v0, v1, v4, v5
stp q2, q3, [x1, #32* 1]
stp q6, q7, [x1, #32* 3]
stp q12, q13, [x1, #32* 5]
stp q16, q17, [x1, #32* 7]
stp q10, q11, [x1, #32* 9]
stp q18, q19, [x1, #32*11]
stp q14, q15, [x1, #32*13]
stp q22, q23, [x1, #32*15]
cmp w20, #64
b.gt 128f
ret
128:
stp x20, x30, [sp, #-16]!
mov w20, #32
add x1, x1, #16*32
bl 32b
add x1, x1, #8*32
bl 32b
ldp x20, x30, [sp], #16
sub x1, x1, #24*32
SETUP_SR_RECOMB 128, x7, x8, x9
cmp w19, #128
b.eq 0f
SR_COMBINE_FULL 128
cmp w20, #128
b.gt 256f
ret
256:
stp x20, x30, [sp, #-16]!
mov w20, #64
add x1, x1, #32*32
bl 32b
add x1, x1, #16*32
bl 32b
ldp x20, x30, [sp], #16
sub x1, x1, #48*32
SETUP_SR_RECOMB 256, x7, x8, x9
cmp w19, #256
b.eq 0f
SR_COMBINE_FULL 256
SR_COMBINE_FULL 256, 8*32
cmp w20, #256
b.gt 512f
ret
512:
stp x20, x30, [sp, #-16]!
mov w20, #128
add x1, x1, #64*32
bl 32b
add x1, x1, #32*32
bl 32b
ldp x20, x30, [sp], #16
sub x1, x1, #96*32
SETUP_SR_RECOMB 512, x7, x8, x9
cmp w19, #512
b.eq 0f
mov x5, 4
5125:
SR_COMBINE_FULL 512
add x1, x1, 8*32
subs w5, w5, 1
b.gt 5125b
cmp w20, #512
b.gt 1024f
ret
1024:
stp x20, x30, [sp, #-16]!
mov w20, #256
add x1, x1, #96*32
bl 32b
add x1, x1, #64*32
bl 32b
ldp x20, x30, [sp], #16
mov x5, #192*32
sub x1, x1, x5
SETUP_SR_RECOMB 1024, x7, x8, x9
cmp w19, #1024
b.eq 0f
mov w5, 8
10245:
SR_COMBINE_FULL 1024
add x1, x1, 8*32
subs w5, w5, 1
b.gt 10245b
cmp w20, #1024
b.gt 2048f
ret
SR_TRANSFORM_DEF 2048, 4096
SR_TRANSFORM_DEF 4096, 8192
SR_TRANSFORM_DEF 8192, 16384
SR_TRANSFORM_DEF 16384, 32768
SR_TRANSFORM_DEF 32768, 65536
SR_TRANSFORM_DEF 65536, 131072
SR_TRANSFORM_DEF 131072
0: // general deinterleave loop
SR_COMBINE_DINT
add x1, x1, #32*8
subs w19, w19, #32*4
b.gt 0b
ldp x19, x20, [sp, #16]
ldp d14, d15, [sp, #16*2]
ldp d12, d13, [sp, #16*3]
ldp d10, d11, [sp, #16*4]
ldp d8, d9, [sp, #16*5]
ldp x21, x22, [sp], #16*6
ret
2: // special case for 64 point deinterleave
mov x10, v23.d[0]
mov x11, v23.d[1]
SR_COMBINE_LITE v0, v1, v8, v9, \
v2, v3, v16, v17, \
v24, v25, v26, v27, \
v28, v29, v30, 0
SR_COMBINE_HALF v4, v5, v12, v13, \
v6, v7, v20, v21, \
v24, v25, v26, v27, \
v28, v29, v30, v23, v24, v26, 1
zip1 v23.2d, v0.2d, v4.2d
zip2 v24.2d, v0.2d, v4.2d
zip1 v25.2d, v1.2d, v20.2d
zip2 v26.2d, v1.2d, v20.2d
zip1 v27.2d, v8.2d, v12.2d
zip2 v28.2d, v8.2d, v12.2d
zip1 v29.2d, v9.2d, v21.2d
zip2 v30.2d, v9.2d, v21.2d
mov v20.16b, v5.16b
mov v21.16b, v7.16b
mov x12, x1
add x13, x1, #32* 4
add x14, x1, #32* 8
add x15, x1, #32*12
zip1 v4.2d, v2.2d, v6.2d
zip2 v5.2d, v2.2d, v6.2d
zip1 v6.2d, v3.2d, v20.2d
zip2 v7.2d, v3.2d, v20.2d
zip1 v0.2d, v16.2d, v21.2d
zip2 v1.2d, v16.2d, v21.2d
zip1 v2.2d, v17.2d, v13.2d
zip2 v3.2d, v17.2d, v13.2d
// stp is faster by a little on A53, but this is faster on M1s (theory)
ldp q8, q9, [x1, #32*1]
ldp q12, q13, [x1, #32*5]
st1 { v23.4s, v24.4s, v25.4s, v26.4s }, [x12], #64 // 32* 0...1
st1 { v27.4s, v28.4s, v29.4s, v30.4s }, [x13], #64 // 32* 4...5
st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x14], #64 // 32* 8...9
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x15], #64 // 32*12..13
mov v23.d[0], x10
mov v23.d[1], x11
ldp q6, q7, [x1, #32*3]
ldp q16, q17, [x1, #32*7]
SR_COMBINE v8, v9, v12, v13, v6, v16, v7, v17, \
v10, v11, v14, v15, v18, v19, v22, v23, \
x7, x8, x9, 0, \
v24, v25, v26, v27, v28, v29, v30, v4, v0, v1, v5, v20
zip1 v0.2d, v8.2d, v6.2d
zip2 v1.2d, v8.2d, v6.2d
zip1 v2.2d, v9.2d, v7.2d
zip2 v3.2d, v9.2d, v7.2d
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x12]
zip1 v4.2d, v12.2d, v16.2d
zip2 v5.2d, v12.2d, v16.2d
zip1 v6.2d, v13.2d, v17.2d
zip2 v7.2d, v13.2d, v17.2d
st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x13]
zip1 v0.2d, v10.2d, v18.2d
zip2 v1.2d, v10.2d, v18.2d
zip1 v2.2d, v11.2d, v19.2d
zip2 v3.2d, v11.2d, v19.2d
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x14]
zip1 v4.2d, v14.2d, v22.2d
zip2 v5.2d, v14.2d, v22.2d
zip1 v6.2d, v15.2d, v23.2d
zip2 v7.2d, v15.2d, v23.2d
st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x15]
ldp x19, x20, [sp, #16]
ldp d14, d15, [sp, #16*2]
ldp d12, d13, [sp, #16*3]
ldp d10, d11, [sp, #16*4]
ldp d8, d9, [sp, #16*5]
ldp x21, x22, [sp], #16*6
ret
endfunc
.endm
FFT_SPLIT_RADIX_FN float, 0
FFT_SPLIT_RADIX_FN ns_float, 1