Source code
Revision control
Copy as Markdown
Other Tools
/*
* Copyright © 2021, VideoLAN and dav1d authors
* Copyright © 2021, Martin Storsjo
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "src/arm/asm.S"
#include "util.S"
#include "src/arm/asm-offsets.h"
#define GRAIN_WIDTH 82
#define GRAIN_HEIGHT 73
#define SUB_GRAIN_WIDTH 44
#define SUB_GRAIN_HEIGHT 38
.macro increment_seed steps, shift=1
lsr r11, r2, #3
lsr r12, r2, #12
lsr lr, r2, #1
eor r11, r2, r11 // (r >> 0) ^ (r >> 3)
eor r12, r12, lr // (r >> 12) ^ (r >> 1)
eor r11, r11, r12 // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1)
.if \shift
lsr r2, r2, #\steps
.endif
and r11, r11, #((1 << \steps) - 1) // bit
.if \shift
orr r2, r2, r11, lsl #(16 - \steps) // *state
.else
orr r2, r2, r11, lsl #16 // *state
.endif
.endm
.macro read_rand dest, bits, age
ubfx \dest, r2, #16 - \bits - \age, #\bits
.endm
.macro read_shift_rand dest, bits
ubfx \dest, r2, #17 - \bits, #\bits
lsr r2, r2, #1
.endm
// special calling convention:
// r2 holds seed
// r3 holds dav1d_gaussian_sequence
// clobbers r11-r12
// returns in d0-d1
function get_gaussian_neon
push {r5-r6,lr}
increment_seed 4
read_rand r5, 11, 3
read_rand r6, 11, 2
add r5, r3, r5, lsl #1
add r6, r3, r6, lsl #1
vld1.16 {d0[0]}, [r5]
read_rand r5, 11, 1
vld1.16 {d0[1]}, [r6]
add r5, r3, r5, lsl #1
read_rand r6, 11, 0
increment_seed 4
add r6, r3, r6, lsl #1
vld1.16 {d0[2]}, [r5]
read_rand r5, 11, 3
vld1.16 {d0[3]}, [r6]
add r5, r3, r5, lsl #1
read_rand r6, 11, 2
vld1.16 {d1[0]}, [r5]
add r6, r3, r6, lsl #1
read_rand r5, 11, 1
vld1.16 {d1[1]}, [r6]
read_rand r6, 11, 0
add r5, r3, r5, lsl #1
add r6, r3, r6, lsl #1
vld1.16 {d1[2]}, [r5]
vld1.16 {d1[3]}, [r6]
pop {r5-r6,pc}
endfunc
function get_grain_2_neon
push {r11,lr}
increment_seed 2
read_rand r11, 11, 1
read_rand r12, 11, 0
add r11, r3, r11, lsl #1
add r12, r3, r12, lsl #1
vld1.16 {d0[0]}, [r11]
vld1.16 {d0[1]}, [r12]
vrshl.s16 d0, d0, d30
pop {r11,pc}
endfunc
.macro get_grain_2 dst
bl get_grain_2_neon
.ifnc \dst, d0
vmov \dst, d0
.endif
.endm
function get_grain_4_neon
push {r11,lr}
increment_seed 4
read_rand r11, 11, 3
read_rand r12, 11, 2
add r11, r3, r11, lsl #1
add r12, r3, r12, lsl #1
vld1.16 {d0[0]}, [r11]
read_rand r11, 11, 1
vld1.16 {d0[1]}, [r12]
read_rand r12, 11, 0
add r11, r3, r11, lsl #1
add r12, r3, r12, lsl #1
vld1.16 {d0[2]}, [r11]
vld1.16 {d0[3]}, [r12]
vrshl.s16 d0, d0, d30
pop {r11,pc}
endfunc
.macro get_grain_4 dst
bl get_grain_4_neon
.ifnc \dst, d0
vmov \dst, d0
.endif
.endm
// r1 holds the number of entries to produce
// r6, r8 and r10 hold the previous output entries
// q0 holds the vector of produced entries
// q1 holds the input vector of sums from above
.macro output_lag n
function output_lag\n\()_neon
push {r0, lr}
.if \n == 1
mvn lr, r5 // grain_min = ~grain_max
.else
mov r0, #1
mov lr, #1
sub r7, r7, #1
sub r9, r9, #1
lsl r0, r0, r7
lsl lr, lr, r9
add r7, r7, #1
add r9, r9, #1
.endif
1:
read_shift_rand r12, 11
vmov.32 r11, d2[0]
lsl r12, r12, #1
vext.8 q0, q0, q0, #2
ldrsh r12, [r3, r12]
.if \n == 1
mla r11, r6, r4, r11 // sum (above) + *coeff * prev output
add r6, r11, r8 // 1 << (ar_coeff_shift - 1)
add r12, r12, r10
asr r6, r6, r7 // >> ar_coeff_shift
asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift)
add r6, r6, r12
cmp r6, r5
.elseif \n == 2
mla r11, r8, r4, r11 // sum (above) + *coeff * prev output 1
mla r11, r6, r10, r11 // += *coeff * prev output 2
mov r8, r6
add r6, r11, r0 // 1 << (ar_coeff_shift - 1)
add r12, r12, lr // 1 << (4 - bitdepth_min_8 + grain_scale_shift - 1)
asr r6, r6, r7 // >> ar_coeff_shift
asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift)
add r6, r6, r12
push {lr}
cmp r6, r5
mvn lr, r5 // grain_min = ~grain_max
.else
push {r1-r3}
sbfx r1, r4, #0, #8
sbfx r2, r4, #8, #8
sbfx r3, r4, #16, #8
mla r11, r10, r1, r11 // sum (above) + *coeff * prev output 1
mla r11, r8, r2, r11 // sum (above) + *coeff * prev output 2
mla r11, r6, r3, r11 // += *coeff * prev output 3
pop {r1-r3}
mov r10, r8
mov r8, r6
add r6, r11, r0 // 1 << (ar_coeff_shift - 1)
add r12, r12, lr // 1 << (4 - bitdepth_min_8 + grain_scale_shift - 1)
asr r6, r6, r7 // >> ar_coeff_shift
asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift)
add r6, r6, r12
push {lr}
cmp r6, r5
mvn lr, r5 // grain_min = ~grain_max
.endif
it gt
movgt r6, r5
cmp r6, lr
it lt
movlt r6, lr
.if \n >= 2
pop {lr}
.endif
subs r1, r1, #1
vext.8 q1, q1, q1, #4
vmov.16 d1[3], r6
bgt 1b
pop {r0, pc}
endfunc
.endm
output_lag 1
output_lag 2
output_lag 3
function sum_lag1_above_neon
sub r12, r0, #1*GRAIN_WIDTH*2 - 16
vld1.16 {q10}, [r12] // load top right
vext.8 q0, q8, q9, #14 // top left, top mid
vext.8 q1, q9, q10, #2 // top left, top mid
vmull.s16 q2, d18, d28
vmlal.s16 q2, d0, d27
vmlal.s16 q2, d2, d29
vmull.s16 q3, d19, d28
vmlal.s16 q3, d1, d27
vmlal.s16 q3, d3, d29
vmov q8, q9
vmov q9, q10
bx lr
endfunc
.macro sum_lag_n_body lag, type, uv_layout, edge, elems, uv_coeff
.ifc \lag\()_\edge, lag3_left
bl sum_lag3_left_above_neon
.else
bl sum_\lag\()_above_neon
.endif
.ifc \type, uv_420
vpush {q6-q7}
add r12, r11, #GRAIN_WIDTH*2
vld1.16 {q0, q1}, [r11]!
vld1.16 {q6, q7}, [r12]!
vpadd.i16 d0, d0, d1
vpadd.i16 d1, d2, d3
vpadd.i16 d12, d12, d13
vpadd.i16 d13, d14, d15
vadd.i16 q0, q0, q6
vpop {q6-q7}
vrshr.s16 q0, q0, #2
.endif
.ifc \type, uv_422
vld1.16 {q0, q1}, [r11]!
vpadd.i16 d0, d0, d1
vpadd.i16 d1, d2, d3
vrshr.s16 q0, q0, #1
.endif
.ifc \type, uv_444
vld1.16 {q0}, [r11]!
.endif
.if \uv_layout
.ifnb \uv_coeff
vdup.8 d13, \uv_coeff
vmovl.s8 q6, d13
.endif
vmlal.s16 q2, d0, d13
vmlal.s16 q3, d1, d13
.endif
.if \uv_layout && \elems == 8
b sum_\lag\()_y_\edge\()_start
.elseif \uv_layout == 444 && \elems == 7
b sum_\lag\()_y_\edge\()_start
.elseif \uv_layout == 422 && \elems == 1
b sum_\lag\()_uv_420_\edge\()_start
.else
sum_\lag\()_\type\()_\edge\()_start:
push {r11}
.if \elems > 4
.ifc \edge, left
increment_seed 4
read_rand r11, 11, 3
read_rand r12, 11, 2
add r11, r3, r11, lsl #1
add r12, r3, r12, lsl #1
vld1.16 {d1[1]}, [r11]
read_rand r11, 11, 1
vld1.16 {d1[2]}, [r12]
add r11, r3, r11, lsl #1
vld1.16 {d1[3]}, [r11]
lsl r2, r2, #1 // shift back the state as if we'd done increment_seed with shift=0
vrshl.s16 d1, d1, d30
vext.8 q2, q2, q2, #12
.ifc \lag, lag3
vmov.s16 r10, d1[1]
.endif
.ifnc \lag, lag1
vmov.s16 r8, d1[2]
.endif
vmov.s16 r6, d1[3]
vmov q1, q2
mov r1, #1
bl output_\lag\()_neon
.else
increment_seed 4, shift=0
vmov q1, q2
mov r1, #4
bl output_\lag\()_neon
.endif
increment_seed 4, shift=0
vmov q1, q3
.ifc \edge, right
mov r1, #3
bl output_\lag\()_neon
read_shift_rand r12, 11
add r12, r3, r12, lsl #1
vld1.16 {d2[0]}, [r12]
vrshl.s16 d2, d2, d30
vext.8 q0, q0, q1, #2
.else
mov r1, #4
bl output_\lag\()_neon
.endif
.else
// elems == 1
increment_seed 4, shift=0
vmov q1, q2
mov r1, #1
bl output_\lag\()_neon
lsr r2, r2, #3
read_rand r11, 11, 2
read_rand r12, 11, 1
add r11, r3, r11, lsl #1
add r12, r3, r12, lsl #1
vld1.16 {d2[0]}, [r11]
read_rand r11, 11, 0
vld1.16 {d2[1]}, [r12]
add r11, r3, r11, lsl #1
vld1.16 {d2[2]}, [r11]
vrshl.s16 d2, d2, d30
vext.8 q0, q0, q1, #14
.endif
vst1.16 {q0}, [r0]!
pop {r11}
pop {r1, pc}
.endif
.endm
.macro sum_lag1_func type, uv_layout, edge, elems=8
function sum_\type\()_lag1_\edge\()_neon
push {r1, lr}
.ifc \edge, left
sub r12, r0, #1*GRAIN_WIDTH*2
vld1.8 {q9}, [r12] // load the previous block right above
.endif
sum_lag_n_body lag1, \type, \uv_layout, \edge, \elems
endfunc
.endm
sum_lag1_func y, 0, left
sum_lag1_func y, 0, mid
sum_lag1_func y, 0, right, 7
sum_lag1_func uv_444, 444, left
sum_lag1_func uv_444, 444, mid
sum_lag1_func uv_444, 444, right, 7
sum_lag1_func uv_422, 422, left
sum_lag1_func uv_422, 422, mid
sum_lag1_func uv_422, 422, right, 1
sum_lag1_func uv_420, 420, left
sum_lag1_func uv_420, 420, mid
sum_lag1_func uv_420, 420, right, 1
function sum_lag2_above_neon
push {lr}
sub r12, r0, #2*GRAIN_WIDTH*2 - 16
sub lr, r0, #1*GRAIN_WIDTH*2 - 16
vld1.16 {q10}, [r12] // load top right
vld1.16 {q13}, [lr]
vdup.8 d10, d28[0]
vext.8 q0, q8, q9, #12 // top left, top mid
vdup.8 d12, d28[1]
vext.8 q1, q8, q9, #14
vdup.8 d14, d28[3]
vext.8 q4, q9, q10, #2 // top mid, top right
vmovl.s8 q5, d10
vmovl.s8 q6, d12
vmovl.s8 q7, d14
vmull.s16 q2, d0, d10
vmlal.s16 q2, d2, d12
vmlal.s16 q2, d8, d14
vmull.s16 q3, d1, d10
vmlal.s16 q3, d3, d12
vmlal.s16 q3, d9, d14
vdup.8 d10, d28[4]
vext.8 q0, q9, q10, #4 // top mid, top right
vdup.8 d12, d28[5]
vext.8 q1, q11, q12, #12 // top left, top mid
vdup.8 d14, d28[6]
vext.8 q4, q11, q12, #14
vmovl.s8 q5, d10
vmovl.s8 q6, d12
vmovl.s8 q7, d14
vmlal.s16 q2, d0, d10
vmlal.s16 q2, d2, d12
vmlal.s16 q2, d8, d14
vmlal.s16 q3, d1, d10
vmlal.s16 q3, d3, d12
vmlal.s16 q3, d9, d14
vdup.8 d10, d29[0]
vext.8 q0, q12, q13, #2 // top mid, top right
vdup.8 d12, d29[1]
vext.8 q1, q12, q13, #4
vdup.8 d14, d28[2]
vdup.8 d8, d28[7]
vmovl.s8 q5, d10
vmovl.s8 q6, d12
vmovl.s8 q7, d14
vmovl.s8 q4, d8
vmlal.s16 q2, d0, d10
vmlal.s16 q2, d2, d12
vmlal.s16 q2, d18, d14
vmlal.s16 q2, d24, d8
vmlal.s16 q3, d1, d10
vmlal.s16 q3, d3, d12
vmlal.s16 q3, d19, d14
vmlal.s16 q3, d25, d8
vmov q8, q9
vmov q9, q10
vmov q11, q12
vmov q12, q13
pop {pc}
endfunc
.macro sum_lag2_func type, uv_layout, edge, elems=8
function sum_\type\()_lag2_\edge\()_neon
push {r1, lr}
.ifc \edge, left
sub r12, r0, #2*GRAIN_WIDTH*2
sub lr, r0, #1*GRAIN_WIDTH*2
vld1.16 {q9}, [r12] // load the previous block right above
vld1.16 {q12}, [lr]
.endif
sum_lag_n_body lag2, \type, \uv_layout, \edge, \elems, uv_coeff=d29[4]
endfunc
.endm
sum_lag2_func y, 0, left
sum_lag2_func y, 0, mid
sum_lag2_func y, 0, right, 7
sum_lag2_func uv_444, 444, left
sum_lag2_func uv_444, 444, mid
sum_lag2_func uv_444, 444, right, 7
sum_lag2_func uv_422, 422, left
sum_lag2_func uv_422, 422, mid
sum_lag2_func uv_422, 422, right, 1
sum_lag2_func uv_420, 420, left
sum_lag2_func uv_420, 420, mid
sum_lag2_func uv_420, 420, right, 1
function sum_lag3_left_above_neon
// A separate codepath for the left edge, to avoid reading outside
// of the edge of the buffer.
sub r12, r0, #3*GRAIN_WIDTH*2
vld1.8 {q11, q12}, [r12]
vext.8 q12, q11, q12, #10
vext.8 q11, q11, q11, #10
b sum_lag3_above_start
endfunc
function sum_lag3_above_neon
movw r12, #(3*GRAIN_WIDTH + 3)*2
sub r12, r0, r12
vld1.8 {q11, q12}, [r12]
sum_lag3_above_start:
vdup.8 d12, d26[0]
vext.8 q1, q11, q12, #2
vdup.8 d14, d26[1]
vext.8 q4, q11, q12, #4
vdup.8 d16, d26[2]
vext.8 q5, q11, q12, #6
vdup.8 d18, d26[3]
vmovl.s8 q6, d12
vmovl.s8 q7, d14
vmovl.s8 q8, d16
vmovl.s8 q9, d18
movw r12, #(2*GRAIN_WIDTH + 3)*2
sub r12, r0, r12
vmull.s16 q2, d22, d12
vmlal.s16 q2, d2, d14
vmlal.s16 q2, d8, d16
vmlal.s16 q2, d10, d18
vmull.s16 q3, d23, d12
vmlal.s16 q3, d3, d14
vmlal.s16 q3, d9, d16
vmlal.s16 q3, d11, d18
vdup.8 d12, d26[4]
vext.8 q0, q11, q12, #8
vdup.8 d14, d26[5]
vext.8 q1, q11, q12, #10
vdup.8 d16, d26[6]
vext.8 q4, q11, q12, #12
vld1.8 {q11, q12}, [r12]
vdup.8 d18, d26[7]
vmovl.s8 q6, d12
vmovl.s8 q7, d14
vmovl.s8 q8, d16
vmovl.s8 q9, d18
vmlal.s16 q2, d0, d12
vmlal.s16 q2, d2, d14
vmlal.s16 q2, d8, d16
vmlal.s16 q2, d22, d18
vmlal.s16 q3, d1, d12
vmlal.s16 q3, d3, d14
vmlal.s16 q3, d9, d16
vmlal.s16 q3, d23, d18
vdup.8 d12, d27[0]
vext.8 q0, q11, q12, #2
vdup.8 d14, d27[1]
vext.8 q1, q11, q12, #4
vdup.8 d16, d27[2]
vext.8 q4, q11, q12, #6
vdup.8 d18, d27[3]
vext.8 q5, q11, q12, #8
vmovl.s8 q6, d12
vmovl.s8 q7, d14
vmovl.s8 q8, d16
vmovl.s8 q9, d18
sub r12, r0, #(1*GRAIN_WIDTH + 3)*2
vmlal.s16 q2, d0, d12
vmlal.s16 q2, d2, d14
vmlal.s16 q2, d8, d16
vmlal.s16 q2, d10, d18
vmlal.s16 q3, d1, d12
vmlal.s16 q3, d3, d14
vmlal.s16 q3, d9, d16
vmlal.s16 q3, d11, d18
vdup.8 d12, d27[4]
vext.8 q0, q11, q12, #10
vdup.8 d14, d27[5]
vext.8 q1, q11, q12, #12
vld1.8 {q11, q12}, [r12]
vdup.8 d16, d27[6]
vdup.8 d18, d27[7]
vmovl.s8 q6, d12
vmovl.s8 q7, d14
vext.8 q5, q11, q12, #2
vmovl.s8 q8, d16
vmovl.s8 q9, d18
vmlal.s16 q2, d0, d12
vmlal.s16 q2, d2, d14
vmlal.s16 q2, d22, d16
vmlal.s16 q2, d10, d18
vmlal.s16 q3, d1, d12
vmlal.s16 q3, d3, d14
vmlal.s16 q3, d23, d16
vmlal.s16 q3, d11, d18
vdup.8 d12, d28[0]
vext.8 q0, q11, q12, #4
vdup.8 d14, d28[1]
vext.8 q1, q11, q12, #6
vdup.8 d16, d28[2]
vext.8 q4, q11, q12, #8
vdup.8 d18, d28[3]
vext.8 q5, q11, q12, #10
vmovl.s8 q6, d12
vmovl.s8 q7, d14
vmovl.s8 q8, d16
vmovl.s8 q9, d18
vmlal.s16 q2, d0, d12
vmlal.s16 q2, d2, d14
vmlal.s16 q2, d8, d16
vmlal.s16 q2, d10, d18
vmlal.s16 q3, d1, d12
vmlal.s16 q3, d3, d14
vmlal.s16 q3, d9, d16
vmlal.s16 q3, d11, d18
vdup.8 d12, d28[4]
vext.8 q0, q11, q12, #12
vmovl.s8 q6, d12
vmlal.s16 q2, d0, d12
vmlal.s16 q3, d1, d12
bx lr
endfunc
.macro sum_lag3_func type, uv_layout, edge, elems=8
function sum_\type\()_lag3_\edge\()_neon
push {r1, lr}
sum_lag_n_body lag3, \type, \uv_layout, \edge, \elems, uv_coeff=d29[0]
endfunc
.endm
sum_lag3_func y, 0, left
sum_lag3_func y, 0, mid
sum_lag3_func y, 0, right, 7
sum_lag3_func uv_444, 444, left
sum_lag3_func uv_444, 444, mid
sum_lag3_func uv_444, 444, right, 7
sum_lag3_func uv_422, 422, left
sum_lag3_func uv_422, 422, mid
sum_lag3_func uv_422, 422, right, 1
sum_lag3_func uv_420, 420, left
sum_lag3_func uv_420, 420, mid
sum_lag3_func uv_420, 420, right, 1
function generate_grain_rows_neon
push {r10-r11,lr}
1:
mov r10, #80
2:
bl get_gaussian_neon
vrshl.s16 q0, q0, q15
subs r10, r10, #8
vst1.16 {q0}, [r0]!
bgt 2b
get_grain_2 d0
subs r1, r1, #1
vst1.32 {d0[0]}, [r0]!
bgt 1b
pop {r10-r11,pc}
endfunc
function generate_grain_rows_44_neon
push {r10-r11,lr}
1:
mov r10, #40
2:
bl get_gaussian_neon
vrshl.s16 q0, q0, q15
subs r10, r10, #8
vst1.16 {q0}, [r0]!
bgt 2b
get_grain_4 d0
subs r1, r1, #1
vst1.16 {d0}, [r0]
add r0, r0, #GRAIN_WIDTH*2-80
bgt 1b
pop {r10-r11,pc}
endfunc
function gen_grain_uv_444_lag0_neon
vld1.16 {q3}, [r11]!
gen_grain_uv_lag0_8_start:
push {r11,lr}
bl get_gaussian_neon
vrshl.s16 q0, q0, q15
gen_grain_uv_lag0_8_add:
vand q3, q3, q1
vmull.s16 q2, d6, d22
vmull.s16 q3, d7, d22
vrshl.s32 q2, q2, q12
vrshl.s32 q3, q3, q12
vqmovn.s32 d4, q2
vqmovn.s32 d5, q3
vqadd.s16 q2, q2, q0
vmin.s16 q2, q2, q9
vmax.s16 q2, q2, q10
vst1.16 {q2}, [r0]!
pop {r11,pc}
endfunc
function gen_grain_uv_420_lag0_8_neon
add r12, r11, #GRAIN_WIDTH*2
vld1.16 {q2,q3}, [r11]!
vld1.16 {q4,q5}, [r12]
vpadd.i16 d4, d4, d5
vpadd.i16 d5, d6, d7
vpadd.i16 d8, d8, d9
vpadd.i16 d9, d10, d11
vadd.i16 q2, q2, q4
vrshr.s16 q3, q2, #2
b gen_grain_uv_lag0_8_start
endfunc
function gen_grain_uv_422_lag0_8_neon
vld1.16 {q2,q3}, [r11]!
vpadd.i16 d4, d4, d5
vpadd.i16 d5, d6, d7
vrshr.s16 q3, q2, #1
b gen_grain_uv_lag0_8_start
endfunc
function gen_grain_uv_420_lag0_4_neon
add r12, r11, #GRAIN_WIDTH*2
vld1.16 {q2}, [r11]
vld1.16 {q0}, [r12]
add r11, r11, #32
vpadd.i16 d4, d4, d5
vpadd.i16 d0, d0, d1
vadd.i16 d4, d4, d0
vrshr.s16 d6, d4, #2
push {r11,lr}
get_grain_4 d0
b gen_grain_uv_lag0_8_add
endfunc
function gen_grain_uv_422_lag0_4_neon
vld1.16 {q2}, [r11]
add r11, r11, #32
vpadd.i16 d4, d4, d5
vrshr.s16 d6, d4, #1
push {r11,lr}
get_grain_4 d0
b gen_grain_uv_lag0_8_add
endfunc
.macro gen_grain_82 type
function generate_grain_\type\()_16bpc_neon, export=1
push {r4-r11,lr}
.ifc \type, uv_444
ldr r4, [sp, #36]
mov r12, r3
mov lr, #28
add r11, r1, #3*GRAIN_WIDTH*2
mov r1, r2
mul r12, r12, lr
clz lr, r4
.else
clz lr, r2
.endif
movrel r3, X(gaussian_sequence)
sub lr, lr, #24 // -bitdepth_min_8
ldr r2, [r1, #FGD_SEED]
ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT]
.ifc \type, y
add r4, r1, #FGD_AR_COEFFS_Y
.else
add r4, r1, #FGD_AR_COEFFS_UV
.endif
add r9, r9, lr // grain_scale_shift - bitdepth_min_8
adr r5, L(gen_grain_\type\()_tbl)
ldr r6, [r1, #FGD_AR_COEFF_LAG]
add r9, r9, #4
ldr r6, [r5, r6, lsl #2]
vdup.16 q15, r9 // 4 - bitdepth_min_8 + data->grain_scale_shift
add r5, r5, r6
vneg.s16 q15, q15
.ifc \type, uv_444
push {lr}
cmp r12, #0
movw r10, #0x49d8
movw lr, #0xb524
// Intentionally using a separate register instead of moveq with an
// immediate constant, to avoid armv8 deprecated it instruction forms.
it eq
moveq r10, lr
add r4, r4, r12 // Add offset to ar_coeffs_uv[1]
eor r2, r2, r10
pop {lr}
.endif
ldr r7, [r1, #FGD_AR_COEFF_SHIFT]
neg lr, lr // bitdepth_min_8
mov r8, #1
mov r10, #1
lsl r8, r8, r7 // 1 << ar_coeff_shift
lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift)
lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1)
lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1)
bx r5
.align 2
L(gen_grain_\type\()_tbl):
.word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB
.word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB
.word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB
.word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB
L(generate_grain_\type\()_lag0):
.ifc \type, y
mov r1, #GRAIN_HEIGHT
bl generate_grain_rows_neon
.else
mov r5, #128
lsl r5, r5, lr // 128 << bitdepth_min_8
sub r5, r5, #1 // (128 << bitdepth_min_8) - 1
mvn r6, r5 // grain_min = ~grain_max
mov r1, #3
bl generate_grain_rows_neon
mov r1, #GRAIN_HEIGHT-3
vdup.32 q12, r7
vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0]
vmov.i8 q0, #0
vmov.i8 q1, #255
vdup.16 q9, r5
vdup.16 q10, r6
vext.8 q13, q0, q1, #10
vext.8 q14, q1, q0, #2
vneg.s32 q12, q12
vmovl.s8 q11, d22
1:
vmov q1, q13
bl gen_grain_uv_444_lag0_neon // 8
vmov.i8 q1, #255
bl gen_grain_uv_444_lag0_neon // 16
bl gen_grain_uv_444_lag0_neon // 24
bl gen_grain_uv_444_lag0_neon // 32
bl gen_grain_uv_444_lag0_neon // 40
bl gen_grain_uv_444_lag0_neon // 48
bl gen_grain_uv_444_lag0_neon // 56
bl gen_grain_uv_444_lag0_neon // 64
bl gen_grain_uv_444_lag0_neon // 72
vmov q1, q14
bl gen_grain_uv_444_lag0_neon // 80
get_grain_2 d16
subs r1, r1, #1
add r11, r11, #4
vst1.32 {d16[0]}, [r0]!
bgt 1b
.endif
pop {r4-r11,pc}
L(generate_grain_\type\()_lag1):
vpush {q4-q7}
mov r5, #128
lsl r5, r5, lr // 128 << bitdepth_min_8
sub r5, r5, #1 // (128 << bitdepth_min_8) - 1
vld1.8 {d27[]}, [r4]! // ar_coeffs_y[0]
vld1.8 {d28[]}, [r4]! // ar_coeffs_y[1]
vld1.8 {d29[]}, [r4] // ar_coeffs_y[2]
.ifc \type, y
ldrsb r4, [r4, #1] // ar_coeffs_y[3]
.else
add r4, r4, #2
.endif
mov r1, #3
.ifc \type, uv_444
vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4]
ldrsb r4, [r4, #-1] // ar_coeffs_uv[3]
.endif
bl generate_grain_rows_neon
vmovl.s8 q13, d27
vmovl.s8 q12, d29
vmovl.s8 q14, d28
vmov d29, d24
.ifc \type, uv_444
vmovl.s8 q6, d13
.endif
mov r1, #GRAIN_HEIGHT - 3
1:
bl sum_\type\()_lag1_left_neon // 8
bl sum_\type\()_lag1_mid_neon // 16
bl sum_\type\()_lag1_mid_neon // 24
bl sum_\type\()_lag1_mid_neon // 32
bl sum_\type\()_lag1_mid_neon // 40
bl sum_\type\()_lag1_mid_neon // 48
bl sum_\type\()_lag1_mid_neon // 56
bl sum_\type\()_lag1_mid_neon // 64
bl sum_\type\()_lag1_mid_neon // 72
bl sum_\type\()_lag1_right_neon // 80
get_grain_2 d16
subs r1, r1, #1
.ifc \type, uv_444
add r11, r11, #4
.endif
vst1.32 {d16[0]}, [r0]!
bgt 1b
vpop {q4-q7}
pop {r4-r11,pc}
L(generate_grain_\type\()_lag2):
vpush {q4-q7}
mov r5, #128
lsl r5, r5, lr // 128 << bitdepth_min_8
sub r5, r5, #1 // (128 << bitdepth_min_8) - 1
vld1.8 {d28,d29}, [r4] // ar_coeffs_y[0-11], ar_coeffs_uv[0-12]
vmov.s8 r4, d29[2]
vmov.s8 r10, d29[3]
mov r1, #3
bl generate_grain_rows_neon
mov r1, #GRAIN_HEIGHT - 3
1:
bl sum_\type\()_lag2_left_neon // 8
bl sum_\type\()_lag2_mid_neon // 16
bl sum_\type\()_lag2_mid_neon // 24
bl sum_\type\()_lag2_mid_neon // 32
bl sum_\type\()_lag2_mid_neon // 40
bl sum_\type\()_lag2_mid_neon // 48
bl sum_\type\()_lag2_mid_neon // 56
bl sum_\type\()_lag2_mid_neon // 64
bl sum_\type\()_lag2_mid_neon // 72
bl sum_\type\()_lag2_right_neon // 80
get_grain_2 d16
subs r1, r1, #1
.ifc \type, uv_444
add r11, r11, #4
.endif
vst1.32 {d16[0]}, [r0]!
bgt 1b
vpop {q4-q7}
pop {r4-r11,pc}
L(generate_grain_\type\()_lag3):
vpush {q4-q7}
mov r5, #128
lsl r5, r5, lr // 128 << bitdepth_min_8
sub r5, r5, #1 // (128 << bitdepth_min_8) - 1
vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24]
vmov.u8 r4, d28[5]
vmov.u8 r10, d28[6]
vmov.u8 r12, d28[7]
orr r4, r4, r10, lsl #8
orr r4, r4, r12, lsl #16
mov r1, #3
vpush {d26}
bl generate_grain_rows_neon
vpop {d26}
mov r1, #GRAIN_HEIGHT - 3
1:
bl sum_\type\()_lag3_left_neon // 8
bl sum_\type\()_lag3_mid_neon // 16
bl sum_\type\()_lag3_mid_neon // 24
bl sum_\type\()_lag3_mid_neon // 32
bl sum_\type\()_lag3_mid_neon // 40
bl sum_\type\()_lag3_mid_neon // 48
bl sum_\type\()_lag3_mid_neon // 56
bl sum_\type\()_lag3_mid_neon // 64
bl sum_\type\()_lag3_mid_neon // 72
bl sum_\type\()_lag3_right_neon // 80
get_grain_2 d16
subs r1, r1, #1
.ifc \type, uv_444
add r11, r11, #4
.endif
vst1.32 {d16[0]}, [r0]!
bgt 1b
vpop {q4-q7}
pop {r4-r11,pc}
endfunc
.endm
gen_grain_82 y
gen_grain_82 uv_444
.macro set_height dst, type
.ifc \type, uv_420
mov \dst, #SUB_GRAIN_HEIGHT-3
.else
mov \dst, #GRAIN_HEIGHT-3
.endif
.endm
.macro increment_y_ptr reg, type
.ifc \type, uv_420
add \reg, \reg, #2*GRAIN_WIDTH*2-(6*32)
.else
sub \reg, \reg, #6*32-GRAIN_WIDTH*2
.endif
.endm
.macro gen_grain_44 type
function generate_grain_\type\()_16bpc_neon, export=1
push {r4-r11,lr}
ldr r4, [sp, #36]
mov r12, r3
movw r11, #(3*GRAIN_WIDTH-3)*2
mov lr, #28
add r11, r1, r11
mov r1, r2
mul r12, r12, lr
clz lr, r4
movrel r3, X(gaussian_sequence)
sub lr, lr, #24 // -bitdepth_min_8
ldr r2, [r1, #FGD_SEED]
ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT]
add r4, r1, #FGD_AR_COEFFS_UV
add r9, r9, lr // grain_scale_shift - bitdepth_min_8
adr r5, L(gen_grain_\type\()_tbl)
ldr r6, [r1, #FGD_AR_COEFF_LAG]
add r9, r9, #4
ldr r6, [r5, r6, lsl #2]
vdup.16 q15, r9 // 4 - bitdepth_min_8 + data->grain_scale_shift
add r5, r5, r6
vneg.s16 q15, q15
push {lr}
cmp r12, #0
movw r10, #0x49d8
movw lr, #0xb524
// Intentionally using a separate register instead of moveq with an
// immediate constant, to avoid armv8 deprecated it instruction forms.
it eq
moveq r10, lr
add r4, r4, r12 // Add offset to ar_coeffs_uv[1]
eor r2, r2, r10
pop {lr}
ldr r7, [r1, #FGD_AR_COEFF_SHIFT]
neg lr, lr
mov r8, #1
mov r10, #1
lsl r8, r8, r7 // 1 << ar_coeff_shift
lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift)
lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1)
lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1)
bx r5
.align 2
L(gen_grain_\type\()_tbl):
.word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB
.word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB
.word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB
.word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB
L(generate_grain_\type\()_lag0):
.ifc \type, uv_420
vpush {q4-q5}
.endif
mov r5, #128
lsl r5, r5, lr // 128 << bitdepth_min_8
sub r5, r5, #1 // (128 << bitdepth_min_8) - 1
mvn r6, r5 // grain_min = ~grain_max
mov r1, #3
bl generate_grain_rows_44_neon
set_height r1, \type
vdup.32 q12, r7
vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0]
vmov.i8 q0, #0
vmov.i8 q1, #255
vdup.16 q9, r5
vdup.16 q10, r6
vext.8 q13, q0, q1, #10
vext.8 q14, q1, q0, #14
vneg.s32 q12, q12
vmovl.s8 q11, d22
1:
vmov q1, q13
bl gen_grain_\type\()_lag0_8_neon // 8
vmov.i8 q1, #255
bl gen_grain_\type\()_lag0_8_neon // 16
bl gen_grain_\type\()_lag0_8_neon // 24
bl gen_grain_\type\()_lag0_8_neon // 32
bl gen_grain_\type\()_lag0_8_neon // 40
vmov q1, q14
bl gen_grain_\type\()_lag0_4_neon // 44
subs r1, r1, #1
increment_y_ptr r11, \type
add r0, r0, #GRAIN_WIDTH*2-6*16
bgt 1b
.ifc \type, uv_420
vpop {q4-q5}
.endif
pop {r4-r11,pc}
L(generate_grain_\type\()_lag1):
vpush {q4-q7}
mov r5, #128
lsl r5, r5, lr // 128 << bitdepth_min_8
sub r5, r5, #1 // (128 << bitdepth_min_8) - 1
vld1.8 {d27[]}, [r4]! // ar_coeffs_uv[0]
vld1.8 {d28[]}, [r4]! // ar_coeffs_uv[1]
vld1.8 {d29[]}, [r4] // ar_coeffs_uv[2]
add r4, r4, #2
mov r1, #3
vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4]
ldrsb r4, [r4, #-1] // ar_coeffs_uv[3]
bl generate_grain_rows_44_neon
vmovl.s8 q13, d27
vmovl.s8 q12, d29
vmovl.s8 q14, d28
vmov d29, d24
vmovl.s8 q6, d13
set_height r1, \type
1:
bl sum_\type\()_lag1_left_neon // 8
bl sum_\type\()_lag1_mid_neon // 16
bl sum_\type\()_lag1_mid_neon // 24
bl sum_\type\()_lag1_mid_neon // 32
bl sum_\type\()_lag1_mid_neon // 40
bl sum_\type\()_lag1_right_neon // 44
subs r1, r1, #1
increment_y_ptr r11, \type
add r0, r0, #GRAIN_WIDTH*2-6*16
bgt 1b
vpop {q4-q7}
pop {r4-r11,pc}
L(generate_grain_\type\()_lag2):
vpush {q4-q7}
mov r5, #128
lsl r5, r5, lr // 128 << bitdepth_min_8
sub r5, r5, #1 // (128 << bitdepth_min_8) - 1
vld1.8 {d28,d29}, [r4] // ar_coeffs_uv[0-12]
vmov.s8 r4, d29[2]
vmov.s8 r10, d29[3]
mov r1, #3
bl generate_grain_rows_44_neon
set_height r1, \type
1:
bl sum_\type\()_lag2_left_neon // 8
bl sum_\type\()_lag2_mid_neon // 16
bl sum_\type\()_lag2_mid_neon // 24
bl sum_\type\()_lag2_mid_neon // 32
bl sum_\type\()_lag2_mid_neon // 40
bl sum_\type\()_lag2_right_neon // 44
subs r1, r1, #1
increment_y_ptr r11, \type
add r0, r0, #GRAIN_WIDTH*2-6*16
bgt 1b
vpop {q4-q7}
pop {r4-r11,pc}
L(generate_grain_\type\()_lag3):
vpush {q4-q7}
mov r5, #128
lsl r5, r5, lr // 128 << bitdepth_min_8
sub r5, r5, #1 // (128 << bitdepth_min_8) - 1
vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24]
vmov.u8 r4, d28[5]
vmov.u8 r10, d28[6]
vmov.u8 r12, d28[7]
orr r4, r4, r10, lsl #8
orr r4, r4, r12, lsl #16
mov r1, #3
bl generate_grain_rows_44_neon
set_height r1, \type
1:
bl sum_\type\()_lag3_left_neon // 8
bl sum_\type\()_lag3_mid_neon // 16
bl sum_\type\()_lag3_mid_neon // 24
bl sum_\type\()_lag3_mid_neon // 32
bl sum_\type\()_lag3_mid_neon // 40
bl sum_\type\()_lag3_right_neon // 44
subs r1, r1, #1
increment_y_ptr r11, \type
add r0, r0, #GRAIN_WIDTH*2-6*16
bgt 1b
vpop {q4-q7}
pop {r4-r11,pc}
endfunc
.endm
gen_grain_44 uv_420
gen_grain_44 uv_422
.macro gather_interleaved dst1, dst2, src1, src2, src3, src4, off
vmov.u16 r11, \src1[0+\off]
vmov.u16 r12, \src3[0+\off]
add r11, r11, r3
vmov.u16 lr, \src1[2+\off]
add r12, r12, r3
vld1.8 {\dst1[0+\off]}, [r11]
vmov.u16 r11, \src3[2+\off]
add lr, lr, r3
vld1.8 {\dst2[0+\off]}, [r12]
vmov.u16 r12, \src2[0+\off]
add r11, r11, r3
vld1.8 {\dst1[2+\off]}, [lr]
vmov.u16 lr, \src4[0+\off]
add r12, r12, r3
vld1.8 {\dst2[2+\off]}, [r11]
vmov.u16 r11, \src2[2+\off]
add lr, lr, r3
vld1.8 {\dst1[4+\off]}, [r12]
vmov.u16 r12, \src4[2+\off]
add r11, r11, r3
vld1.8 {\dst2[4+\off]}, [lr]
add r12, r12, r3
vld1.8 {\dst1[6+\off]}, [r11]
vld1.8 {\dst2[6+\off]}, [r12]
.endm
.macro gather dst1, dst2, dst3, dst4, src1, src2, src3, src4, src5, src6, src7, src8
gather_interleaved \dst1, \dst3, \src1, \src2, \src5, \src6, 0
gather_interleaved \dst1, \dst3, \src1, \src2, \src5, \src6, 1
gather_interleaved \dst2, \dst4, \src3, \src4, \src7, \src8, 0
gather_interleaved \dst2, \dst4, \src3, \src4, \src7, \src8, 1
.endm
function gather32_neon
push {r11-r12,lr}
gather d8, d9, d10, d11, d0, d1, d2, d3, d4, d5, d6, d7
pop {r11-r12,pc}
endfunc
function gather16_neon
push {r11-r12,lr}
gather_interleaved d8, d9, d0, d1, d2, d3, 0
gather_interleaved d8, d9, d0, d1, d2, d3, 1
pop {r11-r12,pc}
endfunc
const overlap_coeffs_0, align=4
.short 27, 17, 0, 0
.short 17, 27, 32, 32
endconst
const overlap_coeffs_1, align=4
.short 23, 0, 0, 0
.short 22, 32, 32, 32
endconst
.macro calc_offset offx, offy, src, sx, sy
and \offy, \src, #0xF // randval & 0xF
lsr \offx, \src, #4 // randval >> 4
.if \sy == 0
add \offy, \offy, \offy // 2 * (randval & 0xF)
.endif
.if \sx == 0
add \offx, \offx, \offx // 2 * (randval >> 4)
.endif
.endm
.macro add_offset dst, offx, offy, src, stride
mla \dst, \stride, \offy, \src // grain_lut += grain_stride * offy
add \dst, \dst, \offx, lsl #1 // grain_lut += offx
.endm
// void dav1d_fgy_32x32_16bpc_neon(pixel *const dst, const pixel *const src,
// const ptrdiff_t stride,
// const uint8_t scaling[SCALING_SIZE],
// const int scaling_shift,
// const entry grain_lut[][GRAIN_WIDTH],
// const int offsets[][2],
// const int h, const ptrdiff_t clip,
// const ptrdiff_t type,
// const int bitdepth_max);
function fgy_32x32_16bpc_neon, export=1
push {r4-r11,lr}
vpush {q4-q7}
ldrd r4, r5, [sp, #100] // scaling_shift, grain_lut
ldrd r6, r7, [sp, #108] // offsets, h
ldr r8, [sp, #116] // clip
mov r9, #GRAIN_WIDTH*2 // grain_lut stride
ldr r10, [sp, #124] // bitdepth_max
eor r4, r4, #15 // 15 - scaling_shift
vdup.16 q6, r10 // bitdepth_max
clz r10, r10
vdup.16 q13, r4 // 15 - scaling_shift
rsb r10, r10, #24 // bitdepth_min_8
cmp r8, #0
vdup.16 q12, r10 // bitdepth_min_8
movrel_local r12, overlap_coeffs_0
beq 1f
// clip
vmov.i16 q14, #16
vmov.i16 q15, #235
vshl.s16 q14, q14, q12
vshl.s16 q15, q15, q12
b 2f
1:
// no clip
vmov.i16 q14, #0
vmov q15, q6
2:
vshr.u16 q6, q6, #1 // grain_max
vld1.16 {d24, d25}, [r12, :128] // overlap_coeffs
add r5, r5, #18 // grain_lut += 9
add r5, r5, r9, lsl #3 // grain_lut += 8 * grain_stride
add r5, r5, r9 // grain_lut += grain_stride
ldr r10, [r6, #8] // offsets[1][0]
calc_offset r10, r4, r10, 0, 0
add_offset r4, r10, r4, r5, r9
ldr r10, [r6, #4] // offsets[0][1]
calc_offset r10, r11, r10, 0, 0
add_offset r11, r10, r11, r5, r9
ldr r10, [r6, #12] // offsets[1][1]
calc_offset r10, r8, r10, 0, 0
add_offset r8, r10, r8, r5, r9
ldr r6, [r6] // offsets[0][0]
calc_offset r6, lr, r6, 0, 0
add_offset r5, r6, lr, r5, r9
add r4, r4, #32*2 // grain_lut += FG_BLOCK_SIZE * bx
add r6, r11, r9, lsl #5 // grain_lut += grain_stride * FG_BLOCK_SIZE * by
ldr r10, [sp, #120] // type
adr r11, L(fgy_loop_tbl)
tst r10, #1
ldr r10, [r11, r10, lsl #2]
add r8, r8, r9, lsl #5 // grain_lut += grain_stride * FG_BLOCK_SIZE * by
add r8, r8, #32*2 // grain_lut += FG_BLOCK_SIZE * bx
add r11, r11, r10
beq 1f
// y overlap
vdup.16 d14, d24[0]
vdup.16 d15, d24[1]
mov r10, r7 // backup actual h
mov r7, #2
1:
sub r2, r2, #32 // src_stride -= 32
sub r9, r9, #32 // grain_stride -= 32
bx r11
endfunc
function fgy_loop_neon
L(fgy_loop_tbl):
.word L(loop_00) - L(fgy_loop_tbl) + CONFIG_THUMB
.word L(loop_01) - L(fgy_loop_tbl) + CONFIG_THUMB
.word L(loop_10) - L(fgy_loop_tbl) + CONFIG_THUMB
.word L(loop_11) - L(fgy_loop_tbl) + CONFIG_THUMB
.macro fgy ox, oy
L(loop_\ox\oy):
1:
.if \ox
vld1.16 {d0}, [r4], r9 // grain_lut old
.endif
.if \oy
vld1.16 {q2, q3}, [r6]! // grain_lut top
.endif
.if \ox && \oy
vld1.16 {d2}, [r8], r9 // grain_lut top old
.endif
.if \oy
vld1.16 {q4, q5}, [r6], r9 // grain_lut top
.endif
.if !\ox && !\oy
vld1.16 {q0, q1}, [r1, :128]! // src
.endif
vld1.16 {q8, q9}, [r5]! // grain_lut
.if !\ox && !\oy
vld1.16 {q2, q3}, [r1, :128], r2 // src
.endif
.if !\oy
vmvn.i16 q5, #0xf000 // 0x0fff
.endif
vld1.16 {q10, q11}, [r5], r9 // grain_lut
.if \ox
add r4, r4, #32
vmull.s16 q0, d0, d24
vmlal.s16 q0, d16, d25
.endif
.if \oy
.if \ox
add r8, r8, #32
vmull.s16 q1, d2, d24
vmlal.s16 q1, d4, d25
vqrshrn.s32 d16, q0, #5
vmvn d0, d12 // grain_min
vqrshrn.s32 d4, q1, #5
vmin.s16 d16, d16, d12
vmin.s16 d4, d4, d12
vmax.s16 d16, d16, d0
vmax.s16 d4, d4, d0
.endif
vmull.s16 q0, d4, d14
vmull.s16 q1, d5, d14
vmull.s16 q2, d6, d14
vmull.s16 q3, d7, d14
vmlal.s16 q0, d16, d15
vmlal.s16 q1, d17, d15
vmlal.s16 q2, d18, d15
vmlal.s16 q3, d19, d15
vmull.s16 q8, d20, d15
vmull.s16 q9, d21, d15
vmull.s16 q10, d22, d15
vmull.s16 q11, d23, d15
vmlal.s16 q8, d8, d14
vmlal.s16 q9, d9, d14
vmlal.s16 q10, d10, d14
vmlal.s16 q11, d11, d14
vmvn q4, q6 // grain_min
vqrshrn.s32 d0, q0, #5
vqrshrn.s32 d1, q1, #5
vqrshrn.s32 d2, q2, #5
vqrshrn.s32 d3, q3, #5
vqrshrn.s32 d4, q8, #5
vqrshrn.s32 d5, q9, #5
vqrshrn.s32 d6, q10, #5
vqrshrn.s32 d7, q11, #5
vmin.s16 q8, q0, q6
vmin.s16 q9, q1, q6
vld1.16 {q0, q1}, [r1, :128]! // src
vmin.s16 q10, q2, q6
vmin.s16 q11, q3, q6
vmax.s16 q8, q8, q4
vmax.s16 q9, q9, q4
vld1.16 {q2, q3}, [r1, :128], r2 // src
vmvn.i16 q5, #0xf000 // 0x0fff
vmax.s16 q10, q10, q4
vmax.s16 q11, q11, q4
.elseif \ox
vmvn d4, d12 // grain_min
vqrshrn.s32 d16, q0, #5
vld1.16 {q0, q1}, [r1, :128]! // src
vmin.s16 d16, d16, d12
vmax.s16 d16, d16, d4
vld1.16 {q2, q3}, [r1, :128], r2 // src
.endif
// Make sure that uninitialized pixels out of range past the right
// edge are in range; their actual values shouldn't matter.
vand q0, q0, q5
vand q1, q1, q5
vand q2, q2, q5
vand q3, q3, q5
bl gather32_neon
.if \ox || \oy
vpush {q6-q7}
.endif
vmovl.u8 q6, d8 // scaling
vmovl.u8 q7, d9
vmovl.u8 q4, d10
vmovl.u8 q5, d11
vshl.u16 q6, q6, q13 // scaling << (15 - scaling_shift)
vshl.u16 q7, q7, q13
vshl.u16 q4, q4, q13
vshl.u16 q5, q5, q13
vqrdmulh.s16 q8, q8, q6 // round2((scaling << (15 - scaling_shift) * grain, 15)
vqrdmulh.s16 q9, q9, q7
vqrdmulh.s16 q10, q10, q4
vqrdmulh.s16 q11, q11, q5
.if \ox || \oy
vpop {q6-q7}
.endif
vqadd.s16 q0, q0, q8 // *src + noise
vqadd.s16 q1, q1, q9
vqadd.s16 q2, q2, q10
vqadd.s16 q3, q3, q11
vmax.s16 q0, q0, q14
vmax.s16 q1, q1, q14
vmax.s16 q2, q2, q14
vmax.s16 q3, q3, q14
vmin.s16 q0, q0, q15
vmin.s16 q1, q1, q15
vmin.s16 q2, q2, q15
vmin.s16 q3, q3, q15
vst1.16 {q0, q1}, [r0, :128]! // dst
subs r7, r7, #1
.if \oy
vdup.16 d14, d25[0]
vdup.16 d15, d25[1]
.endif
vst1.16 {q2, q3}, [r0, :128], r2 // dst
bgt 1b
.if \oy
cmp r10, #2
sub r7, r10, #2 // restore actual remaining h
bgt L(loop_\ox\()0)
.endif
vpop {q4-q7}
pop {r4-r11,pc}
.endm
fgy 0, 0
fgy 0, 1
fgy 1, 0
fgy 1, 1
endfunc
// void dav1d_fguv_32x32_420_16bpc_neon(pixel *const dst,
// const pixel *const src,
// const ptrdiff_t stride,
// const uint8_t scaling[SCALING_SIZE],
// const Dav1dFilmGrainData *const data,
// const entry grain_lut[][GRAIN_WIDTH],
// const pixel *const luma_row,
// const ptrdiff_t luma_stride,
// const int offsets[][2],
// const ptrdiff_t h, const ptrdiff_t uv,
// const ptrdiff_t is_id,
// const ptrdiff_t type,
// const int bitdepth_max);
.macro fguv layout, sx, sy
function fguv_32x32_\layout\()_16bpc_neon, export=1
push {r4-r11,lr}
vpush {q4-q7}
ldrd r4, r5, [sp, #100] // data, grain_lut
ldrd r10, r11, [sp, #124] // uv, is_id
ldr r6, [sp, #136] // bitdepth_max
clz r7, r6
rsb r7, r7, #24 // bitdepth_min_8
// !csfl
add r10, r4, r10, lsl #2 // + 4*uv
add r12, r10, #FGD_UV_LUMA_MULT
add lr, r10, #FGD_UV_MULT
ldrh r10, [r10, #FGD_UV_OFFSET] // uv_offset
vld1.16 {d30[]}, [r12] // uv_luma_mult
lsl r10, r10, r7 // uv_offset << bitdepth_min_8
vld1.16 {d30[1]}, [lr] // uv_mult
ldr lr, [r4, #FGD_SCALING_SHIFT]
ldr r12, [r4, #FGD_CLIP_TO_RESTRICTED_RANGE]
eor lr, lr, #15 // 15 - scaling_shift
vmov.16 d30[2], r10 // uv_offset << bitdepth_min_8
cmp r12, #0
vdup.16 q13, lr // 15 - scaling_shift
beq 1f
// clip
cmp r11, #0
mov r8, #16
mov r9, #240
lsl r8, r8, r7
lsl r9, r9, r7
beq 2f
// is_id
mov r9, #235
lsl r9, r9, r7
b 2f
1:
// no clip
mov r8, #0
mov r9, r6 // bitdepth_max
2:
vmov.16 d30[3], r6 // bitdepth_max
vdup.16 d31, r8 // clip_min
mov r10, #GRAIN_WIDTH*2 // grain_lut stride
.if \sy
mov r6, #23
mov r7, #22
.else
mov r6, #27
mov r7, #17
.endif
vmov.16 d31[1], r9 // clip_max
ldrd r8, r9, [sp, #116] // offsets, h
add r5, r5, #(2*(3 + (2 >> \sx)*3)) // grain_lut += 9 or 6
.if \sy
add r5, r5, r10, lsl #2 // grain_lut += 4 * grain_stride
add r5, r5, r10, lsl #1 // grain_lut += 2 * grain_stride
.else
add r5, r5, r10, lsl #3 // grain_lut += 8 * grain_stride
add r5, r5, r10 // grain_lut += grain_stride
.endif
vmov.16 d31[2], r6 // overlap y [0]
ldr r12, [r8, #8] // offsets[1][0]
calc_offset r12, r4, r12, \sx, \sy
add_offset r4, r12, r4, r5, r10
ldr r12, [r8, #4] // offsets[0][1]
calc_offset r12, lr, r12, \sx, \sy
add_offset lr, r12, lr, r5, r10
ldr r12, [r8, #12] // offsets[1][1]
calc_offset r12, r11, r12, \sx, \sy
add_offset r11, r12, r11, r5, r10
ldr r8, [r8] // offsets[0][0]
calc_offset r8, r12, r8, \sx, \sy
add_offset r5, r8, r12, r5, r10
vmov.16 d31[3], r7 // overlap y [1]
add r4, r4, #2*(32 >> \sx) // grain_lut += FG_BLOCK_SIZE * bx
add r8, lr, r10, lsl #(5 - \sy) // grain_lut += grain_stride * FG_BLOCK_SIZE * by
add r11, r11, r10, lsl #(5 - \sy) // grain_lut += grain_stride * FG_BLOCK_SIZE * by
add r11, r11, #2*(32 >> \sx) // grain_lut += FG_BLOCK_SIZE * bx
movrel_local r12, overlap_coeffs_\sx
ldr lr, [sp, #132] // type
ldrd r6, r7, [sp, #108] // luma_row, luma_stride
vld1.16 {d24, d25}, [r12, :128] // overlap_coeffs
movrel_local r12, L(fguv_loop_sx\sx\()_tbl)
#if CONFIG_THUMB
// This uses movrel_local instead of adr above, because the target
// can be out of range for adr. But movrel_local leaves the thumb bit
// set on COFF (but probably wouldn't if building for thumb on ELF),
// thus try to clear the bit for robustness.
bic r12, r12, #1
#endif
tst lr, #1
ldr lr, [r12, lr, lsl #2]
add r12, r12, lr
beq 1f
// y overlap
sub lr, r9, #(2 >> \sy) // backup remaining h
mov r9, #(2 >> \sy)
1:
.if \sy
add r7, r7, r7 // luma_stride *= 2
.endif
sub r7, r7, #32 // luma_stride -= 32
bx r12
endfunc
.endm
fguv 420, 1, 1
fguv 422, 1, 0
fguv 444, 0, 0
function fguv_loop_sx0_neon
L(fguv_loop_sx0_tbl):
.word L(fguv_loop_sx0_csfl0_00) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
.word L(fguv_loop_sx0_csfl0_01) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
.word L(fguv_loop_sx0_csfl0_10) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
.word L(fguv_loop_sx0_csfl0_11) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
.word L(fguv_loop_sx0_csfl1_00) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
.word L(fguv_loop_sx0_csfl1_01) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
.word L(fguv_loop_sx0_csfl1_10) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
.word L(fguv_loop_sx0_csfl1_11) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB
.macro fguv_loop_sx0 csfl, ox, oy
L(fguv_loop_sx0_csfl\csfl\()_\ox\oy):
sub r2, r2, #32 // src_stride -= 32
sub r10, r10, #32 // grain_stride -= 32
.if \oy
mov r12, lr
.endif
L(fguv_loop_sx0_csfl\csfl\()_\ox\oy\()_loopstart):
1:
.if \ox
vld1.16 {d0}, [r4], r10 // grain_lut old
.endif
.if \oy
vld1.16 {q2, q3}, [r8]! // grain_lut top
.endif
.if \ox && \oy
vld1.16 {d2}, [r11], r10 // grain_lut top old
.endif
.if !\ox && !\oy
vld1.16 {q0, q1}, [r6, :128]! // luma
.endif
vld1.16 {q8, q9}, [r5]! // grain_lut
.if \oy
vld1.16 {q4, q5}, [r8], r10 // grain_lut top
.endif
.if !\ox && !\oy
vld1.16 {q2, q3}, [r6, :128], r7 // luma
.endif
.if \oy
vdup.16 d28, d31[2] // overlap y coeff
vdup.16 d29, d31[3] // overlap y coeff
.endif
vld1.16 {q10, q11}, [r5], r10 // grain_lut
.if \ox
vdup.16 q7, d30[3] // bitdepth_max
add r4, r4, #32
vmull.s16 q0, d0, d24
vshr.u16 q7, q7, #1 // grain_max
vmlal.s16 q0, d16, d25
vmvn q6, q7 // grain_min
.endif
.if \oy
.if \ox
add r11, r11, #32
vmull.s16 q1, d2, d24
vmlal.s16 q1, d4, d25
vqrshrn.s32 d16, q0, #5
vqrshrn.s32 d4, q1, #5
vmin.s16 d4, d4, d14
vmin.s16 d16, d16, d14
vmax.s16 d4, d4, d12
vmax.s16 d16, d16, d12
.endif
vmull.s16 q0, d4, d28
vmull.s16 q1, d5, d28
vmull.s16 q2, d6, d28
vmull.s16 q3, d7, d28
.if !\ox
vdup.16 q7, d30[3] // bitdepth_max
.endif
vmlal.s16 q0, d16, d29
vmlal.s16 q1, d17, d29
vmlal.s16 q2, d18, d29
vmlal.s16 q3, d19, d29
.if !\ox
vshr.u16 q7, q7, #1 // grain_max
.endif
vmull.s16 q8, d20, d29
vmull.s16 q9, d21, d29
vmull.s16 q10, d22, d29
vmull.s16 q11, d23, d29
.if !\ox
vmvn q6, q7 // grain_min
.endif
vmlal.s16 q8, d8, d28
vmlal.s16 q9, d9, d28
vmlal.s16 q10, d10, d28
vmlal.s16 q11, d11, d28
vqrshrn.s32 d0, q0, #5
vqrshrn.s32 d1, q1, #5
vqrshrn.s32 d2, q2, #5
vqrshrn.s32 d3, q3, #5
vqrshrn.s32 d4, q8, #5
vqrshrn.s32 d5, q9, #5
vqrshrn.s32 d6, q10, #5
vqrshrn.s32 d7, q11, #5
vmin.s16 q8, q0, q7
vmin.s16 q9, q1, q7
vld1.16 {q0, q1}, [r6, :128]! // luma
vmin.s16 q10, q2, q7
vmin.s16 q11, q3, q7
vmax.s16 q8, q8, q6
vmax.s16 q9, q9, q6
vld1.16 {q2, q3}, [r6, :128], r7 // luma
vmax.s16 q10, q10, q6
vmax.s16 q11, q11, q6
.elseif \ox
vqrshrn.s32 d16, q0, #5
vld1.16 {q0, q1}, [r6, :128]! // luma
vmin.s16 d16, d16, d14
vld1.16 {q2, q3}, [r6, :128], r7 // luma
vmax.s16 d16, d16, d12
.endif
.if !\csfl
vdup.16 d28, d30[0] // uv_luma_mult
vld1.16 {q4, q5}, [r1, :128]! // src
vdup.16 d29, d30[1] // uv_mult
vmull.s16 q6, d0, d28
vmull.s16 q7, d1, d28
vmull.s16 q0, d2, d28
vmull.s16 q1, d3, d28
vmlal.s16 q6, d8, d29
vmlal.s16 q7, d9, d29
vmlal.s16 q0, d10, d29
vmlal.s16 q1, d11, d29
vld1.16 {q4, q5}, [r1, :128] // src
sub r1, r1, #32
vshrn.s32 d12, q6, #6
vshrn.s32 d13, q7, #6
vshrn.s32 d14, q0, #6
vshrn.s32 d15, q1, #6
vmull.s16 q0, d4, d28
vmull.s16 q1, d5, d28
vmull.s16 q2, d6, d28
vmull.s16 q3, d7, d28
vmlal.s16 q0, d8, d29
vmlal.s16 q1, d9, d29
vmlal.s16 q2, d10, d29
vmlal.s16 q3, d11, d29
vdup.16 q14, d30[2] // uv_offset
vshrn.s32 d0, q0, #6
vshrn.s32 d1, q1, #6
vshrn.s32 d2, q2, #6
vshrn.s32 d3, q3, #6
vdup.16 q4, d30[3] // bitdepth_max
vmov.i16 q5, #0
vadd.i16 q6, q6, q14
vadd.i16 q7, q7, q14
vadd.i16 q2, q0, q14
vadd.i16 q3, q1, q14
vmin.s16 q0, q6, q4
vmin.s16 q1, q7, q4
vmin.s16 q2, q2, q4
vmin.s16 q3, q3, q4
vmax.s16 q0, q0, q5
vmax.s16 q1, q1, q5
vmax.s16 q2, q2, q5
vmax.s16 q3, q3, q5
.else
vdup.16 q14, d30[3] // bitdepth_max
// Make sure that uninitialized pixels out of range past the right
// edge are in range; their actual values shouldn't matter.
vand q0, q0, q14
vand q1, q1, q14
vand q2, q2, q14
vand q3, q3, q14
.endif
bl gather32_neon
vld1.16 {q0, q1}, [r1, :128]! // src
vmovl.u8 q6, d8 // scaling
vmovl.u8 q7, d9
vmovl.u8 q4, d10
vmovl.u8 q5, d11
vld1.16 {q2, q3}, [r1, :128], r2 // src
vshl.u16 q6, q6, q13 // scaling << (15 - scaling_shift)
vshl.u16 q7, q7, q13
vshl.u16 q4, q4, q13
vshl.u16 q5, q5, q13
vqrdmulh.s16 q8, q8, q6 // round2((scaling << (15 - scaling_shift) * grain, 15)
vqrdmulh.s16 q9, q9, q7
vqrdmulh.s16 q10, q10, q4
vqrdmulh.s16 q11, q11, q5
vdup.16 q4, d31[0] // clip_min
vdup.16 q5, d31[1] // clip_max
vqadd.s16 q0, q0, q8 // *src + noise
vqadd.s16 q1, q1, q9
vqadd.s16 q2, q2, q10
vqadd.s16 q3, q3, q11
.if \oy
vmov.32 lr, d25[0] // 2 first 16 bit coeffs from overlap x
.endif
vmax.s16 q0, q0, q4
vmax.s16 q1, q1, q4
vmax.s16 q2, q2, q4
vmax.s16 q3, q3, q4
vmin.s16 q0, q0, q5
vmin.s16 q1, q1, q5
vmin.s16 q2, q2, q5
vmin.s16 q3, q3, q5
vst1.16 {q0, q1}, [r0, :128]! // dst
subs r9, r9, #1
.if \oy
vmov.32 d31[1], lr // new coeffs for overlap y
.endif
vst1.16 {q2, q3}, [r0, :128], r2 // dst
bgt 1b
.if \oy
cmp r12, #0
mov r9, r12 // restore actual remaining h
bgt L(fguv_loop_sx0_csfl\csfl\()_\ox\()0_loopstart)
.endif
b 9f
.endm
fguv_loop_sx0 0, 0, 0
fguv_loop_sx0 0, 0, 1
fguv_loop_sx0 0, 1, 0
fguv_loop_sx0 0, 1, 1
fguv_loop_sx0 1, 0, 0
fguv_loop_sx0 1, 0, 1
fguv_loop_sx0 1, 1, 0
fguv_loop_sx0 1, 1, 1
9:
vpop {q4-q7}
pop {r4-r11,pc}
endfunc
function fguv_loop_sx1_neon
L(fguv_loop_sx1_tbl):
.word L(fguv_loop_sx1_csfl0_00) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
.word L(fguv_loop_sx1_csfl0_01) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
.word L(fguv_loop_sx1_csfl0_10) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
.word L(fguv_loop_sx1_csfl0_11) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
.word L(fguv_loop_sx1_csfl1_00) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
.word L(fguv_loop_sx1_csfl1_01) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
.word L(fguv_loop_sx1_csfl1_10) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
.word L(fguv_loop_sx1_csfl1_11) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB
.macro fguv_loop_sx1 csfl, ox, oy
L(fguv_loop_sx1_csfl\csfl\()_\ox\oy):
.if \oy
mov r12, lr
.endif
1:
.if \ox
vld1.16 {d0}, [r4], r10 // grain_lut old
.endif
.if \ox && \oy
vld1.16 {d2}, [r11], r10 // grain_lut top old
.endif
.if \oy
vld1.16 {q2, q3}, [r8], r10 // grain_lut top
.endif
.if !\ox && !\oy
vld1.16 {q0, q1}, [r6, :128]! // luma
.endif
vld1.16 {q8, q9}, [r5], r10 // grain_lut
.if \oy
vdup.16 d28, d31[2] // overlap y coeff
vdup.16 d29, d31[3] // overlap y coeff
.endif
.if !\ox && !\oy
vld1.16 {q2, q3}, [r6, :128], r7 // luma
.endif
.if \ox
vdup.16 q7, d30[3] // bitdepth_max
vmull.s16 q0, d0, d24
vshr.u16 q7, q7, #1 // grain_max
vmlal.s16 q0, d16, d25
vmvn q6, q7 // grain_min
.endif
.if \oy
.if \ox
vmull.s16 q1, d2, d24
vmlal.s16 q1, d4, d25
vqrshrn.s32 d16, q0, #5
vqrshrn.s32 d4, q1, #5
vmin.s16 d4, d4, d14
vmin.s16 d16, d16, d14
vmax.s16 d4, d4, d12
vmax.s16 d16, d16, d12
.endif
vmull.s16 q0, d4, d28
vmull.s16 q1, d5, d28
vmull.s16 q2, d6, d28
vmull.s16 q3, d7, d28
.if !\ox
vdup.16 q7, d30[3] // bitdepth_max
.endif
vmlal.s16 q0, d16, d29
vmlal.s16 q1, d17, d29
vmlal.s16 q2, d18, d29
vmlal.s16 q3, d19, d29
.if !\ox
vshr.u16 q7, q7, #1 // grain_max
.endif
vqrshrn.s32 d16, q0, #5
vqrshrn.s32 d17, q1, #5
vqrshrn.s32 d18, q2, #5
vqrshrn.s32 d19, q3, #5
.if !\ox
vmvn q6, q7 // grain_min
.endif
vld1.16 {q0, q1}, [r6, :128]! // luma
vmin.s16 q8, q8, q7
vmin.s16 q9, q9, q7
vmax.s16 q8, q8, q6
vmax.s16 q9, q9, q6
vld1.16 {q2, q3}, [r6, :128], r7 // luma
.elseif \ox
vqrshrn.s32 d16, q0, #5
vld1.16 {q0, q1}, [r6, :128]! // luma
vmin.s16 d16, d16, d14
vld1.16 {q2, q3}, [r6, :128], r7 // luma
vmax.s16 d16, d16, d12
.endif
vpadd.i16 d0, d0, d1
vpadd.i16 d1, d2, d3
vpadd.i16 d2, d4, d5
vpadd.i16 d3, d6, d7
vrshr.u16 q0, q0, #1
vrshr.u16 q1, q1, #1
.if !\csfl
vdup.16 d28, d30[0] // uv_luma_mult
vld1.16 {q2, q3}, [r1, :128], r2 // src
vdup.16 d29, d30[1] // uv_mult
vmull.s16 q6, d0, d28
vmull.s16 q7, d1, d28
vmull.s16 q0, d2, d28
vmull.s16 q1, d3, d28
vmlal.s16 q6, d4, d29
vmlal.s16 q7, d5, d29
vmlal.s16 q0, d6, d29
vmlal.s16 q1, d7, d29
vshrn.s32 d12, q6, #6
vshrn.s32 d13, q7, #6
vshrn.s32 d14, q0, #6
vshrn.s32 d15, q1, #6
vdup.16 q14, d30[2] // uv_offset
vdup.16 q4, d30[3] // bitdepth_max
vmov.i16 q5, #0
vadd.i16 q6, q6, q14
vadd.i16 q7, q7, q14
vmin.s16 q0, q6, q4
vmin.s16 q1, q7, q4
vmax.s16 q0, q0, q5
vmax.s16 q1, q1, q5
.else
vdup.16 q14, d30[3] // bitdepth_max
vld1.16 {q2, q3}, [r1, :128], r2 // src
// Make sure that uninitialized pixels out of range past the right
// edge are in range; their actual values shouldn't matter.
vand q0, q0, q14
vand q1, q1, q14
.endif
bl gather16_neon
vmovl.u8 q6, d8 // scaling
vmovl.u8 q7, d9
vshl.u16 q6, q6, q13 // scaling << (15 - scaling_shift)
vshl.u16 q7, q7, q13
vqrdmulh.s16 q8, q8, q6 // round2((scaling << (15 - scaling_shift) * grain, 15)
vqrdmulh.s16 q9, q9, q7
vdup.16 q4, d31[0] // clip_min
vdup.16 q5, d31[1] // clip_max
vqadd.s16 q0, q2, q8 // *src + noise
vqadd.s16 q1, q3, q9
.if \oy
// Swap the two last coefficients of d31, place them first in d28
vrev64.16 d28, d31
.endif
vmax.s16 q0, q0, q4
vmax.s16 q1, q1, q4
vmin.s16 q0, q0, q5
vmin.s16 q1, q1, q5
subs r9, r9, #1
.if \oy
// Take the first two 16 bit coefficients of d28 and place them at the
// end of d31
vtrn.32 d31, d28
.endif
vst1.16 {q0, q1}, [r0, :128], r2 // dst
bgt 1b
.if \oy
cmp r12, #0
mov r9, r12 // restore actual remaining h
bgt L(fguv_loop_sx1_csfl\csfl\()_\ox\()0)
.endif
b 9f
.endm
fguv_loop_sx1 0, 0, 0
fguv_loop_sx1 0, 0, 1
fguv_loop_sx1 0, 1, 0
fguv_loop_sx1 0, 1, 1
fguv_loop_sx1 1, 0, 0
fguv_loop_sx1 1, 0, 1
fguv_loop_sx1 1, 1, 0
fguv_loop_sx1 1, 1, 1
9:
vpop {q4-q7}
pop {r4-r11,pc}
endfunc