Source code
Revision control
Copy as Markdown
Other Tools
/*
* VP8 NEON optimisations
*
* Copyright (c) 2010 Rob Clark <rob@ti.com>
* Copyright (c) 2011 Mans Rullgard <mans@mansr.com>
* Copyright (c) 2018 Magnus Röös <mla2.roos@gmail.com>
* Copyright (c) 2019 Martin Storsjo <martin@martin.st>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
#include "neon.S"
function ff_vp8_luma_dc_wht_neon, export=1
ld1 {v0.4h - v3.4h}, [x1]
movi v30.8h, #0
add v4.4h, v0.4h, v3.4h
add v6.4h, v1.4h, v2.4h
st1 {v30.8h}, [x1], #16
sub v7.4h, v1.4h, v2.4h
sub v5.4h, v0.4h, v3.4h
st1 {v30.8h}, [x1]
add v0.4h, v4.4h, v6.4h
add v1.4h, v5.4h, v7.4h
sub v2.4h, v4.4h, v6.4h
sub v3.4h, v5.4h, v7.4h
movi v16.4h, #3
transpose_4x4H v0, v1, v2, v3, v4, v5, v6, v7
add v0.4h, v0.4h, v16.4h
add v4.4h, v0.4h, v3.4h
add v6.4h, v1.4h, v2.4h
sub v7.4h, v1.4h, v2.4h
sub v5.4h, v0.4h, v3.4h
add v0.4h, v4.4h, v6.4h
add v1.4h, v5.4h, v7.4h
sub v2.4h, v4.4h, v6.4h
sub v3.4h, v5.4h, v7.4h
sshr v0.4h, v0.4h, #3
sshr v1.4h, v1.4h, #3
sshr v2.4h, v2.4h, #3
sshr v3.4h, v3.4h, #3
mov x3, #32
st1 {v0.h}[0], [x0], x3
st1 {v1.h}[0], [x0], x3
st1 {v2.h}[0], [x0], x3
st1 {v3.h}[0], [x0], x3
st1 {v0.h}[1], [x0], x3
st1 {v1.h}[1], [x0], x3
st1 {v2.h}[1], [x0], x3
st1 {v3.h}[1], [x0], x3
st1 {v0.h}[2], [x0], x3
st1 {v1.h}[2], [x0], x3
st1 {v2.h}[2], [x0], x3
st1 {v3.h}[2], [x0], x3
st1 {v0.h}[3], [x0], x3
st1 {v1.h}[3], [x0], x3
st1 {v2.h}[3], [x0], x3
st1 {v3.h}[3], [x0], x3
ret
endfunc
function ff_vp8_idct_add_neon, export=1
ld1 {v0.8b - v3.8b}, [x1]
mov w4, #20091
movk w4, #35468/2, lsl #16
dup v4.2s, w4
smull v26.4s, v1.4h, v4.h[0]
smull v27.4s, v3.4h, v4.h[0]
sqdmulh v20.4h, v1.4h, v4.h[1]
sqdmulh v23.4h, v3.4h, v4.h[1]
shrn v21.4h, v26.4s, #16
shrn v22.4h, v27.4s, #16
add v21.4h, v21.4h, v1.4h
add v22.4h, v22.4h, v3.4h
add v16.4h, v0.4h, v2.4h
sub v17.4h, v0.4h, v2.4h
add v18.4h, v21.4h, v23.4h
sub v19.4h, v20.4h, v22.4h
add v0.4h, v16.4h, v18.4h
add v1.4h, v17.4h, v19.4h
sub v3.4h, v16.4h, v18.4h
sub v2.4h, v17.4h, v19.4h
transpose_4x4H v0, v1, v2, v3, v24, v5, v6, v7
movi v29.8h, #0
smull v26.4s, v1.4h, v4.h[0]
st1 {v29.8h}, [x1], #16
smull v27.4s, v3.4h, v4.h[0]
st1 {v29.16b}, [x1]
sqdmulh v21.4h, v1.4h, v4.h[1]
sqdmulh v23.4h, v3.4h, v4.h[1]
shrn v20.4h, v26.4s, #16
shrn v22.4h, v27.4s, #16
add v20.4h, v20.4h, v1.4h
add v22.4h, v22.4h, v3.4h
add v16.4h, v0.4h, v2.4h
sub v17.4h, v0.4h, v2.4h
add v18.4h, v20.4h, v23.4h
ld1 {v24.s}[0], [x0], x2
sub v19.4h, v21.4h, v22.4h
ld1 {v25.s}[0], [x0], x2
add v0.4h, v16.4h, v18.4h
add v1.4h, v17.4h, v19.4h
ld1 {v26.s}[0], [x0], x2
sub v3.4h, v16.4h, v18.4h
sub v2.4h, v17.4h, v19.4h
ld1 {v27.s}[0], [x0], x2
srshr v0.4h, v0.4h, #3
srshr v1.4h, v1.4h, #3
srshr v2.4h, v2.4h, #3
srshr v3.4h, v3.4h, #3
sub x0, x0, x2, lsl #2
transpose_4x4H v0, v1, v2, v3, v5, v6, v7, v16
uaddw v0.8h, v0.8h, v24.8b
uaddw v1.8h, v1.8h, v25.8b
uaddw v2.8h, v2.8h, v26.8b
uaddw v3.8h, v3.8h, v27.8b
sqxtun v0.8b, v0.8h
sqxtun v1.8b, v1.8h
sqxtun v2.8b, v2.8h
sqxtun v3.8b, v3.8h
st1 {v0.s}[0], [x0], x2
st1 {v1.s}[0], [x0], x2
st1 {v2.s}[0], [x0], x2
st1 {v3.s}[0], [x0], x2
ret
endfunc
function ff_vp8_idct_dc_add4uv_neon, export=1
movi v0.4h, #0
mov x3, #32
ld1r {v16.4h}, [x1]
st1 {v0.h}[0], [x1], x3
ld1r {v17.4h}, [x1]
st1 {v0.h}[0], [x1], x3
ld1r {v18.4h}, [x1]
st1 {v0.h}[0], [x1], x3
ld1r {v19.4h}, [x1]
st1 {v0.h}[0], [x1], x3
ins v16.d[1], v17.d[0]
ins v18.d[1], v19.d[0]
mov x3, x0
srshr v16.8h, v16.8h, #3 // dc >>= 3
ld1 {v0.8b}, [x0], x2
srshr v18.8h, v18.8h, #3
ld1 {v1.8b}, [x0], x2
uaddw v20.8h, v16.8h, v0.8b
ld1 {v2.8b}, [x0], x2
uaddw v0.8h, v16.8h, v1.8b
ld1 {v3.8b}, [x0], x2
uaddw v22.8h, v16.8h, v2.8b
ld1 {v4.8b}, [x0], x2
uaddw v2.8h, v16.8h, v3.8b
ld1 {v5.8b}, [x0], x2
uaddw v24.8h, v18.8h, v4.8b
ld1 {v6.8b}, [x0], x2
uaddw v4.8h, v18.8h, v5.8b
ld1 {v7.8b}, [x0], x2
uaddw v26.8h, v18.8h, v6.8b
sqxtun v20.8b, v20.8h
uaddw v6.8h, v18.8h, v7.8b
sqxtun v21.8b, v0.8h
sqxtun v22.8b, v22.8h
st1 {v20.8b}, [x3], x2
sqxtun v23.8b, v2.8h
st1 {v21.8b}, [x3], x2
sqxtun v24.8b, v24.8h
st1 {v22.8b}, [x3], x2
sqxtun v25.8b, v4.8h
st1 {v23.8b}, [x3], x2
sqxtun v26.8b, v26.8h
st1 {v24.8b}, [x3], x2
sqxtun v27.8b, v6.8h
st1 {v25.8b}, [x3], x2
st1 {v26.8b}, [x3], x2
st1 {v27.8b}, [x3], x2
ret
endfunc
function ff_vp8_idct_dc_add4y_neon, export=1
movi v0.16b, #0
mov x3, #32
ld1r {v16.4h}, [x1]
st1 {v0.h}[0], [x1], x3
ld1r {v17.4h}, [x1]
st1 {v0.h}[0], [x1], x3
zip1 v16.2d, v16.2d, v17.2d
ld1r {v18.4h}, [x1]
st1 {v0.h}[0], [x1], x3
ld1r {v19.4h}, [x1]
st1 {v0.h}[0], [x1], x3
zip1 v18.2d, v18.2d, v19.2d
srshr v16.8h, v16.8h, #3 // dc >>= 3
ld1 {v0.16b}, [x0], x2
srshr v18.8h, v18.8h, #3
ld1 {v1.16b}, [x0], x2
uaddw v20.8h, v16.8h, v0.8b
ld1 {v2.16b}, [x0], x2
uaddw2 v0.8h, v18.8h, v0.16b
ld1 {v3.16b}, [x0], x2
uaddw v21.8h, v16.8h, v1.8b
uaddw2 v1.8h, v18.8h, v1.16b
uaddw v22.8h, v16.8h, v2.8b
uaddw2 v2.8h, v18.8h, v2.16b
uaddw v23.8h, v16.8h, v3.8b
uaddw2 v3.8h, v18.8h, v3.16b
sub x0, x0, x2, lsl #2
sqxtun v20.8b, v20.8h
sqxtun2 v20.16b, v0.8h
sqxtun v21.8b, v21.8h
sqxtun2 v21.16b, v1.8h
sqxtun v22.8b, v22.8h
st1 {v20.16b}, [x0], x2
sqxtun2 v22.16b, v2.8h
st1 {v21.16b}, [x0], x2
sqxtun v23.8b, v23.8h
st1 {v22.16b}, [x0], x2
sqxtun2 v23.16b, v3.8h
st1 {v23.16b}, [x0], x2
ret
endfunc
function ff_vp8_idct_dc_add_neon, export=1
mov w3, #0
ld1r {v2.8h}, [x1]
strh w3, [x1]
srshr v2.8h, v2.8h, #3
ld1 {v0.s}[0], [x0], x2
ld1 {v0.s}[1], [x0], x2
uaddw v3.8h, v2.8h, v0.8b
ld1 {v1.s}[0], [x0], x2
ld1 {v1.s}[1], [x0], x2
uaddw v4.8h, v2.8h, v1.8b
sqxtun v0.8b, v3.8h
sqxtun v1.8b, v4.8h
sub x0, x0, x2, lsl #2
st1 {v0.s}[0], [x0], x2
st1 {v0.s}[1], [x0], x2
st1 {v1.s}[0], [x0], x2
st1 {v1.s}[1], [x0], x2
ret
endfunc
// Register layout:
// P3..Q3 -> v0..v7
// flim_E -> v22
// flim_I -> v23
// hev_thresh -> x5
//
.macro vp8_loop_filter, inner=0, simple=0, hev_thresh
.if \simple
uabd v17.16b, v3.16b, v4.16b // abs(P0-Q0)
uabd v23.16b, v2.16b, v5.16b // abs(P1-Q1)
uqadd v17.16b, v17.16b, v17.16b // abs(P0-Q0) * 2
ushr v18.16b, v23.16b, #1 // abs(P1-Q1) / 2
uqadd v19.16b, v17.16b, v18.16b // (abs(P0-Q0)*2) + (abs(P1-Q1)/2)
movi v21.16b, #0x80
cmhs v16.16b, v22.16b, v19.16b // (abs(P0-Q0)*2) + (abs(P1-Q1)/2) <= flim
.else
// calculate hev and normal_limit:
uabd v20.16b, v2.16b, v3.16b // abs(P1-P0)
uabd v21.16b, v5.16b, v4.16b // abs(Q1-Q0)
uabd v18.16b, v0.16b, v1.16b // abs(P3-P2)
uabd v19.16b, v1.16b, v2.16b // abs(P2-P1)
cmhs v16.16b, v23.16b, v20.16b // abs(P1-P0) <= flim_I
cmhs v17.16b, v23.16b, v21.16b // abs(Q1-Q0) <= flim_I
cmhs v18.16b, v23.16b, v18.16b // abs(P3-P2) <= flim_I
cmhs v19.16b, v23.16b, v19.16b // abs(P2-P1) <= flim_I
and v16.16b, v17.16b, v16.16b
uabd v17.16b, v7.16b, v6.16b // abs(Q3-Q2)
and v16.16b, v16.16b, v19.16b
uabd v19.16b, v6.16b, v5.16b // abs(Q2-Q1)
and v16.16b, v16.16b, v18.16b
cmhs v18.16b, v23.16b, v17.16b // abs(Q3-Q2) <= flim_I
cmhs v19.16b, v23.16b, v19.16b // abs(Q2-Q1) <= flim_I
uabd v17.16b, v3.16b, v4.16b // abs(P0-Q0)
uabd v23.16b, v2.16b, v5.16b // abs(P1-Q1)
and v16.16b, v16.16b, v18.16b
uqadd v17.16b, v17.16b, v17.16b // abs(P0-Q0) * 2
and v16.16b, v16.16b, v19.16b
ushr v18.16b, v23.16b, #1 // abs(P1-Q1) / 2
dup v23.16b, \hev_thresh // hev_thresh
uqadd v19.16b, v17.16b, v18.16b // (abs(P0-Q0)*2) + (abs(P1-Q1)/2)
cmhi v20.16b, v20.16b, v23.16b // abs(P1-P0) > hev_thresh
cmhs v19.16b, v22.16b, v19.16b // (abs(P0-Q0)*2) + (abs(P1-Q1)/2) <= flim_E
cmhi v22.16b, v21.16b, v23.16b // abs(Q1-Q0) > hev_thresh
and v16.16b, v16.16b, v19.16b
movi v21.16b, #0x80
orr v17.16b, v20.16b, v22.16b
.endif
// at this point:
// v16: normal_limit
// v17: hev
// convert to signed value:
eor v3.16b, v3.16b, v21.16b // PS0 = P0 ^ 0x80
eor v4.16b, v4.16b, v21.16b // QS0 = Q0 ^ 0x80
movi v20.8h, #3
ssubl v18.8h, v4.8b, v3.8b // QS0 - PS0
ssubl2 v19.8h, v4.16b, v3.16b // (widened to 16bit)
eor v2.16b, v2.16b, v21.16b // PS1 = P1 ^ 0x80
eor v5.16b, v5.16b, v21.16b // QS1 = Q1 ^ 0x80
mul v18.8h, v18.8h, v20.8h // w = 3 * (QS0 - PS0)
mul v19.8h, v19.8h, v20.8h
sqsub v20.16b, v2.16b, v5.16b // clamp(PS1-QS1)
movi v22.16b, #4
movi v23.16b, #3
.if \inner
and v20.16b, v20.16b, v17.16b // if(hev) w += clamp(PS1-QS1)
.endif
saddw v18.8h, v18.8h, v20.8b // w += clamp(PS1-QS1)
saddw2 v19.8h, v19.8h, v20.16b
sqxtn v18.8b, v18.8h // narrow result back into v18
sqxtn2 v18.16b, v19.8h
.if !\inner && !\simple
eor v1.16b, v1.16b, v21.16b // PS2 = P2 ^ 0x80
eor v6.16b, v6.16b, v21.16b // QS2 = Q2 ^ 0x80
.endif
and v18.16b, v18.16b, v16.16b // w &= normal_limit
// registers used at this point..
// v0 -> P3 (don't corrupt)
// v1-v6 -> PS2-QS2
// v7 -> Q3 (don't corrupt)
// v17 -> hev
// v18 -> w
// v21 -> #0x80
// v22 -> #4
// v23 -> #3
// v16, v19, v29 -> unused
//
// filter_common: is4tap==1
// c1 = clamp(w + 4) >> 3;
// c2 = clamp(w + 3) >> 3;
// Q0 = s2u(QS0 - c1);
// P0 = s2u(PS0 + c2);
.if \simple
sqadd v19.16b, v18.16b, v22.16b // c1 = clamp((w&hev)+4)
sqadd v20.16b, v18.16b, v23.16b // c2 = clamp((w&hev)+3)
sshr v19.16b, v19.16b, #3 // c1 >>= 3
sshr v20.16b, v20.16b, #3 // c2 >>= 3
sqsub v4.16b, v4.16b, v19.16b // QS0 = clamp(QS0-c1)
sqadd v3.16b, v3.16b, v20.16b // PS0 = clamp(PS0+c2)
eor v4.16b, v4.16b, v21.16b // Q0 = QS0 ^ 0x80
eor v3.16b, v3.16b, v21.16b // P0 = PS0 ^ 0x80
eor v5.16b, v5.16b, v21.16b // Q1 = QS1 ^ 0x80
eor v2.16b, v2.16b, v21.16b // P1 = PS1 ^ 0x80
.elseif \inner
// the !is4tap case of filter_common, only used for inner blocks
// c3 = ((c1&~hev) + 1) >> 1;
// Q1 = s2u(QS1 - c3);
// P1 = s2u(PS1 + c3);
sqadd v19.16b, v18.16b, v22.16b // c1 = clamp((w&hev)+4)
sqadd v20.16b, v18.16b, v23.16b // c2 = clamp((w&hev)+3)
sshr v19.16b, v19.16b, #3 // c1 >>= 3
sshr v20.16b, v20.16b, #3 // c2 >>= 3
sqsub v4.16b, v4.16b, v19.16b // QS0 = clamp(QS0-c1)
sqadd v3.16b, v3.16b, v20.16b // PS0 = clamp(PS0+c2)
bic v19.16b, v19.16b, v17.16b // c1 & ~hev
eor v4.16b, v4.16b, v21.16b // Q0 = QS0 ^ 0x80
srshr v19.16b, v19.16b, #1 // c3 >>= 1
eor v3.16b, v3.16b, v21.16b // P0 = PS0 ^ 0x80
sqsub v5.16b, v5.16b, v19.16b // QS1 = clamp(QS1-c3)
sqadd v2.16b, v2.16b, v19.16b // PS1 = clamp(PS1+c3)
eor v5.16b, v5.16b, v21.16b // Q1 = QS1 ^ 0x80
eor v2.16b, v2.16b, v21.16b // P1 = PS1 ^ 0x80
.else
and v20.16b, v18.16b, v17.16b // w & hev
sqadd v19.16b, v20.16b, v22.16b // c1 = clamp((w&hev)+4)
sqadd v20.16b, v20.16b, v23.16b // c2 = clamp((w&hev)+3)
sshr v19.16b, v19.16b, #3 // c1 >>= 3
sshr v20.16b, v20.16b, #3 // c2 >>= 3
bic v18.16b, v18.16b, v17.16b // w &= ~hev
sqsub v4.16b, v4.16b, v19.16b // QS0 = clamp(QS0-c1)
sqadd v3.16b, v3.16b, v20.16b // PS0 = clamp(PS0+c2)
// filter_mbedge:
// a = clamp((27*w + 63) >> 7);
// Q0 = s2u(QS0 - a);
// P0 = s2u(PS0 + a);
// a = clamp((18*w + 63) >> 7);
// Q1 = s2u(QS1 - a);
// P1 = s2u(PS1 + a);
// a = clamp((9*w + 63) >> 7);
// Q2 = s2u(QS2 - a);
// P2 = s2u(PS2 + a);
movi v17.8h, #63
sshll v22.8h, v18.8b, #3
sshll2 v23.8h, v18.16b, #3
saddw v22.8h, v22.8h, v18.8b
saddw2 v23.8h, v23.8h, v18.16b
add v16.8h, v17.8h, v22.8h
add v17.8h, v17.8h, v23.8h // 9*w + 63
add v19.8h, v16.8h, v22.8h
add v20.8h, v17.8h, v23.8h // 18*w + 63
add v22.8h, v19.8h, v22.8h
add v23.8h, v20.8h, v23.8h // 27*w + 63
sqshrn v16.8b, v16.8h, #7
sqshrn2 v16.16b, v17.8h, #7 // clamp(( 9*w + 63)>>7)
sqshrn v19.8b, v19.8h, #7
sqshrn2 v19.16b, v20.8h, #7 // clamp((18*w + 63)>>7)
sqshrn v22.8b, v22.8h, #7
sqshrn2 v22.16b, v23.8h, #7 // clamp((27*w + 63)>>7)
sqadd v1.16b, v1.16b, v16.16b // PS2 = clamp(PS2+a)
sqsub v6.16b, v6.16b, v16.16b // QS2 = clamp(QS2-a)
sqadd v2.16b, v2.16b, v19.16b // PS1 = clamp(PS1+a)
sqsub v5.16b, v5.16b, v19.16b // QS1 = clamp(QS1-a)
sqadd v3.16b, v3.16b, v22.16b // PS0 = clamp(PS0+a)
sqsub v4.16b, v4.16b, v22.16b // QS0 = clamp(QS0-a)
eor v3.16b, v3.16b, v21.16b // P0 = PS0 ^ 0x80
eor v4.16b, v4.16b, v21.16b // Q0 = QS0 ^ 0x80
eor v2.16b, v2.16b, v21.16b // P1 = PS1 ^ 0x80
eor v5.16b, v5.16b, v21.16b // Q1 = QS1 ^ 0x80
eor v1.16b, v1.16b, v21.16b // P2 = PS2 ^ 0x80
eor v6.16b, v6.16b, v21.16b // Q2 = QS2 ^ 0x80
.endif
.endm
.macro vp8_v_loop_filter16 name, inner=0, simple=0
function ff_vp8_v_loop_filter16\name\()_neon, export=1
sub x0, x0, x1, lsl #1+!\simple
// Load pixels:
.if !\simple
ld1 {v0.16b}, [x0], x1 // P3
ld1 {v1.16b}, [x0], x1 // P2
.endif
ld1 {v2.16b}, [x0], x1 // P1
ld1 {v3.16b}, [x0], x1 // P0
ld1 {v4.16b}, [x0], x1 // Q0
ld1 {v5.16b}, [x0], x1 // Q1
.if !\simple
ld1 {v6.16b}, [x0], x1 // Q2
ld1 {v7.16b}, [x0] // Q3
dup v23.16b, w3 // flim_I
.endif
dup v22.16b, w2 // flim_E
vp8_loop_filter inner=\inner, simple=\simple, hev_thresh=w4
// back up to P2: dst -= stride * 6
sub x0, x0, x1, lsl #2
.if !\simple
sub x0, x0, x1, lsl #1
// Store pixels:
st1 {v1.16b}, [x0], x1 // P2
.endif
st1 {v2.16b}, [x0], x1 // P1
st1 {v3.16b}, [x0], x1 // P0
st1 {v4.16b}, [x0], x1 // Q0
st1 {v5.16b}, [x0], x1 // Q1
.if !\simple
st1 {v6.16b}, [x0] // Q2
.endif
ret
endfunc
.endm
vp8_v_loop_filter16
vp8_v_loop_filter16 _inner, inner=1
vp8_v_loop_filter16 _simple, simple=1
.macro vp8_v_loop_filter8uv name, inner=0
function ff_vp8_v_loop_filter8uv\name\()_neon, export=1
sub x0, x0, x2, lsl #2
sub x1, x1, x2, lsl #2
// Load pixels:
ld1 {v0.d}[0], [x0], x2 // P3
ld1 {v0.d}[1], [x1], x2 // P3
ld1 {v1.d}[0], [x0], x2 // P2
ld1 {v1.d}[1], [x1], x2 // P2
ld1 {v2.d}[0], [x0], x2 // P1
ld1 {v2.d}[1], [x1], x2 // P1
ld1 {v3.d}[0], [x0], x2 // P0
ld1 {v3.d}[1], [x1], x2 // P0
ld1 {v4.d}[0], [x0], x2 // Q0
ld1 {v4.d}[1], [x1], x2 // Q0
ld1 {v5.d}[0], [x0], x2 // Q1
ld1 {v5.d}[1], [x1], x2 // Q1
ld1 {v6.d}[0], [x0], x2 // Q2
ld1 {v6.d}[1], [x1], x2 // Q2
ld1 {v7.d}[0], [x0] // Q3
ld1 {v7.d}[1], [x1] // Q3
dup v22.16b, w3 // flim_E
dup v23.16b, w4 // flim_I
vp8_loop_filter inner=\inner, hev_thresh=w5
// back up to P2: u,v -= stride * 6
sub x0, x0, x2, lsl #2
sub x1, x1, x2, lsl #2
sub x0, x0, x2, lsl #1
sub x1, x1, x2, lsl #1
// Store pixels:
st1 {v1.d}[0], [x0], x2 // P2
st1 {v1.d}[1], [x1], x2 // P2
st1 {v2.d}[0], [x0], x2 // P1
st1 {v2.d}[1], [x1], x2 // P1
st1 {v3.d}[0], [x0], x2 // P0
st1 {v3.d}[1], [x1], x2 // P0
st1 {v4.d}[0], [x0], x2 // Q0
st1 {v4.d}[1], [x1], x2 // Q0
st1 {v5.d}[0], [x0], x2 // Q1
st1 {v5.d}[1], [x1], x2 // Q1
st1 {v6.d}[0], [x0] // Q2
st1 {v6.d}[1], [x1] // Q2
ret
endfunc
.endm
vp8_v_loop_filter8uv
vp8_v_loop_filter8uv _inner, inner=1
.macro vp8_h_loop_filter16 name, inner=0, simple=0
function ff_vp8_h_loop_filter16\name\()_neon, export=1
sub x0, x0, #4
// Load pixels:
ld1 {v0.d}[0], [x0], x1
ld1 {v1.d}[0], [x0], x1
ld1 {v2.d}[0], [x0], x1
ld1 {v3.d}[0], [x0], x1
ld1 {v4.d}[0], [x0], x1
ld1 {v5.d}[0], [x0], x1
ld1 {v6.d}[0], [x0], x1
ld1 {v7.d}[0], [x0], x1
ld1 {v0.d}[1], [x0], x1
ld1 {v1.d}[1], [x0], x1
ld1 {v2.d}[1], [x0], x1
ld1 {v3.d}[1], [x0], x1
ld1 {v4.d}[1], [x0], x1
ld1 {v5.d}[1], [x0], x1
ld1 {v6.d}[1], [x0], x1
ld1 {v7.d}[1], [x0], x1
transpose_8x16B v0, v1, v2, v3, v4, v5, v6, v7, v30, v31
dup v22.16b, w2 // flim_E
.if !\simple
dup v23.16b, w3 // flim_I
.endif
vp8_loop_filter inner=\inner, simple=\simple, hev_thresh=w4
sub x0, x0, x1, lsl #4 // backup 16 rows
transpose_8x16B v0, v1, v2, v3, v4, v5, v6, v7, v30, v31
// Store pixels:
st1 {v0.d}[0], [x0], x1
st1 {v1.d}[0], [x0], x1
st1 {v2.d}[0], [x0], x1
st1 {v3.d}[0], [x0], x1
st1 {v4.d}[0], [x0], x1
st1 {v5.d}[0], [x0], x1
st1 {v6.d}[0], [x0], x1
st1 {v7.d}[0], [x0], x1
st1 {v0.d}[1], [x0], x1
st1 {v1.d}[1], [x0], x1
st1 {v2.d}[1], [x0], x1
st1 {v3.d}[1], [x0], x1
st1 {v4.d}[1], [x0], x1
st1 {v5.d}[1], [x0], x1
st1 {v6.d}[1], [x0], x1
st1 {v7.d}[1], [x0]
ret
endfunc
.endm
vp8_h_loop_filter16
vp8_h_loop_filter16 _inner, inner=1
vp8_h_loop_filter16 _simple, simple=1
.macro vp8_h_loop_filter8uv name, inner=0
function ff_vp8_h_loop_filter8uv\name\()_neon, export=1
sub x0, x0, #4
sub x1, x1, #4
// Load pixels:
ld1 {v0.d}[0], [x0], x2 // load u
ld1 {v0.d}[1], [x1], x2 // load v
ld1 {v1.d}[0], [x0], x2
ld1 {v1.d}[1], [x1], x2
ld1 {v2.d}[0], [x0], x2
ld1 {v2.d}[1], [x1], x2
ld1 {v3.d}[0], [x0], x2
ld1 {v3.d}[1], [x1], x2
ld1 {v4.d}[0], [x0], x2
ld1 {v4.d}[1], [x1], x2
ld1 {v5.d}[0], [x0], x2
ld1 {v5.d}[1], [x1], x2
ld1 {v6.d}[0], [x0], x2
ld1 {v6.d}[1], [x1], x2
ld1 {v7.d}[0], [x0], x2
ld1 {v7.d}[1], [x1], x2
transpose_8x16B v0, v1, v2, v3, v4, v5, v6, v7, v30, v31
dup v22.16b, w3 // flim_E
dup v23.16b, w4 // flim_I
vp8_loop_filter inner=\inner, hev_thresh=w5
sub x0, x0, x2, lsl #3 // backup u 8 rows
sub x1, x1, x2, lsl #3 // backup v 8 rows
transpose_8x16B v0, v1, v2, v3, v4, v5, v6, v7, v30, v31
// Store pixels:
st1 {v0.d}[0], [x0], x2 // load u
st1 {v0.d}[1], [x1], x2 // load v
st1 {v1.d}[0], [x0], x2
st1 {v1.d}[1], [x1], x2
st1 {v2.d}[0], [x0], x2
st1 {v2.d}[1], [x1], x2
st1 {v3.d}[0], [x0], x2
st1 {v3.d}[1], [x1], x2
st1 {v4.d}[0], [x0], x2
st1 {v4.d}[1], [x1], x2
st1 {v5.d}[0], [x0], x2
st1 {v5.d}[1], [x1], x2
st1 {v6.d}[0], [x0], x2
st1 {v6.d}[1], [x1], x2
st1 {v7.d}[0], [x0]
st1 {v7.d}[1], [x1]
ret
endfunc
.endm
vp8_h_loop_filter8uv
vp8_h_loop_filter8uv _inner, inner=1
function ff_put_vp8_pixels16_neon, export=1
1:
subs w4, w4, #4
ld1 {v0.16b}, [x2], x3
ld1 {v1.16b}, [x2], x3
ld1 {v2.16b}, [x2], x3
ld1 {v3.16b}, [x2], x3
st1 {v0.16b}, [x0], x1
st1 {v1.16b}, [x0], x1
st1 {v2.16b}, [x0], x1
st1 {v3.16b}, [x0], x1
b.gt 1b
ret
endfunc
function ff_put_vp8_pixels8_neon, export=1
1:
subs w4, w4, #4
ld1 {v0.8b}, [x2], x3
ld1 {v0.d}[1], [x2], x3
ld1 {v1.8b}, [x2], x3
ld1 {v1.d}[1], [x2], x3
st1 {v0.8b}, [x0], x1
st1 {v0.d}[1], [x0], x1
st1 {v1.8b}, [x0], x1
st1 {v1.d}[1], [x0], x1
b.gt 1b
ret
endfunc
/* 4/6-tap 8th-pel MC */
.macro vp8_epel8_h6 d, s0, s1
ext v22.8b, \s0\().8b, \s1\().8b, #1
uxtl v18.8h, \s0\().8b
ext v23.8b, \s0\().8b, \s1\().8b, #2
uxtl v19.8h, v22.8b
ext v24.8b, \s0\().8b, \s1\().8b, #3
uxtl v21.8h, v23.8b
ext v25.8b, \s0\().8b, \s1\().8b, #4
uxtl v22.8h, v24.8b
ext v26.8b, \s0\().8b, \s1\().8b, #5
uxtl v25.8h, v25.8b
mul v21.8h, v21.8h, v0.h[2]
uxtl v26.8h, v26.8b
mul v22.8h, v22.8h, v0.h[3]
mls v21.8h, v19.8h, v0.h[1]
mls v22.8h, v25.8h, v0.h[4]
mla v21.8h, v18.8h, v0.h[0]
mla v22.8h, v26.8h, v0.h[5]
sqadd v22.8h, v21.8h, v22.8h
sqrshrun \d\().8b, v22.8h, #7
.endm
.macro vp8_epel16_h6 d0, v0, v1
ext v22.16b, \v0\().16b, \v1\().16b, #3
ext v23.16b, \v0\().16b, \v1\().16b, #4
uxtl v19.8h, v22.8b
uxtl2 v22.8h, v22.16b
ext v3.16b, \v0\().16b, \v1\().16b, #2
uxtl v20.8h, v23.8b
uxtl2 v23.8h, v23.16b
ext v16.16b, \v0\().16b, \v1\().16b, #1
uxtl v18.8h, v3.8b
uxtl2 v3.8h, v3.16b
ext v2.16b, \v0\().16b, \v1\().16b, #5
uxtl v21.8h, v2.8b
uxtl2 v2.8h, v2.16b
uxtl v17.8h, v16.8b
uxtl2 v16.8h, v16.16b
mul v19.8h, v19.8h, v0.h[3]
mul v18.8h, v18.8h, v0.h[2]
mul v3.8h, v3.8h, v0.h[2]
mul v22.8h, v22.8h, v0.h[3]
mls v19.8h, v20.8h, v0.h[4]
uxtl v20.8h, \v0\().8b
uxtl2 v1.8h, \v0\().16b
mls v18.8h, v17.8h, v0.h[1]
mls v3.8h, v16.8h, v0.h[1]
mls v22.8h, v23.8h, v0.h[4]
mla v18.8h, v20.8h, v0.h[0]
mla v19.8h, v21.8h, v0.h[5]
mla v3.8h, v1.8h, v0.h[0]
mla v22.8h, v2.8h, v0.h[5]
sqadd v19.8h, v18.8h, v19.8h
sqadd v22.8h, v3.8h, v22.8h
sqrshrun \d0\().8b, v19.8h, #7
sqrshrun2 \d0\().16b, v22.8h, #7
.endm
.macro vp8_epel8_v6_y2 d0, d1, s0, s1, s2, s3, s4, s5, s6
uxtl \s0\().8h, \s0\().8b
uxtl \s3\().8h, \s3\().8b
uxtl \s6\().8h, \s6\().8b
uxtl \s1\().8h, \s1\().8b
uxtl \s4\().8h, \s4\().8b
uxtl \s2\().8h, \s2\().8b
uxtl \s5\().8h, \s5\().8b
mul \s0\().8h, \s0\().8h, v0.h[0]
mul v31.8h , \s3\().8h, v0.h[3]
mul \s3\().8h, \s3\().8h, v0.h[2]
mul \s6\().8h, \s6\().8h, v0.h[5]
mls \s0\().8h, \s1\().8h, v0.h[1]
mls v31.8h , \s4\().8h, v0.h[4]
mls \s3\().8h, \s2\().8h, v0.h[1]
mls \s6\().8h, \s5\().8h, v0.h[4]
mla \s0\().8h, \s2\().8h, v0.h[2]
mla v31.8h , \s5\().8h, v0.h[5]
mla \s3\().8h, \s1\().8h, v0.h[0]
mla \s6\().8h, \s4\().8h, v0.h[3]
sqadd v31.8h , \s0\().8h, v31.8h
sqadd \s6\().8h, \s3\().8h, \s6\().8h
sqrshrun \d0\().8b, v31.8h, #7
sqrshrun \d1\().8b, \s6\().8h, #7
.endm
.macro vp8_epel8_h4 d, v0, v1
ext v22.8b, \v0\().8b, \v1\().8b, #1
uxtl v19.8h, \v0\().8b
ext v23.8b, \v0\().8b, \v1\().8b, #2
uxtl v20.8h, v22.8b
ext v25.8b, \v0\().8b, \v1\().8b, #3
uxtl v22.8h, v23.8b
uxtl v25.8h, v25.8b
mul v20.8h, v20.8h, v0.h[2]
mul v22.8h, v22.8h, v0.h[3]
mls v20.8h, v19.8h, v0.h[1]
mls v22.8h, v25.8h, v0.h[4]
sqadd v22.8h, v20.8h, v22.8h
sqrshrun \d\().8b, v22.8h, #7
.endm
.macro vp8_epel8_v4_y2 d0, s0, s1, s2, s3, s4
uxtl \s0\().8h, \s0\().8b
uxtl \s1\().8h, \s1\().8b
uxtl \s2\().8h, \s2\().8b
uxtl \s3\().8h, \s3\().8b
uxtl \s4\().8h, \s4\().8b
mul v21.8h, \s1\().8h, v0.h[2]
mul v23.8h, \s2\().8h, v0.h[3]
mul \s2\().8h, \s2\().8h, v0.h[2]
mul v22.8h, \s3\().8h, v0.h[3]
mls v21.8h, \s0\().8h, v0.h[1]
mls v23.8h, \s3\().8h, v0.h[4]
mls \s2\().8h, \s1\().8h, v0.h[1]
mls v22.8h, \s4\().8h, v0.h[4]
sqadd v21.8h, v21.8h, v23.8h
sqadd \s2\().8h, \s2\().8h, v22.8h
sqrshrun \d0\().8b, v21.8h, #7
sqrshrun2 \d0\().16b, \s2\().8h, #7
.endm
// note: worst case sum of all 6-tap filter values * 255 is 0x7f80 so 16 bit
// arithmetic can be used to apply filters
const subpel_filters, align=4
.short 0, 6, 123, 12, 1, 0, 0, 0
.short 2, 11, 108, 36, 8, 1, 0, 0
.short 0, 9, 93, 50, 6, 0, 0, 0
.short 3, 16, 77, 77, 16, 3, 0, 0
.short 0, 6, 50, 93, 9, 0, 0, 0
.short 1, 8, 36, 108, 11, 2, 0, 0
.short 0, 1, 12, 123, 6, 0, 0, 0
endconst
function ff_put_vp8_epel16_v6_neon, export=1
sub x2, x2, x3, lsl #1
sxtw x4, w4
sxtw x6, w6
movrel x17, subpel_filters, -16
add x6, x17, x6, lsl #4 // y
ld1 {v0.8h}, [x6]
1:
ld1 {v1.1d - v2.1d}, [x2], x3
ld1 {v3.1d - v4.1d}, [x2], x3
ld1 {v16.1d - v17.1d}, [x2], x3
ld1 {v18.1d - v19.1d}, [x2], x3
ld1 {v20.1d - v21.1d}, [x2], x3
ld1 {v22.1d - v23.1d}, [x2], x3
ld1 {v24.1d - v25.1d}, [x2]
sub x2, x2, x3, lsl #2
vp8_epel8_v6_y2 v1, v3, v1, v3, v16, v18, v20, v22, v24
vp8_epel8_v6_y2 v2, v4, v2, v4, v17, v19, v21, v23, v25
st1 {v1.1d - v2.1d}, [x0], x1
st1 {v3.1d - v4.1d}, [x0], x1
subs x4, x4, #2
b.ne 1b
ret
endfunc
function ff_put_vp8_epel16_h6_neon, export=1
sub x2, x2, #2
sxtw x5, w5 // x
// first pass (horizontal):
movrel x17, subpel_filters, -16
add x5, x17, x5, lsl #4 // x
ld1 {v0.8h}, [x5]
1:
ld1 {v1.16b, v2.16b}, [x2], x3
vp8_epel16_h6 v1, v1, v2
st1 {v1.16b}, [x0], x1
subs w4, w4, #1
b.ne 1b
ret
endfunc
function ff_put_vp8_epel16_h6v6_neon, export=1
sub x2, x2, x3, lsl #1
sub x2, x2, #2
// first pass (horizontal):
movrel x17, subpel_filters, -16
sxtw x5, w5 // x
add x16, x17, x5, lsl #4 // x
sub sp, sp, #336+16
ld1 {v0.8h}, [x16]
add x7, sp, #15
sxtw x4, w4
add x16, x4, #5 // h
bic x7, x7, #15
1:
ld1 {v1.16b, v2.16b}, [x2], x3
vp8_epel16_h6 v1, v1, v2
st1 {v1.16b}, [x7], #16
subs x16, x16, #1
b.ne 1b
// second pass (vertical):
sxtw x6, w6
add x6, x17, x6, lsl #4 // y
add x7, sp, #15
ld1 {v0.8h}, [x6]
bic x7, x7, #15
2:
ld1 {v1.8b - v4.8b}, [x7], #32
ld1 {v16.8b - v19.8b}, [x7], #32
ld1 {v20.8b - v23.8b}, [x7], #32
ld1 {v24.8b - v25.8b}, [x7]
sub x7, x7, #64
vp8_epel8_v6_y2 v1, v3, v1, v3, v16, v18, v20, v22, v24
vp8_epel8_v6_y2 v2, v4, v2, v4, v17, v19, v21, v23, v25
trn1 v1.2d, v1.2d, v2.2d
trn1 v3.2d, v3.2d, v4.2d
st1 {v1.16b}, [x0], x1
st1 {v3.16b}, [x0], x1
subs x4, x4, #2
b.ne 2b
add sp, sp, #336+16
ret
endfunc
function ff_put_vp8_epel8_v6_neon, export=1
sub x2, x2, x3, lsl #1
movrel x7, subpel_filters, -16
add x6, x7, w6, uxtw #4
ld1 {v0.8h}, [x6]
1:
ld1 {v2.8b}, [x2], x3
ld1 {v3.8b}, [x2], x3
ld1 {v4.8b}, [x2], x3
ld1 {v5.8b}, [x2], x3
ld1 {v6.8b}, [x2], x3
ld1 {v7.8b}, [x2], x3
ld1 {v28.8b}, [x2]
sub x2, x2, x3, lsl #2
vp8_epel8_v6_y2 v2, v3, v2, v3, v4, v5, v6, v7, v28
st1 {v2.8b}, [x0], x1
st1 {v3.8b}, [x0], x1
subs w4, w4, #2
b.ne 1b
ret
endfunc
function ff_put_vp8_epel8_h6_neon, export=1
sub x2, x2, #2
movrel x7, subpel_filters, -16
add x5, x7, w5, uxtw #4
ld1 {v0.8h}, [x5]
1:
ld1 {v2.8b, v3.8b}, [x2], x3
vp8_epel8_h6 v2, v2, v3
st1 {v2.8b}, [x0], x1
subs w4, w4, #1
b.ne 1b
ret
endfunc
function ff_put_vp8_epel8_h6v6_neon, export=1
sub x2, x2, x3, lsl #1
sub x2, x2, #2
sxtw x4, w4
// first pass (horizontal):
movrel x17, subpel_filters, -16
sxtw x5, w5
add x5, x17, x5, lsl #4 // x
sub sp, sp, #168+16
ld1 {v0.8h}, [x5]
add x7, sp, #15
add x16, x4, #5 // h
bic x7, x7, #15
1:
ld1 {v1.8b, v2.8b}, [x2], x3
vp8_epel8_h6 v1, v1, v2
st1 {v1.8b}, [x7], #8
subs x16, x16, #1
b.ne 1b
// second pass (vertical):
sxtw x6, w6
add x6, x17, x6, lsl #4 // y
add x7, sp, #15
ld1 {v0.8h}, [x6]
bic x7, x7, #15
2:
ld1 {v1.8b - v4.8b}, [x7], #32
ld1 {v5.8b - v7.8b}, [x7]
sub x7, x7, #16
vp8_epel8_v6_y2 v1, v2, v1, v2, v3, v4, v5, v6, v7
st1 {v1.8b}, [x0], x1
st1 {v2.8b}, [x0], x1
subs x4, x4, #2
b.ne 2b
add sp, sp, #168+16
ret
endfunc
function ff_put_vp8_epel8_v4_neon, export=1
sub x2, x2, x3
movrel x7, subpel_filters, -16
add x6, x7, w6, uxtw #4
ld1 {v0.8h}, [x6]
1:
ld1 {v2.8b}, [x2], x3
ld1 {v3.8b}, [x2], x3
ld1 {v4.8b}, [x2], x3
ld1 {v5.8b}, [x2], x3
ld1 {v6.8b}, [x2]
sub x2, x2, x3, lsl #1
vp8_epel8_v4_y2 v2, v2, v3, v4, v5, v6
st1 {v2.d}[0], [x0], x1
st1 {v2.d}[1], [x0], x1
subs w4, w4, #2
b.ne 1b
ret
endfunc
function ff_put_vp8_epel8_h4_neon, export=1
sub x2, x2, #1
movrel x7, subpel_filters, -16
add x5, x7, w5, uxtw #4
ld1 {v0.8h}, [x5]
1:
ld1 {v2.8b,v3.8b}, [x2], x3
vp8_epel8_h4 v2, v2, v3
st1 {v2.8b}, [x0], x1
subs w4, w4, #1
b.ne 1b
ret
endfunc
function ff_put_vp8_epel8_h4v6_neon, export=1
sub x2, x2, x3, lsl #1
sub x2, x2, #1
sxtw x4, w4
// first pass (horizontal):
movrel x17, subpel_filters, -16
sxtw x5, w5
add x5, x17, x5, lsl #4 // x
sub sp, sp, #168+16
ld1 {v0.8h}, [x5]
add x7, sp, #15
add x16, x4, #5 // h
bic x7, x7, #15
1:
ld1 {v1.8b, v2.8b}, [x2], x3
vp8_epel8_h4 v1, v1, v2
st1 {v1.8b}, [x7], #8
subs x16, x16, #1
b.ne 1b
// second pass (vertical):
sxtw x6, w6
add x6, x17, x6, lsl #4 // y
add x7, sp, #15
ld1 {v0.8h}, [x6]
bic x7, x7, #15
2:
ld1 {v1.8b - v4.8b}, [x7], #32
ld1 {v5.8b - v7.8b}, [x7]
sub x7, x7, #16
vp8_epel8_v6_y2 v1, v2, v1, v2, v3, v4, v5, v6, v7
st1 {v1.8b}, [x0], x1
st1 {v2.8b}, [x0], x1
subs x4, x4, #2
b.ne 2b
add sp, sp, #168+16
ret
endfunc
function ff_put_vp8_epel8_h4v4_neon, export=1
sub x2, x2, x3
sub x2, x2, #1
sxtw x4, w4
// first pass (horizontal):
movrel x17, subpel_filters, -16
sxtw x5, w5
add x5, x17, x5, lsl #4 // x
sub sp, sp, #168+16
ld1 {v0.8h}, [x5]
add x7, sp, #15
add x16, x4, #3 // h
bic x7, x7, #15
1:
ld1 {v1.8b, v2.8b}, [x2], x3
vp8_epel8_h4 v1, v1, v2
st1 {v1.8b}, [x7], #8
subs x16, x16, #1
b.ne 1b
// second pass (vertical):
sxtw x6, w6
add x6, x17, x6, lsl #4 // y
add x7, sp, #15
ld1 {v0.8h}, [x6]
bic x7, x7, #15
2:
ld1 {v1.8b - v2.8b}, [x7], #16
ld1 {v3.8b - v5.8b}, [x7]
vp8_epel8_v4_y2 v1, v1, v2, v3, v4, v5
st1 {v1.d}[0], [x0], x1
st1 {v1.d}[1], [x0], x1
subs x4, x4, #2
b.ne 2b
add sp, sp, #168+16
ret
endfunc
function ff_put_vp8_epel8_h6v4_neon, export=1
sub x2, x2, x3
sub x2, x2, #2
sxtw x4, w4
// first pass (horizontal):
movrel x17, subpel_filters, -16
sxtw x5, w5
add x5, x17, x5, lsl #4 // x
sub sp, sp, #168+16
ld1 {v0.8h}, [x5]
add x7, sp, #15
add x16, x4, #3 // h
bic x7, x7, #15
1:
ld1 {v1.8b, v2.8b}, [x2], x3
vp8_epel8_h6 v1, v1, v2
st1 {v1.8b}, [x7], #8
subs x16, x16, #1
b.ne 1b
// second pass (vertical):
sxtw x6, w6
add x6, x17, x6, lsl #4 // y
add x7, sp, #15
ld1 {v0.8h}, [x6]
bic x7, x7, #15
2:
ld1 {v1.8b - v2.8b}, [x7], #16
ld1 {v3.8b - v5.8b}, [x7]
vp8_epel8_v4_y2 v1, v1, v2, v3, v4, v5
st1 {v1.d}[0], [x0], x1
st1 {v1.d}[1], [x0], x1
subs x4, x4, #2
b.ne 2b
add sp, sp, #168+16
ret
endfunc
function ff_put_vp8_epel4_v6_neon, export=1
sub x2, x2, x3, lsl #1
movrel x7, subpel_filters, -16
add x6, x7, w6, uxtw #4
ld1 {v0.8h}, [x6]
1:
ld1r {v2.2s}, [x2], x3
ld1r {v3.2s}, [x2], x3
ld1r {v4.2s}, [x2], x3
ld1r {v5.2s}, [x2], x3
ld1r {v6.2s}, [x2], x3
ld1r {v7.2s}, [x2], x3
ld1r {v28.2s}, [x2]
sub x2, x2, x3, lsl #2
ld1 {v2.s}[1], [x2], x3
ld1 {v3.s}[1], [x2], x3
ld1 {v4.s}[1], [x2], x3
ld1 {v5.s}[1], [x2], x3
ld1 {v6.s}[1], [x2], x3
ld1 {v7.s}[1], [x2], x3
ld1 {v28.s}[1], [x2]
sub x2, x2, x3, lsl #2
vp8_epel8_v6_y2 v2, v3, v2, v3, v4, v5, v6, v7, v28
st1 {v2.s}[0], [x0], x1
st1 {v3.s}[0], [x0], x1
st1 {v2.s}[1], [x0], x1
st1 {v3.s}[1], [x0], x1
subs w4, w4, #4
b.ne 1b
ret
endfunc
function ff_put_vp8_epel4_h6_neon, export=1
sub x2, x2, #2
movrel x7, subpel_filters, -16
add x5, x7, w5, uxtw #4
ld1 {v0.8h}, [x5]
1:
ld1 {v2.8b,v3.8b}, [x2], x3
vp8_epel8_h6 v2, v2, v3
st1 {v2.s}[0], [x0], x1
subs w4, w4, #1
b.ne 1b
ret
endfunc
function ff_put_vp8_epel4_h6v6_neon, export=1
sub x2, x2, x3, lsl #1
sub x2, x2, #2
movrel x7, subpel_filters, -16
add x5, x7, w5, uxtw #4
ld1 {v0.8h}, [x5]
sub sp, sp, #52
add w8, w4, #5
mov x9, sp
1:
ld1 {v2.8b,v3.8b}, [x2], x3
vp8_epel8_h6 v2, v2, v3
st1 {v2.s}[0], [x9], #4
subs w8, w8, #1
b.ne 1b
add x6, x7, w6, uxtw #4
ld1 {v0.8h}, [x6]
mov x9, sp
2:
ld1 {v2.8b,v3.8b}, [x9], #16
ld1 {v6.8b}, [x9], #8
ld1r {v28.2s}, [x9]
sub x9, x9, #16
ld1 {v4.8b,v5.8b}, [x9], #16
ld1 {v7.8b}, [x9], #8
ld1 {v28.s}[1], [x9]
sub x9, x9, #16
trn1 v1.2s, v2.2s, v4.2s
trn2 v4.2s, v2.2s, v4.2s
trn1 v2.2s, v3.2s, v5.2s
trn2 v5.2s, v3.2s, v5.2s
trn1 v3.2s, v6.2s, v7.2s
trn2 v7.2s, v6.2s, v7.2s
vp8_epel8_v6_y2 v2, v3, v1, v4, v2, v5, v3, v7, v28
st1 {v2.s}[0], [x0], x1
st1 {v3.s}[0], [x0], x1
st1 {v2.s}[1], [x0], x1
st1 {v3.s}[1], [x0], x1
subs w4, w4, #4
b.ne 2b
add sp, sp, #52
ret
endfunc
function ff_put_vp8_epel4_h4v6_neon, export=1
sub x2, x2, x3, lsl #1
sub x2, x2, #1
movrel x7, subpel_filters, -16
add x5, x7, w5, uxtw #4
ld1 {v0.8h}, [x5]
sub sp, sp, #52
add w8, w4, #5
mov x9, sp
1:
ld1 {v2.8b}, [x2], x3
vp8_epel8_h4 v2, v2, v2
st1 {v2.s}[0], [x9], #4
subs w8, w8, #1
b.ne 1b
add x6, x7, w6, uxtw #4
ld1 {v0.8h}, [x6]
mov x9, sp
2:
ld1 {v2.8b,v3.8b}, [x9], #16
ld1 {v6.8b}, [x9], #8
ld1r {v28.2s}, [x9]
sub x9, x9, #16
ld1 {v4.8b,v5.8b}, [x9], #16
ld1 {v7.8b}, [x9], #8
ld1 {v28.s}[1], [x9]
sub x9, x9, #16
trn1 v1.2s, v2.2s, v4.2s
trn2 v4.2s, v2.2s, v4.2s
trn1 v2.2s, v3.2s, v5.2s
trn2 v5.2s, v3.2s, v5.2s
trn1 v3.2s, v6.2s, v7.2s
trn2 v7.2s, v6.2s, v7.2s
vp8_epel8_v6_y2 v2, v3, v1, v4, v2, v5, v3, v7, v28
st1 {v2.s}[0], [x0], x1
st1 {v3.s}[0], [x0], x1
st1 {v2.s}[1], [x0], x1
st1 {v3.s}[1], [x0], x1
subs w4, w4, #4
b.ne 2b
add sp, sp, #52
ret
endfunc
function ff_put_vp8_epel4_h6v4_neon, export=1
sub x2, x2, x3
sub x2, x2, #2
movrel x7, subpel_filters, -16
add x5, x7, w5, uxtw #4
ld1 {v0.8h}, [x5]
sub sp, sp, #44
add w8, w4, #3
mov x9, sp
1:
ld1 {v2.8b,v3.8b}, [x2], x3
vp8_epel8_h6 v2, v2, v3
st1 {v2.s}[0], [x9], #4
subs w8, w8, #1
b.ne 1b
add x6, x7, w6, uxtw #4
ld1 {v0.8h}, [x6]
mov x9, sp
2:
ld1 {v2.8b,v3.8b}, [x9], #16
ld1r {v6.2s}, [x9]
sub x9, x9, #8
ld1 {v4.8b,v5.8b}, [x9], #16
ld1 {v6.s}[1], [x9]
sub x9, x9, #8
trn1 v1.2s, v2.2s, v4.2s
trn2 v4.2s, v2.2s, v4.2s
trn1 v2.2s, v3.2s, v5.2s
trn2 v5.2s, v3.2s, v5.2s
vp8_epel8_v4_y2 v1, v1, v4, v2, v5, v6
st1 {v1.s}[0], [x0], x1
st1 {v1.s}[2], [x0], x1
st1 {v1.s}[1], [x0], x1
st1 {v1.s}[3], [x0], x1
subs w4, w4, #4
b.ne 2b
add sp, sp, #44
ret
endfunc
function ff_put_vp8_epel4_h4_neon, export=1
sub x2, x2, #1
movrel x7, subpel_filters, -16
add x5, x7, w5, uxtw #4
ld1 {v0.8h}, [x5]
1:
ld1 {v2.8b}, [x2], x3
vp8_epel8_h4 v2, v2, v2
st1 {v2.s}[0], [x0], x1
subs w4, w4, #1
b.ne 1b
ret
endfunc
function ff_put_vp8_epel4_v4_neon, export=1
sub x2, x2, x3
movrel x7, subpel_filters, -16
add x6, x7, w6, uxtw #4
ld1 {v0.8h}, [x6]
1:
ld1r {v2.2s}, [x2], x3
ld1r {v3.2s}, [x2], x3
ld1r {v4.2s}, [x2], x3
ld1r {v5.2s}, [x2], x3
ld1r {v6.2s}, [x2]
sub x2, x2, x3, lsl #1
ld1 {v2.s}[1], [x2], x3
ld1 {v3.s}[1], [x2], x3
ld1 {v4.s}[1], [x2], x3
ld1 {v5.s}[1], [x2], x3
ld1 {v6.s}[1], [x2]
sub x2, x2, x3, lsl #1
vp8_epel8_v4_y2 v2, v2, v3, v4, v5, v6
st1 {v2.s}[0], [x0], x1
st1 {v2.s}[2], [x0], x1
st1 {v2.s}[1], [x0], x1
st1 {v2.s}[3], [x0], x1
subs w4, w4, #4
b.ne 1b
ret
endfunc
function ff_put_vp8_epel4_h4v4_neon, export=1
sub x2, x2, x3
sub x2, x2, #1
movrel x7, subpel_filters, -16
add x5, x7, w5, uxtw #4
ld1 {v0.8h}, [x5]
sub sp, sp, #44
add w8, w4, #3
mov x9, sp
1:
ld1 {v2.8b}, [x2], x3
vp8_epel8_h4 v2, v2, v3
st1 {v2.s}[0], [x9], #4
subs w8, w8, #1
b.ne 1b
add x6, x7, w6, uxtw #4
ld1 {v0.8h}, [x6]
mov x9, sp
2:
ld1 {v2.8b,v3.8b}, [x9], #16
ld1r {v6.2s}, [x9]
sub x9, x9, #8
ld1 {v4.8b,v5.8b}, [x9], #16
ld1 {v6.s}[1], [x9]
sub x9, x9, #8
trn1 v1.2s, v2.2s, v4.2s
trn2 v4.2s, v2.2s, v4.2s
trn1 v2.2s, v3.2s, v5.2s
trn2 v5.2s, v3.2s, v5.2s
vp8_epel8_v4_y2 v1, v1, v4, v2, v5, v6
st1 {v1.s}[0], [x0], x1
st1 {v1.s}[2], [x0], x1
st1 {v1.s}[1], [x0], x1
st1 {v1.s}[3], [x0], x1
subs w4, w4, #4
b.ne 2b
add sp, sp, #44
ret
endfunc
/* Bilinear MC */
function ff_put_vp8_bilin16_h_neon, export=1
mov w7, #8
dup v0.8b, w5
sub w5, w7, w5
dup v1.8b, w5
1:
subs w4, w4, #2
ld1 {v2.8b,v3.8b,v4.8b}, [x2], x3
ext v5.8b, v3.8b, v4.8b, #1
ext v4.8b, v2.8b, v3.8b, #1
umull v16.8h, v2.8b, v1.8b
umlal v16.8h, v4.8b, v0.8b
ld1 {v18.8b,v19.8b,v20.8b}, [x2], x3
umull v6.8h, v3.8b, v1.8b
umlal v6.8h, v5.8b, v0.8b
ext v21.8b, v19.8b, v20.8b, #1
ext v20.8b, v18.8b, v19.8b, #1
umull v22.8h, v18.8b, v1.8b
umlal v22.8h, v20.8b, v0.8b
umull v24.8h, v19.8b, v1.8b
umlal v24.8h, v21.8b, v0.8b
rshrn v4.8b, v16.8h, #3
rshrn2 v4.16b, v6.8h, #3
rshrn v6.8b, v22.8h, #3
rshrn2 v6.16b, v24.8h, #3
st1 {v4.16b}, [x0], x1
st1 {v6.16b}, [x0], x1
b.gt 1b
ret
endfunc
function ff_put_vp8_bilin16_v_neon, export=1
mov w7, #8
dup v0.16b, w6
sub w6, w7, w6
dup v1.16b, w6
ld1 {v2.16b}, [x2], x3
1:
subs w4, w4, #2
ld1 {v4.16b}, [x2], x3
umull v6.8h, v2.8b, v1.8b
umlal v6.8h, v4.8b, v0.8b
umull2 v16.8h, v2.16b, v1.16b
umlal2 v16.8h, v4.16b, v0.16b
ld1 {v2.16b}, [x2], x3
umull v18.8h, v4.8b, v1.8b
umlal v18.8h, v2.8b, v0.8b
umull2 v20.8h, v4.16b, v1.16b
umlal2 v20.8h, v2.16b, v0.16b
rshrn v4.8b, v6.8h, #3
rshrn2 v4.16b, v16.8h, #3
rshrn v6.8b, v18.8h, #3
rshrn2 v6.16b, v20.8h, #3
st1 {v4.16b}, [x0], x1
st1 {v6.16b}, [x0], x1
b.gt 1b
ret
endfunc
function ff_put_vp8_bilin16_hv_neon, export=1
mov w7, #8
dup v0.8b, w5 // mx
sub w5, w7, w5
dup v1.8b, w5
dup v2.16b, w6 // my
sub w6, w7, w6
dup v3.16b, w6
ld1 {v4.8b,v5.8b,v6.8b}, [x2], x3
ext v7.8b, v5.8b, v6.8b, #1
ext v6.8b, v4.8b, v5.8b, #1
umull v16.8h, v4.8b, v1.8b
umlal v16.8h, v6.8b, v0.8b
umull v18.8h, v5.8b, v1.8b
umlal v18.8h, v7.8b, v0.8b
rshrn v4.8b, v16.8h, #3
rshrn2 v4.16b, v18.8h, #3
1:
subs w4, w4, #2
ld1 {v18.8b,v19.8b,v20.8b}, [x2], x3
ext v21.8b, v19.8b, v20.8b, #1
ext v20.8b, v18.8b, v19.8b, #1
umull v22.8h, v18.8b, v1.8b
umlal v22.8h, v20.8b, v0.8b
ld1 {v26.8b,v27.8b,v28.8b}, [x2], x3
umull v24.8h, v19.8b, v1.8b
umlal v24.8h, v21.8b, v0.8b
ext v29.8b, v27.8b, v28.8b, #1
ext v28.8b, v26.8b, v27.8b, #1
umull v16.8h, v26.8b, v1.8b
umlal v16.8h, v28.8b, v0.8b
umull v18.8h, v27.8b, v1.8b
umlal v18.8h, v29.8b, v0.8b
rshrn v6.8b, v22.8h, #3
rshrn2 v6.16b, v24.8h, #3
umull v24.8h, v4.8b, v3.8b
umlal v24.8h, v6.8b, v2.8b
umull2 v30.8h, v4.16b, v3.16b
umlal2 v30.8h, v6.16b, v2.16b
rshrn v4.8b, v16.8h, #3
rshrn2 v4.16b, v18.8h, #3
umull v20.8h, v6.8b, v3.8b
umlal v20.8h, v4.8b, v2.8b
umull2 v22.8h, v6.16b, v3.16b
umlal2 v22.8h, v4.16b, v2.16b
rshrn v24.8b, v24.8h, #3
rshrn2 v24.16b, v30.8h, #3
st1 {v24.16b}, [x0], x1
rshrn v20.8b, v20.8h, #3
rshrn2 v20.16b, v22.8h, #3
st1 {v20.16b}, [x0], x1
b.gt 1b
ret
endfunc
function ff_put_vp8_bilin8_h_neon, export=1
mov w7, #8
dup v0.8b, w5
sub w5, w7, w5
dup v1.8b, w5
1:
subs w4, w4, #2
ld1 {v2.8b,v3.8b}, [x2], x3
ext v3.8b, v2.8b, v3.8b, #1
umull v4.8h, v2.8b, v1.8b
umlal v4.8h, v3.8b, v0.8b
ld1 {v6.8b,v7.8b}, [x2], x3
ext v7.8b, v6.8b, v7.8b, #1
umull v16.8h, v6.8b, v1.8b
umlal v16.8h, v7.8b, v0.8b
rshrn v4.8b, v4.8h, #3
rshrn v16.8b, v16.8h, #3
st1 {v4.8b}, [x0], x1
st1 {v16.8b}, [x0], x1
b.gt 1b
ret
endfunc
function ff_put_vp8_bilin8_v_neon, export=1
mov w7, #8
dup v0.8b, w6
sub w6, w7, w6
dup v1.8b, w6
ld1 {v2.8b}, [x2], x3
1:
subs w4, w4, #2
ld1 {v3.8b}, [x2], x3
umull v4.8h, v2.8b, v1.8b
umlal v4.8h, v3.8b, v0.8b
ld1 {v2.8b}, [x2], x3
umull v6.8h, v3.8b, v1.8b
umlal v6.8h, v2.8b, v0.8b
rshrn v4.8b, v4.8h, #3
rshrn v6.8b, v6.8h, #3
st1 {v4.8b}, [x0], x1
st1 {v6.8b}, [x0], x1
b.gt 1b
ret
endfunc
function ff_put_vp8_bilin8_hv_neon, export=1
mov w7, #8
dup v0.8b, w5 // mx
sub w5, w7, w5
dup v1.8b, w5
dup v2.8b, w6 // my
sub w6, w7, w6
dup v3.8b, w6
ld1 {v4.8b,v5.8b}, [x2], x3
ext v5.8b, v4.8b, v5.8b, #1
umull v18.8h, v4.8b, v1.8b
umlal v18.8h, v5.8b, v0.8b
rshrn v22.8b, v18.8h, #3
1:
subs w4, w4, #2
ld1 {v6.8b,v7.8b}, [x2], x3
ext v7.8b, v6.8b, v7.8b, #1
umull v16.8h, v6.8b, v1.8b
umlal v16.8h, v7.8b, v0.8b
ld1 {v4.8b,v5.8b}, [x2], x3
ext v5.8b, v4.8b, v5.8b, #1
umull v18.8h, v4.8b, v1.8b
umlal v18.8h, v5.8b, v0.8b
rshrn v16.8b, v16.8h, #3
umull v20.8h, v22.8b, v3.8b
umlal v20.8h, v16.8b, v2.8b
rshrn v22.8b, v18.8h, #3
umull v24.8h, v16.8b, v3.8b
umlal v24.8h, v22.8b, v2.8b
rshrn v20.8b, v20.8h, #3
st1 {v20.8b}, [x0], x1
rshrn v23.8b, v24.8h, #3
st1 {v23.8b}, [x0], x1
b.gt 1b
ret
endfunc
function ff_put_vp8_bilin4_h_neon, export=1
mov w7, #8
dup v0.8b, w5
sub w5, w7, w5
dup v1.8b, w5
1:
subs w4, w4, #2
ld1 {v2.8b}, [x2], x3
ext v3.8b, v2.8b, v3.8b, #1
ld1 {v6.8b}, [x2], x3
ext v7.8b, v6.8b, v7.8b, #1
trn1 v2.2s, v2.2s, v6.2s
trn1 v3.2s, v3.2s, v7.2s
umull v4.8h, v2.8b, v1.8b
umlal v4.8h, v3.8b, v0.8b
rshrn v4.8b, v4.8h, #3
st1 {v4.s}[0], [x0], x1
st1 {v4.s}[1], [x0], x1
b.gt 1b
ret
endfunc
function ff_put_vp8_bilin4_v_neon, export=1
mov w7, #8
dup v0.8b, w6
sub w6, w7, w6
dup v1.8b, w6
ld1r {v2.2s}, [x2], x3
1:
ld1r {v3.2s}, [x2]
ld1 {v2.s}[1], [x2], x3
ld1 {v3.s}[1], [x2], x3
umull v4.8h, v2.8b, v1.8b
umlal v4.8h, v3.8b, v0.8b
trn2 v2.2s, v3.2s, v2.2s
rshrn v4.8b, v4.8h, #3
st1 {v4.s}[0], [x0], x1
st1 {v4.s}[1], [x0], x1
subs w4, w4, #2
b.gt 1b
ret
endfunc
function ff_put_vp8_bilin4_hv_neon, export=1
mov w7, #8
dup v0.8b, w5 // mx
sub w5, w7, w5
dup v1.8b, w5
dup v2.8b, w6 // my
sub w6, w7, w6
dup v3.8b, w6
ld1 {v4.8b}, [x2], x3
ext v5.8b, v4.8b, v4.8b, #1
umull v18.8h, v4.8b, v1.8b
umlal v18.8h, v5.8b, v0.8b
rshrn v22.8b, v18.8h, #3
1:
subs w4, w4, #2
ld1 {v6.8b}, [x2], x3
ext v7.8b, v6.8b, v6.8b, #1
ld1 {v4.8b}, [x2], x3
ext v5.8b, v4.8b, v4.8b, #1
trn1 v6.2s, v6.2s, v4.2s
trn1 v7.2s, v7.2s, v5.2s
umull v16.8h, v6.8b, v1.8b
umlal v16.8h, v7.8b, v0.8b
rshrn v16.8b, v16.8h, #3
umull v20.8h, v16.8b, v2.8b
trn1 v22.2s, v22.2s, v16.2s
umlal v20.8h, v22.8b, v3.8b
rev64 v22.2s, v16.2s
rshrn v20.8b, v20.8h, #3
st1 {v20.s}[0], [x0], x1
st1 {v20.s}[1], [x0], x1
b.gt 1b
ret
endfunc