Source code
Revision control
Copy as Markdown
Other Tools
/*
* Copyright (c) 2023, Alliance for Open Media. All rights reserved.
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#ifndef AOM_AV1_ENCODER_ARM_PICKRST_NEON_H_
#define AOM_AV1_ENCODER_ARM_PICKRST_NEON_H_
#include <arm_neon.h>
#include "av1/common/restoration.h"
#define WIN_7 ((WIENER_WIN - 1) * 2)
#define WIN_CHROMA ((WIENER_WIN_CHROMA - 1) * 2)
// Aligned sizes for Wiener filters.
#define WIENER_WIN2_ALIGN2 ALIGN_POWER_OF_TWO(WIENER_WIN2, 2)
#define WIENER_WIN2_ALIGN3 ALIGN_POWER_OF_TWO(WIENER_WIN2, 3)
#define WIENER_WIN2_REDUCED ((WIENER_WIN_REDUCED) * (WIENER_WIN_REDUCED))
#define WIENER_WIN2_REDUCED_ALIGN2 ALIGN_POWER_OF_TWO(WIENER_WIN2_REDUCED, 2)
#define WIENER_WIN2_REDUCED_ALIGN3 ALIGN_POWER_OF_TWO(WIENER_WIN2_REDUCED, 3)
// Compute 8 values of M (cross correlation) for a single source pixel and
// accumulate.
static inline void update_M_1pixel(int32_t *M_s32, int16x4_t src_avg,
int16x8_t dgd_avg) {
int32x4_t lo = vld1q_s32(M_s32 + 0);
int32x4_t hi = vld1q_s32(M_s32 + 4);
lo = vmlal_s16(lo, vget_low_s16(dgd_avg), src_avg);
hi = vmlal_s16(hi, vget_high_s16(dgd_avg), src_avg);
vst1q_s32(M_s32 + 0, lo);
vst1q_s32(M_s32 + 4, hi);
}
// Compute 8 values of M (cross correlation) for two source pixels and
// accumulate.
static inline void update_M_2pixels(int32_t *M_s32, int16x4_t src_avg0,
int16x4_t src_avg1, int16x8_t dgd_avg0,
int16x8_t dgd_avg1) {
int32x4_t lo = vld1q_s32(M_s32 + 0);
int32x4_t hi = vld1q_s32(M_s32 + 4);
lo = vmlal_s16(lo, vget_low_s16(dgd_avg0), src_avg0);
hi = vmlal_s16(hi, vget_high_s16(dgd_avg0), src_avg0);
lo = vmlal_s16(lo, vget_low_s16(dgd_avg1), src_avg1);
hi = vmlal_s16(hi, vget_high_s16(dgd_avg1), src_avg1);
vst1q_s32(M_s32 + 0, lo);
vst1q_s32(M_s32 + 4, hi);
}
static inline void update_H_1pixel(int32_t *H_s32, const int16_t *dgd_avg,
int width, int height) {
for (int i = 0; i < height; i += 4) {
int16x4_t di = vld1_s16(dgd_avg + i);
for (int j = i; j < width; j += 4) {
int16x4_t dj = vld1_s16(dgd_avg + j);
int32x4_t h0 = vld1q_s32(H_s32 + 0 * width + j);
int32x4_t h1 = vld1q_s32(H_s32 + 1 * width + j);
int32x4_t h2 = vld1q_s32(H_s32 + 2 * width + j);
int32x4_t h3 = vld1q_s32(H_s32 + 3 * width + j);
h0 = vmlal_lane_s16(h0, dj, di, 0);
h1 = vmlal_lane_s16(h1, dj, di, 1);
h2 = vmlal_lane_s16(h2, dj, di, 2);
h3 = vmlal_lane_s16(h3, dj, di, 3);
vst1q_s32(H_s32 + 0 * width + j, h0);
vst1q_s32(H_s32 + 1 * width + j, h1);
vst1q_s32(H_s32 + 2 * width + j, h2);
vst1q_s32(H_s32 + 3 * width + j, h3);
}
H_s32 += 4 * width;
}
}
static inline void update_H_5x5_2pixels(int32_t *H_s32, const int16_t *dgd_avg0,
const int16_t *dgd_avg1) {
for (int i = 0; i < 24; i += 4) {
int16x4_t di0 = vld1_s16(dgd_avg0 + i);
int16x4_t di1 = vld1_s16(dgd_avg1 + i);
for (int j = i + 0; j < WIENER_WIN2_REDUCED_ALIGN2; j += 4) {
int16x4_t dj0 = vld1_s16(dgd_avg0 + j);
int16x4_t dj1 = vld1_s16(dgd_avg1 + j);
int32x4_t h0 = vld1q_s32(H_s32 + 0 * WIENER_WIN2_REDUCED_ALIGN2 + j);
int32x4_t h1 = vld1q_s32(H_s32 + 1 * WIENER_WIN2_REDUCED_ALIGN2 + j);
int32x4_t h2 = vld1q_s32(H_s32 + 2 * WIENER_WIN2_REDUCED_ALIGN2 + j);
int32x4_t h3 = vld1q_s32(H_s32 + 3 * WIENER_WIN2_REDUCED_ALIGN2 + j);
h0 = vmlal_lane_s16(h0, dj0, di0, 0);
h0 = vmlal_lane_s16(h0, dj1, di1, 0);
h1 = vmlal_lane_s16(h1, dj0, di0, 1);
h1 = vmlal_lane_s16(h1, dj1, di1, 1);
h2 = vmlal_lane_s16(h2, dj0, di0, 2);
h2 = vmlal_lane_s16(h2, dj1, di1, 2);
h3 = vmlal_lane_s16(h3, dj0, di0, 3);
h3 = vmlal_lane_s16(h3, dj1, di1, 3);
vst1q_s32(H_s32 + 0 * WIENER_WIN2_REDUCED_ALIGN2 + j, h0);
vst1q_s32(H_s32 + 1 * WIENER_WIN2_REDUCED_ALIGN2 + j, h1);
vst1q_s32(H_s32 + 2 * WIENER_WIN2_REDUCED_ALIGN2 + j, h2);
vst1q_s32(H_s32 + 3 * WIENER_WIN2_REDUCED_ALIGN2 + j, h3);
}
H_s32 += 4 * WIENER_WIN2_REDUCED_ALIGN2;
}
}
static inline void update_H_7x7_2pixels(int32_t *H_s32, const int16_t *dgd_avg0,
const int16_t *dgd_avg1) {
for (int i = 0; i < 48; i += 4) {
int16x4_t di0 = vld1_s16(dgd_avg0 + i);
int16x4_t di1 = vld1_s16(dgd_avg1 + i);
int32x4_t h0 = vld1q_s32(H_s32 + 0 * WIENER_WIN2_ALIGN2 + i);
int32x4_t h1 = vld1q_s32(H_s32 + 1 * WIENER_WIN2_ALIGN2 + i);
int32x4_t h2 = vld1q_s32(H_s32 + 2 * WIENER_WIN2_ALIGN2 + i);
int32x4_t h3 = vld1q_s32(H_s32 + 3 * WIENER_WIN2_ALIGN2 + i);
h0 = vmlal_lane_s16(h0, di0, di0, 0);
h0 = vmlal_lane_s16(h0, di1, di1, 0);
h1 = vmlal_lane_s16(h1, di0, di0, 1);
h1 = vmlal_lane_s16(h1, di1, di1, 1);
h2 = vmlal_lane_s16(h2, di0, di0, 2);
h2 = vmlal_lane_s16(h2, di1, di1, 2);
h3 = vmlal_lane_s16(h3, di0, di0, 3);
h3 = vmlal_lane_s16(h3, di1, di1, 3);
vst1q_s32(H_s32 + 0 * WIENER_WIN2_ALIGN2 + i, h0);
vst1q_s32(H_s32 + 1 * WIENER_WIN2_ALIGN2 + i, h1);
vst1q_s32(H_s32 + 2 * WIENER_WIN2_ALIGN2 + i, h2);
vst1q_s32(H_s32 + 3 * WIENER_WIN2_ALIGN2 + i, h3);
for (int j = i + 4; j < WIENER_WIN2_ALIGN2; j += 4) {
int16x4_t dj0 = vld1_s16(dgd_avg0 + j);
int16x4_t dj1 = vld1_s16(dgd_avg1 + j);
h0 = vld1q_s32(H_s32 + 0 * WIENER_WIN2_ALIGN2 + j);
h1 = vld1q_s32(H_s32 + 1 * WIENER_WIN2_ALIGN2 + j);
h2 = vld1q_s32(H_s32 + 2 * WIENER_WIN2_ALIGN2 + j);
h3 = vld1q_s32(H_s32 + 3 * WIENER_WIN2_ALIGN2 + j);
h0 = vmlal_lane_s16(h0, dj0, di0, 0);
h0 = vmlal_lane_s16(h0, dj1, di1, 0);
h1 = vmlal_lane_s16(h1, dj0, di0, 1);
h1 = vmlal_lane_s16(h1, dj1, di1, 1);
h2 = vmlal_lane_s16(h2, dj0, di0, 2);
h2 = vmlal_lane_s16(h2, dj1, di1, 2);
h3 = vmlal_lane_s16(h3, dj0, di0, 3);
h3 = vmlal_lane_s16(h3, dj1, di1, 3);
vst1q_s32(H_s32 + 0 * WIENER_WIN2_ALIGN2 + j, h0);
vst1q_s32(H_s32 + 1 * WIENER_WIN2_ALIGN2 + j, h1);
vst1q_s32(H_s32 + 2 * WIENER_WIN2_ALIGN2 + j, h2);
vst1q_s32(H_s32 + 3 * WIENER_WIN2_ALIGN2 + j, h3);
}
H_s32 += 4 * WIENER_WIN2_ALIGN2;
}
}
// Widen 32-bit src data and accumulate into 64-bit dst. Clear src data.
static inline void accumulate_and_clear(int64_t *dst, int32_t *src,
int length) {
do {
int32x4_t s32 = vld1q_s32(src);
vst1q_s32(src, vdupq_n_s32(0));
src += 4;
int64x2_t d_lo = vld1q_s64(dst + 0);
int64x2_t d_hi = vld1q_s64(dst + 2);
d_lo = vaddw_s32(d_lo, vget_low_s32(s32));
d_hi = vaddw_s32(d_hi, vget_high_s32(s32));
vst1q_s64(dst + 0, d_lo);
vst1q_s64(dst + 2, d_hi);
dst += 4;
length -= 4;
} while (length > 0);
}
// clang-format off
// Constant pool to act as a mask to zero n top elements in an int16x8_t vector.
// The index we load from depends on n.
static const int16_t mask_16bit[32] = {
0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
};
// clang-format on
static inline void madd_neon_pairwise(int32x4_t *sum, const int16x8_t src,
const int16x8_t dgd) {
const int32x4_t sd =
horizontal_add_2d_s32(vmull_s16(vget_low_s16(src), vget_low_s16(dgd)),
vmull_s16(vget_high_s16(src), vget_high_s16(dgd)));
*sum = vaddq_s32(*sum, sd);
}
static inline void madd_neon(int32x4_t *sum, const int16x8_t src,
const int16x8_t dgd) {
*sum = vmlal_s16(*sum, vget_low_s16(src), vget_low_s16(dgd));
*sum = vmlal_s16(*sum, vget_high_s16(src), vget_high_s16(dgd));
}
static inline void msub_neon(int32x4_t *sum, const int16x8_t src,
const int16x8_t dgd) {
*sum = vmlsl_s16(*sum, vget_low_s16(src), vget_low_s16(dgd));
*sum = vmlsl_s16(*sum, vget_high_s16(src), vget_high_s16(dgd));
}
static inline void compute_delta_step3(int32x4_t *sum0, int32x4_t *sum1,
const int16x8_t src0,
const int16x8_t src1,
const int16x8_t dgd0,
const int16x8_t dgd1) {
*sum0 = vmlsl_s16(*sum0, vget_low_s16(src0), vget_low_s16(dgd0));
*sum0 = vmlal_s16(*sum0, vget_low_s16(src1), vget_low_s16(dgd1));
*sum1 = vmlsl_s16(*sum1, vget_high_s16(src0), vget_high_s16(dgd0));
*sum1 = vmlal_s16(*sum1, vget_high_s16(src1), vget_high_s16(dgd1));
}
static inline int32x4_t hadd_four_32_neon(const int32x4_t src0,
const int32x4_t src1,
const int32x4_t src2,
const int32x4_t src3) {
int32x4_t src[4] = { src0, src1, src2, src3 };
return horizontal_add_4d_s32x4(src);
}
static inline void update_4_stats_neon(const int64_t *const src,
const int32x4_t delta,
int64_t *const dst) {
const int64x2_t s1 = vld1q_s64(src);
const int64x2_t s2 = vld1q_s64(src + 2);
const int64x2_t d1 = vaddw_s32(s1, vget_low_s32(delta));
const int64x2_t d2 = vaddw_s32(s2, vget_high_s32(delta));
vst1q_s64(dst, d1);
vst1q_s64(dst + 2, d2);
}
static inline void load_more_16_neon(const int16_t *const src,
const int32_t width,
const int16x8_t org[2], int16x8_t dst[2]) {
int16x8_t s0 = vld1q_dup_s16(src);
int16x8_t s1 = vld1q_dup_s16(src + width);
dst[0] = vextq_s16(org[0], s0, 1);
dst[1] = vextq_s16(org[1], s1, 1);
}
static inline void stats_top_win5_neon(const int16x8_t src[2],
const int16x8_t dgd[2],
const int16_t *const d,
const int32_t d_stride, int32x4_t *sum_m,
int32x4_t *sum_h) {
int16x8_t dgds[WIENER_WIN_CHROMA * 2];
load_s16_8x5(d + 0, d_stride, &dgds[0], &dgds[2], &dgds[4], &dgds[6],
&dgds[8]);
load_s16_8x5(d + 8, d_stride, &dgds[1], &dgds[3], &dgds[5], &dgds[7],
&dgds[9]);
madd_neon(&sum_m[0], src[0], dgds[0]);
madd_neon(&sum_m[0], src[1], dgds[1]);
madd_neon(&sum_m[1], src[0], dgds[2]);
madd_neon(&sum_m[1], src[1], dgds[3]);
madd_neon(&sum_m[2], src[0], dgds[4]);
madd_neon(&sum_m[2], src[1], dgds[5]);
madd_neon(&sum_m[3], src[0], dgds[6]);
madd_neon(&sum_m[3], src[1], dgds[7]);
madd_neon(&sum_m[4], src[0], dgds[8]);
madd_neon(&sum_m[4], src[1], dgds[9]);
madd_neon(&sum_h[0], dgd[0], dgds[0]);
madd_neon(&sum_h[0], dgd[1], dgds[1]);
madd_neon(&sum_h[1], dgd[0], dgds[2]);
madd_neon(&sum_h[1], dgd[1], dgds[3]);
madd_neon(&sum_h[2], dgd[0], dgds[4]);
madd_neon(&sum_h[2], dgd[1], dgds[5]);
madd_neon(&sum_h[3], dgd[0], dgds[6]);
madd_neon(&sum_h[3], dgd[1], dgds[7]);
madd_neon(&sum_h[4], dgd[0], dgds[8]);
madd_neon(&sum_h[4], dgd[1], dgds[9]);
}
static inline void stats_left_win5_neon(const int16x8_t src[2],
const int16_t *d,
const int32_t d_stride,
int32x4_t *sum) {
int16x8_t dgds[WIN_CHROMA];
load_s16_8x4(d + d_stride + 0, d_stride, &dgds[0], &dgds[2], &dgds[4],
&dgds[6]);
load_s16_8x4(d + d_stride + 8, d_stride, &dgds[1], &dgds[3], &dgds[5],
&dgds[7]);
madd_neon(&sum[0], src[0], dgds[0]);
madd_neon(&sum[0], src[1], dgds[1]);
madd_neon(&sum[1], src[0], dgds[2]);
madd_neon(&sum[1], src[1], dgds[3]);
madd_neon(&sum[2], src[0], dgds[4]);
madd_neon(&sum[2], src[1], dgds[5]);
madd_neon(&sum[3], src[0], dgds[6]);
madd_neon(&sum[3], src[1], dgds[7]);
}
static inline void derive_square_win5_neon(
const int16x8_t *d_is, const int16x8_t *d_ie, const int16x8_t *d_js,
const int16x8_t *d_je,
int32x4_t deltas[WIENER_WIN_CHROMA - 1][WIENER_WIN_CHROMA - 1]) {
msub_neon(&deltas[0][0], d_is[0], d_js[0]);
msub_neon(&deltas[0][0], d_is[1], d_js[1]);
msub_neon(&deltas[0][1], d_is[0], d_js[2]);
msub_neon(&deltas[0][1], d_is[1], d_js[3]);
msub_neon(&deltas[0][2], d_is[0], d_js[4]);
msub_neon(&deltas[0][2], d_is[1], d_js[5]);
msub_neon(&deltas[0][3], d_is[0], d_js[6]);
msub_neon(&deltas[0][3], d_is[1], d_js[7]);
msub_neon(&deltas[1][0], d_is[2], d_js[0]);
msub_neon(&deltas[1][0], d_is[3], d_js[1]);
msub_neon(&deltas[1][1], d_is[2], d_js[2]);
msub_neon(&deltas[1][1], d_is[3], d_js[3]);
msub_neon(&deltas[1][2], d_is[2], d_js[4]);
msub_neon(&deltas[1][2], d_is[3], d_js[5]);
msub_neon(&deltas[1][3], d_is[2], d_js[6]);
msub_neon(&deltas[1][3], d_is[3], d_js[7]);
msub_neon(&deltas[2][0], d_is[4], d_js[0]);
msub_neon(&deltas[2][0], d_is[5], d_js[1]);
msub_neon(&deltas[2][1], d_is[4], d_js[2]);
msub_neon(&deltas[2][1], d_is[5], d_js[3]);
msub_neon(&deltas[2][2], d_is[4], d_js[4]);
msub_neon(&deltas[2][2], d_is[5], d_js[5]);
msub_neon(&deltas[2][3], d_is[4], d_js[6]);
msub_neon(&deltas[2][3], d_is[5], d_js[7]);
msub_neon(&deltas[3][0], d_is[6], d_js[0]);
msub_neon(&deltas[3][0], d_is[7], d_js[1]);
msub_neon(&deltas[3][1], d_is[6], d_js[2]);
msub_neon(&deltas[3][1], d_is[7], d_js[3]);
msub_neon(&deltas[3][2], d_is[6], d_js[4]);
msub_neon(&deltas[3][2], d_is[7], d_js[5]);
msub_neon(&deltas[3][3], d_is[6], d_js[6]);
msub_neon(&deltas[3][3], d_is[7], d_js[7]);
madd_neon(&deltas[0][0], d_ie[0], d_je[0]);
madd_neon(&deltas[0][0], d_ie[1], d_je[1]);
madd_neon(&deltas[0][1], d_ie[0], d_je[2]);
madd_neon(&deltas[0][1], d_ie[1], d_je[3]);
madd_neon(&deltas[0][2], d_ie[0], d_je[4]);
madd_neon(&deltas[0][2], d_ie[1], d_je[5]);
madd_neon(&deltas[0][3], d_ie[0], d_je[6]);
madd_neon(&deltas[0][3], d_ie[1], d_je[7]);
madd_neon(&deltas[1][0], d_ie[2], d_je[0]);
madd_neon(&deltas[1][0], d_ie[3], d_je[1]);
madd_neon(&deltas[1][1], d_ie[2], d_je[2]);
madd_neon(&deltas[1][1], d_ie[3], d_je[3]);
madd_neon(&deltas[1][2], d_ie[2], d_je[4]);
madd_neon(&deltas[1][2], d_ie[3], d_je[5]);
madd_neon(&deltas[1][3], d_ie[2], d_je[6]);
madd_neon(&deltas[1][3], d_ie[3], d_je[7]);
madd_neon(&deltas[2][0], d_ie[4], d_je[0]);
madd_neon(&deltas[2][0], d_ie[5], d_je[1]);
madd_neon(&deltas[2][1], d_ie[4], d_je[2]);
madd_neon(&deltas[2][1], d_ie[5], d_je[3]);
madd_neon(&deltas[2][2], d_ie[4], d_je[4]);
madd_neon(&deltas[2][2], d_ie[5], d_je[5]);
madd_neon(&deltas[2][3], d_ie[4], d_je[6]);
madd_neon(&deltas[2][3], d_ie[5], d_je[7]);
madd_neon(&deltas[3][0], d_ie[6], d_je[0]);
madd_neon(&deltas[3][0], d_ie[7], d_je[1]);
madd_neon(&deltas[3][1], d_ie[6], d_je[2]);
madd_neon(&deltas[3][1], d_ie[7], d_je[3]);
madd_neon(&deltas[3][2], d_ie[6], d_je[4]);
madd_neon(&deltas[3][2], d_ie[7], d_je[5]);
madd_neon(&deltas[3][3], d_ie[6], d_je[6]);
madd_neon(&deltas[3][3], d_ie[7], d_je[7]);
}
static inline void load_square_win5_neon(const int16_t *const di,
const int16_t *const dj,
const int32_t d_stride,
const int32_t height, int16x8_t *d_is,
int16x8_t *d_ie, int16x8_t *d_js,
int16x8_t *d_je) {
load_s16_8x4(di + 0, d_stride, &d_is[0], &d_is[2], &d_is[4], &d_is[6]);
load_s16_8x4(di + 8, d_stride, &d_is[1], &d_is[3], &d_is[5], &d_is[7]);
load_s16_8x4(dj + 0, d_stride, &d_js[0], &d_js[2], &d_js[4], &d_js[6]);
load_s16_8x4(dj + 8, d_stride, &d_js[1], &d_js[3], &d_js[5], &d_js[7]);
load_s16_8x4(di + height * d_stride + 0, d_stride, &d_ie[0], &d_ie[2],
&d_ie[4], &d_ie[6]);
load_s16_8x4(di + height * d_stride + 8, d_stride, &d_ie[1], &d_ie[3],
&d_ie[5], &d_ie[7]);
load_s16_8x4(dj + height * d_stride + 0, d_stride, &d_je[0], &d_je[2],
&d_je[4], &d_je[6]);
load_s16_8x4(dj + height * d_stride + 8, d_stride, &d_je[1], &d_je[3],
&d_je[5], &d_je[7]);
}
static inline void update_5_stats_neon(const int64_t *const src,
const int32x4_t delta,
const int64_t delta4,
int64_t *const dst) {
update_4_stats_neon(src + 0, delta, dst + 0);
dst[4] = src[4] + delta4;
}
static inline void compute_delta_step3_two_lines(int32x4_t *sum,
const int16x8_t src,
const int16x8_t dgd) {
*sum = vmlsl_s16(*sum, vget_low_s16(src), vget_low_s16(dgd));
*sum = vmlal_s16(*sum, vget_high_s16(src), vget_high_s16(dgd));
}
static inline void step3_win5_neon(const int16_t *d, const int32_t d_stride,
const int32_t width, const int32_t height,
int16x8_t *ds, int32x4_t *deltas) {
int32_t y = height;
do {
ds[4] = load_unaligned_s16_4x2(d + 0 * d_stride, width);
ds[5] = load_unaligned_s16_4x2(d + 1 * d_stride, width);
compute_delta_step3_two_lines(&deltas[0], ds[0], ds[0]);
compute_delta_step3_two_lines(&deltas[1], ds[0], ds[1]);
compute_delta_step3_two_lines(&deltas[2], ds[0], ds[2]);
compute_delta_step3_two_lines(&deltas[3], ds[0], ds[3]);
compute_delta_step3_two_lines(&deltas[4], ds[0], ds[4]);
compute_delta_step3_two_lines(&deltas[0], ds[1], ds[1]);
compute_delta_step3_two_lines(&deltas[1], ds[1], ds[2]);
compute_delta_step3_two_lines(&deltas[2], ds[1], ds[3]);
compute_delta_step3_two_lines(&deltas[3], ds[1], ds[4]);
compute_delta_step3_two_lines(&deltas[4], ds[1], ds[5]);
ds[0] = ds[2];
ds[1] = ds[3];
ds[2] = ds[4];
ds[3] = ds[5];
d += 2 * d_stride;
y -= 2;
} while (y);
}
static inline void step3_win5_oneline_neon(const int16_t **const d,
const int32_t d_stride,
const int32_t width,
const int32_t height, int16x8_t *ds,
int32x4_t *deltas) {
int32_t y = height;
do {
ds[8] = vld1q_s16(*d);
ds[9] = vld1q_s16(*d + width);
compute_delta_step3(&deltas[0], &deltas[4], ds[0], ds[1], ds[0], ds[1]);
compute_delta_step3(&deltas[1], &deltas[5], ds[0], ds[1], ds[2], ds[3]);
compute_delta_step3(&deltas[2], &deltas[6], ds[0], ds[1], ds[4], ds[5]);
compute_delta_step3(&deltas[3], &deltas[7], ds[0], ds[1], ds[6], ds[7]);
compute_delta_step3(&deltas[8], &deltas[12], ds[0], ds[1], ds[8], ds[9]);
ds[0] = ds[2];
ds[1] = ds[3];
ds[2] = ds[4];
ds[3] = ds[5];
ds[4] = ds[6];
ds[5] = ds[7];
ds[6] = ds[8];
ds[7] = ds[9];
*d += d_stride;
} while (--y);
}
static inline void derive_triangle_win5_neon(const int16x8_t *d_is,
const int16x8_t *d_ie,
int32x4_t *deltas) {
msub_neon(&deltas[0], d_is[0], d_is[0]);
msub_neon(&deltas[0], d_is[1], d_is[1]);
msub_neon(&deltas[1], d_is[0], d_is[2]);
msub_neon(&deltas[1], d_is[1], d_is[3]);
msub_neon(&deltas[2], d_is[0], d_is[4]);
msub_neon(&deltas[2], d_is[1], d_is[5]);
msub_neon(&deltas[3], d_is[0], d_is[6]);
msub_neon(&deltas[3], d_is[1], d_is[7]);
msub_neon(&deltas[4], d_is[2], d_is[2]);
msub_neon(&deltas[4], d_is[3], d_is[3]);
msub_neon(&deltas[5], d_is[2], d_is[4]);
msub_neon(&deltas[5], d_is[3], d_is[5]);
msub_neon(&deltas[6], d_is[2], d_is[6]);
msub_neon(&deltas[6], d_is[3], d_is[7]);
msub_neon(&deltas[7], d_is[4], d_is[4]);
msub_neon(&deltas[7], d_is[5], d_is[5]);
msub_neon(&deltas[8], d_is[4], d_is[6]);
msub_neon(&deltas[8], d_is[5], d_is[7]);
msub_neon(&deltas[9], d_is[6], d_is[6]);
msub_neon(&deltas[9], d_is[7], d_is[7]);
madd_neon(&deltas[0], d_ie[0], d_ie[0]);
madd_neon(&deltas[0], d_ie[1], d_ie[1]);
madd_neon(&deltas[1], d_ie[0], d_ie[2]);
madd_neon(&deltas[1], d_ie[1], d_ie[3]);
madd_neon(&deltas[2], d_ie[0], d_ie[4]);
madd_neon(&deltas[2], d_ie[1], d_ie[5]);
madd_neon(&deltas[3], d_ie[0], d_ie[6]);
madd_neon(&deltas[3], d_ie[1], d_ie[7]);
madd_neon(&deltas[4], d_ie[2], d_ie[2]);
madd_neon(&deltas[4], d_ie[3], d_ie[3]);
madd_neon(&deltas[5], d_ie[2], d_ie[4]);
madd_neon(&deltas[5], d_ie[3], d_ie[5]);
madd_neon(&deltas[6], d_ie[2], d_ie[6]);
madd_neon(&deltas[6], d_ie[3], d_ie[7]);
madd_neon(&deltas[7], d_ie[4], d_ie[4]);
madd_neon(&deltas[7], d_ie[5], d_ie[5]);
madd_neon(&deltas[8], d_ie[4], d_ie[6]);
madd_neon(&deltas[8], d_ie[5], d_ie[7]);
madd_neon(&deltas[9], d_ie[6], d_ie[6]);
madd_neon(&deltas[9], d_ie[7], d_ie[7]);
}
static inline void load_triangle_win5_neon(const int16_t *const di,
const int32_t d_stride,
const int32_t height,
int16x8_t *d_is, int16x8_t *d_ie) {
load_s16_8x4(di + 0, d_stride, &d_is[0], &d_is[2], &d_is[4], &d_is[6]);
load_s16_8x4(di + 8, d_stride, &d_is[1], &d_is[3], &d_is[5], &d_is[7]);
load_s16_8x4(di + height * d_stride + 0, d_stride, &d_ie[0], &d_ie[2],
&d_ie[4], &d_ie[6]);
load_s16_8x4(di + height * d_stride + 8, d_stride, &d_ie[1], &d_ie[3],
&d_ie[5], &d_ie[7]);
}
static inline void sub_deltas_step4(int16x8_t *A, int16x8_t *B,
int32x4_t *deltas) {
deltas[0] = vmlsl_s16(deltas[0], vget_low_s16(A[0]), vget_low_s16(B[0]));
deltas[0] = vmlsl_s16(deltas[0], vget_high_s16(A[0]), vget_high_s16(B[0]));
deltas[1] = vmlsl_s16(deltas[1], vget_low_s16(A[0]), vget_low_s16(B[1]));
deltas[1] = vmlsl_s16(deltas[1], vget_high_s16(A[0]), vget_high_s16(B[1]));
deltas[2] = vmlsl_s16(deltas[2], vget_low_s16(A[0]), vget_low_s16(B[2]));
deltas[2] = vmlsl_s16(deltas[2], vget_high_s16(A[0]), vget_high_s16(B[2]));
deltas[3] = vmlsl_s16(deltas[3], vget_low_s16(A[0]), vget_low_s16(B[3]));
deltas[3] = vmlsl_s16(deltas[3], vget_high_s16(A[0]), vget_high_s16(B[3]));
deltas[4] = vmlsl_s16(deltas[4], vget_low_s16(A[0]), vget_low_s16(B[4]));
deltas[4] = vmlsl_s16(deltas[4], vget_high_s16(A[0]), vget_high_s16(B[4]));
deltas[5] = vmlsl_s16(deltas[5], vget_low_s16(A[1]), vget_low_s16(B[0]));
deltas[5] = vmlsl_s16(deltas[5], vget_high_s16(A[1]), vget_high_s16(B[0]));
deltas[6] = vmlsl_s16(deltas[6], vget_low_s16(A[2]), vget_low_s16(B[0]));
deltas[6] = vmlsl_s16(deltas[6], vget_high_s16(A[2]), vget_high_s16(B[0]));
deltas[7] = vmlsl_s16(deltas[7], vget_low_s16(A[3]), vget_low_s16(B[0]));
deltas[7] = vmlsl_s16(deltas[7], vget_high_s16(A[3]), vget_high_s16(B[0]));
deltas[8] = vmlsl_s16(deltas[8], vget_low_s16(A[4]), vget_low_s16(B[0]));
deltas[8] = vmlsl_s16(deltas[8], vget_high_s16(A[4]), vget_high_s16(B[0]));
}
static inline void add_deltas_step4(int16x8_t *A, int16x8_t *B,
int32x4_t *deltas) {
deltas[0] = vmlal_s16(deltas[0], vget_low_s16(A[0]), vget_low_s16(B[0]));
deltas[0] = vmlal_s16(deltas[0], vget_high_s16(A[0]), vget_high_s16(B[0]));
deltas[1] = vmlal_s16(deltas[1], vget_low_s16(A[0]), vget_low_s16(B[1]));
deltas[1] = vmlal_s16(deltas[1], vget_high_s16(A[0]), vget_high_s16(B[1]));
deltas[2] = vmlal_s16(deltas[2], vget_low_s16(A[0]), vget_low_s16(B[2]));
deltas[2] = vmlal_s16(deltas[2], vget_high_s16(A[0]), vget_high_s16(B[2]));
deltas[3] = vmlal_s16(deltas[3], vget_low_s16(A[0]), vget_low_s16(B[3]));
deltas[3] = vmlal_s16(deltas[3], vget_high_s16(A[0]), vget_high_s16(B[3]));
deltas[4] = vmlal_s16(deltas[4], vget_low_s16(A[0]), vget_low_s16(B[4]));
deltas[4] = vmlal_s16(deltas[4], vget_high_s16(A[0]), vget_high_s16(B[4]));
deltas[5] = vmlal_s16(deltas[5], vget_low_s16(A[1]), vget_low_s16(B[0]));
deltas[5] = vmlal_s16(deltas[5], vget_high_s16(A[1]), vget_high_s16(B[0]));
deltas[6] = vmlal_s16(deltas[6], vget_low_s16(A[2]), vget_low_s16(B[0]));
deltas[6] = vmlal_s16(deltas[6], vget_high_s16(A[2]), vget_high_s16(B[0]));
deltas[7] = vmlal_s16(deltas[7], vget_low_s16(A[3]), vget_low_s16(B[0]));
deltas[7] = vmlal_s16(deltas[7], vget_high_s16(A[3]), vget_high_s16(B[0]));
deltas[8] = vmlal_s16(deltas[8], vget_low_s16(A[4]), vget_low_s16(B[0]));
deltas[8] = vmlal_s16(deltas[8], vget_high_s16(A[4]), vget_high_s16(B[0]));
}
static inline void stats_top_win7_neon(const int16x8_t src[2],
const int16x8_t dgd[2],
const int16_t *const d,
const int32_t d_stride, int32x4_t *sum_m,
int32x4_t *sum_h) {
int16x8_t dgds[WIENER_WIN * 2];
load_s16_8x7(d + 0, d_stride, &dgds[0], &dgds[2], &dgds[4], &dgds[6],
&dgds[8], &dgds[10], &dgds[12]);
load_s16_8x7(d + 8, d_stride, &dgds[1], &dgds[3], &dgds[5], &dgds[7],
&dgds[9], &dgds[11], &dgds[13]);
madd_neon(&sum_m[0], src[0], dgds[0]);
madd_neon(&sum_m[0], src[1], dgds[1]);
madd_neon(&sum_m[1], src[0], dgds[2]);
madd_neon(&sum_m[1], src[1], dgds[3]);
madd_neon(&sum_m[2], src[0], dgds[4]);
madd_neon(&sum_m[2], src[1], dgds[5]);
madd_neon(&sum_m[3], src[0], dgds[6]);
madd_neon(&sum_m[3], src[1], dgds[7]);
madd_neon(&sum_m[4], src[0], dgds[8]);
madd_neon(&sum_m[4], src[1], dgds[9]);
madd_neon(&sum_m[5], src[0], dgds[10]);
madd_neon(&sum_m[5], src[1], dgds[11]);
madd_neon(&sum_m[6], src[0], dgds[12]);
madd_neon(&sum_m[6], src[1], dgds[13]);
madd_neon(&sum_h[0], dgd[0], dgds[0]);
madd_neon(&sum_h[0], dgd[1], dgds[1]);
madd_neon(&sum_h[1], dgd[0], dgds[2]);
madd_neon(&sum_h[1], dgd[1], dgds[3]);
madd_neon(&sum_h[2], dgd[0], dgds[4]);
madd_neon(&sum_h[2], dgd[1], dgds[5]);
madd_neon(&sum_h[3], dgd[0], dgds[6]);
madd_neon(&sum_h[3], dgd[1], dgds[7]);
madd_neon(&sum_h[4], dgd[0], dgds[8]);
madd_neon(&sum_h[4], dgd[1], dgds[9]);
madd_neon(&sum_h[5], dgd[0], dgds[10]);
madd_neon(&sum_h[5], dgd[1], dgds[11]);
madd_neon(&sum_h[6], dgd[0], dgds[12]);
madd_neon(&sum_h[6], dgd[1], dgds[13]);
}
static inline void derive_square_win7_neon(const int16x8_t *d_is,
const int16x8_t *d_ie,
const int16x8_t *d_js,
const int16x8_t *d_je,
int32x4_t deltas[][WIN_7]) {
msub_neon(&deltas[0][0], d_is[0], d_js[0]);
msub_neon(&deltas[0][0], d_is[1], d_js[1]);
msub_neon(&deltas[0][1], d_is[0], d_js[2]);
msub_neon(&deltas[0][1], d_is[1], d_js[3]);
msub_neon(&deltas[0][2], d_is[0], d_js[4]);
msub_neon(&deltas[0][2], d_is[1], d_js[5]);
msub_neon(&deltas[0][3], d_is[0], d_js[6]);
msub_neon(&deltas[0][3], d_is[1], d_js[7]);
msub_neon(&deltas[0][4], d_is[0], d_js[8]);
msub_neon(&deltas[0][4], d_is[1], d_js[9]);
msub_neon(&deltas[0][5], d_is[0], d_js[10]);
msub_neon(&deltas[0][5], d_is[1], d_js[11]);
msub_neon(&deltas[1][0], d_is[2], d_js[0]);
msub_neon(&deltas[1][0], d_is[3], d_js[1]);
msub_neon(&deltas[1][1], d_is[2], d_js[2]);
msub_neon(&deltas[1][1], d_is[3], d_js[3]);
msub_neon(&deltas[1][2], d_is[2], d_js[4]);
msub_neon(&deltas[1][2], d_is[3], d_js[5]);
msub_neon(&deltas[1][3], d_is[2], d_js[6]);
msub_neon(&deltas[1][3], d_is[3], d_js[7]);
msub_neon(&deltas[1][4], d_is[2], d_js[8]);
msub_neon(&deltas[1][4], d_is[3], d_js[9]);
msub_neon(&deltas[1][5], d_is[2], d_js[10]);
msub_neon(&deltas[1][5], d_is[3], d_js[11]);
msub_neon(&deltas[2][0], d_is[4], d_js[0]);
msub_neon(&deltas[2][0], d_is[5], d_js[1]);
msub_neon(&deltas[2][1], d_is[4], d_js[2]);
msub_neon(&deltas[2][1], d_is[5], d_js[3]);
msub_neon(&deltas[2][2], d_is[4], d_js[4]);
msub_neon(&deltas[2][2], d_is[5], d_js[5]);
msub_neon(&deltas[2][3], d_is[4], d_js[6]);
msub_neon(&deltas[2][3], d_is[5], d_js[7]);
msub_neon(&deltas[2][4], d_is[4], d_js[8]);
msub_neon(&deltas[2][4], d_is[5], d_js[9]);
msub_neon(&deltas[2][5], d_is[4], d_js[10]);
msub_neon(&deltas[2][5], d_is[5], d_js[11]);
msub_neon(&deltas[3][0], d_is[6], d_js[0]);
msub_neon(&deltas[3][0], d_is[7], d_js[1]);
msub_neon(&deltas[3][1], d_is[6], d_js[2]);
msub_neon(&deltas[3][1], d_is[7], d_js[3]);
msub_neon(&deltas[3][2], d_is[6], d_js[4]);
msub_neon(&deltas[3][2], d_is[7], d_js[5]);
msub_neon(&deltas[3][3], d_is[6], d_js[6]);
msub_neon(&deltas[3][3], d_is[7], d_js[7]);
msub_neon(&deltas[3][4], d_is[6], d_js[8]);
msub_neon(&deltas[3][4], d_is[7], d_js[9]);
msub_neon(&deltas[3][5], d_is[6], d_js[10]);
msub_neon(&deltas[3][5], d_is[7], d_js[11]);
msub_neon(&deltas[4][0], d_is[8], d_js[0]);
msub_neon(&deltas[4][0], d_is[9], d_js[1]);
msub_neon(&deltas[4][1], d_is[8], d_js[2]);
msub_neon(&deltas[4][1], d_is[9], d_js[3]);
msub_neon(&deltas[4][2], d_is[8], d_js[4]);
msub_neon(&deltas[4][2], d_is[9], d_js[5]);
msub_neon(&deltas[4][3], d_is[8], d_js[6]);
msub_neon(&deltas[4][3], d_is[9], d_js[7]);
msub_neon(&deltas[4][4], d_is[8], d_js[8]);
msub_neon(&deltas[4][4], d_is[9], d_js[9]);
msub_neon(&deltas[4][5], d_is[8], d_js[10]);
msub_neon(&deltas[4][5], d_is[9], d_js[11]);
msub_neon(&deltas[5][0], d_is[10], d_js[0]);
msub_neon(&deltas[5][0], d_is[11], d_js[1]);
msub_neon(&deltas[5][1], d_is[10], d_js[2]);
msub_neon(&deltas[5][1], d_is[11], d_js[3]);
msub_neon(&deltas[5][2], d_is[10], d_js[4]);
msub_neon(&deltas[5][2], d_is[11], d_js[5]);
msub_neon(&deltas[5][3], d_is[10], d_js[6]);
msub_neon(&deltas[5][3], d_is[11], d_js[7]);
msub_neon(&deltas[5][4], d_is[10], d_js[8]);
msub_neon(&deltas[5][4], d_is[11], d_js[9]);
msub_neon(&deltas[5][5], d_is[10], d_js[10]);
msub_neon(&deltas[5][5], d_is[11], d_js[11]);
madd_neon(&deltas[0][0], d_ie[0], d_je[0]);
madd_neon(&deltas[0][0], d_ie[1], d_je[1]);
madd_neon(&deltas[0][1], d_ie[0], d_je[2]);
madd_neon(&deltas[0][1], d_ie[1], d_je[3]);
madd_neon(&deltas[0][2], d_ie[0], d_je[4]);
madd_neon(&deltas[0][2], d_ie[1], d_je[5]);
madd_neon(&deltas[0][3], d_ie[0], d_je[6]);
madd_neon(&deltas[0][3], d_ie[1], d_je[7]);
madd_neon(&deltas[0][4], d_ie[0], d_je[8]);
madd_neon(&deltas[0][4], d_ie[1], d_je[9]);
madd_neon(&deltas[0][5], d_ie[0], d_je[10]);
madd_neon(&deltas[0][5], d_ie[1], d_je[11]);
madd_neon(&deltas[1][0], d_ie[2], d_je[0]);
madd_neon(&deltas[1][0], d_ie[3], d_je[1]);
madd_neon(&deltas[1][1], d_ie[2], d_je[2]);
madd_neon(&deltas[1][1], d_ie[3], d_je[3]);
madd_neon(&deltas[1][2], d_ie[2], d_je[4]);
madd_neon(&deltas[1][2], d_ie[3], d_je[5]);
madd_neon(&deltas[1][3], d_ie[2], d_je[6]);
madd_neon(&deltas[1][3], d_ie[3], d_je[7]);
madd_neon(&deltas[1][4], d_ie[2], d_je[8]);
madd_neon(&deltas[1][4], d_ie[3], d_je[9]);
madd_neon(&deltas[1][5], d_ie[2], d_je[10]);
madd_neon(&deltas[1][5], d_ie[3], d_je[11]);
madd_neon(&deltas[2][0], d_ie[4], d_je[0]);
madd_neon(&deltas[2][0], d_ie[5], d_je[1]);
madd_neon(&deltas[2][1], d_ie[4], d_je[2]);
madd_neon(&deltas[2][1], d_ie[5], d_je[3]);
madd_neon(&deltas[2][2], d_ie[4], d_je[4]);
madd_neon(&deltas[2][2], d_ie[5], d_je[5]);
madd_neon(&deltas[2][3], d_ie[4], d_je[6]);
madd_neon(&deltas[2][3], d_ie[5], d_je[7]);
madd_neon(&deltas[2][4], d_ie[4], d_je[8]);
madd_neon(&deltas[2][4], d_ie[5], d_je[9]);
madd_neon(&deltas[2][5], d_ie[4], d_je[10]);
madd_neon(&deltas[2][5], d_ie[5], d_je[11]);
madd_neon(&deltas[3][0], d_ie[6], d_je[0]);
madd_neon(&deltas[3][0], d_ie[7], d_je[1]);
madd_neon(&deltas[3][1], d_ie[6], d_je[2]);
madd_neon(&deltas[3][1], d_ie[7], d_je[3]);
madd_neon(&deltas[3][2], d_ie[6], d_je[4]);
madd_neon(&deltas[3][2], d_ie[7], d_je[5]);
madd_neon(&deltas[3][3], d_ie[6], d_je[6]);
madd_neon(&deltas[3][3], d_ie[7], d_je[7]);
madd_neon(&deltas[3][4], d_ie[6], d_je[8]);
madd_neon(&deltas[3][4], d_ie[7], d_je[9]);
madd_neon(&deltas[3][5], d_ie[6], d_je[10]);
madd_neon(&deltas[3][5], d_ie[7], d_je[11]);
madd_neon(&deltas[4][0], d_ie[8], d_je[0]);
madd_neon(&deltas[4][0], d_ie[9], d_je[1]);
madd_neon(&deltas[4][1], d_ie[8], d_je[2]);
madd_neon(&deltas[4][1], d_ie[9], d_je[3]);
madd_neon(&deltas[4][2], d_ie[8], d_je[4]);
madd_neon(&deltas[4][2], d_ie[9], d_je[5]);
madd_neon(&deltas[4][3], d_ie[8], d_je[6]);
madd_neon(&deltas[4][3], d_ie[9], d_je[7]);
madd_neon(&deltas[4][4], d_ie[8], d_je[8]);
madd_neon(&deltas[4][4], d_ie[9], d_je[9]);
madd_neon(&deltas[4][5], d_ie[8], d_je[10]);
madd_neon(&deltas[4][5], d_ie[9], d_je[11]);
madd_neon(&deltas[5][0], d_ie[10], d_je[0]);
madd_neon(&deltas[5][0], d_ie[11], d_je[1]);
madd_neon(&deltas[5][1], d_ie[10], d_je[2]);
madd_neon(&deltas[5][1], d_ie[11], d_je[3]);
madd_neon(&deltas[5][2], d_ie[10], d_je[4]);
madd_neon(&deltas[5][2], d_ie[11], d_je[5]);
madd_neon(&deltas[5][3], d_ie[10], d_je[6]);
madd_neon(&deltas[5][3], d_ie[11], d_je[7]);
madd_neon(&deltas[5][4], d_ie[10], d_je[8]);
madd_neon(&deltas[5][4], d_ie[11], d_je[9]);
madd_neon(&deltas[5][5], d_ie[10], d_je[10]);
madd_neon(&deltas[5][5], d_ie[11], d_je[11]);
}
static inline void update_8_stats_neon(const int64_t *const src,
const int32x4_t delta0,
const int32x4_t delta1,
int64_t *const dst) {
update_4_stats_neon(src + 0, delta0, dst + 0);
update_4_stats_neon(src + 4, delta1, dst + 4);
}
static inline void load_square_win7_neon(const int16_t *const di,
const int16_t *const dj,
const int32_t d_stride,
const int32_t height, int16x8_t *d_is,
int16x8_t *d_ie, int16x8_t *d_js,
int16x8_t *d_je) {
load_s16_8x6(di + 0, d_stride, &d_is[0], &d_is[2], &d_is[4], &d_is[6],
&d_is[8], &d_is[10]);
load_s16_8x6(di + 8, d_stride, &d_is[1], &d_is[3], &d_is[5], &d_is[7],
&d_is[9], &d_is[11]);
load_s16_8x6(dj + 0, d_stride, &d_js[0], &d_js[2], &d_js[4], &d_js[6],
&d_js[8], &d_js[10]);
load_s16_8x6(dj + 8, d_stride, &d_js[1], &d_js[3], &d_js[5], &d_js[7],
&d_js[9], &d_js[11]);
load_s16_8x6(di + height * d_stride + 0, d_stride, &d_ie[0], &d_ie[2],
&d_ie[4], &d_ie[6], &d_ie[8], &d_ie[10]);
load_s16_8x6(di + height * d_stride + 8, d_stride, &d_ie[1], &d_ie[3],
&d_ie[5], &d_ie[7], &d_ie[9], &d_ie[11]);
load_s16_8x6(dj + height * d_stride + 0, d_stride, &d_je[0], &d_je[2],
&d_je[4], &d_je[6], &d_je[8], &d_je[10]);
load_s16_8x6(dj + height * d_stride + 8, d_stride, &d_je[1], &d_je[3],
&d_je[5], &d_je[7], &d_je[9], &d_je[11]);
}
static inline void load_triangle_win7_neon(const int16_t *const di,
const int32_t d_stride,
const int32_t height,
int16x8_t *d_is, int16x8_t *d_ie) {
load_s16_8x6(di, d_stride, &d_is[0], &d_is[2], &d_is[4], &d_is[6], &d_is[8],
&d_is[10]);
load_s16_8x6(di + 8, d_stride, &d_is[1], &d_is[3], &d_is[5], &d_is[7],
&d_is[9], &d_is[11]);
load_s16_8x6(di + height * d_stride, d_stride, &d_ie[0], &d_ie[2], &d_ie[4],
&d_ie[6], &d_ie[8], &d_ie[10]);
load_s16_8x6(di + height * d_stride + 8, d_stride, &d_ie[1], &d_ie[3],
&d_ie[5], &d_ie[7], &d_ie[9], &d_ie[11]);
}
static inline void stats_left_win7_neon(const int16x8_t src[2],
const int16_t *d,
const int32_t d_stride,
int32x4_t *sum) {
int16x8_t dgds[WIN_7];
load_s16_8x6(d + d_stride + 0, d_stride, &dgds[0], &dgds[2], &dgds[4],
&dgds[6], &dgds[8], &dgds[10]);
load_s16_8x6(d + d_stride + 8, d_stride, &dgds[1], &dgds[3], &dgds[5],
&dgds[7], &dgds[9], &dgds[11]);
madd_neon(&sum[0], src[0], dgds[0]);
madd_neon(&sum[0], src[1], dgds[1]);
madd_neon(&sum[1], src[0], dgds[2]);
madd_neon(&sum[1], src[1], dgds[3]);
madd_neon(&sum[2], src[0], dgds[4]);
madd_neon(&sum[2], src[1], dgds[5]);
madd_neon(&sum[3], src[0], dgds[6]);
madd_neon(&sum[3], src[1], dgds[7]);
madd_neon(&sum[4], src[0], dgds[8]);
madd_neon(&sum[4], src[1], dgds[9]);
madd_neon(&sum[5], src[0], dgds[10]);
madd_neon(&sum[5], src[1], dgds[11]);
}
static inline void step3_win7_neon(const int16_t *d, const int32_t d_stride,
const int32_t width, const int32_t height,
int16x8_t *ds, int32x4_t *deltas) {
int32_t y = height;
do {
ds[12] = vld1q_s16(d);
ds[13] = vld1q_s16(d + width);
compute_delta_step3(&deltas[0], &deltas[4], ds[0], ds[1], ds[0], ds[1]);
compute_delta_step3(&deltas[1], &deltas[5], ds[0], ds[1], ds[2], ds[3]);
compute_delta_step3(&deltas[2], &deltas[6], ds[0], ds[1], ds[4], ds[5]);
compute_delta_step3(&deltas[3], &deltas[7], ds[0], ds[1], ds[6], ds[7]);
compute_delta_step3(&deltas[8], &deltas[12], ds[0], ds[1], ds[8], ds[9]);
compute_delta_step3(&deltas[9], &deltas[13], ds[0], ds[1], ds[10], ds[11]);
compute_delta_step3(&deltas[10], &deltas[14], ds[0], ds[1], ds[12], ds[13]);
ds[0] = ds[2];
ds[1] = ds[3];
ds[2] = ds[4];
ds[3] = ds[5];
ds[4] = ds[6];
ds[5] = ds[7];
ds[6] = ds[8];
ds[7] = ds[9];
ds[8] = ds[10];
ds[9] = ds[11];
ds[10] = ds[12];
ds[11] = ds[13];
d += d_stride;
} while (--y);
}
static inline void derive_triangle_win7_neon(const int16x8_t *d_is,
const int16x8_t *d_ie,
int32x4_t *deltas) {
msub_neon(&deltas[0], d_is[0], d_is[0]);
msub_neon(&deltas[0], d_is[1], d_is[1]);
msub_neon(&deltas[1], d_is[0], d_is[2]);
msub_neon(&deltas[1], d_is[1], d_is[3]);
msub_neon(&deltas[2], d_is[0], d_is[4]);
msub_neon(&deltas[2], d_is[1], d_is[5]);
msub_neon(&deltas[3], d_is[0], d_is[6]);
msub_neon(&deltas[3], d_is[1], d_is[7]);
msub_neon(&deltas[4], d_is[0], d_is[8]);
msub_neon(&deltas[4], d_is[1], d_is[9]);
msub_neon(&deltas[5], d_is[0], d_is[10]);
msub_neon(&deltas[5], d_is[1], d_is[11]);
msub_neon(&deltas[6], d_is[2], d_is[2]);
msub_neon(&deltas[6], d_is[3], d_is[3]);
msub_neon(&deltas[7], d_is[2], d_is[4]);
msub_neon(&deltas[7], d_is[3], d_is[5]);
msub_neon(&deltas[8], d_is[2], d_is[6]);
msub_neon(&deltas[8], d_is[3], d_is[7]);
msub_neon(&deltas[9], d_is[2], d_is[8]);
msub_neon(&deltas[9], d_is[3], d_is[9]);
msub_neon(&deltas[10], d_is[2], d_is[10]);
msub_neon(&deltas[10], d_is[3], d_is[11]);
msub_neon(&deltas[11], d_is[4], d_is[4]);
msub_neon(&deltas[11], d_is[5], d_is[5]);
msub_neon(&deltas[12], d_is[4], d_is[6]);
msub_neon(&deltas[12], d_is[5], d_is[7]);
msub_neon(&deltas[13], d_is[4], d_is[8]);
msub_neon(&deltas[13], d_is[5], d_is[9]);
msub_neon(&deltas[14], d_is[4], d_is[10]);
msub_neon(&deltas[14], d_is[5], d_is[11]);
msub_neon(&deltas[15], d_is[6], d_is[6]);
msub_neon(&deltas[15], d_is[7], d_is[7]);
msub_neon(&deltas[16], d_is[6], d_is[8]);
msub_neon(&deltas[16], d_is[7], d_is[9]);
msub_neon(&deltas[17], d_is[6], d_is[10]);
msub_neon(&deltas[17], d_is[7], d_is[11]);
msub_neon(&deltas[18], d_is[8], d_is[8]);
msub_neon(&deltas[18], d_is[9], d_is[9]);
msub_neon(&deltas[19], d_is[8], d_is[10]);
msub_neon(&deltas[19], d_is[9], d_is[11]);
msub_neon(&deltas[20], d_is[10], d_is[10]);
msub_neon(&deltas[20], d_is[11], d_is[11]);
madd_neon(&deltas[0], d_ie[0], d_ie[0]);
madd_neon(&deltas[0], d_ie[1], d_ie[1]);
madd_neon(&deltas[1], d_ie[0], d_ie[2]);
madd_neon(&deltas[1], d_ie[1], d_ie[3]);
madd_neon(&deltas[2], d_ie[0], d_ie[4]);
madd_neon(&deltas[2], d_ie[1], d_ie[5]);
madd_neon(&deltas[3], d_ie[0], d_ie[6]);
madd_neon(&deltas[3], d_ie[1], d_ie[7]);
madd_neon(&deltas[4], d_ie[0], d_ie[8]);
madd_neon(&deltas[4], d_ie[1], d_ie[9]);
madd_neon(&deltas[5], d_ie[0], d_ie[10]);
madd_neon(&deltas[5], d_ie[1], d_ie[11]);
madd_neon(&deltas[6], d_ie[2], d_ie[2]);
madd_neon(&deltas[6], d_ie[3], d_ie[3]);
madd_neon(&deltas[7], d_ie[2], d_ie[4]);
madd_neon(&deltas[7], d_ie[3], d_ie[5]);
madd_neon(&deltas[8], d_ie[2], d_ie[6]);
madd_neon(&deltas[8], d_ie[3], d_ie[7]);
madd_neon(&deltas[9], d_ie[2], d_ie[8]);
madd_neon(&deltas[9], d_ie[3], d_ie[9]);
madd_neon(&deltas[10], d_ie[2], d_ie[10]);
madd_neon(&deltas[10], d_ie[3], d_ie[11]);
madd_neon(&deltas[11], d_ie[4], d_ie[4]);
madd_neon(&deltas[11], d_ie[5], d_ie[5]);
madd_neon(&deltas[12], d_ie[4], d_ie[6]);
madd_neon(&deltas[12], d_ie[5], d_ie[7]);
madd_neon(&deltas[13], d_ie[4], d_ie[8]);
madd_neon(&deltas[13], d_ie[5], d_ie[9]);
madd_neon(&deltas[14], d_ie[4], d_ie[10]);
madd_neon(&deltas[14], d_ie[5], d_ie[11]);
madd_neon(&deltas[15], d_ie[6], d_ie[6]);
madd_neon(&deltas[15], d_ie[7], d_ie[7]);
madd_neon(&deltas[16], d_ie[6], d_ie[8]);
madd_neon(&deltas[16], d_ie[7], d_ie[9]);
madd_neon(&deltas[17], d_ie[6], d_ie[10]);
madd_neon(&deltas[17], d_ie[7], d_ie[11]);
madd_neon(&deltas[18], d_ie[8], d_ie[8]);
madd_neon(&deltas[18], d_ie[9], d_ie[9]);
madd_neon(&deltas[19], d_ie[8], d_ie[10]);
madd_neon(&deltas[19], d_ie[9], d_ie[11]);
madd_neon(&deltas[20], d_ie[10], d_ie[10]);
madd_neon(&deltas[20], d_ie[11], d_ie[11]);
}
static inline void diagonal_copy_stats_neon(const int32_t wiener_win2,
int64_t *const H) {
for (int32_t i = 0; i < wiener_win2 - 1; i += 4) {
int64x2_t in[8], out[8];
in[0] = vld1q_s64(H + (i + 0) * wiener_win2 + i + 1);
in[1] = vld1q_s64(H + (i + 0) * wiener_win2 + i + 3);
in[2] = vld1q_s64(H + (i + 1) * wiener_win2 + i + 1);
in[3] = vld1q_s64(H + (i + 1) * wiener_win2 + i + 3);
in[4] = vld1q_s64(H + (i + 2) * wiener_win2 + i + 1);
in[5] = vld1q_s64(H + (i + 2) * wiener_win2 + i + 3);
in[6] = vld1q_s64(H + (i + 3) * wiener_win2 + i + 1);
in[7] = vld1q_s64(H + (i + 3) * wiener_win2 + i + 3);
transpose_arrays_s64_4x4(in, out);
vst1_s64(H + (i + 1) * wiener_win2 + i, vget_low_s64(out[0]));
vst1q_s64(H + (i + 2) * wiener_win2 + i, out[2]);
vst1q_s64(H + (i + 3) * wiener_win2 + i, out[4]);
vst1q_s64(H + (i + 3) * wiener_win2 + i + 2, out[5]);
vst1q_s64(H + (i + 4) * wiener_win2 + i, out[6]);
vst1q_s64(H + (i + 4) * wiener_win2 + i + 2, out[7]);
for (int32_t j = i + 5; j < wiener_win2; j += 4) {
in[0] = vld1q_s64(H + (i + 0) * wiener_win2 + j);
in[1] = vld1q_s64(H + (i + 0) * wiener_win2 + j + 2);
in[2] = vld1q_s64(H + (i + 1) * wiener_win2 + j);
in[3] = vld1q_s64(H + (i + 1) * wiener_win2 + j + 2);
in[4] = vld1q_s64(H + (i + 2) * wiener_win2 + j);
in[5] = vld1q_s64(H + (i + 2) * wiener_win2 + j + 2);
in[6] = vld1q_s64(H + (i + 3) * wiener_win2 + j);
in[7] = vld1q_s64(H + (i + 3) * wiener_win2 + j + 2);
transpose_arrays_s64_4x4(in, out);
vst1q_s64(H + (j + 0) * wiener_win2 + i, out[0]);
vst1q_s64(H + (j + 0) * wiener_win2 + i + 2, out[1]);
vst1q_s64(H + (j + 1) * wiener_win2 + i, out[2]);
vst1q_s64(H + (j + 1) * wiener_win2 + i + 2, out[3]);
vst1q_s64(H + (j + 2) * wiener_win2 + i, out[4]);
vst1q_s64(H + (j + 2) * wiener_win2 + i + 2, out[5]);
vst1q_s64(H + (j + 3) * wiener_win2 + i, out[6]);
vst1q_s64(H + (j + 3) * wiener_win2 + i + 2, out[7]);
}
}
}
static inline int64x2_t div4_neon(const int64x2_t src) {
#if AOM_ARCH_AARCH64
uint64x2_t sign = vcltzq_s64(src);
int64x2_t abs = vabsq_s64(src);
// divide by 4
abs = vshrq_n_s64(abs, 2);
// re-apply sign
return vbslq_s64(sign, vnegq_s64(abs), abs);
#else
int64x2_t sign = vshrq_n_s64(src, 63);
int64x2_t abs = vsubq_s64(veorq_s64(src, sign), sign);
// divide by 4
abs = vshrq_n_s64(abs, 2);
// re-apply sign
return vsubq_s64(veorq_s64(abs, sign), sign);
#endif // AOM_ARCH_AARCH64
}
static inline void div4_4x4_neon(const int32_t wiener_win2, int64_t *const H,
int64x2_t out[8]) {
out[0] = vld1q_s64(H + 0 * wiener_win2 + 0);
out[1] = vld1q_s64(H + 0 * wiener_win2 + 2);
out[2] = vld1q_s64(H + 1 * wiener_win2 + 0);
out[3] = vld1q_s64(H + 1 * wiener_win2 + 2);
out[4] = vld1q_s64(H + 2 * wiener_win2 + 0);
out[5] = vld1q_s64(H + 2 * wiener_win2 + 2);
out[6] = vld1q_s64(H + 3 * wiener_win2 + 0);
out[7] = vld1q_s64(H + 3 * wiener_win2 + 2);
out[0] = div4_neon(out[0]);
out[1] = div4_neon(out[1]);
out[2] = div4_neon(out[2]);
out[3] = div4_neon(out[3]);
out[4] = div4_neon(out[4]);
out[5] = div4_neon(out[5]);
out[6] = div4_neon(out[6]);
out[7] = div4_neon(out[7]);
vst1q_s64(H + 0 * wiener_win2 + 0, out[0]);
vst1q_s64(H + 0 * wiener_win2 + 2, out[1]);
vst1q_s64(H + 1 * wiener_win2 + 0, out[2]);
vst1q_s64(H + 1 * wiener_win2 + 2, out[3]);
vst1q_s64(H + 2 * wiener_win2 + 0, out[4]);
vst1q_s64(H + 2 * wiener_win2 + 2, out[5]);
vst1q_s64(H + 3 * wiener_win2 + 0, out[6]);
vst1q_s64(H + 3 * wiener_win2 + 2, out[7]);
}
static inline int64x2_t div16_neon(const int64x2_t src) {
#if AOM_ARCH_AARCH64
uint64x2_t sign = vcltzq_s64(src);
int64x2_t abs = vabsq_s64(src);
// divide by 16
abs = vshrq_n_s64(abs, 4);
// re-apply sign
return vbslq_s64(sign, vnegq_s64(abs), abs);
#else
int64x2_t sign = vshrq_n_s64(src, 63);
int64x2_t abs = vsubq_s64(veorq_s64(src, sign), sign);
// divide by 16
abs = vshrq_n_s64(abs, 4);
// re-apply sign
return vsubq_s64(veorq_s64(abs, sign), sign);
#endif // AOM_ARCH_AARCH64
}
static inline void div16_4x4_neon(const int32_t wiener_win2, int64_t *const H,
int64x2_t out[8]) {
out[0] = vld1q_s64(H + 0 * wiener_win2 + 0);
out[1] = vld1q_s64(H + 0 * wiener_win2 + 2);
out[2] = vld1q_s64(H + 1 * wiener_win2 + 0);
out[3] = vld1q_s64(H + 1 * wiener_win2 + 2);
out[4] = vld1q_s64(H + 2 * wiener_win2 + 0);
out[5] = vld1q_s64(H + 2 * wiener_win2 + 2);
out[6] = vld1q_s64(H + 3 * wiener_win2 + 0);
out[7] = vld1q_s64(H + 3 * wiener_win2 + 2);
out[0] = div16_neon(out[0]);
out[1] = div16_neon(out[1]);
out[2] = div16_neon(out[2]);
out[3] = div16_neon(out[3]);
out[4] = div16_neon(out[4]);
out[5] = div16_neon(out[5]);
out[6] = div16_neon(out[6]);
out[7] = div16_neon(out[7]);
vst1q_s64(H + 0 * wiener_win2 + 0, out[0]);
vst1q_s64(H + 0 * wiener_win2 + 2, out[1]);
vst1q_s64(H + 1 * wiener_win2 + 0, out[2]);
vst1q_s64(H + 1 * wiener_win2 + 2, out[3]);
vst1q_s64(H + 2 * wiener_win2 + 0, out[4]);
vst1q_s64(H + 2 * wiener_win2 + 2, out[5]);
vst1q_s64(H + 3 * wiener_win2 + 0, out[6]);
vst1q_s64(H + 3 * wiener_win2 + 2, out[7]);
}
static inline void div4_diagonal_copy_stats_neon(const int32_t wiener_win2,
int64_t *const H) {
for (int32_t i = 0; i < wiener_win2 - 1; i += 4) {
int64x2_t in[8], out[8];
div4_4x4_neon(wiener_win2, H + i * wiener_win2 + i + 1, in);
transpose_arrays_s64_4x4(in, out);
vst1_s64(H + (i + 1) * wiener_win2 + i + 0, vget_low_s64(out[0]));
vst1q_s64(H + (i + 2) * wiener_win2 + i + 0, out[2]);
vst1q_s64(H + (i + 3) * wiener_win2 + i + 0, out[4]);
vst1q_s64(H + (i + 3) * wiener_win2 + i + 2, out[5]);
vst1q_s64(H + (i + 4) * wiener_win2 + i + 0, out[6]);
vst1q_s64(H + (i + 4) * wiener_win2 + i + 2, out[7]);
for (int32_t j = i + 5; j < wiener_win2; j += 4) {
div4_4x4_neon(wiener_win2, H + i * wiener_win2 + j, in);
transpose_arrays_s64_4x4(in, out);
vst1q_s64(H + (j + 0) * wiener_win2 + i + 0, out[0]);
vst1q_s64(H + (j + 0) * wiener_win2 + i + 2, out[1]);
vst1q_s64(H + (j + 1) * wiener_win2 + i + 0, out[2]);
vst1q_s64(H + (j + 1) * wiener_win2 + i + 2, out[3]);
vst1q_s64(H + (j + 2) * wiener_win2 + i + 0, out[4]);
vst1q_s64(H + (j + 2) * wiener_win2 + i + 2, out[5]);
vst1q_s64(H + (j + 3) * wiener_win2 + i + 0, out[6]);
vst1q_s64(H + (j + 3) * wiener_win2 + i + 2, out[7]);
}
}
}
static inline void div16_diagonal_copy_stats_neon(const int32_t wiener_win2,
int64_t *const H) {
for (int32_t i = 0; i < wiener_win2 - 1; i += 4) {
int64x2_t in[8], out[8];
div16_4x4_neon(wiener_win2, H + i * wiener_win2 + i + 1, in);
transpose_arrays_s64_4x4(in, out);
vst1_s64(H + (i + 1) * wiener_win2 + i + 0, vget_low_s64(out[0]));
vst1q_s64(H + (i + 2) * wiener_win2 + i + 0, out[2]);
vst1q_s64(H + (i + 3) * wiener_win2 + i + 0, out[4]);
vst1q_s64(H + (i + 3) * wiener_win2 + i + 2, out[5]);
vst1q_s64(H + (i + 4) * wiener_win2 + i + 0, out[6]);
vst1q_s64(H + (i + 4) * wiener_win2 + i + 2, out[7]);
for (int32_t j = i + 5; j < wiener_win2; j += 4) {
div16_4x4_neon(wiener_win2, H + i * wiener_win2 + j, in);
transpose_arrays_s64_4x4(in, out);
vst1q_s64(H + (j + 0) * wiener_win2 + i + 0, out[0]);
vst1q_s64(H + (j + 0) * wiener_win2 + i + 2, out[1]);
vst1q_s64(H + (j + 1) * wiener_win2 + i + 0, out[2]);
vst1q_s64(H + (j + 1) * wiener_win2 + i + 2, out[3]);
vst1q_s64(H + (j + 2) * wiener_win2 + i + 0, out[4]);
vst1q_s64(H + (j + 2) * wiener_win2 + i + 2, out[5]);
vst1q_s64(H + (j + 3) * wiener_win2 + i + 0, out[6]);
vst1q_s64(H + (j + 3) * wiener_win2 + i + 2, out[7]);
}
}
}
#endif // AOM_AV1_ENCODER_ARM_PICKRST_NEON_H_