Source code
Revision control
Copy as Markdown
Other Tools
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "./vpx_config.h"
#include "./vpx_dsp_rtcd.h"
#include "vpx_util/loongson_intrinsics.h"
static INLINE __m128i sad_ub2_uh(__m128i in0, __m128i in1, __m128i ref0,
__m128i ref1) {
__m128i diff0_m, diff1_m, sad_m0;
__m128i sad_m = __lsx_vldi(0);
diff0_m = __lsx_vabsd_bu(in0, ref0);
diff1_m = __lsx_vabsd_bu(in1, ref1);
sad_m0 = __lsx_vhaddw_hu_bu(diff0_m, diff0_m);
sad_m = __lsx_vadd_h(sad_m, sad_m0);
sad_m0 = __lsx_vhaddw_hu_bu(diff1_m, diff1_m);
sad_m = __lsx_vadd_h(sad_m, sad_m0);
return sad_m;
}
static INLINE uint32_t hadd_uw_u32(__m128i in) {
__m128i res0_m;
uint32_t sum_m;
res0_m = __lsx_vhaddw_du_wu(in, in);
res0_m = __lsx_vhaddw_qu_du(res0_m, res0_m);
sum_m = __lsx_vpickve2gr_w(res0_m, 0);
return sum_m;
}
static INLINE uint32_t hadd_uh_u32(__m128i in) {
__m128i res_m;
uint32_t sum_m;
res_m = __lsx_vhaddw_wu_hu(in, in);
sum_m = hadd_uw_u32(res_m);
return sum_m;
}
static INLINE int32_t hadd_sw_s32(__m128i in) {
__m128i res0_m;
int32_t sum_m;
res0_m = __lsx_vhaddw_d_w(in, in);
res0_m = __lsx_vhaddw_q_d(res0_m, res0_m);
sum_m = __lsx_vpickve2gr_w(res0_m, 0);
return sum_m;
}
static uint32_t sad_8width_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
int32_t height) {
int32_t ht_cnt;
uint32_t res;
__m128i src0, src1, src2, src3, ref0, ref1, ref2, ref3, sad_tmp;
__m128i sad = __lsx_vldi(0);
for (ht_cnt = (height >> 2); ht_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src0, ref0);
src += src_stride;
ref += ref_stride;
DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src1, ref1);
src += src_stride;
ref += ref_stride;
DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src2, ref2);
src += src_stride;
ref += ref_stride;
DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src3, ref3);
src += src_stride;
ref += ref_stride;
DUP4_ARG2(__lsx_vpickev_d, src1, src0, src3, src2, ref1, ref0, ref3, ref2,
src0, src1, ref0, ref1);
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad = __lsx_vadd_h(sad, sad_tmp);
}
res = hadd_uh_u32(sad);
return res;
}
static uint32_t sad_16width_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
int32_t height) {
int32_t ht_cnt = (height >> 2);
uint32_t res;
__m128i src0, src1, ref0, ref1, sad_tmp;
__m128i sad = __lsx_vldi(0);
int32_t src_stride2 = src_stride << 1;
int32_t ref_stride2 = ref_stride << 1;
for (; ht_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src0, ref0);
DUP2_ARG2(__lsx_vldx, src, src_stride, ref, ref_stride, src1, ref1);
src += src_stride2;
ref += ref_stride2;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad = __lsx_vadd_h(sad, sad_tmp);
DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src0, ref0);
DUP2_ARG2(__lsx_vldx, src, src_stride, ref, ref_stride, src1, ref1);
src += src_stride2;
ref += ref_stride2;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad = __lsx_vadd_h(sad, sad_tmp);
}
res = hadd_uh_u32(sad);
return res;
}
static uint32_t sad_32width_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
int32_t height) {
int32_t ht_cnt = (height >> 2);
uint32_t res;
__m128i src0, src1, ref0, ref1;
__m128i sad_tmp;
__m128i sad = __lsx_vldi(0);
for (; ht_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1);
ref += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad = __lsx_vadd_h(sad, sad_tmp);
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1);
ref += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad = __lsx_vadd_h(sad, sad_tmp);
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1);
ref += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad = __lsx_vadd_h(sad, sad_tmp);
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1);
ref += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad = __lsx_vadd_h(sad, sad_tmp);
}
res = hadd_uh_u32(sad);
return res;
}
static uint32_t sad_64width_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
int32_t height) {
int32_t ht_cnt = (height >> 1);
uint32_t sad = 0;
__m128i src0, src1, src2, src3;
__m128i ref0, ref1, ref2, ref3;
__m128i sad_tmp;
__m128i sad0 = __lsx_vldi(0);
__m128i sad1 = sad0;
for (; ht_cnt--;) {
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2,
ref3);
ref += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, ref2, ref3);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2,
ref3);
ref += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, ref2, ref3);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
}
sad = hadd_uh_u32(sad0);
sad += hadd_uh_u32(sad1);
return sad;
}
static void sad_8width_x4d_lsx(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *const aref_ptr[],
int32_t ref_stride, int32_t height,
uint32_t *sad_array) {
int32_t ht_cnt = (height >> 2);
const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr;
__m128i src0, src1, src2, src3, sad_tmp;
__m128i ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7;
__m128i ref8, ref9, ref10, ref11, ref12, ref13, ref14, ref15;
__m128i sad0 = __lsx_vldi(0);
__m128i sad1 = sad0;
__m128i sad2 = sad0;
__m128i sad3 = sad0;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t ref_stride2 = ref_stride << 1;
int32_t ref_stride3 = ref_stride2 + ref_stride;
int32_t ref_stride4 = ref_stride2 << 1;
ref0_ptr = aref_ptr[0];
ref1_ptr = aref_ptr[1];
ref2_ptr = aref_ptr[2];
ref3_ptr = aref_ptr[3];
for (; ht_cnt--;) {
src0 = __lsx_vld(src_ptr, 0);
DUP2_ARG2(__lsx_vldx, src_ptr, src_stride, src_ptr, src_stride2, src1,
src2);
src3 = __lsx_vldx(src_ptr, src_stride3);
src_ptr += src_stride4;
ref0 = __lsx_vld(ref0_ptr, 0);
DUP2_ARG2(__lsx_vldx, ref0_ptr, ref_stride, ref0_ptr, ref_stride2, ref1,
ref2);
ref3 = __lsx_vldx(ref0_ptr, ref_stride3);
ref0_ptr += ref_stride4;
ref4 = __lsx_vld(ref1_ptr, 0);
DUP2_ARG2(__lsx_vldx, ref1_ptr, ref_stride, ref1_ptr, ref_stride2, ref5,
ref6);
ref7 = __lsx_vldx(ref1_ptr, ref_stride3);
ref1_ptr += ref_stride4;
ref8 = __lsx_vld(ref2_ptr, 0);
DUP2_ARG2(__lsx_vldx, ref2_ptr, ref_stride, ref2_ptr, ref_stride2, ref9,
ref10);
ref11 = __lsx_vldx(ref2_ptr, ref_stride3);
ref2_ptr += ref_stride4;
ref12 = __lsx_vld(ref3_ptr, 0);
DUP2_ARG2(__lsx_vldx, ref3_ptr, ref_stride, ref3_ptr, ref_stride2, ref13,
ref14);
ref15 = __lsx_vldx(ref3_ptr, ref_stride3);
ref3_ptr += ref_stride4;
DUP2_ARG2(__lsx_vpickev_d, src1, src0, src3, src2, src0, src1);
DUP2_ARG2(__lsx_vpickev_d, ref1, ref0, ref3, ref2, ref0, ref1);
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
DUP2_ARG2(__lsx_vpickev_d, ref5, ref4, ref7, ref6, ref0, ref1);
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
DUP2_ARG2(__lsx_vpickev_d, ref9, ref8, ref11, ref10, ref0, ref1);
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad2 = __lsx_vadd_h(sad2, sad_tmp);
DUP2_ARG2(__lsx_vpickev_d, ref13, ref12, ref15, ref14, ref0, ref1);
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad3 = __lsx_vadd_h(sad3, sad_tmp);
}
sad_array[0] = hadd_uh_u32(sad0);
sad_array[1] = hadd_uh_u32(sad1);
sad_array[2] = hadd_uh_u32(sad2);
sad_array[3] = hadd_uh_u32(sad3);
}
static void sad_16width_x4d_lsx(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *const aref_ptr[],
int32_t ref_stride, int32_t height,
uint32_t *sad_array) {
int32_t ht_cnt = (height >> 1);
const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr;
__m128i src, ref0, ref1, ref2, ref3, diff, sad_tmp;
__m128i sad0 = __lsx_vldi(0);
__m128i sad1 = sad0;
__m128i sad2 = sad0;
__m128i sad3 = sad0;
ref0_ptr = aref_ptr[0];
ref1_ptr = aref_ptr[1];
ref2_ptr = aref_ptr[2];
ref3_ptr = aref_ptr[3];
for (; ht_cnt--;) {
src = __lsx_vld(src_ptr, 0);
src_ptr += src_stride;
ref0 = __lsx_vld(ref0_ptr, 0);
ref0_ptr += ref_stride;
ref1 = __lsx_vld(ref1_ptr, 0);
ref1_ptr += ref_stride;
ref2 = __lsx_vld(ref2_ptr, 0);
ref2_ptr += ref_stride;
ref3 = __lsx_vld(ref3_ptr, 0);
ref3_ptr += ref_stride;
diff = __lsx_vabsd_bu(src, ref0);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
diff = __lsx_vabsd_bu(src, ref1);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
diff = __lsx_vabsd_bu(src, ref2);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad2 = __lsx_vadd_h(sad2, sad_tmp);
diff = __lsx_vabsd_bu(src, ref3);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad3 = __lsx_vadd_h(sad3, sad_tmp);
src = __lsx_vld(src_ptr, 0);
src_ptr += src_stride;
ref0 = __lsx_vld(ref0_ptr, 0);
ref0_ptr += ref_stride;
ref1 = __lsx_vld(ref1_ptr, 0);
ref1_ptr += ref_stride;
ref2 = __lsx_vld(ref2_ptr, 0);
ref2_ptr += ref_stride;
ref3 = __lsx_vld(ref3_ptr, 0);
ref3_ptr += ref_stride;
diff = __lsx_vabsd_bu(src, ref0);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
diff = __lsx_vabsd_bu(src, ref1);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
diff = __lsx_vabsd_bu(src, ref2);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad2 = __lsx_vadd_h(sad2, sad_tmp);
diff = __lsx_vabsd_bu(src, ref3);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad3 = __lsx_vadd_h(sad3, sad_tmp);
}
sad_array[0] = hadd_uh_u32(sad0);
sad_array[1] = hadd_uh_u32(sad1);
sad_array[2] = hadd_uh_u32(sad2);
sad_array[3] = hadd_uh_u32(sad3);
}
static void sad_32width_x4d_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *const aref_ptr[],
int32_t ref_stride, int32_t height,
uint32_t *sad_array) {
const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr;
int32_t ht_cnt = height;
__m128i src0, src1, ref0, ref1, sad_tmp;
__m128i sad0 = __lsx_vldi(0);
__m128i sad1 = sad0;
__m128i sad2 = sad0;
__m128i sad3 = sad0;
ref0_ptr = aref_ptr[0];
ref1_ptr = aref_ptr[1];
ref2_ptr = aref_ptr[2];
ref3_ptr = aref_ptr[3];
for (; ht_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, ref0_ptr, 0, ref0_ptr, 16, ref0, ref1);
ref0_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
DUP2_ARG2(__lsx_vld, ref1_ptr, 0, ref1_ptr, 16, ref0, ref1);
ref1_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
DUP2_ARG2(__lsx_vld, ref2_ptr, 0, ref2_ptr, 16, ref0, ref1);
ref2_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad2 = __lsx_vadd_h(sad2, sad_tmp);
DUP2_ARG2(__lsx_vld, ref3_ptr, 0, ref3_ptr, 16, ref0, ref1);
ref3_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad3 = __lsx_vadd_h(sad3, sad_tmp);
}
sad_array[0] = hadd_uh_u32(sad0);
sad_array[1] = hadd_uh_u32(sad1);
sad_array[2] = hadd_uh_u32(sad2);
sad_array[3] = hadd_uh_u32(sad3);
}
static void sad_64width_x4d_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *const aref_ptr[],
int32_t ref_stride, int32_t height,
uint32_t *sad_array) {
const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr;
int32_t ht_cnt = height;
__m128i src0, src1, src2, src3;
__m128i ref0, ref1, ref2, ref3;
__m128i sad, sad_tmp;
__m128i sad0_0 = __lsx_vldi(0);
__m128i sad0_1 = sad0_0;
__m128i sad1_0 = sad0_0;
__m128i sad1_1 = sad0_0;
__m128i sad2_0 = sad0_0;
__m128i sad2_1 = sad0_0;
__m128i sad3_0 = sad0_0;
__m128i sad3_1 = sad0_0;
ref0_ptr = aref_ptr[0];
ref1_ptr = aref_ptr[1];
ref2_ptr = aref_ptr[2];
ref3_ptr = aref_ptr[3];
for (; ht_cnt--;) {
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, ref0_ptr, 0, ref0_ptr, 16, ref0_ptr, 32, ref0_ptr, 48,
ref0, ref1, ref2, ref3);
ref0_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad0_0 = __lsx_vadd_h(sad0_0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, ref2, ref3);
sad0_1 = __lsx_vadd_h(sad0_1, sad_tmp);
DUP4_ARG2(__lsx_vld, ref1_ptr, 0, ref1_ptr, 16, ref1_ptr, 32, ref1_ptr, 48,
ref0, ref1, ref2, ref3);
ref1_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad1_0 = __lsx_vadd_h(sad1_0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, ref2, ref3);
sad1_1 = __lsx_vadd_h(sad1_1, sad_tmp);
DUP4_ARG2(__lsx_vld, ref2_ptr, 0, ref2_ptr, 16, ref2_ptr, 32, ref2_ptr, 48,
ref0, ref1, ref2, ref3);
ref2_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad2_0 = __lsx_vadd_h(sad2_0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, ref2, ref3);
sad2_1 = __lsx_vadd_h(sad2_1, sad_tmp);
DUP4_ARG2(__lsx_vld, ref3_ptr, 0, ref3_ptr, 16, ref3_ptr, 32, ref3_ptr, 48,
ref0, ref1, ref2, ref3);
ref3_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad3_0 = __lsx_vadd_h(sad3_0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, ref2, ref3);
sad3_1 = __lsx_vadd_h(sad3_1, sad_tmp);
}
sad = __lsx_vhaddw_wu_hu(sad0_0, sad0_0);
sad_tmp = __lsx_vhaddw_wu_hu(sad0_1, sad0_1);
sad = __lsx_vadd_w(sad, sad_tmp);
sad_array[0] = hadd_uw_u32(sad);
sad = __lsx_vhaddw_wu_hu(sad1_0, sad1_0);
sad_tmp = __lsx_vhaddw_wu_hu(sad1_1, sad1_1);
sad = __lsx_vadd_w(sad, sad_tmp);
sad_array[1] = hadd_uw_u32(sad);
sad = __lsx_vhaddw_wu_hu(sad2_0, sad2_0);
sad_tmp = __lsx_vhaddw_wu_hu(sad2_1, sad2_1);
sad = __lsx_vadd_w(sad, sad_tmp);
sad_array[2] = hadd_uw_u32(sad);
sad = __lsx_vhaddw_wu_hu(sad3_0, sad3_0);
sad_tmp = __lsx_vhaddw_wu_hu(sad3_1, sad3_1);
sad = __lsx_vadd_w(sad, sad_tmp);
sad_array[3] = hadd_uw_u32(sad);
}
static uint32_t avgsad_32width_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
int32_t height, const uint8_t *sec_pred) {
int32_t res, ht_cnt = (height >> 2);
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
__m128i ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7;
__m128i pred0, pred1, pred2, pred3, pred4, pred5, pred6, pred7;
__m128i comp0, comp1, sad_tmp;
__m128i sad = __lsx_vldi(0);
uint8_t *src_tmp, *ref_tmp;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t ref_stride2 = ref_stride << 1;
int32_t ref_stride3 = ref_stride2 + ref_stride;
int32_t ref_stride4 = ref_stride2 << 1;
for (; ht_cnt--;) {
src_tmp = (uint8_t *)src + 16;
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4);
src6 = __lsx_vldx(src, src_stride3);
src1 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src3,
src5);
src7 = __lsx_vldx(src_tmp, src_stride3);
src += src_stride4;
ref_tmp = (uint8_t *)ref + 16;
ref0 = __lsx_vld(ref, 0);
DUP2_ARG2(__lsx_vldx, ref, ref_stride, ref, ref_stride2, ref2, ref4);
ref6 = __lsx_vldx(ref, ref_stride3);
ref1 = __lsx_vld(ref_tmp, 0);
DUP2_ARG2(__lsx_vldx, ref_tmp, ref_stride, ref_tmp, ref_stride2, ref3,
ref5);
ref7 = __lsx_vldx(ref_tmp, ref_stride3);
ref += ref_stride4;
DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 32, sec_pred, 64, sec_pred, 96,
pred0, pred2, pred4, pred6);
DUP4_ARG2(__lsx_vld, sec_pred, 16, sec_pred, 48, sec_pred, 80, sec_pred,
112, pred1, pred3, pred5, pred7);
sec_pred += 128;
DUP2_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, comp0, comp1);
sad_tmp = sad_ub2_uh(src0, src1, comp0, comp1);
sad = __lsx_vadd_h(sad, sad_tmp);
DUP2_ARG2(__lsx_vavgr_bu, pred2, ref2, pred3, ref3, comp0, comp1);
sad_tmp = sad_ub2_uh(src2, src3, comp0, comp1);
sad = __lsx_vadd_h(sad, sad_tmp);
DUP2_ARG2(__lsx_vavgr_bu, pred4, ref4, pred5, ref5, comp0, comp1);
sad_tmp = sad_ub2_uh(src4, src5, comp0, comp1);
sad = __lsx_vadd_h(sad, sad_tmp);
DUP2_ARG2(__lsx_vavgr_bu, pred6, ref6, pred7, ref7, comp0, comp1);
sad_tmp = sad_ub2_uh(src6, src7, comp0, comp1);
sad = __lsx_vadd_h(sad, sad_tmp);
}
res = hadd_uh_u32(sad);
return res;
}
static uint32_t avgsad_64width_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
int32_t height, const uint8_t *sec_pred) {
int32_t res, ht_cnt = (height >> 2);
__m128i src0, src1, src2, src3, ref0, ref1, ref2, ref3;
__m128i comp0, comp1, comp2, comp3, pred0, pred1, pred2, pred3;
__m128i sad, sad_tmp;
__m128i sad0 = __lsx_vldi(0);
__m128i sad1 = sad0;
for (; ht_cnt--;) {
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2,
ref3);
ref += ref_stride;
DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48,
pred0, pred1, pred2, pred3);
sec_pred += 64;
DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3,
ref3, comp0, comp1, comp2, comp3);
sad_tmp = sad_ub2_uh(src0, src1, comp0, comp1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, comp2, comp3);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2,
ref3);
ref += ref_stride;
DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48,
pred0, pred1, pred2, pred3);
sec_pred += 64;
DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3,
ref3, comp0, comp1, comp2, comp3);
sad_tmp = sad_ub2_uh(src0, src1, comp0, comp1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, comp2, comp3);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2,
ref3);
ref += ref_stride;
DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48,
pred0, pred1, pred2, pred3);
sec_pred += 64;
DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3,
ref3, comp0, comp1, comp2, comp3);
sad_tmp = sad_ub2_uh(src0, src1, comp0, comp1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, comp2, comp3);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2,
ref3);
ref += ref_stride;
DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48,
pred0, pred1, pred2, pred3);
sec_pred += 64;
DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3,
ref3, comp0, comp1, comp2, comp3);
sad_tmp = sad_ub2_uh(src0, src1, comp0, comp1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, comp2, comp3);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
}
sad = __lsx_vhaddw_wu_hu(sad0, sad0);
sad_tmp = __lsx_vhaddw_wu_hu(sad1, sad1);
sad = __lsx_vadd_w(sad, sad_tmp);
res = hadd_sw_s32(sad);
return res;
}
#define VPX_SAD_8xHT_LSX(height) \
uint32_t vpx_sad8x##height##_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *ref, int32_t ref_stride) { \
return sad_8width_lsx(src, src_stride, ref, ref_stride, height); \
}
#define VPX_SAD_16xHT_LSX(height) \
uint32_t vpx_sad16x##height##_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *ref, int32_t ref_stride) { \
return sad_16width_lsx(src, src_stride, ref, ref_stride, height); \
}
#define VPX_SAD_32xHT_LSX(height) \
uint32_t vpx_sad32x##height##_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *ref, int32_t ref_stride) { \
return sad_32width_lsx(src, src_stride, ref, ref_stride, height); \
}
#define VPX_SAD_64xHT_LSX(height) \
uint32_t vpx_sad64x##height##_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *ref, int32_t ref_stride) { \
return sad_64width_lsx(src, src_stride, ref, ref_stride, height); \
}
#define VPX_SAD_8xHTx4D_LSX(height) \
void vpx_sad8x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *const refs[4], \
int32_t ref_stride, uint32_t sads[4]) { \
sad_8width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \
}
#define VPX_SAD_16xHTx4D_LSX(height) \
void vpx_sad16x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *const refs[], \
int32_t ref_stride, uint32_t *sads) { \
sad_16width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \
}
#define VPX_SAD_32xHTx4D_LSX(height) \
void vpx_sad32x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *const refs[], \
int32_t ref_stride, uint32_t *sads) { \
sad_32width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \
}
#define VPX_SAD_64xHTx4D_LSX(height) \
void vpx_sad64x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *const refs[], \
int32_t ref_stride, uint32_t *sads) { \
sad_64width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \
}
#define VPX_AVGSAD_32xHT_LSX(height) \
uint32_t vpx_sad32x##height##_avg_lsx( \
const uint8_t *src, int32_t src_stride, const uint8_t *ref, \
int32_t ref_stride, const uint8_t *second_pred) { \
return avgsad_32width_lsx(src, src_stride, ref, ref_stride, height, \
second_pred); \
}
#define VPX_AVGSAD_64xHT_LSX(height) \
uint32_t vpx_sad64x##height##_avg_lsx( \
const uint8_t *src, int32_t src_stride, const uint8_t *ref, \
int32_t ref_stride, const uint8_t *second_pred) { \
return avgsad_64width_lsx(src, src_stride, ref, ref_stride, height, \
second_pred); \
}
#define SAD64 \
VPX_SAD_64xHT_LSX(64) VPX_SAD_64xHTx4D_LSX(64) VPX_SAD_64xHTx4D_LSX(32) \
VPX_AVGSAD_64xHT_LSX(64)
SAD64
#define SAD32 \
VPX_SAD_32xHT_LSX(32) VPX_SAD_32xHTx4D_LSX(32) VPX_SAD_32xHTx4D_LSX(64) \
VPX_AVGSAD_32xHT_LSX(32)
SAD32
#define SAD16 VPX_SAD_16xHT_LSX(16) VPX_SAD_16xHTx4D_LSX(16)
SAD16
#define SAD8 VPX_SAD_8xHT_LSX(8) VPX_SAD_8xHTx4D_LSX(8)
SAD8
#undef SAD64
#undef SAD32
#undef SAD16
#undef SAD8