ben.carrasco/freeswitch
1
0 Fork 0
mirror of https://github.com/signalwire/freeswitch.git synced 2026-11-11 03:57:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

[libvpx] Update to v1.12.0

Browse Source
This commit is contained in:
Andrey Volk committed 2026-07-10 19:38:46 +03:00
1 parent 287a71dc66
commit b5046d2a76
572 files changed
+52900 -29493
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

libs/libvpx/vpx_dsp/arm/avg_neon.c
+3 -8
View File
@@ -22,15 +22,13 @@
uint32_t vpx_avg_4x4_neon(const uint8_t *a, int a_stride) {
const uint8x16_t b = load_unaligned_u8q(a, a_stride);
const uint16x8_t c = vaddl_u8(vget_low_u8(b), vget_high_u8(b));
const uint32x2_t d = horizontal_add_uint16x8(c);
return vget_lane_u32(vrshr_n_u32(d, 4), 0);
return (horizontal_add_uint16x8(c) + (1 << 3)) >> 4;
}
uint32_t vpx_avg_8x8_neon(const uint8_t *a, int a_stride) {
int i;
uint8x8_t b, c;
uint16x8_t sum;
uint32x2_t d;
b = vld1_u8(a);
a += a_stride;
c = vld1_u8(a);
@@ -43,9 +41,7 @@ uint32_t vpx_avg_8x8_neon(const uint8_t *a, int a_stride) {
sum = vaddw_u8(sum, d);
}
d = horizontal_add_uint16x8(sum);
return vget_lane_u32(vrshr_n_u32(d, 6), 0);
return (horizontal_add_uint16x8(sum) + (1 << 5)) >> 6;
}
// coeff: 16 bits, dynamic range [-32640, 32640].
@@ -139,8 +135,7 @@ int16_t vpx_int_pro_col_neon(uint8_t const *ref, const int width) {
ref += 16;
}
return vget_lane_s16(vreinterpret_s16_u32(horizontal_add_uint16x8(vec_sum)),
0);
return (int16_t)horizontal_add_uint16x8(vec_sum);
}
// ref, src = [0, 510] - max diff = 16-bits
libs/libvpx/vpx_dsp/arm/fdct16x16_neon.c
+5 -314
View File
@@ -15,6 +15,7 @@
#include "vpx_dsp/txfm_common.h"
#include "vpx_dsp/arm/mem_neon.h"
#include "vpx_dsp/arm/transpose_neon.h"
#include "vpx_dsp/arm/fdct16x16_neon.h"
// Some builds of gcc 4.9.2 and .3 have trouble with some of the inline
// functions.
@@ -27,316 +28,6 @@ void vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) {
#else
static INLINE void load(const int16_t *a, int stride, int16x8_t *b /*[16]*/) {
b[0] = vld1q_s16(a);
a += stride;
b[1] = vld1q_s16(a);
a += stride;
b[2] = vld1q_s16(a);
a += stride;
b[3] = vld1q_s16(a);
a += stride;
b[4] = vld1q_s16(a);
a += stride;
b[5] = vld1q_s16(a);
a += stride;
b[6] = vld1q_s16(a);
a += stride;
b[7] = vld1q_s16(a);
a += stride;
b[8] = vld1q_s16(a);
a += stride;
b[9] = vld1q_s16(a);
a += stride;
b[10] = vld1q_s16(a);
a += stride;
b[11] = vld1q_s16(a);
a += stride;
b[12] = vld1q_s16(a);
a += stride;
b[13] = vld1q_s16(a);
a += stride;
b[14] = vld1q_s16(a);
a += stride;
b[15] = vld1q_s16(a);
}
// Store 8 16x8 values, assuming stride == 16.
static INLINE void store(tran_low_t *a, const int16x8_t *b /*[8]*/) {
store_s16q_to_tran_low(a, b[0]);
a += 16;
store_s16q_to_tran_low(a, b[1]);
a += 16;
store_s16q_to_tran_low(a, b[2]);
a += 16;
store_s16q_to_tran_low(a, b[3]);
a += 16;
store_s16q_to_tran_low(a, b[4]);
a += 16;
store_s16q_to_tran_low(a, b[5]);
a += 16;
store_s16q_to_tran_low(a, b[6]);
a += 16;
store_s16q_to_tran_low(a, b[7]);
}
// Load step of each pass. Add and subtract clear across the input, requiring
// all 16 values to be loaded. For the first pass it also multiplies by 4.
// To maybe reduce register usage this could be combined with the load() step to
// get the first 4 and last 4 values, cross those, then load the middle 8 values
// and cross them.
static INLINE void cross_input(const int16x8_t *a /*[16]*/,
int16x8_t *b /*[16]*/, const int pass) {
if (pass == 0) {
b[0] = vshlq_n_s16(vaddq_s16(a[0], a[15]), 2);
b[1] = vshlq_n_s16(vaddq_s16(a[1], a[14]), 2);
b[2] = vshlq_n_s16(vaddq_s16(a[2], a[13]), 2);
b[3] = vshlq_n_s16(vaddq_s16(a[3], a[12]), 2);
b[4] = vshlq_n_s16(vaddq_s16(a[4], a[11]), 2);
b[5] = vshlq_n_s16(vaddq_s16(a[5], a[10]), 2);
b[6] = vshlq_n_s16(vaddq_s16(a[6], a[9]), 2);
b[7] = vshlq_n_s16(vaddq_s16(a[7], a[8]), 2);
b[8] = vshlq_n_s16(vsubq_s16(a[7], a[8]), 2);
b[9] = vshlq_n_s16(vsubq_s16(a[6], a[9]), 2);
b[10] = vshlq_n_s16(vsubq_s16(a[5], a[10]), 2);
b[11] = vshlq_n_s16(vsubq_s16(a[4], a[11]), 2);
b[12] = vshlq_n_s16(vsubq_s16(a[3], a[12]), 2);
b[13] = vshlq_n_s16(vsubq_s16(a[2], a[13]), 2);
b[14] = vshlq_n_s16(vsubq_s16(a[1], a[14]), 2);
b[15] = vshlq_n_s16(vsubq_s16(a[0], a[15]), 2);
} else {
b[0] = vaddq_s16(a[0], a[15]);
b[1] = vaddq_s16(a[1], a[14]);
b[2] = vaddq_s16(a[2], a[13]);
b[3] = vaddq_s16(a[3], a[12]);
b[4] = vaddq_s16(a[4], a[11]);
b[5] = vaddq_s16(a[5], a[10]);
b[6] = vaddq_s16(a[6], a[9]);
b[7] = vaddq_s16(a[7], a[8]);
b[8] = vsubq_s16(a[7], a[8]);
b[9] = vsubq_s16(a[6], a[9]);
b[10] = vsubq_s16(a[5], a[10]);
b[11] = vsubq_s16(a[4], a[11]);
b[12] = vsubq_s16(a[3], a[12]);
b[13] = vsubq_s16(a[2], a[13]);
b[14] = vsubq_s16(a[1], a[14]);
b[15] = vsubq_s16(a[0], a[15]);
}
}
// Quarter round at the beginning of the second pass. Can't use vrshr (rounding)
// because this only adds 1, not 1 << 2.
static INLINE void partial_round_shift(int16x8_t *a /*[16]*/) {
const int16x8_t one = vdupq_n_s16(1);
a[0] = vshrq_n_s16(vaddq_s16(a[0], one), 2);
a[1] = vshrq_n_s16(vaddq_s16(a[1], one), 2);
a[2] = vshrq_n_s16(vaddq_s16(a[2], one), 2);
a[3] = vshrq_n_s16(vaddq_s16(a[3], one), 2);
a[4] = vshrq_n_s16(vaddq_s16(a[4], one), 2);
a[5] = vshrq_n_s16(vaddq_s16(a[5], one), 2);
a[6] = vshrq_n_s16(vaddq_s16(a[6], one), 2);
a[7] = vshrq_n_s16(vaddq_s16(a[7], one), 2);
a[8] = vshrq_n_s16(vaddq_s16(a[8], one), 2);
a[9] = vshrq_n_s16(vaddq_s16(a[9], one), 2);
a[10] = vshrq_n_s16(vaddq_s16(a[10], one), 2);
a[11] = vshrq_n_s16(vaddq_s16(a[11], one), 2);
a[12] = vshrq_n_s16(vaddq_s16(a[12], one), 2);
a[13] = vshrq_n_s16(vaddq_s16(a[13], one), 2);
a[14] = vshrq_n_s16(vaddq_s16(a[14], one), 2);
a[15] = vshrq_n_s16(vaddq_s16(a[15], one), 2);
}
// fdct_round_shift((a +/- b) * c)
static INLINE void butterfly_one_coeff(const int16x8_t a, const int16x8_t b,
const tran_high_t c, int16x8_t *add,
int16x8_t *sub) {
const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c);
const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c);
const int32x4_t sum0 = vmlal_n_s16(a0, vget_low_s16(b), c);
const int32x4_t sum1 = vmlal_n_s16(a1, vget_high_s16(b), c);
const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c);
const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c);
const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14);
const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14);
const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14);
const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14);
*add = vcombine_s16(rounded0, rounded1);
*sub = vcombine_s16(rounded2, rounded3);
}
// fdct_round_shift(a * c0 +/- b * c1)
static INLINE void butterfly_two_coeff(const int16x8_t a, const int16x8_t b,
const tran_coef_t c0,
const tran_coef_t c1, int16x8_t *add,
int16x8_t *sub) {
const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c0);
const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c0);
const int32x4_t a2 = vmull_n_s16(vget_low_s16(a), c1);
const int32x4_t a3 = vmull_n_s16(vget_high_s16(a), c1);
const int32x4_t sum0 = vmlal_n_s16(a2, vget_low_s16(b), c0);
const int32x4_t sum1 = vmlal_n_s16(a3, vget_high_s16(b), c0);
const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c1);
const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c1);
const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14);
const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14);
const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14);
const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14);
*add = vcombine_s16(rounded0, rounded1);
*sub = vcombine_s16(rounded2, rounded3);
}
// Transpose 8x8 to a new location. Don't use transpose_neon.h because those
// are all in-place.
static INLINE void transpose_8x8(const int16x8_t *a /*[8]*/,
int16x8_t *b /*[8]*/) {
// Swap 16 bit elements.
const int16x8x2_t c0 = vtrnq_s16(a[0], a[1]);
const int16x8x2_t c1 = vtrnq_s16(a[2], a[3]);
const int16x8x2_t c2 = vtrnq_s16(a[4], a[5]);
const int16x8x2_t c3 = vtrnq_s16(a[6], a[7]);
// Swap 32 bit elements.
const int32x4x2_t d0 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[0]),
vreinterpretq_s32_s16(c1.val[0]));
const int32x4x2_t d1 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[1]),
vreinterpretq_s32_s16(c1.val[1]));
const int32x4x2_t d2 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[0]),
vreinterpretq_s32_s16(c3.val[0]));
const int32x4x2_t d3 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[1]),
vreinterpretq_s32_s16(c3.val[1]));
// Swap 64 bit elements
const int16x8x2_t e0 = vpx_vtrnq_s64_to_s16(d0.val[0], d2.val[0]);
const int16x8x2_t e1 = vpx_vtrnq_s64_to_s16(d1.val[0], d3.val[0]);
const int16x8x2_t e2 = vpx_vtrnq_s64_to_s16(d0.val[1], d2.val[1]);
const int16x8x2_t e3 = vpx_vtrnq_s64_to_s16(d1.val[1], d3.val[1]);
b[0] = e0.val[0];
b[1] = e1.val[0];
b[2] = e2.val[0];
b[3] = e3.val[0];
b[4] = e0.val[1];
b[5] = e1.val[1];
b[6] = e2.val[1];
b[7] = e3.val[1];
}
// Main body of fdct16x16.
static void dct_body(const int16x8_t *in /*[16]*/, int16x8_t *out /*[16]*/) {
int16x8_t s[8];
int16x8_t x[4];
int16x8_t step[8];
// stage 1
// From fwd_txfm.c: Work on the first eight values; fdct8(input,
// even_results);"
s[0] = vaddq_s16(in[0], in[7]);
s[1] = vaddq_s16(in[1], in[6]);
s[2] = vaddq_s16(in[2], in[5]);
s[3] = vaddq_s16(in[3], in[4]);
s[4] = vsubq_s16(in[3], in[4]);
s[5] = vsubq_s16(in[2], in[5]);
s[6] = vsubq_s16(in[1], in[6]);
s[7] = vsubq_s16(in[0], in[7]);
// fdct4(step, step);
x[0] = vaddq_s16(s[0], s[3]);
x[1] = vaddq_s16(s[1], s[2]);
x[2] = vsubq_s16(s[1], s[2]);
x[3] = vsubq_s16(s[0], s[3]);
// out[0] = fdct_round_shift((x0 + x1) * cospi_16_64)
// out[8] = fdct_round_shift((x0 - x1) * cospi_16_64)
butterfly_one_coeff(x[0], x[1], cospi_16_64, &out[0], &out[8]);
// out[4] = fdct_round_shift(x3 * cospi_8_64 + x2 * cospi_24_64);
// out[12] = fdct_round_shift(x3 * cospi_24_64 - x2 * cospi_8_64);
butterfly_two_coeff(x[3], x[2], cospi_24_64, cospi_8_64, &out[4], &out[12]);
// Stage 2
// Re-using source s5/s6
// s5 = fdct_round_shift((s6 - s5) * cospi_16_64)
// s6 = fdct_round_shift((s6 + s5) * cospi_16_64)
butterfly_one_coeff(s[6], s[5], cospi_16_64, &s[6], &s[5]);
// Stage 3
x[0] = vaddq_s16(s[4], s[5]);
x[1] = vsubq_s16(s[4], s[5]);
x[2] = vsubq_s16(s[7], s[6]);
x[3] = vaddq_s16(s[7], s[6]);
// Stage 4
// out[2] = fdct_round_shift(x0 * cospi_28_64 + x3 * cospi_4_64)
// out[14] = fdct_round_shift(x3 * cospi_28_64 + x0 * -cospi_4_64)
butterfly_two_coeff(x[3], x[0], cospi_28_64, cospi_4_64, &out[2], &out[14]);
// out[6] = fdct_round_shift(x1 * cospi_12_64 + x2 * cospi_20_64)
// out[10] = fdct_round_shift(x2 * cospi_12_64 + x1 * -cospi_20_64)
butterfly_two_coeff(x[2], x[1], cospi_12_64, cospi_20_64, &out[10], &out[6]);
// step 2
// From fwd_txfm.c: Work on the next eight values; step1 -> odd_results"
// That file distinguished between "in_high" and "step1" but the only
// difference is that "in_high" is the first 8 values and "step 1" is the
// second. Here, since they are all in one array, "step1" values are += 8.
// step2[2] = fdct_round_shift((step1[5] - step1[2]) * cospi_16_64)
// step2[3] = fdct_round_shift((step1[4] - step1[3]) * cospi_16_64)
// step2[4] = fdct_round_shift((step1[4] + step1[3]) * cospi_16_64)
// step2[5] = fdct_round_shift((step1[5] + step1[2]) * cospi_16_64)
butterfly_one_coeff(in[13], in[10], cospi_16_64, &s[5], &s[2]);
butterfly_one_coeff(in[12], in[11], cospi_16_64, &s[4], &s[3]);
// step 3
s[0] = vaddq_s16(in[8], s[3]);
s[1] = vaddq_s16(in[9], s[2]);
x[0] = vsubq_s16(in[9], s[2]);
x[1] = vsubq_s16(in[8], s[3]);
x[2] = vsubq_s16(in[15], s[4]);
x[3] = vsubq_s16(in[14], s[5]);
s[6] = vaddq_s16(in[14], s[5]);
s[7] = vaddq_s16(in[15], s[4]);
// step 4
// step2[1] = fdct_round_shift(step3[1] *-cospi_8_64 + step3[6] * cospi_24_64)
// step2[6] = fdct_round_shift(step3[1] * cospi_24_64 + step3[6] * cospi_8_64)
butterfly_two_coeff(s[6], s[1], cospi_24_64, cospi_8_64, &s[6], &s[1]);
// step2[2] = fdct_round_shift(step3[2] * cospi_24_64 + step3[5] * cospi_8_64)
// step2[5] = fdct_round_shift(step3[2] * cospi_8_64 - step3[5] * cospi_24_64)
butterfly_two_coeff(x[0], x[3], cospi_8_64, cospi_24_64, &s[2], &s[5]);
// step 5
step[0] = vaddq_s16(s[0], s[1]);
step[1] = vsubq_s16(s[0], s[1]);
step[2] = vaddq_s16(x[1], s[2]);
step[3] = vsubq_s16(x[1], s[2]);
step[4] = vsubq_s16(x[2], s[5]);
step[5] = vaddq_s16(x[2], s[5]);
step[6] = vsubq_s16(s[7], s[6]);
step[7] = vaddq_s16(s[7], s[6]);
// step 6
// out[1] = fdct_round_shift(step1[0] * cospi_30_64 + step1[7] * cospi_2_64)
// out[9] = fdct_round_shift(step1[1] * cospi_14_64 + step1[6] * cospi_18_64)
// out[5] = fdct_round_shift(step1[2] * cospi_22_64 + step1[5] * cospi_10_64)
// out[13] = fdct_round_shift(step1[3] * cospi_6_64 + step1[4] * cospi_26_64)
// out[3] = fdct_round_shift(step1[3] * -cospi_26_64 + step1[4] * cospi_6_64)
// out[11] = fdct_round_shift(step1[2] * -cospi_10_64 + step1[5] *
// cospi_22_64)
// out[7] = fdct_round_shift(step1[1] * -cospi_18_64 + step1[6] * cospi_14_64)
// out[15] = fdct_round_shift(step1[0] * -cospi_2_64 + step1[7] * cospi_30_64)
butterfly_two_coeff(step[6], step[1], cospi_14_64, cospi_18_64, &out[9],
&out[7]);
butterfly_two_coeff(step[7], step[0], cospi_30_64, cospi_2_64, &out[1],
&out[15]);
butterfly_two_coeff(step[4], step[3], cospi_6_64, cospi_26_64, &out[13],
&out[3]);
butterfly_two_coeff(step[5], step[2], cospi_22_64, cospi_10_64, &out[5],
&out[11]);
}
void vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) {
int16x8_t temp0[16];
int16x8_t temp1[16];
@@ -346,12 +37,12 @@ void vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) {
// Left half.
load(input, stride, temp0);
cross_input(temp0, temp1, 0);
dct_body(temp1, temp0);
vpx_fdct16x16_body(temp1, temp0);
// Right half.
load(input + 8, stride, temp1);
cross_input(temp1, temp2, 0);
dct_body(temp2, temp1);
vpx_fdct16x16_body(temp2, temp1);
// Transpose top left and top right quarters into one contiguous location to
// process to the top half.
@@ -359,7 +50,7 @@ void vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) {
transpose_8x8(&temp1[0], &temp2[8]);
partial_round_shift(temp2);
cross_input(temp2, temp3, 1);
dct_body(temp3, temp2);
vpx_fdct16x16_body(temp3, temp2);
transpose_s16_8x8(&temp2[0], &temp2[1], &temp2[2], &temp2[3], &temp2[4],
&temp2[5], &temp2[6], &temp2[7]);
transpose_s16_8x8(&temp2[8], &temp2[9], &temp2[10], &temp2[11], &temp2[12],
@@ -375,7 +66,7 @@ void vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) {
&temp1[13], &temp1[14], &temp1[15]);
partial_round_shift(temp1);
cross_input(temp1, temp0, 1);
dct_body(temp0, temp1);
vpx_fdct16x16_body(temp0, temp1);
transpose_s16_8x8(&temp1[0], &temp1[1], &temp1[2], &temp1[3], &temp1[4],
&temp1[5], &temp1[6], &temp1[7]);
transpose_s16_8x8(&temp1[8], &temp1[9], &temp1[10], &temp1[11], &temp1[12],
libs/libvpx/vpx_dsp/arm/fdct16x16_neon.h
+327
View File
@@ -0,0 +1,327 @@
/*
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VPX_VPX_DSP_ARM_FDCT16X16_NEON_H_
#define VPX_VPX_DSP_ARM_FDCT16X16_NEON_H_
#include <arm_neon.h>
static INLINE void load(const int16_t *a, int stride, int16x8_t *b /*[16]*/) {
b[0] = vld1q_s16(a);
a += stride;
b[1] = vld1q_s16(a);
a += stride;
b[2] = vld1q_s16(a);
a += stride;
b[3] = vld1q_s16(a);
a += stride;
b[4] = vld1q_s16(a);
a += stride;
b[5] = vld1q_s16(a);
a += stride;
b[6] = vld1q_s16(a);
a += stride;
b[7] = vld1q_s16(a);
a += stride;
b[8] = vld1q_s16(a);
a += stride;
b[9] = vld1q_s16(a);
a += stride;
b[10] = vld1q_s16(a);
a += stride;
b[11] = vld1q_s16(a);
a += stride;
b[12] = vld1q_s16(a);
a += stride;
b[13] = vld1q_s16(a);
a += stride;
b[14] = vld1q_s16(a);
a += stride;
b[15] = vld1q_s16(a);
}
// Store 8 16x8 values, assuming stride == 16.
static INLINE void store(tran_low_t *a, const int16x8_t *b /*[8]*/) {
store_s16q_to_tran_low(a, b[0]);
a += 16;
store_s16q_to_tran_low(a, b[1]);
a += 16;
store_s16q_to_tran_low(a, b[2]);
a += 16;
store_s16q_to_tran_low(a, b[3]);
a += 16;
store_s16q_to_tran_low(a, b[4]);
a += 16;
store_s16q_to_tran_low(a, b[5]);
a += 16;
store_s16q_to_tran_low(a, b[6]);
a += 16;
store_s16q_to_tran_low(a, b[7]);
}
// Load step of each pass. Add and subtract clear across the input, requiring
// all 16 values to be loaded. For the first pass it also multiplies by 4.
// To maybe reduce register usage this could be combined with the load() step to
// get the first 4 and last 4 values, cross those, then load the middle 8 values
// and cross them.
static INLINE void cross_input(const int16x8_t *a /*[16]*/,
int16x8_t *b /*[16]*/, const int pass) {
if (pass == 0) {
b[0] = vshlq_n_s16(vaddq_s16(a[0], a[15]), 2);
b[1] = vshlq_n_s16(vaddq_s16(a[1], a[14]), 2);
b[2] = vshlq_n_s16(vaddq_s16(a[2], a[13]), 2);
b[3] = vshlq_n_s16(vaddq_s16(a[3], a[12]), 2);
b[4] = vshlq_n_s16(vaddq_s16(a[4], a[11]), 2);
b[5] = vshlq_n_s16(vaddq_s16(a[5], a[10]), 2);
b[6] = vshlq_n_s16(vaddq_s16(a[6], a[9]), 2);
b[7] = vshlq_n_s16(vaddq_s16(a[7], a[8]), 2);
b[8] = vshlq_n_s16(vsubq_s16(a[7], a[8]), 2);
b[9] = vshlq_n_s16(vsubq_s16(a[6], a[9]), 2);
b[10] = vshlq_n_s16(vsubq_s16(a[5], a[10]), 2);
b[11] = vshlq_n_s16(vsubq_s16(a[4], a[11]), 2);
b[12] = vshlq_n_s16(vsubq_s16(a[3], a[12]), 2);
b[13] = vshlq_n_s16(vsubq_s16(a[2], a[13]), 2);
b[14] = vshlq_n_s16(vsubq_s16(a[1], a[14]), 2);
b[15] = vshlq_n_s16(vsubq_s16(a[0], a[15]), 2);
} else {
b[0] = vaddq_s16(a[0], a[15]);
b[1] = vaddq_s16(a[1], a[14]);
b[2] = vaddq_s16(a[2], a[13]);
b[3] = vaddq_s16(a[3], a[12]);
b[4] = vaddq_s16(a[4], a[11]);
b[5] = vaddq_s16(a[5], a[10]);
b[6] = vaddq_s16(a[6], a[9]);
b[7] = vaddq_s16(a[7], a[8]);
b[8] = vsubq_s16(a[7], a[8]);
b[9] = vsubq_s16(a[6], a[9]);
b[10] = vsubq_s16(a[5], a[10]);
b[11] = vsubq_s16(a[4], a[11]);
b[12] = vsubq_s16(a[3], a[12]);
b[13] = vsubq_s16(a[2], a[13]);
b[14] = vsubq_s16(a[1], a[14]);
b[15] = vsubq_s16(a[0], a[15]);
}
}
// Quarter round at the beginning of the second pass. Can't use vrshr (rounding)
// because this only adds 1, not 1 << 2.
static INLINE void partial_round_shift(int16x8_t *a /*[16]*/) {
const int16x8_t one = vdupq_n_s16(1);
a[0] = vshrq_n_s16(vaddq_s16(a[0], one), 2);
a[1] = vshrq_n_s16(vaddq_s16(a[1], one), 2);
a[2] = vshrq_n_s16(vaddq_s16(a[2], one), 2);
a[3] = vshrq_n_s16(vaddq_s16(a[3], one), 2);
a[4] = vshrq_n_s16(vaddq_s16(a[4], one), 2);
a[5] = vshrq_n_s16(vaddq_s16(a[5], one), 2);
a[6] = vshrq_n_s16(vaddq_s16(a[6], one), 2);
a[7] = vshrq_n_s16(vaddq_s16(a[7], one), 2);
a[8] = vshrq_n_s16(vaddq_s16(a[8], one), 2);
a[9] = vshrq_n_s16(vaddq_s16(a[9], one), 2);
a[10] = vshrq_n_s16(vaddq_s16(a[10], one), 2);
a[11] = vshrq_n_s16(vaddq_s16(a[11], one), 2);
a[12] = vshrq_n_s16(vaddq_s16(a[12], one), 2);
a[13] = vshrq_n_s16(vaddq_s16(a[13], one), 2);
a[14] = vshrq_n_s16(vaddq_s16(a[14], one), 2);
a[15] = vshrq_n_s16(vaddq_s16(a[15], one), 2);
}
// fdct_round_shift((a +/- b) * c)
static INLINE void butterfly_one_coeff(const int16x8_t a, const int16x8_t b,
const tran_high_t c, int16x8_t *add,
int16x8_t *sub) {
const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c);
const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c);
const int32x4_t sum0 = vmlal_n_s16(a0, vget_low_s16(b), c);
const int32x4_t sum1 = vmlal_n_s16(a1, vget_high_s16(b), c);
const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c);
const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c);
const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14);
const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14);
const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14);
const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14);
*add = vcombine_s16(rounded0, rounded1);
*sub = vcombine_s16(rounded2, rounded3);
}
// fdct_round_shift(a * c0 +/- b * c1)
static INLINE void butterfly_two_coeff(const int16x8_t a, const int16x8_t b,
const tran_coef_t c0,
const tran_coef_t c1, int16x8_t *add,
int16x8_t *sub) {
const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c0);
const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c0);
const int32x4_t a2 = vmull_n_s16(vget_low_s16(a), c1);
const int32x4_t a3 = vmull_n_s16(vget_high_s16(a), c1);
const int32x4_t sum0 = vmlal_n_s16(a2, vget_low_s16(b), c0);
const int32x4_t sum1 = vmlal_n_s16(a3, vget_high_s16(b), c0);
const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c1);
const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c1);
const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14);
const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14);
const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14);
const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14);
*add = vcombine_s16(rounded0, rounded1);
*sub = vcombine_s16(rounded2, rounded3);
}
// Transpose 8x8 to a new location. Don't use transpose_neon.h because those
// are all in-place.
static INLINE void transpose_8x8(const int16x8_t *a /*[8]*/,
int16x8_t *b /*[8]*/) {
// Swap 16 bit elements.
const int16x8x2_t c0 = vtrnq_s16(a[0], a[1]);
const int16x8x2_t c1 = vtrnq_s16(a[2], a[3]);
const int16x8x2_t c2 = vtrnq_s16(a[4], a[5]);
const int16x8x2_t c3 = vtrnq_s16(a[6], a[7]);
// Swap 32 bit elements.
const int32x4x2_t d0 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[0]),
vreinterpretq_s32_s16(c1.val[0]));
const int32x4x2_t d1 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[1]),
vreinterpretq_s32_s16(c1.val[1]));
const int32x4x2_t d2 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[0]),
vreinterpretq_s32_s16(c3.val[0]));
const int32x4x2_t d3 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[1]),
vreinterpretq_s32_s16(c3.val[1]));
// Swap 64 bit elements
const int16x8x2_t e0 = vpx_vtrnq_s64_to_s16(d0.val[0], d2.val[0]);
const int16x8x2_t e1 = vpx_vtrnq_s64_to_s16(d1.val[0], d3.val[0]);
const int16x8x2_t e2 = vpx_vtrnq_s64_to_s16(d0.val[1], d2.val[1]);
const int16x8x2_t e3 = vpx_vtrnq_s64_to_s16(d1.val[1], d3.val[1]);
b[0] = e0.val[0];
b[1] = e1.val[0];
b[2] = e2.val[0];
b[3] = e3.val[0];
b[4] = e0.val[1];
b[5] = e1.val[1];
b[6] = e2.val[1];
b[7] = e3.val[1];
}
// Main body of fdct16x16.
static void vpx_fdct16x16_body(const int16x8_t *in /*[16]*/,
int16x8_t *out /*[16]*/) {
int16x8_t s[8];
int16x8_t x[4];
int16x8_t step[8];
// stage 1
// From fwd_txfm.c: Work on the first eight values; fdct8(input,
// even_results);"
s[0] = vaddq_s16(in[0], in[7]);
s[1] = vaddq_s16(in[1], in[6]);
s[2] = vaddq_s16(in[2], in[5]);
s[3] = vaddq_s16(in[3], in[4]);
s[4] = vsubq_s16(in[3], in[4]);
s[5] = vsubq_s16(in[2], in[5]);
s[6] = vsubq_s16(in[1], in[6]);
s[7] = vsubq_s16(in[0], in[7]);
// fdct4(step, step);
x[0] = vaddq_s16(s[0], s[3]);
x[1] = vaddq_s16(s[1], s[2]);
x[2] = vsubq_s16(s[1], s[2]);
x[3] = vsubq_s16(s[0], s[3]);
// out[0] = fdct_round_shift((x0 + x1) * cospi_16_64)
// out[8] = fdct_round_shift((x0 - x1) * cospi_16_64)
butterfly_one_coeff(x[0], x[1], cospi_16_64, &out[0], &out[8]);
// out[4] = fdct_round_shift(x3 * cospi_8_64 + x2 * cospi_24_64);
// out[12] = fdct_round_shift(x3 * cospi_24_64 - x2 * cospi_8_64);
butterfly_two_coeff(x[3], x[2], cospi_24_64, cospi_8_64, &out[4], &out[12]);
// Stage 2
// Re-using source s5/s6
// s5 = fdct_round_shift((s6 - s5) * cospi_16_64)
// s6 = fdct_round_shift((s6 + s5) * cospi_16_64)
butterfly_one_coeff(s[6], s[5], cospi_16_64, &s[6], &s[5]);
// Stage 3
x[0] = vaddq_s16(s[4], s[5]);
x[1] = vsubq_s16(s[4], s[5]);
x[2] = vsubq_s16(s[7], s[6]);
x[3] = vaddq_s16(s[7], s[6]);
// Stage 4
// out[2] = fdct_round_shift(x0 * cospi_28_64 + x3 * cospi_4_64)
// out[14] = fdct_round_shift(x3 * cospi_28_64 + x0 * -cospi_4_64)
butterfly_two_coeff(x[3], x[0], cospi_28_64, cospi_4_64, &out[2], &out[14]);
// out[6] = fdct_round_shift(x1 * cospi_12_64 + x2 * cospi_20_64)
// out[10] = fdct_round_shift(x2 * cospi_12_64 + x1 * -cospi_20_64)
butterfly_two_coeff(x[2], x[1], cospi_12_64, cospi_20_64, &out[10], &out[6]);
// step 2
// From fwd_txfm.c: Work on the next eight values; step1 -> odd_results"
// That file distinguished between "in_high" and "step1" but the only
// difference is that "in_high" is the first 8 values and "step 1" is the
// second. Here, since they are all in one array, "step1" values are += 8.
// step2[2] = fdct_round_shift((step1[5] - step1[2]) * cospi_16_64)
// step2[3] = fdct_round_shift((step1[4] - step1[3]) * cospi_16_64)
// step2[4] = fdct_round_shift((step1[4] + step1[3]) * cospi_16_64)
// step2[5] = fdct_round_shift((step1[5] + step1[2]) * cospi_16_64)
butterfly_one_coeff(in[13], in[10], cospi_16_64, &s[5], &s[2]);
butterfly_one_coeff(in[12], in[11], cospi_16_64, &s[4], &s[3]);
// step 3
s[0] = vaddq_s16(in[8], s[3]);
s[1] = vaddq_s16(in[9], s[2]);
x[0] = vsubq_s16(in[9], s[2]);
x[1] = vsubq_s16(in[8], s[3]);
x[2] = vsubq_s16(in[15], s[4]);
x[3] = vsubq_s16(in[14], s[5]);
s[6] = vaddq_s16(in[14], s[5]);
s[7] = vaddq_s16(in[15], s[4]);
// step 4
// step2[1] = fdct_round_shift(step3[1] *-cospi_8_64 + step3[6] * cospi_24_64)
// step2[6] = fdct_round_shift(step3[1] * cospi_24_64 + step3[6] * cospi_8_64)
butterfly_two_coeff(s[6], s[1], cospi_24_64, cospi_8_64, &s[6], &s[1]);
// step2[2] = fdct_round_shift(step3[2] * cospi_24_64 + step3[5] * cospi_8_64)
// step2[5] = fdct_round_shift(step3[2] * cospi_8_64 - step3[5] * cospi_24_64)
butterfly_two_coeff(x[0], x[3], cospi_8_64, cospi_24_64, &s[2], &s[5]);
// step 5
step[0] = vaddq_s16(s[0], s[1]);
step[1] = vsubq_s16(s[0], s[1]);
step[2] = vaddq_s16(x[1], s[2]);
step[3] = vsubq_s16(x[1], s[2]);
step[4] = vsubq_s16(x[2], s[5]);
step[5] = vaddq_s16(x[2], s[5]);
step[6] = vsubq_s16(s[7], s[6]);
step[7] = vaddq_s16(s[7], s[6]);
// step 6
// out[1] = fdct_round_shift(step1[0] * cospi_30_64 + step1[7] * cospi_2_64)
// out[9] = fdct_round_shift(step1[1] * cospi_14_64 + step1[6] * cospi_18_64)
// out[5] = fdct_round_shift(step1[2] * cospi_22_64 + step1[5] * cospi_10_64)
// out[13] = fdct_round_shift(step1[3] * cospi_6_64 + step1[4] * cospi_26_64)
// out[3] = fdct_round_shift(step1[3] * -cospi_26_64 + step1[4] * cospi_6_64)
// out[11] = fdct_round_shift(step1[2] * -cospi_10_64 + step1[5] *
// cospi_22_64)
// out[7] = fdct_round_shift(step1[1] * -cospi_18_64 + step1[6] * cospi_14_64)
// out[15] = fdct_round_shift(step1[0] * -cospi_2_64 + step1[7] * cospi_30_64)
butterfly_two_coeff(step[6], step[1], cospi_14_64, cospi_18_64, &out[9],
&out[7]);
butterfly_two_coeff(step[7], step[0], cospi_30_64, cospi_2_64, &out[1],
&out[15]);
butterfly_two_coeff(step[4], step[3], cospi_6_64, cospi_26_64, &out[13],
&out[3]);
butterfly_two_coeff(step[5], step[2], cospi_22_64, cospi_10_64, &out[5],
&out[11]);
}
#endif // VPX_VPX_DSP_ARM_FDCT16X16_NEON_H_
libs/libvpx/vpx_dsp/arm/fdct32x32_neon.c
+1 -1
View File
@@ -153,7 +153,7 @@ static INLINE void load(const int16_t *a, int stride, int16x8_t *b) {
do { \
store_s16q_to_tran_low(dest, src[index]); \
dest += 8; \
} while (0);
} while (0)
// Store 32 16x8 values, assuming stride == 32.
// Slight twist: store horizontally in blocks of 8.
libs/libvpx/vpx_dsp/arm/fdct_neon.c
+10 -51
View File
@@ -15,6 +15,7 @@
#include "vpx_dsp/txfm_common.h"
#include "vpx_dsp/vpx_dsp_common.h"
#include "vpx_dsp/arm/idct_neon.h"
#include "vpx_dsp/arm/fdct_neon.h"
#include "vpx_dsp/arm/mem_neon.h"
#include "vpx_dsp/arm/transpose_neon.h"
@@ -22,67 +23,25 @@ void vpx_fdct4x4_neon(const int16_t *input, tran_low_t *final_output,
int stride) {
int i;
// input[M * stride] * 16
int16x4_t input_0 = vshl_n_s16(vld1_s16(input + 0 * stride), 4);
int16x4_t input_1 = vshl_n_s16(vld1_s16(input + 1 * stride), 4);
int16x4_t input_2 = vshl_n_s16(vld1_s16(input + 2 * stride), 4);
int16x4_t input_3 = vshl_n_s16(vld1_s16(input + 3 * stride), 4);
int16x4_t in[4];
in[0] = vshl_n_s16(vld1_s16(input + 0 * stride), 4);
in[1] = vshl_n_s16(vld1_s16(input + 1 * stride), 4);
in[2] = vshl_n_s16(vld1_s16(input + 2 * stride), 4);
in[3] = vshl_n_s16(vld1_s16(input + 3 * stride), 4);
// If the very first value != 0, then add 1.
if (input[0] != 0) {
const int16x4_t one = vreinterpret_s16_s64(vdup_n_s64(1));
input_0 = vadd_s16(input_0, one);
in[0] = vadd_s16(in[0], one);
}
for (i = 0; i < 2; ++i) {
const int16x8_t input_01 = vcombine_s16(input_0, input_1);
const int16x8_t input_32 = vcombine_s16(input_3, input_2);
// in_0 +/- in_3, in_1 +/- in_2
const int16x8_t s_01 = vaddq_s16(input_01, input_32);
const int16x8_t s_32 = vsubq_s16(input_01, input_32);
// step_0 +/- step_1, step_2 +/- step_3
const int16x4_t s_0 = vget_low_s16(s_01);
const int16x4_t s_1 = vget_high_s16(s_01);
const int16x4_t s_2 = vget_high_s16(s_32);
const int16x4_t s_3 = vget_low_s16(s_32);
// (s_0 +/- s_1) * cospi_16_64
// Must expand all elements to s32. See 'needs32' comment in fwd_txfm.c.
const int32x4_t s_0_p_s_1 = vaddl_s16(s_0, s_1);
const int32x4_t s_0_m_s_1 = vsubl_s16(s_0, s_1);
const int32x4_t temp1 = vmulq_n_s32(s_0_p_s_1, cospi_16_64);
const int32x4_t temp2 = vmulq_n_s32(s_0_m_s_1, cospi_16_64);
// fdct_round_shift
int16x4_t out_0 = vrshrn_n_s32(temp1, DCT_CONST_BITS);
int16x4_t out_2 = vrshrn_n_s32(temp2, DCT_CONST_BITS);
// s_3 * cospi_8_64 + s_2 * cospi_24_64
// s_3 * cospi_24_64 - s_2 * cospi_8_64
const int32x4_t s_3_cospi_8_64 = vmull_n_s16(s_3, cospi_8_64);
const int32x4_t s_3_cospi_24_64 = vmull_n_s16(s_3, cospi_24_64);
const int32x4_t temp3 = vmlal_n_s16(s_3_cospi_8_64, s_2, cospi_24_64);
const int32x4_t temp4 = vmlsl_n_s16(s_3_cospi_24_64, s_2, cospi_8_64);
// fdct_round_shift
int16x4_t out_1 = vrshrn_n_s32(temp3, DCT_CONST_BITS);
int16x4_t out_3 = vrshrn_n_s32(temp4, DCT_CONST_BITS);
transpose_s16_4x4d(&out_0, &out_1, &out_2, &out_3);
input_0 = out_0;
input_1 = out_1;
input_2 = out_2;
input_3 = out_3;
vpx_fdct4x4_pass1_neon(in);
}
{
// Not quite a rounding shift. Only add 1 despite shifting by 2.
const int16x8_t one = vdupq_n_s16(1);
int16x8_t out_01 = vcombine_s16(input_0, input_1);
int16x8_t out_23 = vcombine_s16(input_2, input_3);
int16x8_t out_01 = vcombine_s16(in[0], in[1]);
int16x8_t out_23 = vcombine_s16(in[2], in[3]);
out_01 = vshrq_n_s16(vaddq_s16(out_01, one), 2);
out_23 = vshrq_n_s16(vaddq_s16(out_23, one), 2);
store_s16q_to_tran_low(final_output + 0 * 8, out_01);
libs/libvpx/vpx_dsp/arm/fdct_neon.h
+213
View File
@@ -0,0 +1,213 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VPX_VPX_DSP_ARM_FDCT_NEON_H_
#define VPX_VPX_DSP_ARM_FDCT_NEON_H_
#include <arm_neon.h>
static INLINE void vpx_fdct4x4_pass1_neon(int16x4_t *in) {
const int16x8_t input_01 = vcombine_s16(in[0], in[1]);
const int16x8_t input_32 = vcombine_s16(in[3], in[2]);
// in_0 +/- in_3, in_1 +/- in_2
const int16x8_t s_01 = vaddq_s16(input_01, input_32);
const int16x8_t s_32 = vsubq_s16(input_01, input_32);
// step_0 +/- step_1, step_2 +/- step_3
const int16x4_t s_0 = vget_low_s16(s_01);
const int16x4_t s_1 = vget_high_s16(s_01);
const int16x4_t s_2 = vget_high_s16(s_32);
const int16x4_t s_3 = vget_low_s16(s_32);
// (s_0 +/- s_1) * cospi_16_64
// Must expand all elements to s32. See 'needs32' comment in fwd_txfm.c.
const int32x4_t s_0_p_s_1 = vaddl_s16(s_0, s_1);
const int32x4_t s_0_m_s_1 = vsubl_s16(s_0, s_1);
const int32x4_t temp1 = vmulq_n_s32(s_0_p_s_1, cospi_16_64);
const int32x4_t temp2 = vmulq_n_s32(s_0_m_s_1, cospi_16_64);
// fdct_round_shift
int16x4_t out_0 = vrshrn_n_s32(temp1, DCT_CONST_BITS);
int16x4_t out_2 = vrshrn_n_s32(temp2, DCT_CONST_BITS);
// s_3 * cospi_8_64 + s_2 * cospi_24_64
// s_3 * cospi_24_64 - s_2 * cospi_8_64
const int32x4_t s_3_cospi_8_64 = vmull_n_s16(s_3, cospi_8_64);
const int32x4_t s_3_cospi_24_64 = vmull_n_s16(s_3, cospi_24_64);
const int32x4_t temp3 = vmlal_n_s16(s_3_cospi_8_64, s_2, cospi_24_64);
const int32x4_t temp4 = vmlsl_n_s16(s_3_cospi_24_64, s_2, cospi_8_64);
// fdct_round_shift
int16x4_t out_1 = vrshrn_n_s32(temp3, DCT_CONST_BITS);
int16x4_t out_3 = vrshrn_n_s32(temp4, DCT_CONST_BITS);
transpose_s16_4x4d(&out_0, &out_1, &out_2, &out_3);
in[0] = out_0;
in[1] = out_1;
in[2] = out_2;
in[3] = out_3;
}
static INLINE void vpx_fdct8x8_pass1_notranspose_neon(int16x8_t *in,
int16x8_t *out) {
const int16x8_t v_s0 = vaddq_s16(in[0], in[7]);
const int16x8_t v_s1 = vaddq_s16(in[1], in[6]);
const int16x8_t v_s2 = vaddq_s16(in[2], in[5]);
const int16x8_t v_s3 = vaddq_s16(in[3], in[4]);
const int16x8_t v_s4 = vsubq_s16(in[3], in[4]);
const int16x8_t v_s5 = vsubq_s16(in[2], in[5]);
const int16x8_t v_s6 = vsubq_s16(in[1], in[6]);
const int16x8_t v_s7 = vsubq_s16(in[0], in[7]);
// fdct4(step, step);
int16x8_t v_x0 = vaddq_s16(v_s0, v_s3);
int16x8_t v_x1 = vaddq_s16(v_s1, v_s2);
int16x8_t v_x2 = vsubq_s16(v_s1, v_s2);
int16x8_t v_x3 = vsubq_s16(v_s0, v_s3);
// fdct4(step, step);
int32x4_t v_t0_lo = vaddl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1));
int32x4_t v_t0_hi = vaddl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1));
int32x4_t v_t1_lo = vsubl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1));
int32x4_t v_t1_hi = vsubl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1));
int32x4_t v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_24_64);
int32x4_t v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_24_64);
int32x4_t v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_24_64);
int32x4_t v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_24_64);
v_t2_lo = vmlal_n_s16(v_t2_lo, vget_low_s16(v_x3), cospi_8_64);
v_t2_hi = vmlal_n_s16(v_t2_hi, vget_high_s16(v_x3), cospi_8_64);
v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x2), cospi_8_64);
v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x2), cospi_8_64);
v_t0_lo = vmulq_n_s32(v_t0_lo, cospi_16_64);
v_t0_hi = vmulq_n_s32(v_t0_hi, cospi_16_64);
v_t1_lo = vmulq_n_s32(v_t1_lo, cospi_16_64);
v_t1_hi = vmulq_n_s32(v_t1_hi, cospi_16_64);
{
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS);
const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS);
const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS);
const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS);
out[0] = vcombine_s16(a, c); // 00 01 02 03 40 41 42 43
out[2] = vcombine_s16(e, g); // 20 21 22 23 60 61 62 63
out[4] = vcombine_s16(b, d); // 04 05 06 07 44 45 46 47
out[6] = vcombine_s16(f, h); // 24 25 26 27 64 65 66 67
}
// Stage 2
v_x0 = vsubq_s16(v_s6, v_s5);
v_x1 = vaddq_s16(v_s6, v_s5);
v_t0_lo = vmull_n_s16(vget_low_s16(v_x0), cospi_16_64);
v_t0_hi = vmull_n_s16(vget_high_s16(v_x0), cospi_16_64);
v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_16_64);
v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_16_64);
{
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
const int16x8_t ab = vcombine_s16(a, b);
const int16x8_t cd = vcombine_s16(c, d);
// Stage 3
v_x0 = vaddq_s16(v_s4, ab);
v_x1 = vsubq_s16(v_s4, ab);
v_x2 = vsubq_s16(v_s7, cd);
v_x3 = vaddq_s16(v_s7, cd);
}
// Stage 4
v_t0_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_4_64);
v_t0_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_4_64);
v_t0_lo = vmlal_n_s16(v_t0_lo, vget_low_s16(v_x0), cospi_28_64);
v_t0_hi = vmlal_n_s16(v_t0_hi, vget_high_s16(v_x0), cospi_28_64);
v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_12_64);
v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_12_64);
v_t1_lo = vmlal_n_s16(v_t1_lo, vget_low_s16(v_x2), cospi_20_64);
v_t1_hi = vmlal_n_s16(v_t1_hi, vget_high_s16(v_x2), cospi_20_64);
v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_12_64);
v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_12_64);
v_t2_lo = vmlsl_n_s16(v_t2_lo, vget_low_s16(v_x1), cospi_20_64);
v_t2_hi = vmlsl_n_s16(v_t2_hi, vget_high_s16(v_x1), cospi_20_64);
v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_28_64);
v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_28_64);
v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x0), cospi_4_64);
v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x0), cospi_4_64);
{
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS);
const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS);
const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS);
const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS);
out[1] = vcombine_s16(a, c); // 10 11 12 13 50 51 52 53
out[3] = vcombine_s16(e, g); // 30 31 32 33 70 71 72 73
out[5] = vcombine_s16(b, d); // 14 15 16 17 54 55 56 57
out[7] = vcombine_s16(f, h); // 34 35 36 37 74 75 76 77
}
}
static INLINE void vpx_fdct8x8_pass1_neon(int16x8_t *in) {
int16x8_t out[8];
vpx_fdct8x8_pass1_notranspose_neon(in, out);
// transpose 8x8
// Can't use transpose_s16_8x8() because the values are arranged in two 4x8
// columns.
{
// 00 01 02 03 40 41 42 43
// 10 11 12 13 50 51 52 53
// 20 21 22 23 60 61 62 63
// 30 31 32 33 70 71 72 73
// 04 05 06 07 44 45 46 47
// 14 15 16 17 54 55 56 57
// 24 25 26 27 64 65 66 67
// 34 35 36 37 74 75 76 77
const int32x4x2_t r02_s32 =
vtrnq_s32(vreinterpretq_s32_s16(out[0]), vreinterpretq_s32_s16(out[2]));
const int32x4x2_t r13_s32 =
vtrnq_s32(vreinterpretq_s32_s16(out[1]), vreinterpretq_s32_s16(out[3]));
const int32x4x2_t r46_s32 =
vtrnq_s32(vreinterpretq_s32_s16(out[4]), vreinterpretq_s32_s16(out[6]));
const int32x4x2_t r57_s32 =
vtrnq_s32(vreinterpretq_s32_s16(out[5]), vreinterpretq_s32_s16(out[7]));
const int16x8x2_t r01_s16 =
vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[0]),
vreinterpretq_s16_s32(r13_s32.val[0]));
const int16x8x2_t r23_s16 =
vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[1]),
vreinterpretq_s16_s32(r13_s32.val[1]));
const int16x8x2_t r45_s16 =
vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[0]),
vreinterpretq_s16_s32(r57_s32.val[0]));
const int16x8x2_t r67_s16 =
vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[1]),
vreinterpretq_s16_s32(r57_s32.val[1]));
in[0] = r01_s16.val[0];
in[1] = r01_s16.val[1];
in[2] = r23_s16.val[0];
in[3] = r23_s16.val[1];
in[4] = r45_s16.val[0];
in[5] = r45_s16.val[1];
in[6] = r67_s16.val[0];
in[7] = r67_s16.val[1];
// 00 10 20 30 40 50 60 70
// 01 11 21 31 41 51 61 71
// 02 12 22 32 42 52 62 72
// 03 13 23 33 43 53 63 73
// 04 14 24 34 44 54 64 74
// 05 15 25 35 45 55 65 75
// 06 16 26 36 46 56 66 76
// 07 17 27 37 47 57 67 77
}
}
#endif // VPX_VPX_DSP_ARM_FDCT_NEON_H_
libs/libvpx/vpx_dsp/arm/fdct_partial_neon.c
+11 -21
View File
@@ -15,19 +15,10 @@
#include "vpx_dsp/arm/mem_neon.h"
#include "vpx_dsp/arm/sum_neon.h"
static INLINE tran_low_t get_lane(const int32x2_t a) {
#if CONFIG_VP9_HIGHBITDEPTH
return vget_lane_s32(a, 0);
#else
return vget_lane_s16(vreinterpret_s16_s32(a), 0);
#endif // CONFIG_VP9_HIGHBITDETPH
}
void vpx_fdct4x4_1_neon(const int16_t *input, tran_low_t *output, int stride) {
int16x4_t a0, a1, a2, a3;
int16x8_t b0, b1;
int16x8_t c;
int32x2_t d;
a0 = vld1_s16(input);
input += stride;
@@ -42,9 +33,7 @@ void vpx_fdct4x4_1_neon(const int16_t *input, tran_low_t *output, int stride) {
c = vaddq_s16(b0, b1);
d = horizontal_add_int16x8(c);
output[0] = get_lane(vshl_n_s32(d, 1));
output[0] = (tran_low_t)(horizontal_add_int16x8(c) << 1);
output[1] = 0;
}
@@ -57,7 +46,7 @@ void vpx_fdct8x8_1_neon(const int16_t *input, tran_low_t *output, int stride) {
sum = vaddq_s16(sum, input_00);
}
output[0] = get_lane(horizontal_add_int16x8(sum));
output[0] = (tran_low_t)horizontal_add_int16x8(sum);
output[1] = 0;
}
@@ -66,7 +55,7 @@ void vpx_fdct16x16_1_neon(const int16_t *input, tran_low_t *output,
int r;
int16x8_t left = vld1q_s16(input);
int16x8_t right = vld1q_s16(input + 8);
int32x2_t sum;
int32_t sum;
input += stride;
for (r = 1; r < 16; ++r) {
@@ -77,9 +66,9 @@ void vpx_fdct16x16_1_neon(const int16_t *input, tran_low_t *output,
right = vaddq_s16(right, b);
}
sum = vadd_s32(horizontal_add_int16x8(left), horizontal_add_int16x8(right));
sum = horizontal_add_int16x8(left) + horizontal_add_int16x8(right);
output[0] = get_lane(vshr_n_s32(sum, 1));
output[0] = (tran_low_t)(sum >> 1);
output[1] = 0;
}
@@ -90,7 +79,7 @@ void vpx_fdct32x32_1_neon(const int16_t *input, tran_low_t *output,
int16x8_t a1 = vld1q_s16(input + 8);
int16x8_t a2 = vld1q_s16(input + 16);
int16x8_t a3 = vld1q_s16(input + 24);
int32x2_t sum;
int32_t sum;
input += stride;
for (r = 1; r < 32; ++r) {
@@ -105,9 +94,10 @@ void vpx_fdct32x32_1_neon(const int16_t *input, tran_low_t *output,
a3 = vaddq_s16(a3, b3);
}
sum = vadd_s32(horizontal_add_int16x8(a0), horizontal_add_int16x8(a1));
sum = vadd_s32(sum, horizontal_add_int16x8(a2));
sum = vadd_s32(sum, horizontal_add_int16x8(a3));
output[0] = get_lane(vshr_n_s32(sum, 3));
sum = horizontal_add_int16x8(a0);
sum += horizontal_add_int16x8(a1);
sum += horizontal_add_int16x8(a2);
sum += horizontal_add_int16x8(a3);
output[0] = (tran_low_t)(sum >> 3);
output[1] = 0;
}
libs/libvpx/vpx_dsp/arm/fwd_txfm_neon.c
+35 -177
View File
@@ -15,196 +15,54 @@
#include "vpx_dsp/txfm_common.h"
#include "vpx_dsp/vpx_dsp_common.h"
#include "vpx_dsp/arm/idct_neon.h"
#include "vpx_dsp/arm/fdct_neon.h"
#include "vpx_dsp/arm/mem_neon.h"
void vpx_fdct8x8_neon(const int16_t *input, tran_low_t *final_output,
int stride) {
int i;
// stage 1
int16x8_t input_0 = vshlq_n_s16(vld1q_s16(&input[0 * stride]), 2);
int16x8_t input_1 = vshlq_n_s16(vld1q_s16(&input[1 * stride]), 2);
int16x8_t input_2 = vshlq_n_s16(vld1q_s16(&input[2 * stride]), 2);
int16x8_t input_3 = vshlq_n_s16(vld1q_s16(&input[3 * stride]), 2);
int16x8_t input_4 = vshlq_n_s16(vld1q_s16(&input[4 * stride]), 2);
int16x8_t input_5 = vshlq_n_s16(vld1q_s16(&input[5 * stride]), 2);
int16x8_t input_6 = vshlq_n_s16(vld1q_s16(&input[6 * stride]), 2);
int16x8_t input_7 = vshlq_n_s16(vld1q_s16(&input[7 * stride]), 2);
int16x8_t in[8];
in[0] = vshlq_n_s16(vld1q_s16(&input[0 * stride]), 2);
in[1] = vshlq_n_s16(vld1q_s16(&input[1 * stride]), 2);
in[2] = vshlq_n_s16(vld1q_s16(&input[2 * stride]), 2);
in[3] = vshlq_n_s16(vld1q_s16(&input[3 * stride]), 2);
in[4] = vshlq_n_s16(vld1q_s16(&input[4 * stride]), 2);
in[5] = vshlq_n_s16(vld1q_s16(&input[5 * stride]), 2);
in[6] = vshlq_n_s16(vld1q_s16(&input[6 * stride]), 2);
in[7] = vshlq_n_s16(vld1q_s16(&input[7 * stride]), 2);
for (i = 0; i < 2; ++i) {
int16x8_t out_0, out_1, out_2, out_3, out_4, out_5, out_6, out_7;
const int16x8_t v_s0 = vaddq_s16(input_0, input_7);
const int16x8_t v_s1 = vaddq_s16(input_1, input_6);
const int16x8_t v_s2 = vaddq_s16(input_2, input_5);
const int16x8_t v_s3 = vaddq_s16(input_3, input_4);
const int16x8_t v_s4 = vsubq_s16(input_3, input_4);
const int16x8_t v_s5 = vsubq_s16(input_2, input_5);
const int16x8_t v_s6 = vsubq_s16(input_1, input_6);
const int16x8_t v_s7 = vsubq_s16(input_0, input_7);
// fdct4(step, step);
int16x8_t v_x0 = vaddq_s16(v_s0, v_s3);
int16x8_t v_x1 = vaddq_s16(v_s1, v_s2);
int16x8_t v_x2 = vsubq_s16(v_s1, v_s2);
int16x8_t v_x3 = vsubq_s16(v_s0, v_s3);
// fdct4(step, step);
int32x4_t v_t0_lo = vaddl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1));
int32x4_t v_t0_hi = vaddl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1));
int32x4_t v_t1_lo = vsubl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1));
int32x4_t v_t1_hi = vsubl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1));
int32x4_t v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_24_64);
int32x4_t v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_24_64);
int32x4_t v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_24_64);
int32x4_t v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_24_64);
v_t2_lo = vmlal_n_s16(v_t2_lo, vget_low_s16(v_x3), cospi_8_64);
v_t2_hi = vmlal_n_s16(v_t2_hi, vget_high_s16(v_x3), cospi_8_64);
v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x2), cospi_8_64);
v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x2), cospi_8_64);
v_t0_lo = vmulq_n_s32(v_t0_lo, cospi_16_64);
v_t0_hi = vmulq_n_s32(v_t0_hi, cospi_16_64);
v_t1_lo = vmulq_n_s32(v_t1_lo, cospi_16_64);
v_t1_hi = vmulq_n_s32(v_t1_hi, cospi_16_64);
{
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS);
const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS);
const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS);
const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS);
out_0 = vcombine_s16(a, c); // 00 01 02 03 40 41 42 43
out_2 = vcombine_s16(e, g); // 20 21 22 23 60 61 62 63
out_4 = vcombine_s16(b, d); // 04 05 06 07 44 45 46 47
out_6 = vcombine_s16(f, h); // 24 25 26 27 64 65 66 67
}
// Stage 2
v_x0 = vsubq_s16(v_s6, v_s5);
v_x1 = vaddq_s16(v_s6, v_s5);
v_t0_lo = vmull_n_s16(vget_low_s16(v_x0), cospi_16_64);
v_t0_hi = vmull_n_s16(vget_high_s16(v_x0), cospi_16_64);
v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_16_64);
v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_16_64);
{
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
const int16x8_t ab = vcombine_s16(a, b);
const int16x8_t cd = vcombine_s16(c, d);
// Stage 3
v_x0 = vaddq_s16(v_s4, ab);
v_x1 = vsubq_s16(v_s4, ab);
v_x2 = vsubq_s16(v_s7, cd);
v_x3 = vaddq_s16(v_s7, cd);
}
// Stage 4
v_t0_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_4_64);
v_t0_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_4_64);
v_t0_lo = vmlal_n_s16(v_t0_lo, vget_low_s16(v_x0), cospi_28_64);
v_t0_hi = vmlal_n_s16(v_t0_hi, vget_high_s16(v_x0), cospi_28_64);
v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_12_64);
v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_12_64);
v_t1_lo = vmlal_n_s16(v_t1_lo, vget_low_s16(v_x2), cospi_20_64);
v_t1_hi = vmlal_n_s16(v_t1_hi, vget_high_s16(v_x2), cospi_20_64);
v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_12_64);
v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_12_64);
v_t2_lo = vmlsl_n_s16(v_t2_lo, vget_low_s16(v_x1), cospi_20_64);
v_t2_hi = vmlsl_n_s16(v_t2_hi, vget_high_s16(v_x1), cospi_20_64);
v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_28_64);
v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_28_64);
v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x0), cospi_4_64);
v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x0), cospi_4_64);
{
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS);
const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS);
const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS);
const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS);
out_1 = vcombine_s16(a, c); // 10 11 12 13 50 51 52 53
out_3 = vcombine_s16(e, g); // 30 31 32 33 70 71 72 73
out_5 = vcombine_s16(b, d); // 14 15 16 17 54 55 56 57
out_7 = vcombine_s16(f, h); // 34 35 36 37 74 75 76 77
}
// transpose 8x8
// Can't use transpose_s16_8x8() because the values are arranged in two 4x8
// columns.
{
// 00 01 02 03 40 41 42 43
// 10 11 12 13 50 51 52 53
// 20 21 22 23 60 61 62 63
// 30 31 32 33 70 71 72 73
// 04 05 06 07 44 45 46 47
// 14 15 16 17 54 55 56 57
// 24 25 26 27 64 65 66 67
// 34 35 36 37 74 75 76 77
const int32x4x2_t r02_s32 =
vtrnq_s32(vreinterpretq_s32_s16(out_0), vreinterpretq_s32_s16(out_2));
const int32x4x2_t r13_s32 =
vtrnq_s32(vreinterpretq_s32_s16(out_1), vreinterpretq_s32_s16(out_3));
const int32x4x2_t r46_s32 =
vtrnq_s32(vreinterpretq_s32_s16(out_4), vreinterpretq_s32_s16(out_6));
const int32x4x2_t r57_s32 =
vtrnq_s32(vreinterpretq_s32_s16(out_5), vreinterpretq_s32_s16(out_7));
const int16x8x2_t r01_s16 =
vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[0]),
vreinterpretq_s16_s32(r13_s32.val[0]));
const int16x8x2_t r23_s16 =
vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[1]),
vreinterpretq_s16_s32(r13_s32.val[1]));
const int16x8x2_t r45_s16 =
vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[0]),
vreinterpretq_s16_s32(r57_s32.val[0]));
const int16x8x2_t r67_s16 =
vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[1]),
vreinterpretq_s16_s32(r57_s32.val[1]));
input_0 = r01_s16.val[0];
input_1 = r01_s16.val[1];
input_2 = r23_s16.val[0];
input_3 = r23_s16.val[1];
input_4 = r45_s16.val[0];
input_5 = r45_s16.val[1];
input_6 = r67_s16.val[0];
input_7 = r67_s16.val[1];
// 00 10 20 30 40 50 60 70
// 01 11 21 31 41 51 61 71
// 02 12 22 32 42 52 62 72
// 03 13 23 33 43 53 63 73
// 04 14 24 34 44 54 64 74
// 05 15 25 35 45 55 65 75
// 06 16 26 36 46 56 66 76
// 07 17 27 37 47 57 67 77
}
vpx_fdct8x8_pass1_neon(in);
} // for
{
// from vpx_dct_sse2.c
// Post-condition (division by two)
// division of two 16 bits signed numbers using shifts
// n / 2 = (n - (n >> 15)) >> 1
const int16x8_t sign_in0 = vshrq_n_s16(input_0, 15);
const int16x8_t sign_in1 = vshrq_n_s16(input_1, 15);
const int16x8_t sign_in2 = vshrq_n_s16(input_2, 15);
const int16x8_t sign_in3 = vshrq_n_s16(input_3, 15);
const int16x8_t sign_in4 = vshrq_n_s16(input_4, 15);
const int16x8_t sign_in5 = vshrq_n_s16(input_5, 15);
const int16x8_t sign_in6 = vshrq_n_s16(input_6, 15);
const int16x8_t sign_in7 = vshrq_n_s16(input_7, 15);
input_0 = vhsubq_s16(input_0, sign_in0);
input_1 = vhsubq_s16(input_1, sign_in1);
input_2 = vhsubq_s16(input_2, sign_in2);
input_3 = vhsubq_s16(input_3, sign_in3);
input_4 = vhsubq_s16(input_4, sign_in4);
input_5 = vhsubq_s16(input_5, sign_in5);
input_6 = vhsubq_s16(input_6, sign_in6);
input_7 = vhsubq_s16(input_7, sign_in7);
const int16x8_t sign_in0 = vshrq_n_s16(in[0], 15);
const int16x8_t sign_in1 = vshrq_n_s16(in[1], 15);
const int16x8_t sign_in2 = vshrq_n_s16(in[2], 15);
const int16x8_t sign_in3 = vshrq_n_s16(in[3], 15);
const int16x8_t sign_in4 = vshrq_n_s16(in[4], 15);
const int16x8_t sign_in5 = vshrq_n_s16(in[5], 15);
const int16x8_t sign_in6 = vshrq_n_s16(in[6], 15);
const int16x8_t sign_in7 = vshrq_n_s16(in[7], 15);
in[0] = vhsubq_s16(in[0], sign_in0);
in[1] = vhsubq_s16(in[1], sign_in1);
in[2] = vhsubq_s16(in[2], sign_in2);
in[3] = vhsubq_s16(in[3], sign_in3);
in[4] = vhsubq_s16(in[4], sign_in4);
in[5] = vhsubq_s16(in[5], sign_in5);
in[6] = vhsubq_s16(in[6], sign_in6);
in[7] = vhsubq_s16(in[7], sign_in7);
// store results
store_s16q_to_tran_low(final_output + 0 * 8, input_0);
store_s16q_to_tran_low(final_output + 1 * 8, input_1);
store_s16q_to_tran_low(final_output + 2 * 8, input_2);
store_s16q_to_tran_low(final_output + 3 * 8, input_3);
store_s16q_to_tran_low(final_output + 4 * 8, input_4);
store_s16q_to_tran_low(final_output + 5 * 8, input_5);
store_s16q_to_tran_low(final_output + 6 * 8, input_6);
store_s16q_to_tran_low(final_output + 7 * 8, input_7);
store_s16q_to_tran_low(final_output + 0 * 8, in[0]);
store_s16q_to_tran_low(final_output + 1 * 8, in[1]);
store_s16q_to_tran_low(final_output + 2 * 8, in[2]);
store_s16q_to_tran_low(final_output + 3 * 8, in[3]);
store_s16q_to_tran_low(final_output + 4 * 8, in[4]);
store_s16q_to_tran_low(final_output + 5 * 8, in[5]);
store_s16q_to_tran_low(final_output + 6 * 8, in[6]);
store_s16q_to_tran_low(final_output + 7 * 8, in[7]);
}
}
libs/libvpx/vpx_dsp/arm/highbd_loopfilter_neon.c
+15
View File
@@ -661,6 +661,17 @@ void vpx_highbd_lpf_vertical_8_dual_neon(
vpx_highbd_lpf_vertical_8_neon(s + 8 * p, p, blimit1, limit1, thresh1, bd);
}
// Quiet warnings of the form: 'vpx_dsp/arm/highbd_loopfilter_neon.c|675 col 67|
// warning: 'oq1' may be used uninitialized in this function
// [-Wmaybe-uninitialized]', for oq1-op1. Without reworking the code or adding
// an additional branch this warning cannot be silenced otherwise. The
// loopfilter is only called when needed for a block so these output pixels
// will be set.
#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
static void lpf_horizontal_16_kernel(uint16_t *s, int p,
const uint16x8_t blimit_vec,
const uint16x8_t limit_vec,
@@ -723,6 +734,10 @@ static void lpf_vertical_16_kernel(uint16_t *s, int p,
}
}
#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__)
#pragma GCC diagnostic pop
#endif
void vpx_highbd_lpf_horizontal_16_neon(uint16_t *s, int p,
const uint8_t *blimit,
const uint8_t *limit,
libs/libvpx/vpx_dsp/arm/idct4x4_add_neon.asm
+1 -1
View File
@@ -17,7 +17,7 @@
INCLUDE vpx_dsp/arm/idct_neon.asm.S
AREA Block, CODE, READONLY ; name this block of code
AREA Block, CODE, READONLY
;void vpx_idct4x4_16_add_neon(int16_t *input, uint8_t *dest, int stride)
;
; r0 int16_t input
libs/libvpx/vpx_dsp/arm/loopfilter_neon.c
+15
View File
@@ -975,6 +975,17 @@ FUN_LPF_16_KERNEL(_, 8) // lpf_16_kernel
FUN_LPF_16_KERNEL(_dual_, 16) // lpf_16_dual_kernel
#undef FUN_LPF_16_KERNEL
// Quiet warnings of the form: 'vpx_dsp/arm/loopfilter_neon.c|981 col 42|
// warning: 'oq1' may be used uninitialized in this function
// [-Wmaybe-uninitialized]', for oq1-op1. Without reworking the code or adding
// an additional branch this warning cannot be silenced otherwise. The
// loopfilter is only called when needed for a block so these output pixels
// will be set.
#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
void vpx_lpf_horizontal_16_neon(uint8_t *s, int p, const uint8_t *blimit,
const uint8_t *limit, const uint8_t *thresh) {
uint8x8_t p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7, op6,
@@ -1090,3 +1101,7 @@ void vpx_lpf_vertical_16_dual_neon(uint8_t *s, int p, const uint8_t *blimit,
vget_high_u8(oq0), vget_high_u8(oq1));
}
}
#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__)
#pragma GCC diagnostic pop
#endif
libs/libvpx/vpx_dsp/arm/mem_neon.h
+26 -6
View File
@@ -19,6 +19,24 @@
#include "vpx/vpx_integer.h"
#include "vpx_dsp/vpx_dsp_common.h"
// Support for these xN intrinsics is lacking in older versions of GCC.
#if defined(__GNUC__) && !defined(__clang__)
#if __GNUC__ < 8 || defined(__arm__)
static INLINE uint8x16x2_t vld1q_u8_x2(uint8_t const *ptr) {
uint8x16x2_t res = { { vld1q_u8(ptr + 0 * 16), vld1q_u8(ptr + 1 * 16) } };
return res;
}
#endif
#if __GNUC__ < 9 || defined(__arm__)
static INLINE uint8x16x3_t vld1q_u8_x3(uint8_t const *ptr) {
uint8x16x3_t res = { { vld1q_u8(ptr + 0 * 16), vld1q_u8(ptr + 1 * 16),
vld1q_u8(ptr + 2 * 16) } };
return res;
}
#endif
#endif
static INLINE int16x4_t create_s16x4_neon(const int16_t c0, const int16_t c1,
const int16_t c2, const int16_t c3) {
return vcreate_s16((uint16_t)c0 | ((uint32_t)c1 << 16) |
@@ -95,7 +113,8 @@ static INLINE void uint32_to_mem(uint8_t *buf, uint32_t a) {
}
// Load 2 sets of 4 bytes when alignment is not guaranteed.
static INLINE uint8x8_t load_unaligned_u8(const uint8_t *buf, int stride) {
static INLINE uint8x8_t load_unaligned_u8(const uint8_t *buf,
ptrdiff_t stride) {
uint32_t a;
uint32x2_t a_u32 = vdup_n_u32(0);
if (stride == 4) return vld1_u8(buf);
@@ -108,7 +127,7 @@ static INLINE uint8x8_t load_unaligned_u8(const uint8_t *buf, int stride) {
}
// Store 2 sets of 4 bytes when alignment is not guaranteed.
static INLINE void store_unaligned_u8(uint8_t *buf, int stride,
static INLINE void store_unaligned_u8(uint8_t *buf, ptrdiff_t stride,
const uint8x8_t a) {
const uint32x2_t a_u32 = vreinterpret_u32_u8(a);
if (stride == 4) {
@@ -121,7 +140,8 @@ static INLINE void store_unaligned_u8(uint8_t *buf, int stride,
}
// Load 4 sets of 4 bytes when alignment is not guaranteed.
static INLINE uint8x16_t load_unaligned_u8q(const uint8_t *buf, int stride) {
static INLINE uint8x16_t load_unaligned_u8q(const uint8_t *buf,
ptrdiff_t stride) {
uint32_t a;
uint32x4_t a_u32 = vdupq_n_u32(0);
if (stride == 4) return vld1q_u8(buf);
@@ -141,7 +161,7 @@ static INLINE uint8x16_t load_unaligned_u8q(const uint8_t *buf, int stride) {
}
// Store 4 sets of 4 bytes when alignment is not guaranteed.
static INLINE void store_unaligned_u8q(uint8_t *buf, int stride,
static INLINE void store_unaligned_u8q(uint8_t *buf, ptrdiff_t stride,
const uint8x16_t a) {
const uint32x4_t a_u32 = vreinterpretq_u32_u8(a);
if (stride == 4) {
@@ -158,7 +178,7 @@ static INLINE void store_unaligned_u8q(uint8_t *buf, int stride,
}
// Load 2 sets of 4 bytes when alignment is guaranteed.
static INLINE uint8x8_t load_u8(const uint8_t *buf, int stride) {
static INLINE uint8x8_t load_u8(const uint8_t *buf, ptrdiff_t stride) {
uint32x2_t a = vdup_n_u32(0);
assert(!((intptr_t)buf % sizeof(uint32_t)));
@@ -171,7 +191,7 @@ static INLINE uint8x8_t load_u8(const uint8_t *buf, int stride) {
}
// Store 2 sets of 4 bytes when alignment is guaranteed.
static INLINE void store_u8(uint8_t *buf, int stride, const uint8x8_t a) {
static INLINE void store_u8(uint8_t *buf, ptrdiff_t stride, const uint8x8_t a) {
uint32x2_t a_u32 = vreinterpret_u32_u8(a);
assert(!((intptr_t)buf % sizeof(uint32_t)));
libs/libvpx/vpx_dsp/arm/quantize_neon.c
+3 -7
View File
@@ -32,8 +32,8 @@ static INLINE void calculate_dqcoeff_and_store(const int16x8_t qcoeff,
}
void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int skip_block, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
uint16_t *eob_ptr, const int16_t *scan,
@@ -42,8 +42,6 @@ void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16x8_t neg_one = vdupq_n_s16(-1);
uint16x8_t eob_max;
(void)scan;
(void)skip_block;
assert(!skip_block);
// Process first 8 values which include a dc component.
{
@@ -189,7 +187,7 @@ static INLINE void calculate_dqcoeff_and_store_32x32(const int16x8_t qcoeff,
// Main difference is that zbin values are halved before comparison and dqcoeff
// values are divided by 2. zbin is rounded but dqcoeff is not.
void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int skip_block, const int16_t *zbin_ptr,
const int16_t *zbin_ptr,
const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
@@ -202,8 +200,6 @@ void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int i;
(void)scan;
(void)n_coeffs; // Because we will always calculate 32*32.
(void)skip_block;
assert(!skip_block);
// Process first 8 values which include a dc component.
{
libs/libvpx/vpx_dsp/arm/sad4d_neon.c
+321 -49
View File
@@ -31,10 +31,12 @@ static INLINE uint8x8_t load_unaligned_2_buffers(const void *const buf0,
static INLINE void sad4x_4d(const uint8_t *const src_ptr, const int src_stride,
const uint8_t *const ref_array[4],
const int ref_stride, const int height,
uint32_t *const res) {
uint32_t sad_array[4]) {
int i;
uint16x8_t abs[2] = { vdupq_n_u16(0), vdupq_n_u16(0) };
#if !defined(__aarch64__)
uint16x4_t a[2];
#endif
uint32x4_t r;
assert(!((intptr_t)src_ptr % sizeof(uint32_t)));
@@ -51,29 +53,40 @@ static INLINE void sad4x_4d(const uint8_t *const src_ptr, const int src_stride,
abs[1] = vabal_u8(abs[1], s, ref23);
}
#if defined(__aarch64__)
abs[0] = vpaddq_u16(abs[0], abs[1]);
r = vpaddlq_u16(abs[0]);
#else
a[0] = vpadd_u16(vget_low_u16(abs[0]), vget_high_u16(abs[0]));
a[1] = vpadd_u16(vget_low_u16(abs[1]), vget_high_u16(abs[1]));
r = vpaddlq_u16(vcombine_u16(a[0], a[1]));
vst1q_u32(res, r);
#endif
vst1q_u32(sad_array, r);
}
void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res) {
sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 4, res);
uint32_t sad_array[4]) {
sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 4, sad_array);
}
void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res) {
sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 8, res);
uint32_t sad_array[4]) {
sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 8, sad_array);
}
////////////////////////////////////////////////////////////////////////////////
// Can handle 512 pixels' sad sum (such as 16x32 or 32x16)
static INLINE void sad_512_pel_final_neon(const uint16x8_t *sum /*[4]*/,
uint32_t *const res) {
static INLINE void sad_512_pel_final_neon(const uint16x8_t sum[4],
uint32_t sad_array[4]) {
#if defined(__aarch64__)
const uint16x8_t a0 = vpaddq_u16(sum[0], sum[1]);
const uint16x8_t a1 = vpaddq_u16(sum[2], sum[3]);
const uint16x8_t b0 = vpaddq_u16(a0, a1);
const uint32x4_t r = vpaddlq_u16(b0);
#else
const uint16x4_t a0 = vadd_u16(vget_low_u16(sum[0]), vget_high_u16(sum[0]));
const uint16x4_t a1 = vadd_u16(vget_low_u16(sum[1]), vget_high_u16(sum[1]));
const uint16x4_t a2 = vadd_u16(vget_low_u16(sum[2]), vget_high_u16(sum[2]));
@@ -81,12 +94,23 @@ static INLINE void sad_512_pel_final_neon(const uint16x8_t *sum /*[4]*/,
const uint16x4_t b0 = vpadd_u16(a0, a1);
const uint16x4_t b1 = vpadd_u16(a2, a3);
const uint32x4_t r = vpaddlq_u16(vcombine_u16(b0, b1));
vst1q_u32(res, r);
#endif
vst1q_u32(sad_array, r);
}
#if defined(__arm__) || !defined(__ARM_FEATURE_DOTPROD)
// Can handle 1024 pixels' sad sum (such as 32x32)
static INLINE void sad_1024_pel_final_neon(const uint16x8_t *sum /*[4]*/,
uint32_t *const res) {
static INLINE void sad_1024_pel_final_neon(const uint16x8_t sum[4],
uint32_t sad_array[4]) {
#if defined(__aarch64__)
const uint16x8_t a0 = vpaddq_u16(sum[0], sum[1]);
const uint16x8_t a1 = vpaddq_u16(sum[2], sum[3]);
const uint32x4_t b0 = vpaddlq_u16(a0);
const uint32x4_t b1 = vpaddlq_u16(a1);
const uint32x4_t r = vpaddq_u32(b0, b1);
vst1q_u32(sad_array, r);
#else
const uint16x4_t a0 = vpadd_u16(vget_low_u16(sum[0]), vget_high_u16(sum[0]));
const uint16x4_t a1 = vpadd_u16(vget_low_u16(sum[1]), vget_high_u16(sum[1]));
const uint16x4_t a2 = vpadd_u16(vget_low_u16(sum[2]), vget_high_u16(sum[2]));
@@ -95,12 +119,23 @@ static INLINE void sad_1024_pel_final_neon(const uint16x8_t *sum /*[4]*/,
const uint32x4_t b1 = vpaddlq_u16(vcombine_u16(a2, a3));
const uint32x2_t c0 = vpadd_u32(vget_low_u32(b0), vget_high_u32(b0));
const uint32x2_t c1 = vpadd_u32(vget_low_u32(b1), vget_high_u32(b1));
vst1q_u32(res, vcombine_u32(c0, c1));
vst1q_u32(sad_array, vcombine_u32(c0, c1));
#endif
}
// Can handle 2048 pixels' sad sum (such as 32x64 or 64x32)
static INLINE void sad_2048_pel_final_neon(const uint16x8_t *sum /*[4]*/,
uint32_t *const res) {
static INLINE void sad_2048_pel_final_neon(const uint16x8_t sum[4],
uint32_t sad_array[4]) {
#if defined(__aarch64__)
const uint32x4_t a0 = vpaddlq_u16(sum[0]);
const uint32x4_t a1 = vpaddlq_u16(sum[1]);
const uint32x4_t a2 = vpaddlq_u16(sum[2]);
const uint32x4_t a3 = vpaddlq_u16(sum[3]);
const uint32x4_t b0 = vpaddq_u32(a0, a1);
const uint32x4_t b1 = vpaddq_u32(a2, a3);
const uint32x4_t r = vpaddq_u32(b0, b1);
vst1q_u32(sad_array, r);
#else
const uint32x4_t a0 = vpaddlq_u16(sum[0]);
const uint32x4_t a1 = vpaddlq_u16(sum[1]);
const uint32x4_t a2 = vpaddlq_u16(sum[2]);
@@ -111,12 +146,31 @@ static INLINE void sad_2048_pel_final_neon(const uint16x8_t *sum /*[4]*/,
const uint32x2_t b3 = vadd_u32(vget_low_u32(a3), vget_high_u32(a3));
const uint32x2_t c0 = vpadd_u32(b0, b1);
const uint32x2_t c1 = vpadd_u32(b2, b3);
vst1q_u32(res, vcombine_u32(c0, c1));
vst1q_u32(sad_array, vcombine_u32(c0, c1));
#endif
}
// Can handle 4096 pixels' sad sum (such as 64x64)
static INLINE void sad_4096_pel_final_neon(const uint16x8_t *sum /*[8]*/,
uint32_t *const res) {
static INLINE void sad_4096_pel_final_neon(const uint16x8_t sum[8],
uint32_t sad_array[4]) {
#if defined(__aarch64__)
const uint32x4_t a0 = vpaddlq_u16(sum[0]);
const uint32x4_t a1 = vpaddlq_u16(sum[1]);
const uint32x4_t a2 = vpaddlq_u16(sum[2]);
const uint32x4_t a3 = vpaddlq_u16(sum[3]);
const uint32x4_t a4 = vpaddlq_u16(sum[4]);
const uint32x4_t a5 = vpaddlq_u16(sum[5]);
const uint32x4_t a6 = vpaddlq_u16(sum[6]);
const uint32x4_t a7 = vpaddlq_u16(sum[7]);
const uint32x4_t b0 = vaddq_u32(a0, a1);
const uint32x4_t b1 = vaddq_u32(a2, a3);
const uint32x4_t b2 = vaddq_u32(a4, a5);
const uint32x4_t b3 = vaddq_u32(a6, a7);
const uint32x4_t c0 = vpaddq_u32(b0, b1);
const uint32x4_t c1 = vpaddq_u32(b2, b3);
const uint32x4_t r = vpaddq_u32(c0, c1);
vst1q_u32(sad_array, r);
#else
const uint32x4_t a0 = vpaddlq_u16(sum[0]);
const uint32x4_t a1 = vpaddlq_u16(sum[1]);
const uint32x4_t a2 = vpaddlq_u16(sum[2]);
@@ -135,12 +189,15 @@ static INLINE void sad_4096_pel_final_neon(const uint16x8_t *sum /*[8]*/,
const uint32x2_t c3 = vadd_u32(vget_low_u32(b3), vget_high_u32(b3));
const uint32x2_t d0 = vpadd_u32(c0, c1);
const uint32x2_t d1 = vpadd_u32(c2, c3);
vst1q_u32(res, vcombine_u32(d0, d1));
vst1q_u32(sad_array, vcombine_u32(d0, d1));
#endif
}
#endif
static INLINE void sad8x_4d(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res, const int height) {
uint32_t sad_array[4], const int height) {
int i, j;
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
ref_array[3] };
@@ -157,29 +214,64 @@ static INLINE void sad8x_4d(const uint8_t *src_ptr, int src_stride,
}
}
sad_512_pel_final_neon(sum, res);
sad_512_pel_final_neon(sum, sad_array);
}
void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res) {
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 4);
uint32_t sad_array[4]) {
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 4);
}
void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res) {
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 8);
uint32_t sad_array[4]) {
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 8);
}
void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res) {
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 16);
uint32_t sad_array[4]) {
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 16);
}
////////////////////////////////////////////////////////////////////////////////
#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
(__ARM_FEATURE_DOTPROD == 1)
static INLINE void sad16_neon(const uint8_t *ref_ptr, const uint8x16_t src_ptr,
uint32x4_t *const sum) {
const uint8x16_t r = vld1q_u8(ref_ptr);
const uint8x16_t diff = vabdq_u8(src_ptr, r);
*sum = vdotq_u32(*sum, diff, vdupq_n_u8(1));
}
static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t sad_array[4], const int height) {
int i;
uint32x4_t r0, r1;
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
ref_array[3] };
uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
vdupq_n_u32(0) };
for (i = 0; i < height; ++i) {
const uint8x16_t s = vld1q_u8(src_ptr + i * src_stride);
sad16_neon(ref_loop[0] + i * ref_stride, s, &sum[0]);
sad16_neon(ref_loop[1] + i * ref_stride, s, &sum[1]);
sad16_neon(ref_loop[2] + i * ref_stride, s, &sum[2]);
sad16_neon(ref_loop[3] + i * ref_stride, s, &sum[3]);
}
r0 = vpaddq_u32(sum[0], sum[1]);
r1 = vpaddq_u32(sum[2], sum[3]);
vst1q_u32(sad_array, vpaddq_u32(r0, r1));
}
#else
static INLINE void sad16_neon(const uint8_t *ref_ptr, const uint8x16_t src_ptr,
uint16x8_t *const sum) {
const uint8x16_t r = vld1q_u8(ref_ptr);
@@ -189,8 +281,8 @@ static INLINE void sad16_neon(const uint8_t *ref_ptr, const uint8x16_t src_ptr,
static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res, const int height) {
int i, j;
uint32_t sad_array[4], const int height) {
int i;
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
ref_array[3] };
uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
@@ -199,35 +291,103 @@ static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride,
for (i = 0; i < height; ++i) {
const uint8x16_t s = vld1q_u8(src_ptr);
src_ptr += src_stride;
for (j = 0; j < 4; ++j) {
sad16_neon(ref_loop[j], s, &sum[j]);
ref_loop[j] += ref_stride;
}
/* Manual unrolling here stops the compiler from getting confused. */
sad16_neon(ref_loop[0], s, &sum[0]);
ref_loop[0] += ref_stride;
sad16_neon(ref_loop[1], s, &sum[1]);
ref_loop[1] += ref_stride;
sad16_neon(ref_loop[2], s, &sum[2]);
ref_loop[2] += ref_stride;
sad16_neon(ref_loop[3], s, &sum[3]);
ref_loop[3] += ref_stride;
}
sad_512_pel_final_neon(sum, res);
sad_512_pel_final_neon(sum, sad_array);
}
#endif
void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res) {
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 8);
uint32_t sad_array[4]) {
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 8);
}
void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res) {
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 16);
uint32_t sad_array[4]) {
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 16);
}
void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res) {
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 32);
uint32_t sad_array[4]) {
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 32);
}
////////////////////////////////////////////////////////////////////////////////
#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
(__ARM_FEATURE_DOTPROD == 1)
static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t sad_array[4], const int height) {
int i;
uint32x4_t r0, r1;
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
ref_array[3] };
uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
vdupq_n_u32(0) };
for (i = 0; i < height; ++i) {
uint8x16_t s;
s = vld1q_u8(src_ptr + 0 * 16);
sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]);
sad16_neon(ref_loop[1] + 0 * 16, s, &sum[1]);
sad16_neon(ref_loop[2] + 0 * 16, s, &sum[2]);
sad16_neon(ref_loop[3] + 0 * 16, s, &sum[3]);
s = vld1q_u8(src_ptr + 1 * 16);
sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]);
sad16_neon(ref_loop[1] + 1 * 16, s, &sum[1]);
sad16_neon(ref_loop[2] + 1 * 16, s, &sum[2]);
sad16_neon(ref_loop[3] + 1 * 16, s, &sum[3]);
src_ptr += src_stride;
ref_loop[0] += ref_stride;
ref_loop[1] += ref_stride;
ref_loop[2] += ref_stride;
ref_loop[3] += ref_stride;
}
r0 = vpaddq_u32(sum[0], sum[1]);
r1 = vpaddq_u32(sum[2], sum[3]);
vst1q_u32(sad_array, vpaddq_u32(r0, r1));
}
void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t sad_array[4]) {
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 16);
}
void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t sad_array[4]) {
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 32);
}
void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t sad_array[4]) {
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 64);
}
#else
static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
const int height, uint16x8_t *const sum) {
@@ -262,33 +422,143 @@ static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride,
void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res) {
uint32_t sad_array[4]) {
uint16x8_t sum[4];
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 16, sum);
sad_512_pel_final_neon(sum, res);
sad_512_pel_final_neon(sum, sad_array);
}
void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res) {
uint32_t sad_array[4]) {
uint16x8_t sum[4];
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 32, sum);
sad_1024_pel_final_neon(sum, res);
sad_1024_pel_final_neon(sum, sad_array);
}
void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res) {
uint32_t sad_array[4]) {
uint16x8_t sum[4];
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 64, sum);
sad_2048_pel_final_neon(sum, res);
sad_2048_pel_final_neon(sum, sad_array);
}
#endif
////////////////////////////////////////////////////////////////////////////////
#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
(__ARM_FEATURE_DOTPROD == 1)
void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res) {
uint32_t sad_array[4]) {
int i;
uint32x4_t r0, r1;
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
ref_array[3] };
uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
vdupq_n_u32(0) };
for (i = 0; i < 32; ++i) {
uint8x16_t s;
s = vld1q_u8(src_ptr + 0 * 16);
sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]);
sad16_neon(ref_loop[1] + 0 * 16, s, &sum[1]);
sad16_neon(ref_loop[2] + 0 * 16, s, &sum[2]);
sad16_neon(ref_loop[3] + 0 * 16, s, &sum[3]);
s = vld1q_u8(src_ptr + 1 * 16);
sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]);
sad16_neon(ref_loop[1] + 1 * 16, s, &sum[1]);
sad16_neon(ref_loop[2] + 1 * 16, s, &sum[2]);
sad16_neon(ref_loop[3] + 1 * 16, s, &sum[3]);
s = vld1q_u8(src_ptr + 2 * 16);
sad16_neon(ref_loop[0] + 2 * 16, s, &sum[0]);
sad16_neon(ref_loop[1] + 2 * 16, s, &sum[1]);
sad16_neon(ref_loop[2] + 2 * 16, s, &sum[2]);
sad16_neon(ref_loop[3] + 2 * 16, s, &sum[3]);
s = vld1q_u8(src_ptr + 3 * 16);
sad16_neon(ref_loop[0] + 3 * 16, s, &sum[0]);
sad16_neon(ref_loop[1] + 3 * 16, s, &sum[1]);
sad16_neon(ref_loop[2] + 3 * 16, s, &sum[2]);
sad16_neon(ref_loop[3] + 3 * 16, s, &sum[3]);
src_ptr += src_stride;
ref_loop[0] += ref_stride;
ref_loop[1] += ref_stride;
ref_loop[2] += ref_stride;
ref_loop[3] += ref_stride;
}
r0 = vpaddq_u32(sum[0], sum[1]);
r1 = vpaddq_u32(sum[2], sum[3]);
vst1q_u32(sad_array, vpaddq_u32(r0, r1));
}
void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t sad_array[4]) {
int i;
uint32x4_t r0, r1, r2, r3;
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
ref_array[3] };
uint32x4_t sum[8] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
vdupq_n_u32(0), vdupq_n_u32(0) };
for (i = 0; i < 64; ++i) {
uint8x16_t s;
s = vld1q_u8(src_ptr + 0 * 16);
sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]);
sad16_neon(ref_loop[1] + 0 * 16, s, &sum[2]);
sad16_neon(ref_loop[2] + 0 * 16, s, &sum[4]);
sad16_neon(ref_loop[3] + 0 * 16, s, &sum[6]);
s = vld1q_u8(src_ptr + 1 * 16);
sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]);
sad16_neon(ref_loop[1] + 1 * 16, s, &sum[2]);
sad16_neon(ref_loop[2] + 1 * 16, s, &sum[4]);
sad16_neon(ref_loop[3] + 1 * 16, s, &sum[6]);
s = vld1q_u8(src_ptr + 2 * 16);
sad16_neon(ref_loop[0] + 2 * 16, s, &sum[1]);
sad16_neon(ref_loop[1] + 2 * 16, s, &sum[3]);
sad16_neon(ref_loop[2] + 2 * 16, s, &sum[5]);
sad16_neon(ref_loop[3] + 2 * 16, s, &sum[7]);
s = vld1q_u8(src_ptr + 3 * 16);
sad16_neon(ref_loop[0] + 3 * 16, s, &sum[1]);
sad16_neon(ref_loop[1] + 3 * 16, s, &sum[3]);
sad16_neon(ref_loop[2] + 3 * 16, s, &sum[5]);
sad16_neon(ref_loop[3] + 3 * 16, s, &sum[7]);
src_ptr += src_stride;
ref_loop[0] += ref_stride;
ref_loop[1] += ref_stride;
ref_loop[2] += ref_stride;
ref_loop[3] += ref_stride;
}
r0 = vpaddq_u32(sum[0], sum[1]);
r1 = vpaddq_u32(sum[2], sum[3]);
r2 = vpaddq_u32(sum[4], sum[5]);
r3 = vpaddq_u32(sum[6], sum[7]);
r0 = vpaddq_u32(r0, r1);
r1 = vpaddq_u32(r2, r3);
vst1q_u32(sad_array, vpaddq_u32(r0, r1));
}
#else
void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t sad_array[4]) {
int i;
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
ref_array[3] };
@@ -329,12 +599,12 @@ void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride,
ref_loop[3] += ref_stride;
}
sad_2048_pel_final_neon(sum, res);
sad_2048_pel_final_neon(sum, sad_array);
}
void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t *res) {
uint32_t sad_array[4]) {
int i;
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
ref_array[3] };
@@ -376,5 +646,7 @@ void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride,
ref_loop[3] += ref_stride;
}
sad_4096_pel_final_neon(sum, res);
sad_4096_pel_final_neon(sum, sad_array);
}
#endif
libs/libvpx/vpx_dsp/arm/sad_neon.c
+27 -27
View File
@@ -23,7 +23,7 @@ uint32_t vpx_sad4x4_neon(const uint8_t *src_ptr, int src_stride,
const uint8x16_t ref_u8 = load_unaligned_u8q(ref_ptr, ref_stride);
uint16x8_t abs = vabdl_u8(vget_low_u8(src_u8), vget_low_u8(ref_u8));
abs = vabal_u8(abs, vget_high_u8(src_u8), vget_high_u8(ref_u8));
return vget_lane_u32(horizontal_add_uint16x8(abs), 0);
return horizontal_add_uint16x8(abs);
}
uint32_t vpx_sad4x4_avg_neon(const uint8_t *src_ptr, int src_stride,
@@ -35,7 +35,7 @@ uint32_t vpx_sad4x4_avg_neon(const uint8_t *src_ptr, int src_stride,
const uint8x16_t avg = vrhaddq_u8(ref_u8, second_pred_u8);
uint16x8_t abs = vabdl_u8(vget_low_u8(src_u8), vget_low_u8(avg));
abs = vabal_u8(abs, vget_high_u8(src_u8), vget_high_u8(avg));
return vget_lane_u32(horizontal_add_uint16x8(abs), 0);
return horizontal_add_uint16x8(abs);
}
uint32_t vpx_sad4x8_neon(const uint8_t *src_ptr, int src_stride,
@@ -51,7 +51,7 @@ uint32_t vpx_sad4x8_neon(const uint8_t *src_ptr, int src_stride,
abs = vabal_u8(abs, vget_high_u8(src_u8), vget_high_u8(ref_u8));
}
return vget_lane_u32(horizontal_add_uint16x8(abs), 0);
return horizontal_add_uint16x8(abs);
}
uint32_t vpx_sad4x8_avg_neon(const uint8_t *src_ptr, int src_stride,
@@ -71,7 +71,7 @@ uint32_t vpx_sad4x8_avg_neon(const uint8_t *src_ptr, int src_stride,
abs = vabal_u8(abs, vget_high_u8(src_u8), vget_high_u8(avg));
}
return vget_lane_u32(horizontal_add_uint16x8(abs), 0);
return horizontal_add_uint16x8(abs);
}
static INLINE uint16x8_t sad8x(const uint8_t *src_ptr, int src_stride,
@@ -110,11 +110,11 @@ static INLINE uint16x8_t sad8x_avg(const uint8_t *src_ptr, int src_stride,
return abs;
}
#define sad8xN(n) \
#define SAD8XN(n) \
uint32_t vpx_sad8x##n##_neon(const uint8_t *src_ptr, int src_stride, \
const uint8_t *ref_ptr, int ref_stride) { \
const uint16x8_t abs = sad8x(src_ptr, src_stride, ref_ptr, ref_stride, n); \
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
return horizontal_add_uint16x8(abs); \
} \
\
uint32_t vpx_sad8x##n##_avg_neon(const uint8_t *src_ptr, int src_stride, \
@@ -122,12 +122,12 @@ static INLINE uint16x8_t sad8x_avg(const uint8_t *src_ptr, int src_stride,
const uint8_t *second_pred) { \
const uint16x8_t abs = \
sad8x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
return horizontal_add_uint16x8(abs); \
}
sad8xN(4);
sad8xN(8);
sad8xN(16);
SAD8XN(4)
SAD8XN(8)
SAD8XN(16)
static INLINE uint16x8_t sad16x(const uint8_t *src_ptr, int src_stride,
const uint8_t *ref_ptr, int ref_stride,
@@ -167,12 +167,12 @@ static INLINE uint16x8_t sad16x_avg(const uint8_t *src_ptr, int src_stride,
return abs;
}
#define sad16xN(n) \
#define SAD16XN(n) \
uint32_t vpx_sad16x##n##_neon(const uint8_t *src_ptr, int src_stride, \
const uint8_t *ref_ptr, int ref_stride) { \
const uint16x8_t abs = \
sad16x(src_ptr, src_stride, ref_ptr, ref_stride, n); \
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
return horizontal_add_uint16x8(abs); \
} \
\
uint32_t vpx_sad16x##n##_avg_neon(const uint8_t *src_ptr, int src_stride, \
@@ -180,12 +180,12 @@ static INLINE uint16x8_t sad16x_avg(const uint8_t *src_ptr, int src_stride,
const uint8_t *second_pred) { \
const uint16x8_t abs = \
sad16x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
return horizontal_add_uint16x8(abs); \
}
sad16xN(8);
sad16xN(16);
sad16xN(32);
SAD16XN(8)
SAD16XN(16)
SAD16XN(32)
static INLINE uint16x8_t sad32x(const uint8_t *src_ptr, int src_stride,
const uint8_t *ref_ptr, int ref_stride,
@@ -235,12 +235,12 @@ static INLINE uint16x8_t sad32x_avg(const uint8_t *src_ptr, int src_stride,
return abs;
}
#define sad32xN(n) \
#define SAD32XN(n) \
uint32_t vpx_sad32x##n##_neon(const uint8_t *src_ptr, int src_stride, \
const uint8_t *ref_ptr, int ref_stride) { \
const uint16x8_t abs = \
sad32x(src_ptr, src_stride, ref_ptr, ref_stride, n); \
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
return horizontal_add_uint16x8(abs); \
} \
\
uint32_t vpx_sad32x##n##_avg_neon(const uint8_t *src_ptr, int src_stride, \
@@ -248,12 +248,12 @@ static INLINE uint16x8_t sad32x_avg(const uint8_t *src_ptr, int src_stride,
const uint8_t *second_pred) { \
const uint16x8_t abs = \
sad32x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
return horizontal_add_uint16x8(abs); \
}
sad32xN(16);
sad32xN(32);
sad32xN(64);
SAD32XN(16)
SAD32XN(32)
SAD32XN(64)
static INLINE uint32x4_t sad64x(const uint8_t *src_ptr, int src_stride,
const uint8_t *ref_ptr, int ref_stride,
@@ -333,12 +333,12 @@ static INLINE uint32x4_t sad64x_avg(const uint8_t *src_ptr, int src_stride,
}
}
#define sad64xN(n) \
#define SAD64XN(n) \
uint32_t vpx_sad64x##n##_neon(const uint8_t *src_ptr, int src_stride, \
const uint8_t *ref_ptr, int ref_stride) { \
const uint32x4_t abs = \
sad64x(src_ptr, src_stride, ref_ptr, ref_stride, n); \
return vget_lane_u32(horizontal_add_uint32x4(abs), 0); \
return horizontal_add_uint32x4(abs); \
} \
\
uint32_t vpx_sad64x##n##_avg_neon(const uint8_t *src_ptr, int src_stride, \
@@ -346,8 +346,8 @@ static INLINE uint32x4_t sad64x_avg(const uint8_t *src_ptr, int src_stride,
const uint8_t *second_pred) { \
const uint32x4_t abs = \
sad64x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \
return vget_lane_u32(horizontal_add_uint32x4(abs), 0); \
return horizontal_add_uint32x4(abs); \
}
sad64xN(32);
sad64xN(64);
SAD64XN(32)
SAD64XN(64)
libs/libvpx/vpx_dsp/arm/subpel_variance_neon.c
+28 -28
View File
@@ -97,7 +97,7 @@ static void var_filter_block2d_bil_w16(const uint8_t *src_ptr,
// 4xM filter writes an extra row to fdata because it processes two rows at a
// time.
#define sub_pixel_varianceNxM(n, m) \
#define SUB_PIXEL_VARIANCENXM(n, m) \
uint32_t vpx_sub_pixel_variance##n##x##m##_neon( \
const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, \
const uint8_t *ref_ptr, int ref_stride, uint32_t *sse) { \
@@ -123,23 +123,23 @@ static void var_filter_block2d_bil_w16(const uint8_t *src_ptr,
return vpx_variance##n##x##m(temp1, n, ref_ptr, ref_stride, sse); \
}
sub_pixel_varianceNxM(4, 4);
sub_pixel_varianceNxM(4, 8);
sub_pixel_varianceNxM(8, 4);
sub_pixel_varianceNxM(8, 8);
sub_pixel_varianceNxM(8, 16);
sub_pixel_varianceNxM(16, 8);
sub_pixel_varianceNxM(16, 16);
sub_pixel_varianceNxM(16, 32);
sub_pixel_varianceNxM(32, 16);
sub_pixel_varianceNxM(32, 32);
sub_pixel_varianceNxM(32, 64);
sub_pixel_varianceNxM(64, 32);
sub_pixel_varianceNxM(64, 64);
SUB_PIXEL_VARIANCENXM(4, 4)
SUB_PIXEL_VARIANCENXM(4, 8)
SUB_PIXEL_VARIANCENXM(8, 4)
SUB_PIXEL_VARIANCENXM(8, 8)
SUB_PIXEL_VARIANCENXM(8, 16)
SUB_PIXEL_VARIANCENXM(16, 8)
SUB_PIXEL_VARIANCENXM(16, 16)
SUB_PIXEL_VARIANCENXM(16, 32)
SUB_PIXEL_VARIANCENXM(32, 16)
SUB_PIXEL_VARIANCENXM(32, 32)
SUB_PIXEL_VARIANCENXM(32, 64)
SUB_PIXEL_VARIANCENXM(64, 32)
SUB_PIXEL_VARIANCENXM(64, 64)
// 4xM filter writes an extra row to fdata because it processes two rows at a
// time.
#define sub_pixel_avg_varianceNxM(n, m) \
#define SUB_PIXEL_AVG_VARIANCENXM(n, m) \
uint32_t vpx_sub_pixel_avg_variance##n##x##m##_neon( \
const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, \
const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, \
@@ -169,16 +169,16 @@ sub_pixel_varianceNxM(64, 64);
return vpx_variance##n##x##m(temp0, n, ref_ptr, ref_stride, sse); \
}
sub_pixel_avg_varianceNxM(4, 4);
sub_pixel_avg_varianceNxM(4, 8);
sub_pixel_avg_varianceNxM(8, 4);
sub_pixel_avg_varianceNxM(8, 8);
sub_pixel_avg_varianceNxM(8, 16);
sub_pixel_avg_varianceNxM(16, 8);
sub_pixel_avg_varianceNxM(16, 16);
sub_pixel_avg_varianceNxM(16, 32);
sub_pixel_avg_varianceNxM(32, 16);
sub_pixel_avg_varianceNxM(32, 32);
sub_pixel_avg_varianceNxM(32, 64);
sub_pixel_avg_varianceNxM(64, 32);
sub_pixel_avg_varianceNxM(64, 64);
SUB_PIXEL_AVG_VARIANCENXM(4, 4)
SUB_PIXEL_AVG_VARIANCENXM(4, 8)
SUB_PIXEL_AVG_VARIANCENXM(8, 4)
SUB_PIXEL_AVG_VARIANCENXM(8, 8)
SUB_PIXEL_AVG_VARIANCENXM(8, 16)
SUB_PIXEL_AVG_VARIANCENXM(16, 8)
SUB_PIXEL_AVG_VARIANCENXM(16, 16)
SUB_PIXEL_AVG_VARIANCENXM(16, 32)
SUB_PIXEL_AVG_VARIANCENXM(32, 16)
SUB_PIXEL_AVG_VARIANCENXM(32, 32)
SUB_PIXEL_AVG_VARIANCENXM(32, 64)
SUB_PIXEL_AVG_VARIANCENXM(64, 32)
SUB_PIXEL_AVG_VARIANCENXM(64, 64)
libs/libvpx/vpx_dsp/arm/sum_neon.h
+51 -9
View File
@@ -16,23 +16,65 @@
#include "./vpx_config.h"
#include "vpx/vpx_integer.h"
static INLINE int32x2_t horizontal_add_int16x8(const int16x8_t a) {
static INLINE int32_t horizontal_add_int16x8(const int16x8_t a) {
#if defined(__aarch64__)
return vaddlvq_s16(a);
#else
const int32x4_t b = vpaddlq_s16(a);
const int64x2_t c = vpaddlq_s32(b);
return vadd_s32(vreinterpret_s32_s64(vget_low_s64(c)),
vreinterpret_s32_s64(vget_high_s64(c)));
const int32x2_t d = vadd_s32(vreinterpret_s32_s64(vget_low_s64(c)),
vreinterpret_s32_s64(vget_high_s64(c)));
return vget_lane_s32(d, 0);
#endif
}
static INLINE uint32x2_t horizontal_add_uint16x8(const uint16x8_t a) {
static INLINE uint32_t horizontal_add_uint16x8(const uint16x8_t a) {
#if defined(__aarch64__)
return vaddlvq_u16(a);
#else
const uint32x4_t b = vpaddlq_u16(a);
const uint64x2_t c = vpaddlq_u32(b);
return vadd_u32(vreinterpret_u32_u64(vget_low_u64(c)),
vreinterpret_u32_u64(vget_high_u64(c)));
const uint32x2_t d = vadd_u32(vreinterpret_u32_u64(vget_low_u64(c)),
vreinterpret_u32_u64(vget_high_u64(c)));
return vget_lane_u32(d, 0);
#endif
}
static INLINE uint32x2_t horizontal_add_uint32x4(const uint32x4_t a) {
static INLINE int32_t horizontal_add_int32x2(const int32x2_t a) {
#if defined(__aarch64__)
return vaddv_s32(a);
#else
return vget_lane_s32(a, 0) + vget_lane_s32(a, 1);
#endif
}
static INLINE uint32_t horizontal_add_uint32x2(const uint32x2_t a) {
#if defined(__aarch64__)
return vaddv_u32(a);
#else
return vget_lane_u32(a, 0) + vget_lane_u32(a, 1);
#endif
}
static INLINE int32_t horizontal_add_int32x4(const int32x4_t a) {
#if defined(__aarch64__)
return vaddvq_s32(a);
#else
const int64x2_t b = vpaddlq_s32(a);
const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)),
vreinterpret_s32_s64(vget_high_s64(b)));
return vget_lane_s32(c, 0);
#endif
}
static INLINE uint32_t horizontal_add_uint32x4(const uint32x4_t a) {
#if defined(__aarch64__)
return vaddvq_u32(a);
#else
const uint64x2_t b = vpaddlq_u32(a);
return vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)),
vreinterpret_u32_u64(vget_high_u64(b)));
const uint32x2_t c = vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)),
vreinterpret_u32_u64(vget_high_u64(b)));
return vget_lane_u32(c, 0);
#endif
}
#endif // VPX_VPX_DSP_ARM_SUM_NEON_H_
libs/libvpx/vpx_dsp/arm/transpose_neon.h
+43 -4
View File
@@ -138,8 +138,8 @@ static INLINE void transpose_s16_4x4q(int16x8_t *a0, int16x8_t *a1) {
vtrnq_s32(vreinterpretq_s32_s16(*a0), vreinterpretq_s32_s16(*a1));
// Swap 64 bit elements resulting in:
// c0.val[0]: 00 01 20 21 02 03 22 23
// c0.val[1]: 10 11 30 31 12 13 32 33
// c0: 00 01 20 21 02 03 22 23
// c1: 10 11 30 31 12 13 32 33
const int32x4_t c0 =
vcombine_s32(vget_low_s32(b0.val[0]), vget_low_s32(b0.val[1]));
@@ -169,8 +169,8 @@ static INLINE void transpose_u16_4x4q(uint16x8_t *a0, uint16x8_t *a1) {
vtrnq_u32(vreinterpretq_u32_u16(*a0), vreinterpretq_u32_u16(*a1));
// Swap 64 bit elements resulting in:
// c0.val[0]: 00 01 20 21 02 03 22 23
// c0.val[1]: 10 11 30 31 12 13 32 33
// c0: 00 01 20 21 02 03 22 23
// c1: 10 11 30 31 12 13 32 33
const uint32x4_t c0 =
vcombine_u32(vget_low_u32(b0.val[0]), vget_low_u32(b0.val[1]));
@@ -1184,6 +1184,45 @@ static INLINE void transpose_u8_16x16(
*o15 = e7.val[1];
}
static INLINE void transpose_s16_16x16(int16x8_t *in0, int16x8_t *in1) {
int16x8_t t[8];
// transpose the 4 8x8 quadrants separately but first swap quadrants 2 and 3.
t[0] = in0[8];
t[1] = in0[9];
t[2] = in0[10];
t[3] = in0[11];
t[4] = in0[12];
t[5] = in0[13];
t[6] = in0[14];
t[7] = in0[15];
in0[8] = in1[0];
in0[9] = in1[1];
in0[10] = in1[2];
in0[11] = in1[3];
in0[12] = in1[4];
in0[13] = in1[5];
in0[14] = in1[6];
in0[15] = in1[7];
in1[0] = t[0];
in1[1] = t[1];
in1[2] = t[2];
in1[3] = t[3];
in1[4] = t[4];
in1[5] = t[5];
in1[6] = t[6];
in1[7] = t[7];
transpose_s16_8x8(&in0[0], &in0[1], &in0[2], &in0[3], &in0[4], &in0[5],
&in0[6], &in0[7]);
transpose_s16_8x8(&in0[8], &in0[9], &in0[10], &in0[11], &in0[12], &in0[13],
&in0[14], &in0[15]);
transpose_s16_8x8(&in1[0], &in1[1], &in1[2], &in1[3], &in1[4], &in1[5],
&in1[6], &in1[7]);
transpose_s16_8x8(&in1[8], &in1[9], &in1[10], &in1[11], &in1[12], &in1[13],
&in1[14], &in1[15]);
}
static INLINE void load_and_transpose_u8_4x8(const uint8_t *a,
const int a_stride, uint8x8_t *a0,
uint8x8_t *a1, uint8x8_t *a2,
libs/libvpx/vpx_dsp/arm/variance_neon.c
+249 -108
View File
@@ -19,6 +19,100 @@
#include "vpx_dsp/arm/sum_neon.h"
#include "vpx_ports/mem.h"
#if defined(__ARM_FEATURE_DOTPROD) && (__ARM_FEATURE_DOTPROD == 1)
// Process a block of width 4 four rows at a time.
static void variance_neon_w4x4(const uint8_t *src_ptr, int src_stride,
const uint8_t *ref_ptr, int ref_stride, int h,
uint32_t *sse, int *sum) {
int i;
uint32x4_t sum_a = vdupq_n_u32(0);
uint32x4_t sum_b = vdupq_n_u32(0);
uint32x4_t sse_u32 = vdupq_n_u32(0);
for (i = 0; i < h; i += 4) {
const uint8x16_t a = load_unaligned_u8q(src_ptr, src_stride);
const uint8x16_t b = load_unaligned_u8q(ref_ptr, ref_stride);
const uint8x16_t abs_diff = vabdq_u8(a, b);
sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
sum_a = vdotq_u32(sum_a, a, vdupq_n_u8(1));
sum_b = vdotq_u32(sum_b, b, vdupq_n_u8(1));
src_ptr += 4 * src_stride;
ref_ptr += 4 * ref_stride;
}
*sum = horizontal_add_int32x4(vreinterpretq_s32_u32(vsubq_u32(sum_a, sum_b)));
*sse = horizontal_add_uint32x4(sse_u32);
}
// Process a block of any size where the width is divisible by 16.
static void variance_neon_w16(const uint8_t *src_ptr, int src_stride,
const uint8_t *ref_ptr, int ref_stride, int w,
int h, uint32_t *sse, int *sum) {
int i, j;
uint32x4_t sum_a = vdupq_n_u32(0);
uint32x4_t sum_b = vdupq_n_u32(0);
uint32x4_t sse_u32 = vdupq_n_u32(0);
for (i = 0; i < h; ++i) {
for (j = 0; j < w; j += 16) {
const uint8x16_t a = vld1q_u8(src_ptr + j);
const uint8x16_t b = vld1q_u8(ref_ptr + j);
const uint8x16_t abs_diff = vabdq_u8(a, b);
sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
sum_a = vdotq_u32(sum_a, a, vdupq_n_u8(1));
sum_b = vdotq_u32(sum_b, b, vdupq_n_u8(1));
}
src_ptr += src_stride;
ref_ptr += ref_stride;
}
*sum = horizontal_add_int32x4(vreinterpretq_s32_u32(vsubq_u32(sum_a, sum_b)));
*sse = horizontal_add_uint32x4(sse_u32);
}
// Process a block of width 8 two rows at a time.
static void variance_neon_w8x2(const uint8_t *src_ptr, int src_stride,
const uint8_t *ref_ptr, int ref_stride, int h,
uint32_t *sse, int *sum) {
int i = 0;
uint32x2_t sum_a = vdup_n_u32(0);
uint32x2_t sum_b = vdup_n_u32(0);
uint32x2_t sse_lo_u32 = vdup_n_u32(0);
uint32x2_t sse_hi_u32 = vdup_n_u32(0);
do {
const uint8x8_t a_0 = vld1_u8(src_ptr);
const uint8x8_t a_1 = vld1_u8(src_ptr + src_stride);
const uint8x8_t b_0 = vld1_u8(ref_ptr);
const uint8x8_t b_1 = vld1_u8(ref_ptr + ref_stride);
const uint8x8_t abs_diff_0 = vabd_u8(a_0, b_0);
const uint8x8_t abs_diff_1 = vabd_u8(a_1, b_1);
sse_lo_u32 = vdot_u32(sse_lo_u32, abs_diff_0, abs_diff_0);
sse_hi_u32 = vdot_u32(sse_hi_u32, abs_diff_1, abs_diff_1);
sum_a = vdot_u32(sum_a, a_0, vdup_n_u8(1));
sum_b = vdot_u32(sum_b, b_0, vdup_n_u8(1));
sum_a = vdot_u32(sum_a, a_1, vdup_n_u8(1));
sum_b = vdot_u32(sum_b, b_1, vdup_n_u8(1));
src_ptr += src_stride + src_stride;
ref_ptr += ref_stride + ref_stride;
i += 2;
} while (i < h);
*sum = horizontal_add_int32x2(vreinterpret_s32_u32(vsub_u32(sum_a, sum_b)));
*sse = horizontal_add_uint32x2(vadd_u32(sse_lo_u32, sse_hi_u32));
}
#else
// The variance helper functions use int16_t for sum. 8 values are accumulated
// and then added (at which point they expand up to int32_t). To avoid overflow,
// there can be no more than 32767 / 255 ~= 128 values accumulated in each
@@ -66,10 +160,9 @@ static void variance_neon_w4x4(const uint8_t *src_ptr, int src_stride,
ref_ptr += 4 * ref_stride;
}
*sum = vget_lane_s32(horizontal_add_int16x8(sum_s16), 0);
*sse = vget_lane_u32(horizontal_add_uint32x4(vreinterpretq_u32_s32(
vaddq_s32(sse_lo_s32, sse_hi_s32))),
0);
*sum = horizontal_add_int16x8(sum_s16);
*sse = horizontal_add_uint32x4(
vreinterpretq_u32_s32(vaddq_s32(sse_lo_s32, sse_hi_s32)));
}
// Process a block of any size where the width is divisible by 16.
@@ -115,10 +208,9 @@ static void variance_neon_w16(const uint8_t *src_ptr, int src_stride,
ref_ptr += ref_stride;
}
*sum = vget_lane_s32(horizontal_add_int16x8(sum_s16), 0);
*sse = vget_lane_u32(horizontal_add_uint32x4(vreinterpretq_u32_s32(
vaddq_s32(sse_lo_s32, sse_hi_s32))),
0);
*sum = horizontal_add_int16x8(sum_s16);
*sse = horizontal_add_uint32x4(
vreinterpretq_u32_s32(vaddq_s32(sse_lo_s32, sse_hi_s32)));
}
// Process a block of width 8 two rows at a time.
@@ -157,12 +249,13 @@ static void variance_neon_w8x2(const uint8_t *src_ptr, int src_stride,
i += 2;
} while (i < h);
*sum = vget_lane_s32(horizontal_add_int16x8(sum_s16), 0);
*sse = vget_lane_u32(horizontal_add_uint32x4(vreinterpretq_u32_s32(
vaddq_s32(sse_lo_s32, sse_hi_s32))),
0);
*sum = horizontal_add_int16x8(sum_s16);
*sse = horizontal_add_uint32x4(
vreinterpretq_u32_s32(vaddq_s32(sse_lo_s32, sse_hi_s32)));
}
#endif
void vpx_get8x8var_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *ref_ptr, int ref_stride,
unsigned int *sse, int *sum) {
@@ -175,7 +268,7 @@ void vpx_get16x16var_neon(const uint8_t *src_ptr, int src_stride,
variance_neon_w16(src_ptr, src_stride, ref_ptr, ref_stride, 16, 16, sse, sum);
}
#define varianceNxM(n, m, shift) \
#define VARIANCENXM(n, m, shift) \
unsigned int vpx_variance##n##x##m##_neon( \
const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \
int ref_stride, unsigned int *sse) { \
@@ -195,16 +288,16 @@ void vpx_get16x16var_neon(const uint8_t *src_ptr, int src_stride,
return *sse - (uint32_t)(((int64_t)sum * sum) >> shift); \
}
varianceNxM(4, 4, 4);
varianceNxM(4, 8, 5);
varianceNxM(8, 4, 5);
varianceNxM(8, 8, 6);
varianceNxM(8, 16, 7);
varianceNxM(16, 8, 7);
varianceNxM(16, 16, 8);
varianceNxM(16, 32, 9);
varianceNxM(32, 16, 9);
varianceNxM(32, 32, 10);
VARIANCENXM(4, 4, 4)
VARIANCENXM(4, 8, 5)
VARIANCENXM(8, 4, 5)
VARIANCENXM(8, 8, 6)
VARIANCENXM(8, 16, 7)
VARIANCENXM(16, 8, 7)
VARIANCENXM(16, 16, 8)
VARIANCENXM(16, 32, 9)
VARIANCENXM(32, 16, 9)
VARIANCENXM(32, 32, 10)
unsigned int vpx_variance32x64_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *ref_ptr, int ref_stride,
@@ -264,117 +357,165 @@ unsigned int vpx_variance64x64_neon(const uint8_t *src_ptr, int src_stride,
return *sse - (unsigned int)(((int64_t)sum1 * sum1) >> 12);
}
#if defined(__ARM_FEATURE_DOTPROD) && (__ARM_FEATURE_DOTPROD == 1)
unsigned int vpx_mse16x16_neon(const unsigned char *src_ptr, int src_stride,
const unsigned char *ref_ptr, int ref_stride,
unsigned int *sse) {
int i;
int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16;
int64x1_t d0s64;
uint8x16_t q0u8, q1u8, q2u8, q3u8;
int32x4_t q7s32, q8s32, q9s32, q10s32;
uint16x8_t q11u16, q12u16, q13u16, q14u16;
int64x2_t q1s64;
uint8x16_t a[2], b[2], abs_diff[2];
uint32x4_t sse_vec[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
q7s32 = vdupq_n_s32(0);
q8s32 = vdupq_n_s32(0);
q9s32 = vdupq_n_s32(0);
q10s32 = vdupq_n_s32(0);
for (i = 0; i < 8; i++) { // mse16x16_neon_loop
q0u8 = vld1q_u8(src_ptr);
for (i = 0; i < 8; i++) {
a[0] = vld1q_u8(src_ptr);
src_ptr += src_stride;
q1u8 = vld1q_u8(src_ptr);
a[1] = vld1q_u8(src_ptr);
src_ptr += src_stride;
q2u8 = vld1q_u8(ref_ptr);
b[0] = vld1q_u8(ref_ptr);
ref_ptr += ref_stride;
q3u8 = vld1q_u8(ref_ptr);
b[1] = vld1q_u8(ref_ptr);
ref_ptr += ref_stride;
q11u16 = vsubl_u8(vget_low_u8(q0u8), vget_low_u8(q2u8));
q12u16 = vsubl_u8(vget_high_u8(q0u8), vget_high_u8(q2u8));
q13u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q3u8));
q14u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q3u8));
abs_diff[0] = vabdq_u8(a[0], b[0]);
abs_diff[1] = vabdq_u8(a[1], b[1]);
d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
q7s32 = vmlal_s16(q7s32, d22s16, d22s16);
q8s32 = vmlal_s16(q8s32, d23s16, d23s16);
d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
q9s32 = vmlal_s16(q9s32, d24s16, d24s16);
q10s32 = vmlal_s16(q10s32, d25s16, d25s16);
d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
q7s32 = vmlal_s16(q7s32, d26s16, d26s16);
q8s32 = vmlal_s16(q8s32, d27s16, d27s16);
d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16));
d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16));
q9s32 = vmlal_s16(q9s32, d28s16, d28s16);
q10s32 = vmlal_s16(q10s32, d29s16, d29s16);
sse_vec[0] = vdotq_u32(sse_vec[0], abs_diff[0], abs_diff[0]);
sse_vec[1] = vdotq_u32(sse_vec[1], abs_diff[1], abs_diff[1]);
}
q7s32 = vaddq_s32(q7s32, q8s32);
q9s32 = vaddq_s32(q9s32, q10s32);
q10s32 = vaddq_s32(q7s32, q9s32);
q1s64 = vpaddlq_s32(q10s32);
d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d0s64), 0);
return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0);
*sse = horizontal_add_uint32x4(vaddq_u32(sse_vec[0], sse_vec[1]));
return horizontal_add_uint32x4(vaddq_u32(sse_vec[0], sse_vec[1]));
}
unsigned int vpx_get4x4sse_cs_neon(const unsigned char *src_ptr, int src_stride,
const unsigned char *ref_ptr,
int ref_stride) {
int16x4_t d22s16, d24s16, d26s16, d28s16;
int64x1_t d0s64;
uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8;
int32x4_t q7s32, q8s32, q9s32, q10s32;
uint16x8_t q11u16, q12u16, q13u16, q14u16;
int64x2_t q1s64;
uint8x8_t a[4], b[4], abs_diff[4];
uint32x2_t sse = vdup_n_u32(0);
d0u8 = vld1_u8(src_ptr);
a[0] = vld1_u8(src_ptr);
src_ptr += src_stride;
d4u8 = vld1_u8(ref_ptr);
b[0] = vld1_u8(ref_ptr);
ref_ptr += ref_stride;
d1u8 = vld1_u8(src_ptr);
a[1] = vld1_u8(src_ptr);
src_ptr += src_stride;
d5u8 = vld1_u8(ref_ptr);
b[1] = vld1_u8(ref_ptr);
ref_ptr += ref_stride;
d2u8 = vld1_u8(src_ptr);
a[2] = vld1_u8(src_ptr);
src_ptr += src_stride;
d6u8 = vld1_u8(ref_ptr);
ref_ptr += ref_stride;
d3u8 = vld1_u8(src_ptr);
src_ptr += src_stride;
d7u8 = vld1_u8(ref_ptr);
b[2] = vld1_u8(ref_ptr);
ref_ptr += ref_stride;
a[3] = vld1_u8(src_ptr);
b[3] = vld1_u8(ref_ptr);
q11u16 = vsubl_u8(d0u8, d4u8);
q12u16 = vsubl_u8(d1u8, d5u8);
q13u16 = vsubl_u8(d2u8, d6u8);
q14u16 = vsubl_u8(d3u8, d7u8);
abs_diff[0] = vabd_u8(a[0], b[0]);
abs_diff[1] = vabd_u8(a[1], b[1]);
abs_diff[2] = vabd_u8(a[2], b[2]);
abs_diff[3] = vabd_u8(a[3], b[3]);
d22s16 = vget_low_s16(vreinterpretq_s16_u16(q11u16));
d24s16 = vget_low_s16(vreinterpretq_s16_u16(q12u16));
d26s16 = vget_low_s16(vreinterpretq_s16_u16(q13u16));
d28s16 = vget_low_s16(vreinterpretq_s16_u16(q14u16));
sse = vdot_u32(sse, abs_diff[0], abs_diff[0]);
sse = vdot_u32(sse, abs_diff[1], abs_diff[1]);
sse = vdot_u32(sse, abs_diff[2], abs_diff[2]);
sse = vdot_u32(sse, abs_diff[3], abs_diff[3]);
q7s32 = vmull_s16(d22s16, d22s16);
q8s32 = vmull_s16(d24s16, d24s16);
q9s32 = vmull_s16(d26s16, d26s16);
q10s32 = vmull_s16(d28s16, d28s16);
q7s32 = vaddq_s32(q7s32, q8s32);
q9s32 = vaddq_s32(q9s32, q10s32);
q9s32 = vaddq_s32(q7s32, q9s32);
q1s64 = vpaddlq_s32(q9s32);
d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0);
return vget_lane_u32(sse, 0);
}
#else
unsigned int vpx_mse16x16_neon(const unsigned char *src_ptr, int src_stride,
const unsigned char *ref_ptr, int ref_stride,
unsigned int *sse) {
int i;
uint8x16_t a[2], b[2];
int16x4_t diff_lo[4], diff_hi[4];
uint16x8_t diff[4];
int32x4_t sse_vec[4] = { vdupq_n_s32(0), vdupq_n_s32(0), vdupq_n_s32(0),
vdupq_n_s32(0) };
for (i = 0; i < 8; i++) {
a[0] = vld1q_u8(src_ptr);
src_ptr += src_stride;
a[1] = vld1q_u8(src_ptr);
src_ptr += src_stride;
b[0] = vld1q_u8(ref_ptr);
ref_ptr += ref_stride;
b[1] = vld1q_u8(ref_ptr);
ref_ptr += ref_stride;
diff[0] = vsubl_u8(vget_low_u8(a[0]), vget_low_u8(b[0]));
diff[1] = vsubl_u8(vget_high_u8(a[0]), vget_high_u8(b[0]));
diff[2] = vsubl_u8(vget_low_u8(a[1]), vget_low_u8(b[1]));
diff[3] = vsubl_u8(vget_high_u8(a[1]), vget_high_u8(b[1]));
diff_lo[0] = vreinterpret_s16_u16(vget_low_u16(diff[0]));
diff_lo[1] = vreinterpret_s16_u16(vget_low_u16(diff[1]));
sse_vec[0] = vmlal_s16(sse_vec[0], diff_lo[0], diff_lo[0]);
sse_vec[1] = vmlal_s16(sse_vec[1], diff_lo[1], diff_lo[1]);
diff_lo[2] = vreinterpret_s16_u16(vget_low_u16(diff[2]));
diff_lo[3] = vreinterpret_s16_u16(vget_low_u16(diff[3]));
sse_vec[2] = vmlal_s16(sse_vec[2], diff_lo[2], diff_lo[2]);
sse_vec[3] = vmlal_s16(sse_vec[3], diff_lo[3], diff_lo[3]);
diff_hi[0] = vreinterpret_s16_u16(vget_high_u16(diff[0]));
diff_hi[1] = vreinterpret_s16_u16(vget_high_u16(diff[1]));
sse_vec[0] = vmlal_s16(sse_vec[0], diff_hi[0], diff_hi[0]);
sse_vec[1] = vmlal_s16(sse_vec[1], diff_hi[1], diff_hi[1]);
diff_hi[2] = vreinterpret_s16_u16(vget_high_u16(diff[2]));
diff_hi[3] = vreinterpret_s16_u16(vget_high_u16(diff[3]));
sse_vec[2] = vmlal_s16(sse_vec[2], diff_hi[2], diff_hi[2]);
sse_vec[3] = vmlal_s16(sse_vec[3], diff_hi[3], diff_hi[3]);
}
sse_vec[0] = vaddq_s32(sse_vec[0], sse_vec[1]);
sse_vec[2] = vaddq_s32(sse_vec[2], sse_vec[3]);
sse_vec[0] = vaddq_s32(sse_vec[0], sse_vec[2]);
*sse = horizontal_add_uint32x4(vreinterpretq_u32_s32(sse_vec[0]));
return horizontal_add_uint32x4(vreinterpretq_u32_s32(sse_vec[0]));
}
unsigned int vpx_get4x4sse_cs_neon(const unsigned char *src_ptr, int src_stride,
const unsigned char *ref_ptr,
int ref_stride) {
uint8x8_t a[4], b[4];
int16x4_t diff_lo[4];
uint16x8_t diff[4];
int32x4_t sse;
a[0] = vld1_u8(src_ptr);
src_ptr += src_stride;
b[0] = vld1_u8(ref_ptr);
ref_ptr += ref_stride;
a[1] = vld1_u8(src_ptr);
src_ptr += src_stride;
b[1] = vld1_u8(ref_ptr);
ref_ptr += ref_stride;
a[2] = vld1_u8(src_ptr);
src_ptr += src_stride;
b[2] = vld1_u8(ref_ptr);
ref_ptr += ref_stride;
a[3] = vld1_u8(src_ptr);
b[3] = vld1_u8(ref_ptr);
diff[0] = vsubl_u8(a[0], b[0]);
diff[1] = vsubl_u8(a[1], b[1]);
diff[2] = vsubl_u8(a[2], b[2]);
diff[3] = vsubl_u8(a[3], b[3]);
diff_lo[0] = vget_low_s16(vreinterpretq_s16_u16(diff[0]));
diff_lo[1] = vget_low_s16(vreinterpretq_s16_u16(diff[1]));
diff_lo[2] = vget_low_s16(vreinterpretq_s16_u16(diff[2]));
diff_lo[3] = vget_low_s16(vreinterpretq_s16_u16(diff[3]));
sse = vmull_s16(diff_lo[0], diff_lo[0]);
sse = vmlal_s16(sse, diff_lo[1], diff_lo[1]);
sse = vmlal_s16(sse, diff_lo[2], diff_lo[2]);
sse = vmlal_s16(sse, diff_lo[3], diff_lo[3]);
return horizontal_add_uint32x4(vreinterpretq_u32_s32(sse));
}
#endif
libs/libvpx/vpx_dsp/arm/vpx_convolve8_neon.c
+748 -10
View File
@@ -14,6 +14,7 @@
#include "./vpx_config.h"
#include "./vpx_dsp_rtcd.h"
#include "vpx/vpx_integer.h"
#include "vpx_dsp/arm/mem_neon.h"
#include "vpx_dsp/arm/transpose_neon.h"
#include "vpx_dsp/arm/vpx_convolve8_neon.h"
#include "vpx_ports/mem.h"
@@ -30,6 +31,741 @@
// instructions. This optimization is much faster in speed unit test, but slowed
// down the whole decoder by 5%.
#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
(__ARM_FEATURE_DOTPROD == 1)
DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = {
0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
};
DECLARE_ALIGNED(16, static const uint8_t, dot_prod_tran_concat_tbl[32]) = {
0, 8, 16, 24, 1, 9, 17, 25, 2, 10, 18, 26, 3, 11, 19, 27,
4, 12, 20, 28, 5, 13, 21, 29, 6, 14, 22, 30, 7, 15, 23, 31
};
DECLARE_ALIGNED(16, static const uint8_t, dot_prod_merge_block_tbl[48]) = {
/* Shift left and insert new last column in transposed 4x4 block. */
1, 2, 3, 16, 5, 6, 7, 20, 9, 10, 11, 24, 13, 14, 15, 28,
/* Shift left and insert two new columns in transposed 4x4 block. */
2, 3, 16, 17, 6, 7, 20, 21, 10, 11, 24, 25, 14, 15, 28, 29,
/* Shift left and insert three new columns in transposed 4x4 block. */
3, 16, 17, 18, 7, 20, 21, 22, 11, 24, 25, 26, 15, 28, 29, 30
};
static INLINE void transpose_concat_4x4(int8x8_t *a0, int8x8_t *a1,
int8x8_t *a2, int8x8_t *a3,
int8x16_t *b,
const uint8x16_t permute_tbl) {
/* Transpose 8-bit elements and concatenate result rows as follows:
* a0: 00, 01, 02, 03, XX, XX, XX, XX
* a1: 10, 11, 12, 13, XX, XX, XX, XX
* a2: 20, 21, 22, 23, XX, XX, XX, XX
* a3: 30, 31, 32, 33, XX, XX, XX, XX
*
* b: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
*
* The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
* as an argument is preferable to loading it directly from memory as this
* inline helper is called many times from the same parent function.
*/
int8x16x2_t samples = { { vcombine_s8(*a0, *a1), vcombine_s8(*a2, *a3) } };
*b = vqtbl2q_s8(samples, permute_tbl);
}
static INLINE void transpose_concat_8x4(int8x8_t *a0, int8x8_t *a1,
int8x8_t *a2, int8x8_t *a3,
int8x16_t *b0, int8x16_t *b1,
const uint8x16x2_t permute_tbl) {
/* Transpose 8-bit elements and concatenate result rows as follows:
* a0: 00, 01, 02, 03, 04, 05, 06, 07
* a1: 10, 11, 12, 13, 14, 15, 16, 17
* a2: 20, 21, 22, 23, 24, 25, 26, 27
* a3: 30, 31, 32, 33, 34, 35, 36, 37
*
* b0: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
* b1: 04, 14, 24, 34, 05, 15, 25, 35, 06, 16, 26, 36, 07, 17, 27, 37
*
* The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
* as an argument is preferable to loading it directly from memory as this
* inline helper is called many times from the same parent function.
*/
int8x16x2_t samples = { { vcombine_s8(*a0, *a1), vcombine_s8(*a2, *a3) } };
*b0 = vqtbl2q_s8(samples, permute_tbl.val[0]);
*b1 = vqtbl2q_s8(samples, permute_tbl.val[1]);
}
void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const InterpKernel *filter, int x0_q4,
int x_step_q4, int y0_q4, int y_step_q4, int w,
int h) {
const int8x8_t filters = vmovn_s16(vld1q_s16(filter[x0_q4]));
const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[x0_q4]), 128);
const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
const uint8x16_t range_limit = vdupq_n_u8(128);
uint8x16_t s0, s1, s2, s3;
assert(!((intptr_t)dst & 3));
assert(!(dst_stride & 3));
assert(x_step_q4 == 16);
(void)x_step_q4;
(void)y0_q4;
(void)y_step_q4;
src -= 3;
if (w == 4) {
const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
do {
int32x4_t t0, t1, t2, t3;
int16x8_t t01, t23;
uint8x8_t d01, d23;
s0 = vld1q_u8(src);
src += src_stride;
s1 = vld1q_u8(src);
src += src_stride;
s2 = vld1q_u8(src);
src += src_stride;
s3 = vld1q_u8(src);
src += src_stride;
t0 = convolve8_4_dot(s0, filters, correction, range_limit, permute_tbl);
t1 = convolve8_4_dot(s1, filters, correction, range_limit, permute_tbl);
t2 = convolve8_4_dot(s2, filters, correction, range_limit, permute_tbl);
t3 = convolve8_4_dot(s3, filters, correction, range_limit, permute_tbl);
t01 = vcombine_s16(vqmovn_s32(t0), vqmovn_s32(t1));
t23 = vcombine_s16(vqmovn_s32(t2), vqmovn_s32(t3));
d01 = vqrshrun_n_s16(t01, 7);
d23 = vqrshrun_n_s16(t23, 7);
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
dst += dst_stride;
h -= 4;
} while (h > 0);
} else {
const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
const uint8_t *s;
uint8_t *d;
int width;
uint8x8_t d0, d1, d2, d3;
do {
width = w;
s = src;
d = dst;
do {
s0 = vld1q_u8(s + 0 * src_stride);
s1 = vld1q_u8(s + 1 * src_stride);
s2 = vld1q_u8(s + 2 * src_stride);
s3 = vld1q_u8(s + 3 * src_stride);
d0 = convolve8_8_dot(s0, filters, correction, range_limit, permute_tbl);
d1 = convolve8_8_dot(s1, filters, correction, range_limit, permute_tbl);
d2 = convolve8_8_dot(s2, filters, correction, range_limit, permute_tbl);
d3 = convolve8_8_dot(s3, filters, correction, range_limit, permute_tbl);
vst1_u8(d + 0 * dst_stride, d0);
vst1_u8(d + 1 * dst_stride, d1);
vst1_u8(d + 2 * dst_stride, d2);
vst1_u8(d + 3 * dst_stride, d3);
s += 8;
d += 8;
width -= 8;
} while (width > 0);
src += 4 * src_stride;
dst += 4 * dst_stride;
h -= 4;
} while (h > 0);
}
}
void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const InterpKernel *filter, int x0_q4,
int x_step_q4, int y0_q4, int y_step_q4,
int w, int h) {
const int8x8_t filters = vmovn_s16(vld1q_s16(filter[x0_q4]));
const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[x0_q4]), 128);
const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
const uint8x16_t range_limit = vdupq_n_u8(128);
uint8x16_t s0, s1, s2, s3;
assert(!((intptr_t)dst & 3));
assert(!(dst_stride & 3));
assert(x_step_q4 == 16);
(void)x_step_q4;
(void)y0_q4;
(void)y_step_q4;
src -= 3;
if (w == 4) {
const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
do {
int32x4_t t0, t1, t2, t3;
int16x8_t t01, t23;
uint8x8_t d01, d23, dd01, dd23;
dd01 = vdup_n_u8(0);
dd23 = vdup_n_u8(0);
s0 = vld1q_u8(src);
src += src_stride;
s1 = vld1q_u8(src);
src += src_stride;
s2 = vld1q_u8(src);
src += src_stride;
s3 = vld1q_u8(src);
src += src_stride;
t0 = convolve8_4_dot(s0, filters, correction, range_limit, permute_tbl);
t1 = convolve8_4_dot(s1, filters, correction, range_limit, permute_tbl);
t2 = convolve8_4_dot(s2, filters, correction, range_limit, permute_tbl);
t3 = convolve8_4_dot(s3, filters, correction, range_limit, permute_tbl);
t01 = vcombine_s16(vqmovn_s32(t0), vqmovn_s32(t1));
t23 = vcombine_s16(vqmovn_s32(t2), vqmovn_s32(t3));
d01 = vqrshrun_n_s16(t01, 7);
d23 = vqrshrun_n_s16(t23, 7);
dd01 = load_u8(dst + 0 * dst_stride, dst_stride);
dd23 = load_u8(dst + 2 * dst_stride, dst_stride);
d01 = vrhadd_u8(d01, dd01);
d23 = vrhadd_u8(d23, dd23);
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
dst += dst_stride;
h -= 4;
} while (h > 0);
} else {
const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
const uint8_t *s;
uint8_t *d;
int width;
uint8x8_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
do {
width = w;
s = src;
d = dst;
do {
s0 = vld1q_u8(s + 0 * src_stride);
s1 = vld1q_u8(s + 1 * src_stride);
s2 = vld1q_u8(s + 2 * src_stride);
s3 = vld1q_u8(s + 3 * src_stride);
d0 = convolve8_8_dot(s0, filters, correction, range_limit, permute_tbl);
d1 = convolve8_8_dot(s1, filters, correction, range_limit, permute_tbl);
d2 = convolve8_8_dot(s2, filters, correction, range_limit, permute_tbl);
d3 = convolve8_8_dot(s3, filters, correction, range_limit, permute_tbl);
dd0 = vld1_u8(d + 0 * dst_stride);
dd1 = vld1_u8(d + 1 * dst_stride);
dd2 = vld1_u8(d + 2 * dst_stride);
dd3 = vld1_u8(d + 3 * dst_stride);
d0 = vrhadd_u8(d0, dd0);
d1 = vrhadd_u8(d1, dd1);
d2 = vrhadd_u8(d2, dd2);
d3 = vrhadd_u8(d3, dd3);
vst1_u8(d + 0 * dst_stride, d0);
vst1_u8(d + 1 * dst_stride, d1);
vst1_u8(d + 2 * dst_stride, d2);
vst1_u8(d + 3 * dst_stride, d3);
s += 8;
d += 8;
width -= 8;
} while (width > 0);
src += 4 * src_stride;
dst += 4 * dst_stride;
h -= 4;
} while (h > 0);
}
}
void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const InterpKernel *filter, int x0_q4,
int x_step_q4, int y0_q4, int y_step_q4, int w,
int h) {
const int8x8_t filters = vmovn_s16(vld1q_s16(filter[y0_q4]));
const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[y0_q4]), 128);
const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
const uint8x8_t range_limit = vdup_n_u8(128);
const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(dot_prod_merge_block_tbl);
uint8x8_t t0, t1, t2, t3, t4, t5, t6;
int8x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
int8x16x2_t samples_LUT;
assert(!((intptr_t)dst & 3));
assert(!(dst_stride & 3));
assert(y_step_q4 == 16);
(void)x0_q4;
(void)x_step_q4;
(void)y_step_q4;
src -= 3 * src_stride;
if (w == 4) {
const uint8x16_t tran_concat_tbl = vld1q_u8(dot_prod_tran_concat_tbl);
int8x16_t s0123, s1234, s2345, s3456, s4567, s5678, s6789, s78910;
int32x4_t d0, d1, d2, d3;
uint8x8_t d01, d23;
load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
src += 4 * src_stride;
t4 = vld1_u8(src);
src += src_stride;
t5 = vld1_u8(src);
src += src_stride;
t6 = vld1_u8(src);
src += src_stride;
/* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
s7 = vdup_n_s8(0);
s8 = vdup_n_s8(0);
s9 = vdup_n_s8(0);
/* This operation combines a conventional transpose and the sample permute
* (see horizontal case) required before computing the dot product.
*/
transpose_concat_4x4(&s0, &s1, &s2, &s3, &s0123, tran_concat_tbl);
transpose_concat_4x4(&s1, &s2, &s3, &s4, &s1234, tran_concat_tbl);
transpose_concat_4x4(&s2, &s3, &s4, &s5, &s2345, tran_concat_tbl);
transpose_concat_4x4(&s3, &s4, &s5, &s6, &s3456, tran_concat_tbl);
transpose_concat_4x4(&s4, &s5, &s6, &s7, &s4567, tran_concat_tbl);
transpose_concat_4x4(&s5, &s6, &s7, &s8, &s5678, tran_concat_tbl);
transpose_concat_4x4(&s6, &s7, &s8, &s9, &s6789, tran_concat_tbl);
do {
uint8x8_t t7, t8, t9, t10;
load_u8_8x4(src, src_stride, &t7, &t8, &t9, &t10);
s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
transpose_concat_4x4(&s7, &s8, &s9, &s10, &s78910, tran_concat_tbl);
/* Merge new data into block from previous iteration. */
samples_LUT.val[0] = s3456;
samples_LUT.val[1] = s78910;
s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
d0 = convolve8_4_dot_partial(s0123, s4567, correction, filters);
d1 = convolve8_4_dot_partial(s1234, s5678, correction, filters);
d2 = convolve8_4_dot_partial(s2345, s6789, correction, filters);
d3 = convolve8_4_dot_partial(s3456, s78910, correction, filters);
d01 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d0), vqmovn_s32(d1)), 7);
d23 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d2), vqmovn_s32(d3)), 7);
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
dst += dst_stride;
/* Prepare block for next iteration - re-using as much as possible. */
/* Shuffle everything up four rows. */
s0123 = s4567;
s1234 = s5678;
s2345 = s6789;
s3456 = s78910;
src += 4 * src_stride;
h -= 4;
} while (h > 0);
} else {
const uint8x16x2_t tran_concat_tbl = vld1q_u8_x2(dot_prod_tran_concat_tbl);
int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi,
s3456_lo, s3456_hi, s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo,
s6789_hi, s78910_lo, s78910_hi;
uint8x8_t d0, d1, d2, d3;
const uint8_t *s;
uint8_t *d;
int height;
do {
height = h;
s = src;
d = dst;
load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
s += 4 * src_stride;
t4 = vld1_u8(s);
s += src_stride;
t5 = vld1_u8(s);
s += src_stride;
t6 = vld1_u8(s);
s += src_stride;
/* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
s7 = vdup_n_s8(0);
s8 = vdup_n_s8(0);
s9 = vdup_n_s8(0);
/* This operation combines a conventional transpose and the sample permute
* (see horizontal case) required before computing the dot product.
*/
transpose_concat_8x4(&s0, &s1, &s2, &s3, &s0123_lo, &s0123_hi,
tran_concat_tbl);
transpose_concat_8x4(&s1, &s2, &s3, &s4, &s1234_lo, &s1234_hi,
tran_concat_tbl);
transpose_concat_8x4(&s2, &s3, &s4, &s5, &s2345_lo, &s2345_hi,
tran_concat_tbl);
transpose_concat_8x4(&s3, &s4, &s5, &s6, &s3456_lo, &s3456_hi,
tran_concat_tbl);
transpose_concat_8x4(&s4, &s5, &s6, &s7, &s4567_lo, &s4567_hi,
tran_concat_tbl);
transpose_concat_8x4(&s5, &s6, &s7, &s8, &s5678_lo, &s5678_hi,
tran_concat_tbl);
transpose_concat_8x4(&s6, &s7, &s8, &s9, &s6789_lo, &s6789_hi,
tran_concat_tbl);
do {
uint8x8_t t7, t8, t9, t10;
load_u8_8x4(s, src_stride, &t7, &t8, &t9, &t10);
s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
transpose_concat_8x4(&s7, &s8, &s9, &s10, &s78910_lo, &s78910_hi,
tran_concat_tbl);
/* Merge new data into block from previous iteration. */
samples_LUT.val[0] = s3456_lo;
samples_LUT.val[1] = s78910_lo;
s4567_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
s5678_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
s6789_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
samples_LUT.val[0] = s3456_hi;
samples_LUT.val[1] = s78910_hi;
s4567_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
s5678_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
s6789_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
d0 = convolve8_8_dot_partial(s0123_lo, s4567_lo, s0123_hi, s4567_hi,
correction, filters);
d1 = convolve8_8_dot_partial(s1234_lo, s5678_lo, s1234_hi, s5678_hi,
correction, filters);
d2 = convolve8_8_dot_partial(s2345_lo, s6789_lo, s2345_hi, s6789_hi,
correction, filters);
d3 = convolve8_8_dot_partial(s3456_lo, s78910_lo, s3456_hi, s78910_hi,
correction, filters);
vst1_u8(d + 0 * dst_stride, d0);
vst1_u8(d + 1 * dst_stride, d1);
vst1_u8(d + 2 * dst_stride, d2);
vst1_u8(d + 3 * dst_stride, d3);
/* Prepare block for next iteration - re-using as much as possible. */
/* Shuffle everything up four rows. */
s0123_lo = s4567_lo;
s0123_hi = s4567_hi;
s1234_lo = s5678_lo;
s1234_hi = s5678_hi;
s2345_lo = s6789_lo;
s2345_hi = s6789_hi;
s3456_lo = s78910_lo;
s3456_hi = s78910_hi;
s += 4 * src_stride;
d += 4 * dst_stride;
height -= 4;
} while (height > 0);
src += 8;
dst += 8;
w -= 8;
} while (w > 0);
}
}
void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const InterpKernel *filter, int x0_q4,
int x_step_q4, int y0_q4, int y_step_q4, int w,
int h) {
const int8x8_t filters = vmovn_s16(vld1q_s16(filter[y0_q4]));
const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[y0_q4]), 128);
const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
const uint8x8_t range_limit = vdup_n_u8(128);
const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(dot_prod_merge_block_tbl);
uint8x8_t t0, t1, t2, t3, t4, t5, t6;
int8x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
int8x16x2_t samples_LUT;
assert(!((intptr_t)dst & 3));
assert(!(dst_stride & 3));
assert(y_step_q4 == 16);
(void)x0_q4;
(void)x_step_q4;
(void)y_step_q4;
src -= 3 * src_stride;
if (w == 4) {
const uint8x16_t tran_concat_tbl = vld1q_u8(dot_prod_tran_concat_tbl);
int8x16_t s0123, s1234, s2345, s3456, s4567, s5678, s6789, s78910;
int32x4_t d0, d1, d2, d3;
uint8x8_t d01, d23, dd01, dd23;
load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
src += 4 * src_stride;
t4 = vld1_u8(src);
src += src_stride;
t5 = vld1_u8(src);
src += src_stride;
t6 = vld1_u8(src);
src += src_stride;
/* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
s7 = vdup_n_s8(0);
s8 = vdup_n_s8(0);
s9 = vdup_n_s8(0);
/* This operation combines a conventional transpose and the sample permute
* (see horizontal case) required before computing the dot product.
*/
transpose_concat_4x4(&s0, &s1, &s2, &s3, &s0123, tran_concat_tbl);
transpose_concat_4x4(&s1, &s2, &s3, &s4, &s1234, tran_concat_tbl);
transpose_concat_4x4(&s2, &s3, &s4, &s5, &s2345, tran_concat_tbl);
transpose_concat_4x4(&s3, &s4, &s5, &s6, &s3456, tran_concat_tbl);
transpose_concat_4x4(&s4, &s5, &s6, &s7, &s4567, tran_concat_tbl);
transpose_concat_4x4(&s5, &s6, &s7, &s8, &s5678, tran_concat_tbl);
transpose_concat_4x4(&s6, &s7, &s8, &s9, &s6789, tran_concat_tbl);
do {
uint8x8_t t7, t8, t9, t10;
load_u8_8x4(src, src_stride, &t7, &t8, &t9, &t10);
s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
transpose_concat_4x4(&s7, &s8, &s9, &s10, &s78910, tran_concat_tbl);
/* Merge new data into block from previous iteration. */
samples_LUT.val[0] = s3456;
samples_LUT.val[1] = s78910;
s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
d0 = convolve8_4_dot_partial(s0123, s4567, correction, filters);
d1 = convolve8_4_dot_partial(s1234, s5678, correction, filters);
d2 = convolve8_4_dot_partial(s2345, s6789, correction, filters);
d3 = convolve8_4_dot_partial(s3456, s78910, correction, filters);
d01 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d0), vqmovn_s32(d1)), 7);
d23 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d2), vqmovn_s32(d3)), 7);
dd01 = load_u8(dst + 0 * dst_stride, dst_stride);
dd23 = load_u8(dst + 2 * dst_stride, dst_stride);
d01 = vrhadd_u8(d01, dd01);
d23 = vrhadd_u8(d23, dd23);
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
dst += dst_stride;
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
dst += dst_stride;
/* Prepare block for next iteration - re-using as much as possible. */
/* Shuffle everything up four rows. */
s0123 = s4567;
s1234 = s5678;
s2345 = s6789;
s3456 = s78910;
src += 4 * src_stride;
h -= 4;
} while (h > 0);
} else {
const uint8x16x2_t tran_concat_tbl = vld1q_u8_x2(dot_prod_tran_concat_tbl);
int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi,
s3456_lo, s3456_hi, s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo,
s6789_hi, s78910_lo, s78910_hi;
uint8x8_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
const uint8_t *s;
uint8_t *d;
int height;
do {
height = h;
s = src;
d = dst;
load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
s += 4 * src_stride;
t4 = vld1_u8(s);
s += src_stride;
t5 = vld1_u8(s);
s += src_stride;
t6 = vld1_u8(s);
s += src_stride;
/* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
s7 = vdup_n_s8(0);
s8 = vdup_n_s8(0);
s9 = vdup_n_s8(0);
/* This operation combines a conventional transpose and the sample permute
* (see horizontal case) required before computing the dot product.
*/
transpose_concat_8x4(&s0, &s1, &s2, &s3, &s0123_lo, &s0123_hi,
tran_concat_tbl);
transpose_concat_8x4(&s1, &s2, &s3, &s4, &s1234_lo, &s1234_hi,
tran_concat_tbl);
transpose_concat_8x4(&s2, &s3, &s4, &s5, &s2345_lo, &s2345_hi,
tran_concat_tbl);
transpose_concat_8x4(&s3, &s4, &s5, &s6, &s3456_lo, &s3456_hi,
tran_concat_tbl);
transpose_concat_8x4(&s4, &s5, &s6, &s7, &s4567_lo, &s4567_hi,
tran_concat_tbl);
transpose_concat_8x4(&s5, &s6, &s7, &s8, &s5678_lo, &s5678_hi,
tran_concat_tbl);
transpose_concat_8x4(&s6, &s7, &s8, &s9, &s6789_lo, &s6789_hi,
tran_concat_tbl);
do {
uint8x8_t t7, t8, t9, t10;
load_u8_8x4(s, src_stride, &t7, &t8, &t9, &t10);
s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
transpose_concat_8x4(&s7, &s8, &s9, &s10, &s78910_lo, &s78910_hi,
tran_concat_tbl);
/* Merge new data into block from previous iteration. */
samples_LUT.val[0] = s3456_lo;
samples_LUT.val[1] = s78910_lo;
s4567_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
s5678_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
s6789_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
samples_LUT.val[0] = s3456_hi;
samples_LUT.val[1] = s78910_hi;
s4567_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
s5678_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
s6789_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
d0 = convolve8_8_dot_partial(s0123_lo, s4567_lo, s0123_hi, s4567_hi,
correction, filters);
d1 = convolve8_8_dot_partial(s1234_lo, s5678_lo, s1234_hi, s5678_hi,
correction, filters);
d2 = convolve8_8_dot_partial(s2345_lo, s6789_lo, s2345_hi, s6789_hi,
correction, filters);
d3 = convolve8_8_dot_partial(s3456_lo, s78910_lo, s3456_hi, s78910_hi,
correction, filters);
dd0 = vld1_u8(d + 0 * dst_stride);
dd1 = vld1_u8(d + 1 * dst_stride);
dd2 = vld1_u8(d + 2 * dst_stride);
dd3 = vld1_u8(d + 3 * dst_stride);
d0 = vrhadd_u8(d0, dd0);
d1 = vrhadd_u8(d1, dd1);
d2 = vrhadd_u8(d2, dd2);
d3 = vrhadd_u8(d3, dd3);
vst1_u8(d + 0 * dst_stride, d0);
vst1_u8(d + 1 * dst_stride, d1);
vst1_u8(d + 2 * dst_stride, d2);
vst1_u8(d + 3 * dst_stride, d3);
/* Prepare block for next iteration - re-using as much as possible. */
/* Shuffle everything up four rows. */
s0123_lo = s4567_lo;
s0123_hi = s4567_hi;
s1234_lo = s5678_lo;
s1234_hi = s5678_hi;
s2345_lo = s6789_lo;
s2345_hi = s6789_hi;
s3456_lo = s78910_lo;
s3456_hi = s78910_hi;
s += 4 * src_stride;
d += 4 * dst_stride;
height -= 4;
} while (height > 0);
src += 8;
dst += 8;
w -= 8;
} while (w > 0);
}
}
#else
static INLINE void store_u8_8x8(uint8_t *s, const ptrdiff_t p,
const uint8x8_t s0, const uint8x8_t s1,
const uint8x8_t s2, const uint8x8_t s3,
@@ -145,7 +881,7 @@ void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
src += 4;
dst += 4;
w -= 4;
} while (w > 0);
} while (w != 0);
} else {
const int16x8_t filter3 = vdupq_lane_s16(vget_low_s16(filters), 3);
const int16x8_t filter4 = vdupq_lane_s16(vget_high_s16(filters), 0);
@@ -296,7 +1032,7 @@ void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
s += 8;
d += 8;
width -= 8;
} while (width > 0);
} while (width != 0);
src += 8 * src_stride;
dst += 8 * dst_stride;
h -= 8;
@@ -402,7 +1138,7 @@ void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
src += 4;
dst += 4;
w -= 4;
} while (w > 0);
} while (w != 0);
} else {
const int16x8_t filter3 = vdupq_lane_s16(vget_low_s16(filters), 3);
const int16x8_t filter4 = vdupq_lane_s16(vget_high_s16(filters), 0);
@@ -586,7 +1322,7 @@ void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
s += 8;
d += 8;
width -= 8;
} while (width > 0);
} while (width != 0);
src += 8 * src_stride;
dst += 8 * dst_stride;
h -= 8;
@@ -679,7 +1415,7 @@ void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
s5 = s9;
s6 = s10;
h -= 4;
} while (h > 0);
} while (h != 0);
} else {
const int16x8_t filter3 = vdupq_lane_s16(vget_low_s16(filters), 3);
const int16x8_t filter4 = vdupq_lane_s16(vget_high_s16(filters), 0);
@@ -759,11 +1495,11 @@ void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
s5 = s9;
s6 = s10;
height -= 4;
} while (height > 0);
} while (height != 0);
src += 8;
dst += 8;
w -= 8;
} while (w > 0);
} while (w != 0);
}
}
@@ -860,7 +1596,7 @@ void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
s5 = s9;
s6 = s10;
h -= 4;
} while (h > 0);
} while (h != 0);
} else {
const int16x8_t filter3 = vdupq_lane_s16(vget_low_s16(filters), 3);
const int16x8_t filter4 = vdupq_lane_s16(vget_high_s16(filters), 0);
@@ -950,10 +1686,12 @@ void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
s5 = s9;
s6 = s10;
height -= 4;
} while (height > 0);
} while (height != 0);
src += 8;
dst += 8;
w -= 8;
} while (w > 0);
} while (w != 0);
}
}
#endif
libs/libvpx/vpx_dsp/arm/vpx_convolve8_neon.h
+101
View File
@@ -72,6 +72,107 @@ static INLINE void load_u8_16x8(const uint8_t *s, const ptrdiff_t p,
*s7 = vld1q_u8(s);
}
#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
(__ARM_FEATURE_DOTPROD == 1)
static INLINE int32x4_t convolve8_4_dot_partial(const int8x16_t samples_lo,
const int8x16_t samples_hi,
const int32x4_t correction,
const int8x8_t filters) {
/* Sample range-clamping and permutation are performed by the caller. */
int32x4_t sum;
/* Accumulate dot product into 'correction' to account for range clamp. */
sum = vdotq_lane_s32(correction, samples_lo, filters, 0);
sum = vdotq_lane_s32(sum, samples_hi, filters, 1);
/* Narrowing and packing is performed by the caller. */
return sum;
}
static INLINE int32x4_t convolve8_4_dot(uint8x16_t samples,
const int8x8_t filters,
const int32x4_t correction,
const uint8x16_t range_limit,
const uint8x16x2_t permute_tbl) {
int8x16_t clamped_samples, permuted_samples[2];
int32x4_t sum;
/* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
/* Permute samples ready for dot product. */
/* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
/* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
/* Accumulate dot product into 'correction' to account for range clamp. */
sum = vdotq_lane_s32(correction, permuted_samples[0], filters, 0);
sum = vdotq_lane_s32(sum, permuted_samples[1], filters, 1);
/* Narrowing and packing is performed by the caller. */
return sum;
}
static INLINE uint8x8_t convolve8_8_dot_partial(const int8x16_t samples0_lo,
const int8x16_t samples0_hi,
const int8x16_t samples1_lo,
const int8x16_t samples1_hi,
const int32x4_t correction,
const int8x8_t filters) {
/* Sample range-clamping and permutation are performed by the caller. */
int32x4_t sum0, sum1;
int16x8_t sum;
/* Accumulate dot product into 'correction' to account for range clamp. */
/* First 4 output values. */
sum0 = vdotq_lane_s32(correction, samples0_lo, filters, 0);
sum0 = vdotq_lane_s32(sum0, samples0_hi, filters, 1);
/* Second 4 output values. */
sum1 = vdotq_lane_s32(correction, samples1_lo, filters, 0);
sum1 = vdotq_lane_s32(sum1, samples1_hi, filters, 1);
/* Narrow and re-pack. */
sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
return vqrshrun_n_s16(sum, 7);
}
static INLINE uint8x8_t convolve8_8_dot(uint8x16_t samples,
const int8x8_t filters,
const int32x4_t correction,
const uint8x16_t range_limit,
const uint8x16x3_t permute_tbl) {
int8x16_t clamped_samples, permuted_samples[3];
int32x4_t sum0, sum1;
int16x8_t sum;
/* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
/* Permute samples ready for dot product. */
/* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
/* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
/* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
/* Accumulate dot product into 'correction' to account for range clamp. */
/* First 4 output values. */
sum0 = vdotq_lane_s32(correction, permuted_samples[0], filters, 0);
sum0 = vdotq_lane_s32(sum0, permuted_samples[1], filters, 1);
/* Second 4 output values. */
sum1 = vdotq_lane_s32(correction, permuted_samples[1], filters, 0);
sum1 = vdotq_lane_s32(sum1, permuted_samples[2], filters, 1);
/* Narrow and re-pack. */
sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
return vqrshrun_n_s16(sum, 7);
}
#endif
static INLINE int16x4_t convolve8_4(const int16x4_t s0, const int16x4_t s1,
const int16x4_t s2, const int16x4_t s3,
const int16x4_t s4, const int16x4_t s5,
libs/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.c
+4 -4
View File
@@ -35,7 +35,7 @@
} \
}
DEFINE_FILTER(horiz);
DEFINE_FILTER(avg_horiz);
DEFINE_FILTER(vert);
DEFINE_FILTER(avg_vert);
DEFINE_FILTER(horiz)
DEFINE_FILTER(avg_horiz)
DEFINE_FILTER(vert)
DEFINE_FILTER(avg_vert)
libs/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.h
+8 -8
View File
@@ -17,13 +17,13 @@
ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, \
int x_step_q4, int y0_q4, int y_step_q4, int w, int h);
DECLARE_FILTER(horiz, type1);
DECLARE_FILTER(avg_horiz, type1);
DECLARE_FILTER(horiz, type2);
DECLARE_FILTER(avg_horiz, type2);
DECLARE_FILTER(vert, type1);
DECLARE_FILTER(avg_vert, type1);
DECLARE_FILTER(vert, type2);
DECLARE_FILTER(avg_vert, type2);
DECLARE_FILTER(horiz, type1)
DECLARE_FILTER(avg_horiz, type1)
DECLARE_FILTER(horiz, type2)
DECLARE_FILTER(avg_horiz, type2)
DECLARE_FILTER(vert, type1)
DECLARE_FILTER(avg_vert, type1)
DECLARE_FILTER(vert, type2)
DECLARE_FILTER(avg_vert, type2)
#endif // VPX_VPX_DSP_ARM_VPX_CONVOLVE8_NEON_ASM_H_
libs/libvpx/vpx_dsp/arm/vpx_convolve_avg_neon.c
+4 -4
View File
@@ -43,7 +43,7 @@ void vpx_convolve_avg_neon(const uint8_t *src, ptrdiff_t src_stride,
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(dd0), 1);
dst += dst_stride;
h -= 2;
} while (h > 0);
} while (h != 0);
} else if (w == 8) { // avg8
uint8x8_t s0, s1, d0, d1;
uint8x16_t s01, d01;
@@ -64,7 +64,7 @@ void vpx_convolve_avg_neon(const uint8_t *src, ptrdiff_t src_stride,
vst1_u8(dst, vget_high_u8(d01));
dst += dst_stride;
h -= 2;
} while (h > 0);
} while (h != 0);
} else if (w < 32) { // avg16
uint8x16_t s0, s1, d0, d1;
do {
@@ -83,7 +83,7 @@ void vpx_convolve_avg_neon(const uint8_t *src, ptrdiff_t src_stride,
vst1q_u8(dst, d1);
dst += dst_stride;
h -= 2;
} while (h > 0);
} while (h != 0);
} else if (w == 32) { // avg32
uint8x16_t s0, s1, s2, s3, d0, d1, d2, d3;
do {
@@ -110,7 +110,7 @@ void vpx_convolve_avg_neon(const uint8_t *src, ptrdiff_t src_stride,
vst1q_u8(dst + 16, d3);
dst += dst_stride;
h -= 2;
} while (h > 0);
} while (h != 0);
} else { // avg64
uint8x16_t s0, s1, s2, s3, d0, d1, d2, d3;
do {
libs/libvpx/vpx_dsp/arm/vpx_convolve_copy_neon.c
+4 -4
View File
@@ -33,7 +33,7 @@ void vpx_convolve_copy_neon(const uint8_t *src, ptrdiff_t src_stride,
src += src_stride;
dst += dst_stride;
h -= 2;
} while (h > 0);
} while (h != 0);
} else if (w == 8) { // copy8
uint8x8_t s0, s1;
do {
@@ -47,7 +47,7 @@ void vpx_convolve_copy_neon(const uint8_t *src, ptrdiff_t src_stride,
vst1_u8(dst, s1);
dst += dst_stride;
h -= 2;
} while (h > 0);
} while (h != 0);
} else if (w < 32) { // copy16
uint8x16_t s0, s1;
do {
@@ -61,7 +61,7 @@ void vpx_convolve_copy_neon(const uint8_t *src, ptrdiff_t src_stride,
vst1q_u8(dst, s1);
dst += dst_stride;
h -= 2;
} while (h > 0);
} while (h != 0);
} else if (w == 32) { // copy32
uint8x16_t s0, s1, s2, s3;
do {
@@ -79,7 +79,7 @@ void vpx_convolve_copy_neon(const uint8_t *src, ptrdiff_t src_stride,
vst1q_u8(dst + 16, s3);
dst += dst_stride;
h -= 2;
} while (h > 0);
} while (h != 0);
} else { // copy64
uint8x16_t s0, s1, s2, s3;
do {
libs/libvpx/vpx_dsp/avg.c
-1
View File
@@ -344,7 +344,6 @@ void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref,
const int ref_stride, const int height) {
int idx;
const int norm_factor = height >> 1;
if (height == 1) return;
for (idx = 0; idx < 16; ++idx) {
int i;
hbuf[idx] = 0;
libs/libvpx/vpx_dsp/bitwriter.h
+1 -1
View File
@@ -85,7 +85,7 @@ static INLINE void vpx_write(vpx_writer *br, int bit, int probability) {
br->buffer[x] += 1;
}
br->buffer[br->pos++] = (lowvalue >> (24 - offset));
br->buffer[br->pos++] = (lowvalue >> (24 - offset)) & 0xff;
lowvalue <<= offset;
shift = count;
lowvalue &= 0xffffff;
libs/libvpx/vpx_dsp/deblock.c
-5
View File
@@ -52,11 +52,6 @@ void vpx_post_proc_down_and_across_mb_row_c(unsigned char *src,
assert(size >= 8);
assert(cols >= 8);
#ifdef __clang_analyzer__
/* older clang analyzer doesnt understand asserts, insert this until we ugrade analyzer */
if (cols < 8) return;
#endif
for (row = 0; row < size; row++) {
/* post_proc_down for one row */
p_src = src;
libs/libvpx/vpx_dsp/fastssim.c
+4 -2
View File
@@ -47,7 +47,7 @@ struct fs_ctx {
unsigned *col_buf;
};
static void fs_ctx_init(fs_ctx *_ctx, int _w, int _h, int _nlevels) {
static int fs_ctx_init(fs_ctx *_ctx, int _w, int _h, int _nlevels) {
unsigned char *data;
size_t data_size;
int lw;
@@ -71,6 +71,7 @@ static void fs_ctx_init(fs_ctx *_ctx, int _w, int _h, int _nlevels) {
lh = (lh + 1) >> 1;
}
data = (unsigned char *)malloc(data_size);
if (!data) return -1;
_ctx->level = (fs_level *)data;
_ctx->nlevels = _nlevels;
data += _nlevels * sizeof(*_ctx->level);
@@ -95,6 +96,7 @@ static void fs_ctx_init(fs_ctx *_ctx, int _w, int _h, int _nlevels) {
lh = (lh + 1) >> 1;
}
_ctx->col_buf = (unsigned *)data;
return 0;
}
static void fs_ctx_clear(fs_ctx *_ctx) { free(_ctx->level); }
@@ -456,7 +458,7 @@ static double calc_ssim(const uint8_t *_src, int _systride, const uint8_t *_dst,
double ret;
int l;
ret = 1;
fs_ctx_init(&ctx, _w, _h, FS_NLEVELS);
if (fs_ctx_init(&ctx, _w, _h, FS_NLEVELS)) return 99.0;
fs_downsample_level0(&ctx, _src, _systride, _dst, _dystride, _w, _h, _bd,
_shift);
for (l = 0; l < FS_NLEVELS - 1; l++) {
libs/libvpx/vpx_dsp/inv_txfm.c
+16 -16
View File
@@ -701,22 +701,22 @@ void idct16_c(const tran_low_t *input, tran_low_t *output) {
step2[15] = step1[15];
// stage 7
output[0] = WRAPLOW(step2[0] + step2[15]);
output[1] = WRAPLOW(step2[1] + step2[14]);
output[2] = WRAPLOW(step2[2] + step2[13]);
output[3] = WRAPLOW(step2[3] + step2[12]);
output[4] = WRAPLOW(step2[4] + step2[11]);
output[5] = WRAPLOW(step2[5] + step2[10]);
output[6] = WRAPLOW(step2[6] + step2[9]);
output[7] = WRAPLOW(step2[7] + step2[8]);
output[8] = WRAPLOW(step2[7] - step2[8]);
output[9] = WRAPLOW(step2[6] - step2[9]);
output[10] = WRAPLOW(step2[5] - step2[10]);
output[11] = WRAPLOW(step2[4] - step2[11]);
output[12] = WRAPLOW(step2[3] - step2[12]);
output[13] = WRAPLOW(step2[2] - step2[13]);
output[14] = WRAPLOW(step2[1] - step2[14]);
output[15] = WRAPLOW(step2[0] - step2[15]);
output[0] = (tran_low_t)WRAPLOW(step2[0] + step2[15]);
output[1] = (tran_low_t)WRAPLOW(step2[1] + step2[14]);
output[2] = (tran_low_t)WRAPLOW(step2[2] + step2[13]);
output[3] = (tran_low_t)WRAPLOW(step2[3] + step2[12]);
output[4] = (tran_low_t)WRAPLOW(step2[4] + step2[11]);
output[5] = (tran_low_t)WRAPLOW(step2[5] + step2[10]);
output[6] = (tran_low_t)WRAPLOW(step2[6] + step2[9]);
output[7] = (tran_low_t)WRAPLOW(step2[7] + step2[8]);
output[8] = (tran_low_t)WRAPLOW(step2[7] - step2[8]);
output[9] = (tran_low_t)WRAPLOW(step2[6] - step2[9]);
output[10] = (tran_low_t)WRAPLOW(step2[5] - step2[10]);
output[11] = (tran_low_t)WRAPLOW(step2[4] - step2[11]);
output[12] = (tran_low_t)WRAPLOW(step2[3] - step2[12]);
output[13] = (tran_low_t)WRAPLOW(step2[2] - step2[13]);
output[14] = (tran_low_t)WRAPLOW(step2[1] - step2[14]);
output[15] = (tran_low_t)WRAPLOW(step2[0] - step2[15]);
}
void vpx_idct16x16_256_add_c(const tran_low_t *input, uint8_t *dest,
libs/libvpx/vpx_dsp/loongarch/avg_lsx.c
+90
View File
@@ -0,0 +1,90 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stdlib.h>
#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/loongarch/bitdepth_conversion_lsx.h"
void vpx_hadamard_8x8_lsx(const int16_t *src, ptrdiff_t src_stride,
tran_low_t *dst) {
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
ptrdiff_t src_stride2 = src_stride << 1;
ptrdiff_t src_stride3 = src_stride2 + src_stride;
ptrdiff_t src_stride4 = src_stride2 << 1;
ptrdiff_t src_stride6 = src_stride3 << 1;
int16_t *src_tmp = (int16_t *)src;
src0 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride2, src_tmp, src_stride4, src1, src2);
src3 = __lsx_vldx(src_tmp, src_stride6);
src_tmp += src_stride4;
src4 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride2, src_tmp, src_stride4, src5, src6);
src7 = __lsx_vldx(src_tmp, src_stride6);
LSX_BUTTERFLY_8_H(src0, src2, src4, src6, src7, src5, src3, src1, tmp0, tmp2,
tmp4, tmp6, tmp7, tmp5, tmp3, tmp1);
LSX_BUTTERFLY_8_H(tmp0, tmp1, tmp4, tmp5, tmp7, tmp6, tmp3, tmp2, src0, src1,
src4, src5, src7, src6, src3, src2);
LSX_BUTTERFLY_8_H(src0, src1, src2, src3, src7, src6, src5, src4, tmp0, tmp7,
tmp3, tmp4, tmp5, tmp1, tmp6, tmp2);
LSX_TRANSPOSE8x8_H(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, src0, src1,
src2, src3, src4, src5, src6, src7);
LSX_BUTTERFLY_8_H(src0, src2, src4, src6, src7, src5, src3, src1, tmp0, tmp2,
tmp4, tmp6, tmp7, tmp5, tmp3, tmp1);
LSX_BUTTERFLY_8_H(tmp0, tmp1, tmp4, tmp5, tmp7, tmp6, tmp3, tmp2, src0, src1,
src4, src5, src7, src6, src3, src2);
LSX_BUTTERFLY_8_H(src0, src1, src2, src3, src7, src6, src5, src4, tmp0, tmp7,
tmp3, tmp4, tmp5, tmp1, tmp6, tmp2);
store_tran_low(tmp0, dst, 0);
store_tran_low(tmp1, dst, 8);
store_tran_low(tmp2, dst, 16);
store_tran_low(tmp3, dst, 24);
store_tran_low(tmp4, dst, 32);
store_tran_low(tmp5, dst, 40);
store_tran_low(tmp6, dst, 48);
store_tran_low(tmp7, dst, 56);
}
void vpx_hadamard_16x16_lsx(const int16_t *src, ptrdiff_t src_stride,
tran_low_t *dst) {
int i;
__m128i a0, a1, a2, a3, b0, b1, b2, b3;
/* Rearrange 16x16 to 8x32 and remove stride.
* Top left first. */
vpx_hadamard_8x8_lsx(src + 0 + 0 * src_stride, src_stride, dst + 0);
/* Top right. */
vpx_hadamard_8x8_lsx(src + 8 + 0 * src_stride, src_stride, dst + 64);
/* Bottom left. */
vpx_hadamard_8x8_lsx(src + 0 + 8 * src_stride, src_stride, dst + 128);
/* Bottom right. */
vpx_hadamard_8x8_lsx(src + 8 + 8 * src_stride, src_stride, dst + 192);
for (i = 0; i < 64; i += 8) {
a0 = load_tran_low(dst);
a1 = load_tran_low(dst + 64);
a2 = load_tran_low(dst + 128);
a3 = load_tran_low(dst + 192);
LSX_BUTTERFLY_4_H(a0, a2, a3, a1, b0, b2, b3, b1);
DUP4_ARG2(__lsx_vsrai_h, b0, 1, b1, 1, b2, 1, b3, 1, b0, b1, b2, b3);
LSX_BUTTERFLY_4_H(b0, b1, b3, b2, a0, a1, a3, a2);
store_tran_low(a0, dst, 0);
store_tran_low(a1, dst, 64);
store_tran_low(a2, dst, 128);
store_tran_low(a3, dst, 192);
dst += 8;
}
}
libs/libvpx/vpx_dsp/loongarch/avg_pred_lsx.c
+83
View File
@@ -0,0 +1,83 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <assert.h>
#include "./vpx_dsp_rtcd.h"
#include "vpx/vpx_integer.h"
#include "vpx_util/loongson_intrinsics.h"
void vpx_comp_avg_pred_lsx(uint8_t *comp_pred, const uint8_t *pred, int width,
int height, const uint8_t *ref, int ref_stride) {
// width > 8 || width == 8 || width == 4
if (width > 8) {
int i, j;
for (i = 0; i < height; ++i) {
for (j = 0; j < width; j += 16) {
__m128i p, r, avg;
p = __lsx_vld(pred + j, 0);
r = __lsx_vld(ref + j, 0);
avg = __lsx_vavgr_bu(p, r);
__lsx_vst(avg, comp_pred + j, 0);
}
comp_pred += width;
pred += width;
ref += ref_stride;
}
} else if (width == 8) {
int i = height * width;
do {
__m128i p, r, r_0, r_1;
p = __lsx_vld(pred, 0);
r_0 = __lsx_vld(ref, 0);
ref += ref_stride;
r_1 = __lsx_vld(ref, 0);
ref += ref_stride;
r = __lsx_vilvl_d(r_1, r_0);
r = __lsx_vavgr_bu(p, r);
__lsx_vst(r, comp_pred, 0);
pred += 16;
comp_pred += 16;
i -= 16;
} while (i);
} else { // width = 4
int i = height * width;
assert(width == 4);
do {
__m128i p, r, r_0, r_1, r_2, r_3;
p = __lsx_vld(pred, 0);
if (width == ref_stride) {
r = __lsx_vld(ref, 0);
ref += 16;
} else {
r_0 = __lsx_vld(ref, 0);
ref += ref_stride;
r_1 = __lsx_vld(ref, 0);
ref += ref_stride;
r_2 = __lsx_vld(ref, 0);
ref += ref_stride;
r_3 = __lsx_vld(ref, 0);
ref += ref_stride;
DUP2_ARG2(__lsx_vilvl_w, r_1, r_0, r_3, r_2, r_0, r_2);
r = __lsx_vilvl_d(r_2, r_0);
}
r = __lsx_vavgr_bu(p, r);
__lsx_vst(r, comp_pred, 0);
comp_pred += 16;
pred += 16;
i -= 16;
} while (i);
}
}
libs/libvpx/vpx_dsp/loongarch/bitdepth_conversion_lsx.h
+41
View File
@@ -0,0 +1,41 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VPX_VPX_DSP_LOONGARCH_BITDEPTH_CONVERSION_LSX_H_
#define VPX_VPX_DSP_LOONGARCH_BITDEPTH_CONVERSION_LSX_H_
#include "./vpx_config.h"
#include "vpx/vpx_integer.h"
#include "vpx_dsp/vpx_dsp_common.h"
#include "vpx_util/loongson_intrinsics.h"
static INLINE __m128i load_tran_low(const tran_low_t *s) {
#if CONFIG_VP9_HIGHBITDEPTH
__m128i v0_m = __lsx_vld(s, 0);
__m128i v1_m = __lsx_vld(s + 4, 0);
return __lsx_vsrlni_h_w(v0_m, v1_m, 0);
#else
return __lsx_vld(s, 0);
#endif
}
static INLINE void store_tran_low(__m128i v, tran_low_t *s, int32_t c) {
#if CONFIG_VP9_HIGHBITDEPTH
__m128i v0_m, v1_m;
v1_m = __lsx_vexth_w_h(v);
v0_m = __lsx_vsllwil_w_h(v, 0);
__lsx_vst(v0_m, s + c, 0);
__lsx_vst(v1_m, s + c + 4, 0);
#else
__lsx_vst(v, s + c, 0);
#endif
}
#endif // VPX_VPX_DSP_LOONGARCH_BITDEPTH_CONVERSION_LSX_H_
libs/libvpx/vpx_dsp/loongarch/fwd_dct32x32_lsx.c
+1176
View File
File diff suppressed because it is too large. Load diff
libs/libvpx/vpx_dsp/loongarch/fwd_txfm_lsx.c
+350
View File
@@ -0,0 +1,350 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/loongarch/fwd_txfm_lsx.h"
#define LSX_TRANSPOSE4x4_H(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \
do { \
__m128i _s0, _s1, _s2, _s3, _t0, _t1, _t2, _t3; \
\
DUP2_ARG2(__lsx_vilvl_h, _in2, _in0, _in3, _in1, _s0, _s1); \
DUP2_ARG2(__lsx_vilvh_h, _in2, _in0, _in3, _in1, _s2, _s3); \
_t0 = __lsx_vilvl_h(_s1, _s0); \
_t1 = __lsx_vilvh_h(_s1, _s0); \
_t2 = __lsx_vilvl_h(_s3, _s2); \
_t3 = __lsx_vilvh_h(_s3, _s2); \
DUP2_ARG2(__lsx_vpickev_d, _t2, _t0, _t3, _t1, _out0, _out2); \
DUP2_ARG2(__lsx_vpickod_d, _t2, _t0, _t3, _t1, _out1, _out3); \
} while (0)
#if !CONFIG_VP9_HIGHBITDEPTH
void fdct8x16_1d_column(const int16_t *input, int16_t *tmp_ptr,
int32_t src_stride) {
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
__m128i in0, in1, in2, in3, in4, in5, in6, in7;
__m128i in8, in9, in10, in11, in12, in13, in14, in15;
__m128i stp21, stp22, stp23, stp24, stp25, stp26, stp30;
__m128i stp31, stp32, stp33, stp34, stp35, stp36, stp37;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, cnst0, cnst1, cnst4, cnst5;
__m128i coeff = { 0x187e3b21d2bf2d41, 0x238e3537e782c4df };
__m128i coeff1 = { 0x289a317906463fb1, 0x12943d3f1e2b3871 };
__m128i coeff2 = { 0xed6cd766c78fc04f, 0x0 };
int32_t src_stride2 = src_stride << 1;
int32_t src_stride4 = src_stride2 << 1;
int32_t src_stride6 = src_stride4 + src_stride2;
int32_t src_stride8 = src_stride4 << 1;
int16_t *input_tmp = (int16_t *)input;
in0 = __lsx_vld(input_tmp, 0);
DUP4_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4,
input_tmp, src_stride6, input_tmp, src_stride8, in1, in2, in3, in4);
input_tmp += src_stride4;
DUP4_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4,
input_tmp, src_stride6, input_tmp, src_stride8, in5, in6, in7, in8);
input_tmp += src_stride4;
DUP4_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4,
input_tmp, src_stride6, input_tmp, src_stride8, in9, in10, in11,
in12);
input_tmp += src_stride4;
DUP2_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, in13,
in14);
input_tmp += src_stride2;
in15 = __lsx_vldx(input_tmp, src_stride2);
DUP4_ARG2(__lsx_vslli_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3);
DUP4_ARG2(__lsx_vslli_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7);
DUP4_ARG2(__lsx_vslli_h, in8, 2, in9, 2, in10, 2, in11, 2, in8, in9, in10,
in11);
DUP4_ARG2(__lsx_vslli_h, in12, 2, in13, 2, in14, 2, in15, 2, in12, in13, in14,
in15);
DUP4_ARG2(__lsx_vadd_h, in0, in15, in1, in14, in2, in13, in3, in12, tmp0,
tmp1, tmp2, tmp3);
DUP4_ARG2(__lsx_vadd_h, in4, in11, in5, in10, in6, in9, in7, in8, tmp4, tmp5,
tmp6, tmp7);
FDCT8x16_EVEN(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp0, tmp1,
tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
__lsx_vst(tmp0, tmp_ptr, 0);
__lsx_vst(tmp1, tmp_ptr, 64);
__lsx_vst(tmp2, tmp_ptr, 128);
__lsx_vst(tmp3, tmp_ptr, 192);
__lsx_vst(tmp4, tmp_ptr, 256);
__lsx_vst(tmp5, tmp_ptr, 320);
__lsx_vst(tmp6, tmp_ptr, 384);
__lsx_vst(tmp7, tmp_ptr, 448);
DUP4_ARG2(__lsx_vsub_h, in0, in15, in1, in14, in2, in13, in3, in12, in15,
in14, in13, in12);
DUP4_ARG2(__lsx_vsub_h, in4, in11, in5, in10, in6, in9, in7, in8, in11, in10,
in9, in8);
tmp_ptr += 16;
/* stp 1 */
DUP2_ARG2(__lsx_vilvh_h, in10, in13, in11, in12, vec2, vec4);
DUP2_ARG2(__lsx_vilvl_h, in10, in13, in11, in12, vec3, vec5);
cnst4 = __lsx_vreplvei_h(coeff, 0);
DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst4, stp25);
cnst5 = __lsx_vreplvei_h(coeff, 1);
cnst5 = __lsx_vpackev_h(cnst5, cnst4);
DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst5, stp22);
DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst4, stp24);
DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst5, stp23);
/* stp2 */
LSX_BUTTERFLY_4_H(in8, in9, stp22, stp23, stp30, stp31, stp32, stp33);
LSX_BUTTERFLY_4_H(in15, in14, stp25, stp24, stp37, stp36, stp35, stp34);
DUP2_ARG2(__lsx_vilvh_h, stp36, stp31, stp35, stp32, vec2, vec4);
DUP2_ARG2(__lsx_vilvl_h, stp36, stp31, stp35, stp32, vec3, vec5);
DUP2_ARG2(__lsx_vreplvei_h, coeff, 2, coeff, 3, cnst0, cnst1);
cnst0 = __lsx_vpackev_h(cnst0, cnst1);
DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst0, stp26);
cnst0 = __lsx_vreplvei_h(coeff, 4);
cnst1 = __lsx_vpackev_h(cnst1, cnst0);
DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst1, stp21);
LSX_BUTTERFLY_4_H(stp30, stp37, stp26, stp21, in8, in15, in14, in9);
vec1 = __lsx_vilvl_h(in15, in8);
vec0 = __lsx_vilvh_h(in15, in8);
DUP2_ARG2(__lsx_vreplvei_h, coeff1, 0, coeff1, 1, cnst0, cnst1);
cnst0 = __lsx_vpackev_h(cnst0, cnst1);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8);
__lsx_vst(in8, tmp_ptr, 0);
cnst0 = __lsx_vreplvei_h(coeff2, 0);
cnst0 = __lsx_vpackev_h(cnst1, cnst0);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8);
__lsx_vst(in8, tmp_ptr, 448);
vec1 = __lsx_vilvl_h(in14, in9);
vec0 = __lsx_vilvh_h(in14, in9);
DUP2_ARG2(__lsx_vreplvei_h, coeff1, 2, coeff1, 3, cnst0, cnst1);
cnst1 = __lsx_vpackev_h(cnst1, cnst0);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst1, in8);
__lsx_vst(in8, tmp_ptr, 256);
cnst1 = __lsx_vreplvei_h(coeff2, 2);
cnst0 = __lsx_vpackev_h(cnst0, cnst1);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8);
__lsx_vst(in8, tmp_ptr, 192);
DUP2_ARG2(__lsx_vreplvei_h, coeff, 2, coeff, 5, cnst0, cnst1);
cnst1 = __lsx_vpackev_h(cnst1, cnst0);
DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst1, stp25);
cnst1 = __lsx_vreplvei_h(coeff, 3);
cnst1 = __lsx_vpackev_h(cnst0, cnst1);
DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst1, stp22);
/* stp4 */
DUP2_ARG2(__lsx_vadd_h, stp34, stp25, stp33, stp22, in13, in10);
vec1 = __lsx_vilvl_h(in13, in10);
vec0 = __lsx_vilvh_h(in13, in10);
DUP2_ARG2(__lsx_vreplvei_h, coeff1, 4, coeff1, 5, cnst0, cnst1);
cnst0 = __lsx_vpackev_h(cnst0, cnst1);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8);
__lsx_vst(in8, tmp_ptr, 128);
cnst0 = __lsx_vreplvei_h(coeff2, 1);
cnst0 = __lsx_vpackev_h(cnst1, cnst0);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8);
__lsx_vst(in8, tmp_ptr, 320);
DUP2_ARG2(__lsx_vsub_h, stp34, stp25, stp33, stp22, in12, in11);
vec1 = __lsx_vilvl_h(in12, in11);
vec0 = __lsx_vilvh_h(in12, in11);
DUP2_ARG2(__lsx_vreplvei_h, coeff1, 6, coeff1, 7, cnst0, cnst1);
cnst1 = __lsx_vpackev_h(cnst1, cnst0);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst1, in8);
__lsx_vst(in8, tmp_ptr, 384);
cnst1 = __lsx_vreplvei_h(coeff2, 3);
cnst0 = __lsx_vpackev_h(cnst0, cnst1);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8);
__lsx_vst(in8, tmp_ptr, 64);
}
void fdct16x8_1d_row(int16_t *input, int16_t *output) {
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
__m128i in0, in1, in2, in3, in4, in5, in6, in7;
__m128i in8, in9, in10, in11, in12, in13, in14, in15;
int16_t *input_tmp = input;
DUP4_ARG2(__lsx_vld, input, 0, input, 32, input, 64, input, 96, in0, in1, in2,
in3);
DUP4_ARG2(__lsx_vld, input, 128, input, 160, input, 192, input, 224, in4, in5,
in6, in7);
DUP4_ARG2(__lsx_vld, input_tmp, 16, input_tmp, 48, input_tmp, 80, input_tmp,
112, in8, in9, in10, in11);
DUP4_ARG2(__lsx_vld, input_tmp, 144, input_tmp, 176, input_tmp, 208,
input_tmp, 240, in12, in13, in14, in15);
LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3,
in4, in5, in6, in7);
LSX_TRANSPOSE8x8_H(in8, in9, in10, in11, in12, in13, in14, in15, in8, in9,
in10, in11, in12, in13, in14, in15);
DUP4_ARG2(__lsx_vaddi_hu, in0, 1, in1, 1, in2, 1, in3, 1, in0, in1, in2, in3);
DUP4_ARG2(__lsx_vaddi_hu, in4, 1, in5, 1, in6, 1, in7, 1, in4, in5, in6, in7);
DUP4_ARG2(__lsx_vaddi_hu, in8, 1, in9, 1, in10, 1, in11, 1, in8, in9, in10,
in11);
DUP4_ARG2(__lsx_vaddi_hu, in12, 1, in13, 1, in14, 1, in15, 1, in12, in13,
in14, in15);
DUP4_ARG2(__lsx_vsrai_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3);
DUP4_ARG2(__lsx_vsrai_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7);
DUP4_ARG2(__lsx_vsrai_h, in8, 2, in9, 2, in10, 2, in11, 2, in8, in9, in10,
in11);
DUP4_ARG2(__lsx_vsrai_h, in12, 2, in13, 2, in14, 2, in15, 2, in12, in13, in14,
in15);
LSX_BUTTERFLY_16_H(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10,
in11, in12, in13, in14, in15, tmp0, tmp1, tmp2, tmp3, tmp4,
tmp5, tmp6, tmp7, in8, in9, in10, in11, in12, in13, in14,
in15);
__lsx_vst(in8, input, 0);
__lsx_vst(in9, input, 32);
__lsx_vst(in10, input, 64);
__lsx_vst(in11, input, 96);
__lsx_vst(in12, input, 128);
__lsx_vst(in13, input, 160);
__lsx_vst(in14, input, 192);
__lsx_vst(in15, input, 224);
FDCT8x16_EVEN(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp0, tmp1,
tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
DUP4_ARG2(__lsx_vld, input, 0, input, 32, input, 64, input, 96, in8, in9,
in10, in11);
DUP4_ARG2(__lsx_vld, input, 128, input, 160, input, 192, input, 224, in12,
in13, in14, in15);
FDCT8x16_ODD(in8, in9, in10, in11, in12, in13, in14, in15, in0, in1, in2, in3,
in4, in5, in6, in7);
LSX_TRANSPOSE8x8_H(tmp0, in0, tmp1, in1, tmp2, in2, tmp3, in3, tmp0, in0,
tmp1, in1, tmp2, in2, tmp3, in3);
__lsx_vst(tmp0, output, 0);
__lsx_vst(in0, output, 32);
__lsx_vst(tmp1, output, 64);
__lsx_vst(in1, output, 96);
__lsx_vst(tmp2, output, 128);
__lsx_vst(in2, output, 160);
__lsx_vst(tmp3, output, 192);
__lsx_vst(in3, output, 224);
LSX_TRANSPOSE8x8_H(tmp4, in4, tmp5, in5, tmp6, in6, tmp7, in7, tmp4, in4,
tmp5, in5, tmp6, in6, tmp7, in7);
__lsx_vst(tmp4, output, 16);
__lsx_vst(in4, output, 48);
__lsx_vst(tmp5, output, 80);
__lsx_vst(in5, output, 112);
__lsx_vst(tmp6, output, 144);
__lsx_vst(in6, output, 176);
__lsx_vst(tmp7, output, 208);
__lsx_vst(in7, output, 240);
}
void vpx_fdct4x4_lsx(const int16_t *input, int16_t *output,
int32_t src_stride) {
__m128i in0, in1, in2, in3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride4 = src_stride2 << 1;
int32_t src_stride6 = src_stride4 + src_stride2;
in0 = __lsx_vld(input, 0);
DUP2_ARG2(__lsx_vldx, input, src_stride2, input, src_stride4, in1, in2);
in3 = __lsx_vldx(input, src_stride6);
/* fdct4 pre-process */
{
__m128i vec, mask;
__m128i zero = __lsx_vldi(0);
mask = __lsx_vinsgr2vr_b(zero, 1, 0);
DUP4_ARG2(__lsx_vslli_h, in0, 4, in1, 4, in2, 4, in3, 4, in0, in1, in2,
in3);
vec = __lsx_vseqi_h(in0, 0);
vec = __lsx_vxori_b(vec, 255);
vec = __lsx_vand_v(mask, vec);
in0 = __lsx_vadd_h(in0, vec);
}
VP9_FDCT4(in0, in1, in2, in3, in0, in1, in2, in3);
LSX_TRANSPOSE4x4_H(in0, in1, in2, in3, in0, in1, in2, in3);
VP9_FDCT4(in0, in1, in2, in3, in0, in1, in2, in3);
LSX_TRANSPOSE4x4_H(in0, in1, in2, in3, in0, in1, in2, in3);
DUP4_ARG2(__lsx_vaddi_hu, in0, 1, in1, 1, in2, 1, in3, 1, in0, in1, in2, in3);
DUP4_ARG2(__lsx_vsrai_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3);
DUP2_ARG2(__lsx_vpickev_d, in1, in0, in3, in2, in0, in2);
__lsx_vst(in0, output, 0);
__lsx_vst(in2, output, 16);
}
void vpx_fdct8x8_lsx(const int16_t *input, int16_t *output,
int32_t src_stride) {
__m128i in0, in1, in2, in3, in4, in5, in6, in7;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride4 = src_stride2 << 1;
int32_t src_stride6 = src_stride4 + src_stride2;
int16_t *input_tmp = (int16_t *)input;
in0 = __lsx_vld(input_tmp, 0);
DUP2_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, in1,
in2);
in3 = __lsx_vldx(input_tmp, src_stride6);
input_tmp += src_stride4;
in4 = __lsx_vld(input_tmp, 0);
DUP2_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, in5,
in6);
in7 = __lsx_vldx(input_tmp, src_stride6);
DUP4_ARG2(__lsx_vslli_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3);
DUP4_ARG2(__lsx_vslli_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7);
VP9_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4,
in5, in6, in7);
LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3,
in4, in5, in6, in7);
VP9_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4,
in5, in6, in7);
LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3,
in4, in5, in6, in7);
SRLI_AVE_S_4V_H(in0, in1, in2, in3, in4, in5, in6, in7);
__lsx_vst(in0, output, 0);
__lsx_vst(in1, output, 16);
__lsx_vst(in2, output, 32);
__lsx_vst(in3, output, 48);
__lsx_vst(in4, output, 64);
__lsx_vst(in5, output, 80);
__lsx_vst(in6, output, 96);
__lsx_vst(in7, output, 112);
}
void vpx_fdct16x16_lsx(const int16_t *input, int16_t *output,
int32_t src_stride) {
int32_t i;
DECLARE_ALIGNED(32, int16_t, tmp_buf[16 * 16]);
/* column transform */
for (i = 0; i < 2; ++i) {
fdct8x16_1d_column((input + 8 * i), (&tmp_buf[0] + 8 * i), src_stride);
}
/* row transform */
for (i = 0; i < 2; ++i) {
fdct16x8_1d_row((&tmp_buf[0] + (128 * i)), (output + (128 * i)));
}
}
#endif // !CONFIG_VP9_HIGHBITDEPTH
libs/libvpx/vpx_dsp/loongarch/fwd_txfm_lsx.h
+381
View File
@@ -0,0 +1,381 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VPX_VPX_DSP_LOONGARCH_FWD_TXFM_LSX_H_
#define VPX_VPX_DSP_LOONGARCH_FWD_TXFM_LSX_H_
#include "vpx_dsp/loongarch/txfm_macros_lsx.h"
#include "vpx_dsp/txfm_common.h"
#define VP9_FDCT4(in0, in1, in2, in3, out0, out1, out2, out3) \
do { \
__m128i cnst0_m, cnst1_m, cnst2_m, cnst3_m; \
__m128i vec0_m, vec1_m, vec2_m, vec3_m; \
__m128i vec4_m, vec5_m, vec6_m, vec7_m; \
__m128i coeff_m = { 0x187e3b21d2bf2d41, 0x000000000000c4df }; \
\
LSX_BUTTERFLY_4_H(in0, in1, in2, in3, vec0_m, vec1_m, vec2_m, vec3_m); \
DUP2_ARG2(__lsx_vilvl_h, vec1_m, vec0_m, vec3_m, vec2_m, vec0_m, vec2_m); \
DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 0, coeff_m, 1, cnst0_m, cnst1_m); \
cnst1_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \
vec5_m = __lsx_vdp2_w_h(vec0_m, cnst1_m); \
DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 4, coeff_m, 3, cnst2_m, cnst3_m); \
cnst2_m = __lsx_vpackev_h(cnst3_m, cnst2_m); \
vec7_m = __lsx_vdp2_w_h(vec2_m, cnst2_m); \
\
vec4_m = __lsx_vdp2_w_h(vec0_m, cnst0_m); \
cnst2_m = __lsx_vreplvei_h(coeff_m, 2); \
cnst2_m = __lsx_vpackev_h(cnst2_m, cnst3_m); \
vec6_m = __lsx_vdp2_w_h(vec2_m, cnst2_m); \
\
DUP4_ARG3(__lsx_vssrarni_h_w, vec4_m, vec4_m, DCT_CONST_BITS, vec5_m, \
vec5_m, DCT_CONST_BITS, vec6_m, vec6_m, DCT_CONST_BITS, vec7_m, \
vec7_m, DCT_CONST_BITS, out0, out2, out1, out3); \
} while (0)
#define VP9_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, \
out3, out4, out5, out6, out7) \
do { \
__m128i s0_m, s1_m, s2_m, s3_m, s4_m, s5_m, s6_m; \
__m128i s7_m, x0_m, x1_m, x2_m, x3_m; \
__m128i coeff_m = { 0x187e3b21d2bf2d41, 0x238e35370c7c3ec5 }; \
\
/* FDCT stage1 */ \
LSX_BUTTERFLY_8_H(in0, in1, in2, in3, in4, in5, in6, in7, s0_m, s1_m, \
s2_m, s3_m, s4_m, s5_m, s6_m, s7_m); \
LSX_BUTTERFLY_4_H(s0_m, s1_m, s2_m, s3_m, x0_m, x1_m, x2_m, x3_m); \
DUP2_ARG2(__lsx_vilvh_h, x1_m, x0_m, x3_m, x2_m, s0_m, s2_m); \
DUP2_ARG2(__lsx_vilvl_h, x1_m, x0_m, x3_m, x2_m, s1_m, s3_m); \
DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 0, coeff_m, 1, x0_m, x1_m); \
x1_m = __lsx_vpackev_h(x1_m, x0_m); \
DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x1_m, out4); \
\
DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 2, coeff_m, 3, x2_m, x3_m); \
x2_m = __lsx_vneg_h(x2_m); \
x2_m = __lsx_vpackev_h(x3_m, x2_m); \
DOT_SHIFT_RIGHT_PCK_H(s2_m, s3_m, x2_m, out6); \
\
DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x0_m, out0); \
x2_m = __lsx_vreplvei_h(coeff_m, 2); \
x2_m = __lsx_vpackev_h(x2_m, x3_m); \
DOT_SHIFT_RIGHT_PCK_H(s2_m, s3_m, x2_m, out2); \
\
/* stage2 */ \
s1_m = __lsx_vilvl_h(s5_m, s6_m); \
s0_m = __lsx_vilvh_h(s5_m, s6_m); \
\
DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x0_m, s6_m); \
DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x1_m, s5_m); \
\
/* stage3 */ \
LSX_BUTTERFLY_4_H(s4_m, s7_m, s6_m, s5_m, x0_m, x3_m, x2_m, x1_m); \
\
/* stage4 */ \
DUP2_ARG2(__lsx_vilvh_h, x3_m, x0_m, x2_m, x1_m, s4_m, s6_m); \
DUP2_ARG2(__lsx_vilvl_h, x3_m, x0_m, x2_m, x1_m, s5_m, s7_m); \
\
DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 4, coeff_m, 5, x0_m, x1_m); \
x1_m = __lsx_vpackev_h(x0_m, x1_m); \
DOT_SHIFT_RIGHT_PCK_H(s4_m, s5_m, x1_m, out1); \
\
DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 6, coeff_m, 7, x2_m, x3_m); \
x2_m = __lsx_vpackev_h(x3_m, x2_m); \
DOT_SHIFT_RIGHT_PCK_H(s6_m, s7_m, x2_m, out5); \
\
x1_m = __lsx_vreplvei_h(coeff_m, 5); \
x0_m = __lsx_vneg_h(x0_m); \
x0_m = __lsx_vpackev_h(x1_m, x0_m); \
DOT_SHIFT_RIGHT_PCK_H(s4_m, s5_m, x0_m, out7); \
x2_m = __lsx_vreplvei_h(coeff_m, 6); \
x3_m = __lsx_vneg_h(x3_m); \
x2_m = __lsx_vpackev_h(x2_m, x3_m); \
DOT_SHIFT_RIGHT_PCK_H(s6_m, s7_m, x2_m, out3); \
} while (0)
#define SRLI_AVE_S_4V_H(in0, in1, in2, in3, in4, in5, in6, in7) \
do { \
__m128i vec0_m, vec1_m, vec2_m, vec3_m, vec4_m, vec5_m, vec6_m, vec7_m; \
\
DUP4_ARG2(__lsx_vsrli_h, in0, 15, in1, 15, in2, 15, in3, 15, vec0_m, \
vec1_m, vec2_m, vec3_m); \
DUP4_ARG2(__lsx_vsrli_h, in4, 15, in5, 15, in6, 15, in7, 15, vec4_m, \
vec5_m, vec6_m, vec7_m); \
DUP4_ARG2(__lsx_vavg_h, vec0_m, in0, vec1_m, in1, vec2_m, in2, vec3_m, \
in3, in0, in1, in2, in3); \
DUP4_ARG2(__lsx_vavg_h, vec4_m, in4, vec5_m, in5, vec6_m, in6, vec7_m, \
in7, in4, in5, in6, in7); \
} while (0)
#define FDCT32_POSTPROC_2V_POS_H(vec0, vec1) \
do { \
__m128i tp0_m, tp1_m; \
__m128i one = __lsx_vreplgr2vr_h(1); \
\
tp0_m = __lsx_vslei_h(vec0, 0); \
tp1_m = __lsx_vslei_h(vec1, 0); \
tp0_m = __lsx_vxori_b(tp0_m, 255); \
tp1_m = __lsx_vxori_b(tp1_m, 255); \
vec0 = __lsx_vadd_h(vec0, one); \
vec1 = __lsx_vadd_h(vec1, one); \
tp0_m = __lsx_vand_v(one, tp0_m); \
tp1_m = __lsx_vand_v(one, tp1_m); \
vec0 = __lsx_vadd_h(vec0, tp0_m); \
vec1 = __lsx_vadd_h(vec1, tp1_m); \
vec0 = __lsx_vsrai_h(vec0, 2); \
vec1 = __lsx_vsrai_h(vec1, 2); \
} while (0)
#define FDCT_POSTPROC_2V_NEG_H(vec0, vec1) \
do { \
__m128i tp0_m, tp1_m; \
__m128i one_m = __lsx_vldi(0x401); \
\
tp0_m = __lsx_vslti_h(vec0, 0); \
tp1_m = __lsx_vslti_h(vec1, 0); \
vec0 = __lsx_vadd_h(vec0, one_m); \
vec1 = __lsx_vadd_h(vec1, one_m); \
tp0_m = __lsx_vand_v(one_m, tp0_m); \
tp1_m = __lsx_vand_v(one_m, tp1_m); \
vec0 = __lsx_vadd_h(vec0, tp0_m); \
vec1 = __lsx_vadd_h(vec1, tp1_m); \
vec0 = __lsx_vsrai_h(vec0, 2); \
vec1 = __lsx_vsrai_h(vec1, 2); \
} while (0)
#define FDCT32_POSTPROC_NEG_W(vec) \
do { \
__m128i temp_m; \
__m128i one_m = __lsx_vreplgr2vr_w(1); \
\
temp_m = __lsx_vslti_w(vec, 0); \
vec = __lsx_vadd_w(vec, one_m); \
temp_m = __lsx_vand_v(one_m, temp_m); \
vec = __lsx_vadd_w(vec, temp_m); \
vec = __lsx_vsrai_w(vec, 2); \
} while (0)
#define DOTP_CONST_PAIR_W(reg0_left, reg1_left, reg0_right, reg1_right, \
const0, const1, out0, out1, out2, out3) \
do { \
__m128i s0_m, s1_m, s2_m, s3_m, s4_m, s5_m, s6_m, s7_m; \
__m128i tp0_m, tp1_m, tp2_m, tp3_m, _tmp0, _tmp1; \
__m128i k0_m = __lsx_vreplgr2vr_w((int32_t)const0); \
\
s0_m = __lsx_vreplgr2vr_w((int32_t)const1); \
k0_m = __lsx_vpackev_w(s0_m, k0_m); \
\
DUP2_ARG1(__lsx_vneg_w, reg1_left, reg1_right, _tmp0, _tmp1); \
s1_m = __lsx_vilvl_w(_tmp0, reg0_left); \
s0_m = __lsx_vilvh_w(_tmp0, reg0_left); \
s3_m = __lsx_vilvl_w(reg0_left, reg1_left); \
s2_m = __lsx_vilvh_w(reg0_left, reg1_left); \
s5_m = __lsx_vilvl_w(_tmp1, reg0_right); \
s4_m = __lsx_vilvh_w(_tmp1, reg0_right); \
s7_m = __lsx_vilvl_w(reg0_right, reg1_right); \
s6_m = __lsx_vilvh_w(reg0_right, reg1_right); \
DUP2_ARG2(__lsx_vdp2_d_w, s0_m, k0_m, s1_m, k0_m, tp0_m, tp1_m); \
DUP2_ARG2(__lsx_vdp2_d_w, s4_m, k0_m, s5_m, k0_m, tp2_m, tp3_m); \
DUP2_ARG3(__lsx_vssrarni_w_d, tp0_m, tp1_m, DCT_CONST_BITS, tp2_m, tp3_m, \
DCT_CONST_BITS, out0, out1); \
DUP2_ARG2(__lsx_vdp2_d_w, s2_m, k0_m, s3_m, k0_m, tp0_m, tp1_m); \
DUP2_ARG2(__lsx_vdp2_d_w, s6_m, k0_m, s7_m, k0_m, tp2_m, tp3_m); \
DUP2_ARG3(__lsx_vssrarni_w_d, tp0_m, tp1_m, DCT_CONST_BITS, tp2_m, tp3_m, \
DCT_CONST_BITS, out2, out3); \
} while (0)
#define VP9_ADDBLK_ST8x4_UB(dst, _stride, _stride2, _stride3, in0, in1, in2, \
in3) \
do { \
__m128i dst0_m, dst1_m, dst2_m, dst3_m; \
__m128i tmp0_m, tmp1_m; \
__m128i res0_m, res1_m, res2_m, res3_m; \
\
dst0_m = __lsx_vld(dst, 0); \
DUP2_ARG2(__lsx_vldx, dst, _stride, dst, _stride2, dst1_m, dst2_m); \
dst3_m = __lsx_vldx(dst, _stride3); \
DUP4_ARG2(__lsx_vsllwil_hu_bu, dst0_m, 0, dst1_m, 0, dst2_m, 0, dst3_m, 0, \
res0_m, res1_m, res2_m, res3_m); \
DUP4_ARG2(__lsx_vadd_h, res0_m, in0, res1_m, in1, res2_m, in2, res3_m, \
in3, res0_m, res1_m, res2_m, res3_m); \
DUP2_ARG3(__lsx_vssrarni_bu_h, res1_m, res0_m, 0, res3_m, res2_m, 0, \
tmp0_m, tmp1_m); \
__lsx_vstelm_d(tmp0_m, dst, 0, 0); \
__lsx_vstelm_d(tmp0_m, dst + _stride, 0, 1); \
__lsx_vstelm_d(tmp1_m, dst + _stride2, 0, 0); \
__lsx_vstelm_d(tmp1_m, dst + _stride3, 0, 1); \
} while (0)
#define FDCT8x16_EVEN(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \
out2, out3, out4, out5, out6, out7) \
do { \
__m128i s0_m, s1_m, s2_m, s3_m, s4_m, s5_m, s6_m, s7_m; \
__m128i x0_m, x1_m, x2_m, x3_m; \
__m128i coeff_m = { 0x187e3b21d2bf2d41, 0x238e35370c7c3ec5 }; \
\
/* FDCT stage1 */ \
LSX_BUTTERFLY_8_H(in0, in1, in2, in3, in4, in5, in6, in7, s0_m, s1_m, \
s2_m, s3_m, s4_m, s5_m, s6_m, s7_m); \
LSX_BUTTERFLY_4_H(s0_m, s1_m, s2_m, s3_m, x0_m, x1_m, x2_m, x3_m); \
DUP2_ARG2(__lsx_vilvh_h, x1_m, x0_m, x3_m, x2_m, s0_m, s2_m); \
DUP2_ARG2(__lsx_vilvl_h, x1_m, x0_m, x3_m, x2_m, s1_m, s3_m); \
DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 0, coeff_m, 1, x0_m, x1_m); \
x1_m = __lsx_vpackev_h(x1_m, x0_m); \
DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x1_m, out4); \
\
DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 2, coeff_m, 3, x2_m, x3_m); \
x2_m = __lsx_vneg_h(x2_m); \
x2_m = __lsx_vpackev_h(x3_m, x2_m); \
DOT_SHIFT_RIGHT_PCK_H(s2_m, s3_m, x2_m, out6); \
\
DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x0_m, out0); \
x2_m = __lsx_vreplvei_h(coeff_m, 2); \
x2_m = __lsx_vpackev_h(x2_m, x3_m); \
DOT_SHIFT_RIGHT_PCK_H(s2_m, s3_m, x2_m, out2); \
\
/* stage2 */ \
s1_m = __lsx_vilvl_h(s5_m, s6_m); \
s0_m = __lsx_vilvh_h(s5_m, s6_m); \
\
DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x0_m, s6_m); \
DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x1_m, s5_m); \
\
/* stage3 */ \
LSX_BUTTERFLY_4_H(s4_m, s7_m, s6_m, s5_m, x0_m, x3_m, x2_m, x1_m); \
\
/* stage4 */ \
DUP2_ARG2(__lsx_vilvh_h, x3_m, x0_m, x2_m, x1_m, s4_m, s6_m); \
DUP2_ARG2(__lsx_vilvl_h, x3_m, x0_m, x2_m, x1_m, s5_m, s7_m); \
\
DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 4, coeff_m, 5, x0_m, x1_m); \
x1_m = __lsx_vpackev_h(x0_m, x1_m); \
DOT_SHIFT_RIGHT_PCK_H(s4_m, s5_m, x1_m, out1); \
\
DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 6, coeff_m, 7, x2_m, x3_m); \
x2_m = __lsx_vpackev_h(x3_m, x2_m); \
DOT_SHIFT_RIGHT_PCK_H(s6_m, s7_m, x2_m, out5); \
\
x1_m = __lsx_vreplvei_h(coeff_m, 5); \
x0_m = __lsx_vneg_h(x0_m); \
x0_m = __lsx_vpackev_h(x1_m, x0_m); \
DOT_SHIFT_RIGHT_PCK_H(s4_m, s5_m, x0_m, out7); \
\
x2_m = __lsx_vreplvei_h(coeff_m, 6); \
x3_m = __lsx_vneg_h(x3_m); \
x2_m = __lsx_vpackev_h(x2_m, x3_m); \
DOT_SHIFT_RIGHT_PCK_H(s6_m, s7_m, x2_m, out3); \
} while (0)
#define FDCT8x16_ODD(input0, input1, input2, input3, input4, input5, input6, \
input7, out1, out3, out5, out7, out9, out11, out13, \
out15) \
do { \
__m128i stp21_m, stp22_m, stp23_m, stp24_m, stp25_m, stp26_m; \
__m128i stp30_m, stp31_m, stp32_m, stp33_m, stp34_m, stp35_m; \
__m128i stp36_m, stp37_m, vec0_m, vec1_m; \
__m128i vec2_m, vec3_m, vec4_m, vec5_m, vec6_m; \
__m128i cnst0_m, cnst1_m, cnst4_m, cnst5_m; \
__m128i coeff_m = { 0x187e3b21d2bf2d41, 0x238e3537e782c4df }; \
__m128i coeff1_m = { 0x289a317906463fb1, 0x12943d3f1e2b3871 }; \
__m128i coeff2_m = { 0xed6cd766c78fc04f, 0x0 }; \
\
/* stp 1 */ \
DUP2_ARG2(__lsx_vilvh_h, input2, input5, input3, input4, vec2_m, vec4_m); \
DUP2_ARG2(__lsx_vilvl_h, input2, input5, input3, input4, vec3_m, vec5_m); \
\
cnst4_m = __lsx_vreplvei_h(coeff_m, 0); \
DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst4_m, stp25_m); \
\
cnst5_m = __lsx_vreplvei_h(coeff_m, 1); \
cnst5_m = __lsx_vpackev_h(cnst5_m, cnst4_m); \
DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst5_m, stp22_m); \
DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst4_m, stp24_m); \
DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst5_m, stp23_m); \
\
/* stp2 */ \
LSX_BUTTERFLY_4_H(input0, input1, stp22_m, stp23_m, stp30_m, stp31_m, \
stp32_m, stp33_m); \
LSX_BUTTERFLY_4_H(input7, input6, stp25_m, stp24_m, stp37_m, stp36_m, \
stp35_m, stp34_m); \
\
DUP2_ARG2(__lsx_vilvh_h, stp36_m, stp31_m, stp35_m, stp32_m, vec2_m, \
vec4_m); \
DUP2_ARG2(__lsx_vilvl_h, stp36_m, stp31_m, stp35_m, stp32_m, vec3_m, \
vec5_m); \
\
DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 2, coeff_m, 3, cnst0_m, cnst1_m); \
cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \
DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst0_m, stp26_m); \
\
cnst0_m = __lsx_vreplvei_h(coeff_m, 4); \
cnst1_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \
DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst1_m, stp21_m); \
\
DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 5, coeff_m, 2, cnst0_m, cnst1_m); \
cnst1_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \
DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst1_m, stp25_m); \
\
cnst0_m = __lsx_vreplvei_h(coeff_m, 3); \
cnst1_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \
DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst1_m, stp22_m); \
\
/* stp4 */ \
LSX_BUTTERFLY_4_H(stp30_m, stp37_m, stp26_m, stp21_m, vec6_m, vec2_m, \
vec4_m, vec5_m); \
LSX_BUTTERFLY_4_H(stp33_m, stp34_m, stp25_m, stp22_m, stp21_m, stp23_m, \
stp24_m, stp31_m); \
\
vec1_m = __lsx_vilvl_h(vec2_m, vec6_m); \
vec0_m = __lsx_vilvh_h(vec2_m, vec6_m); \
DUP2_ARG2(__lsx_vreplvei_h, coeff1_m, 0, coeff1_m, 1, cnst0_m, cnst1_m); \
cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \
\
DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out1); \
\
cnst0_m = __lsx_vreplvei_h(coeff2_m, 0); \
cnst0_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \
DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out15); \
\
vec1_m = __lsx_vilvl_h(vec4_m, vec5_m); \
vec0_m = __lsx_vilvh_h(vec4_m, vec5_m); \
DUP2_ARG2(__lsx_vreplvei_h, coeff1_m, 2, coeff1_m, 3, cnst0_m, cnst1_m); \
cnst1_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \
\
DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst1_m, out9); \
\
cnst1_m = __lsx_vreplvei_h(coeff2_m, 2); \
cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \
DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out7); \
\
vec1_m = __lsx_vilvl_h(stp23_m, stp21_m); \
vec0_m = __lsx_vilvh_h(stp23_m, stp21_m); \
DUP2_ARG2(__lsx_vreplvei_h, coeff1_m, 4, coeff1_m, 5, cnst0_m, cnst1_m); \
cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \
DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out5); \
\
cnst0_m = __lsx_vreplvei_h(coeff2_m, 1); \
cnst0_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \
DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out11); \
\
vec1_m = __lsx_vilvl_h(stp24_m, stp31_m); \
vec0_m = __lsx_vilvh_h(stp24_m, stp31_m); \
DUP2_ARG2(__lsx_vreplvei_h, coeff1_m, 6, coeff1_m, 7, cnst0_m, cnst1_m); \
cnst1_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \
\
DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst1_m, out13); \
\
cnst1_m = __lsx_vreplvei_h(coeff2_m, 3); \
cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \
DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out3); \
} while (0)
void fdct8x16_1d_column(const int16_t *input, int16_t *tmp_ptr,
int32_t src_stride);
void fdct16x8_1d_row(int16_t *input, int16_t *output);
#endif // VPX_VPX_DSP_LOONGARCH_FWD_TXFM_LSX_H_
libs/libvpx/vpx_dsp/loongarch/idct32x32_lsx.c
+834
View File
@@ -0,0 +1,834 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/loongarch/fwd_txfm_lsx.h"
#define UNPCK_UB_SH(_in, _out0, _out1) \
do { \
_out0 = __lsx_vsllwil_hu_bu(_in, 0); \
_out1 = __lsx_vexth_hu_bu(_in); \
} while (0)
static void idct32x8_row_transpose_store(const int16_t *input,
int16_t *tmp_buf) {
__m128i m0, m1, m2, m3, m4, m5, m6, m7;
__m128i n0, n1, n2, n3, n4, n5, n6, n7;
/* 1st & 2nd 8x8 */
DUP4_ARG2(__lsx_vld, input, 0, input, 64, input, 128, input, 192, m0, n0, m1,
n1);
DUP4_ARG2(__lsx_vld, input, 256, input, 320, input, 384, input, 448, m2, n2,
m3, n3);
DUP4_ARG2(__lsx_vld, input, 16, input, 80, input, 144, input, 208, m4, n4, m5,
n5);
DUP4_ARG2(__lsx_vld, input, 272, input, 336, input, 400, input, 464, m6, n6,
m7, n7);
LSX_TRANSPOSE8x8_H(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3,
n3);
LSX_TRANSPOSE8x8_H(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7,
n7);
__lsx_vst(m0, tmp_buf, 0);
__lsx_vst(n0, tmp_buf, 16);
__lsx_vst(m1, tmp_buf, 32);
__lsx_vst(n1, tmp_buf, 48);
__lsx_vst(m2, tmp_buf, 64);
__lsx_vst(n2, tmp_buf, 80);
__lsx_vst(m3, tmp_buf, 96);
__lsx_vst(n3, tmp_buf, 112);
__lsx_vst(m4, tmp_buf, 128);
__lsx_vst(n4, tmp_buf, 144);
__lsx_vst(m5, tmp_buf, 160);
__lsx_vst(n5, tmp_buf, 176);
__lsx_vst(m6, tmp_buf, 192);
__lsx_vst(n6, tmp_buf, 208);
__lsx_vst(m7, tmp_buf, 224);
__lsx_vst(n7, tmp_buf, 240);
/* 3rd & 4th 8x8 */
DUP4_ARG2(__lsx_vld, input, 32, input, 96, input, 160, input, 224, m0, n0, m1,
n1);
DUP4_ARG2(__lsx_vld, input, 288, input, 352, input, 416, input, 480, m2, n2,
m3, n3);
DUP4_ARG2(__lsx_vld, input, 48, input, 112, input, 176, input, 240, m4, n4,
m5, n5);
DUP4_ARG2(__lsx_vld, input, 304, input, 368, input, 432, input, 496, m6, n6,
m7, n7);
LSX_TRANSPOSE8x8_H(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3,
n3);
LSX_TRANSPOSE8x8_H(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7,
n7);
__lsx_vst(m0, tmp_buf, 256);
__lsx_vst(n0, tmp_buf, 272);
__lsx_vst(m1, tmp_buf, 288);
__lsx_vst(n1, tmp_buf, 304);
__lsx_vst(m2, tmp_buf, 320);
__lsx_vst(n2, tmp_buf, 336);
__lsx_vst(m3, tmp_buf, 352);
__lsx_vst(n3, tmp_buf, 368);
__lsx_vst(m4, tmp_buf, 384);
__lsx_vst(n4, tmp_buf, 400);
__lsx_vst(m5, tmp_buf, 416);
__lsx_vst(n5, tmp_buf, 432);
__lsx_vst(m6, tmp_buf, 448);
__lsx_vst(n6, tmp_buf, 464);
__lsx_vst(m7, tmp_buf, 480);
__lsx_vst(n7, tmp_buf, 496);
}
static void idct32x8_row_even_process_store(int16_t *tmp_buf,
int16_t *tmp_eve_buf) {
__m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3;
__m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
__m128i stp0, stp1, stp2, stp3, stp4, stp5, stp6, stp7;
__m128i tmp0;
/* Even stage 1 */
DUP4_ARG2(__lsx_vld, tmp_buf, 0, tmp_buf, 64, tmp_buf, 128, tmp_buf, 192,
reg0, reg1, reg2, reg3);
DUP4_ARG2(__lsx_vld, tmp_buf, 256, tmp_buf, 320, tmp_buf, 384, tmp_buf, 448,
reg4, reg5, reg6, reg7);
DOTP_CONST_PAIR(reg1, reg7, cospi_28_64, cospi_4_64, reg1, reg7);
DOTP_CONST_PAIR(reg5, reg3, cospi_12_64, cospi_20_64, reg5, reg3);
LSX_BUTTERFLY_4_H(reg1, reg7, reg3, reg5, vec1, vec3, vec2, vec0);
DOTP_CONST_PAIR(vec2, vec0, cospi_16_64, cospi_16_64, loc2, loc3);
loc1 = vec3;
loc0 = vec1;
DOTP_CONST_PAIR(reg0, reg4, cospi_16_64, cospi_16_64, reg0, reg4);
DOTP_CONST_PAIR(reg2, reg6, cospi_24_64, cospi_8_64, reg2, reg6);
LSX_BUTTERFLY_4_H(reg4, reg0, reg2, reg6, vec1, vec3, vec2, vec0);
LSX_BUTTERFLY_4_H(vec0, vec1, loc1, loc0, stp3, stp0, stp7, stp4);
LSX_BUTTERFLY_4_H(vec2, vec3, loc3, loc2, stp2, stp1, stp6, stp5);
/* Even stage 2 */
DUP4_ARG2(__lsx_vld, tmp_buf, 32, tmp_buf, 96, tmp_buf, 160, tmp_buf, 224,
reg0, reg1, reg2, reg3);
DUP4_ARG2(__lsx_vld, tmp_buf, 288, tmp_buf, 352, tmp_buf, 416, tmp_buf, 480,
reg4, reg5, reg6, reg7);
DOTP_CONST_PAIR(reg0, reg7, cospi_30_64, cospi_2_64, reg0, reg7);
DOTP_CONST_PAIR(reg4, reg3, cospi_14_64, cospi_18_64, reg4, reg3);
DOTP_CONST_PAIR(reg2, reg5, cospi_22_64, cospi_10_64, reg2, reg5);
DOTP_CONST_PAIR(reg6, reg1, cospi_6_64, cospi_26_64, reg6, reg1);
vec0 = __lsx_vadd_h(reg0, reg4);
reg0 = __lsx_vsub_h(reg0, reg4);
reg4 = __lsx_vadd_h(reg6, reg2);
reg6 = __lsx_vsub_h(reg6, reg2);
reg2 = __lsx_vadd_h(reg1, reg5);
reg1 = __lsx_vsub_h(reg1, reg5);
reg5 = __lsx_vadd_h(reg7, reg3);
reg7 = __lsx_vsub_h(reg7, reg3);
reg3 = vec0;
vec1 = reg2;
reg2 = __lsx_vadd_h(reg3, reg4);
reg3 = __lsx_vsub_h(reg3, reg4);
reg4 = __lsx_vsub_h(reg5, vec1);
reg5 = __lsx_vadd_h(reg5, vec1);
tmp0 = __lsx_vneg_h(reg6);
DOTP_CONST_PAIR(reg7, reg0, cospi_24_64, cospi_8_64, reg0, reg7);
DOTP_CONST_PAIR(tmp0, reg1, cospi_24_64, cospi_8_64, reg6, reg1);
vec0 = __lsx_vsub_h(reg0, reg6);
reg0 = __lsx_vadd_h(reg0, reg6);
vec1 = __lsx_vsub_h(reg7, reg1);
reg7 = __lsx_vadd_h(reg7, reg1);
DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, reg6, reg1);
DOTP_CONST_PAIR(reg4, reg3, cospi_16_64, cospi_16_64, reg3, reg4);
/* Even stage 3 : Dependency on Even stage 1 & Even stage 2 */
LSX_BUTTERFLY_4_H(stp0, stp1, reg7, reg5, loc1, loc3, loc2, loc0);
__lsx_vst(loc0, tmp_eve_buf, 240);
__lsx_vst(loc1, tmp_eve_buf, 0);
__lsx_vst(loc2, tmp_eve_buf, 224);
__lsx_vst(loc3, tmp_eve_buf, 16);
LSX_BUTTERFLY_4_H(stp2, stp3, reg4, reg1, loc1, loc3, loc2, loc0);
__lsx_vst(loc0, tmp_eve_buf, 208);
__lsx_vst(loc1, tmp_eve_buf, 32);
__lsx_vst(loc2, tmp_eve_buf, 192);
__lsx_vst(loc3, tmp_eve_buf, 48);
/* Store 8 */
LSX_BUTTERFLY_4_H(stp4, stp5, reg6, reg3, loc1, loc3, loc2, loc0);
__lsx_vst(loc0, tmp_eve_buf, 176);
__lsx_vst(loc1, tmp_eve_buf, 64);
__lsx_vst(loc2, tmp_eve_buf, 160);
__lsx_vst(loc3, tmp_eve_buf, 80);
LSX_BUTTERFLY_4_H(stp6, stp7, reg2, reg0, loc1, loc3, loc2, loc0);
__lsx_vst(loc0, tmp_eve_buf, 144);
__lsx_vst(loc1, tmp_eve_buf, 96);
__lsx_vst(loc2, tmp_eve_buf, 128);
__lsx_vst(loc3, tmp_eve_buf, 112);
}
static void idct32x8_row_odd_process_store(int16_t *tmp_buf,
int16_t *tmp_odd_buf) {
__m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3;
__m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
/* Odd stage 1 */
DUP4_ARG2(__lsx_vld, tmp_buf, 16, tmp_buf, 112, tmp_buf, 144, tmp_buf, 240,
reg0, reg1, reg2, reg3);
DUP4_ARG2(__lsx_vld, tmp_buf, 272, tmp_buf, 368, tmp_buf, 400, tmp_buf, 496,
reg4, reg5, reg6, reg7);
DOTP_CONST_PAIR(reg0, reg7, cospi_31_64, cospi_1_64, reg0, reg7);
DOTP_CONST_PAIR(reg4, reg3, cospi_15_64, cospi_17_64, reg3, reg4);
DOTP_CONST_PAIR(reg2, reg5, cospi_23_64, cospi_9_64, reg2, reg5);
DOTP_CONST_PAIR(reg6, reg1, cospi_7_64, cospi_25_64, reg1, reg6);
vec0 = __lsx_vadd_h(reg0, reg3);
reg0 = __lsx_vsub_h(reg0, reg3);
reg3 = __lsx_vadd_h(reg7, reg4);
reg7 = __lsx_vsub_h(reg7, reg4);
reg4 = __lsx_vadd_h(reg1, reg2);
reg1 = __lsx_vsub_h(reg1, reg2);
reg2 = __lsx_vadd_h(reg6, reg5);
reg6 = __lsx_vsub_h(reg6, reg5);
reg5 = vec0;
/* 4 Stores */
DUP2_ARG2(__lsx_vadd_h, reg5, reg4, reg3, reg2, vec0, vec1);
__lsx_vst(vec0, tmp_odd_buf, 64);
__lsx_vst(vec1, tmp_odd_buf, 80);
DUP2_ARG2(__lsx_vsub_h, reg5, reg4, reg3, reg2, vec0, vec1);
DOTP_CONST_PAIR(vec1, vec0, cospi_24_64, cospi_8_64, vec0, vec1);
__lsx_vst(vec0, tmp_odd_buf, 0);
__lsx_vst(vec1, tmp_odd_buf, 16);
/* 4 Stores */
DOTP_CONST_PAIR(reg7, reg0, cospi_28_64, cospi_4_64, reg0, reg7);
DOTP_CONST_PAIR(reg6, reg1, -cospi_4_64, cospi_28_64, reg1, reg6);
LSX_BUTTERFLY_4_H(reg0, reg7, reg6, reg1, vec0, vec1, vec2, vec3);
__lsx_vst(vec0, tmp_odd_buf, 96);
__lsx_vst(vec1, tmp_odd_buf, 112);
DOTP_CONST_PAIR(vec2, vec3, cospi_24_64, cospi_8_64, vec2, vec3);
__lsx_vst(vec2, tmp_odd_buf, 32);
__lsx_vst(vec3, tmp_odd_buf, 48);
/* Odd stage 2 */
/* 8 loads */
DUP4_ARG2(__lsx_vld, tmp_buf, 48, tmp_buf, 80, tmp_buf, 176, tmp_buf, 208,
reg0, reg1, reg2, reg3);
DUP4_ARG2(__lsx_vld, tmp_buf, 304, tmp_buf, 336, tmp_buf, 432, tmp_buf, 464,
reg4, reg5, reg6, reg7);
DOTP_CONST_PAIR(reg1, reg6, cospi_27_64, cospi_5_64, reg1, reg6);
DOTP_CONST_PAIR(reg5, reg2, cospi_11_64, cospi_21_64, reg2, reg5);
DOTP_CONST_PAIR(reg3, reg4, cospi_19_64, cospi_13_64, reg3, reg4);
DOTP_CONST_PAIR(reg7, reg0, cospi_3_64, cospi_29_64, reg0, reg7);
/* 4 Stores */
DUP4_ARG2(__lsx_vsub_h, reg1, reg2, reg6, reg5, reg0, reg3, reg7, reg4, vec0,
vec1, vec2, vec3);
DOTP_CONST_PAIR(vec1, vec0, cospi_12_64, cospi_20_64, loc0, loc1);
DOTP_CONST_PAIR(vec3, vec2, -cospi_20_64, cospi_12_64, loc2, loc3);
LSX_BUTTERFLY_4_H(loc3, loc2, loc0, loc1, vec1, vec0, vec2, vec3);
__lsx_vst(vec0, tmp_odd_buf, 192);
__lsx_vst(vec1, tmp_odd_buf, 240);
DOTP_CONST_PAIR(vec3, vec2, -cospi_8_64, cospi_24_64, vec0, vec1);
__lsx_vst(vec0, tmp_odd_buf, 160);
__lsx_vst(vec1, tmp_odd_buf, 176);
/* 4 Stores */
DUP4_ARG2(__lsx_vadd_h, reg1, reg2, reg6, reg5, reg0, reg3, reg7, reg4, vec1,
vec2, vec0, vec3);
LSX_BUTTERFLY_4_H(vec0, vec3, vec2, vec1, reg0, reg1, reg3, reg2);
__lsx_vst(reg0, tmp_odd_buf, 208);
__lsx_vst(reg1, tmp_odd_buf, 224);
DOTP_CONST_PAIR(reg3, reg2, -cospi_8_64, cospi_24_64, reg0, reg1);
__lsx_vst(reg0, tmp_odd_buf, 128);
__lsx_vst(reg1, tmp_odd_buf, 144);
/* Odd stage 3 : Dependency on Odd stage 1 & Odd stage 2 */
/* Load 8 & Store 8 */
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 0, tmp_odd_buf, 16, tmp_odd_buf, 32,
tmp_odd_buf, 48, reg0, reg1, reg2, reg3);
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 128, tmp_odd_buf, 144, tmp_odd_buf, 160,
tmp_odd_buf, 176, reg4, reg5, reg6, reg7);
DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0,
loc1, loc2, loc3);
__lsx_vst(loc0, tmp_odd_buf, 0);
__lsx_vst(loc1, tmp_odd_buf, 16);
__lsx_vst(loc2, tmp_odd_buf, 32);
__lsx_vst(loc3, tmp_odd_buf, 48);
DUP2_ARG2(__lsx_vsub_h, reg0, reg4, reg1, reg5, vec0, vec1);
DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1);
DUP2_ARG2(__lsx_vsub_h, reg2, reg6, reg3, reg7, vec0, vec1);
DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3);
__lsx_vst(loc0, tmp_odd_buf, 128);
__lsx_vst(loc1, tmp_odd_buf, 144);
__lsx_vst(loc2, tmp_odd_buf, 160);
__lsx_vst(loc3, tmp_odd_buf, 176);
/* Load 8 & Store 8 */
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 64, tmp_odd_buf, 80, tmp_odd_buf, 96,
tmp_odd_buf, 112, reg1, reg2, reg0, reg3);
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 192, tmp_odd_buf, 208, tmp_odd_buf, 224,
tmp_odd_buf, 240, reg4, reg5, reg6, reg7);
DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0,
loc1, loc2, loc3);
__lsx_vst(loc0, tmp_odd_buf, 64);
__lsx_vst(loc1, tmp_odd_buf, 80);
__lsx_vst(loc2, tmp_odd_buf, 96);
__lsx_vst(loc3, tmp_odd_buf, 112);
DUP2_ARG2(__lsx_vsub_h, reg0, reg4, reg3, reg7, vec0, vec1);
DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1);
DUP2_ARG2(__lsx_vsub_h, reg1, reg5, reg2, reg6, vec0, vec1);
DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3);
__lsx_vst(loc0, tmp_odd_buf, 192);
__lsx_vst(loc1, tmp_odd_buf, 208);
__lsx_vst(loc2, tmp_odd_buf, 224);
__lsx_vst(loc3, tmp_odd_buf, 240);
}
static void idct_butterfly_transpose_store(int16_t *tmp_buf,
int16_t *tmp_eve_buf,
int16_t *tmp_odd_buf, int16_t *dst) {
__m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3;
__m128i m0, m1, m2, m3, m4, m5, m6, m7;
__m128i n0, n1, n2, n3, n4, n5, n6, n7;
__m128i reg0, reg1, reg2, reg3;
/* FINAL BUTTERFLY : Dependency on Even & Odd */
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 0, tmp_odd_buf, 144, tmp_odd_buf, 224,
tmp_odd_buf, 96, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vld, tmp_eve_buf, 0, tmp_eve_buf, 128, tmp_eve_buf, 64,
tmp_eve_buf, 192, loc0, loc1, loc2, loc3);
DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m0,
m4, m2, m6);
DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, reg0,
reg1, reg2, reg3);
__lsx_vst(reg0, tmp_buf, 496);
__lsx_vst(reg1, tmp_buf, 368);
__lsx_vst(reg2, tmp_buf, 432);
__lsx_vst(reg3, tmp_buf, 304);
/* Load 8 & Store 8 */
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 64, tmp_odd_buf, 208, tmp_odd_buf, 160,
tmp_odd_buf, 48, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vld, tmp_eve_buf, 32, tmp_eve_buf, 160, tmp_eve_buf, 96,
tmp_eve_buf, 224, loc0, loc1, loc2, loc3);
DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m1,
m5, m3, m7);
DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, reg0,
reg1, reg2, reg3);
__lsx_vst(reg0, tmp_buf, 464);
__lsx_vst(reg1, tmp_buf, 336);
__lsx_vst(reg2, tmp_buf, 400);
__lsx_vst(reg3, tmp_buf, 272);
/* Load 8 & Store 8 */
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 32, tmp_odd_buf, 176, tmp_odd_buf, 192,
tmp_odd_buf, 112, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vld, tmp_eve_buf, 16, tmp_eve_buf, 144, tmp_eve_buf, 80,
tmp_eve_buf, 208, loc0, loc1, loc2, loc3);
DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n0,
n4, n2, n6);
DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, reg0,
reg1, reg2, reg3);
__lsx_vst(reg0, tmp_buf, 480);
__lsx_vst(reg1, tmp_buf, 352);
__lsx_vst(reg2, tmp_buf, 416);
__lsx_vst(reg3, tmp_buf, 288);
/* Load 8 & Store 8 */
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 80, tmp_odd_buf, 240, tmp_odd_buf, 128,
tmp_odd_buf, 16, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vld, tmp_eve_buf, 48, tmp_eve_buf, 176, tmp_eve_buf, 112,
tmp_eve_buf, 240, loc0, loc1, loc2, loc3);
DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n1,
n5, n3, n7);
DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, reg0,
reg1, reg2, reg3);
__lsx_vst(reg0, tmp_buf, 448);
__lsx_vst(reg1, tmp_buf, 320);
__lsx_vst(reg2, tmp_buf, 384);
__lsx_vst(reg3, tmp_buf, 256);
/* Transpose : 16 vectors */
/* 1st & 2nd 8x8 */
LSX_TRANSPOSE8x8_H(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3,
n3);
__lsx_vst(m0, dst, 0);
__lsx_vst(n0, dst, 64);
__lsx_vst(m1, dst, 128);
__lsx_vst(n1, dst, 192);
__lsx_vst(m2, dst, 256);
__lsx_vst(n2, dst, 320);
__lsx_vst(m3, dst, 384);
__lsx_vst(n3, dst, 448);
LSX_TRANSPOSE8x8_H(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7,
n7);
__lsx_vst(m4, dst, 16);
__lsx_vst(n4, dst, 80);
__lsx_vst(m5, dst, 144);
__lsx_vst(n5, dst, 208);
__lsx_vst(m6, dst, 272);
__lsx_vst(n6, dst, 336);
__lsx_vst(m7, dst, 400);
__lsx_vst(n7, dst, 464);
/* 3rd & 4th 8x8 */
DUP4_ARG2(__lsx_vld, tmp_buf, 256, tmp_buf, 272, tmp_buf, 288, tmp_buf, 304,
m0, n0, m1, n1);
DUP4_ARG2(__lsx_vld, tmp_buf, 320, tmp_buf, 336, tmp_buf, 352, tmp_buf, 368,
m2, n2, m3, n3);
DUP4_ARG2(__lsx_vld, tmp_buf, 384, tmp_buf, 400, tmp_buf, 416, tmp_buf, 432,
m4, n4, m5, n5);
DUP4_ARG2(__lsx_vld, tmp_buf, 448, tmp_buf, 464, tmp_buf, 480, tmp_buf, 496,
m6, n6, m7, n7);
LSX_TRANSPOSE8x8_H(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3,
n3);
LSX_TRANSPOSE8x8_H(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7,
n7);
__lsx_vst(m0, dst, 32);
__lsx_vst(n0, dst, 96);
__lsx_vst(m1, dst, 160);
__lsx_vst(n1, dst, 224);
__lsx_vst(m2, dst, 288);
__lsx_vst(n2, dst, 352);
__lsx_vst(m3, dst, 416);
__lsx_vst(n3, dst, 480);
__lsx_vst(m4, dst, 48);
__lsx_vst(n4, dst, 112);
__lsx_vst(m5, dst, 176);
__lsx_vst(n5, dst, 240);
__lsx_vst(m6, dst, 304);
__lsx_vst(n6, dst, 368);
__lsx_vst(m7, dst, 432);
__lsx_vst(n7, dst, 496);
}
static void idct32x8_1d_rows_lsx(const int16_t *input, int16_t *output) {
DECLARE_ALIGNED(32, int16_t, tmp_buf[8 * 32]);
DECLARE_ALIGNED(32, int16_t, tmp_odd_buf[16 * 8]);
DECLARE_ALIGNED(32, int16_t, tmp_eve_buf[16 * 8]);
idct32x8_row_transpose_store(input, &tmp_buf[0]);
idct32x8_row_even_process_store(&tmp_buf[0], &tmp_eve_buf[0]);
idct32x8_row_odd_process_store(&tmp_buf[0], &tmp_odd_buf[0]);
idct_butterfly_transpose_store(&tmp_buf[0], &tmp_eve_buf[0], &tmp_odd_buf[0],
output);
}
static void idct8x32_column_even_process_store(int16_t *tmp_buf,
int16_t *tmp_eve_buf) {
__m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3;
__m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
__m128i stp0, stp1, stp2, stp3, stp4, stp5, stp6, stp7;
__m128i tmp0;
/* Even stage 1 */
DUP4_ARG2(__lsx_vld, tmp_buf, 0, tmp_buf, 256, tmp_buf, 512, tmp_buf, 768,
reg0, reg1, reg2, reg3);
DUP4_ARG2(__lsx_vld, tmp_buf, 1024, tmp_buf, 1280, tmp_buf, 1536, tmp_buf,
1792, reg4, reg5, reg6, reg7);
tmp_buf += 64;
DOTP_CONST_PAIR(reg1, reg7, cospi_28_64, cospi_4_64, reg1, reg7);
DOTP_CONST_PAIR(reg5, reg3, cospi_12_64, cospi_20_64, reg5, reg3);
LSX_BUTTERFLY_4_H(reg1, reg7, reg3, reg5, vec1, vec3, vec2, vec0);
DOTP_CONST_PAIR(vec2, vec0, cospi_16_64, cospi_16_64, loc2, loc3);
loc1 = vec3;
loc0 = vec1;
DOTP_CONST_PAIR(reg0, reg4, cospi_16_64, cospi_16_64, reg0, reg4);
DOTP_CONST_PAIR(reg2, reg6, cospi_24_64, cospi_8_64, reg2, reg6);
LSX_BUTTERFLY_4_H(reg4, reg0, reg2, reg6, vec1, vec3, vec2, vec0);
LSX_BUTTERFLY_4_H(vec0, vec1, loc1, loc0, stp3, stp0, stp7, stp4);
LSX_BUTTERFLY_4_H(vec2, vec3, loc3, loc2, stp2, stp1, stp6, stp5);
/* Even stage 2 */
/* Load 8 */
DUP4_ARG2(__lsx_vld, tmp_buf, 0, tmp_buf, 256, tmp_buf, 512, tmp_buf, 768,
reg0, reg1, reg2, reg3);
DUP4_ARG2(__lsx_vld, tmp_buf, 1024, tmp_buf, 1280, tmp_buf, 1536, tmp_buf,
1792, reg4, reg5, reg6, reg7);
DOTP_CONST_PAIR(reg0, reg7, cospi_30_64, cospi_2_64, reg0, reg7);
DOTP_CONST_PAIR(reg4, reg3, cospi_14_64, cospi_18_64, reg4, reg3);
DOTP_CONST_PAIR(reg2, reg5, cospi_22_64, cospi_10_64, reg2, reg5);
DOTP_CONST_PAIR(reg6, reg1, cospi_6_64, cospi_26_64, reg6, reg1);
vec0 = __lsx_vadd_h(reg0, reg4);
reg0 = __lsx_vsub_h(reg0, reg4);
reg4 = __lsx_vadd_h(reg6, reg2);
reg6 = __lsx_vsub_h(reg6, reg2);
reg2 = __lsx_vadd_h(reg1, reg5);
reg1 = __lsx_vsub_h(reg1, reg5);
reg5 = __lsx_vadd_h(reg7, reg3);
reg7 = __lsx_vsub_h(reg7, reg3);
reg3 = vec0;
vec1 = reg2;
reg2 = __lsx_vadd_h(reg3, reg4);
reg3 = __lsx_vsub_h(reg3, reg4);
reg4 = __lsx_vsub_h(reg5, vec1);
reg5 = __lsx_vadd_h(reg5, vec1);
tmp0 = __lsx_vneg_h(reg6);
DOTP_CONST_PAIR(reg7, reg0, cospi_24_64, cospi_8_64, reg0, reg7);
DOTP_CONST_PAIR(tmp0, reg1, cospi_24_64, cospi_8_64, reg6, reg1);
vec0 = __lsx_vsub_h(reg0, reg6);
reg0 = __lsx_vadd_h(reg0, reg6);
vec1 = __lsx_vsub_h(reg7, reg1);
reg7 = __lsx_vadd_h(reg7, reg1);
DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, reg6, reg1);
DOTP_CONST_PAIR(reg4, reg3, cospi_16_64, cospi_16_64, reg3, reg4);
/* Even stage 3 : Dependency on Even stage 1 & Even stage 2 */
/* Store 8 */
LSX_BUTTERFLY_4_H(stp0, stp1, reg7, reg5, loc1, loc3, loc2, loc0);
__lsx_vst(loc1, tmp_eve_buf, 0);
__lsx_vst(loc3, tmp_eve_buf, 16);
__lsx_vst(loc2, tmp_eve_buf, 224);
__lsx_vst(loc0, tmp_eve_buf, 240);
LSX_BUTTERFLY_4_H(stp2, stp3, reg4, reg1, loc1, loc3, loc2, loc0);
__lsx_vst(loc1, tmp_eve_buf, 32);
__lsx_vst(loc3, tmp_eve_buf, 48);
__lsx_vst(loc2, tmp_eve_buf, 192);
__lsx_vst(loc0, tmp_eve_buf, 208);
/* Store 8 */
LSX_BUTTERFLY_4_H(stp4, stp5, reg6, reg3, loc1, loc3, loc2, loc0);
__lsx_vst(loc1, tmp_eve_buf, 64);
__lsx_vst(loc3, tmp_eve_buf, 80);
__lsx_vst(loc2, tmp_eve_buf, 160);
__lsx_vst(loc0, tmp_eve_buf, 176);
LSX_BUTTERFLY_4_H(stp6, stp7, reg2, reg0, loc1, loc3, loc2, loc0);
__lsx_vst(loc1, tmp_eve_buf, 96);
__lsx_vst(loc3, tmp_eve_buf, 112);
__lsx_vst(loc2, tmp_eve_buf, 128);
__lsx_vst(loc0, tmp_eve_buf, 144);
}
static void idct8x32_column_odd_process_store(int16_t *tmp_buf,
int16_t *tmp_odd_buf) {
__m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3;
__m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
/* Odd stage 1 */
DUP4_ARG2(__lsx_vld, tmp_buf, 64, tmp_buf, 448, tmp_buf, 576, tmp_buf, 960,
reg0, reg1, reg2, reg3);
DUP4_ARG2(__lsx_vld, tmp_buf, 1088, tmp_buf, 1472, tmp_buf, 1600, tmp_buf,
1984, reg4, reg5, reg6, reg7);
DOTP_CONST_PAIR(reg0, reg7, cospi_31_64, cospi_1_64, reg0, reg7);
DOTP_CONST_PAIR(reg4, reg3, cospi_15_64, cospi_17_64, reg3, reg4);
DOTP_CONST_PAIR(reg2, reg5, cospi_23_64, cospi_9_64, reg2, reg5);
DOTP_CONST_PAIR(reg6, reg1, cospi_7_64, cospi_25_64, reg1, reg6);
vec0 = __lsx_vadd_h(reg0, reg3);
reg0 = __lsx_vsub_h(reg0, reg3);
reg3 = __lsx_vadd_h(reg7, reg4);
reg7 = __lsx_vsub_h(reg7, reg4);
reg4 = __lsx_vadd_h(reg1, reg2);
reg1 = __lsx_vsub_h(reg1, reg2);
reg2 = __lsx_vadd_h(reg6, reg5);
reg6 = __lsx_vsub_h(reg6, reg5);
reg5 = vec0;
/* 4 Stores */
DUP2_ARG2(__lsx_vadd_h, reg5, reg4, reg3, reg2, vec0, vec1);
__lsx_vst(vec0, tmp_odd_buf, 64);
__lsx_vst(vec1, tmp_odd_buf, 80);
DUP2_ARG2(__lsx_vsub_h, reg5, reg4, reg3, reg2, vec0, vec1);
DOTP_CONST_PAIR(vec1, vec0, cospi_24_64, cospi_8_64, vec0, vec1);
__lsx_vst(vec0, tmp_odd_buf, 0);
__lsx_vst(vec1, tmp_odd_buf, 16);
/* 4 Stores */
DOTP_CONST_PAIR(reg7, reg0, cospi_28_64, cospi_4_64, reg0, reg7);
DOTP_CONST_PAIR(reg6, reg1, -cospi_4_64, cospi_28_64, reg1, reg6);
LSX_BUTTERFLY_4_H(reg0, reg7, reg6, reg1, vec0, vec1, vec2, vec3);
DOTP_CONST_PAIR(vec2, vec3, cospi_24_64, cospi_8_64, vec2, vec3);
__lsx_vst(vec0, tmp_odd_buf, 96);
__lsx_vst(vec1, tmp_odd_buf, 112);
__lsx_vst(vec2, tmp_odd_buf, 32);
__lsx_vst(vec3, tmp_odd_buf, 48);
/* Odd stage 2 */
/* 8 loads */
DUP4_ARG2(__lsx_vld, tmp_buf, 192, tmp_buf, 320, tmp_buf, 704, tmp_buf, 832,
reg0, reg1, reg2, reg3);
DUP4_ARG2(__lsx_vld, tmp_buf, 1216, tmp_buf, 1344, tmp_buf, 1728, tmp_buf,
1856, reg4, reg5, reg6, reg7);
DOTP_CONST_PAIR(reg1, reg6, cospi_27_64, cospi_5_64, reg1, reg6);
DOTP_CONST_PAIR(reg5, reg2, cospi_11_64, cospi_21_64, reg2, reg5);
DOTP_CONST_PAIR(reg3, reg4, cospi_19_64, cospi_13_64, reg3, reg4);
DOTP_CONST_PAIR(reg7, reg0, cospi_3_64, cospi_29_64, reg0, reg7);
/* 4 Stores */
DUP4_ARG2(__lsx_vsub_h, reg1, reg2, reg6, reg5, reg0, reg3, reg7, reg4, vec0,
vec1, vec2, vec3);
DOTP_CONST_PAIR(vec1, vec0, cospi_12_64, cospi_20_64, loc0, loc1);
DOTP_CONST_PAIR(vec3, vec2, -cospi_20_64, cospi_12_64, loc2, loc3);
LSX_BUTTERFLY_4_H(loc2, loc3, loc1, loc0, vec0, vec1, vec3, vec2);
__lsx_vst(vec0, tmp_odd_buf, 192);
__lsx_vst(vec1, tmp_odd_buf, 240);
DOTP_CONST_PAIR(vec3, vec2, -cospi_8_64, cospi_24_64, vec0, vec1);
__lsx_vst(vec0, tmp_odd_buf, 160);
__lsx_vst(vec1, tmp_odd_buf, 176);
/* 4 Stores */
DUP4_ARG2(__lsx_vadd_h, reg0, reg3, reg1, reg2, reg5, reg6, reg4, reg7, vec0,
vec1, vec2, vec3);
LSX_BUTTERFLY_4_H(vec0, vec3, vec2, vec1, reg0, reg1, reg3, reg2);
__lsx_vst(reg0, tmp_odd_buf, 208);
__lsx_vst(reg1, tmp_odd_buf, 224);
DOTP_CONST_PAIR(reg3, reg2, -cospi_8_64, cospi_24_64, reg0, reg1);
__lsx_vst(reg0, tmp_odd_buf, 128);
__lsx_vst(reg1, tmp_odd_buf, 144);
/* Odd stage 3 : Dependency on Odd stage 1 & Odd stage 2 */
/* Load 8 & Store 8 */
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 0, tmp_odd_buf, 16, tmp_odd_buf, 32,
tmp_odd_buf, 48, reg0, reg1, reg2, reg3);
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 128, tmp_odd_buf, 144, tmp_odd_buf, 160,
tmp_odd_buf, 176, reg4, reg5, reg6, reg7);
DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0,
loc1, loc2, loc3);
__lsx_vst(loc0, tmp_odd_buf, 0);
__lsx_vst(loc1, tmp_odd_buf, 16);
__lsx_vst(loc2, tmp_odd_buf, 32);
__lsx_vst(loc3, tmp_odd_buf, 48);
DUP2_ARG2(__lsx_vsub_h, reg0, reg4, reg1, reg5, vec0, vec1);
DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1);
DUP2_ARG2(__lsx_vsub_h, reg2, reg6, reg3, reg7, vec0, vec1);
DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3);
__lsx_vst(loc0, tmp_odd_buf, 128);
__lsx_vst(loc1, tmp_odd_buf, 144);
__lsx_vst(loc2, tmp_odd_buf, 160);
__lsx_vst(loc3, tmp_odd_buf, 176);
/* Load 8 & Store 8 */
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 64, tmp_odd_buf, 80, tmp_odd_buf, 96,
tmp_odd_buf, 112, reg1, reg2, reg0, reg3);
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 192, tmp_odd_buf, 208, tmp_odd_buf, 224,
tmp_odd_buf, 240, reg4, reg5, reg6, reg7);
DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0,
loc1, loc2, loc3);
__lsx_vst(loc0, tmp_odd_buf, 64);
__lsx_vst(loc1, tmp_odd_buf, 80);
__lsx_vst(loc2, tmp_odd_buf, 96);
__lsx_vst(loc3, tmp_odd_buf, 112);
DUP2_ARG2(__lsx_vsub_h, reg0, reg4, reg3, reg7, vec0, vec1);
DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1);
DUP2_ARG2(__lsx_vsub_h, reg1, reg5, reg2, reg6, vec0, vec1);
DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3);
__lsx_vst(loc0, tmp_odd_buf, 192);
__lsx_vst(loc1, tmp_odd_buf, 208);
__lsx_vst(loc2, tmp_odd_buf, 224);
__lsx_vst(loc3, tmp_odd_buf, 240);
}
static void idct8x32_column_butterfly_addblk(int16_t *tmp_eve_buf,
int16_t *tmp_odd_buf, uint8_t *dst,
int32_t dst_stride) {
__m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3;
__m128i m0, m1, m2, m3, m4, m5, m6, m7;
__m128i n0, n1, n2, n3, n4, n5, n6, n7;
int32_t stride = dst_stride << 2;
int32_t stride2 = stride << 1;
int32_t stride3 = stride + stride2;
/* FINAL BUTTERFLY : Dependency on Even & Odd */
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 0, tmp_odd_buf, 144, tmp_odd_buf, 224,
tmp_odd_buf, 96, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vld, tmp_eve_buf, 0, tmp_eve_buf, 128, tmp_eve_buf, 64,
tmp_eve_buf, 192, loc0, loc1, loc2, loc3);
DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m0,
m4, m2, m6);
DUP4_ARG2(__lsx_vsrari_h, m0, 6, m2, 6, m4, 6, m6, 6, m0, m2, m4, m6);
VP9_ADDBLK_ST8x4_UB(dst, stride, stride2, stride3, m0, m2, m4, m6);
DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m6,
m2, m4, m0);
DUP4_ARG2(__lsx_vsrari_h, m0, 6, m2, 6, m4, 6, m6, 6, m0, m2, m4, m6);
VP9_ADDBLK_ST8x4_UB((dst + 19 * dst_stride), stride, stride2, stride3, m0, m2,
m4, m6);
/* Load 8 & Store 8 */
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 64, tmp_odd_buf, 208, tmp_odd_buf, 160,
tmp_odd_buf, 48, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vld, tmp_eve_buf, 32, tmp_eve_buf, 160, tmp_eve_buf, 96,
tmp_eve_buf, 224, loc0, loc1, loc2, loc3);
DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m1,
m5, m3, m7);
DUP4_ARG2(__lsx_vsrari_h, m1, 6, m3, 6, m5, 6, m7, 6, m1, m3, m5, m7);
VP9_ADDBLK_ST8x4_UB((dst + 2 * dst_stride), stride, stride2, stride3, m1, m3,
m5, m7);
DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m7,
m3, m5, m1);
DUP4_ARG2(__lsx_vsrari_h, m1, 6, m3, 6, m5, 6, m7, 6, m1, m3, m5, m7);
VP9_ADDBLK_ST8x4_UB((dst + 17 * dst_stride), stride, stride2, stride3, m1, m3,
m5, m7);
/* Load 8 & Store 8 */
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 32, tmp_odd_buf, 176, tmp_odd_buf, 192,
tmp_odd_buf, 112, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vld, tmp_eve_buf, 16, tmp_eve_buf, 144, tmp_eve_buf, 80,
tmp_eve_buf, 208, loc0, loc1, loc2, loc3);
DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n0,
n4, n2, n6);
DUP4_ARG2(__lsx_vsrari_h, n0, 6, n2, 6, n4, 6, n6, 6, n0, n2, n4, n6);
VP9_ADDBLK_ST8x4_UB((dst + dst_stride), stride, stride2, stride3, n0, n2, n4,
n6);
DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n6,
n2, n4, n0);
DUP4_ARG2(__lsx_vsrari_h, n0, 6, n2, 6, n4, 6, n6, 6, n0, n2, n4, n6);
VP9_ADDBLK_ST8x4_UB((dst + 18 * dst_stride), stride, stride2, stride3, n0, n2,
n4, n6);
/* Load 8 & Store 8 */
DUP4_ARG2(__lsx_vld, tmp_odd_buf, 80, tmp_odd_buf, 240, tmp_odd_buf, 128,
tmp_odd_buf, 16, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vld, tmp_eve_buf, 48, tmp_eve_buf, 176, tmp_eve_buf, 112,
tmp_eve_buf, 240, loc0, loc1, loc2, loc3);
DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n1,
n5, n3, n7);
DUP4_ARG2(__lsx_vsrari_h, n1, 6, n3, 6, n5, 6, n7, 6, n1, n3, n5, n7);
VP9_ADDBLK_ST8x4_UB((dst + 3 * dst_stride), stride, stride2, stride3, n1, n3,
n5, n7);
DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n7,
n3, n5, n1);
DUP4_ARG2(__lsx_vsrari_h, n1, 6, n3, 6, n5, 6, n7, 6, n1, n3, n5, n7);
VP9_ADDBLK_ST8x4_UB((dst + 16 * dst_stride), stride, stride2, stride3, n1, n3,
n5, n7);
}
static void idct8x32_1d_columns_addblk_lsx(int16_t *input, uint8_t *dst,
int32_t dst_stride) {
DECLARE_ALIGNED(32, int16_t, tmp_odd_buf[16 * 8]);
DECLARE_ALIGNED(32, int16_t, tmp_eve_buf[16 * 8]);
idct8x32_column_even_process_store(input, &tmp_eve_buf[0]);
idct8x32_column_odd_process_store(input, &tmp_odd_buf[0]);
idct8x32_column_butterfly_addblk(&tmp_eve_buf[0], &tmp_odd_buf[0], dst,
dst_stride);
}
void vpx_idct32x32_1024_add_lsx(const int16_t *input, uint8_t *dst,
int32_t dst_stride) {
int32_t i;
DECLARE_ALIGNED(32, int16_t, out_arr[32 * 32]);
int16_t *out_ptr = out_arr;
/* transform rows */
for (i = 0; i < 4; ++i) {
/* process 32 * 8 block */
idct32x8_1d_rows_lsx((input + (i << 8)), (out_ptr + (i << 8)));
}
for (i = 0; i < 4; ++i) {
/* process 8 * 32 block */
idct8x32_1d_columns_addblk_lsx((out_ptr + (i << 3)), (dst + (i << 3)),
dst_stride);
}
}
void vpx_idct32x32_34_add_lsx(const int16_t *input, uint8_t *dst,
int32_t dst_stride) {
int32_t i;
DECLARE_ALIGNED(32, int16_t, out_arr[32 * 32]);
int16_t *out_ptr = out_arr;
__m128i zero = __lsx_vldi(0);
for (i = 32; i--;) {
__lsx_vst(zero, out_ptr, 0);
__lsx_vst(zero, out_ptr, 16);
__lsx_vst(zero, out_ptr, 32);
__lsx_vst(zero, out_ptr, 48);
out_ptr += 32;
}
out_ptr = out_arr;
/* rows: only upper-left 8x8 has non-zero coeff */
idct32x8_1d_rows_lsx(input, out_ptr);
/* transform columns */
for (i = 0; i < 4; ++i) {
/* process 8 * 32 block */
idct8x32_1d_columns_addblk_lsx((out_ptr + (i << 3)), (dst + (i << 3)),
dst_stride);
}
}
void vpx_idct32x32_1_add_lsx(const int16_t *input, uint8_t *dst,
int32_t dst_stride) {
int32_t i;
int16_t out;
__m128i dst0, dst1, dst2, dst3, tmp0, tmp1, tmp2, tmp3;
__m128i res0, res1, res2, res3, res4, res5, res6, res7, vec;
out = ROUND_POWER_OF_TWO((input[0] * cospi_16_64), DCT_CONST_BITS);
out = ROUND_POWER_OF_TWO((out * cospi_16_64), DCT_CONST_BITS);
out = ROUND_POWER_OF_TWO(out, 6);
vec = __lsx_vreplgr2vr_h(out);
for (i = 16; i--;) {
DUP2_ARG2(__lsx_vld, dst, 0, dst, 16, dst0, dst1);
dst2 = __lsx_vldx(dst, dst_stride);
dst3 = __lsx_vldx(dst + 16, dst_stride);
UNPCK_UB_SH(dst0, res0, res4);
UNPCK_UB_SH(dst1, res1, res5);
UNPCK_UB_SH(dst2, res2, res6);
UNPCK_UB_SH(dst3, res3, res7);
DUP4_ARG2(__lsx_vadd_h, res0, vec, res1, vec, res2, vec, res3, vec, res0,
res1, res2, res3);
DUP4_ARG2(__lsx_vadd_h, res4, vec, res5, vec, res6, vec, res7, vec, res4,
res5, res6, res7);
DUP4_ARG3(__lsx_vssrarni_bu_h, res4, res0, 0, res5, res1, 0, res6, res2, 0,
res7, res3, 0, tmp0, tmp1, tmp2, tmp3);
__lsx_vst(tmp0, dst, 0);
__lsx_vst(tmp1, dst, 16);
dst += dst_stride;
__lsx_vst(tmp2, dst, 0);
__lsx_vst(tmp3, dst, 16);
dst += dst_stride;
}
}
libs/libvpx/vpx_dsp/loongarch/intrapred_lsx.c
+98
View File
@@ -0,0 +1,98 @@
/*
* Copyright (c) 2021 Loongson Technology Corporation Limited
* Contributed by Lu Wang <wanglu@loongson.cn>
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "./vpx_dsp_rtcd.h"
#include "vpx_util/loongson_intrinsics.h"
static inline void intra_predict_dc_8x8_lsx(const uint8_t *src_top,
const uint8_t *src_left,
uint8_t *dst, int32_t dst_stride) {
uint64_t val0, val1;
int32_t dst_stride_x2 = dst_stride << 1;
int32_t dst_stride_x3 = dst_stride_x2 + dst_stride;
int32_t dst_stride_x4 = dst_stride << 2;
__m128i store, sum_h, sum_w, sum_d;
__m128i src = { 0 };
val0 = *(const uint64_t *)src_top;
val1 = *(const uint64_t *)src_left;
DUP2_ARG3(__lsx_vinsgr2vr_d, src, val0, 0, src, val1, 1, src, src);
sum_h = __lsx_vhaddw_hu_bu(src, src);
sum_w = __lsx_vhaddw_wu_hu(sum_h, sum_h);
sum_d = __lsx_vhaddw_du_wu(sum_w, sum_w);
sum_w = __lsx_vpickev_w(sum_d, sum_d);
sum_d = __lsx_vhaddw_du_wu(sum_w, sum_w);
sum_w = __lsx_vsrari_w(sum_d, 4);
store = __lsx_vreplvei_b(sum_w, 0);
__lsx_vstelm_d(store, dst, 0, 0);
__lsx_vstelm_d(store, dst + dst_stride, 0, 0);
__lsx_vstelm_d(store, dst + dst_stride_x2, 0, 0);
__lsx_vstelm_d(store, dst + dst_stride_x3, 0, 0);
dst += dst_stride_x4;
__lsx_vstelm_d(store, dst, 0, 0);
__lsx_vstelm_d(store, dst + dst_stride, 0, 0);
__lsx_vstelm_d(store, dst + dst_stride_x2, 0, 0);
__lsx_vstelm_d(store, dst + dst_stride_x3, 0, 0);
}
static inline void intra_predict_dc_16x16_lsx(const uint8_t *src_top,
const uint8_t *src_left,
uint8_t *dst,
int32_t dst_stride) {
int32_t dst_stride_x2 = dst_stride << 1;
int32_t dst_stride_x3 = dst_stride_x2 + dst_stride;
int32_t dst_stride_x4 = dst_stride << 2;
__m128i top, left, out;
__m128i sum_h, sum_top, sum_left;
__m128i sum_w;
__m128i sum_d;
DUP2_ARG2(__lsx_vld, src_top, 0, src_left, 0, top, left);
DUP2_ARG2(__lsx_vhaddw_hu_bu, top, top, left, left, sum_top, sum_left);
sum_h = __lsx_vadd_h(sum_top, sum_left);
sum_w = __lsx_vhaddw_wu_hu(sum_h, sum_h);
sum_d = __lsx_vhaddw_du_wu(sum_w, sum_w);
sum_w = __lsx_vpickev_w(sum_d, sum_d);
sum_d = __lsx_vhaddw_du_wu(sum_w, sum_w);
sum_w = __lsx_vsrari_w(sum_d, 5);
out = __lsx_vreplvei_b(sum_w, 0);
__lsx_vstx(out, dst, 0);
__lsx_vstx(out, dst, dst_stride);
__lsx_vstx(out, dst, dst_stride_x2);
__lsx_vstx(out, dst, dst_stride_x3);
dst += dst_stride_x4;
__lsx_vstx(out, dst, 0);
__lsx_vstx(out, dst, dst_stride);
__lsx_vstx(out, dst, dst_stride_x2);
__lsx_vstx(out, dst, dst_stride_x3);
dst += dst_stride_x4;
__lsx_vstx(out, dst, 0);
__lsx_vstx(out, dst, dst_stride);
__lsx_vstx(out, dst, dst_stride_x2);
__lsx_vstx(out, dst, dst_stride_x3);
dst += dst_stride_x4;
__lsx_vstx(out, dst, 0);
__lsx_vstx(out, dst, dst_stride);
__lsx_vstx(out, dst, dst_stride_x2);
__lsx_vstx(out, dst, dst_stride_x3);
}
void vpx_dc_predictor_8x8_lsx(uint8_t *dst, ptrdiff_t y_stride,
const uint8_t *above, const uint8_t *left) {
intra_predict_dc_8x8_lsx(above, left, dst, y_stride);
}
void vpx_dc_predictor_16x16_lsx(uint8_t *dst, ptrdiff_t y_stride,
const uint8_t *above, const uint8_t *left) {
intra_predict_dc_16x16_lsx(above, left, dst, y_stride);
}
libs/libvpx/vpx_dsp/loongarch/loopfilter_16_lsx.c
+1320
View File
File diff suppressed because it is too large. Load diff
libs/libvpx/vpx_dsp/loongarch/loopfilter_4_lsx.c
+214
View File
@@ -0,0 +1,214 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/loongarch/loopfilter_lsx.h"
void vpx_lpf_horizontal_4_lsx(uint8_t *src, int32_t pitch,
const uint8_t *b_limit_ptr,
const uint8_t *limit_ptr,
const uint8_t *thresh_ptr) {
__m128i mask, hev, flat, thresh, b_limit, limit;
__m128i p3, p2, p1, p0, q3, q2, q1, q0, p1_out, p0_out, q0_out, q1_out;
int32_t pitch2 = pitch << 1;
int32_t pitch3 = pitch2 + pitch;
int32_t pitch4 = pitch2 << 1;
DUP4_ARG2(__lsx_vldx, src, -pitch4, src, -pitch3, src, -pitch2, src, -pitch,
p3, p2, p1, p0);
q0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, pitch, src, pitch2, q1, q2);
q3 = __lsx_vldx(src, pitch3);
thresh = __lsx_vldrepl_b(thresh_ptr, 0);
b_limit = __lsx_vldrepl_b(b_limit_ptr, 0);
limit = __lsx_vldrepl_b(limit_ptr, 0);
LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev,
mask, flat);
VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out);
__lsx_vstelm_d(p1_out, src - pitch2, 0, 0);
__lsx_vstelm_d(p0_out, src - pitch, 0, 0);
__lsx_vstelm_d(q0_out, src, 0, 0);
__lsx_vstelm_d(q1_out, src + pitch, 0, 0);
}
void vpx_lpf_horizontal_4_dual_lsx(uint8_t *src, int32_t pitch,
const uint8_t *b_limit0_ptr,
const uint8_t *limit0_ptr,
const uint8_t *thresh0_ptr,
const uint8_t *b_limit1_ptr,
const uint8_t *limit1_ptr,
const uint8_t *thresh1_ptr) {
__m128i mask, hev, flat, thresh0, b_limit0, limit0, thresh1, b_limit1, limit1;
__m128i p3, p2, p1, p0, q3, q2, q1, q0;
int32_t pitch2 = pitch << 1;
int32_t pitch3 = pitch2 + pitch;
int32_t pitch4 = pitch2 << 1;
DUP4_ARG2(__lsx_vldx, src, -pitch4, src, -pitch3, src, -pitch2, src, -pitch,
p3, p2, p1, p0);
q0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, pitch, src, pitch2, q1, q2);
q3 = __lsx_vldx(src, pitch3);
thresh0 = __lsx_vldrepl_b(thresh0_ptr, 0);
thresh1 = __lsx_vldrepl_b(thresh1_ptr, 0);
thresh0 = __lsx_vilvl_d(thresh1, thresh0);
b_limit0 = __lsx_vldrepl_b(b_limit0_ptr, 0);
b_limit1 = __lsx_vldrepl_b(b_limit1_ptr, 0);
b_limit0 = __lsx_vilvl_d(b_limit1, b_limit0);
limit0 = __lsx_vldrepl_b(limit0_ptr, 0);
limit1 = __lsx_vldrepl_b(limit1_ptr, 0);
limit0 = __lsx_vilvl_d(limit1, limit0);
LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, hev,
mask, flat);
VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1);
__lsx_vstx(p1, src, -pitch2);
__lsx_vstx(p0, src, -pitch);
__lsx_vst(q0, src, 0);
__lsx_vstx(q1, src, pitch);
}
void vpx_lpf_vertical_4_lsx(uint8_t *src, int32_t pitch,
const uint8_t *b_limit_ptr,
const uint8_t *limit_ptr,
const uint8_t *thresh_ptr) {
__m128i mask, hev, flat, limit, thresh, b_limit;
__m128i p3, p2, p1, p0, q3, q2, q1, q0;
__m128i vec0, vec1, vec2, vec3;
int32_t pitch2 = pitch << 1;
int32_t pitch3 = pitch2 + pitch;
int32_t pitch4 = pitch2 << 1;
uint8_t *src_tmp = src - 4;
p3 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, p2, p1);
p0 = __lsx_vldx(src_tmp, pitch3);
src_tmp += pitch4;
q0 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, q1, q2);
q3 = __lsx_vldx(src_tmp, pitch3);
thresh = __lsx_vldrepl_b(thresh_ptr, 0);
b_limit = __lsx_vldrepl_b(b_limit_ptr, 0);
limit = __lsx_vldrepl_b(limit_ptr, 0);
LSX_TRANSPOSE8x8_B(p3, p2, p1, p0, q0, q1, q2, q3, p3, p2, p1, p0, q0, q1, q2,
q3);
LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev,
mask, flat);
VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1);
DUP2_ARG2(__lsx_vilvl_b, p0, p1, q1, q0, vec0, vec1);
vec2 = __lsx_vilvl_h(vec1, vec0);
vec3 = __lsx_vilvh_h(vec1, vec0);
src -= 2;
__lsx_vstelm_w(vec2, src, 0, 0);
src += pitch;
__lsx_vstelm_w(vec2, src, 0, 1);
src += pitch;
__lsx_vstelm_w(vec2, src, 0, 2);
src += pitch;
__lsx_vstelm_w(vec2, src, 0, 3);
src += pitch;
__lsx_vstelm_w(vec3, src, 0, 0);
__lsx_vstelm_w(vec3, src + pitch, 0, 1);
__lsx_vstelm_w(vec3, src + pitch2, 0, 2);
__lsx_vstelm_w(vec3, src + pitch3, 0, 3);
}
void vpx_lpf_vertical_4_dual_lsx(uint8_t *src, int32_t pitch,
const uint8_t *b_limit0_ptr,
const uint8_t *limit0_ptr,
const uint8_t *thresh0_ptr,
const uint8_t *b_limit1_ptr,
const uint8_t *limit1_ptr,
const uint8_t *thresh1_ptr) {
__m128i mask, hev, flat;
__m128i thresh0, b_limit0, limit0, thresh1, b_limit1, limit1;
__m128i p3, p2, p1, p0, q3, q2, q1, q0;
__m128i row0, row1, row2, row3, row4, row5, row6, row7;
__m128i row8, row9, row10, row11, row12, row13, row14, row15;
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
int32_t pitch2 = pitch << 1;
int32_t pitch3 = pitch2 + pitch;
int32_t pitch4 = pitch2 << 1;
uint8_t *src_tmp = src - 4;
row0 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, row1, row2);
row3 = __lsx_vldx(src_tmp, pitch3);
src_tmp += pitch4;
row4 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, row5, row6);
row7 = __lsx_vldx(src_tmp, pitch3);
src_tmp += pitch4;
row8 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, row9, row10);
row11 = __lsx_vldx(src_tmp, pitch3);
src_tmp += pitch4;
row12 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, row13, row14);
row15 = __lsx_vldx(src_tmp, pitch3);
LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, row8,
row9, row10, row11, row12, row13, row14, row15, p3, p2,
p1, p0, q0, q1, q2, q3);
thresh0 = __lsx_vldrepl_b(thresh0_ptr, 0);
thresh1 = __lsx_vldrepl_b(thresh1_ptr, 0);
thresh0 = __lsx_vilvl_d(thresh1, thresh0);
b_limit0 = __lsx_vldrepl_b(b_limit0_ptr, 0);
b_limit1 = __lsx_vldrepl_b(b_limit1_ptr, 0);
b_limit0 = __lsx_vilvl_d(b_limit1, b_limit0);
limit0 = __lsx_vldrepl_b(limit0_ptr, 0);
limit1 = __lsx_vldrepl_b(limit1_ptr, 0);
limit0 = __lsx_vilvl_d(limit1, limit0);
LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, hev,
mask, flat);
VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1);
DUP2_ARG2(__lsx_vilvl_b, p0, p1, q1, q0, tmp0, tmp1);
tmp2 = __lsx_vilvl_h(tmp1, tmp0);
tmp3 = __lsx_vilvh_h(tmp1, tmp0);
DUP2_ARG2(__lsx_vilvh_b, p0, p1, q1, q0, tmp0, tmp1);
tmp4 = __lsx_vilvl_h(tmp1, tmp0);
tmp5 = __lsx_vilvh_h(tmp1, tmp0);
src -= 2;
__lsx_vstelm_w(tmp2, src, 0, 0);
__lsx_vstelm_w(tmp2, src + pitch, 0, 1);
__lsx_vstelm_w(tmp2, src + pitch2, 0, 2);
__lsx_vstelm_w(tmp2, src + pitch3, 0, 3);
src += pitch4;
__lsx_vstelm_w(tmp3, src, 0, 0);
__lsx_vstelm_w(tmp3, src + pitch, 0, 1);
__lsx_vstelm_w(tmp3, src + pitch2, 0, 2);
__lsx_vstelm_w(tmp3, src + pitch3, 0, 3);
src += pitch4;
__lsx_vstelm_w(tmp4, src, 0, 0);
__lsx_vstelm_w(tmp4, src + pitch, 0, 1);
__lsx_vstelm_w(tmp4, src + pitch2, 0, 2);
__lsx_vstelm_w(tmp4, src + pitch3, 0, 3);
src += pitch4;
__lsx_vstelm_w(tmp5, src, 0, 0);
__lsx_vstelm_w(tmp5, src + pitch, 0, 1);
__lsx_vstelm_w(tmp5, src + pitch2, 0, 2);
__lsx_vstelm_w(tmp5, src + pitch3, 0, 3);
}
libs/libvpx/vpx_dsp/loongarch/loopfilter_8_lsx.c
+458
View File
@@ -0,0 +1,458 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/loongarch/loopfilter_lsx.h"
void vpx_lpf_horizontal_8_lsx(uint8_t *dst, int32_t stride,
const uint8_t *b_limit_ptr,
const uint8_t *limit_ptr,
const uint8_t *thresh_ptr) {
__m128i mask, hev, flat, thresh, b_limit, limit;
__m128i p3, p2, p1, p0, q3, q2, q1, q0;
__m128i p2_out, p1_out, p0_out, q0_out, q1_out;
__m128i p2_filter8, p1_filter8, p0_filter8;
__m128i q0_filter8, q1_filter8, q2_filter8;
__m128i p3_l, p2_l, p1_l, p0_l, q3_l, q2_l, q1_l, q0_l;
int32_t stride2 = stride << 1;
int32_t stride3 = stride2 + stride;
int32_t stride4 = stride2 << 1;
/* load vector elements */
DUP4_ARG2(__lsx_vldx, dst, -stride4, dst, -stride3, dst, -stride2, dst,
-stride, p3, p2, p1, p0);
q0 = __lsx_vld(dst, 0);
DUP2_ARG2(__lsx_vldx, dst, stride, dst, stride2, q1, q2);
q3 = __lsx_vldx(dst, stride3);
thresh = __lsx_vldrepl_b(thresh_ptr, 0);
b_limit = __lsx_vldrepl_b(b_limit_ptr, 0);
limit = __lsx_vldrepl_b(limit_ptr, 0);
LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev,
mask, flat);
VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat);
VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out);
flat = __lsx_vilvl_d(flat, flat);
if (__lsx_bz_v(flat)) {
__lsx_vstelm_d(p1_out, dst - stride2, 0, 0);
__lsx_vstelm_d(p0_out, dst - stride, 0, 0);
__lsx_vstelm_d(q0_out, dst, 0, 0);
__lsx_vstelm_d(q1_out, dst + stride, 0, 0);
} else {
DUP4_ARG2(__lsx_vsllwil_hu_bu, p3, 0, p2, 0, p1, 0, p0, 0, p3_l, p2_l, p1_l,
p0_l);
DUP4_ARG2(__lsx_vsllwil_hu_bu, q0, 0, q1, 0, q2, 0, q3, 0, q0_l, q1_l, q2_l,
q3_l);
VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filter8,
p1_filter8, p0_filter8, q0_filter8, q1_filter8, q2_filter8);
DUP2_ARG2(__lsx_vpickev_b, p1_filter8, p2_filter8, q0_filter8, p0_filter8,
p1_filter8, q0_filter8);
q2_filter8 = __lsx_vpickev_b(q2_filter8, q1_filter8);
p2 = __lsx_vilvl_d(p1_out, p2);
p0_out = __lsx_vilvl_d(q0_out, p0_out);
q1_out = __lsx_vilvl_d(q2, q1_out);
DUP2_ARG3(__lsx_vbitsel_v, p2, p1_filter8, flat, p0_out, q0_filter8, flat,
p2_out, p1_out);
p0_out = __lsx_vbitsel_v(q1_out, q2_filter8, flat);
dst -= stride3;
__lsx_vstelm_d(p2_out, dst, 0, 0);
__lsx_vstelm_d(p2_out, dst + stride, 0, 1);
__lsx_vstelm_d(p1_out, dst + stride2, 0, 0);
__lsx_vstelm_d(p1_out, dst + stride3, 0, 1);
dst += stride4;
__lsx_vstelm_d(p0_out, dst, 0, 0);
dst += stride;
__lsx_vstelm_d(p0_out, dst, 0, 1);
}
}
void vpx_lpf_horizontal_8_dual_lsx(
uint8_t *dst, int32_t stride, const uint8_t *b_limit0,
const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *b_limit1,
const uint8_t *limit1, const uint8_t *thresh1) {
__m128i p3, p2, p1, p0, q3, q2, q1, q0;
__m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out;
__m128i flat, mask, hev, thresh, b_limit, limit;
__m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l;
__m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h;
__m128i p2_filt8_l, p1_filt8_l, p0_filt8_l;
__m128i q0_filt8_l, q1_filt8_l, q2_filt8_l;
__m128i p2_filt8_h, p1_filt8_h, p0_filt8_h;
__m128i q0_filt8_h, q1_filt8_h, q2_filt8_h;
int32_t stride2 = stride << 1;
int32_t stride3 = stride2 + stride;
int32_t stride4 = stride2 << 1;
DUP4_ARG2(__lsx_vldx, dst, -stride4, dst, -stride3, dst, -stride2, dst,
-stride, p3, p2, p1, p0);
q0 = __lsx_vld(dst, 0);
DUP2_ARG2(__lsx_vldx, dst, stride, dst, stride2, q1, q2);
q3 = __lsx_vldx(dst, stride3);
thresh = __lsx_vldrepl_b(thresh0, 0);
p2_out = __lsx_vldrepl_b(thresh1, 0);
thresh = __lsx_vilvl_d(p2_out, thresh);
b_limit = __lsx_vldrepl_b(b_limit0, 0);
p2_out = __lsx_vldrepl_b(b_limit1, 0);
b_limit = __lsx_vilvl_d(p2_out, b_limit);
limit = __lsx_vldrepl_b(limit0, 0);
p2_out = __lsx_vldrepl_b(limit1, 0);
limit = __lsx_vilvl_d(p2_out, limit);
/* mask and hev */
LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev,
mask, flat);
VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat);
VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out);
if (__lsx_bz_v(flat)) {
__lsx_vst(p1_out, dst - stride2, 0);
__lsx_vst(p0_out, dst - stride, 0);
__lsx_vst(q0_out, dst, 0);
__lsx_vst(q1_out, dst + stride, 0);
} else {
DUP4_ARG2(__lsx_vsllwil_hu_bu, p3, 0, p2, 0, p1, 0, p0, 0, p3_l, p2_l, p1_l,
p0_l);
DUP4_ARG2(__lsx_vsllwil_hu_bu, q0, 0, q1, 0, q2, 0, q3, 0, q0_l, q1_l, q2_l,
q3_l);
VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l,
p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l);
DUP4_ARG1(__lsx_vexth_hu_bu, p3, p2, p1, p0, p3_h, p2_h, p1_h, p0_h);
DUP4_ARG1(__lsx_vexth_hu_bu, q0, q1, q2, q3, q0_h, q1_h, q2_h, q3_h);
VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h,
p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h);
/* convert 16 bit output data into 8 bit */
DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_l, p1_filt8_h, p1_filt8_l,
p0_filt8_h, p0_filt8_l, q0_filt8_h, q0_filt8_l, p2_filt8_l,
p1_filt8_l, p0_filt8_l, q0_filt8_l);
DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_l, q2_filt8_h, q2_filt8_l,
q1_filt8_l, q2_filt8_l);
/* store pixel values */
p2_out = __lsx_vbitsel_v(p2, p2_filt8_l, flat);
p1_out = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat);
p0_out = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat);
q0_out = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat);
q1_out = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat);
q2_out = __lsx_vbitsel_v(q2, q2_filt8_l, flat);
__lsx_vst(p2_out, dst - stride3, 0);
__lsx_vst(p1_out, dst - stride2, 0);
__lsx_vst(p0_out, dst - stride, 0);
__lsx_vst(q0_out, dst, 0);
__lsx_vst(q1_out, dst + stride, 0);
__lsx_vst(q2_out, dst + stride2, 0);
}
}
void vpx_lpf_vertical_8_lsx(uint8_t *dst, int32_t stride,
const uint8_t *b_limit_ptr,
const uint8_t *limit_ptr,
const uint8_t *thresh_ptr) {
__m128i p3, p2, p1, p0, q3, q2, q1, q0;
__m128i p1_out, p0_out, q0_out, q1_out;
__m128i flat, mask, hev, thresh, b_limit, limit;
__m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l;
__m128i p2_filt8_l, p1_filt8_l, p0_filt8_l;
__m128i q0_filt8_l, q1_filt8_l, q2_filt8_l;
__m128i zero = __lsx_vldi(0);
int32_t stride2 = stride << 1;
int32_t stride3 = stride2 + stride;
int32_t stride4 = stride2 << 1;
uint8_t *dst_tmp = dst - 4;
/* load vector elements */
p3 = __lsx_vld(dst_tmp, 0);
DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, p2, p1);
p0 = __lsx_vldx(dst_tmp, stride3);
dst_tmp += stride4;
q0 = __lsx_vld(dst_tmp, 0);
DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, q1, q2);
q3 = __lsx_vldx(dst_tmp, stride3);
LSX_TRANSPOSE8x8_B(p3, p2, p1, p0, q0, q1, q2, q3, p3, p2, p1, p0, q0, q1, q2,
q3);
thresh = __lsx_vldrepl_b(thresh_ptr, 0);
b_limit = __lsx_vldrepl_b(b_limit_ptr, 0);
limit = __lsx_vldrepl_b(limit_ptr, 0);
/* mask and hev */
LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev,
mask, flat);
/* flat4 */
VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat);
/* filter4 */
VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out);
flat = __lsx_vilvl_d(zero, flat);
/* if flat is zero for all pixels, then no need to calculate other filter */
if (__lsx_bz_v(flat)) {
/* Store 4 pixels p1-_q1 */
DUP2_ARG2(__lsx_vilvl_b, p0_out, p1_out, q1_out, q0_out, p0, p1);
p2 = __lsx_vilvl_h(p1, p0);
p3 = __lsx_vilvh_h(p1, p0);
dst -= 2;
__lsx_vstelm_w(p2, dst, 0, 0);
__lsx_vstelm_w(p2, dst + stride, 0, 1);
__lsx_vstelm_w(p2, dst + stride2, 0, 2);
__lsx_vstelm_w(p2, dst + stride3, 0, 3);
dst += stride4;
__lsx_vstelm_w(p3, dst, 0, 0);
__lsx_vstelm_w(p3, dst + stride, 0, 1);
__lsx_vstelm_w(p3, dst + stride2, 0, 2);
__lsx_vstelm_w(p3, dst + stride3, 0, 3);
} else {
DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l,
p1_l, p0_l);
DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l,
q2_l, q3_l);
VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l,
p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l);
/* convert 16 bit output data into 8 bit */
DUP4_ARG2(__lsx_vpickev_b, p2_filt8_l, p2_filt8_l, p1_filt8_l, p1_filt8_l,
p0_filt8_l, p0_filt8_l, q0_filt8_l, q0_filt8_l, p2_filt8_l,
p1_filt8_l, p0_filt8_l, q0_filt8_l);
DUP2_ARG2(__lsx_vpickev_b, q1_filt8_l, q1_filt8_l, q2_filt8_l, q2_filt8_l,
q1_filt8_l, q2_filt8_l);
/* store pixel values */
p2 = __lsx_vbitsel_v(p2, p2_filt8_l, flat);
p1 = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat);
p0 = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat);
q0 = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat);
q1 = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat);
q2 = __lsx_vbitsel_v(q2, q2_filt8_l, flat);
/* Store 6 pixels p2-_q2 */
DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, p3, q3);
p1 = __lsx_vilvl_h(q3, p3);
p2 = __lsx_vilvh_h(q3, p3);
p3 = __lsx_vilvl_b(q2, q1);
dst -= 3;
__lsx_vstelm_w(p1, dst, 0, 0);
__lsx_vstelm_h(p3, dst, 4, 0);
dst += stride;
__lsx_vstelm_w(p1, dst, 0, 1);
__lsx_vstelm_h(p3, dst, 4, 1);
dst += stride;
__lsx_vstelm_w(p1, dst, 0, 2);
__lsx_vstelm_h(p3, dst, 4, 2);
dst += stride;
__lsx_vstelm_w(p1, dst, 0, 3);
__lsx_vstelm_h(p3, dst, 4, 3);
dst += stride;
__lsx_vstelm_w(p2, dst, 0, 0);
__lsx_vstelm_h(p3, dst, 4, 4);
dst += stride;
__lsx_vstelm_w(p2, dst, 0, 1);
__lsx_vstelm_h(p3, dst, 4, 5);
dst += stride;
__lsx_vstelm_w(p2, dst, 0, 2);
__lsx_vstelm_h(p3, dst, 4, 6);
dst += stride;
__lsx_vstelm_w(p2, dst, 0, 3);
__lsx_vstelm_h(p3, dst, 4, 7);
}
}
void vpx_lpf_vertical_8_dual_lsx(uint8_t *dst, int32_t stride,
const uint8_t *b_limit0, const uint8_t *limit0,
const uint8_t *thresh0,
const uint8_t *b_limit1, const uint8_t *limit1,
const uint8_t *thresh1) {
uint8_t *dst_tmp = dst - 4;
__m128i p3, p2, p1, p0, q3, q2, q1, q0;
__m128i p1_out, p0_out, q0_out, q1_out;
__m128i flat, mask, hev, thresh, b_limit, limit;
__m128i row4, row5, row6, row7, row12, row13, row14, row15;
__m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l;
__m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h;
__m128i p2_filt8_l, p1_filt8_l, p0_filt8_l;
__m128i q0_filt8_l, q1_filt8_l, q2_filt8_l;
__m128i p2_filt8_h, p1_filt8_h, p0_filt8_h;
__m128i q0_filt8_h, q1_filt8_h, q2_filt8_h;
int32_t stride2 = stride << 1;
int32_t stride3 = stride2 + stride;
int32_t stride4 = stride2 << 1;
p0 = __lsx_vld(dst_tmp, 0);
DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, p1, p2);
p3 = __lsx_vldx(dst_tmp, stride3);
dst_tmp += stride4;
row4 = __lsx_vld(dst_tmp, 0);
DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, row5, row6);
row7 = __lsx_vldx(dst_tmp, stride3);
dst_tmp += stride4;
q3 = __lsx_vld(dst_tmp, 0);
DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, q2, q1);
q0 = __lsx_vldx(dst_tmp, stride3);
dst_tmp += stride4;
row12 = __lsx_vld(dst_tmp, 0);
DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, row13, row14);
row15 = __lsx_vldx(dst_tmp, stride3);
/* transpose 16x8 matrix into 8x16 */
LSX_TRANSPOSE16x8_B(p0, p1, p2, p3, row4, row5, row6, row7, q3, q2, q1, q0,
row12, row13, row14, row15, p3, p2, p1, p0, q0, q1, q2,
q3);
thresh = __lsx_vldrepl_b(thresh0, 0);
p1_out = __lsx_vldrepl_b(thresh1, 0);
thresh = __lsx_vilvl_d(p1_out, thresh);
b_limit = __lsx_vldrepl_b(b_limit0, 0);
p1_out = __lsx_vldrepl_b(b_limit1, 0);
b_limit = __lsx_vilvl_d(p1_out, b_limit);
limit = __lsx_vldrepl_b(limit0, 0);
p1_out = __lsx_vldrepl_b(limit1, 0);
limit = __lsx_vilvl_d(p1_out, limit);
/* mask and hev */
LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev,
mask, flat);
/* flat4 */
VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat);
/* filter4 */
VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out);
/* if flat is zero for all pixels, then no need to calculate other filter */
if (__lsx_bz_v(flat)) {
DUP2_ARG2(__lsx_vilvl_b, p0_out, p1_out, q1_out, q0_out, p0, p1);
p2 = __lsx_vilvl_h(p1, p0);
p3 = __lsx_vilvh_h(p1, p0);
DUP2_ARG2(__lsx_vilvh_b, p0_out, p1_out, q1_out, q0_out, p0, p1);
q2 = __lsx_vilvl_h(p1, p0);
q3 = __lsx_vilvh_h(p1, p0);
dst -= 2;
__lsx_vstelm_w(p2, dst, 0, 0);
__lsx_vstelm_w(p2, dst + stride, 0, 1);
__lsx_vstelm_w(p2, dst + stride2, 0, 2);
__lsx_vstelm_w(p2, dst + stride3, 0, 3);
dst += stride4;
__lsx_vstelm_w(p3, dst, 0, 0);
__lsx_vstelm_w(p3, dst + stride, 0, 1);
__lsx_vstelm_w(p3, dst + stride2, 0, 2);
__lsx_vstelm_w(p3, dst + stride3, 0, 3);
dst += stride4;
__lsx_vstelm_w(q2, dst, 0, 0);
__lsx_vstelm_w(q2, dst + stride, 0, 1);
__lsx_vstelm_w(q2, dst + stride2, 0, 2);
__lsx_vstelm_w(q2, dst + stride3, 0, 3);
dst += stride4;
__lsx_vstelm_w(q3, dst, 0, 0);
__lsx_vstelm_w(q3, dst + stride, 0, 1);
__lsx_vstelm_w(q3, dst + stride2, 0, 2);
__lsx_vstelm_w(q3, dst + stride3, 0, 3);
} else {
DUP4_ARG2(__lsx_vsllwil_hu_bu, p3, 0, p2, 0, p1, 0, p0, 0, p3_l, p2_l, p1_l,
p0_l);
DUP4_ARG2(__lsx_vsllwil_hu_bu, q0, 0, q1, 0, q2, 0, q3, 0, q0_l, q1_l, q2_l,
q3_l);
VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l,
p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l);
DUP4_ARG1(__lsx_vexth_hu_bu, p3, p2, p1, p0, p3_h, p2_h, p1_h, p0_h);
DUP4_ARG1(__lsx_vexth_hu_bu, q0, q1, q2, q3, q0_h, q1_h, q2_h, q3_h);
/* filter8 */
VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h,
p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h);
/* convert 16 bit output data into 8 bit */
DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_l, p1_filt8_h, p1_filt8_l,
p0_filt8_h, p0_filt8_l, q0_filt8_h, q0_filt8_l, p2_filt8_l,
p1_filt8_l, p0_filt8_l, q0_filt8_l);
DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_l, q2_filt8_h, q2_filt8_l,
q1_filt8_l, q2_filt8_l);
/* store pixel values */
p2 = __lsx_vbitsel_v(p2, p2_filt8_l, flat);
p1 = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat);
p0 = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat);
q0 = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat);
q1 = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat);
q2 = __lsx_vbitsel_v(q2, q2_filt8_l, flat);
DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, p3, q3);
p2_filt8_l = __lsx_vilvl_h(q3, p3);
p2_filt8_h = __lsx_vilvh_h(q3, p3);
DUP2_ARG2(__lsx_vilvh_b, p1, p2, q0, p0, p3, q3);
p0_filt8_l = __lsx_vilvl_h(q3, p3);
p0_filt8_h = __lsx_vilvh_h(q3, p3);
q1_filt8_l = __lsx_vilvl_b(q2, q1);
q1_filt8_h = __lsx_vilvh_b(q2, q1);
dst -= 3;
__lsx_vstelm_w(p2_filt8_l, dst, 0, 0);
__lsx_vstelm_h(q1_filt8_l, dst, 4, 0);
dst += stride;
__lsx_vstelm_w(p2_filt8_l, dst, 0, 1);
__lsx_vstelm_h(q1_filt8_l, dst, 4, 1);
dst += stride;
__lsx_vstelm_w(p2_filt8_l, dst, 0, 2);
__lsx_vstelm_h(q1_filt8_l, dst, 4, 2);
dst += stride;
__lsx_vstelm_w(p2_filt8_l, dst, 0, 3);
__lsx_vstelm_h(q1_filt8_l, dst, 4, 3);
dst += stride;
__lsx_vstelm_w(p2_filt8_h, dst, 0, 0);
__lsx_vstelm_h(q1_filt8_l, dst, 4, 4);
dst += stride;
__lsx_vstelm_w(p2_filt8_h, dst, 0, 1);
__lsx_vstelm_h(q1_filt8_l, dst, 4, 5);
dst += stride;
__lsx_vstelm_w(p2_filt8_h, dst, 0, 2);
__lsx_vstelm_h(q1_filt8_l, dst, 4, 6);
dst += stride;
__lsx_vstelm_w(p2_filt8_h, dst, 0, 3);
__lsx_vstelm_h(q1_filt8_l, dst, 4, 7);
dst += stride;
__lsx_vstelm_w(p0_filt8_l, dst, 0, 0);
__lsx_vstelm_h(q1_filt8_h, dst, 4, 0);
dst += stride;
__lsx_vstelm_w(p0_filt8_l, dst, 0, 1);
__lsx_vstelm_h(q1_filt8_h, dst, 4, 1);
dst += stride;
__lsx_vstelm_w(p0_filt8_l, dst, 0, 2);
__lsx_vstelm_h(q1_filt8_h, dst, 4, 2);
dst += stride;
__lsx_vstelm_w(p0_filt8_l, dst, 0, 3);
__lsx_vstelm_h(q1_filt8_h, dst, 4, 3);
dst += stride;
__lsx_vstelm_w(p0_filt8_h, dst, 0, 0);
__lsx_vstelm_h(q1_filt8_h, dst, 4, 4);
dst += stride;
__lsx_vstelm_w(p0_filt8_h, dst, 0, 1);
__lsx_vstelm_h(q1_filt8_h, dst, 4, 5);
dst += stride;
__lsx_vstelm_w(p0_filt8_h, dst, 0, 2);
__lsx_vstelm_h(q1_filt8_h, dst, 4, 6);
dst += stride;
__lsx_vstelm_w(p0_filt8_h, dst, 0, 3);
__lsx_vstelm_h(q1_filt8_h, dst, 4, 7);
}
}
libs/libvpx/vpx_dsp/loongarch/loopfilter_lsx.h
+167
View File
@@ -0,0 +1,167 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VPX_VPX_DSP_LOONGARCH_LOOPFILTER_LSX_H_
#define VPX_VPX_DSP_LOONGARCH_LOOPFILTER_LSX_H_
#include "vpx_util/loongson_intrinsics.h"
#define LPF_MASK_HEV(p3_in, p2_in, p1_in, p0_in, q0_in, q1_in, q2_in, q3_in, \
limit_in, b_limit_in, thresh_in, hev_out, mask_out, \
flat_out) \
do { \
__m128i p3_asub_p2_m, p2_asub_p1_m, p1_asub_p0_m, q1_asub_q0_m; \
__m128i p1_asub_q1_m, p0_asub_q0_m, q3_asub_q2_m, q2_asub_q1_m; \
\
/* absolute subtraction of pixel values */ \
p3_asub_p2_m = __lsx_vabsd_bu(p3_in, p2_in); \
p2_asub_p1_m = __lsx_vabsd_bu(p2_in, p1_in); \
p1_asub_p0_m = __lsx_vabsd_bu(p1_in, p0_in); \
q1_asub_q0_m = __lsx_vabsd_bu(q1_in, q0_in); \
q2_asub_q1_m = __lsx_vabsd_bu(q2_in, q1_in); \
q3_asub_q2_m = __lsx_vabsd_bu(q3_in, q2_in); \
p0_asub_q0_m = __lsx_vabsd_bu(p0_in, q0_in); \
p1_asub_q1_m = __lsx_vabsd_bu(p1_in, q1_in); \
\
/* calculation of hev */ \
flat_out = __lsx_vmax_bu(p1_asub_p0_m, q1_asub_q0_m); \
hev_out = __lsx_vslt_bu(thresh_in, flat_out); \
\
/* calculation of mask */ \
p0_asub_q0_m = __lsx_vsadd_bu(p0_asub_q0_m, p0_asub_q0_m); \
p1_asub_q1_m = __lsx_vsrli_b(p1_asub_q1_m, 1); \
p0_asub_q0_m = __lsx_vsadd_bu(p0_asub_q0_m, p1_asub_q1_m); \
mask_out = __lsx_vslt_bu(b_limit_in, p0_asub_q0_m); \
mask_out = __lsx_vmax_bu(flat_out, mask_out); \
p3_asub_p2_m = __lsx_vmax_bu(p3_asub_p2_m, p2_asub_p1_m); \
mask_out = __lsx_vmax_bu(p3_asub_p2_m, mask_out); \
q2_asub_q1_m = __lsx_vmax_bu(q2_asub_q1_m, q3_asub_q2_m); \
mask_out = __lsx_vmax_bu(q2_asub_q1_m, mask_out); \
\
mask_out = __lsx_vslt_bu(limit_in, mask_out); \
mask_out = __lsx_vxori_b(mask_out, 0xff); \
} while (0)
#define VP9_FLAT4(p3_in, p2_in, p0_in, q0_in, q2_in, q3_in, flat_out) \
do { \
__m128i p2_asub_p0, q2_asub_q0, p3_asub_p0, q3_asub_q0; \
__m128i flat4_tmp = __lsx_vldi(1); \
\
DUP4_ARG2(__lsx_vabsd_bu, p2_in, p0_in, q2_in, q0_in, p3_in, p0_in, q3_in, \
q0_in, p2_asub_p0, q2_asub_q0, p3_asub_p0, q3_asub_q0); \
p2_asub_p0 = __lsx_vmax_bu(p2_asub_p0, q2_asub_q0); \
flat_out = __lsx_vmax_bu(p2_asub_p0, flat_out); \
p3_asub_p0 = __lsx_vmax_bu(p3_asub_p0, q3_asub_q0); \
flat_out = __lsx_vmax_bu(p3_asub_p0, flat_out); \
\
flat_out = __lsx_vslt_bu(flat4_tmp, flat_out); \
flat_out = __lsx_vxori_b(flat_out, 0xff); \
flat_out = flat_out & (mask); \
} while (0)
#define VP9_FLAT5(p7_in, p6_in, p5_in, p4_in, p0_in, q0_in, q4_in, q5_in, \
q6_in, q7_in, flat_in, flat2_out) \
do { \
__m128i flat5_tmp = __lsx_vldi(1); \
__m128i p4_asub_p0, q4_asub_q0, p5_asub_p0, q5_asub_q0; \
__m128i p6_asub_p0, q6_asub_q0, p7_asub_p0, q7_asub_q0; \
DUP4_ARG2(__lsx_vabsd_bu, p4_in, p0_in, q4_in, q0_in, p5_in, p0_in, q5_in, \
q0_in, p4_asub_p0, q4_asub_q0, p5_asub_p0, q5_asub_q0); \
DUP4_ARG2(__lsx_vabsd_bu, p6_in, p0_in, q6_in, q0_in, p7_in, p0_in, q7_in, \
q0_in, p6_asub_p0, q6_asub_q0, p7_asub_p0, q7_asub_q0); \
\
DUP2_ARG2(__lsx_vmax_bu, p4_asub_p0, q4_asub_q0, p5_asub_p0, q5_asub_q0, \
p4_asub_p0, flat2_out); \
flat2_out = __lsx_vmax_bu(p4_asub_p0, flat2_out); \
p6_asub_p0 = __lsx_vmax_bu(p6_asub_p0, q6_asub_q0); \
flat2_out = __lsx_vmax_bu(p6_asub_p0, flat2_out); \
p7_asub_p0 = __lsx_vmax_bu(p7_asub_p0, q7_asub_q0); \
flat2_out = __lsx_vmax_bu(p7_asub_p0, flat2_out); \
flat2_out = __lsx_vslt_bu(flat5_tmp, flat2_out); \
flat2_out = __lsx_vxori_b(flat2_out, 0xff); \
flat2_out = flat2_out & flat_in; \
} while (0)
#define VP9_LPF_FILTER4_4W(p1_in, p0_in, q0_in, q1_in, mask, hev, p1_out, \
p0_out, q0_out, q1_out) \
do { \
__m128i p1_m, p0_m, q0_m, q1_m, filt, q0_sub_p0, t1, t2; \
const __m128i cnst4b = __lsx_vldi(4); \
const __m128i cnst3b = __lsx_vldi(3); \
DUP4_ARG2(__lsx_vxori_b, p1_in, 0x80, p0_in, 0x80, q0_in, 0x80, q1_in, \
0x80, p1_m, p0_m, q0_m, q1_m); \
filt = __lsx_vssub_b(p1_m, q1_m); \
filt &= hev; \
\
q0_sub_p0 = __lsx_vssub_b(q0_m, p0_m); \
filt = __lsx_vsadd_b(filt, q0_sub_p0); \
filt = __lsx_vsadd_b(filt, q0_sub_p0); \
filt = __lsx_vsadd_b(filt, q0_sub_p0); \
filt &= mask; \
DUP2_ARG2(__lsx_vsadd_b, filt, cnst4b, filt, cnst3b, t1, t2); \
DUP2_ARG2(__lsx_vsrai_b, t1, 3, t2, 3, t1, t2); \
\
q0_m = __lsx_vssub_b(q0_m, t1); \
p0_m = __lsx_vsadd_b(p0_m, t2); \
DUP2_ARG2(__lsx_vxori_b, q0_m, 0x80, p0_m, 0x80, q0_out, p0_out); \
\
filt = __lsx_vsrari_b(t1, 1); \
hev = __lsx_vxori_b(hev, 0xff); \
filt &= hev; \
q1_m = __lsx_vssub_b(q1_m, filt); \
p1_m = __lsx_vsadd_b(p1_m, filt); \
DUP2_ARG2(__lsx_vxori_b, q1_m, 0x80, p1_m, 0x80, q1_out, p1_out); \
} while (0)
#define VP9_FILTER8(p3_in, p2_in, p1_in, p0_in, q0_in, q1_in, q2_in, q3_in, \
p2_filt8_out, p1_filt8_out, p0_filt8_out, q0_filt8_out, \
q1_filt8_out, q2_filt8_out) \
do { \
__m128i tmp_filt8_0, tmp_filt8_1, tmp_filt8_2; \
\
tmp_filt8_2 = __lsx_vadd_h(p2_in, p1_in); \
tmp_filt8_2 = __lsx_vadd_h(tmp_filt8_2, p0_in); \
tmp_filt8_0 = __lsx_vslli_h(p3_in, 1); \
\
tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_0, tmp_filt8_2); \
tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_0, q0_in); \
tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_0, p3_in); \
tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_1, p2_in); \
p2_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \
\
tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_0, p1_in); \
tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_1, q1_in); \
p1_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \
\
tmp_filt8_1 = __lsx_vadd_h(q2_in, q1_in); \
tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_1, q0_in); \
tmp_filt8_2 = __lsx_vadd_h(tmp_filt8_2, tmp_filt8_1); \
tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_2, p0_in); \
tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_0, p3_in); \
p0_filt8_out = __lsx_vsrari_h(tmp_filt8_0, 3); \
\
tmp_filt8_0 = __lsx_vadd_h(q2_in, q3_in); \
tmp_filt8_0 = __lsx_vadd_h(p0_in, tmp_filt8_0); \
tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_0, tmp_filt8_1); \
tmp_filt8_1 = __lsx_vadd_h(q3_in, q3_in); \
tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_1, tmp_filt8_0); \
q2_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \
\
tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_2, q3_in); \
tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_0, q0_in); \
q0_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \
\
tmp_filt8_1 = __lsx_vsub_h(tmp_filt8_0, p2_in); \
tmp_filt8_0 = __lsx_vadd_h(q1_in, q3_in); \
tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_0, tmp_filt8_1); \
q1_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \
} while (0)
#endif // VPX_VPX_DSP_LOONGARCH_LOOPFILTER_LSX_H_
libs/libvpx/vpx_dsp/loongarch/quantize_lsx.c
+253
View File
@@ -0,0 +1,253 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "./vpx_config.h"
#include "./vpx_dsp_rtcd.h"
#include "vpx_util/loongson_intrinsics.h"
static INLINE __m128i calculate_qcoeff(__m128i coeff, __m128i coeff_abs,
__m128i round, __m128i quant,
__m128i shift, __m128i cmp_mask) {
__m128i rounded, qcoeff;
rounded = __lsx_vsadd_h(coeff_abs, round);
qcoeff = __lsx_vmuh_h(rounded, quant);
qcoeff = __lsx_vadd_h(rounded, qcoeff);
qcoeff = __lsx_vmuh_h(qcoeff, shift);
qcoeff = __lsx_vsigncov_h(coeff, qcoeff);
qcoeff = __lsx_vand_v(qcoeff, cmp_mask);
return qcoeff;
}
static INLINE void calculate_dqcoeff_and_store(__m128i qcoeff, __m128i dequant,
int16_t *dqcoeff) {
__m128i dqcoeff16 = __lsx_vmul_h(qcoeff, dequant);
__lsx_vst(dqcoeff16, dqcoeff, 0);
}
static INLINE void calculate_dqcoeff_and_store_32x32(__m128i qcoeff,
__m128i dequant,
int16_t *dqcoeff) {
// Un-sign to bias rounding like C.
__m128i low, high, dqcoeff32_0, dqcoeff32_1, res;
__m128i zero = __lsx_vldi(0);
__m128i coeff = __lsx_vabsd_h(qcoeff, zero);
const __m128i sign_0 = __lsx_vilvl_h(qcoeff, zero);
const __m128i sign_1 = __lsx_vilvh_h(qcoeff, zero);
low = __lsx_vmul_h(coeff, dequant);
high = __lsx_vmuh_h(coeff, dequant);
dqcoeff32_0 = __lsx_vilvl_h(high, low);
dqcoeff32_1 = __lsx_vilvh_h(high, low);
// "Divide" by 2.
dqcoeff32_0 = __lsx_vsrai_w(dqcoeff32_0, 1);
dqcoeff32_1 = __lsx_vsrai_w(dqcoeff32_1, 1);
dqcoeff32_0 = __lsx_vsigncov_w(sign_0, dqcoeff32_0);
dqcoeff32_1 = __lsx_vsigncov_w(sign_1, dqcoeff32_1);
res = __lsx_vpickev_h(dqcoeff32_1, dqcoeff32_0);
__lsx_vst(res, dqcoeff, 0);
}
static INLINE __m128i scan_for_eob(__m128i coeff0, __m128i coeff1,
__m128i zbin_mask0, __m128i zbin_mask1,
const int16_t *scan, int index,
__m128i zero) {
const __m128i zero_coeff0 = __lsx_vseq_h(coeff0, zero);
const __m128i zero_coeff1 = __lsx_vseq_h(coeff1, zero);
__m128i scan0 = __lsx_vld(scan + index, 0);
__m128i scan1 = __lsx_vld(scan + index + 8, 0);
__m128i eob0, eob1;
scan0 = __lsx_vsub_h(scan0, zbin_mask0);
scan1 = __lsx_vsub_h(scan1, zbin_mask1);
eob0 = __lsx_vandn_v(zero_coeff0, scan0);
eob1 = __lsx_vandn_v(zero_coeff1, scan1);
return __lsx_vmax_h(eob0, eob1);
}
static INLINE int16_t accumulate_eob(__m128i eob) {
__m128i eob_shuffled;
int16_t res_m;
eob_shuffled = __lsx_vshuf4i_w(eob, 0xe);
eob = __lsx_vmax_h(eob, eob_shuffled);
eob_shuffled = __lsx_vshuf4i_h(eob, 0xe);
eob = __lsx_vmax_h(eob, eob_shuffled);
eob_shuffled = __lsx_vshuf4i_h(eob, 0x1);
eob = __lsx_vmax_h(eob, eob_shuffled);
res_m = __lsx_vpickve2gr_h(eob, 1);
return res_m;
}
#if !CONFIG_VP9_HIGHBITDEPTH
void vpx_quantize_b_lsx(const int16_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, int16_t *qcoeff_ptr,
int16_t *dqcoeff_ptr, const int16_t *dequant_ptr,
uint16_t *eob_ptr, const int16_t *scan,
const int16_t *iscan) {
__m128i zero = __lsx_vldi(0);
int index = 16;
__m128i zbin, round, quant, dequant, quant_shift;
__m128i coeff0, coeff1;
__m128i qcoeff0, qcoeff1;
__m128i cmp_mask0, cmp_mask1;
__m128i eob, eob0;
(void)scan;
zbin = __lsx_vld(zbin_ptr, 0);
round = __lsx_vld(round_ptr, 0);
quant = __lsx_vld(quant_ptr, 0);
dequant = __lsx_vld(dequant_ptr, 0);
quant_shift = __lsx_vld(quant_shift_ptr, 0);
// Handle one DC and first 15 AC.
DUP2_ARG2(__lsx_vld, coeff_ptr, 0, coeff_ptr, 16, coeff0, coeff1);
qcoeff0 = __lsx_vabsd_h(coeff0, zero);
qcoeff1 = __lsx_vabsd_h(coeff1, zero);
cmp_mask0 = __lsx_vsle_h(zbin, qcoeff0);
zbin = __lsx_vilvh_d(zbin, zbin);
cmp_mask1 = __lsx_vsle_h(zbin, qcoeff1);
qcoeff0 =
calculate_qcoeff(coeff0, qcoeff0, round, quant, quant_shift, cmp_mask0);
round = __lsx_vilvh_d(round, round);
quant = __lsx_vilvh_d(quant, quant);
quant_shift = __lsx_vilvh_d(quant_shift, quant_shift);
qcoeff1 =
calculate_qcoeff(coeff1, qcoeff1, round, quant, quant_shift, cmp_mask1);
__lsx_vst(qcoeff0, qcoeff_ptr, 0);
__lsx_vst(qcoeff1, qcoeff_ptr, 16);
calculate_dqcoeff_and_store(qcoeff0, dequant, dqcoeff_ptr);
dequant = __lsx_vilvh_d(dequant, dequant);
calculate_dqcoeff_and_store(qcoeff1, dequant, dqcoeff_ptr + 8);
eob = scan_for_eob(qcoeff0, qcoeff1, cmp_mask0, cmp_mask1, iscan, 0, zero);
// AC only loop.
while (index < n_coeffs) {
coeff0 = __lsx_vld(coeff_ptr + index, 0);
coeff1 = __lsx_vld(coeff_ptr + index + 8, 0);
qcoeff0 = __lsx_vabsd_h(coeff0, zero);
qcoeff1 = __lsx_vabsd_h(coeff1, zero);
cmp_mask0 = __lsx_vsle_h(zbin, qcoeff0);
cmp_mask1 = __lsx_vsle_h(zbin, qcoeff1);
qcoeff0 =
calculate_qcoeff(coeff0, qcoeff0, round, quant, quant_shift, cmp_mask0);
qcoeff1 =
calculate_qcoeff(coeff1, qcoeff1, round, quant, quant_shift, cmp_mask1);
__lsx_vst(qcoeff0, qcoeff_ptr + index, 0);
__lsx_vst(qcoeff1, qcoeff_ptr + index + 8, 0);
calculate_dqcoeff_and_store(qcoeff0, dequant, dqcoeff_ptr + index);
calculate_dqcoeff_and_store(qcoeff1, dequant, dqcoeff_ptr + index + 8);
eob0 = scan_for_eob(qcoeff0, qcoeff1, cmp_mask0, cmp_mask1, iscan, index,
zero);
eob = __lsx_vmax_h(eob, eob0);
index += 16;
}
*eob_ptr = accumulate_eob(eob);
}
void vpx_quantize_b_32x32_lsx(const int16_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
int16_t *qcoeff_ptr, int16_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
__m128i zero = __lsx_vldi(0);
int index;
__m128i zbin, round, quant, dequant, quant_shift;
__m128i coeff0, coeff1, qcoeff0, qcoeff1, cmp_mask0, cmp_mask1;
__m128i eob = zero, eob0;
(void)scan;
(void)n_coeffs;
zbin = __lsx_vld(zbin_ptr, 0);
zbin = __lsx_vsrari_h(zbin, 1);
round = __lsx_vld(round_ptr, 0);
round = __lsx_vsrari_h(round, 1);
quant = __lsx_vld(quant_ptr, 0);
dequant = __lsx_vld(dequant_ptr, 0);
quant_shift = __lsx_vld(quant_shift_ptr, 0);
quant_shift = __lsx_vslli_h(quant_shift, 1);
// Handle one DC and first 15 AC.
DUP2_ARG2(__lsx_vld, coeff_ptr, 0, coeff_ptr, 16, coeff0, coeff1);
qcoeff0 = __lsx_vabsd_h(coeff0, zero);
qcoeff1 = __lsx_vabsd_h(coeff1, zero);
cmp_mask0 = __lsx_vsle_h(zbin, qcoeff0);
// remove DC from zbin
zbin = __lsx_vilvh_d(zbin, zbin);
cmp_mask1 = __lsx_vsle_h(zbin, qcoeff1);
qcoeff0 =
calculate_qcoeff(coeff0, qcoeff0, round, quant, quant_shift, cmp_mask0);
// remove DC in quant_shift, quant, quant_shift
round = __lsx_vilvh_d(round, round);
quant = __lsx_vilvh_d(quant, quant);
quant_shift = __lsx_vilvh_d(quant_shift, quant_shift);
qcoeff1 =
calculate_qcoeff(coeff1, qcoeff1, round, quant, quant_shift, cmp_mask1);
__lsx_vst(qcoeff0, qcoeff_ptr, 0);
__lsx_vst(qcoeff1, qcoeff_ptr, 16);
calculate_dqcoeff_and_store_32x32(qcoeff0, dequant, dqcoeff_ptr);
dequant = __lsx_vilvh_d(dequant, dequant);
calculate_dqcoeff_and_store_32x32(qcoeff1, dequant, dqcoeff_ptr + 8);
eob = scan_for_eob(qcoeff0, qcoeff1, cmp_mask0, cmp_mask1, iscan, 0, zero);
// AC only loop.
for (index = 16; index < 32 * 32; index += 16) {
coeff0 = __lsx_vld(coeff_ptr + index, 0);
coeff1 = __lsx_vld(coeff_ptr + index + 8, 0);
qcoeff0 = __lsx_vabsd_h(coeff0, zero);
qcoeff1 = __lsx_vabsd_h(coeff1, zero);
cmp_mask0 = __lsx_vsle_h(zbin, qcoeff0);
cmp_mask1 = __lsx_vsle_h(zbin, qcoeff1);
qcoeff0 =
calculate_qcoeff(coeff0, qcoeff0, round, quant, quant_shift, cmp_mask0);
qcoeff1 =
calculate_qcoeff(coeff1, qcoeff1, round, quant, quant_shift, cmp_mask1);
__lsx_vst(qcoeff0, qcoeff_ptr + index, 0);
__lsx_vst(qcoeff1, qcoeff_ptr + index + 8, 0);
calculate_dqcoeff_and_store_32x32(qcoeff0, dequant, dqcoeff_ptr + index);
calculate_dqcoeff_and_store_32x32(qcoeff1, dequant,
dqcoeff_ptr + 8 + index);
eob0 = scan_for_eob(qcoeff0, qcoeff1, cmp_mask0, cmp_mask1, iscan, index,
zero);
eob = __lsx_vmax_h(eob, eob0);
}
*eob_ptr = accumulate_eob(eob);
}
#endif
libs/libvpx/vpx_dsp/loongarch/sad_lsx.c
+717
View File
@@ -0,0 +1,717 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "./vpx_config.h"
#include "./vpx_dsp_rtcd.h"
#include "vpx_util/loongson_intrinsics.h"
static INLINE __m128i sad_ub2_uh(__m128i in0, __m128i in1, __m128i ref0,
__m128i ref1) {
__m128i diff0_m, diff1_m, sad_m0;
__m128i sad_m = __lsx_vldi(0);
diff0_m = __lsx_vabsd_bu(in0, ref0);
diff1_m = __lsx_vabsd_bu(in1, ref1);
sad_m0 = __lsx_vhaddw_hu_bu(diff0_m, diff0_m);
sad_m = __lsx_vadd_h(sad_m, sad_m0);
sad_m0 = __lsx_vhaddw_hu_bu(diff1_m, diff1_m);
sad_m = __lsx_vadd_h(sad_m, sad_m0);
return sad_m;
}
static INLINE uint32_t hadd_uw_u32(__m128i in) {
__m128i res0_m;
uint32_t sum_m;
res0_m = __lsx_vhaddw_du_wu(in, in);
res0_m = __lsx_vhaddw_qu_du(res0_m, res0_m);
sum_m = __lsx_vpickve2gr_w(res0_m, 0);
return sum_m;
}
static INLINE uint32_t hadd_uh_u32(__m128i in) {
__m128i res_m;
uint32_t sum_m;
res_m = __lsx_vhaddw_wu_hu(in, in);
sum_m = hadd_uw_u32(res_m);
return sum_m;
}
static INLINE int32_t hadd_sw_s32(__m128i in) {
__m128i res0_m;
int32_t sum_m;
res0_m = __lsx_vhaddw_d_w(in, in);
res0_m = __lsx_vhaddw_q_d(res0_m, res0_m);
sum_m = __lsx_vpickve2gr_w(res0_m, 0);
return sum_m;
}
static uint32_t sad_8width_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
int32_t height) {
int32_t ht_cnt;
uint32_t res;
__m128i src0, src1, src2, src3, ref0, ref1, ref2, ref3, sad_tmp;
__m128i sad = __lsx_vldi(0);
for (ht_cnt = (height >> 2); ht_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src0, ref0);
src += src_stride;
ref += ref_stride;
DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src1, ref1);
src += src_stride;
ref += ref_stride;
DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src2, ref2);
src += src_stride;
ref += ref_stride;
DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src3, ref3);
src += src_stride;
ref += ref_stride;
DUP4_ARG2(__lsx_vpickev_d, src1, src0, src3, src2, ref1, ref0, ref3, ref2,
src0, src1, ref0, ref1);
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad = __lsx_vadd_h(sad, sad_tmp);
}
res = hadd_uh_u32(sad);
return res;
}
static uint32_t sad_16width_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
int32_t height) {
int32_t ht_cnt = (height >> 2);
uint32_t res;
__m128i src0, src1, ref0, ref1, sad_tmp;
__m128i sad = __lsx_vldi(0);
int32_t src_stride2 = src_stride << 1;
int32_t ref_stride2 = ref_stride << 1;
for (; ht_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src0, ref0);
DUP2_ARG2(__lsx_vldx, src, src_stride, ref, ref_stride, src1, ref1);
src += src_stride2;
ref += ref_stride2;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad = __lsx_vadd_h(sad, sad_tmp);
DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src0, ref0);
DUP2_ARG2(__lsx_vldx, src, src_stride, ref, ref_stride, src1, ref1);
src += src_stride2;
ref += ref_stride2;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad = __lsx_vadd_h(sad, sad_tmp);
}
res = hadd_uh_u32(sad);
return res;
}
static uint32_t sad_32width_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
int32_t height) {
int32_t ht_cnt = (height >> 2);
uint32_t res;
__m128i src0, src1, ref0, ref1;
__m128i sad_tmp;
__m128i sad = __lsx_vldi(0);
for (; ht_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1);
ref += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad = __lsx_vadd_h(sad, sad_tmp);
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1);
ref += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad = __lsx_vadd_h(sad, sad_tmp);
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1);
ref += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad = __lsx_vadd_h(sad, sad_tmp);
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1);
ref += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad = __lsx_vadd_h(sad, sad_tmp);
}
res = hadd_uh_u32(sad);
return res;
}
static uint32_t sad_64width_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
int32_t height) {
int32_t ht_cnt = (height >> 1);
uint32_t sad = 0;
__m128i src0, src1, src2, src3;
__m128i ref0, ref1, ref2, ref3;
__m128i sad_tmp;
__m128i sad0 = __lsx_vldi(0);
__m128i sad1 = sad0;
for (; ht_cnt--;) {
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2,
ref3);
ref += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, ref2, ref3);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2,
ref3);
ref += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, ref2, ref3);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
}
sad = hadd_uh_u32(sad0);
sad += hadd_uh_u32(sad1);
return sad;
}
static void sad_8width_x4d_lsx(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *const aref_ptr[],
int32_t ref_stride, int32_t height,
uint32_t *sad_array) {
int32_t ht_cnt = (height >> 2);
const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr;
__m128i src0, src1, src2, src3, sad_tmp;
__m128i ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7;
__m128i ref8, ref9, ref10, ref11, ref12, ref13, ref14, ref15;
__m128i sad0 = __lsx_vldi(0);
__m128i sad1 = sad0;
__m128i sad2 = sad0;
__m128i sad3 = sad0;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t ref_stride2 = ref_stride << 1;
int32_t ref_stride3 = ref_stride2 + ref_stride;
int32_t ref_stride4 = ref_stride2 << 1;
ref0_ptr = aref_ptr[0];
ref1_ptr = aref_ptr[1];
ref2_ptr = aref_ptr[2];
ref3_ptr = aref_ptr[3];
for (; ht_cnt--;) {
src0 = __lsx_vld(src_ptr, 0);
DUP2_ARG2(__lsx_vldx, src_ptr, src_stride, src_ptr, src_stride2, src1,
src2);
src3 = __lsx_vldx(src_ptr, src_stride3);
src_ptr += src_stride4;
ref0 = __lsx_vld(ref0_ptr, 0);
DUP2_ARG2(__lsx_vldx, ref0_ptr, ref_stride, ref0_ptr, ref_stride2, ref1,
ref2);
ref3 = __lsx_vldx(ref0_ptr, ref_stride3);
ref0_ptr += ref_stride4;
ref4 = __lsx_vld(ref1_ptr, 0);
DUP2_ARG2(__lsx_vldx, ref1_ptr, ref_stride, ref1_ptr, ref_stride2, ref5,
ref6);
ref7 = __lsx_vldx(ref1_ptr, ref_stride3);
ref1_ptr += ref_stride4;
ref8 = __lsx_vld(ref2_ptr, 0);
DUP2_ARG2(__lsx_vldx, ref2_ptr, ref_stride, ref2_ptr, ref_stride2, ref9,
ref10);
ref11 = __lsx_vldx(ref2_ptr, ref_stride3);
ref2_ptr += ref_stride4;
ref12 = __lsx_vld(ref3_ptr, 0);
DUP2_ARG2(__lsx_vldx, ref3_ptr, ref_stride, ref3_ptr, ref_stride2, ref13,
ref14);
ref15 = __lsx_vldx(ref3_ptr, ref_stride3);
ref3_ptr += ref_stride4;
DUP2_ARG2(__lsx_vpickev_d, src1, src0, src3, src2, src0, src1);
DUP2_ARG2(__lsx_vpickev_d, ref1, ref0, ref3, ref2, ref0, ref1);
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
DUP2_ARG2(__lsx_vpickev_d, ref5, ref4, ref7, ref6, ref0, ref1);
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
DUP2_ARG2(__lsx_vpickev_d, ref9, ref8, ref11, ref10, ref0, ref1);
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad2 = __lsx_vadd_h(sad2, sad_tmp);
DUP2_ARG2(__lsx_vpickev_d, ref13, ref12, ref15, ref14, ref0, ref1);
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad3 = __lsx_vadd_h(sad3, sad_tmp);
}
sad_array[0] = hadd_uh_u32(sad0);
sad_array[1] = hadd_uh_u32(sad1);
sad_array[2] = hadd_uh_u32(sad2);
sad_array[3] = hadd_uh_u32(sad3);
}
static void sad_16width_x4d_lsx(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *const aref_ptr[],
int32_t ref_stride, int32_t height,
uint32_t *sad_array) {
int32_t ht_cnt = (height >> 1);
const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr;
__m128i src, ref0, ref1, ref2, ref3, diff, sad_tmp;
__m128i sad0 = __lsx_vldi(0);
__m128i sad1 = sad0;
__m128i sad2 = sad0;
__m128i sad3 = sad0;
ref0_ptr = aref_ptr[0];
ref1_ptr = aref_ptr[1];
ref2_ptr = aref_ptr[2];
ref3_ptr = aref_ptr[3];
for (; ht_cnt--;) {
src = __lsx_vld(src_ptr, 0);
src_ptr += src_stride;
ref0 = __lsx_vld(ref0_ptr, 0);
ref0_ptr += ref_stride;
ref1 = __lsx_vld(ref1_ptr, 0);
ref1_ptr += ref_stride;
ref2 = __lsx_vld(ref2_ptr, 0);
ref2_ptr += ref_stride;
ref3 = __lsx_vld(ref3_ptr, 0);
ref3_ptr += ref_stride;
diff = __lsx_vabsd_bu(src, ref0);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
diff = __lsx_vabsd_bu(src, ref1);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
diff = __lsx_vabsd_bu(src, ref2);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad2 = __lsx_vadd_h(sad2, sad_tmp);
diff = __lsx_vabsd_bu(src, ref3);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad3 = __lsx_vadd_h(sad3, sad_tmp);
src = __lsx_vld(src_ptr, 0);
src_ptr += src_stride;
ref0 = __lsx_vld(ref0_ptr, 0);
ref0_ptr += ref_stride;
ref1 = __lsx_vld(ref1_ptr, 0);
ref1_ptr += ref_stride;
ref2 = __lsx_vld(ref2_ptr, 0);
ref2_ptr += ref_stride;
ref3 = __lsx_vld(ref3_ptr, 0);
ref3_ptr += ref_stride;
diff = __lsx_vabsd_bu(src, ref0);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
diff = __lsx_vabsd_bu(src, ref1);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
diff = __lsx_vabsd_bu(src, ref2);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad2 = __lsx_vadd_h(sad2, sad_tmp);
diff = __lsx_vabsd_bu(src, ref3);
sad_tmp = __lsx_vhaddw_hu_bu(diff, diff);
sad3 = __lsx_vadd_h(sad3, sad_tmp);
}
sad_array[0] = hadd_uh_u32(sad0);
sad_array[1] = hadd_uh_u32(sad1);
sad_array[2] = hadd_uh_u32(sad2);
sad_array[3] = hadd_uh_u32(sad3);
}
static void sad_32width_x4d_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *const aref_ptr[],
int32_t ref_stride, int32_t height,
uint32_t *sad_array) {
const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr;
int32_t ht_cnt = height;
__m128i src0, src1, ref0, ref1, sad_tmp;
__m128i sad0 = __lsx_vldi(0);
__m128i sad1 = sad0;
__m128i sad2 = sad0;
__m128i sad3 = sad0;
ref0_ptr = aref_ptr[0];
ref1_ptr = aref_ptr[1];
ref2_ptr = aref_ptr[2];
ref3_ptr = aref_ptr[3];
for (; ht_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, ref0_ptr, 0, ref0_ptr, 16, ref0, ref1);
ref0_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
DUP2_ARG2(__lsx_vld, ref1_ptr, 0, ref1_ptr, 16, ref0, ref1);
ref1_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
DUP2_ARG2(__lsx_vld, ref2_ptr, 0, ref2_ptr, 16, ref0, ref1);
ref2_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad2 = __lsx_vadd_h(sad2, sad_tmp);
DUP2_ARG2(__lsx_vld, ref3_ptr, 0, ref3_ptr, 16, ref0, ref1);
ref3_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad3 = __lsx_vadd_h(sad3, sad_tmp);
}
sad_array[0] = hadd_uh_u32(sad0);
sad_array[1] = hadd_uh_u32(sad1);
sad_array[2] = hadd_uh_u32(sad2);
sad_array[3] = hadd_uh_u32(sad3);
}
static void sad_64width_x4d_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *const aref_ptr[],
int32_t ref_stride, int32_t height,
uint32_t *sad_array) {
const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr;
int32_t ht_cnt = height;
__m128i src0, src1, src2, src3;
__m128i ref0, ref1, ref2, ref3;
__m128i sad, sad_tmp;
__m128i sad0_0 = __lsx_vldi(0);
__m128i sad0_1 = sad0_0;
__m128i sad1_0 = sad0_0;
__m128i sad1_1 = sad0_0;
__m128i sad2_0 = sad0_0;
__m128i sad2_1 = sad0_0;
__m128i sad3_0 = sad0_0;
__m128i sad3_1 = sad0_0;
ref0_ptr = aref_ptr[0];
ref1_ptr = aref_ptr[1];
ref2_ptr = aref_ptr[2];
ref3_ptr = aref_ptr[3];
for (; ht_cnt--;) {
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, ref0_ptr, 0, ref0_ptr, 16, ref0_ptr, 32, ref0_ptr, 48,
ref0, ref1, ref2, ref3);
ref0_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad0_0 = __lsx_vadd_h(sad0_0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, ref2, ref3);
sad0_1 = __lsx_vadd_h(sad0_1, sad_tmp);
DUP4_ARG2(__lsx_vld, ref1_ptr, 0, ref1_ptr, 16, ref1_ptr, 32, ref1_ptr, 48,
ref0, ref1, ref2, ref3);
ref1_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad1_0 = __lsx_vadd_h(sad1_0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, ref2, ref3);
sad1_1 = __lsx_vadd_h(sad1_1, sad_tmp);
DUP4_ARG2(__lsx_vld, ref2_ptr, 0, ref2_ptr, 16, ref2_ptr, 32, ref2_ptr, 48,
ref0, ref1, ref2, ref3);
ref2_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad2_0 = __lsx_vadd_h(sad2_0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, ref2, ref3);
sad2_1 = __lsx_vadd_h(sad2_1, sad_tmp);
DUP4_ARG2(__lsx_vld, ref3_ptr, 0, ref3_ptr, 16, ref3_ptr, 32, ref3_ptr, 48,
ref0, ref1, ref2, ref3);
ref3_ptr += ref_stride;
sad_tmp = sad_ub2_uh(src0, src1, ref0, ref1);
sad3_0 = __lsx_vadd_h(sad3_0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, ref2, ref3);
sad3_1 = __lsx_vadd_h(sad3_1, sad_tmp);
}
sad = __lsx_vhaddw_wu_hu(sad0_0, sad0_0);
sad_tmp = __lsx_vhaddw_wu_hu(sad0_1, sad0_1);
sad = __lsx_vadd_w(sad, sad_tmp);
sad_array[0] = hadd_uw_u32(sad);
sad = __lsx_vhaddw_wu_hu(sad1_0, sad1_0);
sad_tmp = __lsx_vhaddw_wu_hu(sad1_1, sad1_1);
sad = __lsx_vadd_w(sad, sad_tmp);
sad_array[1] = hadd_uw_u32(sad);
sad = __lsx_vhaddw_wu_hu(sad2_0, sad2_0);
sad_tmp = __lsx_vhaddw_wu_hu(sad2_1, sad2_1);
sad = __lsx_vadd_w(sad, sad_tmp);
sad_array[2] = hadd_uw_u32(sad);
sad = __lsx_vhaddw_wu_hu(sad3_0, sad3_0);
sad_tmp = __lsx_vhaddw_wu_hu(sad3_1, sad3_1);
sad = __lsx_vadd_w(sad, sad_tmp);
sad_array[3] = hadd_uw_u32(sad);
}
static uint32_t avgsad_32width_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
int32_t height, const uint8_t *sec_pred) {
int32_t res, ht_cnt = (height >> 2);
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
__m128i ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7;
__m128i pred0, pred1, pred2, pred3, pred4, pred5, pred6, pred7;
__m128i comp0, comp1, sad_tmp;
__m128i sad = __lsx_vldi(0);
uint8_t *src_tmp, *ref_tmp;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t ref_stride2 = ref_stride << 1;
int32_t ref_stride3 = ref_stride2 + ref_stride;
int32_t ref_stride4 = ref_stride2 << 1;
for (; ht_cnt--;) {
src_tmp = (uint8_t *)src + 16;
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4);
src6 = __lsx_vldx(src, src_stride3);
src1 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src3,
src5);
src7 = __lsx_vldx(src_tmp, src_stride3);
src += src_stride4;
ref_tmp = (uint8_t *)ref + 16;
ref0 = __lsx_vld(ref, 0);
DUP2_ARG2(__lsx_vldx, ref, ref_stride, ref, ref_stride2, ref2, ref4);
ref6 = __lsx_vldx(ref, ref_stride3);
ref1 = __lsx_vld(ref_tmp, 0);
DUP2_ARG2(__lsx_vldx, ref_tmp, ref_stride, ref_tmp, ref_stride2, ref3,
ref5);
ref7 = __lsx_vldx(ref_tmp, ref_stride3);
ref += ref_stride4;
DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 32, sec_pred, 64, sec_pred, 96,
pred0, pred2, pred4, pred6);
DUP4_ARG2(__lsx_vld, sec_pred, 16, sec_pred, 48, sec_pred, 80, sec_pred,
112, pred1, pred3, pred5, pred7);
sec_pred += 128;
DUP2_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, comp0, comp1);
sad_tmp = sad_ub2_uh(src0, src1, comp0, comp1);
sad = __lsx_vadd_h(sad, sad_tmp);
DUP2_ARG2(__lsx_vavgr_bu, pred2, ref2, pred3, ref3, comp0, comp1);
sad_tmp = sad_ub2_uh(src2, src3, comp0, comp1);
sad = __lsx_vadd_h(sad, sad_tmp);
DUP2_ARG2(__lsx_vavgr_bu, pred4, ref4, pred5, ref5, comp0, comp1);
sad_tmp = sad_ub2_uh(src4, src5, comp0, comp1);
sad = __lsx_vadd_h(sad, sad_tmp);
DUP2_ARG2(__lsx_vavgr_bu, pred6, ref6, pred7, ref7, comp0, comp1);
sad_tmp = sad_ub2_uh(src6, src7, comp0, comp1);
sad = __lsx_vadd_h(sad, sad_tmp);
}
res = hadd_uh_u32(sad);
return res;
}
static uint32_t avgsad_64width_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
int32_t height, const uint8_t *sec_pred) {
int32_t res, ht_cnt = (height >> 2);
__m128i src0, src1, src2, src3, ref0, ref1, ref2, ref3;
__m128i comp0, comp1, comp2, comp3, pred0, pred1, pred2, pred3;
__m128i sad, sad_tmp;
__m128i sad0 = __lsx_vldi(0);
__m128i sad1 = sad0;
for (; ht_cnt--;) {
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2,
ref3);
ref += ref_stride;
DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48,
pred0, pred1, pred2, pred3);
sec_pred += 64;
DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3,
ref3, comp0, comp1, comp2, comp3);
sad_tmp = sad_ub2_uh(src0, src1, comp0, comp1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, comp2, comp3);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2,
ref3);
ref += ref_stride;
DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48,
pred0, pred1, pred2, pred3);
sec_pred += 64;
DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3,
ref3, comp0, comp1, comp2, comp3);
sad_tmp = sad_ub2_uh(src0, src1, comp0, comp1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, comp2, comp3);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2,
ref3);
ref += ref_stride;
DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48,
pred0, pred1, pred2, pred3);
sec_pred += 64;
DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3,
ref3, comp0, comp1, comp2, comp3);
sad_tmp = sad_ub2_uh(src0, src1, comp0, comp1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, comp2, comp3);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2,
ref3);
ref += ref_stride;
DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48,
pred0, pred1, pred2, pred3);
sec_pred += 64;
DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3,
ref3, comp0, comp1, comp2, comp3);
sad_tmp = sad_ub2_uh(src0, src1, comp0, comp1);
sad0 = __lsx_vadd_h(sad0, sad_tmp);
sad_tmp = sad_ub2_uh(src2, src3, comp2, comp3);
sad1 = __lsx_vadd_h(sad1, sad_tmp);
}
sad = __lsx_vhaddw_wu_hu(sad0, sad0);
sad_tmp = __lsx_vhaddw_wu_hu(sad1, sad1);
sad = __lsx_vadd_w(sad, sad_tmp);
res = hadd_sw_s32(sad);
return res;
}
#define VPX_SAD_8xHT_LSX(height) \
uint32_t vpx_sad8x##height##_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *ref, int32_t ref_stride) { \
return sad_8width_lsx(src, src_stride, ref, ref_stride, height); \
}
#define VPX_SAD_16xHT_LSX(height) \
uint32_t vpx_sad16x##height##_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *ref, int32_t ref_stride) { \
return sad_16width_lsx(src, src_stride, ref, ref_stride, height); \
}
#define VPX_SAD_32xHT_LSX(height) \
uint32_t vpx_sad32x##height##_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *ref, int32_t ref_stride) { \
return sad_32width_lsx(src, src_stride, ref, ref_stride, height); \
}
#define VPX_SAD_64xHT_LSX(height) \
uint32_t vpx_sad64x##height##_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *ref, int32_t ref_stride) { \
return sad_64width_lsx(src, src_stride, ref, ref_stride, height); \
}
#define VPX_SAD_8xHTx4D_LSX(height) \
void vpx_sad8x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *const refs[4], \
int32_t ref_stride, uint32_t sads[4]) { \
sad_8width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \
}
#define VPX_SAD_16xHTx4D_LSX(height) \
void vpx_sad16x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *const refs[], \
int32_t ref_stride, uint32_t *sads) { \
sad_16width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \
}
#define VPX_SAD_32xHTx4D_LSX(height) \
void vpx_sad32x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *const refs[], \
int32_t ref_stride, uint32_t *sads) { \
sad_32width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \
}
#define VPX_SAD_64xHTx4D_LSX(height) \
void vpx_sad64x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \
const uint8_t *const refs[], \
int32_t ref_stride, uint32_t *sads) { \
sad_64width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \
}
#define VPX_AVGSAD_32xHT_LSX(height) \
uint32_t vpx_sad32x##height##_avg_lsx( \
const uint8_t *src, int32_t src_stride, const uint8_t *ref, \
int32_t ref_stride, const uint8_t *second_pred) { \
return avgsad_32width_lsx(src, src_stride, ref, ref_stride, height, \
second_pred); \
}
#define VPX_AVGSAD_64xHT_LSX(height) \
uint32_t vpx_sad64x##height##_avg_lsx( \
const uint8_t *src, int32_t src_stride, const uint8_t *ref, \
int32_t ref_stride, const uint8_t *second_pred) { \
return avgsad_64width_lsx(src, src_stride, ref, ref_stride, height, \
second_pred); \
}
#define SAD64 \
VPX_SAD_64xHT_LSX(64) VPX_SAD_64xHTx4D_LSX(64) VPX_SAD_64xHTx4D_LSX(32) \
VPX_AVGSAD_64xHT_LSX(64)
SAD64
#define SAD32 \
VPX_SAD_32xHT_LSX(32) VPX_SAD_32xHTx4D_LSX(32) VPX_SAD_32xHTx4D_LSX(64) \
VPX_AVGSAD_32xHT_LSX(32)
SAD32
#define SAD16 VPX_SAD_16xHT_LSX(16) VPX_SAD_16xHTx4D_LSX(16)
SAD16
#define SAD8 VPX_SAD_8xHT_LSX(8) VPX_SAD_8xHTx4D_LSX(8)
SAD8
#undef SAD64
#undef SAD32
#undef SAD16
#undef SAD8
libs/libvpx/vpx_dsp/loongarch/sub_pixel_variance_lsx.c
+874
View File
@@ -0,0 +1,874 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "./vpx_dsp_rtcd.h"
#include "vpx_ports/mem.h"
#include "vpx_dsp/loongarch/variance_lsx.h"
#include "vpx_dsp/variance.h"
static const uint8_t bilinear_filters_lsx[8][2] = {
{ 128, 0 }, { 112, 16 }, { 96, 32 }, { 80, 48 },
{ 64, 64 }, { 48, 80 }, { 32, 96 }, { 16, 112 },
};
#define VARIANCE_WxH(sse, diff, shift) \
(sse) - (((uint32_t)(diff) * (diff)) >> (shift))
#define VARIANCE_LARGE_WxH(sse, diff, shift) \
(sse) - (((int64_t)(diff) * (diff)) >> (shift))
static uint32_t avg_sse_diff_64x64_lsx(const uint8_t *src_ptr,
int32_t src_stride,
const uint8_t *ref_ptr,
int32_t ref_stride,
const uint8_t *sec_pred, int32_t *diff) {
int32_t res, ht_cnt = 32;
__m128i src0, src1, src2, src3, ref0, ref1, ref2, ref3;
__m128i pred0, pred1, pred2, pred3, vec, vec_tmp;
__m128i avg0, avg1, avg2, avg3;
__m128i var = __lsx_vldi(0);
avg0 = var;
avg1 = var;
avg2 = var;
avg3 = var;
for (; ht_cnt--;) {
DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48,
pred0, pred1, pred2, pred3);
sec_pred += 64;
DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48,
src0, src1, src2, src3);
src_ptr += src_stride;
DUP4_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref_ptr, 32, ref_ptr, 48,
ref0, ref1, ref2, ref3);
ref_ptr += ref_stride;
DUP4_ARG2(__lsx_vavgr_bu, src0, pred0, src1, pred1, src2, pred2, src3,
pred3, src0, src1, src2, src3);
CALC_MSE_AVG_B(src0, ref0, var, avg0);
CALC_MSE_AVG_B(src1, ref1, var, avg1);
CALC_MSE_AVG_B(src2, ref2, var, avg2);
CALC_MSE_AVG_B(src3, ref3, var, avg3);
DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48,
pred0, pred1, pred2, pred3);
sec_pred += 64;
DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48,
src0, src1, src2, src3);
src_ptr += src_stride;
DUP4_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref_ptr, 32, ref_ptr, 48,
ref0, ref1, ref2, ref3);
ref_ptr += ref_stride;
DUP4_ARG2(__lsx_vavgr_bu, src0, pred0, src1, pred1, src2, pred2, src3,
pred3, src0, src1, src2, src3);
CALC_MSE_AVG_B(src0, ref0, var, avg0);
CALC_MSE_AVG_B(src1, ref1, var, avg1);
CALC_MSE_AVG_B(src2, ref2, var, avg2);
CALC_MSE_AVG_B(src3, ref3, var, avg3);
}
vec = __lsx_vhaddw_w_h(avg0, avg0);
vec_tmp = __lsx_vhaddw_w_h(avg1, avg1);
vec = __lsx_vadd_w(vec, vec_tmp);
vec_tmp = __lsx_vhaddw_w_h(avg2, avg2);
vec = __lsx_vadd_w(vec, vec_tmp);
vec_tmp = __lsx_vhaddw_w_h(avg3, avg3);
vec = __lsx_vadd_w(vec, vec_tmp);
HADD_SW_S32(vec, *diff);
HADD_SW_S32(var, res);
return res;
}
static uint32_t sub_pixel_sse_diff_8width_h_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) {
uint32_t loop_cnt = (height >> 2);
int32_t res;
__m128i src0, src1, src2, src3, ref0, ref1, ref2, ref3;
__m128i vec0, vec1, vec2, vec3, filt0, out, vec;
__m128i mask = { 0x0403030202010100, 0x0807070606050504 };
__m128i avg = __lsx_vldi(0);
__m128i var = avg;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
int32_t dst_stride4 = dst_stride2 << 1;
filt0 = __lsx_vldrepl_h(filter, 0);
for (; loop_cnt--;) {
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src += src_stride4;
ref0 = __lsx_vld(dst, 0);
DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, ref1, ref2);
ref3 = __lsx_vldx(dst, dst_stride3);
dst += dst_stride4;
DUP2_ARG2(__lsx_vpickev_d, ref1, ref0, ref3, ref2, ref0, ref1);
DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1);
DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, vec0, vec1, vec2, vec3);
DUP4_ARG3(__lsx_vssrarni_bu_h, vec0, vec0, FILTER_BITS, vec1, vec1,
FILTER_BITS, vec2, vec2, FILTER_BITS, vec3, vec3, FILTER_BITS,
src0, src1, src2, src3);
out = __lsx_vpackev_d(src1, src0);
CALC_MSE_AVG_B(out, ref0, var, avg);
out = __lsx_vpackev_d(src3, src2);
CALC_MSE_AVG_B(out, ref1, var, avg);
}
vec = __lsx_vhaddw_w_h(avg, avg);
HADD_SW_S32(vec, *diff);
HADD_SW_S32(var, res);
return res;
}
static uint32_t sub_pixel_sse_diff_16width_h_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) {
uint32_t loop_cnt = (height >> 2);
int32_t res;
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
__m128i dst0, dst1, dst2, dst3, filt0;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
__m128i out0, out1, out2, out3, out4, out5, out6, out7;
__m128i vec, var = __lsx_vldi(0);
__m128i avg = var;
__m128i mask = { 0x0403030202010100, 0x0807070606050504 };
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
int32_t dst_stride4 = dst_stride2 << 1;
filt0 = __lsx_vldrepl_h(filter, 0);
for (; loop_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3);
src += src_stride;
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src4, src5);
src += src_stride;
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src6, src7);
src += src_stride;
dst0 = __lsx_vld(dst, 0);
DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2);
dst3 = __lsx_vldx(dst, dst_stride3);
dst += dst_stride4;
DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1);
DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3);
DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5);
DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, out0, out1, out2, out3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7,
filt0, out4, out5, out6, out7);
DUP4_ARG3(__lsx_vssrarni_bu_h, out1, out0, FILTER_BITS, out3, out2,
FILTER_BITS, out5, out4, FILTER_BITS, out7, out6, FILTER_BITS,
src0, src1, src2, src3);
CALC_MSE_AVG_B(src0, dst0, var, avg);
CALC_MSE_AVG_B(src1, dst1, var, avg);
CALC_MSE_AVG_B(src2, dst2, var, avg);
CALC_MSE_AVG_B(src3, dst3, var, avg);
}
vec = __lsx_vhaddw_w_h(avg, avg);
HADD_SW_S32(vec, *diff);
HADD_SW_S32(var, res);
return res;
}
static uint32_t sub_pixel_sse_diff_32width_h_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) {
uint32_t sse = 0;
int32_t diff0[2];
sse += sub_pixel_sse_diff_16width_h_lsx(src, src_stride, dst, dst_stride,
filter, height, &diff0[0]);
src += 16;
dst += 16;
sse += sub_pixel_sse_diff_16width_h_lsx(src, src_stride, dst, dst_stride,
filter, height, &diff0[1]);
*diff = diff0[0] + diff0[1];
return sse;
}
static uint32_t sub_pixel_sse_diff_8width_v_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) {
uint32_t loop_cnt = (height >> 2);
int32_t res;
__m128i ref0, ref1, ref2, ref3, src0, src1, src2, src3, src4;
__m128i vec, vec0, vec1, vec2, vec3, tmp0, tmp1, tmp2, tmp3, filt0;
__m128i avg = __lsx_vldi(0);
__m128i var = avg;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
int32_t dst_stride4 = dst_stride2 << 1;
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
src += src_stride;
for (; loop_cnt--;) {
src1 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3);
src4 = __lsx_vldx(src, src_stride3);
src += src_stride4;
ref0 = __lsx_vld(dst, 0);
DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, ref1, ref2);
ref3 = __lsx_vldx(dst, dst_stride3);
dst += dst_stride4;
DUP2_ARG2(__lsx_vpickev_d, ref1, ref0, ref3, ref2, ref0, ref1);
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3,
vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, tmp0, tmp1, tmp2, tmp3);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2,
FILTER_BITS, src0, src1);
CALC_MSE_AVG_B(src0, ref0, var, avg);
CALC_MSE_AVG_B(src1, ref1, var, avg);
src0 = src4;
}
vec = __lsx_vhaddw_w_h(avg, avg);
HADD_SW_S32(vec, *diff);
HADD_SW_S32(var, res);
return res;
}
static uint32_t sub_pixel_sse_diff_16width_v_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) {
uint32_t loop_cnt = (height >> 2);
int32_t res;
__m128i ref0, ref1, ref2, ref3, src0, src1, src2, src3, src4;
__m128i out0, out1, out2, out3, tmp0, tmp1, filt0, vec;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
__m128i var = __lsx_vldi(0);
__m128i avg = var;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
int32_t dst_stride4 = dst_stride2 << 1;
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
src += src_stride;
for (; loop_cnt--;) {
src1 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3);
src4 = __lsx_vldx(src, src_stride3);
src += src_stride4;
ref0 = __lsx_vld(dst, 0);
DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, ref1, ref2);
ref3 = __lsx_vldx(dst, dst_stride3);
dst += dst_stride4;
DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2);
DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1);
out0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6);
DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7);
DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1);
out1 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1);
out2 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1);
out3 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
src0 = src4;
CALC_MSE_AVG_B(out0, ref0, var, avg);
CALC_MSE_AVG_B(out1, ref1, var, avg);
CALC_MSE_AVG_B(out2, ref2, var, avg);
CALC_MSE_AVG_B(out3, ref3, var, avg);
}
vec = __lsx_vhaddw_w_h(avg, avg);
HADD_SW_S32(vec, *diff);
HADD_SW_S32(var, res);
return res;
}
static uint32_t sub_pixel_sse_diff_32width_v_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) {
uint32_t sse = 0;
int32_t diff0[2];
sse += sub_pixel_sse_diff_16width_v_lsx(src, src_stride, dst, dst_stride,
filter, height, &diff0[0]);
src += 16;
dst += 16;
sse += sub_pixel_sse_diff_16width_v_lsx(src, src_stride, dst, dst_stride,
filter, height, &diff0[1]);
*diff = diff0[0] + diff0[1];
return sse;
}
static uint32_t sub_pixel_sse_diff_8width_hv_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *filter_horiz, const uint8_t *filter_vert,
int32_t height, int32_t *diff) {
uint32_t loop_cnt = (height >> 2);
int32_t res;
__m128i ref0, ref1, ref2, ref3, src0, src1, src2, src3, src4, out0, out1;
__m128i hz_out0, hz_out1, tmp0, tmp1, tmp2, tmp3, vec, vec0, filt_hz, filt_vt;
__m128i mask = { 0x0403030202010100, 0x0807070606050504 };
__m128i avg = __lsx_vldi(0);
__m128i var = avg;
filt_hz = __lsx_vldrepl_h(filter_horiz, 0);
filt_vt = __lsx_vldrepl_h(filter_vert, 0);
src0 = __lsx_vld(src, 0);
src += src_stride;
HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS, hz_out0);
for (; loop_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, dst, 0, src1, ref0);
src += src_stride;
dst += dst_stride;
DUP2_ARG2(__lsx_vld, src, 0, dst, 0, src2, ref1);
src += src_stride;
dst += dst_stride;
DUP2_ARG2(__lsx_vld, src, 0, dst, 0, src3, ref2);
src += src_stride;
dst += dst_stride;
DUP2_ARG2(__lsx_vld, src, 0, dst, 0, src4, ref3);
src += src_stride;
dst += dst_stride;
DUP2_ARG2(__lsx_vpickev_d, ref1, ref0, ref3, ref2, ref0, ref1);
HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS, hz_out1);
vec0 = __lsx_vpackev_b(hz_out1, hz_out0);
tmp0 = __lsx_vdp2_h_bu(vec0, filt_vt);
HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS, hz_out0);
vec0 = __lsx_vpackev_b(hz_out0, hz_out1);
tmp1 = __lsx_vdp2_h_bu(vec0, filt_vt);
HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS, hz_out1);
vec0 = __lsx_vpackev_b(hz_out1, hz_out0);
tmp2 = __lsx_vdp2_h_bu(vec0, filt_vt);
HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS, hz_out0);
vec0 = __lsx_vpackev_b(hz_out0, hz_out1);
tmp3 = __lsx_vdp2_h_bu(vec0, filt_vt);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2,
FILTER_BITS, out0, out1);
CALC_MSE_AVG_B(out0, ref0, var, avg);
CALC_MSE_AVG_B(out1, ref1, var, avg);
}
vec = __lsx_vhaddw_w_h(avg, avg);
HADD_SW_S32(vec, *diff);
HADD_SW_S32(var, res);
return res;
}
static uint32_t sub_pixel_sse_diff_16width_hv_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *filter_horiz, const uint8_t *filter_vert,
int32_t height, int32_t *diff) {
uint32_t loop_cnt = (height >> 2);
int32_t res;
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
__m128i ref0, ref1, ref2, ref3, filt_hz, filt_vt, vec0, vec1;
__m128i hz_out0, hz_out1, hz_out2, hz_out3, tmp0, tmp1, vec;
__m128i var = __lsx_vldi(0);
__m128i avg = var;
__m128i mask = { 0x0403030202010100, 0x0807070606050504 };
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
int32_t dst_stride4 = dst_stride2 << 1;
filt_hz = __lsx_vldrepl_h(filter_horiz, 0);
filt_vt = __lsx_vldrepl_h(filter_vert, 0);
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1);
src += src_stride;
HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS, hz_out0);
HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS, hz_out2);
for (; loop_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3);
src += src_stride;
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src4, src5);
src += src_stride;
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src6, src7);
src += src_stride;
ref0 = __lsx_vld(dst, 0);
DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, ref1, ref2);
ref3 = __lsx_vldx(dst, dst_stride3);
dst += dst_stride4;
HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS, hz_out1);
HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS, hz_out3);
DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1);
src0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS, hz_out0);
HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS, hz_out2);
DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1);
src1 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS, hz_out1);
HORIZ_2TAP_FILT_UH(src5, src5, mask, filt_hz, FILTER_BITS, hz_out3);
DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1);
src2 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
HORIZ_2TAP_FILT_UH(src6, src6, mask, filt_hz, FILTER_BITS, hz_out0);
HORIZ_2TAP_FILT_UH(src7, src7, mask, filt_hz, FILTER_BITS, hz_out2);
DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1);
src3 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
CALC_MSE_AVG_B(src0, ref0, var, avg);
CALC_MSE_AVG_B(src1, ref1, var, avg);
CALC_MSE_AVG_B(src2, ref2, var, avg);
CALC_MSE_AVG_B(src3, ref3, var, avg);
}
vec = __lsx_vhaddw_w_h(avg, avg);
HADD_SW_S32(vec, *diff);
HADD_SW_S32(var, res);
return res;
}
static uint32_t sub_pixel_sse_diff_32width_hv_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *filter_horiz, const uint8_t *filter_vert,
int32_t height, int32_t *diff) {
uint32_t sse = 0;
int32_t diff0[2];
sse += sub_pixel_sse_diff_16width_hv_lsx(src, src_stride, dst, dst_stride,
filter_horiz, filter_vert, height,
&diff0[0]);
src += 16;
dst += 16;
sse += sub_pixel_sse_diff_16width_hv_lsx(src, src_stride, dst, dst_stride,
filter_horiz, filter_vert, height,
&diff0[1]);
*diff = diff0[0] + diff0[1];
return sse;
}
static uint32_t subpel_avg_ssediff_16w_h_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter,
int32_t height, int32_t *diff, int32_t width) {
uint32_t loop_cnt = (height >> 2);
int32_t res;
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
__m128i dst0, dst1, dst2, dst3, tmp0, tmp1, tmp2, tmp3;
__m128i pred0, pred1, pred2, pred3, filt0, vec;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
__m128i out0, out1, out2, out3, out4, out5, out6, out7;
__m128i mask = { 0x403030202010100, 0x807070606050504 };
__m128i avg = __lsx_vldi(0);
__m128i var = avg;
filt0 = __lsx_vldrepl_h(filter, 0);
for (; loop_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3);
src += src_stride;
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src4, src5);
src += src_stride;
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src6, src7);
src += src_stride;
dst0 = __lsx_vld(dst, 0);
dst += dst_stride;
dst1 = __lsx_vld(dst, 0);
dst += dst_stride;
dst2 = __lsx_vld(dst, 0);
dst += dst_stride;
dst3 = __lsx_vld(dst, 0);
dst += dst_stride;
pred0 = __lsx_vld(sec_pred, 0);
sec_pred += width;
pred1 = __lsx_vld(sec_pred, 0);
sec_pred += width;
pred2 = __lsx_vld(sec_pred, 0);
sec_pred += width;
pred3 = __lsx_vld(sec_pred, 0);
sec_pred += width;
DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1);
DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3);
DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5);
DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, out0, out1, out2, out3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7,
filt0, out4, out5, out6, out7);
DUP4_ARG3(__lsx_vssrarni_bu_h, out1, out0, FILTER_BITS, out3, out2,
FILTER_BITS, out5, out4, FILTER_BITS, out7, out6, FILTER_BITS,
tmp0, tmp1, tmp2, tmp3);
DUP4_ARG2(__lsx_vavgr_bu, tmp0, pred0, tmp1, pred1, tmp2, pred2, tmp3,
pred3, tmp0, tmp1, tmp2, tmp3);
CALC_MSE_AVG_B(tmp0, dst0, var, avg);
CALC_MSE_AVG_B(tmp1, dst1, var, avg);
CALC_MSE_AVG_B(tmp2, dst2, var, avg);
CALC_MSE_AVG_B(tmp3, dst3, var, avg);
}
vec = __lsx_vhaddw_w_h(avg, avg);
HADD_SW_S32(vec, *diff);
HADD_SW_S32(var, res);
return res;
}
static uint32_t subpel_avg_ssediff_16w_v_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter,
int32_t height, int32_t *diff, int32_t width) {
uint32_t loop_cnt = (height >> 2);
int32_t res;
__m128i ref0, ref1, ref2, ref3, pred0, pred1, pred2, pred3;
__m128i src0, src1, src2, src3, src4, out0, out1, out2, out3;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
__m128i tmp0, tmp1, vec, filt0;
__m128i avg = __lsx_vldi(0);
__m128i var = avg;
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
src += src_stride;
for (; loop_cnt--;) {
src1 = __lsx_vld(src, 0);
src += src_stride;
src2 = __lsx_vld(src, 0);
src += src_stride;
src3 = __lsx_vld(src, 0);
src += src_stride;
src4 = __lsx_vld(src, 0);
src += src_stride;
pred0 = __lsx_vld(sec_pred, 0);
sec_pred += width;
pred1 = __lsx_vld(sec_pred, 0);
sec_pred += width;
pred2 = __lsx_vld(sec_pred, 0);
sec_pred += width;
pred3 = __lsx_vld(sec_pred, 0);
sec_pred += width;
DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2);
DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1);
out0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6);
DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7);
DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1);
out1 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1);
out2 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1);
out3 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
src0 = src4;
ref0 = __lsx_vld(dst, 0);
dst += dst_stride;
ref1 = __lsx_vld(dst, 0);
dst += dst_stride;
ref2 = __lsx_vld(dst, 0);
dst += dst_stride;
ref3 = __lsx_vld(dst, 0);
dst += dst_stride;
DUP4_ARG2(__lsx_vavgr_bu, out0, pred0, out1, pred1, out2, pred2, out3,
pred3, out0, out1, out2, out3);
CALC_MSE_AVG_B(out0, ref0, var, avg);
CALC_MSE_AVG_B(out1, ref1, var, avg);
CALC_MSE_AVG_B(out2, ref2, var, avg);
CALC_MSE_AVG_B(out3, ref3, var, avg);
}
vec = __lsx_vhaddw_w_h(avg, avg);
HADD_SW_S32(vec, *diff);
HADD_SW_S32(var, res);
return res;
}
static uint32_t subpel_avg_ssediff_16w_hv_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter_horiz,
const uint8_t *filter_vert, int32_t height, int32_t *diff, int32_t width) {
uint32_t loop_cnt = (height >> 2);
int32_t res;
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
__m128i ref0, ref1, ref2, ref3, pred0, pred1, pred2, pred3;
__m128i hz_out0, hz_out1, hz_out2, hz_out3, tmp0, tmp1;
__m128i out0, out1, out2, out3, filt_hz, filt_vt, vec, vec0, vec1;
__m128i mask = { 0x403030202010100, 0x807070606050504 };
__m128i avg = __lsx_vldi(0);
__m128i var = avg;
filt_hz = __lsx_vldrepl_h(filter_horiz, 0);
filt_vt = __lsx_vldrepl_h(filter_vert, 0);
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1);
src += src_stride;
HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS, hz_out0);
HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS, hz_out2);
for (; loop_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3);
src += src_stride;
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src4, src5);
src += src_stride;
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src6, src7);
src += src_stride;
pred0 = __lsx_vld(sec_pred, 0);
sec_pred += width;
pred1 = __lsx_vld(sec_pred, 0);
sec_pred += width;
pred2 = __lsx_vld(sec_pred, 0);
sec_pred += width;
pred3 = __lsx_vld(sec_pred, 0);
sec_pred += width;
HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS, hz_out1);
HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS, hz_out3);
DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1);
out0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS, hz_out0);
HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS, hz_out2);
DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1);
out1 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS, hz_out1);
HORIZ_2TAP_FILT_UH(src5, src5, mask, filt_hz, FILTER_BITS, hz_out3);
DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1);
out2 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
HORIZ_2TAP_FILT_UH(src6, src6, mask, filt_hz, FILTER_BITS, hz_out0);
HORIZ_2TAP_FILT_UH(src7, src7, mask, filt_hz, FILTER_BITS, hz_out2);
DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1);
out3 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
ref0 = __lsx_vld(dst, 0);
dst += dst_stride;
ref1 = __lsx_vld(dst, 0);
dst += dst_stride;
ref2 = __lsx_vld(dst, 0);
dst += dst_stride;
ref3 = __lsx_vld(dst, 0);
dst += dst_stride;
DUP4_ARG2(__lsx_vavgr_bu, out0, pred0, out1, pred1, out2, pred2, out3,
pred3, out0, out1, out2, out3);
CALC_MSE_AVG_B(out0, ref0, var, avg);
CALC_MSE_AVG_B(out1, ref1, var, avg);
CALC_MSE_AVG_B(out2, ref2, var, avg);
CALC_MSE_AVG_B(out3, ref3, var, avg);
}
vec = __lsx_vhaddw_w_h(avg, avg);
HADD_SW_S32(vec, *diff);
HADD_SW_S32(var, res);
return res;
}
static uint32_t sub_pixel_avg_sse_diff_64width_h_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter,
int32_t height, int32_t *diff) {
uint32_t loop_cnt, sse = 0;
int32_t diff0[4];
for (loop_cnt = 0; loop_cnt < 4; ++loop_cnt) {
sse +=
subpel_avg_ssediff_16w_h_lsx(src, src_stride, dst, dst_stride, sec_pred,
filter, height, &diff0[loop_cnt], 64);
src += 16;
dst += 16;
sec_pred += 16;
}
*diff = diff0[0] + diff0[1] + diff0[2] + diff0[3];
return sse;
}
static uint32_t sub_pixel_avg_sse_diff_64width_v_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter,
int32_t height, int32_t *diff) {
uint32_t loop_cnt, sse = 0;
int32_t diff0[4];
for (loop_cnt = 0; loop_cnt < 4; ++loop_cnt) {
sse +=
subpel_avg_ssediff_16w_v_lsx(src, src_stride, dst, dst_stride, sec_pred,
filter, height, &diff0[loop_cnt], 64);
src += 16;
dst += 16;
sec_pred += 16;
}
*diff = diff0[0] + diff0[1] + diff0[2] + diff0[3];
return sse;
}
static uint32_t sub_pixel_avg_sse_diff_64width_hv_lsx(
const uint8_t *src, int32_t src_stride, const uint8_t *dst,
int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter_horiz,
const uint8_t *filter_vert, int32_t height, int32_t *diff) {
uint32_t loop_cnt, sse = 0;
int32_t diff0[4];
for (loop_cnt = 0; loop_cnt < 4; ++loop_cnt) {
sse += subpel_avg_ssediff_16w_hv_lsx(src, src_stride, dst, dst_stride,
sec_pred, filter_horiz, filter_vert,
height, &diff0[loop_cnt], 64);
src += 16;
dst += 16;
sec_pred += 16;
}
*diff = diff0[0] + diff0[1] + diff0[2] + diff0[3];
return sse;
}
#define VARIANCE_8Wx8H(sse, diff) VARIANCE_WxH(sse, diff, 6)
#define VARIANCE_16Wx16H(sse, diff) VARIANCE_WxH(sse, diff, 8)
#define VARIANCE_32Wx32H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 10)
#define VARIANCE_64Wx64H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 12)
#define VPX_SUB_PIXEL_VARIANCE_WDXHT_LSX(wd, ht) \
uint32_t vpx_sub_pixel_variance##wd##x##ht##_lsx( \
const uint8_t *src, int32_t src_stride, int32_t x_offset, \
int32_t y_offset, const uint8_t *ref, int32_t ref_stride, \
uint32_t *sse) { \
int32_t diff; \
uint32_t var; \
const uint8_t *h_filter = bilinear_filters_lsx[x_offset]; \
const uint8_t *v_filter = bilinear_filters_lsx[y_offset]; \
\
if (y_offset) { \
if (x_offset) { \
*sse = sub_pixel_sse_diff_##wd##width_hv_lsx( \
src, src_stride, ref, ref_stride, h_filter, v_filter, ht, &diff); \
} else { \
*sse = sub_pixel_sse_diff_##wd##width_v_lsx( \
src, src_stride, ref, ref_stride, v_filter, ht, &diff); \
} \
\
var = VARIANCE_##wd##Wx##ht##H(*sse, diff); \
} else { \
if (x_offset) { \
*sse = sub_pixel_sse_diff_##wd##width_h_lsx( \
src, src_stride, ref, ref_stride, h_filter, ht, &diff); \
\
var = VARIANCE_##wd##Wx##ht##H(*sse, diff); \
} else { \
var = vpx_variance##wd##x##ht##_lsx(src, src_stride, ref, ref_stride, \
sse); \
} \
} \
\
return var; \
}
VPX_SUB_PIXEL_VARIANCE_WDXHT_LSX(8, 8)
VPX_SUB_PIXEL_VARIANCE_WDXHT_LSX(16, 16)
VPX_SUB_PIXEL_VARIANCE_WDXHT_LSX(32, 32)
#define VPX_SUB_PIXEL_AVG_VARIANCE64XHEIGHT_LSX(ht) \
uint32_t vpx_sub_pixel_avg_variance64x##ht##_lsx( \
const uint8_t *src_ptr, int32_t src_stride, int32_t x_offset, \
int32_t y_offset, const uint8_t *ref_ptr, int32_t ref_stride, \
uint32_t *sse, const uint8_t *sec_pred) { \
int32_t diff; \
const uint8_t *h_filter = bilinear_filters_lsx[x_offset]; \
const uint8_t *v_filter = bilinear_filters_lsx[y_offset]; \
\
if (y_offset) { \
if (x_offset) { \
*sse = sub_pixel_avg_sse_diff_64width_hv_lsx( \
src_ptr, src_stride, ref_ptr, ref_stride, sec_pred, h_filter, \
v_filter, ht, &diff); \
} else { \
*sse = sub_pixel_avg_sse_diff_64width_v_lsx( \
src_ptr, src_stride, ref_ptr, ref_stride, sec_pred, v_filter, ht, \
&diff); \
} \
} else { \
if (x_offset) { \
*sse = sub_pixel_avg_sse_diff_64width_h_lsx( \
src_ptr, src_stride, ref_ptr, ref_stride, sec_pred, h_filter, ht, \
&diff); \
} else { \
*sse = avg_sse_diff_64x##ht##_lsx(src_ptr, src_stride, ref_ptr, \
ref_stride, sec_pred, &diff); \
} \
} \
\
return VARIANCE_64Wx##ht##H(*sse, diff); \
}
VPX_SUB_PIXEL_AVG_VARIANCE64XHEIGHT_LSX(64)
libs/libvpx/vpx_dsp/loongarch/subtract_lsx.c
+371
View File
@@ -0,0 +1,371 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "./vpx_dsp_rtcd.h"
#include "vpx_util/loongson_intrinsics.h"
static void sub_blk_4x4_lsx(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *pred_ptr, int32_t pred_stride,
int16_t *diff_ptr, int32_t diff_stride) {
__m128i src0, src1, src2, src3;
__m128i pred0, pred1, pred2, pred3;
__m128i diff0, diff1;
__m128i reg0, reg1;
int32_t src_stride2 = src_stride << 1;
int32_t pred_stride2 = pred_stride << 1;
int32_t diff_stride2 = diff_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t pred_stride3 = pred_stride2 + pred_stride;
int32_t diff_stride3 = diff_stride2 + diff_stride;
DUP4_ARG2(__lsx_vldrepl_w, src_ptr, 0, src_ptr + src_stride, 0,
src_ptr + src_stride2, 0, src_ptr + src_stride3, 0, src0, src1,
src2, src3);
DUP4_ARG2(__lsx_vldrepl_w, pred_ptr, 0, pred_ptr + pred_stride, 0,
pred_ptr + pred_stride2, 0, pred_ptr + pred_stride3, 0, pred0,
pred1, pred2, pred3);
DUP4_ARG2(__lsx_vilvl_w, src1, src0, src3, src2, pred1, pred0, pred3, pred2,
src0, src2, pred0, pred2);
DUP2_ARG2(__lsx_vilvl_d, src2, src0, pred2, pred0, src0, pred0);
reg0 = __lsx_vilvl_b(src0, pred0);
reg1 = __lsx_vilvh_b(src0, pred0);
DUP2_ARG2(__lsx_vhsubw_hu_bu, reg0, reg0, reg1, reg1, diff0, diff1);
__lsx_vstelm_d(diff0, diff_ptr, 0, 0);
__lsx_vstelm_d(diff0, diff_ptr + diff_stride, 0, 1);
__lsx_vstelm_d(diff1, diff_ptr + diff_stride2, 0, 0);
__lsx_vstelm_d(diff1, diff_ptr + diff_stride3, 0, 1);
}
static void sub_blk_8x8_lsx(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *pred_ptr, int32_t pred_stride,
int16_t *diff_ptr, int32_t diff_stride) {
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
__m128i pred0, pred1, pred2, pred3, pred4, pred5, pred6, pred7;
__m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
int32_t src_stride2 = src_stride << 1;
int32_t pred_stride2 = pred_stride << 1;
int32_t dst_stride = diff_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t pred_stride3 = pred_stride2 + pred_stride;
int32_t dst_stride2 = dst_stride << 1;
int32_t src_stride4 = src_stride2 << 1;
int32_t pred_stride4 = pred_stride2 << 1;
int32_t dst_stride3 = dst_stride + dst_stride2;
DUP4_ARG2(__lsx_vldrepl_d, src_ptr, 0, src_ptr + src_stride, 0,
src_ptr + src_stride2, 0, src_ptr + src_stride3, 0, src0, src1,
src2, src3);
DUP4_ARG2(__lsx_vldrepl_d, pred_ptr, 0, pred_ptr + pred_stride, 0,
pred_ptr + pred_stride2, 0, pred_ptr + pred_stride3, 0, pred0,
pred1, pred2, pred3);
src_ptr += src_stride4;
pred_ptr += pred_stride4;
DUP4_ARG2(__lsx_vldrepl_d, src_ptr, 0, src_ptr + src_stride, 0,
src_ptr + src_stride2, 0, src_ptr + src_stride3, 0, src4, src5,
src6, src7);
DUP4_ARG2(__lsx_vldrepl_d, pred_ptr, 0, pred_ptr + pred_stride, 0,
pred_ptr + pred_stride2, 0, pred_ptr + pred_stride3, 0, pred4,
pred5, pred6, pred7);
DUP4_ARG2(__lsx_vilvl_b, src0, pred0, src1, pred1, src2, pred2, src3, pred3,
reg0, reg1, reg2, reg3);
DUP4_ARG2(__lsx_vilvl_b, src4, pred4, src5, pred5, src6, pred6, src7, pred7,
reg4, reg5, reg6, reg7);
DUP4_ARG2(__lsx_vhsubw_hu_bu, reg0, reg0, reg1, reg1, reg2, reg2, reg3, reg3,
src0, src1, src2, src3);
DUP4_ARG2(__lsx_vhsubw_hu_bu, reg4, reg4, reg5, reg5, reg6, reg6, reg7, reg7,
src4, src5, src6, src7);
__lsx_vst(src0, diff_ptr, 0);
__lsx_vstx(src1, diff_ptr, dst_stride);
__lsx_vstx(src2, diff_ptr, dst_stride2);
__lsx_vstx(src3, diff_ptr, dst_stride3);
diff_ptr += dst_stride2;
__lsx_vst(src4, diff_ptr, 0);
__lsx_vstx(src5, diff_ptr, dst_stride);
__lsx_vstx(src6, diff_ptr, dst_stride2);
__lsx_vstx(src7, diff_ptr, dst_stride3);
}
static void sub_blk_16x16_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *pred, int32_t pred_stride,
int16_t *diff, int32_t diff_stride) {
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
__m128i pred0, pred1, pred2, pred3, pred4, pred5, pred6, pred7;
__m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
int32_t src_stride2 = src_stride << 1;
int32_t pred_stride2 = pred_stride << 1;
int32_t dst_stride = diff_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t pred_stride3 = pred_stride2 + pred_stride;
int32_t dst_stride2 = dst_stride << 1;
int32_t src_stride4 = src_stride2 << 1;
int32_t pred_stride4 = pred_stride2 << 1;
int32_t dst_stride3 = dst_stride + dst_stride2;
int16_t *diff_tmp = diff + 8;
DUP2_ARG2(__lsx_vld, src, 0, pred, 0, src0, pred0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
DUP4_ARG2(__lsx_vldx, pred, pred_stride, pred, pred_stride2, pred,
pred_stride3, pred, pred_stride4, pred1, pred2, pred3, pred4);
src += src_stride4;
pred += pred_stride4;
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
pred, pred_stride, src5, src6, src7, pred5);
DUP2_ARG2(__lsx_vldx, pred, pred_stride2, pred, pred_stride3, pred6, pred7);
src += src_stride4;
pred += pred_stride4;
DUP4_ARG2(__lsx_vilvl_b, src0, pred0, src1, pred1, src2, pred2, src3, pred3,
reg0, reg2, reg4, reg6);
DUP4_ARG2(__lsx_vilvh_b, src0, pred0, src1, pred1, src2, pred2, src3, pred3,
reg1, reg3, reg5, reg7);
DUP4_ARG2(__lsx_vilvl_b, src4, pred4, src5, pred5, src6, pred6, src7, pred7,
tmp0, tmp2, tmp4, tmp6);
DUP4_ARG2(__lsx_vilvh_b, src4, pred4, src5, pred5, src6, pred6, src7, pred7,
tmp1, tmp3, tmp5, tmp7);
DUP4_ARG2(__lsx_vhsubw_hu_bu, reg0, reg0, reg1, reg1, reg2, reg2, reg3, reg3,
src0, src1, src2, src3);
DUP4_ARG2(__lsx_vhsubw_hu_bu, reg4, reg4, reg5, reg5, reg6, reg6, reg7, reg7,
src4, src5, src6, src7);
DUP4_ARG2(__lsx_vhsubw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp2, tmp2, tmp3, tmp3,
pred0, pred1, pred2, pred3);
DUP4_ARG2(__lsx_vhsubw_hu_bu, tmp4, tmp4, tmp5, tmp5, tmp6, tmp6, tmp7, tmp7,
pred4, pred5, pred6, pred7);
__lsx_vst(src0, diff, 0);
__lsx_vstx(src2, diff, dst_stride);
__lsx_vstx(src4, diff, dst_stride2);
__lsx_vstx(src6, diff, dst_stride3);
__lsx_vst(src1, diff_tmp, 0);
__lsx_vstx(src3, diff_tmp, dst_stride);
__lsx_vstx(src5, diff_tmp, dst_stride2);
__lsx_vstx(src7, diff_tmp, dst_stride3);
diff += dst_stride2;
diff_tmp += dst_stride2;
__lsx_vst(pred0, diff, 0);
__lsx_vstx(pred2, diff, dst_stride);
__lsx_vstx(pred4, diff, dst_stride2);
__lsx_vstx(pred6, diff, dst_stride3);
__lsx_vst(pred1, diff_tmp, 0);
__lsx_vstx(pred3, diff_tmp, dst_stride);
__lsx_vstx(pred5, diff_tmp, dst_stride2);
__lsx_vstx(pred7, diff_tmp, dst_stride3);
diff += dst_stride2;
diff_tmp += dst_stride2;
DUP2_ARG2(__lsx_vld, src, 0, pred, 0, src0, pred0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
DUP4_ARG2(__lsx_vldx, pred, pred_stride, pred, pred_stride2, pred,
pred_stride3, pred, pred_stride4, pred1, pred2, pred3, pred4);
src += src_stride4;
pred += pred_stride4;
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
pred, pred_stride, src5, src6, src7, pred5);
DUP2_ARG2(__lsx_vldx, pred, pred_stride2, pred, pred_stride3, pred6, pred7);
DUP4_ARG2(__lsx_vilvl_b, src0, pred0, src1, pred1, src2, pred2, src3, pred3,
reg0, reg2, reg4, reg6);
DUP4_ARG2(__lsx_vilvh_b, src0, pred0, src1, pred1, src2, pred2, src3, pred3,
reg1, reg3, reg5, reg7);
DUP4_ARG2(__lsx_vilvl_b, src4, pred4, src5, pred5, src6, pred6, src7, pred7,
tmp0, tmp2, tmp4, tmp6);
DUP4_ARG2(__lsx_vilvh_b, src4, pred4, src5, pred5, src6, pred6, src7, pred7,
tmp1, tmp3, tmp5, tmp7);
DUP4_ARG2(__lsx_vhsubw_hu_bu, reg0, reg0, reg1, reg1, reg2, reg2, reg3, reg3,
src0, src1, src2, src3);
DUP4_ARG2(__lsx_vhsubw_hu_bu, reg4, reg4, reg5, reg5, reg6, reg6, reg7, reg7,
src4, src5, src6, src7);
DUP4_ARG2(__lsx_vhsubw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp2, tmp2, tmp3, tmp3,
pred0, pred1, pred2, pred3);
DUP4_ARG2(__lsx_vhsubw_hu_bu, tmp4, tmp4, tmp5, tmp5, tmp6, tmp6, tmp7, tmp7,
pred4, pred5, pred6, pred7);
__lsx_vst(src0, diff, 0);
__lsx_vstx(src2, diff, dst_stride);
__lsx_vstx(src4, diff, dst_stride2);
__lsx_vstx(src6, diff, dst_stride3);
__lsx_vst(src1, diff_tmp, 0);
__lsx_vstx(src3, diff_tmp, dst_stride);
__lsx_vstx(src5, diff_tmp, dst_stride2);
__lsx_vstx(src7, diff_tmp, dst_stride3);
diff += dst_stride2;
diff_tmp += dst_stride2;
__lsx_vst(pred0, diff, 0);
__lsx_vstx(pred2, diff, dst_stride);
__lsx_vstx(pred4, diff, dst_stride2);
__lsx_vstx(pred6, diff, dst_stride3);
__lsx_vst(pred1, diff_tmp, 0);
__lsx_vstx(pred3, diff_tmp, dst_stride);
__lsx_vstx(pred5, diff_tmp, dst_stride2);
__lsx_vstx(pred7, diff_tmp, dst_stride3);
}
static void sub_blk_32x32_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *pred, int32_t pred_stride,
int16_t *diff, int32_t diff_stride) {
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
__m128i pred0, pred1, pred2, pred3, pred4, pred5, pred6, pred7;
__m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
uint32_t loop_cnt;
int32_t src_stride2 = src_stride << 1;
int32_t pred_stride2 = pred_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t pred_stride3 = pred_stride2 + pred_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t pred_stride4 = pred_stride2 << 1;
for (loop_cnt = 8; loop_cnt--;) {
const uint8_t *src_tmp = src + 16;
const uint8_t *pred_tmp = pred + 16;
DUP4_ARG2(__lsx_vld, src, 0, src_tmp, 0, pred, 0, pred_tmp, 0, src0, src1,
pred0, pred1);
DUP4_ARG2(__lsx_vldx, src, src_stride, src_tmp, src_stride, src,
src_stride2, src_tmp, src_stride2, src2, src3, src4, src5);
DUP4_ARG2(__lsx_vldx, src, src_stride3, src_tmp, src_stride3, pred,
pred_stride, pred_tmp, pred_stride, src6, src7, pred2, pred3);
DUP4_ARG2(__lsx_vldx, pred, pred_stride2, pred_tmp, pred_stride2, pred,
pred_stride3, pred_tmp, pred_stride3, pred4, pred5, pred6, pred7);
DUP4_ARG2(__lsx_vilvl_b, src0, pred0, src1, pred1, src2, pred2, src3, pred3,
reg0, reg2, reg4, reg6);
DUP4_ARG2(__lsx_vilvh_b, src0, pred0, src1, pred1, src2, pred2, src3, pred3,
reg1, reg3, reg5, reg7);
DUP4_ARG2(__lsx_vilvl_b, src4, pred4, src5, pred5, src6, pred6, src7, pred7,
tmp0, tmp2, tmp4, tmp6);
DUP4_ARG2(__lsx_vilvh_b, src4, pred4, src5, pred5, src6, pred6, src7, pred7,
tmp1, tmp3, tmp5, tmp7);
DUP4_ARG2(__lsx_vhsubw_hu_bu, reg0, reg0, reg1, reg1, reg2, reg2, reg3,
reg3, src0, src1, src2, src3);
DUP4_ARG2(__lsx_vhsubw_hu_bu, reg4, reg4, reg5, reg5, reg6, reg6, reg7,
reg7, src4, src5, src6, src7);
DUP4_ARG2(__lsx_vhsubw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp2, tmp2, tmp3,
tmp3, pred0, pred1, pred2, pred3);
DUP4_ARG2(__lsx_vhsubw_hu_bu, tmp4, tmp4, tmp5, tmp5, tmp6, tmp6, tmp7,
tmp7, pred4, pred5, pred6, pred7);
src += src_stride4;
pred += pred_stride4;
__lsx_vst(src0, diff, 0);
__lsx_vst(src1, diff, 16);
__lsx_vst(src2, diff, 32);
__lsx_vst(src3, diff, 48);
diff += diff_stride;
__lsx_vst(src4, diff, 0);
__lsx_vst(src5, diff, 16);
__lsx_vst(src6, diff, 32);
__lsx_vst(src7, diff, 48);
diff += diff_stride;
__lsx_vst(pred0, diff, 0);
__lsx_vst(pred1, diff, 16);
__lsx_vst(pred2, diff, 32);
__lsx_vst(pred3, diff, 48);
diff += diff_stride;
__lsx_vst(pred4, diff, 0);
__lsx_vst(pred5, diff, 16);
__lsx_vst(pred6, diff, 32);
__lsx_vst(pred7, diff, 48);
diff += diff_stride;
}
}
static void sub_blk_64x64_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *pred, int32_t pred_stride,
int16_t *diff, int32_t diff_stride) {
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
__m128i pred0, pred1, pred2, pred3, pred4, pred5, pred6, pred7;
__m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
uint32_t loop_cnt;
for (loop_cnt = 32; loop_cnt--;) {
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
DUP4_ARG2(__lsx_vld, pred, 0, pred, 16, pred, 32, pred, 48, pred0, pred1,
pred2, pred3);
src += src_stride;
pred += pred_stride;
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src4, src5, src6,
src7);
DUP4_ARG2(__lsx_vld, pred, 0, pred, 16, pred, 32, pred, 48, pred4, pred5,
pred6, pred7);
src += src_stride;
pred += pred_stride;
DUP4_ARG2(__lsx_vilvl_b, src0, pred0, src1, pred1, src2, pred2, src3, pred3,
reg0, reg2, reg4, reg6);
DUP4_ARG2(__lsx_vilvh_b, src0, pred0, src1, pred1, src2, pred2, src3, pred3,
reg1, reg3, reg5, reg7);
DUP4_ARG2(__lsx_vilvl_b, src4, pred4, src5, pred5, src6, pred6, src7, pred7,
tmp0, tmp2, tmp4, tmp6);
DUP4_ARG2(__lsx_vilvh_b, src4, pred4, src5, pred5, src6, pred6, src7, pred7,
tmp1, tmp3, tmp5, tmp7);
DUP4_ARG2(__lsx_vhsubw_hu_bu, reg0, reg0, reg1, reg1, reg2, reg2, reg3,
reg3, src0, src1, src2, src3);
DUP4_ARG2(__lsx_vhsubw_hu_bu, reg4, reg4, reg5, reg5, reg6, reg6, reg7,
reg7, src4, src5, src6, src7);
DUP4_ARG2(__lsx_vhsubw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp2, tmp2, tmp3,
tmp3, pred0, pred1, pred2, pred3);
DUP4_ARG2(__lsx_vhsubw_hu_bu, tmp4, tmp4, tmp5, tmp5, tmp6, tmp6, tmp7,
tmp7, pred4, pred5, pred6, pred7);
__lsx_vst(src0, diff, 0);
__lsx_vst(src1, diff, 16);
__lsx_vst(src2, diff, 32);
__lsx_vst(src3, diff, 48);
__lsx_vst(src4, diff, 64);
__lsx_vst(src5, diff, 80);
__lsx_vst(src6, diff, 96);
__lsx_vst(src7, diff, 112);
diff += diff_stride;
__lsx_vst(pred0, diff, 0);
__lsx_vst(pred1, diff, 16);
__lsx_vst(pred2, diff, 32);
__lsx_vst(pred3, diff, 48);
__lsx_vst(pred4, diff, 64);
__lsx_vst(pred5, diff, 80);
__lsx_vst(pred6, diff, 96);
__lsx_vst(pred7, diff, 112);
diff += diff_stride;
}
}
void vpx_subtract_block_lsx(int32_t rows, int32_t cols, int16_t *diff_ptr,
ptrdiff_t diff_stride, const uint8_t *src_ptr,
ptrdiff_t src_stride, const uint8_t *pred_ptr,
ptrdiff_t pred_stride) {
if (rows == cols) {
switch (rows) {
case 4:
sub_blk_4x4_lsx(src_ptr, src_stride, pred_ptr, pred_stride, diff_ptr,
diff_stride);
break;
case 8:
sub_blk_8x8_lsx(src_ptr, src_stride, pred_ptr, pred_stride, diff_ptr,
diff_stride);
break;
case 16:
sub_blk_16x16_lsx(src_ptr, src_stride, pred_ptr, pred_stride, diff_ptr,
diff_stride);
break;
case 32:
sub_blk_32x32_lsx(src_ptr, src_stride, pred_ptr, pred_stride, diff_ptr,
diff_stride);
break;
case 64:
sub_blk_64x64_lsx(src_ptr, src_stride, pred_ptr, pred_stride, diff_ptr,
diff_stride);
break;
default:
vpx_subtract_block_c(rows, cols, diff_ptr, diff_stride, src_ptr,
src_stride, pred_ptr, pred_stride);
break;
}
} else {
vpx_subtract_block_c(rows, cols, diff_ptr, diff_stride, src_ptr, src_stride,
pred_ptr, pred_stride);
}
}
libs/libvpx/vpx_dsp/loongarch/txfm_macros_lsx.h
+48
View File
@@ -0,0 +1,48 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VPX_VPX_DSP_LOONGARCH_TXFM_MACROS_LSX_H_
#define VPX_VPX_DSP_LOONGARCH_TXFM_MACROS_LSX_H_
#include "vpx_util/loongson_intrinsics.h"
#define DOTP_CONST_PAIR(reg0, reg1, cnst0, cnst1, out0, out1) \
do { \
__m128i s0_m, s1_m, s2_m, s3_m, s4_m, s5_m; \
__m128i k0_m, k1_m, k2_m, k3_m; \
\
k0_m = __lsx_vreplgr2vr_h(cnst0); \
k1_m = __lsx_vreplgr2vr_h(cnst1); \
k2_m = __lsx_vpackev_h(k1_m, k0_m); \
\
DUP2_ARG2(__lsx_vilvl_h, reg1, reg0, reg0, reg1, s5_m, s3_m); \
DUP2_ARG2(__lsx_vilvh_h, reg1, reg0, reg0, reg1, s4_m, s2_m); \
\
DUP2_ARG2(__lsx_vmulwev_w_h, s5_m, k0_m, s4_m, k0_m, s1_m, s0_m); \
k3_m = __lsx_vmulwod_w_h(s5_m, k1_m); \
s1_m = __lsx_vsub_w(s1_m, k3_m); \
k3_m = __lsx_vmulwod_w_h(s4_m, k1_m); \
s0_m = __lsx_vsub_w(s0_m, k3_m); \
\
out0 = __lsx_vssrarni_h_w(s0_m, s1_m, DCT_CONST_BITS); \
\
DUP2_ARG2(__lsx_vdp2_w_h, s3_m, k2_m, s2_m, k2_m, s1_m, s0_m); \
out1 = __lsx_vssrarni_h_w(s0_m, s1_m, DCT_CONST_BITS); \
} while (0)
#define DOT_SHIFT_RIGHT_PCK_H(in0, in1, in2, in3) \
do { \
__m128i tp0_m, tp1_m; \
\
DUP2_ARG2(__lsx_vdp2_w_h, in0, in2, in1, in2, tp1_m, tp0_m); \
in3 = __lsx_vssrarni_h_w(tp1_m, tp0_m, DCT_CONST_BITS); \
} while (0)
#endif // VPX_VPX_DSP_LOONGARCH_TXFM_MACROS_LSX_H_
libs/libvpx/vpx_dsp/loongarch/variance_lsx.c
+263
View File
@@ -0,0 +1,263 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/loongarch/variance_lsx.h"
#define VARIANCE_WxH(sse, diff, shift) \
(sse) - (((uint32_t)(diff) * (diff)) >> (shift))
#define VARIANCE_LARGE_WxH(sse, diff, shift) \
(sse) - (((int64_t)(diff) * (diff)) >> (shift))
static uint32_t sse_diff_8width_lsx(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *ref_ptr, int32_t ref_stride,
int32_t height, int32_t *diff) {
int32_t res, ht_cnt = (height >> 2);
__m128i src0, src1, src2, src3, ref0, ref1, ref2, ref3, vec;
__m128i avg = __lsx_vldi(0);
__m128i var = avg;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t ref_stride2 = ref_stride << 1;
int32_t ref_stride3 = ref_stride2 + ref_stride;
int32_t ref_stride4 = ref_stride2 << 1;
for (; ht_cnt--;) {
DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr + src_stride, 0,
src_ptr + src_stride2, 0, src_ptr + src_stride3, 0, src0, src1,
src2, src3);
src_ptr += src_stride4;
DUP4_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr + ref_stride, 0,
ref_ptr + ref_stride2, 0, ref_ptr + ref_stride3, 0, ref0, ref1,
ref2, ref3);
ref_ptr += ref_stride4;
DUP4_ARG2(__lsx_vpickev_d, src1, src0, src3, src2, ref1, ref0, ref3, ref2,
src0, src1, ref0, ref1);
CALC_MSE_AVG_B(src0, ref0, var, avg);
CALC_MSE_AVG_B(src1, ref1, var, avg);
}
vec = __lsx_vhaddw_w_h(avg, avg);
HADD_SW_S32(vec, *diff);
HADD_SW_S32(var, res);
return res;
}
static uint32_t sse_diff_16width_lsx(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *ref_ptr, int32_t ref_stride,
int32_t height, int32_t *diff) {
int32_t res, ht_cnt = (height >> 2);
__m128i src, ref, vec;
__m128i avg = __lsx_vldi(0);
__m128i var = avg;
for (; ht_cnt--;) {
src = __lsx_vld(src_ptr, 0);
src_ptr += src_stride;
ref = __lsx_vld(ref_ptr, 0);
ref_ptr += ref_stride;
CALC_MSE_AVG_B(src, ref, var, avg);
src = __lsx_vld(src_ptr, 0);
src_ptr += src_stride;
ref = __lsx_vld(ref_ptr, 0);
ref_ptr += ref_stride;
CALC_MSE_AVG_B(src, ref, var, avg);
src = __lsx_vld(src_ptr, 0);
src_ptr += src_stride;
ref = __lsx_vld(ref_ptr, 0);
ref_ptr += ref_stride;
CALC_MSE_AVG_B(src, ref, var, avg);
src = __lsx_vld(src_ptr, 0);
src_ptr += src_stride;
ref = __lsx_vld(ref_ptr, 0);
ref_ptr += ref_stride;
CALC_MSE_AVG_B(src, ref, var, avg);
}
vec = __lsx_vhaddw_w_h(avg, avg);
HADD_SW_S32(vec, *diff);
HADD_SW_S32(var, res);
return res;
}
static uint32_t sse_diff_32width_lsx(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *ref_ptr, int32_t ref_stride,
int32_t height, int32_t *diff) {
int32_t res, ht_cnt = (height >> 2);
__m128i avg = __lsx_vldi(0);
__m128i src0, src1, ref0, ref1;
__m128i vec;
__m128i var = avg;
for (; ht_cnt--;) {
DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1);
src_ptr += src_stride;
DUP2_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref0, ref1);
ref_ptr += ref_stride;
CALC_MSE_AVG_B(src0, ref0, var, avg);
CALC_MSE_AVG_B(src1, ref1, var, avg);
DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1);
src_ptr += src_stride;
DUP2_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref0, ref1);
ref_ptr += ref_stride;
CALC_MSE_AVG_B(src0, ref0, var, avg);
CALC_MSE_AVG_B(src1, ref1, var, avg);
DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1);
src_ptr += src_stride;
DUP2_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref0, ref1);
ref_ptr += ref_stride;
CALC_MSE_AVG_B(src0, ref0, var, avg);
CALC_MSE_AVG_B(src1, ref1, var, avg);
DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1);
src_ptr += src_stride;
DUP2_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref0, ref1);
ref_ptr += ref_stride;
CALC_MSE_AVG_B(src0, ref0, var, avg);
CALC_MSE_AVG_B(src1, ref1, var, avg);
}
vec = __lsx_vhaddw_w_h(avg, avg);
HADD_SW_S32(vec, *diff);
HADD_SW_S32(var, res);
return res;
}
static uint32_t sse_diff_64x64_lsx(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *ref_ptr, int32_t ref_stride,
int32_t *diff) {
int32_t res, ht_cnt = 32;
__m128i avg0 = __lsx_vldi(0);
__m128i src0, src1, src2, src3;
__m128i ref0, ref1, ref2, ref3;
__m128i vec0, vec1;
__m128i avg1 = avg0;
__m128i avg2 = avg0;
__m128i avg3 = avg0;
__m128i var = avg0;
for (; ht_cnt--;) {
DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48,
src0, src1, src2, src3);
src_ptr += src_stride;
DUP4_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref_ptr, 32, ref_ptr, 48,
ref0, ref1, ref2, ref3);
ref_ptr += ref_stride;
CALC_MSE_AVG_B(src0, ref0, var, avg0);
CALC_MSE_AVG_B(src1, ref1, var, avg1);
CALC_MSE_AVG_B(src2, ref2, var, avg2);
CALC_MSE_AVG_B(src3, ref3, var, avg3);
DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48,
src0, src1, src2, src3);
src_ptr += src_stride;
DUP4_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref_ptr, 32, ref_ptr, 48,
ref0, ref1, ref2, ref3);
ref_ptr += ref_stride;
CALC_MSE_AVG_B(src0, ref0, var, avg0);
CALC_MSE_AVG_B(src1, ref1, var, avg1);
CALC_MSE_AVG_B(src2, ref2, var, avg2);
CALC_MSE_AVG_B(src3, ref3, var, avg3);
}
vec0 = __lsx_vhaddw_w_h(avg0, avg0);
vec1 = __lsx_vhaddw_w_h(avg1, avg1);
vec0 = __lsx_vadd_w(vec0, vec1);
vec1 = __lsx_vhaddw_w_h(avg2, avg2);
vec0 = __lsx_vadd_w(vec0, vec1);
vec1 = __lsx_vhaddw_w_h(avg3, avg3);
vec0 = __lsx_vadd_w(vec0, vec1);
HADD_SW_S32(vec0, *diff);
HADD_SW_S32(var, res);
return res;
}
#define VARIANCE_8Wx8H(sse, diff) VARIANCE_WxH(sse, diff, 6)
#define VARIANCE_16Wx16H(sse, diff) VARIANCE_WxH(sse, diff, 8)
#define VARIANCE_32Wx32H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 10)
#define VARIANCE_64Wx64H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 12)
#define VPX_VARIANCE_WDXHT_LSX(wd, ht) \
uint32_t vpx_variance##wd##x##ht##_lsx( \
const uint8_t *src, int32_t src_stride, const uint8_t *ref, \
int32_t ref_stride, uint32_t *sse) { \
int32_t diff; \
\
*sse = \
sse_diff_##wd##width_lsx(src, src_stride, ref, ref_stride, ht, &diff); \
\
return VARIANCE_##wd##Wx##ht##H(*sse, diff); \
}
static uint32_t sse_16width_lsx(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *ref_ptr, int32_t ref_stride,
int32_t height) {
int32_t res, ht_cnt = (height >> 2);
__m128i src, ref;
__m128i var = __lsx_vldi(0);
for (; ht_cnt--;) {
DUP2_ARG2(__lsx_vld, src_ptr, 0, ref_ptr, 0, src, ref);
src_ptr += src_stride;
ref_ptr += ref_stride;
CALC_MSE_B(src, ref, var);
DUP2_ARG2(__lsx_vld, src_ptr, 0, ref_ptr, 0, src, ref);
src_ptr += src_stride;
ref_ptr += ref_stride;
CALC_MSE_B(src, ref, var);
DUP2_ARG2(__lsx_vld, src_ptr, 0, ref_ptr, 0, src, ref);
src_ptr += src_stride;
ref_ptr += ref_stride;
CALC_MSE_B(src, ref, var);
DUP2_ARG2(__lsx_vld, src_ptr, 0, ref_ptr, 0, src, ref);
src_ptr += src_stride;
ref_ptr += ref_stride;
CALC_MSE_B(src, ref, var);
}
HADD_SW_S32(var, res);
return res;
}
VPX_VARIANCE_WDXHT_LSX(8, 8)
VPX_VARIANCE_WDXHT_LSX(16, 16)
VPX_VARIANCE_WDXHT_LSX(32, 32)
uint32_t vpx_variance64x64_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
uint32_t *sse) {
int32_t diff;
*sse = sse_diff_64x64_lsx(src, src_stride, ref, ref_stride, &diff);
return VARIANCE_64Wx64H(*sse, diff);
}
uint32_t vpx_mse16x16_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
uint32_t *sse) {
*sse = sse_16width_lsx(src, src_stride, ref, ref_stride, 16);
return *sse;
}
void vpx_get16x16var_lsx(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride, uint32_t *sse,
int32_t *sum) {
*sse = sse_diff_16width_lsx(src, src_stride, ref, ref_stride, 16, sum);
}
libs/libvpx/vpx_dsp/loongarch/variance_lsx.h
+62
View File
@@ -0,0 +1,62 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VPX_VPX_DSP_LOONGARCH_VARIANCE_LSX_H_
#define VPX_VPX_DSP_LOONGARCH_VARIANCE_LSX_H_
#include "vpx_util/loongson_intrinsics.h"
#define HADD_SW_S32(in0, in1) \
do { \
__m128i res0_m; \
\
res0_m = __lsx_vhaddw_d_w(in0, in0); \
res0_m = __lsx_vhaddw_q_d(res0_m, res0_m); \
in1 = __lsx_vpickve2gr_w(res0_m, 0); \
} while (0)
#define HORIZ_2TAP_FILT_UH(in0, in1, mask, coeff, shift, in2) \
do { \
__m128i tmp0_m, tmp1_m; \
\
tmp0_m = __lsx_vshuf_b(in1, in0, mask); \
tmp1_m = __lsx_vdp2_h_bu(tmp0_m, coeff); \
in2 = __lsx_vsrari_h(tmp1_m, shift); \
} while (0)
#define CALC_MSE_B(src, ref, var) \
do { \
__m128i src_l0_m, src_l1_m; \
__m128i res_l0_m, res_l1_m; \
\
src_l0_m = __lsx_vilvl_b(src, ref); \
src_l1_m = __lsx_vilvh_b(src, ref); \
DUP2_ARG2(__lsx_vhsubw_hu_bu, src_l0_m, src_l0_m, src_l1_m, src_l1_m, \
res_l0_m, res_l1_m); \
var = __lsx_vdp2add_w_h(var, res_l0_m, res_l0_m); \
var = __lsx_vdp2add_w_h(var, res_l1_m, res_l1_m); \
} while (0)
#define CALC_MSE_AVG_B(src, ref, var, sub) \
do { \
__m128i src_l0_m, src_l1_m; \
__m128i res_l0_m, res_l1_m; \
\
src_l0_m = __lsx_vilvl_b(src, ref); \
src_l1_m = __lsx_vilvh_b(src, ref); \
DUP2_ARG2(__lsx_vhsubw_hu_bu, src_l0_m, src_l0_m, src_l1_m, src_l1_m, \
res_l0_m, res_l1_m); \
var = __lsx_vdp2add_w_h(var, res_l0_m, res_l0_m); \
var = __lsx_vdp2add_w_h(var, res_l1_m, res_l1_m); \
sub = __lsx_vadd_h(sub, res_l0_m); \
sub = __lsx_vadd_h(sub, res_l1_m); \
} while (0)
#endif // VPX_VPX_DSP_LOONGARCH_VARIANCE_LSX_H_
libs/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_horiz_lsx.c
+972
View File
@@ -0,0 +1,972 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <assert.h>
#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/loongarch/vpx_convolve_lsx.h"
static const uint8_t mc_filt_mask_arr[16 * 3] = {
/* 8 width cases */
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
/* 4 width cases */
0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20,
/* 4 width cases */
8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28
};
static void common_hz_8t_and_aver_dst_4x4_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter) {
uint8_t *dst_tmp = dst;
__m128i src0, src1, src2, src3;
__m128i filter0, filter1, filter2, filter3;
__m128i mask0, mask1, mask2, mask3;
__m128i tmp0, tmp1;
__m128i dst0, dst1, dst2, dst3;
mask0 = __lsx_vld(mc_filt_mask_arr, 16);
src -= 3;
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
LSX_LD_4(src, src_stride, src0, src1, src2, src3);
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3,
filter0, filter1, filter2, filter3, tmp0, tmp1);
dst0 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_w(dst_tmp, 0);
dst0 = __lsx_vilvl_w(dst1, dst0);
dst1 = __lsx_vilvl_w(dst3, dst2);
dst0 = __lsx_vilvl_d(dst1, dst0);
tmp0 = __lsx_vssrarni_b_h(tmp1, tmp0, 7);
tmp0 = __lsx_vxori_b(tmp0, 128);
dst0 = __lsx_vavgr_bu(tmp0, dst0);
__lsx_vstelm_w(dst0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(dst0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(dst0, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(dst0, dst, 0, 3);
}
static void common_hz_8t_and_aver_dst_4x8_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter) {
uint8_t *dst_tmp = dst;
__m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3;
__m128i mask0, mask1, mask2, mask3, tmp0, tmp1, tmp2, tmp3;
__m128i dst0, dst1;
mask0 = __lsx_vld(mc_filt_mask_arr, 16);
src -= 3;
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
LSX_LD_4(src, src_stride, src0, src1, src2, src3);
src += src_stride;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
tmp0 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
tmp1 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
tmp2 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
tmp3 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
tmp0 = __lsx_vilvl_w(tmp1, tmp0);
tmp1 = __lsx_vilvl_w(tmp3, tmp2);
dst0 = __lsx_vilvl_d(tmp1, tmp0);
tmp0 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
tmp1 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
tmp2 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
tmp3 = __lsx_vldrepl_w(dst_tmp, 0);
tmp0 = __lsx_vilvl_w(tmp1, tmp0);
tmp1 = __lsx_vilvl_w(tmp3, tmp2);
dst1 = __lsx_vilvl_d(tmp1, tmp0);
HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3,
filter0, filter1, filter2, filter3, tmp0, tmp1);
LSX_LD_4(src, src_stride, src0, src1, src2, src3);
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3,
filter0, filter1, filter2, filter3, tmp2, tmp3);
DUP4_ARG3(__lsx_vssrarni_b_h, tmp0, tmp0, 7, tmp1, tmp1, 7, tmp2, tmp2, 7,
tmp3, tmp3, 7, tmp0, tmp1, tmp2, tmp3);
DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, tmp0, tmp1);
DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1);
DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp1, dst1, dst0, dst1);
__lsx_vstelm_w(dst0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(dst0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(dst0, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(dst0, dst, 0, 3);
dst += dst_stride;
__lsx_vstelm_w(dst1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(dst1, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(dst1, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(dst1, dst, 0, 3);
}
static void common_hz_8t_and_aver_dst_4w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride, int8_t *filter,
int32_t height) {
if (height == 4) {
common_hz_8t_and_aver_dst_4x4_lsx(src, src_stride, dst, dst_stride, filter);
} else if (height == 8) {
common_hz_8t_and_aver_dst_4x8_lsx(src, src_stride, dst, dst_stride, filter);
}
}
static void common_hz_8t_and_aver_dst_8w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride, int8_t *filter,
int32_t height) {
int32_t loop_cnt = height >> 2;
uint8_t *dst_tmp = dst;
__m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3;
__m128i mask0, mask1, mask2, mask3;
__m128i tmp0, tmp1, tmp2, tmp3;
__m128i dst0, dst1, dst2, dst3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
uint8_t *_src = (uint8_t *)src - 3;
mask0 = __lsx_vld(mc_filt_mask_arr, 0);
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
for (; loop_cnt--;) {
src0 = __lsx_vld(_src, 0);
DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2);
src3 = __lsx_vldx(_src, src_stride3);
_src += src_stride4;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
mask3, filter0, filter1, filter2, filter3, tmp0,
tmp1, tmp2, tmp3);
dst0 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1);
DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, tmp0, tmp1);
DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1);
DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp1, dst1, dst0, dst1);
__lsx_vstelm_d(dst0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(dst0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(dst1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(dst1, dst, 0, 1);
dst += dst_stride;
}
}
static void common_hz_8t_and_aver_dst_16w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter, int32_t height) {
int32_t loop_cnt = height >> 1;
int32_t dst_stride2 = dst_stride << 1;
uint8_t *dst_tmp = dst;
__m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3;
__m128i mask0, mask1, mask2, mask3, dst0, dst1, dst2, dst3;
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
__m128i tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15;
mask0 = __lsx_vld(mc_filt_mask_arr, 0);
src -= 3;
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
for (; loop_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1);
src += src_stride;
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3);
src += src_stride;
dst0 = __lsx_vld(dst_tmp, 0);
dst1 = __lsx_vldx(dst_tmp, dst_stride);
dst_tmp += dst_stride2;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, src2, src2,
mask0, src3, src3, mask0, tmp0, tmp1, tmp2, tmp3);
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, src2, src2,
mask1, src3, src3, mask1, tmp4, tmp5, tmp6, tmp7);
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, src2, src2,
mask2, src3, src3, mask2, tmp8, tmp9, tmp10, tmp11);
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src1, mask3, src2, src2,
mask3, src3, src3, mask3, tmp12, tmp13, tmp14, tmp15);
DUP4_ARG2(__lsx_vdp2_h_b, tmp0, filter0, tmp1, filter0, tmp2, filter0, tmp3,
filter0, tmp0, tmp1, tmp2, tmp3);
DUP4_ARG2(__lsx_vdp2_h_b, tmp8, filter2, tmp9, filter2, tmp10, filter2,
tmp11, filter2, tmp8, tmp9, tmp10, tmp11);
DUP4_ARG3(__lsx_vdp2add_h_b, tmp0, tmp4, filter1, tmp1, tmp5, filter1, tmp2,
tmp6, filter1, tmp3, tmp7, filter1, tmp0, tmp1, tmp2, tmp3);
DUP4_ARG3(__lsx_vdp2add_h_b, tmp8, tmp12, filter3, tmp9, tmp13, filter3,
tmp10, tmp14, filter3, tmp11, tmp15, filter3, tmp4, tmp5, tmp6,
tmp7);
DUP4_ARG2(__lsx_vsadd_h, tmp0, tmp4, tmp1, tmp5, tmp2, tmp6, tmp3, tmp7,
tmp0, tmp1, tmp2, tmp3);
DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, dst2, dst3);
DUP2_ARG2(__lsx_vxori_b, dst2, 128, dst3, 128, dst2, dst3);
DUP2_ARG2(__lsx_vavgr_bu, dst0, dst2, dst1, dst3, dst0, dst1);
__lsx_vst(dst0, dst, 0);
__lsx_vstx(dst1, dst, dst_stride);
dst += dst_stride2;
}
}
static void common_hz_8t_and_aver_dst_32w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = height;
uint8_t *dst_tmp = dst;
__m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3;
__m128i mask0, mask1, mask2, mask3, dst0, dst1;
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
__m128i tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15;
__m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 };
mask0 = __lsx_vld(mc_filt_mask_arr, 0);
src -= 3;
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
for (; loop_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2);
src3 = __lsx_vld(src, 24);
src1 = __lsx_vshuf_b(src2, src0, shuff);
src += src_stride;
DUP2_ARG2(__lsx_vld, dst_tmp, 0, dst, 16, dst0, dst1);
dst_tmp += dst_stride;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, src2, src2,
mask0, src3, src3, mask0, tmp0, tmp1, tmp2, tmp3);
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, src2, src2,
mask1, src3, src3, mask1, tmp4, tmp5, tmp6, tmp7);
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, src2, src2,
mask2, src3, src3, mask2, tmp8, tmp9, tmp10, tmp11);
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src1, mask3, src2, src2,
mask3, src3, src3, mask3, tmp12, tmp13, tmp14, tmp15);
DUP4_ARG2(__lsx_vdp2_h_b, tmp0, filter0, tmp1, filter0, tmp2, filter0, tmp3,
filter0, tmp0, tmp1, tmp2, tmp3);
DUP4_ARG2(__lsx_vdp2_h_b, tmp8, filter2, tmp9, filter2, tmp10, filter2,
tmp11, filter2, tmp8, tmp9, tmp10, tmp11);
DUP4_ARG3(__lsx_vdp2add_h_b, tmp0, tmp4, filter1, tmp1, tmp5, filter1, tmp2,
tmp6, filter1, tmp3, tmp7, filter1, tmp0, tmp1, tmp2, tmp3);
DUP4_ARG3(__lsx_vdp2add_h_b, tmp8, tmp12, filter3, tmp9, tmp13, filter3,
tmp10, tmp14, filter3, tmp11, tmp15, filter3, tmp4, tmp5, tmp6,
tmp7);
DUP4_ARG2(__lsx_vsadd_h, tmp0, tmp4, tmp1, tmp5, tmp2, tmp6, tmp3, tmp7,
tmp0, tmp1, tmp2, tmp3);
DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, tmp0, tmp1);
DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1);
DUP2_ARG2(__lsx_vavgr_bu, dst0, tmp0, dst1, tmp1, dst0, dst1);
__lsx_vst(dst0, dst, 0);
__lsx_vst(dst1, dst, 16);
dst += dst_stride;
}
}
static void common_hz_8t_and_aver_dst_64w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter, int32_t height) {
int32_t loop_cnt = height;
__m128i src0, src1, src2, src3;
__m128i filter0, filter1, filter2, filter3;
__m128i mask0, mask1, mask2, mask3;
__m128i out0, out1, out2, out3, dst0, dst1;
__m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 };
mask0 = __lsx_vld(mc_filt_mask_arr, 0);
src -= 3;
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
for (; loop_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2);
src3 = __lsx_vld(src, 24);
src1 = __lsx_vshuf_b(src2, src0, shuff);
DUP2_ARG2(__lsx_vld, dst, 0, dst, 16, dst0, dst1);
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
mask3, filter0, filter1, filter2, filter3, out0,
out1, out2, out3);
DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1);
DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1);
DUP2_ARG2(__lsx_vavgr_bu, out0, dst0, out1, dst1, out0, out1);
__lsx_vst(out0, dst, 0);
__lsx_vst(out1, dst, 16);
DUP2_ARG2(__lsx_vld, src, 32, src, 48, src0, src2);
src3 = __lsx_vld(src, 56);
src1 = __lsx_vshuf_b(src2, src0, shuff);
DUP2_ARG2(__lsx_vld, dst, 32, dst, 48, dst0, dst1);
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
mask3, filter0, filter1, filter2, filter3, out0,
out1, out2, out3);
DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1);
DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1);
DUP2_ARG2(__lsx_vavgr_bu, out0, dst0, out1, dst1, out0, out1);
__lsx_vst(out0, dst, 32);
__lsx_vst(out1, dst, 48);
src += src_stride;
dst += dst_stride;
}
}
static void common_hz_2t_and_aver_dst_4x4_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter) {
__m128i src0, src1, src2, src3, mask;
__m128i dst0, dst1, dst2, dst3, vec0, vec1, filt0;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
uint8_t *dst_tmp = dst;
mask = __lsx_vld(mc_filt_mask_arr, 16);
/* rearranging filter */
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
dst0 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_w, dst1, dst0, dst3, dst2, dst0, dst1);
dst0 = __lsx_vilvl_d(dst1, dst0);
DUP2_ARG3(__lsx_vshuf_b, src1, src0, mask, src3, src2, mask, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec0, vec1);
vec0 = __lsx_vssrarni_bu_h(vec1, vec0, FILTER_BITS);
vec0 = __lsx_vavgr_bu(vec0, dst0);
__lsx_vstelm_w(vec0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(vec0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(vec0, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(vec0, dst, 0, 3);
}
static void common_hz_2t_and_aver_dst_4x8_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter) {
__m128i src0, src1, src2, src3, src4, src5, src6, src7, mask;
__m128i filt0, vec0, vec1, vec2, vec3, res0, res1, res2, res3;
__m128i dst0, dst1, dst2, dst3, dst4;
__m128i vec4, vec5, vec6, vec7;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
uint8_t *src_tmp1 = (uint8_t *)src + src_stride4;
uint8_t *dst_tmp = dst;
mask = __lsx_vld(mc_filt_mask_arr, 16);
/* rearranging filter */
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src4 = __lsx_vld(src_tmp1, 0);
DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src5,
src6);
src7 = __lsx_vldx(src_tmp1, src_stride3);
dst0 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_w, dst1, dst0, dst3, dst2, dst0, dst1);
dst0 = __lsx_vilvl_d(dst1, dst0);
dst1 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst4 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_w, dst2, dst1, dst4, dst3, dst1, dst2);
dst1 = __lsx_vilvl_d(dst2, dst1);
DUP2_ARG3(__lsx_vshuf_b, src1, src0, mask, src3, src2, mask, vec0, vec1);
DUP2_ARG3(__lsx_vshuf_b, src5, src4, mask, src7, src6, mask, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0,
vec4, vec5, vec6, vec7);
DUP4_ARG3(__lsx_vssrarni_bu_h, vec4, vec4, FILTER_BITS, vec5, vec5,
FILTER_BITS, vec6, vec6, FILTER_BITS, vec7, vec7, FILTER_BITS, res0,
res1, res2, res3);
DUP2_ARG2(__lsx_vilvl_d, res1, res0, res3, res2, res0, res2);
DUP2_ARG2(__lsx_vavgr_bu, res0, dst0, res2, dst1, res0, res2);
__lsx_vstelm_w(res0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(res0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(res0, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(res0, dst, 0, 3);
dst += dst_stride;
__lsx_vstelm_w(res2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(res2, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(res2, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(res2, dst, 0, 3);
dst += dst_stride;
}
static void common_hz_2t_and_aver_dst_4w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride, int8_t *filter,
int32_t height) {
if (height == 4) {
common_hz_2t_and_aver_dst_4x4_lsx(src, src_stride, dst, dst_stride, filter);
} else if (height == 8) {
common_hz_2t_and_aver_dst_4x8_lsx(src, src_stride, dst, dst_stride, filter);
}
}
static void common_hz_2t_and_aver_dst_8x4_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter) {
__m128i src0, src1, src2, src3, mask;
__m128i filt0, dst0, dst1, dst2, dst3;
__m128i vec0, vec1, vec2, vec3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
uint8_t *dst_tmp = dst;
mask = __lsx_vld(mc_filt_mask_arr, 0);
/* rearranging filter */
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1);
DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0,
vec0, vec1, vec2, vec3);
DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2,
FILTER_BITS, vec0, vec1);
dst0 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1);
DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec1, dst1, vec0, vec1);
__lsx_vstelm_d(vec0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(vec0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(vec1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(vec1, dst, 0, 1);
}
static void common_hz_2t_and_aver_dst_8x8mult_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
__m128i src0, src1, src2, src3, mask;
__m128i filt0, dst0, dst1, dst2, dst3;
__m128i vec0, vec1, vec2, vec3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
uint8_t *dst_tmp = dst;
mask = __lsx_vld(mc_filt_mask_arr, 0);
/* rearranging filter */
filt0 = __lsx_vldrepl_h(filter, 0);
LSX_LD_4(src, src_stride, src0, src1, src2, src3);
src += src_stride;
DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1);
DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0,
vec0, vec1, vec2, vec3);
DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2,
FILTER_BITS, vec0, vec2);
dst0 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1);
DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec2, dst1, vec0, vec2);
__lsx_vstelm_d(vec0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(vec0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(vec2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(vec2, dst, 0, 1);
dst += dst_stride;
LSX_LD_4(src, src_stride, src0, src1, src2, src3);
src += src_stride;
DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1);
DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0,
vec0, vec1, vec2, vec3);
DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2,
FILTER_BITS, vec0, vec2);
dst0 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1);
DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec2, dst1, vec0, vec2);
__lsx_vstelm_d(vec0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(vec0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(vec2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(vec2, dst, 0, 1);
dst += dst_stride;
if (height == 16) {
LSX_LD_4(src, src_stride, src0, src1, src2, src3);
src += src_stride;
DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1);
DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, vec0, vec1, vec2, vec3);
DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2,
FILTER_BITS, vec0, vec2);
dst0 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1);
DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec2, dst1, vec0, vec2);
__lsx_vstelm_d(vec0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(vec0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(vec2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(vec2, dst, 0, 1);
dst += dst_stride;
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1);
DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, vec0, vec1, vec2, vec3);
DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2,
FILTER_BITS, vec0, vec2);
dst0 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1);
DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec2, dst1, vec0, vec2);
__lsx_vstelm_d(vec0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(vec0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(vec2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(vec2, dst, 0, 1);
dst += dst_stride;
}
}
static void common_hz_2t_and_aver_dst_8w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride, int8_t *filter,
int32_t height) {
if (height == 4) {
common_hz_2t_and_aver_dst_8x4_lsx(src, src_stride, dst, dst_stride, filter);
} else {
common_hz_2t_and_aver_dst_8x8mult_lsx(src, src_stride, dst, dst_stride,
filter, height);
}
}
static void common_hz_2t_and_aver_dst_16w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = (height >> 2) - 1;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, mask;
__m128i filt0, dst0;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
__m128i res0, res1, res2, res3, res4, res5, res6, res7;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
uint8_t *src_tmp1 = (uint8_t *)src + 8;
mask = __lsx_vld(mc_filt_mask_arr, 0);
/* rearranging filter */
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4);
src6 = __lsx_vldx(src, src_stride3);
src += src_stride4;
src1 = __lsx_vld(src_tmp1, 0);
DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3,
src5);
src7 = __lsx_vldx(src_tmp1, src_stride3);
src_tmp1 += src_stride4;
DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1);
DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3);
DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5);
DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0,
res0, res1, res2, res3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, filt0,
res4, res5, res6, res7);
DUP4_ARG3(__lsx_vssrarni_bu_h, res1, res0, FILTER_BITS, res3, res2,
FILTER_BITS, res5, res4, FILTER_BITS, res7, res6, FILTER_BITS, res0,
res2, res4, res6);
dst0 = __lsx_vld(dst, 0);
res0 = __lsx_vavgr_bu(res0, dst0);
__lsx_vst(res0, dst, 0);
dst += dst_stride;
dst0 = __lsx_vld(dst, 0);
res2 = __lsx_vavgr_bu(res2, dst0);
__lsx_vst(res2, dst, 0);
dst += dst_stride;
dst0 = __lsx_vld(dst, 0);
res4 = __lsx_vavgr_bu(res4, dst0);
__lsx_vst(res4, dst, 0);
dst += dst_stride;
dst0 = __lsx_vld(dst, 0);
res6 = __lsx_vavgr_bu(res6, dst0);
__lsx_vst(res6, dst, 0);
dst += dst_stride;
for (; loop_cnt--;) {
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4);
src6 = __lsx_vldx(src, src_stride3);
src += src_stride4;
src1 = __lsx_vld(src_tmp1, 0);
DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3,
src5);
src7 = __lsx_vldx(src_tmp1, src_stride3);
src_tmp1 += src_stride4;
DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1);
DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3);
DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5);
DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, res0, res1, res2, res3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7,
filt0, res4, res5, res6, res7);
DUP4_ARG3(__lsx_vssrarni_bu_h, res1, res0, FILTER_BITS, res3, res2,
FILTER_BITS, res5, res4, FILTER_BITS, res7, res6, FILTER_BITS,
res0, res2, res4, res6);
dst0 = __lsx_vld(dst, 0);
res0 = __lsx_vavgr_bu(res0, dst0);
__lsx_vst(res0, dst, 0);
dst += dst_stride;
dst0 = __lsx_vld(dst, 0);
res2 = __lsx_vavgr_bu(res2, dst0);
__lsx_vst(res2, dst, 0);
dst += dst_stride;
dst0 = __lsx_vld(dst, 0);
res4 = __lsx_vavgr_bu(res4, dst0);
__lsx_vst(res4, dst, 0);
dst += dst_stride;
dst0 = __lsx_vld(dst, 0);
res6 = __lsx_vavgr_bu(res6, dst0);
__lsx_vst(res6, dst, 0);
dst += dst_stride;
}
}
static void common_hz_2t_and_aver_dst_32w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = (height >> 1);
__m128i src0, src1, src2, src3, src4, src5, src6, src7, mask;
__m128i filt0, dst0, dst1;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
__m128i res0, res1, res2, res3, res4, res5, res6, res7;
__m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 };
mask = __lsx_vld(mc_filt_mask_arr, 0);
/* rearranging filter */
filt0 = __lsx_vldrepl_h(filter, 0);
for (; loop_cnt--;) {
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vld, src, 16, src, 24, src2, src3);
src1 = __lsx_vshuf_b(src2, src0, shuff);
src += src_stride;
src4 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vld, src, 16, src, 24, src6, src7);
src5 = __lsx_vshuf_b(src6, src4, shuff);
src += src_stride;
DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1);
DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3);
DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5);
DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, res0, res1, res2, res3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7,
filt0, res4, res5, res6, res7);
DUP4_ARG3(__lsx_vssrarni_bu_h, res1, res0, FILTER_BITS, res3, res2,
FILTER_BITS, res5, res4, FILTER_BITS, res7, res6, FILTER_BITS,
res0, res2, res4, res6);
DUP2_ARG2(__lsx_vld, dst, 0, dst, 16, dst0, dst1);
res0 = __lsx_vavgr_bu(res0, dst0);
__lsx_vst(res0, dst, 0);
res2 = __lsx_vavgr_bu(res2, dst1);
__lsx_vst(res2, dst, 16);
dst += dst_stride;
DUP2_ARG2(__lsx_vld, dst, 0, dst, 16, dst0, dst1);
res4 = __lsx_vavgr_bu(res4, dst0);
__lsx_vst(res4, dst, 0);
res6 = __lsx_vavgr_bu(res6, dst1);
__lsx_vst(res6, dst, 16);
dst += dst_stride;
}
}
static void common_hz_2t_and_aver_dst_64w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = height;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, mask;
__m128i filt0, dst0, dst1, dst2, dst3;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
__m128i out0, out1, out2, out3, out4, out5, out6, out7;
__m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 };
mask = __lsx_vld(mc_filt_mask_arr, 0);
/* rearranging filter */
filt0 = __lsx_vldrepl_h(filter, 0);
for (; loop_cnt--;) {
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src2, src4,
src6);
src7 = __lsx_vld(src, 56);
DUP2_ARG3(__lsx_vshuf_b, src2, src0, shuff, src4, src2, shuff, src1, src3);
src5 = __lsx_vshuf_b(src6, src4, shuff);
src += src_stride;
DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1);
DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3);
DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5);
DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, out0, out1, out2, out3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7,
filt0, out4, out5, out6, out7);
DUP4_ARG3(__lsx_vssrarni_bu_h, out1, out0, FILTER_BITS, out3, out2,
FILTER_BITS, out5, out4, FILTER_BITS, out7, out6, FILTER_BITS,
out0, out2, out4, out6);
DUP4_ARG2(__lsx_vld, dst, 0, dst, 16, dst, 32, dst, 48, dst0, dst1, dst2,
dst3);
out0 = __lsx_vavgr_bu(out0, dst0);
__lsx_vst(out0, dst, 0);
out2 = __lsx_vavgr_bu(out2, dst1);
__lsx_vst(out2, dst, 16);
out4 = __lsx_vavgr_bu(out4, dst2);
__lsx_vst(out4, dst, 32);
out6 = __lsx_vavgr_bu(out6, dst3);
__lsx_vst(out6, dst, 48);
dst += dst_stride;
}
}
void vpx_convolve8_avg_horiz_lsx(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const InterpKernel *filter, int x0_q4,
int x_step_q4, int y0_q4, int y_step_q4, int w,
int h) {
const int16_t *const filter_x = filter[x0_q4];
int8_t cnt, filt_hor[8];
assert(x_step_q4 == 16);
assert(((const int32_t *)filter_x)[1] != 0x800000);
for (cnt = 0; cnt < 8; ++cnt) {
filt_hor[cnt] = filter_x[cnt];
}
if (vpx_get_filter_taps(filter_x) == 2) {
switch (w) {
case 4:
common_hz_2t_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_hor[3], h);
break;
case 8:
common_hz_2t_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_hor[3], h);
break;
case 16:
common_hz_2t_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_hor[3], h);
break;
case 32:
common_hz_2t_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_hor[3], h);
break;
case 64:
common_hz_2t_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_hor[3], h);
break;
default:
vpx_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride, filter,
x0_q4, x_step_q4, y0_q4, y_step_q4, w, h);
break;
}
} else {
switch (w) {
case 4:
common_hz_8t_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor, h);
break;
case 8:
common_hz_8t_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor, h);
break;
case 16:
common_hz_8t_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor, h);
break;
case 32:
common_hz_8t_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor, h);
break;
case 64:
common_hz_8t_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor, h);
break;
default:
vpx_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride, filter,
x0_q4, x_step_q4, y0_q4, y_step_q4, w, h);
break;
}
}
}
libs/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_lsx.c
+737
View File
@@ -0,0 +1,737 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <assert.h>
#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/loongarch/vpx_convolve_lsx.h"
static const uint8_t mc_filt_mask_arr[16 * 3] = {
/* 8 width cases */
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
/* 4 width cases */
0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20,
/* 4 width cases */
8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28
};
static void common_hv_8ht_8vt_and_aver_dst_4w_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) {
uint32_t loop_cnt = height >> 2;
uint8_t *dst_tmp = dst;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
__m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3;
__m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3;
__m128i mask0, mask1, mask2, mask3;
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
__m128i out0, out1;
__m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 };
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride + src_stride2;
int32_t src_stride4 = src_stride2 << 1;
uint8_t *_src = (uint8_t *)src - 3 - src_stride3;
mask0 = __lsx_vld(mc_filt_mask_arr, 16);
DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4,
filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3);
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
src0 = __lsx_vld(_src, 0);
DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2);
src3 = __lsx_vldx(_src, src_stride3);
_src += src_stride4;
src4 = __lsx_vld(_src, 0);
DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src5, src6);
_src += src_stride3;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5);
src6 = __lsx_vxori_b(src6, 128);
tmp0 = horiz_8tap_filt(src0, src1, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
tmp2 = horiz_8tap_filt(src2, src3, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
tmp4 = horiz_8tap_filt(src4, src5, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
tmp5 = horiz_8tap_filt(src5, src6, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
DUP2_ARG3(__lsx_vshuf_b, tmp2, tmp0, shuff, tmp4, tmp2, shuff, tmp1, tmp3);
DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4,
filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3);
DUP2_ARG2(__lsx_vpackev_b, tmp1, tmp0, tmp3, tmp2, tmp0, tmp1);
tmp2 = __lsx_vpackev_b(tmp5, tmp4);
for (; loop_cnt--;) {
src7 = __lsx_vld(_src, 0);
DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src8, src9);
src10 = __lsx_vldx(_src, src_stride3);
_src += src_stride4;
src2 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
src3 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
src4 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
src5 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_w, src3, src2, src5, src4, src2, src3);
src2 = __lsx_vilvl_d(src3, src2);
DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7,
src8, src9, src10);
tmp3 = horiz_8tap_filt(src7, src8, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
tmp4 = __lsx_vshuf_b(tmp3, tmp5, shuff);
tmp4 = __lsx_vpackev_b(tmp3, tmp4);
out0 = filt_8tap_dpadd_s_h(tmp0, tmp1, tmp2, tmp4, filt_vt0, filt_vt1,
filt_vt2, filt_vt3);
src1 = horiz_8tap_filt(src9, src10, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src0 = __lsx_vshuf_b(src1, tmp3, shuff);
src0 = __lsx_vpackev_b(src1, src0);
out1 = filt_8tap_dpadd_s_h(tmp1, tmp2, tmp4, src0, filt_vt0, filt_vt1,
filt_vt2, filt_vt3);
out0 = __lsx_vssrarni_b_h(out1, out0, FILTER_BITS);
out0 = __lsx_vxori_b(out0, 128);
out0 = __lsx_vavgr_bu(out0, src2);
__lsx_vstelm_w(out0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 3);
dst += dst_stride;
tmp5 = src1;
tmp0 = tmp2;
tmp1 = tmp4;
tmp2 = src0;
}
}
static void common_hv_8ht_8vt_and_aver_dst_8w_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) {
uint32_t loop_cnt = height >> 2;
uint8_t *dst_tmp = dst;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
__m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3;
__m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3;
__m128i mask0, mask1, mask2, mask3;
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6;
__m128i out0, out1;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride + src_stride2;
int32_t src_stride4 = src_stride2 << 1;
uint8_t *_src = (uint8_t *)src - 3 - src_stride3;
mask0 = __lsx_vld(mc_filt_mask_arr, 0);
DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4,
filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3);
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
src0 = __lsx_vld(_src, 0);
DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2);
src3 = __lsx_vldx(_src, src_stride3);
_src += src_stride4;
src4 = __lsx_vld(_src, 0);
DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src5, src6);
_src += src_stride3;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5);
src6 = __lsx_vxori_b(src6, 128);
src0 = horiz_8tap_filt(src0, src0, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src1 = horiz_8tap_filt(src1, src1, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src2 = horiz_8tap_filt(src2, src2, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src3 = horiz_8tap_filt(src3, src3, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src4 = horiz_8tap_filt(src4, src4, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src5 = horiz_8tap_filt(src5, src5, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src6 = horiz_8tap_filt(src6, src6, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4,
filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3);
DUP4_ARG2(__lsx_vpackev_b, src1, src0, src3, src2, src5, src4, src2, src1,
tmp0, tmp1, tmp2, tmp4);
DUP2_ARG2(__lsx_vpackev_b, src4, src3, src6, src5, tmp5, tmp6);
for (; loop_cnt--;) {
src7 = __lsx_vld(_src, 0);
DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src8, src9);
src10 = __lsx_vldx(_src, src_stride3);
_src += src_stride4;
DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7,
src8, src9, src10);
src7 = horiz_8tap_filt(src7, src7, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
tmp3 = __lsx_vpackev_b(src7, src6);
out0 = filt_8tap_dpadd_s_h(tmp0, tmp1, tmp2, tmp3, filt_vt0, filt_vt1,
filt_vt2, filt_vt3);
src8 = horiz_8tap_filt(src8, src8, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src0 = __lsx_vpackev_b(src8, src7);
out1 = filt_8tap_dpadd_s_h(tmp4, tmp5, tmp6, src0, filt_vt0, filt_vt1,
filt_vt2, filt_vt3);
src9 = horiz_8tap_filt(src9, src9, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src1 = __lsx_vpackev_b(src9, src8);
src3 = filt_8tap_dpadd_s_h(tmp1, tmp2, tmp3, src1, filt_vt0, filt_vt1,
filt_vt2, filt_vt3);
src10 = horiz_8tap_filt(src10, src10, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src2 = __lsx_vpackev_b(src10, src9);
src4 = filt_8tap_dpadd_s_h(tmp5, tmp6, src0, src2, filt_vt0, filt_vt1,
filt_vt2, filt_vt3);
DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, FILTER_BITS, src4, src3,
FILTER_BITS, out0, out1);
DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1);
src5 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
src7 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
src8 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
src9 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_d, src7, src5, src9, src8, src5, src7);
DUP2_ARG2(__lsx_vavgr_bu, out0, src5, out1, src7, out0, out1);
__lsx_vstelm_d(out0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 1);
dst += dst_stride;
src6 = src10;
tmp0 = tmp2;
tmp1 = tmp3;
tmp2 = src1;
tmp4 = tmp6;
tmp5 = src0;
tmp6 = src2;
}
}
static void common_hv_8ht_8vt_and_aver_dst_16w_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) {
common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride,
filter_horiz, filter_vert, height);
src += 8;
dst += 8;
common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride,
filter_horiz, filter_vert, height);
}
static void common_hv_8ht_8vt_and_aver_dst_32w_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) {
int32_t multiple8_cnt;
for (multiple8_cnt = 4; multiple8_cnt--;) {
common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride,
filter_horiz, filter_vert, height);
src += 8;
dst += 8;
}
}
static void common_hv_8ht_8vt_and_aver_dst_64w_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) {
int32_t multiple8_cnt;
for (multiple8_cnt = 8; multiple8_cnt--;) {
common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride,
filter_horiz, filter_vert, height);
src += 8;
dst += 8;
}
}
static void common_hv_2ht_2vt_and_aver_dst_4x4_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert) {
__m128i src0, src1, src2, src3, src4, mask;
__m128i filt_hz, filt_vt, vec0, vec1;
__m128i dst0, dst1, dst2, dst3;
__m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, tmp0, tmp1;
__m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 };
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
mask = __lsx_vld(mc_filt_mask_arr, 16);
/* rearranging filter */
DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt);
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
hz_out0 = horiz_2tap_filt_uh(src0, src1, mask, filt_hz);
hz_out2 = horiz_2tap_filt_uh(src2, src3, mask, filt_hz);
hz_out4 = horiz_2tap_filt_uh(src4, src4, mask, filt_hz);
hz_out1 = __lsx_vshuf_b(hz_out2, hz_out0, shuff);
hz_out3 = __lsx_vpickod_d(hz_out4, hz_out2);
DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1);
dst0 = __lsx_vldrepl_w(dst, 0);
dst1 = __lsx_vldrepl_w(dst + dst_stride, 0);
dst2 = __lsx_vldrepl_w(dst + dst_stride2, 0);
dst3 = __lsx_vldrepl_w(dst + dst_stride3, 0);
dst0 = __lsx_vilvl_w(dst1, dst0);
dst1 = __lsx_vilvl_w(dst3, dst2);
dst0 = __lsx_vilvl_d(dst1, dst0);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst0);
__lsx_vstelm_w(tmp0, dst, 0, 0);
__lsx_vstelm_w(tmp0, dst + dst_stride, 0, 1);
__lsx_vstelm_w(tmp0, dst + dst_stride2, 0, 2);
__lsx_vstelm_w(tmp0, dst + dst_stride3, 0, 3);
}
static void common_hv_2ht_2vt_and_aver_dst_4x8_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert) {
uint8_t *dst_tmp = dst;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, mask;
__m128i filt_hz, filt_vt, vec0, vec1, vec2, vec3, res0, res1;
__m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
__m128i hz_out7, hz_out8, tmp0, tmp1, tmp2, tmp3;
__m128i dst0, dst1, dst2, dst3, dst4;
__m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 };
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
mask = __lsx_vld(mc_filt_mask_arr, 16);
/* rearranging filter */
filt_hz = __lsx_vldrepl_h(filter_horiz, 0);
filt_vt = __lsx_vldrepl_h(filter_vert, 0);
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
src += src_stride4;
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src5, src6, src7, src8);
src += src_stride4;
hz_out0 = horiz_2tap_filt_uh(src0, src1, mask, filt_hz);
hz_out2 = horiz_2tap_filt_uh(src2, src3, mask, filt_hz);
hz_out4 = horiz_2tap_filt_uh(src4, src5, mask, filt_hz);
hz_out6 = horiz_2tap_filt_uh(src6, src7, mask, filt_hz);
hz_out8 = horiz_2tap_filt_uh(src8, src8, mask, filt_hz);
DUP2_ARG3(__lsx_vshuf_b, hz_out2, hz_out0, shuff, hz_out4, hz_out2, shuff,
hz_out1, hz_out3);
hz_out5 = __lsx_vshuf_b(hz_out6, hz_out4, shuff);
hz_out7 = __lsx_vpickod_d(hz_out8, hz_out6);
dst0 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst0 = __lsx_vilvl_w(dst1, dst0);
dst1 = __lsx_vilvl_w(dst3, dst2);
dst0 = __lsx_vilvl_d(dst1, dst0);
dst1 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst4 = __lsx_vldrepl_w(dst_tmp, 0);
dst1 = __lsx_vilvl_w(dst2, dst1);
dst2 = __lsx_vilvl_w(dst4, dst3);
dst1 = __lsx_vilvl_d(dst2, dst1);
DUP4_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, hz_out5,
hz_out4, hz_out7, hz_out6, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, vec2, filt_vt, vec3,
filt_vt, tmp0, tmp1, tmp2, tmp3);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2,
FILTER_BITS, res0, res1);
DUP2_ARG2(__lsx_vavgr_bu, res0, dst0, res1, dst1, res0, res1);
__lsx_vstelm_w(res0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(res0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(res0, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(res0, dst, 0, 3);
dst += dst_stride;
__lsx_vstelm_w(res1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(res1, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(res1, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(res1, dst, 0, 3);
}
static void common_hv_2ht_2vt_and_aver_dst_4w_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert, int32_t height) {
if (height == 4) {
common_hv_2ht_2vt_and_aver_dst_4x4_lsx(src, src_stride, dst, dst_stride,
filter_horiz, filter_vert);
} else if (height == 8) {
common_hv_2ht_2vt_and_aver_dst_4x8_lsx(src, src_stride, dst, dst_stride,
filter_horiz, filter_vert);
}
}
static void common_hv_2ht_2vt_and_aver_dst_8x4_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert) {
__m128i src0, src1, src2, src3, src4, mask;
__m128i filt_hz, filt_vt, vec0, vec1, vec2, vec3;
__m128i hz_out0, hz_out1, tmp0, tmp1, tmp2, tmp3;
__m128i dst0, dst1, dst2, dst3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
uint8_t *dst_tmp = dst;
mask = __lsx_vld(mc_filt_mask_arr, 0);
/* rearranging filter */
filt_hz = __lsx_vldrepl_h(filter_horiz, 0);
filt_vt = __lsx_vldrepl_h(filter_vert, 0);
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
dst0 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_d(dst_tmp, 0);
DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1);
hz_out0 = horiz_2tap_filt_uh(src0, src0, mask, filt_hz);
hz_out1 = horiz_2tap_filt_uh(src1, src1, mask, filt_hz);
vec0 = __lsx_vpackev_b(hz_out1, hz_out0);
tmp0 = __lsx_vdp2_h_bu(vec0, filt_vt);
hz_out0 = horiz_2tap_filt_uh(src2, src2, mask, filt_hz);
vec1 = __lsx_vpackev_b(hz_out0, hz_out1);
tmp1 = __lsx_vdp2_h_bu(vec1, filt_vt);
hz_out1 = horiz_2tap_filt_uh(src3, src3, mask, filt_hz);
vec2 = __lsx_vpackev_b(hz_out1, hz_out0);
tmp2 = __lsx_vdp2_h_bu(vec2, filt_vt);
hz_out0 = horiz_2tap_filt_uh(src4, src4, mask, filt_hz);
vec3 = __lsx_vpackev_b(hz_out0, hz_out1);
tmp3 = __lsx_vdp2_h_bu(vec3, filt_vt);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2,
FILTER_BITS, tmp0, tmp1);
AVG_ST4_D(tmp0, tmp1, dst0, dst1, dst, dst_stride);
}
static void common_hv_2ht_2vt_and_aver_dst_8x8mult_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert, int32_t height) {
uint32_t loop_cnt = (height >> 2);
__m128i src0, src1, src2, src3, src4, mask;
__m128i filt_hz, filt_vt, vec0;
__m128i hz_out0, hz_out1, tmp0, tmp1, tmp2, tmp3;
__m128i dst0, dst1, dst2, dst3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
uint8_t *dst_tmp = dst;
/* rearranging filter */
mask = __lsx_vld(mc_filt_mask_arr, 0);
filt_hz = __lsx_vldrepl_h(filter_horiz, 0);
filt_vt = __lsx_vldrepl_h(filter_vert, 0);
src0 = __lsx_vld(src, 0);
src += src_stride;
hz_out0 = horiz_2tap_filt_uh(src0, src0, mask, filt_hz);
for (; loop_cnt--;) {
src1 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3);
src4 = __lsx_vldx(src, src_stride3);
src += src_stride4;
hz_out1 = horiz_2tap_filt_uh(src1, src1, mask, filt_hz);
vec0 = __lsx_vpackev_b(hz_out1, hz_out0);
tmp0 = __lsx_vdp2_h_bu(vec0, filt_vt);
hz_out0 = horiz_2tap_filt_uh(src2, src2, mask, filt_hz);
vec0 = __lsx_vpackev_b(hz_out0, hz_out1);
tmp1 = __lsx_vdp2_h_bu(vec0, filt_vt);
hz_out1 = horiz_2tap_filt_uh(src3, src3, mask, filt_hz);
vec0 = __lsx_vpackev_b(hz_out1, hz_out0);
tmp2 = __lsx_vdp2_h_bu(vec0, filt_vt);
hz_out0 = horiz_2tap_filt_uh(src4, src4, mask, filt_hz);
vec0 = __lsx_vpackev_b(hz_out0, hz_out1);
tmp3 = __lsx_vdp2_h_bu(vec0, filt_vt);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2,
FILTER_BITS, tmp0, tmp1);
dst0 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1);
AVG_ST4_D(tmp0, tmp1, dst0, dst1, dst, dst_stride);
dst += dst_stride;
}
}
static void common_hv_2ht_2vt_and_aver_dst_8w_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert, int32_t height) {
if (height == 4) {
common_hv_2ht_2vt_and_aver_dst_8x4_lsx(src, src_stride, dst, dst_stride,
filter_horiz, filter_vert);
} else {
common_hv_2ht_2vt_and_aver_dst_8x8mult_lsx(
src, src_stride, dst, dst_stride, filter_horiz, filter_vert, height);
}
}
static void common_hv_2ht_2vt_and_aver_dst_16w_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert, int32_t height) {
uint8_t *src_tmp1;
uint32_t loop_cnt = (height >> 2);
__m128i src0, src1, src2, src3, src4, src5, src6, src7, mask;
__m128i filt_hz, filt_vt, vec0, vec1, dst0, dst1, dst2, dst3;
__m128i hz_out0, hz_out1, hz_out2, hz_out3, tmp0, tmp1, tmp3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
int32_t dst_stride4 = dst_stride << 2;
mask = __lsx_vld(mc_filt_mask_arr, 0);
/* rearranging filter */
filt_hz = __lsx_vldrepl_h(filter_horiz, 0);
filt_vt = __lsx_vldrepl_h(filter_vert, 0);
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1);
src += src_stride;
hz_out0 = horiz_2tap_filt_uh(src0, src0, mask, filt_hz);
hz_out2 = horiz_2tap_filt_uh(src1, src1, mask, filt_hz);
for (; loop_cnt--;) {
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4);
src6 = __lsx_vldx(src, src_stride3);
src_tmp1 = (uint8_t *)(src + 8);
src1 = __lsx_vld(src_tmp1, 0);
DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3,
src5);
src7 = __lsx_vldx(src_tmp1, src_stride3);
src += src_stride4;
dst0 = __lsx_vld(dst, 0);
DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2);
dst3 = __lsx_vldx(dst, dst_stride3);
hz_out1 = horiz_2tap_filt_uh(src0, src0, mask, filt_hz);
hz_out3 = horiz_2tap_filt_uh(src1, src1, mask, filt_hz);
DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1);
tmp3 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp3 = __lsx_vavgr_bu(tmp3, dst0);
__lsx_vst(tmp3, dst, 0);
hz_out0 = horiz_2tap_filt_uh(src2, src2, mask, filt_hz);
hz_out2 = horiz_2tap_filt_uh(src3, src3, mask, filt_hz);
DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1);
tmp3 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp3 = __lsx_vavgr_bu(tmp3, dst1);
__lsx_vstx(tmp3, dst, dst_stride);
hz_out1 = horiz_2tap_filt_uh(src4, src4, mask, filt_hz);
hz_out3 = horiz_2tap_filt_uh(src5, src5, mask, filt_hz);
DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1);
tmp3 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp3 = __lsx_vavgr_bu(tmp3, dst2);
__lsx_vstx(tmp3, dst, dst_stride2);
hz_out0 = horiz_2tap_filt_uh(src6, src6, mask, filt_hz);
hz_out2 = horiz_2tap_filt_uh(src7, src7, mask, filt_hz);
DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1);
tmp3 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp3 = __lsx_vavgr_bu(tmp3, dst3);
__lsx_vstx(tmp3, dst, dst_stride3);
dst += dst_stride4;
}
}
static void common_hv_2ht_2vt_and_aver_dst_32w_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert, int32_t height) {
common_hv_2ht_2vt_and_aver_dst_16w_lsx(src, src_stride, dst, dst_stride,
filter_horiz, filter_vert, height);
src += 16;
dst += 16;
common_hv_2ht_2vt_and_aver_dst_16w_lsx(src, src_stride, dst, dst_stride,
filter_horiz, filter_vert, height);
}
static void common_hv_2ht_2vt_and_aver_dst_64w_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert, int32_t height) {
int32_t multiple8_cnt;
for (multiple8_cnt = 4; multiple8_cnt--;) {
common_hv_2ht_2vt_and_aver_dst_16w_lsx(src, src_stride, dst, dst_stride,
filter_horiz, filter_vert, height);
src += 16;
dst += 16;
}
}
void vpx_convolve8_avg_lsx(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const InterpKernel *filter, int x0_q4, int x_step_q4,
int y0_q4, int y_step_q4, int w, int h) {
const int16_t *const filter_x = filter[x0_q4];
const int16_t *const filter_y = filter[y0_q4];
int8_t cnt, filt_hor[8], filt_ver[8];
assert(x_step_q4 == 16);
assert(y_step_q4 == 16);
assert(((const int32_t *)filter_x)[1] != 0x800000);
assert(((const int32_t *)filter_y)[1] != 0x800000);
for (cnt = 0; cnt < 8; ++cnt) {
filt_hor[cnt] = filter_x[cnt];
filt_ver[cnt] = filter_y[cnt];
}
if (vpx_get_filter_taps(filter_x) == 2 &&
vpx_get_filter_taps(filter_y) == 2) {
switch (w) {
case 4:
common_hv_2ht_2vt_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_hor[3],
&filt_ver[3], h);
break;
case 8:
common_hv_2ht_2vt_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_hor[3],
&filt_ver[3], h);
break;
case 16:
common_hv_2ht_2vt_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride,
&filt_hor[3], &filt_ver[3], h);
break;
case 32:
common_hv_2ht_2vt_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride,
&filt_hor[3], &filt_ver[3], h);
break;
case 64:
common_hv_2ht_2vt_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride,
&filt_hor[3], &filt_ver[3], h);
break;
default:
vpx_convolve8_avg_c(src, src_stride, dst, dst_stride, filter, x0_q4,
x_step_q4, y0_q4, y_step_q4, w, h);
break;
}
} else if (vpx_get_filter_taps(filter_x) == 2 ||
vpx_get_filter_taps(filter_y) == 2) {
vpx_convolve8_avg_c(src, src_stride, dst, dst_stride, filter, x0_q4,
x_step_q4, y0_q4, y_step_q4, w, h);
} else {
switch (w) {
case 4:
common_hv_8ht_8vt_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor,
filt_ver, h);
break;
case 8:
common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor,
filt_ver, h);
break;
case 16:
common_hv_8ht_8vt_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor,
filt_ver, h);
break;
case 32:
common_hv_8ht_8vt_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor,
filt_ver, h);
break;
case 64:
common_hv_8ht_8vt_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor,
filt_ver, h);
break;
default:
vpx_convolve8_avg_c(src, src_stride, dst, dst_stride, filter, x0_q4,
x_step_q4, y0_q4, y_step_q4, w, h);
break;
}
}
}
libs/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_vert_lsx.c
+918
View File
@@ -0,0 +1,918 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <assert.h>
#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/loongarch/vpx_convolve_lsx.h"
static void common_vt_8t_and_aver_dst_4w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
const int8_t *filter,
int32_t height) {
uint32_t loop_cnt = (height >> 2);
uint8_t *dst_tmp = dst;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
__m128i reg0, reg1, reg2, reg3, reg4;
__m128i filter0, filter1, filter2, filter3;
__m128i out0, out1;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride + src_stride2;
int32_t src_stride4 = src_stride2 << 1;
uint8_t *src_tmp0 = (uint8_t *)src - src_stride3;
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
src0 = __lsx_vld(src_tmp0, 0);
DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src1,
src2);
src3 = __lsx_vldx(src_tmp0, src_stride3);
src_tmp0 += src_stride4;
src4 = __lsx_vld(src_tmp0, 0);
DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src5,
src6);
src_tmp0 += src_stride3;
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, tmp0,
tmp1, tmp2, tmp3);
DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, tmp4, tmp5);
DUP2_ARG2(__lsx_vilvl_d, tmp3, tmp0, tmp4, tmp1, reg0, reg1);
reg2 = __lsx_vilvl_d(tmp5, tmp2);
DUP2_ARG2(__lsx_vxori_b, reg0, 128, reg1, 128, reg0, reg1);
reg2 = __lsx_vxori_b(reg2, 128);
for (; loop_cnt--;) {
src7 = __lsx_vld(src_tmp0, 0);
DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src8,
src9);
src10 = __lsx_vldx(src_tmp0, src_stride3);
src_tmp0 += src_stride4;
src0 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
src1 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
src2 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
src3 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_w, src1, src0, src3, src2, src0, src1);
src0 = __lsx_vilvl_d(src1, src0);
DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9,
tmp0, tmp1, tmp2, tmp3);
DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, reg3, reg4);
DUP2_ARG2(__lsx_vxori_b, reg3, 128, reg4, 128, reg3, reg4);
out0 = filt_8tap_dpadd_s_h(reg0, reg1, reg2, reg3, filter0, filter1,
filter2, filter3);
out1 = filt_8tap_dpadd_s_h(reg1, reg2, reg3, reg4, filter0, filter1,
filter2, filter3);
out0 = __lsx_vssrarni_b_h(out1, out0, 7);
out0 = __lsx_vxori_b(out0, 128);
out0 = __lsx_vavgr_bu(out0, src0);
__lsx_vstelm_w(out0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 3);
dst += dst_stride;
reg0 = reg2;
reg1 = reg3;
reg2 = reg4;
src6 = src10;
}
}
static void common_vt_8t_and_aver_dst_8w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
const int8_t *filter,
int32_t height) {
uint32_t loop_cnt = height >> 2;
uint8_t *dst_tmp = dst;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
__m128i tmp0, tmp1, tmp2, tmp3;
__m128i reg0, reg1, reg2, reg3, reg4, reg5;
__m128i filter0, filter1, filter2, filter3;
__m128i out0, out1, out2, out3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride + src_stride2;
int32_t src_stride4 = src_stride2 << 1;
uint8_t *src_tmp0 = (uint8_t *)src - src_stride3;
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
src0 = __lsx_vld(src_tmp0, 0);
DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src1,
src2);
src3 = __lsx_vldx(src_tmp0, src_stride3);
src_tmp0 += src_stride4;
src4 = __lsx_vld(src_tmp0, 0);
DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src5,
src6);
src_tmp0 += src_stride3;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5);
src6 = __lsx_vxori_b(src6, 128);
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, reg0,
reg1, reg2, reg3);
DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5);
for (; loop_cnt--;) {
src7 = __lsx_vld(src_tmp0, 0);
DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src8,
src9);
src10 = __lsx_vldx(src_tmp0, src_stride3);
src_tmp0 += src_stride4;
src0 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
src1 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
src2 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
src3 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_d, src1, src0, src3, src2, src0, src1);
DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7,
src8, src9, src10);
DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9,
tmp0, tmp1, tmp2, tmp3);
out0 = filt_8tap_dpadd_s_h(reg0, reg1, reg2, tmp0, filter0, filter1,
filter2, filter3);
out1 = filt_8tap_dpadd_s_h(reg3, reg4, reg5, tmp1, filter0, filter1,
filter2, filter3);
out2 = filt_8tap_dpadd_s_h(reg1, reg2, tmp0, tmp2, filter0, filter1,
filter2, filter3);
out3 = filt_8tap_dpadd_s_h(reg4, reg5, tmp1, tmp3, filter0, filter1,
filter2, filter3);
DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1);
DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1);
DUP2_ARG2(__lsx_vavgr_bu, out0, src0, out1, src1, out0, out1);
__lsx_vstelm_d(out0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 1);
dst += dst_stride;
reg0 = reg2;
reg1 = tmp0;
reg2 = tmp2;
reg3 = reg5;
reg4 = tmp1;
reg5 = tmp3;
src6 = src10;
}
}
static void common_vt_8t_and_aver_dst_16w_mult_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
const int8_t *filter, int32_t height, int32_t width) {
uint8_t *src_tmp;
uint32_t cnt = width >> 4;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
__m128i filter0, filter1, filter2, filter3;
__m128i reg0, reg1, reg2, reg3, reg4, reg5;
__m128i reg6, reg7, reg8, reg9, reg10, reg11;
__m128i tmp0, tmp1, tmp2, tmp3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride + src_stride2;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
int32_t dst_stride4 = dst_stride2 << 1;
uint8_t *src_tmp0 = (uint8_t *)src - src_stride3;
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
for (; cnt--;) {
uint32_t loop_cnt = height >> 2;
uint8_t *dst_reg = dst;
src_tmp = src_tmp0;
src0 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src1,
src2);
src3 = __lsx_vldx(src_tmp, src_stride3);
src_tmp += src_stride4;
src4 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5,
src6);
src_tmp += src_stride3;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5);
src6 = __lsx_vxori_b(src6, 128);
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1,
reg0, reg1, reg2, reg3);
DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5);
DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1,
reg6, reg7, reg8, reg9);
DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, reg10, reg11);
for (; loop_cnt--;) {
src7 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src8,
src9);
src10 = __lsx_vldx(src_tmp, src_stride3);
src_tmp += src_stride4;
DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128,
src7, src8, src9, src10);
DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9,
src0, src1, src2, src3);
DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, src9,
src4, src5, src7, src8);
tmp0 = filt_8tap_dpadd_s_h(reg0, reg1, reg2, src0, filter0, filter1,
filter2, filter3);
tmp1 = filt_8tap_dpadd_s_h(reg3, reg4, reg5, src1, filter0, filter1,
filter2, filter3);
tmp2 = filt_8tap_dpadd_s_h(reg6, reg7, reg8, src4, filter0, filter1,
filter2, filter3);
tmp3 = filt_8tap_dpadd_s_h(reg9, reg10, reg11, src5, filter0, filter1,
filter2, filter3);
DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1);
DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1);
tmp2 = __lsx_vld(dst_reg, 0);
tmp3 = __lsx_vldx(dst_reg, dst_stride);
DUP2_ARG2(__lsx_vavgr_bu, tmp0, tmp2, tmp1, tmp3, tmp0, tmp1);
__lsx_vst(tmp0, dst_reg, 0);
__lsx_vstx(tmp1, dst_reg, dst_stride);
tmp0 = filt_8tap_dpadd_s_h(reg1, reg2, src0, src2, filter0, filter1,
filter2, filter3);
tmp1 = filt_8tap_dpadd_s_h(reg4, reg5, src1, src3, filter0, filter1,
filter2, filter3);
tmp2 = filt_8tap_dpadd_s_h(reg7, reg8, src4, src7, filter0, filter1,
filter2, filter3);
tmp3 = filt_8tap_dpadd_s_h(reg10, reg11, src5, src8, filter0, filter1,
filter2, filter3);
DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1);
DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1);
tmp2 = __lsx_vldx(dst_reg, dst_stride2);
tmp3 = __lsx_vldx(dst_reg, dst_stride3);
DUP2_ARG2(__lsx_vavgr_bu, tmp0, tmp2, tmp1, tmp3, tmp0, tmp1);
__lsx_vstx(tmp0, dst_reg, dst_stride2);
__lsx_vstx(tmp1, dst_reg, dst_stride3);
dst_reg += dst_stride4;
reg0 = reg2;
reg1 = src0;
reg2 = src2;
reg3 = reg5;
reg4 = src1;
reg5 = src3;
reg6 = reg8;
reg7 = src4;
reg8 = src7;
reg9 = reg11;
reg10 = src5;
reg11 = src8;
src6 = src10;
}
src_tmp0 += 16;
dst += 16;
}
}
static void common_vt_8t_and_aver_dst_16w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
const int8_t *filter,
int32_t height) {
common_vt_8t_and_aver_dst_16w_mult_lsx(src, src_stride, dst, dst_stride,
filter, height, 16);
}
static void common_vt_8t_and_aver_dst_32w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
const int8_t *filter,
int32_t height) {
common_vt_8t_and_aver_dst_16w_mult_lsx(src, src_stride, dst, dst_stride,
filter, height, 32);
}
static void common_vt_8t_and_aver_dst_64w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
const int8_t *filter,
int32_t height) {
common_vt_8t_and_aver_dst_16w_mult_lsx(src, src_stride, dst, dst_stride,
filter, height, 64);
}
static void common_vt_2t_and_aver_dst_4x4_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter) {
__m128i src0, src1, src2, src3, src4;
__m128i dst0, dst1, dst2, dst3, out, filt0, src2110, src4332;
__m128i src10_r, src32_r, src21_r, src43_r;
__m128i tmp0, tmp1;
uint8_t *dst_tmp = dst;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src += src_stride4;
src4 = __lsx_vld(src, 0);
src += src_stride;
dst0 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_w(dst_tmp, 0);
dst0 = __lsx_vilvl_w(dst1, dst0);
dst1 = __lsx_vilvl_w(dst3, dst2);
dst0 = __lsx_vilvl_d(dst1, dst0);
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3,
src10_r, src21_r, src32_r, src43_r);
DUP2_ARG2(__lsx_vilvl_d, src21_r, src10_r, src43_r, src32_r, src2110,
src4332);
DUP2_ARG2(__lsx_vdp2_h_bu, src2110, filt0, src4332, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
out = __lsx_vavgr_bu(tmp0, dst0);
__lsx_vstelm_w(out, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(out, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(out, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(out, dst, 0, 3);
dst += dst_stride;
}
static void common_vt_2t_and_aver_dst_4x8_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter) {
__m128i dst0, dst1, dst2, dst3, dst4;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src87_r;
__m128i src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
__m128i src2110, src4332, src6554, src8776, filt0;
__m128i tmp0, tmp1, tmp2, tmp3;
uint8_t *dst_tmp = dst;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src += src_stride4;
src4 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6);
src7 = __lsx_vldx(src, src_stride3);
src += src_stride4;
src8 = __lsx_vld(src, 0);
dst0 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst0 = __lsx_vilvl_w(dst1, dst0);
dst1 = __lsx_vilvl_w(dst3, dst2);
dst0 = __lsx_vilvl_d(dst1, dst0);
dst1 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_w(dst_tmp, 0);
dst_tmp += dst_stride;
dst4 = __lsx_vldrepl_w(dst_tmp, 0);
dst1 = __lsx_vilvl_w(dst2, dst1);
dst2 = __lsx_vilvl_w(dst4, dst3);
dst1 = __lsx_vilvl_d(dst2, dst1);
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3,
src10_r, src21_r, src32_r, src43_r);
DUP4_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src7, src6, src8, src7,
src54_r, src65_r, src76_r, src87_r);
DUP4_ARG2(__lsx_vilvl_d, src21_r, src10_r, src43_r, src32_r, src65_r, src54_r,
src87_r, src76_r, src2110, src4332, src6554, src8776);
DUP4_ARG2(__lsx_vdp2_h_bu, src2110, filt0, src4332, filt0, src6554, filt0,
src8776, filt0, tmp0, tmp1, tmp2, tmp3);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2,
FILTER_BITS, tmp0, tmp2);
DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp2, dst1, tmp0, tmp2);
__lsx_vstelm_w(tmp0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(tmp0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(tmp0, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(tmp0, dst, 0, 3);
dst += dst_stride;
__lsx_vstelm_w(tmp2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(tmp2, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(tmp2, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(tmp2, dst, 0, 3);
}
static void common_vt_2t_and_aver_dst_4w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride, int8_t *filter,
int32_t height) {
if (height == 4) {
common_vt_2t_and_aver_dst_4x4_lsx(src, src_stride, dst, dst_stride, filter);
} else if (height == 8) {
common_vt_2t_and_aver_dst_4x8_lsx(src, src_stride, dst, dst_stride, filter);
}
}
static void common_vt_2t_and_aver_dst_8x4_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter) {
__m128i src0, src1, src2, src3, src4;
__m128i dst0, dst1, dst2, dst3, vec0, vec1, vec2, vec3, filt0;
__m128i tmp0, tmp1, tmp2, tmp3;
uint8_t *dst_tmp = dst;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
dst0 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1);
DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec1);
DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0,
tmp0, tmp1, tmp2, tmp3);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2,
FILTER_BITS, tmp0, tmp2);
DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp2, dst1, tmp0, tmp2);
__lsx_vstelm_d(tmp0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(tmp0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(tmp2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(tmp2, dst, 0, 1);
}
static void common_vt_2t_and_aver_dst_8x8mult_lsx(
const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = (height >> 3);
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8;
__m128i dst0, dst1, dst2, dst3, dst4, dst5;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0;
__m128i tmp0, tmp1, tmp2, tmp3;
uint8_t *dst_tmp = dst;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
src += src_stride;
for (; loop_cnt--;) {
src1 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3);
src4 = __lsx_vldx(src, src_stride3);
src += src_stride4;
src5 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src6, src7);
src8 = __lsx_vldx(src, src_stride3);
src += src_stride4;
dst0 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst1 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst2 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1);
dst2 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst3 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst4 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
dst5 = __lsx_vldrepl_d(dst_tmp, 0);
dst_tmp += dst_stride;
DUP2_ARG2(__lsx_vilvl_d, dst3, dst2, dst5, dst4, dst2, dst3);
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3,
vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src7, src6, src8, src7,
vec4, vec5, vec6, vec7);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, tmp0, tmp1, tmp2, tmp3);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2,
FILTER_BITS, tmp0, tmp2);
DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp2, dst1, tmp0, tmp2);
__lsx_vstelm_d(tmp0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(tmp0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(tmp2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(tmp2, dst, 0, 1);
dst += dst_stride;
DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7,
filt0, tmp0, tmp1, tmp2, tmp3);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2,
FILTER_BITS, tmp0, tmp2);
DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst2, tmp2, dst3, tmp0, tmp2);
__lsx_vstelm_d(tmp0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(tmp0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(tmp2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(tmp2, dst, 0, 1);
dst += dst_stride;
src0 = src8;
}
}
static void common_vt_2t_and_aver_dst_8w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride, int8_t *filter,
int32_t height) {
if (height == 4) {
common_vt_2t_and_aver_dst_8x4_lsx(src, src_stride, dst, dst_stride, filter);
} else {
common_vt_2t_and_aver_dst_8x8mult_lsx(src, src_stride, dst, dst_stride,
filter, height);
}
}
static void common_vt_2t_and_aver_dst_16w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = (height >> 2);
__m128i src0, src1, src2, src3, src4, dst0, dst1, dst2, dst3, filt0;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
__m128i tmp0, tmp1;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
src += src_stride;
for (; loop_cnt--;) {
src1 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3);
src4 = __lsx_vldx(src, src_stride3);
src += src_stride4;
dst0 = __lsx_vld(dst, 0);
DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2);
dst3 = __lsx_vldx(dst, dst_stride3);
DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2);
DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst0);
__lsx_vst(tmp0, dst, 0);
dst += dst_stride;
DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6);
DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7);
DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst1);
__lsx_vst(tmp0, dst, 0);
dst += dst_stride;
DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst2);
__lsx_vst(tmp0, dst, 0);
dst += dst_stride;
DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst3);
__lsx_vst(tmp0, dst, 0);
dst += dst_stride;
src0 = src4;
}
}
static void common_vt_2t_and_aver_dst_32w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = (height >> 2);
uint8_t *src_tmp1;
uint8_t *dst_tmp1;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9;
__m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0;
__m128i tmp0, tmp1;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
filt0 = __lsx_vldrepl_h(filter, 0);
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src5);
src += src_stride;
for (; loop_cnt--;) {
src1 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3);
src4 = __lsx_vldx(src, src_stride3);
dst0 = __lsx_vld(dst, 0);
DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2);
dst3 = __lsx_vldx(dst, dst_stride3);
DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2);
DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3);
src_tmp1 = src + 16;
src6 = __lsx_vld(src_tmp1, 0);
DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src7,
src8);
src9 = __lsx_vldx(src_tmp1, src_stride3);
dst_tmp1 = dst + 16;
dst4 = __lsx_vld(dst_tmp1, 0);
DUP2_ARG2(__lsx_vldx, dst_tmp1, dst_stride, dst_tmp1, dst_stride2, dst5,
dst6);
dst7 = __lsx_vldx(dst_tmp1, dst_stride3);
src += src_stride4;
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst0);
__lsx_vst(tmp0, dst, 0);
DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst1);
__lsx_vstx(tmp0, dst, dst_stride);
DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6);
DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7);
DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst2);
__lsx_vstx(tmp0, dst, dst_stride2);
DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst3);
__lsx_vstx(tmp0, dst, dst_stride3);
DUP2_ARG2(__lsx_vilvl_b, src6, src5, src7, src6, vec0, vec2);
DUP2_ARG2(__lsx_vilvh_b, src6, src5, src7, src6, vec1, vec3);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst4);
__lsx_vst(tmp0, dst, 16);
DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst5);
dst += dst_stride;
__lsx_vst(tmp0, dst, 16);
DUP2_ARG2(__lsx_vilvl_b, src8, src7, src9, src8, vec4, vec6);
DUP2_ARG2(__lsx_vilvh_b, src8, src7, src9, src8, vec5, vec7);
DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst6);
dst += dst_stride;
__lsx_vst(tmp0, dst, 16);
DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst7);
dst += dst_stride;
__lsx_vst(tmp0, dst, 16);
dst += dst_stride;
src0 = src4;
src5 = src9;
}
}
static void common_vt_2t_and_aver_dst_64w_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = (height >> 1);
int32_t src_stride2 = src_stride << 1;
int32_t dst_stride2 = dst_stride << 1;
uint8_t *src_tmp1;
uint8_t *dst_tmp1;
__m128i src0, src1, src2, src3, src4, src5;
__m128i src6, src7, src8, src9, src10, src11, filt0;
__m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
__m128i tmp0, tmp1;
filt0 = __lsx_vldrepl_h(filter, 0);
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src3, src6,
src9);
src += src_stride;
for (; loop_cnt--;) {
src2 = __lsx_vldx(src, src_stride);
dst1 = __lsx_vldx(dst, dst_stride);
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src1, src4, src7,
src10);
DUP4_ARG2(__lsx_vld, dst, 0, dst, 16, dst, 32, dst, 48, dst0, dst2, dst4,
dst6);
src_tmp1 = (uint8_t *)src + 16;
src5 = __lsx_vldx(src_tmp1, src_stride);
src_tmp1 = src_tmp1 + 16;
src8 = __lsx_vldx(src_tmp1, src_stride);
src_tmp1 = src_tmp1 + 16;
src11 = __lsx_vldx(src_tmp1, src_stride);
dst_tmp1 = dst + 16;
dst3 = __lsx_vldx(dst_tmp1, dst_stride);
dst_tmp1 = dst + 32;
dst5 = __lsx_vldx(dst_tmp1, dst_stride);
dst_tmp1 = dst + 48;
dst7 = __lsx_vldx(dst_tmp1, dst_stride);
src += src_stride2;
DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2);
DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst0);
__lsx_vst(tmp0, dst, 0);
DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst1);
__lsx_vstx(tmp0, dst, dst_stride);
DUP2_ARG2(__lsx_vilvl_b, src4, src3, src5, src4, vec4, vec6);
DUP2_ARG2(__lsx_vilvh_b, src4, src3, src5, src4, vec5, vec7);
DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst2);
__lsx_vst(tmp0, dst, 16);
dst_tmp1 = dst + 16;
DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst3);
__lsx_vstx(tmp0, dst_tmp1, dst_stride);
DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, vec0, vec2);
DUP2_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, vec1, vec3);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst4);
__lsx_vst(tmp0, dst, 32);
dst_tmp1 = dst_tmp1 + 16;
DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst5);
__lsx_vstx(tmp0, dst_tmp1, dst_stride);
DUP2_ARG2(__lsx_vilvl_b, src10, src9, src11, src10, vec4, vec6);
DUP2_ARG2(__lsx_vilvh_b, src10, src9, src11, src10, vec5, vec7);
DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst6);
__lsx_vst(tmp0, dst, 48);
dst_tmp1 = dst_tmp1 + 16;
DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
tmp0 = __lsx_vavgr_bu(tmp0, dst7);
__lsx_vstx(tmp0, dst_tmp1, dst_stride);
dst += dst_stride2;
src0 = src2;
src3 = src5;
src6 = src8;
src9 = src11;
}
}
void vpx_convolve8_avg_vert_lsx(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const InterpKernel *filter, int x0_q4,
int x_step_q4, int y0_q4, int y_step_q4, int w,
int h) {
const int16_t *const filter_y = filter[y0_q4];
int8_t cnt, filt_ver[8];
assert(y_step_q4 == 16);
assert(((const int32_t *)filter_y)[1] != 0x800000);
for (cnt = 0; cnt < 8; ++cnt) {
filt_ver[cnt] = filter_y[cnt];
}
if (vpx_get_filter_taps(filter_y) == 2) {
switch (w) {
case 4:
common_vt_2t_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_ver[3], h);
break;
case 8:
common_vt_2t_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_ver[3], h);
break;
case 16:
common_vt_2t_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_ver[3], h);
break;
case 32:
common_vt_2t_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_ver[3], h);
break;
case 64:
common_vt_2t_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_ver[3], h);
break;
default:
vpx_convolve8_avg_vert_c(src, src_stride, dst, dst_stride, filter,
x0_q4, x_step_q4, y0_q4, y_step_q4, w, h);
break;
}
} else {
switch (w) {
case 4:
common_vt_8t_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_ver, h);
break;
case 8:
common_vt_8t_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_ver, h);
break;
case 16:
common_vt_8t_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_ver, h);
break;
case 32:
common_vt_8t_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_ver, h);
break;
case 64:
common_vt_8t_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_ver, h);
break;
default:
vpx_convolve8_avg_vert_c(src, src_stride, dst, dst_stride, filter,
x0_q4, x_step_q4, y0_q4, y_step_q4, w, h);
break;
}
}
}
libs/libvpx/vpx_dsp/loongarch/vpx_convolve8_horiz_lsx.c
+814
View File
@@ -0,0 +1,814 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <assert.h>
#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/loongarch/vpx_convolve_lsx.h"
static const uint8_t mc_filt_mask_arr[16 * 3] = {
/* 8 width cases */
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
/* 4 width cases */
0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20,
/* 4 width cases */
8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28
};
static void common_hz_8t_4x4_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
const int8_t *filter) {
__m128i src0, src1, src2, src3;
__m128i filter0, filter1, filter2, filter3;
__m128i mask0, mask1, mask2, mask3;
__m128i out, out0, out1;
mask0 = __lsx_vld(mc_filt_mask_arr, 16);
src -= 3;
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
LSX_LD_4(src, src_stride, src0, src1, src2, src3);
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3,
filter0, filter1, filter2, filter3, out0, out1);
out = __lsx_vssrarni_b_h(out1, out0, 7);
out = __lsx_vxori_b(out, 128);
__lsx_vstelm_w(out, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(out, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(out, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(out, dst, 0, 3);
}
static void common_hz_8t_4x8_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
const int8_t *filter) {
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride + src_stride2;
int32_t src_stride4 = src_stride2 << 1;
__m128i src0, src1, src2, src3;
__m128i filter0, filter1, filter2, filter3;
__m128i mask0, mask1, mask2, mask3;
__m128i out0, out1, out2, out3;
uint8_t *_src = (uint8_t *)src - 3;
mask0 = __lsx_vld(mc_filt_mask_arr, 16);
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
src0 = __lsx_vld(_src, 0);
DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2);
src3 = __lsx_vldx(_src, src_stride3);
_src += src_stride4;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3,
filter0, filter1, filter2, filter3, out0, out1);
src0 = __lsx_vld(_src, 0);
DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2);
src3 = __lsx_vldx(_src, src_stride3);
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3,
filter0, filter1, filter2, filter3, out2, out3);
DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1);
DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1);
__lsx_vstelm_w(out0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 3);
dst += dst_stride;
__lsx_vstelm_w(out1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(out1, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(out1, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(out1, dst, 0, 3);
}
static void common_hz_8t_4w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
const int8_t *filter, int32_t height) {
if (height == 4) {
common_hz_8t_4x4_lsx(src, src_stride, dst, dst_stride, filter);
} else if (height == 8) {
common_hz_8t_4x8_lsx(src, src_stride, dst, dst_stride, filter);
}
}
static void common_hz_8t_8x4_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
const int8_t *filter) {
__m128i src0, src1, src2, src3;
__m128i filter0, filter1, filter2, filter3;
__m128i mask0, mask1, mask2, mask3;
__m128i out0, out1, out2, out3;
mask0 = __lsx_vld(mc_filt_mask_arr, 0);
src -= 3;
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
LSX_LD_4(src, src_stride, src0, src1, src2, src3);
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3,
filter0, filter1, filter2, filter3, out0, out1,
out2, out3);
DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1);
DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1);
__lsx_vstelm_d(out0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 1);
}
static void common_hz_8t_8x8mult_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
const int8_t *filter, int32_t height) {
uint32_t loop_cnt = height >> 2;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride + src_stride2;
int32_t src_stride4 = src_stride2 << 1;
__m128i src0, src1, src2, src3;
__m128i filter0, filter1, filter2, filter3;
__m128i mask0, mask1, mask2, mask3;
__m128i out0, out1, out2, out3;
uint8_t *_src = (uint8_t *)src - 3;
mask0 = __lsx_vld(mc_filt_mask_arr, 0);
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
for (; loop_cnt--;) {
src0 = __lsx_vld(_src, 0);
DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2);
src3 = __lsx_vldx(_src, src_stride3);
_src += src_stride4;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
mask3, filter0, filter1, filter2, filter3, out0,
out1, out2, out3);
DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1);
DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1);
__lsx_vstelm_d(out0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 1);
dst += dst_stride;
}
}
static void common_hz_8t_8w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
const int8_t *filter, int32_t height) {
if (height == 4) {
common_hz_8t_8x4_lsx(src, src_stride, dst, dst_stride, filter);
} else {
common_hz_8t_8x8mult_lsx(src, src_stride, dst, dst_stride, filter, height);
}
}
static void common_hz_8t_16w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
const int8_t *filter, int32_t height) {
uint32_t loop_cnt = height >> 1;
int32_t stride = src_stride << 1;
__m128i src0, src1, src2, src3;
__m128i filter0, filter1, filter2, filter3;
__m128i mask0, mask1, mask2, mask3;
__m128i out0, out1, out2, out3;
mask0 = __lsx_vld(mc_filt_mask_arr, 0);
src -= 3;
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
for (; loop_cnt--;) {
const uint8_t *_src = src + src_stride;
DUP2_ARG2(__lsx_vld, src, 0, _src, 0, src0, src2);
DUP2_ARG2(__lsx_vld, src, 8, _src, 8, src1, src3);
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
mask3, filter0, filter1, filter2, filter3, out0,
out1, out2, out3);
DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1);
DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1);
__lsx_vst(out0, dst, 0);
dst += dst_stride;
__lsx_vst(out1, dst, 0);
dst += dst_stride;
src += stride;
}
}
static void common_hz_8t_32w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
const int8_t *filter, int32_t height) {
uint32_t loop_cnt = height >> 1;
__m128i src0, src1, src2, src3;
__m128i filter0, filter1, filter2, filter3;
__m128i mask0, mask1, mask2, mask3;
__m128i out0, out1, out2, out3;
__m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 };
mask0 = __lsx_vld(mc_filt_mask_arr, 0);
src -= 3;
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
for (; loop_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2);
src3 = __lsx_vld(src, 24);
src1 = __lsx_vshuf_b(src2, src0, shuff);
src += src_stride;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
mask3, filter0, filter1, filter2, filter3, out0,
out1, out2, out3);
DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1);
DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1);
__lsx_vst(out0, dst, 0);
__lsx_vst(out1, dst, 16);
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2);
src3 = __lsx_vld(src, 24);
src1 = __lsx_vshuf_b(src2, src0, shuff);
src += src_stride;
dst += dst_stride;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
mask3, filter0, filter1, filter2, filter3, out0,
out1, out2, out3);
DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1);
DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1);
__lsx_vst(out0, dst, 0);
__lsx_vst(out1, dst, 16);
dst += dst_stride;
}
}
static void common_hz_8t_64w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
const int8_t *filter, int32_t height) {
int32_t loop_cnt = height;
__m128i src0, src1, src2, src3;
__m128i filter0, filter1, filter2, filter3;
__m128i mask0, mask1, mask2, mask3;
__m128i out0, out1, out2, out3;
__m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 };
mask0 = __lsx_vld(mc_filt_mask_arr, 0);
src -= 3;
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
for (; loop_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2);
src3 = __lsx_vld(src, 24);
src1 = __lsx_vshuf_b(src2, src0, shuff);
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
mask3, filter0, filter1, filter2, filter3, out0,
out1, out2, out3);
DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1);
DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1);
__lsx_vst(out0, dst, 0);
__lsx_vst(out1, dst, 16);
DUP2_ARG2(__lsx_vld, src, 32, src, 48, src0, src2);
src3 = __lsx_vld(src, 56);
src1 = __lsx_vshuf_b(src2, src0, shuff);
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
mask3, filter0, filter1, filter2, filter3, out0,
out1, out2, out3);
DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1);
DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1);
__lsx_vst(out0, dst, 32);
__lsx_vst(out1, dst, 48);
src += src_stride;
dst += dst_stride;
}
}
static void common_hz_2t_4x4_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter) {
__m128i src0, src1, src2, src3, mask;
__m128i filt0, vec0, vec1, vec2, vec3, res0, res1;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride + src_stride2;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride + dst_stride2;
mask = __lsx_vld(mc_filt_mask_arr, 16);
/* rearranging filter */
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
DUP2_ARG3(__lsx_vshuf_b, src1, src0, mask, src3, src2, mask, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, vec3);
DUP2_ARG3(__lsx_vssrarni_bu_h, vec2, vec2, FILTER_BITS, vec3, vec3,
FILTER_BITS, res0, res1);
__lsx_vstelm_w(res0, dst, 0, 0);
__lsx_vstelm_w(res0, dst + dst_stride, 0, 1);
__lsx_vstelm_w(res1, dst + dst_stride2, 0, 0);
__lsx_vstelm_w(res1, dst + dst_stride3, 0, 1);
}
static void common_hz_2t_4x8_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter) {
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, mask;
__m128i res0, res1, res2, res3, filt0;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride + src_stride2;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride + dst_stride2;
uint8_t *src_tmp1 = src + src_stride4;
mask = __lsx_vld(mc_filt_mask_arr, 16);
/* rearranging filter */
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src5,
src6);
src7 = __lsx_vldx(src_tmp1, src_stride3);
DUP4_ARG3(__lsx_vshuf_b, src1, src0, mask, src3, src2, mask, src5, src4, mask,
src7, src6, mask, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0,
vec4, vec5, vec6, vec7);
DUP4_ARG3(__lsx_vssrarni_bu_h, vec4, vec4, FILTER_BITS, vec5, vec5,
FILTER_BITS, vec6, vec6, FILTER_BITS, vec7, vec7, FILTER_BITS, res0,
res1, res2, res3);
__lsx_vstelm_w(res0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(res0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(res1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(res1, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(res2, dst, 0, 0);
__lsx_vstelm_w(res2, dst + dst_stride, 0, 1);
__lsx_vstelm_w(res3, dst + dst_stride2, 0, 0);
__lsx_vstelm_w(res3, dst + dst_stride3, 0, 1);
}
static void common_hz_2t_4w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
if (height == 4) {
common_hz_2t_4x4_lsx(src, src_stride, dst, dst_stride, filter);
} else if (height == 8) {
common_hz_2t_4x8_lsx(src, src_stride, dst, dst_stride, filter);
}
}
static void common_hz_2t_8x4_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter) {
__m128i filt0, mask;
__m128i src0, src1, src2, src3;
__m128i vec0, vec1, vec2, vec3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
mask = __lsx_vld(mc_filt_mask_arr, 0);
/* rearranging filter */
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, mask,
src3, src3, mask, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0,
vec0, vec1, vec2, vec3);
DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2,
FILTER_BITS, vec0, vec1);
__lsx_vstelm_d(vec0, dst, 0, 0);
__lsx_vstelm_d(vec0, dst + dst_stride, 0, 1);
__lsx_vstelm_d(vec1, dst + dst_stride2, 0, 0);
__lsx_vstelm_d(vec1, dst + dst_stride3, 0, 1);
}
static void common_hz_2t_8x8mult_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
__m128i filt0, mask;
__m128i src0, src1, src2, src3, out0, out1;
__m128i vec0, vec1, vec2, vec3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
int32_t dst_stride4 = dst_stride2 << 1;
mask = __lsx_vld(mc_filt_mask_arr, 0);
/* rearranging filter */
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src += src_stride4;
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, mask,
src3, src3, mask, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0,
vec0, vec1, vec2, vec3);
DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2,
FILTER_BITS, out0, out1);
__lsx_vstelm_d(out0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 1);
dst += dst_stride;
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src += src_stride4;
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, mask,
src3, src3, mask, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0,
vec0, vec1, vec2, vec3);
DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2,
FILTER_BITS, out0, out1);
__lsx_vstelm_d(out0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 1);
dst += dst_stride;
if (height == 16) {
uint8_t *dst_tmp1 = dst + dst_stride4;
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src += src_stride4;
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2,
mask, src3, src3, mask, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, vec0, vec1, vec2, vec3);
DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2,
FILTER_BITS, out0, out1);
__lsx_vstelm_d(out0, dst, 0, 0);
__lsx_vstelm_d(out0, dst + dst_stride, 0, 1);
__lsx_vstelm_d(out1, dst + dst_stride2, 0, 0);
__lsx_vstelm_d(out1, dst + dst_stride3, 0, 1);
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src += src_stride4;
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2,
mask, src3, src3, mask, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, vec0, vec1, vec2, vec3);
DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2,
FILTER_BITS, out0, out1);
__lsx_vstelm_d(out0, dst_tmp1, 0, 0);
__lsx_vstelm_d(out0, dst_tmp1 + dst_stride, 0, 1);
__lsx_vstelm_d(out1, dst_tmp1 + dst_stride2, 0, 0);
__lsx_vstelm_d(out1, dst_tmp1 + dst_stride3, 0, 1);
}
}
static void common_hz_2t_8w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
if (height == 4) {
common_hz_2t_8x4_lsx(src, src_stride, dst, dst_stride, filter);
} else {
common_hz_2t_8x8mult_lsx(src, src_stride, dst, dst_stride, filter, height);
}
}
static void common_hz_2t_16w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = (height >> 2) - 1;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, mask;
__m128i filt0, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
__m128i out0, out1, out2, out3, out4, out5, out6, out7;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
uint8_t *src_tmp1 = src + 8;
mask = __lsx_vld(mc_filt_mask_arr, 0);
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4);
src6 = __lsx_vldx(src, src_stride3);
src1 = __lsx_vld(src_tmp1, 0);
DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3,
src5);
src7 = __lsx_vldx(src_tmp1, src_stride3);
src += src_stride4;
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, mask,
src3, src3, mask, vec0, vec1, vec2, vec3);
DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, src6, src6, mask,
src7, src7, mask, vec4, vec5, vec6, vec7);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0,
out0, out1, out2, out3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, filt0,
out4, out5, out6, out7);
DUP4_ARG3(__lsx_vssrarni_bu_h, out1, out0, FILTER_BITS, out3, out2,
FILTER_BITS, out5, out4, FILTER_BITS, out7, out6, FILTER_BITS, out0,
out1, out2, out3);
__lsx_vst(out0, dst, 0);
dst += dst_stride;
__lsx_vst(out1, dst, 0);
dst += dst_stride;
__lsx_vst(out2, dst, 0);
dst += dst_stride;
__lsx_vst(out3, dst, 0);
dst += dst_stride;
for (; loop_cnt--;) {
src_tmp1 += src_stride4;
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4);
src6 = __lsx_vldx(src, src_stride3);
src1 = __lsx_vld(src_tmp1, 0);
DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3,
src5);
src7 = __lsx_vldx(src_tmp1, src_stride3);
src += src_stride4;
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2,
mask, src3, src3, mask, vec0, vec1, vec2, vec3);
DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, src6, src6,
mask, src7, src7, mask, vec4, vec5, vec6, vec7);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, out0, out1, out2, out3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7,
filt0, out4, out5, out6, out7);
DUP4_ARG3(__lsx_vssrarni_bu_h, out1, out0, FILTER_BITS, out3, out2,
FILTER_BITS, out5, out4, FILTER_BITS, out7, out6, FILTER_BITS,
out0, out1, out2, out3);
__lsx_vst(out0, dst, 0);
dst += dst_stride;
__lsx_vst(out1, dst, 0);
dst += dst_stride;
__lsx_vst(out2, dst, 0);
dst += dst_stride;
__lsx_vst(out3, dst, 0);
dst += dst_stride;
}
}
static void common_hz_2t_32w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = (height >> 1);
__m128i src0, src1, src2, src3, src4, src5, src6, src7, mask;
__m128i filt0, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
__m128i out0, out1, out2, out3, out4, out5, out6, out7;
__m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 };
mask = __lsx_vld(mc_filt_mask_arr, 0);
/* rearranging filter */
filt0 = __lsx_vldrepl_h(filter, 0);
for (; loop_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2);
src3 = __lsx_vld(src, 24);
src1 = __lsx_vshuf_b(src2, src0, shuff);
src += src_stride;
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src4, src6);
src7 = __lsx_vld(src, 24);
src5 = __lsx_vshuf_b(src6, src4, shuff);
src += src_stride;
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2,
mask, src3, src3, mask, vec0, vec1, vec2, vec3);
DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, src6, src6,
mask, src7, src7, mask, vec4, vec5, vec6, vec7);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, out0, out1, out2, out3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7,
filt0, out4, out5, out6, out7);
DUP4_ARG3(__lsx_vssrarni_bu_h, out1, out0, FILTER_BITS, out3, out2,
FILTER_BITS, out5, out4, FILTER_BITS, out7, out6, FILTER_BITS,
out0, out1, out2, out3);
__lsx_vst(out0, dst, 0);
__lsx_vst(out1, dst, 16);
dst += dst_stride;
__lsx_vst(out2, dst, 0);
__lsx_vst(out3, dst, 16);
dst += dst_stride;
}
}
static void common_hz_2t_64w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = height;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, mask;
__m128i filt0, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
__m128i out0, out1, out2, out3, out4, out5, out6, out7;
__m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 };
mask = __lsx_vld(mc_filt_mask_arr, 0);
/* rearranging filter */
filt0 = __lsx_vldrepl_h(filter, 0);
for (; loop_cnt--;) {
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src2, src4,
src6);
src7 = __lsx_vld(src, 56);
DUP2_ARG3(__lsx_vshuf_b, src2, src0, shuff, src4, src2, shuff, src1, src3);
src5 = __lsx_vshuf_b(src6, src4, shuff);
src += src_stride;
DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2,
mask, src3, src3, mask, vec0, vec1, vec2, vec3);
DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, src6, src6,
mask, src7, src7, mask, vec4, vec5, vec6, vec7);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, out0, out1, out2, out3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7,
filt0, out4, out5, out6, out7);
DUP4_ARG3(__lsx_vssrarni_bu_h, out1, out0, FILTER_BITS, out3, out2,
FILTER_BITS, out5, out4, FILTER_BITS, out7, out6, FILTER_BITS,
out0, out1, out2, out3);
__lsx_vst(out0, dst, 0);
__lsx_vst(out1, dst, 16);
__lsx_vst(out2, dst, 32);
__lsx_vst(out3, dst, 48);
dst += dst_stride;
}
}
void vpx_convolve8_horiz_lsx(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const InterpKernel *filter, int x0_q4,
int x_step_q4, int y0_q4, int y_step_q4, int w,
int h) {
const int16_t *const filter_x = filter[x0_q4];
int8_t cnt, filt_hor[8];
assert(x_step_q4 == 16);
assert(((const int32_t *)filter_x)[1] != 0x800000);
for (cnt = 0; cnt < 8; ++cnt) {
filt_hor[cnt] = filter_x[cnt];
}
if (vpx_get_filter_taps(filter_x) == 2) {
switch (w) {
case 4:
common_hz_2t_4w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
&filt_hor[3], h);
break;
case 8:
common_hz_2t_8w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
&filt_hor[3], h);
break;
case 16:
common_hz_2t_16w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
&filt_hor[3], h);
break;
case 32:
common_hz_2t_32w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
&filt_hor[3], h);
break;
case 64:
common_hz_2t_64w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
&filt_hor[3], h);
break;
default:
vpx_convolve8_horiz_c(src, src_stride, dst, dst_stride, filter, x0_q4,
x_step_q4, y0_q4, y_step_q4, w, h);
break;
}
} else {
switch (w) {
case 4:
common_hz_8t_4w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
filt_hor, h);
break;
case 8:
common_hz_8t_8w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
filt_hor, h);
break;
case 16:
common_hz_8t_16w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
filt_hor, h);
break;
case 32:
common_hz_8t_32w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
filt_hor, h);
break;
case 64:
common_hz_8t_64w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
filt_hor, h);
break;
default:
vpx_convolve8_horiz_c(src, src_stride, dst, dst_stride, filter, x0_q4,
x_step_q4, y0_q4, y_step_q4, w, h);
break;
}
}
}
libs/libvpx/vpx_dsp/loongarch/vpx_convolve8_lsx.c
+697
View File
@@ -0,0 +1,697 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <assert.h>
#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/loongarch/vpx_convolve_lsx.h"
static const uint8_t mc_filt_mask_arr[16 * 3] = {
/* 8 width cases */
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
/* 4 width cases */
0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20,
/* 4 width cases */
8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28
};
static void common_hv_8ht_8vt_4w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert,
int32_t height) {
uint32_t loop_cnt = (height >> 2);
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
__m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3;
__m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3;
__m128i mask0, mask1, mask2, mask3;
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
__m128i out0, out1;
__m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 };
mask0 = __lsx_vld(mc_filt_mask_arr, 16);
src -= (3 + 3 * src_stride);
DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4,
filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3);
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
LSX_LD_4(src, src_stride, src0, src1, src2, src3);
src += src_stride;
src4 = __lsx_vld(src, 0);
src += src_stride;
src5 = __lsx_vld(src, 0);
src += src_stride;
src6 = __lsx_vld(src, 0);
src += src_stride;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5);
src6 = __lsx_vxori_b(src6, 128);
tmp0 = horiz_8tap_filt(src0, src1, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
tmp2 = horiz_8tap_filt(src2, src3, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
tmp4 = horiz_8tap_filt(src4, src5, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
tmp5 = horiz_8tap_filt(src5, src6, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
DUP2_ARG3(__lsx_vshuf_b, tmp2, tmp0, shuff, tmp4, tmp2, shuff, tmp1, tmp3);
DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4,
filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3);
DUP2_ARG2(__lsx_vpackev_b, tmp1, tmp0, tmp3, tmp2, tmp0, tmp1);
tmp2 = __lsx_vpackev_b(tmp5, tmp4);
for (; loop_cnt--;) {
LSX_LD_4(src, src_stride, src7, src8, src9, src10);
src += src_stride;
DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7,
src8, src9, src10);
tmp3 = horiz_8tap_filt(src7, src8, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
tmp4 = __lsx_vshuf_b(tmp3, tmp5, shuff);
tmp4 = __lsx_vpackev_b(tmp3, tmp4);
out0 = filt_8tap_dpadd_s_h(tmp0, tmp1, tmp2, tmp4, filt_vt0, filt_vt1,
filt_vt2, filt_vt3);
src1 = horiz_8tap_filt(src9, src10, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src0 = __lsx_vshuf_b(src1, tmp3, shuff);
src0 = __lsx_vpackev_b(src1, src0);
out1 = filt_8tap_dpadd_s_h(tmp1, tmp2, tmp4, src0, filt_vt0, filt_vt1,
filt_vt2, filt_vt3);
out0 = __lsx_vssrarni_b_h(out1, out0, 7);
out0 = __lsx_vxori_b(out0, 128);
__lsx_vstelm_w(out0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 3);
dst += dst_stride;
tmp5 = src1;
tmp0 = tmp2;
tmp1 = tmp4;
tmp2 = src0;
}
}
static void common_hv_8ht_8vt_8w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert,
int32_t height) {
uint32_t loop_cnt = (height >> 2);
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
__m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3;
__m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3;
__m128i mask0, mask1, mask2, mask3;
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6;
__m128i out0, out1;
mask0 = __lsx_vld(mc_filt_mask_arr, 0);
src -= (3 + 3 * src_stride);
DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4,
filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3);
DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
mask3 = __lsx_vaddi_bu(mask0, 6);
LSX_LD_4(src, src_stride, src0, src1, src2, src3);
src += src_stride;
src4 = __lsx_vld(src, 0);
src += src_stride;
src5 = __lsx_vld(src, 0);
src += src_stride;
src6 = __lsx_vld(src, 0);
src += src_stride;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5);
src6 = __lsx_vxori_b(src6, 128);
src0 = horiz_8tap_filt(src0, src0, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src1 = horiz_8tap_filt(src1, src1, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src2 = horiz_8tap_filt(src2, src2, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src3 = horiz_8tap_filt(src3, src3, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src4 = horiz_8tap_filt(src4, src4, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src5 = horiz_8tap_filt(src5, src5, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src6 = horiz_8tap_filt(src6, src6, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4,
filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3);
DUP4_ARG2(__lsx_vpackev_b, src1, src0, src3, src2, src5, src4, src2, src1,
tmp0, tmp1, tmp2, tmp4);
DUP2_ARG2(__lsx_vpackev_b, src4, src3, src6, src5, tmp5, tmp6);
for (; loop_cnt--;) {
LSX_LD_4(src, src_stride, src7, src8, src9, src10);
src += src_stride;
DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7,
src8, src9, src10);
src7 = horiz_8tap_filt(src7, src7, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
tmp3 = __lsx_vpackev_b(src7, src6);
out0 = filt_8tap_dpadd_s_h(tmp0, tmp1, tmp2, tmp3, filt_vt0, filt_vt1,
filt_vt2, filt_vt3);
src8 = horiz_8tap_filt(src8, src8, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src0 = __lsx_vpackev_b(src8, src7);
out1 = filt_8tap_dpadd_s_h(tmp4, tmp5, tmp6, src0, filt_vt0, filt_vt1,
filt_vt2, filt_vt3);
src9 = horiz_8tap_filt(src9, src9, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src1 = __lsx_vpackev_b(src9, src8);
src3 = filt_8tap_dpadd_s_h(tmp1, tmp2, tmp3, src1, filt_vt0, filt_vt1,
filt_vt2, filt_vt3);
src10 = horiz_8tap_filt(src10, src10, mask0, mask1, mask2, mask3, filt_hz0,
filt_hz1, filt_hz2, filt_hz3);
src2 = __lsx_vpackev_b(src10, src9);
src4 = filt_8tap_dpadd_s_h(tmp5, tmp6, src0, src2, filt_vt0, filt_vt1,
filt_vt2, filt_vt3);
DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, src4, src3, 7, out0, out1);
DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1);
__lsx_vstelm_d(out0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 1);
dst += dst_stride;
src6 = src10;
tmp0 = tmp2;
tmp1 = tmp3;
tmp2 = src1;
tmp4 = tmp6;
tmp5 = src0;
tmp6 = src2;
}
}
static void common_hv_8ht_8vt_16w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert,
int32_t height) {
common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz,
filter_vert, height);
src += 8;
dst += 8;
common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz,
filter_vert, height);
src += 8;
dst += 8;
}
static void common_hv_8ht_8vt_32w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert,
int32_t height) {
int32_t multiple8_cnt;
for (multiple8_cnt = 4; multiple8_cnt--;) {
common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz,
filter_vert, height);
src += 8;
dst += 8;
}
}
static void common_hv_8ht_8vt_64w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert,
int32_t height) {
int32_t multiple8_cnt;
for (multiple8_cnt = 8; multiple8_cnt--;) {
common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz,
filter_vert, height);
src += 8;
dst += 8;
}
}
static void common_hv_2ht_2vt_4x4_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz,
int8_t *filter_vert) {
__m128i src0, src1, src2, src3, src4, mask;
__m128i filt_vt, filt_hz, vec0, vec1;
__m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, tmp0, tmp1;
__m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 };
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride + src_stride2;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
mask = __lsx_vld(mc_filt_mask_arr, 16);
/* rearranging filter */
filt_hz = __lsx_vldrepl_h(filter_horiz, 0);
filt_vt = __lsx_vldrepl_h(filter_vert, 0);
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
hz_out0 = horiz_2tap_filt_uh(src0, src1, mask, filt_hz);
hz_out2 = horiz_2tap_filt_uh(src2, src3, mask, filt_hz);
hz_out4 = horiz_2tap_filt_uh(src4, src4, mask, filt_hz);
hz_out1 = __lsx_vshuf_b(hz_out2, hz_out0, shuff);
hz_out3 = __lsx_vpickod_d(hz_out4, hz_out2);
DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp0, tmp0, FILTER_BITS, tmp1, tmp1,
FILTER_BITS, tmp0, tmp1);
__lsx_vstelm_w(tmp0, dst, 0, 0);
__lsx_vstelm_w(tmp0, dst + dst_stride, 0, 1);
__lsx_vstelm_w(tmp1, dst + dst_stride2, 0, 0);
__lsx_vstelm_w(tmp1, dst + dst_stride3, 0, 1);
}
static void common_hv_2ht_2vt_4x8_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz,
int8_t *filter_vert) {
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, mask;
__m128i filt_hz, filt_vt, vec0, vec1, vec2, vec3;
__m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
__m128i hz_out7, hz_out8, vec4, vec5, vec6, vec7;
__m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 };
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
int32_t dst_stride4 = dst_stride2 << 1;
mask = __lsx_vld(mc_filt_mask_arr, 16);
/* rearranging filter */
DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt);
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
src += src_stride4;
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src5, src6, src7, src8);
src += src_stride4;
hz_out0 = horiz_2tap_filt_uh(src0, src1, mask, filt_hz);
hz_out2 = horiz_2tap_filt_uh(src2, src3, mask, filt_hz);
hz_out4 = horiz_2tap_filt_uh(src4, src5, mask, filt_hz);
hz_out6 = horiz_2tap_filt_uh(src6, src7, mask, filt_hz);
hz_out8 = horiz_2tap_filt_uh(src8, src8, mask, filt_hz);
DUP2_ARG3(__lsx_vshuf_b, hz_out2, hz_out0, shuff, hz_out4, hz_out2, shuff,
hz_out1, hz_out3);
hz_out5 = __lsx_vshuf_b(hz_out6, hz_out4, shuff);
hz_out7 = __lsx_vpickod_d(hz_out8, hz_out6);
DUP4_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, hz_out5,
hz_out4, hz_out7, hz_out6, vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, vec2, filt_vt, vec3,
filt_vt, vec4, vec5, vec6, vec7);
DUP4_ARG3(__lsx_vssrarni_bu_h, vec4, vec4, FILTER_BITS, vec5, vec5,
FILTER_BITS, vec6, vec6, FILTER_BITS, vec7, vec7, FILTER_BITS, vec4,
vec5, vec6, vec7);
__lsx_vstelm_w(vec4, dst, 0, 0);
__lsx_vstelm_w(vec4, dst + dst_stride, 0, 1);
__lsx_vstelm_w(vec5, dst + dst_stride2, 0, 0);
__lsx_vstelm_w(vec5, dst + dst_stride3, 0, 1);
dst += dst_stride4;
__lsx_vstelm_w(vec6, dst, 0, 0);
__lsx_vstelm_w(vec6, dst + dst_stride, 0, 1);
__lsx_vstelm_w(vec7, dst + dst_stride2, 0, 0);
__lsx_vstelm_w(vec7, dst + dst_stride3, 0, 1);
}
static void common_hv_2ht_2vt_4w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert,
int32_t height) {
if (height == 4) {
common_hv_2ht_2vt_4x4_lsx(src, src_stride, dst, dst_stride, filter_horiz,
filter_vert);
} else if (height == 8) {
common_hv_2ht_2vt_4x8_lsx(src, src_stride, dst, dst_stride, filter_horiz,
filter_vert);
}
}
static void common_hv_2ht_2vt_8x4_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz,
int8_t *filter_vert) {
__m128i src0, src1, src2, src3, src4, mask;
__m128i filt_hz, filt_vt, vec0, vec1, vec2, vec3;
__m128i hz_out0, hz_out1, tmp0, tmp1, tmp2, tmp3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
mask = __lsx_vld(mc_filt_mask_arr, 0);
/* rearranging filter */
DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt);
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
hz_out0 = horiz_2tap_filt_uh(src0, src0, mask, filt_hz);
hz_out1 = horiz_2tap_filt_uh(src1, src1, mask, filt_hz);
vec0 = __lsx_vpackev_b(hz_out1, hz_out0);
tmp0 = __lsx_vdp2_h_bu(vec0, filt_vt);
hz_out0 = horiz_2tap_filt_uh(src2, src2, mask, filt_hz);
vec1 = __lsx_vpackev_b(hz_out0, hz_out1);
tmp1 = __lsx_vdp2_h_bu(vec1, filt_vt);
hz_out1 = horiz_2tap_filt_uh(src3, src3, mask, filt_hz);
vec2 = __lsx_vpackev_b(hz_out1, hz_out0);
tmp2 = __lsx_vdp2_h_bu(vec2, filt_vt);
hz_out0 = horiz_2tap_filt_uh(src4, src4, mask, filt_hz);
vec3 = __lsx_vpackev_b(hz_out0, hz_out1);
tmp3 = __lsx_vdp2_h_bu(vec3, filt_vt);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2,
FILTER_BITS, tmp0, tmp1);
__lsx_vstelm_d(tmp0, dst, 0, 0);
__lsx_vstelm_d(tmp0, dst + dst_stride, 0, 1);
__lsx_vstelm_d(tmp1, dst + dst_stride2, 0, 0);
__lsx_vstelm_d(tmp1, dst + dst_stride3, 0, 1);
}
static void common_hv_2ht_2vt_8x8mult_lsx(const uint8_t *src,
int32_t src_stride, uint8_t *dst,
int32_t dst_stride,
int8_t *filter_horiz,
int8_t *filter_vert, int32_t height) {
uint32_t loop_cnt = (height >> 3);
__m128i src0, src1, src2, src3, src4, mask;
__m128i filt_hz, filt_vt, vec0;
__m128i hz_out0, hz_out1, tmp1, tmp2, tmp3, tmp4;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
mask = __lsx_vld(mc_filt_mask_arr, 0);
/* rearranging filter */
DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt);
src0 = __lsx_vld(src, 0);
src += src_stride;
hz_out0 = horiz_2tap_filt_uh(src0, src0, mask, filt_hz);
for (; loop_cnt--;) {
src1 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3);
src4 = __lsx_vldx(src, src_stride3);
src += src_stride4;
hz_out1 = horiz_2tap_filt_uh(src1, src1, mask, filt_hz);
vec0 = __lsx_vpackev_b(hz_out1, hz_out0);
tmp1 = __lsx_vdp2_h_bu(vec0, filt_vt);
hz_out0 = horiz_2tap_filt_uh(src2, src2, mask, filt_hz);
vec0 = __lsx_vpackev_b(hz_out0, hz_out1);
tmp2 = __lsx_vdp2_h_bu(vec0, filt_vt);
hz_out1 = horiz_2tap_filt_uh(src3, src3, mask, filt_hz);
vec0 = __lsx_vpackev_b(hz_out1, hz_out0);
tmp3 = __lsx_vdp2_h_bu(vec0, filt_vt);
hz_out0 = horiz_2tap_filt_uh(src4, src4, mask, filt_hz);
src1 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3);
src4 = __lsx_vldx(src, src_stride3);
src += src_stride4;
vec0 = __lsx_vpackev_b(hz_out0, hz_out1);
tmp4 = __lsx_vdp2_h_bu(vec0, filt_vt);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp2, tmp1, FILTER_BITS, tmp4, tmp3,
FILTER_BITS, tmp1, tmp2);
__lsx_vstelm_d(tmp1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(tmp1, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(tmp2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(tmp2, dst, 0, 1);
dst += dst_stride;
hz_out1 = horiz_2tap_filt_uh(src1, src1, mask, filt_hz);
vec0 = __lsx_vpackev_b(hz_out1, hz_out0);
tmp1 = __lsx_vdp2_h_bu(vec0, filt_vt);
hz_out0 = horiz_2tap_filt_uh(src2, src2, mask, filt_hz);
vec0 = __lsx_vpackev_b(hz_out0, hz_out1);
tmp2 = __lsx_vdp2_h_bu(vec0, filt_vt);
hz_out1 = horiz_2tap_filt_uh(src3, src3, mask, filt_hz);
vec0 = __lsx_vpackev_b(hz_out1, hz_out0);
tmp3 = __lsx_vdp2_h_bu(vec0, filt_vt);
hz_out0 = horiz_2tap_filt_uh(src4, src4, mask, filt_hz);
vec0 = __lsx_vpackev_b(hz_out0, hz_out1);
tmp4 = __lsx_vdp2_h_bu(vec0, filt_vt);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp2, tmp1, FILTER_BITS, tmp4, tmp3,
FILTER_BITS, tmp1, tmp2);
__lsx_vstelm_d(tmp1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(tmp1, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(tmp2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(tmp2, dst, 0, 1);
dst += dst_stride;
}
}
static void common_hv_2ht_2vt_8w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert,
int32_t height) {
if (height == 4) {
common_hv_2ht_2vt_8x4_lsx(src, src_stride, dst, dst_stride, filter_horiz,
filter_vert);
} else {
common_hv_2ht_2vt_8x8mult_lsx(src, src_stride, dst, dst_stride,
filter_horiz, filter_vert, height);
}
}
static void common_hv_2ht_2vt_16w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert,
int32_t height) {
uint32_t loop_cnt = (height >> 2);
__m128i src0, src1, src2, src3, src4, src5, src6, src7, mask;
__m128i filt_hz, filt_vt, vec0, vec1;
__m128i tmp, tmp1, tmp2, hz_out0, hz_out1, hz_out2, hz_out3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
mask = __lsx_vld(mc_filt_mask_arr, 0);
/* rearranging filter */
DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt);
DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1);
src += src_stride;
hz_out0 = horiz_2tap_filt_uh(src0, src0, mask, filt_hz);
hz_out2 = horiz_2tap_filt_uh(src1, src1, mask, filt_hz);
for (; loop_cnt--;) {
uint8_t *src_tmp0 = src + 8;
DUP2_ARG2(__lsx_vld, src, 0, src_tmp0, 0, src0, src1);
DUP4_ARG2(__lsx_vldx, src, src_stride, src_tmp0, src_stride, src,
src_stride2, src_tmp0, src_stride2, src2, src3, src4, src5);
DUP2_ARG2(__lsx_vldx, src, src_stride3, src_tmp0, src_stride3, src6, src7);
src += src_stride4;
hz_out1 = horiz_2tap_filt_uh(src0, src0, mask, filt_hz);
hz_out3 = horiz_2tap_filt_uh(src1, src1, mask, filt_hz);
DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp1, tmp2);
tmp = __lsx_vssrarni_bu_h(tmp2, tmp1, FILTER_BITS);
__lsx_vst(tmp, dst, 0);
dst += dst_stride;
hz_out0 = horiz_2tap_filt_uh(src2, src2, mask, filt_hz);
hz_out2 = horiz_2tap_filt_uh(src3, src3, mask, filt_hz);
DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp1, tmp2);
tmp = __lsx_vssrarni_bu_h(tmp2, tmp1, FILTER_BITS);
__lsx_vst(tmp, dst, 0);
dst += dst_stride;
hz_out1 = horiz_2tap_filt_uh(src4, src4, mask, filt_hz);
hz_out3 = horiz_2tap_filt_uh(src5, src5, mask, filt_hz);
DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp1, tmp2);
tmp = __lsx_vssrarni_bu_h(tmp2, tmp1, FILTER_BITS);
__lsx_vst(tmp, dst, 0);
dst += dst_stride;
hz_out0 = horiz_2tap_filt_uh(src6, src6, mask, filt_hz);
hz_out2 = horiz_2tap_filt_uh(src7, src7, mask, filt_hz);
DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp1, tmp2);
tmp = __lsx_vssrarni_bu_h(tmp2, tmp1, FILTER_BITS);
__lsx_vst(tmp, dst, 0);
dst += dst_stride;
}
}
static void common_hv_2ht_2vt_32w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert,
int32_t height) {
common_hv_2ht_2vt_16w_lsx(src, src_stride, dst, dst_stride, filter_horiz,
filter_vert, height);
src += 16;
dst += 16;
common_hv_2ht_2vt_16w_lsx(src, src_stride, dst, dst_stride, filter_horiz,
filter_vert, height);
}
static void common_hv_2ht_2vt_64w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter_horiz, int8_t *filter_vert,
int32_t height) {
int32_t multiple8_cnt;
for (multiple8_cnt = 4; multiple8_cnt--;) {
common_hv_2ht_2vt_16w_lsx(src, src_stride, dst, dst_stride, filter_horiz,
filter_vert, height);
src += 16;
dst += 16;
}
}
void vpx_convolve8_lsx(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
ptrdiff_t dst_stride, const InterpKernel *filter,
int x0_q4, int32_t x_step_q4, int y0_q4,
int32_t y_step_q4, int32_t w, int32_t h) {
const int16_t *const filter_x = filter[x0_q4];
const int16_t *const filter_y = filter[y0_q4];
int8_t cnt, filt_hor[8], filt_ver[8];
assert(x_step_q4 == 16);
assert(y_step_q4 == 16);
assert(((const int32_t *)filter_x)[1] != 0x800000);
assert(((const int32_t *)filter_y)[1] != 0x800000);
for (cnt = 0; cnt < 8; ++cnt) {
filt_hor[cnt] = filter_x[cnt];
filt_ver[cnt] = filter_y[cnt];
}
if (vpx_get_filter_taps(filter_x) == 2 &&
vpx_get_filter_taps(filter_y) == 2) {
switch (w) {
case 4:
common_hv_2ht_2vt_4w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_hor[3],
&filt_ver[3], (int32_t)h);
break;
case 8:
common_hv_2ht_2vt_8w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_hor[3],
&filt_ver[3], (int32_t)h);
break;
case 16:
common_hv_2ht_2vt_16w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_hor[3],
&filt_ver[3], (int32_t)h);
break;
case 32:
common_hv_2ht_2vt_32w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_hor[3],
&filt_ver[3], (int32_t)h);
break;
case 64:
common_hv_2ht_2vt_64w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, &filt_hor[3],
&filt_ver[3], (int32_t)h);
break;
default:
vpx_convolve8_c(src, src_stride, dst, dst_stride, filter, x0_q4,
x_step_q4, y0_q4, y_step_q4, w, h);
break;
}
} else if (vpx_get_filter_taps(filter_x) == 2 ||
vpx_get_filter_taps(filter_y) == 2) {
vpx_convolve8_c(src, src_stride, dst, dst_stride, filter, x0_q4, x_step_q4,
y0_q4, y_step_q4, w, h);
} else {
switch (w) {
case 4:
common_hv_8ht_8vt_4w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor, filt_ver,
(int32_t)h);
break;
case 8:
common_hv_8ht_8vt_8w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor, filt_ver,
(int32_t)h);
break;
case 16:
common_hv_8ht_8vt_16w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor, filt_ver,
(int32_t)h);
break;
case 32:
common_hv_8ht_8vt_32w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor, filt_ver,
(int32_t)h);
break;
case 64:
common_hv_8ht_8vt_64w_lsx(src, (int32_t)src_stride, dst,
(int32_t)dst_stride, filt_hor, filt_ver,
(int32_t)h);
break;
default:
vpx_convolve8_c(src, src_stride, dst, dst_stride, filter, x0_q4,
x_step_q4, y0_q4, y_step_q4, w, h);
break;
}
}
}
libs/libvpx/vpx_dsp/loongarch/vpx_convolve8_vert_lsx.c
+825
View File
@@ -0,0 +1,825 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <assert.h>
#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/loongarch/vpx_convolve_lsx.h"
static void common_vt_8t_4w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = height >> 2;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride + src_stride2;
int32_t src_stride4 = src_stride2 << 1;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
__m128i reg0, reg1, reg2, reg3, reg4;
__m128i filter0, filter1, filter2, filter3;
__m128i out0, out1;
uint8_t *_src = (uint8_t *)src - src_stride3;
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
src0 = __lsx_vld(_src, 0);
DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2);
src3 = __lsx_vldx(_src, src_stride3);
_src += src_stride4;
src4 = __lsx_vld(_src, 0);
DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src5, src6);
_src += src_stride3;
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, tmp0,
tmp1, tmp2, tmp3);
DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, tmp4, tmp5);
DUP2_ARG2(__lsx_vilvl_d, tmp3, tmp0, tmp4, tmp1, reg0, reg1);
reg2 = __lsx_vilvl_d(tmp5, tmp2);
DUP2_ARG2(__lsx_vxori_b, reg0, 128, reg1, 128, reg0, reg1);
reg2 = __lsx_vxori_b(reg2, 128);
for (; loop_cnt--;) {
src7 = __lsx_vld(_src, 0);
DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src8, src9);
src10 = __lsx_vldx(_src, src_stride3);
_src += src_stride4;
DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9,
tmp0, tmp1, tmp2, tmp3);
DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, reg3, reg4);
DUP2_ARG2(__lsx_vxori_b, reg3, 128, reg4, 128, reg3, reg4);
out0 = filt_8tap_dpadd_s_h(reg0, reg1, reg2, reg3, filter0, filter1,
filter2, filter3);
out1 = filt_8tap_dpadd_s_h(reg1, reg2, reg3, reg4, filter0, filter1,
filter2, filter3);
out0 = __lsx_vssrarni_b_h(out1, out0, 7);
out0 = __lsx_vxori_b(out0, 128);
__lsx_vstelm_w(out0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 2);
dst += dst_stride;
__lsx_vstelm_w(out0, dst, 0, 3);
dst += dst_stride;
reg0 = reg2;
reg1 = reg3;
reg2 = reg4;
src6 = src10;
}
}
static void common_vt_8t_8w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = height >> 2;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
__m128i tmp0, tmp1, tmp2, tmp3;
__m128i reg0, reg1, reg2, reg3, reg4, reg5;
__m128i filter0, filter1, filter2, filter3;
__m128i out0, out1, out2, out3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride + src_stride2;
int32_t src_stride4 = src_stride2 << 1;
src = src - src_stride3;
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src += src_stride4;
src4 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6);
src += src_stride3;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5);
src6 = __lsx_vxori_b(src6, 128);
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, reg0,
reg1, reg2, reg3);
DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5);
for (; loop_cnt--;) {
src7 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src8, src9);
src10 = __lsx_vldx(src, src_stride3);
src += src_stride4;
DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7,
src8, src9, src10);
DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9,
tmp0, tmp1, tmp2, tmp3);
out0 = filt_8tap_dpadd_s_h(reg0, reg1, reg2, tmp0, filter0, filter1,
filter2, filter3);
out1 = filt_8tap_dpadd_s_h(reg3, reg4, reg5, tmp1, filter0, filter1,
filter2, filter3);
out2 = filt_8tap_dpadd_s_h(reg1, reg2, tmp0, tmp2, filter0, filter1,
filter2, filter3);
out3 = filt_8tap_dpadd_s_h(reg4, reg5, tmp1, tmp3, filter0, filter1,
filter2, filter3);
DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1);
DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1);
__lsx_vstelm_d(out0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out0, dst, 0, 1);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(out1, dst, 0, 1);
dst += dst_stride;
reg0 = reg2;
reg1 = tmp0;
reg2 = tmp2;
reg3 = reg5;
reg4 = tmp1;
reg5 = tmp3;
src6 = src10;
}
}
static void common_vt_8t_16w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = height >> 2;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
__m128i filter0, filter1, filter2, filter3;
__m128i reg0, reg1, reg2, reg3, reg4, reg5;
__m128i reg6, reg7, reg8, reg9, reg10, reg11;
__m128i tmp0, tmp1, tmp2, tmp3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride + src_stride2;
int32_t src_stride4 = src_stride2 << 1;
// uint8_t *_src = (uint8_t *)src - src_stride3;
src -= src_stride3;
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
src += src_stride4;
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6);
src += src_stride3;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5);
src6 = __lsx_vxori_b(src6, 128);
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, reg0,
reg1, reg2, reg3);
DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5);
DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1, reg6,
reg7, reg8, reg9);
DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, reg10, reg11);
for (; loop_cnt--;) {
src7 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src8, src9);
src10 = __lsx_vldx(src, src_stride3);
src += src_stride4;
DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7,
src8, src9, src10);
DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9,
src0, src1, src2, src3);
DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, src9,
src4, src5, src7, src8);
tmp0 = filt_8tap_dpadd_s_h(reg0, reg1, reg2, src0, filter0, filter1,
filter2, filter3);
tmp1 = filt_8tap_dpadd_s_h(reg3, reg4, reg5, src1, filter0, filter1,
filter2, filter3);
tmp2 = filt_8tap_dpadd_s_h(reg6, reg7, reg8, src4, filter0, filter1,
filter2, filter3);
tmp3 = filt_8tap_dpadd_s_h(reg9, reg10, reg11, src5, filter0, filter1,
filter2, filter3);
DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1);
DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1);
__lsx_vst(tmp0, dst, 0);
dst += dst_stride;
__lsx_vst(tmp1, dst, 0);
dst += dst_stride;
tmp0 = filt_8tap_dpadd_s_h(reg1, reg2, src0, src2, filter0, filter1,
filter2, filter3);
tmp1 = filt_8tap_dpadd_s_h(reg4, reg5, src1, src3, filter0, filter1,
filter2, filter3);
tmp2 = filt_8tap_dpadd_s_h(reg7, reg8, src4, src7, filter0, filter1,
filter2, filter3);
tmp3 = filt_8tap_dpadd_s_h(reg10, reg11, src5, src8, filter0, filter1,
filter2, filter3);
DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1);
DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1);
__lsx_vst(tmp0, dst, 0);
dst += dst_stride;
__lsx_vst(tmp1, dst, 0);
dst += dst_stride;
reg0 = reg2;
reg1 = src0;
reg2 = src2;
reg3 = reg5;
reg4 = src1;
reg5 = src3;
reg6 = reg8;
reg7 = src4;
reg8 = src7;
reg9 = reg11;
reg10 = src5;
reg11 = src8;
src6 = src10;
}
}
static void common_vt_8t_16w_mult_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height,
int32_t width) {
uint8_t *src_tmp;
uint8_t *dst_tmp;
uint32_t cnt = width >> 4;
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
__m128i filter0, filter1, filter2, filter3;
__m128i reg0, reg1, reg2, reg3, reg4, reg5;
__m128i reg6, reg7, reg8, reg9, reg10, reg11;
__m128i tmp0, tmp1, tmp2, tmp3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride + src_stride2;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
int32_t dst_stride4 = dst_stride2 << 1;
src -= src_stride3;
DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
filter0, filter1, filter2, filter3);
for (; cnt--;) {
uint32_t loop_cnt = height >> 2;
src_tmp = src;
dst_tmp = dst;
src0 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src1,
src2);
src3 = __lsx_vldx(src_tmp, src_stride3);
src_tmp += src_stride4;
src4 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5,
src6);
src_tmp += src_stride3;
DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0,
src1, src2, src3);
DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5);
src6 = __lsx_vxori_b(src6, 128);
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1,
reg0, reg1, reg2, reg3);
DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5);
DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1,
reg6, reg7, reg8, reg9);
DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, reg10, reg11);
for (; loop_cnt--;) {
src7 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src8,
src9);
src10 = __lsx_vldx(src_tmp, src_stride3);
src_tmp += src_stride4;
DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128,
src7, src8, src9, src10);
DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9,
src0, src1, src2, src3);
DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, src9,
src4, src5, src7, src8);
tmp0 = filt_8tap_dpadd_s_h(reg0, reg1, reg2, src0, filter0, filter1,
filter2, filter3);
tmp1 = filt_8tap_dpadd_s_h(reg3, reg4, reg5, src1, filter0, filter1,
filter2, filter3);
tmp2 = filt_8tap_dpadd_s_h(reg6, reg7, reg8, src4, filter0, filter1,
filter2, filter3);
tmp3 = filt_8tap_dpadd_s_h(reg9, reg10, reg11, src5, filter0, filter1,
filter2, filter3);
DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1);
DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1);
__lsx_vst(tmp0, dst_tmp, 0);
__lsx_vstx(tmp1, dst_tmp, dst_stride);
tmp0 = filt_8tap_dpadd_s_h(reg1, reg2, src0, src2, filter0, filter1,
filter2, filter3);
tmp1 = filt_8tap_dpadd_s_h(reg4, reg5, src1, src3, filter0, filter1,
filter2, filter3);
tmp2 = filt_8tap_dpadd_s_h(reg7, reg8, src4, src7, filter0, filter1,
filter2, filter3);
tmp3 = filt_8tap_dpadd_s_h(reg10, reg11, src5, src8, filter0, filter1,
filter2, filter3);
DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1);
DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1);
__lsx_vstx(tmp0, dst_tmp, dst_stride2);
__lsx_vstx(tmp1, dst_tmp, dst_stride3);
dst_tmp += dst_stride4;
reg0 = reg2;
reg1 = src0;
reg2 = src2;
reg3 = reg5;
reg4 = src1;
reg5 = src3;
reg6 = reg8;
reg7 = src4;
reg8 = src7;
reg9 = reg11;
reg10 = src5;
reg11 = src8;
src6 = src10;
}
src += 16;
dst += 16;
}
}
static void common_vt_8t_32w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
common_vt_8t_16w_mult_lsx(src, src_stride, dst, dst_stride, filter, height,
32);
}
static void common_vt_8t_64w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
common_vt_8t_16w_mult_lsx(src, src_stride, dst, dst_stride, filter, height,
64);
}
static void common_vt_2t_4x4_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter) {
__m128i src0, src1, src2, src3, src4;
__m128i vec0, vec1, vec2, vec3, vec4, vec5;
__m128i filt0, tmp0, tmp1;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
src += (src_stride4 + src_stride);
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, vec0,
vec1, vec2, vec3);
DUP2_ARG2(__lsx_vilvl_d, vec1, vec0, vec3, vec2, vec4, vec5);
DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1);
tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vstelm_w(tmp0, dst, 0, 0);
__lsx_vstelm_w(tmp0, dst + dst_stride, 0, 1);
__lsx_vstelm_w(tmp0, dst + dst_stride2, 0, 2);
__lsx_vstelm_w(tmp0, dst + dst_stride3, 0, 3);
}
static void common_vt_2t_4x8_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter) {
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8;
__m128i vec0, vec1, vec2, vec3, vec4, vec5;
__m128i vec6, vec7, vec8, vec9, vec10, vec11;
__m128i tmp0, tmp1, tmp2, tmp3;
__m128i filt0;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
int32_t dst_stride4 = dst_stride2 << 1;
uint8_t *dst_tmp1 = dst + dst_stride4;
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
src += src_stride4;
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src5, src6, src7, src8);
src += (src_stride4 + src_stride);
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, vec0,
vec1, vec2, vec3);
DUP4_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src7, src6, src8, src7, vec4,
vec5, vec6, vec7);
DUP4_ARG2(__lsx_vilvl_d, vec1, vec0, vec3, vec2, vec5, vec4, vec7, vec6, vec8,
vec9, vec10, vec11);
DUP4_ARG2(__lsx_vdp2_h_bu, vec8, filt0, vec9, filt0, vec10, filt0, vec11,
filt0, tmp0, tmp1, tmp2, tmp3);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2,
FILTER_BITS, tmp0, tmp1);
__lsx_vstelm_w(tmp0, dst, 0, 0);
__lsx_vstelm_w(tmp0, dst + dst_stride, 0, 1);
__lsx_vstelm_w(tmp0, dst + dst_stride2, 0, 2);
__lsx_vstelm_w(tmp0, dst + dst_stride3, 0, 3);
__lsx_vstelm_w(tmp1, dst_tmp1, 0, 0);
__lsx_vstelm_w(tmp1, dst_tmp1 + dst_stride, 0, 1);
__lsx_vstelm_w(tmp1, dst_tmp1 + dst_stride2, 0, 2);
__lsx_vstelm_w(tmp1, dst_tmp1 + dst_stride3, 0, 3);
}
static void common_vt_2t_4w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
if (height == 4) {
common_vt_2t_4x4_lsx(src, src_stride, dst, dst_stride, filter);
} else if (height == 8) {
common_vt_2t_4x8_lsx(src, src_stride, dst, dst_stride, filter);
}
}
static void common_vt_2t_8x4_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter) {
__m128i src0, src1, src2, src3, src4, vec0, vec1, vec2, vec3, filt0;
__m128i out0, out1, tmp0, tmp1, tmp2, tmp3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, vec0,
vec1, vec2, vec3);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0,
tmp0, tmp1, tmp2, tmp3);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2,
FILTER_BITS, out0, out1);
__lsx_vstelm_d(out0, dst, 0, 0);
__lsx_vstelm_d(out0, dst + dst_stride, 0, 1);
__lsx_vstelm_d(out1, dst + dst_stride2, 0, 0);
__lsx_vstelm_d(out1, dst + dst_stride3, 0, 1);
}
static void common_vt_2t_8x8mult_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = (height >> 3);
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0;
__m128i out0, out1, tmp0, tmp1, tmp2, tmp3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
int32_t dst_stride4 = dst_stride2 << 1;
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
src += src_stride;
for (; loop_cnt--;) {
src1 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3);
src4 = __lsx_vldx(src, src_stride3);
src += src_stride4;
src5 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src6, src7)
src8 = __lsx_vldx(src, src_stride3);
src += src_stride4;
DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3,
vec0, vec1, vec2, vec3);
DUP4_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src7, src6, src8, src7,
vec4, vec5, vec6, vec7);
DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3,
filt0, tmp0, tmp1, tmp2, tmp3);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2,
FILTER_BITS, out0, out1);
__lsx_vstelm_d(out0, dst, 0, 0);
__lsx_vstelm_d(out0, dst + dst_stride, 0, 1);
__lsx_vstelm_d(out1, dst + dst_stride2, 0, 0);
__lsx_vstelm_d(out1, dst + dst_stride3, 0, 1);
dst += dst_stride4;
DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7,
filt0, tmp0, tmp1, tmp2, tmp3);
DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2,
FILTER_BITS, out0, out1);
__lsx_vstelm_d(out0, dst, 0, 0);
__lsx_vstelm_d(out0, dst + dst_stride, 0, 1);
__lsx_vstelm_d(out1, dst + dst_stride2, 0, 0);
__lsx_vstelm_d(out1, dst + dst_stride3, 0, 1);
dst += dst_stride4;
src0 = src8;
}
}
static void common_vt_2t_8w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
if (height == 4) {
common_vt_2t_8x4_lsx(src, src_stride, dst, dst_stride, filter);
} else {
common_vt_2t_8x8mult_lsx(src, src_stride, dst, dst_stride, filter, height);
}
}
static void common_vt_2t_16w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = (height >> 2);
__m128i src0, src1, src2, src3, src4, tmp, tmp0, tmp1;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
filt0 = __lsx_vldrepl_h(filter, 0);
src0 = __lsx_vld(src, 0);
src += src_stride;
for (; loop_cnt--;) {
src1 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3);
src4 = __lsx_vldx(src, src_stride3);
src += src_stride4;
DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2);
DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vst(tmp, dst, 0);
dst += dst_stride;
DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6);
DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7);
DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vst(tmp, dst, 0);
dst += dst_stride;
DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vst(tmp, dst, 0);
dst += dst_stride;
DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vst(tmp, dst, 0);
dst += dst_stride;
src0 = src4;
}
}
static void common_vt_2t_32w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = (height >> 2);
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0;
__m128i tmp, tmp0, tmp1;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
uint8_t *src_tmp;
filt0 = __lsx_vldrepl_h(filter, 0);
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src5);
src += src_stride;
src_tmp = src + 16;
for (; loop_cnt--;) {
DUP2_ARG2(__lsx_vld, src, 0, src_tmp, 0, src1, src6);
DUP4_ARG2(__lsx_vldx, src, src_stride, src_tmp, src_stride, src,
src_stride2, src_tmp, src_stride2, src2, src7, src3, src8);
DUP2_ARG2(__lsx_vldx, src, src_stride3, src_tmp, src_stride3, src4, src9);
DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2);
DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3);
src += src_stride4;
src_tmp += src_stride4;
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vst(tmp, dst, 0);
DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vstx(tmp, dst, dst_stride);
DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6);
DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7);
DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vstx(tmp, dst, dst_stride2);
DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vstx(tmp, dst, dst_stride3);
DUP2_ARG2(__lsx_vilvl_b, src6, src5, src7, src6, vec0, vec2);
DUP2_ARG2(__lsx_vilvh_b, src6, src5, src7, src6, vec1, vec3);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vst(tmp, dst, 16);
DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
dst += dst_stride;
__lsx_vst(tmp, dst, 16);
DUP2_ARG2(__lsx_vilvl_b, src8, src7, src9, src8, vec4, vec6);
DUP2_ARG2(__lsx_vilvh_b, src8, src7, src9, src8, vec5, vec7);
DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
dst += dst_stride;
__lsx_vst(tmp, dst, 16);
DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
dst += dst_stride;
__lsx_vst(tmp, dst, 16);
dst += dst_stride;
src0 = src4;
src5 = src9;
}
}
static void common_vt_2t_64w_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int8_t *filter, int32_t height) {
uint32_t loop_cnt = (height >> 1);
__m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
__m128i src11, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0;
__m128i tmp, tmp0, tmp1;
int32_t src_stride2 = src_stride << 1;
int32_t dst_stride2 = dst_stride << 1;
uint8_t *dst_tmp1 = dst + dst_stride;
filt0 = __lsx_vldrepl_h(filter, 0);
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src3, src6,
src9);
src += src_stride;
for (; loop_cnt--;) {
uint8_t *src_tmp0 = src + src_stride;
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src1, src4, src7,
src10);
DUP4_ARG2(__lsx_vld, src_tmp0, 0, src_tmp0, 16, src_tmp0, 32, src_tmp0, 48,
src2, src5, src8, src11);
src += src_stride2;
DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2);
DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vst(tmp, dst, 0);
DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vst(tmp, dst_tmp1, 0);
DUP2_ARG2(__lsx_vilvl_b, src4, src3, src5, src4, vec4, vec6);
DUP2_ARG2(__lsx_vilvh_b, src4, src3, src5, src4, vec5, vec7);
DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vst(tmp, dst, 16);
DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vst(tmp, dst_tmp1, 16);
DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, vec0, vec2);
DUP2_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, vec1, vec3);
DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vst(tmp, dst, 32);
DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vst(tmp, dst_tmp1, 32);
DUP2_ARG2(__lsx_vilvl_b, src10, src9, src11, src10, vec4, vec6);
DUP2_ARG2(__lsx_vilvh_b, src10, src9, src11, src10, vec5, vec7);
DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vst(tmp, dst, 48);
DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1);
tmp = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS);
__lsx_vst(tmp, dst_tmp1, 48);
dst += dst_stride2;
dst_tmp1 += dst_stride2;
src0 = src2;
src3 = src5;
src6 = src8;
src9 = src11;
}
}
void vpx_convolve8_vert_lsx(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const InterpKernel *filter, int x0_q4,
int32_t x_step_q4, int y0_q4, int y_step_q4, int w,
int h) {
const int16_t *const filter_y = filter[y0_q4];
int8_t cnt, filt_ver[8];
assert(y_step_q4 == 16);
assert(((const int32_t *)filter_y)[1] != 0x800000);
for (cnt = 8; cnt--;) {
filt_ver[cnt] = filter_y[cnt];
}
if (vpx_get_filter_taps(filter_y) == 2) {
switch (w) {
case 4:
common_vt_2t_4w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
&filt_ver[3], h);
break;
case 8:
common_vt_2t_8w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
&filt_ver[3], h);
break;
case 16:
common_vt_2t_16w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
&filt_ver[3], h);
break;
case 32:
common_vt_2t_32w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
&filt_ver[3], h);
break;
case 64:
common_vt_2t_64w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
&filt_ver[3], h);
break;
default:
vpx_convolve8_vert_c(src, src_stride, dst, dst_stride, filter, x0_q4,
x_step_q4, y0_q4, y_step_q4, w, h);
break;
}
} else {
switch (w) {
case 4:
common_vt_8t_4w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
filt_ver, h);
break;
case 8:
common_vt_8t_8w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
filt_ver, h);
break;
case 16:
common_vt_8t_16w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
filt_ver, h);
break;
case 32:
common_vt_8t_32w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
filt_ver, h);
break;
case 64:
common_vt_8t_64w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride,
filt_ver, h);
break;
default:
vpx_convolve8_vert_c(src, src_stride, dst, dst_stride, filter, x0_q4,
x_step_q4, y0_q4, y_step_q4, w, h);
break;
}
}
}
libs/libvpx/vpx_dsp/loongarch/vpx_convolve_avg_lsx.c
+321
View File
@@ -0,0 +1,321 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "./vpx_dsp_rtcd.h"
#include "vpx_util/loongson_intrinsics.h"
static void avg_width4_lsx(const uint8_t *src, int32_t src_stride, uint8_t *dst,
int32_t dst_stride, int32_t height) {
int32_t cnt;
__m128i src0, src1;
__m128i dst0, dst1;
int32_t src_stride2 = src_stride << 1;
if ((height % 2) == 0) {
for (cnt = (height / 2); cnt--;) {
src0 = __lsx_vld(src, 0);
src1 = __lsx_vldx(src, src_stride);
src += src_stride2;
dst0 = __lsx_vld(dst, 0);
dst1 = __lsx_vldx(dst, dst_stride);
DUP2_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, dst0, dst1);
__lsx_vstelm_w(dst0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_w(dst1, dst, 0, 0);
dst += dst_stride;
}
}
}
static void avg_width8_lsx(const uint8_t *src, int32_t src_stride, uint8_t *dst,
int32_t dst_stride, int32_t height) {
int32_t cnt = (height / 4);
__m128i src0, src1, src2, src3;
__m128i dst0, dst1, dst2, dst3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
for (; cnt--;) {
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src += src_stride4;
dst0 = __lsx_vld(dst, 0);
DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2);
dst3 = __lsx_vldx(dst, dst_stride3);
DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, src2, dst2, src3, dst3,
dst0, dst1, dst2, dst3);
__lsx_vstelm_d(dst0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(dst1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(dst2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(dst3, dst, 0, 0);
dst += dst_stride;
}
}
static void avg_width16_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride, int32_t height) {
int32_t cnt = (height / 8);
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
__m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
int32_t dst_stride4 = dst_stride2 << 1;
for (; cnt--;) {
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src += src_stride4;
src4 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6);
src7 = __lsx_vldx(src, src_stride3);
src += src_stride4;
dst0 = __lsx_vld(dst, 0);
DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2);
dst3 = __lsx_vldx(dst, dst_stride3);
dst += dst_stride4;
dst4 = __lsx_vld(dst, 0);
DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst5, dst6);
dst7 = __lsx_vldx(dst, dst_stride3);
dst -= dst_stride4;
DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, src2, dst2, src3, dst3,
dst0, dst1, dst2, dst3);
DUP4_ARG2(__lsx_vavgr_bu, src4, dst4, src5, dst5, src6, dst6, src7, dst7,
dst4, dst5, dst6, dst7);
__lsx_vst(dst0, dst, 0);
__lsx_vstx(dst1, dst, dst_stride);
__lsx_vstx(dst2, dst, dst_stride2);
__lsx_vstx(dst3, dst, dst_stride3);
dst += dst_stride4;
__lsx_vst(dst4, dst, 0);
__lsx_vstx(dst5, dst, dst_stride);
__lsx_vstx(dst6, dst, dst_stride2);
__lsx_vstx(dst7, dst, dst_stride3);
dst += dst_stride4;
}
}
static void avg_width32_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride, int32_t height) {
int32_t cnt = (height / 8);
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
__m128i src8, src9, src10, src11, src12, src13, src14, src15;
__m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
__m128i dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
int32_t dst_stride2 = dst_stride << 1;
int32_t dst_stride3 = dst_stride2 + dst_stride;
int32_t dst_stride4 = dst_stride2 << 1;
for (; cnt--;) {
uint8_t *dst_tmp = dst;
uint8_t *dst_tmp1 = dst_tmp + 16;
uint8_t *src_tmp = src + 16;
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vld, src, 0, src_tmp, 0, src0, src1);
DUP4_ARG2(__lsx_vldx, src, src_stride, src_tmp, src_stride, src,
src_stride2, src_tmp, src_stride2, src2, src3, src4, src5);
DUP2_ARG2(__lsx_vldx, src, src_stride3, src_tmp, src_stride3, src6, src7);
src += src_stride4;
DUP2_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp1, 0, dst0, dst1);
DUP4_ARG2(__lsx_vldx, dst_tmp, dst_stride, dst_tmp1, dst_stride, dst_tmp,
dst_stride2, dst_tmp1, dst_stride2, dst2, dst3, dst4, dst5);
DUP2_ARG2(__lsx_vldx, dst_tmp, dst_stride3, dst_tmp1, dst_stride3, dst6,
dst7);
dst_tmp += dst_stride4;
dst_tmp1 += dst_stride4;
src_tmp = src + 16;
DUP2_ARG2(__lsx_vld, src, 0, src_tmp, 0, src8, src9);
DUP4_ARG2(__lsx_vldx, src, src_stride, src_tmp, src_stride, src,
src_stride2, src_tmp, src_stride2, src10, src11, src12, src13);
DUP2_ARG2(__lsx_vldx, src, src_stride3, src_tmp, src_stride3, src14, src15);
src += src_stride4;
DUP2_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp1, 0, dst8, dst9);
DUP4_ARG2(__lsx_vldx, dst_tmp, dst_stride, dst_tmp1, dst_stride, dst_tmp,
dst_stride2, dst_tmp1, dst_stride2, dst10, dst11, dst12, dst13);
DUP2_ARG2(__lsx_vldx, dst_tmp, dst_stride3, dst_tmp1, dst_stride3, dst14,
dst15);
DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, src2, dst2, src3, dst3,
dst0, dst1, dst2, dst3);
DUP4_ARG2(__lsx_vavgr_bu, src4, dst4, src5, dst5, src6, dst6, src7, dst7,
dst4, dst5, dst6, dst7);
DUP4_ARG2(__lsx_vavgr_bu, src8, dst8, src9, dst9, src10, dst10, src11,
dst11, dst8, dst9, dst10, dst11);
DUP4_ARG2(__lsx_vavgr_bu, src12, dst12, src13, dst13, src14, dst14, src15,
dst15, dst12, dst13, dst14, dst15);
dst_tmp = dst + 16;
__lsx_vst(dst0, dst, 0);
__lsx_vstx(dst2, dst, dst_stride);
__lsx_vstx(dst4, dst, dst_stride2);
__lsx_vstx(dst6, dst, dst_stride3);
__lsx_vst(dst1, dst_tmp, 0);
__lsx_vstx(dst3, dst_tmp, dst_stride);
__lsx_vstx(dst5, dst_tmp, dst_stride2);
__lsx_vstx(dst7, dst_tmp, dst_stride3);
dst += dst_stride4;
__lsx_vst(dst8, dst, 0);
__lsx_vstx(dst10, dst, dst_stride);
__lsx_vstx(dst12, dst, dst_stride2);
__lsx_vstx(dst14, dst, dst_stride3);
__lsx_vst(dst9, dst_tmp1, 0);
__lsx_vstx(dst11, dst_tmp1, dst_stride);
__lsx_vstx(dst13, dst_tmp1, dst_stride2);
__lsx_vstx(dst15, dst_tmp1, dst_stride3);
dst += dst_stride4;
}
}
static void avg_width64_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride, int32_t height) {
int32_t cnt = (height / 4);
uint8_t *dst_tmp = dst;
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
__m128i src8, src9, src10, src11, src12, src13, src14, src15;
__m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
__m128i dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
for (; cnt--;) {
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2,
src3);
src += src_stride;
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src4, src5, src6,
src7);
src += src_stride;
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src8, src9, src10,
src11);
src += src_stride;
DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src12, src13, src14,
src15);
src += src_stride;
DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48,
dst0, dst1, dst2, dst3);
dst_tmp += dst_stride;
DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48,
dst4, dst5, dst6, dst7);
dst_tmp += dst_stride;
DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48,
dst8, dst9, dst10, dst11);
dst_tmp += dst_stride;
DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48,
dst12, dst13, dst14, dst15);
dst_tmp += dst_stride;
DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, src2, dst2, src3, dst3,
dst0, dst1, dst2, dst3);
DUP4_ARG2(__lsx_vavgr_bu, src4, dst4, src5, dst5, src6, dst6, src7, dst7,
dst4, dst5, dst6, dst7);
DUP4_ARG2(__lsx_vavgr_bu, src8, dst8, src9, dst9, src10, dst10, src11,
dst11, dst8, dst9, dst10, dst11);
DUP4_ARG2(__lsx_vavgr_bu, src12, dst12, src13, dst13, src14, dst14, src15,
dst15, dst12, dst13, dst14, dst15);
__lsx_vst(dst0, dst, 0);
__lsx_vst(dst1, dst, 16);
__lsx_vst(dst2, dst, 32);
__lsx_vst(dst3, dst, 48);
dst += dst_stride;
__lsx_vst(dst4, dst, 0);
__lsx_vst(dst5, dst, 16);
__lsx_vst(dst6, dst, 32);
__lsx_vst(dst7, dst, 48);
dst += dst_stride;
__lsx_vst(dst8, dst, 0);
__lsx_vst(dst9, dst, 16);
__lsx_vst(dst10, dst, 32);
__lsx_vst(dst11, dst, 48);
dst += dst_stride;
__lsx_vst(dst12, dst, 0);
__lsx_vst(dst13, dst, 16);
__lsx_vst(dst14, dst, 32);
__lsx_vst(dst15, dst, 48);
dst += dst_stride;
}
}
void vpx_convolve_avg_lsx(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const InterpKernel *filter, int x0_q4,
int32_t x_step_q4, int y0_q4, int32_t y_step_q4,
int32_t w, int32_t h) {
(void)filter;
(void)x0_q4;
(void)x_step_q4;
(void)y0_q4;
(void)y_step_q4;
switch (w) {
case 4: {
avg_width4_lsx(src, src_stride, dst, dst_stride, h);
break;
}
case 8: {
avg_width8_lsx(src, src_stride, dst, dst_stride, h);
break;
}
case 16: {
avg_width16_lsx(src, src_stride, dst, dst_stride, h);
break;
}
case 32: {
avg_width32_lsx(src, src_stride, dst, dst_stride, h);
break;
}
case 64: {
avg_width64_lsx(src, src_stride, dst, dst_stride, h);
break;
}
default: {
int32_t lp, cnt;
for (cnt = h; cnt--;) {
for (lp = 0; lp < w; ++lp) {
dst[lp] = (((dst[lp] + src[lp]) + 1) >> 1);
}
src += src_stride;
dst += dst_stride;
}
break;
}
}
}
libs/libvpx/vpx_dsp/loongarch/vpx_convolve_copy_lsx.c
+437
View File
@@ -0,0 +1,437 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <string.h>
#include "./vpx_dsp_rtcd.h"
#include "vpx_util/loongson_intrinsics.h"
static void copy_width8_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride, int32_t height) {
int32_t cnt;
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
if ((height % 12) == 0) {
for (cnt = (height / 12); cnt--;) {
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
src += src_stride4;
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6);
src += src_stride2;
src7 = __lsx_vldx(src, src_stride);
src += src_stride2;
__lsx_vstelm_d(src0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src3, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src4, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src5, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src6, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src7, dst, 0, 0);
dst += dst_stride;
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src += src_stride4;
__lsx_vstelm_d(src0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src3, dst, 0, 0);
dst += dst_stride;
}
} else if ((height % 8) == 0) {
for (cnt = height >> 3; cnt--;) {
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
src += src_stride4;
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6);
src += src_stride2;
src7 = __lsx_vldx(src, src_stride);
src += src_stride2;
__lsx_vstelm_d(src0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src3, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src4, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src5, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src6, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src7, dst, 0, 0);
dst += dst_stride;
}
} else if ((height % 4) == 0) {
for (cnt = (height / 4); cnt--;) {
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src += src_stride4;
__lsx_vstelm_d(src0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src1, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src2, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src3, dst, 0, 0);
dst += dst_stride;
}
} else if ((height % 2) == 0) {
for (cnt = (height / 2); cnt--;) {
src0 = __lsx_vld(src, 0);
src1 = __lsx_vldx(src, src_stride);
src += src_stride2;
__lsx_vstelm_d(src0, dst, 0, 0);
dst += dst_stride;
__lsx_vstelm_d(src1, dst, 0, 0);
dst += dst_stride;
}
}
}
static void copy_16multx8mult_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride,
int32_t height, int32_t width) {
int32_t cnt, loop_cnt;
uint8_t *src_tmp;
uint8_t *dst_tmp;
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
for (cnt = (width >> 4); cnt--;) {
src_tmp = (uint8_t *)src;
dst_tmp = dst;
for (loop_cnt = (height >> 3); loop_cnt--;) {
src0 = __lsx_vld(src_tmp, 0);
DUP4_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src_tmp,
src_stride3, src_tmp, src_stride4, src1, src2, src3, src4);
src_tmp += src_stride4;
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5,
src6);
src_tmp += src_stride2;
src7 = __lsx_vldx(src_tmp, src_stride);
src_tmp += src_stride2;
__lsx_vst(src0, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src1, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src2, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src3, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src4, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src5, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src6, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src7, dst_tmp, 0);
dst_tmp += dst_stride;
}
src += 16;
dst += 16;
}
}
static void copy_width16_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride, int32_t height) {
int32_t cnt;
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
if ((height % 12) == 0) {
for (cnt = (height / 12); cnt--;) {
src0 = __lsx_vld(src, 0);
DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3,
src, src_stride4, src1, src2, src3, src4);
src += src_stride4;
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6);
src += src_stride2;
src7 = __lsx_vldx(src, src_stride);
src += src_stride2;
__lsx_vst(src0, dst, 0);
dst += dst_stride;
__lsx_vst(src1, dst, 0);
dst += dst_stride;
__lsx_vst(src2, dst, 0);
dst += dst_stride;
__lsx_vst(src3, dst, 0);
dst += dst_stride;
__lsx_vst(src4, dst, 0);
dst += dst_stride;
__lsx_vst(src5, dst, 0);
dst += dst_stride;
__lsx_vst(src6, dst, 0);
dst += dst_stride;
__lsx_vst(src7, dst, 0);
dst += dst_stride;
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src += src_stride4;
__lsx_vst(src0, dst, 0);
dst += dst_stride;
__lsx_vst(src1, dst, 0);
dst += dst_stride;
__lsx_vst(src2, dst, 0);
dst += dst_stride;
__lsx_vst(src3, dst, 0);
dst += dst_stride;
}
} else if ((height % 8) == 0) {
copy_16multx8mult_lsx(src, src_stride, dst, dst_stride, height, 16);
} else if ((height % 4) == 0) {
for (cnt = (height >> 2); cnt--;) {
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src += src_stride4;
__lsx_vst(src0, dst, 0);
dst += dst_stride;
__lsx_vst(src1, dst, 0);
dst += dst_stride;
__lsx_vst(src2, dst, 0);
dst += dst_stride;
__lsx_vst(src3, dst, 0);
dst += dst_stride;
}
}
}
static void copy_width32_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride, int32_t height) {
int32_t cnt;
uint8_t *src_tmp;
uint8_t *dst_tmp;
__m128i src0, src1, src2, src3, src4, src5, src6, src7;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride3 = src_stride2 + src_stride;
int32_t src_stride4 = src_stride2 << 1;
if ((height % 12) == 0) {
for (cnt = (height / 12); cnt--;) {
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src_tmp = (uint8_t *)src + 16;
src4 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5,
src6);
src7 = __lsx_vldx(src_tmp, src_stride3);
src += src_stride4;
__lsx_vst(src0, dst, 0);
dst += dst_stride;
__lsx_vst(src1, dst, 0);
dst += dst_stride;
__lsx_vst(src2, dst, 0);
dst += dst_stride;
__lsx_vst(src3, dst, 0);
dst += dst_stride;
dst_tmp = dst + 16;
__lsx_vst(src4, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src5, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src6, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src7, dst_tmp, 0);
dst_tmp += dst_stride;
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src_tmp = (uint8_t *)src + 16;
src4 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5,
src6);
src7 = __lsx_vldx(src_tmp, src_stride3);
src += src_stride4;
__lsx_vst(src0, dst, 0);
dst += dst_stride;
__lsx_vst(src1, dst, 0);
dst += dst_stride;
__lsx_vst(src2, dst, 0);
dst += dst_stride;
__lsx_vst(src3, dst, 0);
dst += dst_stride;
dst_tmp = dst + 16;
__lsx_vst(src4, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src5, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src6, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src7, dst_tmp, 0);
dst_tmp += dst_stride;
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src_tmp = (uint8_t *)src + 16;
src4 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5,
src6);
src7 = __lsx_vldx(src_tmp, src_stride3);
src += src_stride4;
__lsx_vst(src0, dst, 0);
dst += dst_stride;
__lsx_vst(src1, dst, 0);
dst += dst_stride;
__lsx_vst(src2, dst, 0);
dst += dst_stride;
__lsx_vst(src3, dst, 0);
dst += dst_stride;
dst_tmp = dst + 16;
__lsx_vst(src4, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src5, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src6, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src7, dst_tmp, 0);
dst_tmp += dst_stride;
}
} else if ((height % 8) == 0) {
copy_16multx8mult_lsx(src, src_stride, dst, dst_stride, height, 32);
} else if ((height % 4) == 0) {
for (cnt = (height >> 2); cnt--;) {
src0 = __lsx_vld(src, 0);
DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2);
src3 = __lsx_vldx(src, src_stride3);
src_tmp = (uint8_t *)src + 16;
src4 = __lsx_vld(src_tmp, 0);
DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5,
src6);
src7 = __lsx_vldx(src_tmp, src_stride3);
src += src_stride4;
__lsx_vst(src0, dst, 0);
dst += dst_stride;
__lsx_vst(src1, dst, 0);
dst += dst_stride;
__lsx_vst(src2, dst, 0);
dst += dst_stride;
__lsx_vst(src3, dst, 0);
dst += dst_stride;
dst_tmp = dst + 16;
__lsx_vst(src4, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src5, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src6, dst_tmp, 0);
dst_tmp += dst_stride;
__lsx_vst(src7, dst_tmp, 0);
dst_tmp += dst_stride;
}
}
}
static void copy_width64_lsx(const uint8_t *src, int32_t src_stride,
uint8_t *dst, int32_t dst_stride, int32_t height) {
copy_16multx8mult_lsx(src, src_stride, dst, dst_stride, height, 64);
}
void vpx_convolve_copy_lsx(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const InterpKernel *filter, int x0_q4,
int32_t x_step_q4, int y0_q4, int32_t y_step_q4,
int32_t w, int32_t h) {
(void)filter;
(void)x0_q4;
(void)x_step_q4;
(void)y0_q4;
(void)y_step_q4;
switch (w) {
case 4: {
uint32_t cnt;
__m128i tmp;
for (cnt = h; cnt--;) {
tmp = __lsx_vldrepl_w(src, 0);
__lsx_vstelm_w(tmp, dst, 0, 0);
src += src_stride;
dst += dst_stride;
}
break;
}
case 8: {
copy_width8_lsx(src, src_stride, dst, dst_stride, h);
break;
}
case 16: {
copy_width16_lsx(src, src_stride, dst, dst_stride, h);
break;
}
case 32: {
copy_width32_lsx(src, src_stride, dst, dst_stride, h);
break;
}
case 64: {
copy_width64_lsx(src, src_stride, dst, dst_stride, h);
break;
}
default: {
uint32_t cnt;
for (cnt = h; cnt--;) {
memcpy(dst, src, w);
src += src_stride;
dst += dst_stride;
}
break;
}
}
}
libs/libvpx/vpx_dsp/loongarch/vpx_convolve_lsx.h
+138
View File
@@ -0,0 +1,138 @@
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VPX_VPX_DSP_LOONGARCH_VPX_CONVOLVE_LSX_H_
#define VPX_VPX_DSP_LOONGARCH_VPX_CONVOLVE_LSX_H_
#include "./vpx_config.h"
#include "vpx_dsp/vpx_filter.h"
#include "vpx_util/loongson_intrinsics.h"
static INLINE __m128i filt_8tap_dpadd_s_h(__m128i _reg0, __m128i _reg1,
__m128i _reg2, __m128i _reg3,
__m128i _filter0, __m128i _filter1,
__m128i _filter2, __m128i _filter3) {
__m128i _vec0, _vec1;
_vec0 = __lsx_vdp2_h_b(_reg0, _filter0);
_vec0 = __lsx_vdp2add_h_b(_vec0, _reg1, _filter1);
_vec1 = __lsx_vdp2_h_b(_reg2, _filter2);
_vec1 = __lsx_vdp2add_h_b(_vec1, _reg3, _filter3);
return __lsx_vsadd_h(_vec0, _vec1);
}
static INLINE __m128i horiz_8tap_filt(__m128i _src0, __m128i _src1,
__m128i _mask0, __m128i _mask1,
__m128i _mask2, __m128i _mask3,
__m128i _filt_h0, __m128i _filt_h1,
__m128i _filt_h2, __m128i _filt_h3) {
__m128i _tmp0, _tmp1, _tmp2, _tmp3;
__m128i _out;
DUP4_ARG3(__lsx_vshuf_b, _src1, _src0, _mask0, _src1, _src0, _mask1, _src1,
_src0, _mask2, _src1, _src0, _mask3, _tmp0, _tmp1, _tmp2, _tmp3);
_out = filt_8tap_dpadd_s_h(_tmp0, _tmp1, _tmp2, _tmp3, _filt_h0, _filt_h1,
_filt_h2, _filt_h3);
_out = __lsx_vsrari_h(_out, FILTER_BITS);
return __lsx_vsat_h(_out, 7);
}
static INLINE __m128i horiz_2tap_filt_uh(__m128i in0, __m128i in1, __m128i mask,
__m128i coeff) {
__m128i tmp0_m, tmp1_m;
tmp0_m = __lsx_vshuf_b(in1, in0, mask);
tmp1_m = __lsx_vdp2_h_bu(tmp0_m, coeff);
return __lsx_vsrari_h(tmp1_m, FILTER_BITS);
}
#define LSX_LD_4(_src, _stride, _src0, _src1, _src2, _src3) \
do { \
_src0 = __lsx_vld(_src, 0); \
_src += _stride; \
_src1 = __lsx_vld(_src, 0); \
_src += _stride; \
_src2 = __lsx_vld(_src, 0); \
_src += _stride; \
_src3 = __lsx_vld(_src, 0); \
} while (0)
#define HORIZ_8TAP_4WID_4VECS_FILT(_src0, _src1, _src2, _src3, _mask0, _mask1, \
_mask2, _mask3, _filter0, _filter1, \
_filter2, _filter3, _out0, _out1) \
do { \
__m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5, _tmp6, _tmp7; \
__m128i _reg0, _reg1, _reg2, _reg3; \
\
DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask0, _src3, _src2, _mask0, \
_tmp0, _tmp1); \
DUP2_ARG2(__lsx_vdp2_h_b, _tmp0, _filter0, _tmp1, _filter0, _reg0, _reg1); \
DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask1, _src3, _src2, _mask1, \
_tmp2, _tmp3); \
DUP2_ARG3(__lsx_vdp2add_h_b, _reg0, _tmp2, _filter1, _reg1, _tmp3, \
_filter1, _reg0, _reg1); \
DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask2, _src3, _src2, _mask2, \
_tmp4, _tmp5); \
DUP2_ARG2(__lsx_vdp2_h_b, _tmp4, _filter2, _tmp5, _filter2, _reg2, _reg3); \
DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask3, _src3, _src2, _mask3, \
_tmp6, _tmp7); \
DUP2_ARG3(__lsx_vdp2add_h_b, _reg2, _tmp6, _filter3, _reg3, _tmp7, \
_filter3, _reg2, _reg3); \
DUP2_ARG2(__lsx_vsadd_h, _reg0, _reg2, _reg1, _reg3, _out0, _out1); \
} while (0)
#define HORIZ_8TAP_8WID_4VECS_FILT( \
_src0, _src1, _src2, _src3, _mask0, _mask1, _mask2, _mask3, _filter0, \
_filter1, _filter2, _filter3, _out0, _out1, _out2, _out3) \
do { \
__m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5, _tmp6, _tmp7; \
__m128i _reg0, _reg1, _reg2, _reg3, _reg4, _reg5, _reg6, _reg7; \
\
DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask0, _src1, _src1, _mask0, \
_src2, _src2, _mask0, _src3, _src3, _mask0, _tmp0, _tmp1, _tmp2, \
_tmp3); \
DUP4_ARG2(__lsx_vdp2_h_b, _tmp0, _filter0, _tmp1, _filter0, _tmp2, \
_filter0, _tmp3, _filter0, _reg0, _reg1, _reg2, _reg3); \
DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask2, _src1, _src1, _mask2, \
_src2, _src2, _mask2, _src3, _src3, _mask2, _tmp0, _tmp1, _tmp2, \
_tmp3); \
DUP4_ARG2(__lsx_vdp2_h_b, _tmp0, _filter2, _tmp1, _filter2, _tmp2, \
_filter2, _tmp3, _filter2, _reg4, _reg5, _reg6, _reg7); \
DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask1, _src1, _src1, _mask1, \
_src2, _src2, _mask1, _src3, _src3, _mask1, _tmp4, _tmp5, _tmp6, \
_tmp7); \
DUP4_ARG3(__lsx_vdp2add_h_b, _reg0, _tmp4, _filter1, _reg1, _tmp5, \
_filter1, _reg2, _tmp6, _filter1, _reg3, _tmp7, _filter1, _reg0, \
_reg1, _reg2, _reg3); \
DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask3, _src1, _src1, _mask3, \
_src2, _src2, _mask3, _src3, _src3, _mask3, _tmp4, _tmp5, _tmp6, \
_tmp7); \
DUP4_ARG3(__lsx_vdp2add_h_b, _reg4, _tmp4, _filter3, _reg5, _tmp5, \
_filter3, _reg6, _tmp6, _filter3, _reg7, _tmp7, _filter3, _reg4, \
_reg5, _reg6, _reg7); \
DUP4_ARG2(__lsx_vsadd_h, _reg0, _reg4, _reg1, _reg5, _reg2, _reg6, _reg3, \
_reg7, _out0, _out1, _out2, _out3); \
} while (0)
#define AVG_ST4_D(in0, in1, dst0, dst1, pdst, stride) \
do { \
__m128i tmp0_m, tmp1_m; \
\
DUP2_ARG2(__lsx_vavgr_bu, in0, dst0, in1, dst1, tmp0_m, tmp1_m); \
__lsx_vstelm_d(tmp0_m, pdst, 0, 0); \
pdst += stride; \
__lsx_vstelm_d(tmp0_m, pdst, 0, 1); \
pdst += stride; \
__lsx_vstelm_d(tmp1_m, pdst, 0, 0); \
pdst += stride; \
__lsx_vstelm_d(tmp1_m, pdst, 0, 1); \
} while (0)
#endif // VPX_VPX_DSP_LOONGARCH_VPX_CONVOLVE_LSX_H_
libs/libvpx/vpx_dsp/loopfilter.c
+10 -10
View File
@@ -81,11 +81,11 @@ static INLINE void filter4(int8_t mask, uint8_t thresh, uint8_t *op1,
uint8_t *op0, uint8_t *oq0, uint8_t *oq1) {
int8_t filter1, filter2;
const int8_t ps1 = (int8_t)*op1 ^ 0x80;
const int8_t ps0 = (int8_t)*op0 ^ 0x80;
const int8_t qs0 = (int8_t)*oq0 ^ 0x80;
const int8_t qs1 = (int8_t)*oq1 ^ 0x80;
const uint8_t hev = hev_mask(thresh, *op1, *op0, *oq0, *oq1);
const int8_t ps1 = (int8_t)(*op1 ^ 0x80);
const int8_t ps0 = (int8_t)(*op0 ^ 0x80);
const int8_t qs0 = (int8_t)(*oq0 ^ 0x80);
const int8_t qs1 = (int8_t)(*oq1 ^ 0x80);
const int8_t hev = hev_mask(thresh, *op1, *op0, *oq0, *oq1);
// add outer taps if we have high edge variance
int8_t filter = signed_char_clamp(ps1 - qs1) & hev;
@@ -99,14 +99,14 @@ static INLINE void filter4(int8_t mask, uint8_t thresh, uint8_t *op1,
filter1 = signed_char_clamp(filter + 4) >> 3;
filter2 = signed_char_clamp(filter + 3) >> 3;
*oq0 = signed_char_clamp(qs0 - filter1) ^ 0x80;
*op0 = signed_char_clamp(ps0 + filter2) ^ 0x80;
*oq0 = (uint8_t)(signed_char_clamp(qs0 - filter1) ^ 0x80);
*op0 = (uint8_t)(signed_char_clamp(ps0 + filter2) ^ 0x80);
// outer tap adjustments
filter = ROUND_POWER_OF_TWO(filter1, 1) & ~hev;
*oq1 = signed_char_clamp(qs1 - filter) ^ 0x80;
*op1 = signed_char_clamp(ps1 + filter) ^ 0x80;
*oq1 = (uint8_t)(signed_char_clamp(qs1 - filter) ^ 0x80);
*op1 = (uint8_t)(signed_char_clamp(ps1 + filter) ^ 0x80);
}
void vpx_lpf_horizontal_4_c(uint8_t *s, int pitch, const uint8_t *blimit,
@@ -423,7 +423,7 @@ static INLINE void highbd_filter4(int8_t mask, uint8_t thresh, uint16_t *op1,
const int16_t ps0 = (int16_t)*op0 - (0x80 << shift);
const int16_t qs0 = (int16_t)*oq0 - (0x80 << shift);
const int16_t qs1 = (int16_t)*oq1 - (0x80 << shift);
const uint16_t hev = highbd_hev_mask(thresh, *op1, *op0, *oq0, *oq1, bd);
const int16_t hev = highbd_hev_mask(thresh, *op1, *op0, *oq0, *oq1, bd);
// Add outer taps if we have high edge variance.
int16_t filter = signed_char_clamp_high(ps1 - qs1, bd) & hev;
libs/libvpx/vpx_dsp/mips/macros_msa.h
+8 -8
View File
@@ -88,10 +88,10 @@
const uint8_t *psrc_lw_m = (const uint8_t *)(psrc); \
uint32_t val_lw_m; \
\
__asm__ __volatile__("ulw %[val_lw_m], %[psrc_lw_m] \n\t" \
\
: [val_lw_m] "=r"(val_lw_m) \
: [psrc_lw_m] "m"(*psrc_lw_m)); \
__asm__ __volatile__("lwr %[val_lw_m], 0(%[psrc_lw_m]) \n\t" \
"lwl %[val_lw_m], 3(%[psrc_lw_m]) \n\t" \
: [val_lw_m] "=&r"(val_lw_m) \
: [psrc_lw_m] "r"(psrc_lw_m)); \
\
val_lw_m; \
})
@@ -102,10 +102,10 @@
const uint8_t *psrc_ld_m = (const uint8_t *)(psrc); \
uint64_t val_ld_m = 0; \
\
__asm__ __volatile__("uld %[val_ld_m], %[psrc_ld_m] \n\t" \
\
: [val_ld_m] "=r"(val_ld_m) \
: [psrc_ld_m] "m"(*psrc_ld_m)); \
__asm__ __volatile__("ldr %[val_ld_m], 0(%[psrc_ld_m]) \n\t" \
"ldl %[val_ld_m], 7(%[psrc_ld_m]) \n\t" \
: [val_ld_m] "=&r"(val_ld_m) \
: [psrc_ld_m] "r"(psrc_ld_m)); \
\
val_ld_m; \
})
libs/libvpx/vpx_dsp/mips/sad_mmi.c
+40 -38
View File
@@ -334,19 +334,6 @@
"paddw %[ftmp3], %[ftmp3], %[ftmp1] \n\t"
#endif /* _MIPS_SIM == _ABIO32 */
// depending on call sites, pass **ref_array to avoid & in subsequent call and
// de-dup with 4D below.
#define sadMxNxK_mmi(m, n, k) \
void vpx_sad##m##x##n##x##k##_mmi(const uint8_t *src, int src_stride, \
const uint8_t *ref_array, int ref_stride, \
uint32_t *sad_array) { \
int i; \
for (i = 0; i < (k); ++i) \
sad_array[i] = \
vpx_sad##m##x##n##_mmi(src, src_stride, &ref_array[i], ref_stride); \
}
// This appears to be equivalent to the above when k == 4 and refs is const
#define sadMxNx4D_mmi(m, n) \
void vpx_sad##m##x##n##x4d_mmi(const uint8_t *src, int src_stride, \
const uint8_t *const ref_array[], \
@@ -364,8 +351,9 @@ static inline unsigned int vpx_sad64x(const uint8_t *src, int src_stride,
double ftmp1, ftmp2, ftmp3, ftmp4, ftmp5;
mips_reg l_counter = counter;
/* clang-format off */
__asm__ volatile (
"xor %[ftmp5], %[ftmp5], %[ftmp5] \n\t"
"pxor %[ftmp5], %[ftmp5], %[ftmp5] \n\t"
"1: \n\t"
// Include two loop body, to reduce loop time.
SAD_SRC_REF_ABS_SUB_64
@@ -383,6 +371,7 @@ static inline unsigned int vpx_sad64x(const uint8_t *src, int src_stride,
: [src_stride]"r"((mips_reg)src_stride),
[ref_stride]"r"((mips_reg)ref_stride)
);
/* clang-format on */
return sad;
}
@@ -405,9 +394,11 @@ static inline unsigned int vpx_sad_avg64x(const uint8_t *src, int src_stride,
unsigned int sad;
double ftmp1, ftmp2, ftmp3, ftmp4, ftmp5;
mips_reg l_counter = counter;
mips_reg l_second_pred = (mips_reg)second_pred;
/* clang-format off */
__asm__ volatile (
"xor %[ftmp5], %[ftmp5], %[ftmp5] \n\t"
"pxor %[ftmp5], %[ftmp5], %[ftmp5] \n\t"
"1: \n\t"
// Include two loop body, to reduce loop time.
SAD_SRC_AVGREF_ABS_SUB_64
@@ -424,11 +415,12 @@ static inline unsigned int vpx_sad_avg64x(const uint8_t *src, int src_stride,
: [ftmp1]"=&f"(ftmp1), [ftmp2]"=&f"(ftmp2), [ftmp3]"=&f"(ftmp3),
[ftmp4]"=&f"(ftmp4), [ftmp5]"=&f"(ftmp5), [counter]"+&r"(l_counter),
[src]"+&r"(src), [ref]"+&r"(ref),
[second_pred]"+&r"((mips_reg)second_pred),
[second_pred]"+&r"(l_second_pred),
[sad]"=&r"(sad)
: [src_stride]"r"((mips_reg)src_stride),
[ref_stride]"r"((mips_reg)ref_stride)
);
/* clang-format on */
return sad;
}
@@ -450,8 +442,9 @@ static inline unsigned int vpx_sad32x(const uint8_t *src, int src_stride,
double ftmp1, ftmp2, ftmp3, ftmp4, ftmp5;
mips_reg l_counter = counter;
/* clang-format off */
__asm__ volatile (
"xor %[ftmp5], %[ftmp5], %[ftmp5] \n\t"
"pxor %[ftmp5], %[ftmp5], %[ftmp5] \n\t"
"1: \n\t"
// Include two loop body, to reduce loop time.
SAD_SRC_REF_ABS_SUB_32
@@ -469,6 +462,7 @@ static inline unsigned int vpx_sad32x(const uint8_t *src, int src_stride,
: [src_stride]"r"((mips_reg)src_stride),
[ref_stride]"r"((mips_reg)ref_stride)
);
/* clang-format on */
return sad;
}
@@ -493,9 +487,11 @@ static inline unsigned int vpx_sad_avg32x(const uint8_t *src, int src_stride,
unsigned int sad;
double ftmp1, ftmp2, ftmp3, ftmp4, ftmp5;
mips_reg l_counter = counter;
mips_reg l_second_pred = (mips_reg)second_pred;
/* clang-format off */
__asm__ volatile (
"xor %[ftmp5], %[ftmp5], %[ftmp5] \n\t"
"pxor %[ftmp5], %[ftmp5], %[ftmp5] \n\t"
"1: \n\t"
// Include two loop body, to reduce loop time.
SAD_SRC_AVGREF_ABS_SUB_32
@@ -512,11 +508,12 @@ static inline unsigned int vpx_sad_avg32x(const uint8_t *src, int src_stride,
: [ftmp1]"=&f"(ftmp1), [ftmp2]"=&f"(ftmp2), [ftmp3]"=&f"(ftmp3),
[ftmp4]"=&f"(ftmp4), [ftmp5]"=&f"(ftmp5), [counter]"+&r"(l_counter),
[src]"+&r"(src), [ref]"+&r"(ref),
[second_pred]"+&r"((mips_reg)second_pred),
[second_pred]"+&r"(l_second_pred),
[sad]"=&r"(sad)
: [src_stride]"r"((mips_reg)src_stride),
[ref_stride]"r"((mips_reg)ref_stride)
);
/* clang-format on */
return sad;
}
@@ -539,8 +536,9 @@ static inline unsigned int vpx_sad16x(const uint8_t *src, int src_stride,
double ftmp1, ftmp2, ftmp3, ftmp4, ftmp5;
mips_reg l_counter = counter;
/* clang-format off */
__asm__ volatile (
"xor %[ftmp5], %[ftmp5], %[ftmp5] \n\t"
"pxor %[ftmp5], %[ftmp5], %[ftmp5] \n\t"
"1: \n\t"
// Include two loop body, to reduce loop time.
SAD_SRC_REF_ABS_SUB_16
@@ -558,6 +556,7 @@ static inline unsigned int vpx_sad16x(const uint8_t *src, int src_stride,
: [src_stride]"r"((mips_reg)src_stride),
[ref_stride]"r"((mips_reg)ref_stride)
);
/* clang-format on */
return sad;
}
@@ -571,10 +570,6 @@ static inline unsigned int vpx_sad16x(const uint8_t *src, int src_stride,
vpx_sad16xN(32);
vpx_sad16xN(16);
vpx_sad16xN(8);
sadMxNxK_mmi(16, 16, 3);
sadMxNxK_mmi(16, 16, 8);
sadMxNxK_mmi(16, 8, 3);
sadMxNxK_mmi(16, 8, 8);
sadMxNx4D_mmi(16, 32);
sadMxNx4D_mmi(16, 16);
sadMxNx4D_mmi(16, 8);
@@ -586,9 +581,11 @@ static inline unsigned int vpx_sad_avg16x(const uint8_t *src, int src_stride,
unsigned int sad;
double ftmp1, ftmp2, ftmp3, ftmp4, ftmp5;
mips_reg l_counter = counter;
mips_reg l_second_pred = (mips_reg)second_pred;
/* clang-format off */
__asm__ volatile (
"xor %[ftmp5], %[ftmp5], %[ftmp5] \n\t"
"pxor %[ftmp5], %[ftmp5], %[ftmp5] \n\t"
"1: \n\t"
// Include two loop body, to reduce loop time.
SAD_SRC_AVGREF_ABS_SUB_16
@@ -605,11 +602,12 @@ static inline unsigned int vpx_sad_avg16x(const uint8_t *src, int src_stride,
: [ftmp1]"=&f"(ftmp1), [ftmp2]"=&f"(ftmp2), [ftmp3]"=&f"(ftmp3),
[ftmp4]"=&f"(ftmp4), [ftmp5]"=&f"(ftmp5), [counter]"+&r"(l_counter),
[src]"+&r"(src), [ref]"+&r"(ref),
[second_pred]"+&r"((mips_reg)second_pred),
[second_pred]"+&r"(l_second_pred),
[sad]"=&r"(sad)
: [src_stride]"r"((mips_reg)src_stride),
[ref_stride]"r"((mips_reg)ref_stride)
);
/* clang-format on */
return sad;
}
@@ -632,8 +630,9 @@ static inline unsigned int vpx_sad8x(const uint8_t *src, int src_stride,
double ftmp1, ftmp2, ftmp3;
mips_reg l_counter = counter;
/* clang-format off */
__asm__ volatile (
"xor %[ftmp3], %[ftmp3], %[ftmp3] \n\t"
"pxor %[ftmp3], %[ftmp3], %[ftmp3] \n\t"
"1: \n\t"
// Include two loop body, to reduce loop time.
SAD_SRC_REF_ABS_SUB_8
@@ -651,6 +650,7 @@ static inline unsigned int vpx_sad8x(const uint8_t *src, int src_stride,
: [src_stride]"r"((mips_reg)src_stride),
[ref_stride]"r"((mips_reg)ref_stride)
);
/* clang-format on */
return sad;
}
@@ -664,10 +664,6 @@ static inline unsigned int vpx_sad8x(const uint8_t *src, int src_stride,
vpx_sad8xN(16);
vpx_sad8xN(8);
vpx_sad8xN(4);
sadMxNxK_mmi(8, 16, 3);
sadMxNxK_mmi(8, 16, 8);
sadMxNxK_mmi(8, 8, 3);
sadMxNxK_mmi(8, 8, 8);
sadMxNx4D_mmi(8, 16);
sadMxNx4D_mmi(8, 8);
sadMxNx4D_mmi(8, 4);
@@ -679,9 +675,11 @@ static inline unsigned int vpx_sad_avg8x(const uint8_t *src, int src_stride,
unsigned int sad;
double ftmp1, ftmp2, ftmp3;
mips_reg l_counter = counter;
mips_reg l_second_pred = (mips_reg)second_pred;
/* clang-format off */
__asm__ volatile (
"xor %[ftmp3], %[ftmp3], %[ftmp3] \n\t"
"pxor %[ftmp3], %[ftmp3], %[ftmp3] \n\t"
"1: \n\t"
// Include two loop body, to reduce loop time.
SAD_SRC_AVGREF_ABS_SUB_8
@@ -697,11 +695,12 @@ static inline unsigned int vpx_sad_avg8x(const uint8_t *src, int src_stride,
"mfc1 %[sad], %[ftmp3] \n\t"
: [ftmp1]"=&f"(ftmp1), [ftmp2]"=&f"(ftmp2), [ftmp3]"=&f"(ftmp3),
[counter]"+&r"(l_counter), [src]"+&r"(src), [ref]"+&r"(ref),
[second_pred]"+&r"((mips_reg)second_pred),
[second_pred]"+&r"(l_second_pred),
[sad]"=&r"(sad)
: [src_stride]"r"((mips_reg)src_stride),
[ref_stride]"r"((mips_reg)ref_stride)
);
/* clang-format on */
return sad;
}
@@ -724,8 +723,9 @@ static inline unsigned int vpx_sad4x(const uint8_t *src, int src_stride,
double ftmp1, ftmp2, ftmp3;
mips_reg l_counter = counter;
/* clang-format off */
__asm__ volatile (
"xor %[ftmp3], %[ftmp3], %[ftmp3] \n\t"
"pxor %[ftmp3], %[ftmp3], %[ftmp3] \n\t"
"1: \n\t"
// Include two loop body, to reduce loop time.
SAD_SRC_REF_ABS_SUB_4
@@ -743,6 +743,7 @@ static inline unsigned int vpx_sad4x(const uint8_t *src, int src_stride,
: [src_stride]"r"((mips_reg)src_stride),
[ref_stride]"r"((mips_reg)ref_stride)
);
/* clang-format on */
return sad;
}
@@ -755,8 +756,6 @@ static inline unsigned int vpx_sad4x(const uint8_t *src, int src_stride,
vpx_sad4xN(8);
vpx_sad4xN(4);
sadMxNxK_mmi(4, 4, 3);
sadMxNxK_mmi(4, 4, 8);
sadMxNx4D_mmi(4, 8);
sadMxNx4D_mmi(4, 4);
@@ -767,9 +766,11 @@ static inline unsigned int vpx_sad_avg4x(const uint8_t *src, int src_stride,
unsigned int sad;
double ftmp1, ftmp2, ftmp3;
mips_reg l_counter = counter;
mips_reg l_second_pred = (mips_reg)second_pred;
/* clang-format off */
__asm__ volatile (
"xor %[ftmp3], %[ftmp3], %[ftmp3] \n\t"
"pxor %[ftmp3], %[ftmp3], %[ftmp3] \n\t"
"1: \n\t"
// Include two loop body, to reduce loop time.
SAD_SRC_AVGREF_ABS_SUB_4
@@ -785,11 +786,12 @@ static inline unsigned int vpx_sad_avg4x(const uint8_t *src, int src_stride,
"mfc1 %[sad], %[ftmp3] \n\t"
: [ftmp1]"=&f"(ftmp1), [ftmp2]"=&f"(ftmp2), [ftmp3]"=&f"(ftmp3),
[counter]"+&r"(l_counter), [src]"+&r"(src), [ref]"+&r"(ref),
[second_pred]"+&r"((mips_reg)second_pred),
[second_pred]"+&r"(l_second_pred),
[sad]"=&r"(sad)
: [src_stride]"r"((mips_reg)src_stride),
[ref_stride]"r"((mips_reg)ref_stride)
);
/* clang-format on */
return sad;
}
libs/libvpx/vpx_dsp/mips/sad_msa.c
+10 -436
View File
@@ -159,380 +159,6 @@ static uint32_t sad_64width_msa(const uint8_t *src, int32_t src_stride,
return sad;
}
static void sad_4width_x3_msa(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *ref_ptr, int32_t ref_stride,
int32_t height, uint32_t *sad_array) {
int32_t ht_cnt;
uint32_t src0, src1, src2, src3;
v16u8 src = { 0 };
v16u8 ref = { 0 };
v16u8 ref0, ref1, ref2, ref3, diff;
v8u16 sad0 = { 0 };
v8u16 sad1 = { 0 };
v8u16 sad2 = { 0 };
for (ht_cnt = (height >> 2); ht_cnt--;) {
LW4(src_ptr, src_stride, src0, src1, src2, src3);
src_ptr += (4 * src_stride);
INSERT_W4_UB(src0, src1, src2, src3, src);
LD_UB4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3);
ref_ptr += (4 * ref_stride);
SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref);
diff = __msa_asub_u_b(src, ref);
sad0 += __msa_hadd_u_h(diff, diff);
SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1);
SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1);
SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref);
diff = __msa_asub_u_b(src, ref);
sad1 += __msa_hadd_u_h(diff, diff);
SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1);
SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1);
SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref);
diff = __msa_asub_u_b(src, ref);
sad2 += __msa_hadd_u_h(diff, diff);
}
sad_array[0] = HADD_UH_U32(sad0);
sad_array[1] = HADD_UH_U32(sad1);
sad_array[2] = HADD_UH_U32(sad2);
}
static void sad_8width_x3_msa(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
int32_t height, uint32_t *sad_array) {
int32_t ht_cnt;
v16u8 src0, src1, src2, src3;
v16u8 ref0, ref1, ref00, ref11, ref22, ref33;
v8u16 sad0 = { 0 };
v8u16 sad1 = { 0 };
v8u16 sad2 = { 0 };
for (ht_cnt = (height >> 2); ht_cnt--;) {
LD_UB4(src, src_stride, src0, src1, src2, src3);
src += (4 * src_stride);
LD_UB4(ref, ref_stride, ref00, ref11, ref22, ref33);
ref += (4 * ref_stride);
PCKEV_D4_UB(src1, src0, src3, src2, ref11, ref00, ref33, ref22, src0, src1,
ref0, ref1);
sad0 += SAD_UB2_UH(src0, src1, ref0, ref1);
SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1);
SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1);
PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1);
sad1 += SAD_UB2_UH(src0, src1, ref0, ref1);
SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1);
SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1);
PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1);
sad2 += SAD_UB2_UH(src0, src1, ref0, ref1);
}
sad_array[0] = HADD_UH_U32(sad0);
sad_array[1] = HADD_UH_U32(sad1);
sad_array[2] = HADD_UH_U32(sad2);
}
static void sad_16width_x3_msa(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *ref_ptr, int32_t ref_stride,
int32_t height, uint32_t *sad_array) {
int32_t ht_cnt;
v16u8 src, ref, ref0, ref1, diff;
v8u16 sad0 = { 0 };
v8u16 sad1 = { 0 };
v8u16 sad2 = { 0 };
for (ht_cnt = (height >> 1); ht_cnt--;) {
src = LD_UB(src_ptr);
src_ptr += src_stride;
LD_UB2(ref_ptr, 16, ref0, ref1);
ref_ptr += ref_stride;
diff = __msa_asub_u_b(src, ref0);
sad0 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1);
diff = __msa_asub_u_b(src, ref);
sad1 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2);
diff = __msa_asub_u_b(src, ref);
sad2 += __msa_hadd_u_h(diff, diff);
src = LD_UB(src_ptr);
src_ptr += src_stride;
LD_UB2(ref_ptr, 16, ref0, ref1);
ref_ptr += ref_stride;
diff = __msa_asub_u_b(src, ref0);
sad0 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1);
diff = __msa_asub_u_b(src, ref);
sad1 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2);
diff = __msa_asub_u_b(src, ref);
sad2 += __msa_hadd_u_h(diff, diff);
}
sad_array[0] = HADD_UH_U32(sad0);
sad_array[1] = HADD_UH_U32(sad1);
sad_array[2] = HADD_UH_U32(sad2);
}
static void sad_4width_x8_msa(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *ref_ptr, int32_t ref_stride,
int32_t height, uint32_t *sad_array) {
int32_t ht_cnt;
uint32_t src0, src1, src2, src3;
v16u8 ref0, ref1, ref2, ref3, diff;
v16u8 src = { 0 };
v16u8 ref = { 0 };
v8u16 sad0 = { 0 };
v8u16 sad1 = { 0 };
v8u16 sad2 = { 0 };
v8u16 sad3 = { 0 };
v8u16 sad4 = { 0 };
v8u16 sad5 = { 0 };
v8u16 sad6 = { 0 };
v8u16 sad7 = { 0 };
for (ht_cnt = (height >> 2); ht_cnt--;) {
LW4(src_ptr, src_stride, src0, src1, src2, src3);
INSERT_W4_UB(src0, src1, src2, src3, src);
src_ptr += (4 * src_stride);
LD_UB4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3);
ref_ptr += (4 * ref_stride);
SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref);
diff = __msa_asub_u_b(src, ref);
sad0 += __msa_hadd_u_h(diff, diff);
SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1);
SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1);
SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref);
diff = __msa_asub_u_b(src, ref);
sad1 += __msa_hadd_u_h(diff, diff);
SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1);
SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1);
SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref);
diff = __msa_asub_u_b(src, ref);
sad2 += __msa_hadd_u_h(diff, diff);
SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1);
SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1);
SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref);
diff = __msa_asub_u_b(src, ref);
sad3 += __msa_hadd_u_h(diff, diff);
SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1);
SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1);
SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref);
diff = __msa_asub_u_b(src, ref);
sad4 += __msa_hadd_u_h(diff, diff);
SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1);
SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1);
SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref);
diff = __msa_asub_u_b(src, ref);
sad5 += __msa_hadd_u_h(diff, diff);
SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1);
SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1);
SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref);
diff = __msa_asub_u_b(src, ref);
sad6 += __msa_hadd_u_h(diff, diff);
SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1);
SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1);
SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref);
diff = __msa_asub_u_b(src, ref);
sad7 += __msa_hadd_u_h(diff, diff);
}
sad_array[0] = HADD_UH_U32(sad0);
sad_array[1] = HADD_UH_U32(sad1);
sad_array[2] = HADD_UH_U32(sad2);
sad_array[3] = HADD_UH_U32(sad3);
sad_array[4] = HADD_UH_U32(sad4);
sad_array[5] = HADD_UH_U32(sad5);
sad_array[6] = HADD_UH_U32(sad6);
sad_array[7] = HADD_UH_U32(sad7);
}
static void sad_8width_x8_msa(const uint8_t *src, int32_t src_stride,
const uint8_t *ref, int32_t ref_stride,
int32_t height, uint32_t *sad_array) {
int32_t ht_cnt;
v16u8 src0, src1, src2, src3;
v16u8 ref0, ref1, ref00, ref11, ref22, ref33;
v8u16 sad0 = { 0 };
v8u16 sad1 = { 0 };
v8u16 sad2 = { 0 };
v8u16 sad3 = { 0 };
v8u16 sad4 = { 0 };
v8u16 sad5 = { 0 };
v8u16 sad6 = { 0 };
v8u16 sad7 = { 0 };
for (ht_cnt = (height >> 2); ht_cnt--;) {
LD_UB4(src, src_stride, src0, src1, src2, src3);
src += (4 * src_stride);
LD_UB4(ref, ref_stride, ref00, ref11, ref22, ref33);
ref += (4 * ref_stride);
PCKEV_D4_UB(src1, src0, src3, src2, ref11, ref00, ref33, ref22, src0, src1,
ref0, ref1);
sad0 += SAD_UB2_UH(src0, src1, ref0, ref1);
SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1);
SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1);
PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1);
sad1 += SAD_UB2_UH(src0, src1, ref0, ref1);
SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1);
SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1);
PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1);
sad2 += SAD_UB2_UH(src0, src1, ref0, ref1);
SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1);
SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1);
PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1);
sad3 += SAD_UB2_UH(src0, src1, ref0, ref1);
SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1);
SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1);
PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1);
sad4 += SAD_UB2_UH(src0, src1, ref0, ref1);
SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1);
SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1);
PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1);
sad5 += SAD_UB2_UH(src0, src1, ref0, ref1);
SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1);
SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1);
PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1);
sad6 += SAD_UB2_UH(src0, src1, ref0, ref1);
SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1);
SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1);
PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1);
sad7 += SAD_UB2_UH(src0, src1, ref0, ref1);
}
sad_array[0] = HADD_UH_U32(sad0);
sad_array[1] = HADD_UH_U32(sad1);
sad_array[2] = HADD_UH_U32(sad2);
sad_array[3] = HADD_UH_U32(sad3);
sad_array[4] = HADD_UH_U32(sad4);
sad_array[5] = HADD_UH_U32(sad5);
sad_array[6] = HADD_UH_U32(sad6);
sad_array[7] = HADD_UH_U32(sad7);
}
static void sad_16width_x8_msa(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *ref_ptr, int32_t ref_stride,
int32_t height, uint32_t *sad_array) {
int32_t ht_cnt;
v16u8 src, ref0, ref1, ref;
v16u8 diff;
v8u16 sad0 = { 0 };
v8u16 sad1 = { 0 };
v8u16 sad2 = { 0 };
v8u16 sad3 = { 0 };
v8u16 sad4 = { 0 };
v8u16 sad5 = { 0 };
v8u16 sad6 = { 0 };
v8u16 sad7 = { 0 };
for (ht_cnt = (height >> 1); ht_cnt--;) {
src = LD_UB(src_ptr);
src_ptr += src_stride;
LD_UB2(ref_ptr, 16, ref0, ref1);
ref_ptr += ref_stride;
diff = __msa_asub_u_b(src, ref0);
sad0 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1);
diff = __msa_asub_u_b(src, ref);
sad1 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2);
diff = __msa_asub_u_b(src, ref);
sad2 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 3);
diff = __msa_asub_u_b(src, ref);
sad3 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 4);
diff = __msa_asub_u_b(src, ref);
sad4 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 5);
diff = __msa_asub_u_b(src, ref);
sad5 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 6);
diff = __msa_asub_u_b(src, ref);
sad6 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 7);
diff = __msa_asub_u_b(src, ref);
sad7 += __msa_hadd_u_h(diff, diff);
src = LD_UB(src_ptr);
src_ptr += src_stride;
LD_UB2(ref_ptr, 16, ref0, ref1);
ref_ptr += ref_stride;
diff = __msa_asub_u_b(src, ref0);
sad0 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1);
diff = __msa_asub_u_b(src, ref);
sad1 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2);
diff = __msa_asub_u_b(src, ref);
sad2 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 3);
diff = __msa_asub_u_b(src, ref);
sad3 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 4);
diff = __msa_asub_u_b(src, ref);
sad4 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 5);
diff = __msa_asub_u_b(src, ref);
sad5 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 6);
diff = __msa_asub_u_b(src, ref);
sad6 += __msa_hadd_u_h(diff, diff);
ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 7);
diff = __msa_asub_u_b(src, ref);
sad7 += __msa_hadd_u_h(diff, diff);
}
sad_array[0] = HADD_UH_U32(sad0);
sad_array[1] = HADD_UH_U32(sad1);
sad_array[2] = HADD_UH_U32(sad2);
sad_array[3] = HADD_UH_U32(sad3);
sad_array[4] = HADD_UH_U32(sad4);
sad_array[5] = HADD_UH_U32(sad5);
sad_array[6] = HADD_UH_U32(sad6);
sad_array[7] = HADD_UH_U32(sad7);
}
static void sad_4width_x4d_msa(const uint8_t *src_ptr, int32_t src_stride,
const uint8_t *const aref_ptr[],
int32_t ref_stride, int32_t height,
@@ -1037,80 +663,38 @@ static uint32_t avgsad_64width_msa(const uint8_t *src, int32_t src_stride,
return sad_64width_msa(src, src_stride, ref, ref_stride, height); \
}
#define VPX_SAD_4xHEIGHTx3_MSA(height) \
void vpx_sad4x##height##x3_msa(const uint8_t *src, int32_t src_stride, \
const uint8_t *ref, int32_t ref_stride, \
uint32_t *sads) { \
sad_4width_x3_msa(src, src_stride, ref, ref_stride, height, sads); \
}
#define VPX_SAD_8xHEIGHTx3_MSA(height) \
void vpx_sad8x##height##x3_msa(const uint8_t *src, int32_t src_stride, \
const uint8_t *ref, int32_t ref_stride, \
uint32_t *sads) { \
sad_8width_x3_msa(src, src_stride, ref, ref_stride, height, sads); \
}
#define VPX_SAD_16xHEIGHTx3_MSA(height) \
void vpx_sad16x##height##x3_msa(const uint8_t *src, int32_t src_stride, \
const uint8_t *ref, int32_t ref_stride, \
uint32_t *sads) { \
sad_16width_x3_msa(src, src_stride, ref, ref_stride, height, sads); \
}
#define VPX_SAD_4xHEIGHTx8_MSA(height) \
void vpx_sad4x##height##x8_msa(const uint8_t *src, int32_t src_stride, \
const uint8_t *ref, int32_t ref_stride, \
uint32_t *sads) { \
sad_4width_x8_msa(src, src_stride, ref, ref_stride, height, sads); \
}
#define VPX_SAD_8xHEIGHTx8_MSA(height) \
void vpx_sad8x##height##x8_msa(const uint8_t *src, int32_t src_stride, \
const uint8_t *ref, int32_t ref_stride, \
uint32_t *sads) { \
sad_8width_x8_msa(src, src_stride, ref, ref_stride, height, sads); \
}
#define VPX_SAD_16xHEIGHTx8_MSA(height) \
void vpx_sad16x##height##x8_msa(const uint8_t *src, int32_t src_stride, \
const uint8_t *ref, int32_t ref_stride, \
uint32_t *sads) { \
sad_16width_x8_msa(src, src_stride, ref, ref_stride, height, sads); \
}
#define VPX_SAD_4xHEIGHTx4D_MSA(height) \
void vpx_sad4x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \
const uint8_t *const refs[], \
int32_t ref_stride, uint32_t *sads) { \
const uint8_t *const refs[4], \
int32_t ref_stride, uint32_t sads[4]) { \
sad_4width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \
}
#define VPX_SAD_8xHEIGHTx4D_MSA(height) \
void vpx_sad8x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \
const uint8_t *const refs[], \
int32_t ref_stride, uint32_t *sads) { \
const uint8_t *const refs[4], \
int32_t ref_stride, uint32_t sads[4]) { \
sad_8width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \
}
#define VPX_SAD_16xHEIGHTx4D_MSA(height) \
void vpx_sad16x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \
const uint8_t *const refs[], \
int32_t ref_stride, uint32_t *sads) { \
const uint8_t *const refs[4], \
int32_t ref_stride, uint32_t sads[4]) { \
sad_16width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \
}
#define VPX_SAD_32xHEIGHTx4D_MSA(height) \
void vpx_sad32x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \
const uint8_t *const refs[], \
int32_t ref_stride, uint32_t *sads) { \
const uint8_t *const refs[4], \
int32_t ref_stride, uint32_t sads[4]) { \
sad_32width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \
}
#define VPX_SAD_64xHEIGHTx4D_MSA(height) \
void vpx_sad64x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \
const uint8_t *const refs[], \
int32_t ref_stride, uint32_t *sads) { \
const uint8_t *const refs[4], \
int32_t ref_stride, uint32_t sads[4]) { \
sad_64width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \
}
@@ -1186,29 +770,21 @@ VPX_AVGSAD_16xHEIGHT_MSA(32);
// 16x16
VPX_SAD_16xHEIGHT_MSA(16);
VPX_SAD_16xHEIGHTx3_MSA(16);
VPX_SAD_16xHEIGHTx8_MSA(16);
VPX_SAD_16xHEIGHTx4D_MSA(16);
VPX_AVGSAD_16xHEIGHT_MSA(16);
// 16x8
VPX_SAD_16xHEIGHT_MSA(8);
VPX_SAD_16xHEIGHTx3_MSA(8);
VPX_SAD_16xHEIGHTx8_MSA(8);
VPX_SAD_16xHEIGHTx4D_MSA(8);
VPX_AVGSAD_16xHEIGHT_MSA(8);
// 8x16
VPX_SAD_8xHEIGHT_MSA(16);
VPX_SAD_8xHEIGHTx3_MSA(16);
VPX_SAD_8xHEIGHTx8_MSA(16);
VPX_SAD_8xHEIGHTx4D_MSA(16);
VPX_AVGSAD_8xHEIGHT_MSA(16);
// 8x8
VPX_SAD_8xHEIGHT_MSA(8);
VPX_SAD_8xHEIGHTx3_MSA(8);
VPX_SAD_8xHEIGHTx8_MSA(8);
VPX_SAD_8xHEIGHTx4D_MSA(8);
VPX_AVGSAD_8xHEIGHT_MSA(8);
@@ -1224,7 +800,5 @@ VPX_AVGSAD_4xHEIGHT_MSA(8);
// 4x4
VPX_SAD_4xHEIGHT_MSA(4);
VPX_SAD_4xHEIGHTx3_MSA(4);
VPX_SAD_4xHEIGHTx8_MSA(4);
VPX_SAD_4xHEIGHTx4D_MSA(4);
VPX_AVGSAD_4xHEIGHT_MSA(4);
libs/libvpx/vpx_dsp/mips/subtract_mmi.c
+3 -3
View File
@@ -24,7 +24,7 @@ void vpx_subtract_block_mmi(int rows, int cols, int16_t *diff,
switch (rows) {
case 4:
__asm__ volatile(
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
#if _MIPS_SIM == _ABIO32
"ulw %[tmp0], 0x00(%[src]) \n\t"
"mtc1 %[tmp0], %[ftmp1] \n\t"
@@ -118,7 +118,7 @@ void vpx_subtract_block_mmi(int rows, int cols, int16_t *diff,
break;
case 8:
__asm__ volatile(
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"li %[tmp0], 0x02 \n\t"
"1: \n\t"
"gsldlc1 %[ftmp1], 0x07(%[src]) \n\t"
@@ -206,7 +206,7 @@ void vpx_subtract_block_mmi(int rows, int cols, int16_t *diff,
break;
case 16:
__asm__ volatile(
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"li %[tmp0], 0x08 \n\t"
"1: \n\t"
"gsldlc1 %[ftmp1], 0x07(%[src]) \n\t"
libs/libvpx/vpx_dsp/mips/variance_mmi.c
+157 -91
View File
@@ -150,7 +150,7 @@ static const uint8_t bilinear_filters[8][2] = {
"psrlh %[ftmp2], %[ftmp2], %[ftmp6] \n\t" \
\
/* store: temp2[0] ~ temp2[3] */ \
"and %[ftmp2], %[ftmp2], %[mask] \n\t" \
"pand %[ftmp2], %[ftmp2], %[mask] \n\t" \
"packushb %[ftmp2], %[ftmp2], %[ftmp0] \n\t" \
"gssdrc1 %[ftmp2], 0x00(%[temp2_ptr]) \n\t"
@@ -163,7 +163,7 @@ static const uint8_t bilinear_filters[8][2] = {
"psrlh %[ftmp4], %[ftmp4], %[ftmp6] \n\t" \
\
/* store: temp2[0] ~ temp2[3] */ \
"and %[ftmp4], %[ftmp4], %[mask] \n\t" \
"pand %[ftmp4], %[ftmp4], %[mask] \n\t" \
"packushb %[ftmp4], %[ftmp4], %[ftmp0] \n\t" \
"gssdrc1 %[ftmp4], 0x00(%[temp2_ptr]) \n\t"
@@ -225,8 +225,8 @@ static const uint8_t bilinear_filters[8][2] = {
"psrlh %[ftmp3], %[ftmp3], %[ftmp14] \n\t" \
\
/* store: temp2[0] ~ temp2[7] */ \
"and %[ftmp2], %[ftmp2], %[mask] \n\t" \
"and %[ftmp3], %[ftmp3], %[mask] \n\t" \
"pand %[ftmp2], %[ftmp2], %[mask] \n\t" \
"pand %[ftmp3], %[ftmp3], %[mask] \n\t" \
"packushb %[ftmp2], %[ftmp2], %[ftmp3] \n\t" \
"gssdlc1 %[ftmp2], 0x07(%[temp2_ptr]) \n\t" \
"gssdrc1 %[ftmp2], 0x00(%[temp2_ptr]) \n\t"
@@ -247,8 +247,8 @@ static const uint8_t bilinear_filters[8][2] = {
"psrlh %[ftmp9], %[ftmp9], %[ftmp14] \n\t" \
\
/* store: temp2[0] ~ temp2[7] */ \
"and %[ftmp8], %[ftmp8], %[mask] \n\t" \
"and %[ftmp9], %[ftmp9], %[mask] \n\t" \
"pand %[ftmp8], %[ftmp8], %[mask] \n\t" \
"pand %[ftmp9], %[ftmp9], %[mask] \n\t" \
"packushb %[ftmp8], %[ftmp8], %[ftmp9] \n\t" \
"gssdlc1 %[ftmp8], 0x07(%[temp2_ptr]) \n\t" \
"gssdrc1 %[ftmp8], 0x00(%[temp2_ptr]) \n\t"
@@ -319,8 +319,8 @@ static const uint8_t bilinear_filters[8][2] = {
"psrlh %[ftmp5], %[ftmp5], %[ftmp14] \n\t" \
\
/* store: temp2[8] ~ temp2[15] */ \
"and %[ftmp4], %[ftmp4], %[mask] \n\t" \
"and %[ftmp5], %[ftmp5], %[mask] \n\t" \
"pand %[ftmp4], %[ftmp4], %[mask] \n\t" \
"pand %[ftmp5], %[ftmp5], %[mask] \n\t" \
"packushb %[ftmp4], %[ftmp4], %[ftmp5] \n\t" \
"gssdlc1 %[ftmp4], 0x0f(%[temp2_ptr]) \n\t" \
"gssdrc1 %[ftmp4], 0x08(%[temp2_ptr]) \n\t"
@@ -343,8 +343,8 @@ static const uint8_t bilinear_filters[8][2] = {
"psrlh %[ftmp11], %[ftmp11], %[ftmp14] \n\t" \
\
/* store: temp2[8] ~ temp2[15] */ \
"and %[ftmp10], %[ftmp10], %[mask] \n\t" \
"and %[ftmp11], %[ftmp11], %[mask] \n\t" \
"pand %[ftmp10], %[ftmp10], %[mask] \n\t" \
"pand %[ftmp11], %[ftmp11], %[mask] \n\t" \
"packushb %[ftmp10], %[ftmp10], %[ftmp11] \n\t" \
"gssdlc1 %[ftmp10], 0x0f(%[temp2_ptr]) \n\t" \
"gssdrc1 %[ftmp10], 0x08(%[temp2_ptr]) \n\t"
@@ -414,13 +414,14 @@ static inline uint32_t vpx_variance64x(const uint8_t *src_ptr, int src_stride,
*sse = 0;
/* clang-format off */
__asm__ volatile (
"li %[tmp0], 0x20 \n\t"
"mtc1 %[tmp0], %[ftmp11] \n\t"
MMI_L(%[tmp0], %[high], 0x00)
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"xor %[ftmp9], %[ftmp9], %[ftmp9] \n\t"
"xor %[ftmp10], %[ftmp10], %[ftmp10] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp9], %[ftmp9], %[ftmp9] \n\t"
"pxor %[ftmp10], %[ftmp10], %[ftmp10] \n\t"
"1: \n\t"
"gsldlc1 %[ftmp1], 0x07(%[src_ptr]) \n\t"
"gsldrc1 %[ftmp1], 0x00(%[src_ptr]) \n\t"
@@ -478,7 +479,7 @@ static inline uint32_t vpx_variance64x(const uint8_t *src_ptr, int src_stride,
"mfc1 %[tmp1], %[ftmp9] \n\t"
"mfhc1 %[tmp2], %[ftmp9] \n\t"
"addu %[sum], %[tmp1], %[tmp2] \n\t"
"dsrl %[ftmp1], %[ftmp10], %[ftmp11] \n\t"
"ssrld %[ftmp1], %[ftmp10], %[ftmp11] \n\t"
"paddw %[ftmp1], %[ftmp1], %[ftmp10] \n\t"
"swc1 %[ftmp1], 0x00(%[sse]) \n\t"
: [ftmp0]"=&f"(ftmp[0]), [ftmp1]"=&f"(ftmp[1]),
@@ -496,6 +497,7 @@ static inline uint32_t vpx_variance64x(const uint8_t *src_ptr, int src_stride,
[high]"r"(&high), [sse]"r"(sse)
: "memory"
);
/* clang-format on */
return *sse - (((int64_t)sum * sum) / (64 * high));
}
@@ -519,13 +521,14 @@ uint32_t vpx_variance32x64_mmi(const uint8_t *src_ptr, int src_stride,
*sse = 0;
/* clang-format off */
__asm__ volatile (
"li %[tmp0], 0x20 \n\t"
"mtc1 %[tmp0], %[ftmp11] \n\t"
"li %[tmp0], 0x40 \n\t"
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"xor %[ftmp9], %[ftmp9], %[ftmp9] \n\t"
"xor %[ftmp10], %[ftmp10], %[ftmp10] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp9], %[ftmp9], %[ftmp9] \n\t"
"pxor %[ftmp10], %[ftmp10], %[ftmp10] \n\t"
"1: \n\t"
"gsldlc1 %[ftmp1], 0x07(%[src_ptr]) \n\t"
"gsldrc1 %[ftmp1], 0x00(%[src_ptr]) \n\t"
@@ -559,7 +562,7 @@ uint32_t vpx_variance32x64_mmi(const uint8_t *src_ptr, int src_stride,
"mfc1 %[tmp1], %[ftmp9] \n\t"
"mfhc1 %[tmp2], %[ftmp9] \n\t"
"addu %[sum], %[tmp1], %[tmp2] \n\t"
"dsrl %[ftmp1], %[ftmp10], %[ftmp11] \n\t"
"ssrld %[ftmp1], %[ftmp10], %[ftmp11] \n\t"
"paddw %[ftmp1], %[ftmp1], %[ftmp10] \n\t"
"swc1 %[ftmp1], 0x00(%[sse]) \n\t"
: [ftmp0]"=&f"(ftmp[0]), [ftmp1]"=&f"(ftmp[1]),
@@ -577,6 +580,7 @@ uint32_t vpx_variance32x64_mmi(const uint8_t *src_ptr, int src_stride,
[sse]"r"(sse)
: "memory"
);
/* clang-format on */
return *sse - (((int64_t)sum * sum) / 2048);
}
@@ -590,14 +594,15 @@ static inline uint32_t vpx_variance32x(const uint8_t *src_ptr, int src_stride,
*sse = 0;
/* clang-format off */
__asm__ volatile (
"li %[tmp0], 0x20 \n\t"
"mtc1 %[tmp0], %[ftmp11] \n\t"
MMI_L(%[tmp0], %[high], 0x00)
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"xor %[ftmp8], %[ftmp8], %[ftmp8] \n\t"
"xor %[ftmp10], %[ftmp10], %[ftmp10] \n\t"
"xor %[ftmp12], %[ftmp12], %[ftmp12] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp8], %[ftmp8], %[ftmp8] \n\t"
"pxor %[ftmp10], %[ftmp10], %[ftmp10] \n\t"
"pxor %[ftmp12], %[ftmp12], %[ftmp12] \n\t"
"1: \n\t"
"gsldlc1 %[ftmp1], 0x07(%[src_ptr]) \n\t"
"gsldrc1 %[ftmp1], 0x00(%[src_ptr]) \n\t"
@@ -625,7 +630,7 @@ static inline uint32_t vpx_variance32x(const uint8_t *src_ptr, int src_stride,
MMI_ADDU(%[ref_ptr], %[ref_ptr], %[ref_stride])
"bnez %[tmp0], 1b \n\t"
"dsrl %[ftmp9], %[ftmp8], %[ftmp11] \n\t"
"ssrld %[ftmp9], %[ftmp8], %[ftmp11] \n\t"
"paddw %[ftmp9], %[ftmp9], %[ftmp8] \n\t"
"swc1 %[ftmp9], 0x00(%[sse]) \n\t"
@@ -636,7 +641,7 @@ static inline uint32_t vpx_variance32x(const uint8_t *src_ptr, int src_stride,
"paddw %[ftmp3], %[ftmp3], %[ftmp4] \n\t"
"psubw %[ftmp3], %[ftmp3], %[ftmp5] \n\t"
"psubw %[ftmp3], %[ftmp3], %[ftmp6] \n\t"
"dsrl %[ftmp0], %[ftmp3], %[ftmp11] \n\t"
"ssrld %[ftmp0], %[ftmp3], %[ftmp11] \n\t"
"paddw %[ftmp0], %[ftmp0], %[ftmp3] \n\t"
"swc1 %[ftmp0], 0x00(%[sum]) \n\t"
@@ -653,6 +658,7 @@ static inline uint32_t vpx_variance32x(const uint8_t *src_ptr, int src_stride,
[high]"r"(&high), [sse]"r"(sse), [sum]"r"(&sum)
: "memory"
);
/* clang-format on */
return *sse - (((int64_t)sum * sum) / (32 * high));
}
@@ -676,14 +682,15 @@ static inline uint32_t vpx_variance16x(const uint8_t *src_ptr, int src_stride,
*sse = 0;
/* clang-format off */
__asm__ volatile (
"li %[tmp0], 0x20 \n\t"
"mtc1 %[tmp0], %[ftmp11] \n\t"
MMI_L(%[tmp0], %[high], 0x00)
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"xor %[ftmp8], %[ftmp8], %[ftmp8] \n\t"
"xor %[ftmp10], %[ftmp10], %[ftmp10] \n\t"
"xor %[ftmp12], %[ftmp12], %[ftmp12] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp8], %[ftmp8], %[ftmp8] \n\t"
"pxor %[ftmp10], %[ftmp10], %[ftmp10] \n\t"
"pxor %[ftmp12], %[ftmp12], %[ftmp12] \n\t"
"1: \n\t"
"gsldlc1 %[ftmp1], 0x07(%[src_ptr]) \n\t"
"gsldrc1 %[ftmp1], 0x00(%[src_ptr]) \n\t"
@@ -701,7 +708,7 @@ static inline uint32_t vpx_variance16x(const uint8_t *src_ptr, int src_stride,
MMI_ADDU(%[ref_ptr], %[ref_ptr], %[ref_stride])
"bnez %[tmp0], 1b \n\t"
"dsrl %[ftmp9], %[ftmp8], %[ftmp11] \n\t"
"ssrld %[ftmp9], %[ftmp8], %[ftmp11] \n\t"
"paddw %[ftmp9], %[ftmp9], %[ftmp8] \n\t"
"swc1 %[ftmp9], 0x00(%[sse]) \n\t"
@@ -712,7 +719,7 @@ static inline uint32_t vpx_variance16x(const uint8_t *src_ptr, int src_stride,
"paddw %[ftmp3], %[ftmp3], %[ftmp4] \n\t"
"psubw %[ftmp3], %[ftmp3], %[ftmp5] \n\t"
"psubw %[ftmp3], %[ftmp3], %[ftmp6] \n\t"
"dsrl %[ftmp0], %[ftmp3], %[ftmp11] \n\t"
"ssrld %[ftmp0], %[ftmp3], %[ftmp11] \n\t"
"paddw %[ftmp0], %[ftmp0], %[ftmp3] \n\t"
"swc1 %[ftmp0], 0x00(%[sum]) \n\t"
@@ -729,6 +736,7 @@ static inline uint32_t vpx_variance16x(const uint8_t *src_ptr, int src_stride,
[high]"r"(&high), [sse]"r"(sse), [sum]"r"(&sum)
: "memory"
);
/* clang-format on */
return *sse - (((int64_t)sum * sum) / (16 * high));
}
@@ -753,14 +761,15 @@ static inline uint32_t vpx_variance8x(const uint8_t *src_ptr, int src_stride,
*sse = 0;
/* clang-format off */
__asm__ volatile (
"li %[tmp0], 0x20 \n\t"
"mtc1 %[tmp0], %[ftmp11] \n\t"
MMI_L(%[tmp0], %[high], 0x00)
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"xor %[ftmp8], %[ftmp8], %[ftmp8] \n\t"
"xor %[ftmp10], %[ftmp10], %[ftmp10] \n\t"
"xor %[ftmp12], %[ftmp12], %[ftmp12] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp8], %[ftmp8], %[ftmp8] \n\t"
"pxor %[ftmp10], %[ftmp10], %[ftmp10] \n\t"
"pxor %[ftmp12], %[ftmp12], %[ftmp12] \n\t"
"1: \n\t"
"gsldlc1 %[ftmp1], 0x07(%[src_ptr]) \n\t"
"gsldrc1 %[ftmp1], 0x00(%[src_ptr]) \n\t"
@@ -773,7 +782,7 @@ static inline uint32_t vpx_variance8x(const uint8_t *src_ptr, int src_stride,
MMI_ADDU(%[ref_ptr], %[ref_ptr], %[ref_stride])
"bnez %[tmp0], 1b \n\t"
"dsrl %[ftmp9], %[ftmp8], %[ftmp11] \n\t"
"ssrld %[ftmp9], %[ftmp8], %[ftmp11] \n\t"
"paddw %[ftmp9], %[ftmp9], %[ftmp8] \n\t"
"swc1 %[ftmp9], 0x00(%[sse]) \n\t"
@@ -784,7 +793,7 @@ static inline uint32_t vpx_variance8x(const uint8_t *src_ptr, int src_stride,
"paddw %[ftmp3], %[ftmp3], %[ftmp4] \n\t"
"psubw %[ftmp3], %[ftmp3], %[ftmp5] \n\t"
"psubw %[ftmp3], %[ftmp3], %[ftmp6] \n\t"
"dsrl %[ftmp0], %[ftmp3], %[ftmp11] \n\t"
"ssrld %[ftmp0], %[ftmp3], %[ftmp11] \n\t"
"paddw %[ftmp0], %[ftmp0], %[ftmp3] \n\t"
"swc1 %[ftmp0], 0x00(%[sum]) \n\t"
@@ -801,6 +810,7 @@ static inline uint32_t vpx_variance8x(const uint8_t *src_ptr, int src_stride,
[high]"r"(&high), [sse]"r"(sse), [sum]"r"(&sum)
: "memory"
);
/* clang-format on */
return *sse - (((int64_t)sum * sum) / (8 * high));
}
@@ -825,14 +835,15 @@ static inline uint32_t vpx_variance4x(const uint8_t *src_ptr, int src_stride,
*sse = 0;
/* clang-format off */
__asm__ volatile (
"li %[tmp0], 0x20 \n\t"
"mtc1 %[tmp0], %[ftmp10] \n\t"
MMI_L(%[tmp0], %[high], 0x00)
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"xor %[ftmp6], %[ftmp6], %[ftmp6] \n\t"
"xor %[ftmp7], %[ftmp7], %[ftmp7] \n\t"
"xor %[ftmp8], %[ftmp8], %[ftmp8] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp6], %[ftmp6], %[ftmp6] \n\t"
"pxor %[ftmp7], %[ftmp7], %[ftmp7] \n\t"
"pxor %[ftmp8], %[ftmp8], %[ftmp8] \n\t"
"1: \n\t"
"gsldlc1 %[ftmp1], 0x07(%[src_ptr]) \n\t"
"gsldrc1 %[ftmp1], 0x00(%[src_ptr]) \n\t"
@@ -845,7 +856,7 @@ static inline uint32_t vpx_variance4x(const uint8_t *src_ptr, int src_stride,
MMI_ADDU(%[ref_ptr], %[ref_ptr], %[ref_stride])
"bnez %[tmp0], 1b \n\t"
"dsrl %[ftmp9], %[ftmp6], %[ftmp10] \n\t"
"ssrld %[ftmp9], %[ftmp6], %[ftmp10] \n\t"
"paddw %[ftmp9], %[ftmp9], %[ftmp6] \n\t"
"swc1 %[ftmp9], 0x00(%[sse]) \n\t"
@@ -856,7 +867,7 @@ static inline uint32_t vpx_variance4x(const uint8_t *src_ptr, int src_stride,
"paddw %[ftmp3], %[ftmp3], %[ftmp4] \n\t"
"psubw %[ftmp3], %[ftmp3], %[ftmp5] \n\t"
"psubw %[ftmp3], %[ftmp3], %[ftmp6] \n\t"
"dsrl %[ftmp0], %[ftmp3], %[ftmp10] \n\t"
"ssrld %[ftmp0], %[ftmp3], %[ftmp10] \n\t"
"paddw %[ftmp0], %[ftmp0], %[ftmp3] \n\t"
"swc1 %[ftmp0], 0x00(%[sum]) \n\t"
: [ftmp0]"=&f"(ftmp[0]), [ftmp1]"=&f"(ftmp[1]),
@@ -872,6 +883,7 @@ static inline uint32_t vpx_variance4x(const uint8_t *src_ptr, int src_stride,
[high]"r"(&high), [sse]"r"(sse), [sum]"r"(&sum)
: "memory"
);
/* clang-format on */
return *sse - (((int64_t)sum * sum) / (4 * high));
}
@@ -894,12 +906,13 @@ static inline uint32_t vpx_mse16x(const uint8_t *src_ptr, int src_stride,
*sse = 0;
/* clang-format off */
__asm__ volatile (
"li %[tmp0], 0x20 \n\t"
"mtc1 %[tmp0], %[ftmp11] \n\t"
MMI_L(%[tmp0], %[high], 0x00)
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"xor %[ftmp8], %[ftmp8], %[ftmp8] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp8], %[ftmp8], %[ftmp8] \n\t"
"1: \n\t"
VARIANCE_SSE_16
@@ -909,7 +922,7 @@ static inline uint32_t vpx_mse16x(const uint8_t *src_ptr, int src_stride,
MMI_ADDU(%[ref_ptr], %[ref_ptr], %[ref_stride])
"bnez %[tmp0], 1b \n\t"
"dsrl %[ftmp9], %[ftmp8], %[ftmp11] \n\t"
"ssrld %[ftmp9], %[ftmp8], %[ftmp11] \n\t"
"paddw %[ftmp9], %[ftmp9], %[ftmp8] \n\t"
"swc1 %[ftmp9], 0x00(%[sse]) \n\t"
: [ftmp0]"=&f"(ftmp[0]), [ftmp1]"=&f"(ftmp[1]),
@@ -925,6 +938,7 @@ static inline uint32_t vpx_mse16x(const uint8_t *src_ptr, int src_stride,
[high]"r"(&high), [sse]"r"(sse)
: "memory"
);
/* clang-format on */
return *sse;
}
@@ -947,12 +961,13 @@ static inline uint32_t vpx_mse8x(const uint8_t *src_ptr, int src_stride,
*sse = 0;
/* clang-format off */
__asm__ volatile (
"li %[tmp0], 0x20 \n\t"
"mtc1 %[tmp0], %[ftmp11] \n\t"
MMI_L(%[tmp0], %[high], 0x00)
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"xor %[ftmp8], %[ftmp8], %[ftmp8] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp8], %[ftmp8], %[ftmp8] \n\t"
"1: \n\t"
VARIANCE_SSE_8
@@ -962,7 +977,7 @@ static inline uint32_t vpx_mse8x(const uint8_t *src_ptr, int src_stride,
MMI_ADDU(%[ref_ptr], %[ref_ptr], %[ref_stride])
"bnez %[tmp0], 1b \n\t"
"dsrl %[ftmp9], %[ftmp8], %[ftmp11] \n\t"
"ssrld %[ftmp9], %[ftmp8], %[ftmp11] \n\t"
"paddw %[ftmp9], %[ftmp9], %[ftmp8] \n\t"
"swc1 %[ftmp9], 0x00(%[sse]) \n\t"
: [ftmp0]"=&f"(ftmp[0]), [ftmp1]"=&f"(ftmp[1]),
@@ -978,6 +993,7 @@ static inline uint32_t vpx_mse8x(const uint8_t *src_ptr, int src_stride,
[high]"r"(&high), [sse]"r"(sse)
: "memory"
);
/* clang-format on */
return *sse;
}
@@ -1021,22 +1037,39 @@ static inline void var_filter_block2d_bil_16x(const uint8_t *src_ptr,
uint8_t *temp2_ptr = temp2;
mips_reg l_counter = counter;
double ftmp[15];
double ff_ph_40, mask;
double filter_x0, filter_x1, filter_y0, filter_y1;
mips_reg tmp[2];
DECLARE_ALIGNED(8, const uint64_t, ff_ph_40) = { 0x0040004000400040ULL };
DECLARE_ALIGNED(8, const uint64_t, mask) = { 0x00ff00ff00ff00ffULL };
uint64_t x0, x1, y0, y1, all;
const uint8_t *filter_x = bilinear_filters[x_offset];
const uint8_t *filter_y = bilinear_filters[y_offset];
x0 = (uint64_t)filter_x[0];
x1 = (uint64_t)filter_x[1];
y0 = (uint64_t)filter_y[0];
y1 = (uint64_t)filter_y[1];
all = x0 | x1 << 8 | y0 << 16 | y1 << 24;
/* clang-format off */
__asm__ volatile (
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
MMI_MTC1(%[all], %[ftmp14])
"punpcklbh %[ftmp14], %[ftmp14], %[ftmp0] \n\t"
"pshufh %[filter_x0], %[ftmp14], %[ftmp0] \n\t"
MMI_LI(%[tmp0], 0x10)
MMI_MTC1(%[tmp0], %[mask])
"ssrld %[ftmp14], %[ftmp14], %[mask] \n\t"
"pshufh %[filter_x1], %[ftmp14], %[ftmp0] \n\t"
"ssrld %[ftmp14], %[ftmp14], %[mask] \n\t"
"pshufh %[filter_y0], %[ftmp14], %[ftmp0] \n\t"
"ssrld %[ftmp14], %[ftmp14], %[mask] \n\t"
"pshufh %[filter_y1], %[ftmp14], %[ftmp0] \n\t"
MMI_LI(%[tmp0], 0x07)
MMI_MTC1(%[tmp0], %[ftmp14])
"pshufh %[filter_x0], %[filter_x0], %[ftmp0] \n\t"
"pshufh %[filter_x1], %[filter_x1], %[ftmp0] \n\t"
"pshufh %[filter_y0], %[filter_y0], %[ftmp0] \n\t"
"pshufh %[filter_y1], %[filter_y1], %[ftmp0] \n\t"
MMI_LI(%[tmp0], 0x0040004000400040)
MMI_MTC1(%[tmp0], %[ff_ph_40])
MMI_LI(%[tmp0], 0x00ff00ff00ff00ff)
MMI_MTC1(%[tmp0], %[mask])
// fdata3: fdata3[0] ~ fdata3[15]
VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_16_A
@@ -1072,15 +1105,13 @@ static inline void var_filter_block2d_bil_16x(const uint8_t *src_ptr,
[ftmp11] "=&f"(ftmp[11]), [ftmp12] "=&f"(ftmp[12]),
[ftmp13] "=&f"(ftmp[13]), [ftmp14] "=&f"(ftmp[14]),
[tmp0] "=&r"(tmp[0]), [src_ptr] "+&r"(src_ptr), [temp2_ptr] "+&r"(temp2_ptr),
[counter]"+&r"(l_counter)
: [filter_x0] "f"((uint64_t)filter_x[0]),
[filter_x1] "f"((uint64_t)filter_x[1]),
[filter_y0] "f"((uint64_t)filter_y[0]),
[filter_y1] "f"((uint64_t)filter_y[1]),
[src_stride] "r"((mips_reg)src_stride), [ff_ph_40] "f"(ff_ph_40),
[mask] "f"(mask)
[counter]"+&r"(l_counter), [ff_ph_40] "=&f"(ff_ph_40), [mask] "=&f"(mask),
[filter_x0] "=&f"(filter_x0), [filter_x1] "=&f"(filter_x1),
[filter_y0] "=&f"(filter_y0), [filter_y1] "=&f"(filter_y1)
: [src_stride] "r"((mips_reg)src_stride), [all] "r"(all)
: "memory"
);
/* clang-format on */
}
#define SUBPIX_VAR16XN(H) \
@@ -1105,19 +1136,38 @@ static inline void var_filter_block2d_bil_8x(const uint8_t *src_ptr,
mips_reg l_counter = counter;
double ftmp[15];
mips_reg tmp[2];
DECLARE_ALIGNED(8, const uint64_t, ff_ph_40) = { 0x0040004000400040ULL };
DECLARE_ALIGNED(8, const uint64_t, mask) = { 0x00ff00ff00ff00ffULL };
double ff_ph_40, mask;
uint64_t x0, x1, y0, y1, all;
double filter_x0, filter_x1, filter_y0, filter_y1;
const uint8_t *filter_x = bilinear_filters[x_offset];
const uint8_t *filter_y = bilinear_filters[y_offset];
x0 = (uint64_t)filter_x[0];
x1 = (uint64_t)filter_x[1];
y0 = (uint64_t)filter_y[0];
y1 = (uint64_t)filter_y[1];
all = x0 | x1 << 8 | y0 << 16 | y1 << 24;
/* clang-format off */
__asm__ volatile (
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
MMI_MTC1(%[all], %[ftmp14])
"punpcklbh %[ftmp14], %[ftmp14], %[ftmp0] \n\t"
"pshufh %[filter_x0], %[ftmp14], %[ftmp0] \n\t"
MMI_LI(%[tmp0], 0x10)
MMI_MTC1(%[tmp0], %[mask])
"ssrld %[ftmp14], %[ftmp14], %[mask] \n\t"
"pshufh %[filter_x1], %[ftmp14], %[ftmp0] \n\t"
"ssrld %[ftmp14], %[ftmp14], %[mask] \n\t"
"pshufh %[filter_y0], %[ftmp14], %[ftmp0] \n\t"
"ssrld %[ftmp14], %[ftmp14], %[mask] \n\t"
"pshufh %[filter_y1], %[ftmp14], %[ftmp0] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
MMI_LI(%[tmp0], 0x07)
MMI_MTC1(%[tmp0], %[ftmp14])
"pshufh %[filter_x0], %[filter_x0], %[ftmp0] \n\t"
"pshufh %[filter_x1], %[filter_x1], %[ftmp0] \n\t"
"pshufh %[filter_y0], %[filter_y0], %[ftmp0] \n\t"
"pshufh %[filter_y1], %[filter_y1], %[ftmp0] \n\t"
MMI_LI(%[tmp0], 0x0040004000400040)
MMI_MTC1(%[tmp0], %[ff_ph_40])
MMI_LI(%[tmp0], 0x00ff00ff00ff00ff)
MMI_MTC1(%[tmp0], %[mask])
// fdata3: fdata3[0] ~ fdata3[7]
VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_8_A
@@ -1154,15 +1204,13 @@ static inline void var_filter_block2d_bil_8x(const uint8_t *src_ptr,
[ftmp11] "=&f"(ftmp[11]), [ftmp12] "=&f"(ftmp[12]),
[ftmp13] "=&f"(ftmp[13]), [ftmp14] "=&f"(ftmp[14]),
[tmp0] "=&r"(tmp[0]), [src_ptr] "+&r"(src_ptr), [temp2_ptr] "+&r"(temp2_ptr),
[counter]"+&r"(l_counter)
: [filter_x0] "f"((uint64_t)filter_x[0]),
[filter_x1] "f"((uint64_t)filter_x[1]),
[filter_y0] "f"((uint64_t)filter_y[0]),
[filter_y1] "f"((uint64_t)filter_y[1]),
[src_stride] "r"((mips_reg)src_stride), [ff_ph_40] "f"(ff_ph_40),
[mask] "f"(mask)
[counter]"+&r"(l_counter), [ff_ph_40] "=&f"(ff_ph_40), [mask] "=&f"(mask),
[filter_x0] "=&f"(filter_x0), [filter_x1] "=&f"(filter_x1),
[filter_y0] "=&f"(filter_y0), [filter_y1] "=&f"(filter_y1)
: [src_stride] "r"((mips_reg)src_stride), [all] "r"(all)
: "memory"
);
/* clang-format on */
}
#define SUBPIX_VAR8XN(H) \
@@ -1188,19 +1236,38 @@ static inline void var_filter_block2d_bil_4x(const uint8_t *src_ptr,
mips_reg l_counter = counter;
double ftmp[7];
mips_reg tmp[2];
DECLARE_ALIGNED(8, const uint64_t, ff_ph_40) = { 0x0040004000400040ULL };
DECLARE_ALIGNED(8, const uint64_t, mask) = { 0x00ff00ff00ff00ffULL };
double ff_ph_40, mask;
uint64_t x0, x1, y0, y1, all;
double filter_x0, filter_x1, filter_y0, filter_y1;
const uint8_t *filter_x = bilinear_filters[x_offset];
const uint8_t *filter_y = bilinear_filters[y_offset];
x0 = (uint64_t)filter_x[0];
x1 = (uint64_t)filter_x[1];
y0 = (uint64_t)filter_y[0];
y1 = (uint64_t)filter_y[1];
all = x0 | x1 << 8 | y0 << 16 | y1 << 24;
/* clang-format off */
__asm__ volatile (
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
MMI_MTC1(%[all], %[ftmp6])
"punpcklbh %[ftmp6], %[ftmp6], %[ftmp0] \n\t"
"pshufh %[filter_x0], %[ftmp6], %[ftmp0] \n\t"
MMI_LI(%[tmp0], 0x10)
MMI_MTC1(%[tmp0], %[mask])
"ssrld %[ftmp6], %[ftmp6], %[mask] \n\t"
"pshufh %[filter_x1], %[ftmp6], %[ftmp0] \n\t"
"ssrld %[ftmp6], %[ftmp6], %[mask] \n\t"
"pshufh %[filter_y0], %[ftmp6], %[ftmp0] \n\t"
"ssrld %[ftmp6], %[ftmp6], %[mask] \n\t"
"pshufh %[filter_y1], %[ftmp6], %[ftmp0] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
MMI_LI(%[tmp0], 0x07)
MMI_MTC1(%[tmp0], %[ftmp6])
"pshufh %[filter_x0], %[filter_x0], %[ftmp0] \n\t"
"pshufh %[filter_x1], %[filter_x1], %[ftmp0] \n\t"
"pshufh %[filter_y0], %[filter_y0], %[ftmp0] \n\t"
"pshufh %[filter_y1], %[filter_y1], %[ftmp0] \n\t"
MMI_LI(%[tmp0], 0x0040004000400040)
MMI_MTC1(%[tmp0], %[ff_ph_40])
MMI_LI(%[tmp0], 0x00ff00ff00ff00ff)
MMI_MTC1(%[tmp0], %[mask])
// fdata3: fdata3[0] ~ fdata3[3]
VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_4_A
@@ -1232,15 +1299,14 @@ static inline void var_filter_block2d_bil_4x(const uint8_t *src_ptr,
: [ftmp0] "=&f"(ftmp[0]), [ftmp1] "=&f"(ftmp[1]), [ftmp2] "=&f"(ftmp[2]),
[ftmp3] "=&f"(ftmp[3]), [ftmp4] "=&f"(ftmp[4]), [ftmp5] "=&f"(ftmp[5]),
[ftmp6] "=&f"(ftmp[6]), [tmp0] "=&r"(tmp[0]), [src_ptr] "+&r"(src_ptr),
[temp2_ptr] "+&r"(temp2_ptr), [counter]"+&r"(l_counter)
: [filter_x0] "f"((uint64_t)filter_x[0]),
[filter_x1] "f"((uint64_t)filter_x[1]),
[filter_y0] "f"((uint64_t)filter_y[0]),
[filter_y1] "f"((uint64_t)filter_y[1]),
[src_stride] "r"((mips_reg)src_stride), [ff_ph_40] "f"(ff_ph_40),
[mask] "f"(mask)
[temp2_ptr] "+&r"(temp2_ptr), [counter]"+&r"(l_counter),
[ff_ph_40] "=&f"(ff_ph_40), [mask] "=&f"(mask),
[filter_x0] "=&f"(filter_x0), [filter_x1] "=&f"(filter_x1),
[filter_y0] "=&f"(filter_y0), [filter_y1] "=&f"(filter_y1)
: [src_stride] "r"((mips_reg)src_stride), [all] "r"(all)
: "memory"
);
/* clang-format on */
}
#define SUBPIX_VAR4XN(H) \
libs/libvpx/vpx_dsp/mips/vpx_convolve8_mmi.c
+5 -5
View File
@@ -105,7 +105,7 @@ static void convolve_horiz_mmi(const uint8_t *src, ptrdiff_t src_stride,
/* clang-format off */
__asm__ volatile(
"move %[tmp1], %[width] \n\t"
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"gsldlc1 %[filter1], 0x03(%[filter]) \n\t"
"gsldrc1 %[filter1], 0x00(%[filter]) \n\t"
"gsldlc1 %[filter2], 0x0b(%[filter]) \n\t"
@@ -178,7 +178,7 @@ static void convolve_vert_mmi(const uint8_t *src, ptrdiff_t src_stride,
(void)y_step_q4;
__asm__ volatile(
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"gsldlc1 %[ftmp4], 0x03(%[filter]) \n\t"
"gsldrc1 %[ftmp4], 0x00(%[filter]) \n\t"
"gsldlc1 %[ftmp5], 0x0b(%[filter]) \n\t"
@@ -271,7 +271,7 @@ static void convolve_avg_horiz_mmi(const uint8_t *src, ptrdiff_t src_stride,
__asm__ volatile(
"move %[tmp1], %[width] \n\t"
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"gsldlc1 %[filter1], 0x03(%[filter]) \n\t"
"gsldrc1 %[filter1], 0x00(%[filter]) \n\t"
"gsldlc1 %[filter2], 0x0b(%[filter]) \n\t"
@@ -354,7 +354,7 @@ static void convolve_avg_vert_mmi(const uint8_t *src, ptrdiff_t src_stride,
(void)y_step_q4;
__asm__ volatile(
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"gsldlc1 %[ftmp4], 0x03(%[filter]) \n\t"
"gsldrc1 %[ftmp4], 0x00(%[filter]) \n\t"
"gsldlc1 %[ftmp5], 0x0b(%[filter]) \n\t"
@@ -467,7 +467,7 @@ void vpx_convolve_avg_mmi(const uint8_t *src, ptrdiff_t src_stride,
__asm__ volatile(
"move %[tmp1], %[width] \n\t"
"xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t"
"li %[tmp0], 0x10001 \n\t"
MMI_MTC1(%[tmp0], %[ftmp3])
"punpcklhw %[ftmp3], %[ftmp3], %[ftmp3] \n\t"
libs/libvpx/vpx_dsp/ppc/quantize_vsx.c
+10 -12
View File
@@ -95,8 +95,8 @@ static INLINE int16x8_t vec_max_across(int16x8_t a) {
}
void vpx_quantize_b_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int skip_block, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
uint16_t *eob_ptr, const int16_t *scan_ptr,
@@ -122,8 +122,6 @@ void vpx_quantize_b_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
zero_mask1 = vec_cmpge(coeff1_abs, zbin);
(void)scan_ptr;
(void)skip_block;
assert(!skip_block);
qcoeff0 =
quantize_coeff(coeff0, coeff0_abs, round, quant, quant_shift, zero_mask0);
@@ -196,12 +194,14 @@ void vpx_quantize_b_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
*eob_ptr = eob[0];
}
void vpx_quantize_b_32x32_vsx(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block,
const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan_ptr, const int16_t *iscan_ptr) {
void vpx_quantize_b_32x32_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan_ptr,
const int16_t *iscan_ptr) {
// In stage 1, we quantize 16 coeffs (DC + 15 AC)
// In stage 2, we loop 42 times and quantize 24 coeffs per iteration
// (32 * 32 - 16) / 24 = 42
@@ -227,9 +227,7 @@ void vpx_quantize_b_32x32_vsx(
int16x8_t coeff1_abs = vec_abs(coeff1);
(void)scan_ptr;
(void)skip_block;
(void)n_coeffs;
assert(!skip_block);
// 32x32 quantization requires that zbin and round be divided by 2
zbin = vec_sra(vec_add(zbin, vec_ones_s16), vec_ones_u16);
libs/libvpx/vpx_dsp/psnr.h
+2 -5
View File
@@ -12,6 +12,7 @@
#define VPX_VPX_DSP_PSNR_H_
#include "vpx_scale/yv12config.h"
#include "vpx/vpx_encoder.h"
#define MAX_PSNR 100.0
@@ -19,11 +20,7 @@
extern "C" {
#endif
typedef struct {
double psnr[4]; // total/y/u/v
uint64_t sse[4]; // total/y/u/v
uint32_t samples[4]; // total/y/u/v
} PSNR_STATS;
typedef struct vpx_psnr_pkt PSNR_STATS;
// TODO(dkovalev) change vpx_sse_to_psnr signature: double -> int64_t
libs/libvpx/vpx_dsp/quantize.c
+37 -47
View File
@@ -15,7 +15,7 @@
#include "vpx_dsp/vpx_dsp_common.h"
#include "vpx_mem/vpx_mem.h"
void vpx_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, int skip_block,
void vpx_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs,
const int16_t *round_ptr, const int16_t quant,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t dequant, uint16_t *eob_ptr) {
@@ -28,28 +28,26 @@ void vpx_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, int skip_block,
memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
if (!skip_block) {
tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX);
tmp = (tmp * quant) >> 16;
qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant;
if (tmp) eob = 0;
}
tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX);
tmp = (tmp * quant) >> 16;
qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant;
if (tmp) eob = 0;
*eob_ptr = eob + 1;
}
#if CONFIG_VP9_HIGHBITDEPTH
void vpx_highbd_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs,
int skip_block, const int16_t *round_ptr,
const int16_t quant, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t dequant,
uint16_t *eob_ptr) {
const int16_t *round_ptr, const int16_t quant,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t dequant, uint16_t *eob_ptr) {
int eob = -1;
memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
if (!skip_block) {
{
const int coeff = coeff_ptr[0];
const int coeff_sign = (coeff >> 31);
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
@@ -59,11 +57,12 @@ void vpx_highbd_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs,
dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant;
if (abs_qcoeff) eob = 0;
}
*eob_ptr = eob + 1;
}
#endif
void vpx_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block,
void vpx_quantize_dc_32x32(const tran_low_t *coeff_ptr,
const int16_t *round_ptr, const int16_t quant,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t dequant, uint16_t *eob_ptr) {
@@ -77,19 +76,18 @@ void vpx_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block,
memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
if (!skip_block) {
tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1),
INT16_MIN, INT16_MAX);
tmp = (tmp * quant) >> 15;
qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant / 2;
if (tmp) eob = 0;
}
tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1), INT16_MIN,
INT16_MAX);
tmp = (tmp * quant) >> 15;
qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant / 2;
if (tmp) eob = 0;
*eob_ptr = eob + 1;
}
#if CONFIG_VP9_HIGHBITDEPTH
void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block,
void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr,
const int16_t *round_ptr, const int16_t quant,
tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr,
@@ -100,7 +98,7 @@ void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block,
memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
if (!skip_block) {
{
const int coeff = coeff_ptr[0];
const int coeff_sign = (coeff >> 31);
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
@@ -110,23 +108,21 @@ void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block,
dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant / 2;
if (abs_qcoeff) eob = 0;
}
*eob_ptr = eob + 1;
}
#endif
void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int skip_block, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
uint16_t *eob_ptr, const int16_t *scan,
const int16_t *iscan) {
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr, const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
int i, non_zero_count = (int)n_coeffs, eob = -1;
const int zbins[2] = { zbin_ptr[0], zbin_ptr[1] };
const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 };
(void)iscan;
(void)skip_block;
assert(!skip_block);
memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
@@ -156,7 +152,7 @@ void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
quant_shift_ptr[rc != 0]) >>
16; // quantization
qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
dqcoeff_ptr[rc] = (tran_low_t)(qcoeff_ptr[rc] * dequant_ptr[rc != 0]);
if (tmp) eob = i;
}
@@ -166,8 +162,8 @@ void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
#if CONFIG_VP9_HIGHBITDEPTH
void vpx_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int skip_block, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr,
@@ -176,8 +172,6 @@ void vpx_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int zbins[2] = { zbin_ptr[0], zbin_ptr[1] };
const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 };
(void)iscan;
(void)skip_block;
assert(!skip_block);
memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
@@ -215,8 +209,8 @@ void vpx_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
#endif
void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int skip_block, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr,
@@ -229,8 +223,6 @@ void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int idx_arr[1024];
int i, eob = -1;
(void)iscan;
(void)skip_block;
assert(!skip_block);
memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
@@ -260,7 +252,7 @@ void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
15;
qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
#if (ARCH_X86 || ARCH_X86_64) && !CONFIG_VP9_HIGHBITDEPTH
#if (VPX_ARCH_X86 || VPX_ARCH_X86_64) && !CONFIG_VP9_HIGHBITDEPTH
// When tran_low_t is only 16 bits dqcoeff can outrange it. Rather than
// truncating with a cast, saturate the value. This is easier to implement
// on x86 and preserves the sign of the value.
@@ -268,7 +260,7 @@ void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
clamp(qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2, INT16_MIN, INT16_MAX);
#else
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
#endif // ARCH_X86 && CONFIG_VP9_HIGHBITDEPTH
#endif // VPX_ARCH_X86 && CONFIG_VP9_HIGHBITDEPTH
if (tmp) eob = idx_arr[i];
}
@@ -277,8 +269,8 @@ void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
#if CONFIG_VP9_HIGHBITDEPTH
void vpx_highbd_quantize_b_32x32_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block,
const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr,
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
@@ -290,8 +282,6 @@ void vpx_highbd_quantize_b_32x32_c(
int idx_arr[1024];
int i, eob = -1;
(void)iscan;
(void)skip_block;
assert(!skip_block);
memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
libs/libvpx/vpx_dsp/quantize.h
+6 -7
View File
@@ -18,22 +18,21 @@
extern "C" {
#endif
void vpx_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, int skip_block,
void vpx_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs,
const int16_t *round_ptr, const int16_t quant,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t dequant, uint16_t *eob_ptr);
void vpx_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block,
void vpx_quantize_dc_32x32(const tran_low_t *coeff_ptr,
const int16_t *round_ptr, const int16_t quant,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t dequant, uint16_t *eob_ptr);
#if CONFIG_VP9_HIGHBITDEPTH
void vpx_highbd_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs,
int skip_block, const int16_t *round_ptr,
const int16_t quant, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t dequant,
uint16_t *eob_ptr);
void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block,
const int16_t *round_ptr, const int16_t quant,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t dequant, uint16_t *eob_ptr);
void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr,
const int16_t *round_ptr, const int16_t quant,
tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr,
libs/libvpx/vpx_dsp/sad.c
+12 -34
View File
@@ -45,23 +45,11 @@ static INLINE unsigned int sad(const uint8_t *src_ptr, int src_stride,
return sad(src_ptr, src_stride, comp_pred, m, m, n); \
}
// depending on call sites, pass **ref_array to avoid & in subsequent call and
// de-dup with 4D below.
#define sadMxNxK(m, n, k) \
void vpx_sad##m##x##n##x##k##_c(const uint8_t *src_ptr, int src_stride, \
const uint8_t *ref_ptr, int ref_stride, \
uint32_t *sad_array) { \
int i; \
for (i = 0; i < k; ++i) \
sad_array[i] = \
vpx_sad##m##x##n##_c(src_ptr, src_stride, &ref_ptr[i], ref_stride); \
}
// This appears to be equivalent to the above when k == 4 and refs is const
// Compare |src_ptr| to 4 distinct references in |ref_array[4]|
#define sadMxNx4D(m, n) \
void vpx_sad##m##x##n##x4d_c(const uint8_t *src_ptr, int src_stride, \
const uint8_t *const ref_array[], \
int ref_stride, uint32_t *sad_array) { \
const uint8_t *const ref_array[4], \
int ref_stride, uint32_t sad_array[4]) { \
int i; \
for (i = 0; i < 4; ++i) \
sad_array[i] = \
@@ -95,26 +83,18 @@ sadMxNx4D(16, 32)
// 16x16
sadMxN(16, 16)
sadMxNxK(16, 16, 3)
sadMxNxK(16, 16, 8)
sadMxNx4D(16, 16)
// 16x8
sadMxN(16, 8)
sadMxNxK(16, 8, 3)
sadMxNxK(16, 8, 8)
sadMxNx4D(16, 8)
// 8x16
sadMxN(8, 16)
sadMxNxK(8, 16, 3)
sadMxNxK(8, 16, 8)
sadMxNx4D(8, 16)
// 8x8
sadMxN(8, 8)
sadMxNxK(8, 8, 3)
sadMxNxK(8, 8, 8)
sadMxNx4D(8, 8)
// 8x4
@@ -127,8 +107,6 @@ sadMxNx4D(4, 8)
// 4x4
sadMxN(4, 4)
sadMxNxK(4, 4, 3)
sadMxNxK(4, 4, 8)
sadMxNx4D(4, 4)
/* clang-format on */
@@ -180,15 +158,15 @@ static INLINE unsigned int highbd_sadb(const uint8_t *src8_ptr, int src_stride,
return highbd_sadb(src_ptr, src_stride, comp_pred, m, m, n); \
}
#define highbd_sadMxNx4D(m, n) \
void vpx_highbd_sad##m##x##n##x4d_c(const uint8_t *src_ptr, int src_stride, \
const uint8_t *const ref_array[], \
int ref_stride, uint32_t *sad_array) { \
int i; \
for (i = 0; i < 4; ++i) { \
sad_array[i] = vpx_highbd_sad##m##x##n##_c(src_ptr, src_stride, \
ref_array[i], ref_stride); \
} \
#define highbd_sadMxNx4D(m, n) \
void vpx_highbd_sad##m##x##n##x4d_c(const uint8_t *src_ptr, int src_stride, \
const uint8_t *const ref_array[4], \
int ref_stride, uint32_t sad_array[4]) { \
int i; \
for (i = 0; i < 4; ++i) { \
sad_array[i] = vpx_highbd_sad##m##x##n##_c(src_ptr, src_stride, \
ref_array[i], ref_stride); \
} \
}
/* clang-format off */
libs/libvpx/vpx_dsp/variance.h
+1 -3
View File
@@ -59,10 +59,8 @@ typedef struct variance_vtable {
vpx_sad_fn_t sdf;
vpx_variance_fn_t vf;
vpx_subpixvariance_fn_t svf;
vpx_sad_multi_fn_t sdx3f;
vpx_sad_multi_fn_t sdx8f;
vpx_sad_multi_d_fn_t sdx4df;
#if ARCH_X86 || ARCH_X86_64
#if VPX_ARCH_X86 || VPX_ARCH_X86_64
vp8_copy32xn_fn_t copymem;
#endif
} vp8_variance_fn_ptr_t;
libs/libvpx/vpx_dsp/vpx_dsp.mk
+42 -8
View File
@@ -74,6 +74,7 @@ endif # CONFIG_POSTPROC
DSP_SRCS-$(HAVE_NEON_ASM) += arm/intrapred_neon_asm$(ASM)
DSP_SRCS-$(HAVE_NEON) += arm/intrapred_neon.c
DSP_SRCS-$(HAVE_MSA) += mips/intrapred_msa.c
DSP_SRCS-$(HAVE_LSX) += loongarch/intrapred_lsx.c
DSP_SRCS-$(HAVE_DSPR2) += mips/intrapred4_dspr2.c
DSP_SRCS-$(HAVE_DSPR2) += mips/intrapred8_dspr2.c
DSP_SRCS-$(HAVE_DSPR2) += mips/intrapred16_dspr2.c
@@ -87,7 +88,7 @@ ifeq ($(CONFIG_VP9),yes)
DSP_SRCS-yes += vpx_convolve.c
DSP_SRCS-yes += vpx_convolve.h
DSP_SRCS-$(ARCH_X86)$(ARCH_X86_64) += x86/convolve.h
DSP_SRCS-$(VPX_ARCH_X86)$(VPX_ARCH_X86_64) += x86/convolve.h
DSP_SRCS-$(HAVE_SSE2) += x86/convolve_sse2.h
DSP_SRCS-$(HAVE_SSSE3) += x86/convolve_ssse3.h
@@ -162,6 +163,17 @@ DSP_SRCS-$(HAVE_DSPR2) += mips/convolve8_vert_dspr2.c
DSP_SRCS-$(HAVE_VSX) += ppc/vpx_convolve_vsx.c
# common (lsx)
DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_avg_horiz_lsx.c
DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_avg_vert_lsx.c
DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_horiz_lsx.c
DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_vert_lsx.c
DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_lsx.c
DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_avg_lsx.c
DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve_avg_lsx.c
DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve_copy_lsx.c
DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve_lsx.h
# loop filters
DSP_SRCS-yes += loopfilter.c
@@ -188,6 +200,11 @@ DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_dspr2.c
DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_horiz_dspr2.c
DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_vert_dspr2.c
DSP_SRCS-$(HAVE_LSX) += loongarch/loopfilter_lsx.h
DSP_SRCS-$(HAVE_LSX) += loongarch/loopfilter_16_lsx.c
DSP_SRCS-$(HAVE_LSX) += loongarch/loopfilter_8_lsx.c
DSP_SRCS-$(HAVE_LSX) += loongarch/loopfilter_4_lsx.c
ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes)
DSP_SRCS-$(HAVE_NEON) += arm/highbd_loopfilter_neon.c
DSP_SRCS-$(HAVE_SSE2) += x86/highbd_loopfilter_sse2.c
@@ -197,6 +214,7 @@ endif # CONFIG_VP9
DSP_SRCS-yes += txfm_common.h
DSP_SRCS-$(HAVE_SSE2) += x86/txfm_common_sse2.h
DSP_SRCS-$(HAVE_MSA) += mips/txfm_macros_msa.h
DSP_SRCS-$(HAVE_LSX) += loongarch/txfm_macros_lsx.h
# forward transform
ifeq ($(CONFIG_VP9_ENCODER),yes)
DSP_SRCS-yes += fwd_txfm.c
@@ -205,7 +223,7 @@ DSP_SRCS-$(HAVE_SSE2) += x86/fwd_txfm_sse2.h
DSP_SRCS-$(HAVE_SSE2) += x86/fwd_txfm_sse2.c
DSP_SRCS-$(HAVE_SSE2) += x86/fwd_txfm_impl_sse2.h
DSP_SRCS-$(HAVE_SSE2) += x86/fwd_dct32x32_impl_sse2.h
ifeq ($(ARCH_X86_64),yes)
ifeq ($(VPX_ARCH_X86_64),yes)
DSP_SRCS-$(HAVE_SSSE3) += x86/fwd_txfm_ssse3_x86_64.asm
endif
DSP_SRCS-$(HAVE_AVX2) += x86/fwd_txfm_avx2.c
@@ -217,9 +235,12 @@ DSP_SRCS-$(HAVE_NEON) += arm/fdct_partial_neon.c
DSP_SRCS-$(HAVE_NEON) += arm/fwd_txfm_neon.c
DSP_SRCS-$(HAVE_MSA) += mips/fwd_txfm_msa.h
DSP_SRCS-$(HAVE_MSA) += mips/fwd_txfm_msa.c
DSP_SRCS-$(HAVE_LSX) += loongarch/fwd_txfm_lsx.h
DSP_SRCS-$(HAVE_LSX) += loongarch/fwd_txfm_lsx.c
ifneq ($(CONFIG_VP9_HIGHBITDEPTH),yes)
DSP_SRCS-$(HAVE_MSA) += mips/fwd_dct32x32_msa.c
DSP_SRCS-$(HAVE_LSX) += loongarch/fwd_dct32x32_lsx.c
endif # !CONFIG_VP9_HIGHBITDEPTH
DSP_SRCS-$(HAVE_VSX) += ppc/fdct32x32_vsx.c
@@ -252,6 +273,8 @@ DSP_SRCS-$(HAVE_DSPR2) += mips/itrans8_dspr2.c
DSP_SRCS-$(HAVE_DSPR2) += mips/itrans16_dspr2.c
DSP_SRCS-$(HAVE_DSPR2) += mips/itrans32_dspr2.c
DSP_SRCS-$(HAVE_DSPR2) += mips/itrans32_cols_dspr2.c
DSP_SRCS-$(HAVE_LSX) += loongarch/idct32x32_lsx.c
else # CONFIG_VP9_HIGHBITDEPTH
DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct4x4_add_neon.c
DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct8x8_add_neon.c
@@ -305,6 +328,7 @@ DSP_SRCS-$(HAVE_SSSE3) += x86/quantize_ssse3.h
DSP_SRCS-$(HAVE_AVX) += x86/quantize_avx.c
DSP_SRCS-$(HAVE_NEON) += arm/quantize_neon.c
DSP_SRCS-$(HAVE_VSX) += ppc/quantize_vsx.c
DSP_SRCS-$(HAVE_LSX) += loongarch/quantize_lsx.c
ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes)
DSP_SRCS-$(HAVE_SSE2) += x86/highbd_quantize_intrin_sse2.c
endif
@@ -316,7 +340,8 @@ DSP_SRCS-$(HAVE_AVX2) += x86/avg_intrin_avx2.c
DSP_SRCS-$(HAVE_NEON) += arm/avg_neon.c
DSP_SRCS-$(HAVE_NEON) += arm/hadamard_neon.c
DSP_SRCS-$(HAVE_MSA) += mips/avg_msa.c
ifeq ($(ARCH_X86_64),yes)
DSP_SRCS-$(HAVE_LSX) += loongarch/avg_lsx.c
ifeq ($(VPX_ARCH_X86_64),yes)
DSP_SRCS-$(HAVE_SSSE3) += x86/avg_ssse3_x86_64.asm
endif
DSP_SRCS-$(HAVE_VSX) += ppc/hadamard_vsx.c
@@ -342,12 +367,11 @@ DSP_SRCS-$(HAVE_NEON) += arm/subtract_neon.c
DSP_SRCS-$(HAVE_MSA) += mips/sad_msa.c
DSP_SRCS-$(HAVE_MSA) += mips/subtract_msa.c
DSP_SRCS-$(HAVE_LSX) += loongarch/sad_lsx.c
DSP_SRCS-$(HAVE_MMI) += mips/sad_mmi.c
DSP_SRCS-$(HAVE_MMI) += mips/subtract_mmi.c
DSP_SRCS-$(HAVE_SSE3) += x86/sad_sse3.asm
DSP_SRCS-$(HAVE_SSSE3) += x86/sad_ssse3.asm
DSP_SRCS-$(HAVE_SSE4_1) += x86/sad_sse4.asm
DSP_SRCS-$(HAVE_AVX2) += x86/sad4d_avx2.c
DSP_SRCS-$(HAVE_AVX2) += x86/sad_avx2.c
DSP_SRCS-$(HAVE_AVX512) += x86/sad4d_avx512.c
@@ -359,6 +383,8 @@ DSP_SRCS-$(HAVE_SSE2) += x86/subtract_sse2.asm
DSP_SRCS-$(HAVE_VSX) += ppc/sad_vsx.c
DSP_SRCS-$(HAVE_VSX) += ppc/subtract_vsx.c
DSP_SRCS-$(HAVE_LSX) += loongarch/subtract_lsx.c
ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes)
DSP_SRCS-$(HAVE_SSE2) += x86/highbd_sad4d_sse2.asm
DSP_SRCS-$(HAVE_SSE2) += x86/highbd_sad_sse2.asm
@@ -377,6 +403,11 @@ DSP_SRCS-$(HAVE_NEON) += arm/variance_neon.c
DSP_SRCS-$(HAVE_MSA) += mips/variance_msa.c
DSP_SRCS-$(HAVE_MSA) += mips/sub_pixel_variance_msa.c
DSP_SRCS-$(HAVE_LSX) += loongarch/variance_lsx.h
DSP_SRCS-$(HAVE_LSX) += loongarch/variance_lsx.c
DSP_SRCS-$(HAVE_LSX) += loongarch/sub_pixel_variance_lsx.c
DSP_SRCS-$(HAVE_LSX) += loongarch/avg_pred_lsx.c
DSP_SRCS-$(HAVE_MMI) += mips/variance_mmi.c
DSP_SRCS-$(HAVE_SSE2) += x86/avg_pred_sse2.c
@@ -384,9 +415,9 @@ DSP_SRCS-$(HAVE_SSE2) += x86/variance_sse2.c # Contains SSE2 and SSSE3
DSP_SRCS-$(HAVE_AVX2) += x86/variance_avx2.c
DSP_SRCS-$(HAVE_VSX) += ppc/variance_vsx.c
ifeq ($(ARCH_X86_64),yes)
ifeq ($(VPX_ARCH_X86_64),yes)
DSP_SRCS-$(HAVE_SSE2) += x86/ssim_opt_x86_64.asm
endif # ARCH_X86_64
endif # VPX_ARCH_X86_64
DSP_SRCS-$(HAVE_SSE2) += x86/subpel_variance_sse2.asm # Contains SSE2 and SSSE3
@@ -413,6 +444,9 @@ DSP_SRCS-$(HAVE_VSX) += ppc/bitdepth_conversion_vsx.h
DSP_SRCS-$(HAVE_SSE2) += x86/mem_sse2.h
DSP_SRCS-$(HAVE_SSE2) += x86/transpose_sse2.h
# LSX utilities
DSP_SRCS-$(HAVE_LSX) += loongarch/bitdepth_conversion_lsx.h
DSP_SRCS-no += $(DSP_SRCS_REMOVE-yes)
DSP_SRCS-yes += vpx_dsp_rtcd.c
libs/libvpx/vpx_dsp/vpx_dsp_rtcd_defs.pl
+90 -124
View File
@@ -117,7 +117,7 @@ specialize qw/vpx_tm_predictor_8x8 neon dspr2 msa sse2/;
add_proto qw/void vpx_dc_predictor_8x8/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left";
# TODO(crbug.com/webm/1522): Re-enable vsx implementation.
specialize qw/vpx_dc_predictor_8x8 dspr2 neon msa sse2/;
specialize qw/vpx_dc_predictor_8x8 dspr2 neon msa sse2 lsx/;
add_proto qw/void vpx_dc_top_predictor_8x8/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left";
specialize qw/vpx_dc_top_predictor_8x8 neon msa sse2/;
@@ -155,7 +155,7 @@ add_proto qw/void vpx_tm_predictor_16x16/, "uint8_t *dst, ptrdiff_t stride, cons
specialize qw/vpx_tm_predictor_16x16 neon msa sse2 vsx/;
add_proto qw/void vpx_dc_predictor_16x16/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left";
specialize qw/vpx_dc_predictor_16x16 dspr2 neon msa sse2 vsx/;
specialize qw/vpx_dc_predictor_16x16 dspr2 neon msa sse2 vsx lsx/;
add_proto qw/void vpx_dc_top_predictor_16x16/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left";
specialize qw/vpx_dc_top_predictor_16x16 neon msa sse2 vsx/;
@@ -368,28 +368,28 @@ if (vpx_config("CONFIG_VP9") eq "yes") {
# Sub Pixel Filters
#
add_proto qw/void vpx_convolve_copy/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h";
specialize qw/vpx_convolve_copy neon dspr2 msa sse2 vsx/;
specialize qw/vpx_convolve_copy neon dspr2 msa sse2 vsx lsx/;
add_proto qw/void vpx_convolve_avg/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h";
specialize qw/vpx_convolve_avg neon dspr2 msa sse2 vsx mmi/;
specialize qw/vpx_convolve_avg neon dspr2 msa sse2 vsx mmi lsx/;
add_proto qw/void vpx_convolve8/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h";
specialize qw/vpx_convolve8 sse2 ssse3 avx2 neon dspr2 msa vsx mmi/;
specialize qw/vpx_convolve8 sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/;
add_proto qw/void vpx_convolve8_horiz/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h";
specialize qw/vpx_convolve8_horiz sse2 ssse3 avx2 neon dspr2 msa vsx mmi/;
specialize qw/vpx_convolve8_horiz sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/;
add_proto qw/void vpx_convolve8_vert/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h";
specialize qw/vpx_convolve8_vert sse2 ssse3 avx2 neon dspr2 msa vsx mmi/;
specialize qw/vpx_convolve8_vert sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/;
add_proto qw/void vpx_convolve8_avg/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h";
specialize qw/vpx_convolve8_avg sse2 ssse3 avx2 neon dspr2 msa vsx mmi/;
specialize qw/vpx_convolve8_avg sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/;
add_proto qw/void vpx_convolve8_avg_horiz/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h";
specialize qw/vpx_convolve8_avg_horiz sse2 ssse3 avx2 neon dspr2 msa vsx mmi/;
specialize qw/vpx_convolve8_avg_horiz sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/;
add_proto qw/void vpx_convolve8_avg_vert/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h";
specialize qw/vpx_convolve8_avg_vert sse2 ssse3 avx2 neon dspr2 msa vsx mmi/;
specialize qw/vpx_convolve8_avg_vert sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/;
add_proto qw/void vpx_scaled_2d/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h";
specialize qw/vpx_scaled_2d ssse3 neon msa/;
@@ -442,37 +442,37 @@ add_proto qw/void vpx_lpf_vertical_16/, "uint8_t *s, int pitch, const uint8_t *b
specialize qw/vpx_lpf_vertical_16 sse2 neon dspr2 msa/;
add_proto qw/void vpx_lpf_vertical_16_dual/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/vpx_lpf_vertical_16_dual sse2 neon dspr2 msa/;
specialize qw/vpx_lpf_vertical_16_dual sse2 neon dspr2 msa lsx/;
add_proto qw/void vpx_lpf_vertical_8/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/vpx_lpf_vertical_8 sse2 neon dspr2 msa/;
specialize qw/vpx_lpf_vertical_8 sse2 neon dspr2 msa lsx/;
add_proto qw/void vpx_lpf_vertical_8_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1";
specialize qw/vpx_lpf_vertical_8_dual sse2 neon dspr2 msa/;
specialize qw/vpx_lpf_vertical_8_dual sse2 neon dspr2 msa lsx/;
add_proto qw/void vpx_lpf_vertical_4/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/vpx_lpf_vertical_4 sse2 neon dspr2 msa/;
specialize qw/vpx_lpf_vertical_4 sse2 neon dspr2 msa lsx/;
add_proto qw/void vpx_lpf_vertical_4_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1";
specialize qw/vpx_lpf_vertical_4_dual sse2 neon dspr2 msa/;
specialize qw/vpx_lpf_vertical_4_dual sse2 neon dspr2 msa lsx/;
add_proto qw/void vpx_lpf_horizontal_16/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/vpx_lpf_horizontal_16 sse2 avx2 neon dspr2 msa/;
add_proto qw/void vpx_lpf_horizontal_16_dual/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/vpx_lpf_horizontal_16_dual sse2 avx2 neon dspr2 msa/;
specialize qw/vpx_lpf_horizontal_16_dual sse2 avx2 neon dspr2 msa lsx/;
add_proto qw/void vpx_lpf_horizontal_8/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/vpx_lpf_horizontal_8 sse2 neon dspr2 msa/;
specialize qw/vpx_lpf_horizontal_8 sse2 neon dspr2 msa lsx/;
add_proto qw/void vpx_lpf_horizontal_8_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1";
specialize qw/vpx_lpf_horizontal_8_dual sse2 neon dspr2 msa/;
specialize qw/vpx_lpf_horizontal_8_dual sse2 neon dspr2 msa lsx/;
add_proto qw/void vpx_lpf_horizontal_4/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/vpx_lpf_horizontal_4 sse2 neon dspr2 msa/;
specialize qw/vpx_lpf_horizontal_4 sse2 neon dspr2 msa lsx/;
add_proto qw/void vpx_lpf_horizontal_4_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1";
specialize qw/vpx_lpf_horizontal_4_dual sse2 neon dspr2 msa/;
specialize qw/vpx_lpf_horizontal_4_dual sse2 neon dspr2 msa lsx/;
} #CONFIG_VP9
if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
@@ -573,28 +573,28 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
add_proto qw/void vpx_highbd_fdct32x32_1/, "const int16_t *input, tran_low_t *output, int stride";
} else {
add_proto qw/void vpx_fdct4x4/, "const int16_t *input, tran_low_t *output, int stride";
specialize qw/vpx_fdct4x4 neon sse2 msa/;
specialize qw/vpx_fdct4x4 neon sse2 msa lsx/;
add_proto qw/void vpx_fdct4x4_1/, "const int16_t *input, tran_low_t *output, int stride";
specialize qw/vpx_fdct4x4_1 sse2 neon/;
add_proto qw/void vpx_fdct8x8/, "const int16_t *input, tran_low_t *output, int stride";
specialize qw/vpx_fdct8x8 sse2 neon msa/, "$ssse3_x86_64";
specialize qw/vpx_fdct8x8 sse2 neon msa lsx/, "$ssse3_x86_64";
add_proto qw/void vpx_fdct8x8_1/, "const int16_t *input, tran_low_t *output, int stride";
specialize qw/vpx_fdct8x8_1 sse2 neon msa/;
add_proto qw/void vpx_fdct16x16/, "const int16_t *input, tran_low_t *output, int stride";
specialize qw/vpx_fdct16x16 neon sse2 msa/;
specialize qw/vpx_fdct16x16 neon sse2 msa lsx/;
add_proto qw/void vpx_fdct16x16_1/, "const int16_t *input, tran_low_t *output, int stride";
specialize qw/vpx_fdct16x16_1 sse2 neon msa/;
add_proto qw/void vpx_fdct32x32/, "const int16_t *input, tran_low_t *output, int stride";
specialize qw/vpx_fdct32x32 neon sse2 avx2 msa/;
specialize qw/vpx_fdct32x32 neon sse2 avx2 msa lsx/;
add_proto qw/void vpx_fdct32x32_rd/, "const int16_t *input, tran_low_t *output, int stride";
specialize qw/vpx_fdct32x32_rd sse2 avx2 neon msa vsx/;
specialize qw/vpx_fdct32x32_rd sse2 avx2 neon msa vsx lsx/;
add_proto qw/void vpx_fdct32x32_1/, "const int16_t *input, tran_low_t *output, int stride";
specialize qw/vpx_fdct32x32_1 sse2 neon msa/;
@@ -652,12 +652,13 @@ if (vpx_config("CONFIG_EMULATE_HARDWARE") ne "yes") {
$vpx_idct16x16_38_add_msa=vpx_idct16x16_256_add_msa;
specialize qw/vpx_idct16x16_10_add dspr2 msa/;
specialize qw/vpx_idct16x16_1_add dspr2 msa/;
specialize qw/vpx_idct32x32_1024_add dspr2 msa/;
specialize qw/vpx_idct32x32_1024_add dspr2 msa lsx/;
specialize qw/vpx_idct32x32_135_add dspr2 msa/;
$vpx_idct32x32_135_add_dspr2=vpx_idct32x32_1024_add_dspr2;
$vpx_idct32x32_135_add_msa=vpx_idct32x32_1024_add_msa;
specialize qw/vpx_idct32x32_34_add dspr2 msa/;
specialize qw/vpx_idct32x32_1_add dspr2 msa/;
$vpx_idct32x32_135_add_lsx=vpx_idct32x32_1024_add_lsx;
specialize qw/vpx_idct32x32_34_add dspr2 msa lsx/;
specialize qw/vpx_idct32x32_1_add dspr2 msa lsx/;
specialize qw/vpx_iwht4x4_16_add msa/;
specialize qw/vpx_iwht4x4_1_add msa/;
} # !CONFIG_VP9_HIGHBITDEPTH
@@ -709,17 +710,17 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
# Quantization
#
if (vpx_config("CONFIG_VP9_ENCODER") eq "yes") {
add_proto qw/void vpx_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
specialize qw/vpx_quantize_b neon sse2 ssse3 avx vsx/;
add_proto qw/void vpx_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
specialize qw/vpx_quantize_b neon sse2 ssse3 avx vsx lsx/;
add_proto qw/void vpx_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
specialize qw/vpx_quantize_b_32x32 neon ssse3 avx vsx/;
add_proto qw/void vpx_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
specialize qw/vpx_quantize_b_32x32 neon ssse3 avx vsx lsx/;
if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
add_proto qw/void vpx_highbd_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
add_proto qw/void vpx_highbd_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
specialize qw/vpx_highbd_quantize_b sse2/;
add_proto qw/void vpx_highbd_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
add_proto qw/void vpx_highbd_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
specialize qw/vpx_highbd_quantize_b_32x32 sse2/;
} # CONFIG_VP9_HIGHBITDEPTH
} # CONFIG_VP9_ENCODER
@@ -729,13 +730,13 @@ if (vpx_config("CONFIG_ENCODERS") eq "yes") {
# Block subtraction
#
add_proto qw/void vpx_subtract_block/, "int rows, int cols, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr, ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride";
specialize qw/vpx_subtract_block neon msa mmi sse2 vsx/;
specialize qw/vpx_subtract_block neon msa mmi sse2 vsx lsx/;
#
# Single block SAD
#
add_proto qw/unsigned int vpx_sad64x64/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride";
specialize qw/vpx_sad64x64 neon avx2 msa sse2 vsx mmi/;
specialize qw/vpx_sad64x64 neon avx2 msa sse2 vsx mmi lsx/;
add_proto qw/unsigned int vpx_sad64x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride";
specialize qw/vpx_sad64x32 neon avx2 msa sse2 vsx mmi/;
@@ -744,7 +745,7 @@ add_proto qw/unsigned int vpx_sad32x64/, "const uint8_t *src_ptr, int src_stride
specialize qw/vpx_sad32x64 neon avx2 msa sse2 vsx mmi/;
add_proto qw/unsigned int vpx_sad32x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride";
specialize qw/vpx_sad32x32 neon avx2 msa sse2 vsx mmi/;
specialize qw/vpx_sad32x32 neon avx2 msa sse2 vsx mmi lsx/;
add_proto qw/unsigned int vpx_sad32x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride";
specialize qw/vpx_sad32x16 neon avx2 msa sse2 vsx mmi/;
@@ -753,7 +754,7 @@ add_proto qw/unsigned int vpx_sad16x32/, "const uint8_t *src_ptr, int src_stride
specialize qw/vpx_sad16x32 neon msa sse2 vsx mmi/;
add_proto qw/unsigned int vpx_sad16x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride";
specialize qw/vpx_sad16x16 neon msa sse2 vsx mmi/;
specialize qw/vpx_sad16x16 neon msa sse2 vsx mmi lsx/;
add_proto qw/unsigned int vpx_sad16x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride";
specialize qw/vpx_sad16x8 neon msa sse2 vsx mmi/;
@@ -762,7 +763,7 @@ add_proto qw/unsigned int vpx_sad8x16/, "const uint8_t *src_ptr, int src_stride,
specialize qw/vpx_sad8x16 neon msa sse2 vsx mmi/;
add_proto qw/unsigned int vpx_sad8x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride";
specialize qw/vpx_sad8x8 neon msa sse2 vsx mmi/;
specialize qw/vpx_sad8x8 neon msa sse2 vsx mmi lsx/;
add_proto qw/unsigned int vpx_sad8x4/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride";
specialize qw/vpx_sad8x4 neon msa sse2 vsx mmi/;
@@ -788,10 +789,10 @@ if (vpx_config("CONFIG_VP9_ENCODER") eq "yes") {
if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
add_proto qw/void vpx_hadamard_8x8/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff";
specialize qw/vpx_hadamard_8x8 sse2 neon vsx/, "$ssse3_x86_64";
specialize qw/vpx_hadamard_8x8 sse2 neon vsx lsx/, "$ssse3_x86_64";
add_proto qw/void vpx_hadamard_16x16/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff";
specialize qw/vpx_hadamard_16x16 avx2 sse2 neon vsx/;
specialize qw/vpx_hadamard_16x16 avx2 sse2 neon vsx lsx/;
add_proto qw/void vpx_hadamard_32x32/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff";
specialize qw/vpx_hadamard_32x32 sse2 avx2/;
@@ -812,10 +813,10 @@ if (vpx_config("CONFIG_VP9_ENCODER") eq "yes") {
specialize qw/vpx_highbd_satd avx2/;
} else {
add_proto qw/void vpx_hadamard_8x8/, "const int16_t *src_diff, ptrdiff_t src_stride, int16_t *coeff";
specialize qw/vpx_hadamard_8x8 sse2 neon msa vsx/, "$ssse3_x86_64";
specialize qw/vpx_hadamard_8x8 sse2 neon msa vsx lsx/, "$ssse3_x86_64";
add_proto qw/void vpx_hadamard_16x16/, "const int16_t *src_diff, ptrdiff_t src_stride, int16_t *coeff";
specialize qw/vpx_hadamard_16x16 avx2 sse2 neon msa vsx/;
specialize qw/vpx_hadamard_16x16 avx2 sse2 neon msa vsx lsx/;
add_proto qw/void vpx_hadamard_32x32/, "const int16_t *src_diff, ptrdiff_t src_stride, int16_t *coeff";
specialize qw/vpx_hadamard_32x32 sse2 avx2/;
@@ -824,7 +825,7 @@ if (vpx_config("CONFIG_VP9_ENCODER") eq "yes") {
specialize qw/vpx_satd avx2 sse2 neon msa/;
}
add_proto qw/void vpx_int_pro_row/, "int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height";
add_proto qw/void vpx_int_pro_row/, "int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height";
specialize qw/vpx_int_pro_row sse2 neon msa/;
add_proto qw/int16_t vpx_int_pro_col/, "const uint8_t *ref, const int width";
@@ -835,7 +836,7 @@ if (vpx_config("CONFIG_VP9_ENCODER") eq "yes") {
} # CONFIG_VP9_ENCODER
add_proto qw/unsigned int vpx_sad64x64_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred";
specialize qw/vpx_sad64x64_avg neon avx2 msa sse2 vsx mmi/;
specialize qw/vpx_sad64x64_avg neon avx2 msa sse2 vsx mmi lsx/;
add_proto qw/unsigned int vpx_sad64x32_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred";
specialize qw/vpx_sad64x32_avg neon avx2 msa sse2 vsx mmi/;
@@ -844,7 +845,7 @@ add_proto qw/unsigned int vpx_sad32x64_avg/, "const uint8_t *src_ptr, int src_st
specialize qw/vpx_sad32x64_avg neon avx2 msa sse2 vsx mmi/;
add_proto qw/unsigned int vpx_sad32x32_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred";
specialize qw/vpx_sad32x32_avg neon avx2 msa sse2 vsx mmi/;
specialize qw/vpx_sad32x32_avg neon avx2 msa sse2 vsx mmi lsx/;
add_proto qw/unsigned int vpx_sad32x16_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred";
specialize qw/vpx_sad32x16_avg neon avx2 msa sse2 vsx mmi/;
@@ -873,81 +874,46 @@ specialize qw/vpx_sad4x8_avg neon msa sse2 mmi/;
add_proto qw/unsigned int vpx_sad4x4_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred";
specialize qw/vpx_sad4x4_avg neon msa sse2 mmi/;
#
# Multi-block SAD, comparing a reference to N blocks 1 pixel apart horizontally
#
# Blocks of 3
add_proto qw/void vpx_sad16x16x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad16x16x3 sse3 ssse3 msa mmi/;
add_proto qw/void vpx_sad16x8x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad16x8x3 sse3 ssse3 msa mmi/;
add_proto qw/void vpx_sad8x16x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad8x16x3 sse3 msa mmi/;
add_proto qw/void vpx_sad8x8x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad8x8x3 sse3 msa mmi/;
add_proto qw/void vpx_sad4x4x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad4x4x3 sse3 msa mmi/;
# Blocks of 8
add_proto qw/void vpx_sad16x16x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad16x16x8 sse4_1 msa mmi/;
add_proto qw/void vpx_sad16x8x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad16x8x8 sse4_1 msa mmi/;
add_proto qw/void vpx_sad8x16x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad8x16x8 sse4_1 msa mmi/;
add_proto qw/void vpx_sad8x8x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad8x8x8 sse4_1 msa mmi/;
add_proto qw/void vpx_sad4x4x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad4x4x8 sse4_1 msa mmi/;
#
# Multi-block SAD, comparing a reference to N independent blocks
#
add_proto qw/void vpx_sad64x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad64x64x4d avx512 avx2 neon msa sse2 vsx mmi/;
add_proto qw/void vpx_sad64x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_sad64x64x4d avx512 avx2 neon msa sse2 vsx mmi lsx/;
add_proto qw/void vpx_sad64x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad64x32x4d neon msa sse2 vsx mmi/;
add_proto qw/void vpx_sad64x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_sad64x32x4d neon msa sse2 vsx mmi lsx/;
add_proto qw/void vpx_sad32x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad32x64x4d neon msa sse2 vsx mmi/;
add_proto qw/void vpx_sad32x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_sad32x64x4d neon msa sse2 vsx mmi lsx/;
add_proto qw/void vpx_sad32x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad32x32x4d avx2 neon msa sse2 vsx mmi/;
add_proto qw/void vpx_sad32x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_sad32x32x4d avx2 neon msa sse2 vsx mmi lsx/;
add_proto qw/void vpx_sad32x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_sad32x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_sad32x16x4d neon msa sse2 vsx mmi/;
add_proto qw/void vpx_sad16x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_sad16x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_sad16x32x4d neon msa sse2 vsx mmi/;
add_proto qw/void vpx_sad16x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad16x16x4d neon msa sse2 vsx mmi/;
add_proto qw/void vpx_sad16x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_sad16x16x4d neon msa sse2 vsx mmi lsx/;
add_proto qw/void vpx_sad16x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_sad16x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_sad16x8x4d neon msa sse2 vsx mmi/;
add_proto qw/void vpx_sad8x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_sad8x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_sad8x16x4d neon msa sse2 mmi/;
add_proto qw/void vpx_sad8x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array";
specialize qw/vpx_sad8x8x4d neon msa sse2 mmi/;
add_proto qw/void vpx_sad8x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_sad8x8x4d neon msa sse2 mmi lsx/;
add_proto qw/void vpx_sad8x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_sad8x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_sad8x4x4d neon msa sse2 mmi/;
add_proto qw/void vpx_sad4x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_sad4x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_sad4x8x4d neon msa sse2 mmi/;
add_proto qw/void vpx_sad4x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_sad4x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_sad4x4x4d neon msa sse2 mmi/;
add_proto qw/uint64_t vpx_sum_squares_2d_i16/, "const int16_t *src, int stride, int size";
@@ -1061,43 +1027,43 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
#
# Multi-block SAD, comparing a reference to N independent blocks
#
add_proto qw/void vpx_highbd_sad64x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_highbd_sad64x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_highbd_sad64x64x4d sse2/;
add_proto qw/void vpx_highbd_sad64x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_highbd_sad64x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_highbd_sad64x32x4d sse2/;
add_proto qw/void vpx_highbd_sad32x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_highbd_sad32x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_highbd_sad32x64x4d sse2/;
add_proto qw/void vpx_highbd_sad32x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_highbd_sad32x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_highbd_sad32x32x4d sse2/;
add_proto qw/void vpx_highbd_sad32x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_highbd_sad32x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_highbd_sad32x16x4d sse2/;
add_proto qw/void vpx_highbd_sad16x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_highbd_sad16x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_highbd_sad16x32x4d sse2/;
add_proto qw/void vpx_highbd_sad16x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_highbd_sad16x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_highbd_sad16x16x4d sse2/;
add_proto qw/void vpx_highbd_sad16x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_highbd_sad16x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_highbd_sad16x8x4d sse2/;
add_proto qw/void vpx_highbd_sad8x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_highbd_sad8x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_highbd_sad8x16x4d sse2/;
add_proto qw/void vpx_highbd_sad8x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_highbd_sad8x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_highbd_sad8x8x4d sse2/;
add_proto qw/void vpx_highbd_sad8x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_highbd_sad8x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_highbd_sad8x4x4d sse2/;
add_proto qw/void vpx_highbd_sad4x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_highbd_sad4x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_highbd_sad4x8x4d sse2/;
add_proto qw/void vpx_highbd_sad4x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array";
add_proto qw/void vpx_highbd_sad4x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]";
specialize qw/vpx_highbd_sad4x4x4d sse2/;
#
@@ -1115,7 +1081,7 @@ if (vpx_config("CONFIG_ENCODERS") eq "yes" || vpx_config("CONFIG_POSTPROC") eq "
# Variance
#
add_proto qw/unsigned int vpx_variance64x64/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/vpx_variance64x64 sse2 avx2 neon msa mmi vsx/;
specialize qw/vpx_variance64x64 sse2 avx2 neon msa mmi vsx lsx/;
add_proto qw/unsigned int vpx_variance64x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/vpx_variance64x32 sse2 avx2 neon msa mmi vsx/;
@@ -1124,7 +1090,7 @@ add_proto qw/unsigned int vpx_variance32x64/, "const uint8_t *src_ptr, int src_s
specialize qw/vpx_variance32x64 sse2 avx2 neon msa mmi vsx/;
add_proto qw/unsigned int vpx_variance32x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/vpx_variance32x32 sse2 avx2 neon msa mmi vsx/;
specialize qw/vpx_variance32x32 sse2 avx2 neon msa mmi vsx lsx/;
add_proto qw/unsigned int vpx_variance32x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/vpx_variance32x16 sse2 avx2 neon msa mmi vsx/;
@@ -1133,7 +1099,7 @@ add_proto qw/unsigned int vpx_variance16x32/, "const uint8_t *src_ptr, int src_s
specialize qw/vpx_variance16x32 sse2 avx2 neon msa mmi vsx/;
add_proto qw/unsigned int vpx_variance16x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/vpx_variance16x16 sse2 avx2 neon msa mmi vsx/;
specialize qw/vpx_variance16x16 sse2 avx2 neon msa mmi vsx lsx/;
add_proto qw/unsigned int vpx_variance16x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/vpx_variance16x8 sse2 avx2 neon msa mmi vsx/;
@@ -1142,7 +1108,7 @@ add_proto qw/unsigned int vpx_variance8x16/, "const uint8_t *src_ptr, int src_st
specialize qw/vpx_variance8x16 sse2 neon msa mmi vsx/;
add_proto qw/unsigned int vpx_variance8x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/vpx_variance8x8 sse2 neon msa mmi vsx/;
specialize qw/vpx_variance8x8 sse2 neon msa mmi vsx lsx/;
add_proto qw/unsigned int vpx_variance8x4/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/vpx_variance8x4 sse2 neon msa mmi vsx/;
@@ -1157,13 +1123,13 @@ add_proto qw/unsigned int vpx_variance4x4/, "const uint8_t *src_ptr, int src_str
# Specialty Variance
#
add_proto qw/void vpx_get16x16var/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse, int *sum";
specialize qw/vpx_get16x16var sse2 avx2 neon msa vsx/;
specialize qw/vpx_get16x16var sse2 avx2 neon msa vsx lsx/;
add_proto qw/void vpx_get8x8var/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse, int *sum";
specialize qw/vpx_get8x8var sse2 neon msa vsx/;
add_proto qw/unsigned int vpx_mse16x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/vpx_mse16x16 sse2 avx2 neon msa mmi vsx/;
specialize qw/vpx_mse16x16 sse2 avx2 neon msa mmi vsx lsx/;
add_proto qw/unsigned int vpx_mse16x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/vpx_mse16x8 sse2 avx2 msa mmi vsx/;
@@ -1181,7 +1147,7 @@ add_proto qw/unsigned int vpx_get4x4sse_cs/, "const unsigned char *src_ptr, int
specialize qw/vpx_get4x4sse_cs neon msa vsx/;
add_proto qw/void vpx_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride";
specialize qw/vpx_comp_avg_pred neon sse2 vsx/;
specialize qw/vpx_comp_avg_pred neon sse2 vsx lsx/;
#
# Subpixel Variance
@@ -1196,7 +1162,7 @@ add_proto qw/uint32_t vpx_sub_pixel_variance32x64/, "const uint8_t *src_ptr, int
specialize qw/vpx_sub_pixel_variance32x64 neon msa mmi sse2 ssse3/;
add_proto qw/uint32_t vpx_sub_pixel_variance32x32/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
specialize qw/vpx_sub_pixel_variance32x32 avx2 neon msa mmi sse2 ssse3/;
specialize qw/vpx_sub_pixel_variance32x32 avx2 neon msa mmi sse2 ssse3 lsx/;
add_proto qw/uint32_t vpx_sub_pixel_variance32x16/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
specialize qw/vpx_sub_pixel_variance32x16 neon msa mmi sse2 ssse3/;
@@ -1205,7 +1171,7 @@ add_proto qw/uint32_t vpx_sub_pixel_variance16x32/, "const uint8_t *src_ptr, int
specialize qw/vpx_sub_pixel_variance16x32 neon msa mmi sse2 ssse3/;
add_proto qw/uint32_t vpx_sub_pixel_variance16x16/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
specialize qw/vpx_sub_pixel_variance16x16 neon msa mmi sse2 ssse3/;
specialize qw/vpx_sub_pixel_variance16x16 neon msa mmi sse2 ssse3 lsx/;
add_proto qw/uint32_t vpx_sub_pixel_variance16x8/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
specialize qw/vpx_sub_pixel_variance16x8 neon msa mmi sse2 ssse3/;
@@ -1214,7 +1180,7 @@ add_proto qw/uint32_t vpx_sub_pixel_variance8x16/, "const uint8_t *src_ptr, int
specialize qw/vpx_sub_pixel_variance8x16 neon msa mmi sse2 ssse3/;
add_proto qw/uint32_t vpx_sub_pixel_variance8x8/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
specialize qw/vpx_sub_pixel_variance8x8 neon msa mmi sse2 ssse3/;
specialize qw/vpx_sub_pixel_variance8x8 neon msa mmi sse2 ssse3 lsx/;
add_proto qw/uint32_t vpx_sub_pixel_variance8x4/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
specialize qw/vpx_sub_pixel_variance8x4 neon msa mmi sse2 ssse3/;
@@ -1226,7 +1192,7 @@ add_proto qw/uint32_t vpx_sub_pixel_variance4x4/, "const uint8_t *src_ptr, int s
specialize qw/vpx_sub_pixel_variance4x4 neon msa mmi sse2 ssse3/;
add_proto qw/uint32_t vpx_sub_pixel_avg_variance64x64/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
specialize qw/vpx_sub_pixel_avg_variance64x64 neon avx2 msa mmi sse2 ssse3/;
specialize qw/vpx_sub_pixel_avg_variance64x64 neon avx2 msa mmi sse2 ssse3 lsx/;
add_proto qw/uint32_t vpx_sub_pixel_avg_variance64x32/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
specialize qw/vpx_sub_pixel_avg_variance64x32 neon msa mmi sse2 ssse3/;
libs/libvpx/vpx_dsp/x86/add_noise_sse2.asm
+1 -1
View File
@@ -16,7 +16,7 @@ SECTION .text
;void vpx_plane_add_noise_sse2(uint8_t *start, const int8_t *noise,
; int blackclamp, int whiteclamp,
; int width, int height, int pitch)
global sym(vpx_plane_add_noise_sse2) PRIVATE
globalsym(vpx_plane_add_noise_sse2)
sym(vpx_plane_add_noise_sse2):
push rbp
mov rbp, rsp
libs/libvpx/vpx_dsp/x86/avg_intrin_avx2.c
-1
View File
@@ -305,7 +305,6 @@ static void hadamard_8x8x2_avx2(const int16_t *src_diff, ptrdiff_t src_stride,
src[5] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[6] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[7] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
(void)src_diff;
hadamard_col8x2_avx2(src, 0);
hadamard_col8x2_avx2(src, 1);
libs/libvpx/vpx_dsp/x86/avg_intrin_sse2.c
+4 -5
View File
@@ -276,7 +276,6 @@ static INLINE void hadamard_8x8_sse2(const int16_t *src_diff,
src[5] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[6] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[7] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
(void)src_diff;
hadamard_col8_sse2(src, 0);
hadamard_col8_sse2(src, 1);
@@ -465,7 +464,7 @@ int vpx_satd_sse2(const tran_low_t *coeff, int length) {
return _mm_cvtsi128_si32(accum);
}
void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref,
void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref,
const int ref_stride, const int height) {
int idx;
__m128i zero = _mm_setzero_si128();
@@ -516,14 +515,14 @@ void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref,
int16_t vpx_int_pro_col_sse2(const uint8_t *ref, const int width) {
__m128i zero = _mm_setzero_si128();
__m128i src_line = _mm_load_si128((const __m128i *)ref);
__m128i src_line = _mm_loadu_si128((const __m128i *)ref);
__m128i s0 = _mm_sad_epu8(src_line, zero);
__m128i s1;
int i;
for (i = 16; i < width; i += 16) {
ref += 16;
src_line = _mm_load_si128((const __m128i *)ref);
src_line = _mm_loadu_si128((const __m128i *)ref);
s1 = _mm_sad_epu8(src_line, zero);
s0 = _mm_adds_epu16(s0, s1);
}
@@ -572,7 +571,7 @@ int vpx_vector_var_sse2(const int16_t *ref, const int16_t *src, const int bwl) {
v1 = _mm_srli_epi64(sse, 32);
sse = _mm_add_epi32(sse, v1);
mean = _mm_extract_epi16(sum, 0);
mean = (int16_t)_mm_extract_epi16(sum, 0);
return _mm_cvtsi128_si32(sse) - ((mean * mean) >> (bwl + 2));
}
libs/libvpx/vpx_dsp/x86/avg_pred_sse2.c
+3 -3
View File
@@ -46,9 +46,9 @@ void vpx_comp_avg_pred_sse2(uint8_t *comp_pred, const uint8_t *pred, int width,
r = _mm_loadu_si128((const __m128i *)ref);
ref += 16;
} else if (width == 4) {
r = _mm_set_epi32(loadu_uint32(ref + 3 * ref_stride),
loadu_uint32(ref + 2 * ref_stride),
loadu_uint32(ref + ref_stride), loadu_uint32(ref));
r = _mm_set_epi32(loadu_int32(ref + 3 * ref_stride),
loadu_int32(ref + 2 * ref_stride),
loadu_int32(ref + ref_stride), loadu_int32(ref));
ref += 4 * ref_stride;
} else {
libs/libvpx/vpx_dsp/x86/avg_ssse3_x86_64.asm
+1 -1
View File
@@ -13,7 +13,7 @@
SECTION .text
%if ARCH_X86_64
%if VPX_ARCH_X86_64
; matrix transpose
%macro TRANSPOSE8X8 10
; stage 1
libs/libvpx/vpx_dsp/x86/convolve.h
+7 -7
View File
@@ -14,7 +14,7 @@
#include "./vpx_config.h"
#include "vpx/vpx_integer.h"
#include "vpx_ports/mem.h"
#include "vpx_ports/compiler_attributes.h"
// TODO(chiyotsai@google.com): Refactor the code here. Currently this is pretty
// hacky and awful to read. Note that there is a filter_x[3] == 128 check in
@@ -107,7 +107,7 @@ typedef void filter8_1dfunction(const uint8_t *src_ptr, ptrdiff_t src_pitch,
assert(x_step_q4 == 16); \
assert(y_step_q4 == 16); \
if (filter_x[0] | filter_x[1] | filter_x[6] | filter_x[7]) { \
DECLARE_ALIGNED(16, uint8_t, fdata2[64 * 71]); \
DECLARE_ALIGNED(16, uint8_t, fdata2[64 * 71] VPX_UNINITIALIZED); \
vpx_convolve8_horiz_##opt(src - 3 * src_stride, src_stride, fdata2, 64, \
filter, x0_q4, x_step_q4, y0_q4, y_step_q4, w, \
h + 7); \
@@ -116,7 +116,7 @@ typedef void filter8_1dfunction(const uint8_t *src_ptr, ptrdiff_t src_pitch,
y_step_q4, w, h); \
} else if (filter_x[2] | filter_x[5]) { \
const int num_taps = is_avg ? 8 : 4; \
DECLARE_ALIGNED(16, uint8_t, fdata2[64 * 71]); \
DECLARE_ALIGNED(16, uint8_t, fdata2[64 * 71] VPX_UNINITIALIZED); \
vpx_convolve8_horiz_##opt( \
src - (num_taps / 2 - 1) * src_stride, src_stride, fdata2, 64, \
filter, x0_q4, x_step_q4, y0_q4, y_step_q4, w, h + num_taps - 1); \
@@ -124,7 +124,7 @@ typedef void filter8_1dfunction(const uint8_t *src_ptr, ptrdiff_t src_pitch,
dst, dst_stride, filter, x0_q4, \
x_step_q4, y0_q4, y_step_q4, w, h); \
} else { \
DECLARE_ALIGNED(16, uint8_t, fdata2[64 * 65]); \
DECLARE_ALIGNED(16, uint8_t, fdata2[64 * 65] VPX_UNINITIALIZED); \
vpx_convolve8_horiz_##opt(src, src_stride, fdata2, 64, filter, x0_q4, \
x_step_q4, y0_q4, y_step_q4, w, h + 1); \
vpx_convolve8_##avg##vert_##opt(fdata2, 64, dst, dst_stride, filter, \
@@ -242,7 +242,7 @@ typedef void highbd_filter8_1dfunction(const uint16_t *src_ptr,
if (x_step_q4 == 16 && y_step_q4 == 16) { \
if ((filter_x[0] | filter_x[1] | filter_x[6] | filter_x[7]) || \
filter_x[3] == 128) { \
DECLARE_ALIGNED(16, uint16_t, fdata2[64 * 71]); \
DECLARE_ALIGNED(16, uint16_t, fdata2[64 * 71] VPX_UNINITIALIZED); \
vpx_highbd_convolve8_horiz_##opt(src - 3 * src_stride, src_stride, \
fdata2, 64, filter, x0_q4, x_step_q4, \
y0_q4, y_step_q4, w, h + 7, bd); \
@@ -251,7 +251,7 @@ typedef void highbd_filter8_1dfunction(const uint16_t *src_ptr,
y0_q4, y_step_q4, w, h, bd); \
} else if (filter_x[2] | filter_x[5]) { \
const int num_taps = is_avg ? 8 : 4; \
DECLARE_ALIGNED(16, uint16_t, fdata2[64 * 71]); \
DECLARE_ALIGNED(16, uint16_t, fdata2[64 * 71] VPX_UNINITIALIZED); \
vpx_highbd_convolve8_horiz_##opt( \
src - (num_taps / 2 - 1) * src_stride, src_stride, fdata2, 64, \
filter, x0_q4, x_step_q4, y0_q4, y_step_q4, w, h + num_taps - 1, \
@@ -260,7 +260,7 @@ typedef void highbd_filter8_1dfunction(const uint16_t *src_ptr,
fdata2 + 64 * (num_taps / 2 - 1), 64, dst, dst_stride, filter, \
x0_q4, x_step_q4, y0_q4, y_step_q4, w, h, bd); \
} else { \
DECLARE_ALIGNED(16, uint16_t, fdata2[64 * 65]); \
DECLARE_ALIGNED(16, uint16_t, fdata2[64 * 65] VPX_UNINITIALIZED); \
vpx_highbd_convolve8_horiz_##opt(src, src_stride, fdata2, 64, filter, \
x0_q4, x_step_q4, y0_q4, y_step_q4, \
w, h + 1, bd); \
libs/libvpx/vpx_dsp/x86/deblock_sse2.asm
+2 -2
View File
@@ -95,7 +95,7 @@ SECTION .text
; int *flimits,
; int size
;)
global sym(vpx_post_proc_down_and_across_mb_row_sse2) PRIVATE
globalsym(vpx_post_proc_down_and_across_mb_row_sse2)
sym(vpx_post_proc_down_and_across_mb_row_sse2):
push rbp
mov rbp, rsp
@@ -235,7 +235,7 @@ sym(vpx_post_proc_down_and_across_mb_row_sse2):
;void vpx_mbpost_proc_across_ip_sse2(unsigned char *src,
; int pitch, int rows, int cols,int flimit)
global sym(vpx_mbpost_proc_across_ip_sse2) PRIVATE
globalsym(vpx_mbpost_proc_across_ip_sse2)
sym(vpx_mbpost_proc_across_ip_sse2):
push rbp
mov rbp, rsp
libs/libvpx/vpx_dsp/x86/fwd_txfm_sse2.h
+2 -2
View File
@@ -36,7 +36,7 @@ static INLINE __m128i k_packs_epi64(__m128i a, __m128i b) {
static INLINE int check_epi16_overflow_x2(const __m128i *preg0,
const __m128i *preg1) {
const __m128i max_overflow = _mm_set1_epi16(0x7fff);
const __m128i min_overflow = _mm_set1_epi16(0x8000);
const __m128i min_overflow = _mm_set1_epi16((short)0x8000);
__m128i cmp0 = _mm_or_si128(_mm_cmpeq_epi16(*preg0, max_overflow),
_mm_cmpeq_epi16(*preg0, min_overflow));
__m128i cmp1 = _mm_or_si128(_mm_cmpeq_epi16(*preg1, max_overflow),
@@ -50,7 +50,7 @@ static INLINE int check_epi16_overflow_x4(const __m128i *preg0,
const __m128i *preg2,
const __m128i *preg3) {
const __m128i max_overflow = _mm_set1_epi16(0x7fff);
const __m128i min_overflow = _mm_set1_epi16(0x8000);
const __m128i min_overflow = _mm_set1_epi16((short)0x8000);
__m128i cmp0 = _mm_or_si128(_mm_cmpeq_epi16(*preg0, max_overflow),
_mm_cmpeq_epi16(*preg0, min_overflow));
__m128i cmp1 = _mm_or_si128(_mm_cmpeq_epi16(*preg1, max_overflow),
libs/libvpx/vpx_dsp/x86/fwd_txfm_ssse3_x86_64.asm
+1 -1
View File
@@ -27,7 +27,7 @@ TRANSFORM_COEFFS 9102, 13623
SECTION .text
%if ARCH_X86_64
%if VPX_ARCH_X86_64
INIT_XMM ssse3
cglobal fdct8x8, 3, 5, 13, input, output, stride
libs/libvpx/vpx_dsp/x86/highbd_convolve_avx2.c
+6 -6
View File
@@ -1465,10 +1465,10 @@ highbd_filter8_1dfunction vpx_highbd_filter_block1d4_v2_sse2;
#define vpx_highbd_filter_block1d4_h4_avg_avx2 \
vpx_highbd_filter_block1d4_h8_avg_avx2
HIGH_FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , avx2, 0);
HIGH_FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , avx2, 0)
HIGH_FUN_CONV_1D(vert, y0_q4, y_step_q4, v,
src - src_stride * (num_taps / 2 - 1), , avx2, 0);
HIGH_FUN_CONV_2D(, avx2, 0);
src - src_stride * (num_taps / 2 - 1), , avx2, 0)
HIGH_FUN_CONV_2D(, avx2, 0)
// From vpx_dsp/x86/vpx_high_subpixel_8t_sse2.asm.
highbd_filter8_1dfunction vpx_highbd_filter_block1d4_h8_avg_sse2;
@@ -1487,9 +1487,9 @@ highbd_filter8_1dfunction vpx_highbd_filter_block1d4_v2_avg_sse2;
#define vpx_highbd_filter_block1d4_v2_avg_avx2 \
vpx_highbd_filter_block1d4_v2_avg_sse2
HIGH_FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, avx2, 1);
HIGH_FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, avx2, 1)
HIGH_FUN_CONV_1D(avg_vert, y0_q4, y_step_q4, v,
src - src_stride * (num_taps / 2 - 1), avg_, avx2, 1);
HIGH_FUN_CONV_2D(avg_, avx2, 1);
src - src_stride * (num_taps / 2 - 1), avg_, avx2, 1)
HIGH_FUN_CONV_2D(avg_, avx2, 1)
#undef HIGHBD_FUNC
libs/libvpx/vpx_dsp/x86/highbd_idct4x4_add_sse2.c
+2 -2
View File
@@ -112,8 +112,8 @@ void vpx_highbd_idct4x4_16_add_sse2(const tran_low_t *input, uint16_t *dest,
min_input = _mm_min_epi16(min_input, _mm_srli_si128(min_input, 4));
max_input = _mm_max_epi16(max_input, _mm_srli_si128(max_input, 2));
min_input = _mm_min_epi16(min_input, _mm_srli_si128(min_input, 2));
max = _mm_extract_epi16(max_input, 0);
min = _mm_extract_epi16(min_input, 0);
max = (int16_t)_mm_extract_epi16(max_input, 0);
min = (int16_t)_mm_extract_epi16(min_input, 0);
}
if (bd == 8 || (max < 4096 && min >= -4096)) {
libs/libvpx/vpx_dsp/x86/highbd_quantize_intrin_sse2.c
+3 -7
View File
@@ -18,7 +18,7 @@
#if CONFIG_VP9_HIGHBITDEPTH
void vpx_highbd_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t count,
int skip_block, const int16_t *zbin_ptr,
const int16_t *zbin_ptr,
const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
@@ -39,8 +39,6 @@ void vpx_highbd_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t count,
nzbins[1] = _mm_sub_epi32(nzbins[1], zbins[1]);
(void)scan;
(void)skip_block;
assert(!skip_block);
memset(qcoeff_ptr, 0, count * sizeof(*qcoeff_ptr));
memset(dqcoeff_ptr, 0, count * sizeof(*dqcoeff_ptr));
@@ -94,8 +92,8 @@ void vpx_highbd_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t count,
}
void vpx_highbd_quantize_b_32x32_sse2(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block,
const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr,
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
@@ -107,8 +105,6 @@ void vpx_highbd_quantize_b_32x32_sse2(
const int zbin0_tmp = ROUND_POWER_OF_TWO(zbin_ptr[0], 1);
const int zbin1_tmp = ROUND_POWER_OF_TWO(zbin_ptr[1], 1);
(void)scan;
(void)skip_block;
assert(!skip_block);
zbins[0] = _mm_set_epi32(zbin1_tmp, zbin1_tmp, zbin1_tmp, zbin0_tmp);
zbins[1] = _mm_set1_epi32(zbin1_tmp);
libs/libvpx/vpx_dsp/x86/highbd_sad_sse2.asm
+2 -2
View File
@@ -25,11 +25,11 @@ cglobal highbd_sad%1x%2, 4, %3, 7, src, src_stride, ref, ref_stride, \
cglobal highbd_sad%1x%2_avg, 5, 1 + %3, 7, src, src_stride, ref, ref_stride, \
second_pred, n_rows
%else ; %3 == 7
cglobal highbd_sad%1x%2_avg, 5, ARCH_X86_64 + %3, 7, src, src_stride, \
cglobal highbd_sad%1x%2_avg, 5, VPX_ARCH_X86_64 + %3, 7, src, src_stride, \
ref, ref_stride, \
second_pred, \
src_stride3, ref_stride3
%if ARCH_X86_64
%if VPX_ARCH_X86_64
%define n_rowsd r7d
%else ; x86-32
%define n_rowsd dword r0m
libs/libvpx/vpx_dsp/x86/highbd_subpel_variance_impl_sse2.asm
+17 -17
View File
@@ -78,7 +78,7 @@ SECTION .text
%endmacro
%macro INC_SRC_BY_SRC_STRIDE 0
%if ARCH_X86=1 && CONFIG_PIC=1
%if VPX_ARCH_X86=1 && CONFIG_PIC=1
add srcq, src_stridemp
add srcq, src_stridemp
%else
@@ -91,7 +91,7 @@ SECTION .text
%define filter_idx_shift 5
%if ARCH_X86_64
%if VPX_ARCH_X86_64
%if %2 == 1 ; avg
cglobal highbd_sub_pixel_avg_variance%1xh, 9, 10, 13, src, src_stride, \
x_offset, y_offset, \
@@ -268,11 +268,11 @@ SECTION .text
.x_zero_y_nonhalf:
; x_offset == 0 && y_offset == bilin interpolation
%if ARCH_X86_64
%if VPX_ARCH_X86_64
lea bilin_filter, [GLOBAL(bilin_filter_m)]
%endif
shl y_offsetd, filter_idx_shift
%if ARCH_X86_64 && mmsize == 16
%if VPX_ARCH_X86_64 && mmsize == 16
mova m8, [bilin_filter+y_offsetq]
mova m9, [bilin_filter+y_offsetq+16]
mova m10, [GLOBAL(pw_8)]
@@ -280,7 +280,7 @@ SECTION .text
%define filter_y_b m9
%define filter_rnd m10
%else ; x86-32 or mmx
%if ARCH_X86=1 && CONFIG_PIC=1
%if VPX_ARCH_X86=1 && CONFIG_PIC=1
; x_offset == 0, reuse x_offset reg
%define tempq x_offsetq
add y_offsetq, g_bilin_filterm
@@ -495,11 +495,11 @@ SECTION .text
.x_half_y_nonhalf:
; x_offset == 0.5 && y_offset == bilin interpolation
%if ARCH_X86_64
%if VPX_ARCH_X86_64
lea bilin_filter, [GLOBAL(bilin_filter_m)]
%endif
shl y_offsetd, filter_idx_shift
%if ARCH_X86_64 && mmsize == 16
%if VPX_ARCH_X86_64 && mmsize == 16
mova m8, [bilin_filter+y_offsetq]
mova m9, [bilin_filter+y_offsetq+16]
mova m10, [GLOBAL(pw_8)]
@@ -507,7 +507,7 @@ SECTION .text
%define filter_y_b m9
%define filter_rnd m10
%else ; x86_32
%if ARCH_X86=1 && CONFIG_PIC=1
%if VPX_ARCH_X86=1 && CONFIG_PIC=1
; x_offset == 0.5. We can reuse x_offset reg
%define tempq x_offsetq
add y_offsetq, g_bilin_filterm
@@ -617,11 +617,11 @@ SECTION .text
jnz .x_nonhalf_y_nonzero
; x_offset == bilin interpolation && y_offset == 0
%if ARCH_X86_64
%if VPX_ARCH_X86_64
lea bilin_filter, [GLOBAL(bilin_filter_m)]
%endif
shl x_offsetd, filter_idx_shift
%if ARCH_X86_64 && mmsize == 16
%if VPX_ARCH_X86_64 && mmsize == 16
mova m8, [bilin_filter+x_offsetq]
mova m9, [bilin_filter+x_offsetq+16]
mova m10, [GLOBAL(pw_8)]
@@ -629,7 +629,7 @@ SECTION .text
%define filter_x_b m9
%define filter_rnd m10
%else ; x86-32
%if ARCH_X86=1 && CONFIG_PIC=1
%if VPX_ARCH_X86=1 && CONFIG_PIC=1
; y_offset == 0. We can reuse y_offset reg.
%define tempq y_offsetq
add x_offsetq, g_bilin_filterm
@@ -716,11 +716,11 @@ SECTION .text
jne .x_nonhalf_y_nonhalf
; x_offset == bilin interpolation && y_offset == 0.5
%if ARCH_X86_64
%if VPX_ARCH_X86_64
lea bilin_filter, [GLOBAL(bilin_filter_m)]
%endif
shl x_offsetd, filter_idx_shift
%if ARCH_X86_64 && mmsize == 16
%if VPX_ARCH_X86_64 && mmsize == 16
mova m8, [bilin_filter+x_offsetq]
mova m9, [bilin_filter+x_offsetq+16]
mova m10, [GLOBAL(pw_8)]
@@ -728,7 +728,7 @@ SECTION .text
%define filter_x_b m9
%define filter_rnd m10
%else ; x86-32
%if ARCH_X86=1 && CONFIG_PIC=1
%if VPX_ARCH_X86=1 && CONFIG_PIC=1
; y_offset == 0.5. We can reuse y_offset reg.
%define tempq y_offsetq
add x_offsetq, g_bilin_filterm
@@ -843,12 +843,12 @@ SECTION .text
.x_nonhalf_y_nonhalf:
; loading filter - this is same as in 8-bit depth
%if ARCH_X86_64
%if VPX_ARCH_X86_64
lea bilin_filter, [GLOBAL(bilin_filter_m)]
%endif
shl x_offsetd, filter_idx_shift ; filter_idx_shift = 5
shl y_offsetd, filter_idx_shift
%if ARCH_X86_64 && mmsize == 16
%if VPX_ARCH_X86_64 && mmsize == 16
mova m8, [bilin_filter+x_offsetq]
mova m9, [bilin_filter+x_offsetq+16]
mova m10, [bilin_filter+y_offsetq]
@@ -860,7 +860,7 @@ SECTION .text
%define filter_y_b m11
%define filter_rnd m12
%else ; x86-32
%if ARCH_X86=1 && CONFIG_PIC=1
%if VPX_ARCH_X86=1 && CONFIG_PIC=1
; In this case, there is NO unused register. Used src_stride register. Later,
; src_stride has to be loaded from stack when it is needed.
%define tempq src_strideq
libs/libvpx/vpx_dsp/x86/highbd_variance_impl_sse2.asm
+2 -2
View File
@@ -22,7 +22,7 @@ SECTION .text
; unsigned int * SSE,
; int * Sum
;)
global sym(vpx_highbd_calc16x16var_sse2) PRIVATE
globalsym(vpx_highbd_calc16x16var_sse2)
sym(vpx_highbd_calc16x16var_sse2):
push rbp
mov rbp, rsp
@@ -175,7 +175,7 @@ sym(vpx_highbd_calc16x16var_sse2):
; unsigned int * SSE,
; int * Sum
;)
global sym(vpx_highbd_calc8x8var_sse2) PRIVATE
globalsym(vpx_highbd_calc8x8var_sse2)
sym(vpx_highbd_calc8x8var_sse2):
push rbp
mov rbp, rsp
libs/libvpx/vpx_dsp/x86/highbd_variance_sse2.c
+41 -41
View File
@@ -121,8 +121,8 @@ static void highbd_12_variance_sse2(const uint16_t *src, int src_stride,
*sse = ROUND_POWER_OF_TWO(*sse, 8); \
}
HIGH_GET_VAR(16);
HIGH_GET_VAR(8);
HIGH_GET_VAR(16)
HIGH_GET_VAR(8)
#undef HIGH_GET_VAR
@@ -167,16 +167,16 @@ HIGH_GET_VAR(8);
return (var >= 0) ? (uint32_t)var : 0; \
}
VAR_FN(64, 64, 16, 12);
VAR_FN(64, 32, 16, 11);
VAR_FN(32, 64, 16, 11);
VAR_FN(32, 32, 16, 10);
VAR_FN(32, 16, 16, 9);
VAR_FN(16, 32, 16, 9);
VAR_FN(16, 16, 16, 8);
VAR_FN(16, 8, 8, 7);
VAR_FN(8, 16, 8, 7);
VAR_FN(8, 8, 8, 6);
VAR_FN(64, 64, 16, 12)
VAR_FN(64, 32, 16, 11)
VAR_FN(32, 64, 16, 11)
VAR_FN(32, 32, 16, 10)
VAR_FN(32, 16, 16, 9)
VAR_FN(16, 32, 16, 9)
VAR_FN(16, 16, 16, 8)
VAR_FN(16, 8, 8, 7)
VAR_FN(8, 16, 8, 7)
VAR_FN(8, 8, 8, 6)
#undef VAR_FN
@@ -255,10 +255,10 @@ unsigned int vpx_highbd_12_mse8x8_sse2(const uint8_t *src8, int src_stride,
const uint16_t *ref, ptrdiff_t ref_stride, int height, \
unsigned int *sse, void *unused0, void *unused);
#define DECLS(opt) \
DECL(8, opt); \
DECL(8, opt) \
DECL(16, opt)
DECLS(sse2);
DECLS(sse2)
#undef DECLS
#undef DECL
@@ -383,20 +383,20 @@ DECLS(sse2);
return (var >= 0) ? (uint32_t)var : 0; \
}
#define FNS(opt) \
FN(64, 64, 16, 6, 6, opt, (int64_t)); \
FN(64, 32, 16, 6, 5, opt, (int64_t)); \
FN(32, 64, 16, 5, 6, opt, (int64_t)); \
FN(32, 32, 16, 5, 5, opt, (int64_t)); \
FN(32, 16, 16, 5, 4, opt, (int64_t)); \
FN(16, 32, 16, 4, 5, opt, (int64_t)); \
FN(16, 16, 16, 4, 4, opt, (int64_t)); \
FN(16, 8, 16, 4, 3, opt, (int64_t)); \
FN(8, 16, 8, 3, 4, opt, (int64_t)); \
FN(8, 8, 8, 3, 3, opt, (int64_t)); \
FN(8, 4, 8, 3, 2, opt, (int64_t));
#define FNS(opt) \
FN(64, 64, 16, 6, 6, opt, (int64_t)) \
FN(64, 32, 16, 6, 5, opt, (int64_t)) \
FN(32, 64, 16, 5, 6, opt, (int64_t)) \
FN(32, 32, 16, 5, 5, opt, (int64_t)) \
FN(32, 16, 16, 5, 4, opt, (int64_t)) \
FN(16, 32, 16, 4, 5, opt, (int64_t)) \
FN(16, 16, 16, 4, 4, opt, (int64_t)) \
FN(16, 8, 16, 4, 3, opt, (int64_t)) \
FN(8, 16, 8, 3, 4, opt, (int64_t)) \
FN(8, 8, 8, 3, 3, opt, (int64_t)) \
FN(8, 4, 8, 3, 2, opt, (int64_t))
FNS(sse2);
FNS(sse2)
#undef FNS
#undef FN
@@ -412,7 +412,7 @@ FNS(sse2);
DECL(16, opt1) \
DECL(8, opt1)
DECLS(sse2);
DECLS(sse2)
#undef DECL
#undef DECLS
@@ -542,20 +542,20 @@ DECLS(sse2);
return (var >= 0) ? (uint32_t)var : 0; \
}
#define FNS(opt1) \
FN(64, 64, 16, 6, 6, opt1, (int64_t)); \
FN(64, 32, 16, 6, 5, opt1, (int64_t)); \
FN(32, 64, 16, 5, 6, opt1, (int64_t)); \
FN(32, 32, 16, 5, 5, opt1, (int64_t)); \
FN(32, 16, 16, 5, 4, opt1, (int64_t)); \
FN(16, 32, 16, 4, 5, opt1, (int64_t)); \
FN(16, 16, 16, 4, 4, opt1, (int64_t)); \
FN(16, 8, 16, 4, 3, opt1, (int64_t)); \
FN(8, 16, 8, 4, 3, opt1, (int64_t)); \
FN(8, 8, 8, 3, 3, opt1, (int64_t)); \
FN(8, 4, 8, 3, 2, opt1, (int64_t));
#define FNS(opt1) \
FN(64, 64, 16, 6, 6, opt1, (int64_t)) \
FN(64, 32, 16, 6, 5, opt1, (int64_t)) \
FN(32, 64, 16, 5, 6, opt1, (int64_t)) \
FN(32, 32, 16, 5, 5, opt1, (int64_t)) \
FN(32, 16, 16, 5, 4, opt1, (int64_t)) \
FN(16, 32, 16, 4, 5, opt1, (int64_t)) \
FN(16, 16, 16, 4, 4, opt1, (int64_t)) \
FN(16, 8, 16, 4, 3, opt1, (int64_t)) \
FN(8, 16, 8, 4, 3, opt1, (int64_t)) \
FN(8, 8, 8, 3, 3, opt1, (int64_t)) \
FN(8, 4, 8, 3, 2, opt1, (int64_t))
FNS(sse2);
FNS(sse2)
#undef FNS
#undef FN
libs/libvpx/vpx_dsp/x86/loopfilter_sse2.c
+15 -15
View File
@@ -211,21 +211,21 @@ void vpx_lpf_vertical_4_sse2(uint8_t *s, int pitch, const uint8_t *blimit,
// 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
ps1ps0 = _mm_unpacklo_epi8(ps1ps0, x0);
storeu_uint32(s + 0 * pitch - 2, _mm_cvtsi128_si32(ps1ps0));
storeu_int32(s + 0 * pitch - 2, _mm_cvtsi128_si32(ps1ps0));
ps1ps0 = _mm_srli_si128(ps1ps0, 4);
storeu_uint32(s + 1 * pitch - 2, _mm_cvtsi128_si32(ps1ps0));
storeu_int32(s + 1 * pitch - 2, _mm_cvtsi128_si32(ps1ps0));
ps1ps0 = _mm_srli_si128(ps1ps0, 4);
storeu_uint32(s + 2 * pitch - 2, _mm_cvtsi128_si32(ps1ps0));
storeu_int32(s + 2 * pitch - 2, _mm_cvtsi128_si32(ps1ps0));
ps1ps0 = _mm_srli_si128(ps1ps0, 4);
storeu_uint32(s + 3 * pitch - 2, _mm_cvtsi128_si32(ps1ps0));
storeu_int32(s + 3 * pitch - 2, _mm_cvtsi128_si32(ps1ps0));
storeu_uint32(s + 4 * pitch - 2, _mm_cvtsi128_si32(qs1qs0));
storeu_int32(s + 4 * pitch - 2, _mm_cvtsi128_si32(qs1qs0));
qs1qs0 = _mm_srli_si128(qs1qs0, 4);
storeu_uint32(s + 5 * pitch - 2, _mm_cvtsi128_si32(qs1qs0));
storeu_int32(s + 5 * pitch - 2, _mm_cvtsi128_si32(qs1qs0));
qs1qs0 = _mm_srli_si128(qs1qs0, 4);
storeu_uint32(s + 6 * pitch - 2, _mm_cvtsi128_si32(qs1qs0));
storeu_int32(s + 6 * pitch - 2, _mm_cvtsi128_si32(qs1qs0));
qs1qs0 = _mm_srli_si128(qs1qs0, 4);
storeu_uint32(s + 7 * pitch - 2, _mm_cvtsi128_si32(qs1qs0));
storeu_int32(s + 7 * pitch - 2, _mm_cvtsi128_si32(qs1qs0));
}
void vpx_lpf_horizontal_16_sse2(unsigned char *s, int pitch,
@@ -1674,8 +1674,8 @@ void vpx_lpf_vertical_4_dual_sse2(uint8_t *s, int pitch, const uint8_t *blimit0,
transpose8x16(s - 4, s - 4 + pitch * 8, pitch, t_dst, 16);
// Loop filtering
vpx_lpf_horizontal_4_dual_sse2(t_dst + 4 * 16, 16, blimit0, limit0, thresh0,
blimit1, limit1, thresh1);
vpx_lpf_horizontal_4_dual(t_dst + 4 * 16, 16, blimit0, limit0, thresh0,
blimit1, limit1, thresh1);
src[0] = t_dst;
src[1] = t_dst + 8;
dst[0] = s - 4;
@@ -1700,7 +1700,7 @@ void vpx_lpf_vertical_8_sse2(unsigned char *s, int pitch,
transpose(src, pitch, dst, 8, 1);
// Loop filtering
vpx_lpf_horizontal_8_sse2(t_dst + 4 * 8, 8, blimit, limit, thresh);
vpx_lpf_horizontal_8(t_dst + 4 * 8, 8, blimit, limit, thresh);
src[0] = t_dst;
dst[0] = s - 4;
@@ -1721,8 +1721,8 @@ void vpx_lpf_vertical_8_dual_sse2(uint8_t *s, int pitch, const uint8_t *blimit0,
transpose8x16(s - 4, s - 4 + pitch * 8, pitch, t_dst, 16);
// Loop filtering
vpx_lpf_horizontal_8_dual_sse2(t_dst + 4 * 16, 16, blimit0, limit0, thresh0,
blimit1, limit1, thresh1);
vpx_lpf_horizontal_8_dual(t_dst + 4 * 16, 16, blimit0, limit0, thresh0,
blimit1, limit1, thresh1);
src[0] = t_dst;
src[1] = t_dst + 8;
@@ -1750,7 +1750,7 @@ void vpx_lpf_vertical_16_sse2(unsigned char *s, int pitch,
transpose(src, pitch, dst, 8, 2);
// Loop filtering
vpx_lpf_horizontal_16_sse2(t_dst + 8 * 8, 8, blimit, limit, thresh);
vpx_lpf_horizontal_16(t_dst + 8 * 8, 8, blimit, limit, thresh);
src[0] = t_dst;
src[1] = t_dst + 8 * 8;
@@ -1771,7 +1771,7 @@ void vpx_lpf_vertical_16_dual_sse2(unsigned char *s, int pitch,
transpose8x16(s, s + 8 * pitch, pitch, t_dst + 8 * 16, 16);
// Loop filtering
vpx_lpf_horizontal_16_dual_sse2(t_dst + 8 * 16, 16, blimit, limit, thresh);
vpx_lpf_horizontal_16_dual(t_dst + 8 * 16, 16, blimit, limit, thresh);
// Transpose back
transpose8x16(t_dst, t_dst + 8 * 16, 16, s - 8, pitch);
libs/libvpx/vpx_dsp/x86/mem_sse2.h
+3 -3
View File
@@ -16,12 +16,12 @@
#include "./vpx_config.h"
static INLINE void storeu_uint32(void *dst, uint32_t v) {
static INLINE void storeu_int32(void *dst, int32_t v) {
memcpy(dst, &v, sizeof(v));
}
static INLINE uint32_t loadu_uint32(const void *src) {
uint32_t v;
static INLINE int32_t loadu_int32(const void *src) {
int32_t v;
memcpy(&v, src, sizeof(v));
return v;
}
libs/libvpx/vpx_dsp/x86/quantize_avx.c
+3 -8
View File
@@ -21,8 +21,8 @@
#include "vpx_dsp/x86/quantize_ssse3.h"
void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int skip_block, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
uint16_t *eob_ptr, const int16_t *scan,
@@ -39,8 +39,6 @@ void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
__m128i eob = zero, eob0;
(void)scan;
(void)skip_block;
assert(!skip_block);
*eob_ptr = 0;
@@ -145,8 +143,7 @@ void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
}
void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int skip_block, const int16_t *zbin_ptr,
const int16_t *round_ptr,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
@@ -166,8 +163,6 @@ void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
(void)scan;
(void)n_coeffs;
(void)skip_block;
assert(!skip_block);
// Setup global values.
// The 32x32 halves zbin and round.
libs/libvpx/vpx_dsp/x86/quantize_sse2.c
+2 -4
View File
@@ -18,8 +18,8 @@
#include "vpx_dsp/x86/quantize_sse2.h"
void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int skip_block, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
uint16_t *eob_ptr, const int16_t *scan,
@@ -34,8 +34,6 @@ void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
__m128i eob, eob0;
(void)scan;
(void)skip_block;
assert(!skip_block);
// Setup global values.
load_b_values(zbin_ptr, &zbin, round_ptr, &round, quant_ptr, &quant,
libs/libvpx/vpx_dsp/x86/quantize_ssse3.c
+3 -7
View File
@@ -18,8 +18,8 @@
#include "vpx_dsp/x86/quantize_ssse3.h"
void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int skip_block, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr,
@@ -34,8 +34,6 @@ void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
__m128i eob, eob0;
(void)scan;
(void)skip_block;
assert(!skip_block);
load_b_values(zbin_ptr, &zbin, round_ptr, &round, quant_ptr, &quant,
dequant_ptr, &dequant, quant_shift_ptr, &shift);
@@ -111,7 +109,7 @@ void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
}
void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int skip_block, const int16_t *zbin_ptr,
const int16_t *zbin_ptr,
const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
@@ -131,8 +129,6 @@ void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
(void)scan;
(void)n_coeffs;
(void)skip_block;
assert(!skip_block);
// Setup global values.
// The 32x32 halves zbin and round.
libs/libvpx/vpx_dsp/x86/sad4d_avx2.c
+8 -4
View File
@@ -11,8 +11,12 @@
#include "./vpx_dsp_rtcd.h"
#include "vpx/vpx_integer.h"
static INLINE void calc_final(const __m256i *const sums /*[4]*/,
uint32_t sad_array[4]) {
// Note with sums[4] some versions of Visual Studio may fail due to parameter
// alignment, though the functions should be equivalent:
// error C2719: 'sums': formal parameter with requested alignment of 32 won't be
// aligned
static INLINE void calc_final_4(const __m256i *const sums /*[4]*/,
uint32_t sad_array[4]) {
const __m256i t0 = _mm256_hadd_epi32(sums[0], sums[1]);
const __m256i t1 = _mm256_hadd_epi32(sums[2], sums[3]);
const __m256i t2 = _mm256_hadd_epi32(t0, t1);
@@ -66,7 +70,7 @@ void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride,
refs[3] += ref_stride;
}
calc_final(sums, sad_array);
calc_final_4(sums, sad_array);
}
void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride,
@@ -126,5 +130,5 @@ void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride,
refs[3] += ref_stride;
}
calc_final(sums, sad_array);
calc_final_4(sums, sad_array);
}
libs/libvpx/vpx_dsp/x86/sad4d_avx512.c
+2 -2
View File
@@ -13,7 +13,7 @@
void vpx_sad64x64x4d_avx512(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_array[4], int ref_stride,
uint32_t res[4]) {
uint32_t sad_array[4]) {
__m512i src_reg, ref0_reg, ref1_reg, ref2_reg, ref3_reg;
__m512i sum_ref0, sum_ref1, sum_ref2, sum_ref3;
__m512i sum_mlow, sum_mhigh;
@@ -78,6 +78,6 @@ void vpx_sad64x64x4d_avx512(const uint8_t *src_ptr, int src_stride,
sum128 = _mm_add_epi32(_mm256_castsi256_si128(sum256),
_mm256_extractf128_si256(sum256, 1));
_mm_storeu_si128((__m128i *)(res), sum128);
_mm_storeu_si128((__m128i *)(sad_array), sum128);
}
}
libs/libvpx/vpx_dsp/x86/sad_avx2.c
+18 -18
View File
@@ -71,17 +71,17 @@
return res; \
}
#define FSAD64 \
FSAD64_H(64); \
FSAD64_H(32);
#define FSAD64 \
FSAD64_H(64) \
FSAD64_H(32)
#define FSAD32 \
FSAD32_H(64); \
FSAD32_H(32); \
FSAD32_H(16);
#define FSAD32 \
FSAD32_H(64) \
FSAD32_H(32) \
FSAD32_H(16)
FSAD64;
FSAD32;
FSAD64
FSAD32
#undef FSAD64
#undef FSAD32
@@ -160,17 +160,17 @@ FSAD32;
return res; \
}
#define FSADAVG64 \
FSADAVG64_H(64); \
FSADAVG64_H(32);
#define FSADAVG64 \
FSADAVG64_H(64) \
FSADAVG64_H(32)
#define FSADAVG32 \
FSADAVG32_H(64); \
FSADAVG32_H(32); \
FSADAVG32_H(16);
#define FSADAVG32 \
FSADAVG32_H(64) \
FSADAVG32_H(32) \
FSADAVG32_H(16)
FSADAVG64;
FSADAVG32;
FSADAVG64
FSADAVG32
#undef FSADAVG64
#undef FSADAVG32
libs/libvpx/vpx_dsp/x86/sad_sse2.asm
+2 -2
View File
@@ -25,11 +25,11 @@ cglobal sad%1x%2, 4, %3, 6, src, src_stride, ref, ref_stride, \
cglobal sad%1x%2_avg, 5, 1 + %3, 5, src, src_stride, ref, ref_stride, \
second_pred, n_rows
%else ; %3 == 7
cglobal sad%1x%2_avg, 5, ARCH_X86_64 + %3, 6, src, src_stride, \
cglobal sad%1x%2_avg, 5, VPX_ARCH_X86_64 + %3, 6, src, src_stride, \
ref, ref_stride, \
second_pred, \
src_stride3, ref_stride3
%if ARCH_X86_64
%if VPX_ARCH_X86_64
%define n_rowsd r7d
%else ; x86-32
%define n_rowsd dword r0m
libs/libvpx/vpx_dsp/x86/sad_sse3.asm
-376
View File
@@ -1,376 +0,0 @@
;
; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
;
; Use of this source code is governed by a BSD-style license
; that can be found in the LICENSE file in the root of the source
; tree. An additional intellectual property rights grant can be found
; in the file PATENTS. All contributing project authors may
; be found in the AUTHORS file in the root of the source tree.
;
%include "vpx_ports/x86_abi_support.asm"
%macro STACK_FRAME_CREATE_X3 0
%if ABI_IS_32BIT
%define src_ptr rsi
%define src_stride rax
%define ref_ptr rdi
%define ref_stride rdx
%define end_ptr rcx
%define ret_var rbx
%define result_ptr arg(4)
%define height dword ptr arg(4)
push rbp
mov rbp, rsp
push rsi
push rdi
push rbx
mov rsi, arg(0) ; src_ptr
mov rdi, arg(2) ; ref_ptr
movsxd rax, dword ptr arg(1) ; src_stride
movsxd rdx, dword ptr arg(3) ; ref_stride
%else
%if LIBVPX_YASM_WIN64
SAVE_XMM 7, u
%define src_ptr rcx
%define src_stride rdx
%define ref_ptr r8
%define ref_stride r9
%define end_ptr r10
%define ret_var r11
%define result_ptr [rsp+xmm_stack_space+8+4*8]
%define height dword ptr [rsp+xmm_stack_space+8+4*8]
%else
%define src_ptr rdi
%define src_stride rsi
%define ref_ptr rdx
%define ref_stride rcx
%define end_ptr r9
%define ret_var r10
%define result_ptr r8
%define height r8
%endif
%endif
%endmacro
%macro STACK_FRAME_DESTROY_X3 0
%define src_ptr
%define src_stride
%define ref_ptr
%define ref_stride
%define end_ptr
%define ret_var
%define result_ptr
%define height
%if ABI_IS_32BIT
pop rbx
pop rdi
pop rsi
pop rbp
%else
%if LIBVPX_YASM_WIN64
RESTORE_XMM
%endif
%endif
ret
%endmacro
%macro PROCESS_16X2X3 5
%if %1==0
movdqa xmm0, XMMWORD PTR [%2]
lddqu xmm5, XMMWORD PTR [%3]
lddqu xmm6, XMMWORD PTR [%3+1]
lddqu xmm7, XMMWORD PTR [%3+2]
psadbw xmm5, xmm0
psadbw xmm6, xmm0
psadbw xmm7, xmm0
%else
movdqa xmm0, XMMWORD PTR [%2]
lddqu xmm1, XMMWORD PTR [%3]
lddqu xmm2, XMMWORD PTR [%3+1]
lddqu xmm3, XMMWORD PTR [%3+2]
psadbw xmm1, xmm0
psadbw xmm2, xmm0
psadbw xmm3, xmm0
paddw xmm5, xmm1
paddw xmm6, xmm2
paddw xmm7, xmm3
%endif
movdqa xmm0, XMMWORD PTR [%2+%4]
lddqu xmm1, XMMWORD PTR [%3+%5]
lddqu xmm2, XMMWORD PTR [%3+%5+1]
lddqu xmm3, XMMWORD PTR [%3+%5+2]
%if %1==0 || %1==1
lea %2, [%2+%4*2]
lea %3, [%3+%5*2]
%endif
psadbw xmm1, xmm0
psadbw xmm2, xmm0
psadbw xmm3, xmm0
paddw xmm5, xmm1
paddw xmm6, xmm2
paddw xmm7, xmm3
%endmacro
%macro PROCESS_8X2X3 5
%if %1==0
movq mm0, QWORD PTR [%2]
movq mm5, QWORD PTR [%3]
movq mm6, QWORD PTR [%3+1]
movq mm7, QWORD PTR [%3+2]
psadbw mm5, mm0
psadbw mm6, mm0
psadbw mm7, mm0
%else
movq mm0, QWORD PTR [%2]
movq mm1, QWORD PTR [%3]
movq mm2, QWORD PTR [%3+1]
movq mm3, QWORD PTR [%3+2]
psadbw mm1, mm0
psadbw mm2, mm0
psadbw mm3, mm0
paddw mm5, mm1
paddw mm6, mm2
paddw mm7, mm3
%endif
movq mm0, QWORD PTR [%2+%4]
movq mm1, QWORD PTR [%3+%5]
movq mm2, QWORD PTR [%3+%5+1]
movq mm3, QWORD PTR [%3+%5+2]
%if %1==0 || %1==1
lea %2, [%2+%4*2]
lea %3, [%3+%5*2]
%endif
psadbw mm1, mm0
psadbw mm2, mm0
psadbw mm3, mm0
paddw mm5, mm1
paddw mm6, mm2
paddw mm7, mm3
%endmacro
SECTION .text
;void int vpx_sad16x16x3_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr,
; int ref_stride,
; int *results)
global sym(vpx_sad16x16x3_sse3) PRIVATE
sym(vpx_sad16x16x3_sse3):
STACK_FRAME_CREATE_X3
PROCESS_16X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_16X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride
mov rcx, result_ptr
movq xmm0, xmm5
psrldq xmm5, 8
paddw xmm0, xmm5
movd [rcx], xmm0
;-
movq xmm0, xmm6
psrldq xmm6, 8
paddw xmm0, xmm6
movd [rcx+4], xmm0
;-
movq xmm0, xmm7
psrldq xmm7, 8
paddw xmm0, xmm7
movd [rcx+8], xmm0
STACK_FRAME_DESTROY_X3
;void int vpx_sad16x8x3_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr,
; int ref_stride,
; int *results)
global sym(vpx_sad16x8x3_sse3) PRIVATE
sym(vpx_sad16x8x3_sse3):
STACK_FRAME_CREATE_X3
PROCESS_16X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_16X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride
mov rcx, result_ptr
movq xmm0, xmm5
psrldq xmm5, 8
paddw xmm0, xmm5
movd [rcx], xmm0
;-
movq xmm0, xmm6
psrldq xmm6, 8
paddw xmm0, xmm6
movd [rcx+4], xmm0
;-
movq xmm0, xmm7
psrldq xmm7, 8
paddw xmm0, xmm7
movd [rcx+8], xmm0
STACK_FRAME_DESTROY_X3
;void int vpx_sad8x16x3_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr,
; int ref_stride,
; int *results)
global sym(vpx_sad8x16x3_sse3) PRIVATE
sym(vpx_sad8x16x3_sse3):
STACK_FRAME_CREATE_X3
PROCESS_8X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_8X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride
mov rcx, result_ptr
punpckldq mm5, mm6
movq [rcx], mm5
movd [rcx+8], mm7
STACK_FRAME_DESTROY_X3
;void int vpx_sad8x8x3_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr,
; int ref_stride,
; int *results)
global sym(vpx_sad8x8x3_sse3) PRIVATE
sym(vpx_sad8x8x3_sse3):
STACK_FRAME_CREATE_X3
PROCESS_8X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride
PROCESS_8X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride
mov rcx, result_ptr
punpckldq mm5, mm6
movq [rcx], mm5
movd [rcx+8], mm7
STACK_FRAME_DESTROY_X3
;void int vpx_sad4x4x3_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr,
; int ref_stride,
; int *results)
global sym(vpx_sad4x4x3_sse3) PRIVATE
sym(vpx_sad4x4x3_sse3):
STACK_FRAME_CREATE_X3
movd mm0, DWORD PTR [src_ptr]
movd mm1, DWORD PTR [ref_ptr]
movd mm2, DWORD PTR [src_ptr+src_stride]
movd mm3, DWORD PTR [ref_ptr+ref_stride]
punpcklbw mm0, mm2
punpcklbw mm1, mm3
movd mm4, DWORD PTR [ref_ptr+1]
movd mm5, DWORD PTR [ref_ptr+2]
movd mm2, DWORD PTR [ref_ptr+ref_stride+1]
movd mm3, DWORD PTR [ref_ptr+ref_stride+2]
psadbw mm1, mm0
punpcklbw mm4, mm2
punpcklbw mm5, mm3
psadbw mm4, mm0
psadbw mm5, mm0
lea src_ptr, [src_ptr+src_stride*2]
lea ref_ptr, [ref_ptr+ref_stride*2]
movd mm0, DWORD PTR [src_ptr]
movd mm2, DWORD PTR [ref_ptr]
movd mm3, DWORD PTR [src_ptr+src_stride]
movd mm6, DWORD PTR [ref_ptr+ref_stride]
punpcklbw mm0, mm3
punpcklbw mm2, mm6
movd mm3, DWORD PTR [ref_ptr+1]
movd mm7, DWORD PTR [ref_ptr+2]
psadbw mm2, mm0
paddw mm1, mm2
movd mm2, DWORD PTR [ref_ptr+ref_stride+1]
movd mm6, DWORD PTR [ref_ptr+ref_stride+2]
punpcklbw mm3, mm2
punpcklbw mm7, mm6
psadbw mm3, mm0
psadbw mm7, mm0
paddw mm3, mm4
paddw mm7, mm5
mov rcx, result_ptr
punpckldq mm1, mm3
movq [rcx], mm1
movd [rcx+8], mm7
STACK_FRAME_DESTROY_X3
libs/libvpx/vpx_dsp/x86/sad_sse4.asm
-361
View File
@@ -1,361 +0,0 @@
;
; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
;
; Use of this source code is governed by a BSD-style license
; that can be found in the LICENSE file in the root of the source
; tree. An additional intellectual property rights grant can be found
; in the file PATENTS. All contributing project authors may
; be found in the AUTHORS file in the root of the source tree.
;
%include "vpx_ports/x86_abi_support.asm"
%macro PROCESS_16X2X8 1
%if %1
movdqa xmm0, XMMWORD PTR [rsi]
movq xmm1, MMWORD PTR [rdi]
movq xmm3, MMWORD PTR [rdi+8]
movq xmm2, MMWORD PTR [rdi+16]
punpcklqdq xmm1, xmm3
punpcklqdq xmm3, xmm2
movdqa xmm2, xmm1
mpsadbw xmm1, xmm0, 0x0
mpsadbw xmm2, xmm0, 0x5
psrldq xmm0, 8
movdqa xmm4, xmm3
mpsadbw xmm3, xmm0, 0x0
mpsadbw xmm4, xmm0, 0x5
paddw xmm1, xmm2
paddw xmm1, xmm3
paddw xmm1, xmm4
%else
movdqa xmm0, XMMWORD PTR [rsi]
movq xmm5, MMWORD PTR [rdi]
movq xmm3, MMWORD PTR [rdi+8]
movq xmm2, MMWORD PTR [rdi+16]
punpcklqdq xmm5, xmm3
punpcklqdq xmm3, xmm2
movdqa xmm2, xmm5
mpsadbw xmm5, xmm0, 0x0
mpsadbw xmm2, xmm0, 0x5
psrldq xmm0, 8
movdqa xmm4, xmm3
mpsadbw xmm3, xmm0, 0x0
mpsadbw xmm4, xmm0, 0x5
paddw xmm5, xmm2
paddw xmm5, xmm3
paddw xmm5, xmm4
paddw xmm1, xmm5
%endif
movdqa xmm0, XMMWORD PTR [rsi + rax]
movq xmm5, MMWORD PTR [rdi+ rdx]
movq xmm3, MMWORD PTR [rdi+ rdx+8]
movq xmm2, MMWORD PTR [rdi+ rdx+16]
punpcklqdq xmm5, xmm3
punpcklqdq xmm3, xmm2
lea rsi, [rsi+rax*2]
lea rdi, [rdi+rdx*2]
movdqa xmm2, xmm5
mpsadbw xmm5, xmm0, 0x0
mpsadbw xmm2, xmm0, 0x5
psrldq xmm0, 8
movdqa xmm4, xmm3
mpsadbw xmm3, xmm0, 0x0
mpsadbw xmm4, xmm0, 0x5
paddw xmm5, xmm2
paddw xmm5, xmm3
paddw xmm5, xmm4
paddw xmm1, xmm5
%endmacro
%macro PROCESS_8X2X8 1
%if %1
movq xmm0, MMWORD PTR [rsi]
movq xmm1, MMWORD PTR [rdi]
movq xmm3, MMWORD PTR [rdi+8]
punpcklqdq xmm1, xmm3
movdqa xmm2, xmm1
mpsadbw xmm1, xmm0, 0x0
mpsadbw xmm2, xmm0, 0x5
paddw xmm1, xmm2
%else
movq xmm0, MMWORD PTR [rsi]
movq xmm5, MMWORD PTR [rdi]
movq xmm3, MMWORD PTR [rdi+8]
punpcklqdq xmm5, xmm3
movdqa xmm2, xmm5
mpsadbw xmm5, xmm0, 0x0
mpsadbw xmm2, xmm0, 0x5
paddw xmm5, xmm2
paddw xmm1, xmm5
%endif
movq xmm0, MMWORD PTR [rsi + rax]
movq xmm5, MMWORD PTR [rdi+ rdx]
movq xmm3, MMWORD PTR [rdi+ rdx+8]
punpcklqdq xmm5, xmm3
lea rsi, [rsi+rax*2]
lea rdi, [rdi+rdx*2]
movdqa xmm2, xmm5
mpsadbw xmm5, xmm0, 0x0
mpsadbw xmm2, xmm0, 0x5
paddw xmm5, xmm2
paddw xmm1, xmm5
%endmacro
%macro PROCESS_4X2X8 1
%if %1
movd xmm0, [rsi]
movq xmm1, MMWORD PTR [rdi]
movq xmm3, MMWORD PTR [rdi+8]
punpcklqdq xmm1, xmm3
mpsadbw xmm1, xmm0, 0x0
%else
movd xmm0, [rsi]
movq xmm5, MMWORD PTR [rdi]
movq xmm3, MMWORD PTR [rdi+8]
punpcklqdq xmm5, xmm3
mpsadbw xmm5, xmm0, 0x0
paddw xmm1, xmm5
%endif
movd xmm0, [rsi + rax]
movq xmm5, MMWORD PTR [rdi+ rdx]
movq xmm3, MMWORD PTR [rdi+ rdx+8]
punpcklqdq xmm5, xmm3
lea rsi, [rsi+rax*2]
lea rdi, [rdi+rdx*2]
mpsadbw xmm5, xmm0, 0x0
paddw xmm1, xmm5
%endmacro
%macro WRITE_AS_INTS 0
mov rdi, arg(4) ;Results
pxor xmm0, xmm0
movdqa xmm2, xmm1
punpcklwd xmm1, xmm0
punpckhwd xmm2, xmm0
movdqa [rdi], xmm1
movdqa [rdi + 16], xmm2
%endmacro
SECTION .text
;void vpx_sad16x16x8_sse4_1(
; const unsigned char *src_ptr,
; int src_stride,
; const unsigned char *ref_ptr,
; int ref_stride,
; unsigned short *sad_array);
global sym(vpx_sad16x16x8_sse4_1) PRIVATE
sym(vpx_sad16x16x8_sse4_1):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;src_ptr
mov rdi, arg(2) ;ref_ptr
movsxd rax, dword ptr arg(1) ;src_stride
movsxd rdx, dword ptr arg(3) ;ref_stride
PROCESS_16X2X8 1
PROCESS_16X2X8 0
PROCESS_16X2X8 0
PROCESS_16X2X8 0
PROCESS_16X2X8 0
PROCESS_16X2X8 0
PROCESS_16X2X8 0
PROCESS_16X2X8 0
WRITE_AS_INTS
; begin epilog
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret
;void vpx_sad16x8x8_sse4_1(
; const unsigned char *src_ptr,
; int src_stride,
; const unsigned char *ref_ptr,
; int ref_stride,
; unsigned short *sad_array
;);
global sym(vpx_sad16x8x8_sse4_1) PRIVATE
sym(vpx_sad16x8x8_sse4_1):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;src_ptr
mov rdi, arg(2) ;ref_ptr
movsxd rax, dword ptr arg(1) ;src_stride
movsxd rdx, dword ptr arg(3) ;ref_stride
PROCESS_16X2X8 1
PROCESS_16X2X8 0
PROCESS_16X2X8 0
PROCESS_16X2X8 0
WRITE_AS_INTS
; begin epilog
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret
;void vpx_sad8x8x8_sse4_1(
; const unsigned char *src_ptr,
; int src_stride,
; const unsigned char *ref_ptr,
; int ref_stride,
; unsigned short *sad_array
;);
global sym(vpx_sad8x8x8_sse4_1) PRIVATE
sym(vpx_sad8x8x8_sse4_1):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;src_ptr
mov rdi, arg(2) ;ref_ptr
movsxd rax, dword ptr arg(1) ;src_stride
movsxd rdx, dword ptr arg(3) ;ref_stride
PROCESS_8X2X8 1
PROCESS_8X2X8 0
PROCESS_8X2X8 0
PROCESS_8X2X8 0
WRITE_AS_INTS
; begin epilog
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret
;void vpx_sad8x16x8_sse4_1(
; const unsigned char *src_ptr,
; int src_stride,
; const unsigned char *ref_ptr,
; int ref_stride,
; unsigned short *sad_array
;);
global sym(vpx_sad8x16x8_sse4_1) PRIVATE
sym(vpx_sad8x16x8_sse4_1):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;src_ptr
mov rdi, arg(2) ;ref_ptr
movsxd rax, dword ptr arg(1) ;src_stride
movsxd rdx, dword ptr arg(3) ;ref_stride
PROCESS_8X2X8 1
PROCESS_8X2X8 0
PROCESS_8X2X8 0
PROCESS_8X2X8 0
PROCESS_8X2X8 0
PROCESS_8X2X8 0
PROCESS_8X2X8 0
PROCESS_8X2X8 0
WRITE_AS_INTS
; begin epilog
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret
;void vpx_sad4x4x8_sse4_1(
; const unsigned char *src_ptr,
; int src_stride,
; const unsigned char *ref_ptr,
; int ref_stride,
; unsigned short *sad_array
;);
global sym(vpx_sad4x4x8_sse4_1) PRIVATE
sym(vpx_sad4x4x8_sse4_1):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;src_ptr
mov rdi, arg(2) ;ref_ptr
movsxd rax, dword ptr arg(1) ;src_stride
movsxd rdx, dword ptr arg(3) ;ref_stride
PROCESS_4X2X8 1
PROCESS_4X2X8 0
WRITE_AS_INTS
; begin epilog
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret
Loaded 100 of 572 files, more files were not shown because too many files have changed in this diff. Show more