mirror of
https://github.com/signalwire/freeswitch.git
synced 2026-11-12 12:37:05 +00:00
[libvpx] Update to v1.12.0
This commit is contained in:
Andrey Volk committed
1 parent
287a71dc66
commit
b5046d2a76
572 files changed
+52900
-29493
No files matched your search
@@ -22,15 +22,13 @@
|
||||
uint32_t vpx_avg_4x4_neon(const uint8_t *a, int a_stride) {
|
||||
const uint8x16_t b = load_unaligned_u8q(a, a_stride);
|
||||
const uint16x8_t c = vaddl_u8(vget_low_u8(b), vget_high_u8(b));
|
||||
const uint32x2_t d = horizontal_add_uint16x8(c);
|
||||
return vget_lane_u32(vrshr_n_u32(d, 4), 0);
|
||||
return (horizontal_add_uint16x8(c) + (1 << 3)) >> 4;
|
||||
}
|
||||
|
||||
uint32_t vpx_avg_8x8_neon(const uint8_t *a, int a_stride) {
|
||||
int i;
|
||||
uint8x8_t b, c;
|
||||
uint16x8_t sum;
|
||||
uint32x2_t d;
|
||||
b = vld1_u8(a);
|
||||
a += a_stride;
|
||||
c = vld1_u8(a);
|
||||
@@ -43,9 +41,7 @@ uint32_t vpx_avg_8x8_neon(const uint8_t *a, int a_stride) {
|
||||
sum = vaddw_u8(sum, d);
|
||||
}
|
||||
|
||||
d = horizontal_add_uint16x8(sum);
|
||||
|
||||
return vget_lane_u32(vrshr_n_u32(d, 6), 0);
|
||||
return (horizontal_add_uint16x8(sum) + (1 << 5)) >> 6;
|
||||
}
|
||||
|
||||
// coeff: 16 bits, dynamic range [-32640, 32640].
|
||||
@@ -139,8 +135,7 @@ int16_t vpx_int_pro_col_neon(uint8_t const *ref, const int width) {
|
||||
ref += 16;
|
||||
}
|
||||
|
||||
return vget_lane_s16(vreinterpret_s16_u32(horizontal_add_uint16x8(vec_sum)),
|
||||
0);
|
||||
return (int16_t)horizontal_add_uint16x8(vec_sum);
|
||||
}
|
||||
|
||||
// ref, src = [0, 510] - max diff = 16-bits
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
#include "vpx_dsp/arm/fdct16x16_neon.h"
|
||||
|
||||
// Some builds of gcc 4.9.2 and .3 have trouble with some of the inline
|
||||
// functions.
|
||||
@@ -27,316 +28,6 @@ void vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) {
|
||||
|
||||
#else
|
||||
|
||||
static INLINE void load(const int16_t *a, int stride, int16x8_t *b /*[16]*/) {
|
||||
b[0] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[1] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[2] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[3] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[4] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[5] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[6] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[7] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[8] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[9] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[10] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[11] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[12] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[13] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[14] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[15] = vld1q_s16(a);
|
||||
}
|
||||
|
||||
// Store 8 16x8 values, assuming stride == 16.
|
||||
static INLINE void store(tran_low_t *a, const int16x8_t *b /*[8]*/) {
|
||||
store_s16q_to_tran_low(a, b[0]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[1]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[2]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[3]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[4]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[5]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[6]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[7]);
|
||||
}
|
||||
|
||||
// Load step of each pass. Add and subtract clear across the input, requiring
|
||||
// all 16 values to be loaded. For the first pass it also multiplies by 4.
|
||||
|
||||
// To maybe reduce register usage this could be combined with the load() step to
|
||||
// get the first 4 and last 4 values, cross those, then load the middle 8 values
|
||||
// and cross them.
|
||||
static INLINE void cross_input(const int16x8_t *a /*[16]*/,
|
||||
int16x8_t *b /*[16]*/, const int pass) {
|
||||
if (pass == 0) {
|
||||
b[0] = vshlq_n_s16(vaddq_s16(a[0], a[15]), 2);
|
||||
b[1] = vshlq_n_s16(vaddq_s16(a[1], a[14]), 2);
|
||||
b[2] = vshlq_n_s16(vaddq_s16(a[2], a[13]), 2);
|
||||
b[3] = vshlq_n_s16(vaddq_s16(a[3], a[12]), 2);
|
||||
b[4] = vshlq_n_s16(vaddq_s16(a[4], a[11]), 2);
|
||||
b[5] = vshlq_n_s16(vaddq_s16(a[5], a[10]), 2);
|
||||
b[6] = vshlq_n_s16(vaddq_s16(a[6], a[9]), 2);
|
||||
b[7] = vshlq_n_s16(vaddq_s16(a[7], a[8]), 2);
|
||||
|
||||
b[8] = vshlq_n_s16(vsubq_s16(a[7], a[8]), 2);
|
||||
b[9] = vshlq_n_s16(vsubq_s16(a[6], a[9]), 2);
|
||||
b[10] = vshlq_n_s16(vsubq_s16(a[5], a[10]), 2);
|
||||
b[11] = vshlq_n_s16(vsubq_s16(a[4], a[11]), 2);
|
||||
b[12] = vshlq_n_s16(vsubq_s16(a[3], a[12]), 2);
|
||||
b[13] = vshlq_n_s16(vsubq_s16(a[2], a[13]), 2);
|
||||
b[14] = vshlq_n_s16(vsubq_s16(a[1], a[14]), 2);
|
||||
b[15] = vshlq_n_s16(vsubq_s16(a[0], a[15]), 2);
|
||||
} else {
|
||||
b[0] = vaddq_s16(a[0], a[15]);
|
||||
b[1] = vaddq_s16(a[1], a[14]);
|
||||
b[2] = vaddq_s16(a[2], a[13]);
|
||||
b[3] = vaddq_s16(a[3], a[12]);
|
||||
b[4] = vaddq_s16(a[4], a[11]);
|
||||
b[5] = vaddq_s16(a[5], a[10]);
|
||||
b[6] = vaddq_s16(a[6], a[9]);
|
||||
b[7] = vaddq_s16(a[7], a[8]);
|
||||
|
||||
b[8] = vsubq_s16(a[7], a[8]);
|
||||
b[9] = vsubq_s16(a[6], a[9]);
|
||||
b[10] = vsubq_s16(a[5], a[10]);
|
||||
b[11] = vsubq_s16(a[4], a[11]);
|
||||
b[12] = vsubq_s16(a[3], a[12]);
|
||||
b[13] = vsubq_s16(a[2], a[13]);
|
||||
b[14] = vsubq_s16(a[1], a[14]);
|
||||
b[15] = vsubq_s16(a[0], a[15]);
|
||||
}
|
||||
}
|
||||
|
||||
// Quarter round at the beginning of the second pass. Can't use vrshr (rounding)
|
||||
// because this only adds 1, not 1 << 2.
|
||||
static INLINE void partial_round_shift(int16x8_t *a /*[16]*/) {
|
||||
const int16x8_t one = vdupq_n_s16(1);
|
||||
a[0] = vshrq_n_s16(vaddq_s16(a[0], one), 2);
|
||||
a[1] = vshrq_n_s16(vaddq_s16(a[1], one), 2);
|
||||
a[2] = vshrq_n_s16(vaddq_s16(a[2], one), 2);
|
||||
a[3] = vshrq_n_s16(vaddq_s16(a[3], one), 2);
|
||||
a[4] = vshrq_n_s16(vaddq_s16(a[4], one), 2);
|
||||
a[5] = vshrq_n_s16(vaddq_s16(a[5], one), 2);
|
||||
a[6] = vshrq_n_s16(vaddq_s16(a[6], one), 2);
|
||||
a[7] = vshrq_n_s16(vaddq_s16(a[7], one), 2);
|
||||
a[8] = vshrq_n_s16(vaddq_s16(a[8], one), 2);
|
||||
a[9] = vshrq_n_s16(vaddq_s16(a[9], one), 2);
|
||||
a[10] = vshrq_n_s16(vaddq_s16(a[10], one), 2);
|
||||
a[11] = vshrq_n_s16(vaddq_s16(a[11], one), 2);
|
||||
a[12] = vshrq_n_s16(vaddq_s16(a[12], one), 2);
|
||||
a[13] = vshrq_n_s16(vaddq_s16(a[13], one), 2);
|
||||
a[14] = vshrq_n_s16(vaddq_s16(a[14], one), 2);
|
||||
a[15] = vshrq_n_s16(vaddq_s16(a[15], one), 2);
|
||||
}
|
||||
|
||||
// fdct_round_shift((a +/- b) * c)
|
||||
static INLINE void butterfly_one_coeff(const int16x8_t a, const int16x8_t b,
|
||||
const tran_high_t c, int16x8_t *add,
|
||||
int16x8_t *sub) {
|
||||
const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c);
|
||||
const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c);
|
||||
const int32x4_t sum0 = vmlal_n_s16(a0, vget_low_s16(b), c);
|
||||
const int32x4_t sum1 = vmlal_n_s16(a1, vget_high_s16(b), c);
|
||||
const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c);
|
||||
const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c);
|
||||
const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14);
|
||||
const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14);
|
||||
const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14);
|
||||
const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14);
|
||||
*add = vcombine_s16(rounded0, rounded1);
|
||||
*sub = vcombine_s16(rounded2, rounded3);
|
||||
}
|
||||
|
||||
// fdct_round_shift(a * c0 +/- b * c1)
|
||||
static INLINE void butterfly_two_coeff(const int16x8_t a, const int16x8_t b,
|
||||
const tran_coef_t c0,
|
||||
const tran_coef_t c1, int16x8_t *add,
|
||||
int16x8_t *sub) {
|
||||
const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c0);
|
||||
const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c0);
|
||||
const int32x4_t a2 = vmull_n_s16(vget_low_s16(a), c1);
|
||||
const int32x4_t a3 = vmull_n_s16(vget_high_s16(a), c1);
|
||||
const int32x4_t sum0 = vmlal_n_s16(a2, vget_low_s16(b), c0);
|
||||
const int32x4_t sum1 = vmlal_n_s16(a3, vget_high_s16(b), c0);
|
||||
const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c1);
|
||||
const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c1);
|
||||
const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14);
|
||||
const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14);
|
||||
const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14);
|
||||
const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14);
|
||||
*add = vcombine_s16(rounded0, rounded1);
|
||||
*sub = vcombine_s16(rounded2, rounded3);
|
||||
}
|
||||
|
||||
// Transpose 8x8 to a new location. Don't use transpose_neon.h because those
|
||||
// are all in-place.
|
||||
static INLINE void transpose_8x8(const int16x8_t *a /*[8]*/,
|
||||
int16x8_t *b /*[8]*/) {
|
||||
// Swap 16 bit elements.
|
||||
const int16x8x2_t c0 = vtrnq_s16(a[0], a[1]);
|
||||
const int16x8x2_t c1 = vtrnq_s16(a[2], a[3]);
|
||||
const int16x8x2_t c2 = vtrnq_s16(a[4], a[5]);
|
||||
const int16x8x2_t c3 = vtrnq_s16(a[6], a[7]);
|
||||
|
||||
// Swap 32 bit elements.
|
||||
const int32x4x2_t d0 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[0]),
|
||||
vreinterpretq_s32_s16(c1.val[0]));
|
||||
const int32x4x2_t d1 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[1]),
|
||||
vreinterpretq_s32_s16(c1.val[1]));
|
||||
const int32x4x2_t d2 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[0]),
|
||||
vreinterpretq_s32_s16(c3.val[0]));
|
||||
const int32x4x2_t d3 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[1]),
|
||||
vreinterpretq_s32_s16(c3.val[1]));
|
||||
|
||||
// Swap 64 bit elements
|
||||
const int16x8x2_t e0 = vpx_vtrnq_s64_to_s16(d0.val[0], d2.val[0]);
|
||||
const int16x8x2_t e1 = vpx_vtrnq_s64_to_s16(d1.val[0], d3.val[0]);
|
||||
const int16x8x2_t e2 = vpx_vtrnq_s64_to_s16(d0.val[1], d2.val[1]);
|
||||
const int16x8x2_t e3 = vpx_vtrnq_s64_to_s16(d1.val[1], d3.val[1]);
|
||||
|
||||
b[0] = e0.val[0];
|
||||
b[1] = e1.val[0];
|
||||
b[2] = e2.val[0];
|
||||
b[3] = e3.val[0];
|
||||
b[4] = e0.val[1];
|
||||
b[5] = e1.val[1];
|
||||
b[6] = e2.val[1];
|
||||
b[7] = e3.val[1];
|
||||
}
|
||||
|
||||
// Main body of fdct16x16.
|
||||
static void dct_body(const int16x8_t *in /*[16]*/, int16x8_t *out /*[16]*/) {
|
||||
int16x8_t s[8];
|
||||
int16x8_t x[4];
|
||||
int16x8_t step[8];
|
||||
|
||||
// stage 1
|
||||
// From fwd_txfm.c: Work on the first eight values; fdct8(input,
|
||||
// even_results);"
|
||||
s[0] = vaddq_s16(in[0], in[7]);
|
||||
s[1] = vaddq_s16(in[1], in[6]);
|
||||
s[2] = vaddq_s16(in[2], in[5]);
|
||||
s[3] = vaddq_s16(in[3], in[4]);
|
||||
s[4] = vsubq_s16(in[3], in[4]);
|
||||
s[5] = vsubq_s16(in[2], in[5]);
|
||||
s[6] = vsubq_s16(in[1], in[6]);
|
||||
s[7] = vsubq_s16(in[0], in[7]);
|
||||
|
||||
// fdct4(step, step);
|
||||
x[0] = vaddq_s16(s[0], s[3]);
|
||||
x[1] = vaddq_s16(s[1], s[2]);
|
||||
x[2] = vsubq_s16(s[1], s[2]);
|
||||
x[3] = vsubq_s16(s[0], s[3]);
|
||||
|
||||
// out[0] = fdct_round_shift((x0 + x1) * cospi_16_64)
|
||||
// out[8] = fdct_round_shift((x0 - x1) * cospi_16_64)
|
||||
butterfly_one_coeff(x[0], x[1], cospi_16_64, &out[0], &out[8]);
|
||||
// out[4] = fdct_round_shift(x3 * cospi_8_64 + x2 * cospi_24_64);
|
||||
// out[12] = fdct_round_shift(x3 * cospi_24_64 - x2 * cospi_8_64);
|
||||
butterfly_two_coeff(x[3], x[2], cospi_24_64, cospi_8_64, &out[4], &out[12]);
|
||||
|
||||
// Stage 2
|
||||
// Re-using source s5/s6
|
||||
// s5 = fdct_round_shift((s6 - s5) * cospi_16_64)
|
||||
// s6 = fdct_round_shift((s6 + s5) * cospi_16_64)
|
||||
butterfly_one_coeff(s[6], s[5], cospi_16_64, &s[6], &s[5]);
|
||||
|
||||
// Stage 3
|
||||
x[0] = vaddq_s16(s[4], s[5]);
|
||||
x[1] = vsubq_s16(s[4], s[5]);
|
||||
x[2] = vsubq_s16(s[7], s[6]);
|
||||
x[3] = vaddq_s16(s[7], s[6]);
|
||||
|
||||
// Stage 4
|
||||
// out[2] = fdct_round_shift(x0 * cospi_28_64 + x3 * cospi_4_64)
|
||||
// out[14] = fdct_round_shift(x3 * cospi_28_64 + x0 * -cospi_4_64)
|
||||
butterfly_two_coeff(x[3], x[0], cospi_28_64, cospi_4_64, &out[2], &out[14]);
|
||||
// out[6] = fdct_round_shift(x1 * cospi_12_64 + x2 * cospi_20_64)
|
||||
// out[10] = fdct_round_shift(x2 * cospi_12_64 + x1 * -cospi_20_64)
|
||||
butterfly_two_coeff(x[2], x[1], cospi_12_64, cospi_20_64, &out[10], &out[6]);
|
||||
|
||||
// step 2
|
||||
// From fwd_txfm.c: Work on the next eight values; step1 -> odd_results"
|
||||
// That file distinguished between "in_high" and "step1" but the only
|
||||
// difference is that "in_high" is the first 8 values and "step 1" is the
|
||||
// second. Here, since they are all in one array, "step1" values are += 8.
|
||||
|
||||
// step2[2] = fdct_round_shift((step1[5] - step1[2]) * cospi_16_64)
|
||||
// step2[3] = fdct_round_shift((step1[4] - step1[3]) * cospi_16_64)
|
||||
// step2[4] = fdct_round_shift((step1[4] + step1[3]) * cospi_16_64)
|
||||
// step2[5] = fdct_round_shift((step1[5] + step1[2]) * cospi_16_64)
|
||||
butterfly_one_coeff(in[13], in[10], cospi_16_64, &s[5], &s[2]);
|
||||
butterfly_one_coeff(in[12], in[11], cospi_16_64, &s[4], &s[3]);
|
||||
|
||||
// step 3
|
||||
s[0] = vaddq_s16(in[8], s[3]);
|
||||
s[1] = vaddq_s16(in[9], s[2]);
|
||||
x[0] = vsubq_s16(in[9], s[2]);
|
||||
x[1] = vsubq_s16(in[8], s[3]);
|
||||
x[2] = vsubq_s16(in[15], s[4]);
|
||||
x[3] = vsubq_s16(in[14], s[5]);
|
||||
s[6] = vaddq_s16(in[14], s[5]);
|
||||
s[7] = vaddq_s16(in[15], s[4]);
|
||||
|
||||
// step 4
|
||||
// step2[1] = fdct_round_shift(step3[1] *-cospi_8_64 + step3[6] * cospi_24_64)
|
||||
// step2[6] = fdct_round_shift(step3[1] * cospi_24_64 + step3[6] * cospi_8_64)
|
||||
butterfly_two_coeff(s[6], s[1], cospi_24_64, cospi_8_64, &s[6], &s[1]);
|
||||
|
||||
// step2[2] = fdct_round_shift(step3[2] * cospi_24_64 + step3[5] * cospi_8_64)
|
||||
// step2[5] = fdct_round_shift(step3[2] * cospi_8_64 - step3[5] * cospi_24_64)
|
||||
butterfly_two_coeff(x[0], x[3], cospi_8_64, cospi_24_64, &s[2], &s[5]);
|
||||
|
||||
// step 5
|
||||
step[0] = vaddq_s16(s[0], s[1]);
|
||||
step[1] = vsubq_s16(s[0], s[1]);
|
||||
step[2] = vaddq_s16(x[1], s[2]);
|
||||
step[3] = vsubq_s16(x[1], s[2]);
|
||||
step[4] = vsubq_s16(x[2], s[5]);
|
||||
step[5] = vaddq_s16(x[2], s[5]);
|
||||
step[6] = vsubq_s16(s[7], s[6]);
|
||||
step[7] = vaddq_s16(s[7], s[6]);
|
||||
|
||||
// step 6
|
||||
// out[1] = fdct_round_shift(step1[0] * cospi_30_64 + step1[7] * cospi_2_64)
|
||||
// out[9] = fdct_round_shift(step1[1] * cospi_14_64 + step1[6] * cospi_18_64)
|
||||
// out[5] = fdct_round_shift(step1[2] * cospi_22_64 + step1[5] * cospi_10_64)
|
||||
// out[13] = fdct_round_shift(step1[3] * cospi_6_64 + step1[4] * cospi_26_64)
|
||||
// out[3] = fdct_round_shift(step1[3] * -cospi_26_64 + step1[4] * cospi_6_64)
|
||||
// out[11] = fdct_round_shift(step1[2] * -cospi_10_64 + step1[5] *
|
||||
// cospi_22_64)
|
||||
// out[7] = fdct_round_shift(step1[1] * -cospi_18_64 + step1[6] * cospi_14_64)
|
||||
// out[15] = fdct_round_shift(step1[0] * -cospi_2_64 + step1[7] * cospi_30_64)
|
||||
butterfly_two_coeff(step[6], step[1], cospi_14_64, cospi_18_64, &out[9],
|
||||
&out[7]);
|
||||
butterfly_two_coeff(step[7], step[0], cospi_30_64, cospi_2_64, &out[1],
|
||||
&out[15]);
|
||||
butterfly_two_coeff(step[4], step[3], cospi_6_64, cospi_26_64, &out[13],
|
||||
&out[3]);
|
||||
butterfly_two_coeff(step[5], step[2], cospi_22_64, cospi_10_64, &out[5],
|
||||
&out[11]);
|
||||
}
|
||||
|
||||
void vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) {
|
||||
int16x8_t temp0[16];
|
||||
int16x8_t temp1[16];
|
||||
@@ -346,12 +37,12 @@ void vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) {
|
||||
// Left half.
|
||||
load(input, stride, temp0);
|
||||
cross_input(temp0, temp1, 0);
|
||||
dct_body(temp1, temp0);
|
||||
vpx_fdct16x16_body(temp1, temp0);
|
||||
|
||||
// Right half.
|
||||
load(input + 8, stride, temp1);
|
||||
cross_input(temp1, temp2, 0);
|
||||
dct_body(temp2, temp1);
|
||||
vpx_fdct16x16_body(temp2, temp1);
|
||||
|
||||
// Transpose top left and top right quarters into one contiguous location to
|
||||
// process to the top half.
|
||||
@@ -359,7 +50,7 @@ void vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) {
|
||||
transpose_8x8(&temp1[0], &temp2[8]);
|
||||
partial_round_shift(temp2);
|
||||
cross_input(temp2, temp3, 1);
|
||||
dct_body(temp3, temp2);
|
||||
vpx_fdct16x16_body(temp3, temp2);
|
||||
transpose_s16_8x8(&temp2[0], &temp2[1], &temp2[2], &temp2[3], &temp2[4],
|
||||
&temp2[5], &temp2[6], &temp2[7]);
|
||||
transpose_s16_8x8(&temp2[8], &temp2[9], &temp2[10], &temp2[11], &temp2[12],
|
||||
@@ -375,7 +66,7 @@ void vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) {
|
||||
&temp1[13], &temp1[14], &temp1[15]);
|
||||
partial_round_shift(temp1);
|
||||
cross_input(temp1, temp0, 1);
|
||||
dct_body(temp0, temp1);
|
||||
vpx_fdct16x16_body(temp0, temp1);
|
||||
transpose_s16_8x8(&temp1[0], &temp1[1], &temp1[2], &temp1[3], &temp1[4],
|
||||
&temp1[5], &temp1[6], &temp1[7]);
|
||||
transpose_s16_8x8(&temp1[8], &temp1[9], &temp1[10], &temp1[11], &temp1[12],
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
/*
|
||||
* Copyright (c) 2017 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef VPX_VPX_DSP_ARM_FDCT16X16_NEON_H_
|
||||
#define VPX_VPX_DSP_ARM_FDCT16X16_NEON_H_
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
static INLINE void load(const int16_t *a, int stride, int16x8_t *b /*[16]*/) {
|
||||
b[0] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[1] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[2] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[3] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[4] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[5] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[6] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[7] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[8] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[9] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[10] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[11] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[12] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[13] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[14] = vld1q_s16(a);
|
||||
a += stride;
|
||||
b[15] = vld1q_s16(a);
|
||||
}
|
||||
|
||||
// Store 8 16x8 values, assuming stride == 16.
|
||||
static INLINE void store(tran_low_t *a, const int16x8_t *b /*[8]*/) {
|
||||
store_s16q_to_tran_low(a, b[0]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[1]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[2]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[3]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[4]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[5]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[6]);
|
||||
a += 16;
|
||||
store_s16q_to_tran_low(a, b[7]);
|
||||
}
|
||||
|
||||
// Load step of each pass. Add and subtract clear across the input, requiring
|
||||
// all 16 values to be loaded. For the first pass it also multiplies by 4.
|
||||
|
||||
// To maybe reduce register usage this could be combined with the load() step to
|
||||
// get the first 4 and last 4 values, cross those, then load the middle 8 values
|
||||
// and cross them.
|
||||
static INLINE void cross_input(const int16x8_t *a /*[16]*/,
|
||||
int16x8_t *b /*[16]*/, const int pass) {
|
||||
if (pass == 0) {
|
||||
b[0] = vshlq_n_s16(vaddq_s16(a[0], a[15]), 2);
|
||||
b[1] = vshlq_n_s16(vaddq_s16(a[1], a[14]), 2);
|
||||
b[2] = vshlq_n_s16(vaddq_s16(a[2], a[13]), 2);
|
||||
b[3] = vshlq_n_s16(vaddq_s16(a[3], a[12]), 2);
|
||||
b[4] = vshlq_n_s16(vaddq_s16(a[4], a[11]), 2);
|
||||
b[5] = vshlq_n_s16(vaddq_s16(a[5], a[10]), 2);
|
||||
b[6] = vshlq_n_s16(vaddq_s16(a[6], a[9]), 2);
|
||||
b[7] = vshlq_n_s16(vaddq_s16(a[7], a[8]), 2);
|
||||
|
||||
b[8] = vshlq_n_s16(vsubq_s16(a[7], a[8]), 2);
|
||||
b[9] = vshlq_n_s16(vsubq_s16(a[6], a[9]), 2);
|
||||
b[10] = vshlq_n_s16(vsubq_s16(a[5], a[10]), 2);
|
||||
b[11] = vshlq_n_s16(vsubq_s16(a[4], a[11]), 2);
|
||||
b[12] = vshlq_n_s16(vsubq_s16(a[3], a[12]), 2);
|
||||
b[13] = vshlq_n_s16(vsubq_s16(a[2], a[13]), 2);
|
||||
b[14] = vshlq_n_s16(vsubq_s16(a[1], a[14]), 2);
|
||||
b[15] = vshlq_n_s16(vsubq_s16(a[0], a[15]), 2);
|
||||
} else {
|
||||
b[0] = vaddq_s16(a[0], a[15]);
|
||||
b[1] = vaddq_s16(a[1], a[14]);
|
||||
b[2] = vaddq_s16(a[2], a[13]);
|
||||
b[3] = vaddq_s16(a[3], a[12]);
|
||||
b[4] = vaddq_s16(a[4], a[11]);
|
||||
b[5] = vaddq_s16(a[5], a[10]);
|
||||
b[6] = vaddq_s16(a[6], a[9]);
|
||||
b[7] = vaddq_s16(a[7], a[8]);
|
||||
|
||||
b[8] = vsubq_s16(a[7], a[8]);
|
||||
b[9] = vsubq_s16(a[6], a[9]);
|
||||
b[10] = vsubq_s16(a[5], a[10]);
|
||||
b[11] = vsubq_s16(a[4], a[11]);
|
||||
b[12] = vsubq_s16(a[3], a[12]);
|
||||
b[13] = vsubq_s16(a[2], a[13]);
|
||||
b[14] = vsubq_s16(a[1], a[14]);
|
||||
b[15] = vsubq_s16(a[0], a[15]);
|
||||
}
|
||||
}
|
||||
|
||||
// Quarter round at the beginning of the second pass. Can't use vrshr (rounding)
|
||||
// because this only adds 1, not 1 << 2.
|
||||
static INLINE void partial_round_shift(int16x8_t *a /*[16]*/) {
|
||||
const int16x8_t one = vdupq_n_s16(1);
|
||||
a[0] = vshrq_n_s16(vaddq_s16(a[0], one), 2);
|
||||
a[1] = vshrq_n_s16(vaddq_s16(a[1], one), 2);
|
||||
a[2] = vshrq_n_s16(vaddq_s16(a[2], one), 2);
|
||||
a[3] = vshrq_n_s16(vaddq_s16(a[3], one), 2);
|
||||
a[4] = vshrq_n_s16(vaddq_s16(a[4], one), 2);
|
||||
a[5] = vshrq_n_s16(vaddq_s16(a[5], one), 2);
|
||||
a[6] = vshrq_n_s16(vaddq_s16(a[6], one), 2);
|
||||
a[7] = vshrq_n_s16(vaddq_s16(a[7], one), 2);
|
||||
a[8] = vshrq_n_s16(vaddq_s16(a[8], one), 2);
|
||||
a[9] = vshrq_n_s16(vaddq_s16(a[9], one), 2);
|
||||
a[10] = vshrq_n_s16(vaddq_s16(a[10], one), 2);
|
||||
a[11] = vshrq_n_s16(vaddq_s16(a[11], one), 2);
|
||||
a[12] = vshrq_n_s16(vaddq_s16(a[12], one), 2);
|
||||
a[13] = vshrq_n_s16(vaddq_s16(a[13], one), 2);
|
||||
a[14] = vshrq_n_s16(vaddq_s16(a[14], one), 2);
|
||||
a[15] = vshrq_n_s16(vaddq_s16(a[15], one), 2);
|
||||
}
|
||||
|
||||
// fdct_round_shift((a +/- b) * c)
|
||||
static INLINE void butterfly_one_coeff(const int16x8_t a, const int16x8_t b,
|
||||
const tran_high_t c, int16x8_t *add,
|
||||
int16x8_t *sub) {
|
||||
const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c);
|
||||
const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c);
|
||||
const int32x4_t sum0 = vmlal_n_s16(a0, vget_low_s16(b), c);
|
||||
const int32x4_t sum1 = vmlal_n_s16(a1, vget_high_s16(b), c);
|
||||
const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c);
|
||||
const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c);
|
||||
const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14);
|
||||
const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14);
|
||||
const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14);
|
||||
const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14);
|
||||
*add = vcombine_s16(rounded0, rounded1);
|
||||
*sub = vcombine_s16(rounded2, rounded3);
|
||||
}
|
||||
|
||||
// fdct_round_shift(a * c0 +/- b * c1)
|
||||
static INLINE void butterfly_two_coeff(const int16x8_t a, const int16x8_t b,
|
||||
const tran_coef_t c0,
|
||||
const tran_coef_t c1, int16x8_t *add,
|
||||
int16x8_t *sub) {
|
||||
const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c0);
|
||||
const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c0);
|
||||
const int32x4_t a2 = vmull_n_s16(vget_low_s16(a), c1);
|
||||
const int32x4_t a3 = vmull_n_s16(vget_high_s16(a), c1);
|
||||
const int32x4_t sum0 = vmlal_n_s16(a2, vget_low_s16(b), c0);
|
||||
const int32x4_t sum1 = vmlal_n_s16(a3, vget_high_s16(b), c0);
|
||||
const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c1);
|
||||
const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c1);
|
||||
const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14);
|
||||
const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14);
|
||||
const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14);
|
||||
const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14);
|
||||
*add = vcombine_s16(rounded0, rounded1);
|
||||
*sub = vcombine_s16(rounded2, rounded3);
|
||||
}
|
||||
|
||||
// Transpose 8x8 to a new location. Don't use transpose_neon.h because those
|
||||
// are all in-place.
|
||||
static INLINE void transpose_8x8(const int16x8_t *a /*[8]*/,
|
||||
int16x8_t *b /*[8]*/) {
|
||||
// Swap 16 bit elements.
|
||||
const int16x8x2_t c0 = vtrnq_s16(a[0], a[1]);
|
||||
const int16x8x2_t c1 = vtrnq_s16(a[2], a[3]);
|
||||
const int16x8x2_t c2 = vtrnq_s16(a[4], a[5]);
|
||||
const int16x8x2_t c3 = vtrnq_s16(a[6], a[7]);
|
||||
|
||||
// Swap 32 bit elements.
|
||||
const int32x4x2_t d0 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[0]),
|
||||
vreinterpretq_s32_s16(c1.val[0]));
|
||||
const int32x4x2_t d1 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[1]),
|
||||
vreinterpretq_s32_s16(c1.val[1]));
|
||||
const int32x4x2_t d2 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[0]),
|
||||
vreinterpretq_s32_s16(c3.val[0]));
|
||||
const int32x4x2_t d3 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[1]),
|
||||
vreinterpretq_s32_s16(c3.val[1]));
|
||||
|
||||
// Swap 64 bit elements
|
||||
const int16x8x2_t e0 = vpx_vtrnq_s64_to_s16(d0.val[0], d2.val[0]);
|
||||
const int16x8x2_t e1 = vpx_vtrnq_s64_to_s16(d1.val[0], d3.val[0]);
|
||||
const int16x8x2_t e2 = vpx_vtrnq_s64_to_s16(d0.val[1], d2.val[1]);
|
||||
const int16x8x2_t e3 = vpx_vtrnq_s64_to_s16(d1.val[1], d3.val[1]);
|
||||
|
||||
b[0] = e0.val[0];
|
||||
b[1] = e1.val[0];
|
||||
b[2] = e2.val[0];
|
||||
b[3] = e3.val[0];
|
||||
b[4] = e0.val[1];
|
||||
b[5] = e1.val[1];
|
||||
b[6] = e2.val[1];
|
||||
b[7] = e3.val[1];
|
||||
}
|
||||
|
||||
// Main body of fdct16x16.
|
||||
static void vpx_fdct16x16_body(const int16x8_t *in /*[16]*/,
|
||||
int16x8_t *out /*[16]*/) {
|
||||
int16x8_t s[8];
|
||||
int16x8_t x[4];
|
||||
int16x8_t step[8];
|
||||
|
||||
// stage 1
|
||||
// From fwd_txfm.c: Work on the first eight values; fdct8(input,
|
||||
// even_results);"
|
||||
s[0] = vaddq_s16(in[0], in[7]);
|
||||
s[1] = vaddq_s16(in[1], in[6]);
|
||||
s[2] = vaddq_s16(in[2], in[5]);
|
||||
s[3] = vaddq_s16(in[3], in[4]);
|
||||
s[4] = vsubq_s16(in[3], in[4]);
|
||||
s[5] = vsubq_s16(in[2], in[5]);
|
||||
s[6] = vsubq_s16(in[1], in[6]);
|
||||
s[7] = vsubq_s16(in[0], in[7]);
|
||||
|
||||
// fdct4(step, step);
|
||||
x[0] = vaddq_s16(s[0], s[3]);
|
||||
x[1] = vaddq_s16(s[1], s[2]);
|
||||
x[2] = vsubq_s16(s[1], s[2]);
|
||||
x[3] = vsubq_s16(s[0], s[3]);
|
||||
|
||||
// out[0] = fdct_round_shift((x0 + x1) * cospi_16_64)
|
||||
// out[8] = fdct_round_shift((x0 - x1) * cospi_16_64)
|
||||
butterfly_one_coeff(x[0], x[1], cospi_16_64, &out[0], &out[8]);
|
||||
// out[4] = fdct_round_shift(x3 * cospi_8_64 + x2 * cospi_24_64);
|
||||
// out[12] = fdct_round_shift(x3 * cospi_24_64 - x2 * cospi_8_64);
|
||||
butterfly_two_coeff(x[3], x[2], cospi_24_64, cospi_8_64, &out[4], &out[12]);
|
||||
|
||||
// Stage 2
|
||||
// Re-using source s5/s6
|
||||
// s5 = fdct_round_shift((s6 - s5) * cospi_16_64)
|
||||
// s6 = fdct_round_shift((s6 + s5) * cospi_16_64)
|
||||
butterfly_one_coeff(s[6], s[5], cospi_16_64, &s[6], &s[5]);
|
||||
|
||||
// Stage 3
|
||||
x[0] = vaddq_s16(s[4], s[5]);
|
||||
x[1] = vsubq_s16(s[4], s[5]);
|
||||
x[2] = vsubq_s16(s[7], s[6]);
|
||||
x[3] = vaddq_s16(s[7], s[6]);
|
||||
|
||||
// Stage 4
|
||||
// out[2] = fdct_round_shift(x0 * cospi_28_64 + x3 * cospi_4_64)
|
||||
// out[14] = fdct_round_shift(x3 * cospi_28_64 + x0 * -cospi_4_64)
|
||||
butterfly_two_coeff(x[3], x[0], cospi_28_64, cospi_4_64, &out[2], &out[14]);
|
||||
// out[6] = fdct_round_shift(x1 * cospi_12_64 + x2 * cospi_20_64)
|
||||
// out[10] = fdct_round_shift(x2 * cospi_12_64 + x1 * -cospi_20_64)
|
||||
butterfly_two_coeff(x[2], x[1], cospi_12_64, cospi_20_64, &out[10], &out[6]);
|
||||
|
||||
// step 2
|
||||
// From fwd_txfm.c: Work on the next eight values; step1 -> odd_results"
|
||||
// That file distinguished between "in_high" and "step1" but the only
|
||||
// difference is that "in_high" is the first 8 values and "step 1" is the
|
||||
// second. Here, since they are all in one array, "step1" values are += 8.
|
||||
|
||||
// step2[2] = fdct_round_shift((step1[5] - step1[2]) * cospi_16_64)
|
||||
// step2[3] = fdct_round_shift((step1[4] - step1[3]) * cospi_16_64)
|
||||
// step2[4] = fdct_round_shift((step1[4] + step1[3]) * cospi_16_64)
|
||||
// step2[5] = fdct_round_shift((step1[5] + step1[2]) * cospi_16_64)
|
||||
butterfly_one_coeff(in[13], in[10], cospi_16_64, &s[5], &s[2]);
|
||||
butterfly_one_coeff(in[12], in[11], cospi_16_64, &s[4], &s[3]);
|
||||
|
||||
// step 3
|
||||
s[0] = vaddq_s16(in[8], s[3]);
|
||||
s[1] = vaddq_s16(in[9], s[2]);
|
||||
x[0] = vsubq_s16(in[9], s[2]);
|
||||
x[1] = vsubq_s16(in[8], s[3]);
|
||||
x[2] = vsubq_s16(in[15], s[4]);
|
||||
x[3] = vsubq_s16(in[14], s[5]);
|
||||
s[6] = vaddq_s16(in[14], s[5]);
|
||||
s[7] = vaddq_s16(in[15], s[4]);
|
||||
|
||||
// step 4
|
||||
// step2[1] = fdct_round_shift(step3[1] *-cospi_8_64 + step3[6] * cospi_24_64)
|
||||
// step2[6] = fdct_round_shift(step3[1] * cospi_24_64 + step3[6] * cospi_8_64)
|
||||
butterfly_two_coeff(s[6], s[1], cospi_24_64, cospi_8_64, &s[6], &s[1]);
|
||||
|
||||
// step2[2] = fdct_round_shift(step3[2] * cospi_24_64 + step3[5] * cospi_8_64)
|
||||
// step2[5] = fdct_round_shift(step3[2] * cospi_8_64 - step3[5] * cospi_24_64)
|
||||
butterfly_two_coeff(x[0], x[3], cospi_8_64, cospi_24_64, &s[2], &s[5]);
|
||||
|
||||
// step 5
|
||||
step[0] = vaddq_s16(s[0], s[1]);
|
||||
step[1] = vsubq_s16(s[0], s[1]);
|
||||
step[2] = vaddq_s16(x[1], s[2]);
|
||||
step[3] = vsubq_s16(x[1], s[2]);
|
||||
step[4] = vsubq_s16(x[2], s[5]);
|
||||
step[5] = vaddq_s16(x[2], s[5]);
|
||||
step[6] = vsubq_s16(s[7], s[6]);
|
||||
step[7] = vaddq_s16(s[7], s[6]);
|
||||
|
||||
// step 6
|
||||
// out[1] = fdct_round_shift(step1[0] * cospi_30_64 + step1[7] * cospi_2_64)
|
||||
// out[9] = fdct_round_shift(step1[1] * cospi_14_64 + step1[6] * cospi_18_64)
|
||||
// out[5] = fdct_round_shift(step1[2] * cospi_22_64 + step1[5] * cospi_10_64)
|
||||
// out[13] = fdct_round_shift(step1[3] * cospi_6_64 + step1[4] * cospi_26_64)
|
||||
// out[3] = fdct_round_shift(step1[3] * -cospi_26_64 + step1[4] * cospi_6_64)
|
||||
// out[11] = fdct_round_shift(step1[2] * -cospi_10_64 + step1[5] *
|
||||
// cospi_22_64)
|
||||
// out[7] = fdct_round_shift(step1[1] * -cospi_18_64 + step1[6] * cospi_14_64)
|
||||
// out[15] = fdct_round_shift(step1[0] * -cospi_2_64 + step1[7] * cospi_30_64)
|
||||
butterfly_two_coeff(step[6], step[1], cospi_14_64, cospi_18_64, &out[9],
|
||||
&out[7]);
|
||||
butterfly_two_coeff(step[7], step[0], cospi_30_64, cospi_2_64, &out[1],
|
||||
&out[15]);
|
||||
butterfly_two_coeff(step[4], step[3], cospi_6_64, cospi_26_64, &out[13],
|
||||
&out[3]);
|
||||
butterfly_two_coeff(step[5], step[2], cospi_22_64, cospi_10_64, &out[5],
|
||||
&out[11]);
|
||||
}
|
||||
|
||||
#endif // VPX_VPX_DSP_ARM_FDCT16X16_NEON_H_
|
||||
@@ -153,7 +153,7 @@ static INLINE void load(const int16_t *a, int stride, int16x8_t *b) {
|
||||
do { \
|
||||
store_s16q_to_tran_low(dest, src[index]); \
|
||||
dest += 8; \
|
||||
} while (0);
|
||||
} while (0)
|
||||
|
||||
// Store 32 16x8 values, assuming stride == 32.
|
||||
// Slight twist: store horizontally in blocks of 8.
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
#include "vpx_dsp/vpx_dsp_common.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/fdct_neon.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
|
||||
@@ -22,67 +23,25 @@ void vpx_fdct4x4_neon(const int16_t *input, tran_low_t *final_output,
|
||||
int stride) {
|
||||
int i;
|
||||
// input[M * stride] * 16
|
||||
int16x4_t input_0 = vshl_n_s16(vld1_s16(input + 0 * stride), 4);
|
||||
int16x4_t input_1 = vshl_n_s16(vld1_s16(input + 1 * stride), 4);
|
||||
int16x4_t input_2 = vshl_n_s16(vld1_s16(input + 2 * stride), 4);
|
||||
int16x4_t input_3 = vshl_n_s16(vld1_s16(input + 3 * stride), 4);
|
||||
int16x4_t in[4];
|
||||
in[0] = vshl_n_s16(vld1_s16(input + 0 * stride), 4);
|
||||
in[1] = vshl_n_s16(vld1_s16(input + 1 * stride), 4);
|
||||
in[2] = vshl_n_s16(vld1_s16(input + 2 * stride), 4);
|
||||
in[3] = vshl_n_s16(vld1_s16(input + 3 * stride), 4);
|
||||
|
||||
// If the very first value != 0, then add 1.
|
||||
if (input[0] != 0) {
|
||||
const int16x4_t one = vreinterpret_s16_s64(vdup_n_s64(1));
|
||||
input_0 = vadd_s16(input_0, one);
|
||||
in[0] = vadd_s16(in[0], one);
|
||||
}
|
||||
|
||||
for (i = 0; i < 2; ++i) {
|
||||
const int16x8_t input_01 = vcombine_s16(input_0, input_1);
|
||||
const int16x8_t input_32 = vcombine_s16(input_3, input_2);
|
||||
|
||||
// in_0 +/- in_3, in_1 +/- in_2
|
||||
const int16x8_t s_01 = vaddq_s16(input_01, input_32);
|
||||
const int16x8_t s_32 = vsubq_s16(input_01, input_32);
|
||||
|
||||
// step_0 +/- step_1, step_2 +/- step_3
|
||||
const int16x4_t s_0 = vget_low_s16(s_01);
|
||||
const int16x4_t s_1 = vget_high_s16(s_01);
|
||||
const int16x4_t s_2 = vget_high_s16(s_32);
|
||||
const int16x4_t s_3 = vget_low_s16(s_32);
|
||||
|
||||
// (s_0 +/- s_1) * cospi_16_64
|
||||
// Must expand all elements to s32. See 'needs32' comment in fwd_txfm.c.
|
||||
const int32x4_t s_0_p_s_1 = vaddl_s16(s_0, s_1);
|
||||
const int32x4_t s_0_m_s_1 = vsubl_s16(s_0, s_1);
|
||||
const int32x4_t temp1 = vmulq_n_s32(s_0_p_s_1, cospi_16_64);
|
||||
const int32x4_t temp2 = vmulq_n_s32(s_0_m_s_1, cospi_16_64);
|
||||
|
||||
// fdct_round_shift
|
||||
int16x4_t out_0 = vrshrn_n_s32(temp1, DCT_CONST_BITS);
|
||||
int16x4_t out_2 = vrshrn_n_s32(temp2, DCT_CONST_BITS);
|
||||
|
||||
// s_3 * cospi_8_64 + s_2 * cospi_24_64
|
||||
// s_3 * cospi_24_64 - s_2 * cospi_8_64
|
||||
const int32x4_t s_3_cospi_8_64 = vmull_n_s16(s_3, cospi_8_64);
|
||||
const int32x4_t s_3_cospi_24_64 = vmull_n_s16(s_3, cospi_24_64);
|
||||
|
||||
const int32x4_t temp3 = vmlal_n_s16(s_3_cospi_8_64, s_2, cospi_24_64);
|
||||
const int32x4_t temp4 = vmlsl_n_s16(s_3_cospi_24_64, s_2, cospi_8_64);
|
||||
|
||||
// fdct_round_shift
|
||||
int16x4_t out_1 = vrshrn_n_s32(temp3, DCT_CONST_BITS);
|
||||
int16x4_t out_3 = vrshrn_n_s32(temp4, DCT_CONST_BITS);
|
||||
|
||||
transpose_s16_4x4d(&out_0, &out_1, &out_2, &out_3);
|
||||
|
||||
input_0 = out_0;
|
||||
input_1 = out_1;
|
||||
input_2 = out_2;
|
||||
input_3 = out_3;
|
||||
vpx_fdct4x4_pass1_neon(in);
|
||||
}
|
||||
|
||||
{
|
||||
// Not quite a rounding shift. Only add 1 despite shifting by 2.
|
||||
const int16x8_t one = vdupq_n_s16(1);
|
||||
int16x8_t out_01 = vcombine_s16(input_0, input_1);
|
||||
int16x8_t out_23 = vcombine_s16(input_2, input_3);
|
||||
int16x8_t out_01 = vcombine_s16(in[0], in[1]);
|
||||
int16x8_t out_23 = vcombine_s16(in[2], in[3]);
|
||||
out_01 = vshrq_n_s16(vaddq_s16(out_01, one), 2);
|
||||
out_23 = vshrq_n_s16(vaddq_s16(out_23, one), 2);
|
||||
store_s16q_to_tran_low(final_output + 0 * 8, out_01);
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
/*
|
||||
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef VPX_VPX_DSP_ARM_FDCT_NEON_H_
|
||||
#define VPX_VPX_DSP_ARM_FDCT_NEON_H_
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
static INLINE void vpx_fdct4x4_pass1_neon(int16x4_t *in) {
|
||||
const int16x8_t input_01 = vcombine_s16(in[0], in[1]);
|
||||
const int16x8_t input_32 = vcombine_s16(in[3], in[2]);
|
||||
|
||||
// in_0 +/- in_3, in_1 +/- in_2
|
||||
const int16x8_t s_01 = vaddq_s16(input_01, input_32);
|
||||
const int16x8_t s_32 = vsubq_s16(input_01, input_32);
|
||||
|
||||
// step_0 +/- step_1, step_2 +/- step_3
|
||||
const int16x4_t s_0 = vget_low_s16(s_01);
|
||||
const int16x4_t s_1 = vget_high_s16(s_01);
|
||||
const int16x4_t s_2 = vget_high_s16(s_32);
|
||||
const int16x4_t s_3 = vget_low_s16(s_32);
|
||||
|
||||
// (s_0 +/- s_1) * cospi_16_64
|
||||
// Must expand all elements to s32. See 'needs32' comment in fwd_txfm.c.
|
||||
const int32x4_t s_0_p_s_1 = vaddl_s16(s_0, s_1);
|
||||
const int32x4_t s_0_m_s_1 = vsubl_s16(s_0, s_1);
|
||||
const int32x4_t temp1 = vmulq_n_s32(s_0_p_s_1, cospi_16_64);
|
||||
const int32x4_t temp2 = vmulq_n_s32(s_0_m_s_1, cospi_16_64);
|
||||
|
||||
// fdct_round_shift
|
||||
int16x4_t out_0 = vrshrn_n_s32(temp1, DCT_CONST_BITS);
|
||||
int16x4_t out_2 = vrshrn_n_s32(temp2, DCT_CONST_BITS);
|
||||
|
||||
// s_3 * cospi_8_64 + s_2 * cospi_24_64
|
||||
// s_3 * cospi_24_64 - s_2 * cospi_8_64
|
||||
const int32x4_t s_3_cospi_8_64 = vmull_n_s16(s_3, cospi_8_64);
|
||||
const int32x4_t s_3_cospi_24_64 = vmull_n_s16(s_3, cospi_24_64);
|
||||
|
||||
const int32x4_t temp3 = vmlal_n_s16(s_3_cospi_8_64, s_2, cospi_24_64);
|
||||
const int32x4_t temp4 = vmlsl_n_s16(s_3_cospi_24_64, s_2, cospi_8_64);
|
||||
|
||||
// fdct_round_shift
|
||||
int16x4_t out_1 = vrshrn_n_s32(temp3, DCT_CONST_BITS);
|
||||
int16x4_t out_3 = vrshrn_n_s32(temp4, DCT_CONST_BITS);
|
||||
|
||||
transpose_s16_4x4d(&out_0, &out_1, &out_2, &out_3);
|
||||
|
||||
in[0] = out_0;
|
||||
in[1] = out_1;
|
||||
in[2] = out_2;
|
||||
in[3] = out_3;
|
||||
}
|
||||
|
||||
static INLINE void vpx_fdct8x8_pass1_notranspose_neon(int16x8_t *in,
|
||||
int16x8_t *out) {
|
||||
const int16x8_t v_s0 = vaddq_s16(in[0], in[7]);
|
||||
const int16x8_t v_s1 = vaddq_s16(in[1], in[6]);
|
||||
const int16x8_t v_s2 = vaddq_s16(in[2], in[5]);
|
||||
const int16x8_t v_s3 = vaddq_s16(in[3], in[4]);
|
||||
const int16x8_t v_s4 = vsubq_s16(in[3], in[4]);
|
||||
const int16x8_t v_s5 = vsubq_s16(in[2], in[5]);
|
||||
const int16x8_t v_s6 = vsubq_s16(in[1], in[6]);
|
||||
const int16x8_t v_s7 = vsubq_s16(in[0], in[7]);
|
||||
// fdct4(step, step);
|
||||
int16x8_t v_x0 = vaddq_s16(v_s0, v_s3);
|
||||
int16x8_t v_x1 = vaddq_s16(v_s1, v_s2);
|
||||
int16x8_t v_x2 = vsubq_s16(v_s1, v_s2);
|
||||
int16x8_t v_x3 = vsubq_s16(v_s0, v_s3);
|
||||
// fdct4(step, step);
|
||||
int32x4_t v_t0_lo = vaddl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1));
|
||||
int32x4_t v_t0_hi = vaddl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1));
|
||||
int32x4_t v_t1_lo = vsubl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1));
|
||||
int32x4_t v_t1_hi = vsubl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1));
|
||||
int32x4_t v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_24_64);
|
||||
int32x4_t v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_24_64);
|
||||
int32x4_t v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_24_64);
|
||||
int32x4_t v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_24_64);
|
||||
v_t2_lo = vmlal_n_s16(v_t2_lo, vget_low_s16(v_x3), cospi_8_64);
|
||||
v_t2_hi = vmlal_n_s16(v_t2_hi, vget_high_s16(v_x3), cospi_8_64);
|
||||
v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x2), cospi_8_64);
|
||||
v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x2), cospi_8_64);
|
||||
v_t0_lo = vmulq_n_s32(v_t0_lo, cospi_16_64);
|
||||
v_t0_hi = vmulq_n_s32(v_t0_hi, cospi_16_64);
|
||||
v_t1_lo = vmulq_n_s32(v_t1_lo, cospi_16_64);
|
||||
v_t1_hi = vmulq_n_s32(v_t1_hi, cospi_16_64);
|
||||
{
|
||||
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
|
||||
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
|
||||
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
|
||||
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
|
||||
const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS);
|
||||
const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS);
|
||||
const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS);
|
||||
const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS);
|
||||
out[0] = vcombine_s16(a, c); // 00 01 02 03 40 41 42 43
|
||||
out[2] = vcombine_s16(e, g); // 20 21 22 23 60 61 62 63
|
||||
out[4] = vcombine_s16(b, d); // 04 05 06 07 44 45 46 47
|
||||
out[6] = vcombine_s16(f, h); // 24 25 26 27 64 65 66 67
|
||||
}
|
||||
// Stage 2
|
||||
v_x0 = vsubq_s16(v_s6, v_s5);
|
||||
v_x1 = vaddq_s16(v_s6, v_s5);
|
||||
v_t0_lo = vmull_n_s16(vget_low_s16(v_x0), cospi_16_64);
|
||||
v_t0_hi = vmull_n_s16(vget_high_s16(v_x0), cospi_16_64);
|
||||
v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_16_64);
|
||||
v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_16_64);
|
||||
{
|
||||
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
|
||||
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
|
||||
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
|
||||
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
|
||||
const int16x8_t ab = vcombine_s16(a, b);
|
||||
const int16x8_t cd = vcombine_s16(c, d);
|
||||
// Stage 3
|
||||
v_x0 = vaddq_s16(v_s4, ab);
|
||||
v_x1 = vsubq_s16(v_s4, ab);
|
||||
v_x2 = vsubq_s16(v_s7, cd);
|
||||
v_x3 = vaddq_s16(v_s7, cd);
|
||||
}
|
||||
// Stage 4
|
||||
v_t0_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_4_64);
|
||||
v_t0_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_4_64);
|
||||
v_t0_lo = vmlal_n_s16(v_t0_lo, vget_low_s16(v_x0), cospi_28_64);
|
||||
v_t0_hi = vmlal_n_s16(v_t0_hi, vget_high_s16(v_x0), cospi_28_64);
|
||||
v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_12_64);
|
||||
v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_12_64);
|
||||
v_t1_lo = vmlal_n_s16(v_t1_lo, vget_low_s16(v_x2), cospi_20_64);
|
||||
v_t1_hi = vmlal_n_s16(v_t1_hi, vget_high_s16(v_x2), cospi_20_64);
|
||||
v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_12_64);
|
||||
v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_12_64);
|
||||
v_t2_lo = vmlsl_n_s16(v_t2_lo, vget_low_s16(v_x1), cospi_20_64);
|
||||
v_t2_hi = vmlsl_n_s16(v_t2_hi, vget_high_s16(v_x1), cospi_20_64);
|
||||
v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_28_64);
|
||||
v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_28_64);
|
||||
v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x0), cospi_4_64);
|
||||
v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x0), cospi_4_64);
|
||||
{
|
||||
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
|
||||
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
|
||||
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
|
||||
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
|
||||
const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS);
|
||||
const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS);
|
||||
const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS);
|
||||
const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS);
|
||||
out[1] = vcombine_s16(a, c); // 10 11 12 13 50 51 52 53
|
||||
out[3] = vcombine_s16(e, g); // 30 31 32 33 70 71 72 73
|
||||
out[5] = vcombine_s16(b, d); // 14 15 16 17 54 55 56 57
|
||||
out[7] = vcombine_s16(f, h); // 34 35 36 37 74 75 76 77
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void vpx_fdct8x8_pass1_neon(int16x8_t *in) {
|
||||
int16x8_t out[8];
|
||||
vpx_fdct8x8_pass1_notranspose_neon(in, out);
|
||||
// transpose 8x8
|
||||
// Can't use transpose_s16_8x8() because the values are arranged in two 4x8
|
||||
// columns.
|
||||
{
|
||||
// 00 01 02 03 40 41 42 43
|
||||
// 10 11 12 13 50 51 52 53
|
||||
// 20 21 22 23 60 61 62 63
|
||||
// 30 31 32 33 70 71 72 73
|
||||
// 04 05 06 07 44 45 46 47
|
||||
// 14 15 16 17 54 55 56 57
|
||||
// 24 25 26 27 64 65 66 67
|
||||
// 34 35 36 37 74 75 76 77
|
||||
const int32x4x2_t r02_s32 =
|
||||
vtrnq_s32(vreinterpretq_s32_s16(out[0]), vreinterpretq_s32_s16(out[2]));
|
||||
const int32x4x2_t r13_s32 =
|
||||
vtrnq_s32(vreinterpretq_s32_s16(out[1]), vreinterpretq_s32_s16(out[3]));
|
||||
const int32x4x2_t r46_s32 =
|
||||
vtrnq_s32(vreinterpretq_s32_s16(out[4]), vreinterpretq_s32_s16(out[6]));
|
||||
const int32x4x2_t r57_s32 =
|
||||
vtrnq_s32(vreinterpretq_s32_s16(out[5]), vreinterpretq_s32_s16(out[7]));
|
||||
const int16x8x2_t r01_s16 =
|
||||
vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[0]),
|
||||
vreinterpretq_s16_s32(r13_s32.val[0]));
|
||||
const int16x8x2_t r23_s16 =
|
||||
vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[1]),
|
||||
vreinterpretq_s16_s32(r13_s32.val[1]));
|
||||
const int16x8x2_t r45_s16 =
|
||||
vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[0]),
|
||||
vreinterpretq_s16_s32(r57_s32.val[0]));
|
||||
const int16x8x2_t r67_s16 =
|
||||
vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[1]),
|
||||
vreinterpretq_s16_s32(r57_s32.val[1]));
|
||||
in[0] = r01_s16.val[0];
|
||||
in[1] = r01_s16.val[1];
|
||||
in[2] = r23_s16.val[0];
|
||||
in[3] = r23_s16.val[1];
|
||||
in[4] = r45_s16.val[0];
|
||||
in[5] = r45_s16.val[1];
|
||||
in[6] = r67_s16.val[0];
|
||||
in[7] = r67_s16.val[1];
|
||||
// 00 10 20 30 40 50 60 70
|
||||
// 01 11 21 31 41 51 61 71
|
||||
// 02 12 22 32 42 52 62 72
|
||||
// 03 13 23 33 43 53 63 73
|
||||
// 04 14 24 34 44 54 64 74
|
||||
// 05 15 25 35 45 55 65 75
|
||||
// 06 16 26 36 46 56 66 76
|
||||
// 07 17 27 37 47 57 67 77
|
||||
}
|
||||
}
|
||||
#endif // VPX_VPX_DSP_ARM_FDCT_NEON_H_
|
||||
@@ -15,19 +15,10 @@
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/sum_neon.h"
|
||||
|
||||
static INLINE tran_low_t get_lane(const int32x2_t a) {
|
||||
#if CONFIG_VP9_HIGHBITDEPTH
|
||||
return vget_lane_s32(a, 0);
|
||||
#else
|
||||
return vget_lane_s16(vreinterpret_s16_s32(a), 0);
|
||||
#endif // CONFIG_VP9_HIGHBITDETPH
|
||||
}
|
||||
|
||||
void vpx_fdct4x4_1_neon(const int16_t *input, tran_low_t *output, int stride) {
|
||||
int16x4_t a0, a1, a2, a3;
|
||||
int16x8_t b0, b1;
|
||||
int16x8_t c;
|
||||
int32x2_t d;
|
||||
|
||||
a0 = vld1_s16(input);
|
||||
input += stride;
|
||||
@@ -42,9 +33,7 @@ void vpx_fdct4x4_1_neon(const int16_t *input, tran_low_t *output, int stride) {
|
||||
|
||||
c = vaddq_s16(b0, b1);
|
||||
|
||||
d = horizontal_add_int16x8(c);
|
||||
|
||||
output[0] = get_lane(vshl_n_s32(d, 1));
|
||||
output[0] = (tran_low_t)(horizontal_add_int16x8(c) << 1);
|
||||
output[1] = 0;
|
||||
}
|
||||
|
||||
@@ -57,7 +46,7 @@ void vpx_fdct8x8_1_neon(const int16_t *input, tran_low_t *output, int stride) {
|
||||
sum = vaddq_s16(sum, input_00);
|
||||
}
|
||||
|
||||
output[0] = get_lane(horizontal_add_int16x8(sum));
|
||||
output[0] = (tran_low_t)horizontal_add_int16x8(sum);
|
||||
output[1] = 0;
|
||||
}
|
||||
|
||||
@@ -66,7 +55,7 @@ void vpx_fdct16x16_1_neon(const int16_t *input, tran_low_t *output,
|
||||
int r;
|
||||
int16x8_t left = vld1q_s16(input);
|
||||
int16x8_t right = vld1q_s16(input + 8);
|
||||
int32x2_t sum;
|
||||
int32_t sum;
|
||||
input += stride;
|
||||
|
||||
for (r = 1; r < 16; ++r) {
|
||||
@@ -77,9 +66,9 @@ void vpx_fdct16x16_1_neon(const int16_t *input, tran_low_t *output,
|
||||
right = vaddq_s16(right, b);
|
||||
}
|
||||
|
||||
sum = vadd_s32(horizontal_add_int16x8(left), horizontal_add_int16x8(right));
|
||||
sum = horizontal_add_int16x8(left) + horizontal_add_int16x8(right);
|
||||
|
||||
output[0] = get_lane(vshr_n_s32(sum, 1));
|
||||
output[0] = (tran_low_t)(sum >> 1);
|
||||
output[1] = 0;
|
||||
}
|
||||
|
||||
@@ -90,7 +79,7 @@ void vpx_fdct32x32_1_neon(const int16_t *input, tran_low_t *output,
|
||||
int16x8_t a1 = vld1q_s16(input + 8);
|
||||
int16x8_t a2 = vld1q_s16(input + 16);
|
||||
int16x8_t a3 = vld1q_s16(input + 24);
|
||||
int32x2_t sum;
|
||||
int32_t sum;
|
||||
input += stride;
|
||||
|
||||
for (r = 1; r < 32; ++r) {
|
||||
@@ -105,9 +94,10 @@ void vpx_fdct32x32_1_neon(const int16_t *input, tran_low_t *output,
|
||||
a3 = vaddq_s16(a3, b3);
|
||||
}
|
||||
|
||||
sum = vadd_s32(horizontal_add_int16x8(a0), horizontal_add_int16x8(a1));
|
||||
sum = vadd_s32(sum, horizontal_add_int16x8(a2));
|
||||
sum = vadd_s32(sum, horizontal_add_int16x8(a3));
|
||||
output[0] = get_lane(vshr_n_s32(sum, 3));
|
||||
sum = horizontal_add_int16x8(a0);
|
||||
sum += horizontal_add_int16x8(a1);
|
||||
sum += horizontal_add_int16x8(a2);
|
||||
sum += horizontal_add_int16x8(a3);
|
||||
output[0] = (tran_low_t)(sum >> 3);
|
||||
output[1] = 0;
|
||||
}
|
||||
@@ -15,196 +15,54 @@
|
||||
#include "vpx_dsp/txfm_common.h"
|
||||
#include "vpx_dsp/vpx_dsp_common.h"
|
||||
#include "vpx_dsp/arm/idct_neon.h"
|
||||
#include "vpx_dsp/arm/fdct_neon.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
|
||||
void vpx_fdct8x8_neon(const int16_t *input, tran_low_t *final_output,
|
||||
int stride) {
|
||||
int i;
|
||||
// stage 1
|
||||
int16x8_t input_0 = vshlq_n_s16(vld1q_s16(&input[0 * stride]), 2);
|
||||
int16x8_t input_1 = vshlq_n_s16(vld1q_s16(&input[1 * stride]), 2);
|
||||
int16x8_t input_2 = vshlq_n_s16(vld1q_s16(&input[2 * stride]), 2);
|
||||
int16x8_t input_3 = vshlq_n_s16(vld1q_s16(&input[3 * stride]), 2);
|
||||
int16x8_t input_4 = vshlq_n_s16(vld1q_s16(&input[4 * stride]), 2);
|
||||
int16x8_t input_5 = vshlq_n_s16(vld1q_s16(&input[5 * stride]), 2);
|
||||
int16x8_t input_6 = vshlq_n_s16(vld1q_s16(&input[6 * stride]), 2);
|
||||
int16x8_t input_7 = vshlq_n_s16(vld1q_s16(&input[7 * stride]), 2);
|
||||
int16x8_t in[8];
|
||||
in[0] = vshlq_n_s16(vld1q_s16(&input[0 * stride]), 2);
|
||||
in[1] = vshlq_n_s16(vld1q_s16(&input[1 * stride]), 2);
|
||||
in[2] = vshlq_n_s16(vld1q_s16(&input[2 * stride]), 2);
|
||||
in[3] = vshlq_n_s16(vld1q_s16(&input[3 * stride]), 2);
|
||||
in[4] = vshlq_n_s16(vld1q_s16(&input[4 * stride]), 2);
|
||||
in[5] = vshlq_n_s16(vld1q_s16(&input[5 * stride]), 2);
|
||||
in[6] = vshlq_n_s16(vld1q_s16(&input[6 * stride]), 2);
|
||||
in[7] = vshlq_n_s16(vld1q_s16(&input[7 * stride]), 2);
|
||||
for (i = 0; i < 2; ++i) {
|
||||
int16x8_t out_0, out_1, out_2, out_3, out_4, out_5, out_6, out_7;
|
||||
const int16x8_t v_s0 = vaddq_s16(input_0, input_7);
|
||||
const int16x8_t v_s1 = vaddq_s16(input_1, input_6);
|
||||
const int16x8_t v_s2 = vaddq_s16(input_2, input_5);
|
||||
const int16x8_t v_s3 = vaddq_s16(input_3, input_4);
|
||||
const int16x8_t v_s4 = vsubq_s16(input_3, input_4);
|
||||
const int16x8_t v_s5 = vsubq_s16(input_2, input_5);
|
||||
const int16x8_t v_s6 = vsubq_s16(input_1, input_6);
|
||||
const int16x8_t v_s7 = vsubq_s16(input_0, input_7);
|
||||
// fdct4(step, step);
|
||||
int16x8_t v_x0 = vaddq_s16(v_s0, v_s3);
|
||||
int16x8_t v_x1 = vaddq_s16(v_s1, v_s2);
|
||||
int16x8_t v_x2 = vsubq_s16(v_s1, v_s2);
|
||||
int16x8_t v_x3 = vsubq_s16(v_s0, v_s3);
|
||||
// fdct4(step, step);
|
||||
int32x4_t v_t0_lo = vaddl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1));
|
||||
int32x4_t v_t0_hi = vaddl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1));
|
||||
int32x4_t v_t1_lo = vsubl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1));
|
||||
int32x4_t v_t1_hi = vsubl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1));
|
||||
int32x4_t v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_24_64);
|
||||
int32x4_t v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_24_64);
|
||||
int32x4_t v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_24_64);
|
||||
int32x4_t v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_24_64);
|
||||
v_t2_lo = vmlal_n_s16(v_t2_lo, vget_low_s16(v_x3), cospi_8_64);
|
||||
v_t2_hi = vmlal_n_s16(v_t2_hi, vget_high_s16(v_x3), cospi_8_64);
|
||||
v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x2), cospi_8_64);
|
||||
v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x2), cospi_8_64);
|
||||
v_t0_lo = vmulq_n_s32(v_t0_lo, cospi_16_64);
|
||||
v_t0_hi = vmulq_n_s32(v_t0_hi, cospi_16_64);
|
||||
v_t1_lo = vmulq_n_s32(v_t1_lo, cospi_16_64);
|
||||
v_t1_hi = vmulq_n_s32(v_t1_hi, cospi_16_64);
|
||||
{
|
||||
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
|
||||
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
|
||||
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
|
||||
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
|
||||
const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS);
|
||||
const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS);
|
||||
const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS);
|
||||
const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS);
|
||||
out_0 = vcombine_s16(a, c); // 00 01 02 03 40 41 42 43
|
||||
out_2 = vcombine_s16(e, g); // 20 21 22 23 60 61 62 63
|
||||
out_4 = vcombine_s16(b, d); // 04 05 06 07 44 45 46 47
|
||||
out_6 = vcombine_s16(f, h); // 24 25 26 27 64 65 66 67
|
||||
}
|
||||
// Stage 2
|
||||
v_x0 = vsubq_s16(v_s6, v_s5);
|
||||
v_x1 = vaddq_s16(v_s6, v_s5);
|
||||
v_t0_lo = vmull_n_s16(vget_low_s16(v_x0), cospi_16_64);
|
||||
v_t0_hi = vmull_n_s16(vget_high_s16(v_x0), cospi_16_64);
|
||||
v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_16_64);
|
||||
v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_16_64);
|
||||
{
|
||||
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
|
||||
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
|
||||
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
|
||||
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
|
||||
const int16x8_t ab = vcombine_s16(a, b);
|
||||
const int16x8_t cd = vcombine_s16(c, d);
|
||||
// Stage 3
|
||||
v_x0 = vaddq_s16(v_s4, ab);
|
||||
v_x1 = vsubq_s16(v_s4, ab);
|
||||
v_x2 = vsubq_s16(v_s7, cd);
|
||||
v_x3 = vaddq_s16(v_s7, cd);
|
||||
}
|
||||
// Stage 4
|
||||
v_t0_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_4_64);
|
||||
v_t0_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_4_64);
|
||||
v_t0_lo = vmlal_n_s16(v_t0_lo, vget_low_s16(v_x0), cospi_28_64);
|
||||
v_t0_hi = vmlal_n_s16(v_t0_hi, vget_high_s16(v_x0), cospi_28_64);
|
||||
v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_12_64);
|
||||
v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_12_64);
|
||||
v_t1_lo = vmlal_n_s16(v_t1_lo, vget_low_s16(v_x2), cospi_20_64);
|
||||
v_t1_hi = vmlal_n_s16(v_t1_hi, vget_high_s16(v_x2), cospi_20_64);
|
||||
v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_12_64);
|
||||
v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_12_64);
|
||||
v_t2_lo = vmlsl_n_s16(v_t2_lo, vget_low_s16(v_x1), cospi_20_64);
|
||||
v_t2_hi = vmlsl_n_s16(v_t2_hi, vget_high_s16(v_x1), cospi_20_64);
|
||||
v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_28_64);
|
||||
v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_28_64);
|
||||
v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x0), cospi_4_64);
|
||||
v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x0), cospi_4_64);
|
||||
{
|
||||
const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS);
|
||||
const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS);
|
||||
const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS);
|
||||
const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS);
|
||||
const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS);
|
||||
const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS);
|
||||
const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS);
|
||||
const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS);
|
||||
out_1 = vcombine_s16(a, c); // 10 11 12 13 50 51 52 53
|
||||
out_3 = vcombine_s16(e, g); // 30 31 32 33 70 71 72 73
|
||||
out_5 = vcombine_s16(b, d); // 14 15 16 17 54 55 56 57
|
||||
out_7 = vcombine_s16(f, h); // 34 35 36 37 74 75 76 77
|
||||
}
|
||||
// transpose 8x8
|
||||
// Can't use transpose_s16_8x8() because the values are arranged in two 4x8
|
||||
// columns.
|
||||
{
|
||||
// 00 01 02 03 40 41 42 43
|
||||
// 10 11 12 13 50 51 52 53
|
||||
// 20 21 22 23 60 61 62 63
|
||||
// 30 31 32 33 70 71 72 73
|
||||
// 04 05 06 07 44 45 46 47
|
||||
// 14 15 16 17 54 55 56 57
|
||||
// 24 25 26 27 64 65 66 67
|
||||
// 34 35 36 37 74 75 76 77
|
||||
const int32x4x2_t r02_s32 =
|
||||
vtrnq_s32(vreinterpretq_s32_s16(out_0), vreinterpretq_s32_s16(out_2));
|
||||
const int32x4x2_t r13_s32 =
|
||||
vtrnq_s32(vreinterpretq_s32_s16(out_1), vreinterpretq_s32_s16(out_3));
|
||||
const int32x4x2_t r46_s32 =
|
||||
vtrnq_s32(vreinterpretq_s32_s16(out_4), vreinterpretq_s32_s16(out_6));
|
||||
const int32x4x2_t r57_s32 =
|
||||
vtrnq_s32(vreinterpretq_s32_s16(out_5), vreinterpretq_s32_s16(out_7));
|
||||
const int16x8x2_t r01_s16 =
|
||||
vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[0]),
|
||||
vreinterpretq_s16_s32(r13_s32.val[0]));
|
||||
const int16x8x2_t r23_s16 =
|
||||
vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[1]),
|
||||
vreinterpretq_s16_s32(r13_s32.val[1]));
|
||||
const int16x8x2_t r45_s16 =
|
||||
vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[0]),
|
||||
vreinterpretq_s16_s32(r57_s32.val[0]));
|
||||
const int16x8x2_t r67_s16 =
|
||||
vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[1]),
|
||||
vreinterpretq_s16_s32(r57_s32.val[1]));
|
||||
input_0 = r01_s16.val[0];
|
||||
input_1 = r01_s16.val[1];
|
||||
input_2 = r23_s16.val[0];
|
||||
input_3 = r23_s16.val[1];
|
||||
input_4 = r45_s16.val[0];
|
||||
input_5 = r45_s16.val[1];
|
||||
input_6 = r67_s16.val[0];
|
||||
input_7 = r67_s16.val[1];
|
||||
// 00 10 20 30 40 50 60 70
|
||||
// 01 11 21 31 41 51 61 71
|
||||
// 02 12 22 32 42 52 62 72
|
||||
// 03 13 23 33 43 53 63 73
|
||||
// 04 14 24 34 44 54 64 74
|
||||
// 05 15 25 35 45 55 65 75
|
||||
// 06 16 26 36 46 56 66 76
|
||||
// 07 17 27 37 47 57 67 77
|
||||
}
|
||||
vpx_fdct8x8_pass1_neon(in);
|
||||
} // for
|
||||
{
|
||||
// from vpx_dct_sse2.c
|
||||
// Post-condition (division by two)
|
||||
// division of two 16 bits signed numbers using shifts
|
||||
// n / 2 = (n - (n >> 15)) >> 1
|
||||
const int16x8_t sign_in0 = vshrq_n_s16(input_0, 15);
|
||||
const int16x8_t sign_in1 = vshrq_n_s16(input_1, 15);
|
||||
const int16x8_t sign_in2 = vshrq_n_s16(input_2, 15);
|
||||
const int16x8_t sign_in3 = vshrq_n_s16(input_3, 15);
|
||||
const int16x8_t sign_in4 = vshrq_n_s16(input_4, 15);
|
||||
const int16x8_t sign_in5 = vshrq_n_s16(input_5, 15);
|
||||
const int16x8_t sign_in6 = vshrq_n_s16(input_6, 15);
|
||||
const int16x8_t sign_in7 = vshrq_n_s16(input_7, 15);
|
||||
input_0 = vhsubq_s16(input_0, sign_in0);
|
||||
input_1 = vhsubq_s16(input_1, sign_in1);
|
||||
input_2 = vhsubq_s16(input_2, sign_in2);
|
||||
input_3 = vhsubq_s16(input_3, sign_in3);
|
||||
input_4 = vhsubq_s16(input_4, sign_in4);
|
||||
input_5 = vhsubq_s16(input_5, sign_in5);
|
||||
input_6 = vhsubq_s16(input_6, sign_in6);
|
||||
input_7 = vhsubq_s16(input_7, sign_in7);
|
||||
const int16x8_t sign_in0 = vshrq_n_s16(in[0], 15);
|
||||
const int16x8_t sign_in1 = vshrq_n_s16(in[1], 15);
|
||||
const int16x8_t sign_in2 = vshrq_n_s16(in[2], 15);
|
||||
const int16x8_t sign_in3 = vshrq_n_s16(in[3], 15);
|
||||
const int16x8_t sign_in4 = vshrq_n_s16(in[4], 15);
|
||||
const int16x8_t sign_in5 = vshrq_n_s16(in[5], 15);
|
||||
const int16x8_t sign_in6 = vshrq_n_s16(in[6], 15);
|
||||
const int16x8_t sign_in7 = vshrq_n_s16(in[7], 15);
|
||||
in[0] = vhsubq_s16(in[0], sign_in0);
|
||||
in[1] = vhsubq_s16(in[1], sign_in1);
|
||||
in[2] = vhsubq_s16(in[2], sign_in2);
|
||||
in[3] = vhsubq_s16(in[3], sign_in3);
|
||||
in[4] = vhsubq_s16(in[4], sign_in4);
|
||||
in[5] = vhsubq_s16(in[5], sign_in5);
|
||||
in[6] = vhsubq_s16(in[6], sign_in6);
|
||||
in[7] = vhsubq_s16(in[7], sign_in7);
|
||||
// store results
|
||||
store_s16q_to_tran_low(final_output + 0 * 8, input_0);
|
||||
store_s16q_to_tran_low(final_output + 1 * 8, input_1);
|
||||
store_s16q_to_tran_low(final_output + 2 * 8, input_2);
|
||||
store_s16q_to_tran_low(final_output + 3 * 8, input_3);
|
||||
store_s16q_to_tran_low(final_output + 4 * 8, input_4);
|
||||
store_s16q_to_tran_low(final_output + 5 * 8, input_5);
|
||||
store_s16q_to_tran_low(final_output + 6 * 8, input_6);
|
||||
store_s16q_to_tran_low(final_output + 7 * 8, input_7);
|
||||
store_s16q_to_tran_low(final_output + 0 * 8, in[0]);
|
||||
store_s16q_to_tran_low(final_output + 1 * 8, in[1]);
|
||||
store_s16q_to_tran_low(final_output + 2 * 8, in[2]);
|
||||
store_s16q_to_tran_low(final_output + 3 * 8, in[3]);
|
||||
store_s16q_to_tran_low(final_output + 4 * 8, in[4]);
|
||||
store_s16q_to_tran_low(final_output + 5 * 8, in[5]);
|
||||
store_s16q_to_tran_low(final_output + 6 * 8, in[6]);
|
||||
store_s16q_to_tran_low(final_output + 7 * 8, in[7]);
|
||||
}
|
||||
}
|
||||
@@ -661,6 +661,17 @@ void vpx_highbd_lpf_vertical_8_dual_neon(
|
||||
vpx_highbd_lpf_vertical_8_neon(s + 8 * p, p, blimit1, limit1, thresh1, bd);
|
||||
}
|
||||
|
||||
// Quiet warnings of the form: 'vpx_dsp/arm/highbd_loopfilter_neon.c|675 col 67|
|
||||
// warning: 'oq1' may be used uninitialized in this function
|
||||
// [-Wmaybe-uninitialized]', for oq1-op1. Without reworking the code or adding
|
||||
// an additional branch this warning cannot be silenced otherwise. The
|
||||
// loopfilter is only called when needed for a block so these output pixels
|
||||
// will be set.
|
||||
#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__)
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
|
||||
#endif
|
||||
|
||||
static void lpf_horizontal_16_kernel(uint16_t *s, int p,
|
||||
const uint16x8_t blimit_vec,
|
||||
const uint16x8_t limit_vec,
|
||||
@@ -723,6 +734,10 @@ static void lpf_vertical_16_kernel(uint16_t *s, int p,
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__)
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
void vpx_highbd_lpf_horizontal_16_neon(uint16_t *s, int p,
|
||||
const uint8_t *blimit,
|
||||
const uint8_t *limit,
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
|
||||
INCLUDE vpx_dsp/arm/idct_neon.asm.S
|
||||
|
||||
AREA Block, CODE, READONLY ; name this block of code
|
||||
AREA Block, CODE, READONLY
|
||||
;void vpx_idct4x4_16_add_neon(int16_t *input, uint8_t *dest, int stride)
|
||||
;
|
||||
; r0 int16_t input
|
||||
|
||||
@@ -975,6 +975,17 @@ FUN_LPF_16_KERNEL(_, 8) // lpf_16_kernel
|
||||
FUN_LPF_16_KERNEL(_dual_, 16) // lpf_16_dual_kernel
|
||||
#undef FUN_LPF_16_KERNEL
|
||||
|
||||
// Quiet warnings of the form: 'vpx_dsp/arm/loopfilter_neon.c|981 col 42|
|
||||
// warning: 'oq1' may be used uninitialized in this function
|
||||
// [-Wmaybe-uninitialized]', for oq1-op1. Without reworking the code or adding
|
||||
// an additional branch this warning cannot be silenced otherwise. The
|
||||
// loopfilter is only called when needed for a block so these output pixels
|
||||
// will be set.
|
||||
#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__)
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
|
||||
#endif
|
||||
|
||||
void vpx_lpf_horizontal_16_neon(uint8_t *s, int p, const uint8_t *blimit,
|
||||
const uint8_t *limit, const uint8_t *thresh) {
|
||||
uint8x8_t p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7, op6,
|
||||
@@ -1090,3 +1101,7 @@ void vpx_lpf_vertical_16_dual_neon(uint8_t *s, int p, const uint8_t *blimit,
|
||||
vget_high_u8(oq0), vget_high_u8(oq1));
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__)
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
@@ -19,6 +19,24 @@
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/vpx_dsp_common.h"
|
||||
|
||||
// Support for these xN intrinsics is lacking in older versions of GCC.
|
||||
#if defined(__GNUC__) && !defined(__clang__)
|
||||
#if __GNUC__ < 8 || defined(__arm__)
|
||||
static INLINE uint8x16x2_t vld1q_u8_x2(uint8_t const *ptr) {
|
||||
uint8x16x2_t res = { { vld1q_u8(ptr + 0 * 16), vld1q_u8(ptr + 1 * 16) } };
|
||||
return res;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if __GNUC__ < 9 || defined(__arm__)
|
||||
static INLINE uint8x16x3_t vld1q_u8_x3(uint8_t const *ptr) {
|
||||
uint8x16x3_t res = { { vld1q_u8(ptr + 0 * 16), vld1q_u8(ptr + 1 * 16),
|
||||
vld1q_u8(ptr + 2 * 16) } };
|
||||
return res;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
static INLINE int16x4_t create_s16x4_neon(const int16_t c0, const int16_t c1,
|
||||
const int16_t c2, const int16_t c3) {
|
||||
return vcreate_s16((uint16_t)c0 | ((uint32_t)c1 << 16) |
|
||||
@@ -95,7 +113,8 @@ static INLINE void uint32_to_mem(uint8_t *buf, uint32_t a) {
|
||||
}
|
||||
|
||||
// Load 2 sets of 4 bytes when alignment is not guaranteed.
|
||||
static INLINE uint8x8_t load_unaligned_u8(const uint8_t *buf, int stride) {
|
||||
static INLINE uint8x8_t load_unaligned_u8(const uint8_t *buf,
|
||||
ptrdiff_t stride) {
|
||||
uint32_t a;
|
||||
uint32x2_t a_u32 = vdup_n_u32(0);
|
||||
if (stride == 4) return vld1_u8(buf);
|
||||
@@ -108,7 +127,7 @@ static INLINE uint8x8_t load_unaligned_u8(const uint8_t *buf, int stride) {
|
||||
}
|
||||
|
||||
// Store 2 sets of 4 bytes when alignment is not guaranteed.
|
||||
static INLINE void store_unaligned_u8(uint8_t *buf, int stride,
|
||||
static INLINE void store_unaligned_u8(uint8_t *buf, ptrdiff_t stride,
|
||||
const uint8x8_t a) {
|
||||
const uint32x2_t a_u32 = vreinterpret_u32_u8(a);
|
||||
if (stride == 4) {
|
||||
@@ -121,7 +140,8 @@ static INLINE void store_unaligned_u8(uint8_t *buf, int stride,
|
||||
}
|
||||
|
||||
// Load 4 sets of 4 bytes when alignment is not guaranteed.
|
||||
static INLINE uint8x16_t load_unaligned_u8q(const uint8_t *buf, int stride) {
|
||||
static INLINE uint8x16_t load_unaligned_u8q(const uint8_t *buf,
|
||||
ptrdiff_t stride) {
|
||||
uint32_t a;
|
||||
uint32x4_t a_u32 = vdupq_n_u32(0);
|
||||
if (stride == 4) return vld1q_u8(buf);
|
||||
@@ -141,7 +161,7 @@ static INLINE uint8x16_t load_unaligned_u8q(const uint8_t *buf, int stride) {
|
||||
}
|
||||
|
||||
// Store 4 sets of 4 bytes when alignment is not guaranteed.
|
||||
static INLINE void store_unaligned_u8q(uint8_t *buf, int stride,
|
||||
static INLINE void store_unaligned_u8q(uint8_t *buf, ptrdiff_t stride,
|
||||
const uint8x16_t a) {
|
||||
const uint32x4_t a_u32 = vreinterpretq_u32_u8(a);
|
||||
if (stride == 4) {
|
||||
@@ -158,7 +178,7 @@ static INLINE void store_unaligned_u8q(uint8_t *buf, int stride,
|
||||
}
|
||||
|
||||
// Load 2 sets of 4 bytes when alignment is guaranteed.
|
||||
static INLINE uint8x8_t load_u8(const uint8_t *buf, int stride) {
|
||||
static INLINE uint8x8_t load_u8(const uint8_t *buf, ptrdiff_t stride) {
|
||||
uint32x2_t a = vdup_n_u32(0);
|
||||
|
||||
assert(!((intptr_t)buf % sizeof(uint32_t)));
|
||||
@@ -171,7 +191,7 @@ static INLINE uint8x8_t load_u8(const uint8_t *buf, int stride) {
|
||||
}
|
||||
|
||||
// Store 2 sets of 4 bytes when alignment is guaranteed.
|
||||
static INLINE void store_u8(uint8_t *buf, int stride, const uint8x8_t a) {
|
||||
static INLINE void store_u8(uint8_t *buf, ptrdiff_t stride, const uint8x8_t a) {
|
||||
uint32x2_t a_u32 = vreinterpret_u32_u8(a);
|
||||
|
||||
assert(!((intptr_t)buf % sizeof(uint32_t)));
|
||||
|
||||
@@ -32,8 +32,8 @@ static INLINE void calculate_dqcoeff_and_store(const int16x8_t qcoeff,
|
||||
}
|
||||
|
||||
void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
|
||||
int skip_block, const int16_t *zbin_ptr,
|
||||
const int16_t *round_ptr, const int16_t *quant_ptr,
|
||||
const int16_t *zbin_ptr, const int16_t *round_ptr,
|
||||
const int16_t *quant_ptr,
|
||||
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
|
||||
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
|
||||
uint16_t *eob_ptr, const int16_t *scan,
|
||||
@@ -42,8 +42,6 @@ void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
|
||||
const int16x8_t neg_one = vdupq_n_s16(-1);
|
||||
uint16x8_t eob_max;
|
||||
(void)scan;
|
||||
(void)skip_block;
|
||||
assert(!skip_block);
|
||||
|
||||
// Process first 8 values which include a dc component.
|
||||
{
|
||||
@@ -189,7 +187,7 @@ static INLINE void calculate_dqcoeff_and_store_32x32(const int16x8_t qcoeff,
|
||||
// Main difference is that zbin values are halved before comparison and dqcoeff
|
||||
// values are divided by 2. zbin is rounded but dqcoeff is not.
|
||||
void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
|
||||
int skip_block, const int16_t *zbin_ptr,
|
||||
const int16_t *zbin_ptr,
|
||||
const int16_t *round_ptr,
|
||||
const int16_t *quant_ptr,
|
||||
const int16_t *quant_shift_ptr,
|
||||
@@ -202,8 +200,6 @@ void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
|
||||
int i;
|
||||
(void)scan;
|
||||
(void)n_coeffs; // Because we will always calculate 32*32.
|
||||
(void)skip_block;
|
||||
assert(!skip_block);
|
||||
|
||||
// Process first 8 values which include a dc component.
|
||||
{
|
||||
|
||||
@@ -31,10 +31,12 @@ static INLINE uint8x8_t load_unaligned_2_buffers(const void *const buf0,
|
||||
static INLINE void sad4x_4d(const uint8_t *const src_ptr, const int src_stride,
|
||||
const uint8_t *const ref_array[4],
|
||||
const int ref_stride, const int height,
|
||||
uint32_t *const res) {
|
||||
uint32_t sad_array[4]) {
|
||||
int i;
|
||||
uint16x8_t abs[2] = { vdupq_n_u16(0), vdupq_n_u16(0) };
|
||||
#if !defined(__aarch64__)
|
||||
uint16x4_t a[2];
|
||||
#endif
|
||||
uint32x4_t r;
|
||||
|
||||
assert(!((intptr_t)src_ptr % sizeof(uint32_t)));
|
||||
@@ -51,29 +53,40 @@ static INLINE void sad4x_4d(const uint8_t *const src_ptr, const int src_stride,
|
||||
abs[1] = vabal_u8(abs[1], s, ref23);
|
||||
}
|
||||
|
||||
#if defined(__aarch64__)
|
||||
abs[0] = vpaddq_u16(abs[0], abs[1]);
|
||||
r = vpaddlq_u16(abs[0]);
|
||||
#else
|
||||
a[0] = vpadd_u16(vget_low_u16(abs[0]), vget_high_u16(abs[0]));
|
||||
a[1] = vpadd_u16(vget_low_u16(abs[1]), vget_high_u16(abs[1]));
|
||||
r = vpaddlq_u16(vcombine_u16(a[0], a[1]));
|
||||
vst1q_u32(res, r);
|
||||
#endif
|
||||
vst1q_u32(sad_array, r);
|
||||
}
|
||||
|
||||
void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 4, res);
|
||||
uint32_t sad_array[4]) {
|
||||
sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 4, sad_array);
|
||||
}
|
||||
|
||||
void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 8, res);
|
||||
uint32_t sad_array[4]) {
|
||||
sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 8, sad_array);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Can handle 512 pixels' sad sum (such as 16x32 or 32x16)
|
||||
static INLINE void sad_512_pel_final_neon(const uint16x8_t *sum /*[4]*/,
|
||||
uint32_t *const res) {
|
||||
static INLINE void sad_512_pel_final_neon(const uint16x8_t sum[4],
|
||||
uint32_t sad_array[4]) {
|
||||
#if defined(__aarch64__)
|
||||
const uint16x8_t a0 = vpaddq_u16(sum[0], sum[1]);
|
||||
const uint16x8_t a1 = vpaddq_u16(sum[2], sum[3]);
|
||||
const uint16x8_t b0 = vpaddq_u16(a0, a1);
|
||||
const uint32x4_t r = vpaddlq_u16(b0);
|
||||
#else
|
||||
const uint16x4_t a0 = vadd_u16(vget_low_u16(sum[0]), vget_high_u16(sum[0]));
|
||||
const uint16x4_t a1 = vadd_u16(vget_low_u16(sum[1]), vget_high_u16(sum[1]));
|
||||
const uint16x4_t a2 = vadd_u16(vget_low_u16(sum[2]), vget_high_u16(sum[2]));
|
||||
@@ -81,12 +94,23 @@ static INLINE void sad_512_pel_final_neon(const uint16x8_t *sum /*[4]*/,
|
||||
const uint16x4_t b0 = vpadd_u16(a0, a1);
|
||||
const uint16x4_t b1 = vpadd_u16(a2, a3);
|
||||
const uint32x4_t r = vpaddlq_u16(vcombine_u16(b0, b1));
|
||||
vst1q_u32(res, r);
|
||||
#endif
|
||||
vst1q_u32(sad_array, r);
|
||||
}
|
||||
|
||||
#if defined(__arm__) || !defined(__ARM_FEATURE_DOTPROD)
|
||||
|
||||
// Can handle 1024 pixels' sad sum (such as 32x32)
|
||||
static INLINE void sad_1024_pel_final_neon(const uint16x8_t *sum /*[4]*/,
|
||||
uint32_t *const res) {
|
||||
static INLINE void sad_1024_pel_final_neon(const uint16x8_t sum[4],
|
||||
uint32_t sad_array[4]) {
|
||||
#if defined(__aarch64__)
|
||||
const uint16x8_t a0 = vpaddq_u16(sum[0], sum[1]);
|
||||
const uint16x8_t a1 = vpaddq_u16(sum[2], sum[3]);
|
||||
const uint32x4_t b0 = vpaddlq_u16(a0);
|
||||
const uint32x4_t b1 = vpaddlq_u16(a1);
|
||||
const uint32x4_t r = vpaddq_u32(b0, b1);
|
||||
vst1q_u32(sad_array, r);
|
||||
#else
|
||||
const uint16x4_t a0 = vpadd_u16(vget_low_u16(sum[0]), vget_high_u16(sum[0]));
|
||||
const uint16x4_t a1 = vpadd_u16(vget_low_u16(sum[1]), vget_high_u16(sum[1]));
|
||||
const uint16x4_t a2 = vpadd_u16(vget_low_u16(sum[2]), vget_high_u16(sum[2]));
|
||||
@@ -95,12 +119,23 @@ static INLINE void sad_1024_pel_final_neon(const uint16x8_t *sum /*[4]*/,
|
||||
const uint32x4_t b1 = vpaddlq_u16(vcombine_u16(a2, a3));
|
||||
const uint32x2_t c0 = vpadd_u32(vget_low_u32(b0), vget_high_u32(b0));
|
||||
const uint32x2_t c1 = vpadd_u32(vget_low_u32(b1), vget_high_u32(b1));
|
||||
vst1q_u32(res, vcombine_u32(c0, c1));
|
||||
vst1q_u32(sad_array, vcombine_u32(c0, c1));
|
||||
#endif
|
||||
}
|
||||
|
||||
// Can handle 2048 pixels' sad sum (such as 32x64 or 64x32)
|
||||
static INLINE void sad_2048_pel_final_neon(const uint16x8_t *sum /*[4]*/,
|
||||
uint32_t *const res) {
|
||||
static INLINE void sad_2048_pel_final_neon(const uint16x8_t sum[4],
|
||||
uint32_t sad_array[4]) {
|
||||
#if defined(__aarch64__)
|
||||
const uint32x4_t a0 = vpaddlq_u16(sum[0]);
|
||||
const uint32x4_t a1 = vpaddlq_u16(sum[1]);
|
||||
const uint32x4_t a2 = vpaddlq_u16(sum[2]);
|
||||
const uint32x4_t a3 = vpaddlq_u16(sum[3]);
|
||||
const uint32x4_t b0 = vpaddq_u32(a0, a1);
|
||||
const uint32x4_t b1 = vpaddq_u32(a2, a3);
|
||||
const uint32x4_t r = vpaddq_u32(b0, b1);
|
||||
vst1q_u32(sad_array, r);
|
||||
#else
|
||||
const uint32x4_t a0 = vpaddlq_u16(sum[0]);
|
||||
const uint32x4_t a1 = vpaddlq_u16(sum[1]);
|
||||
const uint32x4_t a2 = vpaddlq_u16(sum[2]);
|
||||
@@ -111,12 +146,31 @@ static INLINE void sad_2048_pel_final_neon(const uint16x8_t *sum /*[4]*/,
|
||||
const uint32x2_t b3 = vadd_u32(vget_low_u32(a3), vget_high_u32(a3));
|
||||
const uint32x2_t c0 = vpadd_u32(b0, b1);
|
||||
const uint32x2_t c1 = vpadd_u32(b2, b3);
|
||||
vst1q_u32(res, vcombine_u32(c0, c1));
|
||||
vst1q_u32(sad_array, vcombine_u32(c0, c1));
|
||||
#endif
|
||||
}
|
||||
|
||||
// Can handle 4096 pixels' sad sum (such as 64x64)
|
||||
static INLINE void sad_4096_pel_final_neon(const uint16x8_t *sum /*[8]*/,
|
||||
uint32_t *const res) {
|
||||
static INLINE void sad_4096_pel_final_neon(const uint16x8_t sum[8],
|
||||
uint32_t sad_array[4]) {
|
||||
#if defined(__aarch64__)
|
||||
const uint32x4_t a0 = vpaddlq_u16(sum[0]);
|
||||
const uint32x4_t a1 = vpaddlq_u16(sum[1]);
|
||||
const uint32x4_t a2 = vpaddlq_u16(sum[2]);
|
||||
const uint32x4_t a3 = vpaddlq_u16(sum[3]);
|
||||
const uint32x4_t a4 = vpaddlq_u16(sum[4]);
|
||||
const uint32x4_t a5 = vpaddlq_u16(sum[5]);
|
||||
const uint32x4_t a6 = vpaddlq_u16(sum[6]);
|
||||
const uint32x4_t a7 = vpaddlq_u16(sum[7]);
|
||||
const uint32x4_t b0 = vaddq_u32(a0, a1);
|
||||
const uint32x4_t b1 = vaddq_u32(a2, a3);
|
||||
const uint32x4_t b2 = vaddq_u32(a4, a5);
|
||||
const uint32x4_t b3 = vaddq_u32(a6, a7);
|
||||
const uint32x4_t c0 = vpaddq_u32(b0, b1);
|
||||
const uint32x4_t c1 = vpaddq_u32(b2, b3);
|
||||
const uint32x4_t r = vpaddq_u32(c0, c1);
|
||||
vst1q_u32(sad_array, r);
|
||||
#else
|
||||
const uint32x4_t a0 = vpaddlq_u16(sum[0]);
|
||||
const uint32x4_t a1 = vpaddlq_u16(sum[1]);
|
||||
const uint32x4_t a2 = vpaddlq_u16(sum[2]);
|
||||
@@ -135,12 +189,15 @@ static INLINE void sad_4096_pel_final_neon(const uint16x8_t *sum /*[8]*/,
|
||||
const uint32x2_t c3 = vadd_u32(vget_low_u32(b3), vget_high_u32(b3));
|
||||
const uint32x2_t d0 = vpadd_u32(c0, c1);
|
||||
const uint32x2_t d1 = vpadd_u32(c2, c3);
|
||||
vst1q_u32(res, vcombine_u32(d0, d1));
|
||||
vst1q_u32(sad_array, vcombine_u32(d0, d1));
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
static INLINE void sad8x_4d(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res, const int height) {
|
||||
uint32_t sad_array[4], const int height) {
|
||||
int i, j;
|
||||
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
|
||||
ref_array[3] };
|
||||
@@ -157,29 +214,64 @@ static INLINE void sad8x_4d(const uint8_t *src_ptr, int src_stride,
|
||||
}
|
||||
}
|
||||
|
||||
sad_512_pel_final_neon(sum, res);
|
||||
sad_512_pel_final_neon(sum, sad_array);
|
||||
}
|
||||
|
||||
void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 4);
|
||||
uint32_t sad_array[4]) {
|
||||
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 4);
|
||||
}
|
||||
|
||||
void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 8);
|
||||
uint32_t sad_array[4]) {
|
||||
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 8);
|
||||
}
|
||||
|
||||
void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 16);
|
||||
uint32_t sad_array[4]) {
|
||||
sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
|
||||
(__ARM_FEATURE_DOTPROD == 1)
|
||||
|
||||
static INLINE void sad16_neon(const uint8_t *ref_ptr, const uint8x16_t src_ptr,
|
||||
uint32x4_t *const sum) {
|
||||
const uint8x16_t r = vld1q_u8(ref_ptr);
|
||||
const uint8x16_t diff = vabdq_u8(src_ptr, r);
|
||||
*sum = vdotq_u32(*sum, diff, vdupq_n_u8(1));
|
||||
}
|
||||
|
||||
static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t sad_array[4], const int height) {
|
||||
int i;
|
||||
uint32x4_t r0, r1;
|
||||
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
|
||||
ref_array[3] };
|
||||
uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
|
||||
vdupq_n_u32(0) };
|
||||
|
||||
for (i = 0; i < height; ++i) {
|
||||
const uint8x16_t s = vld1q_u8(src_ptr + i * src_stride);
|
||||
sad16_neon(ref_loop[0] + i * ref_stride, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + i * ref_stride, s, &sum[1]);
|
||||
sad16_neon(ref_loop[2] + i * ref_stride, s, &sum[2]);
|
||||
sad16_neon(ref_loop[3] + i * ref_stride, s, &sum[3]);
|
||||
}
|
||||
|
||||
r0 = vpaddq_u32(sum[0], sum[1]);
|
||||
r1 = vpaddq_u32(sum[2], sum[3]);
|
||||
vst1q_u32(sad_array, vpaddq_u32(r0, r1));
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static INLINE void sad16_neon(const uint8_t *ref_ptr, const uint8x16_t src_ptr,
|
||||
uint16x8_t *const sum) {
|
||||
const uint8x16_t r = vld1q_u8(ref_ptr);
|
||||
@@ -189,8 +281,8 @@ static INLINE void sad16_neon(const uint8_t *ref_ptr, const uint8x16_t src_ptr,
|
||||
|
||||
static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res, const int height) {
|
||||
int i, j;
|
||||
uint32_t sad_array[4], const int height) {
|
||||
int i;
|
||||
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
|
||||
ref_array[3] };
|
||||
uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
|
||||
@@ -199,35 +291,103 @@ static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride,
|
||||
for (i = 0; i < height; ++i) {
|
||||
const uint8x16_t s = vld1q_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
for (j = 0; j < 4; ++j) {
|
||||
sad16_neon(ref_loop[j], s, &sum[j]);
|
||||
ref_loop[j] += ref_stride;
|
||||
}
|
||||
/* Manual unrolling here stops the compiler from getting confused. */
|
||||
sad16_neon(ref_loop[0], s, &sum[0]);
|
||||
ref_loop[0] += ref_stride;
|
||||
sad16_neon(ref_loop[1], s, &sum[1]);
|
||||
ref_loop[1] += ref_stride;
|
||||
sad16_neon(ref_loop[2], s, &sum[2]);
|
||||
ref_loop[2] += ref_stride;
|
||||
sad16_neon(ref_loop[3], s, &sum[3]);
|
||||
ref_loop[3] += ref_stride;
|
||||
}
|
||||
|
||||
sad_512_pel_final_neon(sum, res);
|
||||
sad_512_pel_final_neon(sum, sad_array);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 8);
|
||||
uint32_t sad_array[4]) {
|
||||
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 8);
|
||||
}
|
||||
|
||||
void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 16);
|
||||
uint32_t sad_array[4]) {
|
||||
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 16);
|
||||
}
|
||||
|
||||
void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 32);
|
||||
uint32_t sad_array[4]) {
|
||||
sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
|
||||
(__ARM_FEATURE_DOTPROD == 1)
|
||||
|
||||
static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t sad_array[4], const int height) {
|
||||
int i;
|
||||
uint32x4_t r0, r1;
|
||||
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
|
||||
ref_array[3] };
|
||||
|
||||
uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
|
||||
vdupq_n_u32(0) };
|
||||
|
||||
for (i = 0; i < height; ++i) {
|
||||
uint8x16_t s;
|
||||
|
||||
s = vld1q_u8(src_ptr + 0 * 16);
|
||||
sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 0 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[2] + 0 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[3] + 0 * 16, s, &sum[3]);
|
||||
|
||||
s = vld1q_u8(src_ptr + 1 * 16);
|
||||
sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 1 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[2] + 1 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[3] + 1 * 16, s, &sum[3]);
|
||||
|
||||
src_ptr += src_stride;
|
||||
ref_loop[0] += ref_stride;
|
||||
ref_loop[1] += ref_stride;
|
||||
ref_loop[2] += ref_stride;
|
||||
ref_loop[3] += ref_stride;
|
||||
}
|
||||
|
||||
r0 = vpaddq_u32(sum[0], sum[1]);
|
||||
r1 = vpaddq_u32(sum[2], sum[3]);
|
||||
vst1q_u32(sad_array, vpaddq_u32(r0, r1));
|
||||
}
|
||||
|
||||
void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t sad_array[4]) {
|
||||
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 16);
|
||||
}
|
||||
|
||||
void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t sad_array[4]) {
|
||||
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 32);
|
||||
}
|
||||
|
||||
void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t sad_array[4]) {
|
||||
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 64);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
const int height, uint16x8_t *const sum) {
|
||||
@@ -262,33 +422,143 @@ static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride,
|
||||
|
||||
void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
uint32_t sad_array[4]) {
|
||||
uint16x8_t sum[4];
|
||||
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 16, sum);
|
||||
sad_512_pel_final_neon(sum, res);
|
||||
sad_512_pel_final_neon(sum, sad_array);
|
||||
}
|
||||
|
||||
void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
uint32_t sad_array[4]) {
|
||||
uint16x8_t sum[4];
|
||||
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 32, sum);
|
||||
sad_1024_pel_final_neon(sum, res);
|
||||
sad_1024_pel_final_neon(sum, sad_array);
|
||||
}
|
||||
|
||||
void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
uint32_t sad_array[4]) {
|
||||
uint16x8_t sum[4];
|
||||
sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 64, sum);
|
||||
sad_2048_pel_final_neon(sum, res);
|
||||
sad_2048_pel_final_neon(sum, sad_array);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
|
||||
(__ARM_FEATURE_DOTPROD == 1)
|
||||
|
||||
void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
uint32_t sad_array[4]) {
|
||||
int i;
|
||||
uint32x4_t r0, r1;
|
||||
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
|
||||
ref_array[3] };
|
||||
uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
|
||||
vdupq_n_u32(0) };
|
||||
|
||||
for (i = 0; i < 32; ++i) {
|
||||
uint8x16_t s;
|
||||
|
||||
s = vld1q_u8(src_ptr + 0 * 16);
|
||||
sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 0 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[2] + 0 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[3] + 0 * 16, s, &sum[3]);
|
||||
|
||||
s = vld1q_u8(src_ptr + 1 * 16);
|
||||
sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 1 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[2] + 1 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[3] + 1 * 16, s, &sum[3]);
|
||||
|
||||
s = vld1q_u8(src_ptr + 2 * 16);
|
||||
sad16_neon(ref_loop[0] + 2 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 2 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[2] + 2 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[3] + 2 * 16, s, &sum[3]);
|
||||
|
||||
s = vld1q_u8(src_ptr + 3 * 16);
|
||||
sad16_neon(ref_loop[0] + 3 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 3 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[2] + 3 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[3] + 3 * 16, s, &sum[3]);
|
||||
|
||||
src_ptr += src_stride;
|
||||
ref_loop[0] += ref_stride;
|
||||
ref_loop[1] += ref_stride;
|
||||
ref_loop[2] += ref_stride;
|
||||
ref_loop[3] += ref_stride;
|
||||
}
|
||||
|
||||
r0 = vpaddq_u32(sum[0], sum[1]);
|
||||
r1 = vpaddq_u32(sum[2], sum[3]);
|
||||
vst1q_u32(sad_array, vpaddq_u32(r0, r1));
|
||||
}
|
||||
|
||||
void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t sad_array[4]) {
|
||||
int i;
|
||||
uint32x4_t r0, r1, r2, r3;
|
||||
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
|
||||
ref_array[3] };
|
||||
uint32x4_t sum[8] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
|
||||
vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
|
||||
vdupq_n_u32(0), vdupq_n_u32(0) };
|
||||
|
||||
for (i = 0; i < 64; ++i) {
|
||||
uint8x16_t s;
|
||||
|
||||
s = vld1q_u8(src_ptr + 0 * 16);
|
||||
sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 0 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[2] + 0 * 16, s, &sum[4]);
|
||||
sad16_neon(ref_loop[3] + 0 * 16, s, &sum[6]);
|
||||
|
||||
s = vld1q_u8(src_ptr + 1 * 16);
|
||||
sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]);
|
||||
sad16_neon(ref_loop[1] + 1 * 16, s, &sum[2]);
|
||||
sad16_neon(ref_loop[2] + 1 * 16, s, &sum[4]);
|
||||
sad16_neon(ref_loop[3] + 1 * 16, s, &sum[6]);
|
||||
|
||||
s = vld1q_u8(src_ptr + 2 * 16);
|
||||
sad16_neon(ref_loop[0] + 2 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[1] + 2 * 16, s, &sum[3]);
|
||||
sad16_neon(ref_loop[2] + 2 * 16, s, &sum[5]);
|
||||
sad16_neon(ref_loop[3] + 2 * 16, s, &sum[7]);
|
||||
|
||||
s = vld1q_u8(src_ptr + 3 * 16);
|
||||
sad16_neon(ref_loop[0] + 3 * 16, s, &sum[1]);
|
||||
sad16_neon(ref_loop[1] + 3 * 16, s, &sum[3]);
|
||||
sad16_neon(ref_loop[2] + 3 * 16, s, &sum[5]);
|
||||
sad16_neon(ref_loop[3] + 3 * 16, s, &sum[7]);
|
||||
|
||||
src_ptr += src_stride;
|
||||
ref_loop[0] += ref_stride;
|
||||
ref_loop[1] += ref_stride;
|
||||
ref_loop[2] += ref_stride;
|
||||
ref_loop[3] += ref_stride;
|
||||
}
|
||||
|
||||
r0 = vpaddq_u32(sum[0], sum[1]);
|
||||
r1 = vpaddq_u32(sum[2], sum[3]);
|
||||
r2 = vpaddq_u32(sum[4], sum[5]);
|
||||
r3 = vpaddq_u32(sum[6], sum[7]);
|
||||
r0 = vpaddq_u32(r0, r1);
|
||||
r1 = vpaddq_u32(r2, r3);
|
||||
vst1q_u32(sad_array, vpaddq_u32(r0, r1));
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t sad_array[4]) {
|
||||
int i;
|
||||
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
|
||||
ref_array[3] };
|
||||
@@ -329,12 +599,12 @@ void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
ref_loop[3] += ref_stride;
|
||||
}
|
||||
|
||||
sad_2048_pel_final_neon(sum, res);
|
||||
sad_2048_pel_final_neon(sum, sad_array);
|
||||
}
|
||||
|
||||
void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *const ref_array[4], int ref_stride,
|
||||
uint32_t *res) {
|
||||
uint32_t sad_array[4]) {
|
||||
int i;
|
||||
const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2],
|
||||
ref_array[3] };
|
||||
@@ -376,5 +646,7 @@ void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride,
|
||||
ref_loop[3] += ref_stride;
|
||||
}
|
||||
|
||||
sad_4096_pel_final_neon(sum, res);
|
||||
sad_4096_pel_final_neon(sum, sad_array);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -23,7 +23,7 @@ uint32_t vpx_sad4x4_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8x16_t ref_u8 = load_unaligned_u8q(ref_ptr, ref_stride);
|
||||
uint16x8_t abs = vabdl_u8(vget_low_u8(src_u8), vget_low_u8(ref_u8));
|
||||
abs = vabal_u8(abs, vget_high_u8(src_u8), vget_high_u8(ref_u8));
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0);
|
||||
return horizontal_add_uint16x8(abs);
|
||||
}
|
||||
|
||||
uint32_t vpx_sad4x4_avg_neon(const uint8_t *src_ptr, int src_stride,
|
||||
@@ -35,7 +35,7 @@ uint32_t vpx_sad4x4_avg_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8x16_t avg = vrhaddq_u8(ref_u8, second_pred_u8);
|
||||
uint16x8_t abs = vabdl_u8(vget_low_u8(src_u8), vget_low_u8(avg));
|
||||
abs = vabal_u8(abs, vget_high_u8(src_u8), vget_high_u8(avg));
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0);
|
||||
return horizontal_add_uint16x8(abs);
|
||||
}
|
||||
|
||||
uint32_t vpx_sad4x8_neon(const uint8_t *src_ptr, int src_stride,
|
||||
@@ -51,7 +51,7 @@ uint32_t vpx_sad4x8_neon(const uint8_t *src_ptr, int src_stride,
|
||||
abs = vabal_u8(abs, vget_high_u8(src_u8), vget_high_u8(ref_u8));
|
||||
}
|
||||
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0);
|
||||
return horizontal_add_uint16x8(abs);
|
||||
}
|
||||
|
||||
uint32_t vpx_sad4x8_avg_neon(const uint8_t *src_ptr, int src_stride,
|
||||
@@ -71,7 +71,7 @@ uint32_t vpx_sad4x8_avg_neon(const uint8_t *src_ptr, int src_stride,
|
||||
abs = vabal_u8(abs, vget_high_u8(src_u8), vget_high_u8(avg));
|
||||
}
|
||||
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0);
|
||||
return horizontal_add_uint16x8(abs);
|
||||
}
|
||||
|
||||
static INLINE uint16x8_t sad8x(const uint8_t *src_ptr, int src_stride,
|
||||
@@ -110,11 +110,11 @@ static INLINE uint16x8_t sad8x_avg(const uint8_t *src_ptr, int src_stride,
|
||||
return abs;
|
||||
}
|
||||
|
||||
#define sad8xN(n) \
|
||||
#define SAD8XN(n) \
|
||||
uint32_t vpx_sad8x##n##_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride) { \
|
||||
const uint16x8_t abs = sad8x(src_ptr, src_stride, ref_ptr, ref_stride, n); \
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
|
||||
return horizontal_add_uint16x8(abs); \
|
||||
} \
|
||||
\
|
||||
uint32_t vpx_sad8x##n##_avg_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
@@ -122,12 +122,12 @@ static INLINE uint16x8_t sad8x_avg(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *second_pred) { \
|
||||
const uint16x8_t abs = \
|
||||
sad8x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
|
||||
return horizontal_add_uint16x8(abs); \
|
||||
}
|
||||
|
||||
sad8xN(4);
|
||||
sad8xN(8);
|
||||
sad8xN(16);
|
||||
SAD8XN(4)
|
||||
SAD8XN(8)
|
||||
SAD8XN(16)
|
||||
|
||||
static INLINE uint16x8_t sad16x(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
@@ -167,12 +167,12 @@ static INLINE uint16x8_t sad16x_avg(const uint8_t *src_ptr, int src_stride,
|
||||
return abs;
|
||||
}
|
||||
|
||||
#define sad16xN(n) \
|
||||
#define SAD16XN(n) \
|
||||
uint32_t vpx_sad16x##n##_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride) { \
|
||||
const uint16x8_t abs = \
|
||||
sad16x(src_ptr, src_stride, ref_ptr, ref_stride, n); \
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
|
||||
return horizontal_add_uint16x8(abs); \
|
||||
} \
|
||||
\
|
||||
uint32_t vpx_sad16x##n##_avg_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
@@ -180,12 +180,12 @@ static INLINE uint16x8_t sad16x_avg(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *second_pred) { \
|
||||
const uint16x8_t abs = \
|
||||
sad16x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
|
||||
return horizontal_add_uint16x8(abs); \
|
||||
}
|
||||
|
||||
sad16xN(8);
|
||||
sad16xN(16);
|
||||
sad16xN(32);
|
||||
SAD16XN(8)
|
||||
SAD16XN(16)
|
||||
SAD16XN(32)
|
||||
|
||||
static INLINE uint16x8_t sad32x(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
@@ -235,12 +235,12 @@ static INLINE uint16x8_t sad32x_avg(const uint8_t *src_ptr, int src_stride,
|
||||
return abs;
|
||||
}
|
||||
|
||||
#define sad32xN(n) \
|
||||
#define SAD32XN(n) \
|
||||
uint32_t vpx_sad32x##n##_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride) { \
|
||||
const uint16x8_t abs = \
|
||||
sad32x(src_ptr, src_stride, ref_ptr, ref_stride, n); \
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
|
||||
return horizontal_add_uint16x8(abs); \
|
||||
} \
|
||||
\
|
||||
uint32_t vpx_sad32x##n##_avg_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
@@ -248,12 +248,12 @@ static INLINE uint16x8_t sad32x_avg(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *second_pred) { \
|
||||
const uint16x8_t abs = \
|
||||
sad32x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \
|
||||
return vget_lane_u32(horizontal_add_uint16x8(abs), 0); \
|
||||
return horizontal_add_uint16x8(abs); \
|
||||
}
|
||||
|
||||
sad32xN(16);
|
||||
sad32xN(32);
|
||||
sad32xN(64);
|
||||
SAD32XN(16)
|
||||
SAD32XN(32)
|
||||
SAD32XN(64)
|
||||
|
||||
static INLINE uint32x4_t sad64x(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
@@ -333,12 +333,12 @@ static INLINE uint32x4_t sad64x_avg(const uint8_t *src_ptr, int src_stride,
|
||||
}
|
||||
}
|
||||
|
||||
#define sad64xN(n) \
|
||||
#define SAD64XN(n) \
|
||||
uint32_t vpx_sad64x##n##_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
const uint8_t *ref_ptr, int ref_stride) { \
|
||||
const uint32x4_t abs = \
|
||||
sad64x(src_ptr, src_stride, ref_ptr, ref_stride, n); \
|
||||
return vget_lane_u32(horizontal_add_uint32x4(abs), 0); \
|
||||
return horizontal_add_uint32x4(abs); \
|
||||
} \
|
||||
\
|
||||
uint32_t vpx_sad64x##n##_avg_neon(const uint8_t *src_ptr, int src_stride, \
|
||||
@@ -346,8 +346,8 @@ static INLINE uint32x4_t sad64x_avg(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *second_pred) { \
|
||||
const uint32x4_t abs = \
|
||||
sad64x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \
|
||||
return vget_lane_u32(horizontal_add_uint32x4(abs), 0); \
|
||||
return horizontal_add_uint32x4(abs); \
|
||||
}
|
||||
|
||||
sad64xN(32);
|
||||
sad64xN(64);
|
||||
SAD64XN(32)
|
||||
SAD64XN(64)
|
||||
@@ -97,7 +97,7 @@ static void var_filter_block2d_bil_w16(const uint8_t *src_ptr,
|
||||
|
||||
// 4xM filter writes an extra row to fdata because it processes two rows at a
|
||||
// time.
|
||||
#define sub_pixel_varianceNxM(n, m) \
|
||||
#define SUB_PIXEL_VARIANCENXM(n, m) \
|
||||
uint32_t vpx_sub_pixel_variance##n##x##m##_neon( \
|
||||
const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, \
|
||||
const uint8_t *ref_ptr, int ref_stride, uint32_t *sse) { \
|
||||
@@ -123,23 +123,23 @@ static void var_filter_block2d_bil_w16(const uint8_t *src_ptr,
|
||||
return vpx_variance##n##x##m(temp1, n, ref_ptr, ref_stride, sse); \
|
||||
}
|
||||
|
||||
sub_pixel_varianceNxM(4, 4);
|
||||
sub_pixel_varianceNxM(4, 8);
|
||||
sub_pixel_varianceNxM(8, 4);
|
||||
sub_pixel_varianceNxM(8, 8);
|
||||
sub_pixel_varianceNxM(8, 16);
|
||||
sub_pixel_varianceNxM(16, 8);
|
||||
sub_pixel_varianceNxM(16, 16);
|
||||
sub_pixel_varianceNxM(16, 32);
|
||||
sub_pixel_varianceNxM(32, 16);
|
||||
sub_pixel_varianceNxM(32, 32);
|
||||
sub_pixel_varianceNxM(32, 64);
|
||||
sub_pixel_varianceNxM(64, 32);
|
||||
sub_pixel_varianceNxM(64, 64);
|
||||
SUB_PIXEL_VARIANCENXM(4, 4)
|
||||
SUB_PIXEL_VARIANCENXM(4, 8)
|
||||
SUB_PIXEL_VARIANCENXM(8, 4)
|
||||
SUB_PIXEL_VARIANCENXM(8, 8)
|
||||
SUB_PIXEL_VARIANCENXM(8, 16)
|
||||
SUB_PIXEL_VARIANCENXM(16, 8)
|
||||
SUB_PIXEL_VARIANCENXM(16, 16)
|
||||
SUB_PIXEL_VARIANCENXM(16, 32)
|
||||
SUB_PIXEL_VARIANCENXM(32, 16)
|
||||
SUB_PIXEL_VARIANCENXM(32, 32)
|
||||
SUB_PIXEL_VARIANCENXM(32, 64)
|
||||
SUB_PIXEL_VARIANCENXM(64, 32)
|
||||
SUB_PIXEL_VARIANCENXM(64, 64)
|
||||
|
||||
// 4xM filter writes an extra row to fdata because it processes two rows at a
|
||||
// time.
|
||||
#define sub_pixel_avg_varianceNxM(n, m) \
|
||||
#define SUB_PIXEL_AVG_VARIANCENXM(n, m) \
|
||||
uint32_t vpx_sub_pixel_avg_variance##n##x##m##_neon( \
|
||||
const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, \
|
||||
const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, \
|
||||
@@ -169,16 +169,16 @@ sub_pixel_varianceNxM(64, 64);
|
||||
return vpx_variance##n##x##m(temp0, n, ref_ptr, ref_stride, sse); \
|
||||
}
|
||||
|
||||
sub_pixel_avg_varianceNxM(4, 4);
|
||||
sub_pixel_avg_varianceNxM(4, 8);
|
||||
sub_pixel_avg_varianceNxM(8, 4);
|
||||
sub_pixel_avg_varianceNxM(8, 8);
|
||||
sub_pixel_avg_varianceNxM(8, 16);
|
||||
sub_pixel_avg_varianceNxM(16, 8);
|
||||
sub_pixel_avg_varianceNxM(16, 16);
|
||||
sub_pixel_avg_varianceNxM(16, 32);
|
||||
sub_pixel_avg_varianceNxM(32, 16);
|
||||
sub_pixel_avg_varianceNxM(32, 32);
|
||||
sub_pixel_avg_varianceNxM(32, 64);
|
||||
sub_pixel_avg_varianceNxM(64, 32);
|
||||
sub_pixel_avg_varianceNxM(64, 64);
|
||||
SUB_PIXEL_AVG_VARIANCENXM(4, 4)
|
||||
SUB_PIXEL_AVG_VARIANCENXM(4, 8)
|
||||
SUB_PIXEL_AVG_VARIANCENXM(8, 4)
|
||||
SUB_PIXEL_AVG_VARIANCENXM(8, 8)
|
||||
SUB_PIXEL_AVG_VARIANCENXM(8, 16)
|
||||
SUB_PIXEL_AVG_VARIANCENXM(16, 8)
|
||||
SUB_PIXEL_AVG_VARIANCENXM(16, 16)
|
||||
SUB_PIXEL_AVG_VARIANCENXM(16, 32)
|
||||
SUB_PIXEL_AVG_VARIANCENXM(32, 16)
|
||||
SUB_PIXEL_AVG_VARIANCENXM(32, 32)
|
||||
SUB_PIXEL_AVG_VARIANCENXM(32, 64)
|
||||
SUB_PIXEL_AVG_VARIANCENXM(64, 32)
|
||||
SUB_PIXEL_AVG_VARIANCENXM(64, 64)
|
||||
@@ -16,23 +16,65 @@
|
||||
#include "./vpx_config.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
|
||||
static INLINE int32x2_t horizontal_add_int16x8(const int16x8_t a) {
|
||||
static INLINE int32_t horizontal_add_int16x8(const int16x8_t a) {
|
||||
#if defined(__aarch64__)
|
||||
return vaddlvq_s16(a);
|
||||
#else
|
||||
const int32x4_t b = vpaddlq_s16(a);
|
||||
const int64x2_t c = vpaddlq_s32(b);
|
||||
return vadd_s32(vreinterpret_s32_s64(vget_low_s64(c)),
|
||||
vreinterpret_s32_s64(vget_high_s64(c)));
|
||||
const int32x2_t d = vadd_s32(vreinterpret_s32_s64(vget_low_s64(c)),
|
||||
vreinterpret_s32_s64(vget_high_s64(c)));
|
||||
return vget_lane_s32(d, 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
static INLINE uint32x2_t horizontal_add_uint16x8(const uint16x8_t a) {
|
||||
static INLINE uint32_t horizontal_add_uint16x8(const uint16x8_t a) {
|
||||
#if defined(__aarch64__)
|
||||
return vaddlvq_u16(a);
|
||||
#else
|
||||
const uint32x4_t b = vpaddlq_u16(a);
|
||||
const uint64x2_t c = vpaddlq_u32(b);
|
||||
return vadd_u32(vreinterpret_u32_u64(vget_low_u64(c)),
|
||||
vreinterpret_u32_u64(vget_high_u64(c)));
|
||||
const uint32x2_t d = vadd_u32(vreinterpret_u32_u64(vget_low_u64(c)),
|
||||
vreinterpret_u32_u64(vget_high_u64(c)));
|
||||
return vget_lane_u32(d, 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
static INLINE uint32x2_t horizontal_add_uint32x4(const uint32x4_t a) {
|
||||
static INLINE int32_t horizontal_add_int32x2(const int32x2_t a) {
|
||||
#if defined(__aarch64__)
|
||||
return vaddv_s32(a);
|
||||
#else
|
||||
return vget_lane_s32(a, 0) + vget_lane_s32(a, 1);
|
||||
#endif
|
||||
}
|
||||
|
||||
static INLINE uint32_t horizontal_add_uint32x2(const uint32x2_t a) {
|
||||
#if defined(__aarch64__)
|
||||
return vaddv_u32(a);
|
||||
#else
|
||||
return vget_lane_u32(a, 0) + vget_lane_u32(a, 1);
|
||||
#endif
|
||||
}
|
||||
|
||||
static INLINE int32_t horizontal_add_int32x4(const int32x4_t a) {
|
||||
#if defined(__aarch64__)
|
||||
return vaddvq_s32(a);
|
||||
#else
|
||||
const int64x2_t b = vpaddlq_s32(a);
|
||||
const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)),
|
||||
vreinterpret_s32_s64(vget_high_s64(b)));
|
||||
return vget_lane_s32(c, 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
static INLINE uint32_t horizontal_add_uint32x4(const uint32x4_t a) {
|
||||
#if defined(__aarch64__)
|
||||
return vaddvq_u32(a);
|
||||
#else
|
||||
const uint64x2_t b = vpaddlq_u32(a);
|
||||
return vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)),
|
||||
vreinterpret_u32_u64(vget_high_u64(b)));
|
||||
const uint32x2_t c = vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)),
|
||||
vreinterpret_u32_u64(vget_high_u64(b)));
|
||||
return vget_lane_u32(c, 0);
|
||||
#endif
|
||||
}
|
||||
#endif // VPX_VPX_DSP_ARM_SUM_NEON_H_
|
||||
@@ -138,8 +138,8 @@ static INLINE void transpose_s16_4x4q(int16x8_t *a0, int16x8_t *a1) {
|
||||
vtrnq_s32(vreinterpretq_s32_s16(*a0), vreinterpretq_s32_s16(*a1));
|
||||
|
||||
// Swap 64 bit elements resulting in:
|
||||
// c0.val[0]: 00 01 20 21 02 03 22 23
|
||||
// c0.val[1]: 10 11 30 31 12 13 32 33
|
||||
// c0: 00 01 20 21 02 03 22 23
|
||||
// c1: 10 11 30 31 12 13 32 33
|
||||
|
||||
const int32x4_t c0 =
|
||||
vcombine_s32(vget_low_s32(b0.val[0]), vget_low_s32(b0.val[1]));
|
||||
@@ -169,8 +169,8 @@ static INLINE void transpose_u16_4x4q(uint16x8_t *a0, uint16x8_t *a1) {
|
||||
vtrnq_u32(vreinterpretq_u32_u16(*a0), vreinterpretq_u32_u16(*a1));
|
||||
|
||||
// Swap 64 bit elements resulting in:
|
||||
// c0.val[0]: 00 01 20 21 02 03 22 23
|
||||
// c0.val[1]: 10 11 30 31 12 13 32 33
|
||||
// c0: 00 01 20 21 02 03 22 23
|
||||
// c1: 10 11 30 31 12 13 32 33
|
||||
|
||||
const uint32x4_t c0 =
|
||||
vcombine_u32(vget_low_u32(b0.val[0]), vget_low_u32(b0.val[1]));
|
||||
@@ -1184,6 +1184,45 @@ static INLINE void transpose_u8_16x16(
|
||||
*o15 = e7.val[1];
|
||||
}
|
||||
|
||||
static INLINE void transpose_s16_16x16(int16x8_t *in0, int16x8_t *in1) {
|
||||
int16x8_t t[8];
|
||||
|
||||
// transpose the 4 8x8 quadrants separately but first swap quadrants 2 and 3.
|
||||
t[0] = in0[8];
|
||||
t[1] = in0[9];
|
||||
t[2] = in0[10];
|
||||
t[3] = in0[11];
|
||||
t[4] = in0[12];
|
||||
t[5] = in0[13];
|
||||
t[6] = in0[14];
|
||||
t[7] = in0[15];
|
||||
in0[8] = in1[0];
|
||||
in0[9] = in1[1];
|
||||
in0[10] = in1[2];
|
||||
in0[11] = in1[3];
|
||||
in0[12] = in1[4];
|
||||
in0[13] = in1[5];
|
||||
in0[14] = in1[6];
|
||||
in0[15] = in1[7];
|
||||
in1[0] = t[0];
|
||||
in1[1] = t[1];
|
||||
in1[2] = t[2];
|
||||
in1[3] = t[3];
|
||||
in1[4] = t[4];
|
||||
in1[5] = t[5];
|
||||
in1[6] = t[6];
|
||||
in1[7] = t[7];
|
||||
|
||||
transpose_s16_8x8(&in0[0], &in0[1], &in0[2], &in0[3], &in0[4], &in0[5],
|
||||
&in0[6], &in0[7]);
|
||||
transpose_s16_8x8(&in0[8], &in0[9], &in0[10], &in0[11], &in0[12], &in0[13],
|
||||
&in0[14], &in0[15]);
|
||||
transpose_s16_8x8(&in1[0], &in1[1], &in1[2], &in1[3], &in1[4], &in1[5],
|
||||
&in1[6], &in1[7]);
|
||||
transpose_s16_8x8(&in1[8], &in1[9], &in1[10], &in1[11], &in1[12], &in1[13],
|
||||
&in1[14], &in1[15]);
|
||||
}
|
||||
|
||||
static INLINE void load_and_transpose_u8_4x8(const uint8_t *a,
|
||||
const int a_stride, uint8x8_t *a0,
|
||||
uint8x8_t *a1, uint8x8_t *a2,
|
||||
|
||||
@@ -19,6 +19,100 @@
|
||||
#include "vpx_dsp/arm/sum_neon.h"
|
||||
#include "vpx_ports/mem.h"
|
||||
|
||||
#if defined(__ARM_FEATURE_DOTPROD) && (__ARM_FEATURE_DOTPROD == 1)
|
||||
|
||||
// Process a block of width 4 four rows at a time.
|
||||
static void variance_neon_w4x4(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride, int h,
|
||||
uint32_t *sse, int *sum) {
|
||||
int i;
|
||||
uint32x4_t sum_a = vdupq_n_u32(0);
|
||||
uint32x4_t sum_b = vdupq_n_u32(0);
|
||||
uint32x4_t sse_u32 = vdupq_n_u32(0);
|
||||
|
||||
for (i = 0; i < h; i += 4) {
|
||||
const uint8x16_t a = load_unaligned_u8q(src_ptr, src_stride);
|
||||
const uint8x16_t b = load_unaligned_u8q(ref_ptr, ref_stride);
|
||||
|
||||
const uint8x16_t abs_diff = vabdq_u8(a, b);
|
||||
sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
|
||||
|
||||
sum_a = vdotq_u32(sum_a, a, vdupq_n_u8(1));
|
||||
sum_b = vdotq_u32(sum_b, b, vdupq_n_u8(1));
|
||||
|
||||
src_ptr += 4 * src_stride;
|
||||
ref_ptr += 4 * ref_stride;
|
||||
}
|
||||
|
||||
*sum = horizontal_add_int32x4(vreinterpretq_s32_u32(vsubq_u32(sum_a, sum_b)));
|
||||
*sse = horizontal_add_uint32x4(sse_u32);
|
||||
}
|
||||
|
||||
// Process a block of any size where the width is divisible by 16.
|
||||
static void variance_neon_w16(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride, int w,
|
||||
int h, uint32_t *sse, int *sum) {
|
||||
int i, j;
|
||||
uint32x4_t sum_a = vdupq_n_u32(0);
|
||||
uint32x4_t sum_b = vdupq_n_u32(0);
|
||||
uint32x4_t sse_u32 = vdupq_n_u32(0);
|
||||
|
||||
for (i = 0; i < h; ++i) {
|
||||
for (j = 0; j < w; j += 16) {
|
||||
const uint8x16_t a = vld1q_u8(src_ptr + j);
|
||||
const uint8x16_t b = vld1q_u8(ref_ptr + j);
|
||||
|
||||
const uint8x16_t abs_diff = vabdq_u8(a, b);
|
||||
sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
|
||||
|
||||
sum_a = vdotq_u32(sum_a, a, vdupq_n_u8(1));
|
||||
sum_b = vdotq_u32(sum_b, b, vdupq_n_u8(1));
|
||||
}
|
||||
src_ptr += src_stride;
|
||||
ref_ptr += ref_stride;
|
||||
}
|
||||
|
||||
*sum = horizontal_add_int32x4(vreinterpretq_s32_u32(vsubq_u32(sum_a, sum_b)));
|
||||
*sse = horizontal_add_uint32x4(sse_u32);
|
||||
}
|
||||
|
||||
// Process a block of width 8 two rows at a time.
|
||||
static void variance_neon_w8x2(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride, int h,
|
||||
uint32_t *sse, int *sum) {
|
||||
int i = 0;
|
||||
uint32x2_t sum_a = vdup_n_u32(0);
|
||||
uint32x2_t sum_b = vdup_n_u32(0);
|
||||
uint32x2_t sse_lo_u32 = vdup_n_u32(0);
|
||||
uint32x2_t sse_hi_u32 = vdup_n_u32(0);
|
||||
|
||||
do {
|
||||
const uint8x8_t a_0 = vld1_u8(src_ptr);
|
||||
const uint8x8_t a_1 = vld1_u8(src_ptr + src_stride);
|
||||
const uint8x8_t b_0 = vld1_u8(ref_ptr);
|
||||
const uint8x8_t b_1 = vld1_u8(ref_ptr + ref_stride);
|
||||
|
||||
const uint8x8_t abs_diff_0 = vabd_u8(a_0, b_0);
|
||||
const uint8x8_t abs_diff_1 = vabd_u8(a_1, b_1);
|
||||
sse_lo_u32 = vdot_u32(sse_lo_u32, abs_diff_0, abs_diff_0);
|
||||
sse_hi_u32 = vdot_u32(sse_hi_u32, abs_diff_1, abs_diff_1);
|
||||
|
||||
sum_a = vdot_u32(sum_a, a_0, vdup_n_u8(1));
|
||||
sum_b = vdot_u32(sum_b, b_0, vdup_n_u8(1));
|
||||
sum_a = vdot_u32(sum_a, a_1, vdup_n_u8(1));
|
||||
sum_b = vdot_u32(sum_b, b_1, vdup_n_u8(1));
|
||||
|
||||
src_ptr += src_stride + src_stride;
|
||||
ref_ptr += ref_stride + ref_stride;
|
||||
i += 2;
|
||||
} while (i < h);
|
||||
|
||||
*sum = horizontal_add_int32x2(vreinterpret_s32_u32(vsub_u32(sum_a, sum_b)));
|
||||
*sse = horizontal_add_uint32x2(vadd_u32(sse_lo_u32, sse_hi_u32));
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
// The variance helper functions use int16_t for sum. 8 values are accumulated
|
||||
// and then added (at which point they expand up to int32_t). To avoid overflow,
|
||||
// there can be no more than 32767 / 255 ~= 128 values accumulated in each
|
||||
@@ -66,10 +160,9 @@ static void variance_neon_w4x4(const uint8_t *src_ptr, int src_stride,
|
||||
ref_ptr += 4 * ref_stride;
|
||||
}
|
||||
|
||||
*sum = vget_lane_s32(horizontal_add_int16x8(sum_s16), 0);
|
||||
*sse = vget_lane_u32(horizontal_add_uint32x4(vreinterpretq_u32_s32(
|
||||
vaddq_s32(sse_lo_s32, sse_hi_s32))),
|
||||
0);
|
||||
*sum = horizontal_add_int16x8(sum_s16);
|
||||
*sse = horizontal_add_uint32x4(
|
||||
vreinterpretq_u32_s32(vaddq_s32(sse_lo_s32, sse_hi_s32)));
|
||||
}
|
||||
|
||||
// Process a block of any size where the width is divisible by 16.
|
||||
@@ -115,10 +208,9 @@ static void variance_neon_w16(const uint8_t *src_ptr, int src_stride,
|
||||
ref_ptr += ref_stride;
|
||||
}
|
||||
|
||||
*sum = vget_lane_s32(horizontal_add_int16x8(sum_s16), 0);
|
||||
*sse = vget_lane_u32(horizontal_add_uint32x4(vreinterpretq_u32_s32(
|
||||
vaddq_s32(sse_lo_s32, sse_hi_s32))),
|
||||
0);
|
||||
*sum = horizontal_add_int16x8(sum_s16);
|
||||
*sse = horizontal_add_uint32x4(
|
||||
vreinterpretq_u32_s32(vaddq_s32(sse_lo_s32, sse_hi_s32)));
|
||||
}
|
||||
|
||||
// Process a block of width 8 two rows at a time.
|
||||
@@ -157,12 +249,13 @@ static void variance_neon_w8x2(const uint8_t *src_ptr, int src_stride,
|
||||
i += 2;
|
||||
} while (i < h);
|
||||
|
||||
*sum = vget_lane_s32(horizontal_add_int16x8(sum_s16), 0);
|
||||
*sse = vget_lane_u32(horizontal_add_uint32x4(vreinterpretq_u32_s32(
|
||||
vaddq_s32(sse_lo_s32, sse_hi_s32))),
|
||||
0);
|
||||
*sum = horizontal_add_int16x8(sum_s16);
|
||||
*sse = horizontal_add_uint32x4(
|
||||
vreinterpretq_u32_s32(vaddq_s32(sse_lo_s32, sse_hi_s32)));
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
void vpx_get8x8var_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
unsigned int *sse, int *sum) {
|
||||
@@ -175,7 +268,7 @@ void vpx_get16x16var_neon(const uint8_t *src_ptr, int src_stride,
|
||||
variance_neon_w16(src_ptr, src_stride, ref_ptr, ref_stride, 16, 16, sse, sum);
|
||||
}
|
||||
|
||||
#define varianceNxM(n, m, shift) \
|
||||
#define VARIANCENXM(n, m, shift) \
|
||||
unsigned int vpx_variance##n##x##m##_neon( \
|
||||
const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \
|
||||
int ref_stride, unsigned int *sse) { \
|
||||
@@ -195,16 +288,16 @@ void vpx_get16x16var_neon(const uint8_t *src_ptr, int src_stride,
|
||||
return *sse - (uint32_t)(((int64_t)sum * sum) >> shift); \
|
||||
}
|
||||
|
||||
varianceNxM(4, 4, 4);
|
||||
varianceNxM(4, 8, 5);
|
||||
varianceNxM(8, 4, 5);
|
||||
varianceNxM(8, 8, 6);
|
||||
varianceNxM(8, 16, 7);
|
||||
varianceNxM(16, 8, 7);
|
||||
varianceNxM(16, 16, 8);
|
||||
varianceNxM(16, 32, 9);
|
||||
varianceNxM(32, 16, 9);
|
||||
varianceNxM(32, 32, 10);
|
||||
VARIANCENXM(4, 4, 4)
|
||||
VARIANCENXM(4, 8, 5)
|
||||
VARIANCENXM(8, 4, 5)
|
||||
VARIANCENXM(8, 8, 6)
|
||||
VARIANCENXM(8, 16, 7)
|
||||
VARIANCENXM(16, 8, 7)
|
||||
VARIANCENXM(16, 16, 8)
|
||||
VARIANCENXM(16, 32, 9)
|
||||
VARIANCENXM(32, 16, 9)
|
||||
VARIANCENXM(32, 32, 10)
|
||||
|
||||
unsigned int vpx_variance32x64_neon(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *ref_ptr, int ref_stride,
|
||||
@@ -264,117 +357,165 @@ unsigned int vpx_variance64x64_neon(const uint8_t *src_ptr, int src_stride,
|
||||
return *sse - (unsigned int)(((int64_t)sum1 * sum1) >> 12);
|
||||
}
|
||||
|
||||
#if defined(__ARM_FEATURE_DOTPROD) && (__ARM_FEATURE_DOTPROD == 1)
|
||||
|
||||
unsigned int vpx_mse16x16_neon(const unsigned char *src_ptr, int src_stride,
|
||||
const unsigned char *ref_ptr, int ref_stride,
|
||||
unsigned int *sse) {
|
||||
int i;
|
||||
int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16;
|
||||
int64x1_t d0s64;
|
||||
uint8x16_t q0u8, q1u8, q2u8, q3u8;
|
||||
int32x4_t q7s32, q8s32, q9s32, q10s32;
|
||||
uint16x8_t q11u16, q12u16, q13u16, q14u16;
|
||||
int64x2_t q1s64;
|
||||
uint8x16_t a[2], b[2], abs_diff[2];
|
||||
uint32x4_t sse_vec[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
|
||||
|
||||
q7s32 = vdupq_n_s32(0);
|
||||
q8s32 = vdupq_n_s32(0);
|
||||
q9s32 = vdupq_n_s32(0);
|
||||
q10s32 = vdupq_n_s32(0);
|
||||
|
||||
for (i = 0; i < 8; i++) { // mse16x16_neon_loop
|
||||
q0u8 = vld1q_u8(src_ptr);
|
||||
for (i = 0; i < 8; i++) {
|
||||
a[0] = vld1q_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
q1u8 = vld1q_u8(src_ptr);
|
||||
a[1] = vld1q_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
q2u8 = vld1q_u8(ref_ptr);
|
||||
b[0] = vld1q_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
q3u8 = vld1q_u8(ref_ptr);
|
||||
b[1] = vld1q_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
|
||||
q11u16 = vsubl_u8(vget_low_u8(q0u8), vget_low_u8(q2u8));
|
||||
q12u16 = vsubl_u8(vget_high_u8(q0u8), vget_high_u8(q2u8));
|
||||
q13u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q3u8));
|
||||
q14u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q3u8));
|
||||
abs_diff[0] = vabdq_u8(a[0], b[0]);
|
||||
abs_diff[1] = vabdq_u8(a[1], b[1]);
|
||||
|
||||
d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
|
||||
d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
|
||||
q7s32 = vmlal_s16(q7s32, d22s16, d22s16);
|
||||
q8s32 = vmlal_s16(q8s32, d23s16, d23s16);
|
||||
|
||||
d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
|
||||
d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
|
||||
q9s32 = vmlal_s16(q9s32, d24s16, d24s16);
|
||||
q10s32 = vmlal_s16(q10s32, d25s16, d25s16);
|
||||
|
||||
d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
|
||||
d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
|
||||
q7s32 = vmlal_s16(q7s32, d26s16, d26s16);
|
||||
q8s32 = vmlal_s16(q8s32, d27s16, d27s16);
|
||||
|
||||
d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16));
|
||||
d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16));
|
||||
q9s32 = vmlal_s16(q9s32, d28s16, d28s16);
|
||||
q10s32 = vmlal_s16(q10s32, d29s16, d29s16);
|
||||
sse_vec[0] = vdotq_u32(sse_vec[0], abs_diff[0], abs_diff[0]);
|
||||
sse_vec[1] = vdotq_u32(sse_vec[1], abs_diff[1], abs_diff[1]);
|
||||
}
|
||||
|
||||
q7s32 = vaddq_s32(q7s32, q8s32);
|
||||
q9s32 = vaddq_s32(q9s32, q10s32);
|
||||
q10s32 = vaddq_s32(q7s32, q9s32);
|
||||
|
||||
q1s64 = vpaddlq_s32(q10s32);
|
||||
d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
|
||||
|
||||
vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d0s64), 0);
|
||||
return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0);
|
||||
*sse = horizontal_add_uint32x4(vaddq_u32(sse_vec[0], sse_vec[1]));
|
||||
return horizontal_add_uint32x4(vaddq_u32(sse_vec[0], sse_vec[1]));
|
||||
}
|
||||
|
||||
unsigned int vpx_get4x4sse_cs_neon(const unsigned char *src_ptr, int src_stride,
|
||||
const unsigned char *ref_ptr,
|
||||
int ref_stride) {
|
||||
int16x4_t d22s16, d24s16, d26s16, d28s16;
|
||||
int64x1_t d0s64;
|
||||
uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8;
|
||||
int32x4_t q7s32, q8s32, q9s32, q10s32;
|
||||
uint16x8_t q11u16, q12u16, q13u16, q14u16;
|
||||
int64x2_t q1s64;
|
||||
uint8x8_t a[4], b[4], abs_diff[4];
|
||||
uint32x2_t sse = vdup_n_u32(0);
|
||||
|
||||
d0u8 = vld1_u8(src_ptr);
|
||||
a[0] = vld1_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
d4u8 = vld1_u8(ref_ptr);
|
||||
b[0] = vld1_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
d1u8 = vld1_u8(src_ptr);
|
||||
a[1] = vld1_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
d5u8 = vld1_u8(ref_ptr);
|
||||
b[1] = vld1_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
d2u8 = vld1_u8(src_ptr);
|
||||
a[2] = vld1_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
d6u8 = vld1_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
d3u8 = vld1_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
d7u8 = vld1_u8(ref_ptr);
|
||||
b[2] = vld1_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
a[3] = vld1_u8(src_ptr);
|
||||
b[3] = vld1_u8(ref_ptr);
|
||||
|
||||
q11u16 = vsubl_u8(d0u8, d4u8);
|
||||
q12u16 = vsubl_u8(d1u8, d5u8);
|
||||
q13u16 = vsubl_u8(d2u8, d6u8);
|
||||
q14u16 = vsubl_u8(d3u8, d7u8);
|
||||
abs_diff[0] = vabd_u8(a[0], b[0]);
|
||||
abs_diff[1] = vabd_u8(a[1], b[1]);
|
||||
abs_diff[2] = vabd_u8(a[2], b[2]);
|
||||
abs_diff[3] = vabd_u8(a[3], b[3]);
|
||||
|
||||
d22s16 = vget_low_s16(vreinterpretq_s16_u16(q11u16));
|
||||
d24s16 = vget_low_s16(vreinterpretq_s16_u16(q12u16));
|
||||
d26s16 = vget_low_s16(vreinterpretq_s16_u16(q13u16));
|
||||
d28s16 = vget_low_s16(vreinterpretq_s16_u16(q14u16));
|
||||
sse = vdot_u32(sse, abs_diff[0], abs_diff[0]);
|
||||
sse = vdot_u32(sse, abs_diff[1], abs_diff[1]);
|
||||
sse = vdot_u32(sse, abs_diff[2], abs_diff[2]);
|
||||
sse = vdot_u32(sse, abs_diff[3], abs_diff[3]);
|
||||
|
||||
q7s32 = vmull_s16(d22s16, d22s16);
|
||||
q8s32 = vmull_s16(d24s16, d24s16);
|
||||
q9s32 = vmull_s16(d26s16, d26s16);
|
||||
q10s32 = vmull_s16(d28s16, d28s16);
|
||||
|
||||
q7s32 = vaddq_s32(q7s32, q8s32);
|
||||
q9s32 = vaddq_s32(q9s32, q10s32);
|
||||
q9s32 = vaddq_s32(q7s32, q9s32);
|
||||
|
||||
q1s64 = vpaddlq_s32(q9s32);
|
||||
d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
|
||||
|
||||
return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0);
|
||||
return vget_lane_u32(sse, 0);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
unsigned int vpx_mse16x16_neon(const unsigned char *src_ptr, int src_stride,
|
||||
const unsigned char *ref_ptr, int ref_stride,
|
||||
unsigned int *sse) {
|
||||
int i;
|
||||
uint8x16_t a[2], b[2];
|
||||
int16x4_t diff_lo[4], diff_hi[4];
|
||||
uint16x8_t diff[4];
|
||||
int32x4_t sse_vec[4] = { vdupq_n_s32(0), vdupq_n_s32(0), vdupq_n_s32(0),
|
||||
vdupq_n_s32(0) };
|
||||
|
||||
for (i = 0; i < 8; i++) {
|
||||
a[0] = vld1q_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
a[1] = vld1q_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
b[0] = vld1q_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
b[1] = vld1q_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
|
||||
diff[0] = vsubl_u8(vget_low_u8(a[0]), vget_low_u8(b[0]));
|
||||
diff[1] = vsubl_u8(vget_high_u8(a[0]), vget_high_u8(b[0]));
|
||||
diff[2] = vsubl_u8(vget_low_u8(a[1]), vget_low_u8(b[1]));
|
||||
diff[3] = vsubl_u8(vget_high_u8(a[1]), vget_high_u8(b[1]));
|
||||
|
||||
diff_lo[0] = vreinterpret_s16_u16(vget_low_u16(diff[0]));
|
||||
diff_lo[1] = vreinterpret_s16_u16(vget_low_u16(diff[1]));
|
||||
sse_vec[0] = vmlal_s16(sse_vec[0], diff_lo[0], diff_lo[0]);
|
||||
sse_vec[1] = vmlal_s16(sse_vec[1], diff_lo[1], diff_lo[1]);
|
||||
|
||||
diff_lo[2] = vreinterpret_s16_u16(vget_low_u16(diff[2]));
|
||||
diff_lo[3] = vreinterpret_s16_u16(vget_low_u16(diff[3]));
|
||||
sse_vec[2] = vmlal_s16(sse_vec[2], diff_lo[2], diff_lo[2]);
|
||||
sse_vec[3] = vmlal_s16(sse_vec[3], diff_lo[3], diff_lo[3]);
|
||||
|
||||
diff_hi[0] = vreinterpret_s16_u16(vget_high_u16(diff[0]));
|
||||
diff_hi[1] = vreinterpret_s16_u16(vget_high_u16(diff[1]));
|
||||
sse_vec[0] = vmlal_s16(sse_vec[0], diff_hi[0], diff_hi[0]);
|
||||
sse_vec[1] = vmlal_s16(sse_vec[1], diff_hi[1], diff_hi[1]);
|
||||
|
||||
diff_hi[2] = vreinterpret_s16_u16(vget_high_u16(diff[2]));
|
||||
diff_hi[3] = vreinterpret_s16_u16(vget_high_u16(diff[3]));
|
||||
sse_vec[2] = vmlal_s16(sse_vec[2], diff_hi[2], diff_hi[2]);
|
||||
sse_vec[3] = vmlal_s16(sse_vec[3], diff_hi[3], diff_hi[3]);
|
||||
}
|
||||
|
||||
sse_vec[0] = vaddq_s32(sse_vec[0], sse_vec[1]);
|
||||
sse_vec[2] = vaddq_s32(sse_vec[2], sse_vec[3]);
|
||||
sse_vec[0] = vaddq_s32(sse_vec[0], sse_vec[2]);
|
||||
|
||||
*sse = horizontal_add_uint32x4(vreinterpretq_u32_s32(sse_vec[0]));
|
||||
return horizontal_add_uint32x4(vreinterpretq_u32_s32(sse_vec[0]));
|
||||
}
|
||||
|
||||
unsigned int vpx_get4x4sse_cs_neon(const unsigned char *src_ptr, int src_stride,
|
||||
const unsigned char *ref_ptr,
|
||||
int ref_stride) {
|
||||
uint8x8_t a[4], b[4];
|
||||
int16x4_t diff_lo[4];
|
||||
uint16x8_t diff[4];
|
||||
int32x4_t sse;
|
||||
|
||||
a[0] = vld1_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
b[0] = vld1_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
a[1] = vld1_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
b[1] = vld1_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
a[2] = vld1_u8(src_ptr);
|
||||
src_ptr += src_stride;
|
||||
b[2] = vld1_u8(ref_ptr);
|
||||
ref_ptr += ref_stride;
|
||||
a[3] = vld1_u8(src_ptr);
|
||||
b[3] = vld1_u8(ref_ptr);
|
||||
|
||||
diff[0] = vsubl_u8(a[0], b[0]);
|
||||
diff[1] = vsubl_u8(a[1], b[1]);
|
||||
diff[2] = vsubl_u8(a[2], b[2]);
|
||||
diff[3] = vsubl_u8(a[3], b[3]);
|
||||
|
||||
diff_lo[0] = vget_low_s16(vreinterpretq_s16_u16(diff[0]));
|
||||
diff_lo[1] = vget_low_s16(vreinterpretq_s16_u16(diff[1]));
|
||||
diff_lo[2] = vget_low_s16(vreinterpretq_s16_u16(diff[2]));
|
||||
diff_lo[3] = vget_low_s16(vreinterpretq_s16_u16(diff[3]));
|
||||
|
||||
sse = vmull_s16(diff_lo[0], diff_lo[0]);
|
||||
sse = vmlal_s16(sse, diff_lo[1], diff_lo[1]);
|
||||
sse = vmlal_s16(sse, diff_lo[2], diff_lo[2]);
|
||||
sse = vmlal_s16(sse, diff_lo[3], diff_lo[3]);
|
||||
|
||||
return horizontal_add_uint32x4(vreinterpretq_u32_s32(sse));
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -14,6 +14,7 @@
|
||||
#include "./vpx_config.h"
|
||||
#include "./vpx_dsp_rtcd.h"
|
||||
#include "vpx/vpx_integer.h"
|
||||
#include "vpx_dsp/arm/mem_neon.h"
|
||||
#include "vpx_dsp/arm/transpose_neon.h"
|
||||
#include "vpx_dsp/arm/vpx_convolve8_neon.h"
|
||||
#include "vpx_ports/mem.h"
|
||||
@@ -30,6 +31,741 @@
|
||||
// instructions. This optimization is much faster in speed unit test, but slowed
|
||||
// down the whole decoder by 5%.
|
||||
|
||||
#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
|
||||
(__ARM_FEATURE_DOTPROD == 1)
|
||||
DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = {
|
||||
0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
|
||||
4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
|
||||
8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
|
||||
};
|
||||
|
||||
DECLARE_ALIGNED(16, static const uint8_t, dot_prod_tran_concat_tbl[32]) = {
|
||||
0, 8, 16, 24, 1, 9, 17, 25, 2, 10, 18, 26, 3, 11, 19, 27,
|
||||
4, 12, 20, 28, 5, 13, 21, 29, 6, 14, 22, 30, 7, 15, 23, 31
|
||||
};
|
||||
|
||||
DECLARE_ALIGNED(16, static const uint8_t, dot_prod_merge_block_tbl[48]) = {
|
||||
/* Shift left and insert new last column in transposed 4x4 block. */
|
||||
1, 2, 3, 16, 5, 6, 7, 20, 9, 10, 11, 24, 13, 14, 15, 28,
|
||||
/* Shift left and insert two new columns in transposed 4x4 block. */
|
||||
2, 3, 16, 17, 6, 7, 20, 21, 10, 11, 24, 25, 14, 15, 28, 29,
|
||||
/* Shift left and insert three new columns in transposed 4x4 block. */
|
||||
3, 16, 17, 18, 7, 20, 21, 22, 11, 24, 25, 26, 15, 28, 29, 30
|
||||
};
|
||||
|
||||
static INLINE void transpose_concat_4x4(int8x8_t *a0, int8x8_t *a1,
|
||||
int8x8_t *a2, int8x8_t *a3,
|
||||
int8x16_t *b,
|
||||
const uint8x16_t permute_tbl) {
|
||||
/* Transpose 8-bit elements and concatenate result rows as follows:
|
||||
* a0: 00, 01, 02, 03, XX, XX, XX, XX
|
||||
* a1: 10, 11, 12, 13, XX, XX, XX, XX
|
||||
* a2: 20, 21, 22, 23, XX, XX, XX, XX
|
||||
* a3: 30, 31, 32, 33, XX, XX, XX, XX
|
||||
*
|
||||
* b: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
|
||||
*
|
||||
* The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
|
||||
* as an argument is preferable to loading it directly from memory as this
|
||||
* inline helper is called many times from the same parent function.
|
||||
*/
|
||||
|
||||
int8x16x2_t samples = { { vcombine_s8(*a0, *a1), vcombine_s8(*a2, *a3) } };
|
||||
*b = vqtbl2q_s8(samples, permute_tbl);
|
||||
}
|
||||
|
||||
static INLINE void transpose_concat_8x4(int8x8_t *a0, int8x8_t *a1,
|
||||
int8x8_t *a2, int8x8_t *a3,
|
||||
int8x16_t *b0, int8x16_t *b1,
|
||||
const uint8x16x2_t permute_tbl) {
|
||||
/* Transpose 8-bit elements and concatenate result rows as follows:
|
||||
* a0: 00, 01, 02, 03, 04, 05, 06, 07
|
||||
* a1: 10, 11, 12, 13, 14, 15, 16, 17
|
||||
* a2: 20, 21, 22, 23, 24, 25, 26, 27
|
||||
* a3: 30, 31, 32, 33, 34, 35, 36, 37
|
||||
*
|
||||
* b0: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
|
||||
* b1: 04, 14, 24, 34, 05, 15, 25, 35, 06, 16, 26, 36, 07, 17, 27, 37
|
||||
*
|
||||
* The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
|
||||
* as an argument is preferable to loading it directly from memory as this
|
||||
* inline helper is called many times from the same parent function.
|
||||
*/
|
||||
|
||||
int8x16x2_t samples = { { vcombine_s8(*a0, *a1), vcombine_s8(*a2, *a3) } };
|
||||
*b0 = vqtbl2q_s8(samples, permute_tbl.val[0]);
|
||||
*b1 = vqtbl2q_s8(samples, permute_tbl.val[1]);
|
||||
}
|
||||
|
||||
void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w,
|
||||
int h) {
|
||||
const int8x8_t filters = vmovn_s16(vld1q_s16(filter[x0_q4]));
|
||||
const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[x0_q4]), 128);
|
||||
const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
|
||||
const uint8x16_t range_limit = vdupq_n_u8(128);
|
||||
uint8x16_t s0, s1, s2, s3;
|
||||
|
||||
assert(!((intptr_t)dst & 3));
|
||||
assert(!(dst_stride & 3));
|
||||
assert(x_step_q4 == 16);
|
||||
|
||||
(void)x_step_q4;
|
||||
(void)y0_q4;
|
||||
(void)y_step_q4;
|
||||
|
||||
src -= 3;
|
||||
|
||||
if (w == 4) {
|
||||
const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
|
||||
do {
|
||||
int32x4_t t0, t1, t2, t3;
|
||||
int16x8_t t01, t23;
|
||||
uint8x8_t d01, d23;
|
||||
|
||||
s0 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
s1 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
s2 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
s3 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
|
||||
t0 = convolve8_4_dot(s0, filters, correction, range_limit, permute_tbl);
|
||||
t1 = convolve8_4_dot(s1, filters, correction, range_limit, permute_tbl);
|
||||
t2 = convolve8_4_dot(s2, filters, correction, range_limit, permute_tbl);
|
||||
t3 = convolve8_4_dot(s3, filters, correction, range_limit, permute_tbl);
|
||||
|
||||
t01 = vcombine_s16(vqmovn_s32(t0), vqmovn_s32(t1));
|
||||
t23 = vcombine_s16(vqmovn_s32(t2), vqmovn_s32(t3));
|
||||
d01 = vqrshrun_n_s16(t01, 7);
|
||||
d23 = vqrshrun_n_s16(t23, 7);
|
||||
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
|
||||
dst += dst_stride;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
} else {
|
||||
const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
|
||||
const uint8_t *s;
|
||||
uint8_t *d;
|
||||
int width;
|
||||
uint8x8_t d0, d1, d2, d3;
|
||||
|
||||
do {
|
||||
width = w;
|
||||
s = src;
|
||||
d = dst;
|
||||
do {
|
||||
s0 = vld1q_u8(s + 0 * src_stride);
|
||||
s1 = vld1q_u8(s + 1 * src_stride);
|
||||
s2 = vld1q_u8(s + 2 * src_stride);
|
||||
s3 = vld1q_u8(s + 3 * src_stride);
|
||||
|
||||
d0 = convolve8_8_dot(s0, filters, correction, range_limit, permute_tbl);
|
||||
d1 = convolve8_8_dot(s1, filters, correction, range_limit, permute_tbl);
|
||||
d2 = convolve8_8_dot(s2, filters, correction, range_limit, permute_tbl);
|
||||
d3 = convolve8_8_dot(s3, filters, correction, range_limit, permute_tbl);
|
||||
|
||||
vst1_u8(d + 0 * dst_stride, d0);
|
||||
vst1_u8(d + 1 * dst_stride, d1);
|
||||
vst1_u8(d + 2 * dst_stride, d2);
|
||||
vst1_u8(d + 3 * dst_stride, d3);
|
||||
|
||||
s += 8;
|
||||
d += 8;
|
||||
width -= 8;
|
||||
} while (width > 0);
|
||||
src += 4 * src_stride;
|
||||
dst += 4 * dst_stride;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4,
|
||||
int w, int h) {
|
||||
const int8x8_t filters = vmovn_s16(vld1q_s16(filter[x0_q4]));
|
||||
const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[x0_q4]), 128);
|
||||
const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
|
||||
const uint8x16_t range_limit = vdupq_n_u8(128);
|
||||
uint8x16_t s0, s1, s2, s3;
|
||||
|
||||
assert(!((intptr_t)dst & 3));
|
||||
assert(!(dst_stride & 3));
|
||||
assert(x_step_q4 == 16);
|
||||
|
||||
(void)x_step_q4;
|
||||
(void)y0_q4;
|
||||
(void)y_step_q4;
|
||||
|
||||
src -= 3;
|
||||
|
||||
if (w == 4) {
|
||||
const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
|
||||
do {
|
||||
int32x4_t t0, t1, t2, t3;
|
||||
int16x8_t t01, t23;
|
||||
uint8x8_t d01, d23, dd01, dd23;
|
||||
dd01 = vdup_n_u8(0);
|
||||
dd23 = vdup_n_u8(0);
|
||||
|
||||
s0 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
s1 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
s2 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
s3 = vld1q_u8(src);
|
||||
src += src_stride;
|
||||
|
||||
t0 = convolve8_4_dot(s0, filters, correction, range_limit, permute_tbl);
|
||||
t1 = convolve8_4_dot(s1, filters, correction, range_limit, permute_tbl);
|
||||
t2 = convolve8_4_dot(s2, filters, correction, range_limit, permute_tbl);
|
||||
t3 = convolve8_4_dot(s3, filters, correction, range_limit, permute_tbl);
|
||||
|
||||
t01 = vcombine_s16(vqmovn_s32(t0), vqmovn_s32(t1));
|
||||
t23 = vcombine_s16(vqmovn_s32(t2), vqmovn_s32(t3));
|
||||
d01 = vqrshrun_n_s16(t01, 7);
|
||||
d23 = vqrshrun_n_s16(t23, 7);
|
||||
|
||||
dd01 = load_u8(dst + 0 * dst_stride, dst_stride);
|
||||
dd23 = load_u8(dst + 2 * dst_stride, dst_stride);
|
||||
d01 = vrhadd_u8(d01, dd01);
|
||||
d23 = vrhadd_u8(d23, dd23);
|
||||
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
|
||||
dst += dst_stride;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
} else {
|
||||
const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
|
||||
const uint8_t *s;
|
||||
uint8_t *d;
|
||||
int width;
|
||||
uint8x8_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
|
||||
|
||||
do {
|
||||
width = w;
|
||||
s = src;
|
||||
d = dst;
|
||||
do {
|
||||
s0 = vld1q_u8(s + 0 * src_stride);
|
||||
s1 = vld1q_u8(s + 1 * src_stride);
|
||||
s2 = vld1q_u8(s + 2 * src_stride);
|
||||
s3 = vld1q_u8(s + 3 * src_stride);
|
||||
|
||||
d0 = convolve8_8_dot(s0, filters, correction, range_limit, permute_tbl);
|
||||
d1 = convolve8_8_dot(s1, filters, correction, range_limit, permute_tbl);
|
||||
d2 = convolve8_8_dot(s2, filters, correction, range_limit, permute_tbl);
|
||||
d3 = convolve8_8_dot(s3, filters, correction, range_limit, permute_tbl);
|
||||
|
||||
dd0 = vld1_u8(d + 0 * dst_stride);
|
||||
dd1 = vld1_u8(d + 1 * dst_stride);
|
||||
dd2 = vld1_u8(d + 2 * dst_stride);
|
||||
dd3 = vld1_u8(d + 3 * dst_stride);
|
||||
d0 = vrhadd_u8(d0, dd0);
|
||||
d1 = vrhadd_u8(d1, dd1);
|
||||
d2 = vrhadd_u8(d2, dd2);
|
||||
d3 = vrhadd_u8(d3, dd3);
|
||||
|
||||
vst1_u8(d + 0 * dst_stride, d0);
|
||||
vst1_u8(d + 1 * dst_stride, d1);
|
||||
vst1_u8(d + 2 * dst_stride, d2);
|
||||
vst1_u8(d + 3 * dst_stride, d3);
|
||||
|
||||
s += 8;
|
||||
d += 8;
|
||||
width -= 8;
|
||||
} while (width > 0);
|
||||
src += 4 * src_stride;
|
||||
dst += 4 * dst_stride;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w,
|
||||
int h) {
|
||||
const int8x8_t filters = vmovn_s16(vld1q_s16(filter[y0_q4]));
|
||||
const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[y0_q4]), 128);
|
||||
const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
|
||||
const uint8x8_t range_limit = vdup_n_u8(128);
|
||||
const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(dot_prod_merge_block_tbl);
|
||||
uint8x8_t t0, t1, t2, t3, t4, t5, t6;
|
||||
int8x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
|
||||
int8x16x2_t samples_LUT;
|
||||
|
||||
assert(!((intptr_t)dst & 3));
|
||||
assert(!(dst_stride & 3));
|
||||
assert(y_step_q4 == 16);
|
||||
|
||||
(void)x0_q4;
|
||||
(void)x_step_q4;
|
||||
(void)y_step_q4;
|
||||
|
||||
src -= 3 * src_stride;
|
||||
|
||||
if (w == 4) {
|
||||
const uint8x16_t tran_concat_tbl = vld1q_u8(dot_prod_tran_concat_tbl);
|
||||
int8x16_t s0123, s1234, s2345, s3456, s4567, s5678, s6789, s78910;
|
||||
int32x4_t d0, d1, d2, d3;
|
||||
uint8x8_t d01, d23;
|
||||
|
||||
load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
|
||||
src += 4 * src_stride;
|
||||
t4 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
t5 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
t6 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
|
||||
/* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
|
||||
s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
|
||||
s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
|
||||
s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
|
||||
s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
|
||||
s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
|
||||
s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
|
||||
s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
|
||||
s7 = vdup_n_s8(0);
|
||||
s8 = vdup_n_s8(0);
|
||||
s9 = vdup_n_s8(0);
|
||||
|
||||
/* This operation combines a conventional transpose and the sample permute
|
||||
* (see horizontal case) required before computing the dot product.
|
||||
*/
|
||||
transpose_concat_4x4(&s0, &s1, &s2, &s3, &s0123, tran_concat_tbl);
|
||||
transpose_concat_4x4(&s1, &s2, &s3, &s4, &s1234, tran_concat_tbl);
|
||||
transpose_concat_4x4(&s2, &s3, &s4, &s5, &s2345, tran_concat_tbl);
|
||||
transpose_concat_4x4(&s3, &s4, &s5, &s6, &s3456, tran_concat_tbl);
|
||||
transpose_concat_4x4(&s4, &s5, &s6, &s7, &s4567, tran_concat_tbl);
|
||||
transpose_concat_4x4(&s5, &s6, &s7, &s8, &s5678, tran_concat_tbl);
|
||||
transpose_concat_4x4(&s6, &s7, &s8, &s9, &s6789, tran_concat_tbl);
|
||||
|
||||
do {
|
||||
uint8x8_t t7, t8, t9, t10;
|
||||
|
||||
load_u8_8x4(src, src_stride, &t7, &t8, &t9, &t10);
|
||||
|
||||
s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
|
||||
s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
|
||||
s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
|
||||
s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
|
||||
|
||||
transpose_concat_4x4(&s7, &s8, &s9, &s10, &s78910, tran_concat_tbl);
|
||||
|
||||
/* Merge new data into block from previous iteration. */
|
||||
samples_LUT.val[0] = s3456;
|
||||
samples_LUT.val[1] = s78910;
|
||||
s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
|
||||
s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
|
||||
s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
|
||||
|
||||
d0 = convolve8_4_dot_partial(s0123, s4567, correction, filters);
|
||||
d1 = convolve8_4_dot_partial(s1234, s5678, correction, filters);
|
||||
d2 = convolve8_4_dot_partial(s2345, s6789, correction, filters);
|
||||
d3 = convolve8_4_dot_partial(s3456, s78910, correction, filters);
|
||||
|
||||
d01 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d0), vqmovn_s32(d1)), 7);
|
||||
d23 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d2), vqmovn_s32(d3)), 7);
|
||||
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
|
||||
dst += dst_stride;
|
||||
|
||||
/* Prepare block for next iteration - re-using as much as possible. */
|
||||
/* Shuffle everything up four rows. */
|
||||
s0123 = s4567;
|
||||
s1234 = s5678;
|
||||
s2345 = s6789;
|
||||
s3456 = s78910;
|
||||
|
||||
src += 4 * src_stride;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
} else {
|
||||
const uint8x16x2_t tran_concat_tbl = vld1q_u8_x2(dot_prod_tran_concat_tbl);
|
||||
int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi,
|
||||
s3456_lo, s3456_hi, s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo,
|
||||
s6789_hi, s78910_lo, s78910_hi;
|
||||
uint8x8_t d0, d1, d2, d3;
|
||||
const uint8_t *s;
|
||||
uint8_t *d;
|
||||
int height;
|
||||
|
||||
do {
|
||||
height = h;
|
||||
s = src;
|
||||
d = dst;
|
||||
|
||||
load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
|
||||
s += 4 * src_stride;
|
||||
t4 = vld1_u8(s);
|
||||
s += src_stride;
|
||||
t5 = vld1_u8(s);
|
||||
s += src_stride;
|
||||
t6 = vld1_u8(s);
|
||||
s += src_stride;
|
||||
|
||||
/* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
|
||||
s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
|
||||
s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
|
||||
s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
|
||||
s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
|
||||
s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
|
||||
s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
|
||||
s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
|
||||
s7 = vdup_n_s8(0);
|
||||
s8 = vdup_n_s8(0);
|
||||
s9 = vdup_n_s8(0);
|
||||
|
||||
/* This operation combines a conventional transpose and the sample permute
|
||||
* (see horizontal case) required before computing the dot product.
|
||||
*/
|
||||
transpose_concat_8x4(&s0, &s1, &s2, &s3, &s0123_lo, &s0123_hi,
|
||||
tran_concat_tbl);
|
||||
transpose_concat_8x4(&s1, &s2, &s3, &s4, &s1234_lo, &s1234_hi,
|
||||
tran_concat_tbl);
|
||||
transpose_concat_8x4(&s2, &s3, &s4, &s5, &s2345_lo, &s2345_hi,
|
||||
tran_concat_tbl);
|
||||
transpose_concat_8x4(&s3, &s4, &s5, &s6, &s3456_lo, &s3456_hi,
|
||||
tran_concat_tbl);
|
||||
transpose_concat_8x4(&s4, &s5, &s6, &s7, &s4567_lo, &s4567_hi,
|
||||
tran_concat_tbl);
|
||||
transpose_concat_8x4(&s5, &s6, &s7, &s8, &s5678_lo, &s5678_hi,
|
||||
tran_concat_tbl);
|
||||
transpose_concat_8x4(&s6, &s7, &s8, &s9, &s6789_lo, &s6789_hi,
|
||||
tran_concat_tbl);
|
||||
|
||||
do {
|
||||
uint8x8_t t7, t8, t9, t10;
|
||||
|
||||
load_u8_8x4(s, src_stride, &t7, &t8, &t9, &t10);
|
||||
|
||||
s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
|
||||
s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
|
||||
s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
|
||||
s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
|
||||
|
||||
transpose_concat_8x4(&s7, &s8, &s9, &s10, &s78910_lo, &s78910_hi,
|
||||
tran_concat_tbl);
|
||||
|
||||
/* Merge new data into block from previous iteration. */
|
||||
samples_LUT.val[0] = s3456_lo;
|
||||
samples_LUT.val[1] = s78910_lo;
|
||||
s4567_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
|
||||
s5678_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
|
||||
s6789_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
|
||||
|
||||
samples_LUT.val[0] = s3456_hi;
|
||||
samples_LUT.val[1] = s78910_hi;
|
||||
s4567_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
|
||||
s5678_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
|
||||
s6789_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
|
||||
|
||||
d0 = convolve8_8_dot_partial(s0123_lo, s4567_lo, s0123_hi, s4567_hi,
|
||||
correction, filters);
|
||||
d1 = convolve8_8_dot_partial(s1234_lo, s5678_lo, s1234_hi, s5678_hi,
|
||||
correction, filters);
|
||||
d2 = convolve8_8_dot_partial(s2345_lo, s6789_lo, s2345_hi, s6789_hi,
|
||||
correction, filters);
|
||||
d3 = convolve8_8_dot_partial(s3456_lo, s78910_lo, s3456_hi, s78910_hi,
|
||||
correction, filters);
|
||||
vst1_u8(d + 0 * dst_stride, d0);
|
||||
vst1_u8(d + 1 * dst_stride, d1);
|
||||
vst1_u8(d + 2 * dst_stride, d2);
|
||||
vst1_u8(d + 3 * dst_stride, d3);
|
||||
|
||||
/* Prepare block for next iteration - re-using as much as possible. */
|
||||
/* Shuffle everything up four rows. */
|
||||
s0123_lo = s4567_lo;
|
||||
s0123_hi = s4567_hi;
|
||||
s1234_lo = s5678_lo;
|
||||
s1234_hi = s5678_hi;
|
||||
s2345_lo = s6789_lo;
|
||||
s2345_hi = s6789_hi;
|
||||
s3456_lo = s78910_lo;
|
||||
s3456_hi = s78910_hi;
|
||||
|
||||
s += 4 * src_stride;
|
||||
d += 4 * dst_stride;
|
||||
height -= 4;
|
||||
} while (height > 0);
|
||||
src += 8;
|
||||
dst += 8;
|
||||
w -= 8;
|
||||
} while (w > 0);
|
||||
}
|
||||
}
|
||||
|
||||
void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
uint8_t *dst, ptrdiff_t dst_stride,
|
||||
const InterpKernel *filter, int x0_q4,
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w,
|
||||
int h) {
|
||||
const int8x8_t filters = vmovn_s16(vld1q_s16(filter[y0_q4]));
|
||||
const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[y0_q4]), 128);
|
||||
const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
|
||||
const uint8x8_t range_limit = vdup_n_u8(128);
|
||||
const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(dot_prod_merge_block_tbl);
|
||||
uint8x8_t t0, t1, t2, t3, t4, t5, t6;
|
||||
int8x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
|
||||
int8x16x2_t samples_LUT;
|
||||
|
||||
assert(!((intptr_t)dst & 3));
|
||||
assert(!(dst_stride & 3));
|
||||
assert(y_step_q4 == 16);
|
||||
|
||||
(void)x0_q4;
|
||||
(void)x_step_q4;
|
||||
(void)y_step_q4;
|
||||
|
||||
src -= 3 * src_stride;
|
||||
|
||||
if (w == 4) {
|
||||
const uint8x16_t tran_concat_tbl = vld1q_u8(dot_prod_tran_concat_tbl);
|
||||
int8x16_t s0123, s1234, s2345, s3456, s4567, s5678, s6789, s78910;
|
||||
int32x4_t d0, d1, d2, d3;
|
||||
uint8x8_t d01, d23, dd01, dd23;
|
||||
|
||||
load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
|
||||
src += 4 * src_stride;
|
||||
t4 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
t5 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
t6 = vld1_u8(src);
|
||||
src += src_stride;
|
||||
|
||||
/* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
|
||||
s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
|
||||
s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
|
||||
s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
|
||||
s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
|
||||
s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
|
||||
s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
|
||||
s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
|
||||
s7 = vdup_n_s8(0);
|
||||
s8 = vdup_n_s8(0);
|
||||
s9 = vdup_n_s8(0);
|
||||
|
||||
/* This operation combines a conventional transpose and the sample permute
|
||||
* (see horizontal case) required before computing the dot product.
|
||||
*/
|
||||
transpose_concat_4x4(&s0, &s1, &s2, &s3, &s0123, tran_concat_tbl);
|
||||
transpose_concat_4x4(&s1, &s2, &s3, &s4, &s1234, tran_concat_tbl);
|
||||
transpose_concat_4x4(&s2, &s3, &s4, &s5, &s2345, tran_concat_tbl);
|
||||
transpose_concat_4x4(&s3, &s4, &s5, &s6, &s3456, tran_concat_tbl);
|
||||
transpose_concat_4x4(&s4, &s5, &s6, &s7, &s4567, tran_concat_tbl);
|
||||
transpose_concat_4x4(&s5, &s6, &s7, &s8, &s5678, tran_concat_tbl);
|
||||
transpose_concat_4x4(&s6, &s7, &s8, &s9, &s6789, tran_concat_tbl);
|
||||
|
||||
do {
|
||||
uint8x8_t t7, t8, t9, t10;
|
||||
|
||||
load_u8_8x4(src, src_stride, &t7, &t8, &t9, &t10);
|
||||
|
||||
s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
|
||||
s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
|
||||
s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
|
||||
s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
|
||||
|
||||
transpose_concat_4x4(&s7, &s8, &s9, &s10, &s78910, tran_concat_tbl);
|
||||
|
||||
/* Merge new data into block from previous iteration. */
|
||||
samples_LUT.val[0] = s3456;
|
||||
samples_LUT.val[1] = s78910;
|
||||
s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
|
||||
s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
|
||||
s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
|
||||
|
||||
d0 = convolve8_4_dot_partial(s0123, s4567, correction, filters);
|
||||
d1 = convolve8_4_dot_partial(s1234, s5678, correction, filters);
|
||||
d2 = convolve8_4_dot_partial(s2345, s6789, correction, filters);
|
||||
d3 = convolve8_4_dot_partial(s3456, s78910, correction, filters);
|
||||
|
||||
d01 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d0), vqmovn_s32(d1)), 7);
|
||||
d23 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d2), vqmovn_s32(d3)), 7);
|
||||
|
||||
dd01 = load_u8(dst + 0 * dst_stride, dst_stride);
|
||||
dd23 = load_u8(dst + 2 * dst_stride, dst_stride);
|
||||
d01 = vrhadd_u8(d01, dd01);
|
||||
d23 = vrhadd_u8(d23, dd23);
|
||||
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
|
||||
dst += dst_stride;
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
|
||||
dst += dst_stride;
|
||||
|
||||
/* Prepare block for next iteration - re-using as much as possible. */
|
||||
/* Shuffle everything up four rows. */
|
||||
s0123 = s4567;
|
||||
s1234 = s5678;
|
||||
s2345 = s6789;
|
||||
s3456 = s78910;
|
||||
|
||||
src += 4 * src_stride;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
} else {
|
||||
const uint8x16x2_t tran_concat_tbl = vld1q_u8_x2(dot_prod_tran_concat_tbl);
|
||||
int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi,
|
||||
s3456_lo, s3456_hi, s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo,
|
||||
s6789_hi, s78910_lo, s78910_hi;
|
||||
uint8x8_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
|
||||
const uint8_t *s;
|
||||
uint8_t *d;
|
||||
int height;
|
||||
|
||||
do {
|
||||
height = h;
|
||||
s = src;
|
||||
d = dst;
|
||||
|
||||
load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
|
||||
s += 4 * src_stride;
|
||||
t4 = vld1_u8(s);
|
||||
s += src_stride;
|
||||
t5 = vld1_u8(s);
|
||||
s += src_stride;
|
||||
t6 = vld1_u8(s);
|
||||
s += src_stride;
|
||||
|
||||
/* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
|
||||
s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
|
||||
s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
|
||||
s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
|
||||
s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
|
||||
s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
|
||||
s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
|
||||
s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
|
||||
s7 = vdup_n_s8(0);
|
||||
s8 = vdup_n_s8(0);
|
||||
s9 = vdup_n_s8(0);
|
||||
|
||||
/* This operation combines a conventional transpose and the sample permute
|
||||
* (see horizontal case) required before computing the dot product.
|
||||
*/
|
||||
transpose_concat_8x4(&s0, &s1, &s2, &s3, &s0123_lo, &s0123_hi,
|
||||
tran_concat_tbl);
|
||||
transpose_concat_8x4(&s1, &s2, &s3, &s4, &s1234_lo, &s1234_hi,
|
||||
tran_concat_tbl);
|
||||
transpose_concat_8x4(&s2, &s3, &s4, &s5, &s2345_lo, &s2345_hi,
|
||||
tran_concat_tbl);
|
||||
transpose_concat_8x4(&s3, &s4, &s5, &s6, &s3456_lo, &s3456_hi,
|
||||
tran_concat_tbl);
|
||||
transpose_concat_8x4(&s4, &s5, &s6, &s7, &s4567_lo, &s4567_hi,
|
||||
tran_concat_tbl);
|
||||
transpose_concat_8x4(&s5, &s6, &s7, &s8, &s5678_lo, &s5678_hi,
|
||||
tran_concat_tbl);
|
||||
transpose_concat_8x4(&s6, &s7, &s8, &s9, &s6789_lo, &s6789_hi,
|
||||
tran_concat_tbl);
|
||||
|
||||
do {
|
||||
uint8x8_t t7, t8, t9, t10;
|
||||
|
||||
load_u8_8x4(s, src_stride, &t7, &t8, &t9, &t10);
|
||||
|
||||
s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
|
||||
s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
|
||||
s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
|
||||
s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
|
||||
|
||||
transpose_concat_8x4(&s7, &s8, &s9, &s10, &s78910_lo, &s78910_hi,
|
||||
tran_concat_tbl);
|
||||
|
||||
/* Merge new data into block from previous iteration. */
|
||||
samples_LUT.val[0] = s3456_lo;
|
||||
samples_LUT.val[1] = s78910_lo;
|
||||
s4567_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
|
||||
s5678_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
|
||||
s6789_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
|
||||
|
||||
samples_LUT.val[0] = s3456_hi;
|
||||
samples_LUT.val[1] = s78910_hi;
|
||||
s4567_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
|
||||
s5678_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
|
||||
s6789_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
|
||||
|
||||
d0 = convolve8_8_dot_partial(s0123_lo, s4567_lo, s0123_hi, s4567_hi,
|
||||
correction, filters);
|
||||
d1 = convolve8_8_dot_partial(s1234_lo, s5678_lo, s1234_hi, s5678_hi,
|
||||
correction, filters);
|
||||
d2 = convolve8_8_dot_partial(s2345_lo, s6789_lo, s2345_hi, s6789_hi,
|
||||
correction, filters);
|
||||
d3 = convolve8_8_dot_partial(s3456_lo, s78910_lo, s3456_hi, s78910_hi,
|
||||
correction, filters);
|
||||
|
||||
dd0 = vld1_u8(d + 0 * dst_stride);
|
||||
dd1 = vld1_u8(d + 1 * dst_stride);
|
||||
dd2 = vld1_u8(d + 2 * dst_stride);
|
||||
dd3 = vld1_u8(d + 3 * dst_stride);
|
||||
d0 = vrhadd_u8(d0, dd0);
|
||||
d1 = vrhadd_u8(d1, dd1);
|
||||
d2 = vrhadd_u8(d2, dd2);
|
||||
d3 = vrhadd_u8(d3, dd3);
|
||||
|
||||
vst1_u8(d + 0 * dst_stride, d0);
|
||||
vst1_u8(d + 1 * dst_stride, d1);
|
||||
vst1_u8(d + 2 * dst_stride, d2);
|
||||
vst1_u8(d + 3 * dst_stride, d3);
|
||||
|
||||
/* Prepare block for next iteration - re-using as much as possible. */
|
||||
/* Shuffle everything up four rows. */
|
||||
s0123_lo = s4567_lo;
|
||||
s0123_hi = s4567_hi;
|
||||
s1234_lo = s5678_lo;
|
||||
s1234_hi = s5678_hi;
|
||||
s2345_lo = s6789_lo;
|
||||
s2345_hi = s6789_hi;
|
||||
s3456_lo = s78910_lo;
|
||||
s3456_hi = s78910_hi;
|
||||
|
||||
s += 4 * src_stride;
|
||||
d += 4 * dst_stride;
|
||||
height -= 4;
|
||||
} while (height > 0);
|
||||
src += 8;
|
||||
dst += 8;
|
||||
w -= 8;
|
||||
} while (w > 0);
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static INLINE void store_u8_8x8(uint8_t *s, const ptrdiff_t p,
|
||||
const uint8x8_t s0, const uint8x8_t s1,
|
||||
const uint8x8_t s2, const uint8x8_t s3,
|
||||
@@ -145,7 +881,7 @@ void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
src += 4;
|
||||
dst += 4;
|
||||
w -= 4;
|
||||
} while (w > 0);
|
||||
} while (w != 0);
|
||||
} else {
|
||||
const int16x8_t filter3 = vdupq_lane_s16(vget_low_s16(filters), 3);
|
||||
const int16x8_t filter4 = vdupq_lane_s16(vget_high_s16(filters), 0);
|
||||
@@ -296,7 +1032,7 @@ void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
s += 8;
|
||||
d += 8;
|
||||
width -= 8;
|
||||
} while (width > 0);
|
||||
} while (width != 0);
|
||||
src += 8 * src_stride;
|
||||
dst += 8 * dst_stride;
|
||||
h -= 8;
|
||||
@@ -402,7 +1138,7 @@ void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
src += 4;
|
||||
dst += 4;
|
||||
w -= 4;
|
||||
} while (w > 0);
|
||||
} while (w != 0);
|
||||
} else {
|
||||
const int16x8_t filter3 = vdupq_lane_s16(vget_low_s16(filters), 3);
|
||||
const int16x8_t filter4 = vdupq_lane_s16(vget_high_s16(filters), 0);
|
||||
@@ -586,7 +1322,7 @@ void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
s += 8;
|
||||
d += 8;
|
||||
width -= 8;
|
||||
} while (width > 0);
|
||||
} while (width != 0);
|
||||
src += 8 * src_stride;
|
||||
dst += 8 * dst_stride;
|
||||
h -= 8;
|
||||
@@ -679,7 +1415,7 @@ void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
} while (h != 0);
|
||||
} else {
|
||||
const int16x8_t filter3 = vdupq_lane_s16(vget_low_s16(filters), 3);
|
||||
const int16x8_t filter4 = vdupq_lane_s16(vget_high_s16(filters), 0);
|
||||
@@ -759,11 +1495,11 @@ void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
height -= 4;
|
||||
} while (height > 0);
|
||||
} while (height != 0);
|
||||
src += 8;
|
||||
dst += 8;
|
||||
w -= 8;
|
||||
} while (w > 0);
|
||||
} while (w != 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -860,7 +1596,7 @@ void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
h -= 4;
|
||||
} while (h > 0);
|
||||
} while (h != 0);
|
||||
} else {
|
||||
const int16x8_t filter3 = vdupq_lane_s16(vget_low_s16(filters), 3);
|
||||
const int16x8_t filter4 = vdupq_lane_s16(vget_high_s16(filters), 0);
|
||||
@@ -950,10 +1686,12 @@ void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
s5 = s9;
|
||||
s6 = s10;
|
||||
height -= 4;
|
||||
} while (height > 0);
|
||||
} while (height != 0);
|
||||
src += 8;
|
||||
dst += 8;
|
||||
w -= 8;
|
||||
} while (w > 0);
|
||||
} while (w != 0);
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -72,6 +72,107 @@ static INLINE void load_u8_16x8(const uint8_t *s, const ptrdiff_t p,
|
||||
*s7 = vld1q_u8(s);
|
||||
}
|
||||
|
||||
#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
|
||||
(__ARM_FEATURE_DOTPROD == 1)
|
||||
|
||||
static INLINE int32x4_t convolve8_4_dot_partial(const int8x16_t samples_lo,
|
||||
const int8x16_t samples_hi,
|
||||
const int32x4_t correction,
|
||||
const int8x8_t filters) {
|
||||
/* Sample range-clamping and permutation are performed by the caller. */
|
||||
int32x4_t sum;
|
||||
|
||||
/* Accumulate dot product into 'correction' to account for range clamp. */
|
||||
sum = vdotq_lane_s32(correction, samples_lo, filters, 0);
|
||||
sum = vdotq_lane_s32(sum, samples_hi, filters, 1);
|
||||
|
||||
/* Narrowing and packing is performed by the caller. */
|
||||
return sum;
|
||||
}
|
||||
|
||||
static INLINE int32x4_t convolve8_4_dot(uint8x16_t samples,
|
||||
const int8x8_t filters,
|
||||
const int32x4_t correction,
|
||||
const uint8x16_t range_limit,
|
||||
const uint8x16x2_t permute_tbl) {
|
||||
int8x16_t clamped_samples, permuted_samples[2];
|
||||
int32x4_t sum;
|
||||
|
||||
/* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
|
||||
clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
|
||||
|
||||
/* Permute samples ready for dot product. */
|
||||
/* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
|
||||
permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
|
||||
/* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
|
||||
permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
|
||||
|
||||
/* Accumulate dot product into 'correction' to account for range clamp. */
|
||||
sum = vdotq_lane_s32(correction, permuted_samples[0], filters, 0);
|
||||
sum = vdotq_lane_s32(sum, permuted_samples[1], filters, 1);
|
||||
|
||||
/* Narrowing and packing is performed by the caller. */
|
||||
return sum;
|
||||
}
|
||||
|
||||
static INLINE uint8x8_t convolve8_8_dot_partial(const int8x16_t samples0_lo,
|
||||
const int8x16_t samples0_hi,
|
||||
const int8x16_t samples1_lo,
|
||||
const int8x16_t samples1_hi,
|
||||
const int32x4_t correction,
|
||||
const int8x8_t filters) {
|
||||
/* Sample range-clamping and permutation are performed by the caller. */
|
||||
int32x4_t sum0, sum1;
|
||||
int16x8_t sum;
|
||||
|
||||
/* Accumulate dot product into 'correction' to account for range clamp. */
|
||||
/* First 4 output values. */
|
||||
sum0 = vdotq_lane_s32(correction, samples0_lo, filters, 0);
|
||||
sum0 = vdotq_lane_s32(sum0, samples0_hi, filters, 1);
|
||||
/* Second 4 output values. */
|
||||
sum1 = vdotq_lane_s32(correction, samples1_lo, filters, 0);
|
||||
sum1 = vdotq_lane_s32(sum1, samples1_hi, filters, 1);
|
||||
|
||||
/* Narrow and re-pack. */
|
||||
sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
|
||||
return vqrshrun_n_s16(sum, 7);
|
||||
}
|
||||
|
||||
static INLINE uint8x8_t convolve8_8_dot(uint8x16_t samples,
|
||||
const int8x8_t filters,
|
||||
const int32x4_t correction,
|
||||
const uint8x16_t range_limit,
|
||||
const uint8x16x3_t permute_tbl) {
|
||||
int8x16_t clamped_samples, permuted_samples[3];
|
||||
int32x4_t sum0, sum1;
|
||||
int16x8_t sum;
|
||||
|
||||
/* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
|
||||
clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
|
||||
|
||||
/* Permute samples ready for dot product. */
|
||||
/* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
|
||||
permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
|
||||
/* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
|
||||
permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
|
||||
/* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
|
||||
permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
|
||||
|
||||
/* Accumulate dot product into 'correction' to account for range clamp. */
|
||||
/* First 4 output values. */
|
||||
sum0 = vdotq_lane_s32(correction, permuted_samples[0], filters, 0);
|
||||
sum0 = vdotq_lane_s32(sum0, permuted_samples[1], filters, 1);
|
||||
/* Second 4 output values. */
|
||||
sum1 = vdotq_lane_s32(correction, permuted_samples[1], filters, 0);
|
||||
sum1 = vdotq_lane_s32(sum1, permuted_samples[2], filters, 1);
|
||||
|
||||
/* Narrow and re-pack. */
|
||||
sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
|
||||
return vqrshrun_n_s16(sum, 7);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
static INLINE int16x4_t convolve8_4(const int16x4_t s0, const int16x4_t s1,
|
||||
const int16x4_t s2, const int16x4_t s3,
|
||||
const int16x4_t s4, const int16x4_t s5,
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
} \
|
||||
}
|
||||
|
||||
DEFINE_FILTER(horiz);
|
||||
DEFINE_FILTER(avg_horiz);
|
||||
DEFINE_FILTER(vert);
|
||||
DEFINE_FILTER(avg_vert);
|
||||
DEFINE_FILTER(horiz)
|
||||
DEFINE_FILTER(avg_horiz)
|
||||
DEFINE_FILTER(vert)
|
||||
DEFINE_FILTER(avg_vert)
|
||||
@@ -17,13 +17,13 @@
|
||||
ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, \
|
||||
int x_step_q4, int y0_q4, int y_step_q4, int w, int h);
|
||||
|
||||
DECLARE_FILTER(horiz, type1);
|
||||
DECLARE_FILTER(avg_horiz, type1);
|
||||
DECLARE_FILTER(horiz, type2);
|
||||
DECLARE_FILTER(avg_horiz, type2);
|
||||
DECLARE_FILTER(vert, type1);
|
||||
DECLARE_FILTER(avg_vert, type1);
|
||||
DECLARE_FILTER(vert, type2);
|
||||
DECLARE_FILTER(avg_vert, type2);
|
||||
DECLARE_FILTER(horiz, type1)
|
||||
DECLARE_FILTER(avg_horiz, type1)
|
||||
DECLARE_FILTER(horiz, type2)
|
||||
DECLARE_FILTER(avg_horiz, type2)
|
||||
DECLARE_FILTER(vert, type1)
|
||||
DECLARE_FILTER(avg_vert, type1)
|
||||
DECLARE_FILTER(vert, type2)
|
||||
DECLARE_FILTER(avg_vert, type2)
|
||||
|
||||
#endif // VPX_VPX_DSP_ARM_VPX_CONVOLVE8_NEON_ASM_H_
|
||||
@@ -43,7 +43,7 @@ void vpx_convolve_avg_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(dd0), 1);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} while (h != 0);
|
||||
} else if (w == 8) { // avg8
|
||||
uint8x8_t s0, s1, d0, d1;
|
||||
uint8x16_t s01, d01;
|
||||
@@ -64,7 +64,7 @@ void vpx_convolve_avg_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
vst1_u8(dst, vget_high_u8(d01));
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} while (h != 0);
|
||||
} else if (w < 32) { // avg16
|
||||
uint8x16_t s0, s1, d0, d1;
|
||||
do {
|
||||
@@ -83,7 +83,7 @@ void vpx_convolve_avg_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
vst1q_u8(dst, d1);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} while (h != 0);
|
||||
} else if (w == 32) { // avg32
|
||||
uint8x16_t s0, s1, s2, s3, d0, d1, d2, d3;
|
||||
do {
|
||||
@@ -110,7 +110,7 @@ void vpx_convolve_avg_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
vst1q_u8(dst + 16, d3);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} while (h != 0);
|
||||
} else { // avg64
|
||||
uint8x16_t s0, s1, s2, s3, d0, d1, d2, d3;
|
||||
do {
|
||||
|
||||
@@ -33,7 +33,7 @@ void vpx_convolve_copy_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} while (h != 0);
|
||||
} else if (w == 8) { // copy8
|
||||
uint8x8_t s0, s1;
|
||||
do {
|
||||
@@ -47,7 +47,7 @@ void vpx_convolve_copy_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
vst1_u8(dst, s1);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} while (h != 0);
|
||||
} else if (w < 32) { // copy16
|
||||
uint8x16_t s0, s1;
|
||||
do {
|
||||
@@ -61,7 +61,7 @@ void vpx_convolve_copy_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
vst1q_u8(dst, s1);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} while (h != 0);
|
||||
} else if (w == 32) { // copy32
|
||||
uint8x16_t s0, s1, s2, s3;
|
||||
do {
|
||||
@@ -79,7 +79,7 @@ void vpx_convolve_copy_neon(const uint8_t *src, ptrdiff_t src_stride,
|
||||
vst1q_u8(dst + 16, s3);
|
||||
dst += dst_stride;
|
||||
h -= 2;
|
||||
} while (h > 0);
|
||||
} while (h != 0);
|
||||
} else { // copy64
|
||||
uint8x16_t s0, s1, s2, s3;
|
||||
do {
|
||||
|
||||
Reference in new issue
Block a user