Files
freeswitch/libs/libyuv/source/scale_win.cc
T

2622 lines
106 KiB
C++

/*
* Copyright 2013 The LibYuv Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/row.h"
#include "libyuv/scale_row.h"
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
defined(_M_IX86)) && \
((defined(_MSC_VER) && !defined(__clang__)) || \
defined(LIBYUV_ENABLE_ROWWIN))
#include <emmintrin.h>
#include <immintrin.h>
#include <string.h>
#include <tmmintrin.h>
#if defined(_MSC_VER) && !defined(__clang__)
#include <intrin.h>
#endif
#endif
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
// This module is for 32 bit Visual C x86
#if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && defined(_M_IX86) && \
!defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN)
#ifdef HAS_SCALEROWDOWN2_SSSE3
// Reads 32 pixels, throws half away and writes 16 pixels.
__declspec(naked) void ScaleRowDown2_SSSE3(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
__asm {
mov eax, [esp + 4] // src_ptr
// src_stride ignored
mov edx, [esp + 12] // dst_ptr
mov ecx, [esp + 16] // dst_width
wloop:
movdqu xmm0, [eax]
movdqu xmm1, [eax + 16]
lea eax, [eax + 32]
psrlw xmm0, 8 // isolate odd pixels.
psrlw xmm1, 8
packuswb xmm0, xmm1
movdqu [edx], xmm0
lea edx, [edx + 16]
sub ecx, 16
jg wloop
ret
}
}
// Blends 32x1 rectangle to 16x1.
__declspec(naked) void ScaleRowDown2Linear_SSSE3(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
__asm {
mov eax, [esp + 4] // src_ptr
// src_stride
mov edx, [esp + 12] // dst_ptr
mov ecx, [esp + 16] // dst_width
pcmpeqb xmm4, xmm4 // constant 0x0101
psrlw xmm4, 15
packuswb xmm4, xmm4
pxor xmm5, xmm5 // constant 0
wloop:
movdqu xmm0, [eax]
movdqu xmm1, [eax + 16]
lea eax, [eax + 32]
pmaddubsw xmm0, xmm4 // horizontal add
pmaddubsw xmm1, xmm4
pavgw xmm0, xmm5 // (x + 1) / 2
pavgw xmm1, xmm5
packuswb xmm0, xmm1
movdqu [edx], xmm0
lea edx, [edx + 16]
sub ecx, 16
jg wloop
ret
}
}
// Blends 32x2 rectangle to 16x1.
__declspec(naked) void ScaleRowDown2Box_SSSE3(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
__asm {
push esi
mov eax, [esp + 4 + 4] // src_ptr
mov esi, [esp + 4 + 8] // src_stride
mov edx, [esp + 4 + 12] // dst_ptr
mov ecx, [esp + 4 + 16] // dst_width
pcmpeqb xmm4, xmm4 // constant 0x0101
psrlw xmm4, 15
packuswb xmm4, xmm4
pxor xmm5, xmm5 // constant 0
wloop:
movdqu xmm0, [eax]
movdqu xmm1, [eax + 16]
movdqu xmm2, [eax + esi]
movdqu xmm3, [eax + esi + 16]
lea eax, [eax + 32]
pmaddubsw xmm0, xmm4 // horizontal add
pmaddubsw xmm1, xmm4
pmaddubsw xmm2, xmm4
pmaddubsw xmm3, xmm4
paddw xmm0, xmm2 // vertical add
paddw xmm1, xmm3
psrlw xmm0, 1
psrlw xmm1, 1
pavgw xmm0, xmm5 // (x + 1) / 2
pavgw xmm1, xmm5
packuswb xmm0, xmm1
movdqu [edx], xmm0
lea edx, [edx + 16]
sub ecx, 16
jg wloop
pop esi
ret
}
}
#endif // HAS_SCALEROWDOWN2_SSSE3
#ifdef HAS_SCALEROWDOWN4_SSSE3
// Point samples 32 pixels to 8 pixels.
__declspec(naked) void ScaleRowDown4_SSSE3(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
__asm {
mov eax, [esp + 4] // src_ptr
// src_stride ignored
mov edx, [esp + 12] // dst_ptr
mov ecx, [esp + 16] // dst_width
pcmpeqb xmm5, xmm5 // generate mask 0x00ff0000
psrld xmm5, 24
pslld xmm5, 16
wloop:
movdqu xmm0, [eax]
movdqu xmm1, [eax + 16]
lea eax, [eax + 32]
pand xmm0, xmm5
pand xmm1, xmm5
packuswb xmm0, xmm1
psrlw xmm0, 8
packuswb xmm0, xmm0
movq qword ptr [edx], xmm0
lea edx, [edx + 8]
sub ecx, 8
jg wloop
ret
}
}
// Blends 32x4 rectangle to 8x1.
__declspec(naked) void ScaleRowDown4Box_SSSE3(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
__asm {
push esi
push edi
mov eax, [esp + 8 + 4] // src_ptr
mov esi, [esp + 8 + 8] // src_stride
mov edx, [esp + 8 + 12] // dst_ptr
mov ecx, [esp + 8 + 16] // dst_width
lea edi, [esi + esi * 2] // src_stride * 3
pcmpeqb xmm4, xmm4 // constant 0x0101
psrlw xmm4, 15
movdqa xmm5, xmm4
packuswb xmm4, xmm4
psllw xmm5, 3 // constant 0x0008
wloop:
movdqu xmm0, [eax] // average rows
movdqu xmm1, [eax + 16]
movdqu xmm2, [eax + esi]
movdqu xmm3, [eax + esi + 16]
pmaddubsw xmm0, xmm4 // horizontal add
pmaddubsw xmm1, xmm4
pmaddubsw xmm2, xmm4
pmaddubsw xmm3, xmm4
paddw xmm0, xmm2 // vertical add rows 0, 1
paddw xmm1, xmm3
movdqu xmm2, [eax + esi * 2]
movdqu xmm3, [eax + esi * 2 + 16]
pmaddubsw xmm2, xmm4
pmaddubsw xmm3, xmm4
paddw xmm0, xmm2 // add row 2
paddw xmm1, xmm3
movdqu xmm2, [eax + edi]
movdqu xmm3, [eax + edi + 16]
lea eax, [eax + 32]
pmaddubsw xmm2, xmm4
pmaddubsw xmm3, xmm4
paddw xmm0, xmm2 // add row 3
paddw xmm1, xmm3
phaddw xmm0, xmm1
paddw xmm0, xmm5 // + 8 for round
psrlw xmm0, 4 // /16 for average of 4 * 4
packuswb xmm0, xmm0
movq qword ptr [edx], xmm0
lea edx, [edx + 8]
sub ecx, 8
jg wloop
pop edi
pop esi
ret
}
}
#endif // HAS_SCALEROWDOWN4_SSSE3
#ifdef HAS_FIXEDDIV_X86
// Divide num by div and return as 16.16 fixed point result.
__declspec(naked) int FixedDiv_X86(int num, int div) {
__asm {
mov eax, [esp + 4] // num
cdq // extend num to 64 bits
shld edx, eax, 16 // 32.16
shl eax, 16
idiv dword ptr [esp + 8]
ret
}
}
// Divide num by div and return as 16.16 fixed point result.
__declspec(naked) int FixedDiv1_X86(int num, int div) {
__asm {
mov eax, [esp + 4] // num
mov ecx, [esp + 8] // denom
cdq // extend num to 64 bits
shld edx, eax, 16 // 32.16
shl eax, 16
sub eax, 0x00010001
sbb edx, 0
sub ecx, 1
idiv ecx
ret
}
}
#endif // HAS_FIXEDDIV_X86
#endif // !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && defined(_M_IX86)
// The following are available for Visual C 32/64 bit, and for clang with
// LIBYUV_ENABLE_ROWWIN:
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
defined(_M_IX86)) && \
((defined(_MSC_VER) && !defined(__clang__)) || \
defined(LIBYUV_ENABLE_ROWWIN))
#if defined(__clang__) || defined(__GNUC__)
#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
#define LIBYUV_TARGET_AVX512BW \
__attribute__((target("avx512bw,avx512vl,avx512f")))
#define LIBYUV_TARGET_AVX512VBMI \
__attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f")))
#else
#define LIBYUV_TARGET_AVX2
#define LIBYUV_TARGET_AVX512BW
#define LIBYUV_TARGET_AVX512VBMI
#endif // defined(__clang__) || defined(__GNUC__)
static inline uint16_t LoadU16(const void* p) {
uint16_t v;
memcpy(&v, p, sizeof(v));
return v;
}
static inline uint32_t LoadU32(const void* p) {
uint32_t v;
memcpy(&v, p, sizeof(v));
return v;
}
static inline uint64_t LoadU64(const void* p) {
uint64_t v;
memcpy(&v, p, sizeof(v));
return v;
}
static inline void StoreU32(void* p, uint32_t v) {
memcpy(p, &v, sizeof(v));
}
#if defined(HAS_FIXEDDIV_X86) && \
!(defined(_MSC_VER) && defined(_M_IX86) && !defined(__clang__) && \
!defined(LIBYUV_ENABLE_ROWWIN))
int FixedDiv_X86(int num, int div) {
return (int)(((int64_t)num * 65536) / div);
}
#endif
#if defined(HAS_FIXEDDIV1_X86) && \
!(defined(_MSC_VER) && defined(_M_IX86) && !defined(__clang__) && \
!defined(LIBYUV_ENABLE_ROWWIN))
int FixedDiv1_X86(int num, int div) {
return (int)((((int64_t)num * 65536) - 0x00010001) / (div - 1));
}
#endif
#ifdef HAS_SCALEROWDOWN2_AVX2
LIBYUV_TARGET_AVX2
void ScaleRowDown2_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
(void)src_stride;
while (dst_width > 0) {
__m256i s0 = _mm256_srli_epi16(
_mm256_loadu_si256((const __m256i*)src_ptr), 8);
__m256i s1 = _mm256_srli_epi16(
_mm256_loadu_si256((const __m256i*)(src_ptr + 32)), 8);
src_ptr += 64;
__m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8);
_mm256_storeu_si256((__m256i*)dst_ptr, d);
dst_ptr += 32;
dst_width -= 32;
}
}
LIBYUV_TARGET_AVX2
void ScaleRowDown2Linear_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
(void)src_stride;
const __m256i ones = _mm256_set1_epi8(1);
const __m256i zero = _mm256_setzero_si256();
while (dst_width > 0) {
__m256i s0 = _mm256_maddubs_epi16(
_mm256_loadu_si256((const __m256i*)src_ptr), ones);
__m256i s1 = _mm256_maddubs_epi16(
_mm256_loadu_si256((const __m256i*)(src_ptr + 32)), ones);
src_ptr += 64;
s0 = _mm256_avg_epu16(s0, zero);
s1 = _mm256_avg_epu16(s1, zero);
__m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8);
_mm256_storeu_si256((__m256i*)dst_ptr, d);
dst_ptr += 32;
dst_width -= 32;
}
}
LIBYUV_TARGET_AVX2
void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
const __m256i ones = _mm256_set1_epi8(1);
const __m256i zero = _mm256_setzero_si256();
while (dst_width > 0) {
__m256i s0 = _mm256_maddubs_epi16(
_mm256_loadu_si256((const __m256i*)src_ptr), ones);
__m256i s1 = _mm256_maddubs_epi16(
_mm256_loadu_si256((const __m256i*)(src_ptr + 32)), ones);
__m256i s2 = _mm256_maddubs_epi16(
_mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)), ones);
__m256i s3 = _mm256_maddubs_epi16(
_mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32)), ones);
src_ptr += 64;
s0 = _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(s0, s2), 1), zero);
s1 = _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(s1, s3), 1), zero);
__m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8);
_mm256_storeu_si256((__m256i*)dst_ptr, d);
dst_ptr += 32;
dst_width -= 32;
}
}
#endif // HAS_SCALEROWDOWN2_AVX2
#ifdef HAS_SCALEROWDOWN2_16_AVX2
LIBYUV_TARGET_AVX2
void ScaleRowDown2_16_AVX2(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst_ptr,
int dst_width) {
(void)src_stride;
while (dst_width > 0) {
__m256i s0 = _mm256_srli_epi32(
_mm256_loadu_si256((const __m256i*)src_ptr), 16);
__m256i s1 = _mm256_srli_epi32(
_mm256_loadu_si256((const __m256i*)(src_ptr + 16)), 16);
src_ptr += 32;
__m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8);
_mm256_storeu_si256((__m256i*)dst_ptr, d);
dst_ptr += 16;
dst_width -= 16;
}
}
LIBYUV_TARGET_AVX2
void ScaleRowDown2Linear_16_AVX2(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst_ptr,
int dst_width) {
(void)src_stride;
const __m256i mask = _mm256_set1_epi32(0x0000ffff);
while (dst_width > 0) {
__m256i r0 = _mm256_loadu_si256((const __m256i*)src_ptr);
__m256i r1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 16));
src_ptr += 32;
__m256i e = _mm256_packus_epi32(_mm256_and_si256(r0, mask),
_mm256_and_si256(r1, mask));
__m256i o = _mm256_packus_epi32(_mm256_srli_epi32(r0, 16),
_mm256_srli_epi32(r1, 16));
__m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu16(e, o), 0xd8);
_mm256_storeu_si256((__m256i*)dst_ptr, d);
dst_ptr += 16;
dst_width -= 16;
}
}
LIBYUV_TARGET_AVX2
void ScaleRowDown2Box_16_AVX2(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst_ptr,
int dst_width) {
const __m256i mask = _mm256_set1_epi32(0x0000ffff);
const __m256i round2 = _mm256_set1_epi32(2);
while (dst_width > 0) {
__m256i r0 = _mm256_loadu_si256((const __m256i*)src_ptr);
__m256i r1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 16));
__m256i r2 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride));
__m256i r3 =
_mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 16));
src_ptr += 32;
__m256i s0 = _mm256_add_epi32(
_mm256_add_epi32(_mm256_and_si256(r0, mask), _mm256_srli_epi32(r0, 16)),
_mm256_add_epi32(_mm256_and_si256(r2, mask),
_mm256_srli_epi32(r2, 16)));
__m256i s1 = _mm256_add_epi32(
_mm256_add_epi32(_mm256_and_si256(r1, mask), _mm256_srli_epi32(r1, 16)),
_mm256_add_epi32(_mm256_and_si256(r3, mask),
_mm256_srli_epi32(r3, 16)));
s0 = _mm256_srli_epi32(_mm256_add_epi32(s0, round2), 2);
s1 = _mm256_srli_epi32(_mm256_add_epi32(s1, round2), 2);
__m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8);
_mm256_storeu_si256((__m256i*)dst_ptr, d);
dst_ptr += 16;
dst_width -= 16;
}
}
#endif // HAS_SCALEROWDOWN2_16_AVX2
#ifdef HAS_SCALEROWDOWN2_AVX512BW
LIBYUV_TARGET_AVX512BW
void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
(void)src_stride;
while (dst_width >= 32) {
__m512i s =
_mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)src_ptr), 8);
src_ptr += 64;
_mm256_storeu_si256((__m256i*)dst_ptr, _mm512_cvtepi16_epi8(s));
dst_ptr += 32;
dst_width -= 32;
}
if (dst_width > 0) {
__mmask32 k = (__mmask32)((1ULL << dst_width) - 1);
__m512i s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, src_ptr), 8);
_mm256_mask_storeu_epi8(dst_ptr, k, _mm512_cvtepi16_epi8(s));
}
}
LIBYUV_TARGET_AVX512BW
void ScaleRowDown2Linear_AVX512BW(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
(void)src_stride;
const __m512i ones = _mm512_set1_epi8(1);
const __m512i zero = _mm512_setzero_si512();
while (dst_width >= 32) {
__m512i s = _mm512_avg_epu16(
_mm512_maddubs_epi16(_mm512_loadu_si512((const __m512i*)src_ptr), ones),
zero);
src_ptr += 64;
_mm256_storeu_si256((__m256i*)dst_ptr, _mm512_cvtepi16_epi8(s));
dst_ptr += 32;
dst_width -= 32;
}
if (dst_width > 0) {
__mmask32 k = (__mmask32)((1ULL << dst_width) - 1);
__m512i s = _mm512_avg_epu16(
_mm512_maddubs_epi16(_mm512_maskz_loadu_epi16(k, src_ptr), ones),
zero);
_mm256_mask_storeu_epi8(dst_ptr, k, _mm512_cvtepi16_epi8(s));
}
}
LIBYUV_TARGET_AVX512BW
void ScaleRowDown2Box_AVX512BW(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
const __m512i ones = _mm512_set1_epi8(1);
const __m512i zero = _mm512_setzero_si512();
while (dst_width >= 32) {
__m512i s0 = _mm512_maddubs_epi16(
_mm512_loadu_si512((const __m512i*)src_ptr), ones);
__m512i s1 = _mm512_maddubs_epi16(
_mm512_loadu_si512((const __m512i*)(src_ptr + src_stride)), ones);
src_ptr += 64;
__m512i s =
_mm512_avg_epu16(_mm512_srli_epi16(_mm512_add_epi16(s0, s1), 1), zero);
_mm256_storeu_si256((__m256i*)dst_ptr, _mm512_cvtepi16_epi8(s));
dst_ptr += 32;
dst_width -= 32;
}
if (dst_width > 0) {
__mmask32 k = (__mmask32)((1ULL << dst_width) - 1);
__m512i s0 =
_mm512_maddubs_epi16(_mm512_maskz_loadu_epi16(k, src_ptr), ones);
__m512i s1 = _mm512_maddubs_epi16(
_mm512_maskz_loadu_epi16(k, src_ptr + src_stride), ones);
__m512i s =
_mm512_avg_epu16(_mm512_srli_epi16(_mm512_add_epi16(s0, s1), 1), zero);
_mm256_mask_storeu_epi8(dst_ptr, k, _mm512_cvtepi16_epi8(s));
}
}
LIBYUV_TARGET_AVX512BW
void ScaleRowDown2Box_Odd_AVX512BW(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
dst_width -= 1;
if (dst_width > 0) {
ScaleRowDown2Box_AVX512BW(src_ptr, src_stride, dst_ptr, dst_width);
src_ptr += dst_width * 2;
dst_ptr += dst_width;
}
dst_ptr[0] = (uint8_t)((src_ptr[0] + src_ptr[src_stride] + 1) >> 1);
}
#endif // HAS_SCALEROWDOWN2_AVX512BW
#ifdef HAS_SCALEROWDOWN4_AVX2
LIBYUV_TARGET_AVX2
void ScaleRowDown4_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
(void)src_stride;
const __m256i mask = _mm256_set1_epi32(0x00ff0000);
while (dst_width > 0) {
__m256i s0 =
_mm256_and_si256(_mm256_loadu_si256((const __m256i*)src_ptr), mask);
__m256i s1 = _mm256_and_si256(
_mm256_loadu_si256((const __m256i*)(src_ptr + 32)), mask);
src_ptr += 64;
__m256i t = _mm256_srli_epi16(
_mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8), 8);
__m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(t, t), 0xd8);
_mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d));
dst_ptr += 16;
dst_width -= 16;
}
}
LIBYUV_TARGET_AVX2
void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
const __m256i ones = _mm256_set1_epi8(1);
const __m256i round8 = _mm256_set1_epi16(8);
const ptrdiff_t stridex3 = src_stride * 3;
while (dst_width > 0) {
__m256i s0 = _mm256_maddubs_epi16(
_mm256_loadu_si256((const __m256i*)src_ptr), ones);
__m256i s1 = _mm256_maddubs_epi16(
_mm256_loadu_si256((const __m256i*)(src_ptr + 32)), ones);
s0 = _mm256_add_epi16(
s0, _mm256_maddubs_epi16(
_mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)),
ones));
s1 = _mm256_add_epi16(
s1, _mm256_maddubs_epi16(
_mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32)),
ones));
s0 = _mm256_add_epi16(
s0, _mm256_maddubs_epi16(
_mm256_loadu_si256((const __m256i*)(src_ptr + src_stride * 2)),
ones));
s1 = _mm256_add_epi16(
s1,
_mm256_maddubs_epi16(
_mm256_loadu_si256((const __m256i*)(src_ptr + src_stride * 2 + 32)),
ones));
s0 = _mm256_add_epi16(
s0,
_mm256_maddubs_epi16(
_mm256_loadu_si256((const __m256i*)(src_ptr + stridex3)), ones));
s1 = _mm256_add_epi16(
s1, _mm256_maddubs_epi16(
_mm256_loadu_si256((const __m256i*)(src_ptr + stridex3 + 32)),
ones));
src_ptr += 64;
__m256i t = _mm256_permute4x64_epi64(_mm256_hadd_epi16(s0, s1), 0xd8);
t = _mm256_srli_epi16(_mm256_add_epi16(t, round8), 4);
__m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(t, t), 0xd8);
_mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d));
dst_ptr += 16;
dst_width -= 16;
}
}
#endif // HAS_SCALEROWDOWN4_AVX2
#ifdef HAS_SCALEROWDOWN34_AVX2
static const uvec8 kShuf0 = {0, 1, 3, 4, 5, 7, 8, 9,
128, 128, 128, 128, 128, 128, 128, 128};
static const uvec8 kShuf1 = {3, 4, 5, 7, 8, 9, 11, 12,
128, 128, 128, 128, 128, 128, 128, 128};
static const uvec8 kShuf2 = {5, 7, 8, 9, 11, 12, 13, 15,
128, 128, 128, 128, 128, 128, 128, 128};
static const uvec8 kShuf01 = {0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10};
static const uvec8 kShuf11 = {2, 3, 4, 5, 5, 6, 6, 7,
8, 9, 9, 10, 10, 11, 12, 13};
static const uvec8 kShuf21 = {5, 6, 6, 7, 8, 9, 9, 10,
10, 11, 12, 13, 13, 14, 14, 15};
static const uvec8 kMadd01 = {3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2};
static const uvec8 kMadd11 = {1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1};
static const uvec8 kMadd21 = {2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3};
LIBYUV_TARGET_AVX2
void ScaleRowDown34_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
(void)src_stride;
const __m256i shuf0 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf0));
const __m256i shuf1 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf1));
const __m256i shuf2 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf2));
while (dst_width >= 48) {
__m256i s0 = _mm256_loadu_si256((const __m256i*)src_ptr);
__m256i s1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 32));
src_ptr += 64;
__m256i lo = _mm256_permute2x128_si256(s0, s1, 0x20);
__m256i hi = _mm256_permute2x128_si256(s0, s1, 0x31);
__m256i mid = _mm256_alignr_epi8(hi, lo, 8);
__m256i d0 = _mm256_shuffle_epi8(lo, shuf0);
__m256i d1 = _mm256_shuffle_epi8(mid, shuf1);
__m256i d2 = _mm256_shuffle_epi8(hi, shuf2);
__m256i d01 = _mm256_unpacklo_epi64(d0, d1);
_mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d01));
_mm_storel_epi64((__m128i*)(dst_ptr + 16), _mm256_castsi256_si128(d2));
_mm_storeu_si128((__m128i*)(dst_ptr + 24), _mm256_extracti128_si256(d01, 1));
_mm_storel_epi64((__m128i*)(dst_ptr + 40), _mm256_extracti128_si256(d2, 1));
dst_ptr += 48;
dst_width -= 48;
}
if (dst_width > 0) {
__m256i s0 = _mm256_loadu_si256((const __m256i*)src_ptr);
__m128i lo = _mm256_castsi256_si128(s0);
__m128i hi = _mm256_extracti128_si256(s0, 1);
__m128i mid = _mm_alignr_epi8(hi, lo, 8);
__m128i d0 = _mm_shuffle_epi8(lo, _mm256_castsi256_si128(shuf0));
__m128i d1 = _mm_shuffle_epi8(mid, _mm256_castsi256_si128(shuf1));
__m128i d2 = _mm_shuffle_epi8(hi, _mm256_castsi256_si128(shuf2));
_mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi64(d0, d1));
_mm_storel_epi64((__m128i*)(dst_ptr + 16), d2);
}
}
LIBYUV_TARGET_AVX2
void ScaleRowDown34_1_Box_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
const __m256i shuf01 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf01));
const __m256i shuf11 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf11));
const __m256i shuf21 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf21));
const __m256i madd01 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd01));
const __m256i madd11 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd11));
const __m256i madd21 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd21));
const __m256i round2 = _mm256_set1_epi16(2);
while (dst_width >= 48) {
__m256i s0 = _mm256_avg_epu8(
_mm256_loadu_si256((const __m256i*)src_ptr),
_mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)));
__m256i s1 = _mm256_avg_epu8(
_mm256_loadu_si256((const __m256i*)(src_ptr + 32)),
_mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32)));
src_ptr += 64;
__m256i lo = _mm256_permute2x128_si256(s0, s1, 0x20);
__m256i hi = _mm256_permute2x128_si256(s0, s1, 0x31);
__m256i mid = _mm256_alignr_epi8(hi, lo, 8);
__m256i p0 = _mm256_srli_epi16(
_mm256_adds_epi16(
_mm256_maddubs_epi16(_mm256_shuffle_epi8(lo, shuf01), madd01),
round2),
2);
__m256i p1 = _mm256_srli_epi16(
_mm256_adds_epi16(
_mm256_maddubs_epi16(_mm256_shuffle_epi8(mid, shuf11), madd11),
round2),
2);
__m256i p2 = _mm256_srli_epi16(
_mm256_adds_epi16(
_mm256_maddubs_epi16(_mm256_shuffle_epi8(hi, shuf21), madd21),
round2),
2);
p0 = _mm256_packus_epi16(p0, p0);
p1 = _mm256_packus_epi16(p1, p1);
p2 = _mm256_packus_epi16(p2, p2);
__m256i d01 = _mm256_unpacklo_epi64(p0, p1);
_mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d01));
_mm_storel_epi64((__m128i*)(dst_ptr + 16), _mm256_castsi256_si128(p2));
_mm_storeu_si128((__m128i*)(dst_ptr + 24), _mm256_extracti128_si256(d01, 1));
_mm_storel_epi64((__m128i*)(dst_ptr + 40), _mm256_extracti128_si256(p2, 1));
dst_ptr += 48;
dst_width -= 48;
}
if (dst_width > 0) {
__m256i s0 = _mm256_avg_epu8(
_mm256_loadu_si256((const __m256i*)src_ptr),
_mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)));
__m128i lo = _mm256_castsi256_si128(s0);
__m128i hi = _mm256_extracti128_si256(s0, 1);
__m128i mid = _mm_alignr_epi8(hi, lo, 8);
__m128i r2 = _mm256_castsi256_si128(round2);
__m128i p0 = _mm_srli_epi16(
_mm_adds_epi16(
_mm_maddubs_epi16(_mm_shuffle_epi8(lo, _mm256_castsi256_si128(shuf01)),
_mm256_castsi256_si128(madd01)),
r2),
2);
__m128i p1 = _mm_srli_epi16(
_mm_adds_epi16(
_mm_maddubs_epi16(
_mm_shuffle_epi8(mid, _mm256_castsi256_si128(shuf11)),
_mm256_castsi256_si128(madd11)),
r2),
2);
__m128i p2 = _mm_srli_epi16(
_mm_adds_epi16(
_mm_maddubs_epi16(_mm_shuffle_epi8(hi, _mm256_castsi256_si128(shuf21)),
_mm256_castsi256_si128(madd21)),
r2),
2);
p0 = _mm_packus_epi16(p0, p0);
p1 = _mm_packus_epi16(p1, p1);
p2 = _mm_packus_epi16(p2, p2);
_mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi64(p0, p1));
_mm_storel_epi64((__m128i*)(dst_ptr + 16), p2);
}
}
LIBYUV_TARGET_AVX2
void ScaleRowDown34_0_Box_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
const __m256i shuf01 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf01));
const __m256i shuf11 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf11));
const __m256i shuf21 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf21));
const __m256i madd01 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd01));
const __m256i madd11 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd11));
const __m256i madd21 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd21));
const __m256i round2 = _mm256_set1_epi16(2);
while (dst_width >= 48) {
__m256i a0 = _mm256_loadu_si256((const __m256i*)src_ptr);
__m256i b0 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride));
__m256i s0 = _mm256_avg_epu8(a0, _mm256_avg_epu8(a0, b0));
__m256i a1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 32));
__m256i b1 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32));
__m256i s1 = _mm256_avg_epu8(a1, _mm256_avg_epu8(a1, b1));
src_ptr += 64;
__m256i lo = _mm256_permute2x128_si256(s0, s1, 0x20);
__m256i hi = _mm256_permute2x128_si256(s0, s1, 0x31);
__m256i mid = _mm256_alignr_epi8(hi, lo, 8);
__m256i p0 = _mm256_srli_epi16(
_mm256_adds_epi16(
_mm256_maddubs_epi16(_mm256_shuffle_epi8(lo, shuf01), madd01),
round2),
2);
__m256i p1 = _mm256_srli_epi16(
_mm256_adds_epi16(
_mm256_maddubs_epi16(_mm256_shuffle_epi8(mid, shuf11), madd11),
round2),
2);
__m256i p2 = _mm256_srli_epi16(
_mm256_adds_epi16(
_mm256_maddubs_epi16(_mm256_shuffle_epi8(hi, shuf21), madd21),
round2),
2);
p0 = _mm256_packus_epi16(p0, p0);
p1 = _mm256_packus_epi16(p1, p1);
p2 = _mm256_packus_epi16(p2, p2);
__m256i d01 = _mm256_unpacklo_epi64(p0, p1);
_mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d01));
_mm_storel_epi64((__m128i*)(dst_ptr + 16), _mm256_castsi256_si128(p2));
_mm_storeu_si128((__m128i*)(dst_ptr + 24), _mm256_extracti128_si256(d01, 1));
_mm_storel_epi64((__m128i*)(dst_ptr + 40), _mm256_extracti128_si256(p2, 1));
dst_ptr += 48;
dst_width -= 48;
}
if (dst_width > 0) {
__m256i a0 = _mm256_loadu_si256((const __m256i*)src_ptr);
__m256i b0 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride));
__m256i s0 = _mm256_avg_epu8(a0, _mm256_avg_epu8(a0, b0));
__m128i lo = _mm256_castsi256_si128(s0);
__m128i hi = _mm256_extracti128_si256(s0, 1);
__m128i mid = _mm_alignr_epi8(hi, lo, 8);
__m128i r2 = _mm256_castsi256_si128(round2);
__m128i p0 = _mm_srli_epi16(
_mm_adds_epi16(
_mm_maddubs_epi16(_mm_shuffle_epi8(lo, _mm256_castsi256_si128(shuf01)),
_mm256_castsi256_si128(madd01)),
r2),
2);
__m128i p1 = _mm_srli_epi16(
_mm_adds_epi16(
_mm_maddubs_epi16(
_mm_shuffle_epi8(mid, _mm256_castsi256_si128(shuf11)),
_mm256_castsi256_si128(madd11)),
r2),
2);
__m128i p2 = _mm_srli_epi16(
_mm_adds_epi16(
_mm_maddubs_epi16(_mm_shuffle_epi8(hi, _mm256_castsi256_si128(shuf21)),
_mm256_castsi256_si128(madd21)),
r2),
2);
p0 = _mm_packus_epi16(p0, p0);
p1 = _mm_packus_epi16(p1, p1);
p2 = _mm_packus_epi16(p2, p2);
_mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi64(p0, p1));
_mm_storel_epi64((__m128i*)(dst_ptr + 16), p2);
}
}
#endif // HAS_SCALEROWDOWN34_AVX2
#ifdef HAS_SCALEROWDOWN38_AVX2
static const uvec8 kShuf38a = {0, 3, 6, 8, 11, 14, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128};
static const uvec8 kShuf38b = {128, 128, 128, 128, 128, 128, 0, 3,
6, 8, 11, 14, 128, 128, 128, 128};
static const uvec8 kShufAc = {0, 1, 6, 7, 12, 13, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128};
static const uvec8 kShufAc3 = {128, 128, 128, 128, 128, 128, 0, 1,
6, 7, 12, 13, 128, 128, 128, 128};
static const uvec16 kScaleAc33 = {65536 / 9, 65536 / 9, 65536 / 6, 65536 / 9,
65536 / 9, 65536 / 6, 0, 0};
static const uvec8 kShufAb0 = {0, 128, 3, 128, 6, 128, 8, 128,
11, 128, 14, 128, 128, 128, 128, 128};
static const uvec8 kShufAb1 = {1, 128, 4, 128, 7, 128, 9, 128,
12, 128, 15, 128, 128, 128, 128, 128};
static const uvec8 kShufAb2 = {2, 128, 5, 128, 128, 128, 10, 128,
13, 128, 128, 128, 128, 128, 128, 128};
static const uvec16 kScaleAb2 = {65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3,
65536 / 3, 65536 / 2, 0, 0};
LIBYUV_TARGET_AVX2
void ScaleRowDown38_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
(void)src_stride;
const __m256i shuf38 = _mm256_inserti128_si256(
_mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf38a)),
_mm_loadu_si128((const __m128i*)&kShuf38b), 1);
while (dst_width >= 24) {
__m256i s0 =
_mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_ptr), shuf38);
__m256i s1 = _mm256_shuffle_epi8(
_mm256_loadu_si256((const __m256i*)(src_ptr + 32)), shuf38);
src_ptr += 64;
__m128i d0 =
_mm_or_si128(_mm256_castsi256_si128(s0), _mm256_extracti128_si256(s0, 1));
__m128i d1 =
_mm_or_si128(_mm256_castsi256_si128(s1), _mm256_extracti128_si256(s1, 1));
_mm_storel_epi64((__m128i*)dst_ptr, d0);
StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d0, 8)));
_mm_storel_epi64((__m128i*)(dst_ptr + 12), d1);
StoreU32(dst_ptr + 20, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d1, 8)));
dst_ptr += 24;
dst_width -= 24;
}
if (dst_width > 0) {
__m256i s0 =
_mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_ptr), shuf38);
__m128i d0 =
_mm_or_si128(_mm256_castsi256_si128(s0), _mm256_extracti128_si256(s0, 1));
_mm_storel_epi64((__m128i*)dst_ptr, d0);
StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d0, 8)));
}
}
LIBYUV_TARGET_AVX2
void ScaleRowDown38_2_Box_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
const __m256i shuf_ab0 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAb0));
const __m256i shuf_ab1 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAb1));
const __m256i shuf_ab2 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAb2));
const __m256i scale_ab2 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kScaleAb2));
const __m256i zero = _mm256_setzero_si256();
while (dst_width > 0) {
__m256i r = _mm256_avg_epu8(
_mm256_loadu_si256((const __m256i*)src_ptr),
_mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)));
src_ptr += 32;
__m256i t = _mm256_adds_epu16(
_mm256_adds_epu16(_mm256_shuffle_epi8(r, shuf_ab0),
_mm256_shuffle_epi8(r, shuf_ab1)),
_mm256_shuffle_epi8(r, shuf_ab2));
t = _mm256_packus_epi16(_mm256_mulhi_epu16(t, scale_ab2), zero);
__m128i d = _mm_or_si128(_mm256_castsi256_si128(t),
_mm_slli_si128(_mm256_extracti128_si256(t, 1), 6));
_mm_storel_epi64((__m128i*)dst_ptr, d);
StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d, 8)));
dst_ptr += 12;
dst_width -= 12;
}
}
LIBYUV_TARGET_AVX2
void ScaleRowDown38_3_Box_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
const __m256i shuf_ac =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAc));
const __m256i shuf_ac3 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAc3));
const __m256i scale_ac33 =
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kScaleAc33));
const __m256i zero = _mm256_setzero_si256();
while (dst_width > 0) {
__m256i r0 = _mm256_loadu_si256((const __m256i*)src_ptr);
__m256i r1 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride));
__m256i r2 =
_mm256_loadu_si256((const __m256i*)(src_ptr + src_stride * 2));
src_ptr += 32;
__m256i s0 =
_mm256_adds_epu16(_mm256_adds_epu16(_mm256_unpacklo_epi8(r0, zero),
_mm256_unpacklo_epi8(r1, zero)),
_mm256_unpacklo_epi8(r2, zero));
__m256i s1 =
_mm256_adds_epu16(_mm256_adds_epu16(_mm256_unpackhi_epi8(r0, zero),
_mm256_unpackhi_epi8(r1, zero)),
_mm256_unpackhi_epi8(r2, zero));
__m256i t0 = _mm256_shuffle_epi8(
_mm256_adds_epu16(_mm256_adds_epu16(s0, _mm256_srli_si256(s0, 2)),
_mm256_srli_si256(s0, 4)),
shuf_ac);
__m256i t1 = _mm256_shuffle_epi8(
_mm256_adds_epu16(_mm256_adds_epu16(s1, _mm256_srli_si256(s1, 2)),
_mm256_srli_si256(s1, 4)),
shuf_ac3);
__m256i t = _mm256_packus_epi16(
_mm256_mulhi_epu16(_mm256_adds_epu16(t0, t1), scale_ac33), zero);
__m128i d = _mm_or_si128(_mm256_castsi256_si128(t),
_mm_slli_si128(_mm256_extracti128_si256(t, 1), 6));
_mm_storel_epi64((__m128i*)dst_ptr, d);
StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d, 8)));
dst_ptr += 12;
dst_width -= 12;
}
}
#endif // HAS_SCALEROWDOWN38_AVX2
#ifdef HAS_SCALEADDROW_SSE2
// Reads 16 bytes and accumulates to 16 shorts at a time.
void ScaleAddRow_SSE2(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width) {
const __m128i zero = _mm_setzero_si128();
while (src_width > 0) {
__m128i s = _mm_loadu_si128((const __m128i*)src_ptr);
src_ptr += 16;
__m128i d0 = _mm_loadu_si128((const __m128i*)dst_ptr);
__m128i d1 = _mm_loadu_si128((const __m128i*)(dst_ptr + 8));
d0 = _mm_adds_epu16(d0, _mm_unpacklo_epi8(s, zero));
d1 = _mm_adds_epu16(d1, _mm_unpackhi_epi8(s, zero));
_mm_storeu_si128((__m128i*)dst_ptr, d0);
_mm_storeu_si128((__m128i*)(dst_ptr + 8), d1);
dst_ptr += 16;
src_width -= 16;
}
}
#endif // HAS_SCALEADDROW_SSE2
#ifdef HAS_SCALEADDROW_AVX2
// Reads 32 bytes and accumulates to 32 shorts at a time.
LIBYUV_TARGET_AVX2
void ScaleAddRow_AVX2(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width) {
while (src_width > 0) {
__m256i s0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_ptr));
__m256i s1 =
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_ptr + 16)));
src_ptr += 32;
__m256i d0 = _mm256_loadu_si256((const __m256i*)dst_ptr);
__m256i d1 = _mm256_loadu_si256((const __m256i*)(dst_ptr + 16));
d0 = _mm256_adds_epu16(d0, s0);
d1 = _mm256_adds_epu16(d1, s1);
_mm256_storeu_si256((__m256i*)dst_ptr, d0);
_mm256_storeu_si256((__m256i*)(dst_ptr + 16), d1);
dst_ptr += 32;
src_width -= 32;
}
}
#endif // HAS_SCALEADDROW_AVX2
#ifdef HAS_SCALEADDROW_AVX512BW
// Reads 64 bytes and accumulates to 64 shorts at a time, with masked tail.
LIBYUV_TARGET_AVX512BW
void ScaleAddRow_AVX512BW(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width) {
while (src_width >= 64) {
__m512i s0 =
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_ptr));
__m512i s1 = _mm512_cvtepu8_epi16(
_mm256_loadu_si256((const __m256i*)(src_ptr + 32)));
src_ptr += 64;
__m512i d0 = _mm512_loadu_si512((const __m512i*)dst_ptr);
__m512i d1 = _mm512_loadu_si512((const __m512i*)(dst_ptr + 32));
d0 = _mm512_adds_epu16(d0, s0);
d1 = _mm512_adds_epu16(d1, s1);
_mm512_storeu_si512((__m512i*)dst_ptr, d0);
_mm512_storeu_si512((__m512i*)(dst_ptr + 32), d1);
dst_ptr += 64;
src_width -= 64;
}
if (src_width >= 32) {
__m512i s0 =
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_ptr));
src_ptr += 32;
__m512i d0 = _mm512_loadu_si512((const __m512i*)dst_ptr);
d0 = _mm512_adds_epu16(d0, s0);
_mm512_storeu_si512((__m512i*)dst_ptr, d0);
dst_ptr += 32;
src_width -= 32;
}
if (src_width > 0) {
__mmask32 k = (__mmask32)((1ULL << src_width) - 1);
__m512i s0 = _mm512_maskz_cvtepu8_epi16(
k, _mm256_maskz_loadu_epi8(k, (const void*)src_ptr));
__m512i d0 = _mm512_maskz_loadu_epi16(k, (const void*)dst_ptr);
d0 = _mm512_maskz_adds_epu16(k, d0, s0);
_mm512_mask_storeu_epi16((void*)dst_ptr, k, d0);
}
}
#endif // HAS_SCALEADDROW_AVX512BW
#if defined(HAS_SCALEADDCOLS1_AVX512BW) || defined(HAS_SCALEADDCOLS2_AVX512BW)
static const uint8_t kSequence0_15[16] = {0, 1, 2, 3, 4, 5, 6, 7,
8, 9, 10, 11, 12, 13, 14, 15};
#endif
#ifdef HAS_SCALEADDCOLS1_AVX512BW
LIBYUV_TARGET_AVX512BW
void ScaleAddCols1_AVX512BW(int dst_width,
int boxheight,
int x,
int dx,
const uint16_t* src_ptr,
uint8_t* dst_ptr) {
int boxwidth = dx >> 16;
if (boxwidth < 1) {
boxwidth = 1;
}
int scaleval = 65536 / (boxwidth * boxheight);
src_ptr += (x >> 16);
const __m512i mask_ffff = _mm512_set1_epi32(0x0000ffff);
const __m512i vscale = _mm512_set1_epi32(scaleval);
if (boxwidth <= 4) {
__m512i seq =
_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kSequence0_15));
seq = _mm512_or_si512(seq, _mm512_slli_epi32(seq, 16));
__m512i vbw = _mm512_set1_epi16((int16_t)boxwidth);
__m512i base = _mm512_mullo_epi16(seq, vbw);
__m512i c01 = _mm512_set1_epi32(0x00010000);
__m512i c23 = _mm512_set1_epi32(0x00030002);
__m512i idx01 = _mm512_add_epi16(base, c01);
__m512i idx23 = _mm512_add_epi16(base, c23);
__mmask32 k2 = _mm512_cmpgt_epu16_mask(vbw, c01);
__mmask32 k3 = _mm512_cmpgt_epu16_mask(vbw, c23);
int span = boxwidth * 16;
uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1);
__mmask32 k1 = (__mmask32)m64;
__mmask32 k4 = (__mmask32)(m64 >> 32);
while (dst_width >= 16) {
__m512i s0 = _mm512_maskz_loadu_epi16(k1, src_ptr);
__m512i s1 = _mm512_maskz_loadu_epi16(k4, src_ptr + 32);
src_ptr += span;
__m512i p01 = _mm512_maskz_permutex2var_epi16(k2, s0, idx01, s1);
__m512i p23 = _mm512_maskz_permutex2var_epi16(k3, s0, idx23, s1);
__m512i sum01 = _mm512_add_epi32(_mm512_and_si512(p01, mask_ffff),
_mm512_srli_epi32(p01, 16));
__m512i sum23 = _mm512_add_epi32(_mm512_and_si512(p23, mask_ffff),
_mm512_srli_epi32(p23, 16));
__m512i sum = _mm512_add_epi32(sum01, sum23);
sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vscale), 16);
_mm_storeu_si128((__m128i*)dst_ptr, _mm512_cvtepi32_epi8(sum));
dst_ptr += 16;
dst_width -= 16;
}
if (dst_width > 0) {
int rem_span = dst_width * boxwidth;
uint64_t rm64 = (rem_span >= 64) ? ~0ULL : ((1ULL << rem_span) - 1);
__m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)rm64, src_ptr);
__m512i s1 =
_mm512_maskz_loadu_epi16((__mmask32)(rm64 >> 32), src_ptr + 32);
__m512i p01 = _mm512_maskz_permutex2var_epi16(k2, s0, idx01, s1);
__m512i p23 = _mm512_maskz_permutex2var_epi16(k3, s0, idx23, s1);
__m512i sum01 = _mm512_add_epi32(_mm512_and_si512(p01, mask_ffff),
_mm512_srli_epi32(p01, 16));
__m512i sum23 = _mm512_add_epi32(_mm512_and_si512(p23, mask_ffff),
_mm512_srli_epi32(p23, 16));
__m512i sum = _mm512_add_epi32(sum01, sum23);
sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vscale), 16);
__mmask16 km = (__mmask16)((1U << dst_width) - 1);
// Use _mm_mask_storeu_epi8 instead of _mm512_mask_cvtepi32_storeu_epi8
// because LLVM MSan does not instrument @llvm.x86.avx512.mask.pmov.db.mem
// to unpoison destination shadow memory.
_mm_mask_storeu_epi8(dst_ptr, km, _mm512_cvtepi32_epi8(sum));
}
} else if (boxwidth <= 8) {
__m512i seq =
_mm512_cvtepu8_epi64(_mm_loadu_si128((const __m128i*)kSequence0_15));
seq = _mm512_or_si512(seq, _mm512_slli_epi64(seq, 16));
seq = _mm512_or_si512(seq, _mm512_slli_epi64(seq, 32));
__m512i vbw = _mm512_set1_epi16((int16_t)boxwidth);
__m512i base = _mm512_mullo_epi16(seq, vbw);
__m512i c03 = _mm512_set1_epi64(0x0003000200010000ULL);
__m512i c47 = _mm512_set1_epi64(0x0007000600050004ULL);
__m512i idx03 = _mm512_add_epi16(base, c03);
__m512i idx47 = _mm512_add_epi16(base, c47);
__mmask32 k3 = _mm512_cmpgt_epu16_mask(vbw, c47);
int span = boxwidth * 8;
uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1);
__mmask32 k1 = (__mmask32)m64;
__mmask32 k4 = (__mmask32)(m64 >> 32);
while (dst_width >= 8) {
__m512i s0 = _mm512_maskz_loadu_epi16(k1, src_ptr);
__m512i s1 = _mm512_maskz_loadu_epi16(k4, src_ptr + 32);
src_ptr += span;
__m512i p03 = _mm512_permutex2var_epi16(s0, idx03, s1);
__m512i p47 = _mm512_maskz_permutex2var_epi16(k3, s0, idx47, s1);
__m512i sum03 = _mm512_add_epi32(_mm512_and_si512(p03, mask_ffff),
_mm512_srli_epi32(p03, 16));
__m512i sum47 = _mm512_add_epi32(_mm512_and_si512(p47, mask_ffff),
_mm512_srli_epi32(p47, 16));
__m512i sum = _mm512_add_epi32(sum03, sum47);
sum = _mm512_add_epi32(sum, _mm512_srli_epi64(sum, 32));
sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vscale), 16);
_mm_storel_epi64((__m128i*)dst_ptr, _mm512_cvtepi64_epi8(sum));
dst_ptr += 8;
dst_width -= 8;
}
if (dst_width > 0) {
int rem_span = dst_width * boxwidth;
uint64_t rm64 = (rem_span >= 64) ? ~0ULL : ((1ULL << rem_span) - 1);
__m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)rm64, src_ptr);
__m512i s1 =
_mm512_maskz_loadu_epi16((__mmask32)(rm64 >> 32), src_ptr + 32);
__m512i p03 = _mm512_permutex2var_epi16(s0, idx03, s1);
__m512i p47 = _mm512_maskz_permutex2var_epi16(k3, s0, idx47, s1);
__m512i sum03 = _mm512_add_epi32(_mm512_and_si512(p03, mask_ffff),
_mm512_srli_epi32(p03, 16));
__m512i sum47 = _mm512_add_epi32(_mm512_and_si512(p47, mask_ffff),
_mm512_srli_epi32(p47, 16));
__m512i sum = _mm512_add_epi32(sum03, sum47);
sum = _mm512_add_epi32(sum, _mm512_srli_epi64(sum, 32));
sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vscale), 16);
__mmask8 km = (__mmask8)((1U << dst_width) - 1);
// Use _mm_mask_storeu_epi8 instead of _mm512_mask_cvtepi64_storeu_epi8
// because LLVM MSan does not instrument @llvm.x86.avx512.mask.pmov.qb.mem
// to unpoison destination shadow memory.
_mm_mask_storeu_epi8(dst_ptr, km, _mm512_cvtepi64_epi8(sum));
}
} else {
int rem = boxwidth & 31;
__mmask32 k1 = (__mmask32)((1ULL << rem) - 1);
while (dst_width > 0) {
__m512i acc = _mm512_setzero_si512();
const uint16_t* p = src_ptr;
src_ptr += boxwidth;
int n = boxwidth;
while (n >= 32) {
__m512i s = _mm512_loadu_si512((const __m512i*)p);
p += 32;
acc = _mm512_add_epi32(acc,
_mm512_add_epi32(_mm512_and_si512(s, mask_ffff),
_mm512_srli_epi32(s, 16)));
n -= 32;
}
if (n > 0) {
__m512i s = _mm512_maskz_loadu_epi16(k1, p);
acc = _mm512_add_epi32(acc,
_mm512_add_epi32(_mm512_and_si512(s, mask_ffff),
_mm512_srli_epi32(s, 16)));
}
uint32_t total = (uint32_t)_mm512_reduce_add_epi32(acc);
*dst_ptr++ = (uint8_t)((total * (uint32_t)scaleval) >> 16);
--dst_width;
}
}
}
#endif // HAS_SCALEADDCOLS1_AVX512BW
#ifdef HAS_SCALEADDCOLS2_AVX512BW
LIBYUV_TARGET_AVX512BW
void ScaleAddCols2_AVX512BW(int dst_width,
int boxheight,
int x,
int dx,
const uint16_t* src_ptr,
uint8_t* dst_ptr) {
int minboxwidth = dx >> 16;
int scale0 = 65536 / ((minboxwidth < 1 ? 1 : minboxwidth) * boxheight);
int scale1 =
65536 / (((minboxwidth + 1) < 1 ? 1 : (minboxwidth + 1)) * boxheight);
const __m512i mask_ffff = _mm512_set1_epi32(0x0000ffff);
const __m512i vscale0 = _mm512_set1_epi32(scale0);
const __m512i vscale1 = _mm512_set1_epi32(scale1);
if (minboxwidth >= 1 && minboxwidth <= 3) {
__m512i seq =
_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kSequence0_15));
__m512i vdx = _mm512_set1_epi32(dx);
__m512i step0 = _mm512_mullo_epi32(seq, vdx);
__m512i step1 = _mm512_add_epi32(step0, vdx);
__m512i c01 = _mm512_set1_epi32(0x00010000);
__m512i c23 = _mm512_set1_epi32(0x00030002);
__m512i vminbw = _mm512_set1_epi32(minboxwidth);
while (dst_width >= 16) {
int base_ix = x >> 16;
__m512i xfrac = _mm512_set1_epi32(x & 0xffff);
x += dx * 16;
int span = (x >> 16) - base_ix;
uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1);
__m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)m64, src_ptr + base_ix);
__m512i s1 = _mm512_maskz_loadu_epi16((__mmask32)(m64 >> 32),
src_ptr + base_ix + 32);
__m512i ix0 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step0), 16);
__m512i ix1 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step1), 16);
__m512i bw = _mm512_sub_epi32(ix1, ix0);
__mmask16 k4 = _mm512_cmpgt_epi32_mask(bw, vminbw);
__m512i ix0_16 = _mm512_or_si512(ix0, _mm512_slli_epi32(ix0, 16));
__m512i bw_16 = _mm512_or_si512(bw, _mm512_slli_epi32(bw, 16));
__m512i idx01 = _mm512_add_epi16(ix0_16, c01);
__m512i idx23 = _mm512_add_epi16(ix0_16, c23);
__mmask32 k2 = _mm512_cmpgt_epu16_mask(bw_16, c01);
__mmask32 k3 = _mm512_cmpgt_epu16_mask(bw_16, c23);
__m512i p01 = _mm512_maskz_permutex2var_epi16(k2, s0, idx01, s1);
__m512i p23 = _mm512_maskz_permutex2var_epi16(k3, s0, idx23, s1);
__m512i sum01 = _mm512_add_epi32(_mm512_and_si512(p01, mask_ffff),
_mm512_srli_epi32(p01, 16));
__m512i sum23 = _mm512_add_epi32(_mm512_and_si512(p23, mask_ffff),
_mm512_srli_epi32(p23, 16));
__m512i sum = _mm512_add_epi32(sum01, sum23);
__m512i vsc = _mm512_mask_blend_epi32(k4, vscale0, vscale1);
sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vsc), 16);
_mm_storeu_si128((__m128i*)dst_ptr, _mm512_cvtepi32_epi8(sum));
dst_ptr += 16;
dst_width -= 16;
}
if (dst_width > 0) {
int base_ix = x >> 16;
__m512i xfrac = _mm512_set1_epi32(x & 0xffff);
int end_x = x + dx * dst_width;
int span = (end_x >> 16) - base_ix;
uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1);
__m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)m64, src_ptr + base_ix);
__m512i s1 = _mm512_maskz_loadu_epi16((__mmask32)(m64 >> 32),
src_ptr + base_ix + 32);
__m512i ix0 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step0), 16);
__m512i ix1 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step1), 16);
__m512i bw = _mm512_sub_epi32(ix1, ix0);
__mmask16 k4 = _mm512_cmpgt_epi32_mask(bw, vminbw);
__m512i ix0_16 = _mm512_or_si512(ix0, _mm512_slli_epi32(ix0, 16));
__m512i bw_16 = _mm512_or_si512(bw, _mm512_slli_epi32(bw, 16));
__m512i idx01 = _mm512_add_epi16(ix0_16, c01);
__m512i idx23 = _mm512_add_epi16(ix0_16, c23);
__mmask32 k2 = _mm512_cmpgt_epu16_mask(bw_16, c01);
__mmask32 k3 = _mm512_cmpgt_epu16_mask(bw_16, c23);
__m512i p01 = _mm512_maskz_permutex2var_epi16(k2, s0, idx01, s1);
__m512i p23 = _mm512_maskz_permutex2var_epi16(k3, s0, idx23, s1);
__m512i sum01 = _mm512_add_epi32(_mm512_and_si512(p01, mask_ffff),
_mm512_srli_epi32(p01, 16));
__m512i sum23 = _mm512_add_epi32(_mm512_and_si512(p23, mask_ffff),
_mm512_srli_epi32(p23, 16));
__m512i sum = _mm512_add_epi32(sum01, sum23);
__m512i vsc = _mm512_mask_blend_epi32(k4, vscale0, vscale1);
sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vsc), 16);
__mmask16 km = (__mmask16)((1U << dst_width) - 1);
// Use _mm_mask_storeu_epi8 instead of _mm512_mask_cvtepi32_storeu_epi8
// because LLVM MSan does not instrument @llvm.x86.avx512.mask.pmov.db.mem
// to unpoison destination shadow memory.
_mm_mask_storeu_epi8(dst_ptr, km, _mm512_cvtepi32_epi8(sum));
}
} else if (minboxwidth >= 4 && minboxwidth <= 7) {
__m512i seq =
_mm512_cvtepu8_epi64(_mm_loadu_si128((const __m128i*)kSequence0_15));
__m512i vdx = _mm512_set1_epi64(dx);
__m512i step0 = _mm512_mullo_epi32(seq, vdx);
__m512i step1 = _mm512_add_epi64(step0, vdx);
__m512i c03 = _mm512_set1_epi64(0x0003000200010000ULL);
__m512i c47 = _mm512_set1_epi64(0x0007000600050004ULL);
__m512i vminbw = _mm512_set1_epi64(minboxwidth);
while (dst_width >= 8) {
int base_ix = x >> 16;
__m512i xfrac = _mm512_set1_epi64(x & 0xffff);
x += dx * 8;
int span = (x >> 16) - base_ix;
uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1);
__m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)m64, src_ptr + base_ix);
__m512i s1 = _mm512_maskz_loadu_epi16((__mmask32)(m64 >> 32),
src_ptr + base_ix + 32);
__m512i ix0 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step0), 16);
__m512i ix1 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step1), 16);
__m512i bw = _mm512_sub_epi32(ix1, ix0);
__mmask8 k4 = _mm512_cmpgt_epi64_mask(bw, vminbw);
__m512i ix0_16 = _mm512_or_si512(ix0, _mm512_slli_epi64(ix0, 16));
ix0_16 = _mm512_or_si512(ix0_16, _mm512_slli_epi64(ix0_16, 32));
__m512i bw_16 = _mm512_or_si512(bw, _mm512_slli_epi64(bw, 16));
bw_16 = _mm512_or_si512(bw_16, _mm512_slli_epi64(bw_16, 32));
__m512i idx03 = _mm512_add_epi16(ix0_16, c03);
__m512i idx47 = _mm512_add_epi16(ix0_16, c47);
__mmask32 k3 = _mm512_cmpgt_epu16_mask(bw_16, c47);
__m512i p03 = _mm512_permutex2var_epi16(s0, idx03, s1);
__m512i p47 = _mm512_maskz_permutex2var_epi16(k3, s0, idx47, s1);
__m512i sum03 = _mm512_add_epi32(_mm512_and_si512(p03, mask_ffff),
_mm512_srli_epi32(p03, 16));
__m512i sum47 = _mm512_add_epi32(_mm512_and_si512(p47, mask_ffff),
_mm512_srli_epi32(p47, 16));
__m512i sum = _mm512_add_epi32(sum03, sum47);
sum = _mm512_add_epi32(sum, _mm512_srli_epi64(sum, 32));
__m512i vsc = _mm512_mask_blend_epi64(k4, vscale0, vscale1);
sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vsc), 16);
_mm_storel_epi64((__m128i*)dst_ptr, _mm512_cvtepi64_epi8(sum));
dst_ptr += 8;
dst_width -= 8;
}
if (dst_width > 0) {
int base_ix = x >> 16;
__m512i xfrac = _mm512_set1_epi64(x & 0xffff);
int end_x = x + dx * dst_width;
int span = (end_x >> 16) - base_ix;
uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1);
__m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)m64, src_ptr + base_ix);
__m512i s1 = _mm512_maskz_loadu_epi16((__mmask32)(m64 >> 32),
src_ptr + base_ix + 32);
__m512i ix0 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step0), 16);
__m512i ix1 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step1), 16);
__m512i bw = _mm512_sub_epi32(ix1, ix0);
__mmask8 k4 = _mm512_cmpgt_epi64_mask(bw, vminbw);
__m512i ix0_16 = _mm512_or_si512(ix0, _mm512_slli_epi64(ix0, 16));
ix0_16 = _mm512_or_si512(ix0_16, _mm512_slli_epi64(ix0_16, 32));
__m512i bw_16 = _mm512_or_si512(bw, _mm512_slli_epi64(bw, 16));
bw_16 = _mm512_or_si512(bw_16, _mm512_slli_epi64(bw_16, 32));
__m512i idx03 = _mm512_add_epi16(ix0_16, c03);
__m512i idx47 = _mm512_add_epi16(ix0_16, c47);
__mmask32 k3 = _mm512_cmpgt_epu16_mask(bw_16, c47);
__m512i p03 = _mm512_permutex2var_epi16(s0, idx03, s1);
__m512i p47 = _mm512_maskz_permutex2var_epi16(k3, s0, idx47, s1);
__m512i sum03 = _mm512_add_epi32(_mm512_and_si512(p03, mask_ffff),
_mm512_srli_epi32(p03, 16));
__m512i sum47 = _mm512_add_epi32(_mm512_and_si512(p47, mask_ffff),
_mm512_srli_epi32(p47, 16));
__m512i sum = _mm512_add_epi32(sum03, sum47);
sum = _mm512_add_epi32(sum, _mm512_srli_epi64(sum, 32));
__m512i vsc = _mm512_mask_blend_epi64(k4, vscale0, vscale1);
sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vsc), 16);
__mmask8 km = (__mmask8)((1U << dst_width) - 1);
// Use _mm_mask_storeu_epi8 instead of _mm512_mask_cvtepi64_storeu_epi8
// because LLVM MSan does not instrument @llvm.x86.avx512.mask.pmov.qb.mem
// to unpoison destination shadow memory.
_mm_mask_storeu_epi8(dst_ptr, km, _mm512_cvtepi64_epi8(sum));
}
} else {
while (dst_width > 0) {
int ix = x >> 16;
x += dx;
int bw = (x >> 16) - ix;
if (bw < 1) {
bw = 1;
}
int sc = (bw > minboxwidth) ? scale1 : scale0;
__m512i acc = _mm512_setzero_si512();
const uint16_t* p = src_ptr + ix;
int n = bw;
while (n >= 32) {
__m512i s = _mm512_loadu_si512((const __m512i*)p);
p += 32;
acc = _mm512_add_epi32(acc,
_mm512_add_epi32(_mm512_and_si512(s, mask_ffff),
_mm512_srli_epi32(s, 16)));
n -= 32;
}
if (n > 0) {
__mmask32 k1 = (__mmask32)((1ULL << n) - 1);
__m512i s = _mm512_maskz_loadu_epi16(k1, p);
acc = _mm512_add_epi32(acc,
_mm512_add_epi32(_mm512_and_si512(s, mask_ffff),
_mm512_srli_epi32(s, 16)));
}
uint32_t total = (uint32_t)_mm512_reduce_add_epi32(acc);
*dst_ptr++ = (uint8_t)((total * (uint32_t)sc) >> 16);
--dst_width;
}
}
}
#endif // HAS_SCALEADDCOLS2_AVX512BW
#ifdef HAS_SCALEUVADDCOLS3_AVX2
static const ulvec32 kPermUV3_0 = {0, 3, 6, 1, 4, 7, 2, 5};
static const ulvec32 kPermUV3_1 = {3, 4, 5, 6, 7, 0, 1, 2};
static const ulvec32 kPermUV3_2 = {6, 7, 0, 1, 2, 3, 4, 5};
// Scale UV columns by 3 box filter.
// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per block.
LIBYUV_TARGET_AVX2
void ScaleUVAddCols3_AVX2(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
const __m256i vscale = _mm256_set1_epi16((int16_t)scaleval);
const __m256i perm0 = _mm256_loadu_si256((const __m256i*)&kPermUV3_0);
const __m256i perm1 = _mm256_loadu_si256((const __m256i*)&kPermUV3_1);
const __m256i perm2 = _mm256_loadu_si256((const __m256i*)&kPermUV3_2);
while (dst_width >= 16) {
__m256i a0 = _mm256_permutevar8x32_epi32(
_mm256_loadu_si256((const __m256i*)src_uv), perm0);
__m256i a1 = _mm256_permutevar8x32_epi32(
_mm256_loadu_si256((const __m256i*)(src_uv + 16)), perm0);
__m256i a2 = _mm256_permutevar8x32_epi32(
_mm256_loadu_si256((const __m256i*)(src_uv + 32)), perm0);
__m256i c0 = _mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0x38), a2, 0xc0);
__m256i c1 = _mm256_permutevar8x32_epi32(
_mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0xc0), a2, 0x07), perm1);
__m256i c2 = _mm256_permutevar8x32_epi32(
_mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0x07), a2, 0x38), perm2);
__m256i sum0 = _mm256_mulhi_epu16(
_mm256_add_epi16(_mm256_add_epi16(c0, c1), c2), vscale);
__m256i b0 = _mm256_permutevar8x32_epi32(
_mm256_loadu_si256((const __m256i*)(src_uv + 48)), perm0);
__m256i b1 = _mm256_permutevar8x32_epi32(
_mm256_loadu_si256((const __m256i*)(src_uv + 64)), perm0);
__m256i b2 = _mm256_permutevar8x32_epi32(
_mm256_loadu_si256((const __m256i*)(src_uv + 80)), perm0);
src_uv += 96;
__m256i d0 = _mm256_blend_epi32(_mm256_blend_epi32(b0, b1, 0x38), b2, 0xc0);
__m256i d1 = _mm256_permutevar8x32_epi32(
_mm256_blend_epi32(_mm256_blend_epi32(b0, b1, 0xc0), b2, 0x07), perm1);
__m256i d2 = _mm256_permutevar8x32_epi32(
_mm256_blend_epi32(_mm256_blend_epi32(b0, b1, 0x07), b2, 0x38), perm2);
__m256i sum1 = _mm256_mulhi_epu16(
_mm256_add_epi16(_mm256_add_epi16(d0, d1), d2), vscale);
__m256i packed =
_mm256_permute4x64_epi64(_mm256_packus_epi16(sum0, sum1), 0xd8);
_mm256_storeu_si256((__m256i*)dst_uv, packed);
dst_uv += 32;
dst_width -= 16;
}
while (dst_width > 0) {
__m256i a0 = _mm256_permutevar8x32_epi32(
_mm256_loadu_si256((const __m256i*)src_uv), perm0);
__m256i a1 = _mm256_permutevar8x32_epi32(
_mm256_loadu_si256((const __m256i*)(src_uv + 16)), perm0);
__m256i a2 = _mm256_permutevar8x32_epi32(
_mm256_loadu_si256((const __m256i*)(src_uv + 32)), perm0);
src_uv += 48;
__m256i c0 = _mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0x38), a2, 0xc0);
__m256i c1 = _mm256_permutevar8x32_epi32(
_mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0xc0), a2, 0x07), perm1);
__m256i c2 = _mm256_permutevar8x32_epi32(
_mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0x07), a2, 0x38), perm2);
__m256i sum = _mm256_mulhi_epu16(
_mm256_add_epi16(_mm256_add_epi16(c0, c1), c2), vscale);
__m128i lo = _mm256_castsi256_si128(sum);
__m128i hi = _mm256_extracti128_si256(sum, 1);
_mm_storeu_si128((__m128i*)dst_uv, _mm_packus_epi16(lo, hi));
dst_uv += 16;
dst_width -= 8;
}
}
#endif // HAS_SCALEUVADDCOLS3_AVX2
#ifdef HAS_SCALEUVADDCOLS3_AVX512VBMI
static const uint8_t kPerm512_Col0[16] = {0, 3, 6, 9, 12, 15, 18, 21,
24, 27, 30, 1, 4, 7, 10, 13};
static const uint8_t kPerm512_Col1[16] = {1, 4, 7, 10, 13, 16, 19, 22,
25, 28, 31, 2, 5, 8, 11, 14};
static const uint8_t kPerm512_Col2[16] = {2, 5, 8, 11, 14, 17, 20, 23,
26, 29, 0, 3, 6, 9, 12, 15};
static const uint8_t kPackEvenBytes[64] = {
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24,
26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50,
52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76,
78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102,
104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126};
// Under MemorySanitizer, use a portable lane-by-lane C loop so shadow
// propagation relies only on standard C memory/array semantics rather than
// target-specific LLVM intrinsic modeling. In non-MSan builds, call the native
// AVX-512 intrinsic directly.
LIBYUV_TARGET_AVX512VBMI
static inline __m512i MaskPermutexvarEpi32(__m512i src,
__mmask16 k,
__m512i idx,
__m512i v) {
#if defined(LIBYUV_MSAN)
uint32_t src_arr[16];
uint32_t idx_arr[16];
uint32_t v_arr[16];
uint32_t out_arr[16];
_mm512_storeu_si512((__m512i*)src_arr, src);
_mm512_storeu_si512((__m512i*)idx_arr, idx);
_mm512_storeu_si512((__m512i*)v_arr, v);
for (int i = 0; i < 16; ++i) {
out_arr[i] = ((k >> i) & 1) ? v_arr[idx_arr[i] & 15] : src_arr[i];
}
return _mm512_loadu_si512((const __m512i*)out_arr);
#else
return _mm512_mask_permutexvar_epi32(src, k, idx, v);
#endif
}
// Scale UV columns by 3 box filter using AVX512VBMI.
// Processes 32 UV pixels (384 input bytes -> 64 output bytes) per main loop
// iteration, followed by a 16-pixel block and a k-masked tail (1..15 pixels).
LIBYUV_TARGET_AVX512VBMI
void ScaleUVAddCols3_AVX512VBMI(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
const __m512i vscale = _mm512_set1_epi16((int16_t)scaleval);
const __m512i col0 =
_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kPerm512_Col0));
const __m512i col1 =
_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kPerm512_Col1));
const __m512i col2 =
_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kPerm512_Col2));
const __m512i pack_even = _mm512_loadu_si512((const __m512i*)kPackEvenBytes);
const __mmask16 k5 = 0xf800;
const __mmask16 k6 = 0xfc00;
while (dst_width >= 32) {
__m512i s0 = _mm512_loadu_si512((const __m512i*)src_uv);
__m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32));
__m512i s2 = _mm512_loadu_si512((const __m512i*)(src_uv + 64));
__m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1),
k5, col0, s2);
__m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1),
k5, col1, s2);
__m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1),
k6, col2, s2);
__m512i sum0 = _mm512_mulhi_epu16(
_mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale);
__m512i s3 = _mm512_loadu_si512((const __m512i*)(src_uv + 96));
__m512i s4 = _mm512_loadu_si512((const __m512i*)(src_uv + 128));
__m512i s5 = _mm512_loadu_si512((const __m512i*)(src_uv + 160));
src_uv += 192;
__m512i b0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col0, s4),
k5, col0, s5);
__m512i b1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col1, s4),
k5, col1, s5);
__m512i b2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col2, s4),
k6, col2, s5);
__m512i sum1 = _mm512_mulhi_epu16(
_mm512_add_epi16(_mm512_add_epi16(b0, b1), b2), vscale);
__m512i out = _mm512_permutex2var_epi8(sum0, pack_even, sum1);
_mm512_storeu_si512((__m512i*)dst_uv, out);
dst_uv += 64;
dst_width -= 32;
}
if (dst_width >= 16) {
__m512i s0 = _mm512_loadu_si512((const __m512i*)src_uv);
__m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32));
__m512i s2 = _mm512_loadu_si512((const __m512i*)(src_uv + 64));
src_uv += 96;
__m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1),
k5, col0, s2);
__m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1),
k5, col1, s2);
__m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1),
k6, col2, s2);
__m512i sum0 = _mm512_mulhi_epu16(
_mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale);
_mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(sum0));
dst_uv += 32;
dst_width -= 16;
}
if (dst_width > 0) {
uint64_t m64 = (1ULL << (dst_width * 3)) - 1;
__mmask16 k1 = (__mmask16)m64;
__mmask16 k2 = (__mmask16)(m64 >> 16);
__mmask16 k3 = (__mmask16)(m64 >> 32);
__mmask16 k4 = (__mmask16)((1U << dst_width) - 1);
__m512i s0 = _mm512_maskz_loadu_epi32(k1, src_uv);
__m512i s1 = _mm512_maskz_loadu_epi32(k2, src_uv + 32);
__m512i s2 = _mm512_maskz_loadu_epi32(k3, src_uv + 64);
__m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1),
k5, col0, s2);
__m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1),
k5, col1, s2);
__m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1),
k6, col2, s2);
__m512i sum0 = _mm512_mulhi_epu16(
_mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale);
_mm256_mask_storeu_epi16(dst_uv, k4, _mm512_cvtepi16_epi8(sum0));
}
}
#endif // HAS_SCALEUVADDCOLS3_AVX512VBMI
#if defined(HAS_SCALEROWUP2_LINEAR_AVX2) || \
defined(HAS_SCALEROWUP2_BILINEAR_AVX2)
static const uvec8 kLinearMadd31 = {3, 1, 1, 3, 3, 1, 1, 3,
3, 1, 1, 3, 3, 1, 1, 3};
#endif
#if defined(HAS_SCALEROWUP2_LINEAR_12_AVX2) || \
defined(HAS_SCALEROWUP2_BILINEAR_12_AVX2)
static const uvec8 kLinearShuffleFar = {2, 3, 0, 1, 6, 7, 4, 5,
10, 11, 8, 9, 14, 15, 12, 13};
#endif
#ifdef HAS_SCALEROWUP2_LINEAR_AVX2
LIBYUV_TARGET_AVX2
void ScaleRowUp2_Linear_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width) {
const __m256i madd31 = _mm256_broadcastsi128_si256(
_mm_loadu_si128((const __m128i*)&kLinearMadd31));
const __m256i round2 = _mm256_set1_epi16(2);
while (dst_width > 0) {
__m256i s0 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8);
__m256i s1 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 1))),
0xd8);
src_ptr += 16;
s0 = _mm256_unpacklo_epi16(s0, s0);
s1 = _mm256_unpacklo_epi16(s1, s1);
__m256i lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(s0, s1), madd31);
__m256i hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(s0, s1), madd31);
lo = _mm256_srli_epi16(_mm256_add_epi16(lo, round2), 2);
hi = _mm256_srli_epi16(_mm256_add_epi16(hi, round2), 2);
_mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(lo, hi));
dst_ptr += 32;
dst_width -= 32;
}
}
#endif // HAS_SCALEROWUP2_LINEAR_AVX2
#ifdef HAS_SCALEROWUP2_BILINEAR_AVX2
LIBYUV_TARGET_AVX2
void ScaleRowUp2_Bilinear_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
ptrdiff_t dst_stride,
int dst_width) {
const __m256i madd31 = _mm256_broadcastsi128_si256(
_mm_loadu_si128((const __m128i*)&kLinearMadd31));
const __m256i round8 = _mm256_set1_epi16(8);
while (dst_width > 0) {
__m256i a0 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8);
__m256i a1 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 1))),
0xd8);
a0 = _mm256_unpacklo_epi16(a0, a0);
a1 = _mm256_unpacklo_epi16(a1, a1);
__m256i r0_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(a0, a1), madd31);
__m256i r0_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(a0, a1), madd31);
__m256i b0 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(
_mm_loadu_si128((const __m128i*)(src_ptr + src_stride))),
0xd8);
__m256i b1 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(
_mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 1))),
0xd8);
src_ptr += 16;
b0 = _mm256_unpacklo_epi16(b0, b0);
b1 = _mm256_unpacklo_epi16(b1, b1);
__m256i r1_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(b0, b1), madd31);
__m256i r1_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(b0, b1), madd31);
__m256i d0_lo = _mm256_srli_epi16(
_mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_lo, r0_lo), r0_lo),
_mm256_add_epi16(r1_lo, round8)),
4);
__m256i d1_lo = _mm256_srli_epi16(
_mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_lo, r1_lo), r1_lo),
_mm256_add_epi16(r0_lo, round8)),
4);
__m256i d0_hi = _mm256_srli_epi16(
_mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_hi, r0_hi), r0_hi),
_mm256_add_epi16(r1_hi, round8)),
4);
__m256i d1_hi = _mm256_srli_epi16(
_mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_hi, r1_hi), r1_hi),
_mm256_add_epi16(r0_hi, round8)),
4);
_mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(d0_lo, d0_hi));
_mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride),
_mm256_packus_epi16(d1_lo, d1_hi));
dst_ptr += 32;
dst_width -= 32;
}
}
#endif // HAS_SCALEROWUP2_BILINEAR_AVX2
#ifdef HAS_SCALEROWUP2_LINEAR_12_AVX2
LIBYUV_TARGET_AVX2
void ScaleRowUp2_Linear_12_AVX2(const uint16_t* src_ptr,
uint16_t* dst_ptr,
int dst_width) {
const __m256i shuf_far = _mm256_broadcastsi128_si256(
_mm_loadu_si128((const __m128i*)&kLinearShuffleFar));
const __m256i round2 = _mm256_set1_epi16(2);
while (dst_width > 0) {
__m256i s0 =
_mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)src_ptr),
0xd8);
__m256i s1 = _mm256_permute4x64_epi64(
_mm256_loadu_si256((const __m256i*)(src_ptr + 1)), 0xd8);
src_ptr += 16;
__m256i near_lo = _mm256_unpacklo_epi16(s0, s1);
__m256i near_hi = _mm256_unpackhi_epi16(s0, s1);
__m256i far_lo = _mm256_shuffle_epi8(near_lo, shuf_far);
__m256i far_hi = _mm256_shuffle_epi8(near_hi, shuf_far);
__m256i d_lo = _mm256_srli_epi16(
_mm256_add_epi16(_mm256_add_epi16(near_lo, near_lo),
_mm256_add_epi16(near_lo, _mm256_add_epi16(far_lo, round2))),
2);
__m256i d_hi = _mm256_srli_epi16(
_mm256_add_epi16(_mm256_add_epi16(near_hi, near_hi),
_mm256_add_epi16(near_hi, _mm256_add_epi16(far_hi, round2))),
2);
_mm256_storeu_si256((__m256i*)dst_ptr, d_lo);
_mm256_storeu_si256((__m256i*)(dst_ptr + 16), d_hi);
dst_ptr += 32;
dst_width -= 32;
}
}
#endif // HAS_SCALEROWUP2_LINEAR_12_AVX2
#ifdef HAS_SCALEROWUP2_BILINEAR_12_AVX2
LIBYUV_TARGET_AVX2
void ScaleRowUp2_Bilinear_12_AVX2(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst_ptr,
ptrdiff_t dst_stride,
int dst_width) {
const __m256i shuf_far = _mm256_broadcastsi128_si256(
_mm_loadu_si128((const __m128i*)&kLinearShuffleFar));
const __m256i round8 = _mm256_set1_epi16(8);
while (dst_width > 0) {
__m256i a0 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8);
__m256i a1 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 1))),
0xd8);
__m256i near0 = _mm256_unpacklo_epi16(a0, a1);
__m256i far0 = _mm256_shuffle_epi8(near0, shuf_far);
__m256i r0 = _mm256_add_epi16(_mm256_add_epi16(near0, near0),
_mm256_add_epi16(near0, far0));
__m256i b0 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(
_mm_loadu_si128((const __m128i*)(src_ptr + src_stride))),
0xd8);
__m256i b1 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(
_mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 1))),
0xd8);
src_ptr += 8;
__m256i near1 = _mm256_unpacklo_epi16(b0, b1);
__m256i far1 = _mm256_shuffle_epi8(near1, shuf_far);
__m256i r1 = _mm256_add_epi16(_mm256_add_epi16(near1, near1),
_mm256_add_epi16(near1, far1));
__m256i d0 = _mm256_srli_epi16(
_mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0, r0), r0),
_mm256_add_epi16(r1, round8)),
4);
__m256i d1 = _mm256_srli_epi16(
_mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1, r1), r1),
_mm256_add_epi16(r0, round8)),
4);
_mm256_storeu_si256((__m256i*)dst_ptr, d0);
_mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride), d1);
dst_ptr += 16;
dst_width -= 16;
}
}
#endif // HAS_SCALEROWUP2_BILINEAR_12_AVX2
#ifdef HAS_SCALEROWUP2_LINEAR_16_AVX2
LIBYUV_TARGET_AVX2
void ScaleRowUp2_Linear_16_AVX2(const uint16_t* src_ptr,
uint16_t* dst_ptr,
int dst_width) {
const __m256i round2 = _mm256_set1_epi32(2);
while (dst_width > 0) {
__m256i near_lo =
_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr));
__m256i near_hi =
_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 1)));
src_ptr += 8;
__m256i far_lo = _mm256_shuffle_epi32(near_lo, 0xb1);
__m256i far_hi = _mm256_shuffle_epi32(near_hi, 0xb1);
__m256i d_lo = _mm256_srli_epi32(
_mm256_add_epi32(_mm256_add_epi32(near_lo, near_lo),
_mm256_add_epi32(near_lo, _mm256_add_epi32(far_lo, round2))),
2);
__m256i d_hi = _mm256_srli_epi32(
_mm256_add_epi32(_mm256_add_epi32(near_hi, near_hi),
_mm256_add_epi32(near_hi, _mm256_add_epi32(far_hi, round2))),
2);
__m256i d = _mm256_shuffle_epi32(_mm256_packus_epi32(d_lo, d_hi), 0xd8);
_mm256_storeu_si256((__m256i*)dst_ptr, d);
dst_ptr += 16;
dst_width -= 16;
}
}
#endif // HAS_SCALEROWUP2_LINEAR_16_AVX2
#ifdef HAS_SCALEROWUP2_BILINEAR_16_AVX2
LIBYUV_TARGET_AVX2
void ScaleRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst_ptr,
ptrdiff_t dst_stride,
int dst_width) {
const __m256i round8 = _mm256_set1_epi32(8);
while (dst_width > 0) {
__m256i a0 =
_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr));
__m256i a1 =
_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 1)));
__m256i r0_lo = _mm256_add_epi32(
_mm256_add_epi32(a0, a0),
_mm256_add_epi32(a0, _mm256_shuffle_epi32(a0, 0xb1)));
__m256i r0_hi = _mm256_add_epi32(
_mm256_add_epi32(a1, a1),
_mm256_add_epi32(a1, _mm256_shuffle_epi32(a1, 0xb1)));
__m256i b0 = _mm256_cvtepu16_epi32(
_mm_loadu_si128((const __m128i*)(src_ptr + src_stride)));
__m256i b1 = _mm256_cvtepu16_epi32(
_mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 1)));
src_ptr += 8;
__m256i r1_lo = _mm256_add_epi32(
_mm256_add_epi32(b0, b0),
_mm256_add_epi32(b0, _mm256_shuffle_epi32(b0, 0xb1)));
__m256i r1_hi = _mm256_add_epi32(
_mm256_add_epi32(b1, b1),
_mm256_add_epi32(b1, _mm256_shuffle_epi32(b1, 0xb1)));
__m256i d0_lo = _mm256_srli_epi32(
_mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_lo, r0_lo), r0_lo),
_mm256_add_epi32(r1_lo, round8)),
4);
__m256i d1_lo = _mm256_srli_epi32(
_mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_lo, r1_lo), r1_lo),
_mm256_add_epi32(r0_lo, round8)),
4);
__m256i d0_hi = _mm256_srli_epi32(
_mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_hi, r0_hi), r0_hi),
_mm256_add_epi32(r1_hi, round8)),
4);
__m256i d1_hi = _mm256_srli_epi32(
_mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_hi, r1_hi), r1_hi),
_mm256_add_epi32(r0_hi, round8)),
4);
_mm256_storeu_si256(
(__m256i*)dst_ptr,
_mm256_shuffle_epi32(_mm256_packus_epi32(d0_lo, d0_hi), 0xd8));
_mm256_storeu_si256(
(__m256i*)(dst_ptr + dst_stride),
_mm256_shuffle_epi32(_mm256_packus_epi32(d1_lo, d1_hi), 0xd8));
dst_ptr += 16;
dst_width -= 16;
}
}
#endif // HAS_SCALEROWUP2_BILINEAR_16_AVX2
#ifdef HAS_SCALEFILTERCOLS_AVX2
static const int32_t kStep0_7_Filter[8] = {0, 1, 2, 3, 4, 5, 6, 7};
static const uvec8 kShufFracCols = {0, 0, 4, 4, 8, 8, 12, 12,
128, 128, 128, 128, 128, 128, 128, 128};
LIBYUV_TARGET_AVX2
void ScaleFilterCols_AVX2(uint8_t* dst_ptr,
const uint8_t* src_ptr,
int dst_width,
int x,
int dx) {
int n = dst_width & ~7;
int r = dst_width & 7;
if (n > 0) {
SIMD_ALIGNED(int32_t xi_buf[8]);
SIMD_ALIGNED(uint16_t pix_buf[8]);
__m256i vx = _mm256_add_epi32(
_mm256_set1_epi32(x),
_mm256_mullo_epi32(
_mm256_set1_epi32(dx),
_mm256_loadu_si256((const __m256i*)kStep0_7_Filter)));
const __m256i vdx8 = _mm256_slli_epi32(_mm256_set1_epi32(dx), 3);
const __m256i shuf_frac = _mm256_broadcastsi128_si256(
_mm_loadu_si128((const __m128i*)&kShufFracCols));
const __m128i mask_7f = _mm_set1_epi16(0x007f);
const __m128i mask_ff = _mm_set1_epi16(0x00ff);
const __m128i round_40 = _mm_set1_epi16(0x0040);
for (int i = 0; i < n; i += 8) {
_mm256_store_si256((__m256i*)xi_buf, _mm256_srai_epi32(vx, 16));
__m256i f256 = _mm256_permute4x64_epi64(
_mm256_shuffle_epi8(_mm256_srli_epi16(vx, 9), shuf_frac), 0xd8);
vx = _mm256_add_epi32(vx, vdx8);
__m128i weights = _mm_xor_si128(_mm256_castsi256_si128(f256), mask_7f);
pix_buf[0] = LoadU16(src_ptr + xi_buf[0]);
pix_buf[1] = LoadU16(src_ptr + xi_buf[1]);
pix_buf[2] = LoadU16(src_ptr + xi_buf[2]);
pix_buf[3] = LoadU16(src_ptr + xi_buf[3]);
pix_buf[4] = LoadU16(src_ptr + xi_buf[4]);
pix_buf[5] = LoadU16(src_ptr + xi_buf[5]);
pix_buf[6] = LoadU16(src_ptr + xi_buf[6]);
pix_buf[7] = LoadU16(src_ptr + xi_buf[7]);
__m128i pix = _mm_load_si128((const __m128i*)pix_buf);
__m128i a = _mm_and_si128(pix, mask_ff);
__m128i res = _mm_srli_epi16(
_mm_add_epi16(_mm_add_epi16(_mm_maddubs_epi16(pix, weights), a),
round_40),
7);
_mm_storel_epi64((__m128i*)dst_ptr, _mm_packus_epi16(res, res));
dst_ptr += 8;
}
x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx);
}
for (int j = 0; j < r; ++j) {
int xi = x >> 16;
int a = src_ptr[xi];
int b = src_ptr[xi + 1];
dst_ptr[j] = (uint8_t)(a + ((((x & 0xffff) >> 9) * (b - a) + 0x40) >> 7));
x = (int)((uint32_t)x + (uint32_t)dx);
}
}
#endif // HAS_SCALEFILTERCOLS_AVX2
#ifdef HAS_SCALECOLSUP2_AVX2
LIBYUV_TARGET_AVX2
void ScaleColsUp2_AVX2(uint8_t* dst_ptr,
const uint8_t* src_ptr,
int dst_width,
int x,
int dx) {
(void)x;
(void)dx;
while (dst_width >= 32) {
__m256i s = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8);
src_ptr += 16;
_mm256_storeu_si256((__m256i*)dst_ptr, _mm256_unpacklo_epi8(s, s));
dst_ptr += 32;
dst_width -= 32;
}
if (dst_width & 16) {
__m128i s = _mm_loadl_epi64((const __m128i*)src_ptr);
src_ptr += 8;
_mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi8(s, s));
dst_ptr += 16;
}
if (dst_width & 8) {
__m128i s = _mm_cvtsi32_si128((int)LoadU32(src_ptr));
_mm_storel_epi64((__m128i*)dst_ptr, _mm_unpacklo_epi8(s, s));
}
}
#endif // HAS_SCALECOLSUP2_AVX2
#ifdef HAS_SCALEARGBROWDOWN2_AVX2
LIBYUV_TARGET_AVX2
void ScaleARGBRowDown2_AVX2(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width) {
(void)src_stride;
while (dst_width > 0) {
__m256 s0 = _mm256_loadu_ps((const float*)src_argb);
__m256 s1 = _mm256_loadu_ps((const float*)(src_argb + 32));
src_argb += 64;
__m256i d = _mm256_permute4x64_epi64(
_mm256_castps_si256(_mm256_shuffle_ps(s0, s1, 0xdd)), 0xd8);
_mm256_storeu_si256((__m256i*)dst_argb, d);
dst_argb += 32;
dst_width -= 8;
}
}
LIBYUV_TARGET_AVX2
void ScaleARGBRowDown2Linear_AVX2(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width) {
(void)src_stride;
while (dst_width > 0) {
__m256 s0 = _mm256_loadu_ps((const float*)src_argb);
__m256 s1 = _mm256_loadu_ps((const float*)(src_argb + 32));
src_argb += 64;
__m256i e = _mm256_castps_si256(_mm256_shuffle_ps(s0, s1, 0x88));
__m256i o = _mm256_castps_si256(_mm256_shuffle_ps(s0, s1, 0xdd));
__m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu8(e, o), 0xd8);
_mm256_storeu_si256((__m256i*)dst_argb, d);
dst_argb += 32;
dst_width -= 8;
}
}
LIBYUV_TARGET_AVX2
void ScaleARGBRowDown2Box_AVX2(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width) {
while (dst_width > 0) {
__m256i r0 = _mm256_avg_epu8(
_mm256_loadu_si256((const __m256i*)src_argb),
_mm256_loadu_si256((const __m256i*)(src_argb + src_stride)));
__m256i r1 = _mm256_avg_epu8(
_mm256_loadu_si256((const __m256i*)(src_argb + 32)),
_mm256_loadu_si256((const __m256i*)(src_argb + src_stride + 32)));
src_argb += 64;
__m256i e = _mm256_castps_si256(
_mm256_shuffle_ps(_mm256_castsi256_ps(r0), _mm256_castsi256_ps(r1),
0x88));
__m256i o = _mm256_castps_si256(
_mm256_shuffle_ps(_mm256_castsi256_ps(r0), _mm256_castsi256_ps(r1),
0xdd));
__m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu8(e, o), 0xd8);
_mm256_storeu_si256((__m256i*)dst_argb, d);
dst_argb += 32;
dst_width -= 8;
}
}
#endif // HAS_SCALEARGBROWDOWN2_AVX2
#ifdef HAS_SCALEARGBROWDOWNEVEN_AVX2
LIBYUV_TARGET_AVX2
void ScaleARGBRowDownEven_AVX2(const uint8_t* src_argb,
ptrdiff_t src_stride,
int src_stepx,
uint8_t* dst_argb,
int dst_width) {
(void)src_stride;
const ptrdiff_t step = (ptrdiff_t)src_stepx * 4;
SIMD_ALIGNED(uint32_t pix_buf[8]);
while (dst_width > 0) {
pix_buf[0] = LoadU32(src_argb);
pix_buf[1] = LoadU32(src_argb + step);
pix_buf[2] = LoadU32(src_argb + step * 2);
pix_buf[3] = LoadU32(src_argb + step * 3);
pix_buf[4] = LoadU32(src_argb + step * 4);
pix_buf[5] = LoadU32(src_argb + step * 5);
pix_buf[6] = LoadU32(src_argb + step * 6);
pix_buf[7] = LoadU32(src_argb + step * 7);
src_argb += step * 8;
_mm256_storeu_si256((__m256i*)dst_argb,
_mm256_load_si256((const __m256i*)pix_buf));
dst_argb += 32;
dst_width -= 8;
}
}
LIBYUV_TARGET_AVX2
void ScaleARGBRowDownEvenBox_AVX2(const uint8_t* src_argb,
ptrdiff_t src_stride,
int src_stepx,
uint8_t* dst_argb,
int dst_width) {
const uint8_t* row1 = src_argb + src_stride;
const ptrdiff_t step = (ptrdiff_t)src_stepx * 4;
SIMD_ALIGNED(uint64_t r0_lo[4]);
SIMD_ALIGNED(uint64_t r0_hi[4]);
SIMD_ALIGNED(uint64_t r1_lo[4]);
SIMD_ALIGNED(uint64_t r1_hi[4]);
while (dst_width > 0) {
r0_lo[0] = LoadU64(src_argb);
r0_lo[1] = LoadU64(src_argb + step);
r0_lo[2] = LoadU64(src_argb + step * 2);
r0_lo[3] = LoadU64(src_argb + step * 3);
r0_hi[0] = LoadU64(src_argb + step * 4);
r0_hi[1] = LoadU64(src_argb + step * 5);
r0_hi[2] = LoadU64(src_argb + step * 6);
r0_hi[3] = LoadU64(src_argb + step * 7);
r1_lo[0] = LoadU64(row1);
r1_lo[1] = LoadU64(row1 + step);
r1_lo[2] = LoadU64(row1 + step * 2);
r1_lo[3] = LoadU64(row1 + step * 3);
r1_hi[0] = LoadU64(row1 + step * 4);
r1_hi[1] = LoadU64(row1 + step * 5);
r1_hi[2] = LoadU64(row1 + step * 6);
r1_hi[3] = LoadU64(row1 + step * 7);
src_argb += step * 8;
row1 += step * 8;
__m256i a0 = _mm256_avg_epu8(_mm256_load_si256((const __m256i*)r0_lo),
_mm256_load_si256((const __m256i*)r1_lo));
__m256i a1 = _mm256_avg_epu8(_mm256_load_si256((const __m256i*)r0_hi),
_mm256_load_si256((const __m256i*)r1_hi));
__m256i e = _mm256_castps_si256(
_mm256_shuffle_ps(_mm256_castsi256_ps(a0), _mm256_castsi256_ps(a1),
0x88));
__m256i o = _mm256_castps_si256(
_mm256_shuffle_ps(_mm256_castsi256_ps(a0), _mm256_castsi256_ps(a1),
0xdd));
__m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu8(e, o), 0xd8);
_mm256_storeu_si256((__m256i*)dst_argb, d);
dst_argb += 32;
dst_width -= 8;
}
}
#endif // HAS_SCALEARGBROWDOWNEVEN_AVX2
#ifdef HAS_SCALEARGBCOLS_AVX2
static const int32_t kStep0_7_Cols[8] = {0, 1, 2, 3, 4, 5, 6, 7};
LIBYUV_TARGET_AVX2
void ScaleARGBCols_AVX2(uint8_t* dst_argb,
const uint8_t* src_argb,
int dst_width,
int x,
int dx) {
int n = dst_width & ~7;
int r = dst_width & 7;
if (n > 0) {
SIMD_ALIGNED(int32_t xi_buf[8]);
SIMD_ALIGNED(uint32_t pix_buf[8]);
__m256i vx = _mm256_add_epi32(
_mm256_set1_epi32(x),
_mm256_mullo_epi32(
_mm256_set1_epi32(dx),
_mm256_loadu_si256((const __m256i*)kStep0_7_Cols)));
const __m256i vdx8 = _mm256_slli_epi32(_mm256_set1_epi32(dx), 3);
for (int i = 0; i < n; i += 8) {
_mm256_store_si256((__m256i*)xi_buf, _mm256_srai_epi32(vx, 16));
vx = _mm256_add_epi32(vx, vdx8);
pix_buf[0] = LoadU32(src_argb + (ptrdiff_t)xi_buf[0] * 4);
pix_buf[1] = LoadU32(src_argb + (ptrdiff_t)xi_buf[1] * 4);
pix_buf[2] = LoadU32(src_argb + (ptrdiff_t)xi_buf[2] * 4);
pix_buf[3] = LoadU32(src_argb + (ptrdiff_t)xi_buf[3] * 4);
pix_buf[4] = LoadU32(src_argb + (ptrdiff_t)xi_buf[4] * 4);
pix_buf[5] = LoadU32(src_argb + (ptrdiff_t)xi_buf[5] * 4);
pix_buf[6] = LoadU32(src_argb + (ptrdiff_t)xi_buf[6] * 4);
pix_buf[7] = LoadU32(src_argb + (ptrdiff_t)xi_buf[7] * 4);
_mm256_storeu_si256((__m256i*)dst_argb,
_mm256_load_si256((const __m256i*)pix_buf));
dst_argb += 32;
}
x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx);
}
for (int j = 0; j < r; ++j) {
const uint8_t* src = src_argb + (ptrdiff_t)(x >> 16) * 4;
dst_argb[0] = src[0];
dst_argb[1] = src[1];
dst_argb[2] = src[2];
dst_argb[3] = src[3];
dst_argb += 4;
x = (int)((uint32_t)x + (uint32_t)dx);
}
}
#endif // HAS_SCALEARGBCOLS_AVX2
#ifdef HAS_SCALEARGBCOLSUP2_AVX2
LIBYUV_TARGET_AVX2
void ScaleARGBColsUp2_AVX2(uint8_t* dst_argb,
const uint8_t* src_argb,
int dst_width,
int x,
int dx) {
(void)x;
(void)dx;
while (dst_width > 0) {
__m256i s = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_argb)),
0xd8);
src_argb += 16;
_mm256_storeu_si256((__m256i*)dst_argb, _mm256_unpacklo_epi32(s, s));
dst_argb += 32;
dst_width -= 8;
}
}
#endif // HAS_SCALEARGBCOLSUP2_AVX2
#ifdef HAS_SCALEARGBFILTERCOLS_AVX2
static const int32_t kStep0_3_ARGB[4] = {0, 1, 2, 3};
static const uvec8 kShuffleColARGB = {
0u, 4u, 1u, 5u, 2u, 6u, 3u, 7u,
8u, 12u, 9u, 13u, 10u, 14u, 11u, 15u};
static const uint8_t kShuffleFractionsAVX2[32] = {
0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u,
8u, 8u, 8u, 8u, 8u, 8u, 8u, 8u, 12u, 12u, 12u, 12u, 12u, 12u, 12u, 12u,
};
LIBYUV_TARGET_AVX2
void ScaleARGBFilterCols_AVX2(uint8_t* dst_argb,
const uint8_t* src_argb,
int dst_width,
int x,
int dx) {
int n = dst_width & ~3;
int r = dst_width & 3;
if (n > 0) {
SIMD_ALIGNED(int32_t xi_buf[4]);
SIMD_ALIGNED(uint64_t pix_buf[4]);
const __m256i shuf_argb = _mm256_broadcastsi128_si256(
_mm_loadu_si128((const __m128i*)&kShuffleColARGB));
const __m256i shuf_frac =
_mm256_loadu_si256((const __m256i*)kShuffleFractionsAVX2);
const __m256i mask_7f = _mm256_set1_epi16(0x007f);
__m128i vx = _mm_add_epi32(
_mm_set1_epi32(x),
_mm_mullo_epi32(_mm_set1_epi32(dx),
_mm_loadu_si128((const __m128i*)kStep0_3_ARGB)));
const __m128i vdx4 = _mm_slli_epi32(_mm_set1_epi32(dx), 2);
for (int i = 0; i < n; i += 4) {
_mm_store_si128((__m128i*)xi_buf, _mm_srai_epi32(vx, 16));
__m128i f128 = _mm_srli_epi16(vx, 9);
vx = _mm_add_epi32(vx, vdx4);
pix_buf[0] = LoadU64(src_argb + (ptrdiff_t)xi_buf[0] * 4);
pix_buf[1] = LoadU64(src_argb + (ptrdiff_t)xi_buf[1] * 4);
pix_buf[2] = LoadU64(src_argb + (ptrdiff_t)xi_buf[2] * 4);
pix_buf[3] = LoadU64(src_argb + (ptrdiff_t)xi_buf[3] * 4);
__m256i pix = _mm256_shuffle_epi8(
_mm256_load_si256((const __m256i*)pix_buf), shuf_argb);
__m256i weights = _mm256_xor_si256(
_mm256_shuffle_epi8(
_mm256_inserti128_si256(_mm256_zextsi128_si256(f128), f128, 1),
shuf_frac),
mask_7f);
__m256i res = _mm256_srli_epi16(_mm256_maddubs_epi16(pix, weights), 7);
res = _mm256_permute4x64_epi64(_mm256_packus_epi16(res, res), 0xd8);
_mm_storeu_si128((__m128i*)dst_argb, _mm256_castsi256_si128(res));
dst_argb += 16;
}
x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx);
}
for (int j = 0; j < r; ++j) {
ptrdiff_t xi = (ptrdiff_t)(x >> 16) * 4;
int xf = (x >> 9) & 0x7f;
const uint8_t* a = src_argb + xi;
const uint8_t* b = a + 4;
dst_argb[0] = (uint8_t)((a[0] * (0x7f ^ xf) + b[0] * xf) >> 7);
dst_argb[1] = (uint8_t)((a[1] * (0x7f ^ xf) + b[1] * xf) >> 7);
dst_argb[2] = (uint8_t)((a[2] * (0x7f ^ xf) + b[2] * xf) >> 7);
dst_argb[3] = (uint8_t)((a[3] * (0x7f ^ xf) + b[3] * xf) >> 7);
dst_argb += 4;
x = (int)((uint32_t)x + (uint32_t)dx);
}
}
#endif // HAS_SCALEARGBFILTERCOLS_AVX2
#ifdef HAS_SCALEUVROWDOWN2_AVX2
LIBYUV_TARGET_AVX2
void ScaleUVRowDown2_AVX2(const uint8_t* src_uv,
ptrdiff_t src_stride,
uint8_t* dst_uv,
int dst_width) {
(void)src_stride;
while (dst_width >= 16) {
__m256i s0 =
_mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)src_uv), 16);
__m256i s1 = _mm256_srli_epi32(
_mm256_loadu_si256((const __m256i*)(src_uv + 32)), 16);
src_uv += 64;
__m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8);
_mm256_storeu_si256((__m256i*)dst_uv, d);
dst_uv += 32;
dst_width -= 16;
}
if (dst_width > 0) {
__m256i s0 =
_mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)src_uv), 16);
__m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s0), 0xd8);
_mm_storeu_si128((__m128i*)dst_uv, _mm256_castsi256_si128(d));
}
}
#endif // HAS_SCALEUVROWDOWN2_AVX2
#ifdef HAS_SCALEUVROWDOWN2LINEAR_AVX2
LIBYUV_TARGET_AVX2
void ScaleUVRowDown2Linear_AVX2(const uint8_t* src_uv,
ptrdiff_t src_stride,
uint8_t* dst_uv,
int dst_width) {
(void)src_stride;
const __m256i mask = _mm256_set1_epi32(0x0000ffff);
while (dst_width >= 16) {
__m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv);
__m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + 32));
src_uv += 64;
__m256i s0 =
_mm256_and_si256(_mm256_avg_epu8(r0, _mm256_srli_epi32(r0, 16)), mask);
__m256i s1 =
_mm256_and_si256(_mm256_avg_epu8(r1, _mm256_srli_epi32(r1, 16)), mask);
__m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8);
_mm256_storeu_si256((__m256i*)dst_uv, d);
dst_uv += 32;
dst_width -= 16;
}
if (dst_width > 0) {
__m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv);
__m256i s0 =
_mm256_and_si256(_mm256_avg_epu8(r0, _mm256_srli_epi32(r0, 16)), mask);
__m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s0), 0xd8);
_mm_storeu_si128((__m128i*)dst_uv, _mm256_castsi256_si128(d));
}
}
#endif // HAS_SCALEUVROWDOWN2LINEAR_AVX2
#ifdef HAS_SCALEUVROWDOWN2BOX_AVX2
static const uvec8 kShuffleSplitUV = {0u, 2u, 4u, 6u, 8u, 10u, 12u, 14u,
1u, 3u, 5u, 7u, 9u, 11u, 13u, 15u};
static const uvec8 kShuffleMergeUV = {0u, 8u, 2u, 10u, 4u, 12u,
6u, 14u, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80};
LIBYUV_TARGET_AVX2
void ScaleUVRowDown2Box_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
const __m256i split = _mm256_broadcastsi128_si256(
_mm_loadu_si128((const __m128i*)&kShuffleSplitUV));
const __m256i merge = _mm256_broadcastsi128_si256(
_mm_loadu_si128((const __m128i*)&kShuffleMergeUV));
const __m256i ones = _mm256_set1_epi8(1);
const __m256i zero = _mm256_setzero_si256();
while (dst_width > 0) {
__m256i r0 = _mm256_maddubs_epi16(
_mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_ptr), split),
ones);
__m256i r1 = _mm256_maddubs_epi16(
_mm256_shuffle_epi8(
_mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)), split),
ones);
src_ptr += 32;
__m256i s =
_mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(r0, r1), 1), zero);
__m256i d = _mm256_permute4x64_epi64(_mm256_shuffle_epi8(s, merge), 0xd8);
_mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d));
dst_ptr += 16;
dst_width -= 8;
}
}
#endif // HAS_SCALEUVROWDOWN2BOX_AVX2
#ifdef HAS_SCALEUVROWDOWNEVEN_AVX2
LIBYUV_TARGET_AVX2
void ScaleUVRowDownEven_AVX2(const uint8_t* src_uv,
ptrdiff_t src_stride,
int src_stepx,
uint8_t* dst_uv,
int dst_width) {
(void)src_stride;
const ptrdiff_t step = (ptrdiff_t)src_stepx * 2;
SIMD_ALIGNED(uint16_t uv_buf[8]);
while (dst_width > 0) {
uv_buf[0] = LoadU16(src_uv);
uv_buf[1] = LoadU16(src_uv + step);
uv_buf[2] = LoadU16(src_uv + step * 2);
uv_buf[3] = LoadU16(src_uv + step * 3);
uv_buf[4] = LoadU16(src_uv + step * 4);
uv_buf[5] = LoadU16(src_uv + step * 5);
uv_buf[6] = LoadU16(src_uv + step * 6);
uv_buf[7] = LoadU16(src_uv + step * 7);
src_uv += step * 8;
_mm_storeu_si128((__m128i*)dst_uv, _mm_load_si128((const __m128i*)uv_buf));
dst_uv += 16;
dst_width -= 8;
}
}
#endif // HAS_SCALEUVROWDOWNEVEN_AVX2
#if defined(HAS_SCALEUVROWUP2_LINEAR_AVX2) || \
defined(HAS_SCALEUVROWUP2_BILINEAR_AVX2)
static const uvec8 kUVLinearMadd31 = {3, 1, 3, 1, 1, 3, 1, 3,
3, 1, 3, 1, 1, 3, 1, 3};
#endif
#ifdef HAS_SCALEUVROWUP2_LINEAR_AVX2
LIBYUV_TARGET_AVX2
void ScaleUVRowUp2_Linear_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width) {
const __m256i madd31 = _mm256_broadcastsi128_si256(
_mm_loadu_si128((const __m128i*)&kUVLinearMadd31));
const __m256i round2 = _mm256_set1_epi16(2);
while (dst_width > 0) {
__m256i s0 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8);
__m256i s1 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 2))),
0xd8);
src_ptr += 16;
__m256i p = _mm256_unpacklo_epi8(s0, s1);
__m256i lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(p, p), madd31);
__m256i hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(p, p), madd31);
lo = _mm256_srli_epi16(_mm256_add_epi16(lo, round2), 2);
hi = _mm256_srli_epi16(_mm256_add_epi16(hi, round2), 2);
_mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(lo, hi));
dst_ptr += 32;
dst_width -= 16;
}
}
#endif // HAS_SCALEUVROWUP2_LINEAR_AVX2
#ifdef HAS_SCALEUVROWUP2_BILINEAR_AVX2
LIBYUV_TARGET_AVX2
void ScaleUVRowUp2_Bilinear_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
ptrdiff_t dst_stride,
int dst_width) {
const __m256i madd31 = _mm256_broadcastsi128_si256(
_mm_loadu_si128((const __m128i*)&kUVLinearMadd31));
const __m256i round8 = _mm256_set1_epi16(8);
while (dst_width > 0) {
__m256i a0 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8);
__m256i a1 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 2))),
0xd8);
__m256i pa = _mm256_unpacklo_epi8(a0, a1);
__m256i r0_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(pa, pa), madd31);
__m256i r0_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(pa, pa), madd31);
__m256i b0 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(
_mm_loadu_si128((const __m128i*)(src_ptr + src_stride))),
0xd8);
__m256i b1 = _mm256_permute4x64_epi64(
_mm256_zextsi128_si256(
_mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 2))),
0xd8);
src_ptr += 16;
__m256i pb = _mm256_unpacklo_epi8(b0, b1);
__m256i r1_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(pb, pb), madd31);
__m256i r1_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(pb, pb), madd31);
__m256i d0_lo = _mm256_srli_epi16(
_mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_lo, r0_lo), r0_lo),
_mm256_add_epi16(r1_lo, round8)),
4);
__m256i d1_lo = _mm256_srli_epi16(
_mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_lo, r1_lo), r1_lo),
_mm256_add_epi16(r0_lo, round8)),
4);
__m256i d0_hi = _mm256_srli_epi16(
_mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_hi, r0_hi), r0_hi),
_mm256_add_epi16(r1_hi, round8)),
4);
__m256i d1_hi = _mm256_srli_epi16(
_mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_hi, r1_hi), r1_hi),
_mm256_add_epi16(r0_hi, round8)),
4);
_mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(d0_lo, d0_hi));
_mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride),
_mm256_packus_epi16(d1_lo, d1_hi));
dst_ptr += 32;
dst_width -= 16;
}
}
#endif // HAS_SCALEUVROWUP2_BILINEAR_AVX2
#ifdef HAS_SCALEUVROWUP2_LINEAR_16_AVX2
LIBYUV_TARGET_AVX2
void ScaleUVRowUp2_Linear_16_AVX2(const uint16_t* src_ptr,
uint16_t* dst_ptr,
int dst_width) {
const __m256i round2 = _mm256_set1_epi32(2);
while (dst_width > 0) {
__m256i near_lo =
_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr));
__m256i near_hi =
_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 2)));
src_ptr += 8;
__m256i far_lo = _mm256_shuffle_epi32(near_lo, 0x4e);
__m256i far_hi = _mm256_shuffle_epi32(near_hi, 0x4e);
__m256i d_lo = _mm256_srli_epi32(
_mm256_add_epi32(_mm256_add_epi32(near_lo, near_lo),
_mm256_add_epi32(near_lo, _mm256_add_epi32(far_lo, round2))),
2);
__m256i d_hi = _mm256_srli_epi32(
_mm256_add_epi32(_mm256_add_epi32(near_hi, near_hi),
_mm256_add_epi32(near_hi, _mm256_add_epi32(far_hi, round2))),
2);
_mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi32(d_lo, d_hi));
dst_ptr += 16;
dst_width -= 8;
}
}
#endif // HAS_SCALEUVROWUP2_LINEAR_16_AVX2
#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_AVX2
LIBYUV_TARGET_AVX2
void ScaleUVRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst_ptr,
ptrdiff_t dst_stride,
int dst_width) {
const __m256i round8 = _mm256_set1_epi32(8);
while (dst_width > 0) {
__m256i a0 =
_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr));
__m256i a1 =
_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 2)));
__m256i r0_lo = _mm256_add_epi32(
_mm256_add_epi32(a0, a0),
_mm256_add_epi32(a0, _mm256_shuffle_epi32(a0, 0x4e)));
__m256i r0_hi = _mm256_add_epi32(
_mm256_add_epi32(a1, a1),
_mm256_add_epi32(a1, _mm256_shuffle_epi32(a1, 0x4e)));
__m256i b0 = _mm256_cvtepu16_epi32(
_mm_loadu_si128((const __m128i*)(src_ptr + src_stride)));
__m256i b1 = _mm256_cvtepu16_epi32(
_mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 2)));
src_ptr += 8;
__m256i r1_lo = _mm256_add_epi32(
_mm256_add_epi32(b0, b0),
_mm256_add_epi32(b0, _mm256_shuffle_epi32(b0, 0x4e)));
__m256i r1_hi = _mm256_add_epi32(
_mm256_add_epi32(b1, b1),
_mm256_add_epi32(b1, _mm256_shuffle_epi32(b1, 0x4e)));
__m256i d0_lo = _mm256_srli_epi32(
_mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_lo, r0_lo), r0_lo),
_mm256_add_epi32(r1_lo, round8)),
4);
__m256i d1_lo = _mm256_srli_epi32(
_mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_lo, r1_lo), r1_lo),
_mm256_add_epi32(r0_lo, round8)),
4);
__m256i d0_hi = _mm256_srli_epi32(
_mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_hi, r0_hi), r0_hi),
_mm256_add_epi32(r1_hi, round8)),
4);
__m256i d1_hi = _mm256_srli_epi32(
_mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_hi, r1_hi), r1_hi),
_mm256_add_epi32(r0_hi, round8)),
4);
_mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi32(d0_lo, d0_hi));
_mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride),
_mm256_packus_epi32(d1_lo, d1_hi));
dst_ptr += 16;
dst_width -= 8;
}
}
#endif // HAS_SCALEUVROWUP2_BILINEAR_16_AVX2
#undef LIBYUV_TARGET_AVX2
#undef LIBYUV_TARGET_AVX512BW
#undef LIBYUV_TARGET_AVX512VBMI
#endif // !defined(LIBYUV_DISABLE_X86) && ...
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif