mirror of
https://github.com/signalwire/freeswitch.git
synced 2026-11-18 07:18:34 +00:00
10858 lines
435 KiB
C++
10858 lines
435 KiB
C++
/*
|
|
* Copyright 2011 The LibYuv Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "libyuv/convert_from_argb.h" // For ArgbConstants
|
|
#include "libyuv/row.h"
|
|
|
|
// This module is for Visual C 32/64 bit
|
|
#if !defined(LIBYUV_DISABLE_X86) && \
|
|
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
|
|
defined(_M_IX86)) && \
|
|
((defined(_MSC_VER) && !defined(__clang__)) || \
|
|
defined(LIBYUV_ENABLE_ROWWIN))
|
|
|
|
#include <emmintrin.h>
|
|
#include <immintrin.h> // For AVX2 intrinsics
|
|
#include <string.h> // For memcpy
|
|
#include <tmmintrin.h> // For _mm_maddubs_epi16
|
|
#if defined(_MSC_VER) && !defined(__clang__)
|
|
#include <intrin.h>
|
|
#endif
|
|
|
|
#ifdef __cplusplus
|
|
namespace libyuv {
|
|
extern "C" {
|
|
#endif
|
|
|
|
#if defined(__clang__) || defined(__GNUC__)
|
|
#define LIBYUV_TARGET_SSSE3 __attribute__((target("ssse3")))
|
|
#define LIBYUV_TARGET_AVX __attribute__((target("avx")))
|
|
#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
|
|
#define LIBYUV_TARGET_AVX2_FMA __attribute__((target("avx2,fma")))
|
|
#define LIBYUV_TARGET_AVX512BW \
|
|
__attribute__((target("avx512bw,avx512vl,avx512f")))
|
|
#define LIBYUV_TARGET_AVX512VBMI \
|
|
__attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f")))
|
|
#define LIBYUV_TARGET_AVX512VNNI \
|
|
__attribute__((target("avx512vnni,avx512bw,avx512vl,avx512f")))
|
|
#define LIBYUV_TARGET_AVX512VNNIVBMI \
|
|
__attribute__((target("avx512vnni,avx512vbmi,avx512bw,avx512vl,avx512f")))
|
|
#else
|
|
#define LIBYUV_TARGET_SSSE3
|
|
#define LIBYUV_TARGET_AVX
|
|
#define LIBYUV_TARGET_AVX2
|
|
#define LIBYUV_TARGET_AVX2_FMA
|
|
#define LIBYUV_TARGET_AVX512BW
|
|
#define LIBYUV_TARGET_AVX512VBMI
|
|
#define LIBYUV_TARGET_AVX512VNNI
|
|
#define LIBYUV_TARGET_AVX512VNNIVBMI
|
|
#endif // defined(__clang__) || defined(__GNUC__)
|
|
|
|
#if defined(__x86_64__) || defined(_M_X64)
|
|
// Under MemorySanitizer, use portable lane-by-lane C loops for single-source
|
|
// variable permutes and 512-bit float-to-int truncation so shadow propagation
|
|
// relies only on standard C memory/array semantics rather than target-specific
|
|
// LLVM intrinsic modeling. In non-MSan builds, call the native AVX-512
|
|
// intrinsics directly.
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
static inline __m512i PermutexvarEpi8(__m512i idx, __m512i v) {
|
|
#if defined(LIBYUV_MSAN)
|
|
uint8_t idx_arr[64];
|
|
uint8_t v_arr[64];
|
|
uint8_t out_arr[64];
|
|
_mm512_storeu_si512((__m512i*)idx_arr, idx);
|
|
_mm512_storeu_si512((__m512i*)v_arr, v);
|
|
for (int i = 0; i < 64; ++i) {
|
|
out_arr[i] = v_arr[idx_arr[i] & 63];
|
|
}
|
|
return _mm512_loadu_si512((const __m512i*)out_arr);
|
|
#else
|
|
return _mm512_permutexvar_epi8(idx, v);
|
|
#endif
|
|
}
|
|
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
static inline __m512i MaskPermutexvarEpi8(__m512i src,
|
|
__mmask64 k,
|
|
__m512i idx,
|
|
__m512i v) {
|
|
#if defined(LIBYUV_MSAN)
|
|
uint8_t src_arr[64];
|
|
uint8_t idx_arr[64];
|
|
uint8_t v_arr[64];
|
|
uint8_t out_arr[64];
|
|
_mm512_storeu_si512((__m512i*)src_arr, src);
|
|
_mm512_storeu_si512((__m512i*)idx_arr, idx);
|
|
_mm512_storeu_si512((__m512i*)v_arr, v);
|
|
for (int i = 0; i < 64; ++i) {
|
|
out_arr[i] = ((k >> i) & 1) ? v_arr[idx_arr[i] & 63] : src_arr[i];
|
|
}
|
|
return _mm512_loadu_si512((const __m512i*)out_arr);
|
|
#else
|
|
return _mm512_mask_permutexvar_epi8(src, k, idx, v);
|
|
#endif
|
|
}
|
|
|
|
LIBYUV_TARGET_AVX512BW
|
|
static inline __m512i PermutexvarEpi16(__m512i idx, __m512i v) {
|
|
#if defined(LIBYUV_MSAN)
|
|
uint16_t idx_arr[32];
|
|
uint16_t v_arr[32];
|
|
uint16_t out_arr[32];
|
|
_mm512_storeu_si512((__m512i*)idx_arr, idx);
|
|
_mm512_storeu_si512((__m512i*)v_arr, v);
|
|
for (int i = 0; i < 32; ++i) {
|
|
out_arr[i] = v_arr[idx_arr[i] & 31];
|
|
}
|
|
return _mm512_loadu_si512((const __m512i*)out_arr);
|
|
#else
|
|
return _mm512_permutexvar_epi16(idx, v);
|
|
#endif
|
|
}
|
|
|
|
LIBYUV_TARGET_AVX512BW
|
|
static inline __m512i PermutexvarEpi32(__m512i idx, __m512i v) {
|
|
#if defined(LIBYUV_MSAN)
|
|
uint32_t idx_arr[16];
|
|
uint32_t v_arr[16];
|
|
uint32_t out_arr[16];
|
|
_mm512_storeu_si512((__m512i*)idx_arr, idx);
|
|
_mm512_storeu_si512((__m512i*)v_arr, v);
|
|
for (int i = 0; i < 16; ++i) {
|
|
out_arr[i] = v_arr[idx_arr[i] & 15];
|
|
}
|
|
return _mm512_loadu_si512((const __m512i*)out_arr);
|
|
#else
|
|
return _mm512_permutexvar_epi32(idx, v);
|
|
#endif
|
|
}
|
|
|
|
LIBYUV_TARGET_AVX512BW
|
|
static inline __m512i PermutexvarEpi64(__m512i idx, __m512i v) {
|
|
#if defined(LIBYUV_MSAN)
|
|
uint64_t idx_arr[8];
|
|
uint64_t v_arr[8];
|
|
uint64_t out_arr[8];
|
|
_mm512_storeu_si512((__m512i*)idx_arr, idx);
|
|
_mm512_storeu_si512((__m512i*)v_arr, v);
|
|
for (int i = 0; i < 8; ++i) {
|
|
out_arr[i] = v_arr[idx_arr[i] & 7];
|
|
}
|
|
return _mm512_loadu_si512((const __m512i*)out_arr);
|
|
#else
|
|
return _mm512_permutexvar_epi64(idx, v);
|
|
#endif
|
|
}
|
|
|
|
LIBYUV_TARGET_AVX512BW
|
|
static inline __m512i CvttpsEpi32(__m512 v) {
|
|
#if defined(LIBYUV_MSAN)
|
|
float v_arr[16];
|
|
int32_t out_arr[16];
|
|
_mm512_storeu_ps(v_arr, v);
|
|
for (int i = 0; i < 16; ++i) {
|
|
out_arr[i] = (int32_t)v_arr[i];
|
|
}
|
|
return _mm512_loadu_si512((const __m512i*)out_arr);
|
|
#else
|
|
return _mm512_cvttps_epi32(v);
|
|
#endif
|
|
}
|
|
#endif // defined(__x86_64__) || defined(_M_X64)
|
|
|
|
// Read 8 UV from 444
|
|
#define READYUV444 \
|
|
xmm3 = _mm_loadl_epi64((__m128i*)u_buf); \
|
|
xmm1 = _mm_loadl_epi64((__m128i*)(u_buf + offset)); \
|
|
xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
|
|
u_buf += 8; \
|
|
xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
|
|
xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
|
|
y_buf += 8;
|
|
|
|
// Read 8 UV from 444, With 8 Alpha.
|
|
#define READYUVA444 \
|
|
xmm3 = _mm_loadl_epi64((__m128i*)u_buf); \
|
|
xmm1 = _mm_loadl_epi64((__m128i*)(u_buf + offset)); \
|
|
xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
|
|
u_buf += 8; \
|
|
xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
|
|
xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
|
|
y_buf += 8; \
|
|
xmm5 = _mm_loadl_epi64((__m128i*)a_buf); \
|
|
a_buf += 8;
|
|
|
|
// Read 4 UV from 422, upsample to 8 UV.
|
|
#define READYUV422 \
|
|
xmm3 = _mm_cvtsi32_si128(*(uint32_t*)u_buf); \
|
|
xmm1 = _mm_cvtsi32_si128(*(uint32_t*)(u_buf + offset)); \
|
|
xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
|
|
xmm3 = _mm_unpacklo_epi16(xmm3, xmm3); \
|
|
u_buf += 4; \
|
|
xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
|
|
xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
|
|
y_buf += 8;
|
|
|
|
// Read 4 UV from 422, upsample to 8 UV. With 8 Alpha.
|
|
#define READYUVA422 \
|
|
xmm3 = _mm_cvtsi32_si128(*(uint32_t*)u_buf); \
|
|
xmm1 = _mm_cvtsi32_si128(*(uint32_t*)(u_buf + offset)); \
|
|
xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
|
|
xmm3 = _mm_unpacklo_epi16(xmm3, xmm3); \
|
|
u_buf += 4; \
|
|
xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
|
|
xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
|
|
y_buf += 8; \
|
|
xmm5 = _mm_loadl_epi64((__m128i*)a_buf); \
|
|
a_buf += 8;
|
|
|
|
// Convert 8 pixels: 8 UV and 8 Y.
|
|
#define YUVTORGB(yuvconstants) \
|
|
xmm3 = _mm_sub_epi8(xmm3, _mm_set1_epi8((char)0x80)); \
|
|
xmm4 = _mm_mulhi_epu16(xmm4, *(__m128i*)yuvconstants->kYToRgb); \
|
|
xmm4 = _mm_add_epi16(xmm4, *(__m128i*)yuvconstants->kYBiasToRgb); \
|
|
xmm0 = _mm_maddubs_epi16(*(__m128i*)yuvconstants->kUVToB, xmm3); \
|
|
xmm1 = _mm_maddubs_epi16(*(__m128i*)yuvconstants->kUVToG, xmm3); \
|
|
xmm2 = _mm_maddubs_epi16(*(__m128i*)yuvconstants->kUVToR, xmm3); \
|
|
xmm0 = _mm_adds_epi16(xmm4, xmm0); \
|
|
xmm1 = _mm_subs_epi16(xmm4, xmm1); \
|
|
xmm2 = _mm_adds_epi16(xmm4, xmm2); \
|
|
xmm0 = _mm_srai_epi16(xmm0, 6); \
|
|
xmm1 = _mm_srai_epi16(xmm1, 6); \
|
|
xmm2 = _mm_srai_epi16(xmm2, 6); \
|
|
xmm0 = _mm_packus_epi16(xmm0, xmm0); \
|
|
xmm1 = _mm_packus_epi16(xmm1, xmm1); \
|
|
xmm2 = _mm_packus_epi16(xmm2, xmm2);
|
|
|
|
// Store 8 ARGB values.
|
|
#define STOREARGB \
|
|
xmm0 = _mm_unpacklo_epi8(xmm0, xmm1); \
|
|
xmm2 = _mm_unpacklo_epi8(xmm2, xmm5); \
|
|
xmm1 = _mm_loadu_si128(&xmm0); \
|
|
xmm0 = _mm_unpacklo_epi16(xmm0, xmm2); \
|
|
xmm1 = _mm_unpackhi_epi16(xmm1, xmm2); \
|
|
_mm_storeu_si128((__m128i*)dst_argb, xmm0); \
|
|
_mm_storeu_si128((__m128i*)(dst_argb + 16), xmm1); \
|
|
dst_argb += 32;
|
|
|
|
// Store 8 AR30 values.
|
|
#define STOREAR30 \
|
|
xmm1 = _mm_and_si128(xmm1, xmm7); \
|
|
xmm2 = _mm_and_si128(xmm2, xmm7); \
|
|
xmm0 = _mm_srli_epi16(xmm0, 4); \
|
|
xmm3 = _mm_slli_epi16(xmm1, 6); \
|
|
xmm0 = _mm_or_si128(xmm0, xmm3); \
|
|
xmm1 = _mm_srli_epi16(xmm1, 10); \
|
|
xmm2 = _mm_or_si128(xmm2, xmm6); \
|
|
xmm2 = _mm_or_si128(xmm2, xmm1); \
|
|
xmm1 = _mm_unpackhi_epi16(xmm0, xmm2); \
|
|
xmm0 = _mm_unpacklo_epi16(xmm0, xmm2); \
|
|
_mm_storeu_si128((__m128i*)dst_ar30, xmm0); \
|
|
_mm_storeu_si128((__m128i*)(dst_ar30 + 16), xmm1); \
|
|
dst_ar30 += 32;
|
|
|
|
// Convert 32 ARGB pixels (128 bytes) to 32 UV444 values.
|
|
#if defined(HAS_ARGBTOYMATRIXROW_AVX2) || defined(HAS_ARGBTOUV444MATRIXROW_AVX2)
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
__m256i ymm_u =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToU));
|
|
__m256i ymm_v =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToV));
|
|
__m256i ymm5 =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kAddUV));
|
|
__m256i perm_mask = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_argb + 32));
|
|
__m256i ymm2 = _mm256_loadu_si256((const __m256i*)(src_argb + 64));
|
|
__m256i ymm3 = _mm256_loadu_si256((const __m256i*)(src_argb + 96));
|
|
src_argb += 128;
|
|
|
|
__m256i ymm0_u = _mm256_maddubs_epi16(ymm0, ymm_u);
|
|
__m256i ymm1_u = _mm256_maddubs_epi16(ymm1, ymm_u);
|
|
__m256i ymm2_u = _mm256_maddubs_epi16(ymm2, ymm_u);
|
|
__m256i ymm3_u = _mm256_maddubs_epi16(ymm3, ymm_u);
|
|
|
|
__m256i ymm0_v = _mm256_maddubs_epi16(ymm0, ymm_v);
|
|
__m256i ymm1_v = _mm256_maddubs_epi16(ymm1, ymm_v);
|
|
__m256i ymm2_v = _mm256_maddubs_epi16(ymm2, ymm_v);
|
|
__m256i ymm3_v = _mm256_maddubs_epi16(ymm3, ymm_v);
|
|
|
|
ymm0_u = _mm256_hadd_epi16(ymm0_u, ymm1_u);
|
|
ymm2_u = _mm256_hadd_epi16(ymm2_u, ymm3_u);
|
|
|
|
ymm0_v = _mm256_hadd_epi16(ymm0_v, ymm1_v);
|
|
ymm2_v = _mm256_hadd_epi16(ymm2_v, ymm3_v);
|
|
|
|
ymm0_u = _mm256_sub_epi16(ymm5, ymm0_u);
|
|
ymm2_u = _mm256_sub_epi16(ymm5, ymm2_u);
|
|
|
|
ymm0_v = _mm256_sub_epi16(ymm5, ymm0_v);
|
|
ymm2_v = _mm256_sub_epi16(ymm5, ymm2_v);
|
|
|
|
ymm0_u = _mm256_srli_epi16(ymm0_u, 8);
|
|
ymm2_u = _mm256_srli_epi16(ymm2_u, 8);
|
|
|
|
ymm0_v = _mm256_srli_epi16(ymm0_v, 8);
|
|
ymm2_v = _mm256_srli_epi16(ymm2_v, 8);
|
|
|
|
ymm0_u = _mm256_packus_epi16(ymm0_u, ymm2_u);
|
|
ymm0_u = _mm256_permutevar8x32_epi32(ymm0_u, perm_mask);
|
|
|
|
ymm0_v = _mm256_packus_epi16(ymm0_v, ymm2_v);
|
|
ymm0_v = _mm256_permutevar8x32_epi32(ymm0_v, perm_mask);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_u, ymm0_u);
|
|
_mm256_storeu_si256((__m256i*)dst_v, ymm0_v);
|
|
dst_u += 32;
|
|
dst_v += 32;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_ARGBTOYMATRIXROW_AVX2) ||
|
|
// defined(HAS_ARGBTOUV444MATRIXROW_AVX2)
|
|
|
|
#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_y,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
__m256i ymm5 = _mm256_set1_epi8((char)0x80);
|
|
__m256i ymm4 =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToY));
|
|
__m256i ymm7 =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kAddY));
|
|
__m256i ymm6 = _mm256_maddubs_epi16(ymm4, ymm5);
|
|
ymm6 = _mm256_hadd_epi16(ymm6, ymm6);
|
|
ymm7 = _mm256_sub_epi16(ymm7, ymm6);
|
|
__m256i perm_mask = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_argb + 32));
|
|
__m256i ymm2 = _mm256_loadu_si256((const __m256i*)(src_argb + 64));
|
|
__m256i ymm3 = _mm256_loadu_si256((const __m256i*)(src_argb + 96));
|
|
src_argb += 128;
|
|
|
|
ymm0 = _mm256_sub_epi8(ymm0, ymm5);
|
|
ymm1 = _mm256_sub_epi8(ymm1, ymm5);
|
|
ymm2 = _mm256_sub_epi8(ymm2, ymm5);
|
|
ymm3 = _mm256_sub_epi8(ymm3, ymm5);
|
|
|
|
ymm0 = _mm256_maddubs_epi16(ymm4, ymm0);
|
|
ymm1 = _mm256_maddubs_epi16(ymm4, ymm1);
|
|
ymm2 = _mm256_maddubs_epi16(ymm4, ymm2);
|
|
ymm3 = _mm256_maddubs_epi16(ymm4, ymm3);
|
|
|
|
ymm0 = _mm256_hadd_epi16(ymm0, ymm1);
|
|
ymm2 = _mm256_hadd_epi16(ymm2, ymm3);
|
|
|
|
ymm0 = _mm256_add_epi16(ymm0, ymm7);
|
|
ymm2 = _mm256_add_epi16(ymm2, ymm7);
|
|
|
|
ymm0 = _mm256_srli_epi16(ymm0, 8);
|
|
ymm2 = _mm256_srli_epi16(ymm2, 8);
|
|
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm2);
|
|
ymm0 = _mm256_permutevar8x32_epi32(ymm0, perm_mask);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_y, ymm0);
|
|
dst_y += 32;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_ARGBTOYMATRIXROW_AVX2)
|
|
|
|
#ifdef HAS_RAWTOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
|
|
__m256i ymm_alpha = _mm256_set1_epi32(0xff000000);
|
|
__m128i shuf_low =
|
|
_mm_set_epi8(-1, 9, 10, 11, -1, 6, 7, 8, -1, 3, 4, 5, -1, 0, 1, 2);
|
|
__m128i shuf_high =
|
|
_mm_set_epi8(-1, 13, 14, 15, -1, 10, 11, 12, -1, 7, 8, 9, -1, 4, 5, 6);
|
|
__m256i ymm_shuf = _mm256_broadcastsi128_si256(shuf_low);
|
|
__m256i ymm_shuf2 = _mm256_broadcastsi128_si256(shuf_high);
|
|
|
|
while (width > 0) {
|
|
__m128i xmm0 = _mm_loadu_si128((const __m128i*)src_raw);
|
|
__m256i ymm0 = _mm256_castsi128_si256(xmm0);
|
|
ymm0 = _mm256_inserti128_si256(
|
|
ymm0, _mm_loadu_si128((const __m128i*)(src_raw + 12)), 1);
|
|
|
|
__m128i xmm1 = _mm_loadu_si128((const __m128i*)(src_raw + 24));
|
|
__m256i ymm1 = _mm256_castsi128_si256(xmm1);
|
|
ymm1 = _mm256_inserti128_si256(
|
|
ymm1, _mm_loadu_si128((const __m128i*)(src_raw + 36)), 1);
|
|
|
|
__m128i xmm2 = _mm_loadu_si128((const __m128i*)(src_raw + 48));
|
|
__m256i ymm2 = _mm256_castsi128_si256(xmm2);
|
|
ymm2 = _mm256_inserti128_si256(
|
|
ymm2, _mm_loadu_si128((const __m128i*)(src_raw + 60)), 1);
|
|
|
|
__m128i xmm3 = _mm_loadu_si128((const __m128i*)(src_raw + 68));
|
|
__m256i ymm3 = _mm256_castsi128_si256(xmm3);
|
|
ymm3 = _mm256_inserti128_si256(
|
|
ymm3, _mm_loadu_si128((const __m128i*)(src_raw + 80)), 1);
|
|
|
|
ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
|
|
ymm1 = _mm256_shuffle_epi8(ymm1, ymm_shuf);
|
|
ymm2 = _mm256_shuffle_epi8(ymm2, ymm_shuf);
|
|
ymm3 = _mm256_shuffle_epi8(ymm3, ymm_shuf2);
|
|
|
|
ymm0 = _mm256_or_si256(ymm0, ymm_alpha);
|
|
ymm1 = _mm256_or_si256(ymm1, ymm_alpha);
|
|
ymm2 = _mm256_or_si256(ymm2, ymm_alpha);
|
|
ymm3 = _mm256_or_si256(ymm3, ymm_alpha);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, ymm0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 64), ymm2);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 96), ymm3);
|
|
|
|
src_raw += 96;
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RAWTOARGBROW_AVX2
|
|
|
|
#if defined(HAS_RAWTOARGBROW_AVX512BW) || \
|
|
defined(HAS_RGB24TOARGBROW_AVX512BW) || \
|
|
defined(HAS_RAWTORGB24ROW_AVX512BW) || \
|
|
defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
|
static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2, 3, 4, 4, 5};
|
|
static const uint64_t kPermqRGB24ToARGB1_AVX512BW[8] = {6, 7, 7, 8,
|
|
9, 10, 10, 11};
|
|
static const uint64_t kPermqRGB24ToARGB2_AVX512BW[8] = {4, 5, 5, 6, 7, 8, 8, 9};
|
|
static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4, 5, 6, 6, 7};
|
|
#endif // defined(HAS_RAWTOARGBROW_AVX512BW) ||
|
|
// defined(HAS_RGB24TOARGBROW_AVX512BW) ||
|
|
// defined(HAS_RAWTORGB24ROW_AVX512BW) ||
|
|
// defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
|
|
|
#if defined(HAS_RGB24TOARGBROW_AVX512BW) || \
|
|
defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
|
static const uint8_t kShuffleMaskRGB24ToARGB_AVX512BW[32] = {
|
|
0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u,
|
|
128u, 9u, 10u, 11u, 128u, 4u, 5u, 6u, 128u, 7u, 8u,
|
|
9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u, 128u};
|
|
#endif // defined(HAS_RGB24TOARGBROW_AVX512BW) ||
|
|
// defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
|
|
|
#if defined(HAS_RAWTOARGBROW_AVX512BW)
|
|
static const uint8_t kShuffleMaskRAWToARGB_AVX512BW[32] = {
|
|
2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u,
|
|
128u, 11u, 10u, 9u, 128u, 6u, 5u, 4u, 128u, 9u, 8u,
|
|
7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u, 128u};
|
|
#endif // defined(HAS_RAWTOARGBROW_AVX512BW)
|
|
|
|
#if defined(HAS_RAWTORGB24ROW_SSSE3)
|
|
// Shuffle table for converting RAW to RGB24. First 8.
|
|
static const uint8_t kShuffleMaskRAWToRGB24_0[16] = {
|
|
2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u,
|
|
128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
|
|
|
|
// Shuffle table for converting RAW to RGB24. Middle 8.
|
|
static const uint8_t kShuffleMaskRAWToRGB24_1[16] = {
|
|
2u, 7u, 6u, 5u, 10u, 9u, 8u, 13u,
|
|
128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
|
|
|
|
// Shuffle table for converting RAW to RGB24. Last 8.
|
|
static const uint8_t kShuffleMaskRAWToRGB24_2[16] = {
|
|
8u, 7u, 12u, 11u, 10u, 15u, 14u, 13u,
|
|
128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
|
|
#endif // defined(HAS_RAWTORGB24ROW_SSSE3)
|
|
|
|
#if defined(HAS_RAWTORGB24ROW_AVX2) || defined(HAS_RAWTORGB24ROW_AVX512BW)
|
|
static const uint8_t kShuffleMaskRAWToRGB24_AVX2[32] = {
|
|
2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u, 6u, 11u, 10u,
|
|
9u, 128u, 128u, 128u, 128u, 6u, 5u, 4u, 9u, 8u, 7u,
|
|
12u, 11u, 10u, 15u, 14u, 13u, 128u, 128u, 128u, 128u};
|
|
#endif // defined(HAS_RAWTORGB24ROW_AVX2) ||
|
|
// defined(HAS_RAWTORGB24ROW_AVX512BW)
|
|
|
|
#if defined(HAS_RAWTORGB24ROW_AVX2)
|
|
static const uint32_t kPermd0_AVX2[8] = {0, 1, 2, 4, 5, 6, 3, 7};
|
|
static const uint32_t kPermd1_AVX2[8] = {2, 4, 5, 6, 3, 7, 0, 1};
|
|
static const uint32_t kPermd2_AVX2[8] = {5, 6, 3, 7, 0, 1, 2, 4};
|
|
#endif // defined(HAS_RAWTORGB24ROW_AVX2)
|
|
|
|
#if defined(HAS_RAWTORGB24ROW_AVX512BW) || \
|
|
defined(HAS_ARGBTORGB24ROW_AVX512BW) || \
|
|
defined(HAS_ARGBTORAWROW_AVX512BW)
|
|
static const uint32_t kPermdRGB24_0_AVX512BW[16] = {
|
|
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20};
|
|
static const uint32_t kPermdRGB24_1_AVX512BW[16] = {
|
|
5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25};
|
|
static const uint32_t kPermdRGB24_2_AVX512BW[16] = {
|
|
10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25, 26, 28, 29, 30};
|
|
#endif
|
|
#if defined(HAS_RAWTORGB24ROW_AVX512VBMI)
|
|
static const uint8_t kPermRAWToRGB24_0[64] = {
|
|
0x02, 0x01, 0x00, 0x05, 0x04, 0x03, 0x08, 0x07, 0x06, 0x0b, 0x0a,
|
|
0x09, 0x0e, 0x0d, 0x0c, 0x11, 0x10, 0x0f, 0x14, 0x13, 0x12, 0x17,
|
|
0x16, 0x15, 0x1a, 0x19, 0x18, 0x1d, 0x1c, 0x1b, 0x20, 0x1f, 0x1e,
|
|
0x23, 0x22, 0x21, 0x26, 0x25, 0x24, 0x29, 0x28, 0x27, 0x2c, 0x2b,
|
|
0x2a, 0x2f, 0x2e, 0x2d, 0x32, 0x31, 0x30, 0x35, 0x34, 0x33, 0x38,
|
|
0x37, 0x36, 0x3b, 0x3a, 0x39, 0x3e, 0x3d, 0x3c, 0x41};
|
|
static const uint8_t kPermRAWToRGB24_1[64] = {
|
|
0x00, 0x7f, 0x04, 0x03, 0x02, 0x07, 0x06, 0x05, 0x0a, 0x09, 0x08,
|
|
0x0d, 0x0c, 0x0b, 0x10, 0x0f, 0x0e, 0x13, 0x12, 0x11, 0x16, 0x15,
|
|
0x14, 0x19, 0x18, 0x17, 0x1c, 0x1b, 0x1a, 0x1f, 0x1e, 0x1d, 0x22,
|
|
0x21, 0x20, 0x25, 0x24, 0x23, 0x28, 0x27, 0x26, 0x2b, 0x2a, 0x29,
|
|
0x2e, 0x2d, 0x2c, 0x31, 0x30, 0x2f, 0x34, 0x33, 0x32, 0x37, 0x36,
|
|
0x35, 0x3a, 0x39, 0x38, 0x3d, 0x3c, 0x3b, 0x00, 0x3f};
|
|
static const uint8_t kPermRAWToRGB24_2[64] = {
|
|
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a,
|
|
0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15,
|
|
0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20,
|
|
0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b,
|
|
0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36,
|
|
0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x40, 0x3f};
|
|
static const uint8_t kPermRAWToRGB24_3[64] = {
|
|
0x3e, 0x43, 0x42, 0x41, 0x46, 0x45, 0x44, 0x49, 0x48, 0x47, 0x4c,
|
|
0x4b, 0x4a, 0x4f, 0x4e, 0x4d, 0x52, 0x51, 0x50, 0x55, 0x54, 0x53,
|
|
0x58, 0x57, 0x56, 0x5b, 0x5a, 0x59, 0x5e, 0x5d, 0x5c, 0x61, 0x60,
|
|
0x5f, 0x64, 0x63, 0x62, 0x67, 0x66, 0x65, 0x6a, 0x69, 0x68, 0x6d,
|
|
0x6c, 0x6b, 0x70, 0x6f, 0x6e, 0x73, 0x72, 0x71, 0x76, 0x75, 0x74,
|
|
0x79, 0x78, 0x77, 0x7c, 0x7b, 0x7a, 0x7f, 0x7e, 0x7d};
|
|
#endif // defined(HAS_RAWTORGB24ROW_AVX512VBMI)
|
|
|
|
#if defined(HAS_RAWTOARGBROW_AVX512BW) || defined(HAS_RGB24TOARGBROW_AVX512BW)
|
|
LIBYUV_TARGET_AVX512BW
|
|
static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
|
|
uint8_t* dst_argb,
|
|
const uint32_t* shuffler,
|
|
int width) {
|
|
__m512i zmm_alpha = _mm512_set1_epi32(0xff000000);
|
|
__m512i zmm_q0 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB0_AVX512BW);
|
|
__m512i zmm_q1 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB1_AVX512BW);
|
|
__m512i zmm_q2 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB2_AVX512BW);
|
|
__m512i zmm_q3 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB3_AVX512BW);
|
|
__m512i zmm_shuf =
|
|
_mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)shuffler));
|
|
|
|
while (width >= 64) {
|
|
__m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw);
|
|
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
|
|
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
|
|
|
|
__m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
|
|
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
|
|
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
|
|
__m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
zmm0 = _mm512_or_si512(zmm0, zmm_alpha);
|
|
zmm1 = _mm512_or_si512(zmm1, zmm_alpha);
|
|
zmm2 = _mm512_or_si512(zmm2, zmm_alpha);
|
|
zmm3 = _mm512_or_si512(zmm3, zmm_alpha);
|
|
|
|
_mm512_storeu_si512(dst_argb, zmm0);
|
|
_mm512_storeu_si512(dst_argb + 64, zmm1);
|
|
_mm512_storeu_si512(dst_argb + 128, zmm2);
|
|
_mm512_storeu_si512(dst_argb + 192, zmm3);
|
|
|
|
src_raw += 192;
|
|
dst_argb += 256;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
uint64_t dst_mask = (1ULL << width) - 1ULL;
|
|
int src_bytes = width * 3;
|
|
__mmask64 src_mask0 =
|
|
(src_bytes >= 64) ? ~0ULL : ((1ULL << src_bytes) - 1ULL);
|
|
__mmask64 src_mask1 =
|
|
(src_bytes >= 128)
|
|
? ~0ULL
|
|
: ((src_bytes > 64) ? ((1ULL << (src_bytes - 64)) - 1ULL) : 0ULL);
|
|
__mmask64 src_mask2 =
|
|
(src_bytes > 128) ? ((1ULL << (src_bytes - 128)) - 1ULL) : 0ULL;
|
|
|
|
__m512i zmm_raw0 = _mm512_maskz_loadu_epi8(src_mask0, src_raw);
|
|
__m512i zmm_raw1 = _mm512_maskz_loadu_epi8(src_mask1, src_raw + 64);
|
|
__m512i zmm_raw2 = _mm512_maskz_loadu_epi8(src_mask2, src_raw + 128);
|
|
|
|
__m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
|
|
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
|
|
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
|
|
__m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
zmm0 = _mm512_or_si512(zmm0, zmm_alpha);
|
|
zmm1 = _mm512_or_si512(zmm1, zmm_alpha);
|
|
zmm2 = _mm512_or_si512(zmm2, zmm_alpha);
|
|
zmm3 = _mm512_or_si512(zmm3, zmm_alpha);
|
|
|
|
_mm512_mask_storeu_epi32(dst_argb, (__mmask16)dst_mask, zmm0);
|
|
_mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(dst_mask >> 16), zmm1);
|
|
_mm512_mask_storeu_epi32(dst_argb + 128, (__mmask16)(dst_mask >> 32), zmm2);
|
|
_mm512_mask_storeu_epi32(dst_argb + 192, (__mmask16)(dst_mask >> 48), zmm3);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_RAWTOARGBROW_AVX512BW) ||
|
|
// defined(HAS_RGB24TOARGBROW_AVX512BW)
|
|
|
|
#ifdef HAS_RAWTOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void RAWToARGBRow_AVX512BW(const uint8_t* src_raw,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
RGBToARGBRow_AVX512BW(src_raw, dst_argb,
|
|
(const uint32_t*)kShuffleMaskRAWToARGB_AVX512BW, width);
|
|
}
|
|
#endif // HAS_RAWTOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_RGB24TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
RGBToARGBRow_AVX512BW(src_rgb24, dst_argb,
|
|
(const uint32_t*)kShuffleMaskRGB24ToARGB_AVX512BW,
|
|
width);
|
|
}
|
|
#endif // HAS_RGB24TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_RAWTORGB24ROW_SSSE3
|
|
LIBYUV_TARGET_SSSE3
|
|
void RAWToRGB24Row_SSSE3(const uint8_t* src_raw,
|
|
uint8_t* dst_rgb24,
|
|
int width) {
|
|
__m128i shuf0 = _mm_loadu_si128((const __m128i*)kShuffleMaskRAWToRGB24_0);
|
|
__m128i shuf1 = _mm_loadu_si128((const __m128i*)kShuffleMaskRAWToRGB24_1);
|
|
__m128i shuf2 = _mm_loadu_si128((const __m128i*)kShuffleMaskRAWToRGB24_2);
|
|
|
|
while (width > 0) {
|
|
__m128i raw0 = _mm_loadu_si128((const __m128i*)src_raw);
|
|
__m128i raw1 = _mm_loadu_si128((const __m128i*)(src_raw + 4));
|
|
__m128i raw2 = _mm_loadu_si128((const __m128i*)(src_raw + 8));
|
|
|
|
raw0 = _mm_shuffle_epi8(raw0, shuf0);
|
|
raw1 = _mm_shuffle_epi8(raw1, shuf1);
|
|
raw2 = _mm_shuffle_epi8(raw2, shuf2);
|
|
|
|
_mm_storel_epi64((__m128i*)dst_rgb24, raw0);
|
|
_mm_storel_epi64((__m128i*)(dst_rgb24 + 8), raw1);
|
|
_mm_storel_epi64((__m128i*)(dst_rgb24 + 16), raw2);
|
|
|
|
src_raw += 24;
|
|
dst_rgb24 += 24;
|
|
width -= 8;
|
|
}
|
|
}
|
|
#endif // HAS_RAWTORGB24ROW_SSSE3
|
|
|
|
#ifdef HAS_RAWTORGB24ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
|
|
__m256i ymm_shuf =
|
|
_mm256_loadu_si256((const __m256i*)kShuffleMaskRAWToRGB24_AVX2);
|
|
__m256i ymm_p0 = _mm256_loadu_si256((const __m256i*)kPermd0_AVX2);
|
|
__m256i ymm_p1 = _mm256_loadu_si256((const __m256i*)kPermd1_AVX2);
|
|
__m256i ymm_p2 = _mm256_loadu_si256((const __m256i*)kPermd2_AVX2);
|
|
|
|
while (width > 0) {
|
|
__m256i raw0 = _mm256_loadu_si256((const __m256i*)src_raw);
|
|
__m256i raw1 = _mm256_loadu_si256((const __m256i*)(src_raw + 32));
|
|
__m256i raw2 = _mm256_loadu_si256((const __m256i*)(src_raw + 64));
|
|
|
|
__m256i m01 = _mm256_permute2x128_si256(raw0, raw1, 0x21);
|
|
__m256i b0 = _mm256_permute4x64_epi64(raw0, 0x94);
|
|
__m256i m12 = _mm256_permute2x128_si256(raw1, raw2, 0x21);
|
|
__m256i b3 = _mm256_permute4x64_epi64(raw2, 0xe9);
|
|
__m256i b1 = _mm256_permute4x64_epi64(m01, 0xe9);
|
|
__m256i b2 = _mm256_permute4x64_epi64(m12, 0x94);
|
|
|
|
b0 = _mm256_shuffle_epi8(b0, ymm_shuf);
|
|
b1 = _mm256_shuffle_epi8(b1, ymm_shuf);
|
|
b2 = _mm256_shuffle_epi8(b2, ymm_shuf);
|
|
b3 = _mm256_shuffle_epi8(b3, ymm_shuf);
|
|
|
|
__m256i d0 = _mm256_permutevar8x32_epi32(b0, ymm_p0);
|
|
__m256i d1 = _mm256_permutevar8x32_epi32(b1, ymm_p1);
|
|
__m256i d2 = _mm256_permutevar8x32_epi32(b2, ymm_p2);
|
|
__m256i d3 = _mm256_permutevar8x32_epi32(b3, ymm_p0);
|
|
|
|
// Dst0 = blend(d0, d1)
|
|
__m256i dst0 = _mm256_blend_epi32(d0, d1, 0xc0);
|
|
_mm256_storeu_si256((__m256i*)dst_rgb24, dst0);
|
|
|
|
// Dst1 = blend(d1, d2)
|
|
__m256i dst1 = _mm256_blend_epi32(d1, d2, 0xf0);
|
|
d3 = _mm256_permute4x64_epi64(d3, 0x90);
|
|
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 32), dst1);
|
|
|
|
// Dst2 = blend(d2, d3)
|
|
__m256i dst2 = _mm256_blend_epi32(d2, d3, 0xfc);
|
|
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), dst2);
|
|
|
|
src_raw += 96;
|
|
dst_rgb24 += 96;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RAWTORGB24ROW_AVX2
|
|
|
|
#ifdef HAS_RAWTORGB24ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw,
|
|
uint8_t* dst_rgb24,
|
|
int width) {
|
|
__m512i zmm_q0 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB0_AVX512BW);
|
|
__m512i zmm_q1 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB1_AVX512BW);
|
|
__m512i zmm_q2 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB2_AVX512BW);
|
|
__m512i zmm_q3 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB3_AVX512BW);
|
|
__m512i zmm_shuf = _mm512_broadcast_i64x4(
|
|
_mm256_loadu_si256((const __m256i*)kShuffleMaskRAWToRGB24_AVX2));
|
|
__m512i zmm_d0 = _mm512_loadu_si512((const __m512i*)kPermdRGB24_0_AVX512BW);
|
|
__m512i zmm_d1 = _mm512_loadu_si512((const __m512i*)kPermdRGB24_1_AVX512BW);
|
|
__m512i zmm_d2 = _mm512_loadu_si512((const __m512i*)kPermdRGB24_2_AVX512BW);
|
|
|
|
while (width >= 64) {
|
|
__m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw);
|
|
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
|
|
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
|
|
|
|
__m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
|
|
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
|
|
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
|
|
__m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
__m512i dst0 = _mm512_permutex2var_epi32(zmm0, zmm_d0, zmm1);
|
|
__m512i dst1 = _mm512_permutex2var_epi32(zmm1, zmm_d1, zmm2);
|
|
__m512i dst2 = _mm512_permutex2var_epi32(zmm2, zmm_d2, zmm3);
|
|
|
|
_mm512_storeu_si512((__m512i*)dst_rgb24, dst0);
|
|
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 64), dst1);
|
|
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 128), dst2);
|
|
|
|
src_raw += 192;
|
|
dst_rgb24 += 192;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
int nbytes = width * 3;
|
|
__mmask64 k0 =
|
|
(nbytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << nbytes) - 1ULL);
|
|
__mmask64 k1 = (nbytes <= 64) ? 0
|
|
: (nbytes >= 128)
|
|
? ~(__mmask64)0
|
|
: (__mmask64)((1ULL << (nbytes - 64)) - 1ULL);
|
|
__mmask64 k2 =
|
|
(nbytes <= 128) ? 0 : (__mmask64)((1ULL << (nbytes - 128)) - 1ULL);
|
|
__m512i zmm_raw0 = _mm512_maskz_loadu_epi8(k0, src_raw);
|
|
__m512i zmm_raw1 =
|
|
k1 ? _mm512_maskz_loadu_epi8(k1, src_raw + 64) : _mm512_setzero_si512();
|
|
__m512i zmm_raw2 = k2 ? _mm512_maskz_loadu_epi8(k2, src_raw + 128)
|
|
: _mm512_setzero_si512();
|
|
|
|
__m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
|
|
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
|
|
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
|
|
__m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
__m512i dst0 = _mm512_permutex2var_epi32(zmm0, zmm_d0, zmm1);
|
|
__m512i dst1 = _mm512_permutex2var_epi32(zmm1, zmm_d1, zmm2);
|
|
__m512i dst2 = _mm512_permutex2var_epi32(zmm2, zmm_d2, zmm3);
|
|
|
|
_mm512_mask_storeu_epi8(dst_rgb24, k0, dst0);
|
|
if (k1) {
|
|
_mm512_mask_storeu_epi8(dst_rgb24 + 64, k1, dst1);
|
|
}
|
|
if (k2) {
|
|
_mm512_mask_storeu_epi8(dst_rgb24 + 128, k2, dst2);
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RAWTORGB24ROW_AVX512BW
|
|
|
|
#ifdef HAS_RAWTORGB24ROW_AVX512VBMI
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw,
|
|
uint8_t* dst_rgb24,
|
|
int width) {
|
|
__m512i zmm_p0 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_0);
|
|
__m512i zmm_p1 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_1);
|
|
__m512i zmm_p2 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_2);
|
|
__m512i zmm_p3 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_3);
|
|
|
|
while (width >= 64) {
|
|
__m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw);
|
|
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
|
|
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
|
|
|
|
__m512i tmp1 = _mm512_permutex2var_epi8(zmm_raw1, zmm_p1, zmm_raw0);
|
|
__m512i dst0 = _mm512_permutex2var_epi8(zmm_raw0, zmm_p0, zmm_raw1);
|
|
__m512i dst2 = _mm512_permutex2var_epi8(zmm_raw1, zmm_p3, zmm_raw2);
|
|
__m512i dst1 = _mm512_permutex2var_epi8(tmp1, zmm_p2, zmm_raw2);
|
|
|
|
_mm512_storeu_si512((__m512i*)dst_rgb24, dst0);
|
|
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 64), dst1);
|
|
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 128), dst2);
|
|
|
|
src_raw += 192;
|
|
dst_rgb24 += 192;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
int nbytes = width * 3;
|
|
__mmask64 k0 =
|
|
(nbytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << nbytes) - 1ULL);
|
|
__mmask64 k1 = (nbytes <= 64) ? 0
|
|
: (nbytes >= 128)
|
|
? ~(__mmask64)0
|
|
: (__mmask64)((1ULL << (nbytes - 64)) - 1ULL);
|
|
__mmask64 k2 =
|
|
(nbytes <= 128) ? 0 : (__mmask64)((1ULL << (nbytes - 128)) - 1ULL);
|
|
__m512i zmm_raw0 = _mm512_maskz_loadu_epi8(k0, src_raw);
|
|
__m512i zmm_raw1 =
|
|
k1 ? _mm512_maskz_loadu_epi8(k1, src_raw + 64) : _mm512_setzero_si512();
|
|
__m512i zmm_raw2 = k2 ? _mm512_maskz_loadu_epi8(k2, src_raw + 128)
|
|
: _mm512_setzero_si512();
|
|
|
|
__m512i tmp1 = _mm512_permutex2var_epi8(zmm_raw1, zmm_p1, zmm_raw0);
|
|
__m512i dst0 = _mm512_permutex2var_epi8(zmm_raw0, zmm_p0, zmm_raw1);
|
|
__m512i dst2 = _mm512_permutex2var_epi8(zmm_raw1, zmm_p3, zmm_raw2);
|
|
__m512i dst1 = _mm512_permutex2var_epi8(tmp1, zmm_p2, zmm_raw2);
|
|
|
|
_mm512_mask_storeu_epi8(dst_rgb24, k0, dst0);
|
|
if (k1) {
|
|
_mm512_mask_storeu_epi8(dst_rgb24 + 64, k1, dst1);
|
|
}
|
|
if (k2) {
|
|
_mm512_mask_storeu_epi8(dst_rgb24 + 128, k2, dst2);
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RAWTORGB24ROW_AVX512VBMI
|
|
|
|
#if defined(HAS_RGBTOYMATRIXROW_AVX512BW) || \
|
|
defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \
|
|
defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) || \
|
|
defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || \
|
|
defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) || \
|
|
defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) || \
|
|
defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI)
|
|
static const uint32_t kPermdARGBToY_AVX512BW[16] = {0, 4, 8, 12, 1, 5, 9, 13,
|
|
2, 6, 10, 14, 3, 7, 11, 15};
|
|
#endif
|
|
|
|
#ifdef HAS_RGBTOYMATRIXROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb,
|
|
uint8_t* dst_y,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
__m512i zmm_q0 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB0_AVX512BW);
|
|
__m512i zmm_q1 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB1_AVX512BW);
|
|
__m512i zmm_q2 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB2_AVX512BW);
|
|
__m512i zmm_q3 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB3_AVX512BW);
|
|
__m512i zmm_shuf = _mm512_broadcast_i64x4(
|
|
_mm256_loadu_si256((const __m256i*)kShuffleMaskRGB24ToARGB_AVX512BW));
|
|
__m512i zmm_perm_y =
|
|
_mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
|
|
|
|
__m512i zmm_80 = _mm512_set1_epi8((char)0x80);
|
|
__m512i zmm_one = _mm512_set1_epi16(1);
|
|
__m512i zmm_coeff =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToY));
|
|
__m512i zmm_add =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddY));
|
|
|
|
// Compute bias adjusted for 0x80 subtraction:
|
|
// bias = kAddY - pmaddwd(pmaddubsw(kRGBToY, 0x80), 1)
|
|
__m512i zmm_bias = _mm512_maddubs_epi16(zmm_coeff, zmm_80);
|
|
zmm_bias = _mm512_madd_epi16(zmm_bias, zmm_one);
|
|
zmm_bias = _mm512_packs_epi32(zmm_bias, zmm_bias);
|
|
zmm_add = _mm512_sub_epi16(zmm_add, zmm_bias);
|
|
|
|
while (width >= 64) {
|
|
__m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_rgb);
|
|
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_rgb + 64));
|
|
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_rgb + 128));
|
|
|
|
__m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
|
|
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
|
|
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
|
|
__m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
zmm0 = _mm512_xor_si512(zmm0, zmm_80);
|
|
zmm1 = _mm512_xor_si512(zmm1, zmm_80);
|
|
zmm2 = _mm512_xor_si512(zmm2, zmm_80);
|
|
zmm3 = _mm512_xor_si512(zmm3, zmm_80);
|
|
|
|
zmm0 = _mm512_maddubs_epi16(zmm_coeff, zmm0);
|
|
zmm1 = _mm512_maddubs_epi16(zmm_coeff, zmm1);
|
|
zmm2 = _mm512_maddubs_epi16(zmm_coeff, zmm2);
|
|
zmm3 = _mm512_maddubs_epi16(zmm_coeff, zmm3);
|
|
|
|
zmm0 = _mm512_madd_epi16(zmm0, zmm_one);
|
|
zmm1 = _mm512_madd_epi16(zmm1, zmm_one);
|
|
zmm0 = _mm512_packs_epi32(zmm0, zmm1);
|
|
|
|
zmm2 = _mm512_madd_epi16(zmm2, zmm_one);
|
|
zmm3 = _mm512_madd_epi16(zmm3, zmm_one);
|
|
zmm2 = _mm512_packs_epi32(zmm2, zmm3);
|
|
|
|
zmm0 = _mm512_add_epi16(zmm0, zmm_add);
|
|
zmm2 = _mm512_add_epi16(zmm2, zmm_add);
|
|
|
|
zmm0 = _mm512_srli_epi16(zmm0, 8);
|
|
zmm2 = _mm512_srli_epi16(zmm2, 8);
|
|
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm2);
|
|
zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
|
|
|
|
_mm512_storeu_si512((__m512i*)dst_y, zmm0);
|
|
|
|
src_rgb += 192;
|
|
dst_y += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
__mmask64 ky = (__mmask64)((1ULL << width) - 1ULL);
|
|
int nbytes = width * 3;
|
|
__mmask64 k0 =
|
|
(nbytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << nbytes) - 1ULL);
|
|
__mmask64 k1 = (nbytes <= 64) ? 0
|
|
: (nbytes >= 128)
|
|
? ~(__mmask64)0
|
|
: (__mmask64)((1ULL << (nbytes - 64)) - 1ULL);
|
|
__mmask64 k2 =
|
|
(nbytes <= 128) ? 0 : (__mmask64)((1ULL << (nbytes - 128)) - 1ULL);
|
|
__m512i zmm_raw0 = _mm512_maskz_loadu_epi8(k0, src_rgb);
|
|
__m512i zmm_raw1 =
|
|
k1 ? _mm512_maskz_loadu_epi8(k1, src_rgb + 64) : _mm512_setzero_si512();
|
|
__m512i zmm_raw2 = k2 ? _mm512_maskz_loadu_epi8(k2, src_rgb + 128)
|
|
: _mm512_setzero_si512();
|
|
|
|
__m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
|
|
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
|
|
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
|
|
__m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
|
|
|
|
zmm0 = _mm512_xor_si512(_mm512_shuffle_epi8(zmm0, zmm_shuf), zmm_80);
|
|
zmm1 = _mm512_xor_si512(_mm512_shuffle_epi8(zmm1, zmm_shuf), zmm_80);
|
|
zmm2 = _mm512_xor_si512(_mm512_shuffle_epi8(zmm2, zmm_shuf), zmm_80);
|
|
zmm3 = _mm512_xor_si512(_mm512_shuffle_epi8(zmm3, zmm_shuf), zmm_80);
|
|
|
|
zmm0 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm0), zmm_one);
|
|
zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm1), zmm_one);
|
|
zmm0 = _mm512_packs_epi32(zmm0, zmm1);
|
|
|
|
zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm2), zmm_one);
|
|
zmm3 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm3), zmm_one);
|
|
zmm2 = _mm512_packs_epi32(zmm2, zmm3);
|
|
|
|
zmm0 = _mm512_srli_epi16(_mm512_add_epi16(zmm0, zmm_add), 8);
|
|
zmm2 = _mm512_srli_epi16(_mm512_add_epi16(zmm2, zmm_add), 8);
|
|
|
|
zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2));
|
|
_mm512_mask_storeu_epi8(dst_y, ky, zmm0);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RGBTOYMATRIXROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNI
|
|
LIBYUV_TARGET_AVX512VNNI
|
|
void ARGBToYMatrixRow_AVX512VNNI(const uint8_t* src_argb,
|
|
uint8_t* dst_y,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
__m512i zmm_perm_y =
|
|
_mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
|
|
__m512i zmm_80 = _mm512_set1_epi8((char)0x80);
|
|
__m512i zmm_coeff =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToY));
|
|
__m512i zmm_add =
|
|
_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddY));
|
|
|
|
__m512i zmm_bias =
|
|
_mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_coeff, zmm_80);
|
|
zmm_add = _mm512_sub_epi32(zmm_add, zmm_bias);
|
|
|
|
while (width >= 64) {
|
|
__m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
__m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
|
|
__m512i zmm2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128));
|
|
__m512i zmm3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192));
|
|
|
|
zmm0 = _mm512_xor_si512(zmm0, zmm_80);
|
|
zmm1 = _mm512_xor_si512(zmm1, zmm_80);
|
|
zmm2 = _mm512_xor_si512(zmm2, zmm_80);
|
|
zmm3 = _mm512_xor_si512(zmm3, zmm_80);
|
|
|
|
zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm0);
|
|
zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm1);
|
|
zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm2);
|
|
zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm3);
|
|
|
|
zmm0 = _mm512_packus_epi32(zmm0, zmm1);
|
|
zmm2 = _mm512_packus_epi32(zmm2, zmm3);
|
|
|
|
zmm0 = _mm512_srli_epi16(zmm0, 8);
|
|
zmm2 = _mm512_srli_epi16(zmm2, 8);
|
|
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm2);
|
|
zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
|
|
|
|
_mm512_storeu_si512((__m512i*)dst_y, zmm0);
|
|
|
|
src_argb += 256;
|
|
dst_y += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
uint64_t mask = (1ULL << width) - 1ULL;
|
|
__mmask16 k0 = (__mmask16)mask;
|
|
__mmask16 k1 = (__mmask16)(mask >> 16);
|
|
__mmask16 k2 = (__mmask16)(mask >> 32);
|
|
__mmask16 k3 = (__mmask16)(mask >> 48);
|
|
__m512i zmm0 =
|
|
_mm512_xor_si512(_mm512_maskz_loadu_epi32(k0, src_argb), zmm_80);
|
|
__m512i zmm1 =
|
|
_mm512_xor_si512(k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64)
|
|
: _mm512_setzero_si512(),
|
|
zmm_80);
|
|
__m512i zmm2 =
|
|
_mm512_xor_si512(k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128)
|
|
: _mm512_setzero_si512(),
|
|
zmm_80);
|
|
__m512i zmm3 =
|
|
_mm512_xor_si512(k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192)
|
|
: _mm512_setzero_si512(),
|
|
zmm_80);
|
|
|
|
zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm0);
|
|
zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm1);
|
|
zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm2);
|
|
zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm3);
|
|
|
|
zmm0 = _mm512_srli_epi16(_mm512_packus_epi32(zmm0, zmm1), 8);
|
|
zmm2 = _mm512_srli_epi16(_mm512_packus_epi32(zmm2, zmm3), 8);
|
|
|
|
zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2));
|
|
_mm512_mask_storeu_epi8(dst_y, (__mmask64)mask, zmm0);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOYMATRIXROW_AVX512VNNI
|
|
|
|
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) || \
|
|
defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI) || \
|
|
defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
|
|
// AVX512VNNIVBMI kernels use both AVX512VNNI (vpdpbusd) and AVX512VBMI
|
|
// (vpermt2b) and require both CPU features. There are no AVX512VBMI
|
|
// versions without AVX512VNNI.
|
|
|
|
// vpermt2b indices selecting byte 1 of each int32 from 2 vectors. The 32
|
|
// byte pattern is broadcast so each 256 bit half holds the same 32 results.
|
|
// Clang constant folds the indices and lowers _mm512_permutex2var_epi8 to a
|
|
// longer vpshufb + vpor + vpermd sequence, so hide the indices from it.
|
|
#if defined(__clang__)
|
|
#define LIBYUV_HIDE_CONSTANT(v) __asm__("" : "+v"(v))
|
|
#else
|
|
#define LIBYUV_HIDE_CONSTANT(v)
|
|
#endif
|
|
static const uint8_t kPermbDWordByte1_AVX512VNNIVBMI[32] = {
|
|
1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61,
|
|
65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125};
|
|
#endif
|
|
|
|
#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI
|
|
// Requires AVX512VNNI and AVX512VBMI.
|
|
// The 32 bit sums are kAddY + dot(kRGBToY, argb) which is always in
|
|
// [0, 65535], so Y is byte 1 of each int32 and no saturation is needed.
|
|
LIBYUV_TARGET_AVX512VNNIVBMI
|
|
void ARGBToYMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
|
|
uint8_t* dst_y,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
__m512i zmm_perm = _mm512_broadcast_i64x4(
|
|
_mm256_loadu_si256((const __m256i*)kPermbDWordByte1_AVX512VNNIVBMI));
|
|
LIBYUV_HIDE_CONSTANT(zmm_perm);
|
|
__m512i zmm_80 = _mm512_set1_epi8((char)0x80);
|
|
__m512i zmm_coeff =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToY));
|
|
__m512i zmm_add =
|
|
_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddY));
|
|
|
|
__m512i zmm_bias =
|
|
_mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_coeff, zmm_80);
|
|
zmm_add = _mm512_sub_epi32(zmm_add, zmm_bias);
|
|
|
|
while (width >= 64) {
|
|
__m512i zmm0 =
|
|
_mm512_xor_si512(_mm512_loadu_si512((const __m512i*)src_argb), zmm_80);
|
|
__m512i zmm1 = _mm512_xor_si512(
|
|
_mm512_loadu_si512((const __m512i*)(src_argb + 64)), zmm_80);
|
|
__m512i zmm2 = _mm512_xor_si512(
|
|
_mm512_loadu_si512((const __m512i*)(src_argb + 128)), zmm_80);
|
|
__m512i zmm3 = _mm512_xor_si512(
|
|
_mm512_loadu_si512((const __m512i*)(src_argb + 192)), zmm_80);
|
|
|
|
zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm0);
|
|
zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm1);
|
|
zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm2);
|
|
zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm3);
|
|
|
|
zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); // Y 0..31
|
|
zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3); // Y 32..63
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_y, _mm512_castsi512_si256(zmm0));
|
|
_mm256_storeu_si256((__m256i*)(dst_y + 32), _mm512_castsi512_si256(zmm2));
|
|
|
|
src_argb += 256;
|
|
dst_y += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
uint64_t mask = (1ULL << width) - 1ULL;
|
|
__mmask16 k0 = (__mmask16)mask;
|
|
__mmask16 k1 = (__mmask16)(mask >> 16);
|
|
__mmask16 k2 = (__mmask16)(mask >> 32);
|
|
__mmask16 k3 = (__mmask16)(mask >> 48);
|
|
__m512i zmm0 =
|
|
_mm512_xor_si512(_mm512_maskz_loadu_epi32(k0, src_argb), zmm_80);
|
|
__m512i zmm1 =
|
|
_mm512_xor_si512(k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64)
|
|
: _mm512_setzero_si512(),
|
|
zmm_80);
|
|
__m512i zmm2 =
|
|
_mm512_xor_si512(k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128)
|
|
: _mm512_setzero_si512(),
|
|
zmm_80);
|
|
__m512i zmm3 =
|
|
_mm512_xor_si512(k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192)
|
|
: _mm512_setzero_si512(),
|
|
zmm_80);
|
|
|
|
zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm0);
|
|
zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm1);
|
|
zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm2);
|
|
zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm3);
|
|
|
|
zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1);
|
|
zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3);
|
|
|
|
_mm256_mask_storeu_epi8(dst_y, (__mmask32)mask,
|
|
_mm512_castsi512_si256(zmm0));
|
|
if (mask >> 32) {
|
|
_mm256_mask_storeu_epi8(dst_y + 32, (__mmask32)(mask >> 32),
|
|
_mm512_castsi512_si256(zmm2));
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI
|
|
|
|
#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNI
|
|
LIBYUV_TARGET_AVX512VNNI
|
|
void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
__m512i zmm_u =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU));
|
|
__m512i zmm_v =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV));
|
|
__m512i zmm_8000 =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddUV));
|
|
__m512i zmm_perm_y =
|
|
_mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
|
|
|
|
while (width >= 64) {
|
|
__m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
__m512i zmm_s1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
|
|
__m512i zmm_s2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128));
|
|
__m512i zmm_s3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192));
|
|
|
|
__m512i zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_u);
|
|
__m512i zmm1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s1, zmm_u);
|
|
__m512i zmm2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s2, zmm_u);
|
|
__m512i zmm3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s3, zmm_u);
|
|
|
|
zmm0 = _mm512_packs_epi32(zmm0, zmm1);
|
|
zmm2 = _mm512_packs_epi32(zmm2, zmm3);
|
|
zmm0 = _mm512_sub_epi16(zmm_8000, zmm0);
|
|
zmm2 = _mm512_sub_epi16(zmm_8000, zmm2);
|
|
zmm0 = _mm512_srli_epi16(zmm0, 8);
|
|
zmm2 = _mm512_srli_epi16(zmm2, 8);
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm2);
|
|
zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
|
|
_mm512_storeu_si512((__m512i*)dst_u, zmm0);
|
|
|
|
zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_v);
|
|
zmm1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s1, zmm_v);
|
|
zmm2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s2, zmm_v);
|
|
zmm3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s3, zmm_v);
|
|
|
|
zmm0 = _mm512_packs_epi32(zmm0, zmm1);
|
|
zmm2 = _mm512_packs_epi32(zmm2, zmm3);
|
|
zmm0 = _mm512_sub_epi16(zmm_8000, zmm0);
|
|
zmm2 = _mm512_sub_epi16(zmm_8000, zmm2);
|
|
zmm0 = _mm512_srli_epi16(zmm0, 8);
|
|
zmm2 = _mm512_srli_epi16(zmm2, 8);
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm2);
|
|
zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
|
|
_mm512_storeu_si512((__m512i*)dst_v, zmm0);
|
|
|
|
src_argb += 256;
|
|
dst_u += 64;
|
|
dst_v += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
uint64_t mask = (1ULL << width) - 1ULL;
|
|
__mmask16 k0 = (__mmask16)mask;
|
|
__mmask16 k1 = (__mmask16)(mask >> 16);
|
|
__mmask16 k2 = (__mmask16)(mask >> 32);
|
|
__mmask16 k3 = (__mmask16)(mask >> 48);
|
|
__m512i zmm_s0 = _mm512_maskz_loadu_epi32(k0, src_argb);
|
|
__m512i zmm_s1 = k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64)
|
|
: _mm512_setzero_si512();
|
|
__m512i zmm_s2 = k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128)
|
|
: _mm512_setzero_si512();
|
|
__m512i zmm_s3 = k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192)
|
|
: _mm512_setzero_si512();
|
|
|
|
__m512i zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_u);
|
|
__m512i zmm1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s1, zmm_u);
|
|
__m512i zmm2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s2, zmm_u);
|
|
__m512i zmm3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s3, zmm_u);
|
|
|
|
zmm0 = _mm512_srli_epi16(
|
|
_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(zmm0, zmm1)), 8);
|
|
zmm2 = _mm512_srli_epi16(
|
|
_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(zmm2, zmm3)), 8);
|
|
zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2));
|
|
_mm512_mask_storeu_epi8(dst_u, (__mmask64)mask, zmm0);
|
|
|
|
zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_v);
|
|
zmm1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s1, zmm_v);
|
|
zmm2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s2, zmm_v);
|
|
zmm3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s3, zmm_v);
|
|
|
|
zmm0 = _mm512_srli_epi16(
|
|
_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(zmm0, zmm1)), 8);
|
|
zmm2 = _mm512_srli_epi16(
|
|
_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(zmm2, zmm3)), 8);
|
|
zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2));
|
|
_mm512_mask_storeu_epi8(dst_v, (__mmask64)mask, zmm0);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOUV444MATRIXROW_AVX512VNNI
|
|
|
|
#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI
|
|
// Requires AVX512VNNI and AVX512VBMI.
|
|
// U = (kAddUV - dot(kRGBToU, argb)) >> 8. The accumulators start at
|
|
// ~kAddUV so they hold ~(kAddUV - dot) and U is the inverse of byte 1 of each
|
|
// int32.
|
|
LIBYUV_TARGET_AVX512VNNIVBMI
|
|
void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
__m512i zmm_perm = _mm512_broadcast_i64x4(
|
|
_mm256_loadu_si256((const __m256i*)kPermbDWordByte1_AVX512VNNIVBMI));
|
|
LIBYUV_HIDE_CONSTANT(zmm_perm);
|
|
__m512i zmm_u =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU));
|
|
__m512i zmm_v =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV));
|
|
const __m512i zmm_neg1 = _mm512_set1_epi32(-1);
|
|
__m512i zmm_add = _mm512_xor_si512(
|
|
_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddUV)),
|
|
zmm_neg1); // ~kAddUV
|
|
|
|
while (width >= 64) {
|
|
__m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
__m512i zmm_s1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
|
|
__m512i zmm_s2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128));
|
|
__m512i zmm_s3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192));
|
|
|
|
__m512i zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_s0, zmm_u);
|
|
__m512i zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_s1, zmm_u);
|
|
__m512i zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_u);
|
|
__m512i zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_u);
|
|
__m512i zmm4 = _mm512_dpbusd_epi32(zmm_add, zmm_s0, zmm_v);
|
|
__m512i zmm5 = _mm512_dpbusd_epi32(zmm_add, zmm_s1, zmm_v);
|
|
__m512i zmm6 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_v);
|
|
__m512i zmm7 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_v);
|
|
|
|
zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); // ~U 0..31
|
|
zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3); // ~U 32..63
|
|
zmm4 = _mm512_permutex2var_epi8(zmm4, zmm_perm, zmm5); // ~V 0..31
|
|
zmm6 = _mm512_permutex2var_epi8(zmm6, zmm_perm, zmm7); // ~V 32..63
|
|
// Avoid _mm512_ternarylogic_epi32 (unmodeled by LLVM MSan; see GEMINI.md).
|
|
zmm0 = _mm512_xor_si512(_mm512_shuffle_i64x2(zmm0, zmm2, 0x44),
|
|
zmm_neg1); // U
|
|
zmm4 = _mm512_xor_si512(_mm512_shuffle_i64x2(zmm4, zmm6, 0x44),
|
|
zmm_neg1); // V
|
|
|
|
_mm512_storeu_si512((__m512i*)dst_u, zmm0);
|
|
_mm512_storeu_si512((__m512i*)dst_v, zmm4);
|
|
|
|
src_argb += 256;
|
|
dst_u += 64;
|
|
dst_v += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
uint64_t mask = (1ULL << width) - 1ULL;
|
|
__mmask16 k0 = (__mmask16)mask;
|
|
__mmask16 k1 = (__mmask16)(mask >> 16);
|
|
__mmask16 k2 = (__mmask16)(mask >> 32);
|
|
__mmask16 k3 = (__mmask16)(mask >> 48);
|
|
__m512i zmm_s0 = _mm512_maskz_loadu_epi32(k0, src_argb);
|
|
__m512i zmm_s1 = k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64)
|
|
: _mm512_setzero_si512();
|
|
__m512i zmm_s2 = k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128)
|
|
: _mm512_setzero_si512();
|
|
__m512i zmm_s3 = k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192)
|
|
: _mm512_setzero_si512();
|
|
|
|
__m512i zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_s0, zmm_u);
|
|
__m512i zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_s1, zmm_u);
|
|
__m512i zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_u);
|
|
__m512i zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_u);
|
|
__m512i zmm4 = _mm512_dpbusd_epi32(zmm_add, zmm_s0, zmm_v);
|
|
__m512i zmm5 = _mm512_dpbusd_epi32(zmm_add, zmm_s1, zmm_v);
|
|
__m512i zmm6 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_v);
|
|
__m512i zmm7 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_v);
|
|
|
|
zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1);
|
|
zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3);
|
|
zmm4 = _mm512_permutex2var_epi8(zmm4, zmm_perm, zmm5);
|
|
zmm6 = _mm512_permutex2var_epi8(zmm6, zmm_perm, zmm7);
|
|
zmm0 = _mm512_xor_si512(
|
|
_mm512_inserti64x4(zmm0, _mm512_castsi512_si256(zmm2), 1), zmm_neg1);
|
|
zmm4 = _mm512_xor_si512(
|
|
_mm512_inserti64x4(zmm4, _mm512_castsi512_si256(zmm6), 1), zmm_neg1);
|
|
|
|
_mm512_mask_storeu_epi8(dst_u, (__mmask64)mask, zmm0);
|
|
_mm512_mask_storeu_epi8(dst_v, (__mmask64)mask, zmm4);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI
|
|
|
|
#ifdef HAS_ARGBTOUVMATRIXROW_AVX512VNNI
|
|
LIBYUV_TARGET_AVX512VNNI
|
|
void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb,
|
|
int src_stride_argb,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
static const uint32_t kPermdARGBToUV_AVX512BW[16] = {
|
|
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15};
|
|
static const uint8_t kShuffleAARRGGBB_AVX512[16] = {
|
|
0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15};
|
|
__m512i zmm_u =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU));
|
|
__m512i zmm_v =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV));
|
|
__m512i zmm_0101 = _mm512_set1_epi16(0x0101);
|
|
__m512i zmm_8000 =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddUV));
|
|
__m512i zmm_zero = _mm512_setzero_si512();
|
|
__m512i zmm_shuf = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)kShuffleAARRGGBB_AVX512));
|
|
__m512i zmm_perm_y =
|
|
_mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
|
|
__m512i zmm_perm_uv =
|
|
_mm512_loadu_si512((const __m512i*)kPermdARGBToUV_AVX512BW);
|
|
|
|
while (width >= 32) {
|
|
__m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
__m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
|
|
__m512i zmm2 =
|
|
_mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb));
|
|
__m512i zmm3 =
|
|
_mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb + 64));
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101);
|
|
zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101);
|
|
zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101);
|
|
zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101);
|
|
|
|
zmm0 = _mm512_add_epi16(zmm0, zmm2);
|
|
zmm1 = _mm512_add_epi16(zmm1, zmm3);
|
|
|
|
zmm0 = _mm512_srli_epi16(zmm0, 1);
|
|
zmm1 = _mm512_srli_epi16(zmm1, 1);
|
|
zmm0 = _mm512_avg_epu16(zmm0, zmm_zero);
|
|
zmm1 = _mm512_avg_epu16(zmm1, zmm_zero);
|
|
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm1);
|
|
zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0);
|
|
|
|
zmm1 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_u);
|
|
zmm2 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_v);
|
|
zmm0 = _mm512_packs_epi32(zmm1, zmm2);
|
|
zmm0 = _mm512_sub_epi16(zmm_8000, zmm0);
|
|
zmm0 = _mm512_srli_epi16(zmm0, 8);
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
|
|
zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
|
|
|
|
_mm_storeu_si128((__m128i*)dst_u, _mm512_castsi512_si128(zmm0));
|
|
_mm_storeu_si128((__m128i*)dst_v, _mm512_extracti32x4_epi32(zmm0, 1));
|
|
|
|
src_argb += 128;
|
|
dst_u += 16;
|
|
dst_v += 16;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t mask = (1u << width) - 1u;
|
|
__mmask16 k1 = (__mmask16)mask;
|
|
__mmask16 k2 = (__mmask16)(mask >> 16);
|
|
__mmask16 k3 = ~k1;
|
|
__mmask16 k4 = ~k2;
|
|
__m512i zmm0 = _mm512_maskz_loadu_epi32(k1, src_argb);
|
|
__m512i zmm1 = _mm512_maskz_loadu_epi32(k2, src_argb + 64);
|
|
__m512i zmm2 = _mm512_maskz_loadu_epi32(k1, src_argb + src_stride_argb);
|
|
__m512i zmm3 =
|
|
_mm512_maskz_loadu_epi32(k2, src_argb + src_stride_argb + 64);
|
|
|
|
zmm0 = _mm512_mask_shuffle_epi32(zmm0, k3, zmm0, (_MM_PERM_ENUM)0xa0);
|
|
zmm1 = _mm512_mask_shuffle_epi32(zmm1, k4, zmm1, (_MM_PERM_ENUM)0xa0);
|
|
zmm2 = _mm512_mask_shuffle_epi32(zmm2, k3, zmm2, (_MM_PERM_ENUM)0xa0);
|
|
zmm3 = _mm512_mask_shuffle_epi32(zmm3, k4, zmm3, (_MM_PERM_ENUM)0xa0);
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101);
|
|
zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101);
|
|
zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101);
|
|
zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101);
|
|
|
|
zmm0 = _mm512_add_epi16(zmm0, zmm2);
|
|
zmm1 = _mm512_add_epi16(zmm1, zmm3);
|
|
|
|
zmm0 = _mm512_srli_epi16(zmm0, 1);
|
|
zmm1 = _mm512_srli_epi16(zmm1, 1);
|
|
zmm0 = _mm512_avg_epu16(zmm0, zmm_zero);
|
|
zmm1 = _mm512_avg_epu16(zmm1, zmm_zero);
|
|
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm1);
|
|
zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0);
|
|
|
|
zmm1 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_u);
|
|
zmm2 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_v);
|
|
zmm0 = _mm512_packs_epi32(zmm1, zmm2);
|
|
zmm0 = _mm512_sub_epi16(zmm_8000, zmm0);
|
|
zmm0 = _mm512_srli_epi16(zmm0, 8);
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
|
|
zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
|
|
|
|
int halfwidth = (width + 1) >> 1;
|
|
__mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u);
|
|
_mm_mask_storeu_epi8(dst_u, k_out, _mm512_castsi512_si128(zmm0));
|
|
_mm_mask_storeu_epi8(dst_v, k_out, _mm512_extracti32x4_epi32(zmm0, 1));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOUVMATRIXROW_AVX512VNNI
|
|
|
|
#ifdef HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI
|
|
// Requires AVX512VNNI and AVX512VBMI.
|
|
LIBYUV_TARGET_AVX512VNNIVBMI
|
|
void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
|
|
int src_stride_argb,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
static const uint32_t kPermdARGBToUV_AVX512BW[16] = {
|
|
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15};
|
|
static const uint8_t kShuffleAARRGGBB_AVX512[16] = {
|
|
0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15};
|
|
__m512i zmm_u =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU));
|
|
__m512i zmm_v =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV));
|
|
__m512i zmm_0101 = _mm512_set1_epi16(0x0101);
|
|
__m512i zmm_add = _mm512_xor_si512(
|
|
_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddUV)),
|
|
_mm512_set1_epi32(-1)); // ~kAddUV
|
|
__m512i zmm_zero = _mm512_setzero_si512();
|
|
__m512i zmm_shuf = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)kShuffleAARRGGBB_AVX512));
|
|
__m512i zmm_perm = _mm512_broadcast_i64x4(
|
|
_mm256_loadu_si256((const __m256i*)kPermbDWordByte1_AVX512VNNIVBMI));
|
|
LIBYUV_HIDE_CONSTANT(zmm_perm);
|
|
__m512i zmm_perm_uv =
|
|
_mm512_loadu_si512((const __m512i*)kPermdARGBToUV_AVX512BW);
|
|
|
|
while (width >= 32) {
|
|
__m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
__m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
|
|
__m512i zmm2 =
|
|
_mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb));
|
|
__m512i zmm3 =
|
|
_mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb + 64));
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101);
|
|
zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101);
|
|
zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101);
|
|
zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101);
|
|
|
|
zmm0 = _mm512_add_epi16(zmm0, zmm2);
|
|
zmm1 = _mm512_add_epi16(zmm1, zmm3);
|
|
|
|
zmm0 = _mm512_srli_epi16(zmm0, 1);
|
|
zmm1 = _mm512_srli_epi16(zmm1, 1);
|
|
zmm0 = _mm512_avg_epu16(zmm0, zmm_zero);
|
|
zmm1 = _mm512_avg_epu16(zmm1, zmm_zero);
|
|
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm1);
|
|
zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0);
|
|
|
|
zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_u); // ~U
|
|
zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_v); // ~V
|
|
zmm0 = _mm512_permutex2var_epi8(zmm1, zmm_perm, zmm2);
|
|
__m256i ymm_uv =
|
|
_mm256_xor_si256(_mm512_castsi512_si256(zmm0), _mm256_set1_epi32(-1));
|
|
|
|
_mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(ymm_uv));
|
|
_mm_storeu_si128((__m128i*)dst_v, _mm256_extracti128_si256(ymm_uv, 1));
|
|
|
|
src_argb += 128;
|
|
dst_u += 16;
|
|
dst_v += 16;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t mask = (1u << width) - 1u;
|
|
__mmask16 k1 = (__mmask16)mask;
|
|
__mmask16 k2 = (__mmask16)(mask >> 16);
|
|
__mmask16 k3 = ~k1;
|
|
__mmask16 k4 = ~k2;
|
|
__m512i zmm0 = _mm512_maskz_loadu_epi32(k1, src_argb);
|
|
__m512i zmm1 = _mm512_maskz_loadu_epi32(k2, src_argb + 64);
|
|
__m512i zmm2 = _mm512_maskz_loadu_epi32(k1, src_argb + src_stride_argb);
|
|
__m512i zmm3 =
|
|
_mm512_maskz_loadu_epi32(k2, src_argb + src_stride_argb + 64);
|
|
|
|
zmm0 = _mm512_mask_shuffle_epi32(zmm0, k3, zmm0, (_MM_PERM_ENUM)0xa0);
|
|
zmm1 = _mm512_mask_shuffle_epi32(zmm1, k4, zmm1, (_MM_PERM_ENUM)0xa0);
|
|
zmm2 = _mm512_mask_shuffle_epi32(zmm2, k3, zmm2, (_MM_PERM_ENUM)0xa0);
|
|
zmm3 = _mm512_mask_shuffle_epi32(zmm3, k4, zmm3, (_MM_PERM_ENUM)0xa0);
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101);
|
|
zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101);
|
|
zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101);
|
|
zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101);
|
|
|
|
zmm0 = _mm512_add_epi16(zmm0, zmm2);
|
|
zmm1 = _mm512_add_epi16(zmm1, zmm3);
|
|
|
|
zmm0 = _mm512_srli_epi16(zmm0, 1);
|
|
zmm1 = _mm512_srli_epi16(zmm1, 1);
|
|
zmm0 = _mm512_avg_epu16(zmm0, zmm_zero);
|
|
zmm1 = _mm512_avg_epu16(zmm1, zmm_zero);
|
|
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm1);
|
|
zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0);
|
|
|
|
zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_u); // ~U
|
|
zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_v); // ~V
|
|
zmm0 = _mm512_permutex2var_epi8(zmm1, zmm_perm, zmm2);
|
|
__m256i ymm_uv =
|
|
_mm256_xor_si256(_mm512_castsi512_si256(zmm0), _mm256_set1_epi32(-1));
|
|
|
|
int halfwidth = (width + 1) >> 1;
|
|
__mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u);
|
|
_mm_mask_storeu_epi8(dst_u, k_out, _mm256_castsi256_si128(ymm_uv));
|
|
_mm_mask_storeu_epi8(dst_v, k_out, _mm256_extracti128_si256(ymm_uv, 1));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI
|
|
|
|
#ifdef HAS_ARGBTOUVMATRIXROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb,
|
|
int src_stride_argb,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
__m256i ymm_u =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToU));
|
|
__m256i ymm_v =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToV));
|
|
__m256i ymm_0101 = _mm256_set1_epi16(0x0101);
|
|
__m256i ymm_shuf =
|
|
_mm256_setr_epi8(0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15, 0,
|
|
4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15);
|
|
__m256i ymm_8000 =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kAddUV));
|
|
__m256i ymm_zero = _mm256_setzero_si256();
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_argb + 32));
|
|
__m256i ymm2 =
|
|
_mm256_loadu_si256((const __m256i*)(src_argb + src_stride_argb));
|
|
__m256i ymm3 =
|
|
_mm256_loadu_si256((const __m256i*)(src_argb + src_stride_argb + 32));
|
|
|
|
ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
|
|
ymm1 = _mm256_shuffle_epi8(ymm1, ymm_shuf);
|
|
ymm2 = _mm256_shuffle_epi8(ymm2, ymm_shuf);
|
|
ymm3 = _mm256_shuffle_epi8(ymm3, ymm_shuf);
|
|
|
|
ymm0 = _mm256_maddubs_epi16(ymm0, ymm_0101);
|
|
ymm1 = _mm256_maddubs_epi16(ymm1, ymm_0101);
|
|
ymm2 = _mm256_maddubs_epi16(ymm2, ymm_0101);
|
|
ymm3 = _mm256_maddubs_epi16(ymm3, ymm_0101);
|
|
|
|
ymm0 = _mm256_add_epi16(ymm0, ymm2);
|
|
ymm1 = _mm256_add_epi16(ymm1, ymm3);
|
|
|
|
ymm0 = _mm256_srli_epi16(ymm0, 1);
|
|
ymm1 = _mm256_srli_epi16(ymm1, 1);
|
|
ymm0 = _mm256_avg_epu16(ymm0, ymm_zero);
|
|
ymm1 = _mm256_avg_epu16(ymm1, ymm_zero);
|
|
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm1);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
|
|
ymm1 = _mm256_maddubs_epi16(ymm0, ymm_v);
|
|
ymm0 = _mm256_maddubs_epi16(ymm0, ymm_u);
|
|
|
|
ymm0 = _mm256_hadd_epi16(ymm0, ymm1);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
ymm0 = _mm256_sub_epi16(ymm_8000, ymm0);
|
|
ymm0 = _mm256_srli_epi16(ymm0, 8);
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
|
|
|
|
__m128i xmm_u = _mm256_castsi256_si128(ymm0);
|
|
__m128i xmm_v = _mm256_extracti128_si256(ymm0, 1);
|
|
|
|
_mm_storel_epi64((__m128i*)dst_u, xmm_u);
|
|
_mm_storel_epi64((__m128i*)dst_v, xmm_v);
|
|
|
|
src_argb += 64;
|
|
dst_u += 8;
|
|
dst_v += 8;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOUVMATRIXROW_AVX2
|
|
|
|
#ifdef HAS_CONVERT8TO16ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void Convert8To16Row_AVX2(const uint8_t* src_y,
|
|
uint16_t* dst_y,
|
|
int bits,
|
|
int width) {
|
|
const int shift = 16 - bits;
|
|
__m128i shift128 = _mm_cvtsi32_si128(shift);
|
|
do {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_y);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
__m256i row0 = _mm256_unpacklo_epi8(ymm0, ymm0);
|
|
__m256i row1 = _mm256_unpackhi_epi8(ymm0, ymm0);
|
|
row0 = _mm256_srl_epi16(row0, shift128);
|
|
row1 = _mm256_srl_epi16(row1, shift128);
|
|
_mm256_storeu_si256((__m256i*)dst_y, row0);
|
|
_mm256_storeu_si256((__m256i*)(dst_y + 16), row1);
|
|
src_y += 32;
|
|
dst_y += 32;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_CONVERT8TO16ROW_AVX2
|
|
|
|
#ifdef HAS_MULTIPLYROW_16_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MultiplyRow_16_AVX2(const uint16_t* src_y,
|
|
uint16_t* dst_y,
|
|
int scale,
|
|
int width) {
|
|
__m256i scale256 = _mm256_set1_epi16((short)scale);
|
|
do {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_y);
|
|
__m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_y + 16));
|
|
ymm0 = _mm256_mullo_epi16(ymm0, scale256);
|
|
ymm1 = _mm256_mullo_epi16(ymm1, scale256);
|
|
_mm256_storeu_si256((__m256i*)dst_y, ymm0);
|
|
_mm256_storeu_si256((__m256i*)(dst_y + 16), ymm1);
|
|
src_y += 32;
|
|
dst_y += 32;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MULTIPLYROW_16_AVX2
|
|
|
|
#ifdef HAS_MERGEUVROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MergeUVRow_AVX2(const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_u));
|
|
__m256i ymm1 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_v));
|
|
|
|
ymm1 = _mm256_slli_epi16(ymm1, 8);
|
|
ymm0 = _mm256_or_si256(ymm0, ymm1);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_uv, ymm0);
|
|
|
|
src_u += 16;
|
|
src_v += 16;
|
|
dst_uv += 32;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGEUVROW_AVX2
|
|
|
|
#ifdef HAS_MIRRORROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
|
|
__m256i ymm_shuf = _mm256_broadcastsi128_si256(
|
|
_mm_setr_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
|
|
src += width;
|
|
while (width > 0) {
|
|
src -= 32;
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src);
|
|
ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0x4e);
|
|
_mm256_storeu_si256((__m256i*)dst, ymm0);
|
|
dst += 32;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MIRRORROW_AVX2
|
|
|
|
#ifdef HAS_MIRRORUVROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
|
|
__m256i ymm_shuf = _mm256_broadcastsi128_si256(
|
|
_mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1));
|
|
src_uv += width * 2;
|
|
while (width > 0) {
|
|
src_uv -= 32;
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_uv);
|
|
ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0x4e);
|
|
_mm256_storeu_si256((__m256i*)dst_uv, ymm0);
|
|
dst_uv += 32;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MIRRORUVROW_AVX2
|
|
|
|
#ifdef HAS_MIRRORROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
|
|
const __m512i zmm_shuf = _mm512_broadcast_i32x4(
|
|
_mm_setr_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
|
|
const __m512i zmm_rev = _mm512_set_epi16(
|
|
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
|
|
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31);
|
|
const uint8_t* s = src + width;
|
|
while (width >= 64) {
|
|
s -= 64;
|
|
__m512i v = _mm512_loadu_si512((const __m512i*)s);
|
|
v = _mm512_shuffle_epi8(v, zmm_shuf);
|
|
v = _mm512_shuffle_i64x2(v, v, 0x1b);
|
|
_mm512_storeu_si512((__m512i*)dst, v);
|
|
dst += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
__mmask64 k = (1ULL << width) - 1ULL;
|
|
__m512i v = _mm512_maskz_loadu_epi8(k, src);
|
|
v = _mm512_or_si512(_mm512_srli_epi16(v, 8), _mm512_slli_epi16(v, 8));
|
|
__m512i idx =
|
|
_mm512_sub_epi16(zmm_rev, _mm512_set1_epi16((short)((64 - width) >> 1)));
|
|
v = PermutexvarEpi16(idx, v);
|
|
if (width & 1) {
|
|
__m512i v_hi = _mm512_alignr_epi64(v, v, 2);
|
|
v = _mm512_alignr_epi8(v_hi, v, 1);
|
|
}
|
|
_mm512_mask_storeu_epi8(dst, k, v);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MIRRORROW_AVX512BW
|
|
|
|
#ifdef HAS_MIRRORSPLITUVROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void MirrorSplitUVRow_AVX512BW(const uint8_t* src_uv,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
const __m512i zmm_shuf = _mm512_broadcast_i32x4(
|
|
_mm_setr_epi8(14, 12, 10, 8, 6, 4, 2, 0, 15, 13, 11, 9, 7, 5, 3, 1));
|
|
const __m512i zmm_perm = _mm512_setr_epi64(6, 4, 2, 0, 7, 5, 3, 1);
|
|
const __m512i zmm_rev = _mm512_set_epi16(
|
|
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
|
|
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31);
|
|
const uint8_t* s = src_uv + (ptrdiff_t)width * 2;
|
|
while (width >= 32) {
|
|
s -= 64;
|
|
__m512i v = _mm512_loadu_si512((const __m512i*)s);
|
|
v = _mm512_shuffle_epi8(v, zmm_shuf);
|
|
v = PermutexvarEpi64(zmm_perm, v);
|
|
_mm256_storeu_si256((__m256i*)dst_u, _mm512_castsi512_si256(v));
|
|
_mm256_storeu_si256((__m256i*)dst_v, _mm512_extracti64x4_epi64(v, 1));
|
|
dst_u += 32;
|
|
dst_v += 32;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask32 k = (__mmask32)((1u << width) - 1u);
|
|
__m512i idx =
|
|
_mm512_sub_epi16(zmm_rev, _mm512_set1_epi16((short)(32 - width)));
|
|
__m512i v = _mm512_maskz_loadu_epi16(k, src_uv);
|
|
v = PermutexvarEpi16(idx, v);
|
|
_mm256_mask_storeu_epi8(dst_u, k, _mm512_cvtepi16_epi8(v));
|
|
_mm256_mask_storeu_epi8(dst_v, k,
|
|
_mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8)));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MIRRORSPLITUVROW_AVX512BW
|
|
|
|
#ifdef HAS_MIRRORUVROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
|
|
const __m512i zmm_idx = _mm512_set_epi16(
|
|
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
|
|
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31);
|
|
const uint8_t* s = src_uv + (ptrdiff_t)width * 2;
|
|
while (width >= 32) {
|
|
s -= 64;
|
|
__m512i zmm0 = _mm512_loadu_si512((const __m512i*)s);
|
|
zmm0 = PermutexvarEpi16(zmm_idx, zmm0);
|
|
_mm512_storeu_si512((__m512i*)dst_uv, zmm0);
|
|
dst_uv += 64;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask32 k = (__mmask32)((1u << width) - 1u);
|
|
__m512i idx =
|
|
_mm512_sub_epi16(zmm_idx, _mm512_set1_epi16((short)(32 - width)));
|
|
__m512i zmm0 = _mm512_maskz_loadu_epi16(k, src_uv);
|
|
_mm512_mask_storeu_epi16(dst_uv, k, PermutexvarEpi16(idx, zmm0));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MIRRORUVROW_AVX512BW
|
|
|
|
#ifdef HAS_SWAPUVROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
|
|
__m256i ymm_shuf = _mm256_broadcastsi128_si256(
|
|
_mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14));
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_uv);
|
|
__m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_uv + 32));
|
|
ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
|
|
ymm1 = _mm256_shuffle_epi8(ymm1, ymm_shuf);
|
|
_mm256_storeu_si256((__m256i*)dst_vu, ymm0);
|
|
_mm256_storeu_si256((__m256i*)(dst_vu + 32), ymm1);
|
|
src_uv += 64;
|
|
dst_vu += 64;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SWAPUVROW_AVX2
|
|
|
|
#ifdef HAS_MIRRORSPLITUVROW_SSSE3
|
|
LIBYUV_TARGET_SSSE3
|
|
void MirrorSplitUVRow_SSSE3(const uint8_t* src_uv,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
const __m128i xmm_shuf =
|
|
_mm_setr_epi8(14, 12, 10, 8, 6, 4, 2, 0, 15, 13, 11, 9, 7, 5, 3, 1);
|
|
src_uv += (ptrdiff_t)width * 2;
|
|
while (width > 0) {
|
|
src_uv -= 16;
|
|
__m128i xmm0 =
|
|
_mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)src_uv), xmm_shuf);
|
|
_mm_storel_epi64((__m128i*)dst_u, xmm0);
|
|
_mm_storel_epi64((__m128i*)dst_v, _mm_unpackhi_epi64(xmm0, xmm0));
|
|
dst_u += 8;
|
|
dst_v += 8;
|
|
width -= 8;
|
|
}
|
|
}
|
|
#endif // HAS_MIRRORSPLITUVROW_SSSE3
|
|
|
|
#ifdef HAS_MIRRORSPLITUVROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MirrorSplitUVRow_AVX2(const uint8_t* src_uv,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
__m256i ymm_shuf = _mm256_broadcastsi128_si256(
|
|
_mm_setr_epi8(14, 12, 10, 8, 6, 4, 2, 0, 15, 13, 11, 9, 7, 5, 3, 1));
|
|
src_uv += (ptrdiff_t)width * 2;
|
|
while (width > 0) {
|
|
src_uv -= 32;
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_uv);
|
|
ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0x72);
|
|
_mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(ymm0));
|
|
_mm_storeu_si128((__m128i*)dst_v, _mm256_extracti128_si256(ymm0, 1));
|
|
dst_u += 16;
|
|
dst_v += 16;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MIRRORSPLITUVROW_AVX2
|
|
|
|
#ifdef HAS_RGB24MIRRORROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24,
|
|
uint8_t* dst_rgb24,
|
|
int width) {
|
|
__m256i shuf0 =
|
|
_mm256_setr_epi8(-1, 12, 13, 14, 9, 10, 11, 6, 7, 8, 3, 4, 5, 0, 1, 2, -1,
|
|
12, 13, 14, 9, 10, 11, 6, 7, 8, 3, 4, 5, 0, 1, 2);
|
|
__m128i shuf1 =
|
|
_mm_setr_epi8(13, 14, 15, 10, 11, 12, 7, 8, 9, 4, 5, 6, 1, 2, 3, -1);
|
|
|
|
src_rgb24 += (ptrdiff_t)width * 3 - 96;
|
|
while (width > 0) {
|
|
__m128i v0_lo = _mm_loadu_si128((const __m128i*)(src_rgb24 + 0));
|
|
__m128i v0_hi = _mm_loadu_si128((const __m128i*)(src_rgb24 + 15));
|
|
__m256i v0 =
|
|
_mm256_inserti128_si256(_mm256_castsi128_si256(v0_lo), v0_hi, 1);
|
|
|
|
__m128i v1_lo = _mm_loadu_si128((const __m128i*)(src_rgb24 + 30));
|
|
__m128i v1_hi = _mm_loadu_si128((const __m128i*)(src_rgb24 + 45));
|
|
__m256i v1 =
|
|
_mm256_inserti128_si256(_mm256_castsi128_si256(v1_lo), v1_hi, 1);
|
|
|
|
__m128i v2_lo = _mm_loadu_si128((const __m128i*)(src_rgb24 + 60));
|
|
__m128i v2_hi = _mm_loadu_si128((const __m128i*)(src_rgb24 + 75));
|
|
__m256i v2 =
|
|
_mm256_inserti128_si256(_mm256_castsi128_si256(v2_lo), v2_hi, 1);
|
|
|
|
__m128i v3 = _mm_loadu_si128((const __m128i*)(src_rgb24 + 80));
|
|
|
|
v0 = _mm256_shuffle_epi8(v0, shuf0);
|
|
v1 = _mm256_shuffle_epi8(v1, shuf0);
|
|
v2 = _mm256_shuffle_epi8(v2, shuf0);
|
|
v3 = _mm_shuffle_epi8(v3, shuf1);
|
|
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 80), _mm256_castsi256_si128(v0));
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 65),
|
|
_mm256_extracti128_si256(v0, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 50), _mm256_castsi256_si128(v1));
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 35),
|
|
_mm256_extracti128_si256(v1, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 20), _mm256_castsi256_si128(v2));
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 5),
|
|
_mm256_extracti128_si256(v2, 1));
|
|
_mm_storel_epi64((__m128i*)(dst_rgb24 + 0), v3);
|
|
|
|
src_rgb24 -= 96;
|
|
dst_rgb24 += 96;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RGB24MIRRORROW_AVX2
|
|
|
|
#ifdef HAS_INTERPOLATEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void InterpolateRow_AVX2(uint8_t* dst_ptr,
|
|
const uint8_t* src_ptr,
|
|
ptrdiff_t src_stride,
|
|
int width,
|
|
int source_y_fraction) {
|
|
int y1 = source_y_fraction;
|
|
int y0 = 256 - y1;
|
|
const uint8_t* src_ptr1 = src_ptr + src_stride;
|
|
__m256i ymm_y = _mm256_set1_epi16((y1 << 8) | y0);
|
|
__m256i ymm_8080 = _mm256_set1_epi16(0x8080);
|
|
int i;
|
|
|
|
if (y1 == 0) {
|
|
for (i = 0; i < width; i += 32) {
|
|
_mm256_storeu_si256((__m256i*)(dst_ptr + i),
|
|
_mm256_loadu_si256((const __m256i*)(src_ptr + i)));
|
|
}
|
|
} else if (y1 == 128) {
|
|
for (i = 0; i < width; i += 32) {
|
|
__m256i row0 = _mm256_loadu_si256((const __m256i*)(src_ptr + i));
|
|
__m256i row1 = _mm256_loadu_si256((const __m256i*)(src_ptr1 + i));
|
|
_mm256_storeu_si256((__m256i*)(dst_ptr + i), _mm256_avg_epu8(row0, row1));
|
|
}
|
|
} else {
|
|
for (i = 0; i < width; i += 32) {
|
|
__m256i row0 = _mm256_loadu_si256((const __m256i*)(src_ptr + i));
|
|
__m256i row1 = _mm256_loadu_si256((const __m256i*)(src_ptr1 + i));
|
|
__m256i low = _mm256_unpacklo_epi8(row0, row1);
|
|
__m256i high = _mm256_unpackhi_epi8(row0, row1);
|
|
low = _mm256_sub_epi8(low, ymm_8080);
|
|
high = _mm256_sub_epi8(high, ymm_8080);
|
|
low = _mm256_maddubs_epi16(ymm_y, low);
|
|
high = _mm256_maddubs_epi16(ymm_y, high);
|
|
low = _mm256_add_epi16(low, ymm_8080);
|
|
high = _mm256_add_epi16(high, ymm_8080);
|
|
low = _mm256_srli_epi16(low, 8);
|
|
high = _mm256_srli_epi16(high, 8);
|
|
_mm256_storeu_si256((__m256i*)(dst_ptr + i),
|
|
_mm256_packus_epi16(low, high));
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_INTERPOLATEROW_AVX2
|
|
|
|
#ifdef HAS_INTERPOLATEROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void InterpolateRow_AVX512BW(uint8_t* dst_ptr,
|
|
const uint8_t* src_ptr,
|
|
ptrdiff_t src_stride,
|
|
int width,
|
|
int source_y_fraction) {
|
|
int y1 = source_y_fraction;
|
|
int y0 = 256 - y1;
|
|
const uint8_t* src_ptr1 = src_ptr + src_stride;
|
|
__m512i zmm_y = _mm512_set1_epi16((short)((y1 << 8) | y0));
|
|
__m512i zmm_8080 = _mm512_set1_epi16((short)0x8080);
|
|
int i;
|
|
|
|
if (y1 == 0) {
|
|
for (i = 0; i <= width - 64; i += 64) {
|
|
_mm512_storeu_si512((__m512i*)(dst_ptr + i),
|
|
_mm512_loadu_si512((const __m512i*)(src_ptr + i)));
|
|
}
|
|
if (i < width) {
|
|
__mmask64 mask = (__mmask64)((1ull << (width - i)) - 1ull);
|
|
_mm512_mask_storeu_epi8(dst_ptr + i, mask,
|
|
_mm512_maskz_loadu_epi8(mask, src_ptr + i));
|
|
}
|
|
} else if (y1 == 128) {
|
|
for (i = 0; i <= width - 64; i += 64) {
|
|
__m512i row0 = _mm512_loadu_si512((const __m512i*)(src_ptr + i));
|
|
__m512i row1 = _mm512_loadu_si512((const __m512i*)(src_ptr1 + i));
|
|
_mm512_storeu_si512((__m512i*)(dst_ptr + i), _mm512_avg_epu8(row0, row1));
|
|
}
|
|
if (i < width) {
|
|
__mmask64 mask = (__mmask64)((1ull << (width - i)) - 1ull);
|
|
__m512i row0 = _mm512_maskz_loadu_epi8(mask, src_ptr + i);
|
|
__m512i row1 = _mm512_maskz_loadu_epi8(mask, src_ptr1 + i);
|
|
_mm512_mask_storeu_epi8(dst_ptr + i, mask, _mm512_avg_epu8(row0, row1));
|
|
}
|
|
} else {
|
|
for (i = 0; i <= width - 64; i += 64) {
|
|
__m512i row0 = _mm512_loadu_si512((const __m512i*)(src_ptr + i));
|
|
__m512i row1 = _mm512_loadu_si512((const __m512i*)(src_ptr1 + i));
|
|
__m512i low = _mm512_unpacklo_epi8(row0, row1);
|
|
__m512i high = _mm512_unpackhi_epi8(row0, row1);
|
|
low = _mm512_sub_epi8(low, zmm_8080);
|
|
high = _mm512_sub_epi8(high, zmm_8080);
|
|
low = _mm512_maddubs_epi16(zmm_y, low);
|
|
high = _mm512_maddubs_epi16(zmm_y, high);
|
|
low = _mm512_add_epi16(low, zmm_8080);
|
|
high = _mm512_add_epi16(high, zmm_8080);
|
|
low = _mm512_srli_epi16(low, 8);
|
|
high = _mm512_srli_epi16(high, 8);
|
|
_mm512_storeu_si512((__m512i*)(dst_ptr + i),
|
|
_mm512_packus_epi16(low, high));
|
|
}
|
|
if (i < width) {
|
|
__mmask64 mask = (__mmask64)((1ull << (width - i)) - 1ull);
|
|
__m512i row0 = _mm512_maskz_loadu_epi8(mask, src_ptr + i);
|
|
__m512i row1 = _mm512_maskz_loadu_epi8(mask, src_ptr1 + i);
|
|
__m512i low = _mm512_unpacklo_epi8(row0, row1);
|
|
__m512i high = _mm512_unpackhi_epi8(row0, row1);
|
|
low = _mm512_sub_epi8(low, zmm_8080);
|
|
high = _mm512_sub_epi8(high, zmm_8080);
|
|
low = _mm512_maddubs_epi16(zmm_y, low);
|
|
high = _mm512_maddubs_epi16(zmm_y, high);
|
|
low = _mm512_add_epi16(low, zmm_8080);
|
|
high = _mm512_add_epi16(high, zmm_8080);
|
|
low = _mm512_srli_epi16(low, 8);
|
|
high = _mm512_srli_epi16(high, 8);
|
|
_mm512_mask_storeu_epi8(dst_ptr + i, mask,
|
|
_mm512_packus_epi16(low, high));
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_INTERPOLATEROW_AVX512BW
|
|
|
|
#ifdef HAS_INTERPOLATEROW_16_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void InterpolateRow_16_AVX2(uint16_t* dst_ptr,
|
|
const uint16_t* src_ptr,
|
|
ptrdiff_t src_stride,
|
|
int width,
|
|
int source_y_fraction) {
|
|
int y1 = source_y_fraction;
|
|
int y0 = 256 - y1;
|
|
const uint16_t* src_ptr1 = src_ptr + src_stride;
|
|
__m256i ymm_y = _mm256_set1_epi32((y1 << 16) | y0);
|
|
__m256i ymm_8000 = _mm256_set1_epi16((short)0x8000);
|
|
__m256i ymm_round = _mm256_set1_epi32(8388736); // 0x800000 + 128
|
|
int i;
|
|
|
|
if (y1 == 0) {
|
|
for (i = 0; i < width; i += 16) {
|
|
_mm256_storeu_si256((__m256i*)(dst_ptr + i),
|
|
_mm256_loadu_si256((const __m256i*)(src_ptr + i)));
|
|
}
|
|
} else if (y1 == 128) {
|
|
for (i = 0; i < width; i += 16) {
|
|
__m256i row0 = _mm256_loadu_si256((const __m256i*)(src_ptr + i));
|
|
__m256i row1 = _mm256_loadu_si256((const __m256i*)(src_ptr1 + i));
|
|
_mm256_storeu_si256((__m256i*)(dst_ptr + i),
|
|
_mm256_avg_epu16(row0, row1));
|
|
}
|
|
} else {
|
|
for (i = 0; i < width; i += 16) {
|
|
__m256i row0 = _mm256_loadu_si256((const __m256i*)(src_ptr + i));
|
|
__m256i row1 = _mm256_loadu_si256((const __m256i*)(src_ptr1 + i));
|
|
__m256i row0l = _mm256_unpacklo_epi16(row0, row1);
|
|
__m256i row0h = _mm256_unpackhi_epi16(row0, row1);
|
|
row0l = _mm256_sub_epi16(row0l, ymm_8000);
|
|
row0h = _mm256_sub_epi16(row0h, ymm_8000);
|
|
__m256i resl = _mm256_madd_epi16(row0l, ymm_y);
|
|
__m256i resh = _mm256_madd_epi16(row0h, ymm_y);
|
|
resl = _mm256_add_epi32(resl, ymm_round);
|
|
resh = _mm256_add_epi32(resh, ymm_round);
|
|
resl = _mm256_srai_epi32(resl, 8);
|
|
resh = _mm256_srai_epi32(resh, 8);
|
|
_mm256_storeu_si256((__m256i*)(dst_ptr + i),
|
|
_mm256_packus_epi32(resl, resh));
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_INTERPOLATEROW_16_AVX2
|
|
|
|
#ifdef HAS_ARGBMIRRORROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
|
|
__m256i ymm_shuf = _mm256_setr_epi32(7, 6, 5, 4, 3, 2, 1, 0);
|
|
src += width * 4;
|
|
while (width > 0) {
|
|
src -= 32;
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src);
|
|
ymm0 = _mm256_permutevar8x32_epi32(ymm0, ymm_shuf);
|
|
_mm256_storeu_si256((__m256i*)dst, ymm0);
|
|
dst += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBMIRRORROW_AVX2
|
|
|
|
#ifdef HAS_J400TOARGBROW_AVX2
|
|
static const uint8_t kShuffleMaskJ400ToARGB_0[32] = {
|
|
0u, 0u, 0u, 128u, 1u, 1u, 1u, 128u, 2u, 2u, 2u, 128u, 3u, 3u, 3u, 128u,
|
|
4u, 4u, 4u, 128u, 5u, 5u, 5u, 128u, 6u, 6u, 6u, 128u, 7u, 7u, 7u, 128u};
|
|
static const uint8_t kShuffleMaskJ400ToARGB_1[32] = {
|
|
8u, 8u, 8u, 128u, 9u, 9u, 9u, 128u, 10u, 10u, 10u,
|
|
128u, 11u, 11u, 11u, 128u, 12u, 12u, 12u, 128u, 13u, 13u,
|
|
13u, 128u, 14u, 14u, 14u, 128u, 15u, 15u, 15u, 128u};
|
|
|
|
LIBYUV_TARGET_AVX2
|
|
void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) {
|
|
__m256i ymm_mask0 =
|
|
_mm256_loadu_si256((const __m256i*)kShuffleMaskJ400ToARGB_0);
|
|
__m256i ymm_mask1 =
|
|
_mm256_loadu_si256((const __m256i*)kShuffleMaskJ400ToARGB_1);
|
|
__m256i ymm_alpha = _mm256_set1_epi32((int)0xff000000u);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)src_y));
|
|
|
|
__m256i ymm1 = _mm256_shuffle_epi8(ymm0, ymm_mask0);
|
|
__m256i ymm2 = _mm256_shuffle_epi8(ymm0, ymm_mask1);
|
|
|
|
ymm1 = _mm256_or_si256(ymm1, ymm_alpha);
|
|
ymm2 = _mm256_or_si256(ymm2, ymm_alpha);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, ymm1);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm2);
|
|
|
|
src_y += 16;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_J400TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_RGB24TOARGBROW_AVX2
|
|
static const uint8_t kShuffleMaskRGB24ToARGB[2][16] = {
|
|
{0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u},
|
|
{4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u,
|
|
128u}};
|
|
#endif // HAS_RGB24TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_RGB565TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void RGB565ToARGBRow_AVX2(const uint8_t* src_rgb565,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
__m256i ymm_scale_rb = _mm256_set1_epi32(0x01080108);
|
|
__m256i ymm_scale_g = _mm256_set1_epi32(0x20802080);
|
|
__m256i ymm_mask_b = _mm256_set1_epi16((short)0xf800);
|
|
__m256i ymm_mask_g = _mm256_set1_epi16(0x07e0);
|
|
__m256i ymm_mask_a = _mm256_set1_epi16((short)0xff00);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_rgb565);
|
|
__m256i ymm1 = ymm0;
|
|
__m256i ymm2 = ymm0;
|
|
|
|
ymm1 = _mm256_and_si256(ymm1, ymm_mask_b);
|
|
ymm2 = _mm256_slli_epi16(ymm2, 11);
|
|
ymm1 = _mm256_mulhi_epu16(ymm1, ymm_scale_rb);
|
|
ymm2 = _mm256_mulhi_epu16(ymm2, ymm_scale_rb);
|
|
ymm1 = _mm256_slli_epi16(ymm1, 8);
|
|
ymm1 = _mm256_or_si256(ymm1, ymm2); // RB
|
|
|
|
ymm0 = _mm256_and_si256(ymm0, ymm_mask_g);
|
|
ymm0 = _mm256_mulhi_epu16(ymm0, ymm_scale_g);
|
|
ymm0 = _mm256_or_si256(ymm0, ymm_mask_a); // GA
|
|
|
|
ymm2 = _mm256_unpacklo_epi8(ymm1, ymm0);
|
|
ymm1 = _mm256_unpackhi_epi8(ymm1, ymm0);
|
|
|
|
ymm0 = _mm256_permute2x128_si256(ymm2, ymm1, 0x20);
|
|
ymm1 = _mm256_permute2x128_si256(ymm2, ymm1, 0x31);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, ymm0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
|
|
|
|
src_rgb565 += 32;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RGB565TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_ARGB1555TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGB1555ToARGBRow_AVX2(const uint8_t* src_argb1555,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
__m256i ymm_scale_rb = _mm256_set1_epi32(0x01080108);
|
|
__m256i ymm_scale_g = _mm256_set1_epi32(0x42004200);
|
|
__m256i ymm_mask_b = _mm256_set1_epi16((short)0xf800);
|
|
__m256i ymm_mask_g = _mm256_set1_epi16(0x03e0);
|
|
__m256i ymm_mask_a = _mm256_set1_epi16((short)0xff00);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb1555);
|
|
__m256i ymm1 = ymm0;
|
|
__m256i ymm2 = ymm0;
|
|
|
|
ymm1 = _mm256_slli_epi16(ymm1, 1);
|
|
ymm2 = _mm256_slli_epi16(ymm2, 11);
|
|
ymm1 = _mm256_and_si256(ymm1, ymm_mask_b);
|
|
ymm2 = _mm256_mulhi_epu16(ymm2, ymm_scale_rb);
|
|
ymm1 = _mm256_mulhi_epu16(ymm1, ymm_scale_rb);
|
|
ymm1 = _mm256_slli_epi16(ymm1, 8);
|
|
ymm1 = _mm256_or_si256(ymm1, ymm2); // RB
|
|
|
|
ymm2 = ymm0;
|
|
ymm0 = _mm256_and_si256(ymm0, ymm_mask_g);
|
|
ymm2 = _mm256_srai_epi16(ymm2, 8);
|
|
ymm0 = _mm256_mulhi_epu16(ymm0, ymm_scale_g);
|
|
ymm2 = _mm256_and_si256(ymm2, ymm_mask_a);
|
|
ymm0 = _mm256_or_si256(ymm0, ymm2); // GA
|
|
|
|
ymm2 = _mm256_unpacklo_epi8(ymm1, ymm0);
|
|
ymm1 = _mm256_unpackhi_epi8(ymm1, ymm0);
|
|
|
|
ymm0 = _mm256_permute2x128_si256(ymm2, ymm1, 0x20);
|
|
ymm1 = _mm256_permute2x128_si256(ymm2, ymm1, 0x31);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, ymm0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
|
|
|
|
src_argb1555 += 32;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGB1555TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_ARGB4444TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGB4444ToARGBRow_AVX2(const uint8_t* src_argb4444,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
__m256i ymm_mask = _mm256_set1_epi32(0x0f0f0f0f);
|
|
__m256i ymm_mask2 = _mm256_slli_epi32(ymm_mask, 4);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb4444);
|
|
__m256i ymm2 = ymm0;
|
|
|
|
ymm0 = _mm256_and_si256(ymm0, ymm_mask);
|
|
ymm2 = _mm256_and_si256(ymm2, ymm_mask2);
|
|
|
|
__m256i ymm1 = ymm0;
|
|
__m256i ymm3 = ymm2;
|
|
|
|
ymm1 = _mm256_slli_epi16(ymm1, 4);
|
|
ymm3 = _mm256_srli_epi16(ymm3, 4);
|
|
|
|
ymm0 = _mm256_or_si256(ymm0, ymm1);
|
|
ymm2 = _mm256_or_si256(ymm2, ymm3);
|
|
|
|
ymm1 = ymm0;
|
|
ymm0 = _mm256_unpacklo_epi8(ymm0, ymm2);
|
|
ymm1 = _mm256_unpackhi_epi8(ymm1, ymm2);
|
|
|
|
ymm2 = _mm256_permute2x128_si256(ymm0, ymm1, 0x20);
|
|
ymm1 = _mm256_permute2x128_si256(ymm0, ymm1, 0x31);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, ymm2);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
|
|
|
|
src_argb4444 += 32;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGB4444TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_ARGBTORGB565DITHERROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToRGB565DitherRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_rgb,
|
|
uint32_t dither4,
|
|
int width) {
|
|
__m128i dither_shuf =
|
|
_mm_setr_epi8(0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3);
|
|
__m128i d128 = _mm_shuffle_epi8(_mm_cvtsi32_si128((int)dither4), dither_shuf);
|
|
__m256i ymm_dither = _mm256_broadcastsi128_si256(d128);
|
|
__m256i mask_b = _mm256_set1_epi32(0x0000001f);
|
|
__m256i mask_g = _mm256_set1_epi32(0x000007e0);
|
|
__m256i mask_r = _mm256_set1_epi32(0x0000f800);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
ymm0 = _mm256_adds_epu8(ymm0, ymm_dither);
|
|
__m256i ymm2 = _mm256_srli_epi32(ymm0, 5);
|
|
__m256i ymm1 = _mm256_srli_epi32(ymm0, 3);
|
|
ymm0 = _mm256_srli_epi32(ymm0, 8);
|
|
ymm2 = _mm256_and_si256(ymm2, mask_g);
|
|
ymm1 = _mm256_and_si256(ymm1, mask_b);
|
|
ymm0 = _mm256_and_si256(ymm0, mask_r);
|
|
ymm1 = _mm256_or_si256(ymm2, ymm1);
|
|
ymm0 = _mm256_or_si256(ymm1, ymm0);
|
|
ymm0 = _mm256_packus_epi32(ymm0, ymm0);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_rgb, _mm256_castsi256_si128(ymm0));
|
|
src_argb += 32;
|
|
dst_rgb += 16;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTORGB565DITHERROW_AVX2
|
|
|
|
#ifdef HAS_ARGBTORGB565ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToRGB565Row_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_rgb,
|
|
int width) {
|
|
__m256i mask_b = _mm256_set1_epi32(0x0000001f);
|
|
__m256i mask_g = _mm256_set1_epi32(0x000007e0);
|
|
__m256i mask_r = _mm256_set1_epi32(0x0000f800);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i ymm2 = _mm256_srli_epi32(ymm0, 5);
|
|
__m256i ymm1 = _mm256_srli_epi32(ymm0, 3);
|
|
ymm0 = _mm256_srli_epi32(ymm0, 8);
|
|
ymm2 = _mm256_and_si256(ymm2, mask_g);
|
|
ymm1 = _mm256_and_si256(ymm1, mask_b);
|
|
ymm0 = _mm256_and_si256(ymm0, mask_r);
|
|
ymm1 = _mm256_or_si256(ymm2, ymm1);
|
|
ymm0 = _mm256_or_si256(ymm1, ymm0);
|
|
ymm0 = _mm256_packus_epi32(ymm0, ymm0);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_rgb, _mm256_castsi256_si128(ymm0));
|
|
src_argb += 32;
|
|
dst_rgb += 16;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTORGB565ROW_AVX2
|
|
|
|
#ifdef HAS_ARGBTOARGB1555ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToARGB1555Row_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_argb1555,
|
|
int width) {
|
|
__m256i mask_b = _mm256_set1_epi32(0x0000001f);
|
|
__m256i mask_g = _mm256_set1_epi32(0x000003e0);
|
|
__m256i mask_r = _mm256_set1_epi32(0x00007c00);
|
|
__m256i mask_a = _mm256_set1_epi32(0x00008000);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i ymm2 = _mm256_srli_epi32(ymm0, 6);
|
|
__m256i ymm1 = _mm256_srli_epi32(ymm0, 3);
|
|
__m256i ymm7 = _mm256_srli_epi32(ymm0, 16);
|
|
ymm0 = _mm256_srli_epi32(ymm0, 9);
|
|
ymm2 = _mm256_and_si256(ymm2, mask_g);
|
|
ymm1 = _mm256_and_si256(ymm1, mask_b);
|
|
ymm7 = _mm256_and_si256(ymm7, mask_a);
|
|
ymm0 = _mm256_and_si256(ymm0, mask_r);
|
|
ymm1 = _mm256_or_si256(ymm2, ymm1);
|
|
ymm0 = _mm256_or_si256(ymm7, ymm0);
|
|
ymm0 = _mm256_or_si256(ymm1, ymm0);
|
|
ymm0 = _mm256_packus_epi32(ymm0, ymm0);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_argb1555, _mm256_castsi256_si128(ymm0));
|
|
src_argb += 32;
|
|
dst_argb1555 += 16;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOARGB1555ROW_AVX2
|
|
|
|
#ifdef HAS_ARGBTOARGB4444ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToARGB4444Row_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_argb4444,
|
|
int width) {
|
|
__m256i mask_b = _mm256_set1_epi32(0x0000000f);
|
|
__m256i mask_g = _mm256_set1_epi32(0x000000f0);
|
|
__m256i mask_r = _mm256_set1_epi32(0x00000f00);
|
|
__m256i mask_a = _mm256_set1_epi32(0x0000f000);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i ymm2 = _mm256_srli_epi32(ymm0, 8);
|
|
__m256i ymm1 = _mm256_srli_epi32(ymm0, 4);
|
|
__m256i ymm7 = _mm256_srli_epi32(ymm0, 16);
|
|
ymm0 = _mm256_srli_epi32(ymm0, 12);
|
|
ymm2 = _mm256_and_si256(ymm2, mask_g);
|
|
ymm1 = _mm256_and_si256(ymm1, mask_b);
|
|
ymm7 = _mm256_and_si256(ymm7, mask_a);
|
|
ymm0 = _mm256_and_si256(ymm0, mask_r);
|
|
ymm1 = _mm256_or_si256(ymm2, ymm1);
|
|
ymm0 = _mm256_or_si256(ymm7, ymm0);
|
|
ymm0 = _mm256_or_si256(ymm1, ymm0);
|
|
ymm0 = _mm256_packus_epi32(ymm0, ymm0);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_argb4444, _mm256_castsi256_si128(ymm0));
|
|
src_argb += 32;
|
|
dst_argb4444 += 16;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOARGB4444ROW_AVX2
|
|
|
|
#ifdef HAS_RGB24TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
__m256i ymm_alpha = _mm256_set1_epi32(0xff000000);
|
|
__m256i ymm_shuf = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShuffleMaskRGB24ToARGB[0]));
|
|
__m256i ymm_shuf2 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShuffleMaskRGB24ToARGB[1]));
|
|
|
|
while (width > 0) {
|
|
__m128i xmm0 = _mm_loadu_si128((const __m128i*)src_rgb24);
|
|
__m256i ymm0 = _mm256_castsi128_si256(xmm0);
|
|
ymm0 = _mm256_inserti128_si256(
|
|
ymm0, _mm_loadu_si128((const __m128i*)(src_rgb24 + 12)), 1);
|
|
|
|
__m128i xmm1 = _mm_loadu_si128((const __m128i*)(src_rgb24 + 24));
|
|
__m256i ymm1 = _mm256_castsi128_si256(xmm1);
|
|
ymm1 = _mm256_inserti128_si256(
|
|
ymm1, _mm_loadu_si128((const __m128i*)(src_rgb24 + 36)), 1);
|
|
|
|
__m128i xmm2 = _mm_loadu_si128((const __m128i*)(src_rgb24 + 48));
|
|
__m256i ymm2 = _mm256_castsi128_si256(xmm2);
|
|
ymm2 = _mm256_inserti128_si256(
|
|
ymm2, _mm_loadu_si128((const __m128i*)(src_rgb24 + 60)), 1);
|
|
|
|
__m128i xmm3 = _mm_loadu_si128((const __m128i*)(src_rgb24 + 68));
|
|
__m256i ymm3 = _mm256_castsi128_si256(xmm3);
|
|
ymm3 = _mm256_inserti128_si256(
|
|
ymm3, _mm_loadu_si128((const __m128i*)(src_rgb24 + 80)), 1);
|
|
|
|
ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
|
|
ymm1 = _mm256_shuffle_epi8(ymm1, ymm_shuf);
|
|
ymm2 = _mm256_shuffle_epi8(ymm2, ymm_shuf);
|
|
ymm3 = _mm256_shuffle_epi8(ymm3, ymm_shuf2);
|
|
|
|
ymm0 = _mm256_or_si256(ymm0, ymm_alpha);
|
|
ymm1 = _mm256_or_si256(ymm1, ymm_alpha);
|
|
ymm2 = _mm256_or_si256(ymm2, ymm_alpha);
|
|
ymm3 = _mm256_or_si256(ymm3, ymm_alpha);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, ymm0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 64), ymm2);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 96), ymm3);
|
|
|
|
src_rgb24 += 96;
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RGB24TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_ARGBSHUFFLEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBShuffleRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
const uint8_t* shuffler,
|
|
int width) {
|
|
__m256i control =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)shuffler));
|
|
while (width >= 16) {
|
|
__m256i row = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i row1 = _mm256_loadu_si256((const __m256i*)(src_argb + 32));
|
|
row = _mm256_shuffle_epi8(row, control);
|
|
row1 = _mm256_shuffle_epi8(row1, control);
|
|
_mm256_storeu_si256((__m256i*)dst_argb, row);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), row1);
|
|
src_argb += 64;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBSHUFFLEROW_AVX2
|
|
|
|
#ifdef HAS_ARGBSHUFFLEROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
const uint8_t* shuffler,
|
|
int width) {
|
|
__m512i control =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)shuffler));
|
|
while (width >= 32) {
|
|
__m512i row = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
__m512i row1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
|
|
row = _mm512_shuffle_epi8(row, control);
|
|
row1 = _mm512_shuffle_epi8(row1, control);
|
|
_mm512_storeu_si512((__m512i*)dst_argb, row);
|
|
_mm512_storeu_si512((__m512i*)(dst_argb + 64), row1);
|
|
src_argb += 128;
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t mask = (1u << width) - 1u;
|
|
__mmask16 mask0 = (__mmask16)mask;
|
|
__mmask16 mask1 = (__mmask16)(mask >> 16);
|
|
__m512i row = _mm512_maskz_loadu_epi32(mask0, src_argb);
|
|
__m512i row1 = _mm512_maskz_loadu_epi32(mask1, src_argb + 64);
|
|
row = _mm512_shuffle_epi8(row, control);
|
|
row1 = _mm512_shuffle_epi8(row1, control);
|
|
_mm512_mask_storeu_epi32(dst_argb, mask0, row);
|
|
_mm512_mask_storeu_epi32(dst_argb + 64, mask1, row1);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBSHUFFLEROW_AVX512BW
|
|
|
|
#ifdef HAS_I422TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void I422ToARGBRow_AVX2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
|
|
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
|
|
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
|
|
__m256i ymm_kYToRgb =
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
|
|
__m256i ymm_kYBiasToRgb =
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
|
|
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
|
|
__m256i ymm_alpha = _mm256_set1_epi8((char)0xff);
|
|
__m256i ymm_u_zero = _mm256_setzero_si256();
|
|
|
|
ptrdiff_t offset = src_v - src_u;
|
|
|
|
while (width >= 16) {
|
|
__m128i xmm_u = _mm_loadl_epi64((const __m128i*)src_u);
|
|
__m128i xmm_v = _mm_loadl_epi64((const __m128i*)(src_u + offset));
|
|
src_u += 8;
|
|
|
|
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_u, 0);
|
|
__m256i ymm1 = _mm256_insertf128_si256(ymm_u_zero, xmm_v, 0);
|
|
|
|
ymm3 = _mm256_unpacklo_epi8(ymm3, ymm1);
|
|
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
|
|
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
|
|
|
|
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
|
|
src_y += 16;
|
|
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
|
|
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
|
|
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
|
|
|
|
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
|
|
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
|
|
|
|
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
|
|
ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
|
|
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
|
|
|
|
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
|
|
|
|
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
|
|
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
|
|
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
|
|
|
|
ymm0 = _mm256_srai_epi16(ymm0, 6);
|
|
ymm1 = _mm256_srai_epi16(ymm1, 6);
|
|
ymm2 = _mm256_srai_epi16(ymm2, 6);
|
|
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
|
|
ymm1 = _mm256_packus_epi16(ymm1, ymm1);
|
|
ymm2 = _mm256_packus_epi16(ymm2, ymm2);
|
|
|
|
ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha);
|
|
ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
|
|
|
|
__m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2);
|
|
__m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, argb0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1);
|
|
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_I422TOAR30ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void I422ToAR30Row_AVX2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
|
|
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
|
|
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
|
|
__m256i ymm_kYToRgb =
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
|
|
__m256i ymm_kYBiasToRgb = _mm256_sub_epi16(
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb),
|
|
_mm256_set1_epi16(24));
|
|
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
|
|
__m256i ymm_3ff0 = _mm256_set1_epi16((short)0x3ff0);
|
|
__m256i ymm_c000 = _mm256_set1_epi16((short)0xc000);
|
|
__m256i ymm_u_zero = _mm256_setzero_si256();
|
|
|
|
ptrdiff_t offset = src_v - src_u;
|
|
|
|
while (width >= 16) {
|
|
__m128i xmm_u = _mm_loadl_epi64((const __m128i*)src_u);
|
|
__m128i xmm_v = _mm_loadl_epi64((const __m128i*)(src_u + offset));
|
|
src_u += 8;
|
|
|
|
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_u, 0);
|
|
__m256i ymm1 = _mm256_insertf128_si256(ymm_u_zero, xmm_v, 0);
|
|
|
|
ymm3 = _mm256_unpacklo_epi8(ymm3, ymm1);
|
|
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
|
|
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
|
|
|
|
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
|
|
src_y += 16;
|
|
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
|
|
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
|
|
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
|
|
|
|
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
|
|
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
|
|
|
|
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
|
|
ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
|
|
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
|
|
|
|
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
|
|
|
|
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
|
|
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
|
|
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
|
|
|
|
ymm0 = _mm256_min_epi16(ymm0, ymm_3ff0);
|
|
ymm1 = _mm256_min_epi16(ymm1, ymm_3ff0);
|
|
ymm2 = _mm256_min_epi16(ymm2, ymm_3ff0);
|
|
ymm0 = _mm256_max_epi16(ymm0, ymm_u_zero);
|
|
ymm1 = _mm256_max_epi16(ymm1, ymm_u_zero);
|
|
ymm2 = _mm256_max_epi16(ymm2, ymm_u_zero);
|
|
|
|
__m256i ymm_g_masked = _mm256_and_si256(ymm1, ymm_3ff0);
|
|
__m256i ymm_r_masked = _mm256_and_si256(ymm2, ymm_3ff0);
|
|
ymm0 = _mm256_srli_epi16(ymm0, 4);
|
|
ymm3 = _mm256_slli_epi16(ymm_g_masked, 6);
|
|
ymm0 = _mm256_or_si256(ymm0, ymm3);
|
|
ymm1 = _mm256_srli_epi16(ymm1, 10);
|
|
ymm2 = _mm256_or_si256(ymm_r_masked, ymm_c000);
|
|
ymm2 = _mm256_or_si256(ymm2, ymm1);
|
|
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
|
|
|
|
__m256i ar30_lo = _mm256_unpacklo_epi16(ymm0, ymm2);
|
|
__m256i ar30_hi = _mm256_unpackhi_epi16(ymm0, ymm2);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_ar30, ar30_lo);
|
|
_mm256_storeu_si256((__m256i*)(dst_ar30 + 32), ar30_hi);
|
|
|
|
dst_ar30 += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422TOAR30ROW_AVX2
|
|
|
|
#ifdef HAS_NV12TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void NV12ToARGBRow_AVX2(const uint8_t* src_y,
|
|
const uint8_t* src_uv,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
|
|
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
|
|
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
|
|
__m256i ymm_kYToRgb =
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
|
|
__m256i ymm_kYBiasToRgb =
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
|
|
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
|
|
__m256i ymm_alpha = _mm256_set1_epi8((char)0xff);
|
|
__m256i ymm_u_zero = _mm256_setzero_si256();
|
|
|
|
while (width >= 16) {
|
|
__m128i xmm_uv = _mm_loadu_si128((const __m128i*)src_uv);
|
|
src_uv += 16;
|
|
|
|
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_uv, 0);
|
|
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
|
|
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
|
|
|
|
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
|
|
src_y += 16;
|
|
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
|
|
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
|
|
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
|
|
|
|
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
|
|
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
|
|
|
|
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
|
|
__m256i ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
|
|
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
|
|
|
|
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
|
|
|
|
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
|
|
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
|
|
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
|
|
|
|
ymm0 = _mm256_srai_epi16(ymm0, 6);
|
|
ymm1 = _mm256_srai_epi16(ymm1, 6);
|
|
ymm2 = _mm256_srai_epi16(ymm2, 6);
|
|
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
|
|
ymm1 = _mm256_packus_epi16(ymm1, ymm1);
|
|
ymm2 = _mm256_packus_epi16(ymm2, ymm2);
|
|
|
|
ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha);
|
|
ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
|
|
|
|
__m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2);
|
|
__m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, argb0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1);
|
|
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_NV12TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_NV21TOARGBROW_AVX2
|
|
static const uint8_t kShuffleNV21_AVX2[32] = {
|
|
1u, 0u, 3u, 2u, 5u, 4u, 7u, 6u, 9u, 8u, 11u, 10u, 13u, 12u, 15u, 14u,
|
|
1u, 0u, 3u, 2u, 5u, 4u, 7u, 6u, 9u, 8u, 11u, 10u, 13u, 12u, 15u, 14u};
|
|
|
|
LIBYUV_TARGET_AVX2
|
|
void NV21ToARGBRow_AVX2(const uint8_t* src_y,
|
|
const uint8_t* src_vu,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
|
|
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
|
|
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
|
|
__m256i ymm_kYToRgb =
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
|
|
__m256i ymm_kYBiasToRgb =
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
|
|
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
|
|
__m256i ymm_alpha = _mm256_set1_epi8((char)0xff);
|
|
__m256i ymm_shuf_nv21 = _mm256_loadu_si256((const __m256i*)kShuffleNV21_AVX2);
|
|
__m256i ymm_u_zero = _mm256_setzero_si256();
|
|
|
|
while (width >= 16) {
|
|
__m128i xmm_vu = _mm_loadu_si128((const __m128i*)src_vu);
|
|
src_vu += 16;
|
|
|
|
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_vu, 0);
|
|
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
|
|
ymm3 = _mm256_shuffle_epi8(ymm3, ymm_shuf_nv21);
|
|
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
|
|
|
|
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
|
|
src_y += 16;
|
|
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
|
|
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
|
|
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
|
|
|
|
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
|
|
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
|
|
|
|
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
|
|
__m256i ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
|
|
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
|
|
|
|
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
|
|
|
|
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
|
|
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
|
|
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
|
|
|
|
ymm0 = _mm256_srai_epi16(ymm0, 6);
|
|
ymm1 = _mm256_srai_epi16(ymm1, 6);
|
|
ymm2 = _mm256_srai_epi16(ymm2, 6);
|
|
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
|
|
ymm1 = _mm256_packus_epi16(ymm1, ymm1);
|
|
ymm2 = _mm256_packus_epi16(ymm2, ymm2);
|
|
|
|
ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha);
|
|
ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
|
|
|
|
__m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2);
|
|
__m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, argb0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1);
|
|
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_NV21TOARGBROW_AVX2
|
|
|
|
#if defined(HAS_I422TORGB24ROW_AVX2) || \
|
|
defined(HAS_I422TORGB24ROW_AVX512BW) || \
|
|
defined(HAS_ARGBTORGB24ROW_AVX2) || \
|
|
defined(HAS_ARGBTORGB24ROW_AVX512BW)
|
|
static const uint8_t kShuffleMaskARGBToRGB24[2][16] = {
|
|
{0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u,
|
|
128u},
|
|
{0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u,
|
|
14u}};
|
|
#endif
|
|
|
|
#if defined(HAS_I422TORGB24ROW_AVX2)
|
|
LIBYUV_TARGET_AVX2
|
|
void I422ToRGB24Row_AVX2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
// Constants
|
|
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
|
|
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
|
|
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
|
|
__m256i ymm_kYToRgb =
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
|
|
__m256i ymm_kYBiasToRgb =
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
|
|
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
|
|
|
|
__m256i ymm_shuf0 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[1]));
|
|
__m256i ymm_shuf1 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[0]));
|
|
__m256i ymm_u_zero = _mm256_setzero_si256();
|
|
|
|
ptrdiff_t offset = src_v - src_u;
|
|
|
|
while (width >= 16) {
|
|
// READYUV422_AVX2
|
|
__m128i xmm_u = _mm_loadl_epi64((const __m128i*)src_u);
|
|
__m128i xmm_v = _mm_loadl_epi64((const __m128i*)(src_u + offset));
|
|
src_u += 8;
|
|
|
|
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_u, 0);
|
|
__m256i ymm1 = _mm256_insertf128_si256(ymm_u_zero, xmm_v, 0);
|
|
|
|
ymm3 = _mm256_unpacklo_epi8(ymm3, ymm1);
|
|
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
|
|
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
|
|
|
|
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
|
|
src_y += 16;
|
|
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
|
|
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
|
|
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
|
|
|
|
// YUVTORGB_AVX2
|
|
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
|
|
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
|
|
|
|
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
|
|
ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
|
|
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
|
|
|
|
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
|
|
|
|
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
|
|
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
|
|
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
|
|
|
|
ymm0 = _mm256_srai_epi16(ymm0, 6);
|
|
ymm1 = _mm256_srai_epi16(ymm1, 6);
|
|
ymm2 = _mm256_srai_epi16(ymm2, 6);
|
|
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
|
|
ymm1 = _mm256_packus_epi16(ymm1, ymm1);
|
|
ymm2 = _mm256_packus_epi16(ymm2, ymm2);
|
|
|
|
// STORERGB24_AVX2
|
|
__m256i ymm0_packed = _mm256_unpacklo_epi8(ymm0, ymm1);
|
|
__m256i ymm2_packed = _mm256_unpacklo_epi8(ymm2, ymm2);
|
|
__m256i ymm1_packed = ymm0_packed;
|
|
|
|
ymm0_packed = _mm256_unpacklo_epi16(ymm0_packed, ymm2_packed);
|
|
ymm1_packed = _mm256_unpackhi_epi16(ymm1_packed, ymm2_packed);
|
|
|
|
ymm0_packed = _mm256_shuffle_epi8(ymm0_packed, ymm_shuf0);
|
|
ymm1_packed = _mm256_shuffle_epi8(ymm1_packed, ymm_shuf1);
|
|
|
|
ymm1_packed = _mm256_alignr_epi8(ymm1_packed, ymm0_packed, 0xc);
|
|
|
|
__m128i xmm0_store = _mm256_castsi256_si128(ymm0_packed);
|
|
__m128i xmm1_store = _mm256_castsi256_si128(ymm1_packed);
|
|
__m128i xmm2_store = _mm256_extractf128_si256(ymm0_packed, 1);
|
|
__m128i xmm3_store = _mm256_extractf128_si256(ymm1_packed, 1);
|
|
|
|
xmm0_store = _mm_unpacklo_epi64(xmm0_store, xmm1_store);
|
|
xmm2_store = _mm_alignr_epi8(xmm2_store, xmm1_store, 8);
|
|
|
|
_mm_storeu_si128((__m128i*)dst_rgb24, xmm0_store);
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 16), xmm2_store);
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 32), xmm3_store);
|
|
|
|
dst_rgb24 += 48;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_I422TORGB24ROW_AVX2) ||
|
|
// defined(HAS_I422TORGB24ROW_AVX512BW)
|
|
|
|
#if defined(HAS_I422TORGB24ROW_AVX512VBMI) || \
|
|
defined(HAS_I422TORGB24ROW_AVX512BW)
|
|
#define READYUV422_AVX512BW \
|
|
__m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u); \
|
|
__m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset)); \
|
|
src_u += 16; \
|
|
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \
|
|
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \
|
|
zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val); \
|
|
zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val); \
|
|
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \
|
|
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \
|
|
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \
|
|
__m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y); \
|
|
src_y += 32; \
|
|
__m512i zmm4 = _mm512_castsi256_si512(ymm_y); \
|
|
zmm4 = PermutexvarEpi64(zmm_split_y, zmm4); \
|
|
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \
|
|
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
|
|
|
|
#define READYUV422_MASK_AVX512BW \
|
|
__mmask16 mask_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); \
|
|
__mmask32 mask_y = (__mmask32)((1u << width) - 1u); \
|
|
__m128i xmm_u = _mm_maskz_loadu_epi8(mask_uv, src_u); \
|
|
__m128i xmm_v = _mm_maskz_loadu_epi8(mask_uv, src_u + offset); \
|
|
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \
|
|
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \
|
|
zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val); \
|
|
zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val); \
|
|
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \
|
|
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \
|
|
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \
|
|
__m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y); \
|
|
__m512i zmm4 = _mm512_castsi256_si512(ymm_y); \
|
|
zmm4 = PermutexvarEpi64(zmm_split_y, zmm4); \
|
|
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \
|
|
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
|
|
|
|
#define YUVTORGB_AVX512BW \
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128); \
|
|
zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb); \
|
|
__m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3); \
|
|
__m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3); \
|
|
__m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3); \
|
|
zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb); \
|
|
zmm0 = _mm512_adds_epi16(zmm0, zmm4); \
|
|
zmm1 = _mm512_subs_epi16(zmm4, zmm1); \
|
|
zmm2 = _mm512_adds_epi16(zmm2, zmm4); \
|
|
zmm0 = _mm512_srai_epi16(zmm0, 6); \
|
|
zmm1 = _mm512_srai_epi16(zmm1, 6); \
|
|
zmm2 = _mm512_srai_epi16(zmm2, 6); \
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm0); \
|
|
zmm1 = _mm512_packus_epi16(zmm1, zmm1); \
|
|
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
|
|
|
|
#define STORERGB24_AVX512VBMI \
|
|
__m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); \
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); \
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); \
|
|
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); \
|
|
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), \
|
|
_mm512_castsi512_si256(zmm_dst1)); \
|
|
dst_rgb24 += 96;
|
|
|
|
#define STORERGB24_MASK_AVX512VBMI \
|
|
int dst_bytes = width * 3; \
|
|
__mmask64 mask_dst0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; \
|
|
__mmask32 mask_dst1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; \
|
|
__m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); \
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); \
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); \
|
|
_mm512_mask_storeu_epi8(dst_rgb24, mask_dst0, zmm_dst0); \
|
|
_mm256_mask_storeu_epi8(dst_rgb24 + 64, mask_dst1, \
|
|
_mm512_castsi512_si256(zmm_dst1));
|
|
|
|
#define STORERGB24_AVX512BW \
|
|
__m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1); \
|
|
__m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2); \
|
|
__m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr); \
|
|
__m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr); \
|
|
zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0); \
|
|
zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1); \
|
|
zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12); \
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi); \
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi); \
|
|
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); \
|
|
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), \
|
|
_mm512_castsi512_si256(zmm_dst1)); \
|
|
dst_rgb24 += 96;
|
|
|
|
#define STORERGB24_MASK_AVX512BW \
|
|
int dst_bytes = width * 3; \
|
|
__mmask64 mask_dst0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; \
|
|
__mmask32 mask_dst1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; \
|
|
__m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1); \
|
|
__m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2); \
|
|
__m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr); \
|
|
__m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr); \
|
|
zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0); \
|
|
zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1); \
|
|
zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12); \
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi); \
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi); \
|
|
_mm512_mask_storeu_epi8(dst_rgb24, mask_dst0, zmm_dst0); \
|
|
_mm256_mask_storeu_epi8(dst_rgb24 + 64, mask_dst1, \
|
|
_mm512_castsi512_si256(zmm_dst1));
|
|
#endif // defined(HAS_I422TORGB24ROW_AVX512VBMI) ||
|
|
// defined(HAS_I422TORGB24ROW_AVX512BW)
|
|
|
|
#ifdef HAS_I422TORGB24ROW_AVX512VBMI
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
// Masks
|
|
static const uint8_t kMaskBG[64] = {
|
|
0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05,
|
|
0x45, 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52,
|
|
0x13, 0x53, 0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20,
|
|
0x60, 0x21, 0x61, 0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65,
|
|
0x26, 0x66, 0x27, 0x67, 0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33,
|
|
0x73, 0x34, 0x74, 0x35, 0x75, 0x36, 0x76, 0x37, 0x77};
|
|
static const uint8_t kMaskDST0[64] = {
|
|
0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07,
|
|
0x43, 0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e,
|
|
0x0f, 0x47, 0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52,
|
|
0x16, 0x17, 0x53, 0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d,
|
|
0x56, 0x1e, 0x1f, 0x57, 0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24,
|
|
0x25, 0x62, 0x26, 0x27, 0x63, 0x28, 0x29, 0x64, 0x2a};
|
|
static const uint8_t kMaskDST1[64] = {
|
|
0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70,
|
|
0x32, 0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39,
|
|
0x74, 0x3a, 0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00,
|
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
|
|
|
|
static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2};
|
|
static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
|
|
|
|
// Constants
|
|
__m512i zmm_kUVToB = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
|
|
__m512i zmm_kUVToG = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
|
|
__m512i zmm_kUVToR = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
|
|
__m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
|
|
__m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
|
|
__m512i zmm_128 = _mm512_set1_epi8((char)0x80);
|
|
|
|
__m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG);
|
|
__m512i zmm_mask_DST0 = _mm512_loadu_si512((const __m512i*)kMaskDST0);
|
|
__m512i zmm_mask_DST1 = _mm512_loadu_si512((const __m512i*)kMaskDST1);
|
|
__m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords);
|
|
__m512i zmm_split_y =
|
|
_mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords);
|
|
|
|
ptrdiff_t offset = src_v - src_u;
|
|
|
|
while (width >= 32) {
|
|
READYUV422_AVX512BW
|
|
YUVTORGB_AVX512BW
|
|
STORERGB24_AVX512VBMI
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
READYUV422_MASK_AVX512BW
|
|
YUVTORGB_AVX512BW
|
|
STORERGB24_MASK_AVX512VBMI
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422TORGB24ROW_AVX512VBMI
|
|
|
|
#ifdef HAS_I422TORGB24ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I422ToRGB24Row_AVX512BW(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
static const uint64_t kStitchRGB24_0[8] = {0, 8, 9, 2, 10, 11, 4, 12};
|
|
static const uint64_t kStitchRGB24_1[8] = {13, 6, 14, 15, 0, 0, 0, 0};
|
|
static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2};
|
|
static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
|
|
|
|
// Constants
|
|
__m512i zmm_kUVToB = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
|
|
__m512i zmm_kUVToG = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
|
|
__m512i zmm_kUVToR = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
|
|
__m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
|
|
__m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
|
|
__m512i zmm_128 = _mm512_set1_epi8((char)0x80);
|
|
|
|
__m512i zmm_shuf0 = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[1]));
|
|
__m512i zmm_shuf1 = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[0]));
|
|
__m512i zmm_stitch0 = _mm512_loadu_si512((const __m512i*)kStitchRGB24_0);
|
|
__m512i zmm_stitch1 = _mm512_loadu_si512((const __m512i*)kStitchRGB24_1);
|
|
__m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords);
|
|
__m512i zmm_split_y =
|
|
_mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords);
|
|
|
|
ptrdiff_t offset = src_v - src_u;
|
|
|
|
while (width >= 32) {
|
|
READYUV422_AVX512BW
|
|
YUVTORGB_AVX512BW
|
|
STORERGB24_AVX512BW
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
READYUV422_MASK_AVX512BW
|
|
YUVTORGB_AVX512BW
|
|
STORERGB24_MASK_AVX512BW
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422TORGB24ROW_AVX512BW
|
|
|
|
#if defined(HAS_I422TORGB24ROW_AVX512VBMI) || \
|
|
defined(HAS_I422TORGB24ROW_AVX512BW)
|
|
#undef READYUV422_AVX512BW
|
|
#undef READYUV422_MASK_AVX512BW
|
|
#undef YUVTORGB_AVX512BW
|
|
#undef STORERGB24_AVX512VBMI
|
|
#undef STORERGB24_MASK_AVX512VBMI
|
|
#undef STORERGB24_AVX512BW
|
|
#undef STORERGB24_MASK_AVX512BW
|
|
#endif
|
|
|
|
#ifdef HAS_I422TOAR30ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I422ToAR30Row_AVX512BW(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2};
|
|
static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
|
|
static const uint16_t kPermAR30_0[32] = {
|
|
0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39,
|
|
8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47};
|
|
static const uint16_t kPermAR30_1[32] = {
|
|
16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55,
|
|
24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63};
|
|
|
|
// Constants
|
|
__m512i zmm_kUVToB = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
|
|
__m512i zmm_kUVToG = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
|
|
__m512i zmm_kUVToR = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
|
|
__m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
|
|
__m512i zmm_kYBiasToRgb = _mm512_sub_epi16(
|
|
_mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)),
|
|
_mm512_set1_epi16(24));
|
|
__m512i zmm_128 = _mm512_set1_epi8((char)0x80);
|
|
__m512i zmm_3ff0 = _mm512_set1_epi16((short)0x3ff0);
|
|
__m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00);
|
|
__m512i zmm_c000 = _mm512_set1_epi16((short)0xc000);
|
|
__m512i zmm_zero = _mm512_setzero_si512();
|
|
|
|
__m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0);
|
|
__m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1);
|
|
__m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords);
|
|
__m512i zmm_split_y =
|
|
_mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords);
|
|
|
|
ptrdiff_t offset = src_v - src_u;
|
|
|
|
while (width >= 32) {
|
|
// READYUV422_AVX512BW
|
|
__m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u);
|
|
__m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset));
|
|
src_u += 16;
|
|
|
|
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
|
|
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
|
|
|
|
zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val);
|
|
zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val);
|
|
|
|
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
|
|
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
|
|
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3);
|
|
|
|
__m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y);
|
|
src_y += 32;
|
|
__m512i zmm4 = _mm512_castsi256_si512(ymm_y);
|
|
zmm4 = PermutexvarEpi64(zmm_split_y, zmm4);
|
|
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
|
|
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
|
|
|
|
// YUVTORGB_AVX512BW
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb);
|
|
|
|
__m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3);
|
|
__m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3);
|
|
__m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3);
|
|
|
|
zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb);
|
|
|
|
zmm0 = _mm512_adds_epi16(zmm0, zmm4);
|
|
zmm1 = _mm512_subs_epi16(zmm4, zmm1);
|
|
zmm2 = _mm512_adds_epi16(zmm2, zmm4);
|
|
|
|
// Clamp to [0, 0x3ff0] (1023 * 16)
|
|
zmm0 = _mm512_min_epi16(zmm0, zmm_3ff0);
|
|
zmm1 = _mm512_min_epi16(zmm1, zmm_3ff0);
|
|
zmm2 = _mm512_min_epi16(zmm2, zmm_3ff0);
|
|
zmm0 = _mm512_max_epi16(zmm0, zmm_zero);
|
|
zmm1 = _mm512_max_epi16(zmm1, zmm_zero);
|
|
zmm2 = _mm512_max_epi16(zmm2, zmm_zero);
|
|
|
|
// STOREAR30_AVX512BW:
|
|
zmm0 = _mm512_srli_epi16(zmm0, 4);
|
|
zmm3 = _mm512_srli_epi16(zmm1, 10);
|
|
zmm1 = _mm512_slli_epi16(zmm1, 6);
|
|
zmm0 = _mm512_or_si512(zmm0, _mm512_and_si512(zmm1, zmm_fc00));
|
|
zmm2 = _mm512_or_si512(_mm512_and_si512(zmm2, zmm_3ff0),
|
|
_mm512_andnot_si512(zmm_3ff0, zmm3));
|
|
zmm2 = _mm512_or_si512(zmm2, zmm_c000);
|
|
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi16(zmm0, zmm_perm0, zmm2);
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi16(zmm0, zmm_perm1, zmm2);
|
|
|
|
_mm512_storeu_si512((__m512i*)dst_ar30, zmm_dst0);
|
|
_mm512_storeu_si512((__m512i*)(dst_ar30 + 64), zmm_dst1);
|
|
|
|
dst_ar30 += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 mask_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__mmask32 mask_y = (__mmask32)((1u << width) - 1u);
|
|
__m128i xmm_u = _mm_maskz_loadu_epi8(mask_uv, src_u);
|
|
__m128i xmm_v = _mm_maskz_loadu_epi8(mask_uv, src_u + offset);
|
|
|
|
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
|
|
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
|
|
|
|
zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val);
|
|
zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val);
|
|
|
|
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
|
|
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
|
|
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3);
|
|
|
|
__m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y);
|
|
__m512i zmm4 = _mm512_castsi256_si512(ymm_y);
|
|
zmm4 = PermutexvarEpi64(zmm_split_y, zmm4);
|
|
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
|
|
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
|
|
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb);
|
|
|
|
__m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3);
|
|
__m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3);
|
|
__m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3);
|
|
|
|
zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb);
|
|
|
|
zmm0 = _mm512_adds_epi16(zmm0, zmm4);
|
|
zmm1 = _mm512_subs_epi16(zmm4, zmm1);
|
|
zmm2 = _mm512_adds_epi16(zmm2, zmm4);
|
|
|
|
zmm0 = _mm512_min_epi16(zmm0, zmm_3ff0);
|
|
zmm1 = _mm512_min_epi16(zmm1, zmm_3ff0);
|
|
zmm2 = _mm512_min_epi16(zmm2, zmm_3ff0);
|
|
zmm0 = _mm512_max_epi16(zmm0, zmm_zero);
|
|
zmm1 = _mm512_max_epi16(zmm1, zmm_zero);
|
|
zmm2 = _mm512_max_epi16(zmm2, zmm_zero);
|
|
|
|
zmm0 = _mm512_srli_epi16(zmm0, 4);
|
|
zmm3 = _mm512_srli_epi16(zmm1, 10);
|
|
zmm1 = _mm512_slli_epi16(zmm1, 6);
|
|
zmm0 = _mm512_or_si512(zmm0, _mm512_and_si512(zmm1, zmm_fc00));
|
|
zmm2 = _mm512_or_si512(_mm512_and_si512(zmm2, zmm_3ff0),
|
|
_mm512_andnot_si512(zmm_3ff0, zmm3));
|
|
zmm2 = _mm512_or_si512(zmm2, zmm_c000);
|
|
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi16(zmm0, zmm_perm0, zmm2);
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi16(zmm0, zmm_perm1, zmm2);
|
|
|
|
_mm512_mask_storeu_epi32(dst_ar30, (__mmask16)mask_y, zmm_dst0);
|
|
_mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(mask_y >> 16),
|
|
zmm_dst1);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422TOAR30ROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBBLENDROW_AVX2
|
|
static const uint8_t kShuffleAlpha[16] = {3u, 0x80, 3u, 0x80, 7u, 0x80,
|
|
7u, 0x80, 11u, 0x80, 11u, 0x80,
|
|
15u, 0x80, 15u, 0x80};
|
|
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBBlendRow_AVX2(const uint8_t* src_argb0,
|
|
const uint8_t* src_argb1,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
__m256i shuff =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kShuffleAlpha));
|
|
__m256i ymm_1 = _mm256_set1_epi16(1);
|
|
__m256i mask_lo = _mm256_set1_epi16(0x00ff);
|
|
__m256i mask_hi = _mm256_set1_epi16((short)0xff00);
|
|
__m256i mask_alpha = _mm256_set1_epi32((int)0xff000000);
|
|
|
|
while (width >= 8) {
|
|
__m256i ymm_s0 = _mm256_loadu_si256((const __m256i*)src_argb0);
|
|
__m256i ymm_s1 = _mm256_loadu_si256((const __m256i*)src_argb1);
|
|
src_argb0 += 32;
|
|
src_argb1 += 32;
|
|
|
|
__m256i ymm_a = _mm256_xor_si256(ymm_s0, mask_alpha);
|
|
ymm_a = _mm256_shuffle_epi8(ymm_a, shuff);
|
|
ymm_a = _mm256_add_epi16(ymm_a, ymm_1);
|
|
|
|
__m256i ymm_s1_lo = _mm256_and_si256(ymm_s1, mask_lo);
|
|
__m256i ymm_s1_hi = _mm256_srli_epi16(ymm_s1, 8);
|
|
|
|
ymm_s1_lo = _mm256_mullo_epi16(ymm_s1_lo, ymm_a);
|
|
ymm_s1_hi = _mm256_mullo_epi16(ymm_s1_hi, ymm_a);
|
|
|
|
ymm_s1_lo = _mm256_srli_epi16(ymm_s1_lo, 8);
|
|
ymm_s1_hi = _mm256_and_si256(ymm_s1_hi, mask_hi);
|
|
|
|
__m256i res = _mm256_or_si256(ymm_s0, mask_alpha);
|
|
res = _mm256_adds_epu8(res, ymm_s1_lo);
|
|
res = _mm256_adds_epu8(res, ymm_s1_hi);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, res);
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
|
|
while (width > 0) {
|
|
uint32_t s0 = *(const uint32_t*)src_argb0;
|
|
uint32_t s1 = *(const uint32_t*)src_argb1;
|
|
src_argb0 += 4;
|
|
src_argb1 += 4;
|
|
|
|
uint32_t a = 256 - (s0 >> 24);
|
|
uint32_t b = (((s1 & 0x000000ff) * a) >> 8) + (s0 & 0x000000ff);
|
|
uint32_t g = (((s1 & 0x0000ff00) * a) >> 8) + (s0 & 0x0000ff00);
|
|
uint32_t r = (((s1 & 0x00ff0000) >> 8) * a) + (s0 & 0x00ff0000);
|
|
|
|
b = (b > 255) ? 255 : b;
|
|
g = (g > 0xff00) ? 0xff00 : (g & 0xff00);
|
|
r = (r > 0x00ff0000) ? 0x00ff0000 : (r & 0x00ff0000);
|
|
|
|
*(uint32_t*)dst_argb = 0xff000000 | r | g | b;
|
|
dst_argb += 4;
|
|
--width;
|
|
}
|
|
}
|
|
#endif // HAS_ARGBBLENDROW_AVX2
|
|
|
|
#ifdef HAS_BLENDPLANEROW_SSSE3
|
|
void BlendPlaneRow_SSSE3(const uint8_t* src0,
|
|
const uint8_t* src1,
|
|
const uint8_t* alpha,
|
|
uint8_t* dst,
|
|
int width) {
|
|
__m128i xmm_ff00 = _mm_set1_epi16((short)0xff00);
|
|
__m128i xmm_80 = _mm_set1_epi8((char)0x80);
|
|
__m128i xmm_round = _mm_set1_epi16((short)0x807f);
|
|
|
|
while (width > 0) {
|
|
__m128i xmm_a = _mm_loadl_epi64((const __m128i*)alpha);
|
|
__m128i xmm_s0 = _mm_loadl_epi64((const __m128i*)src0);
|
|
__m128i xmm_s1 = _mm_loadl_epi64((const __m128i*)src1);
|
|
alpha += 8;
|
|
src0 += 8;
|
|
src1 += 8;
|
|
|
|
xmm_a = _mm_unpacklo_epi8(xmm_a, xmm_a);
|
|
xmm_a = _mm_xor_si128(xmm_a, xmm_ff00);
|
|
|
|
__m128i xmm_s = _mm_unpacklo_epi8(xmm_s0, xmm_s1);
|
|
xmm_s = _mm_sub_epi8(xmm_s, xmm_80);
|
|
|
|
__m128i xmm_p = _mm_maddubs_epi16(xmm_a, xmm_s);
|
|
xmm_p = _mm_add_epi16(xmm_p, xmm_round);
|
|
xmm_p = _mm_srli_epi16(xmm_p, 8);
|
|
|
|
__m128i xmm_d = _mm_packus_epi16(xmm_p, xmm_p);
|
|
_mm_storel_epi64((__m128i*)dst, xmm_d);
|
|
dst += 8;
|
|
width -= 8;
|
|
}
|
|
}
|
|
#endif // HAS_BLENDPLANEROW_SSSE3
|
|
|
|
#ifdef HAS_BLENDPLANEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void BlendPlaneRow_AVX2(const uint8_t* src0,
|
|
const uint8_t* src1,
|
|
const uint8_t* alpha,
|
|
uint8_t* dst,
|
|
int width) {
|
|
__m256i ymm_ff00 = _mm256_set1_epi16((short)0xff00);
|
|
__m256i ymm_80 = _mm256_set1_epi8((char)0x80);
|
|
__m256i ymm_round = _mm256_set1_epi16((short)0x807f);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm_a = _mm256_loadu_si256((const __m256i*)alpha);
|
|
__m256i ymm_s0 = _mm256_loadu_si256((const __m256i*)src0);
|
|
__m256i ymm_s1 = _mm256_loadu_si256((const __m256i*)src1);
|
|
alpha += 32;
|
|
src0 += 32;
|
|
src1 += 32;
|
|
|
|
__m256i ymm_a_lo = _mm256_unpacklo_epi8(ymm_a, ymm_a);
|
|
__m256i ymm_a_hi = _mm256_unpackhi_epi8(ymm_a, ymm_a);
|
|
ymm_a_lo = _mm256_xor_si256(ymm_a_lo, ymm_ff00);
|
|
ymm_a_hi = _mm256_xor_si256(ymm_a_hi, ymm_ff00);
|
|
|
|
__m256i ymm_s_lo = _mm256_unpacklo_epi8(ymm_s0, ymm_s1);
|
|
__m256i ymm_s_hi = _mm256_unpackhi_epi8(ymm_s0, ymm_s1);
|
|
ymm_s_lo = _mm256_sub_epi8(ymm_s_lo, ymm_80);
|
|
ymm_s_hi = _mm256_sub_epi8(ymm_s_hi, ymm_80);
|
|
|
|
__m256i ymm_p_lo = _mm256_maddubs_epi16(ymm_a_lo, ymm_s_lo);
|
|
__m256i ymm_p_hi = _mm256_maddubs_epi16(ymm_a_hi, ymm_s_hi);
|
|
|
|
ymm_p_lo = _mm256_add_epi16(ymm_p_lo, ymm_round);
|
|
ymm_p_hi = _mm256_add_epi16(ymm_p_hi, ymm_round);
|
|
|
|
ymm_p_lo = _mm256_srli_epi16(ymm_p_lo, 8);
|
|
ymm_p_hi = _mm256_srli_epi16(ymm_p_hi, 8);
|
|
|
|
__m256i ymm_d = _mm256_packus_epi16(ymm_p_lo, ymm_p_hi);
|
|
_mm256_storeu_si256((__m256i*)dst, ymm_d);
|
|
dst += 32;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_BLENDPLANEROW_AVX2
|
|
|
|
#ifdef HAS_BLENDPLANEROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void BlendPlaneRow_AVX512BW(const uint8_t* src0,
|
|
const uint8_t* src1,
|
|
const uint8_t* alpha,
|
|
uint8_t* dst,
|
|
int width) {
|
|
__m512i zmm_ff00 = _mm512_set1_epi16((short)0xff00);
|
|
__m512i zmm_80 = _mm512_set1_epi8((char)0x80);
|
|
__m512i zmm_round = _mm512_set1_epi16((short)0x807f);
|
|
|
|
while (width >= 64) {
|
|
__m512i zmm_a = _mm512_loadu_si512((const __m512i*)alpha);
|
|
__m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src0);
|
|
__m512i zmm_s1 = _mm512_loadu_si512((const __m512i*)src1);
|
|
alpha += 64;
|
|
src0 += 64;
|
|
src1 += 64;
|
|
|
|
__m512i zmm_a_lo = _mm512_unpacklo_epi8(zmm_a, zmm_a);
|
|
__m512i zmm_a_hi = _mm512_unpackhi_epi8(zmm_a, zmm_a);
|
|
zmm_a_lo = _mm512_xor_si512(zmm_a_lo, zmm_ff00);
|
|
zmm_a_hi = _mm512_xor_si512(zmm_a_hi, zmm_ff00);
|
|
|
|
__m512i zmm_s_lo = _mm512_unpacklo_epi8(zmm_s0, zmm_s1);
|
|
__m512i zmm_s_hi = _mm512_unpackhi_epi8(zmm_s0, zmm_s1);
|
|
zmm_s_lo = _mm512_sub_epi8(zmm_s_lo, zmm_80);
|
|
zmm_s_hi = _mm512_sub_epi8(zmm_s_hi, zmm_80);
|
|
|
|
__m512i zmm_p_lo = _mm512_maddubs_epi16(zmm_a_lo, zmm_s_lo);
|
|
__m512i zmm_p_hi = _mm512_maddubs_epi16(zmm_a_hi, zmm_s_hi);
|
|
|
|
zmm_p_lo = _mm512_add_epi16(zmm_p_lo, zmm_round);
|
|
zmm_p_hi = _mm512_add_epi16(zmm_p_hi, zmm_round);
|
|
|
|
zmm_p_lo = _mm512_srli_epi16(zmm_p_lo, 8);
|
|
zmm_p_hi = _mm512_srli_epi16(zmm_p_hi, 8);
|
|
|
|
__m512i zmm_d = _mm512_packus_epi16(zmm_p_lo, zmm_p_hi);
|
|
_mm512_storeu_si512((__m512i*)dst, zmm_d);
|
|
dst += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
__mmask64 mask = (1ULL << width) - 1ULL;
|
|
__m512i zmm_a = _mm512_maskz_loadu_epi8(mask, alpha);
|
|
__m512i zmm_s0 = _mm512_maskz_loadu_epi8(mask, src0);
|
|
__m512i zmm_s1 = _mm512_maskz_loadu_epi8(mask, src1);
|
|
|
|
__m512i zmm_a_lo = _mm512_unpacklo_epi8(zmm_a, zmm_a);
|
|
__m512i zmm_a_hi = _mm512_unpackhi_epi8(zmm_a, zmm_a);
|
|
zmm_a_lo = _mm512_xor_si512(zmm_a_lo, zmm_ff00);
|
|
zmm_a_hi = _mm512_xor_si512(zmm_a_hi, zmm_ff00);
|
|
|
|
__m512i zmm_s_lo = _mm512_unpacklo_epi8(zmm_s0, zmm_s1);
|
|
__m512i zmm_s_hi = _mm512_unpackhi_epi8(zmm_s0, zmm_s1);
|
|
zmm_s_lo = _mm512_sub_epi8(zmm_s_lo, zmm_80);
|
|
zmm_s_hi = _mm512_sub_epi8(zmm_s_hi, zmm_80);
|
|
|
|
__m512i zmm_p_lo = _mm512_maddubs_epi16(zmm_a_lo, zmm_s_lo);
|
|
__m512i zmm_p_hi = _mm512_maddubs_epi16(zmm_a_hi, zmm_s_hi);
|
|
|
|
zmm_p_lo = _mm512_add_epi16(zmm_p_lo, zmm_round);
|
|
zmm_p_hi = _mm512_add_epi16(zmm_p_hi, zmm_round);
|
|
|
|
zmm_p_lo = _mm512_srli_epi16(zmm_p_lo, 8);
|
|
zmm_p_hi = _mm512_srli_epi16(zmm_p_hi, 8);
|
|
|
|
__m512i zmm_d = _mm512_packus_epi16(zmm_p_lo, zmm_p_hi);
|
|
_mm512_mask_storeu_epi8(dst, mask, zmm_d);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_BLENDPLANEROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBEXTRACTALPHAROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBExtractAlphaRow_AVX512BW(const uint8_t* src_argb,
|
|
uint8_t* dst_a,
|
|
int width) {
|
|
while (width >= 32) {
|
|
__m512i z0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
__m512i z1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
|
|
__m128i a0 = _mm512_cvtepi32_epi8(_mm512_srli_epi32(z0, 24));
|
|
__m128i a1 = _mm512_cvtepi32_epi8(_mm512_srli_epi32(z1, 24));
|
|
_mm_storeu_si128((__m128i*)dst_a, a0);
|
|
_mm_storeu_si128((__m128i*)(dst_a + 16), a1);
|
|
src_argb += 128;
|
|
dst_a += 32;
|
|
width -= 32;
|
|
}
|
|
if (width >= 16) {
|
|
__m512i z0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
__m128i a0 = _mm512_cvtepi32_epi8(_mm512_srli_epi32(z0, 24));
|
|
_mm_storeu_si128((__m128i*)dst_a, a0);
|
|
src_argb += 64;
|
|
dst_a += 16;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 mask = (__mmask16)((1u << width) - 1u);
|
|
__m512i z0 = _mm512_maskz_loadu_epi32(mask, src_argb);
|
|
__m128i a0 = _mm512_cvtepi32_epi8(_mm512_srli_epi32(z0, 24));
|
|
// Use _mm_mask_storeu_epi8 instead of _mm512_mask_cvtepi32_storeu_epi8
|
|
// because LLVM MSan does not instrument @llvm.x86.avx512.mask.pmov.db.mem
|
|
// to unpoison destination shadow memory.
|
|
_mm_mask_storeu_epi8(dst_a, mask, a0);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBEXTRACTALPHAROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBCOPYYTOALPHAROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBCopyYToAlphaRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
|
|
// Store only the alpha byte (every 4th byte) using a byte mask instead of
|
|
// loading dst and blending with _mm512_ternarylogic_epi32. This avoids
|
|
// reading uninitialized destination bytes and avoids
|
|
// @llvm.x86.avx512.pternlog which LLVM MSan does not model bitwise.
|
|
const __mmask64 alpha_mask = 0x8888888888888888ull;
|
|
while (width >= 32) {
|
|
__m512i a0 = _mm512_slli_epi32(
|
|
_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)src)), 24);
|
|
__m512i a1 = _mm512_slli_epi32(
|
|
_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)(src + 16))), 24);
|
|
_mm512_mask_storeu_epi8(dst, alpha_mask, a0);
|
|
_mm512_mask_storeu_epi8(dst + 64, alpha_mask, a1);
|
|
src += 32;
|
|
dst += 128;
|
|
width -= 32;
|
|
}
|
|
if (width >= 16) {
|
|
__m512i a0 = _mm512_slli_epi32(
|
|
_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)src)), 24);
|
|
_mm512_mask_storeu_epi8(dst, alpha_mask, a0);
|
|
src += 16;
|
|
dst += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 mask = (__mmask16)((1u << width) - 1u);
|
|
__mmask64 tail_alpha_mask = alpha_mask >> (64 - width * 4);
|
|
__m512i a0 = _mm512_slli_epi32(
|
|
_mm512_maskz_cvtepu8_epi32(mask, _mm_maskz_loadu_epi8(mask, src)), 24);
|
|
_mm512_mask_storeu_epi8(dst, tail_alpha_mask, a0);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBCOPYYTOALPHAROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBMIRRORROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
|
|
const __m512i rev =
|
|
_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
|
|
const uint8_t* s = src + (ptrdiff_t)width * 4;
|
|
while (width >= 16) {
|
|
s -= 64;
|
|
_mm512_storeu_si512(
|
|
(__m512i*)dst,
|
|
PermutexvarEpi32(rev, _mm512_loadu_si512((const __m512i*)s)));
|
|
dst += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
// Remaining 1 to 15 pixels are at the start of src.
|
|
__mmask16 mask = (__mmask16)((1u << width) - 1u);
|
|
__m512i idx = _mm512_sub_epi32(rev, _mm512_set1_epi32(16 - width));
|
|
__m512i v = _mm512_maskz_loadu_epi32(mask, src);
|
|
_mm512_mask_storeu_epi32(dst, mask, PermutexvarEpi32(idx, v));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBMIRRORROW_AVX512BW
|
|
|
|
#ifdef HAS_RGB24MIRRORROW_AVX512VBMI
|
|
static const uint8_t kPermMirrorRGB24_0[64] = {
|
|
125u, 126u, 127u, 122u, 123u, 124u, 119u, 120u, 121u, 116u, 117u,
|
|
118u, 113u, 114u, 115u, 110u, 111u, 112u, 107u, 108u, 109u, 104u,
|
|
105u, 106u, 101u, 102u, 103u, 98u, 99u, 100u, 95u, 96u, 97u,
|
|
92u, 93u, 94u, 89u, 90u, 91u, 86u, 87u, 88u, 83u, 84u,
|
|
85u, 80u, 81u, 82u, 77u, 78u, 79u, 74u, 75u, 76u, 71u,
|
|
72u, 73u, 68u, 69u, 70u, 65u, 66u, 67u, 62u};
|
|
static const uint8_t kPermMirrorRGB24_1[64] = {
|
|
127u, 128u, 123u, 124u, 125u, 120u, 121u, 122u, 117u, 118u, 119u,
|
|
114u, 115u, 116u, 111u, 112u, 113u, 108u, 109u, 110u, 105u, 106u,
|
|
107u, 102u, 103u, 104u, 99u, 100u, 101u, 96u, 97u, 98u, 93u,
|
|
94u, 95u, 90u, 91u, 92u, 87u, 88u, 89u, 84u, 85u, 86u,
|
|
81u, 82u, 83u, 78u, 79u, 80u, 75u, 76u, 77u, 72u, 73u,
|
|
74u, 69u, 70u, 71u, 66u, 67u, 68u, 63u, 64u};
|
|
static const uint8_t kPermMirrorRGB24_2[64] = {
|
|
65u, 60u, 61u, 62u, 57u, 58u, 59u, 54u, 55u, 56u, 51u, 52u, 53u,
|
|
48u, 49u, 50u, 45u, 46u, 47u, 42u, 43u, 44u, 39u, 40u, 41u, 36u,
|
|
37u, 38u, 33u, 34u, 35u, 30u, 31u, 32u, 27u, 28u, 29u, 24u, 25u,
|
|
26u, 21u, 22u, 23u, 18u, 19u, 20u, 15u, 16u, 17u, 12u, 13u, 14u,
|
|
9u, 10u, 11u, 6u, 7u, 8u, 3u, 4u, 5u, 0u, 1u, 2u};
|
|
|
|
// Byte mask with the low n bits set, clamped to [0, 64].
|
|
static inline __mmask64 RGB24MirrorMask(int n) {
|
|
return n <= 0 ? 0ull : n >= 64 ? ~0ull : (1ull << n) - 1ull;
|
|
}
|
|
|
|
// Mirror 64 RGB24 pixels (3 full zmm vectors) per loop. The remaining 1 to
|
|
// 63 pixels use masked loads and stores that access only valid bytes.
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24,
|
|
uint8_t* dst_rgb24,
|
|
int width) {
|
|
const __m512i idx0 = _mm512_loadu_si512((const __m512i*)kPermMirrorRGB24_0);
|
|
const __m512i idx1 = _mm512_loadu_si512((const __m512i*)kPermMirrorRGB24_1);
|
|
const __m512i idx2 = _mm512_loadu_si512((const __m512i*)kPermMirrorRGB24_2);
|
|
int r = width & 63;
|
|
int w = width - r;
|
|
if (w > 0) {
|
|
const uint8_t* src = src_rgb24 + (ptrdiff_t)width * 3 - 192;
|
|
do {
|
|
__m512i a = _mm512_loadu_si512((const __m512i*)src);
|
|
__m512i b = _mm512_loadu_si512((const __m512i*)(src + 64));
|
|
__m512i c = _mm512_loadu_si512((const __m512i*)(src + 128));
|
|
__m512i d0 = _mm512_permutex2var_epi8(b, idx0, c);
|
|
__m512i d1 = _mm512_permutex2var_epi8(a, idx1, b);
|
|
__m512i d2 = _mm512_permutex2var_epi8(a, idx2, b);
|
|
d1 = _mm512_mask_broadcastb_epi8(d1, (__mmask64)2,
|
|
_mm512_castsi512_si128(c));
|
|
_mm512_storeu_si512((__m512i*)dst_rgb24, d0);
|
|
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 64), d1);
|
|
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 128), d2);
|
|
src -= 192;
|
|
dst_rgb24 += 192;
|
|
w -= 64;
|
|
} while (w > 0);
|
|
}
|
|
if (r) {
|
|
int nb = r * 3;
|
|
int off = 192 - nb;
|
|
__mmask64 m0 = RGB24MirrorMask(nb);
|
|
__mmask64 m1 = RGB24MirrorMask(nb - 64);
|
|
__mmask64 m2 = RGB24MirrorMask(nb - 128);
|
|
__m512i a = _mm512_maskz_loadu_epi8(m0, src_rgb24);
|
|
__m512i b = _mm512_maskz_loadu_epi8(m1, src_rgb24 + 64);
|
|
__m512i c = _mm512_maskz_loadu_epi8(m2, src_rgb24 + 128);
|
|
__m512i t0 = _mm512_sub_epi8(idx0, _mm512_set1_epi8((char)(off - 64)));
|
|
__m512i t1 = _mm512_sub_epi8(idx1, _mm512_set1_epi8((char)off));
|
|
__m512i t2 = _mm512_sub_epi8(idx2, _mm512_set1_epi8((char)off));
|
|
__m512i d0 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t0, b),
|
|
_mm512_movepi8_mask(t0), t0, c);
|
|
__m512i d1 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t1, b),
|
|
_mm512_movepi8_mask(t1), t1, c);
|
|
__m512i d2 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t2, b),
|
|
_mm512_movepi8_mask(t2), t2, c);
|
|
_mm512_mask_storeu_epi8(dst_rgb24, m0, d0);
|
|
_mm512_mask_storeu_epi8(dst_rgb24 + 64, m1, d1);
|
|
_mm512_mask_storeu_epi8(dst_rgb24 + 128, m2, d2);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RGB24MIRRORROW_AVX512VBMI
|
|
|
|
// ===========================================================================
|
|
// Module 1: AYUV to Planar Y / UV / VU (AVX2 / AVX512VBMI)
|
|
// ===========================================================================
|
|
|
|
#ifdef HAS_AYUVTOYROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void AYUVToYRow_AVX2(const uint8_t* src_ayuv, uint8_t* dst_y, int width) {
|
|
const __m256i ymm_mask = _mm256_set1_epi32(0x000000ff);
|
|
while (width >= 16) {
|
|
__m256i ymm0 = _mm256_and_si256(
|
|
_mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)src_ayuv), 16),
|
|
ymm_mask);
|
|
__m256i ymm1 = _mm256_and_si256(
|
|
_mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)(src_ayuv + 32)),
|
|
16),
|
|
ymm_mask);
|
|
src_ayuv += 64;
|
|
ymm0 = _mm256_permute4x64_epi64(_mm256_packus_epi32(ymm0, ymm1), 0xd8);
|
|
__m128i out = _mm_packus_epi16(_mm256_castsi256_si128(ymm0),
|
|
_mm256_extracti128_si256(ymm0, 1));
|
|
_mm_storeu_si128((__m128i*)dst_y, out);
|
|
dst_y += 16;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_AYUVTOYROW_AVX2
|
|
|
|
#if defined(HAS_AYUVTOUVROW_AVX2) || defined(HAS_AYUVTOVUROW_AVX2)
|
|
static const uint8_t kShufAYUVToUV16_Win[16] = {
|
|
1, 5, 0, 4, 9, 13, 8, 12, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
|
|
|
|
static const uint8_t kShufAYUVToVU16_Win[16] = {
|
|
0, 4, 1, 5, 8, 12, 9, 13, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
|
|
|
|
LIBYUV_TARGET_AVX2
|
|
static inline void AYUVToUVOrVURow_Win_AVX2(const uint8_t* src_ayuv,
|
|
int src_stride_ayuv,
|
|
uint8_t* dst_uv,
|
|
const uint8_t* shuf_table,
|
|
int width) {
|
|
const __m256i ymm_shuf =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)shuf_table));
|
|
const __m256i ymm_0101 = _mm256_set1_epi16(0x0101);
|
|
const __m256i ymm_zero = _mm256_setzero_si256();
|
|
while (width >= 16) {
|
|
__m256i s00 = _mm256_shuffle_epi8(
|
|
_mm256_loadu_si256((const __m256i*)src_ayuv), ymm_shuf);
|
|
__m256i s01 = _mm256_shuffle_epi8(
|
|
_mm256_loadu_si256((const __m256i*)(src_ayuv + 32)), ymm_shuf);
|
|
__m256i s10 = _mm256_shuffle_epi8(
|
|
_mm256_loadu_si256((const __m256i*)(src_ayuv + src_stride_ayuv)),
|
|
ymm_shuf);
|
|
__m256i s11 = _mm256_shuffle_epi8(
|
|
_mm256_loadu_si256((const __m256i*)(src_ayuv + src_stride_ayuv + 32)),
|
|
ymm_shuf);
|
|
src_ayuv += 64;
|
|
__m256i p0 =
|
|
_mm256_maddubs_epi16(_mm256_unpacklo_epi64(s00, s01), ymm_0101);
|
|
__m256i p1 =
|
|
_mm256_maddubs_epi16(_mm256_unpacklo_epi64(s10, s11), ymm_0101);
|
|
p0 = _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(p0, p1), 1),
|
|
ymm_zero);
|
|
p0 = _mm256_permute4x64_epi64(p0, 0xd8);
|
|
__m128i out = _mm_packus_epi16(_mm256_castsi256_si128(p0),
|
|
_mm256_extracti128_si256(p0, 1));
|
|
_mm_storeu_si128((__m128i*)dst_uv, out);
|
|
dst_uv += 16;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif
|
|
|
|
#ifdef HAS_AYUVTOUVROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void AYUVToUVRow_AVX2(const uint8_t* src_ayuv,
|
|
int src_stride_ayuv,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
AYUVToUVOrVURow_Win_AVX2(src_ayuv, src_stride_ayuv, dst_uv,
|
|
kShufAYUVToUV16_Win, width);
|
|
}
|
|
#endif // HAS_AYUVTOUVROW_AVX2
|
|
|
|
#ifdef HAS_AYUVTOVUROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void AYUVToVURow_AVX2(const uint8_t* src_ayuv,
|
|
int src_stride_ayuv,
|
|
uint8_t* dst_vu,
|
|
int width) {
|
|
AYUVToUVOrVURow_Win_AVX2(src_ayuv, src_stride_ayuv, dst_vu,
|
|
kShufAYUVToVU16_Win, width);
|
|
}
|
|
#endif // HAS_AYUVTOVUROW_AVX2
|
|
|
|
#ifdef HAS_AYUVTOYROW_AVX512VBMI
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
void AYUVToYRow_AVX512VBMI(const uint8_t* src_ayuv, uint8_t* dst_y, int width) {
|
|
static const uint8_t kPermAYUVToY32[64] = {
|
|
2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62,
|
|
66, 70, 74, 78, 82, 86, 90, 94, 98, 102, 106, 110, 114, 118, 122, 126,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
|
|
const __m512i zmm_perm = _mm512_loadu_si512((const __m512i*)kPermAYUVToY32);
|
|
while (width >= 32) {
|
|
__m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_ayuv);
|
|
__m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_ayuv + 64));
|
|
src_ayuv += 128;
|
|
__m512i out = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1);
|
|
_mm256_storeu_si256((__m256i*)dst_y, _mm512_castsi512_si256(out));
|
|
dst_y += 32;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k1 = (__mmask16)m32;
|
|
__mmask16 k2 = (__mmask16)(m32 >> 16);
|
|
__m512i zmm0 = _mm512_maskz_loadu_epi32(k1, src_ayuv);
|
|
__m512i zmm1 = _mm512_maskz_loadu_epi32(k2, src_ayuv + 64);
|
|
__m512i out = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1);
|
|
_mm256_mask_storeu_epi8(dst_y, (__mmask32)m32, _mm512_castsi512_si256(out));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_AYUVTOYROW_AVX512VBMI
|
|
|
|
#if defined(HAS_AYUVTOUVROW_AVX512VBMI) || defined(HAS_AYUVTOVUROW_AVX512VBMI)
|
|
static const uint8_t kPermAYUVToUV_Win[64] = {
|
|
1, 5, 0, 4, 9, 13, 8, 12, 17, 21, 16, 20, 25,
|
|
29, 24, 28, 33, 37, 32, 36, 41, 45, 40, 44, 49, 53,
|
|
48, 52, 57, 61, 56, 60, 65, 69, 64, 68, 73, 77, 72,
|
|
76, 81, 85, 80, 84, 89, 93, 88, 92, 97, 101, 96, 100,
|
|
105, 109, 104, 108, 113, 117, 112, 116, 121, 125, 120, 124};
|
|
|
|
static const uint8_t kPermAYUVToVU_Win[64] = {
|
|
0, 4, 1, 5, 8, 12, 9, 13, 16, 20, 17, 21, 24,
|
|
28, 25, 29, 32, 36, 33, 37, 40, 44, 41, 45, 48, 52,
|
|
49, 53, 56, 60, 57, 61, 64, 68, 65, 69, 72, 76, 73,
|
|
77, 80, 84, 81, 85, 88, 92, 89, 93, 96, 100, 97, 101,
|
|
104, 108, 105, 109, 112, 116, 113, 117, 120, 124, 121, 125};
|
|
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
static inline void AYUVToUVOrVURow_Win_AVX512VBMI(const uint8_t* src_ayuv,
|
|
int src_stride_ayuv,
|
|
uint8_t* dst_uv,
|
|
const uint8_t* perm_table,
|
|
int width) {
|
|
const __m512i zmm_perm = _mm512_loadu_si512((const __m512i*)perm_table);
|
|
const __m512i zmm_0101 = _mm512_set1_epi16(0x0101);
|
|
const __m512i zmm_zero = _mm512_setzero_si512();
|
|
while (width >= 32) {
|
|
__m512i s00 = _mm512_loadu_si512((const __m512i*)src_ayuv);
|
|
__m512i s01 = _mm512_loadu_si512((const __m512i*)(src_ayuv + 64));
|
|
__m512i s10 =
|
|
_mm512_loadu_si512((const __m512i*)(src_ayuv + src_stride_ayuv));
|
|
__m512i s11 =
|
|
_mm512_loadu_si512((const __m512i*)(src_ayuv + src_stride_ayuv + 64));
|
|
src_ayuv += 128;
|
|
__m512i p0 = _mm512_permutex2var_epi8(s00, zmm_perm, s01);
|
|
__m512i p1 = _mm512_permutex2var_epi8(s10, zmm_perm, s11);
|
|
p0 = _mm512_maddubs_epi16(p0, zmm_0101);
|
|
p1 = _mm512_maddubs_epi16(p1, zmm_0101);
|
|
p0 = _mm512_avg_epu16(_mm512_srli_epi16(_mm512_add_epi16(p0, p1), 1),
|
|
zmm_zero);
|
|
_mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(p0));
|
|
dst_uv += 32;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k1 = (__mmask16)m32;
|
|
__mmask16 k2 = (__mmask16)(m32 >> 16);
|
|
__m512i s00 = _mm512_maskz_loadu_epi32(k1, src_ayuv);
|
|
__m512i s01 = _mm512_maskz_loadu_epi32(k2, src_ayuv + 64);
|
|
__m512i s10 = _mm512_maskz_loadu_epi32(k1, src_ayuv + src_stride_ayuv);
|
|
__m512i s11 = _mm512_maskz_loadu_epi32(k2, src_ayuv + src_stride_ayuv + 64);
|
|
__m512i p0 = _mm512_permutex2var_epi8(s00, zmm_perm, s01);
|
|
__m512i p1 = _mm512_permutex2var_epi8(s10, zmm_perm, s11);
|
|
__mmask16 k_odd = (__mmask16)((width & 1) << (width >> 1));
|
|
p0 = _mm512_mask_or_epi32(p0, k_odd, p0, _mm512_slli_epi32(p0, 8));
|
|
p1 = _mm512_mask_or_epi32(p1, k_odd, p1, _mm512_slli_epi32(p1, 8));
|
|
p0 = _mm512_maddubs_epi16(p0, zmm_0101);
|
|
p1 = _mm512_maddubs_epi16(p1, zmm_0101);
|
|
p0 = _mm512_avg_epu16(_mm512_srli_epi16(_mm512_add_epi16(p0, p1), 1),
|
|
zmm_zero);
|
|
uint32_t out_bytes = (uint32_t)((width + 1) & ~1);
|
|
__mmask32 k_out =
|
|
out_bytes >= 32 ? ~0u : (__mmask32)((1u << out_bytes) - 1u);
|
|
_mm256_mask_storeu_epi8(dst_uv, k_out, _mm512_cvtepi16_epi8(p0));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif
|
|
|
|
#ifdef HAS_AYUVTOUVROW_AVX512VBMI
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
void AYUVToUVRow_AVX512VBMI(const uint8_t* src_ayuv,
|
|
int src_stride_ayuv,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
AYUVToUVOrVURow_Win_AVX512VBMI(src_ayuv, src_stride_ayuv, dst_uv,
|
|
kPermAYUVToUV_Win, width);
|
|
}
|
|
#endif // HAS_AYUVTOUVROW_AVX512VBMI
|
|
|
|
#ifdef HAS_AYUVTOVUROW_AVX512VBMI
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
void AYUVToVURow_AVX512VBMI(const uint8_t* src_ayuv,
|
|
int src_stride_ayuv,
|
|
uint8_t* dst_vu,
|
|
int width) {
|
|
AYUVToUVOrVURow_Win_AVX512VBMI(src_ayuv, src_stride_ayuv, dst_vu,
|
|
kPermAYUVToVU_Win, width);
|
|
}
|
|
#endif // HAS_AYUVTOVUROW_AVX512VBMI
|
|
|
|
// ===========================================================================
|
|
// Module 2: 10-bit / 30-bit AR30 & XR30 Pack / Unpack (AVX2 / AVX512BW)
|
|
// ===========================================================================
|
|
|
|
#ifdef HAS_AR30TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void AR30ToARGBRow_AVX2(const uint8_t* src_ar30,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
const __m256i m0 = _mm256_set1_epi32((int)0xc0000000u);
|
|
const __m256i m1 = _mm256_set1_epi32(0x300000ff);
|
|
const __m256i m2 = _mm256_set1_epi32(0x0c00ff00);
|
|
const __m256i m3 = _mm256_set1_epi32(0x03ff0000);
|
|
while (width >= 8) {
|
|
__m256i v = _mm256_loadu_si256((const __m256i*)src_ar30);
|
|
src_ar30 += 32;
|
|
__m256i out = _mm256_or_si256(
|
|
_mm256_or_si256(_mm256_and_si256(v, m0),
|
|
_mm256_and_si256(_mm256_srli_epi32(v, 2), m1)),
|
|
_mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v, 4), m2),
|
|
_mm256_and_si256(_mm256_srli_epi32(v, 6), m3)));
|
|
_mm256_storeu_si256((__m256i*)dst_argb, out);
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_AR30TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_AR30TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
const __m512i m0 = _mm512_set1_epi32((int)0xc0000000u);
|
|
const __m512i m1 = _mm512_set1_epi32(0x300000ff);
|
|
const __m512i m2 = _mm512_set1_epi32(0x0c00ff00);
|
|
const __m512i m3 = _mm512_set1_epi32(0x03ff0000);
|
|
while (width >= 16) {
|
|
__m512i v = _mm512_loadu_si512((const __m512i*)src_ar30);
|
|
src_ar30 += 64;
|
|
__m512i out = _mm512_or_si512(
|
|
_mm512_or_si512(_mm512_and_si512(v, m0),
|
|
_mm512_and_si512(_mm512_srli_epi32(v, 2), m1)),
|
|
_mm512_or_si512(_mm512_and_si512(_mm512_srli_epi32(v, 4), m2),
|
|
_mm512_and_si512(_mm512_srli_epi32(v, 6), m3)));
|
|
_mm512_storeu_si512((__m512i*)dst_argb, out);
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i v = _mm512_maskz_loadu_epi32(k, src_ar30);
|
|
__m512i out = _mm512_or_si512(
|
|
_mm512_or_si512(_mm512_and_si512(v, m0),
|
|
_mm512_and_si512(_mm512_srli_epi32(v, 2), m1)),
|
|
_mm512_or_si512(_mm512_and_si512(_mm512_srli_epi32(v, 4), m2),
|
|
_mm512_and_si512(_mm512_srli_epi32(v, 6), m3)));
|
|
_mm512_mask_storeu_epi32(dst_argb, k, out);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_AR30TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_AR30TOABGRROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void AR30ToABGRRow_AVX2(const uint8_t* src_ar30,
|
|
uint8_t* dst_abgr,
|
|
int width) {
|
|
static const uint8_t kShufSwapRB16[16] = {2, 1, 0, 3, 6, 5, 4, 7,
|
|
10, 9, 8, 11, 14, 13, 12, 15};
|
|
const __m256i m0 = _mm256_set1_epi32((int)0xc0000000u);
|
|
const __m256i m1 = _mm256_set1_epi32(0x300000ff);
|
|
const __m256i m2 = _mm256_set1_epi32(0x0c00ff00);
|
|
const __m256i m3 = _mm256_set1_epi32(0x03ff0000);
|
|
const __m256i shuf =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kShufSwapRB16));
|
|
while (width >= 8) {
|
|
__m256i v = _mm256_loadu_si256((const __m256i*)src_ar30);
|
|
src_ar30 += 32;
|
|
__m256i out = _mm256_or_si256(
|
|
_mm256_or_si256(_mm256_and_si256(v, m0),
|
|
_mm256_and_si256(_mm256_srli_epi32(v, 2), m1)),
|
|
_mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v, 4), m2),
|
|
_mm256_and_si256(_mm256_srli_epi32(v, 6), m3)));
|
|
out = _mm256_shuffle_epi8(out, shuf);
|
|
_mm256_storeu_si256((__m256i*)dst_abgr, out);
|
|
dst_abgr += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_AR30TOABGRROW_AVX2
|
|
|
|
#ifdef HAS_AR30TOABGRROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30,
|
|
uint8_t* dst_abgr,
|
|
int width) {
|
|
static const uint8_t kShufSwapRB16[16] = {2, 1, 0, 3, 6, 5, 4, 7,
|
|
10, 9, 8, 11, 14, 13, 12, 15};
|
|
const __m512i m0 = _mm512_set1_epi32((int)0xc0000000u);
|
|
const __m512i m1 = _mm512_set1_epi32(0x300000ff);
|
|
const __m512i m2 = _mm512_set1_epi32(0x0c00ff00);
|
|
const __m512i m3 = _mm512_set1_epi32(0x03ff0000);
|
|
const __m512i shuf =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kShufSwapRB16));
|
|
while (width >= 16) {
|
|
__m512i v = _mm512_loadu_si512((const __m512i*)src_ar30);
|
|
src_ar30 += 64;
|
|
__m512i out = _mm512_or_si512(
|
|
_mm512_or_si512(_mm512_and_si512(v, m0),
|
|
_mm512_and_si512(_mm512_srli_epi32(v, 2), m1)),
|
|
_mm512_or_si512(_mm512_and_si512(_mm512_srli_epi32(v, 4), m2),
|
|
_mm512_and_si512(_mm512_srli_epi32(v, 6), m3)));
|
|
out = _mm512_shuffle_epi8(out, shuf);
|
|
_mm512_storeu_si512((__m512i*)dst_abgr, out);
|
|
dst_abgr += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i v = _mm512_maskz_loadu_epi32(k, src_ar30);
|
|
__m512i out = _mm512_or_si512(
|
|
_mm512_or_si512(_mm512_and_si512(v, m0),
|
|
_mm512_and_si512(_mm512_srli_epi32(v, 2), m1)),
|
|
_mm512_or_si512(_mm512_and_si512(_mm512_srli_epi32(v, 4), m2),
|
|
_mm512_and_si512(_mm512_srli_epi32(v, 6), m3)));
|
|
out = _mm512_shuffle_epi8(out, shuf);
|
|
_mm512_mask_storeu_epi32(dst_abgr, k, out);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_AR30TOABGRROW_AVX512BW
|
|
|
|
#ifdef HAS_AR30TOAB30ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void AR30ToAB30Row_AVX2(const uint8_t* src_ar30,
|
|
uint8_t* dst_ab30,
|
|
int width) {
|
|
const __m256i m0 = _mm256_set1_epi32((int)0xc00ffc00u);
|
|
const __m256i m1 = _mm256_set1_epi32(0x3ff00000);
|
|
const __m256i m2 = _mm256_set1_epi32(0x000003ff);
|
|
while (width >= 8) {
|
|
__m256i v = _mm256_loadu_si256((const __m256i*)src_ar30);
|
|
src_ar30 += 32;
|
|
__m256i out = _mm256_or_si256(
|
|
_mm256_and_si256(v, m0),
|
|
_mm256_or_si256(_mm256_and_si256(_mm256_slli_epi32(v, 20), m1),
|
|
_mm256_and_si256(_mm256_srli_epi32(v, 20), m2)));
|
|
_mm256_storeu_si256((__m256i*)dst_ab30, out);
|
|
dst_ab30 += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_AR30TOAB30ROW_AVX2
|
|
|
|
#ifdef HAS_AR30TOAB30ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30,
|
|
uint8_t* dst_ab30,
|
|
int width) {
|
|
const __m512i m0 = _mm512_set1_epi32((int)0xc00ffc00u);
|
|
const __m512i m1 = _mm512_set1_epi32(0x3ff00000);
|
|
const __m512i m2 = _mm512_set1_epi32(0x000003ff);
|
|
while (width >= 16) {
|
|
__m512i v = _mm512_loadu_si512((const __m512i*)src_ar30);
|
|
src_ar30 += 64;
|
|
__m512i out = _mm512_or_si512(
|
|
_mm512_and_si512(v, m0),
|
|
_mm512_or_si512(_mm512_and_si512(_mm512_slli_epi32(v, 20), m1),
|
|
_mm512_and_si512(_mm512_srli_epi32(v, 20), m2)));
|
|
_mm512_storeu_si512((__m512i*)dst_ab30, out);
|
|
dst_ab30 += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i v = _mm512_maskz_loadu_epi32(k, src_ar30);
|
|
__m512i out = _mm512_or_si512(
|
|
_mm512_and_si512(v, m0),
|
|
_mm512_or_si512(_mm512_and_si512(_mm512_slli_epi32(v, 20), m1),
|
|
_mm512_and_si512(_mm512_srli_epi32(v, 20), m2)));
|
|
_mm512_mask_storeu_epi32(dst_ab30, k, out);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_AR30TOAB30ROW_AVX512BW
|
|
|
|
#ifdef HAS_MERGEXR30ROW_10_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void MergeXR30Row_10_AVX512BW(const uint16_t* src_r,
|
|
const uint16_t* src_g,
|
|
const uint16_t* src_b,
|
|
uint8_t* dst_ar30,
|
|
int depth,
|
|
int width) {
|
|
(void)depth;
|
|
const __m512i max10 = _mm512_set1_epi32(1023);
|
|
const __m512i alpha = _mm512_set1_epi32((int)0xc0000000u);
|
|
while (width >= 16) {
|
|
__m512i r = _mm512_min_epu32(
|
|
_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_r)),
|
|
max10);
|
|
__m512i g = _mm512_min_epu32(
|
|
_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_g)),
|
|
max10);
|
|
__m512i b = _mm512_min_epu32(
|
|
_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_b)),
|
|
max10);
|
|
src_r += 16;
|
|
src_g += 16;
|
|
src_b += 16;
|
|
__m512i out = _mm512_or_si512(
|
|
_mm512_or_si512(_mm512_slli_epi32(r, 20), _mm512_slli_epi32(g, 10)),
|
|
_mm512_or_si512(b, alpha));
|
|
_mm512_storeu_si512((__m512i*)dst_ar30, out);
|
|
dst_ar30 += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i r = _mm512_min_epu32(
|
|
_mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_r)), max10);
|
|
__m512i g = _mm512_min_epu32(
|
|
_mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_g)), max10);
|
|
__m512i b = _mm512_min_epu32(
|
|
_mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_b)), max10);
|
|
__m512i out = _mm512_or_si512(
|
|
_mm512_or_si512(_mm512_slli_epi32(r, 20), _mm512_slli_epi32(g, 10)),
|
|
_mm512_or_si512(b, alpha));
|
|
_mm512_mask_storeu_epi32(dst_ar30, k, out);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGEXR30ROW_10_AVX512BW
|
|
|
|
#if defined(HAS_ARGBTOAR30ROW_AVX512BW) || defined(HAS_ABGRTOAR30ROW_AVX512BW)
|
|
static const uint8_t kShuffleRB30_Win[16] = {128u, 0u, 128u, 2u, 128u, 4u,
|
|
128u, 6u, 128u, 8u, 128u, 10u,
|
|
128u, 12u, 128u, 14u};
|
|
|
|
static const uint8_t kShuffleBR30_Win[16] = {128u, 2u, 128u, 0u, 128u, 6u,
|
|
128u, 4u, 128u, 10u, 128u, 8u,
|
|
128u, 14u, 128u, 12u};
|
|
|
|
LIBYUV_TARGET_AVX512BW
|
|
static inline void ARGBOrABGRToAR30Row_Win_AVX512BW(const uint8_t* src,
|
|
uint8_t* dst,
|
|
const uint8_t* shuf_rb,
|
|
int width) {
|
|
const __m512i zmm_shuf =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)shuf_rb));
|
|
const __m512i zmm_mul_rb = _mm512_set1_epi32(1028 * 16 * 65536 + 1028);
|
|
const __m512i zmm_mask_rb = _mm512_set1_epi32(0x3ff003ff);
|
|
const __m512i zmm_mask_ag = _mm512_set1_epi32((int)0xc000ff00u);
|
|
const __m512i zmm_mul_ag = _mm512_set1_epi32(64 * 65536 + 1028);
|
|
while (width >= 16) {
|
|
__m512i v = _mm512_loadu_si512((const __m512i*)src);
|
|
src += 64;
|
|
__m512i rb = _mm512_and_si512(
|
|
_mm512_mulhi_epu16(_mm512_shuffle_epi8(v, zmm_shuf), zmm_mul_rb),
|
|
zmm_mask_rb);
|
|
__m512i ag = _mm512_slli_epi32(
|
|
_mm512_mulhi_epu16(_mm512_and_si512(v, zmm_mask_ag), zmm_mul_ag), 10);
|
|
_mm512_storeu_si512((__m512i*)dst, _mm512_or_si512(ag, rb));
|
|
dst += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i v = _mm512_maskz_loadu_epi32(k, src);
|
|
__m512i rb = _mm512_and_si512(
|
|
_mm512_mulhi_epu16(_mm512_shuffle_epi8(v, zmm_shuf), zmm_mul_rb),
|
|
zmm_mask_rb);
|
|
__m512i ag = _mm512_slli_epi32(
|
|
_mm512_mulhi_epu16(_mm512_and_si512(v, zmm_mask_ag), zmm_mul_ag), 10);
|
|
_mm512_mask_storeu_epi32(dst, k, _mm512_or_si512(ag, rb));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif
|
|
|
|
#ifdef HAS_ARGBTOAR30ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBToAR30Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
|
|
ARGBOrABGRToAR30Row_Win_AVX512BW(src, dst, kShuffleRB30_Win, width);
|
|
}
|
|
#endif // HAS_ARGBTOAR30ROW_AVX512BW
|
|
|
|
#ifdef HAS_ABGRTOAR30ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ABGRToAR30Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
|
|
ARGBOrABGRToAR30Row_Win_AVX512BW(src, dst, kShuffleBR30_Win, width);
|
|
}
|
|
#endif // HAS_ABGRTOAR30ROW_AVX512BW
|
|
|
|
// ===========================================================================
|
|
// Module 3: Planar / Channel Pack & Mirror / Interpolate_16 (AVX512VBMI / BW)
|
|
// ===========================================================================
|
|
|
|
#ifdef HAS_MERGERGBROW_AVX512VBMI
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
void MergeRGBRow_AVX512VBMI(const uint8_t* src_r,
|
|
const uint8_t* src_g,
|
|
const uint8_t* src_b,
|
|
uint8_t* dst_rgb,
|
|
int width) {
|
|
static const uint8_t kPermMergeRGB64_0[64] = {
|
|
0x00, 0x20, 0x40, 0x01, 0x21, 0x41, 0x02, 0x22, 0x42, 0x03, 0x23,
|
|
0x43, 0x04, 0x24, 0x44, 0x05, 0x25, 0x45, 0x06, 0x26, 0x46, 0x07,
|
|
0x27, 0x47, 0x08, 0x28, 0x48, 0x09, 0x29, 0x49, 0x0a, 0x2a, 0x4a,
|
|
0x0b, 0x2b, 0x4b, 0x0c, 0x2c, 0x4c, 0x0d, 0x2d, 0x4d, 0x0e, 0x2e,
|
|
0x4e, 0x0f, 0x2f, 0x4f, 0x10, 0x30, 0x50, 0x11, 0x31, 0x51, 0x12,
|
|
0x32, 0x52, 0x13, 0x33, 0x53, 0x14, 0x34, 0x54, 0x15};
|
|
static const uint8_t kPermMergeRGB64_1[64] = {
|
|
0x35, 0x55, 0x16, 0x36, 0x56, 0x17, 0x37, 0x57, 0x18, 0x38, 0x58,
|
|
0x19, 0x39, 0x59, 0x1a, 0x3a, 0x5a, 0x1b, 0x3b, 0x5b, 0x1c, 0x3c,
|
|
0x5c, 0x1d, 0x3d, 0x5d, 0x1e, 0x3e, 0x5e, 0x1f, 0x3f, 0x5f, 0x60,
|
|
0x00, 0x20, 0x61, 0x01, 0x21, 0x62, 0x02, 0x22, 0x63, 0x03, 0x23,
|
|
0x64, 0x04, 0x24, 0x65, 0x05, 0x25, 0x66, 0x06, 0x26, 0x67, 0x07,
|
|
0x27, 0x68, 0x08, 0x28, 0x69, 0x09, 0x29, 0x6a, 0x0a};
|
|
static const uint8_t kPermMergeRGB64_2[64] = {
|
|
0x6a, 0x2b, 0x4b, 0x6b, 0x2c, 0x4c, 0x6c, 0x2d, 0x4d, 0x6d, 0x2e,
|
|
0x4e, 0x6e, 0x2f, 0x4f, 0x6f, 0x30, 0x50, 0x70, 0x31, 0x51, 0x71,
|
|
0x32, 0x52, 0x72, 0x33, 0x53, 0x73, 0x34, 0x54, 0x74, 0x35, 0x55,
|
|
0x75, 0x36, 0x56, 0x76, 0x37, 0x57, 0x77, 0x38, 0x58, 0x78, 0x39,
|
|
0x59, 0x79, 0x3a, 0x5a, 0x7a, 0x3b, 0x5b, 0x7b, 0x3c, 0x5c, 0x7c,
|
|
0x3d, 0x5d, 0x7d, 0x3e, 0x5e, 0x7e, 0x3f, 0x5f, 0x7f};
|
|
const __m512i m_rgb0 = _mm512_loadu_si512((const __m512i*)kPermMergeRGB64_0);
|
|
const __m512i m_rgb1 = _mm512_loadu_si512((const __m512i*)kPermMergeRGB64_1);
|
|
const __m512i m_rgb2 = _mm512_loadu_si512((const __m512i*)kPermMergeRGB64_2);
|
|
while (width >= 64) {
|
|
__m512i rg0 = _mm512_inserti64x4(
|
|
_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)src_r)),
|
|
_mm256_loadu_si256((const __m256i*)src_g), 1);
|
|
__m512i br = _mm512_inserti64x4(
|
|
_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)src_b)),
|
|
_mm256_loadu_si256((const __m256i*)(src_r + 32)), 1);
|
|
__m512i gb1 = _mm512_inserti64x4(
|
|
_mm512_castsi256_si512(
|
|
_mm256_loadu_si256((const __m256i*)(src_g + 32))),
|
|
_mm256_loadu_si256((const __m256i*)(src_b + 32)), 1);
|
|
src_r += 64;
|
|
src_g += 64;
|
|
src_b += 64;
|
|
__m512i d0 = _mm512_permutex2var_epi8(rg0, m_rgb0, br);
|
|
__m512i d1 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(rg0, m_rgb1, br),
|
|
0xb6db6db600000000ull, m_rgb1, gb1);
|
|
__m512i d2 = _mm512_permutex2var_epi8(br, m_rgb2, gb1);
|
|
_mm512_storeu_si512((__m512i*)dst_rgb, d0);
|
|
_mm512_storeu_si512((__m512i*)(dst_rgb + 64), d1);
|
|
_mm512_storeu_si512((__m512i*)(dst_rgb + 128), d2);
|
|
dst_rgb += 192;
|
|
width -= 64;
|
|
}
|
|
if (width >= 32) {
|
|
__m512i rg = _mm512_inserti64x4(
|
|
_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)src_r)),
|
|
_mm256_loadu_si256((const __m256i*)src_g), 1);
|
|
__m512i b =
|
|
_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)src_b));
|
|
src_r += 32;
|
|
src_g += 32;
|
|
src_b += 32;
|
|
__m512i d0 = _mm512_permutex2var_epi8(rg, m_rgb0, b);
|
|
__m512i d1 = _mm512_permutex2var_epi8(rg, m_rgb1, b);
|
|
_mm512_storeu_si512((__m512i*)dst_rgb, d0);
|
|
_mm256_storeu_si256((__m256i*)(dst_rgb + 64), _mm512_castsi512_si256(d1));
|
|
dst_rgb += 96;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask32 k_in = (__mmask32)((1u << width) - 1u);
|
|
__m512i rg = _mm512_inserti64x4(
|
|
_mm512_castsi256_si512(_mm256_maskz_loadu_epi8(k_in, src_r)),
|
|
_mm256_maskz_loadu_epi8(k_in, src_g), 1);
|
|
__m512i b = _mm512_castsi256_si512(_mm256_maskz_loadu_epi8(k_in, src_b));
|
|
__m512i d0 = _mm512_permutex2var_epi8(rg, m_rgb0, b);
|
|
__m512i d1 = _mm512_permutex2var_epi8(rg, m_rgb1, b);
|
|
int dst_bytes = width * 3;
|
|
__mmask64 k0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull;
|
|
__mmask32 k1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u;
|
|
_mm512_mask_storeu_epi8(dst_rgb, k0, d0);
|
|
_mm256_mask_storeu_epi8(dst_rgb + 64, k1, _mm512_castsi512_si256(d1));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGERGBROW_AVX512VBMI
|
|
|
|
#ifdef HAS_RAWTORGBAROW_AVX512VBMI
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
void RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw,
|
|
uint8_t* dst_rgba,
|
|
int width) {
|
|
static const uint8_t kPermRAWToRGBA0[64] = {
|
|
0, 2, 1, 0, 0, 5, 4, 3, 0, 8, 7, 6, 0, 11, 10, 9,
|
|
0, 14, 13, 12, 0, 17, 16, 15, 0, 20, 19, 18, 0, 23, 22, 21,
|
|
0, 26, 25, 24, 0, 29, 28, 27, 0, 32, 31, 30, 0, 35, 34, 33,
|
|
0, 38, 37, 36, 0, 41, 40, 39, 0, 44, 43, 42, 0, 47, 46, 45};
|
|
static const uint8_t kPermRAWToRGBA1[64] = {
|
|
0, 50, 49, 48, 0, 53, 52, 51, 0, 56, 55, 54, 0, 59, 58, 57,
|
|
0, 62, 61, 60, 0, 65, 64, 63, 0, 68, 67, 66, 0, 71, 70, 69,
|
|
0, 74, 73, 72, 0, 77, 76, 75, 0, 80, 79, 78, 0, 83, 82, 81,
|
|
0, 86, 85, 84, 0, 89, 88, 87, 0, 92, 91, 90, 0, 95, 94, 93};
|
|
const __m512i p0 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGBA0);
|
|
const __m512i p1 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGBA1);
|
|
const __m512i alpha = _mm512_set1_epi32(0x000000ff);
|
|
while (width >= 32) {
|
|
__m512i s0 = _mm512_loadu_si512((const __m512i*)src_raw);
|
|
__m512i s1 = _mm512_castsi256_si512(
|
|
_mm256_loadu_si256((const __m256i*)(src_raw + 64)));
|
|
src_raw += 96;
|
|
__m512i d0 = _mm512_or_si512(PermutexvarEpi8(p0, s0), alpha);
|
|
__m512i d1 = _mm512_or_si512(_mm512_permutex2var_epi8(s0, p1, s1), alpha);
|
|
_mm512_storeu_si512((__m512i*)dst_rgba, d0);
|
|
_mm512_storeu_si512((__m512i*)(dst_rgba + 64), d1);
|
|
dst_rgba += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
int src_bytes = width * 3;
|
|
__mmask64 ks0 = src_bytes >= 64 ? ~0ull : (1ull << src_bytes) - 1ull;
|
|
__mmask32 ks1 = src_bytes > 64 ? (1u << (src_bytes - 64)) - 1u : 0u;
|
|
__m512i s0 = _mm512_maskz_loadu_epi8(ks0, src_raw);
|
|
__m512i s1 =
|
|
_mm512_castsi256_si512(_mm256_maskz_loadu_epi8(ks1, src_raw + 64));
|
|
__m512i d0 = _mm512_or_si512(PermutexvarEpi8(p0, s0), alpha);
|
|
__m512i d1 = _mm512_or_si512(_mm512_permutex2var_epi8(s0, p1, s1), alpha);
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
_mm512_mask_storeu_epi32(dst_rgba, (__mmask16)m32, d0);
|
|
_mm512_mask_storeu_epi32(dst_rgba + 64, (__mmask16)(m32 >> 16), d1);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RAWTORGBAROW_AVX512VBMI
|
|
|
|
#ifdef HAS_SWAPUVROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void SwapUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
|
|
static const uint8_t kSwapUV16[16] = {1, 0, 3, 2, 5, 4, 7, 6,
|
|
9, 8, 11, 10, 13, 12, 15, 14};
|
|
const __m512i swap =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kSwapUV16));
|
|
while (width >= 32) {
|
|
__m512i v = _mm512_loadu_si512((const __m512i*)src_uv);
|
|
src_uv += 64;
|
|
_mm512_storeu_si512((__m512i*)dst_vu, _mm512_shuffle_epi8(v, swap));
|
|
dst_vu += 64;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask32 k = (__mmask32)((1u << width) - 1u);
|
|
__m512i v = _mm512_maskz_loadu_epi16(k, src_uv);
|
|
_mm512_mask_storeu_epi16(dst_vu, k, _mm512_shuffle_epi8(v, swap));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SWAPUVROW_AVX512BW
|
|
|
|
#ifdef HAS_MIRRORROW_16_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width) {
|
|
static const uint16_t kMirror16Idx[32] = {
|
|
31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16,
|
|
15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
|
|
const __m512i rev = _mm512_loadu_si512((const __m512i*)kMirror16Idx);
|
|
src += width;
|
|
while (width >= 32) {
|
|
src -= 32;
|
|
__m512i v = _mm512_loadu_si512((const __m512i*)src);
|
|
_mm512_storeu_si512((__m512i*)dst, PermutexvarEpi16(rev, v));
|
|
dst += 32;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask32 k = (__mmask32)((1u << width) - 1u);
|
|
__m512i idx = _mm512_sub_epi16(rev, _mm512_set1_epi16((short)(32 - width)));
|
|
__m512i v = _mm512_maskz_loadu_epi16(k, src - width);
|
|
_mm512_mask_storeu_epi16(dst, k, PermutexvarEpi16(idx, v));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MIRRORROW_16_AVX512BW
|
|
|
|
#ifdef HAS_HALFMERGEUVROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void HalfMergeUVRow_AVX512BW(const uint8_t* src_u,
|
|
int src_stride_u,
|
|
const uint8_t* src_v,
|
|
int src_stride_v,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
const __m512i c0101 = _mm512_set1_epi16(0x0101);
|
|
const __m512i czero = _mm512_setzero_si512();
|
|
const __m512i c00ff = _mm512_set1_epi16(0x00ff);
|
|
while (width >= 64) {
|
|
__m512i u0 = _mm512_loadu_si512((const __m512i*)src_u);
|
|
__m512i u1 = _mm512_loadu_si512((const __m512i*)(src_u + src_stride_u));
|
|
__m512i v0 = _mm512_loadu_si512((const __m512i*)src_v);
|
|
__m512i v1 = _mm512_loadu_si512((const __m512i*)(src_v + src_stride_v));
|
|
src_u += 64;
|
|
src_v += 64;
|
|
u0 = _mm512_avg_epu16(
|
|
_mm512_srli_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(u0, c0101),
|
|
_mm512_maddubs_epi16(u1, c0101)),
|
|
1),
|
|
czero);
|
|
v0 = _mm512_avg_epu16(
|
|
_mm512_srli_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(v0, c0101),
|
|
_mm512_maddubs_epi16(v1, c0101)),
|
|
1),
|
|
czero);
|
|
_mm512_storeu_si512((__m512i*)dst_uv,
|
|
_mm512_or_si512(u0, _mm512_slli_epi16(v0, 8)));
|
|
dst_uv += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
__mmask64 k = width >= 64 ? ~0ull : (1ull << width) - 1ull;
|
|
__m512i u0 = _mm512_maskz_loadu_epi8(k, src_u);
|
|
__m512i u1 = _mm512_maskz_loadu_epi8(k, src_u + src_stride_u);
|
|
__m512i v0 = _mm512_maskz_loadu_epi8(k, src_v);
|
|
__m512i v1 = _mm512_maskz_loadu_epi8(k, src_v + src_stride_v);
|
|
__mmask32 k_odd = (__mmask32)((uint32_t)(width & 1) << (width >> 1));
|
|
u0 = _mm512_mask_mullo_epi16(u0, k_odd, _mm512_and_si512(u0, c00ff), c0101);
|
|
u1 = _mm512_mask_mullo_epi16(u1, k_odd, _mm512_and_si512(u1, c00ff), c0101);
|
|
v0 = _mm512_mask_mullo_epi16(v0, k_odd, _mm512_and_si512(v0, c00ff), c0101);
|
|
v1 = _mm512_mask_mullo_epi16(v1, k_odd, _mm512_and_si512(v1, c00ff), c0101);
|
|
u0 = _mm512_avg_epu16(
|
|
_mm512_srli_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(u0, c0101),
|
|
_mm512_maddubs_epi16(u1, c0101)),
|
|
1),
|
|
czero);
|
|
v0 = _mm512_avg_epu16(
|
|
_mm512_srli_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(v0, c0101),
|
|
_mm512_maddubs_epi16(v1, c0101)),
|
|
1),
|
|
czero);
|
|
int out_words = (width + 1) >> 1;
|
|
__mmask32 k_out =
|
|
out_words >= 32 ? ~0u : (__mmask32)((1u << out_words) - 1u);
|
|
_mm512_mask_storeu_epi16(dst_uv, k_out,
|
|
_mm512_or_si512(u0, _mm512_slli_epi16(v0, 8)));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_HALFMERGEUVROW_AVX512BW
|
|
|
|
#ifdef HAS_INTERPOLATEROW_16_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void InterpolateRow_16_AVX512BW(uint16_t* dst_ptr,
|
|
const uint16_t* src_ptr,
|
|
ptrdiff_t src_stride,
|
|
int width,
|
|
int source_y_fraction) {
|
|
int y1 = source_y_fraction;
|
|
int y0 = 256 - y1;
|
|
const uint16_t* src_ptr1 = src_ptr + src_stride;
|
|
__m512i zmm_y = _mm512_set1_epi32((y1 << 16) | y0);
|
|
__m512i zmm_8000 = _mm512_set1_epi16((short)0x8000);
|
|
__m512i zmm_round = _mm512_set1_epi32(8388736); // 0x800000 + 128
|
|
int i;
|
|
|
|
if (y1 == 0) {
|
|
for (i = 0; i <= width - 32; i += 32) {
|
|
_mm512_storeu_si512((__m512i*)(dst_ptr + i),
|
|
_mm512_loadu_si512((const __m512i*)(src_ptr + i)));
|
|
}
|
|
if (i < width) {
|
|
__mmask32 mask = (__mmask32)((1u << (width - i)) - 1u);
|
|
_mm512_mask_storeu_epi16(dst_ptr + i, mask,
|
|
_mm512_maskz_loadu_epi16(mask, src_ptr + i));
|
|
}
|
|
} else if (y1 == 128) {
|
|
for (i = 0; i <= width - 32; i += 32) {
|
|
__m512i row0 = _mm512_loadu_si512((const __m512i*)(src_ptr + i));
|
|
__m512i row1 = _mm512_loadu_si512((const __m512i*)(src_ptr1 + i));
|
|
_mm512_storeu_si512((__m512i*)(dst_ptr + i),
|
|
_mm512_avg_epu16(row0, row1));
|
|
}
|
|
if (i < width) {
|
|
__mmask32 mask = (__mmask32)((1u << (width - i)) - 1u);
|
|
__m512i row0 = _mm512_maskz_loadu_epi16(mask, src_ptr + i);
|
|
__m512i row1 = _mm512_maskz_loadu_epi16(mask, src_ptr1 + i);
|
|
_mm512_mask_storeu_epi16(dst_ptr + i, mask, _mm512_avg_epu16(row0, row1));
|
|
}
|
|
} else {
|
|
for (i = 0; i <= width - 32; i += 32) {
|
|
__m512i row0 = _mm512_loadu_si512((const __m512i*)(src_ptr + i));
|
|
__m512i row1 = _mm512_loadu_si512((const __m512i*)(src_ptr1 + i));
|
|
__m512i row0l = _mm512_unpacklo_epi16(row0, row1);
|
|
__m512i row0h = _mm512_unpackhi_epi16(row0, row1);
|
|
row0l = _mm512_sub_epi16(row0l, zmm_8000);
|
|
row0h = _mm512_sub_epi16(row0h, zmm_8000);
|
|
__m512i resl = _mm512_madd_epi16(row0l, zmm_y);
|
|
__m512i resh = _mm512_madd_epi16(row0h, zmm_y);
|
|
resl = _mm512_add_epi32(resl, zmm_round);
|
|
resh = _mm512_add_epi32(resh, zmm_round);
|
|
resl = _mm512_srai_epi32(resl, 8);
|
|
resh = _mm512_srai_epi32(resh, 8);
|
|
_mm512_storeu_si512((__m512i*)(dst_ptr + i),
|
|
_mm512_packus_epi32(resl, resh));
|
|
}
|
|
if (i < width) {
|
|
__mmask32 mask = (__mmask32)((1u << (width - i)) - 1u);
|
|
__m512i row0 = _mm512_maskz_loadu_epi16(mask, src_ptr + i);
|
|
__m512i row1 = _mm512_maskz_loadu_epi16(mask, src_ptr1 + i);
|
|
__m512i row0l = _mm512_unpacklo_epi16(row0, row1);
|
|
__m512i row0h = _mm512_unpackhi_epi16(row0, row1);
|
|
row0l = _mm512_sub_epi16(row0l, zmm_8000);
|
|
row0h = _mm512_sub_epi16(row0h, zmm_8000);
|
|
__m512i resl = _mm512_madd_epi16(row0l, zmm_y);
|
|
__m512i resh = _mm512_madd_epi16(row0h, zmm_y);
|
|
resl = _mm512_add_epi32(resl, zmm_round);
|
|
resh = _mm512_add_epi32(resh, zmm_round);
|
|
resl = _mm512_srai_epi32(resl, 8);
|
|
resh = _mm512_srai_epi32(resh, 8);
|
|
_mm512_mask_storeu_epi16(dst_ptr + i, mask,
|
|
_mm512_packus_epi32(resl, resh));
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_INTERPOLATEROW_16_AVX512BW
|
|
|
|
// ===========================================================================
|
|
// Module 4: NV12 / NV21 & I422Alpha to ARGB / RGB24 (AVX512BW / AVX512VBMI)
|
|
// ===========================================================================
|
|
|
|
#if defined(HAS_NV12TOARGBROW_AVX512BW) || \
|
|
defined(HAS_NV21TOARGBROW_AVX512BW) || \
|
|
defined(HAS_NV12TORGB24ROW_AVX512VBMI) || \
|
|
defined(HAS_NV21TORGB24ROW_AVX512VBMI) || \
|
|
defined(HAS_I422ALPHATOARGBROW_AVX512BW)
|
|
static const uint16_t kSplitQuadWords_Win[32] = {
|
|
0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
|
|
8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
|
|
static const uint64_t kUnpermuteAVX512_Win[8] = {0, 4, 1, 5, 2, 6, 3, 7};
|
|
#endif
|
|
|
|
#ifdef HAS_NV12TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf,
|
|
const uint8_t* uv_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
const __m512i zmm_kUVToB = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
|
|
const __m512i zmm_kUVToG = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
|
|
const __m512i zmm_kUVToR = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
|
|
const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
|
|
const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
|
|
const __m512i zmm_128 = _mm512_set1_epi8((char)0x80);
|
|
const __m512i zmm_quad =
|
|
_mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win);
|
|
const __m512i zmm5 = _mm512_set1_epi32(-1);
|
|
while (width >= 32) {
|
|
__m512i zmm3 = PermutexvarEpi16(
|
|
zmm_quad,
|
|
_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)uv_buf)));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
|
|
uv_buf += 32;
|
|
y_buf += 32;
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
|
|
zmm_kYBiasToRgb);
|
|
__m512i zmm0 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6),
|
|
zmm5);
|
|
__m512i zmm1 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6),
|
|
zmm5);
|
|
__m512i zmm2 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
|
|
zmm5);
|
|
__m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
|
|
__m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
|
|
_mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra));
|
|
_mm512_storeu_si512((__m512i*)(dst_argb + 64),
|
|
_mm512_unpackhi_epi16(bg, ra));
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m512i zmm3 = PermutexvarEpi16(
|
|
zmm_quad,
|
|
_mm512_castsi256_si512(_mm256_maskz_loadu_epi16(k_uv, uv_buf)));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
|
|
zmm_kYBiasToRgb);
|
|
__m512i zmm0 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6),
|
|
zmm5);
|
|
__m512i zmm1 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6),
|
|
zmm5);
|
|
__m512i zmm2 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
|
|
zmm5);
|
|
__m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
|
|
__m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
|
|
_mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32,
|
|
_mm512_unpacklo_epi16(bg, ra));
|
|
_mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16),
|
|
_mm512_unpackhi_epi16(bg, ra));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_NV12TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_NV21TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf,
|
|
const uint8_t* vu_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
const __m512i zmm_kUVToB = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
|
|
const __m512i zmm_kUVToG = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
|
|
const __m512i zmm_kUVToR = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
|
|
const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
|
|
const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
|
|
const __m512i zmm_128 = _mm512_set1_epi8((char)0x80);
|
|
const __m512i zmm_quad =
|
|
_mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win);
|
|
const __m512i zmm5 = _mm512_set1_epi32(-1);
|
|
while (width >= 32) {
|
|
__m512i zmm3 = PermutexvarEpi16(
|
|
zmm_quad,
|
|
_mm512_castsi256_si512(_mm256_or_si256(
|
|
_mm256_slli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8),
|
|
_mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8))));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
|
|
vu_buf += 32;
|
|
y_buf += 32;
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
|
|
zmm_kYBiasToRgb);
|
|
__m512i zmm0 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6),
|
|
zmm5);
|
|
__m512i zmm1 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6),
|
|
zmm5);
|
|
__m512i zmm2 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
|
|
zmm5);
|
|
__m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
|
|
__m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
|
|
_mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra));
|
|
_mm512_storeu_si512((__m512i*)(dst_argb + 64),
|
|
_mm512_unpackhi_epi16(bg, ra));
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m512i zmm3 = PermutexvarEpi16(
|
|
zmm_quad,
|
|
_mm512_castsi256_si512(_mm256_or_si256(
|
|
_mm256_slli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8),
|
|
_mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8))));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
|
|
zmm_kYBiasToRgb);
|
|
__m512i zmm0 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6),
|
|
zmm5);
|
|
__m512i zmm1 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6),
|
|
zmm5);
|
|
__m512i zmm2 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
|
|
zmm5);
|
|
__m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
|
|
__m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
|
|
_mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32,
|
|
_mm512_unpacklo_epi16(bg, ra));
|
|
_mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16),
|
|
_mm512_unpackhi_epi16(bg, ra));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_NV21TOARGBROW_AVX512BW
|
|
|
|
#if defined(HAS_NV12TORGB24ROW_AVX512VBMI) || \
|
|
defined(HAS_NV21TORGB24ROW_AVX512VBMI)
|
|
static const uint8_t kMaskBG_NV[64] = {
|
|
0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05,
|
|
0x45, 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52,
|
|
0x13, 0x53, 0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20,
|
|
0x60, 0x21, 0x61, 0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65,
|
|
0x26, 0x66, 0x27, 0x67, 0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33,
|
|
0x73, 0x34, 0x74, 0x35, 0x75, 0x36, 0x76, 0x37, 0x77};
|
|
static const uint8_t kMaskDST0_NV[64] = {
|
|
0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07,
|
|
0x43, 0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e,
|
|
0x0f, 0x47, 0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52,
|
|
0x16, 0x17, 0x53, 0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d,
|
|
0x56, 0x1e, 0x1f, 0x57, 0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24,
|
|
0x25, 0x62, 0x26, 0x27, 0x63, 0x28, 0x29, 0x64, 0x2a};
|
|
static const uint8_t kMaskDST1_NV[64] = {
|
|
0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70,
|
|
0x32, 0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39,
|
|
0x74, 0x3a, 0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00,
|
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
|
|
#endif
|
|
|
|
#ifdef HAS_NV12TORGB24ROW_AVX512VBMI
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
void NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
|
|
const uint8_t* uv_buf,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
const __m512i zmm_kUVToB = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
|
|
const __m512i zmm_kUVToG = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
|
|
const __m512i zmm_kUVToR = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
|
|
const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
|
|
const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
|
|
const __m512i zmm_128 = _mm512_set1_epi8((char)0x80);
|
|
const __m512i zmm_quad =
|
|
_mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win);
|
|
const __m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG_NV);
|
|
const __m512i zmm_mask_DST0 =
|
|
_mm512_loadu_si512((const __m512i*)kMaskDST0_NV);
|
|
const __m512i zmm_mask_DST1 =
|
|
_mm512_loadu_si512((const __m512i*)kMaskDST1_NV);
|
|
while (width >= 32) {
|
|
__m512i zmm3 = PermutexvarEpi16(
|
|
zmm_quad,
|
|
_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)uv_buf)));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
|
|
uv_buf += 32;
|
|
y_buf += 32;
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
|
|
zmm_kYBiasToRgb);
|
|
__m512i zmm0 = _mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6);
|
|
__m512i zmm1 = _mm512_srai_epi16(
|
|
_mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6);
|
|
__m512i zmm2 = _mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6);
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
|
|
zmm1 = _mm512_packus_epi16(zmm1, zmm1);
|
|
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
|
|
__m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
|
|
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0);
|
|
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64),
|
|
_mm512_castsi512_si256(zmm_dst1));
|
|
dst_rgb24 += 96;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m512i zmm3 = PermutexvarEpi16(
|
|
zmm_quad,
|
|
_mm512_castsi256_si512(_mm256_maskz_loadu_epi16(k_uv, uv_buf)));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
|
|
zmm_kYBiasToRgb);
|
|
__m512i zmm0 = _mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6);
|
|
__m512i zmm1 = _mm512_srai_epi16(
|
|
_mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6);
|
|
__m512i zmm2 = _mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6);
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
|
|
zmm1 = _mm512_packus_epi16(zmm1, zmm1);
|
|
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
|
|
__m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
|
|
int dst_bytes = width * 3;
|
|
__mmask64 k0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull;
|
|
__mmask32 k1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u;
|
|
_mm512_mask_storeu_epi8(dst_rgb24, k0, zmm_dst0);
|
|
_mm256_mask_storeu_epi8(dst_rgb24 + 64, k1,
|
|
_mm512_castsi512_si256(zmm_dst1));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_NV12TORGB24ROW_AVX512VBMI
|
|
|
|
#ifdef HAS_NV21TORGB24ROW_AVX512VBMI
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
void NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
|
|
const uint8_t* vu_buf,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
const __m512i zmm_kUVToB = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
|
|
const __m512i zmm_kUVToG = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
|
|
const __m512i zmm_kUVToR = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
|
|
const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
|
|
const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
|
|
const __m512i zmm_128 = _mm512_set1_epi8((char)0x80);
|
|
const __m512i zmm_quad =
|
|
_mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win);
|
|
const __m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG_NV);
|
|
const __m512i zmm_mask_DST0 =
|
|
_mm512_loadu_si512((const __m512i*)kMaskDST0_NV);
|
|
const __m512i zmm_mask_DST1 =
|
|
_mm512_loadu_si512((const __m512i*)kMaskDST1_NV);
|
|
while (width >= 32) {
|
|
__m512i zmm3 = PermutexvarEpi16(
|
|
zmm_quad,
|
|
_mm512_castsi256_si512(_mm256_or_si256(
|
|
_mm256_slli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8),
|
|
_mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8))));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
|
|
vu_buf += 32;
|
|
y_buf += 32;
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
|
|
zmm_kYBiasToRgb);
|
|
__m512i zmm0 = _mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6);
|
|
__m512i zmm1 = _mm512_srai_epi16(
|
|
_mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6);
|
|
__m512i zmm2 = _mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6);
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
|
|
zmm1 = _mm512_packus_epi16(zmm1, zmm1);
|
|
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
|
|
__m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
|
|
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0);
|
|
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64),
|
|
_mm512_castsi512_si256(zmm_dst1));
|
|
dst_rgb24 += 96;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m512i zmm3 = PermutexvarEpi16(
|
|
zmm_quad,
|
|
_mm512_castsi256_si512(_mm256_or_si256(
|
|
_mm256_slli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8),
|
|
_mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8))));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
|
|
zmm_kYBiasToRgb);
|
|
__m512i zmm0 = _mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6);
|
|
__m512i zmm1 = _mm512_srai_epi16(
|
|
_mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6);
|
|
__m512i zmm2 = _mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6);
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
|
|
zmm1 = _mm512_packus_epi16(zmm1, zmm1);
|
|
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
|
|
__m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
|
|
int dst_bytes = width * 3;
|
|
__mmask64 k0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull;
|
|
__mmask32 k1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u;
|
|
_mm512_mask_storeu_epi8(dst_rgb24, k0, zmm_dst0);
|
|
_mm256_mask_storeu_epi8(dst_rgb24 + 64, k1,
|
|
_mm512_castsi512_si256(zmm_dst1));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_NV21TORGB24ROW_AVX512VBMI
|
|
|
|
#ifdef HAS_I422ALPHATOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf,
|
|
const uint8_t* u_buf,
|
|
const uint8_t* v_buf,
|
|
const uint8_t* a_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
const __m512i zmm_kUVToB = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
|
|
const __m512i zmm_kUVToG = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
|
|
const __m512i zmm_kUVToR = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
|
|
const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
|
|
const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
|
|
const __m512i zmm_128 = _mm512_set1_epi8((char)0x80);
|
|
const __m512i zmm_quad =
|
|
_mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win);
|
|
ptrdiff_t offset = v_buf - u_buf;
|
|
while (width >= 32) {
|
|
__m128i xu = _mm_loadu_si128((const __m128i*)u_buf);
|
|
__m128i xv = _mm_loadu_si128((const __m128i*)(u_buf + offset));
|
|
u_buf += 16;
|
|
__m256i yuv = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_unpacklo_epi8(xu, xv)),
|
|
_mm_unpackhi_epi8(xu, xv), 1);
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_castsi256_si512(yuv));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
|
|
__m512i zmm5 = _mm512_packus_epi16(
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)a_buf)),
|
|
_mm512_setzero_si512());
|
|
y_buf += 32;
|
|
a_buf += 32;
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
|
|
zmm_kYBiasToRgb);
|
|
__m512i zmm0 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6),
|
|
zmm5);
|
|
__m512i zmm1 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6),
|
|
zmm5);
|
|
__m512i zmm2 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
|
|
zmm5);
|
|
__m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
|
|
__m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
|
|
_mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra));
|
|
_mm512_storeu_si512((__m512i*)(dst_argb + 64),
|
|
_mm512_unpackhi_epi16(bg, ra));
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m128i xu = _mm_maskz_loadu_epi8(k_uv, u_buf);
|
|
__m128i xv = _mm_maskz_loadu_epi8(k_uv, u_buf + offset);
|
|
__m256i yuv = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_unpacklo_epi8(xu, xv)),
|
|
_mm_unpackhi_epi8(xu, xv), 1);
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_castsi256_si512(yuv));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
|
|
__m512i zmm5 = _mm512_packus_epi16(
|
|
_mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, a_buf)),
|
|
_mm512_setzero_si512());
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
|
|
zmm_kYBiasToRgb);
|
|
__m512i zmm0 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6),
|
|
zmm5);
|
|
__m512i zmm1 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6),
|
|
zmm5);
|
|
__m512i zmm2 = _mm512_packus_epi16(
|
|
_mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
|
|
zmm5);
|
|
__m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
|
|
__m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
|
|
_mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32,
|
|
_mm512_unpacklo_epi16(bg, ra));
|
|
_mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16),
|
|
_mm512_unpackhi_epi16(bg, ra));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422ALPHATOARGBROW_AVX512BW
|
|
|
|
// ===========================================================================
|
|
// Module 5: I400ToARGB & ARGB Shaders (Attenuate, Multiply, Gray, Blend)
|
|
// ===========================================================================
|
|
|
|
#ifdef HAS_I400TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I400ToARGBRow_AVX512BW(const uint8_t* y_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
static const uint64_t kLinearizePackQ[8] = {0, 2, 4, 6, 1, 3, 5, 7};
|
|
static const uint8_t kBcastGrayByte[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80,
|
|
8, 8, 8, 0x80, 12, 12, 12, 0x80};
|
|
const __m512i zmm_yg =
|
|
_mm512_set1_epi16(*(const int16_t*)yuvconstants->kYToRgb);
|
|
const __m512i zmm_ygb =
|
|
_mm512_set1_epi16(*(const int16_t*)yuvconstants->kYBiasToRgb);
|
|
const __m512i zmm_lin = _mm512_loadu_si512((const __m512i*)kLinearizePackQ);
|
|
const __m512i zmm_bcast =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kBcastGrayByte));
|
|
const __m512i zmm_alpha = _mm512_set1_epi32((int)0xff000000u);
|
|
while (width >= 32) {
|
|
__m512i y16 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
|
|
y_buf += 32;
|
|
y16 = _mm512_or_si512(y16, _mm512_slli_epi16(y16, 8));
|
|
y16 = _mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_mulhi_epu16(y16, zmm_yg), zmm_ygb), 6);
|
|
__m512i g = PermutexvarEpi64(zmm_lin, _mm512_packus_epi16(y16, y16));
|
|
__m256i g256 = _mm512_castsi512_si256(g);
|
|
__m512i d0 = _mm512_or_si512(
|
|
_mm512_shuffle_epi8(_mm512_cvtepu8_epi32(_mm256_castsi256_si128(g256)),
|
|
zmm_bcast),
|
|
zmm_alpha);
|
|
__m512i d1 = _mm512_or_si512(
|
|
_mm512_shuffle_epi8(
|
|
_mm512_cvtepu8_epi32(_mm256_extracti128_si256(g256, 1)), zmm_bcast),
|
|
zmm_alpha);
|
|
_mm512_storeu_si512((__m512i*)dst_argb, d0);
|
|
_mm512_storeu_si512((__m512i*)(dst_argb + 64), d1);
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__m512i y16 =
|
|
_mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
|
|
y16 = _mm512_or_si512(y16, _mm512_slli_epi16(y16, 8));
|
|
y16 = _mm512_srai_epi16(
|
|
_mm512_adds_epi16(_mm512_mulhi_epu16(y16, zmm_yg), zmm_ygb), 6);
|
|
__m512i g = PermutexvarEpi64(zmm_lin, _mm512_packus_epi16(y16, y16));
|
|
__m256i g256 = _mm512_castsi512_si256(g);
|
|
__m512i d0 = _mm512_or_si512(
|
|
_mm512_shuffle_epi8(_mm512_cvtepu8_epi32(_mm256_castsi256_si128(g256)),
|
|
zmm_bcast),
|
|
zmm_alpha);
|
|
__m512i d1 = _mm512_or_si512(
|
|
_mm512_shuffle_epi8(
|
|
_mm512_cvtepu8_epi32(_mm256_extracti128_si256(g256, 1)), zmm_bcast),
|
|
zmm_alpha);
|
|
_mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32, d0);
|
|
_mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16), d1);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I400TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBATTENUATEROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBAttenuateRow_AVX512BW(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
static const int8_t kAttenShuf16[16] = {6, -128, 6, -128, 6, -128,
|
|
-128, -128, 14, -128, 14, -128,
|
|
14, -128, -128, -128};
|
|
const __m512i zmm_shuf =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kAttenShuf16));
|
|
const __m512i zmm_alpha = _mm512_set1_epi32((int)0xff000000u);
|
|
const __m512i zmm_bias = _mm512_set1_epi16(0x00ff);
|
|
while (width >= 16) {
|
|
__m512i s = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
src_argb += 64;
|
|
__m512i lo = _mm512_unpacklo_epi8(s, zmm_alpha);
|
|
__m512i hi = _mm512_unpackhi_epi8(s, zmm_alpha);
|
|
lo = _mm512_srli_epi16(
|
|
_mm512_add_epi16(
|
|
_mm512_mullo_epi16(lo, _mm512_shuffle_epi8(lo, zmm_shuf)),
|
|
zmm_bias),
|
|
8);
|
|
hi = _mm512_srli_epi16(
|
|
_mm512_add_epi16(
|
|
_mm512_mullo_epi16(hi, _mm512_shuffle_epi8(hi, zmm_shuf)),
|
|
zmm_bias),
|
|
8);
|
|
__m512i out = _mm512_or_si512(_mm512_packus_epi16(lo, hi),
|
|
_mm512_and_si512(s, zmm_alpha));
|
|
_mm512_storeu_si512((__m512i*)dst_argb, out);
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i s = _mm512_maskz_loadu_epi32(k, src_argb);
|
|
__m512i lo = _mm512_unpacklo_epi8(s, zmm_alpha);
|
|
__m512i hi = _mm512_unpackhi_epi8(s, zmm_alpha);
|
|
lo = _mm512_srli_epi16(
|
|
_mm512_add_epi16(
|
|
_mm512_mullo_epi16(lo, _mm512_shuffle_epi8(lo, zmm_shuf)),
|
|
zmm_bias),
|
|
8);
|
|
hi = _mm512_srli_epi16(
|
|
_mm512_add_epi16(
|
|
_mm512_mullo_epi16(hi, _mm512_shuffle_epi8(hi, zmm_shuf)),
|
|
zmm_bias),
|
|
8);
|
|
__m512i out = _mm512_or_si512(_mm512_packus_epi16(lo, hi),
|
|
_mm512_and_si512(s, zmm_alpha));
|
|
_mm512_mask_storeu_epi32(dst_argb, k, out);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBATTENUATEROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBMULTIPLYROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBMultiplyRow_AVX512BW(const uint8_t* src_argb,
|
|
const uint8_t* src_argb1,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
const __m512i zero = _mm512_setzero_si512();
|
|
while (width >= 16) {
|
|
__m512i s0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
__m512i s1 = _mm512_loadu_si512((const __m512i*)src_argb1);
|
|
src_argb += 64;
|
|
src_argb1 += 64;
|
|
__m512i lo = _mm512_mulhrs_epi16(
|
|
_mm512_srli_epi16(_mm512_unpacklo_epi8(zero, s0), 1),
|
|
_mm512_unpacklo_epi8(s1, zero));
|
|
__m512i hi = _mm512_mulhrs_epi16(
|
|
_mm512_srli_epi16(_mm512_unpackhi_epi8(zero, s0), 1),
|
|
_mm512_unpackhi_epi8(s1, zero));
|
|
_mm512_storeu_si512((__m512i*)dst_argb, _mm512_packus_epi16(lo, hi));
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i s0 = _mm512_maskz_loadu_epi32(k, src_argb);
|
|
__m512i s1 = _mm512_maskz_loadu_epi32(k, src_argb1);
|
|
__m512i lo = _mm512_mulhrs_epi16(
|
|
_mm512_srli_epi16(_mm512_unpacklo_epi8(zero, s0), 1),
|
|
_mm512_unpacklo_epi8(s1, zero));
|
|
__m512i hi = _mm512_mulhrs_epi16(
|
|
_mm512_srli_epi16(_mm512_unpackhi_epi8(zero, s0), 1),
|
|
_mm512_unpackhi_epi8(s1, zero));
|
|
_mm512_mask_storeu_epi32(dst_argb, k, _mm512_packus_epi16(lo, hi));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBMULTIPLYROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBGRAYROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBGrayRow_AVX512BW(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80,
|
|
8, 8, 8, 0x80, 12, 12, 12, 0x80};
|
|
const __m512i c_coeff = _mm512_set1_epi32(0x004d961d);
|
|
const __m512i c_sub128 = _mm512_set1_epi8((char)0x80);
|
|
const __m512i c_one16 = _mm512_set1_epi16(1);
|
|
const __m512i c_bias = _mm512_set1_epi32(32896);
|
|
const __m512i c_shuf =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kShufGrayDword));
|
|
const __m512i c_alpha = _mm512_set1_epi32((int)0xff000000u);
|
|
while (width >= 16) {
|
|
__m512i s = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
src_argb += 64;
|
|
__m512i y16 = _mm512_maddubs_epi16(c_coeff, _mm512_sub_epi8(s, c_sub128));
|
|
__m512i y32 = _mm512_srli_epi32(
|
|
_mm512_add_epi32(_mm512_madd_epi16(y16, c_one16), c_bias), 8);
|
|
__m512i out = _mm512_or_si512(_mm512_and_si512(s, c_alpha),
|
|
_mm512_shuffle_epi8(y32, c_shuf));
|
|
_mm512_storeu_si512((__m512i*)dst_argb, out);
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i s = _mm512_maskz_loadu_epi32(k, src_argb);
|
|
__m512i y16 = _mm512_maddubs_epi16(c_coeff, _mm512_sub_epi8(s, c_sub128));
|
|
__m512i y32 = _mm512_srli_epi32(
|
|
_mm512_add_epi32(_mm512_madd_epi16(y16, c_one16), c_bias), 8);
|
|
__m512i out = _mm512_or_si512(_mm512_and_si512(s, c_alpha),
|
|
_mm512_shuffle_epi8(y32, c_shuf));
|
|
_mm512_mask_storeu_epi32(dst_argb, k, out);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBGRAYROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBGRAYROW_AVX512VNNI
|
|
LIBYUV_TARGET_AVX512VNNI
|
|
void ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80,
|
|
8, 8, 8, 0x80, 12, 12, 12, 0x80};
|
|
const __m512i c_coeff = _mm512_set1_epi32(0x004d961d);
|
|
const __m512i c_sub128 = _mm512_set1_epi8((char)0x80);
|
|
const __m512i c_bias = _mm512_set1_epi32(32896);
|
|
const __m512i c_shuf =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kShufGrayDword));
|
|
const __m512i c_alpha = _mm512_set1_epi32((int)0xff000000u);
|
|
// Avoid _mm512_ternarylogic_epi32 (unmodeled by LLVM MSan on RGBX inputs;
|
|
// see GEMINI.md).
|
|
while (width >= 16) {
|
|
__m512i s = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
src_argb += 64;
|
|
__m512i y32 = _mm512_srli_epi32(
|
|
_mm512_dpbusd_epi32(c_bias, c_coeff, _mm512_xor_si512(s, c_sub128)), 8);
|
|
__m512i out = _mm512_or_si512(_mm512_shuffle_epi8(y32, c_shuf),
|
|
_mm512_and_si512(s, c_alpha));
|
|
_mm512_storeu_si512((__m512i*)dst_argb, out);
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i s = _mm512_maskz_loadu_epi32(k, src_argb);
|
|
__m512i y32 = _mm512_srli_epi32(
|
|
_mm512_dpbusd_epi32(c_bias, c_coeff, _mm512_xor_si512(s, c_sub128)), 8);
|
|
__m512i out = _mm512_or_si512(_mm512_shuffle_epi8(y32, c_shuf),
|
|
_mm512_and_si512(s, c_alpha));
|
|
_mm512_mask_storeu_epi32(dst_argb, k, out);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBGRAYROW_AVX512VNNI
|
|
|
|
#ifdef HAS_ARGBBLENDROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBBlendRow_AVX512BW(const uint8_t* src_argb,
|
|
const uint8_t* src_argb1,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
static const uint8_t kShuffleAlpha16[16] = {3u, 0x80, 3u, 0x80, 7u, 0x80,
|
|
7u, 0x80, 11u, 0x80, 11u, 0x80,
|
|
15u, 0x80, 15u, 0x80};
|
|
const __m512i zmm_shuf =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kShuffleAlpha16));
|
|
const __m512i zmm_alpha = _mm512_set1_epi32((int)0xff000000u);
|
|
const __m512i zmm_ff00 = _mm512_set1_epi16((short)0xff00u);
|
|
const __m512i zmm_00ff = _mm512_set1_epi16(0x00ff);
|
|
const __m512i zmm_0001 = _mm512_set1_epi16(0x0001);
|
|
while (width >= 16) {
|
|
__m512i s0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
__m512i s1 = _mm512_loadu_si512((const __m512i*)src_argb1);
|
|
src_argb += 64;
|
|
src_argb1 += 64;
|
|
__m512i inv_a = _mm512_add_epi16(
|
|
_mm512_shuffle_epi8(_mm512_xor_si512(s0, zmm_alpha), zmm_shuf),
|
|
zmm_0001);
|
|
__m512i even = _mm512_srli_epi16(
|
|
_mm512_mullo_epi16(_mm512_and_si512(s1, zmm_00ff), inv_a), 8);
|
|
__m512i odd = _mm512_and_si512(
|
|
_mm512_mullo_epi16(_mm512_srli_epi16(s1, 8), inv_a), zmm_ff00);
|
|
__m512i out = _mm512_adds_epu8(
|
|
_mm512_adds_epu8(_mm512_or_si512(s0, zmm_alpha), even), odd);
|
|
_mm512_storeu_si512((__m512i*)dst_argb, out);
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i s0 = _mm512_maskz_loadu_epi32(k, src_argb);
|
|
__m512i s1 = _mm512_maskz_loadu_epi32(k, src_argb1);
|
|
__m512i inv_a = _mm512_add_epi16(
|
|
_mm512_shuffle_epi8(_mm512_xor_si512(s0, zmm_alpha), zmm_shuf),
|
|
zmm_0001);
|
|
__m512i even = _mm512_srli_epi16(
|
|
_mm512_mullo_epi16(_mm512_and_si512(s1, zmm_00ff), inv_a), 8);
|
|
__m512i odd = _mm512_and_si512(
|
|
_mm512_mullo_epi16(_mm512_srli_epi16(s1, 8), inv_a), zmm_ff00);
|
|
__m512i out = _mm512_adds_epu8(
|
|
_mm512_adds_epu8(_mm512_or_si512(s0, zmm_alpha), even), odd);
|
|
_mm512_mask_storeu_epi32(dst_argb, k, out);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBBLENDROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBORROW_SSE2
|
|
void ARGBOrRow_SSE2(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
uint32_t value,
|
|
int width) {
|
|
const __m128i val = _mm_set1_epi32((int)value);
|
|
while (width >= 4) {
|
|
__m128i s = _mm_loadu_si128((const __m128i*)src_argb);
|
|
src_argb += 16;
|
|
_mm_storeu_si128((__m128i*)dst_argb, _mm_or_si128(s, val));
|
|
dst_argb += 16;
|
|
width -= 4;
|
|
}
|
|
}
|
|
#endif // HAS_ARGBORROW_SSE2
|
|
|
|
#ifdef HAS_ARGBORROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBOrRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
uint32_t value,
|
|
int width) {
|
|
const __m256i val = _mm256_set1_epi32((int)value);
|
|
while (width >= 8) {
|
|
__m256i s = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
src_argb += 32;
|
|
_mm256_storeu_si256((__m256i*)dst_argb, _mm256_or_si256(s, val));
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBORROW_AVX2
|
|
|
|
#ifdef HAS_ARGBORROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBOrRow_AVX512BW(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
uint32_t value,
|
|
int width) {
|
|
const __m512i val = _mm512_set1_epi32((int)value);
|
|
while (width >= 16) {
|
|
__m512i s = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
src_argb += 64;
|
|
_mm512_storeu_si512((__m512i*)dst_argb, _mm512_or_si512(s, val));
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i s = _mm512_maskz_loadu_epi32(k, src_argb);
|
|
_mm512_mask_storeu_epi32(dst_argb, k, _mm512_or_si512(s, val));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBORROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBALPHABLENDROW_SSSE3
|
|
LIBYUV_TARGET_SSSE3
|
|
void ARGBAlphaBlendRow_SSSE3(const uint8_t* src_argb0,
|
|
const uint8_t* src_argb1,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
const __m128i zero = _mm_setzero_si128();
|
|
const __m128i k0001 = _mm_set1_epi16(0x0001);
|
|
const __m128i k0101 = _mm_set1_epi16(0x0101);
|
|
const __m128i k00ff = _mm_set1_epi16(0x00ff);
|
|
const __m128i k8080 = _mm_set1_epi8((char)0x80);
|
|
const __m128i k7f81 = _mm_set1_epi16(0x7f81);
|
|
const __m128i k00ffffff = _mm_set1_epi32(0x00ffffff);
|
|
while (width >= 4) {
|
|
__m128i s = _mm_loadu_si128((const __m128i*)src_argb0);
|
|
__m128i d = _mm_loadu_si128((const __m128i*)src_argb1);
|
|
src_argb0 += 16;
|
|
src_argb1 += 16;
|
|
__m128i sa = _mm_srli_epi32(s, 24);
|
|
__m128i da = _mm_srli_epi32(d, 24);
|
|
__m128i da_sa = _mm_mulhi_epu16(
|
|
_mm_add_epi16(_mm_mullo_epi16(da, sa), k0001), k0101);
|
|
__m128i da2 = _mm_add_epi32(da, _mm_sub_epi32(sa, da_sa));
|
|
__m128i t = _mm_srli_epi32(_mm_cmpeq_epi32(da, zero), 24);
|
|
__m128i sa_eff = _mm_or_si128(sa, t);
|
|
__m128i den_eff = _mm_or_si128(da2, t);
|
|
__m128i num = _mm_sub_epi32(_mm_slli_epi32(sa_eff, 8), sa_eff);
|
|
__m128i ratio = _mm_cvttps_epi32(
|
|
_mm_div_ps(_mm_cvtepi32_ps(num), _mm_cvtepi32_ps(den_eff)));
|
|
__m128i r_rep = _mm_shufflelo_epi16(_mm_shufflehi_epi16(ratio, 0xa0), 0xa0);
|
|
__m128i w =
|
|
_mm_or_si128(r_rep, _mm_slli_epi16(_mm_sub_epi16(k00ff, r_rep), 8));
|
|
__m128i w_lo = _mm_unpacklo_epi16(w, w);
|
|
__m128i w_hi = _mm_unpackhi_epi16(w, w);
|
|
__m128i sb = _mm_sub_epi8(s, k8080);
|
|
__m128i db = _mm_sub_epi8(d, k8080);
|
|
__m128i lo = _mm_mulhi_epu16(
|
|
_mm_add_epi16(_mm_maddubs_epi16(w_lo, _mm_unpacklo_epi8(sb, db)),
|
|
k7f81),
|
|
k0101);
|
|
__m128i hi = _mm_mulhi_epu16(
|
|
_mm_add_epi16(_mm_maddubs_epi16(w_hi, _mm_unpackhi_epi8(sb, db)),
|
|
k7f81),
|
|
k0101);
|
|
__m128i rgb = _mm_packus_epi16(lo, hi);
|
|
__m128i out =
|
|
_mm_or_si128(_mm_and_si128(rgb, k00ffffff), _mm_slli_epi32(da2, 24));
|
|
_mm_storeu_si128((__m128i*)dst_argb, out);
|
|
dst_argb += 16;
|
|
width -= 4;
|
|
}
|
|
}
|
|
#endif // HAS_ARGBALPHABLENDROW_SSSE3
|
|
|
|
#ifdef HAS_ARGBALPHABLENDROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBAlphaBlendRow_AVX2(const uint8_t* src_argb0,
|
|
const uint8_t* src_argb1,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
const __m256i zero = _mm256_setzero_si256();
|
|
const __m256i k0001 = _mm256_set1_epi16(0x0001);
|
|
const __m256i k0101 = _mm256_set1_epi16(0x0101);
|
|
const __m256i k00ff = _mm256_set1_epi16(0x00ff);
|
|
const __m256i k8080 = _mm256_set1_epi8((char)0x80);
|
|
const __m256i k7f81 = _mm256_set1_epi16(0x7f81);
|
|
const __m256i k00ffffff = _mm256_set1_epi32(0x00ffffff);
|
|
while (width >= 8) {
|
|
__m256i s = _mm256_loadu_si256((const __m256i*)src_argb0);
|
|
__m256i d = _mm256_loadu_si256((const __m256i*)src_argb1);
|
|
src_argb0 += 32;
|
|
src_argb1 += 32;
|
|
__m256i sa = _mm256_srli_epi32(s, 24);
|
|
__m256i da = _mm256_srli_epi32(d, 24);
|
|
__m256i da_sa = _mm256_mulhi_epu16(
|
|
_mm256_add_epi16(_mm256_mullo_epi16(da, sa), k0001), k0101);
|
|
__m256i da2 = _mm256_add_epi32(da, _mm256_sub_epi32(sa, da_sa));
|
|
__m256i t = _mm256_srli_epi32(_mm256_cmpeq_epi32(da, zero), 24);
|
|
__m256i sa_eff = _mm256_or_si256(sa, t);
|
|
__m256i den_eff = _mm256_or_si256(da2, t);
|
|
__m256i num = _mm256_sub_epi32(_mm256_slli_epi32(sa_eff, 8), sa_eff);
|
|
__m256i ratio = _mm256_cvttps_epi32(
|
|
_mm256_div_ps(_mm256_cvtepi32_ps(num), _mm256_cvtepi32_ps(den_eff)));
|
|
__m256i r_rep =
|
|
_mm256_shufflelo_epi16(_mm256_shufflehi_epi16(ratio, 0xa0), 0xa0);
|
|
__m256i w = _mm256_or_si256(
|
|
r_rep, _mm256_slli_epi16(_mm256_sub_epi16(k00ff, r_rep), 8));
|
|
__m256i w_lo = _mm256_unpacklo_epi16(w, w);
|
|
__m256i w_hi = _mm256_unpackhi_epi16(w, w);
|
|
__m256i sb = _mm256_sub_epi8(s, k8080);
|
|
__m256i db = _mm256_sub_epi8(d, k8080);
|
|
__m256i lo = _mm256_mulhi_epu16(
|
|
_mm256_add_epi16(
|
|
_mm256_maddubs_epi16(w_lo, _mm256_unpacklo_epi8(sb, db)), k7f81),
|
|
k0101);
|
|
__m256i hi = _mm256_mulhi_epu16(
|
|
_mm256_add_epi16(
|
|
_mm256_maddubs_epi16(w_hi, _mm256_unpackhi_epi8(sb, db)), k7f81),
|
|
k0101);
|
|
__m256i rgb = _mm256_packus_epi16(lo, hi);
|
|
__m256i out = _mm256_or_si256(_mm256_and_si256(rgb, k00ffffff),
|
|
_mm256_slli_epi32(da2, 24));
|
|
_mm256_storeu_si256((__m256i*)dst_argb, out);
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBALPHABLENDROW_AVX2
|
|
|
|
#ifdef HAS_ARGBALPHABLENDROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb0,
|
|
const uint8_t* src_argb1,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
const __m512i zero = _mm512_setzero_si512();
|
|
const __m512i k0001 = _mm512_set1_epi16(0x0001);
|
|
const __m512i k0101 = _mm512_set1_epi16(0x0101);
|
|
const __m512i k00ff = _mm512_set1_epi16(0x00ff);
|
|
const __m512i k8080 = _mm512_set1_epi8((char)0x80);
|
|
const __m512i k7f81 = _mm512_set1_epi16(0x7f81);
|
|
const __m512i k00ffffff = _mm512_set1_epi32(0x00ffffff);
|
|
while (width >= 16) {
|
|
__m512i s = _mm512_loadu_si512((const __m512i*)src_argb0);
|
|
__m512i d = _mm512_loadu_si512((const __m512i*)src_argb1);
|
|
src_argb0 += 64;
|
|
src_argb1 += 64;
|
|
__m512i sa = _mm512_srli_epi32(s, 24);
|
|
__m512i da = _mm512_srli_epi32(d, 24);
|
|
__m512i da_sa = _mm512_mulhi_epu16(
|
|
_mm512_add_epi16(_mm512_mullo_epi16(da, sa), k0001), k0101);
|
|
__m512i da2 = _mm512_add_epi32(da, _mm512_sub_epi32(sa, da_sa));
|
|
__mmask16 k_da0 = _mm512_cmpeq_epi32_mask(da, zero);
|
|
__m512i t = _mm512_maskz_set1_epi32(k_da0, 255);
|
|
__m512i sa_eff = _mm512_or_si512(sa, t);
|
|
__m512i den_eff = _mm512_or_si512(da2, t);
|
|
__m512i num = _mm512_sub_epi32(_mm512_slli_epi32(sa_eff, 8), sa_eff);
|
|
__m512i ratio = CvttpsEpi32(
|
|
_mm512_div_ps(_mm512_cvtepi32_ps(num), _mm512_cvtepi32_ps(den_eff)));
|
|
__m512i r_rep =
|
|
_mm512_shufflelo_epi16(_mm512_shufflehi_epi16(ratio, 0xa0), 0xa0);
|
|
__m512i w = _mm512_or_si512(
|
|
r_rep, _mm512_slli_epi16(_mm512_sub_epi16(k00ff, r_rep), 8));
|
|
__m512i w_lo = _mm512_unpacklo_epi16(w, w);
|
|
__m512i w_hi = _mm512_unpackhi_epi16(w, w);
|
|
__m512i sb = _mm512_sub_epi8(s, k8080);
|
|
__m512i db = _mm512_sub_epi8(d, k8080);
|
|
__m512i lo = _mm512_mulhi_epu16(
|
|
_mm512_add_epi16(
|
|
_mm512_maddubs_epi16(w_lo, _mm512_unpacklo_epi8(sb, db)), k7f81),
|
|
k0101);
|
|
__m512i hi = _mm512_mulhi_epu16(
|
|
_mm512_add_epi16(
|
|
_mm512_maddubs_epi16(w_hi, _mm512_unpackhi_epi8(sb, db)), k7f81),
|
|
k0101);
|
|
__m512i rgb = _mm512_packus_epi16(lo, hi);
|
|
__m512i out = _mm512_or_si512(_mm512_and_si512(rgb, k00ffffff),
|
|
_mm512_slli_epi32(da2, 24));
|
|
_mm512_storeu_si512((__m512i*)dst_argb, out);
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i s = _mm512_maskz_loadu_epi32(k, src_argb0);
|
|
__m512i d = _mm512_maskz_loadu_epi32(k, src_argb1);
|
|
__m512i sa = _mm512_srli_epi32(s, 24);
|
|
__m512i da = _mm512_srli_epi32(d, 24);
|
|
__m512i da_sa = _mm512_mulhi_epu16(
|
|
_mm512_add_epi16(_mm512_mullo_epi16(da, sa), k0001), k0101);
|
|
__m512i da2 = _mm512_add_epi32(da, _mm512_sub_epi32(sa, da_sa));
|
|
__mmask16 k_da0 = _mm512_cmpeq_epi32_mask(da, zero);
|
|
__m512i t = _mm512_maskz_set1_epi32(k_da0, 255);
|
|
__m512i sa_eff = _mm512_or_si512(sa, t);
|
|
__m512i den_eff = _mm512_or_si512(da2, t);
|
|
__m512i num = _mm512_sub_epi32(_mm512_slli_epi32(sa_eff, 8), sa_eff);
|
|
__m512i ratio = CvttpsEpi32(
|
|
_mm512_div_ps(_mm512_cvtepi32_ps(num), _mm512_cvtepi32_ps(den_eff)));
|
|
__m512i r_rep =
|
|
_mm512_shufflelo_epi16(_mm512_shufflehi_epi16(ratio, 0xa0), 0xa0);
|
|
__m512i w = _mm512_or_si512(
|
|
r_rep, _mm512_slli_epi16(_mm512_sub_epi16(k00ff, r_rep), 8));
|
|
__m512i w_lo = _mm512_unpacklo_epi16(w, w);
|
|
__m512i w_hi = _mm512_unpackhi_epi16(w, w);
|
|
__m512i sb = _mm512_sub_epi8(s, k8080);
|
|
__m512i db = _mm512_sub_epi8(d, k8080);
|
|
__m512i lo = _mm512_mulhi_epu16(
|
|
_mm512_add_epi16(
|
|
_mm512_maddubs_epi16(w_lo, _mm512_unpacklo_epi8(sb, db)), k7f81),
|
|
k0101);
|
|
__m512i hi = _mm512_mulhi_epu16(
|
|
_mm512_add_epi16(
|
|
_mm512_maddubs_epi16(w_hi, _mm512_unpackhi_epi8(sb, db)), k7f81),
|
|
k0101);
|
|
__m512i rgb = _mm512_packus_epi16(lo, hi);
|
|
__m512i out = _mm512_or_si512(_mm512_and_si512(rgb, k00ffffff),
|
|
_mm512_slli_epi32(da2, 24));
|
|
_mm512_mask_storeu_epi32(dst_argb, k, out);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBALPHABLENDROW_AVX512BW
|
|
|
|
// ===========================================================================
|
|
// Module 6: AVX2 Row Functions (Minimum AVX2 Coverage)
|
|
// ===========================================================================
|
|
|
|
#ifdef HAS_ARGBSEPIAROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBSepiaRow_AVX2(uint8_t* dst_argb, int width) {
|
|
static const int8_t kSepiaB[16] = {17, 68, 35, 0, 17, 68, 35, 0,
|
|
17, 68, 35, 0, 17, 68, 35, 0};
|
|
static const int8_t kSepiaG[16] = {22, 88, 45, 0, 22, 88, 45, 0,
|
|
22, 88, 45, 0, 22, 88, 45, 0};
|
|
static const int8_t kSepiaR[16] = {24, 98, 50, 0, 24, 98, 50, 0,
|
|
24, 98, 50, 0, 24, 98, 50, 0};
|
|
if (width >= 8) {
|
|
const __m256i vb =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kSepiaB));
|
|
const __m256i vg =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kSepiaG));
|
|
const __m256i vr =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kSepiaR));
|
|
while (width >= 8) {
|
|
__m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb);
|
|
__m256i b = _mm256_maddubs_epi16(v0, vb);
|
|
__m256i g = _mm256_maddubs_epi16(v0, vg);
|
|
__m256i r = _mm256_maddubs_epi16(v0, vr);
|
|
__m256i bg = _mm256_srli_epi16(_mm256_hadd_epi16(b, g), 7);
|
|
__m256i rr = _mm256_srli_epi16(_mm256_hadd_epi16(r, r), 7);
|
|
bg = _mm256_packus_epi16(bg, bg);
|
|
rr = _mm256_packus_epi16(rr, rr);
|
|
bg = _mm256_unpacklo_epi8(bg, _mm256_shuffle_epi32(bg, 0x01));
|
|
__m256i a = _mm256_srli_epi32(v0, 24);
|
|
a = _mm256_packus_epi16(a, a);
|
|
a = _mm256_packus_epi16(a, a);
|
|
__m256i ra = _mm256_unpacklo_epi8(rr, a);
|
|
__m256i out = _mm256_unpacklo_epi16(bg, ra);
|
|
_mm256_storeu_si256((__m256i*)dst_argb, out);
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
if (width > 0) {
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
ARGBSepiaRow_C(dst_argb, width);
|
|
}
|
|
}
|
|
#endif // HAS_ARGBSEPIAROW_AVX2
|
|
|
|
#ifdef HAS_ARGBCOLORMATRIXROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBColorMatrixRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
const int8_t* matrix_argb,
|
|
int width) {
|
|
static const int8_t kShufTranspose4x4[16] = {0, 4, 8, 12, 1, 5, 9, 13,
|
|
2, 6, 10, 14, 3, 7, 11, 15};
|
|
if (width >= 8) {
|
|
const __m256i m = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)matrix_argb));
|
|
const __m256i shuf = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShufTranspose4x4));
|
|
const __m256i m0 = _mm256_shuffle_epi32(m, 0x00);
|
|
const __m256i m1 = _mm256_shuffle_epi32(m, 0x55);
|
|
const __m256i m2 = _mm256_shuffle_epi32(m, 0xaa);
|
|
const __m256i m3 = _mm256_shuffle_epi32(m, 0xff);
|
|
while (width >= 8) {
|
|
__m256i v0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
src_argb += 32;
|
|
__m256i p01 = _mm256_hadds_epi16(_mm256_maddubs_epi16(v0, m0),
|
|
_mm256_maddubs_epi16(v0, m1));
|
|
__m256i p23 = _mm256_hadds_epi16(_mm256_maddubs_epi16(v0, m2),
|
|
_mm256_maddubs_epi16(v0, m3));
|
|
p01 = _mm256_srai_epi16(p01, 6);
|
|
p23 = _mm256_srai_epi16(p23, 6);
|
|
__m256i out = _mm256_shuffle_epi8(_mm256_packus_epi16(p01, p23), shuf);
|
|
_mm256_storeu_si256((__m256i*)dst_argb, out);
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
if (width > 0) {
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
ARGBColorMatrixRow_C(src_argb, dst_argb, matrix_argb, width);
|
|
}
|
|
}
|
|
#endif // HAS_ARGBCOLORMATRIXROW_AVX2
|
|
|
|
#ifdef HAS_ARGBQUANTIZEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBQuantizeRow_AVX2(uint8_t* dst_argb,
|
|
int scale,
|
|
int interval_size,
|
|
int interval_offset,
|
|
int width) {
|
|
if (width >= 8) {
|
|
const __m256i v_scale = _mm256_set1_epi16((short)scale);
|
|
const __m256i v_size = _mm256_set1_epi16((short)interval_size);
|
|
const __m256i v_off = _mm256_set1_epi16((short)interval_offset);
|
|
const __m256i zero = _mm256_setzero_si256();
|
|
const __m256i alpha_mask = _mm256_set1_epi32((int)0xff000000u);
|
|
while (width >= 8) {
|
|
__m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb);
|
|
__m256i a = _mm256_and_si256(v0, alpha_mask);
|
|
__m256i lo = _mm256_unpacklo_epi8(v0, zero);
|
|
__m256i hi = _mm256_unpackhi_epi8(v0, zero);
|
|
lo = _mm256_add_epi16(
|
|
_mm256_mullo_epi16(_mm256_mulhi_epu16(lo, v_scale), v_size), v_off);
|
|
hi = _mm256_add_epi16(
|
|
_mm256_mullo_epi16(_mm256_mulhi_epu16(hi, v_scale), v_size), v_off);
|
|
__m256i out = _mm256_or_si256(
|
|
_mm256_andnot_si256(alpha_mask, _mm256_packus_epi16(lo, hi)), a);
|
|
_mm256_storeu_si256((__m256i*)dst_argb, out);
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
if (width > 0) {
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
ARGBQuantizeRow_C(dst_argb, scale, interval_size, interval_offset, width);
|
|
}
|
|
}
|
|
#endif // HAS_ARGBQUANTIZEROW_AVX2
|
|
|
|
#ifdef HAS_ARGBSHADEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBShadeRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
int width,
|
|
uint32_t value) {
|
|
if (width >= 8) {
|
|
__m256i v_val = _mm256_set1_epi32((int)value);
|
|
v_val = _mm256_unpacklo_epi8(v_val, v_val);
|
|
while (width >= 8) {
|
|
__m256i v0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
src_argb += 32;
|
|
__m256i lo = _mm256_unpacklo_epi8(v0, v0);
|
|
__m256i hi = _mm256_unpackhi_epi8(v0, v0);
|
|
lo = _mm256_srli_epi16(_mm256_mulhi_epu16(lo, v_val), 8);
|
|
hi = _mm256_srli_epi16(_mm256_mulhi_epu16(hi, v_val), 8);
|
|
_mm256_storeu_si256((__m256i*)dst_argb, _mm256_packus_epi16(lo, hi));
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
if (width > 0) {
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
ARGBShadeRow_C(src_argb, dst_argb, width, value);
|
|
}
|
|
}
|
|
#endif // HAS_ARGBSHADEROW_AVX2
|
|
|
|
#ifdef HAS_SOBELXROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void SobelXRow_AVX2(const uint8_t* src_y0,
|
|
const uint8_t* src_y1,
|
|
const uint8_t* src_y2,
|
|
uint8_t* dst_sobelx,
|
|
int width) {
|
|
while (width > 0) {
|
|
__m256i a0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y0));
|
|
__m256i b0 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y0 + 2)));
|
|
__m256i a1 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y1));
|
|
__m256i b1 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y1 + 2)));
|
|
__m256i a2 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y2));
|
|
__m256i b2 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y2 + 2)));
|
|
__m256i d0 = _mm256_sub_epi16(a0, b0);
|
|
__m256i d1 = _mm256_sub_epi16(a1, b1);
|
|
__m256i d2 = _mm256_sub_epi16(a2, b2);
|
|
__m256i sum =
|
|
_mm256_add_epi16(_mm256_add_epi16(d0, d2), _mm256_add_epi16(d1, d1));
|
|
sum = _mm256_abs_epi16(sum);
|
|
sum = _mm256_permute4x64_epi64(_mm256_packus_epi16(sum, sum), 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_sobelx, _mm256_castsi256_si128(sum));
|
|
src_y0 += 16;
|
|
src_y1 += 16;
|
|
src_y2 += 16;
|
|
dst_sobelx += 16;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SOBELXROW_AVX2
|
|
|
|
#ifdef HAS_SOBELYROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void SobelYRow_AVX2(const uint8_t* src_y0,
|
|
const uint8_t* src_y1,
|
|
uint8_t* dst_sobely,
|
|
int width) {
|
|
while (width > 0) {
|
|
__m256i a0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y0));
|
|
__m256i b0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y1));
|
|
__m256i a1 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y0 + 1)));
|
|
__m256i b1 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y1 + 1)));
|
|
__m256i a2 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y0 + 2)));
|
|
__m256i b2 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y1 + 2)));
|
|
__m256i d0 = _mm256_sub_epi16(a0, b0);
|
|
__m256i d1 = _mm256_sub_epi16(a1, b1);
|
|
__m256i d2 = _mm256_sub_epi16(a2, b2);
|
|
__m256i sum =
|
|
_mm256_add_epi16(_mm256_add_epi16(d0, d2), _mm256_add_epi16(d1, d1));
|
|
sum = _mm256_abs_epi16(sum);
|
|
sum = _mm256_permute4x64_epi64(_mm256_packus_epi16(sum, sum), 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_sobely, _mm256_castsi256_si128(sum));
|
|
src_y0 += 16;
|
|
src_y1 += 16;
|
|
dst_sobely += 16;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SOBELYROW_AVX2
|
|
|
|
#ifdef HAS_SOBELROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void SobelRow_AVX2(const uint8_t* src_sobelx,
|
|
const uint8_t* src_sobely,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
const __m256i v_alpha = _mm256_set1_epi32((int)0xff000000u);
|
|
while (width > 0) {
|
|
__m128i vx = _mm_loadu_si128((const __m128i*)src_sobelx);
|
|
__m128i vy = _mm_loadu_si128((const __m128i*)src_sobely);
|
|
__m256i s16 = _mm256_cvtepu8_epi16(_mm_adds_epu8(vx, vy));
|
|
s16 = _mm256_or_si256(s16, _mm256_slli_epi16(s16, 8));
|
|
s16 = _mm256_permute4x64_epi64(s16, 0xd8);
|
|
__m256i p0 = _mm256_or_si256(_mm256_unpacklo_epi16(s16, s16), v_alpha);
|
|
__m256i p1 = _mm256_or_si256(_mm256_unpackhi_epi16(s16, s16), v_alpha);
|
|
_mm256_storeu_si256((__m256i*)dst_argb, p0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), p1);
|
|
src_sobelx += 16;
|
|
src_sobely += 16;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SOBELROW_AVX2
|
|
|
|
#ifdef HAS_SOBELTOPLANEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void SobelToPlaneRow_AVX2(const uint8_t* src_sobelx,
|
|
const uint8_t* src_sobely,
|
|
uint8_t* dst_y,
|
|
int width) {
|
|
while (width > 0) {
|
|
__m256i vx = _mm256_loadu_si256((const __m256i*)src_sobelx);
|
|
__m256i vy = _mm256_loadu_si256((const __m256i*)src_sobely);
|
|
_mm256_storeu_si256((__m256i*)dst_y, _mm256_adds_epu8(vx, vy));
|
|
src_sobelx += 32;
|
|
src_sobely += 32;
|
|
dst_y += 32;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SOBELTOPLANEROW_AVX2
|
|
|
|
#ifdef HAS_SOBELXYROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void SobelXYRow_AVX2(const uint8_t* src_sobelx,
|
|
const uint8_t* src_sobely,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
const __m256i v_ff00 = _mm256_set1_epi16((short)0xff00u);
|
|
while (width > 0) {
|
|
__m128i vx = _mm_loadu_si128((const __m128i*)src_sobelx);
|
|
__m128i vy = _mm_loadu_si128((const __m128i*)src_sobely);
|
|
__m128i vs = _mm_adds_epu8(vx, vy);
|
|
__m256i bg =
|
|
_mm256_or_si256(_mm256_cvtepu8_epi16(vy),
|
|
_mm256_slli_epi16(_mm256_cvtepu8_epi16(vs), 8));
|
|
__m256i ra = _mm256_or_si256(_mm256_cvtepu8_epi16(vx), v_ff00);
|
|
bg = _mm256_permute4x64_epi64(bg, 0xd8);
|
|
ra = _mm256_permute4x64_epi64(ra, 0xd8);
|
|
_mm256_storeu_si256((__m256i*)dst_argb, _mm256_unpacklo_epi16(bg, ra));
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32),
|
|
_mm256_unpackhi_epi16(bg, ra));
|
|
src_sobelx += 16;
|
|
src_sobely += 16;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SOBELXYROW_AVX2
|
|
|
|
#ifdef HAS_ARGBAFFINEROW_AVX2
|
|
LIBYUV_API
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBAffineRow_AVX2(const uint8_t* src_argb,
|
|
int src_argb_stride,
|
|
uint8_t* dst_argb,
|
|
const float* src_dudv,
|
|
int width) {
|
|
static const float kRamp[8] = {0.0f, 1.0f, 2.0f, 3.0f,
|
|
4.0f, 5.0f, 6.0f, 7.0f};
|
|
float uv_tail[4] = {src_dudv[0], src_dudv[1], src_dudv[2], src_dudv[3]};
|
|
if (width >= 8) {
|
|
const __m256 ramp = _mm256_loadu_ps(kRamp);
|
|
const __m256 step8 = _mm256_set1_ps(8.0f);
|
|
__m256 du = _mm256_set1_ps(uv_tail[2]);
|
|
__m256 dv = _mm256_set1_ps(uv_tail[3]);
|
|
__m256 u =
|
|
_mm256_add_ps(_mm256_set1_ps(uv_tail[0]), _mm256_mul_ps(ramp, du));
|
|
__m256 v =
|
|
_mm256_add_ps(_mm256_set1_ps(uv_tail[1]), _mm256_mul_ps(ramp, dv));
|
|
du = _mm256_mul_ps(du, step8);
|
|
dv = _mm256_mul_ps(dv, step8);
|
|
const __m256i v_stride = _mm256_set1_epi32(src_argb_stride);
|
|
while (width >= 8) {
|
|
__m256i iu = _mm256_cvttps_epi32(u);
|
|
__m256i iv = _mm256_cvttps_epi32(v);
|
|
__m256i idx = _mm256_add_epi32(_mm256_slli_epi32(iu, 2),
|
|
_mm256_mullo_epi32(iv, v_stride));
|
|
__m256i pix = _mm256_i32gather_epi32((const int*)src_argb, idx, 1);
|
|
_mm256_storeu_si256((__m256i*)dst_argb, pix);
|
|
u = _mm256_add_ps(u, du);
|
|
v = _mm256_add_ps(v, dv);
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
}
|
|
uv_tail[0] = _mm256_cvtss_f32(u);
|
|
uv_tail[1] = _mm256_cvtss_f32(v);
|
|
_mm256_zeroupper();
|
|
}
|
|
if (width > 0) {
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
ARGBAffineRow_C(src_argb, src_argb_stride, dst_argb, uv_tail, width);
|
|
}
|
|
}
|
|
#endif // HAS_ARGBAFFINEROW_AVX2
|
|
|
|
#ifdef HAS_ARGBCOLORTABLEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBColorTableRow_AVX2(uint8_t* dst_argb,
|
|
const uint8_t* table_argb,
|
|
int width) {
|
|
if (width >= 8) {
|
|
const __m256i m0 = _mm256_set1_epi32(0x000000ff);
|
|
const __m256i m1 = _mm256_set1_epi32(0x0000ff00);
|
|
const __m256i m2 = _mm256_set1_epi32(0x00ff0000);
|
|
const __m256i m3 = _mm256_set1_epi32((int)0xff000000u);
|
|
while (width >= 8) {
|
|
__m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb);
|
|
__m256i c0 =
|
|
_mm256_and_si256(_mm256_i32gather_epi32((const int*)table_argb,
|
|
_mm256_and_si256(v0, m0), 4),
|
|
m0);
|
|
__m256i c1 = _mm256_and_si256(
|
|
_mm256_i32gather_epi32((const int*)table_argb,
|
|
_mm256_and_si256(_mm256_srli_epi32(v0, 8), m0),
|
|
4),
|
|
m1);
|
|
__m256i c2 = _mm256_and_si256(
|
|
_mm256_i32gather_epi32(
|
|
(const int*)table_argb,
|
|
_mm256_and_si256(_mm256_srli_epi32(v0, 16), m0), 4),
|
|
m2);
|
|
__m256i c3 =
|
|
_mm256_and_si256(_mm256_i32gather_epi32((const int*)table_argb,
|
|
_mm256_srli_epi32(v0, 24), 4),
|
|
m3);
|
|
__m256i out =
|
|
_mm256_or_si256(_mm256_or_si256(c0, c1), _mm256_or_si256(c2, c3));
|
|
_mm256_storeu_si256((__m256i*)dst_argb, out);
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
if (width > 0) {
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
ARGBColorTableRow_C(dst_argb, table_argb, width);
|
|
}
|
|
}
|
|
#endif // HAS_ARGBCOLORTABLEROW_AVX2
|
|
|
|
#ifdef HAS_RGBCOLORTABLEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void RGBColorTableRow_AVX2(uint8_t* dst_argb,
|
|
const uint8_t* table_argb,
|
|
int width) {
|
|
if (width >= 8) {
|
|
const __m256i m0 = _mm256_set1_epi32(0x000000ff);
|
|
const __m256i m1 = _mm256_set1_epi32(0x0000ff00);
|
|
const __m256i m2 = _mm256_set1_epi32(0x00ff0000);
|
|
const __m256i m3 = _mm256_set1_epi32((int)0xff000000u);
|
|
while (width >= 8) {
|
|
__m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb);
|
|
__m256i c0 =
|
|
_mm256_and_si256(_mm256_i32gather_epi32((const int*)table_argb,
|
|
_mm256_and_si256(v0, m0), 4),
|
|
m0);
|
|
__m256i c1 = _mm256_and_si256(
|
|
_mm256_i32gather_epi32((const int*)table_argb,
|
|
_mm256_and_si256(_mm256_srli_epi32(v0, 8), m0),
|
|
4),
|
|
m1);
|
|
__m256i c2 = _mm256_and_si256(
|
|
_mm256_i32gather_epi32(
|
|
(const int*)table_argb,
|
|
_mm256_and_si256(_mm256_srli_epi32(v0, 16), m0), 4),
|
|
m2);
|
|
__m256i c3 = _mm256_and_si256(v0, m3);
|
|
__m256i out =
|
|
_mm256_or_si256(_mm256_or_si256(c0, c1), _mm256_or_si256(c2, c3));
|
|
_mm256_storeu_si256((__m256i*)dst_argb, out);
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
if (width > 0) {
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
RGBColorTableRow_C(dst_argb, table_argb, width);
|
|
}
|
|
}
|
|
#endif // HAS_RGBCOLORTABLEROW_AVX2
|
|
|
|
#ifdef HAS_ARGBLUMACOLORTABLEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
int width,
|
|
const uint8_t* luma,
|
|
uint32_t lumacoeff) {
|
|
if (width >= 8) {
|
|
const __m256i c_br = _mm256_set1_epi32((int)(lumacoeff & 0x00ff00ffu));
|
|
const __m256i c_ga =
|
|
_mm256_set1_epi32((int)((lumacoeff >> 8) & 0x000000ffu));
|
|
const __m256i m_00ff = _mm256_set1_epi16(0x00ff);
|
|
const __m256i m_ff = _mm256_set1_epi32(0x000000ff);
|
|
const __m256i m_7f00 = _mm256_set1_epi32(0x00007f00);
|
|
const __m256i m_18 = _mm256_set1_epi32(0x00000018);
|
|
const __m256i m_alpha = _mm256_set1_epi32((int)0xff000000u);
|
|
while (width >= 8) {
|
|
__m256i v0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
src_argb += 32;
|
|
__m256i y = _mm256_add_epi32(
|
|
_mm256_madd_epi16(_mm256_and_si256(v0, m_00ff), c_br),
|
|
_mm256_madd_epi16(_mm256_srli_epi16(v0, 8), c_ga));
|
|
y = _mm256_and_si256(y, m_7f00);
|
|
|
|
__m256i b_idx =
|
|
_mm256_srli_epi32(_mm256_or_si256(_mm256_and_si256(v0, m_ff), y), 2);
|
|
__m256i b_val = _mm256_i32gather_epi32((const int*)luma, b_idx, 4);
|
|
__m256i b_sh = _mm256_and_si256(_mm256_slli_epi32(v0, 3), m_18);
|
|
b_val = _mm256_and_si256(_mm256_srlv_epi32(b_val, b_sh), m_ff);
|
|
|
|
__m256i g_idx = _mm256_srli_epi32(
|
|
_mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v0, 8), m_ff), y),
|
|
2);
|
|
__m256i g_val = _mm256_i32gather_epi32((const int*)luma, g_idx, 4);
|
|
__m256i g_sh = _mm256_and_si256(_mm256_srli_epi32(v0, 5), m_18);
|
|
g_val = _mm256_slli_epi32(
|
|
_mm256_and_si256(_mm256_srlv_epi32(g_val, g_sh), m_ff), 8);
|
|
|
|
__m256i r_idx = _mm256_srli_epi32(
|
|
_mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v0, 16), m_ff), y),
|
|
2);
|
|
__m256i r_val = _mm256_i32gather_epi32((const int*)luma, r_idx, 4);
|
|
__m256i r_sh = _mm256_and_si256(_mm256_srli_epi32(v0, 13), m_18);
|
|
r_val = _mm256_slli_epi32(
|
|
_mm256_and_si256(_mm256_srlv_epi32(r_val, r_sh), m_ff), 16);
|
|
|
|
__m256i a_val = _mm256_and_si256(v0, m_alpha);
|
|
__m256i out = _mm256_or_si256(_mm256_or_si256(b_val, g_val),
|
|
_mm256_or_si256(r_val, a_val));
|
|
_mm256_storeu_si256((__m256i*)dst_argb, out);
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
if (width > 0) {
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
ARGBLumaColorTableRow_C(src_argb, dst_argb, width, luma, lumacoeff);
|
|
}
|
|
}
|
|
#endif // HAS_ARGBLUMACOLORTABLEROW_AVX2
|
|
|
|
#ifdef HAS_BYTETOFLOATROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ByteToFloatRow_AVX2(const uint8_t* src,
|
|
float* dst,
|
|
float scale,
|
|
int width) {
|
|
const __m256 v_scale = _mm256_set1_ps(scale);
|
|
while (width > 0) {
|
|
__m256i i0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)src));
|
|
__m256i i1 =
|
|
_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)(src + 8)));
|
|
_mm256_storeu_ps(dst, _mm256_mul_ps(_mm256_cvtepi32_ps(i0), v_scale));
|
|
_mm256_storeu_ps(dst + 8, _mm256_mul_ps(_mm256_cvtepi32_ps(i1), v_scale));
|
|
src += 16;
|
|
dst += 16;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_BYTETOFLOATROW_AVX2
|
|
|
|
#ifdef HAS_GAUSSROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void GaussRow_AVX2(const uint32_t* src, uint16_t* dst, int width) {
|
|
static const int8_t kShuf32To16[16] = {0, 1, 4, 5, 8, 9, 12, 13,
|
|
-1, -1, -1, -1, -1, -1, -1, -1};
|
|
if (width >= 8) {
|
|
const __m256i v_128 = _mm256_set1_epi32(128);
|
|
const __m256i v_shuf = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShuf32To16));
|
|
while (width >= 8) {
|
|
__m256i s0 = _mm256_loadu_si256((const __m256i*)src);
|
|
__m256i s1 = _mm256_loadu_si256((const __m256i*)(src + 1));
|
|
__m256i s2 = _mm256_loadu_si256((const __m256i*)(src + 2));
|
|
__m256i s3 = _mm256_loadu_si256((const __m256i*)(src + 3));
|
|
__m256i s4 = _mm256_loadu_si256((const __m256i*)(src + 4));
|
|
src += 8;
|
|
__m256i sum04 = _mm256_add_epi32(s0, s4);
|
|
__m256i sum13 = _mm256_slli_epi32(_mm256_add_epi32(s1, s3), 2);
|
|
__m256i sum2 =
|
|
_mm256_add_epi32(_mm256_slli_epi32(s2, 2), _mm256_slli_epi32(s2, 1));
|
|
__m256i acc =
|
|
_mm256_srli_epi32(_mm256_add_epi32(_mm256_add_epi32(sum04, sum13),
|
|
_mm256_add_epi32(sum2, v_128)),
|
|
8);
|
|
acc = _mm256_permute4x64_epi64(_mm256_shuffle_epi8(acc, v_shuf), 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst, _mm256_castsi256_si128(acc));
|
|
dst += 8;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
if (width > 0) {
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
GaussRow_C(src, dst, width);
|
|
}
|
|
}
|
|
#endif // HAS_GAUSSROW_AVX2
|
|
|
|
#ifdef HAS_GAUSSROW_F32_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void GaussRow_F32_AVX2(const float* src, float* dst, int width) {
|
|
if (width >= 8) {
|
|
const __m256 v4 = _mm256_set1_ps(4.0f);
|
|
const __m256 v6 = _mm256_set1_ps(6.0f);
|
|
const __m256 v_scale = _mm256_set1_ps(1.0f / 256.0f);
|
|
while (width >= 8) {
|
|
__m256 acc = _mm256_loadu_ps(src);
|
|
acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_loadu_ps(src + 1), v4));
|
|
acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_loadu_ps(src + 2), v6));
|
|
acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_loadu_ps(src + 3), v4));
|
|
acc = _mm256_add_ps(acc, _mm256_loadu_ps(src + 4));
|
|
acc = _mm256_mul_ps(acc, v_scale);
|
|
_mm256_storeu_ps(dst, acc);
|
|
src += 8;
|
|
dst += 8;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
if (width > 0) {
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
GaussRow_F32_C(src, dst, width);
|
|
}
|
|
}
|
|
#endif // HAS_GAUSSROW_F32_AVX2
|
|
|
|
#ifdef HAS_SETROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void SetRow_AVX2(uint8_t* dst, uint8_t v8, int width) {
|
|
const __m256i v = _mm256_set1_epi8((char)v8);
|
|
while (width >= 64) {
|
|
_mm256_storeu_si256((__m256i*)dst, v);
|
|
_mm256_storeu_si256((__m256i*)(dst + 32), v);
|
|
dst += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
_mm256_storeu_si256((__m256i*)dst, v);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SETROW_AVX2
|
|
|
|
#ifdef HAS_ARGBSETROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBSetRow_AVX2(uint8_t* dst_argb, uint32_t v32, int width) {
|
|
const __m256i v = _mm256_set1_epi32((int)v32);
|
|
while (width >= 16) {
|
|
_mm256_storeu_si256((__m256i*)dst_argb, v);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), v);
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
_mm256_storeu_si256((__m256i*)dst_argb, v);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBSETROW_AVX2
|
|
|
|
#ifdef HAS_COPYROW_16_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void CopyRow_16_AVX2(const uint16_t* src, uint16_t* dst, int width) {
|
|
while (width > 0) {
|
|
__m256i v0 = _mm256_loadu_si256((const __m256i*)src);
|
|
__m256i v1 = _mm256_loadu_si256((const __m256i*)(src + 16));
|
|
_mm256_storeu_si256((__m256i*)dst, v0);
|
|
_mm256_storeu_si256((__m256i*)(dst + 16), v1);
|
|
src += 32;
|
|
dst += 32;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_COPYROW_16_AVX2
|
|
|
|
#ifdef HAS_DETILEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void DetileRow_AVX2(const uint8_t* src,
|
|
ptrdiff_t src_tile_stride,
|
|
uint8_t* dst,
|
|
int width) {
|
|
while (width >= 32) {
|
|
__m128i t0 = _mm_loadu_si128((const __m128i*)src);
|
|
__m128i t1 = _mm_loadu_si128((const __m128i*)(src + src_tile_stride));
|
|
src += 2 * src_tile_stride;
|
|
_mm256_storeu_si256((__m256i*)dst, _mm256_set_m128i(t1, t0));
|
|
dst += 32;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
_mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_DETILEROW_AVX2
|
|
|
|
#ifdef HAS_DETILEROW_16_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void DetileRow_16_AVX2(const uint16_t* src,
|
|
ptrdiff_t src_tile_stride,
|
|
uint16_t* dst,
|
|
int width) {
|
|
while (width >= 32) {
|
|
__m256i t0 = _mm256_loadu_si256((const __m256i*)src);
|
|
__m256i t1 = _mm256_loadu_si256((const __m256i*)(src + src_tile_stride));
|
|
src += 2 * src_tile_stride;
|
|
_mm256_storeu_si256((__m256i*)dst, t0);
|
|
_mm256_storeu_si256((__m256i*)(dst + 16), t1);
|
|
dst += 32;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
_mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_DETILEROW_16_AVX2
|
|
|
|
#ifdef HAS_DETILESPLITUVROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void DetileSplitUVRow_AVX2(const uint8_t* src_uv,
|
|
ptrdiff_t src_tile_stride,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
static const uint8_t kDeinterlaceUV_Win[16] = {0, 2, 4, 6, 8, 10, 12, 14,
|
|
1, 3, 5, 7, 9, 11, 13, 15};
|
|
const __m128i shuf128 = _mm_loadu_si128((const __m128i*)kDeinterlaceUV_Win);
|
|
const __m256i shuf256 = _mm256_broadcastsi128_si256(shuf128);
|
|
while (width >= 32) {
|
|
__m128i t0 = _mm_loadu_si128((const __m128i*)src_uv);
|
|
__m128i t1 = _mm_loadu_si128((const __m128i*)(src_uv + src_tile_stride));
|
|
src_uv += 2 * src_tile_stride;
|
|
__m256i v = _mm256_permute4x64_epi64(
|
|
_mm256_shuffle_epi8(_mm256_set_m128i(t1, t0), shuf256), 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(v));
|
|
_mm_storeu_si128((__m128i*)dst_v, _mm256_extracti128_si256(v, 1));
|
|
dst_u += 16;
|
|
dst_v += 16;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__m128i v =
|
|
_mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)src_uv), shuf128);
|
|
_mm_storel_epi64((__m128i*)dst_u, v);
|
|
_mm_storel_epi64((__m128i*)dst_v, _mm_unpackhi_epi64(v, v));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_DETILESPLITUVROW_AVX2
|
|
|
|
#if defined(__x86_64__) || defined(_M_X64)
|
|
|
|
#ifdef HAS_ARGB1555TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGB1555ToARGBRow_AVX512BW(const uint8_t* src_argb1555,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
const __m512i zmm_f800 = _mm512_set1_epi16((short)0xf800);
|
|
const __m512i zmm_03e0 = _mm512_set1_epi16(0x03e0);
|
|
const __m512i zmm_ff00 = _mm512_set1_epi16((short)0xff00);
|
|
const __m512i zmm_0108 = _mm512_set1_epi16(0x0108);
|
|
const __m512i zmm_4200 = _mm512_set1_epi16(0x4200);
|
|
while (width >= 32) {
|
|
__m512i v0 = _mm512_loadu_si512((const __m512i*)src_argb1555);
|
|
src_argb1555 += 64;
|
|
__m512i r = _mm512_mulhi_epu16(
|
|
_mm512_and_si512(_mm512_slli_epi16(v0, 1), zmm_f800), zmm_0108);
|
|
__m512i b = _mm512_mulhi_epu16(_mm512_slli_epi16(v0, 11), zmm_0108);
|
|
__m512i rb = _mm512_or_si512(_mm512_slli_epi16(r, 8), b);
|
|
__m512i g = _mm512_mulhi_epu16(_mm512_and_si512(v0, zmm_03e0), zmm_4200);
|
|
__m512i a = _mm512_and_si512(_mm512_srai_epi16(v0, 8), zmm_ff00);
|
|
__m512i ga = _mm512_or_si512(g, a);
|
|
__m512i lo = _mm512_unpacklo_epi8(rb, ga);
|
|
__m512i hi = _mm512_unpackhi_epi8(rb, ga);
|
|
__m512i out0 = _mm512_shuffle_i64x2(lo, hi, 0x44);
|
|
__m512i out1 = _mm512_shuffle_i64x2(lo, hi, 0xee);
|
|
out0 = _mm512_shuffle_i64x2(out0, out0, 0xd8);
|
|
out1 = _mm512_shuffle_i64x2(out1, out1, 0xd8);
|
|
_mm512_storeu_si512((__m512i*)dst_argb, out0);
|
|
_mm512_storeu_si512((__m512i*)(dst_argb + 64), out1);
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask32 k = (__mmask32)((1ULL << width) - 1ULL);
|
|
__m512i v0 = _mm512_maskz_loadu_epi16(k, src_argb1555);
|
|
__m512i r = _mm512_mulhi_epu16(
|
|
_mm512_and_si512(_mm512_slli_epi16(v0, 1), zmm_f800), zmm_0108);
|
|
__m512i b = _mm512_mulhi_epu16(_mm512_slli_epi16(v0, 11), zmm_0108);
|
|
__m512i rb = _mm512_or_si512(_mm512_slli_epi16(r, 8), b);
|
|
__m512i g = _mm512_mulhi_epu16(_mm512_and_si512(v0, zmm_03e0), zmm_4200);
|
|
__m512i a = _mm512_and_si512(_mm512_srai_epi16(v0, 8), zmm_ff00);
|
|
__m512i ga = _mm512_or_si512(g, a);
|
|
__m512i lo = _mm512_unpacklo_epi8(rb, ga);
|
|
__m512i hi = _mm512_unpackhi_epi8(rb, ga);
|
|
__m512i out0 = _mm512_shuffle_i64x2(lo, hi, 0x44);
|
|
__m512i out1 = _mm512_shuffle_i64x2(lo, hi, 0xee);
|
|
out0 = _mm512_shuffle_i64x2(out0, out0, 0xd8);
|
|
out1 = _mm512_shuffle_i64x2(out1, out1, 0xd8);
|
|
if (width > 16) {
|
|
_mm512_storeu_si512((__m512i*)dst_argb, out0);
|
|
_mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(k >> 16), out1);
|
|
} else {
|
|
_mm512_mask_storeu_epi32(dst_argb, (__mmask16)k, out0);
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGB1555TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBTORGB565DITHERROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBToRGB565DitherRow_AVX512BW(const uint8_t* src_argb,
|
|
uint8_t* dst_rgb,
|
|
uint32_t dither4,
|
|
int width) {
|
|
__m128i d128 = _mm_cvtsi32_si128((int)dither4);
|
|
d128 = _mm_unpacklo_epi8(d128, d128);
|
|
d128 = _mm_unpacklo_epi16(d128, d128);
|
|
const __m512i zmm_dither = _mm512_broadcast_i32x4(d128);
|
|
const __m512i zmm_001f = _mm512_set1_epi32(0x0000001f);
|
|
const __m512i zmm_07e0 = _mm512_set1_epi32(0x000007e0);
|
|
const __m512i zmm_f800 = _mm512_set1_epi32(0x0000f800);
|
|
while (width >= 16) {
|
|
__m512i v0 = _mm512_adds_epu8(_mm512_loadu_si512((const __m512i*)src_argb),
|
|
zmm_dither);
|
|
src_argb += 64;
|
|
__m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f);
|
|
__m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0);
|
|
__m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800);
|
|
__m256i rgb16 =
|
|
_mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r));
|
|
_mm256_storeu_si256((__m256i*)dst_rgb, rgb16);
|
|
dst_rgb += 32;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i v0 =
|
|
_mm512_adds_epu8(_mm512_maskz_loadu_epi32(k, src_argb), zmm_dither);
|
|
__m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f);
|
|
__m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0);
|
|
__m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800);
|
|
__m256i rgb16 =
|
|
_mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r));
|
|
_mm256_mask_storeu_epi16(dst_rgb, k, rgb16);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTORGB565DITHERROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBTORGB565ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBToRGB565Row_AVX512BW(const uint8_t* src_argb,
|
|
uint8_t* dst_rgb,
|
|
int width) {
|
|
const __m512i zmm_001f = _mm512_set1_epi32(0x0000001f);
|
|
const __m512i zmm_07e0 = _mm512_set1_epi32(0x000007e0);
|
|
const __m512i zmm_f800 = _mm512_set1_epi32(0x0000f800);
|
|
while (width >= 16) {
|
|
__m512i v0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
src_argb += 64;
|
|
__m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f);
|
|
__m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0);
|
|
__m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800);
|
|
__m256i rgb16 =
|
|
_mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r));
|
|
_mm256_storeu_si256((__m256i*)dst_rgb, rgb16);
|
|
dst_rgb += 32;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i v0 = _mm512_maskz_loadu_epi32(k, src_argb);
|
|
__m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f);
|
|
__m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0);
|
|
__m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800);
|
|
__m256i rgb16 =
|
|
_mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r));
|
|
_mm256_mask_storeu_epi16(dst_rgb, k, rgb16);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTORGB565ROW_AVX512BW
|
|
|
|
#ifdef HAS_MERGEUVROW_16_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void MergeUVRow_16_AVX512BW(const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
uint16_t* dst_uv,
|
|
int depth,
|
|
int width) {
|
|
const __m128i sh_u = _mm_cvtsi32_si128(16 - depth);
|
|
const __m128i sh_v = _mm_cvtsi32_si128(32 - depth);
|
|
while (width >= 16) {
|
|
__m512i u32 =
|
|
_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_u));
|
|
__m512i v32 =
|
|
_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_v));
|
|
src_u += 16;
|
|
src_v += 16;
|
|
u32 = _mm512_sll_epi16(u32, sh_u);
|
|
v32 = _mm512_sll_epi32(v32, sh_v);
|
|
_mm512_storeu_si512((__m512i*)dst_uv, _mm512_or_si512(u32, v32));
|
|
dst_uv += 32;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i u32 = _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_u));
|
|
__m512i v32 = _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_v));
|
|
u32 = _mm512_sll_epi16(u32, sh_u);
|
|
v32 = _mm512_sll_epi32(v32, sh_v);
|
|
_mm512_mask_storeu_epi32(dst_uv, k, _mm512_or_si512(u32, v32));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGEUVROW_16_AVX512BW
|
|
|
|
#ifdef HAS_DIVIDEROW_16_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void DivideRow_16_AVX512BW(const uint16_t* src_y,
|
|
uint16_t* dst_y,
|
|
int scale,
|
|
int width) {
|
|
if (scale == 65536) { // Divide by 1, so do memcpy.
|
|
if (src_y != dst_y) {
|
|
memcpy(dst_y, src_y, (size_t)width * 2);
|
|
}
|
|
return;
|
|
}
|
|
const __m512i zmm_scale = _mm512_set1_epi16((short)scale);
|
|
while (width >= 32) {
|
|
__m512i v = _mm512_loadu_si512((const __m512i*)src_y);
|
|
src_y += 32;
|
|
_mm512_storeu_si512((__m512i*)dst_y, _mm512_mulhi_epu16(v, zmm_scale));
|
|
dst_y += 32;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask32 k = (__mmask32)((1u << width) - 1u);
|
|
__m512i v = _mm512_maskz_loadu_epi16(k, src_y);
|
|
_mm512_mask_storeu_epi16(dst_y, k, _mm512_mulhi_epu16(v, zmm_scale));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_DIVIDEROW_16_AVX512BW
|
|
|
|
#ifdef HAS_YUY2TOYROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
|
|
while (width >= 32) {
|
|
__m512i v = _mm512_loadu_si512((const __m512i*)src_yuy2);
|
|
src_yuy2 += 64;
|
|
_mm256_storeu_si256((__m256i*)dst_y, _mm512_cvtepi16_epi8(v));
|
|
dst_y += 32;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask32 k = (__mmask32)((1u << width) - 1u);
|
|
__m512i v = _mm512_maskz_loadu_epi16(k, src_yuy2);
|
|
_mm256_mask_storeu_epi8(dst_y, k, _mm512_cvtepi16_epi8(v));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_YUY2TOYROW_AVX512BW
|
|
|
|
#ifdef HAS_YUY2TOUVROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2,
|
|
int stride_yuy2,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
const ptrdiff_t stride = (ptrdiff_t)stride_yuy2;
|
|
while (width >= 32) {
|
|
__m512i r0 = _mm512_loadu_si512((const __m512i*)src_yuy2);
|
|
__m512i r1 = _mm512_loadu_si512((const __m512i*)(src_yuy2 + stride));
|
|
src_yuy2 += 64;
|
|
__m512i avg = _mm512_avg_epu8(r0, r1);
|
|
_mm_storeu_si128((__m128i*)dst_u,
|
|
_mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 8)));
|
|
_mm_storeu_si128((__m128i*)dst_v,
|
|
_mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 24)));
|
|
dst_u += 16;
|
|
dst_v += 16;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m512i r0 = _mm512_maskz_loadu_epi32(k, src_yuy2);
|
|
__m512i r1 = _mm512_maskz_loadu_epi32(k, src_yuy2 + stride);
|
|
__m512i avg = _mm512_avg_epu8(r0, r1);
|
|
_mm_mask_storeu_epi8(dst_u, k,
|
|
_mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 8)));
|
|
_mm_mask_storeu_epi8(dst_v, k,
|
|
_mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 24)));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_YUY2TOUVROW_AVX512BW
|
|
|
|
#ifdef HAS_UYVYTOYROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
|
|
while (width >= 32) {
|
|
__m512i v = _mm512_loadu_si512((const __m512i*)src_uyvy);
|
|
src_uyvy += 64;
|
|
_mm256_storeu_si256((__m256i*)dst_y,
|
|
_mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8)));
|
|
dst_y += 32;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask32 k = (__mmask32)((1u << width) - 1u);
|
|
__m512i v = _mm512_maskz_loadu_epi16(k, src_uyvy);
|
|
_mm256_mask_storeu_epi8(dst_y, k,
|
|
_mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8)));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_UYVYTOYROW_AVX512BW
|
|
|
|
#ifdef HAS_UYVYTOUVROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy,
|
|
int stride_uyvy,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
const ptrdiff_t stride = (ptrdiff_t)stride_uyvy;
|
|
while (width >= 32) {
|
|
__m512i r0 = _mm512_loadu_si512((const __m512i*)src_uyvy);
|
|
__m512i r1 = _mm512_loadu_si512((const __m512i*)(src_uyvy + stride));
|
|
src_uyvy += 64;
|
|
__m512i avg = _mm512_avg_epu8(r0, r1);
|
|
_mm_storeu_si128((__m128i*)dst_u, _mm512_cvtepi32_epi8(avg));
|
|
_mm_storeu_si128((__m128i*)dst_v,
|
|
_mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 16)));
|
|
dst_u += 16;
|
|
dst_v += 16;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m512i r0 = _mm512_maskz_loadu_epi32(k, src_uyvy);
|
|
__m512i r1 = _mm512_maskz_loadu_epi32(k, src_uyvy + stride);
|
|
__m512i avg = _mm512_avg_epu8(r0, r1);
|
|
_mm_mask_storeu_epi8(dst_u, k, _mm512_cvtepi32_epi8(avg));
|
|
_mm_mask_storeu_epi8(dst_v, k,
|
|
_mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 16)));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_UYVYTOUVROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBSEPIAROW_AVX512VNNI
|
|
LIBYUV_TARGET_AVX512VNNI
|
|
void ARGBSepiaRow_AVX512VNNI(uint8_t* dst_argb, int width) {
|
|
const __m512i zero = _mm512_setzero_si512();
|
|
const __m512i c_b = _mm512_set1_epi32(0x00234411);
|
|
const __m512i c_g = _mm512_set1_epi32(0x002d5816);
|
|
const __m512i c_r = _mm512_set1_epi32(0x00326218);
|
|
const __m512i max255 = _mm512_set1_epi32(255);
|
|
const __m512i a_mask = _mm512_set1_epi32((int)0xff000000u);
|
|
while (width >= 16) {
|
|
__m512i v0 = _mm512_loadu_si512((const __m512i*)dst_argb);
|
|
__m512i b = _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_b), 7);
|
|
__m512i g = _mm512_min_epu32(
|
|
_mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_g), 7), max255);
|
|
__m512i r = _mm512_min_epu32(
|
|
_mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_r), 7), max255);
|
|
__m512i a = _mm512_and_si512(v0, a_mask);
|
|
__m512i out = _mm512_or_si512(_mm512_or_si512(b, _mm512_slli_epi32(g, 8)),
|
|
_mm512_or_si512(_mm512_slli_epi32(r, 16), a));
|
|
_mm512_storeu_si512((__m512i*)dst_argb, out);
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i v0 = _mm512_maskz_loadu_epi32(k, dst_argb);
|
|
__m512i b = _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_b), 7);
|
|
__m512i g = _mm512_min_epu32(
|
|
_mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_g), 7), max255);
|
|
__m512i r = _mm512_min_epu32(
|
|
_mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_r), 7), max255);
|
|
__m512i a = _mm512_and_si512(v0, a_mask);
|
|
__m512i out = _mm512_or_si512(_mm512_or_si512(b, _mm512_slli_epi32(g, 8)),
|
|
_mm512_or_si512(_mm512_slli_epi32(r, 16), a));
|
|
_mm512_mask_storeu_epi32(dst_argb, k, out);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBSEPIAROW_AVX512VNNI
|
|
|
|
#ifdef HAS_ARGBCOLORMATRIXROW_AVX512VNNI
|
|
LIBYUV_TARGET_AVX512VNNI
|
|
void ARGBColorMatrixRow_AVX512VNNI(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
const int8_t* matrix_argb,
|
|
int width) {
|
|
const __m512i zero = _mm512_setzero_si512();
|
|
const __m512i max255 = _mm512_set1_epi32(255);
|
|
const __m128i m128 = _mm_loadu_si128((const __m128i*)matrix_argb);
|
|
const __m512i m0 = _mm512_set1_epi32(_mm_cvtsi128_si32(m128));
|
|
const __m512i m1 =
|
|
_mm512_set1_epi32(_mm_cvtsi128_si32(_mm_srli_si128(m128, 4)));
|
|
const __m512i m2 =
|
|
_mm512_set1_epi32(_mm_cvtsi128_si32(_mm_srli_si128(m128, 8)));
|
|
const __m512i m3 =
|
|
_mm512_set1_epi32(_mm_cvtsi128_si32(_mm_srli_si128(m128, 12)));
|
|
while (width >= 16) {
|
|
__m512i v0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
src_argb += 64;
|
|
__m512i b = _mm512_min_epu32(
|
|
_mm512_max_epi32(
|
|
_mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m0), 6), zero),
|
|
max255);
|
|
__m512i g = _mm512_min_epu32(
|
|
_mm512_max_epi32(
|
|
_mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m1), 6), zero),
|
|
max255);
|
|
__m512i r = _mm512_min_epu32(
|
|
_mm512_max_epi32(
|
|
_mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m2), 6), zero),
|
|
max255);
|
|
__m512i a = _mm512_min_epu32(
|
|
_mm512_max_epi32(
|
|
_mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m3), 6), zero),
|
|
max255);
|
|
__m512i out = _mm512_or_si512(
|
|
_mm512_or_si512(b, _mm512_slli_epi32(g, 8)),
|
|
_mm512_or_si512(_mm512_slli_epi32(r, 16), _mm512_slli_epi32(a, 24)));
|
|
_mm512_storeu_si512((__m512i*)dst_argb, out);
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i v0 = _mm512_maskz_loadu_epi32(k, src_argb);
|
|
__m512i b = _mm512_min_epu32(
|
|
_mm512_max_epi32(
|
|
_mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m0), 6), zero),
|
|
max255);
|
|
__m512i g = _mm512_min_epu32(
|
|
_mm512_max_epi32(
|
|
_mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m1), 6), zero),
|
|
max255);
|
|
__m512i r = _mm512_min_epu32(
|
|
_mm512_max_epi32(
|
|
_mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m2), 6), zero),
|
|
max255);
|
|
__m512i a = _mm512_min_epu32(
|
|
_mm512_max_epi32(
|
|
_mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m3), 6), zero),
|
|
max255);
|
|
__m512i out = _mm512_or_si512(
|
|
_mm512_or_si512(b, _mm512_slli_epi32(g, 8)),
|
|
_mm512_or_si512(_mm512_slli_epi32(r, 16), _mm512_slli_epi32(a, 24)));
|
|
_mm512_mask_storeu_epi32(dst_argb, k, out);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBCOLORMATRIXROW_AVX512VNNI
|
|
|
|
#ifdef HAS_HALFFLOATROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void HalfFloatRow_AVX512BW(const uint16_t* src,
|
|
uint16_t* dst,
|
|
float scale,
|
|
int width) {
|
|
const __m512 vscale = _mm512_set1_ps(scale);
|
|
while (width >= 16) {
|
|
__m512i s32 =
|
|
_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src));
|
|
src += 16;
|
|
__m512 f = _mm512_mul_ps(_mm512_cvtepi32_ps(s32), vscale);
|
|
__m256i h16 = _mm512_cvtps_ph(f, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
|
|
_mm256_storeu_si256((__m256i*)dst, h16);
|
|
dst += 16;
|
|
width -= 16;
|
|
}
|
|
if (width > 0) {
|
|
__mmask16 k = (__mmask16)((1u << width) - 1u);
|
|
__m512i s32 = _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src));
|
|
__m512 f = _mm512_mul_ps(_mm512_cvtepi32_ps(s32), vscale);
|
|
__m256i h16 = _mm512_cvtps_ph(f, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
|
|
_mm256_mask_storeu_epi16(dst, k, h16);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_HALFFLOATROW_AVX512BW
|
|
|
|
#ifdef HAS_CONVERT8TO8ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void Convert8To8Row_AVX512BW(const uint8_t* src_y,
|
|
uint8_t* dst_y,
|
|
int scale,
|
|
int bias,
|
|
int width) {
|
|
const __m512i zmm_scale =
|
|
_mm512_slli_epi16(_mm512_set1_epi16((short)scale), 8);
|
|
const __m512i zmm_bias = _mm512_set1_epi8((char)bias);
|
|
const __m512i zmm_zero = _mm512_setzero_si512();
|
|
while (width >= 64) {
|
|
__m512i y8 = _mm512_loadu_si512((const __m512i*)src_y);
|
|
src_y += 64;
|
|
__m512i y_lo =
|
|
_mm512_mulhi_epu16(_mm512_unpacklo_epi8(y8, zmm_zero), zmm_scale);
|
|
__m512i y_hi =
|
|
_mm512_mulhi_epu16(_mm512_unpackhi_epi8(y8, zmm_zero), zmm_scale);
|
|
__m512i out8 = _mm512_add_epi8(_mm512_packus_epi16(y_lo, y_hi), zmm_bias);
|
|
_mm512_storeu_si512((__m512i*)dst_y, out8);
|
|
dst_y += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
__mmask64 k = (__mmask64)((1ULL << width) - 1ULL);
|
|
__m512i y8 = _mm512_maskz_loadu_epi8(k, src_y);
|
|
__m512i y_lo =
|
|
_mm512_mulhi_epu16(_mm512_unpacklo_epi8(y8, zmm_zero), zmm_scale);
|
|
__m512i y_hi =
|
|
_mm512_mulhi_epu16(_mm512_unpackhi_epi8(y8, zmm_zero), zmm_scale);
|
|
__m512i out8 = _mm512_add_epi8(_mm512_packus_epi16(y_lo, y_hi), zmm_bias);
|
|
_mm512_mask_storeu_epi8(dst_y, k, out8);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_CONVERT8TO8ROW_AVX512BW
|
|
|
|
// ===========================================================================
|
|
// Extended YUV (I422, I444, YUY2, UYVY, I210, I212, I410, P210, P410)
|
|
// to ARGB / RGBA / RGB24 / AR30 (AVX512BW / AVX512VBMI)
|
|
// ===========================================================================
|
|
|
|
static const uint16_t kSplitQuadWords_ExtWin[32] = {
|
|
0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
|
|
8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
|
|
static const uint64_t kUnpermuteAVX512_ExtWin[8] = {0, 4, 1, 5, 2, 6, 3, 7};
|
|
static const uint16_t kPermAR30_0_Win[32] = {
|
|
0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39,
|
|
8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47};
|
|
static const uint16_t kPermAR30_1_Win[32] = {
|
|
16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55,
|
|
24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63};
|
|
static const uint8_t kShuffleYUY2Y_Win[16] = {0, 0, 2, 2, 4, 4, 6, 6,
|
|
8, 8, 10, 10, 12, 12, 14, 14};
|
|
static const uint8_t kShuffleYUY2UV_Win[16] = {1, 3, 1, 3, 5, 7, 5, 7,
|
|
9, 11, 9, 11, 13, 15, 13, 15};
|
|
static const uint8_t kShuffleUYVYY_Win[16] = {1, 1, 3, 3, 5, 5, 7, 7,
|
|
9, 9, 11, 11, 13, 13, 15, 15};
|
|
static const uint8_t kShuffleUYVYUV_Win[16] = {0, 2, 0, 2, 4, 6, 4, 6,
|
|
8, 10, 8, 10, 12, 14, 12, 14};
|
|
|
|
#define YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants) \
|
|
const __m512i zmm_kUVToB = _mm512_broadcast_i32x4( \
|
|
_mm_loadu_si128((const __m128i*)(yuvconstants)->kUVToB)); \
|
|
const __m512i zmm_kUVToG = _mm512_broadcast_i32x4( \
|
|
_mm_loadu_si128((const __m128i*)(yuvconstants)->kUVToG)); \
|
|
const __m512i zmm_kUVToR = _mm512_broadcast_i32x4( \
|
|
_mm_loadu_si128((const __m128i*)(yuvconstants)->kUVToR)); \
|
|
const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4( \
|
|
_mm_loadu_si128((const __m128i*)(yuvconstants)->kYToRgb)); \
|
|
const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4( \
|
|
_mm_loadu_si128((const __m128i*)(yuvconstants)->kYBiasToRgb)); \
|
|
const __m512i zmm_128 = _mm512_set1_epi8((char)0x80)
|
|
|
|
#define YUVTORGB16_WIN_AVX512BW \
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128); \
|
|
zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb), \
|
|
zmm_kYBiasToRgb); \
|
|
__m512i b16 = \
|
|
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4); \
|
|
__m512i g16 = \
|
|
_mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)); \
|
|
__m512i r16 = _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4)
|
|
|
|
#define YUVTORGB8_WIN_AVX512BW \
|
|
YUVTORGB16_WIN_AVX512BW; \
|
|
__m512i zmm0 = _mm512_packus_epi16(_mm512_srai_epi16(b16, 6), zmm5); \
|
|
__m512i zmm1 = _mm512_packus_epi16(_mm512_srai_epi16(g16, 6), zmm5); \
|
|
__m512i zmm2 = _mm512_packus_epi16(_mm512_srai_epi16(r16, 6), zmm5)
|
|
|
|
#define STOREARGB_WIN_AVX512BW(dst_ptr) \
|
|
do { \
|
|
__m512i bg = \
|
|
PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); \
|
|
__m512i ra = \
|
|
PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); \
|
|
_mm512_storeu_si512((__m512i*)(dst_ptr), _mm512_unpacklo_epi16(bg, ra)); \
|
|
_mm512_storeu_si512((__m512i*)((dst_ptr) + 64), \
|
|
_mm512_unpackhi_epi16(bg, ra)); \
|
|
} while (0)
|
|
|
|
#define STOREARGB_MASK_WIN_AVX512BW(dst_ptr, m32) \
|
|
do { \
|
|
__m512i bg = \
|
|
PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); \
|
|
__m512i ra = \
|
|
PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); \
|
|
_mm512_mask_storeu_epi32((dst_ptr), (__mmask16)(m32), \
|
|
_mm512_unpacklo_epi16(bg, ra)); \
|
|
_mm512_mask_storeu_epi32((dst_ptr) + 64, (__mmask16)((m32) >> 16), \
|
|
_mm512_unpackhi_epi16(bg, ra)); \
|
|
} while (0)
|
|
|
|
#define PACKAR30_WIN_AVX512BW(dst0, dst1) \
|
|
b16 = _mm512_max_epi16(_mm512_min_epi16(b16, zmm_3ff0), zmm_zero); \
|
|
g16 = _mm512_max_epi16(_mm512_min_epi16(g16, zmm_3ff0), zmm_zero); \
|
|
r16 = _mm512_max_epi16(_mm512_min_epi16(r16, zmm_3ff0), zmm_zero); \
|
|
__m512i lo16 = \
|
|
_mm512_or_si512(_mm512_srli_epi16(b16, 4), \
|
|
_mm512_and_si512(_mm512_slli_epi16(g16, 6), zmm_fc00)); \
|
|
__m512i hi16 = \
|
|
_mm512_or_si512(_mm512_or_si512(_mm512_and_si512(r16, zmm_3ff0), \
|
|
_mm512_srli_epi16(g16, 10)), \
|
|
zmm_c000); \
|
|
__m512i dst0 = _mm512_permutex2var_epi16(lo16, zmm_perm0, hi16); \
|
|
__m512i dst1 = _mm512_permutex2var_epi16(lo16, zmm_perm1, hi16)
|
|
|
|
#ifdef HAS_I422TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I422ToARGBRow_AVX512BW(const uint8_t* y_buf,
|
|
const uint8_t* u_buf,
|
|
const uint8_t* v_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_quad =
|
|
_mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
|
|
const __m512i zmm5 = _mm512_set1_epi32(-1);
|
|
while (width >= 32) {
|
|
__m128i xu = _mm_loadu_si128((const __m128i*)u_buf);
|
|
__m128i xv = _mm_loadu_si128((const __m128i*)v_buf);
|
|
u_buf += 16;
|
|
v_buf += 16;
|
|
__m256i uv256 = _mm256_inserti128_si256(
|
|
_mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)),
|
|
_mm_unpackhi_epi8(xu, xv), 1);
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
|
|
y_buf += 32;
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_WIN_AVX512BW(dst_argb);
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m128i xu = _mm_maskz_loadu_epi8(k_uv, u_buf);
|
|
__m128i xv = _mm_maskz_loadu_epi8(k_uv, v_buf);
|
|
__m256i uv256 = _mm256_inserti128_si256(
|
|
_mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)),
|
|
_mm_unpackhi_epi8(xu, xv), 1);
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_I422TORGBAROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I422ToRGBARow_AVX512BW(const uint8_t* y_buf,
|
|
const uint8_t* u_buf,
|
|
const uint8_t* v_buf,
|
|
uint8_t* dst_rgba,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_quad =
|
|
_mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
|
|
const __m512i zmm5 = _mm512_set1_epi32(-1);
|
|
while (width >= 32) {
|
|
__m128i xu = _mm_loadu_si128((const __m128i*)u_buf);
|
|
__m128i xv = _mm_loadu_si128((const __m128i*)v_buf);
|
|
u_buf += 16;
|
|
v_buf += 16;
|
|
__m256i uv256 = _mm256_inserti128_si256(
|
|
_mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)),
|
|
_mm_unpackhi_epi8(xu, xv), 1);
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
|
|
y_buf += 32;
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
__m512i ab = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0));
|
|
__m512i gr = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2));
|
|
_mm512_storeu_si512((__m512i*)dst_rgba, _mm512_unpacklo_epi16(ab, gr));
|
|
_mm512_storeu_si512((__m512i*)(dst_rgba + 64),
|
|
_mm512_unpackhi_epi16(ab, gr));
|
|
dst_rgba += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m128i xu = _mm_maskz_loadu_epi8(k_uv, u_buf);
|
|
__m128i xv = _mm_maskz_loadu_epi8(k_uv, v_buf);
|
|
__m256i uv256 = _mm256_inserti128_si256(
|
|
_mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)),
|
|
_mm_unpackhi_epi8(xu, xv), 1);
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
__m512i ab = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0));
|
|
__m512i gr = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2));
|
|
_mm512_mask_storeu_epi32(dst_rgba, (__mmask16)m32,
|
|
_mm512_unpacklo_epi16(ab, gr));
|
|
_mm512_mask_storeu_epi32(dst_rgba + 64, (__mmask16)(m32 >> 16),
|
|
_mm512_unpackhi_epi16(ab, gr));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422TORGBAROW_AVX512BW
|
|
|
|
#ifdef HAS_I444TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I444ToARGBRow_AVX512BW(const uint8_t* y_buf,
|
|
const uint8_t* u_buf,
|
|
const uint8_t* v_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
|
|
const __m512i zmm5 = _mm512_set1_epi32(-1);
|
|
while (width >= 32) {
|
|
__m512i u16 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)u_buf));
|
|
__m512i v16 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)v_buf));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
|
|
u_buf += 32;
|
|
v_buf += 32;
|
|
y_buf += 32;
|
|
__m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_WIN_AVX512BW(dst_argb);
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask32 k = (__mmask32)m32;
|
|
__m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, u_buf));
|
|
__m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, v_buf));
|
|
__m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, y_buf));
|
|
__m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I444TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_I444ALPHATOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I444AlphaToARGBRow_AVX512BW(const uint8_t* y_buf,
|
|
const uint8_t* u_buf,
|
|
const uint8_t* v_buf,
|
|
const uint8_t* a_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
|
|
while (width >= 32) {
|
|
__m512i u16 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)u_buf));
|
|
__m512i v16 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)v_buf));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
|
|
__m512i a16 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)a_buf));
|
|
u_buf += 32;
|
|
v_buf += 32;
|
|
y_buf += 32;
|
|
a_buf += 32;
|
|
__m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
__m512i zmm5 = _mm512_packus_epi16(a16, a16);
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_WIN_AVX512BW(dst_argb);
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask32 k = (__mmask32)m32;
|
|
__m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, u_buf));
|
|
__m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, v_buf));
|
|
__m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, y_buf));
|
|
__m512i a16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, a_buf));
|
|
__m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
__m512i zmm5 = _mm512_packus_epi16(a16, a16);
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I444ALPHATOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_I444TORGB24ROW_AVX512VBMI
|
|
static const uint8_t kMaskBG_I444[64] = {
|
|
0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05,
|
|
0x45, 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52,
|
|
0x13, 0x53, 0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20,
|
|
0x60, 0x21, 0x61, 0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65,
|
|
0x26, 0x66, 0x27, 0x67, 0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33,
|
|
0x73, 0x34, 0x74, 0x35, 0x75, 0x36, 0x76, 0x37, 0x77};
|
|
static const uint8_t kMaskDST0_I444[64] = {
|
|
0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07,
|
|
0x43, 0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e,
|
|
0x0f, 0x47, 0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52,
|
|
0x16, 0x17, 0x53, 0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d,
|
|
0x56, 0x1e, 0x1f, 0x57, 0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24,
|
|
0x25, 0x62, 0x26, 0x27, 0x63, 0x28, 0x29, 0x64, 0x2a};
|
|
static const uint8_t kMaskDST1_I444[64] = {
|
|
0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70,
|
|
0x32, 0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39,
|
|
0x74, 0x3a, 0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00,
|
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
|
|
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
void I444ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
|
|
const uint8_t* u_buf,
|
|
const uint8_t* v_buf,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG_I444);
|
|
const __m512i zmm_mask_DST0 =
|
|
_mm512_loadu_si512((const __m512i*)kMaskDST0_I444);
|
|
const __m512i zmm_mask_DST1 =
|
|
_mm512_loadu_si512((const __m512i*)kMaskDST1_I444);
|
|
while (width >= 32) {
|
|
__m512i u16 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)u_buf));
|
|
__m512i v16 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)v_buf));
|
|
__m512i zmm4 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
|
|
u_buf += 32;
|
|
v_buf += 32;
|
|
y_buf += 32;
|
|
__m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
YUVTORGB16_WIN_AVX512BW;
|
|
__m512i zmm0 = _mm512_srai_epi16(b16, 6);
|
|
__m512i zmm1 = _mm512_srai_epi16(g16, 6);
|
|
__m512i zmm2 = _mm512_srai_epi16(r16, 6);
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
|
|
zmm1 = _mm512_packus_epi16(zmm1, zmm1);
|
|
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
|
|
__m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
|
|
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0);
|
|
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64),
|
|
_mm512_castsi512_si256(zmm_dst1));
|
|
dst_rgb24 += 96;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask32 k = (__mmask32)((1u << width) - 1u);
|
|
__m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, u_buf));
|
|
__m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, v_buf));
|
|
__m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, y_buf));
|
|
__m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
|
|
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
|
|
YUVTORGB16_WIN_AVX512BW;
|
|
__m512i zmm0 = _mm512_srai_epi16(b16, 6);
|
|
__m512i zmm1 = _mm512_srai_epi16(g16, 6);
|
|
__m512i zmm2 = _mm512_srai_epi16(r16, 6);
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
|
|
zmm1 = _mm512_packus_epi16(zmm1, zmm1);
|
|
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
|
|
__m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
|
|
int dst_bytes = width * 3;
|
|
__mmask64 k0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull;
|
|
__mmask32 k1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u;
|
|
_mm512_mask_storeu_epi8(dst_rgb24, k0, zmm_dst0);
|
|
_mm256_mask_storeu_epi8(dst_rgb24 + 64, k1,
|
|
_mm512_castsi512_si256(zmm_dst1));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I444TORGB24ROW_AVX512VBMI
|
|
|
|
#ifdef HAS_YUY2TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void YUY2ToARGBRow_AVX512BW(const uint8_t* yuy2_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
|
|
const __m512i zmm_shuf_y = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)kShuffleYUY2Y_Win));
|
|
const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)kShuffleYUY2UV_Win));
|
|
const __m512i zmm5 = _mm512_set1_epi32(-1);
|
|
while (width >= 32) {
|
|
__m512i in = _mm512_loadu_si512((const __m512i*)yuy2_buf);
|
|
yuy2_buf += 64;
|
|
__m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y);
|
|
__m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv);
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_WIN_AVX512BW(dst_argb);
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k2 = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m512i in = _mm512_maskz_loadu_epi32(k2, yuy2_buf);
|
|
__m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y);
|
|
__m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv);
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_YUY2TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_UYVYTOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void UYVYToARGBRow_AVX512BW(const uint8_t* uyvy_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
|
|
const __m512i zmm_shuf_y = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)kShuffleUYVYY_Win));
|
|
const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)kShuffleUYVYUV_Win));
|
|
const __m512i zmm5 = _mm512_set1_epi32(-1);
|
|
while (width >= 32) {
|
|
__m512i in = _mm512_loadu_si512((const __m512i*)uyvy_buf);
|
|
uyvy_buf += 64;
|
|
__m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y);
|
|
__m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv);
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_WIN_AVX512BW(dst_argb);
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k2 = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m512i in = _mm512_maskz_loadu_epi32(k2, uyvy_buf);
|
|
__m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y);
|
|
__m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv);
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_UYVYTOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_I210TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I210ToARGBRow_AVX512BW(const uint16_t* y_buf,
|
|
const uint16_t* u_buf,
|
|
const uint16_t* v_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_quad =
|
|
_mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
|
|
const __m512i zmm5 = _mm512_set1_epi32(-1);
|
|
while (width >= 32) {
|
|
__m256i u_s =
|
|
_mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 2);
|
|
__m256i v_s =
|
|
_mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 2);
|
|
__m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
|
|
u_buf += 16;
|
|
v_buf += 16;
|
|
y_buf += 32;
|
|
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
|
|
_mm256_packus_epi16(v_s, v_s));
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_WIN_AVX512BW(dst_argb);
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m256i u_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 2);
|
|
__m256i v_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 2);
|
|
__m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
|
|
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
|
|
_mm256_packus_epi16(v_s, v_s));
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I210TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_I210TOAR30ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I210ToAR30Row_AVX512BW(const uint16_t* y_buf,
|
|
const uint16_t* u_buf,
|
|
const uint16_t* v_buf,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_quad =
|
|
_mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin);
|
|
const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win);
|
|
const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win);
|
|
const __m512i zmm_zero = _mm512_setzero_si512();
|
|
const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0);
|
|
const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00);
|
|
const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000);
|
|
while (width >= 32) {
|
|
__m256i u_s =
|
|
_mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 2);
|
|
__m256i v_s =
|
|
_mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 2);
|
|
__m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
|
|
u_buf += 16;
|
|
v_buf += 16;
|
|
y_buf += 32;
|
|
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
|
|
_mm256_packus_epi16(v_s, v_s));
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
|
|
YUVTORGB16_WIN_AVX512BW;
|
|
PACKAR30_WIN_AVX512BW(dst0, dst1);
|
|
_mm512_storeu_si512((__m512i*)dst_ar30, dst0);
|
|
_mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1);
|
|
dst_ar30 += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m256i u_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 2);
|
|
__m256i v_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 2);
|
|
__m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
|
|
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
|
|
_mm256_packus_epi16(v_s, v_s));
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
|
|
YUVTORGB16_WIN_AVX512BW;
|
|
PACKAR30_WIN_AVX512BW(dst0, dst1);
|
|
_mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0);
|
|
_mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I210TOAR30ROW_AVX512BW
|
|
|
|
#ifdef HAS_I210ALPHATOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf,
|
|
const uint16_t* u_buf,
|
|
const uint16_t* v_buf,
|
|
const uint16_t* a_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_quad =
|
|
_mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
|
|
while (width >= 32) {
|
|
__m256i u_s =
|
|
_mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 2);
|
|
__m256i v_s =
|
|
_mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 2);
|
|
__m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
|
|
__m512i a16 =
|
|
_mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)a_buf), 2);
|
|
u_buf += 16;
|
|
v_buf += 16;
|
|
y_buf += 32;
|
|
a_buf += 32;
|
|
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
|
|
_mm256_packus_epi16(v_s, v_s));
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
|
|
__m512i zmm5 = _mm512_packus_epi16(a16, a16);
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_WIN_AVX512BW(dst_argb);
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m256i u_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 2);
|
|
__m256i v_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 2);
|
|
__m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
|
|
__m512i a16 =
|
|
_mm512_srli_epi16(_mm512_maskz_loadu_epi16((__mmask32)m32, a_buf), 2);
|
|
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
|
|
_mm256_packus_epi16(v_s, v_s));
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
|
|
__m512i zmm5 = _mm512_packus_epi16(a16, a16);
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I210ALPHATOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_I212TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I212ToARGBRow_AVX512BW(const uint16_t* y_buf,
|
|
const uint16_t* u_buf,
|
|
const uint16_t* v_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_quad =
|
|
_mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
|
|
const __m512i zmm5 = _mm512_set1_epi32(-1);
|
|
while (width >= 32) {
|
|
__m256i u_s =
|
|
_mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 4);
|
|
__m256i v_s =
|
|
_mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 4);
|
|
__m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
|
|
u_buf += 16;
|
|
v_buf += 16;
|
|
y_buf += 32;
|
|
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
|
|
_mm256_packus_epi16(v_s, v_s));
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8));
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_WIN_AVX512BW(dst_argb);
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m256i u_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 4);
|
|
__m256i v_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 4);
|
|
__m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
|
|
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
|
|
_mm256_packus_epi16(v_s, v_s));
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8));
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I212TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_I212TOAR30ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I212ToAR30Row_AVX512BW(const uint16_t* y_buf,
|
|
const uint16_t* u_buf,
|
|
const uint16_t* v_buf,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_quad =
|
|
_mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin);
|
|
const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win);
|
|
const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win);
|
|
const __m512i zmm_zero = _mm512_setzero_si512();
|
|
const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0);
|
|
const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00);
|
|
const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000);
|
|
while (width >= 32) {
|
|
__m256i u_s =
|
|
_mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 4);
|
|
__m256i v_s =
|
|
_mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 4);
|
|
__m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
|
|
u_buf += 16;
|
|
v_buf += 16;
|
|
y_buf += 32;
|
|
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
|
|
_mm256_packus_epi16(v_s, v_s));
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8));
|
|
YUVTORGB16_WIN_AVX512BW;
|
|
PACKAR30_WIN_AVX512BW(dst0, dst1);
|
|
_mm512_storeu_si512((__m512i*)dst_ar30, dst0);
|
|
_mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1);
|
|
dst_ar30 += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m256i u_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 4);
|
|
__m256i v_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 4);
|
|
__m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
|
|
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
|
|
_mm256_packus_epi16(v_s, v_s));
|
|
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8));
|
|
YUVTORGB16_WIN_AVX512BW;
|
|
PACKAR30_WIN_AVX512BW(dst0, dst1);
|
|
_mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0);
|
|
_mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I212TOAR30ROW_AVX512BW
|
|
|
|
#ifdef HAS_I410TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I410ToARGBRow_AVX512BW(const uint16_t* y_buf,
|
|
const uint16_t* u_buf,
|
|
const uint16_t* v_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
|
|
const __m512i zmm5 = _mm512_set1_epi32(-1);
|
|
while (width >= 32) {
|
|
__m512i u_s =
|
|
_mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)u_buf), 2);
|
|
__m512i v_s =
|
|
_mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)v_buf), 2);
|
|
__m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
|
|
u_buf += 32;
|
|
v_buf += 32;
|
|
y_buf += 32;
|
|
__m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s),
|
|
_mm512_packus_epi16(v_s, v_s));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_WIN_AVX512BW(dst_argb);
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask32 k = (__mmask32)m32;
|
|
__m512i u_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, u_buf), 2);
|
|
__m512i v_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, v_buf), 2);
|
|
__m512i y16 = _mm512_maskz_loadu_epi16(k, y_buf);
|
|
__m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s),
|
|
_mm512_packus_epi16(v_s, v_s));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I410TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_I410TOAR30ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I410ToAR30Row_AVX512BW(const uint16_t* y_buf,
|
|
const uint16_t* u_buf,
|
|
const uint16_t* v_buf,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win);
|
|
const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win);
|
|
const __m512i zmm_zero = _mm512_setzero_si512();
|
|
const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0);
|
|
const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00);
|
|
const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000);
|
|
while (width >= 32) {
|
|
__m512i u_s =
|
|
_mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)u_buf), 2);
|
|
__m512i v_s =
|
|
_mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)v_buf), 2);
|
|
__m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
|
|
u_buf += 32;
|
|
v_buf += 32;
|
|
y_buf += 32;
|
|
__m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s),
|
|
_mm512_packus_epi16(v_s, v_s));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
|
|
YUVTORGB16_WIN_AVX512BW;
|
|
PACKAR30_WIN_AVX512BW(dst0, dst1);
|
|
_mm512_storeu_si512((__m512i*)dst_ar30, dst0);
|
|
_mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1);
|
|
dst_ar30 += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask32 k = (__mmask32)m32;
|
|
__m512i u_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, u_buf), 2);
|
|
__m512i v_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, v_buf), 2);
|
|
__m512i y16 = _mm512_maskz_loadu_epi16(k, y_buf);
|
|
__m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s),
|
|
_mm512_packus_epi16(v_s, v_s));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
|
|
YUVTORGB16_WIN_AVX512BW;
|
|
PACKAR30_WIN_AVX512BW(dst0, dst1);
|
|
_mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0);
|
|
_mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I410TOAR30ROW_AVX512BW
|
|
|
|
#ifdef HAS_I410ALPHATOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I410AlphaToARGBRow_AVX512BW(const uint16_t* y_buf,
|
|
const uint16_t* u_buf,
|
|
const uint16_t* v_buf,
|
|
const uint16_t* a_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
|
|
while (width >= 32) {
|
|
__m512i u_s =
|
|
_mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)u_buf), 2);
|
|
__m512i v_s =
|
|
_mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)v_buf), 2);
|
|
__m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
|
|
__m512i a16 =
|
|
_mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)a_buf), 2);
|
|
u_buf += 32;
|
|
v_buf += 32;
|
|
y_buf += 32;
|
|
a_buf += 32;
|
|
__m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s),
|
|
_mm512_packus_epi16(v_s, v_s));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
|
|
__m512i zmm5 = _mm512_packus_epi16(a16, a16);
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_WIN_AVX512BW(dst_argb);
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask32 k = (__mmask32)m32;
|
|
__m512i u_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, u_buf), 2);
|
|
__m512i v_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, v_buf), 2);
|
|
__m512i y16 = _mm512_maskz_loadu_epi16(k, y_buf);
|
|
__m512i a16 = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, a_buf), 2);
|
|
__m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s),
|
|
_mm512_packus_epi16(v_s, v_s));
|
|
__m512i zmm4 =
|
|
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
|
|
__m512i zmm5 = _mm512_packus_epi16(a16, a16);
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I410ALPHATOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_P210TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void P210ToARGBRow_AVX512BW(const uint16_t* y_buf,
|
|
const uint16_t* uv_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
|
|
const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)kShuffleYUY2UV_Win));
|
|
const __m512i zmm5 = _mm512_set1_epi32(-1);
|
|
while (width >= 32) {
|
|
__m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf);
|
|
__m512i zmm3 = _mm512_shuffle_epi8(
|
|
_mm512_loadu_si512((const __m512i*)uv_buf), zmm_shuf_uv);
|
|
y_buf += 32;
|
|
uv_buf += 32;
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_WIN_AVX512BW(dst_argb);
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
|
|
__m512i zmm3 = _mm512_shuffle_epi8(_mm512_maskz_loadu_epi32(k_uv, uv_buf),
|
|
zmm_shuf_uv);
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_P210TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_P210TOAR30ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void P210ToAR30Row_AVX512BW(const uint16_t* y_buf,
|
|
const uint16_t* uv_buf,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win);
|
|
const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win);
|
|
const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)kShuffleYUY2UV_Win));
|
|
const __m512i zmm_zero = _mm512_setzero_si512();
|
|
const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0);
|
|
const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00);
|
|
const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000);
|
|
while (width >= 32) {
|
|
__m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf);
|
|
__m512i zmm3 = _mm512_shuffle_epi8(
|
|
_mm512_loadu_si512((const __m512i*)uv_buf), zmm_shuf_uv);
|
|
y_buf += 32;
|
|
uv_buf += 32;
|
|
YUVTORGB16_WIN_AVX512BW;
|
|
PACKAR30_WIN_AVX512BW(dst0, dst1);
|
|
_mm512_storeu_si512((__m512i*)dst_ar30, dst0);
|
|
_mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1);
|
|
dst_ar30 += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
|
|
__m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
|
|
__m512i zmm3 = _mm512_shuffle_epi8(_mm512_maskz_loadu_epi32(k_uv, uv_buf),
|
|
zmm_shuf_uv);
|
|
YUVTORGB16_WIN_AVX512BW;
|
|
PACKAR30_WIN_AVX512BW(dst0, dst1);
|
|
_mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0);
|
|
_mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_P210TOAR30ROW_AVX512BW
|
|
|
|
#ifdef HAS_P410TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void P410ToARGBRow_AVX512BW(const uint16_t* y_buf,
|
|
const uint16_t* uv_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_unperm =
|
|
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
|
|
const __m512i zmm5 = _mm512_set1_epi32(-1);
|
|
while (width >= 32) {
|
|
__m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf);
|
|
__m512i uv0 = _mm512_loadu_si512((const __m512i*)uv_buf);
|
|
__m512i uv1 = _mm512_loadu_si512((const __m512i*)(uv_buf + 32));
|
|
y_buf += 32;
|
|
uv_buf += 64;
|
|
__m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8));
|
|
__m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8));
|
|
__m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1);
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_WIN_AVX512BW(dst_argb);
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
|
|
__m512i uv0 = _mm512_maskz_loadu_epi32((__mmask16)m32, uv_buf);
|
|
__m512i uv1 = _mm512_maskz_loadu_epi32((__mmask16)(m32 >> 16), uv_buf + 32);
|
|
__m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8));
|
|
__m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8));
|
|
__m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1);
|
|
YUVTORGB8_WIN_AVX512BW;
|
|
STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_P410TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_P410TOAR30ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void P410ToAR30Row_AVX512BW(const uint16_t* y_buf,
|
|
const uint16_t* uv_buf,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
|
|
const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win);
|
|
const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win);
|
|
const __m512i zmm_zero = _mm512_setzero_si512();
|
|
const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0);
|
|
const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00);
|
|
const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000);
|
|
while (width >= 32) {
|
|
__m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf);
|
|
__m512i uv0 = _mm512_loadu_si512((const __m512i*)uv_buf);
|
|
__m512i uv1 = _mm512_loadu_si512((const __m512i*)(uv_buf + 32));
|
|
y_buf += 32;
|
|
uv_buf += 64;
|
|
__m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8));
|
|
__m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8));
|
|
__m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1);
|
|
YUVTORGB16_WIN_AVX512BW;
|
|
PACKAR30_WIN_AVX512BW(dst0, dst1);
|
|
_mm512_storeu_si512((__m512i*)dst_ar30, dst0);
|
|
_mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1);
|
|
dst_ar30 += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t m32 = (1u << width) - 1u;
|
|
__m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
|
|
__m512i uv0 = _mm512_maskz_loadu_epi32((__mmask16)m32, uv_buf);
|
|
__m512i uv1 = _mm512_maskz_loadu_epi32((__mmask16)(m32 >> 16), uv_buf + 32);
|
|
__m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8));
|
|
__m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8));
|
|
__m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1);
|
|
YUVTORGB16_WIN_AVX512BW;
|
|
PACKAR30_WIN_AVX512BW(dst0, dst1);
|
|
_mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0);
|
|
_mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_P410TOAR30ROW_AVX512BW
|
|
|
|
#if defined(HAS_ARGBTORGB24ROW_AVX512BW) || defined(HAS_ARGBTORAWROW_AVX512BW)
|
|
LIBYUV_TARGET_AVX512BW
|
|
static inline void ARGBToRGBRow_AVX512BW(const uint8_t* src,
|
|
uint8_t* dst,
|
|
int width,
|
|
const uint8_t* shuffler) {
|
|
const __m512i zmm_shuf =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)shuffler));
|
|
const __m512i zmm_p0 =
|
|
_mm512_loadu_si512((const __m512i*)kPermdRGB24_0_AVX512BW);
|
|
const __m512i zmm_p1 =
|
|
_mm512_loadu_si512((const __m512i*)kPermdRGB24_1_AVX512BW);
|
|
const __m512i zmm_p2 =
|
|
_mm512_loadu_si512((const __m512i*)kPermdRGB24_2_AVX512BW);
|
|
|
|
while (width >= 64) {
|
|
__m512i zmm0 = _mm512_loadu_si512((const __m512i*)src);
|
|
__m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src + 64));
|
|
__m512i zmm2 = _mm512_loadu_si512((const __m512i*)(src + 128));
|
|
__m512i zmm3 = _mm512_loadu_si512((const __m512i*)(src + 192));
|
|
src += 256;
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
__m512i out0 = _mm512_permutex2var_epi32(zmm0, zmm_p0, zmm1);
|
|
__m512i out1 = _mm512_permutex2var_epi32(zmm1, zmm_p1, zmm2);
|
|
__m512i out2 = _mm512_permutex2var_epi32(zmm2, zmm_p2, zmm3);
|
|
|
|
_mm512_storeu_si512((__m512i*)dst, out0);
|
|
_mm512_storeu_si512((__m512i*)(dst + 64), out1);
|
|
_mm512_storeu_si512((__m512i*)(dst + 128), out2);
|
|
dst += 192;
|
|
width -= 64;
|
|
}
|
|
|
|
if (width > 0) {
|
|
uint64_t m_in = (1ULL << width) - 1ULL;
|
|
__m512i zmm0 = _mm512_maskz_loadu_epi32((__mmask16)m_in, src);
|
|
__m512i zmm1 = _mm512_maskz_loadu_epi32((__mmask16)(m_in >> 16), src + 64);
|
|
__m512i zmm2 = _mm512_maskz_loadu_epi32((__mmask16)(m_in >> 32), src + 128);
|
|
__m512i zmm3 = _mm512_maskz_loadu_epi32((__mmask16)(m_in >> 48), src + 192);
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
__m512i out0 = _mm512_permutex2var_epi32(zmm0, zmm_p0, zmm1);
|
|
__m512i out1 = _mm512_permutex2var_epi32(zmm1, zmm_p1, zmm2);
|
|
__m512i out2 = _mm512_permutex2var_epi32(zmm2, zmm_p2, zmm3);
|
|
|
|
int bytes = width * 3;
|
|
__mmask64 k_out0 =
|
|
(bytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << bytes) - 1ULL);
|
|
__mmask64 k_out1 =
|
|
(bytes <= 64)
|
|
? 0
|
|
: ((bytes >= 128) ? ~(__mmask64)0
|
|
: (__mmask64)((1ULL << (bytes - 64)) - 1ULL));
|
|
__mmask64 k_out2 =
|
|
(bytes <= 128) ? 0 : (__mmask64)((1ULL << (bytes - 128)) - 1ULL);
|
|
|
|
_mm512_mask_storeu_epi8(dst, k_out0, out0);
|
|
_mm512_mask_storeu_epi8(dst + 64, k_out1, out1);
|
|
_mm512_mask_storeu_epi8(dst + 128, k_out2, out2);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_ARGBTORGB24ROW_AVX512BW) ||
|
|
// defined(HAS_ARGBTORAWROW_AVX512BW)
|
|
|
|
#ifdef HAS_ARGBTORGB24ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBToRGB24Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
|
|
ARGBToRGBRow_AVX512BW(src, dst, width, kShuffleMaskARGBToRGB24[0]);
|
|
}
|
|
#endif // HAS_ARGBTORGB24ROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBTORAWROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBToRAWRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
|
|
static const uint8_t kShuffleMaskARGBToRAW_AVX512[16] = {
|
|
2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u,
|
|
128u};
|
|
ARGBToRGBRow_AVX512BW(src, dst, width, kShuffleMaskARGBToRAW_AVX512);
|
|
}
|
|
#endif // HAS_ARGBTORAWROW_AVX512BW
|
|
|
|
#ifdef HAS_CONVERT16TO8ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void Convert16To8Row_AVX512BW(const uint16_t* src_y,
|
|
uint8_t* dst_y,
|
|
int scale,
|
|
int width) {
|
|
const __m512i zmm_scale = _mm512_set1_epi16((short)scale);
|
|
const __m512i zmm_255 = _mm512_set1_epi16(255);
|
|
while (width >= 64) {
|
|
__m512i y0 = _mm512_loadu_si512((const __m512i*)src_y);
|
|
__m512i y1 = _mm512_loadu_si512((const __m512i*)(src_y + 32));
|
|
y0 = _mm512_min_epu16(_mm512_mulhi_epu16(y0, zmm_scale), zmm_255);
|
|
y1 = _mm512_min_epu16(_mm512_mulhi_epu16(y1, zmm_scale), zmm_255);
|
|
_mm256_storeu_si256((__m256i*)dst_y, _mm512_cvtepi16_epi8(y0));
|
|
_mm256_storeu_si256((__m256i*)(dst_y + 32), _mm512_cvtepi16_epi8(y1));
|
|
src_y += 64;
|
|
dst_y += 64;
|
|
width -= 64;
|
|
}
|
|
if (width >= 32) {
|
|
__m512i y0 = _mm512_loadu_si512((const __m512i*)src_y);
|
|
y0 = _mm512_min_epu16(_mm512_mulhi_epu16(y0, zmm_scale), zmm_255);
|
|
_mm256_storeu_si256((__m256i*)dst_y, _mm512_cvtepi16_epi8(y0));
|
|
src_y += 32;
|
|
dst_y += 32;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask32 k = (__mmask32)((1u << width) - 1u);
|
|
__m512i y0 = _mm512_maskz_loadu_epi16(k, src_y);
|
|
y0 = _mm512_min_epu16(_mm512_mulhi_epu16(y0, zmm_scale), zmm_255);
|
|
_mm256_mask_storeu_epi8(dst_y, k, _mm512_cvtepi16_epi8(y0));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_CONVERT16TO8ROW_AVX512BW
|
|
|
|
#ifdef HAS_COPYROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
|
|
while (width >= 128) {
|
|
__m512i z0 = _mm512_loadu_si512((const __m512i*)src);
|
|
__m512i z1 = _mm512_loadu_si512((const __m512i*)(src + 64));
|
|
_mm512_storeu_si512((__m512i*)dst, z0);
|
|
_mm512_storeu_si512((__m512i*)(dst + 64), z1);
|
|
src += 128;
|
|
dst += 128;
|
|
width -= 128;
|
|
}
|
|
if (width >= 64) {
|
|
__m512i z0 = _mm512_loadu_si512((const __m512i*)src);
|
|
_mm512_storeu_si512((__m512i*)dst, z0);
|
|
src += 64;
|
|
dst += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
__mmask64 k = (__mmask64)((1ULL << width) - 1ULL);
|
|
__m512i z0 = _mm512_maskz_loadu_epi8(k, src);
|
|
_mm512_mask_storeu_epi8(dst, k, z0);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_COPYROW_AVX512BW
|
|
|
|
#ifdef HAS_HALFROW_16TO8_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void HalfRow_16To8_AVX512BW(const uint16_t* src_uv,
|
|
ptrdiff_t src_uv_stride,
|
|
uint8_t* dst_uv,
|
|
int scale,
|
|
int width) {
|
|
const __m512i zmm_scale = _mm512_set1_epi16((short)scale);
|
|
const __m512i zmm_255 = _mm512_set1_epi16(255);
|
|
while (width >= 64) {
|
|
__m512i r0 = _mm512_loadu_si512((const __m512i*)src_uv);
|
|
__m512i r1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32));
|
|
__m512i s0 = _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride));
|
|
__m512i s1 =
|
|
_mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride + 32));
|
|
r0 = _mm512_avg_epu16(r0, s0);
|
|
r1 = _mm512_avg_epu16(r1, s1);
|
|
r0 = _mm512_min_epu16(_mm512_mulhi_epu16(r0, zmm_scale), zmm_255);
|
|
r1 = _mm512_min_epu16(_mm512_mulhi_epu16(r1, zmm_scale), zmm_255);
|
|
_mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(r0));
|
|
_mm256_storeu_si256((__m256i*)(dst_uv + 32), _mm512_cvtepi16_epi8(r1));
|
|
src_uv += 64;
|
|
dst_uv += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
uint64_t mask = (1ULL << width) - 1ULL;
|
|
__mmask32 k1 = (__mmask32)mask;
|
|
__mmask32 k2 = (__mmask32)(mask >> 32);
|
|
__m512i r0 = _mm512_maskz_loadu_epi16(k1, src_uv);
|
|
__m512i r1 = _mm512_maskz_loadu_epi16(k2, src_uv + 32);
|
|
__m512i s0 = _mm512_maskz_loadu_epi16(k1, src_uv + src_uv_stride);
|
|
__m512i s1 = _mm512_maskz_loadu_epi16(k2, src_uv + src_uv_stride + 32);
|
|
r0 = _mm512_avg_epu16(r0, s0);
|
|
r1 = _mm512_avg_epu16(r1, s1);
|
|
r0 = _mm512_min_epu16(_mm512_mulhi_epu16(r0, zmm_scale), zmm_255);
|
|
r1 = _mm512_min_epu16(_mm512_mulhi_epu16(r1, zmm_scale), zmm_255);
|
|
_mm256_mask_storeu_epi8(dst_uv, k1, _mm512_cvtepi16_epi8(r0));
|
|
_mm256_mask_storeu_epi8(dst_uv + 32, k2, _mm512_cvtepi16_epi8(r1));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_HALFROW_16TO8_AVX512BW
|
|
|
|
#ifdef HAS_HALFWIDTHROW_16TO8_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv,
|
|
ptrdiff_t src_uv_stride,
|
|
uint8_t* dst_uv,
|
|
int scale,
|
|
int width) {
|
|
const __m512i zmm_mask16 = _mm512_set1_epi32(0x0000ffff);
|
|
const __m512i zmm_2 = _mm512_set1_epi32(2);
|
|
const __m512i zmm_scale = _mm512_set1_epi16((short)scale);
|
|
const __m512i zmm_255 = _mm512_set1_epi16(255);
|
|
while (width >= 32) {
|
|
__m512i s0 = _mm512_loadu_si512((const __m512i*)src_uv);
|
|
__m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32));
|
|
__m512i t0 = _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride));
|
|
__m512i t1 =
|
|
_mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride + 32));
|
|
__m512i sum0 = _mm512_add_epi32(_mm512_and_si512(s0, zmm_mask16),
|
|
_mm512_srli_epi32(s0, 16));
|
|
sum0 = _mm512_add_epi32(sum0, _mm512_and_si512(t0, zmm_mask16));
|
|
sum0 = _mm512_add_epi32(sum0, _mm512_srli_epi32(t0, 16));
|
|
sum0 = _mm512_srli_epi32(_mm512_add_epi32(sum0, zmm_2), 2);
|
|
|
|
__m512i sum1 = _mm512_add_epi32(_mm512_and_si512(s1, zmm_mask16),
|
|
_mm512_srli_epi32(s1, 16));
|
|
sum1 = _mm512_add_epi32(sum1, _mm512_and_si512(t1, zmm_mask16));
|
|
sum1 = _mm512_add_epi32(sum1, _mm512_srli_epi32(t1, 16));
|
|
sum1 = _mm512_srli_epi32(_mm512_add_epi32(sum1, zmm_2), 2);
|
|
|
|
__m512i avg16 =
|
|
_mm512_inserti64x4(_mm512_zextsi256_si512(_mm512_cvtepi32_epi16(sum0)),
|
|
_mm512_cvtepi32_epi16(sum1), 1);
|
|
avg16 = _mm512_min_epu16(_mm512_mulhi_epu16(avg16, zmm_scale), zmm_255);
|
|
_mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(avg16));
|
|
src_uv += 64;
|
|
dst_uv += 32;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t mask = (1u << width) - 1u;
|
|
__mmask16 k1 = (__mmask16)mask;
|
|
__mmask16 k2 = (__mmask16)(mask >> 16);
|
|
__mmask32 k3 = (__mmask32)mask;
|
|
__m512i s0 = _mm512_maskz_loadu_epi32(k1, src_uv);
|
|
__m512i s1 = _mm512_maskz_loadu_epi32(k2, src_uv + 32);
|
|
__m512i t0 = _mm512_maskz_loadu_epi32(k1, src_uv + src_uv_stride);
|
|
__m512i t1 = _mm512_maskz_loadu_epi32(k2, src_uv + src_uv_stride + 32);
|
|
__m512i sum0 = _mm512_add_epi32(_mm512_and_si512(s0, zmm_mask16),
|
|
_mm512_srli_epi32(s0, 16));
|
|
sum0 = _mm512_add_epi32(sum0, _mm512_and_si512(t0, zmm_mask16));
|
|
sum0 = _mm512_add_epi32(sum0, _mm512_srli_epi32(t0, 16));
|
|
sum0 = _mm512_srli_epi32(_mm512_add_epi32(sum0, zmm_2), 2);
|
|
|
|
__m512i sum1 = _mm512_add_epi32(_mm512_and_si512(s1, zmm_mask16),
|
|
_mm512_srli_epi32(s1, 16));
|
|
sum1 = _mm512_add_epi32(sum1, _mm512_and_si512(t1, zmm_mask16));
|
|
sum1 = _mm512_add_epi32(sum1, _mm512_srli_epi32(t1, 16));
|
|
sum1 = _mm512_srli_epi32(_mm512_add_epi32(sum1, zmm_2), 2);
|
|
|
|
__m512i avg16 =
|
|
_mm512_inserti64x4(_mm512_zextsi256_si512(_mm512_cvtepi32_epi16(sum0)),
|
|
_mm512_cvtepi32_epi16(sum1), 1);
|
|
avg16 = _mm512_min_epu16(_mm512_mulhi_epu16(avg16, zmm_scale), zmm_255);
|
|
_mm256_mask_storeu_epi8(dst_uv, k3, _mm512_cvtepi16_epi8(avg16));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_HALFWIDTHROW_16TO8_AVX512BW
|
|
|
|
#ifdef HAS_SPLITUVROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void SplitUVRow_AVX512BW(const uint8_t* src_uv,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
while (width >= 64) {
|
|
__m512i uv0 = _mm512_loadu_si512((const __m512i*)src_uv);
|
|
__m512i uv1 = _mm512_loadu_si512((const __m512i*)(src_uv + 64));
|
|
_mm256_storeu_si256((__m256i*)dst_u, _mm512_cvtepi16_epi8(uv0));
|
|
_mm256_storeu_si256((__m256i*)(dst_u + 32), _mm512_cvtepi16_epi8(uv1));
|
|
_mm256_storeu_si256((__m256i*)dst_v,
|
|
_mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8)));
|
|
_mm256_storeu_si256((__m256i*)(dst_v + 32),
|
|
_mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8)));
|
|
src_uv += 128;
|
|
dst_u += 64;
|
|
dst_v += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
uint64_t mask = (1ULL << width) - 1ULL;
|
|
__mmask32 k0 = (__mmask32)mask;
|
|
__mmask32 k1 = (__mmask32)(mask >> 32);
|
|
__m512i uv0 = _mm512_maskz_loadu_epi16(k0, src_uv);
|
|
_mm256_mask_storeu_epi8(dst_u, k0, _mm512_cvtepi16_epi8(uv0));
|
|
_mm256_mask_storeu_epi8(dst_v, k0,
|
|
_mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8)));
|
|
if (k1) {
|
|
__m512i uv1 = _mm512_maskz_loadu_epi16(k1, src_uv + 64);
|
|
_mm256_mask_storeu_epi8(dst_u + 32, k1, _mm512_cvtepi16_epi8(uv1));
|
|
_mm256_mask_storeu_epi8(dst_v + 32, k1,
|
|
_mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8)));
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SPLITUVROW_AVX512BW
|
|
|
|
#ifdef HAS_CONVERT8TO16ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void Convert8To16Row_AVX512BW(const uint8_t* src_y,
|
|
uint16_t* dst_y,
|
|
int bits,
|
|
int width) {
|
|
const __m128i shift128 = _mm_cvtsi32_si128(16 - bits);
|
|
while (width >= 64) {
|
|
__m256i y0_8 = _mm256_loadu_si256((const __m256i*)src_y);
|
|
__m256i y1_8 = _mm256_loadu_si256((const __m256i*)(src_y + 32));
|
|
__m512i y0 = _mm512_cvtepu8_epi16(y0_8);
|
|
__m512i y1 = _mm512_cvtepu8_epi16(y1_8);
|
|
y0 = _mm512_srl_epi16(_mm512_or_si512(y0, _mm512_slli_epi16(y0, 8)),
|
|
shift128);
|
|
y1 = _mm512_srl_epi16(_mm512_or_si512(y1, _mm512_slli_epi16(y1, 8)),
|
|
shift128);
|
|
_mm512_storeu_si512((__m512i*)dst_y, y0);
|
|
_mm512_storeu_si512((__m512i*)(dst_y + 32), y1);
|
|
src_y += 64;
|
|
dst_y += 64;
|
|
width -= 64;
|
|
}
|
|
if (width >= 32) {
|
|
__m256i y0_8 = _mm256_loadu_si256((const __m256i*)src_y);
|
|
__m512i y0 = _mm512_cvtepu8_epi16(y0_8);
|
|
y0 = _mm512_srl_epi16(_mm512_or_si512(y0, _mm512_slli_epi16(y0, 8)),
|
|
shift128);
|
|
_mm512_storeu_si512((__m512i*)dst_y, y0);
|
|
src_y += 32;
|
|
dst_y += 32;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask32 k = (__mmask32)((1u << width) - 1u);
|
|
__m512i y0 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, src_y));
|
|
y0 = _mm512_srl_epi16(_mm512_or_si512(y0, _mm512_slli_epi16(y0, 8)),
|
|
shift128);
|
|
_mm512_mask_storeu_epi16(dst_y, k, y0);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_CONVERT8TO16ROW_AVX512BW
|
|
|
|
#ifdef HAS_J400TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void J400ToARGBRow_AVX512BW(const uint8_t* src_y,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
static const uint8_t kShuffleMaskJ400ToARGB_AVX512[64] = {
|
|
0u, 0u, 0u, 128u, 1u, 1u, 1u, 128u, 2u, 2u, 2u, 128u, 3u,
|
|
3u, 3u, 128u, 4u, 4u, 4u, 128u, 5u, 5u, 5u, 128u, 6u, 6u,
|
|
6u, 128u, 7u, 7u, 7u, 128u, 8u, 8u, 8u, 128u, 9u, 9u, 9u,
|
|
128u, 10u, 10u, 10u, 128u, 11u, 11u, 11u, 128u, 12u, 12u, 12u, 128u,
|
|
13u, 13u, 13u, 128u, 14u, 14u, 14u, 128u, 15u, 15u, 15u, 128u};
|
|
const __m512i zmm_mask =
|
|
_mm512_loadu_si512((const __m512i*)kShuffleMaskJ400ToARGB_AVX512);
|
|
const __m512i zmm_alpha = _mm512_set1_epi32((int)0xff000000u);
|
|
while (width >= 32) {
|
|
__m512i z0 =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)src_y));
|
|
__m512i z1 =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)(src_y + 16)));
|
|
z0 = _mm512_or_si512(_mm512_shuffle_epi8(z0, zmm_mask), zmm_alpha);
|
|
z1 = _mm512_or_si512(_mm512_shuffle_epi8(z1, zmm_mask), zmm_alpha);
|
|
_mm512_storeu_si512((__m512i*)dst_argb, z0);
|
|
_mm512_storeu_si512((__m512i*)(dst_argb + 64), z1);
|
|
src_y += 32;
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t mask = (1u << width) - 1u;
|
|
__mmask16 k0 = (__mmask16)mask;
|
|
__mmask16 k1 = (__mmask16)(mask >> 16);
|
|
__m512i z0 = _mm512_broadcast_i32x4(_mm_maskz_loadu_epi8(k0, src_y));
|
|
z0 = _mm512_or_si512(_mm512_shuffle_epi8(z0, zmm_mask), zmm_alpha);
|
|
_mm512_mask_storeu_epi32(dst_argb, k0, z0);
|
|
if (k1) {
|
|
__m512i z1 = _mm512_broadcast_i32x4(_mm_maskz_loadu_epi8(k1, src_y + 16));
|
|
z1 = _mm512_or_si512(_mm512_shuffle_epi8(z1, zmm_mask), zmm_alpha);
|
|
_mm512_mask_storeu_epi32(dst_argb + 64, k1, z1);
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_J400TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_MERGEUVROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void MergeUVRow_AVX512BW(const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
while (width >= 32) {
|
|
__m512i u16 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_u));
|
|
__m512i v16 =
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_v));
|
|
__m512i uv = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
|
|
_mm512_storeu_si512((__m512i*)dst_uv, uv);
|
|
src_u += 32;
|
|
src_v += 32;
|
|
dst_uv += 64;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask32 k = (__mmask32)((1u << width) - 1u);
|
|
__m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, src_u));
|
|
__m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, src_v));
|
|
__m512i uv = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
|
|
_mm512_mask_storeu_epi16(dst_uv, k, uv);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGEUVROW_AVX512BW
|
|
|
|
#ifdef HAS_MULTIPLYROW_16_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void MultiplyRow_16_AVX512BW(const uint16_t* src_y,
|
|
uint16_t* dst_y,
|
|
int scale,
|
|
int width) {
|
|
const __m512i zmm_scale = _mm512_set1_epi16((short)scale);
|
|
while (width >= 64) {
|
|
__m512i z0 = _mm512_loadu_si512((const __m512i*)src_y);
|
|
__m512i z1 = _mm512_loadu_si512((const __m512i*)(src_y + 32));
|
|
z0 = _mm512_mullo_epi16(z0, zmm_scale);
|
|
z1 = _mm512_mullo_epi16(z1, zmm_scale);
|
|
_mm512_storeu_si512((__m512i*)dst_y, z0);
|
|
_mm512_storeu_si512((__m512i*)(dst_y + 32), z1);
|
|
src_y += 64;
|
|
dst_y += 64;
|
|
width -= 64;
|
|
}
|
|
if (width >= 32) {
|
|
__m512i z0 = _mm512_loadu_si512((const __m512i*)src_y);
|
|
z0 = _mm512_mullo_epi16(z0, zmm_scale);
|
|
_mm512_storeu_si512((__m512i*)dst_y, z0);
|
|
src_y += 32;
|
|
dst_y += 32;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
__mmask32 k = (__mmask32)((1u << width) - 1u);
|
|
__m512i z0 = _mm512_maskz_loadu_epi16(k, src_y);
|
|
z0 = _mm512_mullo_epi16(z0, zmm_scale);
|
|
_mm512_mask_storeu_epi16(dst_y, k, z0);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MULTIPLYROW_16_AVX512BW
|
|
|
|
#ifdef HAS_ARGBTOYMATRIXROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb,
|
|
uint8_t* dst_y,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
const __m512i zmm_perm_y =
|
|
_mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
|
|
const __m512i zmm_80 = _mm512_set1_epi8((char)0x80);
|
|
const __m512i zmm_one = _mm512_set1_epi16(1);
|
|
const __m512i zmm_coeff =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToY));
|
|
__m512i zmm_add =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddY));
|
|
|
|
__m512i zmm_bias = _mm512_maddubs_epi16(zmm_coeff, zmm_80);
|
|
zmm_bias = _mm512_madd_epi16(zmm_bias, zmm_one);
|
|
zmm_bias = _mm512_packs_epi32(zmm_bias, zmm_bias);
|
|
zmm_add = _mm512_sub_epi16(zmm_add, zmm_bias);
|
|
|
|
while (width >= 64) {
|
|
__m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
__m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
|
|
__m512i zmm2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128));
|
|
__m512i zmm3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192));
|
|
|
|
zmm0 = _mm512_xor_si512(zmm0, zmm_80);
|
|
zmm1 = _mm512_xor_si512(zmm1, zmm_80);
|
|
zmm2 = _mm512_xor_si512(zmm2, zmm_80);
|
|
zmm3 = _mm512_xor_si512(zmm3, zmm_80);
|
|
|
|
zmm0 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm0), zmm_one);
|
|
zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm1), zmm_one);
|
|
zmm0 = _mm512_packs_epi32(zmm0, zmm1);
|
|
|
|
zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm2), zmm_one);
|
|
zmm3 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm3), zmm_one);
|
|
zmm2 = _mm512_packs_epi32(zmm2, zmm3);
|
|
|
|
zmm0 = _mm512_srli_epi16(_mm512_add_epi16(zmm0, zmm_add), 8);
|
|
zmm2 = _mm512_srli_epi16(_mm512_add_epi16(zmm2, zmm_add), 8);
|
|
|
|
zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2));
|
|
_mm512_storeu_si512((__m512i*)dst_y, zmm0);
|
|
|
|
src_argb += 256;
|
|
dst_y += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
uint64_t mask = (1ULL << width) - 1ULL;
|
|
__mmask16 k0 = (__mmask16)mask;
|
|
__mmask16 k1 = (__mmask16)(mask >> 16);
|
|
__mmask16 k2 = (__mmask16)(mask >> 32);
|
|
__mmask16 k3 = (__mmask16)(mask >> 48);
|
|
__m512i zmm0 =
|
|
_mm512_xor_si512(_mm512_maskz_loadu_epi32(k0, src_argb), zmm_80);
|
|
__m512i zmm1 =
|
|
_mm512_xor_si512(k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64)
|
|
: _mm512_setzero_si512(),
|
|
zmm_80);
|
|
__m512i zmm2 =
|
|
_mm512_xor_si512(k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128)
|
|
: _mm512_setzero_si512(),
|
|
zmm_80);
|
|
__m512i zmm3 =
|
|
_mm512_xor_si512(k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192)
|
|
: _mm512_setzero_si512(),
|
|
zmm_80);
|
|
|
|
zmm0 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm0), zmm_one);
|
|
zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm1), zmm_one);
|
|
zmm0 = _mm512_packs_epi32(zmm0, zmm1);
|
|
|
|
zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm2), zmm_one);
|
|
zmm3 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm3), zmm_one);
|
|
zmm2 = _mm512_packs_epi32(zmm2, zmm3);
|
|
|
|
zmm0 = _mm512_srli_epi16(_mm512_add_epi16(zmm0, zmm_add), 8);
|
|
zmm2 = _mm512_srli_epi16(_mm512_add_epi16(zmm2, zmm_add), 8);
|
|
|
|
zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2));
|
|
_mm512_mask_storeu_epi8(dst_y, (__mmask64)mask, zmm0);
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOYMATRIXROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
const __m512i zmm_u =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU));
|
|
const __m512i zmm_v =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV));
|
|
const __m512i zmm_one = _mm512_set1_epi16(1);
|
|
const __m512i zmm_8000 =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddUV));
|
|
const __m512i zmm_perm_y =
|
|
_mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
|
|
|
|
while (width >= 64) {
|
|
__m512i s0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
__m512i s1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
|
|
__m512i s2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128));
|
|
__m512i s3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192));
|
|
|
|
__m512i u0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_u), zmm_one);
|
|
__m512i u1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_u), zmm_one);
|
|
__m512i u2 = _mm512_madd_epi16(_mm512_maddubs_epi16(s2, zmm_u), zmm_one);
|
|
__m512i u3 = _mm512_madd_epi16(_mm512_maddubs_epi16(s3, zmm_u), zmm_one);
|
|
__m512i u01 =
|
|
_mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(u0, u1)),
|
|
8);
|
|
__m512i u23 =
|
|
_mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(u2, u3)),
|
|
8);
|
|
_mm512_storeu_si512(
|
|
(__m512i*)dst_u,
|
|
PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(u01, u23)));
|
|
|
|
__m512i v0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_v), zmm_one);
|
|
__m512i v1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_v), zmm_one);
|
|
__m512i v2 = _mm512_madd_epi16(_mm512_maddubs_epi16(s2, zmm_v), zmm_one);
|
|
__m512i v3 = _mm512_madd_epi16(_mm512_maddubs_epi16(s3, zmm_v), zmm_one);
|
|
__m512i v01 =
|
|
_mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(v0, v1)),
|
|
8);
|
|
__m512i v23 =
|
|
_mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(v2, v3)),
|
|
8);
|
|
_mm512_storeu_si512(
|
|
(__m512i*)dst_v,
|
|
PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(v01, v23)));
|
|
|
|
src_argb += 256;
|
|
dst_u += 64;
|
|
dst_v += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
uint64_t mask = (1ULL << width) - 1ULL;
|
|
__mmask16 k0 = (__mmask16)mask;
|
|
__mmask16 k1 = (__mmask16)(mask >> 16);
|
|
__mmask16 k2 = (__mmask16)(mask >> 32);
|
|
__mmask16 k3 = (__mmask16)(mask >> 48);
|
|
__m512i s0 = _mm512_maskz_loadu_epi32(k0, src_argb);
|
|
__m512i s1 = k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64)
|
|
: _mm512_setzero_si512();
|
|
__m512i s2 = k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128)
|
|
: _mm512_setzero_si512();
|
|
__m512i s3 = k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192)
|
|
: _mm512_setzero_si512();
|
|
|
|
__m512i u0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_u), zmm_one);
|
|
__m512i u1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_u), zmm_one);
|
|
__m512i u2 = _mm512_madd_epi16(_mm512_maddubs_epi16(s2, zmm_u), zmm_one);
|
|
__m512i u3 = _mm512_madd_epi16(_mm512_maddubs_epi16(s3, zmm_u), zmm_one);
|
|
__m512i u01 = _mm512_srli_epi16(
|
|
_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(u0, u1)), 8);
|
|
__m512i u23 = _mm512_srli_epi16(
|
|
_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(u2, u3)), 8);
|
|
_mm512_mask_storeu_epi8(
|
|
dst_u, (__mmask64)mask,
|
|
PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(u01, u23)));
|
|
|
|
__m512i v0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_v), zmm_one);
|
|
__m512i v1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_v), zmm_one);
|
|
__m512i v2 = _mm512_madd_epi16(_mm512_maddubs_epi16(s2, zmm_v), zmm_one);
|
|
__m512i v3 = _mm512_madd_epi16(_mm512_maddubs_epi16(s3, zmm_v), zmm_one);
|
|
__m512i v01 = _mm512_srli_epi16(
|
|
_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(v0, v1)), 8);
|
|
__m512i v23 = _mm512_srli_epi16(
|
|
_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(v2, v3)), 8);
|
|
_mm512_mask_storeu_epi8(
|
|
dst_v, (__mmask64)mask,
|
|
PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(v01, v23)));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOUV444MATRIXROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBTOUVMATRIXROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb,
|
|
int src_stride_argb,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
static const uint32_t kPermdARGBToUV_AVX512BW[16] = {
|
|
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15};
|
|
static const uint8_t kShuffleAARRGGBB_AVX512BW[16] = {
|
|
0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15};
|
|
const __m512i zmm_u =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU));
|
|
const __m512i zmm_v =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV));
|
|
const __m512i zmm_0101 = _mm512_set1_epi16(0x0101);
|
|
const __m512i zmm_one = _mm512_set1_epi16(1);
|
|
const __m512i zmm_8000 =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddUV));
|
|
const __m512i zmm_zero = _mm512_setzero_si512();
|
|
const __m512i zmm_shuf = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)kShuffleAARRGGBB_AVX512BW));
|
|
const __m512i zmm_perm_y =
|
|
_mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
|
|
const __m512i zmm_perm_uv =
|
|
_mm512_loadu_si512((const __m512i*)kPermdARGBToUV_AVX512BW);
|
|
|
|
while (width >= 32) {
|
|
__m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
__m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
|
|
__m512i zmm2 =
|
|
_mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb));
|
|
__m512i zmm3 =
|
|
_mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb + 64));
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101);
|
|
zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101);
|
|
zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101);
|
|
zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101);
|
|
|
|
zmm0 = _mm512_add_epi16(zmm0, zmm2);
|
|
zmm1 = _mm512_add_epi16(zmm1, zmm3);
|
|
|
|
zmm0 = _mm512_avg_epu16(_mm512_srli_epi16(zmm0, 1), zmm_zero);
|
|
zmm1 = _mm512_avg_epu16(_mm512_srli_epi16(zmm1, 1), zmm_zero);
|
|
|
|
zmm0 = PermutexvarEpi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1));
|
|
|
|
zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_u), zmm_one);
|
|
zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_v), zmm_one);
|
|
zmm0 = _mm512_packs_epi32(zmm1, zmm2);
|
|
zmm0 = _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, zmm0), 8);
|
|
zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0));
|
|
|
|
_mm_storeu_si128((__m128i*)dst_u, _mm512_castsi512_si128(zmm0));
|
|
_mm_storeu_si128((__m128i*)dst_v, _mm512_extracti32x4_epi32(zmm0, 1));
|
|
|
|
src_argb += 128;
|
|
dst_u += 16;
|
|
dst_v += 16;
|
|
width -= 32;
|
|
}
|
|
if (width > 0) {
|
|
uint32_t mask = (1u << width) - 1u;
|
|
__mmask16 k1 = (__mmask16)mask;
|
|
__mmask16 k2 = (__mmask16)(mask >> 16);
|
|
__mmask16 k3 = ~k1;
|
|
__mmask16 k4 = ~k2;
|
|
__m512i zmm0 = _mm512_maskz_loadu_epi32(k1, src_argb);
|
|
__m512i zmm1 = _mm512_maskz_loadu_epi32(k2, src_argb + 64);
|
|
__m512i zmm2 = _mm512_maskz_loadu_epi32(k1, src_argb + src_stride_argb);
|
|
__m512i zmm3 =
|
|
_mm512_maskz_loadu_epi32(k2, src_argb + src_stride_argb + 64);
|
|
|
|
zmm0 = _mm512_mask_shuffle_epi32(zmm0, k3, zmm0, (_MM_PERM_ENUM)0xa0);
|
|
zmm1 = _mm512_mask_shuffle_epi32(zmm1, k4, zmm1, (_MM_PERM_ENUM)0xa0);
|
|
zmm2 = _mm512_mask_shuffle_epi32(zmm2, k3, zmm2, (_MM_PERM_ENUM)0xa0);
|
|
zmm3 = _mm512_mask_shuffle_epi32(zmm3, k4, zmm3, (_MM_PERM_ENUM)0xa0);
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101);
|
|
zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101);
|
|
zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101);
|
|
zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101);
|
|
|
|
zmm0 = _mm512_add_epi16(zmm0, zmm2);
|
|
zmm1 = _mm512_add_epi16(zmm1, zmm3);
|
|
|
|
zmm0 = _mm512_avg_epu16(_mm512_srli_epi16(zmm0, 1), zmm_zero);
|
|
zmm1 = _mm512_avg_epu16(_mm512_srli_epi16(zmm1, 1), zmm_zero);
|
|
|
|
zmm0 = PermutexvarEpi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1));
|
|
|
|
zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_u), zmm_one);
|
|
zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_v), zmm_one);
|
|
zmm0 = _mm512_packs_epi32(zmm1, zmm2);
|
|
zmm0 = _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, zmm0), 8);
|
|
zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0));
|
|
|
|
int halfwidth = (width + 1) >> 1;
|
|
__mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u);
|
|
_mm_mask_storeu_epi8(dst_u, k_out, _mm512_castsi512_si128(zmm0));
|
|
_mm_mask_storeu_epi8(dst_v, k_out, _mm512_extracti32x4_epi32(zmm0, 1));
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOUVMATRIXROW_AVX512BW
|
|
|
|
#endif // defined(__x86_64__) || defined(_M_X64)
|
|
|
|
#ifdef HAS_COPYROW_AVX
|
|
LIBYUV_TARGET_AVX
|
|
void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width) {
|
|
do {
|
|
__m256i y0 = _mm256_loadu_si256((const __m256i*)src);
|
|
__m256i y1 = _mm256_loadu_si256((const __m256i*)(src + 32));
|
|
_mm256_storeu_si256((__m256i*)dst, y0);
|
|
_mm256_storeu_si256((__m256i*)(dst + 32), y1);
|
|
src += 64;
|
|
dst += 64;
|
|
width -= 64;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_COPYROW_AVX
|
|
|
|
#ifdef HAS_DETILETOYUY2_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void DetileToYUY2_AVX2(const uint8_t* src_y,
|
|
ptrdiff_t src_y_tile_stride,
|
|
const uint8_t* src_uv,
|
|
ptrdiff_t src_uv_tile_stride,
|
|
uint8_t* dst_yuy2,
|
|
int width) {
|
|
do {
|
|
__m128i y0 = _mm_loadu_si128((const __m128i*)src_y);
|
|
__m128i y1 = _mm_loadu_si128((const __m128i*)(src_y + src_y_tile_stride));
|
|
__m128i uv0 = _mm_loadu_si128((const __m128i*)src_uv);
|
|
__m128i uv1 =
|
|
_mm_loadu_si128((const __m128i*)(src_uv + src_uv_tile_stride));
|
|
__m256i y = _mm256_inserti128_si256(_mm256_castsi128_si256(y0), y1, 1);
|
|
__m256i uv = _mm256_inserti128_si256(_mm256_castsi128_si256(uv0), uv1, 1);
|
|
__m256i lo = _mm256_unpacklo_epi8(y, uv);
|
|
__m256i hi = _mm256_unpackhi_epi8(y, uv);
|
|
_mm256_storeu_si256((__m256i*)dst_yuy2,
|
|
_mm256_permute2x128_si256(lo, hi, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst_yuy2 + 32),
|
|
_mm256_permute2x128_si256(lo, hi, 0x31));
|
|
src_y += 2 * src_y_tile_stride;
|
|
src_uv += 2 * src_uv_tile_stride;
|
|
dst_yuy2 += 64;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_DETILETOYUY2_AVX2
|
|
|
|
#ifdef HAS_UNPACKMT2T_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void UnpackMT2T_AVX2(const uint8_t* src, uint16_t* dst, size_t size) {
|
|
const __m256i mask_00c0 = _mm256_set1_epi16(0x00c0);
|
|
while (size >= 80) {
|
|
__m256i low16 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src));
|
|
__m256i l0 = _mm256_and_si256(_mm256_slli_epi16(low16, 6), mask_00c0);
|
|
__m256i l1 = _mm256_and_si256(_mm256_slli_epi16(low16, 4), mask_00c0);
|
|
__m256i l2 = _mm256_and_si256(_mm256_slli_epi16(low16, 2), mask_00c0);
|
|
__m256i l3 = _mm256_and_si256(low16, mask_00c0);
|
|
|
|
__m256i u0 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 16)));
|
|
__m256i u1 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 32)));
|
|
__m256i u2 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 48)));
|
|
__m256i u3 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 64)));
|
|
|
|
u0 = _mm256_or_si256(
|
|
_mm256_or_si256(_mm256_slli_epi16(u0, 8), _mm256_srli_epi16(u0, 2)),
|
|
l0);
|
|
u1 = _mm256_or_si256(
|
|
_mm256_or_si256(_mm256_slli_epi16(u1, 8), _mm256_srli_epi16(u1, 2)),
|
|
l1);
|
|
u2 = _mm256_or_si256(
|
|
_mm256_or_si256(_mm256_slli_epi16(u2, 8), _mm256_srli_epi16(u2, 2)),
|
|
l2);
|
|
u3 = _mm256_or_si256(
|
|
_mm256_or_si256(_mm256_slli_epi16(u3, 8), _mm256_srli_epi16(u3, 2)),
|
|
l3);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst, u0);
|
|
_mm256_storeu_si256((__m256i*)(dst + 16), u1);
|
|
_mm256_storeu_si256((__m256i*)(dst + 32), u2);
|
|
_mm256_storeu_si256((__m256i*)(dst + 48), u3);
|
|
|
|
src += 80;
|
|
dst += 64;
|
|
size -= 80;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_UNPACKMT2T_AVX2
|
|
|
|
#ifdef HAS_ARGBGRAYROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBGrayRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
|
|
static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80,
|
|
8, 8, 8, 0x80, 12, 12, 12, 0x80};
|
|
const __m256i c_coeff = _mm256_set1_epi32(0x004d961d);
|
|
const __m256i c_sub128 = _mm256_set1_epi8((char)0x80);
|
|
const __m256i c_one16 = _mm256_set1_epi16(1);
|
|
const __m256i c_bias = _mm256_set1_epi32(32896);
|
|
const __m256i c_shuf = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShufGrayDword));
|
|
const __m256i c_alpha = _mm256_set1_epi32((int)0xff000000u);
|
|
|
|
while (width >= 8) {
|
|
__m256i s = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
src_argb += 32;
|
|
__m256i y16 = _mm256_maddubs_epi16(c_coeff, _mm256_sub_epi8(s, c_sub128));
|
|
__m256i y32 = _mm256_srli_epi32(
|
|
_mm256_add_epi32(_mm256_madd_epi16(y16, c_one16), c_bias), 8);
|
|
__m256i out = _mm256_or_si256(_mm256_and_si256(s, c_alpha),
|
|
_mm256_shuffle_epi8(y32, c_shuf));
|
|
_mm256_storeu_si256((__m256i*)dst_argb, out);
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
if (width > 0) {
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
ARGBGrayRow_C(src_argb, dst_argb, width);
|
|
}
|
|
}
|
|
#endif // HAS_ARGBGRAYROW_AVX2
|
|
|
|
#ifdef HAS_MERGERGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MergeRGBRow_AVX2(const uint8_t* src_r,
|
|
const uint8_t* src_g,
|
|
const uint8_t* src_b,
|
|
uint8_t* dst_rgb,
|
|
int width) {
|
|
static const uint8_t kMergeRGBShuf[5][16] = {
|
|
{0u, 11u, 6u, 1u, 12u, 7u, 2u, 13u, 8u, 3u, 14u, 9u, 4u, 15u, 10u, 5u},
|
|
{5u, 0u, 11u, 6u, 1u, 12u, 7u, 2u, 13u, 8u, 3u, 14u, 9u, 4u, 15u, 10u},
|
|
{10u, 5u, 0u, 11u, 6u, 1u, 12u, 7u, 2u, 13u, 8u, 3u, 14u, 9u, 4u, 15u},
|
|
{0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u,
|
|
0u},
|
|
{0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u,
|
|
0u}};
|
|
const __m256i shuf_r = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kMergeRGBShuf[0]));
|
|
const __m256i shuf_g = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kMergeRGBShuf[1]));
|
|
const __m256i shuf_b = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kMergeRGBShuf[2]));
|
|
const __m256i m1 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kMergeRGBShuf[3]));
|
|
const __m256i m2 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kMergeRGBShuf[4]));
|
|
|
|
do {
|
|
__m256i r =
|
|
_mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_r), shuf_r);
|
|
__m256i g =
|
|
_mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_g), shuf_g);
|
|
__m256i b =
|
|
_mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_b), shuf_b);
|
|
|
|
__m256i c0 = _mm256_blendv_epi8(_mm256_blendv_epi8(r, g, m1), b, m2);
|
|
__m256i c1 = _mm256_blendv_epi8(_mm256_blendv_epi8(g, b, m1), r, m2);
|
|
__m256i c2 = _mm256_blendv_epi8(_mm256_blendv_epi8(b, r, m1), g, m2);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_rgb,
|
|
_mm256_permute2x128_si256(c0, c1, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst_rgb + 32),
|
|
_mm256_permute2x128_si256(c2, c0, 0x30));
|
|
_mm256_storeu_si256((__m256i*)(dst_rgb + 64),
|
|
_mm256_permute2x128_si256(c1, c2, 0x31));
|
|
|
|
src_r += 32;
|
|
src_g += 32;
|
|
src_b += 32;
|
|
dst_rgb += 96;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGERGBROW_AVX2
|
|
|
|
#ifdef HAS_RAWTORGBAROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void RAWToRGBARow_AVX2(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
|
|
static const uint8_t kShufRAWToRGBA_Lo[16] = {0x80, 2u, 1u, 0u, 0x80, 5u,
|
|
4u, 3u, 0x80, 8u, 7u, 6u,
|
|
0x80, 11u, 10u, 9u};
|
|
static const uint8_t kShufRAWToRGBA_Hi[16] = {0x80, 6u, 5u, 4u, 0x80, 9u,
|
|
8u, 7u, 0x80, 12u, 11u, 10u,
|
|
0x80, 15u, 14u, 13u};
|
|
const __m256i shuf_lo = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShufRAWToRGBA_Lo));
|
|
const __m256i shuf_hi = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShufRAWToRGBA_Hi));
|
|
const __m256i alpha = _mm256_set1_epi32(0x000000ff);
|
|
|
|
do {
|
|
__m128i s0 = _mm_loadu_si128((const __m128i*)src_raw);
|
|
__m128i s1 = _mm_loadu_si128((const __m128i*)(src_raw + 12));
|
|
__m128i s2 = _mm_loadu_si128((const __m128i*)(src_raw + 20));
|
|
__m128i s3 = _mm_loadu_si128((const __m128i*)(src_raw + 32));
|
|
__m256i y0 = _mm256_inserti128_si256(_mm256_castsi128_si256(s0), s1, 1);
|
|
__m256i y1 = _mm256_inserti128_si256(_mm256_castsi128_si256(s2), s3, 1);
|
|
y0 = _mm256_or_si256(_mm256_shuffle_epi8(y0, shuf_lo), alpha);
|
|
y1 = _mm256_or_si256(_mm256_shuffle_epi8(y1, shuf_hi), alpha);
|
|
_mm256_storeu_si256((__m256i*)dst_rgba, y0);
|
|
_mm256_storeu_si256((__m256i*)(dst_rgba + 32), y1);
|
|
src_raw += 48;
|
|
dst_rgba += 64;
|
|
width -= 16;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RAWTORGBAROW_AVX2
|
|
|
|
#ifdef HAS_HALFROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void HalfRow_AVX2(const uint8_t* src_uv,
|
|
ptrdiff_t src_uv_stride,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
do {
|
|
__m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv);
|
|
__m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride));
|
|
_mm256_storeu_si256((__m256i*)dst_uv, _mm256_avg_epu8(r0, r1));
|
|
src_uv += 32;
|
|
dst_uv += 32;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_HALFROW_AVX2
|
|
|
|
#ifdef HAS_HALFROW_16_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void HalfRow_16_AVX2(const uint16_t* src_uv,
|
|
ptrdiff_t src_uv_stride,
|
|
uint16_t* dst_uv,
|
|
int width) {
|
|
do {
|
|
__m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv);
|
|
__m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride));
|
|
_mm256_storeu_si256((__m256i*)dst_uv, _mm256_avg_epu16(r0, r1));
|
|
src_uv += 16;
|
|
dst_uv += 16;
|
|
width -= 16;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_HALFROW_16_AVX2
|
|
|
|
#ifdef HAS_HALFMERGEUVROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void HalfMergeUVRow_AVX2(const uint8_t* src_u,
|
|
int src_stride_u,
|
|
const uint8_t* src_v,
|
|
int src_stride_v,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
const __m256i ymm_0101 = _mm256_set1_epi16(0x0101);
|
|
const __m256i ymm_zero = _mm256_setzero_si256();
|
|
do {
|
|
__m256i u0 = _mm256_loadu_si256((const __m256i*)src_u);
|
|
__m256i v0 = _mm256_loadu_si256((const __m256i*)src_v);
|
|
__m256i u1 = _mm256_loadu_si256((const __m256i*)(src_u + src_stride_u));
|
|
__m256i v1 = _mm256_loadu_si256((const __m256i*)(src_v + src_stride_v));
|
|
u0 = _mm256_add_epi16(_mm256_maddubs_epi16(u0, ymm_0101),
|
|
_mm256_maddubs_epi16(u1, ymm_0101));
|
|
v0 = _mm256_add_epi16(_mm256_maddubs_epi16(v0, ymm_0101),
|
|
_mm256_maddubs_epi16(v1, ymm_0101));
|
|
u0 = _mm256_avg_epu16(_mm256_srli_epi16(u0, 1), ymm_zero);
|
|
v0 = _mm256_avg_epu16(_mm256_srli_epi16(v0, 1), ymm_zero);
|
|
u0 = _mm256_packus_epi16(u0, u0);
|
|
v0 = _mm256_packus_epi16(v0, v0);
|
|
_mm256_storeu_si256((__m256i*)dst_uv, _mm256_unpacklo_epi8(u0, v0));
|
|
src_u += 32;
|
|
src_v += 32;
|
|
dst_uv += 32;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_HALFMERGEUVROW_AVX2
|
|
|
|
#ifdef HAS_CONVERT8TO8ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void Convert8To8Row_AVX2(const uint8_t* src_y,
|
|
uint8_t* dst_y,
|
|
int scale,
|
|
int bias,
|
|
int width) {
|
|
const __m256i ymm_scale =
|
|
_mm256_slli_epi16(_mm256_set1_epi16((short)scale), 8);
|
|
const __m256i ymm_bias = _mm256_set1_epi8((char)bias);
|
|
const __m256i ymm_zero = _mm256_setzero_si256();
|
|
do {
|
|
__m256i y = _mm256_loadu_si256((const __m256i*)src_y);
|
|
__m256i lo =
|
|
_mm256_mulhi_epu16(_mm256_unpacklo_epi8(y, ymm_zero), ymm_scale);
|
|
__m256i hi =
|
|
_mm256_mulhi_epu16(_mm256_unpackhi_epi8(y, ymm_zero), ymm_scale);
|
|
__m256i out = _mm256_add_epi8(_mm256_packus_epi16(lo, hi), ymm_bias);
|
|
_mm256_storeu_si256((__m256i*)dst_y, out);
|
|
src_y += 32;
|
|
dst_y += 32;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_CONVERT8TO8ROW_AVX2
|
|
|
|
#ifdef HAS_DIVIDEROW_16_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void DivideRow_16_AVX2(const uint16_t* src_y,
|
|
uint16_t* dst_y,
|
|
int scale,
|
|
int width) {
|
|
if (scale == 65536) { // Divide by 1, so do memcpy.
|
|
if (src_y != dst_y) {
|
|
memcpy(dst_y, src_y, (size_t)width * 2);
|
|
}
|
|
return;
|
|
}
|
|
const __m256i ymm_scale = _mm256_set1_epi16((short)scale);
|
|
while (width > 0) {
|
|
__m256i v0 = _mm256_mulhi_epu16(_mm256_loadu_si256((const __m256i*)src_y),
|
|
ymm_scale);
|
|
__m256i v1 = _mm256_mulhi_epu16(
|
|
_mm256_loadu_si256((const __m256i*)(src_y + 16)), ymm_scale);
|
|
_mm256_storeu_si256((__m256i*)dst_y, v0);
|
|
_mm256_storeu_si256((__m256i*)(dst_y + 16), v1);
|
|
src_y += 32;
|
|
dst_y += 32;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_DIVIDEROW_16_AVX2
|
|
|
|
#ifdef HAS_CONVERT16TO8ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void Convert16To8Row_AVX2(const uint16_t* src_y,
|
|
uint8_t* dst_y,
|
|
int scale,
|
|
int width) {
|
|
const __m256i ymm_scale = _mm256_set1_epi16((short)scale);
|
|
const __m256i ymm_255 = _mm256_set1_epi16(255);
|
|
do {
|
|
__m256i y0 = _mm256_loadu_si256((const __m256i*)src_y);
|
|
__m256i y1 = _mm256_loadu_si256((const __m256i*)(src_y + 16));
|
|
y0 = _mm256_min_epu16(_mm256_mulhi_epu16(y0, ymm_scale), ymm_255);
|
|
y1 = _mm256_min_epu16(_mm256_mulhi_epu16(y1, ymm_scale), ymm_255);
|
|
__m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(y0, y1), 0xd8);
|
|
_mm256_storeu_si256((__m256i*)dst_y, out);
|
|
src_y += 32;
|
|
dst_y += 32;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_CONVERT16TO8ROW_AVX2
|
|
|
|
#ifdef HAS_HALFROW_16TO8_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void HalfRow_16To8_AVX2(const uint16_t* src_uv,
|
|
ptrdiff_t src_uv_stride,
|
|
uint8_t* dst_uv,
|
|
int scale,
|
|
int width) {
|
|
const __m256i ymm_scale = _mm256_set1_epi16((short)scale);
|
|
const __m256i ymm_255 = _mm256_set1_epi16(255);
|
|
do {
|
|
__m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv);
|
|
__m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + 16));
|
|
__m256i s0 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride));
|
|
__m256i s1 =
|
|
_mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride + 16));
|
|
r0 = _mm256_min_epu16(_mm256_mulhi_epu16(_mm256_avg_epu16(r0, s0), ymm_scale),
|
|
ymm_255);
|
|
r1 = _mm256_min_epu16(_mm256_mulhi_epu16(_mm256_avg_epu16(r1, s1), ymm_scale),
|
|
ymm_255);
|
|
__m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(r0, r1), 0xd8);
|
|
_mm256_storeu_si256((__m256i*)dst_uv, out);
|
|
src_uv += 32;
|
|
dst_uv += 32;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_HALFROW_16TO8_AVX2
|
|
|
|
#ifdef HAS_HALFWIDTHROW_16TO8_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv,
|
|
ptrdiff_t src_uv_stride,
|
|
uint8_t* dst_uv,
|
|
int scale,
|
|
int width) {
|
|
const __m256i ymm_mask16 = _mm256_set1_epi32(0x0000ffff);
|
|
const __m256i ymm_2 = _mm256_set1_epi32(2);
|
|
const __m256i ymm_scale = _mm256_set1_epi16((short)scale);
|
|
const __m256i ymm_255 = _mm256_set1_epi16(255);
|
|
do {
|
|
__m256i s0 = _mm256_loadu_si256((const __m256i*)src_uv);
|
|
__m256i s1 = _mm256_loadu_si256((const __m256i*)(src_uv + 16));
|
|
__m256i t0 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride));
|
|
__m256i t1 =
|
|
_mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride + 16));
|
|
|
|
__m256i sum0 = _mm256_add_epi32(_mm256_and_si256(s0, ymm_mask16),
|
|
_mm256_srli_epi32(s0, 16));
|
|
sum0 = _mm256_add_epi32(sum0, _mm256_and_si256(t0, ymm_mask16));
|
|
sum0 = _mm256_add_epi32(sum0, _mm256_srli_epi32(t0, 16));
|
|
sum0 = _mm256_srli_epi32(_mm256_add_epi32(sum0, ymm_2), 2);
|
|
|
|
__m256i sum1 = _mm256_add_epi32(_mm256_and_si256(s1, ymm_mask16),
|
|
_mm256_srli_epi32(s1, 16));
|
|
sum1 = _mm256_add_epi32(sum1, _mm256_and_si256(t1, ymm_mask16));
|
|
sum1 = _mm256_add_epi32(sum1, _mm256_srli_epi32(t1, 16));
|
|
sum1 = _mm256_srli_epi32(_mm256_add_epi32(sum1, ymm_2), 2);
|
|
|
|
__m256i avg16 =
|
|
_mm256_permute4x64_epi64(_mm256_packus_epi32(sum0, sum1), 0xd8);
|
|
avg16 = _mm256_min_epu16(_mm256_mulhi_epu16(avg16, ymm_scale), ymm_255);
|
|
__m256i out =
|
|
_mm256_permute4x64_epi64(_mm256_packus_epi16(avg16, avg16), 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_uv, _mm256_castsi256_si128(out));
|
|
src_uv += 32;
|
|
dst_uv += 16;
|
|
width -= 16;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_HALFWIDTHROW_16TO8_AVX2
|
|
|
|
#ifdef HAS_HALFWIDTHROW_16TO8_ODD_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void HalfWidthRow_16To8_Odd_AVX2(const uint16_t* src_uv,
|
|
ptrdiff_t src_uv_stride,
|
|
uint8_t* dst_uv,
|
|
int scale,
|
|
int width) {
|
|
int n = (width - 1) & ~15;
|
|
int r = (width - 1) & 15;
|
|
SIMD_ALIGNED(uint16_t pair[64]);
|
|
SIMD_ALIGNED(uint8_t out[16]);
|
|
if (n > 0) {
|
|
HalfWidthRow_16To8_AVX2(src_uv, src_uv_stride, dst_uv, scale, n);
|
|
src_uv += n * 2;
|
|
dst_uv += n;
|
|
}
|
|
if (r > 0) {
|
|
HalfWidthRow_16To8_Any_AVX2(src_uv, src_uv_stride, dst_uv, scale, r);
|
|
src_uv += r * 2;
|
|
dst_uv += r;
|
|
}
|
|
memset(pair, 0, sizeof(pair));
|
|
pair[0] = src_uv[0];
|
|
pair[1] = src_uv[0];
|
|
pair[32] = src_uv[src_uv_stride];
|
|
pair[33] = src_uv[src_uv_stride];
|
|
HalfWidthRow_16To8_AVX2(pair, 32, out, scale, 16);
|
|
dst_uv[0] = out[0];
|
|
}
|
|
#endif // HAS_HALFWIDTHROW_16TO8_ODD_AVX2
|
|
|
|
#ifdef HAS_GAUSSCOL_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void GaussCol_AVX2(const uint16_t* src0,
|
|
const uint16_t* src1,
|
|
const uint16_t* src2,
|
|
const uint16_t* src3,
|
|
const uint16_t* src4,
|
|
uint32_t* dst,
|
|
int width) {
|
|
while (width >= 8) {
|
|
__m256i v0 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src0));
|
|
__m256i v1 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src1));
|
|
__m256i v2 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src2));
|
|
__m256i v3 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src3));
|
|
__m256i v4 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src4));
|
|
__m256i v04 = _mm256_add_epi32(v0, v4);
|
|
__m256i v13 = _mm256_slli_epi32(_mm256_add_epi32(v1, v3), 2);
|
|
__m256i v2_6 =
|
|
_mm256_add_epi32(_mm256_slli_epi32(v2, 2), _mm256_slli_epi32(v2, 1));
|
|
_mm256_storeu_si256((__m256i*)dst,
|
|
_mm256_add_epi32(_mm256_add_epi32(v04, v13), v2_6));
|
|
src0 += 8;
|
|
src1 += 8;
|
|
src2 += 8;
|
|
src3 += 8;
|
|
src4 += 8;
|
|
dst += 8;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
if (width > 0) {
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
GaussCol_C(src0, src1, src2, src3, src4, dst, width);
|
|
}
|
|
}
|
|
#endif // HAS_GAUSSCOL_AVX2
|
|
|
|
#ifdef HAS_GAUSSCOL_F32_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void GaussCol_F32_AVX2(const float* src0,
|
|
const float* src1,
|
|
const float* src2,
|
|
const float* src3,
|
|
const float* src4,
|
|
float* dst,
|
|
int width) {
|
|
const __m256 c4 = _mm256_set1_ps(4.0f);
|
|
const __m256 c6 = _mm256_set1_ps(6.0f);
|
|
while (width >= 8) {
|
|
__m256 s0 = _mm256_loadu_ps(src0);
|
|
__m256 s1 = _mm256_loadu_ps(src1);
|
|
__m256 s2 = _mm256_loadu_ps(src2);
|
|
__m256 s3 = _mm256_loadu_ps(src3);
|
|
__m256 s4 = _mm256_loadu_ps(src4);
|
|
__m256 sum = _mm256_add_ps(s0, _mm256_mul_ps(s1, c4));
|
|
sum = _mm256_add_ps(sum, _mm256_mul_ps(s2, c6));
|
|
sum = _mm256_add_ps(sum, _mm256_mul_ps(s3, c4));
|
|
sum = _mm256_add_ps(sum, s4);
|
|
_mm256_storeu_ps(dst, sum);
|
|
src0 += 8;
|
|
src1 += 8;
|
|
src2 += 8;
|
|
src3 += 8;
|
|
src4 += 8;
|
|
dst += 8;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
if (width > 0) {
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
GaussCol_F32_C(src0, src1, src2, src3, src4, dst, width);
|
|
}
|
|
}
|
|
#endif // HAS_GAUSSCOL_F32_AVX2
|
|
|
|
#ifdef HAS_SCALEMAXSAMPLES_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
float ScaleMaxSamples_AVX2(const float* src,
|
|
float* dst,
|
|
float scale,
|
|
int width) {
|
|
__m256 vmax = _mm256_setzero_ps();
|
|
const __m256 vscale = _mm256_set1_ps(scale);
|
|
while (width >= 8) {
|
|
__m256 v = _mm256_loadu_ps(src);
|
|
src += 8;
|
|
vmax = _mm256_max_ps(v, vmax);
|
|
_mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale));
|
|
dst += 8;
|
|
width -= 8;
|
|
}
|
|
__m128 m128 =
|
|
_mm_max_ps(_mm256_extractf128_ps(vmax, 1), _mm256_castps256_ps128(vmax));
|
|
m128 = _mm_max_ps(_mm_movehl_ps(m128, m128), m128);
|
|
m128 = _mm_max_ss(_mm_shuffle_ps(m128, m128, 1), m128);
|
|
float fmax = _mm_cvtss_f32(m128);
|
|
_mm256_zeroupper();
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
for (; width > 0; --width) {
|
|
float v = *src++;
|
|
float vs = v * scale;
|
|
fmax = (v > fmax) ? v : fmax;
|
|
*dst++ = vs;
|
|
}
|
|
return fmax;
|
|
}
|
|
#endif // HAS_SCALEMAXSAMPLES_AVX2
|
|
|
|
#ifdef HAS_SCALESUMSAMPLES_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
float ScaleSumSamples_AVX2(const float* src,
|
|
float* dst,
|
|
float scale,
|
|
int width) {
|
|
__m256 vsum = _mm256_setzero_ps();
|
|
const __m256 vscale = _mm256_set1_ps(scale);
|
|
while (width >= 8) {
|
|
__m256 v = _mm256_loadu_ps(src);
|
|
src += 8;
|
|
vsum = _mm256_add_ps(vsum, _mm256_mul_ps(v, v));
|
|
_mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale));
|
|
dst += 8;
|
|
width -= 8;
|
|
}
|
|
__m128 s128 =
|
|
_mm_add_ps(_mm256_castps256_ps128(vsum), _mm256_extractf128_ps(vsum, 1));
|
|
s128 = _mm_add_ps(s128, _mm_movehl_ps(s128, s128));
|
|
s128 = _mm_add_ss(s128, _mm_shuffle_ps(s128, s128, 1));
|
|
float fsum = _mm_cvtss_f32(s128);
|
|
_mm256_zeroupper();
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
for (; width > 0; --width) {
|
|
float v = *src++;
|
|
fsum += v * v;
|
|
*dst++ = v * scale;
|
|
}
|
|
return fsum;
|
|
}
|
|
#endif // HAS_SCALESUMSAMPLES_AVX2
|
|
|
|
#ifdef HAS_SCALESAMPLES_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width) {
|
|
const __m256 vscale = _mm256_set1_ps(scale);
|
|
while (width >= 8) {
|
|
__m256 v = _mm256_loadu_ps(src);
|
|
src += 8;
|
|
_mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale));
|
|
dst += 8;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
if (width > 0) {
|
|
// TODO(fbarchard): Handle remainder with masking or SIMD Any function.
|
|
ScaleSamples_C(src, dst, scale, width);
|
|
}
|
|
}
|
|
#endif // HAS_SCALESAMPLES_AVX2
|
|
|
|
#if defined(HAS_ARGBTORGB24ROW_AVX2) || defined(HAS_ARGBTORAWROW_AVX2)
|
|
LIBYUV_TARGET_AVX2
|
|
static inline void ARGBToRGBRow_AVX2(const uint8_t* src,
|
|
uint8_t* dst,
|
|
int width,
|
|
const uint8_t* shuffler) {
|
|
static const uint32_t kPermdRGB24_AVX[8] = {0, 1, 2, 4, 5, 6, 3, 7};
|
|
const __m256i ymm_shuf =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)shuffler));
|
|
const __m256i ymm_perm =
|
|
_mm256_loadu_si256((const __m256i*)kPermdRGB24_AVX);
|
|
do {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src);
|
|
__m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src + 32));
|
|
__m256i ymm2 = _mm256_loadu_si256((const __m256i*)(src + 64));
|
|
__m256i ymm3 = _mm256_loadu_si256((const __m256i*)(src + 96));
|
|
src += 128;
|
|
|
|
ymm0 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm0, ymm_shuf),
|
|
ymm_perm);
|
|
ymm1 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm1, ymm_shuf),
|
|
ymm_perm);
|
|
ymm2 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm2, ymm_shuf),
|
|
ymm_perm);
|
|
ymm3 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm3, ymm_shuf),
|
|
ymm_perm);
|
|
|
|
_mm256_storeu_si256(
|
|
(__m256i*)dst,
|
|
_mm256_or_si256(ymm0, _mm256_permute4x64_epi64(ymm1, 0x3f)));
|
|
_mm256_storeu_si256(
|
|
(__m256i*)(dst + 32),
|
|
_mm256_or_si256(_mm256_permute4x64_epi64(ymm1, 0xf9),
|
|
_mm256_permute4x64_epi64(ymm2, 0x4f)));
|
|
_mm256_storeu_si256(
|
|
(__m256i*)(dst + 64),
|
|
_mm256_or_si256(_mm256_permute4x64_epi64(ymm2, 0xfe),
|
|
_mm256_permute4x64_epi64(ymm3, 0x93)));
|
|
dst += 96;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_ARGBTORGB24ROW_AVX2) || defined(HAS_ARGBTORAWROW_AVX2)
|
|
|
|
#ifdef HAS_ARGBTORGB24ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToRGB24Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
|
|
ARGBToRGBRow_AVX2(src, dst, width, kShuffleMaskARGBToRGB24[0]);
|
|
}
|
|
#endif // HAS_ARGBTORGB24ROW_AVX2
|
|
|
|
#ifdef HAS_ARGBTORAWROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToRAWRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
|
|
static const uint8_t kShuffleMaskARGBToRAW_AVX2[16] = {
|
|
2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u,
|
|
128u};
|
|
ARGBToRGBRow_AVX2(src, dst, width, kShuffleMaskARGBToRAW_AVX2);
|
|
}
|
|
#endif // HAS_ARGBTORAWROW_AVX2
|
|
|
|
#ifdef HAS_SPLITUVROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void SplitUVRow_AVX2(const uint8_t* src_uv,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
const __m256i mask_00ff = _mm256_set1_epi16(0x00ff);
|
|
do {
|
|
__m256i uv0 = _mm256_loadu_si256((const __m256i*)src_uv);
|
|
__m256i uv1 = _mm256_loadu_si256((const __m256i*)(src_uv + 32));
|
|
__m256i u = _mm256_permute4x64_epi64(
|
|
_mm256_packus_epi16(_mm256_and_si256(uv0, mask_00ff),
|
|
_mm256_and_si256(uv1, mask_00ff)),
|
|
0xd8);
|
|
__m256i v = _mm256_permute4x64_epi64(
|
|
_mm256_packus_epi16(_mm256_srli_epi16(uv0, 8),
|
|
_mm256_srli_epi16(uv1, 8)),
|
|
0xd8);
|
|
_mm256_storeu_si256((__m256i*)dst_u, u);
|
|
_mm256_storeu_si256((__m256i*)dst_v, v);
|
|
src_uv += 64;
|
|
dst_u += 32;
|
|
dst_v += 32;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SPLITUVROW_AVX2
|
|
|
|
#ifdef HAS_AR64TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void AR64ToARGBRow_AVX2(const uint16_t* src_ar64,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
do {
|
|
__m256i s0 = _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)src_ar64),
|
|
8);
|
|
__m256i s1 = _mm256_srli_epi16(
|
|
_mm256_loadu_si256((const __m256i*)(src_ar64 + 16)), 8);
|
|
__m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8);
|
|
_mm256_storeu_si256((__m256i*)dst_argb, out);
|
|
src_ar64 += 32;
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_AR64TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_AB64TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void AB64ToARGBRow_AVX2(const uint16_t* src_ab64,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
static const uint8_t kShufSwapRB[16] = {2u, 1u, 0u, 3u, 6u, 5u, 4u, 7u,
|
|
10u, 9u, 8u, 11u, 14u, 13u, 12u, 15u};
|
|
const __m256i shuf = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShufSwapRB));
|
|
do {
|
|
__m256i s0 = _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)src_ab64),
|
|
8);
|
|
__m256i s1 = _mm256_srli_epi16(
|
|
_mm256_loadu_si256((const __m256i*)(src_ab64 + 16)), 8);
|
|
__m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8);
|
|
out = _mm256_shuffle_epi8(out, shuf);
|
|
_mm256_storeu_si256((__m256i*)dst_argb, out);
|
|
src_ab64 += 32;
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_AB64TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_ARGBTOAR64ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToAR64Row_AVX2(const uint8_t* src_argb,
|
|
uint16_t* dst_ar64,
|
|
int width) {
|
|
do {
|
|
__m256i s = _mm256_permute4x64_epi64(
|
|
_mm256_loadu_si256((const __m256i*)src_argb), 0xd8);
|
|
_mm256_storeu_si256((__m256i*)dst_ar64, _mm256_unpacklo_epi8(s, s));
|
|
_mm256_storeu_si256((__m256i*)(dst_ar64 + 16), _mm256_unpackhi_epi8(s, s));
|
|
src_argb += 32;
|
|
dst_ar64 += 32;
|
|
width -= 8;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOAR64ROW_AVX2
|
|
|
|
#ifdef HAS_ARGBTOAB64ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToAB64Row_AVX2(const uint8_t* src_argb,
|
|
uint16_t* dst_ab64,
|
|
int width) {
|
|
static const uint8_t kShufSwapRB[16] = {2u, 1u, 0u, 3u, 6u, 5u, 4u, 7u,
|
|
10u, 9u, 8u, 11u, 14u, 13u, 12u, 15u};
|
|
const __m256i shuf = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShufSwapRB));
|
|
do {
|
|
__m256i s = _mm256_shuffle_epi8(
|
|
_mm256_loadu_si256((const __m256i*)src_argb), shuf);
|
|
s = _mm256_permute4x64_epi64(s, 0xd8);
|
|
_mm256_storeu_si256((__m256i*)dst_ab64, _mm256_unpacklo_epi8(s, s));
|
|
_mm256_storeu_si256((__m256i*)(dst_ab64 + 16), _mm256_unpackhi_epi8(s, s));
|
|
src_argb += 32;
|
|
dst_ab64 += 32;
|
|
width -= 8;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOAB64ROW_AVX2
|
|
|
|
#ifdef HAS_ARGBADDROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBAddRow_AVX2(const uint8_t* src_argb,
|
|
const uint8_t* src_argb1,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
do {
|
|
__m256i s0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i s1 = _mm256_loadu_si256((const __m256i*)src_argb1);
|
|
_mm256_storeu_si256((__m256i*)dst_argb, _mm256_adds_epu8(s0, s1));
|
|
src_argb += 32;
|
|
src_argb1 += 32;
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBADDROW_AVX2
|
|
|
|
#ifdef HAS_ARGBSUBTRACTROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBSubtractRow_AVX2(const uint8_t* src_argb,
|
|
const uint8_t* src_argb1,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
do {
|
|
__m256i s0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i s1 = _mm256_loadu_si256((const __m256i*)src_argb1);
|
|
_mm256_storeu_si256((__m256i*)dst_argb, _mm256_subs_epu8(s0, s1));
|
|
src_argb += 32;
|
|
src_argb1 += 32;
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBSUBTRACTROW_AVX2
|
|
|
|
#ifdef HAS_ARGBCOPYALPHAROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBCopyAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
|
|
const __m256i mask_alpha = _mm256_set1_epi32((int)0xff000000u);
|
|
do {
|
|
__m256i s0 = _mm256_loadu_si256((const __m256i*)src);
|
|
__m256i s1 = _mm256_loadu_si256((const __m256i*)(src + 32));
|
|
__m256i d0 = _mm256_loadu_si256((const __m256i*)dst);
|
|
__m256i d1 = _mm256_loadu_si256((const __m256i*)(dst + 32));
|
|
_mm256_storeu_si256((__m256i*)dst, _mm256_blendv_epi8(d0, s0, mask_alpha));
|
|
_mm256_storeu_si256((__m256i*)(dst + 32),
|
|
_mm256_blendv_epi8(d1, s1, mask_alpha));
|
|
src += 64;
|
|
dst += 64;
|
|
width -= 16;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBCOPYALPHAROW_AVX2
|
|
|
|
#ifdef HAS_ARGBPOLYNOMIALROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBPolynomialRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
const float* poly,
|
|
int width) {
|
|
const __m256 c0 = _mm256_broadcast_ps((const __m128*)(poly + 0));
|
|
const __m256 c1 = _mm256_broadcast_ps((const __m128*)(poly + 4));
|
|
const __m256 c2 = _mm256_broadcast_ps((const __m128*)(poly + 8));
|
|
const __m256 c3 = _mm256_broadcast_ps((const __m128*)(poly + 12));
|
|
do {
|
|
__m128i s8 = _mm_loadl_epi64((const __m128i*)src_argb);
|
|
src_argb += 8;
|
|
__m256 x = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(s8));
|
|
__m256 x2 = _mm256_mul_ps(x, x);
|
|
__m256 res = _mm256_add_ps(c0, _mm256_mul_ps(c1, x));
|
|
res = _mm256_add_ps(res, _mm256_mul_ps(c2, x2));
|
|
res = _mm256_add_ps(res, _mm256_mul_ps(_mm256_mul_ps(c3, x), x2));
|
|
__m256i i32 = _mm256_cvttps_epi32(res);
|
|
__m256i u16 = _mm256_permute4x64_epi64(_mm256_packs_epi32(i32, i32), 0xd8);
|
|
__m128i u8 = _mm_packus_epi16(_mm256_castsi256_si128(u16),
|
|
_mm256_castsi256_si128(u16));
|
|
_mm_storel_epi64((__m128i*)dst_argb, u8);
|
|
dst_argb += 8;
|
|
width -= 2;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBPOLYNOMIALROW_AVX2
|
|
|
|
#ifdef HAS_ARGBUNATTENUATEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBUnattenuateRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
static const uint8_t kUnattenShuffleAlpha[16] = {
|
|
0u, 1u, 0u, 1u, 0u, 1u, 6u, 7u, 8u, 9u, 8u, 9u, 8u, 9u, 14u, 15u};
|
|
const __m256i ymm_shuf = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kUnattenShuffleAlpha));
|
|
do {
|
|
__m256i ymm3 = _mm256_setr_epi32(
|
|
(int)fixed_invtbl8[src_argb[3]], (int)fixed_invtbl8[src_argb[7]],
|
|
(int)fixed_invtbl8[src_argb[11]], (int)fixed_invtbl8[src_argb[15]],
|
|
(int)fixed_invtbl8[src_argb[19]], (int)fixed_invtbl8[src_argb[23]],
|
|
(int)fixed_invtbl8[src_argb[27]], (int)fixed_invtbl8[src_argb[31]]);
|
|
__m256i ymm6 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i ymm0 = _mm256_unpacklo_epi8(ymm6, ymm6);
|
|
__m256i ymm1 = _mm256_unpackhi_epi8(ymm6, ymm6);
|
|
__m256i ymm2 =
|
|
_mm256_shuffle_epi8(_mm256_unpacklo_epi16(ymm3, ymm3), ymm_shuf);
|
|
ymm3 = _mm256_shuffle_epi8(_mm256_unpackhi_epi16(ymm3, ymm3), ymm_shuf);
|
|
ymm0 = _mm256_mulhi_epu16(ymm0, ymm2);
|
|
ymm1 = _mm256_mulhi_epu16(ymm1, ymm3);
|
|
_mm256_storeu_si256((__m256i*)dst_argb, _mm256_packus_epi16(ymm0, ymm1));
|
|
src_argb += 32;
|
|
dst_argb += 32;
|
|
width -= 8;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBUNATTENUATEROW_AVX2
|
|
|
|
#ifdef HAS_COMPUTECUMULATIVESUMROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ComputeCumulativeSumRow_AVX2(const uint8_t* row,
|
|
int32_t* cumsum,
|
|
const int32_t* previous_cumsum,
|
|
int width) {
|
|
static const uint8_t kCumSumTranspose[16] = {0, 4, 8, 12, 1, 5, 9, 13,
|
|
2, 6, 10, 14, 3, 7, 11, 15};
|
|
static const int8_t kCumSumMaddLo[32] = {
|
|
1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0,
|
|
1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0};
|
|
static const int8_t kCumSumMaddHi[32] = {
|
|
1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0,
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
|
|
__m256i ymm0 = _mm256_setzero_si256();
|
|
const __m256i ymm5 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kCumSumTranspose));
|
|
const __m256i ymm6 = _mm256_loadu_si256((const __m256i*)kCumSumMaddLo);
|
|
const __m256i ymm7 = _mm256_loadu_si256((const __m256i*)kCumSumMaddHi);
|
|
const __m256i ymm8 = _mm256_set1_epi16(1);
|
|
|
|
while (width >= 4) {
|
|
__m256i ymm2 =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)row));
|
|
row += 16;
|
|
ymm2 = _mm256_shuffle_epi8(ymm2, ymm5);
|
|
__m256i ymm3 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm2, ymm6), ymm8);
|
|
__m256i ymm4 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm2, ymm7), ymm8);
|
|
__m256i sum4 = _mm256_permute2x128_si256(ymm4, ymm4, 0x11);
|
|
ymm3 = _mm256_add_epi32(
|
|
_mm256_add_epi32(ymm3,
|
|
_mm256_loadu_si256((const __m256i*)previous_cumsum)),
|
|
ymm0);
|
|
ymm4 = _mm256_add_epi32(
|
|
_mm256_add_epi32(
|
|
ymm4, _mm256_loadu_si256((const __m256i*)(previous_cumsum + 8))),
|
|
ymm0);
|
|
previous_cumsum += 16;
|
|
ymm0 = _mm256_add_epi32(ymm0, sum4);
|
|
_mm256_storeu_si256((__m256i*)cumsum, ymm3);
|
|
_mm256_storeu_si256((__m256i*)(cumsum + 8), ymm4);
|
|
cumsum += 16;
|
|
width -= 4;
|
|
}
|
|
|
|
__m128i xmm0 = _mm256_castsi256_si128(ymm0);
|
|
while (width > 0) {
|
|
int32_t px;
|
|
memcpy(&px, row, sizeof(px));
|
|
row += 4;
|
|
__m128i xmm2 = _mm_cvtepu8_epi32(_mm_cvtsi32_si128(px));
|
|
xmm0 = _mm_add_epi32(xmm0, xmm2);
|
|
xmm2 = _mm_add_epi32(xmm0,
|
|
_mm_loadu_si128((const __m128i*)previous_cumsum));
|
|
previous_cumsum += 4;
|
|
_mm_storeu_si128((__m128i*)cumsum, xmm2);
|
|
cumsum += 4;
|
|
--width;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_COMPUTECUMULATIVESUMROW_AVX2
|
|
|
|
#ifdef HAS_CUMULATIVESUMTOAVERAGEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void CumulativeSumToAverageRow_AVX2(const int32_t* topleft,
|
|
const int32_t* botleft,
|
|
int width,
|
|
int area,
|
|
uint8_t* dst,
|
|
int count) {
|
|
__m128 xmm5 = _mm_cvtsi32_ss(_mm_setzero_ps(), area);
|
|
__m128 xmm4 = _mm_div_ss(_mm_set_ss(1.0f), xmm5);
|
|
xmm4 = _mm_shuffle_ps(xmm4, xmm4, 0);
|
|
|
|
if (count >= 4) {
|
|
__m256 ymm4 = _mm256_insertf128_ps(_mm256_castps128_ps256(xmm4), xmm4, 1);
|
|
if (area > 1 && area <= 0x80) {
|
|
__m128 xmm6 = _mm_cvtepi32_ps(_mm_set1_epi32(0x0000ffff));
|
|
xmm6 = _mm_mul_ss(_mm_add_ss(xmm5, xmm6), xmm4);
|
|
__m128i i6 = _mm_cvtps_epi32(xmm6);
|
|
i6 = _mm_packus_epi32(i6, i6);
|
|
__m256i ymm6 = _mm256_broadcastw_epi16(i6);
|
|
while (count >= 4) {
|
|
__m256i ymm0 = _mm256_sub_epi32(
|
|
_mm256_loadu_si256((const __m256i*)topleft),
|
|
_mm256_loadu_si256((const __m256i*)(topleft + width)));
|
|
__m256i ymm1 = _mm256_sub_epi32(
|
|
_mm256_loadu_si256((const __m256i*)(topleft + 8)),
|
|
_mm256_loadu_si256((const __m256i*)(topleft + width + 8)));
|
|
topleft += 16;
|
|
ymm0 = _mm256_sub_epi32(ymm0,
|
|
_mm256_loadu_si256((const __m256i*)botleft));
|
|
ymm1 = _mm256_sub_epi32(
|
|
ymm1, _mm256_loadu_si256((const __m256i*)(botleft + 8)));
|
|
ymm0 = _mm256_add_epi32(
|
|
ymm0, _mm256_loadu_si256((const __m256i*)(botleft + width)));
|
|
ymm1 = _mm256_add_epi32(
|
|
ymm1, _mm256_loadu_si256((const __m256i*)(botleft + width + 8)));
|
|
botleft += 16;
|
|
ymm0 = _mm256_mulhi_epu16(_mm256_packus_epi32(ymm0, ymm1), ymm6);
|
|
__m128i out = _mm_packus_epi16(_mm256_castsi256_si128(ymm0),
|
|
_mm256_extracti128_si256(ymm0, 1));
|
|
_mm_storeu_si128((__m128i*)dst, _mm_shuffle_epi32(out, 0xd8));
|
|
dst += 16;
|
|
count -= 4;
|
|
}
|
|
} else {
|
|
while (count >= 4) {
|
|
__m256i ymm0 = _mm256_sub_epi32(
|
|
_mm256_loadu_si256((const __m256i*)topleft),
|
|
_mm256_loadu_si256((const __m256i*)(topleft + width)));
|
|
__m256i ymm1 = _mm256_sub_epi32(
|
|
_mm256_loadu_si256((const __m256i*)(topleft + 8)),
|
|
_mm256_loadu_si256((const __m256i*)(topleft + width + 8)));
|
|
topleft += 16;
|
|
ymm0 = _mm256_sub_epi32(ymm0,
|
|
_mm256_loadu_si256((const __m256i*)botleft));
|
|
ymm1 = _mm256_sub_epi32(
|
|
ymm1, _mm256_loadu_si256((const __m256i*)(botleft + 8)));
|
|
ymm0 = _mm256_add_epi32(
|
|
ymm0, _mm256_loadu_si256((const __m256i*)(botleft + width)));
|
|
ymm1 = _mm256_add_epi32(
|
|
ymm1, _mm256_loadu_si256((const __m256i*)(botleft + width + 8)));
|
|
botleft += 16;
|
|
ymm0 =
|
|
_mm256_cvttps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(ymm0), ymm4));
|
|
ymm1 =
|
|
_mm256_cvttps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(ymm1), ymm4));
|
|
ymm0 = _mm256_packus_epi32(ymm0, ymm1);
|
|
__m128i out = _mm_packus_epi16(_mm256_castsi256_si128(ymm0),
|
|
_mm256_extracti128_si256(ymm0, 1));
|
|
_mm_storeu_si128((__m128i*)dst, _mm_shuffle_epi32(out, 0xd8));
|
|
dst += 16;
|
|
count -= 4;
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
|
|
while (count > 0) {
|
|
__m128i d0 = _mm_sub_epi32(
|
|
_mm_loadu_si128((const __m128i*)topleft),
|
|
_mm_loadu_si128((const __m128i*)(topleft + width)));
|
|
topleft += 4;
|
|
__m128i d1 = _mm_sub_epi32(
|
|
_mm_loadu_si128((const __m128i*)(botleft + width)),
|
|
_mm_loadu_si128((const __m128i*)botleft));
|
|
botleft += 4;
|
|
d0 = _mm_add_epi32(d0, d1);
|
|
d0 = _mm_cvttps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(d0), xmm4));
|
|
d0 = _mm_packus_epi32(d0, d0);
|
|
d0 = _mm_packus_epi16(d0, d0);
|
|
int32_t out32 = _mm_cvtsi128_si32(d0);
|
|
memcpy(dst, &out32, sizeof(out32));
|
|
dst += 4;
|
|
--count;
|
|
}
|
|
}
|
|
#endif // HAS_CUMULATIVESUMTOAVERAGEROW_AVX2
|
|
|
|
#ifdef HAS_I422TOYUY2ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void I422ToYUY2Row_AVX2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_yuy2,
|
|
int width) {
|
|
do {
|
|
__m128i u = _mm_loadu_si128((const __m128i*)src_u);
|
|
__m128i v = _mm_loadu_si128((const __m128i*)src_v);
|
|
__m256i uv = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_unpacklo_epi8(u, v)),
|
|
_mm_unpackhi_epi8(u, v), 1);
|
|
__m256i y = _mm256_loadu_si256((const __m256i*)src_y);
|
|
__m256i lo = _mm256_unpacklo_epi8(y, uv);
|
|
__m256i hi = _mm256_unpackhi_epi8(y, uv);
|
|
_mm256_storeu_si256((__m256i*)dst_yuy2,
|
|
_mm256_permute2x128_si256(lo, hi, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst_yuy2 + 32),
|
|
_mm256_permute2x128_si256(lo, hi, 0x31));
|
|
src_y += 32;
|
|
src_u += 16;
|
|
src_v += 16;
|
|
dst_yuy2 += 64;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422TOYUY2ROW_AVX2
|
|
|
|
#ifdef HAS_I422TOUYVYROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void I422ToUYVYRow_AVX2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_uyvy,
|
|
int width) {
|
|
do {
|
|
__m128i u = _mm_loadu_si128((const __m128i*)src_u);
|
|
__m128i v = _mm_loadu_si128((const __m128i*)src_v);
|
|
__m256i uv = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_unpacklo_epi8(u, v)),
|
|
_mm_unpackhi_epi8(u, v), 1);
|
|
__m256i y = _mm256_loadu_si256((const __m256i*)src_y);
|
|
__m256i lo = _mm256_unpacklo_epi8(uv, y);
|
|
__m256i hi = _mm256_unpackhi_epi8(uv, y);
|
|
_mm256_storeu_si256((__m256i*)dst_uyvy,
|
|
_mm256_permute2x128_si256(lo, hi, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst_uyvy + 32),
|
|
_mm256_permute2x128_si256(lo, hi, 0x31));
|
|
src_y += 32;
|
|
src_u += 16;
|
|
src_v += 16;
|
|
dst_uyvy += 64;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422TOUYVYROW_AVX2
|
|
|
|
#ifdef HAS_MERGEARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MergeARGBRow_AVX2(const uint8_t* src_r,
|
|
const uint8_t* src_g,
|
|
const uint8_t* src_b,
|
|
const uint8_t* src_a,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
do {
|
|
__m128i r = _mm_loadu_si128((const __m128i*)src_r);
|
|
__m128i g = _mm_loadu_si128((const __m128i*)src_g);
|
|
__m128i b = _mm_loadu_si128((const __m128i*)src_b);
|
|
__m128i a = _mm_loadu_si128((const __m128i*)src_a);
|
|
__m256i bg = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)),
|
|
_mm_unpackhi_epi8(b, g), 1);
|
|
__m256i ra = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)),
|
|
_mm_unpackhi_epi8(r, a), 1);
|
|
__m256i lo = _mm256_unpacklo_epi16(bg, ra);
|
|
__m256i hi = _mm256_unpackhi_epi16(bg, ra);
|
|
_mm256_storeu_si256((__m256i*)dst_argb,
|
|
_mm256_permute2x128_si256(lo, hi, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32),
|
|
_mm256_permute2x128_si256(lo, hi, 0x31));
|
|
src_r += 16;
|
|
src_g += 16;
|
|
src_b += 16;
|
|
src_a += 16;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGEARGBROW_AVX2
|
|
|
|
#ifdef HAS_MERGEXRGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MergeXRGBRow_AVX2(const uint8_t* src_r,
|
|
const uint8_t* src_g,
|
|
const uint8_t* src_b,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
const __m128i a = _mm_set1_epi8((char)0xff);
|
|
do {
|
|
__m128i r = _mm_loadu_si128((const __m128i*)src_r);
|
|
__m128i g = _mm_loadu_si128((const __m128i*)src_g);
|
|
__m128i b = _mm_loadu_si128((const __m128i*)src_b);
|
|
__m256i bg = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)),
|
|
_mm_unpackhi_epi8(b, g), 1);
|
|
__m256i ra = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)),
|
|
_mm_unpackhi_epi8(r, a), 1);
|
|
__m256i lo = _mm256_unpacklo_epi16(bg, ra);
|
|
__m256i hi = _mm256_unpackhi_epi16(bg, ra);
|
|
_mm256_storeu_si256((__m256i*)dst_argb,
|
|
_mm256_permute2x128_si256(lo, hi, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32),
|
|
_mm256_permute2x128_si256(lo, hi, 0x31));
|
|
src_r += 16;
|
|
src_g += 16;
|
|
src_b += 16;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGEXRGBROW_AVX2
|
|
|
|
#ifdef HAS_MERGEARGB16TO8ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MergeARGB16To8Row_AVX2(const uint16_t* src_r,
|
|
const uint16_t* src_g,
|
|
const uint16_t* src_b,
|
|
const uint16_t* src_a,
|
|
uint8_t* dst_argb,
|
|
int depth,
|
|
int width) {
|
|
const __m128i shift128 = _mm_cvtsi32_si128(depth - 8);
|
|
const __m256i ymm_255 = _mm256_set1_epi16(255);
|
|
do {
|
|
__m256i r16 = _mm256_min_epu16(
|
|
_mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_r), shift128),
|
|
ymm_255);
|
|
__m256i g16 = _mm256_min_epu16(
|
|
_mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_g), shift128),
|
|
ymm_255);
|
|
__m256i b16 = _mm256_min_epu16(
|
|
_mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_b), shift128),
|
|
ymm_255);
|
|
__m256i a16 = _mm256_min_epu16(
|
|
_mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_a), shift128),
|
|
ymm_255);
|
|
__m256i br8 =
|
|
_mm256_permute4x64_epi64(_mm256_packus_epi16(b16, r16), 0xd8);
|
|
__m256i ga8 =
|
|
_mm256_permute4x64_epi64(_mm256_packus_epi16(g16, a16), 0xd8);
|
|
__m128i b = _mm256_castsi256_si128(br8);
|
|
__m128i r = _mm256_extracti128_si256(br8, 1);
|
|
__m128i g = _mm256_castsi256_si128(ga8);
|
|
__m128i a = _mm256_extracti128_si256(ga8, 1);
|
|
__m256i bg = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)),
|
|
_mm_unpackhi_epi8(b, g), 1);
|
|
__m256i ra = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)),
|
|
_mm_unpackhi_epi8(r, a), 1);
|
|
__m256i lo = _mm256_unpacklo_epi16(bg, ra);
|
|
__m256i hi = _mm256_unpackhi_epi16(bg, ra);
|
|
_mm256_storeu_si256((__m256i*)dst_argb,
|
|
_mm256_permute2x128_si256(lo, hi, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32),
|
|
_mm256_permute2x128_si256(lo, hi, 0x31));
|
|
src_r += 16;
|
|
src_g += 16;
|
|
src_b += 16;
|
|
src_a += 16;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGEARGB16TO8ROW_AVX2
|
|
|
|
#ifdef HAS_MERGEXRGB16TO8ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MergeXRGB16To8Row_AVX2(const uint16_t* src_r,
|
|
const uint16_t* src_g,
|
|
const uint16_t* src_b,
|
|
uint8_t* dst_argb,
|
|
int depth,
|
|
int width) {
|
|
const __m128i shift128 = _mm_cvtsi32_si128(depth - 8);
|
|
const __m256i ymm_255 = _mm256_set1_epi16(255);
|
|
const __m128i a = _mm_set1_epi8((char)0xff);
|
|
do {
|
|
__m256i r16 = _mm256_min_epu16(
|
|
_mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_r), shift128),
|
|
ymm_255);
|
|
__m256i g16 = _mm256_min_epu16(
|
|
_mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_g), shift128),
|
|
ymm_255);
|
|
__m256i b16 = _mm256_min_epu16(
|
|
_mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_b), shift128),
|
|
ymm_255);
|
|
__m256i br8 =
|
|
_mm256_permute4x64_epi64(_mm256_packus_epi16(b16, r16), 0xd8);
|
|
__m256i g8 =
|
|
_mm256_permute4x64_epi64(_mm256_packus_epi16(g16, g16), 0xd8);
|
|
__m128i b = _mm256_castsi256_si128(br8);
|
|
__m128i r = _mm256_extracti128_si256(br8, 1);
|
|
__m128i g = _mm256_castsi256_si128(g8);
|
|
__m256i bg = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)),
|
|
_mm_unpackhi_epi8(b, g), 1);
|
|
__m256i ra = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)),
|
|
_mm_unpackhi_epi8(r, a), 1);
|
|
__m256i lo = _mm256_unpacklo_epi16(bg, ra);
|
|
__m256i hi = _mm256_unpackhi_epi16(bg, ra);
|
|
_mm256_storeu_si256((__m256i*)dst_argb,
|
|
_mm256_permute2x128_si256(lo, hi, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32),
|
|
_mm256_permute2x128_si256(lo, hi, 0x31));
|
|
src_r += 16;
|
|
src_g += 16;
|
|
src_b += 16;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGEXRGB16TO8ROW_AVX2
|
|
|
|
#ifdef HAS_MERGEAR64ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MergeAR64Row_AVX2(const uint16_t* src_r,
|
|
const uint16_t* src_g,
|
|
const uint16_t* src_b,
|
|
const uint16_t* src_a,
|
|
uint16_t* dst_ar64,
|
|
int depth,
|
|
int width) {
|
|
const __m128i shift128 = _mm_cvtsi32_si128(16 - depth);
|
|
const uint32_t max_val = (depth >= 16) ? 0xffffu : ((1u << depth) - 1u);
|
|
const __m256i ymm_max = _mm256_set1_epi16((short)max_val);
|
|
do {
|
|
__m256i r = _mm256_sll_epi16(
|
|
_mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_r), ymm_max),
|
|
shift128);
|
|
__m256i g = _mm256_sll_epi16(
|
|
_mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_g), ymm_max),
|
|
shift128);
|
|
__m256i b = _mm256_sll_epi16(
|
|
_mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_b), ymm_max),
|
|
shift128);
|
|
__m256i a = _mm256_sll_epi16(
|
|
_mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_a), ymm_max),
|
|
shift128);
|
|
__m256i bg_lo = _mm256_unpacklo_epi16(b, g);
|
|
__m256i bg_hi = _mm256_unpackhi_epi16(b, g);
|
|
__m256i ra_lo = _mm256_unpacklo_epi16(r, a);
|
|
__m256i ra_hi = _mm256_unpackhi_epi16(r, a);
|
|
__m256i p0_8 = _mm256_unpacklo_epi32(bg_lo, ra_lo);
|
|
__m256i p2_10 = _mm256_unpackhi_epi32(bg_lo, ra_lo);
|
|
__m256i p4_12 = _mm256_unpacklo_epi32(bg_hi, ra_hi);
|
|
__m256i p6_14 = _mm256_unpackhi_epi32(bg_hi, ra_hi);
|
|
_mm256_storeu_si256((__m256i*)dst_ar64,
|
|
_mm256_permute2x128_si256(p0_8, p2_10, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst_ar64 + 16),
|
|
_mm256_permute2x128_si256(p4_12, p6_14, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst_ar64 + 32),
|
|
_mm256_permute2x128_si256(p0_8, p2_10, 0x31));
|
|
_mm256_storeu_si256((__m256i*)(dst_ar64 + 48),
|
|
_mm256_permute2x128_si256(p4_12, p6_14, 0x31));
|
|
src_r += 16;
|
|
src_g += 16;
|
|
src_b += 16;
|
|
src_a += 16;
|
|
dst_ar64 += 64;
|
|
width -= 16;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGEAR64ROW_AVX2
|
|
|
|
#ifdef HAS_MERGEXR64ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MergeXR64Row_AVX2(const uint16_t* src_r,
|
|
const uint16_t* src_g,
|
|
const uint16_t* src_b,
|
|
uint16_t* dst_ar64,
|
|
int depth,
|
|
int width) {
|
|
const __m128i shift128 = _mm_cvtsi32_si128(16 - depth);
|
|
const uint32_t max_val = (depth >= 16) ? 0xffffu : ((1u << depth) - 1u);
|
|
const __m256i ymm_max = _mm256_set1_epi16((short)max_val);
|
|
const __m256i a = _mm256_set1_epi16((short)0xffff);
|
|
do {
|
|
__m256i r = _mm256_sll_epi16(
|
|
_mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_r), ymm_max),
|
|
shift128);
|
|
__m256i g = _mm256_sll_epi16(
|
|
_mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_g), ymm_max),
|
|
shift128);
|
|
__m256i b = _mm256_sll_epi16(
|
|
_mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_b), ymm_max),
|
|
shift128);
|
|
__m256i bg_lo = _mm256_unpacklo_epi16(b, g);
|
|
__m256i bg_hi = _mm256_unpackhi_epi16(b, g);
|
|
__m256i ra_lo = _mm256_unpacklo_epi16(r, a);
|
|
__m256i ra_hi = _mm256_unpackhi_epi16(r, a);
|
|
__m256i p0_8 = _mm256_unpacklo_epi32(bg_lo, ra_lo);
|
|
__m256i p2_10 = _mm256_unpackhi_epi32(bg_lo, ra_lo);
|
|
__m256i p4_12 = _mm256_unpacklo_epi32(bg_hi, ra_hi);
|
|
__m256i p6_14 = _mm256_unpackhi_epi32(bg_hi, ra_hi);
|
|
_mm256_storeu_si256((__m256i*)dst_ar64,
|
|
_mm256_permute2x128_si256(p0_8, p2_10, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst_ar64 + 16),
|
|
_mm256_permute2x128_si256(p4_12, p6_14, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst_ar64 + 32),
|
|
_mm256_permute2x128_si256(p0_8, p2_10, 0x31));
|
|
_mm256_storeu_si256((__m256i*)(dst_ar64 + 48),
|
|
_mm256_permute2x128_si256(p4_12, p6_14, 0x31));
|
|
src_r += 16;
|
|
src_g += 16;
|
|
src_b += 16;
|
|
dst_ar64 += 64;
|
|
width -= 16;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGEXR64ROW_AVX2
|
|
|
|
#ifdef HAS_NV21TOYUV24ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void NV21ToYUV24Row_AVX2(const uint8_t* src_y,
|
|
const uint8_t* src_vu,
|
|
uint8_t* dst_yuv24,
|
|
int width) {
|
|
static const uint8_t kYUV24Shuffle[3][16] = {
|
|
{8, 9, 0, 8, 9, 1, 10, 11, 2, 10, 11, 3, 12, 13, 4, 12},
|
|
{9, 1, 10, 11, 2, 10, 11, 3, 12, 13, 4, 12, 13, 5, 14, 15},
|
|
{2, 10, 11, 3, 12, 13, 4, 12, 13, 5, 14, 15, 6, 14, 15, 7}};
|
|
const __m256i ymm4 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kYUV24Shuffle[0]));
|
|
const __m256i ymm5 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kYUV24Shuffle[1]));
|
|
const __m256i ymm6 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kYUV24Shuffle[2]));
|
|
|
|
do {
|
|
__m256 ymm2 = _mm256_castsi256_ps(_mm256_loadu_si256((const __m256i*)src_y));
|
|
__m256 ymm3 =
|
|
_mm256_castsi256_ps(_mm256_loadu_si256((const __m256i*)src_vu));
|
|
__m256i c0 = _mm256_shuffle_epi8(
|
|
_mm256_castps_si256(_mm256_shuffle_ps(ymm2, ymm3, 0x44)), ymm4);
|
|
__m256i c1 = _mm256_shuffle_epi8(
|
|
_mm256_castps_si256(_mm256_shuffle_ps(ymm2, ymm3, 0x99)), ymm5);
|
|
__m256i c2 = _mm256_shuffle_epi8(
|
|
_mm256_castps_si256(_mm256_shuffle_ps(ymm2, ymm3, 0xee)), ymm6);
|
|
_mm256_storeu_si256((__m256i*)dst_yuv24,
|
|
_mm256_permute2x128_si256(c0, c1, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst_yuv24 + 32),
|
|
_mm256_permute2x128_si256(c2, c0, 0x30));
|
|
_mm256_storeu_si256((__m256i*)(dst_yuv24 + 64),
|
|
_mm256_permute2x128_si256(c1, c2, 0x31));
|
|
src_y += 32;
|
|
src_vu += 32;
|
|
dst_yuv24 += 96;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_NV21TOYUV24ROW_AVX2
|
|
|
|
#ifdef HAS_NV21TOYUV24ROW_AVX512VBMI
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
void NV21ToYUV24Row_AVX512VBMI(const uint8_t* src_y,
|
|
const uint8_t* src_vu,
|
|
uint8_t* dst_yuv24,
|
|
int width) {
|
|
static const uint8_t kNV21ToYUV24Perm_AVX512VBMI[3][64] = {
|
|
{64, 65, 0, 64, 65, 1, 66, 67, 2, 66, 67, 3, 68, 69, 4, 68,
|
|
69, 5, 70, 71, 6, 70, 71, 7, 72, 73, 8, 72, 73, 9, 74, 75,
|
|
10, 74, 75, 11, 76, 77, 12, 76, 77, 13, 78, 79, 14, 78, 79, 15,
|
|
80, 81, 16, 80, 81, 17, 82, 83, 18, 82, 83, 19, 84, 85, 20, 84},
|
|
{85, 21, 86, 87, 22, 86, 87, 23, 88, 89, 24, 88, 89,
|
|
25, 90, 91, 26, 90, 91, 27, 92, 93, 28, 92, 93, 29,
|
|
94, 95, 30, 94, 95, 31, 96, 97, 32, 96, 97, 33, 98,
|
|
99, 34, 98, 99, 35, 100, 101, 36, 100, 101, 37, 102, 103,
|
|
38, 102, 103, 39, 104, 105, 40, 104, 105, 41, 106, 107},
|
|
{42, 106, 107, 43, 108, 109, 44, 108, 109, 45, 110, 111, 46,
|
|
110, 111, 47, 112, 113, 48, 112, 113, 49, 114, 115, 50, 114,
|
|
115, 51, 116, 117, 52, 116, 117, 53, 118, 119, 54, 118, 119,
|
|
55, 120, 121, 56, 120, 121, 57, 122, 123, 58, 122, 123, 59,
|
|
124, 125, 60, 124, 125, 61, 126, 127, 62, 126, 127, 63}};
|
|
__m512i zmm_p0 =
|
|
_mm512_loadu_si512((const __m512i*)kNV21ToYUV24Perm_AVX512VBMI[0]);
|
|
__m512i zmm_p1 =
|
|
_mm512_loadu_si512((const __m512i*)kNV21ToYUV24Perm_AVX512VBMI[1]);
|
|
__m512i zmm_p2 =
|
|
_mm512_loadu_si512((const __m512i*)kNV21ToYUV24Perm_AVX512VBMI[2]);
|
|
LIBYUV_HIDE_CONSTANT(zmm_p0);
|
|
LIBYUV_HIDE_CONSTANT(zmm_p1);
|
|
LIBYUV_HIDE_CONSTANT(zmm_p2);
|
|
|
|
while (width >= 64) {
|
|
__m512i zy = _mm512_loadu_si512((const __m512i*)src_y);
|
|
__m512i zvu = _mm512_loadu_si512((const __m512i*)src_vu);
|
|
__m512i dst0 = _mm512_permutex2var_epi8(zy, zmm_p0, zvu);
|
|
__m512i dst1 = _mm512_permutex2var_epi8(zy, zmm_p1, zvu);
|
|
__m512i dst2 = _mm512_permutex2var_epi8(zy, zmm_p2, zvu);
|
|
_mm512_storeu_si512((__m512i*)dst_yuv24, dst0);
|
|
_mm512_storeu_si512((__m512i*)(dst_yuv24 + 64), dst1);
|
|
_mm512_storeu_si512((__m512i*)(dst_yuv24 + 128), dst2);
|
|
src_y += 64;
|
|
src_vu += 64;
|
|
dst_yuv24 += 192;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
__mmask64 ky = (__mmask64)((1ULL << width) - 1ULL);
|
|
int vu_bytes = (width + 1) & ~1;
|
|
__mmask64 kvu = (vu_bytes >= 64) ? ~(__mmask64)0
|
|
: (__mmask64)((1ULL << vu_bytes) - 1ULL);
|
|
__m512i zy = _mm512_maskz_loadu_epi8(ky, src_y);
|
|
__m512i zvu = _mm512_maskz_loadu_epi8(kvu, src_vu);
|
|
__m512i dst0 = _mm512_permutex2var_epi8(zy, zmm_p0, zvu);
|
|
__m512i dst1 = _mm512_permutex2var_epi8(zy, zmm_p1, zvu);
|
|
__m512i dst2 = _mm512_permutex2var_epi8(zy, zmm_p2, zvu);
|
|
int nbytes = width * 3;
|
|
__mmask64 k0 =
|
|
(nbytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << nbytes) - 1ULL);
|
|
__mmask64 k1 = (nbytes <= 64) ? 0
|
|
: (nbytes >= 128)
|
|
? ~(__mmask64)0
|
|
: (__mmask64)((1ULL << (nbytes - 64)) - 1ULL);
|
|
__mmask64 k2 =
|
|
(nbytes <= 128) ? 0 : (__mmask64)((1ULL << (nbytes - 128)) - 1ULL);
|
|
_mm512_mask_storeu_epi8(dst_yuv24, k0, dst0);
|
|
if (k1) {
|
|
_mm512_mask_storeu_epi8(dst_yuv24 + 64, k1, dst1);
|
|
}
|
|
if (k2) {
|
|
_mm512_mask_storeu_epi8(dst_yuv24 + 128, k2, dst2);
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_NV21TOYUV24ROW_AVX512VBMI
|
|
|
|
#ifdef HAS_SPLITARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void SplitARGBRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_r,
|
|
uint8_t* dst_g,
|
|
uint8_t* dst_b,
|
|
uint8_t* dst_a,
|
|
int width) {
|
|
static const uint8_t kShufSplitARGB[16] = {0, 4, 8, 12, 1, 5, 9, 13,
|
|
2, 6, 10, 14, 3, 7, 11, 15};
|
|
const __m256i shuf = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShufSplitARGB));
|
|
do {
|
|
__m256i t0 = _mm256_shuffle_epi8(
|
|
_mm256_loadu_si256((const __m256i*)src_argb), shuf);
|
|
__m256i t1 = _mm256_shuffle_epi8(
|
|
_mm256_loadu_si256((const __m256i*)(src_argb + 32)), shuf);
|
|
__m256i p0 = _mm256_permute2x128_si256(t0, t1, 0x20);
|
|
__m256i p1 = _mm256_permute2x128_si256(t0, t1, 0x31);
|
|
__m256i bg = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(p0, p1), 0xd8);
|
|
__m256i ra = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(p0, p1), 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_b, _mm256_castsi256_si128(bg));
|
|
_mm_storeu_si128((__m128i*)dst_g, _mm256_extracti128_si256(bg, 1));
|
|
_mm_storeu_si128((__m128i*)dst_r, _mm256_castsi256_si128(ra));
|
|
_mm_storeu_si128((__m128i*)dst_a, _mm256_extracti128_si256(ra, 1));
|
|
src_argb += 64;
|
|
dst_r += 16;
|
|
dst_g += 16;
|
|
dst_b += 16;
|
|
dst_a += 16;
|
|
width -= 16;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SPLITARGBROW_AVX2
|
|
|
|
#ifdef HAS_SPLITXRGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void SplitXRGBRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_r,
|
|
uint8_t* dst_g,
|
|
uint8_t* dst_b,
|
|
int width) {
|
|
static const uint8_t kShufSplitARGB[16] = {0, 4, 8, 12, 1, 5, 9, 13,
|
|
2, 6, 10, 14, 3, 7, 11, 15};
|
|
const __m256i shuf = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShufSplitARGB));
|
|
do {
|
|
__m256i t0 = _mm256_shuffle_epi8(
|
|
_mm256_loadu_si256((const __m256i*)src_argb), shuf);
|
|
__m256i t1 = _mm256_shuffle_epi8(
|
|
_mm256_loadu_si256((const __m256i*)(src_argb + 32)), shuf);
|
|
__m256i p0 = _mm256_permute2x128_si256(t0, t1, 0x20);
|
|
__m256i p1 = _mm256_permute2x128_si256(t0, t1, 0x31);
|
|
__m256i bg = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(p0, p1), 0xd8);
|
|
__m256i ra = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(p0, p1), 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_b, _mm256_castsi256_si128(bg));
|
|
_mm_storeu_si128((__m128i*)dst_g, _mm256_extracti128_si256(bg, 1));
|
|
_mm_storeu_si128((__m128i*)dst_r, _mm256_castsi256_si128(ra));
|
|
src_argb += 64;
|
|
dst_r += 16;
|
|
dst_g += 16;
|
|
dst_b += 16;
|
|
width -= 16;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SPLITXRGBROW_AVX2
|
|
|
|
#ifdef HAS_SPLITRGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void SplitRGBRow_AVX2(const uint8_t* src_rgb,
|
|
uint8_t* dst_r,
|
|
uint8_t* dst_g,
|
|
uint8_t* dst_b,
|
|
int width) {
|
|
static const uint8_t kSplitRGBShuffle[5][16] = {
|
|
{0u, 3u, 6u, 9u, 12u, 15u, 2u, 5u, 8u, 11u, 14u, 1u, 4u, 7u, 10u, 13u},
|
|
{1u, 4u, 7u, 10u, 13u, 0u, 3u, 6u, 9u, 12u, 15u, 2u, 5u, 8u, 11u, 14u},
|
|
{2u, 5u, 8u, 11u, 14u, 1u, 4u, 7u, 10u, 13u, 0u, 3u, 6u, 9u, 12u, 15u},
|
|
{0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u,
|
|
0u},
|
|
{0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u,
|
|
0u}};
|
|
const __m256i s0 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kSplitRGBShuffle[0]));
|
|
const __m256i s1 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kSplitRGBShuffle[1]));
|
|
const __m256i s2 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kSplitRGBShuffle[2]));
|
|
const __m256i m0 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kSplitRGBShuffle[3]));
|
|
const __m256i m1 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kSplitRGBShuffle[4]));
|
|
|
|
do {
|
|
__m256i ymm4 = _mm256_loadu_si256((const __m256i*)src_rgb);
|
|
__m256i ymm5 = _mm256_loadu_si256((const __m256i*)(src_rgb + 32));
|
|
__m256i ymm6 = _mm256_loadu_si256((const __m256i*)(src_rgb + 64));
|
|
src_rgb += 96;
|
|
__m256i ymm1 = _mm256_blend_epi32(ymm4, ymm5, 0xf0);
|
|
__m256i ymm2 = _mm256_permute2x128_si256(ymm4, ymm6, 0x21);
|
|
__m256i ymm3 = _mm256_blend_epi32(ymm5, ymm6, 0xf0);
|
|
ymm4 = _mm256_blendv_epi8(ymm1, ymm3, m0);
|
|
ymm5 = _mm256_blendv_epi8(ymm2, ymm1, m0);
|
|
ymm6 = _mm256_blendv_epi8(ymm3, ymm2, m0);
|
|
ymm1 = _mm256_shuffle_epi8(_mm256_blendv_epi8(ymm4, ymm5, m1), s0);
|
|
ymm2 = _mm256_shuffle_epi8(_mm256_blendv_epi8(ymm5, ymm6, m1), s1);
|
|
ymm3 = _mm256_shuffle_epi8(_mm256_blendv_epi8(ymm6, ymm4, m1), s2);
|
|
_mm256_storeu_si256((__m256i*)dst_r, ymm1);
|
|
_mm256_storeu_si256((__m256i*)dst_g, ymm2);
|
|
_mm256_storeu_si256((__m256i*)dst_b, ymm3);
|
|
dst_r += 32;
|
|
dst_g += 32;
|
|
dst_b += 32;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SPLITRGBROW_AVX2
|
|
|
|
#ifdef HAS_SPLITUVROW_16_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void SplitUVRow_16_AVX2(const uint16_t* src_uv,
|
|
uint16_t* dst_u,
|
|
uint16_t* dst_v,
|
|
int depth,
|
|
int width) {
|
|
const __m128i shift128 = _mm_cvtsi32_si128(16 - depth);
|
|
const __m256i mask16 = _mm256_set1_epi32(0x0000ffff);
|
|
do {
|
|
__m256i uv0 = _mm256_loadu_si256((const __m256i*)src_uv);
|
|
__m256i uv1 = _mm256_loadu_si256((const __m256i*)(src_uv + 16));
|
|
__m256i u = _mm256_permute4x64_epi64(
|
|
_mm256_packus_epi32(_mm256_and_si256(uv0, mask16),
|
|
_mm256_and_si256(uv1, mask16)),
|
|
0xd8);
|
|
__m256i v = _mm256_permute4x64_epi64(
|
|
_mm256_packus_epi32(_mm256_srli_epi32(uv0, 16),
|
|
_mm256_srli_epi32(uv1, 16)),
|
|
0xd8);
|
|
_mm256_storeu_si256((__m256i*)dst_u, _mm256_srl_epi16(u, shift128));
|
|
_mm256_storeu_si256((__m256i*)dst_v, _mm256_srl_epi16(v, shift128));
|
|
src_uv += 32;
|
|
dst_u += 16;
|
|
dst_v += 16;
|
|
width -= 16;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SPLITUVROW_16_AVX2
|
|
|
|
#ifdef HAS_YUY2TOYROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void YUY2ToYRow_AVX2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
|
|
const __m256i mask_00ff = _mm256_set1_epi16(0x00ff);
|
|
do {
|
|
__m256i s0 =
|
|
_mm256_and_si256(_mm256_loadu_si256((const __m256i*)src_yuy2),
|
|
mask_00ff);
|
|
__m256i s1 =
|
|
_mm256_and_si256(_mm256_loadu_si256((const __m256i*)(src_yuy2 + 32)),
|
|
mask_00ff);
|
|
_mm256_storeu_si256(
|
|
(__m256i*)dst_y,
|
|
_mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8));
|
|
src_yuy2 += 64;
|
|
dst_y += 32;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_YUY2TOYROW_AVX2
|
|
|
|
#ifdef HAS_UYVYTOYROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void UYVYToYRow_AVX2(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
|
|
do {
|
|
__m256i s0 =
|
|
_mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)src_uyvy), 8);
|
|
__m256i s1 =
|
|
_mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)(src_uyvy + 32)),
|
|
8);
|
|
_mm256_storeu_si256(
|
|
(__m256i*)dst_y,
|
|
_mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8));
|
|
src_uyvy += 64;
|
|
dst_y += 32;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_UYVYTOYROW_AVX2
|
|
|
|
#ifdef HAS_YUY2TOUVROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void YUY2ToUVRow_AVX2(const uint8_t* src_yuy2,
|
|
int stride_yuy2,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
const __m256i mask_00ff = _mm256_set1_epi16(0x00ff);
|
|
do {
|
|
__m256i y0 = _mm256_avg_epu8(
|
|
_mm256_loadu_si256((const __m256i*)src_yuy2),
|
|
_mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2)));
|
|
__m256i y1 = _mm256_avg_epu8(
|
|
_mm256_loadu_si256((const __m256i*)(src_yuy2 + 32)),
|
|
_mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2 + 32)));
|
|
__m256i uv = _mm256_permute4x64_epi64(
|
|
_mm256_packus_epi16(_mm256_srli_epi16(y0, 8), _mm256_srli_epi16(y1, 8)),
|
|
0xd8);
|
|
__m256i u = _mm256_and_si256(uv, mask_00ff);
|
|
__m256i v = _mm256_srli_epi16(uv, 8);
|
|
u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8);
|
|
v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u));
|
|
_mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v));
|
|
src_yuy2 += 64;
|
|
dst_u += 16;
|
|
dst_v += 16;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_YUY2TOUVROW_AVX2
|
|
|
|
#ifdef HAS_YUY2TOUV422ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void YUY2ToUV422Row_AVX2(const uint8_t* src_yuy2,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
const __m256i mask_00ff = _mm256_set1_epi16(0x00ff);
|
|
do {
|
|
__m256i y0 = _mm256_loadu_si256((const __m256i*)src_yuy2);
|
|
__m256i y1 = _mm256_loadu_si256((const __m256i*)(src_yuy2 + 32));
|
|
__m256i uv = _mm256_permute4x64_epi64(
|
|
_mm256_packus_epi16(_mm256_srli_epi16(y0, 8), _mm256_srli_epi16(y1, 8)),
|
|
0xd8);
|
|
__m256i u = _mm256_and_si256(uv, mask_00ff);
|
|
__m256i v = _mm256_srli_epi16(uv, 8);
|
|
u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8);
|
|
v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u));
|
|
_mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v));
|
|
src_yuy2 += 64;
|
|
dst_u += 16;
|
|
dst_v += 16;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_YUY2TOUV422ROW_AVX2
|
|
|
|
#ifdef HAS_YUY2TONVUVROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void YUY2ToNVUVRow_AVX2(const uint8_t* src_yuy2,
|
|
int stride_yuy2,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
do {
|
|
__m256i y0 = _mm256_avg_epu8(
|
|
_mm256_loadu_si256((const __m256i*)src_yuy2),
|
|
_mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2)));
|
|
__m256i y1 = _mm256_avg_epu8(
|
|
_mm256_loadu_si256((const __m256i*)(src_yuy2 + 32)),
|
|
_mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2 + 32)));
|
|
__m256i uv = _mm256_permute4x64_epi64(
|
|
_mm256_packus_epi16(_mm256_srli_epi16(y0, 8), _mm256_srli_epi16(y1, 8)),
|
|
0xd8);
|
|
_mm256_storeu_si256((__m256i*)dst_uv, uv);
|
|
src_yuy2 += 64;
|
|
dst_uv += 32;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_YUY2TONVUVROW_AVX2
|
|
|
|
#if defined(HAS_UYVYTOUVROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2)
|
|
LIBYUV_TARGET_AVX2
|
|
void UYVYToUVRow_AVX2(const uint8_t* src_uyvy,
|
|
int stride_uyvy,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
const __m256i mask_00ff = _mm256_set1_epi16(0x00ff);
|
|
do {
|
|
__m256i y0 = _mm256_avg_epu8(
|
|
_mm256_loadu_si256((const __m256i*)src_uyvy),
|
|
_mm256_loadu_si256((const __m256i*)(src_uyvy + stride_uyvy)));
|
|
__m256i y1 = _mm256_avg_epu8(
|
|
_mm256_loadu_si256((const __m256i*)(src_uyvy + 32)),
|
|
_mm256_loadu_si256((const __m256i*)(src_uyvy + stride_uyvy + 32)));
|
|
__m256i uv = _mm256_permute4x64_epi64(
|
|
_mm256_packus_epi16(_mm256_and_si256(y0, mask_00ff),
|
|
_mm256_and_si256(y1, mask_00ff)),
|
|
0xd8);
|
|
__m256i u = _mm256_and_si256(uv, mask_00ff);
|
|
__m256i v = _mm256_srli_epi16(uv, 8);
|
|
u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8);
|
|
v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u));
|
|
_mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v));
|
|
src_uyvy += 64;
|
|
dst_u += 16;
|
|
dst_v += 16;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_UYVYTOUVROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2)
|
|
|
|
#if defined(HAS_UYVYTOUV422ROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2)
|
|
LIBYUV_TARGET_AVX2
|
|
void UYVYToUV422Row_AVX2(const uint8_t* src_uyvy,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
const __m256i mask_00ff = _mm256_set1_epi16(0x00ff);
|
|
do {
|
|
__m256i y0 = _mm256_loadu_si256((const __m256i*)src_uyvy);
|
|
__m256i y1 = _mm256_loadu_si256((const __m256i*)(src_uyvy + 32));
|
|
__m256i uv = _mm256_permute4x64_epi64(
|
|
_mm256_packus_epi16(_mm256_and_si256(y0, mask_00ff),
|
|
_mm256_and_si256(y1, mask_00ff)),
|
|
0xd8);
|
|
__m256i u = _mm256_and_si256(uv, mask_00ff);
|
|
__m256i v = _mm256_srli_epi16(uv, 8);
|
|
u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8);
|
|
v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u));
|
|
_mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v));
|
|
src_uyvy += 64;
|
|
dst_u += 16;
|
|
dst_v += 16;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_UYVYTOUV422ROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2)
|
|
|
|
#ifdef HAS_I444TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void I444ToARGBRow_AVX2(const uint8_t* y_buf,
|
|
const uint8_t* u_buf,
|
|
const uint8_t* v_buf,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
const __m256i ymm_kUVToB =
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
|
|
const __m256i ymm_kUVToG =
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
|
|
const __m256i ymm_kUVToR =
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
|
|
const __m256i ymm_kYToRgb =
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
|
|
const __m256i ymm_kYBiasToRgb =
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
|
|
const __m256i ymm_128 = _mm256_set1_epi8((char)0x80);
|
|
const __m256i ymm_alpha = _mm256_set1_epi8((char)0xff);
|
|
|
|
while (width >= 16) {
|
|
__m128i xmm_u = _mm_loadu_si128((const __m128i*)u_buf);
|
|
__m128i xmm_v = _mm_loadu_si128((const __m128i*)v_buf);
|
|
__m128i xmm_y = _mm_loadu_si128((const __m128i*)y_buf);
|
|
u_buf += 16;
|
|
v_buf += 16;
|
|
y_buf += 16;
|
|
|
|
__m256i ymm3 = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_unpacklo_epi8(xmm_u, xmm_v)),
|
|
_mm_unpackhi_epi8(xmm_u, xmm_v), 1);
|
|
__m256i ymm4 = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_unpacklo_epi8(xmm_y, xmm_y)),
|
|
_mm_unpackhi_epi8(xmm_y, xmm_y), 1);
|
|
|
|
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
|
|
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
|
|
|
|
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
|
|
__m256i ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
|
|
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
|
|
|
|
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
|
|
|
|
ymm0 = _mm256_srai_epi16(_mm256_adds_epi16(ymm0, ymm4), 6);
|
|
ymm1 = _mm256_srai_epi16(_mm256_subs_epi16(ymm4, ymm1), 6);
|
|
ymm2 = _mm256_srai_epi16(_mm256_adds_epi16(ymm2, ymm4), 6);
|
|
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
|
|
ymm1 = _mm256_packus_epi16(ymm1, ymm1);
|
|
ymm2 = _mm256_packus_epi16(ymm2, ymm2);
|
|
|
|
ymm0 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi8(ymm0, ymm1), 0xd8);
|
|
ymm2 =
|
|
_mm256_permute4x64_epi64(_mm256_unpacklo_epi8(ymm2, ymm_alpha), 0xd8);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, _mm256_unpacklo_epi16(ymm0, ymm2));
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32),
|
|
_mm256_unpackhi_epi16(ymm0, ymm2));
|
|
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I444TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_MERGEXR30ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MergeXR30Row_AVX2(const uint16_t* src_r,
|
|
const uint16_t* src_g,
|
|
const uint16_t* src_b,
|
|
uint8_t* dst_ar30,
|
|
int depth,
|
|
int width) {
|
|
int shift = depth - 10;
|
|
__m128i v_shift = _mm_cvtsi32_si128(shift);
|
|
__m256i v_alpha = _mm256_set1_epi16(0x0030);
|
|
__m256i v_max = _mm256_set1_epi16(1023);
|
|
while (width > 0) {
|
|
__m256i r = _mm256_min_epu16(
|
|
_mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_r), v_shift),
|
|
v_max);
|
|
__m256i g = _mm256_min_epu16(
|
|
_mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_g), v_shift),
|
|
v_max);
|
|
__m256i b = _mm256_min_epu16(
|
|
_mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_b), v_shift),
|
|
v_max);
|
|
r = _mm256_slli_epi16(_mm256_permute4x64_epi64(r, 0xd8), 4);
|
|
g = _mm256_permute4x64_epi64(g, 0xd8);
|
|
b = _mm256_permute4x64_epi64(b, 0xd8);
|
|
__m256i rb_lo = _mm256_unpacklo_epi16(b, r);
|
|
__m256i rb_hi = _mm256_unpackhi_epi16(b, r);
|
|
__m256i ag_lo = _mm256_slli_epi32(_mm256_unpacklo_epi16(g, v_alpha), 10);
|
|
__m256i ag_hi = _mm256_slli_epi32(_mm256_unpackhi_epi16(g, v_alpha), 10);
|
|
_mm256_storeu_si256((__m256i*)dst_ar30, _mm256_or_si256(rb_lo, ag_lo));
|
|
_mm256_storeu_si256((__m256i*)(dst_ar30 + 32),
|
|
_mm256_or_si256(rb_hi, ag_hi));
|
|
src_r += 16;
|
|
src_g += 16;
|
|
src_b += 16;
|
|
dst_ar30 += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGEXR30ROW_AVX2
|
|
|
|
#ifdef HAS_MERGEUVROW_16_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MergeUVRow_16_AVX2(const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
uint16_t* dst_uv,
|
|
int depth,
|
|
int width) {
|
|
const __m128i sh_u = _mm_cvtsi32_si128(16 - depth);
|
|
const __m128i sh_v = _mm_cvtsi32_si128(32 - depth);
|
|
while (width > 0) {
|
|
__m256i u32 =
|
|
_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_u));
|
|
__m256i v32 =
|
|
_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_v));
|
|
src_u += 8;
|
|
src_v += 8;
|
|
u32 = _mm256_sll_epi16(u32, sh_u);
|
|
v32 = _mm256_sll_epi32(v32, sh_v);
|
|
_mm256_storeu_si256((__m256i*)dst_uv, _mm256_or_si256(u32, v32));
|
|
dst_uv += 16;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGEUVROW_16_AVX2
|
|
|
|
#ifdef __cplusplus
|
|
} // extern "C"
|
|
} // namespace libyuv
|
|
#endif
|
|
|
|
#endif // !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) ||
|
|
// defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) &&
|
|
// ((defined(_MSC_VER) && !defined(__clang__)) ||
|
|
// defined(LIBYUV_ENABLE_ROWWIN))
|