Commit 8a1038536c for aom
commit 8a1038536cde81c548341145f40d6dfddc3077d6
Author: Lin Zheng <linzhen@google.com>
Date: Sun Nov 8 22:05:30 2026 +0000
Highway: Extend AVX512 SAD to 16xN, 32xN, 64x16
Function / Block Size │ Base (ns) │ AVX512 (ns) │ Speedup (x)
──────────────────────┼───────────┼─────────────┼────────────
sad 16x8 │ 2.88 │ 2.89 │ 1.00x
sad 16x16 │ 5.86 │ 5.49 │ 1.07x
sad 16x32 │ 10.60 │ 10.59 │ 1.00x
sad 16x64 │ 20.25 │ 20.25 │ 1.00x
sad 32x8 │ 5.46 │ 3.20 │ 1.71x
sad 32x16 │ 5.60 │ 5.79 │ 0.97x
sad 32x32 │ 13.33 │ 10.38 │ 1.28x
sad 64x16 │ 20.41 │ 7.49 │ 2.73x
sad 64x32 │ 20.23 │ 14.26 │ 1.42x
sad 64x64 │ 39.51 │ 26.81 │ 1.47x
sad 64x128 │ 83.79 │ 60.83 │ 1.38x
sad 128x64 │ 87.19 │ 42.02 │ 2.07x
sad 128x128 │ 155.42 │ 80.96 │ 1.92x
sad_avg 16x8 │ 4.07 │ 3.21 │ 1.27x
sad_avg 16x16 │ 7.66 │ 6.05 │ 1.27x
sad_avg 16x32 │ 15.15 │ 12.13 │ 1.25x
sad_avg 32x16 │ 7.96 │ 7.55 │ 1.05x
sad_avg 32x32 │ 18.40 │ 13.50 │ 1.36x
sad_avg 64x32 │ 29.95 │ 21.02 │ 1.42x
sad_avg 64x64 │ 58.91 │ 42.36 │ 1.39x
sad_avg 128x128 │ 314.82 │ 123.18 │ 2.56x
Change-Id: I5889bd349765abda7a6a34769bcbf25672304f6e
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl
index 440aac1f79..f741044e04 100755
--- a/aom_dsp/aom_dsp_rtcd_defs.pl
+++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -878,6 +878,14 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_sad64x128 avx512/;
specialize qw/aom_sad64x64 avx512/;
specialize qw/aom_sad64x32 avx512/;
+ specialize qw/aom_sad64x16 avx512/;
+ specialize qw/aom_sad32x32 avx512/;
+ specialize qw/aom_sad32x16 avx512/;
+ specialize qw/aom_sad32x8 avx512/;
+ specialize qw/aom_sad16x64 avx512/;
+ specialize qw/aom_sad16x32 avx512/;
+ specialize qw/aom_sad16x16 avx512/;
+ specialize qw/aom_sad16x8 avx512/;
}
specialize qw/aom_sad32x64 avx2 neon neon_dotprod/, "$sse2_asm";
@@ -911,6 +919,12 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_sad_skip_64x128 avx512/;
specialize qw/aom_sad_skip_64x64 avx512/;
specialize qw/aom_sad_skip_64x32 avx512/;
+ specialize qw/aom_sad_skip_64x16 avx512/;
+ specialize qw/aom_sad_skip_32x32 avx512/;
+ specialize qw/aom_sad_skip_32x16 avx512/;
+ specialize qw/aom_sad_skip_16x64 avx512/;
+ specialize qw/aom_sad_skip_16x32 avx512/;
+ specialize qw/aom_sad_skip_16x16 avx512/;
}
specialize qw/aom_sad_skip_32x64 avx2 neon neon_dotprod/, "$sse2_asm";
@@ -939,6 +953,14 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_sad64x128_avg avx512/;
specialize qw/aom_sad64x64_avg avx512/;
specialize qw/aom_sad64x32_avg avx512/;
+ specialize qw/aom_sad64x16_avg avx512/;
+ specialize qw/aom_sad32x32_avg avx512/;
+ specialize qw/aom_sad32x16_avg avx512/;
+ specialize qw/aom_sad32x8_avg avx512/;
+ specialize qw/aom_sad16x64_avg avx512/;
+ specialize qw/aom_sad16x32_avg avx512/;
+ specialize qw/aom_sad16x16_avg avx512/;
+ specialize qw/aom_sad16x8_avg avx512/;
}
specialize qw/aom_sad32x32_avg avx2 neon neon_dotprod/, "$sse2_asm";
@@ -1094,6 +1116,15 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_sad64x128x4d avx512/;
specialize qw/aom_sad64x64x4d avx512/;
specialize qw/aom_sad64x32x4d avx512/;
+ specialize qw/aom_sad64x16x4d avx512/;
+ specialize qw/aom_sad32x32x4d avx512/;
+ specialize qw/aom_sad32x16x4d avx512/;
+ specialize qw/aom_sad32x8x4d avx512/;
+ specialize qw/aom_sad16x64x4d avx512/;
+ specialize qw/aom_sad16x32x4d avx512/;
+ specialize qw/aom_sad16x16x4d avx512/;
+ specialize qw/aom_sad16x8x4d avx512/;
+ specialize qw/aom_sad16x4x4d avx512/;
}
specialize qw/aom_sad32x32x4d avx2 neon neon_dotprod/, "$sse2_asm";
@@ -1131,6 +1162,12 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_sad_skip_64x128x4d avx512/;
specialize qw/aom_sad_skip_64x64x4d avx512/;
specialize qw/aom_sad_skip_64x32x4d avx512/;
+ specialize qw/aom_sad_skip_64x16x4d avx512/;
+ specialize qw/aom_sad_skip_32x32x4d avx512/;
+ specialize qw/aom_sad_skip_32x16x4d avx512/;
+ specialize qw/aom_sad_skip_16x64x4d avx512/;
+ specialize qw/aom_sad_skip_16x32x4d avx512/;
+ specialize qw/aom_sad_skip_16x16x4d avx512/;
}
specialize qw/aom_sad_skip_16x64x4d avx2 neon neon_dotprod/, "$sse2_asm";
@@ -1153,6 +1190,15 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_sad64x128x3d avx512/;
specialize qw/aom_sad64x64x3d avx512/;
specialize qw/aom_sad64x32x3d avx512/;
+ specialize qw/aom_sad64x16x3d avx512/;
+ specialize qw/aom_sad32x32x3d avx512/;
+ specialize qw/aom_sad32x16x3d avx512/;
+ specialize qw/aom_sad32x8x3d avx512/;
+ specialize qw/aom_sad16x64x3d avx512/;
+ specialize qw/aom_sad16x32x3d avx512/;
+ specialize qw/aom_sad16x16x3d avx512/;
+ specialize qw/aom_sad16x8x3d avx512/;
+ specialize qw/aom_sad16x4x3d avx512/;
}
specialize qw/aom_sad32x64x3d avx2 neon neon_dotprod/;
diff --git a/aom_dsp/sad_hwy.h b/aom_dsp/sad_hwy.h
index 7d35ce5264..f33a35f279 100644
--- a/aom_dsp/sad_hwy.h
+++ b/aom_dsp/sad_hwy.h
@@ -11,6 +11,11 @@
#ifndef AOM_AOM_DSP_SAD_HWY_H_
#define AOM_AOM_DSP_SAD_HWY_H_
+#include <assert.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
#include "aom_dsp/reduce_sum_hwy.h"
#include "third_party/highway/hwy/highway.h"
@@ -21,30 +26,78 @@ namespace HWY_NAMESPACE {
namespace hn = hwy::HWY_NAMESPACE;
-template <int BlockWidth>
-HWY_MAYBE_UNUSED unsigned int SumOfAbsoluteDiff(
+template <class D, class VHalf>
+HWY_ATTR HWY_INLINE hn::VFromD<D> Combine2Rows(D d, VHalf r0, VHalf r1) {
+ if constexpr (hn::MaxLanes(d) * sizeof(hn::TFromD<D>) >= 32) {
+ return hn::ConcatLowerLower(d, hn::ResizeBitCast(d, r1),
+ hn::ResizeBitCast(d, r0));
+ } else {
+ return hn::Combine(d, r1, r0);
+ }
+}
+
+template <class D, class VQuarter>
+HWY_ATTR HWY_INLINE hn::VFromD<D> Combine4Rows(D d, VQuarter r0, VQuarter r1,
+ VQuarter r2, VQuarter r3) {
+ if constexpr (hn::MaxLanes(d) * sizeof(hn::TFromD<D>) == 64) {
+ auto v = hn::ResizeBitCast(d, r0);
+ v = hn::InsertBlock<1>(v, r1);
+ v = hn::InsertBlock<2>(v, r2);
+ return hn::InsertBlock<3>(v, r3);
+ } else {
+ constexpr hn::Half<D> d_half;
+ auto r01 = Combine2Rows(d_half, r0, r1);
+ auto r23 = Combine2Rows(d_half, r2, r3);
+ return Combine2Rows(d, r01, r23);
+ }
+}
+
+template <int RowsPerVec, class D>
+HWY_ATTR HWY_INLINE hn::VFromD<D> LoadRowsU(D full_tag, const uint8_t *ptr,
+ ptrdiff_t stride) {
+ static_assert(RowsPerVec == 1 || RowsPerVec == 2 || RowsPerVec == 4,
+ "RowsPerVec must be 1, 2, or 4.");
+ if constexpr (RowsPerVec == 4) {
+ constexpr hn::Half<hn::Half<D>> row_tag;
+ const uint8_t *ptr2 = ptr + stride * 2;
+ return Combine4Rows(
+ full_tag, hn::LoadU(row_tag, ptr), hn::LoadU(row_tag, ptr + stride),
+ hn::LoadU(row_tag, ptr2), hn::LoadU(row_tag, ptr2 + stride));
+ } else if constexpr (RowsPerVec == 2) {
+ constexpr hn::Half<D> row_tag;
+ return Combine2Rows(full_tag, hn::LoadU(row_tag, ptr),
+ hn::LoadU(row_tag, ptr + stride));
+ } else {
+ return hn::LoadU(full_tag, ptr);
+ }
+}
+
+template <int BlockWidth, bool IsSadAvg = false>
+HWY_ATTR HWY_INLINE unsigned int SumOfAbsoluteDiff(
const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
int ref_stride, int h, const uint8_t *second_pred = nullptr) {
- constexpr hn::CappedTag<uint8_t, BlockWidth> pixel_tag;
+ constexpr hn::ScalableTag<uint8_t> pixel_tag;
+ constexpr int vw = hn::MaxLanes(pixel_tag);
+ constexpr int kRowsPerVec = (BlockWidth < vw) ? (vw / BlockWidth) : 1;
+ constexpr int kRowSpan = (BlockWidth < vw) ? vw : BlockWidth;
+ assert(h % kRowsPerVec == 0);
constexpr hn::Repartition<uint64_t, decltype(pixel_tag)> intermediate_sum_tag;
- const int vw = hn::Lanes(pixel_tag);
auto sum_sad = hn::Zero(intermediate_sum_tag);
- const bool is_sad_avg = second_pred != nullptr;
- for (int i = 0; i < h; ++i) {
- for (int j = 0; j < BlockWidth; j += vw) {
- auto src_vec = hn::LoadU(pixel_tag, &src_ptr[j]);
- auto ref_vec = hn::LoadU(pixel_tag, &ref_ptr[j]);
- if (is_sad_avg) {
+ for (int i = 0; i < h; i += kRowsPerVec) {
+ for (int j = 0; j < kRowSpan; j += vw) {
+ auto src_vec = LoadRowsU<kRowsPerVec>(pixel_tag, &src_ptr[j], src_stride);
+ auto ref_vec = LoadRowsU<kRowsPerVec>(pixel_tag, &ref_ptr[j], ref_stride);
+ if constexpr (IsSadAvg) {
auto sec_pred_vec = hn::LoadU(pixel_tag, &second_pred[j]);
ref_vec = hn::AverageRound(ref_vec, sec_pred_vec);
}
auto sad = hn::SumsOf8AbsDiff(src_vec, ref_vec);
sum_sad = hn::Add(sum_sad, sad);
}
- src_ptr += src_stride;
- ref_ptr += ref_stride;
- if (is_sad_avg) {
- second_pred += BlockWidth;
+ src_ptr += src_stride * kRowsPerVec;
+ ref_ptr += ref_stride * kRowsPerVec;
+ if constexpr (IsSadAvg) {
+ second_pred += BlockWidth * kRowsPerVec;
}
}
return static_cast<unsigned int>(
@@ -52,50 +105,55 @@ HWY_MAYBE_UNUSED unsigned int SumOfAbsoluteDiff(
}
template <int BlockWidth, int NumRef>
-HWY_MAYBE_UNUSED void SumOfAbsoluteDiffND(const uint8_t *src_ptr,
- int src_stride,
- const uint8_t *const ref_ptr[4],
- int ref_stride, int h,
- uint32_t res[4]) {
+HWY_ATTR HWY_INLINE void SumOfAbsoluteDiffND(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, int h,
+ uint32_t res[4]) {
static_assert(NumRef == 3 || NumRef == 4, "NumRef must be 3 or 4.");
- constexpr hn::CappedTag<uint8_t, BlockWidth> pixel_tag;
+ constexpr hn::ScalableTag<uint8_t> pixel_tag;
+ constexpr int vw = hn::MaxLanes(pixel_tag);
+ constexpr int kRowsPerVec = (BlockWidth < vw) ? (vw / BlockWidth) : 1;
+ constexpr int kRowSpan = (BlockWidth < vw) ? vw : BlockWidth;
+ assert(h % kRowsPerVec == 0);
constexpr hn::Repartition<uint64_t, decltype(pixel_tag)> intermediate_sum_tag;
- const int vw = hn::Lanes(pixel_tag);
auto sum_sad_0 = hn::Zero(intermediate_sum_tag);
auto sum_sad_1 = hn::Zero(intermediate_sum_tag);
auto sum_sad_2 = hn::Zero(intermediate_sum_tag);
auto sum_sad_3 = hn::Zero(intermediate_sum_tag);
- const uint8_t *ref_0, *ref_1, *ref_2, *ref_3;
+ const uint8_t *ref_0, *ref_1, *ref_2;
+ const uint8_t *ref_3 = nullptr;
ref_0 = ref_ptr[0];
ref_1 = ref_ptr[1];
ref_2 = ref_ptr[2];
- if (NumRef == 4) {
+ if constexpr (NumRef == 4) {
ref_3 = ref_ptr[3];
}
- for (int i = 0; i < h; ++i) {
- for (int j = 0; j < BlockWidth; j += vw) {
- auto src_vec = hn::LoadU(pixel_tag, &src_ptr[j]);
- auto ref_vec_0 = hn::LoadU(pixel_tag, &ref_0[j]);
- auto ref_vec_1 = hn::LoadU(pixel_tag, &ref_1[j]);
- auto ref_vec_2 = hn::LoadU(pixel_tag, &ref_2[j]);
+ for (int i = 0; i < h; i += kRowsPerVec) {
+ for (int j = 0; j < kRowSpan; j += vw) {
+ auto src_vec = LoadRowsU<kRowsPerVec>(pixel_tag, &src_ptr[j], src_stride);
+ auto ref_vec_0 = LoadRowsU<kRowsPerVec>(pixel_tag, &ref_0[j], ref_stride);
+ auto ref_vec_1 = LoadRowsU<kRowsPerVec>(pixel_tag, &ref_1[j], ref_stride);
+ auto ref_vec_2 = LoadRowsU<kRowsPerVec>(pixel_tag, &ref_2[j], ref_stride);
auto sad_0 = hn::SumsOf8AbsDiff(src_vec, ref_vec_0);
auto sad_1 = hn::SumsOf8AbsDiff(src_vec, ref_vec_1);
auto sad_2 = hn::SumsOf8AbsDiff(src_vec, ref_vec_2);
sum_sad_0 = hn::Add(sum_sad_0, sad_0);
sum_sad_1 = hn::Add(sum_sad_1, sad_1);
sum_sad_2 = hn::Add(sum_sad_2, sad_2);
- if (NumRef == 4) {
- auto ref_vec_3 = hn::LoadU(pixel_tag, &ref_3[j]);
+ if constexpr (NumRef == 4) {
+ auto ref_vec_3 =
+ LoadRowsU<kRowsPerVec>(pixel_tag, &ref_3[j], ref_stride);
auto sad_3 = hn::SumsOf8AbsDiff(src_vec, ref_vec_3);
sum_sad_3 = hn::Add(sum_sad_3, sad_3);
}
}
- src_ptr += src_stride;
- ref_0 += ref_stride;
- ref_1 += ref_stride;
- ref_2 += ref_stride;
- if (NumRef == 4) {
- ref_3 += ref_stride;
+ src_ptr += src_stride * kRowsPerVec;
+ ref_0 += ref_stride * kRowsPerVec;
+ ref_1 += ref_stride * kRowsPerVec;
+ ref_2 += ref_stride * kRowsPerVec;
+ if constexpr (NumRef == 4) {
+ ref_3 += ref_stride * kRowsPerVec;
}
}
constexpr hn::Repartition<uint32_t, decltype(pixel_tag)> uint32_tag;
@@ -173,15 +231,15 @@ HWY_MAYBE_UNUSED void SumOfAbsoluteDiffND(const uint8_t *src_ptr,
res[3] <<= 1; \
}
-#define FSAD_AVG(w, h, suffix) \
- extern "C" unsigned int aom_sad##w##x##h##_avg_##suffix( \
- const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \
- int ref_stride, const uint8_t *second_pred); \
- HWY_ATTR unsigned int aom_sad##w##x##h##_avg_##suffix( \
- const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \
- int ref_stride, const uint8_t *second_pred) { \
- return HWY_NAMESPACE::SumOfAbsoluteDiff<w>(src_ptr, src_stride, ref_ptr, \
- ref_stride, h, second_pred); \
+#define FSAD_AVG(w, h, suffix) \
+ extern "C" unsigned int aom_sad##w##x##h##_avg_##suffix( \
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \
+ int ref_stride, const uint8_t *second_pred); \
+ HWY_ATTR unsigned int aom_sad##w##x##h##_avg_##suffix( \
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \
+ int ref_stride, const uint8_t *second_pred) { \
+ return HWY_NAMESPACE::SumOfAbsoluteDiff<w, true>( \
+ src_ptr, src_stride, ref_ptr, ref_stride, h, second_pred); \
}
#define FOR_EACH_SAD_BLOCK_SIZE(X, suffix) \
@@ -191,6 +249,31 @@ HWY_MAYBE_UNUSED void SumOfAbsoluteDiffND(const uint8_t *src_ptr,
X(64, 64, suffix) \
X(64, 32, suffix)
+#if !CONFIG_REALTIME_ONLY
+#define FOR_EACH_SAD_SMALL_SKIP_RECT_BLOCK_SIZE(X, suffix) \
+ X(64, 16, suffix) \
+ X(16, 64, suffix)
+
+#define FOR_EACH_SAD_H8_RECT_BLOCK_SIZE(X, suffix) X(32, 8, suffix)
+
+#define FOR_EACH_SAD_H4_BLOCK_SIZE(X, suffix) X(16, 4, suffix)
+#else
+#define FOR_EACH_SAD_SMALL_SKIP_RECT_BLOCK_SIZE(X, suffix)
+#define FOR_EACH_SAD_H8_RECT_BLOCK_SIZE(X, suffix)
+#define FOR_EACH_SAD_H4_BLOCK_SIZE(X, suffix)
+#endif // !CONFIG_REALTIME_ONLY
+
+#define FOR_EACH_SAD_SMALL_SKIP_BLOCK_SIZE(X, suffix) \
+ X(32, 32, suffix) \
+ X(32, 16, suffix) \
+ X(16, 32, suffix) \
+ X(16, 16, suffix) \
+ FOR_EACH_SAD_SMALL_SKIP_RECT_BLOCK_SIZE(X, suffix)
+
+#define FOR_EACH_SAD_H8_BLOCK_SIZE(X, suffix) \
+ X(16, 8, suffix) \
+ FOR_EACH_SAD_H8_RECT_BLOCK_SIZE(X, suffix)
+
HWY_AFTER_NAMESPACE();
#endif // AOM_AOM_DSP_SAD_HWY_H_
diff --git a/aom_dsp/x86/sad_hwy_avx512.cc b/aom_dsp/x86/sad_hwy_avx512.cc
index c3adf214ca..1a9192e1d4 100644
--- a/aom_dsp/x86/sad_hwy_avx512.cc
+++ b/aom_dsp/x86/sad_hwy_avx512.cc
@@ -20,3 +20,20 @@ FOR_EACH_SAD_BLOCK_SIZE(FSAD_AVG, avx512)
FOR_EACH_SAD_BLOCK_SIZE(FSAD_4D, avx512)
FOR_EACH_SAD_BLOCK_SIZE(FSAD_4D_SKIP, avx512)
FOR_EACH_SAD_BLOCK_SIZE(FSAD_3D, avx512)
+
+FOR_EACH_SAD_SMALL_SKIP_BLOCK_SIZE(FSAD, avx512)
+FOR_EACH_SAD_SMALL_SKIP_BLOCK_SIZE(FSAD_SKIP, avx512)
+FOR_EACH_SAD_SMALL_SKIP_BLOCK_SIZE(FSAD_AVG, avx512)
+FOR_EACH_SAD_SMALL_SKIP_BLOCK_SIZE(FSAD_4D, avx512)
+FOR_EACH_SAD_SMALL_SKIP_BLOCK_SIZE(FSAD_4D_SKIP, avx512)
+FOR_EACH_SAD_SMALL_SKIP_BLOCK_SIZE(FSAD_3D, avx512)
+
+FOR_EACH_SAD_H8_BLOCK_SIZE(FSAD, avx512)
+FOR_EACH_SAD_H8_BLOCK_SIZE(FSAD_AVG, avx512)
+FOR_EACH_SAD_H8_BLOCK_SIZE(FSAD_4D, avx512)
+FOR_EACH_SAD_H8_BLOCK_SIZE(FSAD_3D, avx512)
+
+#if !CONFIG_REALTIME_ONLY
+FOR_EACH_SAD_H4_BLOCK_SIZE(FSAD_4D, avx512)
+FOR_EACH_SAD_H4_BLOCK_SIZE(FSAD_3D, avx512)
+#endif // !CONFIG_REALTIME_ONLY
diff --git a/test/sad_test.cc b/test/sad_test.cc
index 2f344ae833..ae416ac1a6 100644
--- a/test/sad_test.cc
+++ b/test/sad_test.cc
@@ -2932,15 +2932,33 @@ const SadMxNParam avx512_tests[] = {
make_tuple(128, 128, &aom_sad128x128_avx512, -1),
make_tuple(64, 64, &aom_sad64x64_avx512, -1),
make_tuple(64, 32, &aom_sad64x32_avx512, -1),
+ make_tuple(32, 32, &aom_sad32x32_avx512, -1),
+ make_tuple(32, 16, &aom_sad32x16_avx512, -1),
+ make_tuple(16, 32, &aom_sad16x32_avx512, -1),
+ make_tuple(16, 16, &aom_sad16x16_avx512, -1),
+ make_tuple(16, 8, &aom_sad16x8_avx512, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_sad64x16_avx512, -1),
+ make_tuple(32, 8, &aom_sad32x8_avx512, -1),
+ make_tuple(16, 64, &aom_sad16x64_avx512, -1),
+#endif
};
INSTANTIATE_TEST_SUITE_P(AVX512, SADTest, ::testing::ValuesIn(avx512_tests));
const SadSkipMxNParam skip_avx512_tests[] = {
- make_tuple(128, 128, &aom_sad_skip_128x128_avx2, -1),
- make_tuple(128, 64, &aom_sad_skip_128x64_avx2, -1),
- make_tuple(64, 128, &aom_sad_skip_64x128_avx2, -1),
- make_tuple(64, 64, &aom_sad_skip_64x64_avx2, -1),
- make_tuple(64, 32, &aom_sad_skip_64x32_avx2, -1),
+ make_tuple(128, 128, &aom_sad_skip_128x128_avx512, -1),
+ make_tuple(128, 64, &aom_sad_skip_128x64_avx512, -1),
+ make_tuple(64, 128, &aom_sad_skip_64x128_avx512, -1),
+ make_tuple(64, 64, &aom_sad_skip_64x64_avx512, -1),
+ make_tuple(64, 32, &aom_sad_skip_64x32_avx512, -1),
+ make_tuple(32, 32, &aom_sad_skip_32x32_avx512, -1),
+ make_tuple(32, 16, &aom_sad_skip_32x16_avx512, -1),
+ make_tuple(16, 32, &aom_sad_skip_16x32_avx512, -1),
+ make_tuple(16, 16, &aom_sad_skip_16x16_avx512, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_sad_skip_64x16_avx512, -1),
+ make_tuple(16, 64, &aom_sad_skip_16x64_avx512, -1),
+#endif
};
INSTANTIATE_TEST_SUITE_P(AVX512, SADSkipTest,
::testing::ValuesIn(skip_avx512_tests));
@@ -2951,6 +2969,16 @@ const SadMxNAvgParam avg_avx512_tests[] = {
make_tuple(128, 128, &aom_sad128x128_avg_avx512, -1),
make_tuple(64, 64, &aom_sad64x64_avg_avx512, -1),
make_tuple(64, 32, &aom_sad64x32_avg_avx512, -1),
+ make_tuple(32, 32, &aom_sad32x32_avg_avx512, -1),
+ make_tuple(32, 16, &aom_sad32x16_avg_avx512, -1),
+ make_tuple(16, 32, &aom_sad16x32_avg_avx512, -1),
+ make_tuple(16, 16, &aom_sad16x16_avg_avx512, -1),
+ make_tuple(16, 8, &aom_sad16x8_avg_avx512, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_sad64x16_avg_avx512, -1),
+ make_tuple(32, 8, &aom_sad32x8_avg_avx512, -1),
+ make_tuple(16, 64, &aom_sad16x64_avg_avx512, -1),
+#endif
};
INSTANTIATE_TEST_SUITE_P(AVX512, SADavgTest,
::testing::ValuesIn(avg_avx512_tests));
@@ -2961,6 +2989,17 @@ const SadMxNx4Param x4d_avx512_tests[] = {
make_tuple(64, 128, &aom_sad64x128x4d_avx512, -1),
make_tuple(64, 64, &aom_sad64x64x4d_avx512, -1),
make_tuple(64, 32, &aom_sad64x32x4d_avx512, -1),
+ make_tuple(32, 32, &aom_sad32x32x4d_avx512, -1),
+ make_tuple(32, 16, &aom_sad32x16x4d_avx512, -1),
+ make_tuple(16, 32, &aom_sad16x32x4d_avx512, -1),
+ make_tuple(16, 16, &aom_sad16x16x4d_avx512, -1),
+ make_tuple(16, 8, &aom_sad16x8x4d_avx512, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_sad64x16x4d_avx512, -1),
+ make_tuple(32, 8, &aom_sad32x8x4d_avx512, -1),
+ make_tuple(16, 64, &aom_sad16x64x4d_avx512, -1),
+ make_tuple(16, 4, &aom_sad16x4x4d_avx512, -1),
+#endif
};
INSTANTIATE_TEST_SUITE_P(AVX512, SADx4Test,
::testing::ValuesIn(x4d_avx512_tests));
@@ -2971,6 +3010,17 @@ const SadMxNx4Param x3d_avx512_tests[] = {
make_tuple(64, 128, &aom_sad64x128x3d_avx512, -1),
make_tuple(64, 64, &aom_sad64x64x3d_avx512, -1),
make_tuple(64, 32, &aom_sad64x32x3d_avx512, -1),
+ make_tuple(32, 32, &aom_sad32x32x3d_avx512, -1),
+ make_tuple(32, 16, &aom_sad32x16x3d_avx512, -1),
+ make_tuple(16, 32, &aom_sad16x32x3d_avx512, -1),
+ make_tuple(16, 16, &aom_sad16x16x3d_avx512, -1),
+ make_tuple(16, 8, &aom_sad16x8x3d_avx512, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_sad64x16x3d_avx512, -1),
+ make_tuple(32, 8, &aom_sad32x8x3d_avx512, -1),
+ make_tuple(16, 64, &aom_sad16x64x3d_avx512, -1),
+ make_tuple(16, 4, &aom_sad16x4x3d_avx512, -1),
+#endif
};
INSTANTIATE_TEST_SUITE_P(AVX512, SADx3Test,
::testing::ValuesIn(x3d_avx512_tests));
@@ -2981,6 +3031,14 @@ const SadSkipMxNx4Param skip_x4d_avx512_tests[] = {
make_tuple(64, 128, &aom_sad_skip_64x128x4d_avx512, -1),
make_tuple(64, 64, &aom_sad_skip_64x64x4d_avx512, -1),
make_tuple(64, 32, &aom_sad_skip_64x32x4d_avx512, -1),
+ make_tuple(32, 32, &aom_sad_skip_32x32x4d_avx512, -1),
+ make_tuple(32, 16, &aom_sad_skip_32x16x4d_avx512, -1),
+ make_tuple(16, 32, &aom_sad_skip_16x32x4d_avx512, -1),
+ make_tuple(16, 16, &aom_sad_skip_16x16x4d_avx512, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_sad_skip_64x16x4d_avx512, -1),
+ make_tuple(16, 64, &aom_sad_skip_16x64x4d_avx512, -1),
+#endif
};
INSTANTIATE_TEST_SUITE_P(AVX512, SADSkipx4Test,
::testing::ValuesIn(skip_x4d_avx512_tests));