Commit 1f409177a3 for aom
commit 1f409177a3d12b20565c527ff49501cfc3c3fb74
Author: Lin Zheng <linzhen@google.com>
Date: Sun Nov 8 23:00:30 2026 +0000
Highway: Add AVX512 masked_sad and obmc_sad
Function / Block Size │ AVX2 (ns) │ AVX512 (ns) │ Speedup (x)
──────────────────────┼───────────┼─────────────┼────────────
masked_sad 16x4 │ 18.46 │ 14.30 │ 1.29x
masked_sad 16x8 │ 28.72 │ 26.61 │ 1.08x
masked_sad 16x16 │ 48.18 │ 41.36 │ 1.16x
masked_sad 16x32 │ 92.18 │ 74.99 │ 1.23x
masked_sad 16x64 │ 178.60 │ 142.02 │ 1.26x
masked_sad 32x8 │ 33.42 │ 27.61 │ 1.21x
masked_sad 32x16 │ 68.18 │ 47.79 │ 1.43x
masked_sad 32x32 │ 129.39 │ 95.47 │ 1.36x
masked_sad 32x64 │ 258.21 │ 188.68 │ 1.37x
masked_sad 64x16 │ 118.76 │ 70.13 │ 1.69x
masked_sad 64x32 │ 229.35 │ 112.96 │ 2.03x
masked_sad 64x64 │ 370.24 │ 203.44 │ 1.82x
masked_sad 64x128 │ 966.72 │ 631.33 │ 1.53x
masked_sad 128x64 │ 789.82 │ 379.41 │ 2.08x
masked_sad 128x128 │ 1368.82 │ 1086.04 │ 1.26x
obmc_sad 16x4 │ 18.33 │ 11.93 │ 1.54x
obmc_sad 16x8 │ 34.18 │ 19.72 │ 1.73x
obmc_sad 16x16 │ 100.99 │ 34.55 │ 2.92x
obmc_sad 16x32 │ 192.54 │ 81.81 │ 2.35x
obmc_sad 16x64 │ 376.46 │ 100.31 │ 3.75x
obmc_sad 32x8 │ 55.47 │ 24.63 │ 2.25x
obmc_sad 32x16 │ 108.20 │ 50.73 │ 2.13x
obmc_sad 32x32 │ 220.78 │ 97.44 │ 2.27x
obmc_sad 32x64 │ 432.88 │ 189.34 │ 2.29x
obmc_sad 64x16 │ 222.09 │ 91.54 │ 2.43x
obmc_sad 64x32 │ 433.42 │ 179.12 │ 2.42x
obmc_sad 64x64 │ 962.89 │ 379.40 │ 2.54x
obmc_sad 64x128 │ 1856.34 │ 1195.38 │ 1.55x
obmc_sad 128x64 │ 1753.11 │ 835.49 │ 2.10x
obmc_sad 128x128 │ 3526.30 │ 1708.90 │ 2.06x
Change-Id: I7ed6c8d0f2f0aae47e584846ba8114324a42b883
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl
index f741044e04..d1325c9041 100755
--- a/aom_dsp/aom_dsp_rtcd_defs.pl
+++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -1057,7 +1057,11 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
foreach (@encoder_block_sizes) {
($w, $h) = @$_;
add_proto qw/unsigned int/, "aom_masked_sad${w}x${h}", "const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, const uint8_t *second_pred, const uint8_t *msk, int msk_stride, int invert_mask";
- specialize "aom_masked_sad${w}x${h}", qw/ssse3 avx2 neon/;
+ if (aom_config("CONFIG_HIGHWAY") eq "yes" && $w >= 16) {
+ specialize "aom_masked_sad${w}x${h}", qw/ssse3 avx2 avx512 neon/;
+ } else {
+ specialize "aom_masked_sad${w}x${h}", qw/ssse3 avx2 neon/;
+ }
}
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
@@ -1076,7 +1080,11 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
($w, $h) = @$_;
add_proto qw/unsigned int/, "aom_obmc_sad${w}x${h}", "const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask";
if (! (($w == 128 && $h == 32) || ($w == 32 && $h == 128))) {
- specialize "aom_obmc_sad${w}x${h}", qw/sse4_1 avx2 neon/;
+ if (aom_config("CONFIG_HIGHWAY") eq "yes" && $w >= 16) {
+ specialize "aom_obmc_sad${w}x${h}", qw/sse4_1 avx2 avx512 neon/;
+ } else {
+ specialize "aom_obmc_sad${w}x${h}", qw/sse4_1 avx2 neon/;
+ }
}
}
diff --git a/aom_dsp/sad_hwy.h b/aom_dsp/sad_hwy.h
index f33a35f279..23bb04a461 100644
--- a/aom_dsp/sad_hwy.h
+++ b/aom_dsp/sad_hwy.h
@@ -16,6 +16,7 @@
#include "config/aom_config.h"
#include "config/aom_dsp_rtcd.h"
+#include "aom_dsp/blend.h"
#include "aom_dsp/reduce_sum_hwy.h"
#include "third_party/highway/hwy/highway.h"
@@ -169,6 +170,103 @@ HWY_ATTR HWY_INLINE void SumOfAbsoluteDiffND(const uint8_t *src_ptr,
hn::StoreU(block_sum, block_sum_tag, res);
}
+template <int BlockWidth>
+HWY_ATTR HWY_INLINE unsigned int MaskedSumOfAbsoluteDiff(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, const uint8_t *second_pred, const uint8_t *msk_ptr,
+ int msk_stride, int invert_mask, int h) {
+ constexpr hn::ScalableTag<uint8_t> pixel_tag;
+ constexpr hn::Repartition<int8_t, decltype(pixel_tag)> int8_tag;
+ constexpr hn::Repartition<int16_t, decltype(pixel_tag)> int16_tag;
+ constexpr hn::Repartition<uint64_t, decltype(pixel_tag)> intermediate_sum_tag;
+ constexpr int vw = hn::MaxLanes(pixel_tag);
+ constexpr int kRowsPerVec = (BlockWidth < vw) ? (vw / BlockWidth) : 1;
+ constexpr int kRowSpan = (BlockWidth < vw) ? vw : BlockWidth;
+ assert(h % kRowsPerVec == 0);
+
+ const auto mask_max = hn::Set(pixel_tag, 1 << AOM_BLEND_A64_ROUND_BITS);
+ const auto round_scale =
+ hn::Set(int16_tag, 1 << (15 - AOM_BLEND_A64_ROUND_BITS));
+ auto sum_sad = hn::Zero(intermediate_sum_tag);
+
+ for (int y = 0; y < h; y += kRowsPerVec) {
+ for (int x = 0; x < kRowSpan; x += vw) {
+ auto src = LoadRowsU<kRowsPerVec>(pixel_tag, &src_ptr[x], src_stride);
+ auto ref = LoadRowsU<kRowsPerVec>(pixel_tag, &ref_ptr[x], ref_stride);
+ auto sec = hn::LoadU(pixel_tag, &second_pred[x]);
+ auto m = LoadRowsU<kRowsPerVec>(pixel_tag, &msk_ptr[x], msk_stride);
+ auto m_inv = hn::Sub(mask_max, m);
+ auto a = invert_mask ? sec : ref;
+ auto b = invert_mask ? ref : sec;
+
+ auto data_l = hn::InterleaveLower(pixel_tag, a, b);
+ auto mask_l = hn::InterleaveLower(pixel_tag, m, m_inv);
+ auto pred_l = hn::MulFixedPoint15(
+ hn::SatWidenMulPairwiseAdd(int16_tag, data_l,
+ hn::BitCast(int8_tag, mask_l)),
+ round_scale);
+
+ auto data_r = hn::InterleaveUpper(pixel_tag, a, b);
+ auto mask_r = hn::InterleaveUpper(pixel_tag, m, m_inv);
+ auto pred_r = hn::MulFixedPoint15(
+ hn::SatWidenMulPairwiseAdd(int16_tag, data_r,
+ hn::BitCast(int8_tag, mask_r)),
+ round_scale);
+
+ auto pred = hn::ReorderDemote2To(pixel_tag, pred_l, pred_r);
+ sum_sad = hn::Add(sum_sad, hn::SumsOf8AbsDiff(pred, src));
+ }
+ src_ptr += src_stride * kRowsPerVec;
+ ref_ptr += ref_stride * kRowsPerVec;
+ second_pred += BlockWidth * kRowsPerVec;
+ msk_ptr += msk_stride * kRowsPerVec;
+ }
+ return static_cast<unsigned int>(
+ hn::ReduceSum(intermediate_sum_tag, sum_sad));
+}
+
+#if !CONFIG_REALTIME_ONLY
+template <int BlockWidth>
+HWY_ATTR HWY_INLINE unsigned int ObmcSumOfAbsoluteDiff(const uint8_t *pre,
+ int pre_stride,
+ const int32_t *wsrc,
+ const int32_t *mask,
+ int h) {
+ constexpr hn::ScalableTag<int32_t> d32;
+ constexpr hn::Repartition<int16_t, decltype(d32)> d16;
+ constexpr hn::Rebind<uint8_t, decltype(d32)> du8;
+ constexpr int kLanes = hn::MaxLanes(d32);
+ constexpr int kRowsPerVec = (BlockWidth < kLanes) ? (kLanes / BlockWidth) : 1;
+ constexpr int kColsPerRow = (BlockWidth < kLanes) ? kLanes : BlockWidth;
+ assert(h % kRowsPerVec == 0);
+
+ const auto v_bias_d = hn::Set(d32, (1 << 12) >> 1);
+ auto v_sad_d = hn::Zero(d32);
+ int n = 0;
+
+ for (int r = 0; r < h; r += kRowsPerVec) {
+ for (int c = 0; c < kColsPerRow; c += kLanes) {
+ auto v_p_b = LoadRowsU<kRowsPerVec>(du8, pre + c, pre_stride);
+ auto v_p_d = hn::PromoteTo(d32, v_p_b);
+ auto v_m_d = hn::LoadU(d32, mask + n);
+ auto v_w_d = hn::LoadU(d32, wsrc + n);
+ // Values in both pre and mask fit in signed 15 bits and are packed at
+ // 32-bit boundaries, so WidenMulPairwiseAdd (vpmaddwd) produces the
+ // exact product without overflow (matching obmc_sad_avx2.c).
+ auto v_pm_d = hn::WidenMulPairwiseAdd(d32, hn::BitCast(d16, v_p_d),
+ hn::BitCast(d16, v_m_d));
+ auto v_diff_d = hn::Sub(v_w_d, v_pm_d);
+ auto v_absdiff_d = hn::Abs(v_diff_d);
+ auto v_rad_d = hn::ShiftRight<12>(hn::Add(v_absdiff_d, v_bias_d));
+ v_sad_d = hn::Add(v_sad_d, v_rad_d);
+ n += kLanes;
+ }
+ pre += pre_stride * kRowsPerVec;
+ }
+ return static_cast<unsigned int>(hn::ReduceSum(d32, v_sad_d));
+}
+#endif // !CONFIG_REALTIME_ONLY
+
} // namespace HWY_NAMESPACE
} // namespace
@@ -242,6 +340,33 @@ HWY_ATTR HWY_INLINE void SumOfAbsoluteDiffND(const uint8_t *src_ptr,
src_ptr, src_stride, ref_ptr, ref_stride, h, second_pred); \
}
+#define FMASKED_SAD(w, h, suffix) \
+ extern "C" unsigned int aom_masked_sad##w##x##h##_##suffix( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
+ const uint8_t *second_pred, const uint8_t *msk, int msk_stride, \
+ int invert_mask); \
+ HWY_ATTR unsigned int aom_masked_sad##w##x##h##_##suffix( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
+ const uint8_t *second_pred, const uint8_t *msk, int msk_stride, \
+ int invert_mask) { \
+ return HWY_NAMESPACE::MaskedSumOfAbsoluteDiff<w>( \
+ src, src_stride, ref, ref_stride, second_pred, msk, msk_stride, \
+ invert_mask, h); \
+ }
+
+#if !CONFIG_REALTIME_ONLY
+#define FOBMC_SAD(w, h, suffix) \
+ extern "C" unsigned int aom_obmc_sad##w##x##h##_##suffix( \
+ const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
+ const int32_t *mask); \
+ HWY_ATTR unsigned int aom_obmc_sad##w##x##h##_##suffix( \
+ const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
+ const int32_t *mask) { \
+ return HWY_NAMESPACE::ObmcSumOfAbsoluteDiff<w>(pre, pre_stride, wsrc, \
+ mask, h); \
+ }
+#endif // !CONFIG_REALTIME_ONLY
+
#define FOR_EACH_SAD_BLOCK_SIZE(X, suffix) \
X(128, 128, suffix) \
X(128, 64, suffix) \
@@ -257,10 +382,17 @@ HWY_ATTR HWY_INLINE void SumOfAbsoluteDiffND(const uint8_t *src_ptr,
#define FOR_EACH_SAD_H8_RECT_BLOCK_SIZE(X, suffix) X(32, 8, suffix)
#define FOR_EACH_SAD_H4_BLOCK_SIZE(X, suffix) X(16, 4, suffix)
+
+#define FOR_EACH_MASKED_OBMC_RECT_BLOCK_SIZE(X, suffix) \
+ X(64, 16, suffix) \
+ X(16, 64, suffix) \
+ X(32, 8, suffix) \
+ X(16, 4, suffix)
#else
#define FOR_EACH_SAD_SMALL_SKIP_RECT_BLOCK_SIZE(X, suffix)
#define FOR_EACH_SAD_H8_RECT_BLOCK_SIZE(X, suffix)
#define FOR_EACH_SAD_H4_BLOCK_SIZE(X, suffix)
+#define FOR_EACH_MASKED_OBMC_RECT_BLOCK_SIZE(X, suffix)
#endif // !CONFIG_REALTIME_ONLY
#define FOR_EACH_SAD_SMALL_SKIP_BLOCK_SIZE(X, suffix) \
@@ -274,6 +406,20 @@ HWY_ATTR HWY_INLINE void SumOfAbsoluteDiffND(const uint8_t *src_ptr,
X(16, 8, suffix) \
FOR_EACH_SAD_H8_RECT_BLOCK_SIZE(X, suffix)
+#define FOR_EACH_MASKED_OBMC_SAD_BLOCK_SIZE(X, suffix) \
+ X(128, 128, suffix) \
+ X(128, 64, suffix) \
+ X(64, 128, suffix) \
+ X(64, 64, suffix) \
+ X(64, 32, suffix) \
+ X(32, 64, suffix) \
+ X(32, 32, suffix) \
+ X(32, 16, suffix) \
+ X(16, 32, suffix) \
+ X(16, 16, suffix) \
+ X(16, 8, suffix) \
+ FOR_EACH_MASKED_OBMC_RECT_BLOCK_SIZE(X, suffix)
+
HWY_AFTER_NAMESPACE();
#endif // AOM_AOM_DSP_SAD_HWY_H_
diff --git a/aom_dsp/x86/sad_hwy_avx512.cc b/aom_dsp/x86/sad_hwy_avx512.cc
index 1a9192e1d4..7a9bfb3421 100644
--- a/aom_dsp/x86/sad_hwy_avx512.cc
+++ b/aom_dsp/x86/sad_hwy_avx512.cc
@@ -37,3 +37,8 @@ FOR_EACH_SAD_H8_BLOCK_SIZE(FSAD_3D, avx512)
FOR_EACH_SAD_H4_BLOCK_SIZE(FSAD_4D, avx512)
FOR_EACH_SAD_H4_BLOCK_SIZE(FSAD_3D, avx512)
#endif // !CONFIG_REALTIME_ONLY
+
+FOR_EACH_MASKED_OBMC_SAD_BLOCK_SIZE(FMASKED_SAD, avx512)
+#if !CONFIG_REALTIME_ONLY
+FOR_EACH_MASKED_OBMC_SAD_BLOCK_SIZE(FOBMC_SAD, avx512)
+#endif // !CONFIG_REALTIME_ONLY
diff --git a/test/masked_sad_test.cc b/test/masked_sad_test.cc
index 4de17aeeea..c9bce55817 100644
--- a/test/masked_sad_test.cc
+++ b/test/masked_sad_test.cc
@@ -433,6 +433,31 @@ INSTANTIATE_TEST_SUITE_P(AVX2, HighbdMaskedSADTest,
#endif // CONFIG_AV1_HIGHBITDEPTH
#endif // HAVE_AVX2
+#if CONFIG_HIGHWAY && HAVE_AVX512
+const MaskedSADParam msad_avx512_test[] = {
+ make_tuple(&aom_masked_sad16x8_avx512, &aom_masked_sad16x8_c),
+ make_tuple(&aom_masked_sad16x16_avx512, &aom_masked_sad16x16_c),
+ make_tuple(&aom_masked_sad16x32_avx512, &aom_masked_sad16x32_c),
+ make_tuple(&aom_masked_sad32x16_avx512, &aom_masked_sad32x16_c),
+ make_tuple(&aom_masked_sad32x32_avx512, &aom_masked_sad32x32_c),
+ make_tuple(&aom_masked_sad32x64_avx512, &aom_masked_sad32x64_c),
+ make_tuple(&aom_masked_sad64x32_avx512, &aom_masked_sad64x32_c),
+ make_tuple(&aom_masked_sad64x64_avx512, &aom_masked_sad64x64_c),
+ make_tuple(&aom_masked_sad64x128_avx512, &aom_masked_sad64x128_c),
+ make_tuple(&aom_masked_sad128x64_avx512, &aom_masked_sad128x64_c),
+ make_tuple(&aom_masked_sad128x128_avx512, &aom_masked_sad128x128_c),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(&aom_masked_sad16x4_avx512, &aom_masked_sad16x4_c),
+ make_tuple(&aom_masked_sad32x8_avx512, &aom_masked_sad32x8_c),
+ make_tuple(&aom_masked_sad16x64_avx512, &aom_masked_sad16x64_c),
+ make_tuple(&aom_masked_sad64x16_avx512, &aom_masked_sad64x16_c)
+#endif
+};
+
+INSTANTIATE_TEST_SUITE_P(AVX512, MaskedSADTest,
+ ::testing::ValuesIn(msad_avx512_test));
+#endif // CONFIG_HIGHWAY && HAVE_AVX512
+
#if HAVE_NEON
const MaskedSADParam msad_test[] = {
make_tuple(&aom_masked_sad4x4_neon, &aom_masked_sad4x4_c),
diff --git a/test/obmc_sad_test.cc b/test/obmc_sad_test.cc
index 17f32516b6..20db07de42 100644
--- a/test/obmc_sad_test.cc
+++ b/test/obmc_sad_test.cc
@@ -147,6 +147,30 @@ INSTANTIATE_TEST_SUITE_P(AVX2, ObmcSadTest,
::testing::ValuesIn(avx2_functions));
#endif // HAVE_AVX2
+#if CONFIG_HIGHWAY && HAVE_AVX512
+const ObmcSadTest::ParamType avx512_functions[] = {
+ TestFuncs(aom_obmc_sad128x128_c, aom_obmc_sad128x128_avx512),
+ TestFuncs(aom_obmc_sad128x64_c, aom_obmc_sad128x64_avx512),
+ TestFuncs(aom_obmc_sad64x128_c, aom_obmc_sad64x128_avx512),
+ TestFuncs(aom_obmc_sad64x64_c, aom_obmc_sad64x64_avx512),
+ TestFuncs(aom_obmc_sad64x32_c, aom_obmc_sad64x32_avx512),
+ TestFuncs(aom_obmc_sad32x64_c, aom_obmc_sad32x64_avx512),
+ TestFuncs(aom_obmc_sad32x32_c, aom_obmc_sad32x32_avx512),
+ TestFuncs(aom_obmc_sad32x16_c, aom_obmc_sad32x16_avx512),
+ TestFuncs(aom_obmc_sad16x32_c, aom_obmc_sad16x32_avx512),
+ TestFuncs(aom_obmc_sad16x16_c, aom_obmc_sad16x16_avx512),
+ TestFuncs(aom_obmc_sad16x8_c, aom_obmc_sad16x8_avx512),
+
+ TestFuncs(aom_obmc_sad64x16_c, aom_obmc_sad64x16_avx512),
+ TestFuncs(aom_obmc_sad16x64_c, aom_obmc_sad16x64_avx512),
+ TestFuncs(aom_obmc_sad32x8_c, aom_obmc_sad32x8_avx512),
+ TestFuncs(aom_obmc_sad16x4_c, aom_obmc_sad16x4_avx512),
+};
+
+INSTANTIATE_TEST_SUITE_P(AVX512, ObmcSadTest,
+ ::testing::ValuesIn(avx512_functions));
+#endif // CONFIG_HIGHWAY && HAVE_AVX512
+
#if HAVE_NEON
const ObmcSadTest::ParamType neon_functions[] = {
TestFuncs(aom_obmc_sad128x128_c, aom_obmc_sad128x128_neon),