Commit ecb1d1d561 for aom
commit ecb1d1d5613bdb7310ef1b735b9c40190b8384c2
Author: Lin Zheng <linzhen@google.com>
Date: Sun Nov 8 23:00:37 2026 +0000
Highway: Add AVX512 sub_pixel_{avg_}variance
Function / Block Size │ Base (ns) │ AVX512 (ns) │ Speedup (x)
──────────────────────────┼───────────┼─────────────┼────────────
sub_pixel_var 32x8 │ 47.44 │ 16.81 │ 2.82x
sub_pixel_var 32x16 │ 36.27 │ 31.02 │ 1.17x
sub_pixel_var 32x32 │ 69.47 │ 57.79 │ 1.20x
sub_pixel_var 32x64 │ 135.61 │ 111.03 │ 1.22x
sub_pixel_var 64x16 │ 178.83 │ 45.71 │ 3.91x
sub_pixel_var 64x32 │ 138.85 │ 90.10 │ 1.54x
sub_pixel_var 64x64 │ 314.25 │ 187.96 │ 1.67x
sub_pixel_var 64x128 │ 610.23 │ 377.10 │ 1.62x
sub_pixel_var 128x64 │ 603.34 │ 382.84 │ 1.58x
sub_pixel_var 128x128 │ 1089.61 │ 712.94 │ 1.53x
sub_pixel_avg_var 32x8 │ 60.50 │ 20.10 │ 3.01x
sub_pixel_avg_var 32x16 │ 58.79 │ 38.98 │ 1.51x
sub_pixel_avg_var 32x32 │ 114.11 │ 71.65 │ 1.59x
sub_pixel_avg_var 32x64 │ 225.75 │ 141.90 │ 1.59x
sub_pixel_avg_var 64x16 │ 233.96 │ 59.94 │ 3.90x
sub_pixel_avg_var 64x32 │ 232.92 │ 113.35 │ 2.05x
sub_pixel_avg_var 64x64 │ 451.68 │ 226.90 │ 1.99x
sub_pixel_avg_var 64x128 │ 862.32 │ 428.44 │ 2.01x
sub_pixel_avg_var 128x64 │ 873.18 │ 432.90 │ 2.02x
sub_pixel_avg_var 128x128│ 1722.62 │ 854.85 │ 2.02x
Change-Id: I82c5193e18e9fcc30c966d43d7401e0763e618ef
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl
index c8b28deb93..7358e4d21f 100755
--- a/aom_dsp/aom_dsp_rtcd_defs.pl
+++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -1537,6 +1537,17 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_sub_pixel_variance4x8 neon neon_dotprod/, "$ssse3_asm";
specialize qw/aom_sub_pixel_variance4x4 neon/, "$ssse3_asm";
+ if (aom_config("CONFIG_HIGHWAY") eq "yes") {
+ specialize qw/aom_sub_pixel_variance128x128 avx512/;
+ specialize qw/aom_sub_pixel_variance128x64 avx512/;
+ specialize qw/aom_sub_pixel_variance64x128 avx512/;
+ specialize qw/aom_sub_pixel_variance64x64 avx512/;
+ specialize qw/aom_sub_pixel_variance64x32 avx512/;
+ specialize qw/aom_sub_pixel_variance32x64 avx512/;
+ specialize qw/aom_sub_pixel_variance32x32 avx512/;
+ specialize qw/aom_sub_pixel_variance32x16 avx512/;
+ }
+
specialize qw/aom_sub_pixel_avg_variance128x128 avx2 neon/, "$ssse3_asm";
specialize qw/aom_sub_pixel_avg_variance128x64 avx2 neon/, "$ssse3_asm";
specialize qw/aom_sub_pixel_avg_variance64x128 avx2 neon/, "$ssse3_asm";
@@ -1554,6 +1565,17 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_sub_pixel_avg_variance4x8 neon/, "$ssse3_asm";
specialize qw/aom_sub_pixel_avg_variance4x4 neon/, "$ssse3_asm";
+ if (aom_config("CONFIG_HIGHWAY") eq "yes") {
+ specialize qw/aom_sub_pixel_avg_variance128x128 avx512/;
+ specialize qw/aom_sub_pixel_avg_variance128x64 avx512/;
+ specialize qw/aom_sub_pixel_avg_variance64x128 avx512/;
+ specialize qw/aom_sub_pixel_avg_variance64x64 avx512/;
+ specialize qw/aom_sub_pixel_avg_variance64x32 avx512/;
+ specialize qw/aom_sub_pixel_avg_variance32x64 avx512/;
+ specialize qw/aom_sub_pixel_avg_variance32x32 avx512/;
+ specialize qw/aom_sub_pixel_avg_variance32x16 avx512/;
+ }
+
if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
specialize qw/aom_variance4x16 neon neon_dotprod sse2/;
specialize qw/aom_variance16x4 neon neon_dotprod sse2 avx2/;
@@ -1581,6 +1603,13 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_sub_pixel_avg_variance32x8 neon/, "$ssse3_asm";
specialize qw/aom_sub_pixel_avg_variance16x64 neon/, "$ssse3_asm";
specialize qw/aom_sub_pixel_avg_variance64x16 neon/, "$ssse3_asm";
+
+ if (aom_config("CONFIG_HIGHWAY") eq "yes") {
+ specialize qw/aom_sub_pixel_variance32x8 avx512/;
+ specialize qw/aom_sub_pixel_variance64x16 avx512/;
+ specialize qw/aom_sub_pixel_avg_variance32x8 avx512/;
+ specialize qw/aom_sub_pixel_avg_variance64x16 avx512/;
+ }
}
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
diff --git a/aom_dsp/variance_hwy.h b/aom_dsp/variance_hwy.h
index 73d2cd09d3..4d5331529e 100644
--- a/aom_dsp/variance_hwy.h
+++ b/aom_dsp/variance_hwy.h
@@ -430,9 +430,287 @@ HWY_ATTR HWY_INLINE int64_t HighbdCalcVarianceStat(const uint16_t *src,
}
#endif // CONFIG_AV1_HIGHBITDEPTH
+// -----------------------------------------------------------------------------
+// Subpixel Variance & Subpixel Avg Variance
+// -----------------------------------------------------------------------------
+
+template <class DI8>
+HWY_ATTR HWY_INLINE hn::VFromD<DI8> BilinearFilter(DI8 di8, int offset) {
+ assert(offset >= 0 && offset < 8);
+ constexpr hn::Repartition<int16_t, DI8> di16;
+ const int f0 = 16 - 2 * offset;
+ const int f1 = 2 * offset;
+ return hn::BitCast(di8, hn::Set(di16, static_cast<int16_t>(f0 | (f1 << 8))));
+}
+
+template <class DU8, class DI16>
+HWY_ATTR HWY_INLINE void FilterPair(
+ DU8 du8, DI16 di16, hn::VFromD<DU8> s0, hn::VFromD<DU8> s1,
+ hn::VFromD<hn::Repartition<int8_t, DU8>> filt, hn::VFromD<DI16> *out_lo,
+ hn::VFromD<DI16> *out_hi) {
+ const auto pw8 = hn::Set(di16, int16_t{ 8 });
+ const auto lo =
+ hn::SatWidenMulPairwiseAdd(di16, hn::InterleaveLower(du8, s0, s1), filt);
+ const auto hi =
+ hn::SatWidenMulPairwiseAdd(di16, hn::InterleaveUpper(du8, s0, s1), filt);
+ *out_lo = hn::ShiftRight<4>(hn::Add(lo, pw8));
+ *out_hi = hn::ShiftRight<4>(hn::Add(hi, pw8));
+}
+
+template <class DU8>
+HWY_ATTR HWY_INLINE hn::VFromD<DU8> FilterPairPack(
+ DU8 du8, hn::VFromD<DU8> s0, hn::VFromD<DU8> s1,
+ hn::VFromD<hn::Repartition<int8_t, DU8>> filt) {
+ constexpr hn::Repartition<int16_t, DU8> di16;
+ hn::VFromD<decltype(di16)> lo, hi;
+ FilterPair(du8, di16, s0, s1, filt, &lo, &hi);
+ return hn::ReorderDemote2To(du8, lo, hi);
+}
+
+template <class DU8, class DI16, class DI32>
+HWY_ATTR HWY_INLINE void AccumDiff(DU8 du8, DI16 di16, DI32 di32,
+ hn::VFromD<DI16> exp_src_lo,
+ hn::VFromD<DI16> exp_src_hi,
+ hn::VFromD<DU8> dst_reg,
+ hn::VFromD<DI32> *sse_reg,
+ hn::VFromD<DI16> *sum_reg) {
+ const auto zero = hn::Zero(du8);
+ const auto exp_dst_lo =
+ hn::BitCast(di16, hn::InterleaveLower(du8, dst_reg, zero));
+ const auto exp_dst_hi =
+ hn::BitCast(di16, hn::InterleaveUpper(du8, dst_reg, zero));
+ const auto d_lo = hn::Sub(exp_src_lo, exp_dst_lo);
+ const auto d_hi = hn::Sub(exp_src_hi, exp_dst_hi);
+ *sum_reg = hn::Add(*sum_reg, hn::Add(d_lo, d_hi));
+ const auto madd_lo = hn::WidenMulPairwiseAdd(di32, d_lo, d_lo);
+ const auto madd_hi = hn::WidenMulPairwiseAdd(di32, d_hi, d_hi);
+ *sse_reg = hn::Add(*sse_reg, hn::Add(madd_lo, madd_hi));
+}
+
+template <int Width, class D>
+HWY_ATTR HWY_INLINE hn::VFromD<D> SubPelLoad(D d, const uint8_t *ptr,
+ ptrdiff_t half_stride) {
+ HWY_IF_CONSTEXPR(Width == 32 && hn::MaxLanes(d) == 64) {
+ constexpr hn::Half<D> d_half;
+ return Combine2Rows(d, hn::LoadU(d_half, ptr),
+ hn::LoadU(d_half, ptr + half_stride));
+ }
+ return hn::LoadU(d, ptr);
+}
+
+template <int Width, bool HasSec, class LoadRowFn>
+HWY_ATTR HWY_INLINE void SubPelVertLoop(
+ const uint8_t *s_ptr, int src_stride, int y_offset, const uint8_t *d_ptr,
+ int dst_stride, ptrdiff_t half_dst_stride, const uint8_t *sec_ptr,
+ int sec_stride, ptrdiff_t half_sec_stride, int loop_h, LoadRowFn load_row,
+ hn::Vec<hn::ScalableTag<int32_t>> *sse_reg,
+ hn::Vec<hn::ScalableTag<int16_t>> *sum_reg) {
+ constexpr hn::ScalableTag<uint8_t> du8;
+ constexpr hn::Repartition<int8_t, decltype(du8)> di8;
+ constexpr hn::Repartition<int16_t, decltype(du8)> di16;
+ constexpr hn::Repartition<int32_t, decltype(du8)> di32;
+
+ if (y_offset == 0) {
+ for (int i = 0; i < loop_h; ++i) {
+ auto s = load_row(s_ptr);
+ HWY_IF_CONSTEXPR(HasSec) {
+ s = hn::AverageRound(s,
+ SubPelLoad<Width>(du8, sec_ptr, half_sec_stride));
+ sec_ptr += sec_stride;
+ }
+ const auto d = SubPelLoad<Width>(du8, d_ptr, half_dst_stride);
+ VarianceKernel(du8, di32, di16, s, d, sse_reg, sum_reg);
+ s_ptr += src_stride;
+ d_ptr += dst_stride;
+ }
+ } else if (y_offset == 4) {
+ auto s_prev = load_row(s_ptr);
+ for (int i = 0; i < loop_h; ++i) {
+ s_ptr += src_stride;
+ const auto s_curr = load_row(s_ptr);
+ auto s_avg = hn::AverageRound(s_prev, s_curr);
+ HWY_IF_CONSTEXPR(HasSec) {
+ s_avg = hn::AverageRound(
+ s_avg, SubPelLoad<Width>(du8, sec_ptr, half_sec_stride));
+ sec_ptr += sec_stride;
+ }
+ const auto d = SubPelLoad<Width>(du8, d_ptr, half_dst_stride);
+ VarianceKernel(du8, di32, di16, s_avg, d, sse_reg, sum_reg);
+ s_prev = s_curr;
+ d_ptr += dst_stride;
+ }
+ } else {
+ const auto yfilter = BilinearFilter(di8, y_offset);
+ auto s_prev = load_row(s_ptr);
+ for (int i = 0; i < loop_h; ++i) {
+ s_ptr += src_stride;
+ const auto s_curr = load_row(s_ptr);
+ const auto d = SubPelLoad<Width>(du8, d_ptr, half_dst_stride);
+ HWY_IF_CONSTEXPR(HasSec) {
+ const auto s_avg =
+ hn::AverageRound(FilterPairPack(du8, s_prev, s_curr, yfilter),
+ SubPelLoad<Width>(du8, sec_ptr, half_sec_stride));
+ VarianceKernel(du8, di32, di16, s_avg, d, sse_reg, sum_reg);
+ sec_ptr += sec_stride;
+ }
+ HWY_IF_CONSTEXPR(!HasSec) {
+ hn::VFromD<decltype(di16)> exp_lo, exp_hi;
+ FilterPair(du8, di16, s_prev, s_curr, yfilter, &exp_lo, &exp_hi);
+ AccumDiff(du8, di16, di32, exp_lo, exp_hi, d, sse_reg, sum_reg);
+ }
+ s_prev = s_curr;
+ d_ptr += dst_stride;
+ }
+ }
+}
+
+template <int Width, bool HasSec>
+HWY_ATTR HWY_INLINE int SubPixelVarianceWxHImp(
+ const uint8_t *src, int src_stride, int x_offset, int y_offset,
+ const uint8_t *dst, int dst_stride, const uint8_t *sec, int sec_stride,
+ int height, unsigned int *sse) {
+ constexpr hn::ScalableTag<uint8_t> du8;
+ constexpr hn::Repartition<int8_t, decltype(du8)> di8;
+ constexpr hn::Repartition<int16_t, decltype(du8)> di16;
+ constexpr hn::Repartition<int32_t, decltype(du8)> di32;
+ constexpr int vw = hn::MaxLanes(du8);
+ constexpr int kRowsPerIter = (Width == 32 && vw == 64) ? 2 : 1;
+ constexpr int kColStep = (Width == 32 && vw == 64) ? 32 : vw;
+ static_assert(Width == 32 || Width % kColStep == 0,
+ "Width must be 32 or a multiple of kColStep.");
+
+ const int loop_h = height / kRowsPerIter;
+ // Each int16_t lane in sum_reg accumulates 2 pixel diffs per iteration, so
+ // loop_h must be <= 64 (128 * 255 = 32640 <= 32767) to avoid overflow.
+ assert(height > 0 && height % kRowsPerIter == 0 && loop_h <= 64);
+ const ptrdiff_t half_src_stride = static_cast<ptrdiff_t>(loop_h) * src_stride;
+ const ptrdiff_t half_dst_stride = static_cast<ptrdiff_t>(loop_h) * dst_stride;
+ const ptrdiff_t half_sec_stride = static_cast<ptrdiff_t>(loop_h) * sec_stride;
+
+ const auto one = hn::Set(di16, int16_t{ 1 });
+ auto sum32_reg = hn::Zero(di32);
+ auto sse_reg = hn::Zero(di32);
+
+ for (int col = 0; col < Width; col += kColStep) {
+ auto sum_reg = hn::Zero(di16);
+ const uint8_t *s_ptr = src + col;
+ const uint8_t *d_ptr = dst + col;
+ const uint8_t *sec_ptr = HasSec ? (sec + col) : nullptr;
+
+ if (x_offset == 0) {
+ SubPelVertLoop<Width, HasSec>(
+ s_ptr, src_stride, y_offset, d_ptr, dst_stride, half_dst_stride,
+ sec_ptr, sec_stride, half_sec_stride, loop_h,
+ [=](const uint8_t *p)
+ HWY_ATTR { return SubPelLoad<Width>(du8, p, half_src_stride); },
+ &sse_reg, &sum_reg);
+ } else if (x_offset == 4) {
+ SubPelVertLoop<Width, HasSec>(
+ s_ptr, src_stride, y_offset, d_ptr, dst_stride, half_dst_stride,
+ sec_ptr, sec_stride, half_sec_stride, loop_h,
+ [=](const uint8_t *p) HWY_ATTR {
+ return hn::AverageRound(
+ SubPelLoad<Width>(du8, p, half_src_stride),
+ SubPelLoad<Width>(du8, p + 1, half_src_stride));
+ },
+ &sse_reg, &sum_reg);
+ } else {
+ const auto xfilter = BilinearFilter(di8, x_offset);
+ if (y_offset == 0) {
+ for (int i = 0; i < loop_h; ++i) {
+ const auto s0 = SubPelLoad<Width>(du8, s_ptr, half_src_stride);
+ const auto s1 = SubPelLoad<Width>(du8, s_ptr + 1, half_src_stride);
+ const auto d = SubPelLoad<Width>(du8, d_ptr, half_dst_stride);
+ HWY_IF_CONSTEXPR(HasSec) {
+ const auto s = hn::AverageRound(
+ FilterPairPack(du8, s0, s1, xfilter),
+ SubPelLoad<Width>(du8, sec_ptr, half_sec_stride));
+ VarianceKernel(du8, di32, di16, s, d, &sse_reg, &sum_reg);
+ sec_ptr += sec_stride;
+ }
+ HWY_IF_CONSTEXPR(!HasSec) {
+ hn::VFromD<decltype(di16)> exp_lo, exp_hi;
+ FilterPair(du8, di16, s0, s1, xfilter, &exp_lo, &exp_hi);
+ AccumDiff(du8, di16, di32, exp_lo, exp_hi, d, &sse_reg, &sum_reg);
+ }
+ s_ptr += src_stride;
+ d_ptr += dst_stride;
+ }
+ } else {
+ SubPelVertLoop<Width, HasSec>(
+ s_ptr, src_stride, y_offset, d_ptr, dst_stride, half_dst_stride,
+ sec_ptr, sec_stride, half_sec_stride, loop_h,
+ [=](const uint8_t *p) HWY_ATTR {
+ return FilterPairPack(
+ du8, SubPelLoad<Width>(du8, p, half_src_stride),
+ SubPelLoad<Width>(du8, p + 1, half_src_stride), xfilter);
+ },
+ &sse_reg, &sum_reg);
+ }
+ }
+ sum32_reg = hn::Add(sum32_reg, hn::WidenMulPairwiseAdd(di32, sum_reg, one));
+ }
+ return VarianceFinalFrom32BitSum(di32, sse_reg, sum32_reg, sse);
+}
+
+template <int BlockWidth, int BlockHeight, int ChunkHeight, int WLog2,
+ int HLog2, bool HasSec>
+HWY_ATTR HWY_INLINE unsigned int SubPixelVarianceWxH(
+ const uint8_t *src, int src_stride, int x_offset, int y_offset,
+ const uint8_t *dst, int dst_stride, unsigned int *sse_ptr,
+ const uint8_t *sec) {
+ constexpr int kChunkWidth = (BlockWidth == 32) ? 32 : 64;
+ static_assert(BlockWidth == 32 || BlockWidth % 64 == 0,
+ "BlockWidth must be 32 or a multiple of 64.");
+ static_assert(BlockHeight % ChunkHeight == 0,
+ "BlockHeight must be a multiple of ChunkHeight.");
+ // ChunkHeight <= 64 keeps the int16_t accumulator in SubPixelVarianceWxHImp
+ // within bounds (at most 64 iterations * 2 diffs = 128 diffs per lane).
+ static_assert(ChunkHeight > 0 && ChunkHeight <= 64,
+ "ChunkHeight must be in (0, 64] to avoid int16_t overflow.");
+ static_assert((1 << (WLog2 + HLog2)) == BlockWidth * BlockHeight,
+ "(1 << (WLog2 + HLog2)) must equal BlockWidth * BlockHeight.");
+ HWY_IF_CONSTEXPR(BlockWidth == kChunkWidth && BlockHeight == ChunkHeight) {
+ const int sum = SubPixelVarianceWxHImp<kChunkWidth, HasSec>(
+ src, src_stride, x_offset, y_offset, dst, dst_stride,
+ HasSec ? sec : nullptr, HasSec ? BlockWidth : 0, BlockHeight, sse_ptr);
+ return *sse_ptr - static_cast<unsigned int>(
+ (static_cast<int64_t>(sum) * sum) >> (WLog2 + HLog2));
+ }
+ unsigned int sse = 0;
+ int se = 0;
+ for (int i = 0; i < BlockWidth / kChunkWidth; ++i) {
+ const uint8_t *src_ptr = src;
+ const uint8_t *dst_ptr = dst;
+ const uint8_t *sec_ptr = sec;
+ for (int j = 0; j < BlockHeight / ChunkHeight; ++j) {
+ unsigned int sse2;
+ const int se2 = SubPixelVarianceWxHImp<kChunkWidth, HasSec>(
+ src_ptr, src_stride, x_offset, y_offset, dst_ptr, dst_stride,
+ HasSec ? sec_ptr : nullptr, HasSec ? BlockWidth : 0, ChunkHeight,
+ &sse2);
+ dst_ptr += ChunkHeight * dst_stride;
+ src_ptr += ChunkHeight * src_stride;
+ if (HasSec) sec_ptr += ChunkHeight * BlockWidth;
+ se += se2;
+ sse += sse2;
+ }
+ src += kChunkWidth;
+ dst += kChunkWidth;
+ if (HasSec) sec += kChunkWidth;
+ }
+ *sse_ptr = sse;
+ return sse - static_cast<unsigned int>((static_cast<int64_t>(se) * se) >>
+ (WLog2 + HLog2));
+}
+
} // namespace HWY_NAMESPACE
} // namespace
+// -----------------------------------------------------------------------------
+// Export Macros
+// -----------------------------------------------------------------------------
+
#define FVAR(w, h, bits, suffix) \
extern "C" unsigned int aom_variance##w##x##h##_##suffix( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
@@ -512,6 +790,73 @@ HWY_ATTR HWY_INLINE int64_t HighbdCalcVarianceStat(const uint16_t *src,
X(16, 8, 7, suffix) \
FOR_EACH_VAR_RECT_BLOCK_SIZE(X, suffix)
+#define FSUBPIX_VAR_32XH(h, log2h, suffix) \
+ extern "C" unsigned int aom_sub_pixel_variance32x##h##_##suffix( \
+ const uint8_t *src, int src_stride, int x_offset, int y_offset, \
+ const uint8_t *dst, int dst_stride, unsigned int *sse); \
+ HWY_ATTR unsigned int aom_sub_pixel_variance32x##h##_##suffix( \
+ const uint8_t *src, int src_stride, int x_offset, int y_offset, \
+ const uint8_t *dst, int dst_stride, unsigned int *sse) { \
+ return HWY_NAMESPACE::SubPixelVarianceWxH<32, h, h, 5, log2h, false>( \
+ src, src_stride, x_offset, y_offset, dst, dst_stride, sse, nullptr); \
+ } \
+ extern "C" unsigned int aom_sub_pixel_avg_variance32x##h##_##suffix( \
+ const uint8_t *src, int src_stride, int x_offset, int y_offset, \
+ const uint8_t *dst, int dst_stride, unsigned int *sse, \
+ const uint8_t *sec_ptr); \
+ HWY_ATTR unsigned int aom_sub_pixel_avg_variance32x##h##_##suffix( \
+ const uint8_t *src, int src_stride, int x_offset, int y_offset, \
+ const uint8_t *dst, int dst_stride, unsigned int *sse, \
+ const uint8_t *sec_ptr) { \
+ return HWY_NAMESPACE::SubPixelVarianceWxH<32, h, h, 5, log2h, true>( \
+ src, src_stride, x_offset, y_offset, dst, dst_stride, sse, sec_ptr); \
+ }
+
+#define FSUBPIX_VAR_64_128(w, h, hf, wlog2, hlog2, suffix) \
+ extern "C" unsigned int aom_sub_pixel_variance##w##x##h##_##suffix( \
+ const uint8_t *src, int src_stride, int x_offset, int y_offset, \
+ const uint8_t *dst, int dst_stride, unsigned int *sse_ptr); \
+ HWY_ATTR unsigned int aom_sub_pixel_variance##w##x##h##_##suffix( \
+ const uint8_t *src, int src_stride, int x_offset, int y_offset, \
+ const uint8_t *dst, int dst_stride, unsigned int *sse_ptr) { \
+ return HWY_NAMESPACE::SubPixelVarianceWxH<w, h, hf, wlog2, hlog2, false>( \
+ src, src_stride, x_offset, y_offset, dst, dst_stride, sse_ptr, \
+ nullptr); \
+ } \
+ extern "C" unsigned int aom_sub_pixel_avg_variance##w##x##h##_##suffix( \
+ const uint8_t *src, int src_stride, int x_offset, int y_offset, \
+ const uint8_t *dst, int dst_stride, unsigned int *sse_ptr, \
+ const uint8_t *sec); \
+ HWY_ATTR unsigned int aom_sub_pixel_avg_variance##w##x##h##_##suffix( \
+ const uint8_t *src, int src_stride, int x_offset, int y_offset, \
+ const uint8_t *dst, int dst_stride, unsigned int *sse_ptr, \
+ const uint8_t *sec) { \
+ return HWY_NAMESPACE::SubPixelVarianceWxH<w, h, hf, wlog2, hlog2, true>( \
+ src, src_stride, x_offset, y_offset, dst, dst_stride, sse_ptr, sec); \
+ }
+
+#if !CONFIG_REALTIME_ONLY
+#define FOR_EACH_SUBPIX_VAR_32XH_RECT(X, suffix) X(8, 3, suffix)
+#define FOR_EACH_SUBPIX_VAR_64_128_RECT(X, suffix) X(64, 16, 16, 6, 4, suffix)
+#else
+#define FOR_EACH_SUBPIX_VAR_32XH_RECT(X, suffix)
+#define FOR_EACH_SUBPIX_VAR_64_128_RECT(X, suffix)
+#endif
+
+#define FOR_EACH_SUBPIX_VAR_32XH(X, suffix) \
+ X(64, 6, suffix) \
+ X(32, 5, suffix) \
+ X(16, 4, suffix) \
+ FOR_EACH_SUBPIX_VAR_32XH_RECT(X, suffix)
+
+#define FOR_EACH_SUBPIX_VAR_64_128(X, suffix) \
+ X(128, 128, 64, 7, 7, suffix) \
+ X(128, 64, 64, 7, 6, suffix) \
+ X(64, 128, 64, 6, 7, suffix) \
+ X(64, 64, 64, 6, 6, suffix) \
+ X(64, 32, 32, 6, 5, suffix) \
+ FOR_EACH_SUBPIX_VAR_64_128_RECT(X, suffix)
+
HWY_AFTER_NAMESPACE();
#endif // AOM_AOM_DSP_VARIANCE_HWY_H_
diff --git a/aom_dsp/x86/variance_hwy_avx512.cc b/aom_dsp/x86/variance_hwy_avx512.cc
index 8532c333bb..d9ab225131 100644
--- a/aom_dsp/x86/variance_hwy_avx512.cc
+++ b/aom_dsp/x86/variance_hwy_avx512.cc
@@ -18,3 +18,6 @@ FOR_EACH_VAR_BLOCK_SIZE(FVAR, avx512)
FMSE16X16(avx512)
FGET_BLK_SSE_SUM(avx512)
FCALC_VAR_STAT(avx512)
+
+FOR_EACH_SUBPIX_VAR_32XH(FSUBPIX_VAR_32XH, avx512)
+FOR_EACH_SUBPIX_VAR_64_128(FSUBPIX_VAR_64_128, avx512)
diff --git a/test/variance_test.cc b/test/variance_test.cc
index abf109491e..84d2185de0 100644
--- a/test/variance_test.cc
+++ b/test/variance_test.cc
@@ -3992,6 +3992,40 @@ INSTANTIATE_TEST_SUITE_P(
AVX512, CalcVarianceStatHbdTest,
::testing::Values(&aom_highbd_calc_variance_stat_avx512));
#endif // CONFIG_AV1_HIGHBITDEPTH
+
+const SubpelVarianceParams kArraySubpelVariance_avx512[] = {
+ SubpelVarianceParams(7, 7, &aom_sub_pixel_variance128x128_avx512, 0),
+ SubpelVarianceParams(7, 6, &aom_sub_pixel_variance128x64_avx512, 0),
+ SubpelVarianceParams(6, 7, &aom_sub_pixel_variance64x128_avx512, 0),
+ SubpelVarianceParams(6, 6, &aom_sub_pixel_variance64x64_avx512, 0),
+ SubpelVarianceParams(6, 5, &aom_sub_pixel_variance64x32_avx512, 0),
+ SubpelVarianceParams(5, 6, &aom_sub_pixel_variance32x64_avx512, 0),
+ SubpelVarianceParams(5, 5, &aom_sub_pixel_variance32x32_avx512, 0),
+ SubpelVarianceParams(5, 4, &aom_sub_pixel_variance32x16_avx512, 0),
+#if !CONFIG_REALTIME_ONLY
+ SubpelVarianceParams(6, 4, &aom_sub_pixel_variance64x16_avx512, 0),
+ SubpelVarianceParams(5, 3, &aom_sub_pixel_variance32x8_avx512, 0),
+#endif
+};
+INSTANTIATE_TEST_SUITE_P(AVX512, AvxSubpelVarianceTest,
+ ::testing::ValuesIn(kArraySubpelVariance_avx512));
+
+const SubpelAvgVarianceParams kArraySubpelAvgVariance_avx512[] = {
+ SubpelAvgVarianceParams(7, 7, &aom_sub_pixel_avg_variance128x128_avx512, 0),
+ SubpelAvgVarianceParams(7, 6, &aom_sub_pixel_avg_variance128x64_avx512, 0),
+ SubpelAvgVarianceParams(6, 7, &aom_sub_pixel_avg_variance64x128_avx512, 0),
+ SubpelAvgVarianceParams(6, 6, &aom_sub_pixel_avg_variance64x64_avx512, 0),
+ SubpelAvgVarianceParams(6, 5, &aom_sub_pixel_avg_variance64x32_avx512, 0),
+ SubpelAvgVarianceParams(5, 6, &aom_sub_pixel_avg_variance32x64_avx512, 0),
+ SubpelAvgVarianceParams(5, 5, &aom_sub_pixel_avg_variance32x32_avx512, 0),
+ SubpelAvgVarianceParams(5, 4, &aom_sub_pixel_avg_variance32x16_avx512, 0),
+#if !CONFIG_REALTIME_ONLY
+ SubpelAvgVarianceParams(6, 4, &aom_sub_pixel_avg_variance64x16_avx512, 0),
+ SubpelAvgVarianceParams(5, 3, &aom_sub_pixel_avg_variance32x8_avx512, 0),
+#endif
+};
+INSTANTIATE_TEST_SUITE_P(AVX512, AvxSubpelAvgVarianceTest,
+ ::testing::ValuesIn(kArraySubpelAvgVariance_avx512));
#endif // CONFIG_HIGHWAY && HAVE_AVX512
} // namespace