Commit 1713dda4ee for aom
commit 1713dda4ee47faeba50e231c7bc9bd9d55d37e8a
Author: Lin Zheng <linzhen@google.com>
Date: Sun Nov 8 23:00:42 2026 +0000
Highway: Add AVX512 hadamard, satd, and int_pro
Function │ AVX2 (ns) │ AVX512 (ns) │ Speedup (x)
───────────────────┼───────────┼─────────────┼────────────
hadamard_16x16 │ 31.49 │ 21.12 │ 1.49x
hadamard_lp_16x16 │ 23.92 │ 16.38 │ 1.46x
hadamard_32x32 │ 184.06 │ 107.02 │ 1.72x
satd (256) │ 19.06 │ 9.38 │ 2.03x
satd (1024) │ 62.11 │ 28.01 │ 2.22x
satd_lp (256) │ 11.27 │ 5.87 │ 1.92x
satd_lp (1024) │ 38.70 │ 21.93 │ 1.76x
int_pro_row 64x64 │ 76.46 │ 46.78 │ 1.63x
int_pro_col 64x64 │ 34.65 │ 27.17 │ 1.28x
Change-Id: Icc4e24f2a27ba9f7a363ce5ff38c90d4dee4eef8
diff --git a/aom_dsp/aom_dsp.cmake b/aom_dsp/aom_dsp.cmake
index 2c6ee2537a..1639cd1fce 100644
--- a/aom_dsp/aom_dsp.cmake
+++ b/aom_dsp/aom_dsp.cmake
@@ -186,7 +186,8 @@ if(CONFIG_AV1_ENCODER)
"${AOM_ROOT}/aom_dsp/variance.h")
if(CONFIG_HIGHWAY)
- list(APPEND AOM_DSP_ENCODER_SOURCES "${AOM_ROOT}/aom_dsp/reduce_sum_hwy.h"
+ list(APPEND AOM_DSP_ENCODER_SOURCES "${AOM_ROOT}/aom_dsp/avg_hwy.h"
+ "${AOM_ROOT}/aom_dsp/reduce_sum_hwy.h"
"${AOM_ROOT}/aom_dsp/sad_hwy.h"
"${AOM_ROOT}/aom_dsp/subtract_hwy.h"
"${AOM_ROOT}/aom_dsp/variance_hwy.h")
@@ -274,6 +275,7 @@ if(CONFIG_AV1_ENCODER)
list(APPEND AOM_DSP_ENCODER_INTRIN_AVX2
"${AOM_ROOT}/aom_dsp/x86/sad_hwy_avx2.cc")
list(APPEND AOM_DSP_ENCODER_INTRIN_AVX512
+ "${AOM_ROOT}/aom_dsp/x86/avg_hwy_avx512.cc"
"${AOM_ROOT}/aom_dsp/x86/sad_hwy_avx512.cc"
"${AOM_ROOT}/aom_dsp/x86/subtract_hwy_avx512.cc"
"${AOM_ROOT}/aom_dsp/x86/variance_hwy_avx512.cc")
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl
index 3c38010ef4..1d4826b059 100755
--- a/aom_dsp/aom_dsp_rtcd_defs.pl
+++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -1389,6 +1389,20 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/int aom_satd_lp/, "const int16_t *coeff, int length";
specialize qw/aom_satd_lp sse2 avx2 neon/;
+ if (aom_config("CONFIG_HIGHWAY") eq "yes") {
+ specialize qw/aom_int_pro_row avx512/;
+ specialize qw/aom_int_pro_col avx512/;
+ specialize qw/aom_hadamard_16x16 avx512/;
+ specialize qw/aom_hadamard_32x32 avx512/;
+ specialize qw/aom_hadamard_lp_16x16 avx512/;
+ if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
+ specialize qw/aom_highbd_hadamard_16x16 avx512/;
+ specialize qw/aom_highbd_hadamard_32x32 avx512/;
+ }
+ specialize qw/aom_satd avx512/;
+ specialize qw/aom_satd_lp avx512/;
+ }
+
#
# Structured Similarity (SSIM)
diff --git a/aom_dsp/avg_hwy.h b/aom_dsp/avg_hwy.h
new file mode 100644
index 0000000000..19293f9d94
--- /dev/null
+++ b/aom_dsp/avg_hwy.h
@@ -0,0 +1,616 @@
+/*
+ * Copyright (c) 2026, Alliance for Open Media. All rights reserved.
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#ifndef AOM_AOM_DSP_AVG_HWY_H_
+#define AOM_AOM_DSP_AVG_HWY_H_
+
+#include <assert.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include <type_traits>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+#include "aom/aom_integer.h"
+#include "third_party/highway/hwy/highway.h"
+
+HWY_BEFORE_NAMESPACE();
+
+namespace {
+namespace HWY_NAMESPACE {
+
+namespace hn = hwy::HWY_NAMESPACE;
+
+// Sign-extends 32 int16_t values from a 512-bit register into 32 tran_low_t
+// (int32_t) slots linearly.
+template <class D16>
+HWY_ATTR HWY_INLINE void StoreTranLow512(D16 d16, hn::VFromD<D16> a,
+ tran_low_t *b) {
+ (void)d16;
+ constexpr hn::Repartition<tran_low_t, D16> d32;
+ constexpr hn::Half<D16> d16_half;
+ const auto lo = hn::PromoteLowerTo(d32, a);
+ const auto hi = hn::PromoteTo(d32, hn::UpperHalf(d16_half, a));
+ hn::StoreU(lo, d32, b);
+ hn::StoreU(hi, d32, b + 16);
+}
+
+// 8-point 1D Hadamard butterfly across in[0..7] (int16_t or int32_t lanes).
+template <class D>
+HWY_ATTR HWY_INLINE void Hadamard8x8Butterfly(D d, hn::VFromD<D> *in) {
+ (void)d;
+ auto a0 = in[0];
+ auto a1 = in[1];
+ auto a2 = in[2];
+ auto a3 = in[3];
+ auto a4 = in[4];
+ auto a5 = in[5];
+ auto a6 = in[6];
+ auto a7 = in[7];
+
+ const auto b0 = hn::Add(a0, a1);
+ const auto b1 = hn::Sub(a0, a1);
+ const auto b2 = hn::Add(a2, a3);
+ const auto b3 = hn::Sub(a2, a3);
+ const auto b4 = hn::Add(a4, a5);
+ const auto b5 = hn::Sub(a4, a5);
+ const auto b6 = hn::Add(a6, a7);
+ const auto b7 = hn::Sub(a6, a7);
+
+ a0 = hn::Add(b0, b2);
+ a1 = hn::Add(b1, b3);
+ a2 = hn::Sub(b0, b2);
+ a3 = hn::Sub(b1, b3);
+ a4 = hn::Add(b4, b6);
+ a5 = hn::Add(b5, b7);
+ a6 = hn::Sub(b4, b6);
+ a7 = hn::Sub(b5, b7);
+
+ in[0] = hn::Add(a0, a4);
+ in[7] = hn::Add(a1, a5);
+ in[3] = hn::Add(a2, a6);
+ in[4] = hn::Add(a3, a7);
+ in[2] = hn::Sub(a0, a4);
+ in[6] = hn::Sub(a1, a5);
+ in[1] = hn::Sub(a2, a6);
+ in[5] = hn::Sub(a3, a7);
+}
+
+// Transposes 8x8 blocks in-place across in[0..7] (for 16-bit elements per
+// 128-bit lane or 32-bit elements per 256-bit half).
+template <class D>
+HWY_ATTR HWY_INLINE void Transpose8x8(D d, hn::VFromD<D> *in) {
+ using T = hn::TFromD<D>;
+ constexpr hn::RepartitionToWide<D> d_wide;
+ constexpr hn::Repartition<int64_t, D> d64;
+
+ const auto a0 = hn::InterleaveLower(d, in[0], in[1]);
+ const auto a1 = hn::InterleaveLower(d, in[2], in[3]);
+ const auto a2 = hn::InterleaveUpper(d, in[0], in[1]);
+ const auto a3 = hn::InterleaveUpper(d, in[2], in[3]);
+ const auto a4 = hn::InterleaveLower(d, in[4], in[5]);
+ const auto a5 = hn::InterleaveLower(d, in[6], in[7]);
+ const auto a6 = hn::InterleaveUpper(d, in[4], in[5]);
+ const auto a7 = hn::InterleaveUpper(d, in[6], in[7]);
+
+ const auto c0 =
+ hn::BitCast(d64, hn::InterleaveLower(d_wide, hn::BitCast(d_wide, a0),
+ hn::BitCast(d_wide, a1)));
+ const auto c1 =
+ hn::BitCast(d64, hn::InterleaveLower(d_wide, hn::BitCast(d_wide, a4),
+ hn::BitCast(d_wide, a5)));
+ const auto c2 =
+ hn::BitCast(d64, hn::InterleaveUpper(d_wide, hn::BitCast(d_wide, a0),
+ hn::BitCast(d_wide, a1)));
+ const auto c3 =
+ hn::BitCast(d64, hn::InterleaveUpper(d_wide, hn::BitCast(d_wide, a4),
+ hn::BitCast(d_wide, a5)));
+ const auto c4 =
+ hn::BitCast(d64, hn::InterleaveLower(d_wide, hn::BitCast(d_wide, a2),
+ hn::BitCast(d_wide, a3)));
+ const auto c5 =
+ hn::BitCast(d64, hn::InterleaveLower(d_wide, hn::BitCast(d_wide, a6),
+ hn::BitCast(d_wide, a7)));
+ const auto c6 =
+ hn::BitCast(d64, hn::InterleaveUpper(d_wide, hn::BitCast(d_wide, a2),
+ hn::BitCast(d_wide, a3)));
+ const auto c7 =
+ hn::BitCast(d64, hn::InterleaveUpper(d_wide, hn::BitCast(d_wide, a6),
+ hn::BitCast(d_wide, a7)));
+
+ HWY_IF_CONSTEXPR(sizeof(T) == 2) {
+ in[0] = hn::BitCast(d, hn::InterleaveLower(d64, c0, c1));
+ in[1] = hn::BitCast(d, hn::InterleaveUpper(d64, c0, c1));
+ in[2] = hn::BitCast(d, hn::InterleaveLower(d64, c2, c3));
+ in[3] = hn::BitCast(d, hn::InterleaveUpper(d64, c2, c3));
+ in[4] = hn::BitCast(d, hn::InterleaveLower(d64, c4, c5));
+ in[5] = hn::BitCast(d, hn::InterleaveUpper(d64, c4, c5));
+ in[6] = hn::BitCast(d, hn::InterleaveLower(d64, c6, c7));
+ in[7] = hn::BitCast(d, hn::InterleaveUpper(d64, c6, c7));
+ }
+ HWY_IF_CONSTEXPR(sizeof(T) == 4) {
+ HWY_ALIGN static constexpr int64_t kIdxLo[8] = { 0, 1, 8, 9, 4, 5, 12, 13 };
+ HWY_ALIGN static constexpr int64_t kIdxHi[8] = {
+ 2, 3, 10, 11, 6, 7, 14, 15
+ };
+ const auto idx_lo = hn::SetTableIndices(d64, kIdxLo);
+ const auto idx_hi = hn::SetTableIndices(d64, kIdxHi);
+ in[0] = hn::BitCast(d, hn::TwoTablesLookupLanes(d64, c0, c1, idx_lo));
+ in[1] = hn::BitCast(d, hn::TwoTablesLookupLanes(d64, c0, c1, idx_hi));
+ in[2] = hn::BitCast(d, hn::TwoTablesLookupLanes(d64, c2, c3, idx_lo));
+ in[3] = hn::BitCast(d, hn::TwoTablesLookupLanes(d64, c2, c3, idx_hi));
+ in[4] = hn::BitCast(d, hn::TwoTablesLookupLanes(d64, c4, c5, idx_lo));
+ in[5] = hn::BitCast(d, hn::TwoTablesLookupLanes(d64, c4, c5, idx_hi));
+ in[6] = hn::BitCast(d, hn::TwoTablesLookupLanes(d64, c6, c7, idx_lo));
+ in[7] = hn::BitCast(d, hn::TwoTablesLookupLanes(d64, c6, c7, idx_hi));
+ }
+}
+
+template <int Shift, class D>
+HWY_ATTR HWY_INLINE void Hadamard2x2Butterfly(
+ D d, hn::VFromD<D> c0, hn::VFromD<D> c1, hn::VFromD<D> c2, hn::VFromD<D> c3,
+ hn::VFromD<D> *out0, hn::VFromD<D> *out1, hn::VFromD<D> *out2,
+ hn::VFromD<D> *out3) {
+ (void)d;
+ const auto b0 = hn::ShiftRight<Shift>(hn::Add(c0, c1));
+ const auto b1 = hn::ShiftRight<Shift>(hn::Sub(c0, c1));
+ const auto b2 = hn::ShiftRight<Shift>(hn::Add(c2, c3));
+ const auto b3 = hn::ShiftRight<Shift>(hn::Sub(c2, c3));
+
+ *out0 = hn::Add(b0, b2);
+ *out1 = hn::Add(b1, b3);
+ *out2 = hn::Sub(b0, b2);
+ *out3 = hn::Sub(b1, b3);
+}
+
+// Transposes 4x4 128-bit lanes across 4 ZMM registers so coeff0..coeff3 hold 32
+// int16_t coefficients from quadrants 0..3 respectively (optionally swapping
+// [4..7] with [8..11] in each 16-element group when DoShift is true to match
+// aom_hadamard_16x16_c / aom_hadamard_16x16_avx2), then runs the 2x2
+// inter-quadrant Hadamard butterfly in-register.
+template <bool DoShift, class D16>
+HWY_ATTR HWY_INLINE void Hadamard16x16QuadrantButterfly512(
+ D16 d16, hn::VFromD<D16> s0, hn::VFromD<D16> s1, hn::VFromD<D16> s2,
+ hn::VFromD<D16> s3, hn::VFromD<D16> *out0, hn::VFromD<D16> *out1,
+ hn::VFromD<D16> *out2, hn::VFromD<D16> *out3) {
+ constexpr hn::Repartition<int64_t, D16> d64;
+ HWY_ALIGN static constexpr int64_t kEvenBlocks[8] = {
+ 0, DoShift ? 4 : 1, DoShift ? 1 : 4, 5,
+ 8 + 0, DoShift ? 8 + 4 : 8 + 1, DoShift ? 8 + 1 : 8 + 4, 8 + 5
+ };
+ HWY_ALIGN static constexpr int64_t kOddBlocks[8] = {
+ 2, DoShift ? 6 : 3, DoShift ? 3 : 6, 7,
+ 8 + 2, DoShift ? 8 + 6 : 8 + 3, DoShift ? 8 + 3 : 8 + 6, 8 + 7
+ };
+ const auto idx_even = hn::SetTableIndices(d64, kEvenBlocks);
+ const auto idx_odd = hn::SetTableIndices(d64, kOddBlocks);
+
+ const auto t0 = hn::BitCast(d64, hn::ConcatLowerLower(d16, s1, s0));
+ const auto t1 = hn::BitCast(d64, hn::ConcatLowerLower(d16, s3, s2));
+ const auto t2 = hn::BitCast(d64, hn::ConcatUpperUpper(d16, s1, s0));
+ const auto t3 = hn::BitCast(d64, hn::ConcatUpperUpper(d16, s3, s2));
+
+ const auto coeff0 =
+ hn::BitCast(d16, hn::TwoTablesLookupLanes(d64, t0, t1, idx_even));
+ const auto coeff1 =
+ hn::BitCast(d16, hn::TwoTablesLookupLanes(d64, t0, t1, idx_odd));
+ const auto coeff2 =
+ hn::BitCast(d16, hn::TwoTablesLookupLanes(d64, t2, t3, idx_even));
+ const auto coeff3 =
+ hn::BitCast(d16, hn::TwoTablesLookupLanes(d64, t2, t3, idx_odd));
+
+ Hadamard2x2Butterfly<1>(d16, coeff0, coeff1, coeff2, coeff3, out0, out1, out2,
+ out3);
+}
+
+template <bool DoShift, typename CoeffType>
+HWY_ATTR HWY_INLINE void Hadamard16x16Impl(const int16_t *src_diff,
+ ptrdiff_t src_stride,
+ CoeffType *coeff) {
+ constexpr hn::ScalableTag<int16_t> d16;
+ constexpr hn::Half<decltype(d16)> d16_half;
+ hn::VFromD<decltype(d16)> src[8];
+ const int16_t *src_top = src_diff;
+ const int16_t *src_bot = src_diff + 8 * src_stride;
+
+ for (int i = 0; i < 8; ++i) {
+ const auto top = hn::LoadU(d16_half, src_top);
+ const auto bot = hn::LoadU(d16_half, src_bot);
+ src[i] = hn::Combine(d16, bot, top);
+ src_top += src_stride;
+ src_bot += src_stride;
+ }
+
+ Hadamard8x8Butterfly(d16, src);
+ Transpose8x8(d16, src);
+ Hadamard8x8Butterfly(d16, src);
+
+ for (int half = 0; half < 2; ++half) {
+ hn::VFromD<decltype(d16)> out0, out1, out2, out3;
+ const auto *s = src + 4 * half;
+ CoeffType *c = coeff + 32 * half;
+ Hadamard16x16QuadrantButterfly512<DoShift>(d16, s[0], s[1], s[2], s[3],
+ &out0, &out1, &out2, &out3);
+ HWY_IF_CONSTEXPR(std::is_same<CoeffType, tran_low_t>::value) {
+ StoreTranLow512(d16, out0, c);
+ StoreTranLow512(d16, out1, c + 64);
+ StoreTranLow512(d16, out2, c + 128);
+ StoreTranLow512(d16, out3, c + 192);
+ }
+ HWY_IF_CONSTEXPR(std::is_same<CoeffType, int16_t>::value) {
+ hn::StoreU(out0, d16, c);
+ hn::StoreU(out1, d16, c + 64);
+ hn::StoreU(out2, d16, c + 128);
+ hn::StoreU(out3, d16, c + 192);
+ }
+ }
+}
+
+HWY_ATTR HWY_INLINE void Hadamard16x16(const int16_t *src_diff,
+ ptrdiff_t src_stride,
+ tran_low_t *coeff) {
+ Hadamard16x16Impl<true>(src_diff, src_stride, coeff);
+}
+
+HWY_ATTR HWY_INLINE void HadamardLp16x16(const int16_t *src_diff,
+ ptrdiff_t src_stride, int16_t *coeff) {
+ Hadamard16x16Impl<false>(src_diff, src_stride, coeff);
+}
+
+HWY_ATTR HWY_INLINE void Hadamard32x32(const int16_t *src_diff,
+ ptrdiff_t src_stride,
+ tran_low_t *coeff) {
+ constexpr hn::ScalableTag<int32_t> d32;
+ constexpr hn::Rebind<int16_t, decltype(d32)> d16_half;
+ HWY_ALIGN int16_t temp_coeff[32 * 32];
+ int16_t *t_coeff = temp_coeff;
+ for (int idx = 0; idx < 4; ++idx) {
+ const int16_t *src_ptr =
+ src_diff + (idx >> 1) * 16 * src_stride + (idx & 0x01) * 16;
+ Hadamard16x16Impl<true>(src_ptr, src_stride, t_coeff + idx * 256);
+ }
+
+ for (int idx = 0; idx < 256; idx += 16) {
+ const auto coeff0 = hn::PromoteTo(d32, hn::Load(d16_half, t_coeff));
+ const auto coeff1 = hn::PromoteTo(d32, hn::Load(d16_half, t_coeff + 256));
+ const auto coeff2 = hn::PromoteTo(d32, hn::Load(d16_half, t_coeff + 512));
+ const auto coeff3 = hn::PromoteTo(d32, hn::Load(d16_half, t_coeff + 768));
+
+ hn::VFromD<decltype(d32)> out0, out1, out2, out3;
+ Hadamard2x2Butterfly<2>(d32, coeff0, coeff1, coeff2, coeff3, &out0, &out1,
+ &out2, &out3);
+
+ hn::StoreU(out0, d32, coeff);
+ hn::StoreU(out1, d32, coeff + 256);
+ hn::StoreU(out2, d32, coeff + 512);
+ hn::StoreU(out3, d32, coeff + 768);
+
+ coeff += 16;
+ t_coeff += 16;
+ }
+}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+// Processes two 8x8 32-bit Hadamard blocks simultaneously (block 0 in the low
+// 256-bit half, block 1 in the high 256-bit half of src32[0..7]).
+HWY_ATTR HWY_INLINE void HighbdHadamard8x8Dual512(const int16_t *src_diff,
+ ptrdiff_t src_stride,
+ tran_low_t *coeff) {
+ constexpr hn::ScalableTag<int32_t> d32;
+ constexpr hn::Rebind<int16_t, decltype(d32)> d16_half;
+ hn::VFromD<decltype(d32)> src32[8];
+ for (int i = 0; i < 8; ++i) {
+ src32[i] = hn::PromoteTo(d32, hn::LoadU(d16_half, src_diff));
+ src_diff += src_stride;
+ }
+
+ Hadamard8x8Butterfly(d32, src32);
+ Transpose8x8(d32, src32);
+ Hadamard8x8Butterfly(d32, src32);
+
+ hn::StoreU(hn::ConcatLowerLower(d32, src32[1], src32[0]), d32, coeff + 0);
+ hn::StoreU(hn::ConcatLowerLower(d32, src32[3], src32[2]), d32, coeff + 16);
+ hn::StoreU(hn::ConcatLowerLower(d32, src32[5], src32[4]), d32, coeff + 32);
+ hn::StoreU(hn::ConcatLowerLower(d32, src32[7], src32[6]), d32, coeff + 48);
+ hn::StoreU(hn::ConcatUpperUpper(d32, src32[1], src32[0]), d32, coeff + 64);
+ hn::StoreU(hn::ConcatUpperUpper(d32, src32[3], src32[2]), d32, coeff + 80);
+ hn::StoreU(hn::ConcatUpperUpper(d32, src32[5], src32[4]), d32, coeff + 96);
+ hn::StoreU(hn::ConcatUpperUpper(d32, src32[7], src32[6]), d32, coeff + 112);
+}
+
+HWY_ATTR HWY_INLINE void HighbdHadamard16x16(const int16_t *src_diff,
+ ptrdiff_t src_stride,
+ tran_low_t *coeff) {
+ constexpr hn::ScalableTag<int32_t> d32;
+ tran_low_t *t_coeff = coeff;
+ HighbdHadamard8x8Dual512(src_diff, src_stride, t_coeff);
+ HighbdHadamard8x8Dual512(src_diff + 8 * src_stride, src_stride,
+ t_coeff + 128);
+
+ for (int idx = 0; idx < 64; idx += 16) {
+ const auto coeff0 = hn::LoadU(d32, t_coeff);
+ const auto coeff1 = hn::LoadU(d32, t_coeff + 64);
+ const auto coeff2 = hn::LoadU(d32, t_coeff + 128);
+ const auto coeff3 = hn::LoadU(d32, t_coeff + 192);
+
+ hn::VFromD<decltype(d32)> out0, out1, out2, out3;
+ Hadamard2x2Butterfly<1>(d32, coeff0, coeff1, coeff2, coeff3, &out0, &out1,
+ &out2, &out3);
+
+ hn::StoreU(out0, d32, coeff);
+ hn::StoreU(out1, d32, coeff + 64);
+ hn::StoreU(out2, d32, coeff + 128);
+ hn::StoreU(out3, d32, coeff + 192);
+
+ coeff += 16;
+ t_coeff += 16;
+ }
+}
+
+HWY_ATTR HWY_INLINE void HighbdHadamard32x32(const int16_t *src_diff,
+ ptrdiff_t src_stride,
+ tran_low_t *coeff) {
+ constexpr hn::ScalableTag<int32_t> d32;
+ tran_low_t *t_coeff = coeff;
+ for (int idx = 0; idx < 4; ++idx) {
+ const int16_t *src_ptr =
+ src_diff + (idx >> 1) * 16 * src_stride + (idx & 0x01) * 16;
+ HighbdHadamard16x16(src_ptr, src_stride, t_coeff + idx * 256);
+ }
+
+ for (int idx = 0; idx < 256; idx += 16) {
+ const auto coeff0 = hn::LoadU(d32, t_coeff);
+ const auto coeff1 = hn::LoadU(d32, t_coeff + 256);
+ const auto coeff2 = hn::LoadU(d32, t_coeff + 512);
+ const auto coeff3 = hn::LoadU(d32, t_coeff + 768);
+
+ hn::VFromD<decltype(d32)> out0, out1, out2, out3;
+ Hadamard2x2Butterfly<2>(d32, coeff0, coeff1, coeff2, coeff3, &out0, &out1,
+ &out2, &out3);
+
+ hn::StoreU(out0, d32, coeff);
+ hn::StoreU(out1, d32, coeff + 256);
+ hn::StoreU(out2, d32, coeff + 512);
+ hn::StoreU(out3, d32, coeff + 768);
+
+ coeff += 16;
+ t_coeff += 16;
+ }
+}
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
+HWY_ATTR HWY_INLINE int Satd(const tran_low_t *coeff, int length) {
+ constexpr hn::ScalableTag<int32_t> d32;
+ constexpr int kLanes = static_cast<int>(hn::MaxLanes(d32));
+ auto accum0 = hn::Zero(d32);
+ auto accum1 = hn::Zero(d32);
+ int i = 0;
+ for (; i + 2 * kLanes <= length; i += 2 * kLanes, coeff += 2 * kLanes) {
+ const auto s0 = hn::LoadU(d32, coeff);
+ const auto s1 = hn::LoadU(d32, coeff + kLanes);
+ accum0 = hn::Add(accum0, hn::Abs(s0));
+ accum1 = hn::Add(accum1, hn::Abs(s1));
+ }
+ if (i < length) {
+ const auto s0 = hn::LoadU(d32, coeff);
+ accum0 = hn::Add(accum0, hn::Abs(s0));
+ }
+ accum0 = hn::Add(accum0, accum1);
+ return static_cast<int>(hn::ReduceSum(d32, accum0));
+}
+
+HWY_ATTR HWY_INLINE int SatdLp(const int16_t *coeff, int length) {
+ constexpr hn::ScalableTag<int16_t> d16;
+ constexpr int kLanes = static_cast<int>(hn::MaxLanes(d16));
+ if (length < kLanes) {
+ return aom_satd_lp_avx2(coeff, length);
+ }
+ constexpr hn::Repartition<int32_t, decltype(d16)> d32;
+ const auto one = hn::Set(d16, static_cast<int16_t>(1));
+ auto accum0 = hn::Zero(d32);
+ auto accum1 = hn::Zero(d32);
+ auto accum2 = hn::Zero(d32);
+ auto accum3 = hn::Zero(d32);
+ int i = 0;
+ for (; i + 2 * kLanes <= length; i += 2 * kLanes, coeff += 2 * kLanes) {
+ const auto s0 = hn::Abs(hn::LoadU(d16, coeff));
+ const auto s1 = hn::Abs(hn::LoadU(d16, coeff + kLanes));
+ accum0 = hn::ReorderWidenMulAccumulate(d32, s0, one, accum0, accum1);
+ accum2 = hn::ReorderWidenMulAccumulate(d32, s1, one, accum2, accum3);
+ }
+ if (i < length) {
+ const auto s0 = hn::Abs(hn::LoadU(d16, coeff));
+ accum0 = hn::ReorderWidenMulAccumulate(d32, s0, one, accum0, accum1);
+ }
+ const auto sum01 = hn::Add(accum0, accum1);
+ const auto sum23 = hn::Add(accum2, accum3);
+ return static_cast<int>(hn::ReduceSum(d32, hn::Add(sum01, sum23)));
+}
+
+HWY_ATTR HWY_INLINE void IntProRow(int16_t *hbuf, const uint8_t *ref,
+ const int ref_stride, const int width,
+ const int height, int norm_factor) {
+ assert(width % 16 == 0 && height % 2 == 0);
+ constexpr hn::ScalableTag<int16_t> d16;
+ constexpr hn::Rebind<uint8_t, decltype(d16)> d8_half;
+ int wd = 0;
+ for (; wd + 64 <= width; wd += 64) {
+ const uint8_t *ref_tmp = ref + wd;
+ auto s0 = hn::Zero(d16);
+ auto s1 = hn::Zero(d16);
+ for (int idx = 0; idx < height; idx += 2) {
+ const auto r0_lo = hn::LoadU(d8_half, ref_tmp);
+ const auto r0_hi = hn::LoadU(d8_half, ref_tmp + 32);
+ ref_tmp += ref_stride;
+ const auto r1_lo = hn::LoadU(d8_half, ref_tmp);
+ const auto r1_hi = hn::LoadU(d8_half, ref_tmp + 32);
+ ref_tmp += ref_stride;
+
+ s0 = hn::Add(s0, hn::PromoteTo(d16, r0_lo));
+ s1 = hn::Add(s1, hn::PromoteTo(d16, r0_hi));
+ s0 = hn::Add(s0, hn::PromoteTo(d16, r1_lo));
+ s1 = hn::Add(s1, hn::PromoteTo(d16, r1_hi));
+ }
+ s0 = hn::ShiftRightSame(s0, norm_factor);
+ s1 = hn::ShiftRightSame(s1, norm_factor);
+ hn::StoreU(s0, d16, hbuf + wd);
+ hn::StoreU(s1, d16, hbuf + wd + 32);
+ }
+ if (wd + 32 <= width) {
+ const uint8_t *ref_tmp = ref + wd;
+ auto s0 = hn::Zero(d16);
+ for (int idx = 0; idx < height; idx += 2) {
+ const auto r0 = hn::LoadU(d8_half, ref_tmp);
+ ref_tmp += ref_stride;
+ const auto r1 = hn::LoadU(d8_half, ref_tmp);
+ ref_tmp += ref_stride;
+ s0 = hn::Add(s0, hn::PromoteTo(d16, r0));
+ s0 = hn::Add(s0, hn::PromoteTo(d16, r1));
+ }
+ s0 = hn::ShiftRightSame(s0, norm_factor);
+ hn::StoreU(s0, d16, hbuf + wd);
+ wd += 32;
+ }
+ if (wd < width) {
+ aom_int_pro_row_sse2(hbuf + wd, ref + wd, ref_stride, width - wd, height,
+ norm_factor);
+ }
+}
+
+template <class D16>
+HWY_ATTR HWY_INLINE void CalcTotSadAndStore512(D16 d16, hn::VFromD<D16> r0,
+ hn::VFromD<D16> r1,
+ hn::VFromD<D16> r2,
+ hn::VFromD<D16> r3,
+ int norm_factor, int16_t *vbuf) {
+ constexpr hn::Half<D16> d16_256;
+ constexpr hn::Half<decltype(d16_256)> d16_128;
+ constexpr hn::Half<decltype(d16_128)> d16_64;
+ constexpr hn::RebindToUnsigned<decltype(d16_128)> du16_128;
+ const auto r01 = hn::Add(hn::ShiftLeftBytes<2>(d16, r1), r0);
+ const auto r012 = hn::Add(hn::ShiftLeftBytes<4>(d16, r2), r01);
+ const auto result0 = hn::Add(hn::ShiftLeftBytes<6>(d16, r3), r012);
+ const auto res256 =
+ hn::Add(hn::LowerHalf(d16_256, result0), hn::UpperHalf(d16_256, result0));
+ const auto res128 =
+ hn::Add(hn::LowerHalf(d16_128, res256), hn::UpperHalf(d16_128, res256));
+ const auto results1 =
+ hn::Add(res128, hn::ShiftRightBytes<8>(d16_128, res128));
+ const auto shifted = hn::BitCast(
+ d16_128,
+ hn::ShiftRightSame(hn::BitCast(du16_128, results1), norm_factor));
+ hn::StoreU(hn::LowerHalf(d16_64, shifted), d16_64, vbuf);
+}
+
+HWY_ATTR HWY_INLINE void IntProCol(int16_t *vbuf, const uint8_t *ref,
+ const int ref_stride, const int width,
+ const int height, int norm_factor) {
+ assert(width % 16 == 0 && height % 4 == 0);
+ constexpr hn::ScalableTag<uint8_t> d8;
+ constexpr hn::Repartition<int16_t, decltype(d8)> d16;
+ if (width == 128) {
+ const auto zero = hn::Zero(d8);
+ for (int ht = 0; ht < height; ht += 4) {
+ const auto s0_0 = hn::LoadU(d8, ref);
+ const auto s0_1 = hn::LoadU(d8, ref + 64);
+ const auto s1_0 = hn::LoadU(d8, ref + ref_stride);
+ const auto s1_1 = hn::LoadU(d8, ref + ref_stride + 64);
+ const auto s2_0 = hn::LoadU(d8, ref + 2 * ref_stride);
+ const auto s2_1 = hn::LoadU(d8, ref + 2 * ref_stride + 64);
+ const auto s3_0 = hn::LoadU(d8, ref + 3 * ref_stride);
+ const auto s3_1 = hn::LoadU(d8, ref + 3 * ref_stride + 64);
+
+ const auto r0 = hn::Add(hn::BitCast(d16, hn::SumsOf8AbsDiff(s0_0, zero)),
+ hn::BitCast(d16, hn::SumsOf8AbsDiff(s0_1, zero)));
+ const auto r1 = hn::Add(hn::BitCast(d16, hn::SumsOf8AbsDiff(s1_0, zero)),
+ hn::BitCast(d16, hn::SumsOf8AbsDiff(s1_1, zero)));
+ const auto r2 = hn::Add(hn::BitCast(d16, hn::SumsOf8AbsDiff(s2_0, zero)),
+ hn::BitCast(d16, hn::SumsOf8AbsDiff(s2_1, zero)));
+ const auto r3 = hn::Add(hn::BitCast(d16, hn::SumsOf8AbsDiff(s3_0, zero)),
+ hn::BitCast(d16, hn::SumsOf8AbsDiff(s3_1, zero)));
+
+ CalcTotSadAndStore512(d16, r0, r1, r2, r3, norm_factor, vbuf);
+ vbuf += 4;
+ ref += ref_stride << 2;
+ }
+ } else if (width == 64) {
+ const auto zero = hn::Zero(d8);
+ for (int ht = 0; ht < height; ht += 4) {
+ const auto s0 = hn::LoadU(d8, ref);
+ const auto s1 = hn::LoadU(d8, ref + ref_stride);
+ const auto s2 = hn::LoadU(d8, ref + 2 * ref_stride);
+ const auto s3 = hn::LoadU(d8, ref + 3 * ref_stride);
+
+ const auto r0 = hn::BitCast(d16, hn::SumsOf8AbsDiff(s0, zero));
+ const auto r1 = hn::BitCast(d16, hn::SumsOf8AbsDiff(s1, zero));
+ const auto r2 = hn::BitCast(d16, hn::SumsOf8AbsDiff(s2, zero));
+ const auto r3 = hn::BitCast(d16, hn::SumsOf8AbsDiff(s3, zero));
+
+ CalcTotSadAndStore512(d16, r0, r1, r2, r3, norm_factor, vbuf);
+ vbuf += 4;
+ ref += ref_stride << 2;
+ }
+ } else {
+ aom_int_pro_col_avx2(vbuf, ref, ref_stride, width, height, norm_factor);
+ }
+}
+
+} // namespace HWY_NAMESPACE
+} // namespace
+
+HWY_AFTER_NAMESPACE();
+
+#define MAKE_HADAMARD(w, h, suffix) \
+ extern "C" HWY_ATTR void aom_hadamard_##w##x##h##_##suffix( \
+ const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff) { \
+ HWY_NAMESPACE::Hadamard##w##x##h(src_diff, src_stride, coeff); \
+ }
+
+#define MAKE_HADAMARD_LP(w, h, suffix) \
+ extern "C" HWY_ATTR void aom_hadamard_lp_##w##x##h##_##suffix( \
+ const int16_t *src_diff, ptrdiff_t src_stride, int16_t *coeff) { \
+ HWY_NAMESPACE::HadamardLp##w##x##h(src_diff, src_stride, coeff); \
+ }
+
+#define MAKE_SATD_INT_PRO_FUNCS(suffix) \
+ extern "C" HWY_ATTR int aom_satd_##suffix(const tran_low_t *coeff, \
+ int length) { \
+ return HWY_NAMESPACE::Satd(coeff, length); \
+ } \
+ extern "C" HWY_ATTR int aom_satd_lp_##suffix(const int16_t *coeff, \
+ int length) { \
+ return HWY_NAMESPACE::SatdLp(coeff, length); \
+ } \
+ extern "C" HWY_ATTR void aom_int_pro_row_##suffix( \
+ int16_t *hbuf, const uint8_t *ref, const int ref_stride, \
+ const int width, const int height, int norm_factor) { \
+ HWY_NAMESPACE::IntProRow(hbuf, ref, ref_stride, width, height, \
+ norm_factor); \
+ } \
+ extern "C" HWY_ATTR void aom_int_pro_col_##suffix( \
+ int16_t *vbuf, const uint8_t *ref, const int ref_stride, \
+ const int width, const int height, int norm_factor) { \
+ HWY_NAMESPACE::IntProCol(vbuf, ref, ref_stride, width, height, \
+ norm_factor); \
+ }
+
+#if CONFIG_AV1_HIGHBITDEPTH
+#define MAKE_HIGHBD_HADAMARD(w, h, suffix) \
+ extern "C" HWY_ATTR void aom_highbd_hadamard_##w##x##h##_##suffix( \
+ const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff) { \
+ HWY_NAMESPACE::HighbdHadamard##w##x##h(src_diff, src_stride, coeff); \
+ }
+#else
+#define MAKE_HIGHBD_HADAMARD(w, h, suffix)
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
+#endif // AOM_AOM_DSP_AVG_HWY_H_
diff --git a/aom_dsp/x86/avg_hwy_avx512.cc b/aom_dsp/x86/avg_hwy_avx512.cc
new file mode 100644
index 0000000000..0498824a52
--- /dev/null
+++ b/aom_dsp/x86/avg_hwy_avx512.cc
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2026, Alliance for Open Media. All rights reserved.
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#define HWY_BASELINE_TARGETS HWY_AVX3_DL
+#define HWY_BROKEN_32BIT 0
+
+#include "aom_dsp/avg_hwy.h"
+
+MAKE_HADAMARD(16, 16, avx512)
+MAKE_HADAMARD(32, 32, avx512)
+MAKE_HADAMARD_LP(16, 16, avx512)
+MAKE_SATD_INT_PRO_FUNCS(avx512)
+
+MAKE_HIGHBD_HADAMARD(16, 16, avx512)
+MAKE_HIGHBD_HADAMARD(32, 32, avx512)
diff --git a/test/avg_test.cc b/test/avg_test.cc
index d2b4dbcbba..37cbdf7cfb 100644
--- a/test/avg_test.cc
+++ b/test/avg_test.cc
@@ -800,6 +800,28 @@ INSTANTIATE_TEST_SUITE_P(
make_tuple(128, 128, &aom_int_pro_col_avx2, &aom_int_pro_col_c)));
#endif
+#if CONFIG_HIGHWAY && HAVE_AVX512
+INSTANTIATE_TEST_SUITE_P(
+ AVX512, IntProRowTest,
+ ::testing::Values(
+ make_tuple(16, 16, &aom_int_pro_row_avx512, &aom_int_pro_row_c),
+ make_tuple(32, 32, &aom_int_pro_row_avx512, &aom_int_pro_row_c),
+ make_tuple(48, 32, &aom_int_pro_row_avx512, &aom_int_pro_row_c),
+ make_tuple(64, 64, &aom_int_pro_row_avx512, &aom_int_pro_row_c),
+ make_tuple(80, 64, &aom_int_pro_row_avx512, &aom_int_pro_row_c),
+ make_tuple(96, 64, &aom_int_pro_row_avx512, &aom_int_pro_row_c),
+ make_tuple(112, 64, &aom_int_pro_row_avx512, &aom_int_pro_row_c),
+ make_tuple(128, 128, &aom_int_pro_row_avx512, &aom_int_pro_row_c)));
+
+INSTANTIATE_TEST_SUITE_P(
+ AVX512, IntProColTest,
+ ::testing::Values(
+ make_tuple(16, 16, &aom_int_pro_col_avx512, &aom_int_pro_col_c),
+ make_tuple(32, 32, &aom_int_pro_col_avx512, &aom_int_pro_col_c),
+ make_tuple(64, 64, &aom_int_pro_col_avx512, &aom_int_pro_col_c),
+ make_tuple(128, 128, &aom_int_pro_col_avx512, &aom_int_pro_col_c)));
+#endif // CONFIG_HIGHWAY && HAVE_AVX512
+
#if HAVE_NEON
INSTANTIATE_TEST_SUITE_P(
NEON, AverageTest8bpp,
@@ -1058,6 +1080,16 @@ INSTANTIATE_TEST_SUITE_P(
make_tuple(5, &aom_vector_var_c, &aom_vector_var_avx2)));
#endif // HAVE_AVX2
+#if CONFIG_HIGHWAY && HAVE_AVX512
+INSTANTIATE_TEST_SUITE_P(
+ AVX512, SatdTest,
+ ::testing::Values(
+ SatdTestParam<SatdFunc>(16, &aom_satd_c, &aom_satd_avx512),
+ SatdTestParam<SatdFunc>(64, &aom_satd_c, &aom_satd_avx512),
+ SatdTestParam<SatdFunc>(256, &aom_satd_c, &aom_satd_avx512),
+ SatdTestParam<SatdFunc>(1024, &aom_satd_c, &aom_satd_avx512)));
+#endif // CONFIG_HIGHWAY && HAVE_AVX512
+
#if HAVE_SSE2
INSTANTIATE_TEST_SUITE_P(
SSE2, SatdTest,
@@ -1138,6 +1170,16 @@ INSTANTIATE_TEST_SUITE_P(
SatdTestParam<SatdLpFunc>(1024, &aom_satd_lp_c, &aom_satd_lp_avx2)));
#endif
+#if CONFIG_HIGHWAY && HAVE_AVX512
+INSTANTIATE_TEST_SUITE_P(
+ AVX512, SatdLpTest,
+ ::testing::Values(
+ SatdTestParam<SatdLpFunc>(16, &aom_satd_lp_c, &aom_satd_lp_avx512),
+ SatdTestParam<SatdLpFunc>(64, &aom_satd_lp_c, &aom_satd_lp_avx512),
+ SatdTestParam<SatdLpFunc>(256, &aom_satd_lp_c, &aom_satd_lp_avx512),
+ SatdTestParam<SatdLpFunc>(1024, &aom_satd_lp_c, &aom_satd_lp_avx512)));
+#endif // CONFIG_HIGHWAY && HAVE_AVX512
+
#if HAVE_SSE2
INSTANTIATE_TEST_SUITE_P(
SSE2, SatdLpTest,
diff --git a/test/hadamard_test.cc b/test/hadamard_test.cc
index b867d0a91b..ea6775c320 100644
--- a/test/hadamard_test.cc
+++ b/test/hadamard_test.cc
@@ -393,6 +393,14 @@ INSTANTIATE_TEST_SUITE_P(
HadamardFuncWithSize(&aom_hadamard_32x32_avx2, 32, 32)));
#endif // HAVE_AVX2
+#if CONFIG_HIGHWAY && HAVE_AVX512
+INSTANTIATE_TEST_SUITE_P(
+ AVX512, HadamardLowbdTest,
+ ::testing::Values(HadamardFuncWithSize(&aom_hadamard_16x16_avx512, 16, 16),
+ HadamardFuncWithSize(&aom_hadamard_32x32_avx512, 32,
+ 32)));
+#endif // CONFIG_HIGHWAY && HAVE_AVX512
+
// TODO(aomedia:3314): Disable NEON unit test for now, since hadamard 16x16 NEON
// need modifications to match C/AVX2 behavior.
#if HAVE_NEON
@@ -442,6 +450,14 @@ INSTANTIATE_TEST_SUITE_P(
HadamardFuncWithSize(&aom_highbd_hadamard_32x32_avx2, 32, 32)));
#endif // HAVE_AVX2
+#if CONFIG_HIGHWAY && HAVE_AVX512
+INSTANTIATE_TEST_SUITE_P(
+ AVX512, HadamardHighbdTest,
+ ::testing::Values(
+ HadamardFuncWithSize(&aom_highbd_hadamard_16x16_avx512, 16, 16),
+ HadamardFuncWithSize(&aom_highbd_hadamard_32x32_avx512, 32, 32)));
+#endif // CONFIG_HIGHWAY && HAVE_AVX512
+
#if HAVE_NEON
INSTANTIATE_TEST_SUITE_P(
NEON, HadamardHighbdTest,
@@ -493,6 +509,12 @@ INSTANTIATE_TEST_SUITE_P(AVX2, HadamardLowbdLPTest,
&aom_hadamard_lp_16x16_avx2, 16, 16)));
#endif // HAVE_AVX2
+#if CONFIG_HIGHWAY && HAVE_AVX512
+INSTANTIATE_TEST_SUITE_P(AVX512, HadamardLowbdLPTest,
+ ::testing::Values(HadamardLPFuncWithSize(
+ &aom_hadamard_lp_16x16_avx512, 16, 16)));
+#endif // CONFIG_HIGHWAY && HAVE_AVX512
+
#if HAVE_NEON
INSTANTIATE_TEST_SUITE_P(
NEON, HadamardLowbdLPTest,