Commit a52c0da614 for aom
commit a52c0da6147f60305a5a3c30f2d33ce5998a9f6a
Author: Lin Zheng <linzhen@google.com>
Date: Sun Nov 8 23:00:32 2026 +0000
Highway: Add AVX512 aom_subtract_block
Use 256-bit vectors (`CappedTag<int16_t, 16>`) for `Cols == 128` to
avoid 64-byte split-cache-line stores when `diff_ptr` is 32-byte
aligned.
Block Size │ AVX2 (ns) │ AVX512 (ns) │ Speedup (x)
───────────┼───────────┼─────────────┼────────────
4x4 │ 16.62 │ 9.71 │ 1.71x
4x8 │ 19.63 │ 13.15 │ 1.49x
4x16 │ 23.06 │ 20.38 │ 1.13x
8x4 │ 7.72 │ 7.41 │ 1.04x
8x8 │ 11.50 │ 10.68 │ 1.08x
8x16 │ 17.08 │ 16.89 │ 1.01x
8x32 │ 29.37 │ 29.07 │ 1.01x
16x4 │ 13.07 │ 11.00 │ 1.19x
16x8 │ 14.80 │ 13.33 │ 1.11x
16x16 │ 22.02 │ 20.63 │ 1.07x
16x32 │ 37.98 │ 35.40 │ 1.07x
16x64 │ 67.84 │ 66.12 │ 1.03x
32x8 │ 20.72 │ 16.27 │ 1.27x
32x16 │ 36.68 │ 29.78 │ 1.23x
32x32 │ 66.74 │ 54.31 │ 1.23x
32x64 │ 117.53 │ 104.51 │ 1.12x
64x16 │ 53.74 │ 44.44 │ 1.21x
64x32 │ 224.72 │ 204.61 │ 1.10x
64x64 │ 212.84 │ 175.49 │ 1.21x
64x128 │ 501.69 │ 416.06 │ 1.21x
128x64 │ 488.10 │ 490.45 │ 1.00x
128x128 │ 2344.06 │ 1278.32 │ 1.83x
Change-Id: I02bf2c771b27dc841d0c31c73985b9bf5b7a73ff
diff --git a/aom_dsp/aom_dsp.cmake b/aom_dsp/aom_dsp.cmake
index e249af360b..869152568a 100644
--- a/aom_dsp/aom_dsp.cmake
+++ b/aom_dsp/aom_dsp.cmake
@@ -187,7 +187,8 @@ if(CONFIG_AV1_ENCODER)
if(CONFIG_HIGHWAY)
list(APPEND AOM_DSP_ENCODER_SOURCES "${AOM_ROOT}/aom_dsp/reduce_sum_hwy.h"
- "${AOM_ROOT}/aom_dsp/sad_hwy.h")
+ "${AOM_ROOT}/aom_dsp/sad_hwy.h"
+ "${AOM_ROOT}/aom_dsp/subtract_hwy.h")
endif()
# Flow estimation library and grain/noise table/model.
@@ -272,7 +273,8 @@ if(CONFIG_AV1_ENCODER)
list(APPEND AOM_DSP_ENCODER_INTRIN_AVX2
"${AOM_ROOT}/aom_dsp/x86/sad_hwy_avx2.cc")
list(APPEND AOM_DSP_ENCODER_INTRIN_AVX512
- "${AOM_ROOT}/aom_dsp/x86/sad_hwy_avx512.cc")
+ "${AOM_ROOT}/aom_dsp/x86/sad_hwy_avx512.cc"
+ "${AOM_ROOT}/aom_dsp/x86/subtract_hwy_avx512.cc")
list(REMOVE_ITEM AOM_DSP_ENCODER_INTRIN_AVX2
"${AOM_ROOT}/aom_dsp/x86/sad_impl_avx2.c")
endif()
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl
index d1325c9041..ed27e7d0fe 100755
--- a/aom_dsp/aom_dsp_rtcd_defs.pl
+++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -817,7 +817,11 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
# Block subtraction
#
add_proto qw/void aom_subtract_block/, "int rows, int cols, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr, ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride";
- specialize qw/aom_subtract_block neon avx2/, "$sse2_asm";
+ if (aom_config("CONFIG_HIGHWAY") eq "yes") {
+ specialize qw/aom_subtract_block neon avx2 avx512/, "$sse2_asm";
+ } else {
+ specialize qw/aom_subtract_block neon avx2/, "$sse2_asm";
+ }
add_proto qw/int64_t/, "aom_sse", "const uint8_t *a, int a_stride, const uint8_t *b,int b_stride, int width, int height";
specialize qw/aom_sse sse4_1 avx2 neon neon_dotprod/;
diff --git a/aom_dsp/subtract_hwy.h b/aom_dsp/subtract_hwy.h
new file mode 100644
index 0000000000..77995ba7fe
--- /dev/null
+++ b/aom_dsp/subtract_hwy.h
@@ -0,0 +1,182 @@
+/*
+ * Copyright (c) 2026, Alliance for Open Media. All rights reserved.
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#ifndef AOM_AOM_DSP_SUBTRACT_HWY_H_
+#define AOM_AOM_DSP_SUBTRACT_HWY_H_
+
+#include <assert.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+#include "third_party/highway/hwy/highway.h"
+
+HWY_BEFORE_NAMESPACE();
+
+namespace {
+namespace HWY_NAMESPACE {
+
+namespace hn = hwy::HWY_NAMESPACE;
+
+HWY_ATTR HWY_INLINE void SubtractBlock4xN(
+ int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
+ ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
+ assert(rows >= 0 && rows % 2 == 0);
+ constexpr hn::FixedTag<uint8_t, 4> u8x4_tag;
+ constexpr hn::FixedTag<uint8_t, 8> u8x8_tag;
+ constexpr hn::FixedTag<int16_t, 8> i16x8_tag;
+ constexpr hn::FixedTag<int16_t, 4> i16x4_tag;
+
+ for (int r = 0; r < rows; r += 2) {
+ const auto s01 =
+ hn::Combine(u8x8_tag, hn::LoadU(u8x4_tag, src_ptr + src_stride),
+ hn::LoadU(u8x4_tag, src_ptr));
+ const auto p01 =
+ hn::Combine(u8x8_tag, hn::LoadU(u8x4_tag, pred_ptr + pred_stride),
+ hn::LoadU(u8x4_tag, pred_ptr));
+ const auto d01 =
+ hn::Sub(hn::PromoteTo(i16x8_tag, s01), hn::PromoteTo(i16x8_tag, p01));
+ hn::StoreU(hn::LowerHalf(i16x4_tag, d01), i16x4_tag, diff_ptr);
+ hn::StoreU(hn::UpperHalf(i16x4_tag, d01), i16x4_tag,
+ diff_ptr + diff_stride);
+ src_ptr += src_stride * 2;
+ pred_ptr += pred_stride * 2;
+ diff_ptr += diff_stride * 2;
+ }
+}
+
+HWY_ATTR HWY_INLINE void SubtractBlock8xN(
+ int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
+ ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
+ assert(rows >= 0 && rows % 2 == 0);
+ constexpr hn::FixedTag<uint8_t, 8> u8x8_tag;
+ constexpr hn::FixedTag<uint8_t, 16> u8x16_tag;
+ constexpr hn::FixedTag<int16_t, 16> i16x16_tag;
+ constexpr hn::FixedTag<int16_t, 8> i16x8_tag;
+
+ for (int r = 0; r < rows; r += 2) {
+ const auto s01 =
+ hn::Combine(u8x16_tag, hn::LoadU(u8x8_tag, src_ptr + src_stride),
+ hn::LoadU(u8x8_tag, src_ptr));
+ const auto p01 =
+ hn::Combine(u8x16_tag, hn::LoadU(u8x8_tag, pred_ptr + pred_stride),
+ hn::LoadU(u8x8_tag, pred_ptr));
+ const auto d01 =
+ hn::Sub(hn::PromoteTo(i16x16_tag, s01), hn::PromoteTo(i16x16_tag, p01));
+ hn::StoreU(hn::LowerHalf(i16x8_tag, d01), i16x8_tag, diff_ptr);
+ hn::StoreU(hn::UpperHalf(i16x8_tag, d01), i16x8_tag,
+ diff_ptr + diff_stride);
+ src_ptr += src_stride * 2;
+ pred_ptr += pred_stride * 2;
+ diff_ptr += diff_stride * 2;
+ }
+}
+
+HWY_ATTR HWY_INLINE void SubtractBlock16xN(
+ int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
+ ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
+ assert(rows >= 0 && rows % 2 == 0);
+ constexpr hn::FixedTag<uint8_t, 16> u8x16_tag;
+ constexpr hn::FixedTag<int16_t, 16> i16x16_tag;
+ for (int r = 0; r < rows; r += 2) {
+ const auto s0 = hn::PromoteTo(i16x16_tag, hn::LoadU(u8x16_tag, src_ptr));
+ const auto p0 = hn::PromoteTo(i16x16_tag, hn::LoadU(u8x16_tag, pred_ptr));
+ const auto s1 =
+ hn::PromoteTo(i16x16_tag, hn::LoadU(u8x16_tag, src_ptr + src_stride));
+ const auto p1 =
+ hn::PromoteTo(i16x16_tag, hn::LoadU(u8x16_tag, pred_ptr + pred_stride));
+ hn::StoreU(hn::Sub(s0, p0), i16x16_tag, diff_ptr);
+ hn::StoreU(hn::Sub(s1, p1), i16x16_tag, diff_ptr + diff_stride);
+ src_ptr += src_stride * 2;
+ pred_ptr += pred_stride * 2;
+ diff_ptr += diff_stride * 2;
+ }
+}
+
+template <int Cols>
+HWY_ATTR HWY_INLINE void SubtractBlockWide(
+ int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
+ ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
+ constexpr int kMaxLanes = (Cols == 128) ? 16 : 32;
+ constexpr hn::CappedTag<int16_t, kMaxLanes> i16_tag;
+ constexpr hn::Rebind<uint8_t, decltype(i16_tag)> u8_tag;
+ constexpr int kLanes = hn::MaxLanes(i16_tag);
+ for (int r = 0; r < rows; ++r) {
+ HWY_UNROLL(8)
+ for (int c = 0; c < Cols; c += kLanes) {
+ const auto s = hn::PromoteTo(i16_tag, hn::LoadU(u8_tag, src_ptr + c));
+ const auto p = hn::PromoteTo(i16_tag, hn::LoadU(u8_tag, pred_ptr + c));
+ hn::StoreU(hn::Sub(s, p), i16_tag, diff_ptr + c);
+ }
+ src_ptr += src_stride;
+ pred_ptr += pred_stride;
+ diff_ptr += diff_stride;
+ }
+}
+
+HWY_ATTR HWY_INLINE void SubtractBlock(int rows, int cols, int16_t *diff_ptr,
+ ptrdiff_t diff_stride,
+ const uint8_t *src_ptr,
+ ptrdiff_t src_stride,
+ const uint8_t *pred_ptr,
+ ptrdiff_t pred_stride) {
+ switch (cols) {
+ case 4:
+ SubtractBlock4xN(rows, diff_ptr, diff_stride, src_ptr, src_stride,
+ pred_ptr, pred_stride);
+ break;
+ case 8:
+ SubtractBlock8xN(rows, diff_ptr, diff_stride, src_ptr, src_stride,
+ pred_ptr, pred_stride);
+ break;
+ case 16:
+ SubtractBlock16xN(rows, diff_ptr, diff_stride, src_ptr, src_stride,
+ pred_ptr, pred_stride);
+ break;
+ case 32:
+ SubtractBlockWide<32>(rows, diff_ptr, diff_stride, src_ptr, src_stride,
+ pred_ptr, pred_stride);
+ break;
+ case 64:
+ SubtractBlockWide<64>(rows, diff_ptr, diff_stride, src_ptr, src_stride,
+ pred_ptr, pred_stride);
+ break;
+ case 128:
+ SubtractBlockWide<128>(rows, diff_ptr, diff_stride, src_ptr, src_stride,
+ pred_ptr, pred_stride);
+ break;
+ default:
+ aom_subtract_block_c(rows, cols, diff_ptr, diff_stride, src_ptr,
+ src_stride, pred_ptr, pred_stride);
+ break;
+ }
+}
+
+} // namespace HWY_NAMESPACE
+} // namespace
+
+HWY_AFTER_NAMESPACE();
+
+#define MAKE_SUBTRACT_BLOCK(suffix) \
+ extern "C" void aom_subtract_block_##suffix( \
+ int rows, int cols, int16_t *diff_ptr, ptrdiff_t diff_stride, \
+ const uint8_t *src_ptr, ptrdiff_t src_stride, const uint8_t *pred_ptr, \
+ ptrdiff_t pred_stride); \
+ HWY_ATTR void aom_subtract_block_##suffix( \
+ int rows, int cols, int16_t *diff_ptr, ptrdiff_t diff_stride, \
+ const uint8_t *src_ptr, ptrdiff_t src_stride, const uint8_t *pred_ptr, \
+ ptrdiff_t pred_stride) { \
+ HWY_NAMESPACE::SubtractBlock(rows, cols, diff_ptr, diff_stride, src_ptr, \
+ src_stride, pred_ptr, pred_stride); \
+ }
+
+#endif // AOM_AOM_DSP_SUBTRACT_HWY_H_
diff --git a/aom_dsp/x86/subtract_hwy_avx512.cc b/aom_dsp/x86/subtract_hwy_avx512.cc
new file mode 100644
index 0000000000..6a8d3d5df0
--- /dev/null
+++ b/aom_dsp/x86/subtract_hwy_avx512.cc
@@ -0,0 +1,17 @@
+/*
+ * Copyright (c) 2026, Alliance for Open Media. All rights reserved.
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#define HWY_BASELINE_TARGETS HWY_AVX3_DL
+#define HWY_BROKEN_32BIT 0
+
+#include "aom_dsp/subtract_hwy.h"
+
+MAKE_SUBTRACT_BLOCK(avx512)
diff --git a/test/subtract_test.cc b/test/subtract_test.cc
index 2522db063a..18f05bd5cd 100644
--- a/test/subtract_test.cc
+++ b/test/subtract_test.cc
@@ -311,6 +311,13 @@ INSTANTIATE_TEST_SUITE_P(
::testing::Values(&aom_subtract_block_c)));
#endif
+#if CONFIG_HIGHWAY && HAVE_AVX512
+INSTANTIATE_TEST_SUITE_P(
+ AVX512, AV1SubtractBlockTest,
+ ::testing::Combine(::testing::ValuesIn(kValidBlockSize),
+ ::testing::Values(&aom_subtract_block_avx512),
+ ::testing::Values(&aom_subtract_block_c)));
+#endif
#if HAVE_NEON
INSTANTIATE_TEST_SUITE_P(
NEON, AV1SubtractBlockTest,