Commit 73c7ae678c for aom
commit 73c7ae678c007afee911a9d77926a668ca3a016e
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Nov 14 18:29:06 2026 -0800
aom_dsp,neon: Optimize highbd dc intra predictors
Use vaddlv_u16 and vaddlvq_u16 scalar reductions on AArch64 in
aom_highbd_dc_predictor_*_neon, aom_highbd_dc_left_predictor_*_neon,
and aom_highbd_dc_top_predictor_*_neon, and unroll
highbd_dc_store_{16,32,64}xh across 4 rows.
Speedup: -0.9% (HighbdIntraPredTest 8x16) in test_libaom on Neoverse
N3; -0.27% (1T) / -0.50% (4T, -1.38% min) 10b avifenc on Neoverse N2
(bit-exact).
TAG=agy
CONV=490488d4-7005-402a-8cc4-c245227f80fe
Change-Id: I6480e404f55aba06fe3b741a7b1c74762c3646dc
diff --git a/aom_dsp/arm/highbd_intrapred_neon.c b/aom_dsp/arm/highbd_intrapred_neon.c
index 0f8af118d4..7e05e8b19a 100644
--- a/aom_dsp/arm/highbd_intrapred_neon.c
+++ b/aom_dsp/arm/highbd_intrapred_neon.c
@@ -10,6 +10,7 @@
*/
#include <arm_neon.h>
+#include <assert.h>
#include "config/aom_config.h"
#include "config/aom_dsp_rtcd.h"
@@ -40,24 +41,75 @@ static inline void highbd_dc_store_8xh(uint16_t *dst, ptrdiff_t stride, int h,
static inline void highbd_dc_store_16xh(uint16_t *dst, ptrdiff_t stride, int h,
uint16x8_t dc) {
+#if AOM_ARCH_AARCH64
+ assert(h >= 4 && (h & 3) == 0);
+ const uint16x8x2_t dc2 = { { dc, dc } };
+ do {
+ vst1q_u16_x2(dst, dc2);
+ dst += stride;
+ vst1q_u16_x2(dst, dc2);
+ dst += stride;
+ vst1q_u16_x2(dst, dc2);
+ dst += stride;
+ vst1q_u16_x2(dst, dc2);
+ dst += stride;
+ h -= 4;
+ } while (h != 0);
+#else
for (int i = 0; i < h; ++i) {
vst1q_u16(dst + i * stride, dc);
vst1q_u16(dst + i * stride + 8, dc);
}
+#endif
}
static inline void highbd_dc_store_32xh(uint16_t *dst, ptrdiff_t stride, int h,
uint16x8_t dc) {
+#if AOM_ARCH_AARCH64
+ assert(h >= 4 && (h & 3) == 0);
+ const uint16x8x4_t dc4 = { { dc, dc, dc, dc } };
+ do {
+ vst1q_u16_x4(dst, dc4);
+ dst += stride;
+ vst1q_u16_x4(dst, dc4);
+ dst += stride;
+ vst1q_u16_x4(dst, dc4);
+ dst += stride;
+ vst1q_u16_x4(dst, dc4);
+ dst += stride;
+ h -= 4;
+ } while (h != 0);
+#else
for (int i = 0; i < h; ++i) {
vst1q_u16(dst + i * stride, dc);
vst1q_u16(dst + i * stride + 8, dc);
vst1q_u16(dst + i * stride + 16, dc);
vst1q_u16(dst + i * stride + 24, dc);
}
+#endif
}
static inline void highbd_dc_store_64xh(uint16_t *dst, ptrdiff_t stride, int h,
uint16x8_t dc) {
+#if AOM_ARCH_AARCH64
+ assert(h >= 4 && (h & 3) == 0);
+ const uint16x8x4_t dc4 = { { dc, dc, dc, dc } };
+ do {
+ vst1q_u16_x4(dst, dc4);
+ vst1q_u16_x4(dst + 32, dc4);
+ dst += stride;
+ vst1q_u16_x4(dst, dc4);
+ vst1q_u16_x4(dst + 32, dc4);
+ dst += stride;
+ vst1q_u16_x4(dst, dc4);
+ vst1q_u16_x4(dst + 32, dc4);
+ dst += stride;
+ vst1q_u16_x4(dst, dc4);
+ vst1q_u16_x4(dst + 32, dc4);
+ dst += stride;
+ h -= 4;
+ } while (h != 0);
+#else
for (int i = 0; i < h; ++i) {
vst1q_u16(dst + i * stride, dc);
vst1q_u16(dst + i * stride + 8, dc);
@@ -68,21 +120,19 @@ static inline void highbd_dc_store_64xh(uint16_t *dst, ptrdiff_t stride, int h,
vst1q_u16(dst + i * stride + 48, dc);
vst1q_u16(dst + i * stride + 56, dc);
}
+#endif
}
+#if !AOM_ARCH_AARCH64
static inline uint32x4_t horizontal_add_and_broadcast_long_u16x8(uint16x8_t a) {
// Need to assume input is up to 16 bits wide from dc 64x64 partial sum, so
// promote first.
const uint32x4_t b = vpaddlq_u16(a);
-#if AOM_ARCH_AARCH64
- const uint32x4_t c = vpaddq_u32(b, b);
- return vpaddq_u32(c, c);
-#else
const uint32x2_t c = vadd_u32(vget_low_u32(b), vget_high_u32(b));
const uint32x2_t d = vpadd_u32(c, c);
return vcombine_u32(d, d);
-#endif
}
+#endif // !AOM_ARCH_AARCH64
static inline uint16x8_t highbd_dc_load_partial_sum_4(const uint16_t *left) {
// Nothing to do since sum is already one vector, but saves needing to
@@ -132,6 +182,29 @@ static inline uint16x8_t highbd_dc_load_partial_sum_64(const uint16_t *left) {
return vaddq_u16(c0, c1); // up to 15 bits
}
+#if AOM_ARCH_AARCH64
+#define HIGHBD_DC_PREDICTOR(w, h, shift) \
+ void aom_highbd_dc_predictor_##w##x##h##_neon( \
+ uint16_t *dst, ptrdiff_t stride, const uint16_t *above, \
+ const uint16_t *left, int bd) { \
+ (void)bd; \
+ const uint16x8_t a = highbd_dc_load_partial_sum_##w(above); \
+ const uint16x8_t l = highbd_dc_load_partial_sum_##h(left); \
+ const uint32_t sum = vaddlvq_u16(vaddq_u16(a, l)); \
+ const uint16_t dc0 = (uint16_t)((sum + (1 << ((shift) - 1))) >> (shift)); \
+ highbd_dc_store_##w##xh(dst, stride, (h), vdupq_n_u16(dc0)); \
+ }
+
+void aom_highbd_dc_predictor_4x4_neon(uint16_t *dst, ptrdiff_t stride,
+ const uint16_t *above,
+ const uint16_t *left, int bd) {
+ (void)bd;
+ const uint16x8_t al = vcombine_u16(vld1_u16(above), vld1_u16(left));
+ const uint32_t sum = vaddlvq_u16(al);
+ const uint16_t dc0 = (uint16_t)((sum + 4) >> 3);
+ highbd_dc_store_4xh(dst, stride, 4, vdup_n_u16(dc0));
+}
+#else
#define HIGHBD_DC_PREDICTOR(w, h, shift) \
void aom_highbd_dc_predictor_##w##x##h##_neon( \
uint16_t *dst, ptrdiff_t stride, const uint16_t *above, \
@@ -161,6 +234,7 @@ void aom_highbd_dc_predictor_4x4_neon(uint16_t *dst, ptrdiff_t stride,
const uint16x4_t dc = vrshr_n_u16(sum, 3);
highbd_dc_store_4xh(dst, stride, 4, dc);
}
+#endif // AOM_ARCH_AARCH64
HIGHBD_DC_PREDICTOR(8, 8, 4)
HIGHBD_DC_PREDICTOR(16, 16, 5)
@@ -285,6 +359,38 @@ HIGHBD_DC_PREDICTOR_128(64, 64, q)
// -----------------------------------------------------------------------------
// DC_LEFT
+#if AOM_ARCH_AARCH64
+static inline uint32_t highbd_dc_sum_scalar_4(const uint16_t *left) {
+ return vaddlv_u16(vld1_u16(left));
+}
+
+static inline uint32_t highbd_dc_sum_scalar_8(const uint16_t *left) {
+ return vaddlvq_u16(vld1q_u16(left));
+}
+
+static inline uint32_t highbd_dc_sum_scalar_16(const uint16_t *left) {
+ return vaddlvq_u16(highbd_dc_load_partial_sum_16(left));
+}
+
+static inline uint32_t highbd_dc_sum_scalar_32(const uint16_t *left) {
+ return vaddlvq_u16(highbd_dc_load_partial_sum_32(left));
+}
+
+static inline uint32_t highbd_dc_sum_scalar_64(const uint16_t *left) {
+ return vaddlvq_u16(highbd_dc_load_partial_sum_64(left));
+}
+
+#define DC_PREDICTOR_LEFT(w, h, shift, q) \
+ void aom_highbd_dc_left_predictor_##w##x##h##_neon( \
+ uint16_t *dst, ptrdiff_t stride, const uint16_t *above, \
+ const uint16_t *left, int bd) { \
+ (void)above; \
+ (void)bd; \
+ const uint32_t sum = highbd_dc_sum_scalar_##h(left); \
+ const uint16_t dc0 = (uint16_t)((sum + (1 << ((shift) - 1))) >> (shift)); \
+ highbd_dc_store_##w##xh(dst, stride, (h), vdup##q##_n_u16(dc0)); \
+ }
+#else
static inline uint32x4_t highbd_dc_load_sum_4(const uint16_t *left) {
const uint16x4_t a = vld1_u16(left); // up to 12 bits
const uint16x4_t b = vpadd_u16(a, a); // up to 13 bits
@@ -320,6 +426,7 @@ static inline uint32x4_t highbd_dc_load_sum_64(const uint16_t *left) {
const uint16x4_t dc0 = vrshrn_n_u32(sum, (shift)); \
highbd_dc_store_##w##xh(dst, stride, (h), vdup##q##_lane_u16(dc0, 0)); \
}
+#endif // AOM_ARCH_AARCH64
#if !CONFIG_REALTIME_ONLY || CONFIG_AV1_DECODER
DC_PREDICTOR_LEFT(4, 4, 2, )
@@ -362,6 +469,18 @@ DC_PREDICTOR_LEFT(64, 64, 6, q)
// -----------------------------------------------------------------------------
// DC_TOP
+#if AOM_ARCH_AARCH64
+#define DC_PREDICTOR_TOP(w, h, shift, q) \
+ void aom_highbd_dc_top_predictor_##w##x##h##_neon( \
+ uint16_t *dst, ptrdiff_t stride, const uint16_t *above, \
+ const uint16_t *left, int bd) { \
+ (void)bd; \
+ (void)left; \
+ const uint32_t sum = highbd_dc_sum_scalar_##w(above); \
+ const uint16_t dc0 = (uint16_t)((sum + (1 << ((shift) - 1))) >> (shift)); \
+ highbd_dc_store_##w##xh(dst, stride, (h), vdup##q##_n_u16(dc0)); \
+ }
+#else
#define DC_PREDICTOR_TOP(w, h, shift, q) \
void aom_highbd_dc_top_predictor_##w##x##h##_neon( \
uint16_t *dst, ptrdiff_t stride, const uint16_t *above, \
@@ -372,6 +491,7 @@ DC_PREDICTOR_LEFT(64, 64, 6, q)
const uint16x4_t dc0 = vrshrn_n_u32(sum, (shift)); \
highbd_dc_store_##w##xh(dst, stride, (h), vdup##q##_lane_u16(dc0, 0)); \
}
+#endif // AOM_ARCH_AARCH64
#if !CONFIG_REALTIME_ONLY || CONFIG_AV1_DECODER
DC_PREDICTOR_TOP(4, 4, 2, )