Commit 95f2f18a2b for aom
commit 95f2f18a2beb57e16b196fdffc00e4cf82e4ace4
Author: Jonathan Wright <jonathan.wright@arm.com>
Date: Fri Jul 10 14:54:44 2026 +0100
Unroll Arm Neon DotProd variance kernels for w >= 32
Unroll the Arm Neon DotProd variance kernels and split the
accumulators for blocks having width >= 32. This change reduces the
time required to execute these kernels by up to 40% on a Neoverse V3
when compiled with Clang 22.
Change-Id: Ibc8541bde727cbba5df460704877dc5e61455d1f
diff --git a/aom_dsp/arm/variance_neon_dotprod.c b/aom_dsp/arm/variance_neon_dotprod.c
index 68a644551c..68bb93f685 100644
--- a/aom_dsp/arm/variance_neon_dotprod.c
+++ b/aom_dsp/arm/variance_neon_dotprod.c
@@ -103,47 +103,110 @@ static inline void variance_16xh_neon_dotprod(const uint8_t *src,
*sse = horizontal_add_u32x4(sse_u32);
}
+static inline void variance_32xh_neon_dotprod(const uint8_t *src,
+ int src_stride,
+ const uint8_t *ref,
+ int ref_stride, int h,
+ uint32_t *sse, int *sum) {
+ uint32x4_t src_sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+ uint32x4_t ref_sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+ uint32x4_t sse_u32[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src);
+ uint8x16_t r0 = vld1q_u8(ref);
+
+ src_sum[0] = vdotq_u32(src_sum[0], s0, vdupq_n_u8(1));
+ ref_sum[0] = vdotq_u32(ref_sum[0], r0, vdupq_n_u8(1));
+
+ uint8x16_t abs_diff0 = vabdq_u8(s0, r0);
+ sse_u32[0] = vdotq_u32(sse_u32[0], abs_diff0, abs_diff0);
+
+ uint8x16_t s1 = vld1q_u8(src + 16);
+ uint8x16_t r1 = vld1q_u8(ref + 16);
+
+ src_sum[1] = vdotq_u32(src_sum[1], s1, vdupq_n_u8(1));
+ ref_sum[1] = vdotq_u32(ref_sum[1], r1, vdupq_n_u8(1));
+
+ uint8x16_t abs_diff1 = vabdq_u8(s1, r1);
+ sse_u32[1] = vdotq_u32(sse_u32[1], abs_diff1, abs_diff1);
+
+ src += src_stride;
+ ref += ref_stride;
+ } while (--h != 0);
+
+ src_sum[0] = vaddq_u32(src_sum[0], src_sum[1]);
+ ref_sum[0] = vaddq_u32(ref_sum[0], ref_sum[1]);
+ int32x4_t sum_diff = vsubq_s32(vreinterpretq_s32_u32(src_sum[0]),
+ vreinterpretq_s32_u32(ref_sum[0]));
+ *sum = horizontal_add_s32x4(sum_diff);
+ *sse = horizontal_add_u32x4(vaddq_u32(sse_u32[0], sse_u32[1]));
+}
+
static inline void variance_large_neon_dotprod(const uint8_t *src,
int src_stride,
const uint8_t *ref,
int ref_stride, int w, int h,
uint32_t *sse, int *sum) {
- uint32x4_t src_sum = vdupq_n_u32(0);
- uint32x4_t ref_sum = vdupq_n_u32(0);
- uint32x4_t sse_u32 = vdupq_n_u32(0);
+ uint32x4_t src_sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+ uint32x4_t ref_sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+ uint32x4_t sse_u32[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
do {
int i = 0;
do {
- uint8x16_t s = vld1q_u8(src + i);
- uint8x16_t r = vld1q_u8(ref + i);
+ uint8x16_t s0 = vld1q_u8(src + i + 0);
+ uint8x16_t r0 = vld1q_u8(ref + i + 0);
+
+ src_sum[0] = vdotq_u32(src_sum[0], s0, vdupq_n_u8(1));
+ ref_sum[0] = vdotq_u32(ref_sum[0], r0, vdupq_n_u8(1));
+
+ uint8x16_t abs_diff0 = vabdq_u8(s0, r0);
+ sse_u32[0] = vdotq_u32(sse_u32[0], abs_diff0, abs_diff0);
+
+ uint8x16_t s1 = vld1q_u8(src + i + 16);
+ uint8x16_t r1 = vld1q_u8(ref + i + 16);
+
+ src_sum[1] = vdotq_u32(src_sum[1], s1, vdupq_n_u8(1));
+ ref_sum[1] = vdotq_u32(ref_sum[1], r1, vdupq_n_u8(1));
+
+ uint8x16_t abs_diff1 = vabdq_u8(s1, r1);
+ sse_u32[1] = vdotq_u32(sse_u32[1], abs_diff1, abs_diff1);
+
+ uint8x16_t s2 = vld1q_u8(src + i + 32);
+ uint8x16_t r2 = vld1q_u8(ref + i + 32);
+
+ src_sum[2] = vdotq_u32(src_sum[2], s2, vdupq_n_u8(1));
+ ref_sum[2] = vdotq_u32(ref_sum[2], r2, vdupq_n_u8(1));
+
+ uint8x16_t abs_diff2 = vabdq_u8(s2, r2);
+ sse_u32[2] = vdotq_u32(sse_u32[2], abs_diff2, abs_diff2);
- src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1));
- ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1));
+ uint8x16_t s3 = vld1q_u8(src + i + 48);
+ uint8x16_t r3 = vld1q_u8(ref + i + 48);
- uint8x16_t abs_diff = vabdq_u8(s, r);
- sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
+ src_sum[3] = vdotq_u32(src_sum[3], s3, vdupq_n_u8(1));
+ ref_sum[3] = vdotq_u32(ref_sum[3], r3, vdupq_n_u8(1));
- i += 16;
+ uint8x16_t abs_diff3 = vabdq_u8(s3, r3);
+ sse_u32[3] = vdotq_u32(sse_u32[3], abs_diff3, abs_diff3);
+
+ i += 64;
} while (i < w);
src += src_stride;
ref += ref_stride;
} while (--h != 0);
- int32x4_t sum_diff =
- vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum));
+ src_sum[0] = horizontal_add_4d_u32x4(src_sum);
+ ref_sum[0] = horizontal_add_4d_u32x4(ref_sum);
+ int32x4_t sum_diff = vsubq_s32(vreinterpretq_s32_u32(src_sum[0]),
+ vreinterpretq_s32_u32(ref_sum[0]));
*sum = horizontal_add_s32x4(sum_diff);
- *sse = horizontal_add_u32x4(sse_u32);
-}
-
-static inline void variance_32xh_neon_dotprod(const uint8_t *src,
- int src_stride,
- const uint8_t *ref,
- int ref_stride, int h,
- uint32_t *sse, int *sum) {
- variance_large_neon_dotprod(src, src_stride, ref, ref_stride, 32, h, sse,
- sum);
+ *sse = horizontal_add_u32x4(horizontal_add_4d_u32x4(sse_u32));
}
static inline void variance_64xh_neon_dotprod(const uint8_t *src,