Commit f2b8680cd4 for aom
commit f2b8680cd4f73a285f3f97a57afd3c914c86419d
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Nov 14 11:44:54 2026 -0800
av1,neon: Unroll av1_highbd_block_error_neon by 16
Unroll av1_highbd_block_error_neon to process 16 coefficients per
iteration with dual 64-bit error and ssz accumulators.
Speedup: 1.18x-1.84x (-15.5% to -45.6% on AmpereOne) and 1.42x-1.63x
(-29.8% to -38.7% on Neoverse N2) for av1_highbd_block_error_neon in
test_libaom; -0.23% (1T, -0.86% min) on Neoverse N2 and -0.22% mean
(1T Cortex-X4) / -0.06% (4T) 10b avifenc on Pixel 9 (bit-exact).
TAG=agy
CONV=490488d4-7005-402a-8cc4-c245227f80fe
Change-Id: I17b4789ba9cd4ec37c5136eacce1690cff493911
diff --git a/av1/encoder/arm/highbd_rdopt_neon.c b/av1/encoder/arm/highbd_rdopt_neon.c
index f5a0921e27..c1e3ba67d0 100644
--- a/av1/encoder/arm/highbd_rdopt_neon.c
+++ b/av1/encoder/arm/highbd_rdopt_neon.c
@@ -13,37 +13,88 @@
#include <arm_neon.h>
#include "config/av1_rtcd.h"
+#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/arm/sum_neon.h"
-int64_t av1_highbd_block_error_neon(const tran_low_t *coeff,
- const tran_low_t *dqcoeff,
- intptr_t block_size, int64_t *ssz, int bd) {
- uint64x2_t err_u64 = vdupq_n_u64(0);
- int64x2_t ssz_s64 = vdupq_n_s64(0);
+// Compute squared error and squared coefficient sum for 8 highbd coefficients,
+// initializing the dual 64-bit accumulators directly via vmull_u32 / vmull_s32
+// on the first 4 coefficients without zero-register setup.
+static AOM_FORCE_INLINE void highbd_block_error_8_init_neon(
+ const tran_low_t *coeff, const tran_low_t *dqcoeff, uint64x2_t *err_u64_0,
+ uint64x2_t *err_u64_1, int64x2_t *ssz_s64_0, int64x2_t *ssz_s64_1) {
+ const int32x4_t c0 = vld1q_s32(coeff);
+ const int32x4_t c1 = vld1q_s32(coeff + 4);
+ const int32x4_t d0 = vld1q_s32(dqcoeff);
+ const int32x4_t d1 = vld1q_s32(dqcoeff + 4);
- const int shift = 2 * (bd - 8);
- const int rounding = (1 << shift) >> 1;
+ const uint32x4_t diff0 = vreinterpretq_u32_s32(vabdq_s32(c0, d0));
+ const uint32x4_t diff1 = vreinterpretq_u32_s32(vabdq_s32(c1, d1));
- assert(block_size >= 16);
- assert((block_size % 16) == 0);
+ *err_u64_0 = vmull_u32(vget_low_u32(diff0), vget_low_u32(diff0));
+ *err_u64_1 = vmull_u32(vget_high_u32(diff0), vget_high_u32(diff0));
+ *err_u64_0 = vmlal_u32(*err_u64_0, vget_low_u32(diff1), vget_low_u32(diff1));
+ *err_u64_1 =
+ vmlal_u32(*err_u64_1, vget_high_u32(diff1), vget_high_u32(diff1));
- do {
- const int32x4_t c = vld1q_s32(coeff);
- const int32x4_t d = vld1q_s32(dqcoeff);
+ *ssz_s64_0 = vmull_s32(vget_low_s32(c0), vget_low_s32(c0));
+ *ssz_s64_1 = vmull_s32(vget_high_s32(c0), vget_high_s32(c0));
+ *ssz_s64_0 = vmlal_s32(*ssz_s64_0, vget_low_s32(c1), vget_low_s32(c1));
+ *ssz_s64_1 = vmlal_s32(*ssz_s64_1, vget_high_s32(c1), vget_high_s32(c1));
+}
- const uint32x4_t diff = vreinterpretq_u32_s32(vabdq_s32(c, d));
+// Accumulate 8 highbd coefficients into dual 64-bit accumulators (10 Q-regs
+// total, fitting within the 16-register ARMv7 NEON budget without spills).
+static AOM_FORCE_INLINE void highbd_block_error_8_accum_neon(
+ const tran_low_t *coeff, const tran_low_t *dqcoeff, uint64x2_t *err_u64_0,
+ uint64x2_t *err_u64_1, int64x2_t *ssz_s64_0, int64x2_t *ssz_s64_1) {
+ const int32x4_t c0 = vld1q_s32(coeff);
+ const int32x4_t c1 = vld1q_s32(coeff + 4);
+ const int32x4_t d0 = vld1q_s32(dqcoeff);
+ const int32x4_t d1 = vld1q_s32(dqcoeff + 4);
- err_u64 = vmlal_u32(err_u64, vget_low_u32(diff), vget_low_u32(diff));
- err_u64 = vmlal_u32(err_u64, vget_high_u32(diff), vget_high_u32(diff));
+ const uint32x4_t diff0 = vreinterpretq_u32_s32(vabdq_s32(c0, d0));
+ const uint32x4_t diff1 = vreinterpretq_u32_s32(vabdq_s32(c1, d1));
- ssz_s64 = vmlal_s32(ssz_s64, vget_low_s32(c), vget_low_s32(c));
- ssz_s64 = vmlal_s32(ssz_s64, vget_high_s32(c), vget_high_s32(c));
+ *err_u64_0 = vmlal_u32(*err_u64_0, vget_low_u32(diff0), vget_low_u32(diff0));
+ *err_u64_1 =
+ vmlal_u32(*err_u64_1, vget_high_u32(diff0), vget_high_u32(diff0));
+ *err_u64_0 = vmlal_u32(*err_u64_0, vget_low_u32(diff1), vget_low_u32(diff1));
+ *err_u64_1 =
+ vmlal_u32(*err_u64_1, vget_high_u32(diff1), vget_high_u32(diff1));
- coeff += 4;
- dqcoeff += 4;
- block_size -= 4;
- } while (block_size != 0);
+ *ssz_s64_0 = vmlal_s32(*ssz_s64_0, vget_low_s32(c0), vget_low_s32(c0));
+ *ssz_s64_1 = vmlal_s32(*ssz_s64_1, vget_high_s32(c0), vget_high_s32(c0));
+ *ssz_s64_0 = vmlal_s32(*ssz_s64_0, vget_low_s32(c1), vget_low_s32(c1));
+ *ssz_s64_1 = vmlal_s32(*ssz_s64_1, vget_high_s32(c1), vget_high_s32(c1));
+}
+int64_t av1_highbd_block_error_neon(const tran_low_t *coeff,
+ const tran_low_t *dqcoeff,
+ intptr_t block_size, int64_t *ssz, int bd) {
+ assert(block_size >= 16);
+ assert((block_size % 16) == 0);
+ assert(bd == 8 || bd == 10 || bd == 12);
+
+ uint64x2_t err_u64_0, err_u64_1;
+ int64x2_t ssz_s64_0, ssz_s64_1;
+ highbd_block_error_8_init_neon(coeff, dqcoeff, &err_u64_0, &err_u64_1,
+ &ssz_s64_0, &ssz_s64_1);
+ highbd_block_error_8_accum_neon(coeff + 8, dqcoeff + 8, &err_u64_0,
+ &err_u64_1, &ssz_s64_0, &ssz_s64_1);
+
+ for (block_size -= 16; block_size != 0; block_size -= 16) {
+ coeff += 16;
+ dqcoeff += 16;
+ highbd_block_error_8_accum_neon(coeff, dqcoeff, &err_u64_0, &err_u64_1,
+ &ssz_s64_0, &ssz_s64_1);
+ highbd_block_error_8_accum_neon(coeff + 8, dqcoeff + 8, &err_u64_0,
+ &err_u64_1, &ssz_s64_0, &ssz_s64_1);
+ }
+
+ const int shift = 2 * (bd - 8);
+ const int rounding = (1 << shift) >> 1;
+ const uint64x2_t err_u64 = vaddq_u64(err_u64_0, err_u64_1);
+ const int64x2_t ssz_s64 = vaddq_s64(ssz_s64_0, ssz_s64_1);
*ssz = (horizontal_add_s64x2(ssz_s64) + rounding) >> shift;
return ((int64_t)horizontal_add_u64x2(err_u64) + rounding) >> shift;
}