Commit fcc5bf455d for aom
commit fcc5bf455d21ead1b8343a07a1503342d8b4307a
Author: Frank Galligan <fgalligan@google.com>
Date: Tue Nov 17 06:28:30 2026 -0800
aom_dsp,neon: Optimize satd & hadamard_16x16
- aom_satd_neon(): assert length invariants, add a length == 16 fast
path, and unroll length >= 32 loops to 32 elements with 4 accumulators
(vabsq_s32 + vaddq_s32 vs 2 vabaq_s32 accumulators).
- aom_satd_lp_neon(): assert length invariants, add a length == 16 fast
path reducing in uint16, and unroll length >= 32 loops to 32 elements
with pair-wise uint16 sums before vpadalq_u16.
- aom_hadamard_16x16_neon(): compute the four 8x8 sub-transforms in
16-bit via aom_hadamard_lp_8x8_neon() and combine quadrants with
vhaddq_s16/vhsubq_s16 before widening via vaddl_s16/vsubl_s16.
Speedup (Pixel 9 Tensor G4, 1T, test_libaom DISABLED_Speed vs
origin/main, bit-exact across tier1 768x1024 realtime speed 6/7):
- SatdLpTest (16/64/256/1024):
A720: -7.77% / +0.65% / -0.39% / -0.73% (839->767us at 16, 1.09x)
A520: -8.72% / -11.63% / -26.24% / -31.31% (1.10x..1.46x)
- SatdTest (16/64/256/1024):
A720: +0.00% / +0.14% / +0.03% / -0.02% (neutral)
A520: +0.13% / -21.74% / -27.35% / -29.74% (1.28x..1.42x for >=64)
- HadamardLowbdTest (16x16 / 32x32):
A720: -21.58% (128850->102838us, 1.25x) / -14.21% (778973->669592us)
TAG=agy
CONV=bc5228f2-7fa6-4522-aad6-34935d739e28
Change-Id: Icef8651208537da1de24e2d6c37bd8fe67023029
diff --git a/aom_dsp/arm/avg_neon.c b/aom_dsp/arm/avg_neon.c
index 7d6054cb0a..5fb6ed7b99 100644
--- a/aom_dsp/arm/avg_neon.c
+++ b/aom_dsp/arm/avg_neon.c
@@ -55,35 +55,39 @@ void aom_avg_8x8_quad_neon(const uint8_t *s, int p, int x16_idx, int y16_idx,
avg[3] = aom_avg_8x8_neon(s + (y16_idx + 8) * p + (x16_idx + 8), p);
}
-int aom_satd_lp_neon(const int16_t *coeff, int length) {
- int16x8_t s0 = vld1q_s16(coeff);
- int16x8_t s1 = vld1q_s16(coeff + 8);
+static inline uint16x8_t load_abs_s16(const int16_t *p) {
+ return vreinterpretq_u16_s16(vabsq_s16(vld1q_s16(p)));
+}
- int16x8_t abs0 = vabsq_s16(s0);
- int16x8_t abs1 = vabsq_s16(s1);
+// coeff: 16 bits, dynamic range [-32640, 32640].
+// |coeff[i]| + |coeff[i + 8]| <= 65280 fits in uint16_t without wrapping.
+// length: value range {16, 32, 64, 128, 256, 512, 1024}.
+int aom_satd_lp_neon(const int16_t *coeff, int length) {
+ assert(length == 16 || (length >= 32 && length % 32 == 0));
+ const uint16x8_t sum0 =
+ vaddq_u16(load_abs_s16(coeff), load_abs_s16(coeff + 8));
+ if (length == 16) {
+ return (int)horizontal_add_u16x8(sum0);
+ }
- int32x4_t acc0 = vpaddlq_s16(abs0);
- int32x4_t acc1 = vpaddlq_s16(abs1);
+ uint32x4_t acc0 = vpaddlq_u16(sum0);
+ uint32x4_t acc1 = vpaddlq_u16(
+ vaddq_u16(load_abs_s16(coeff + 16), load_abs_s16(coeff + 24)));
- length -= 16;
- coeff += 16;
+ length -= 32;
+ coeff += 32;
while (length != 0) {
- s0 = vld1q_s16(coeff);
- s1 = vld1q_s16(coeff + 8);
-
- abs0 = vabsq_s16(s0);
- abs1 = vabsq_s16(s1);
+ acc0 = vpadalq_u16(acc0,
+ vaddq_u16(load_abs_s16(coeff), load_abs_s16(coeff + 8)));
+ acc1 = vpadalq_u16(
+ acc1, vaddq_u16(load_abs_s16(coeff + 16), load_abs_s16(coeff + 24)));
- acc0 = vpadalq_s16(acc0, abs0);
- acc1 = vpadalq_s16(acc1, abs1);
-
- length -= 16;
- coeff += 16;
+ length -= 32;
+ coeff += 32;
}
- int32x4_t accum = vaddq_s32(acc0, acc1);
- return horizontal_add_s32x4(accum);
+ return (int)horizontal_add_u32x4(vaddq_u32(acc0, acc1));
}
void aom_int_pro_row_neon(int16_t *hbuf, const uint8_t *ref,
@@ -178,38 +182,66 @@ void aom_int_pro_col_neon(int16_t *vbuf, const uint8_t *ref,
// coeff: 20 bits, dynamic range [-524287, 524287].
// length: value range {16, 32, 64, 128, 256, 512, 1024}.
int aom_satd_neon(const tran_low_t *coeff, int length) {
- const int32x4_t zero = vdupq_n_s32(0);
+ assert(length == 16 || (length >= 32 && length % 32 == 0));
+ if (length == 16) {
+ const int32x4_t s0 = vld1q_s32(&coeff[0]);
+ const int32x4_t s1 = vld1q_s32(&coeff[4]);
+ const int32x4_t s2 = vld1q_s32(&coeff[8]);
+ const int32x4_t s3 = vld1q_s32(&coeff[12]);
+
+ const int32x4_t a0 = vabsq_s32(s0);
+ const int32x4_t a1 = vabsq_s32(s1);
+ const int32x4_t a2 = vabsq_s32(s2);
+ const int32x4_t a3 = vabsq_s32(s3);
+
+ return horizontal_add_s32x4(
+ vaddq_s32(vaddq_s32(a0, a1), vaddq_s32(a2, a3)));
+ }
- int32x4_t s0 = vld1q_s32(&coeff[0]);
- int32x4_t s1 = vld1q_s32(&coeff[4]);
- int32x4_t s2 = vld1q_s32(&coeff[8]);
- int32x4_t s3 = vld1q_s32(&coeff[12]);
+ const int32x4_t init0 = vld1q_s32(&coeff[0]);
+ const int32x4_t init1 = vld1q_s32(&coeff[4]);
+ const int32x4_t init2 = vld1q_s32(&coeff[8]);
+ const int32x4_t init3 = vld1q_s32(&coeff[12]);
+ const int32x4_t init4 = vld1q_s32(&coeff[16]);
+ const int32x4_t init5 = vld1q_s32(&coeff[20]);
+ const int32x4_t init6 = vld1q_s32(&coeff[24]);
+ const int32x4_t init7 = vld1q_s32(&coeff[28]);
- int32x4_t accum0 = vabsq_s32(s0);
- int32x4_t accum1 = vabsq_s32(s2);
- accum0 = vabaq_s32(accum0, s1, zero);
- accum1 = vabaq_s32(accum1, s3, zero);
+ int32x4_t acc0 = vaddq_s32(vabsq_s32(init0), vabsq_s32(init4));
+ int32x4_t acc1 = vaddq_s32(vabsq_s32(init1), vabsq_s32(init5));
+ int32x4_t acc2 = vaddq_s32(vabsq_s32(init2), vabsq_s32(init6));
+ int32x4_t acc3 = vaddq_s32(vabsq_s32(init3), vabsq_s32(init7));
- length -= 16;
- coeff += 16;
+ length -= 32;
+ coeff += 32;
while (length != 0) {
- s0 = vld1q_s32(&coeff[0]);
- s1 = vld1q_s32(&coeff[4]);
- s2 = vld1q_s32(&coeff[8]);
- s3 = vld1q_s32(&coeff[12]);
-
- accum0 = vabaq_s32(accum0, s0, zero);
- accum1 = vabaq_s32(accum1, s1, zero);
- accum0 = vabaq_s32(accum0, s2, zero);
- accum1 = vabaq_s32(accum1, s3, zero);
-
- length -= 16;
- coeff += 16;
+ const int32x4_t s0 = vld1q_s32(&coeff[0]);
+ const int32x4_t s1 = vld1q_s32(&coeff[4]);
+ const int32x4_t s2 = vld1q_s32(&coeff[8]);
+ const int32x4_t s3 = vld1q_s32(&coeff[12]);
+ const int32x4_t s4 = vld1q_s32(&coeff[16]);
+ const int32x4_t s5 = vld1q_s32(&coeff[20]);
+ const int32x4_t s6 = vld1q_s32(&coeff[24]);
+ const int32x4_t s7 = vld1q_s32(&coeff[28]);
+
+ const int32x4_t sum0 = vaddq_s32(vabsq_s32(s0), vabsq_s32(s4));
+ const int32x4_t sum1 = vaddq_s32(vabsq_s32(s1), vabsq_s32(s5));
+ const int32x4_t sum2 = vaddq_s32(vabsq_s32(s2), vabsq_s32(s6));
+ const int32x4_t sum3 = vaddq_s32(vabsq_s32(s3), vabsq_s32(s7));
+
+ acc0 = vaddq_s32(acc0, sum0);
+ acc1 = vaddq_s32(acc1, sum1);
+ acc2 = vaddq_s32(acc2, sum2);
+ acc3 = vaddq_s32(acc3, sum3);
+
+ length -= 32;
+ coeff += 32;
}
// satd: 30 bits, dynamic range [-524287 * 1024, 524287 * 1024]
- return horizontal_add_s32x4(vaddq_s32(accum0, accum1));
+ return horizontal_add_s32x4(
+ vaddq_s32(vaddq_s32(acc0, acc1), vaddq_s32(acc2, acc3)));
}
int aom_vector_var_neon(const int16_t *ref, const int16_t *src, int bwl) {
diff --git a/aom_dsp/arm/hadamard_neon.c b/aom_dsp/arm/hadamard_neon.c
index 4b100d23e8..83c94cfab7 100644
--- a/aom_dsp/arm/hadamard_neon.c
+++ b/aom_dsp/arm/hadamard_neon.c
@@ -187,103 +187,87 @@ void aom_hadamard_lp_16x16_neon(const int16_t *src_diff, ptrdiff_t src_stride,
void aom_hadamard_16x16_neon(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
+ int16_t t_coeff[256];
/* Rearrange 16x16 to 8x32 and remove stride.
* Top left first. */
- aom_hadamard_8x8_neon(src_diff + 0 + 0 * src_stride, src_stride, coeff + 0);
+ aom_hadamard_lp_8x8_neon(src_diff + 0 + 0 * src_stride, src_stride,
+ t_coeff + 0);
/* Top right. */
- aom_hadamard_8x8_neon(src_diff + 8 + 0 * src_stride, src_stride, coeff + 64);
+ aom_hadamard_lp_8x8_neon(src_diff + 8 + 0 * src_stride, src_stride,
+ t_coeff + 64);
/* Bottom left. */
- aom_hadamard_8x8_neon(src_diff + 0 + 8 * src_stride, src_stride, coeff + 128);
+ aom_hadamard_lp_8x8_neon(src_diff + 0 + 8 * src_stride, src_stride,
+ t_coeff + 128);
/* Bottom right. */
- aom_hadamard_8x8_neon(src_diff + 8 + 8 * src_stride, src_stride, coeff + 192);
+ aom_hadamard_lp_8x8_neon(src_diff + 8 + 8 * src_stride, src_stride,
+ t_coeff + 192);
// Each iteration of the loop operates on entire rows (16 samples each)
// because we need to swap the second and third quarters of every row in the
// output to match AVX2 output (i.e., aom_hadamard_16x16_avx2). See the for
// loop at the end of aom_hadamard_16x16_c.
- for (int i = 0; i < 64; i += 16) {
- const int32x4_t a00 = vld1q_s32(coeff + 0);
- const int32x4_t a01 = vld1q_s32(coeff + 64);
- const int32x4_t a02 = vld1q_s32(coeff + 128);
- const int32x4_t a03 = vld1q_s32(coeff + 192);
-
- const int32x4_t b00 = vhaddq_s32(a00, a01);
- const int32x4_t b01 = vhsubq_s32(a00, a01);
- const int32x4_t b02 = vhaddq_s32(a02, a03);
- const int32x4_t b03 = vhsubq_s32(a02, a03);
-
- const int32x4_t c00 = vaddq_s32(b00, b02);
- const int32x4_t c01 = vaddq_s32(b01, b03);
- const int32x4_t c02 = vsubq_s32(b00, b02);
- const int32x4_t c03 = vsubq_s32(b01, b03);
-
- const int32x4_t a10 = vld1q_s32(coeff + 4 + 0);
- const int32x4_t a11 = vld1q_s32(coeff + 4 + 64);
- const int32x4_t a12 = vld1q_s32(coeff + 4 + 128);
- const int32x4_t a13 = vld1q_s32(coeff + 4 + 192);
-
- const int32x4_t b10 = vhaddq_s32(a10, a11);
- const int32x4_t b11 = vhsubq_s32(a10, a11);
- const int32x4_t b12 = vhaddq_s32(a12, a13);
- const int32x4_t b13 = vhsubq_s32(a12, a13);
-
- const int32x4_t c10 = vaddq_s32(b10, b12);
- const int32x4_t c11 = vaddq_s32(b11, b13);
- const int32x4_t c12 = vsubq_s32(b10, b12);
- const int32x4_t c13 = vsubq_s32(b11, b13);
-
- const int32x4_t a20 = vld1q_s32(coeff + 8 + 0);
- const int32x4_t a21 = vld1q_s32(coeff + 8 + 64);
- const int32x4_t a22 = vld1q_s32(coeff + 8 + 128);
- const int32x4_t a23 = vld1q_s32(coeff + 8 + 192);
-
- const int32x4_t b20 = vhaddq_s32(a20, a21);
- const int32x4_t b21 = vhsubq_s32(a20, a21);
- const int32x4_t b22 = vhaddq_s32(a22, a23);
- const int32x4_t b23 = vhsubq_s32(a22, a23);
-
- const int32x4_t c20 = vaddq_s32(b20, b22);
- const int32x4_t c21 = vaddq_s32(b21, b23);
- const int32x4_t c22 = vsubq_s32(b20, b22);
- const int32x4_t c23 = vsubq_s32(b21, b23);
-
- const int32x4_t a30 = vld1q_s32(coeff + 12 + 0);
- const int32x4_t a31 = vld1q_s32(coeff + 12 + 64);
- const int32x4_t a32 = vld1q_s32(coeff + 12 + 128);
- const int32x4_t a33 = vld1q_s32(coeff + 12 + 192);
-
- const int32x4_t b30 = vhaddq_s32(a30, a31);
- const int32x4_t b31 = vhsubq_s32(a30, a31);
- const int32x4_t b32 = vhaddq_s32(a32, a33);
- const int32x4_t b33 = vhsubq_s32(a32, a33);
-
- const int32x4_t c30 = vaddq_s32(b30, b32);
- const int32x4_t c31 = vaddq_s32(b31, b33);
- const int32x4_t c32 = vsubq_s32(b30, b32);
- const int32x4_t c33 = vsubq_s32(b31, b33);
-
- vst1q_s32(coeff + 0 + 0, c00);
- vst1q_s32(coeff + 0 + 4, c20);
- vst1q_s32(coeff + 0 + 8, c10);
- vst1q_s32(coeff + 0 + 12, c30);
-
- vst1q_s32(coeff + 64 + 0, c01);
- vst1q_s32(coeff + 64 + 4, c21);
- vst1q_s32(coeff + 64 + 8, c11);
- vst1q_s32(coeff + 64 + 12, c31);
-
- vst1q_s32(coeff + 128 + 0, c02);
- vst1q_s32(coeff + 128 + 4, c22);
- vst1q_s32(coeff + 128 + 8, c12);
- vst1q_s32(coeff + 128 + 12, c32);
-
- vst1q_s32(coeff + 192 + 0, c03);
- vst1q_s32(coeff + 192 + 4, c23);
- vst1q_s32(coeff + 192 + 8, c13);
- vst1q_s32(coeff + 192 + 12, c33);
-
+ // For 8-bit inputs aom_hadamard_lp_8x8_neon() outputs 15-bit values in
+ // [-16320, 16320], and for any int16_t input vhaddq_s16/vhsubq_s16 halve
+ // exactly before widening via vaddl_s16/vsubl_s16.
+ const int16_t *t_ptr = t_coeff;
+ int h = 4;
+ do {
+ const int16x8_t a0_e = vld1q_s16(t_ptr + 0);
+ const int16x8_t a0_o = vld1q_s16(t_ptr + 8);
+ const int16x8_t a1_e = vld1q_s16(t_ptr + 64);
+ const int16x8_t a1_o = vld1q_s16(t_ptr + 72);
+ const int16x8_t a2_e = vld1q_s16(t_ptr + 128);
+ const int16x8_t a2_o = vld1q_s16(t_ptr + 136);
+ const int16x8_t a3_e = vld1q_s16(t_ptr + 192);
+ const int16x8_t a3_o = vld1q_s16(t_ptr + 200);
+
+ const int16x8_t b0_e = vhaddq_s16(a0_e, a1_e);
+ const int16x8_t b1_e = vhsubq_s16(a0_e, a1_e);
+ const int16x8_t b2_e = vhaddq_s16(a2_e, a3_e);
+ const int16x8_t b3_e = vhsubq_s16(a2_e, a3_e);
+
+ const int16x8_t b0_o = vhaddq_s16(a0_o, a1_o);
+ const int16x8_t b1_o = vhsubq_s16(a0_o, a1_o);
+ const int16x8_t b2_o = vhaddq_s16(a2_o, a3_o);
+ const int16x8_t b3_o = vhsubq_s16(a2_o, a3_o);
+
+ vst1q_s32(coeff + 0 + 0, vaddl_s16(vget_low_s16(b0_e), vget_low_s16(b2_e)));
+ vst1q_s32(coeff + 0 + 4, vaddl_s16(vget_low_s16(b0_o), vget_low_s16(b2_o)));
+ vst1q_s32(coeff + 0 + 8,
+ vaddl_s16(vget_high_s16(b0_e), vget_high_s16(b2_e)));
+ vst1q_s32(coeff + 0 + 12,
+ vaddl_s16(vget_high_s16(b0_o), vget_high_s16(b2_o)));
+
+ vst1q_s32(coeff + 64 + 0,
+ vaddl_s16(vget_low_s16(b1_e), vget_low_s16(b3_e)));
+ vst1q_s32(coeff + 64 + 4,
+ vaddl_s16(vget_low_s16(b1_o), vget_low_s16(b3_o)));
+ vst1q_s32(coeff + 64 + 8,
+ vaddl_s16(vget_high_s16(b1_e), vget_high_s16(b3_e)));
+ vst1q_s32(coeff + 64 + 12,
+ vaddl_s16(vget_high_s16(b1_o), vget_high_s16(b3_o)));
+
+ vst1q_s32(coeff + 128 + 0,
+ vsubl_s16(vget_low_s16(b0_e), vget_low_s16(b2_e)));
+ vst1q_s32(coeff + 128 + 4,
+ vsubl_s16(vget_low_s16(b0_o), vget_low_s16(b2_o)));
+ vst1q_s32(coeff + 128 + 8,
+ vsubl_s16(vget_high_s16(b0_e), vget_high_s16(b2_e)));
+ vst1q_s32(coeff + 128 + 12,
+ vsubl_s16(vget_high_s16(b0_o), vget_high_s16(b2_o)));
+
+ vst1q_s32(coeff + 192 + 0,
+ vsubl_s16(vget_low_s16(b1_e), vget_low_s16(b3_e)));
+ vst1q_s32(coeff + 192 + 4,
+ vsubl_s16(vget_low_s16(b1_o), vget_low_s16(b3_o)));
+ vst1q_s32(coeff + 192 + 8,
+ vsubl_s16(vget_high_s16(b1_e), vget_high_s16(b3_e)));
+ vst1q_s32(coeff + 192 + 12,
+ vsubl_s16(vget_high_s16(b1_o), vget_high_s16(b3_o)));
+
+ t_ptr += 16;
coeff += 16;
- }
+ } while (--h != 0);
}
void aom_hadamard_32x32_neon(const int16_t *src_diff, ptrdiff_t src_stride,