Commit fcc5bf455d for aom

commit fcc5bf455d21ead1b8343a07a1503342d8b4307a
Author: Frank Galligan <fgalligan@google.com>
Date:   Tue Nov 17 06:28:30 2026 -0800

    aom_dsp,neon: Optimize satd & hadamard_16x16

    - aom_satd_neon(): assert length invariants, add a length == 16 fast
      path, and unroll length >= 32 loops to 32 elements with 4 accumulators
      (vabsq_s32 + vaddq_s32 vs 2 vabaq_s32 accumulators).
    - aom_satd_lp_neon(): assert length invariants, add a length == 16 fast
      path reducing in uint16, and unroll length >= 32 loops to 32 elements
      with pair-wise uint16 sums before vpadalq_u16.
    - aom_hadamard_16x16_neon(): compute the four 8x8 sub-transforms in
      16-bit via aom_hadamard_lp_8x8_neon() and combine quadrants with
      vhaddq_s16/vhsubq_s16 before widening via vaddl_s16/vsubl_s16.

    Speedup (Pixel 9 Tensor G4, 1T, test_libaom DISABLED_Speed vs
    origin/main, bit-exact across tier1 768x1024 realtime speed 6/7):
    - SatdLpTest (16/64/256/1024):
      A720: -7.77% / +0.65% / -0.39% / -0.73% (839->767us at 16, 1.09x)
      A520: -8.72% / -11.63% / -26.24% / -31.31% (1.10x..1.46x)
    - SatdTest (16/64/256/1024):
      A720: +0.00% / +0.14% / +0.03% / -0.02% (neutral)
      A520: +0.13% / -21.74% / -27.35% / -29.74% (1.28x..1.42x for >=64)
    - HadamardLowbdTest (16x16 / 32x32):
      A720: -21.58% (128850->102838us, 1.25x) / -14.21% (778973->669592us)

    TAG=agy
    CONV=bc5228f2-7fa6-4522-aad6-34935d739e28
    Change-Id: Icef8651208537da1de24e2d6c37bd8fe67023029

diff --git a/aom_dsp/arm/avg_neon.c b/aom_dsp/arm/avg_neon.c
index 7d6054cb0a..5fb6ed7b99 100644
--- a/aom_dsp/arm/avg_neon.c
+++ b/aom_dsp/arm/avg_neon.c
@@ -55,35 +55,39 @@ void aom_avg_8x8_quad_neon(const uint8_t *s, int p, int x16_idx, int y16_idx,
   avg[3] = aom_avg_8x8_neon(s + (y16_idx + 8) * p + (x16_idx + 8), p);
 }

-int aom_satd_lp_neon(const int16_t *coeff, int length) {
-  int16x8_t s0 = vld1q_s16(coeff);
-  int16x8_t s1 = vld1q_s16(coeff + 8);
+static inline uint16x8_t load_abs_s16(const int16_t *p) {
+  return vreinterpretq_u16_s16(vabsq_s16(vld1q_s16(p)));
+}

-  int16x8_t abs0 = vabsq_s16(s0);
-  int16x8_t abs1 = vabsq_s16(s1);
+// coeff: 16 bits, dynamic range [-32640, 32640].
+// |coeff[i]| + |coeff[i + 8]| <= 65280 fits in uint16_t without wrapping.
+// length: value range {16, 32, 64, 128, 256, 512, 1024}.
+int aom_satd_lp_neon(const int16_t *coeff, int length) {
+  assert(length == 16 || (length >= 32 && length % 32 == 0));
+  const uint16x8_t sum0 =
+      vaddq_u16(load_abs_s16(coeff), load_abs_s16(coeff + 8));
+  if (length == 16) {
+    return (int)horizontal_add_u16x8(sum0);
+  }

-  int32x4_t acc0 = vpaddlq_s16(abs0);
-  int32x4_t acc1 = vpaddlq_s16(abs1);
+  uint32x4_t acc0 = vpaddlq_u16(sum0);
+  uint32x4_t acc1 = vpaddlq_u16(
+      vaddq_u16(load_abs_s16(coeff + 16), load_abs_s16(coeff + 24)));

-  length -= 16;
-  coeff += 16;
+  length -= 32;
+  coeff += 32;

   while (length != 0) {
-    s0 = vld1q_s16(coeff);
-    s1 = vld1q_s16(coeff + 8);
-
-    abs0 = vabsq_s16(s0);
-    abs1 = vabsq_s16(s1);
+    acc0 = vpadalq_u16(acc0,
+                       vaddq_u16(load_abs_s16(coeff), load_abs_s16(coeff + 8)));
+    acc1 = vpadalq_u16(
+        acc1, vaddq_u16(load_abs_s16(coeff + 16), load_abs_s16(coeff + 24)));

-    acc0 = vpadalq_s16(acc0, abs0);
-    acc1 = vpadalq_s16(acc1, abs1);
-
-    length -= 16;
-    coeff += 16;
+    length -= 32;
+    coeff += 32;
   }

-  int32x4_t accum = vaddq_s32(acc0, acc1);
-  return horizontal_add_s32x4(accum);
+  return (int)horizontal_add_u32x4(vaddq_u32(acc0, acc1));
 }

 void aom_int_pro_row_neon(int16_t *hbuf, const uint8_t *ref,
@@ -178,38 +182,66 @@ void aom_int_pro_col_neon(int16_t *vbuf, const uint8_t *ref,
 // coeff: 20 bits, dynamic range [-524287, 524287].
 // length: value range {16, 32, 64, 128, 256, 512, 1024}.
 int aom_satd_neon(const tran_low_t *coeff, int length) {
-  const int32x4_t zero = vdupq_n_s32(0);
+  assert(length == 16 || (length >= 32 && length % 32 == 0));
+  if (length == 16) {
+    const int32x4_t s0 = vld1q_s32(&coeff[0]);
+    const int32x4_t s1 = vld1q_s32(&coeff[4]);
+    const int32x4_t s2 = vld1q_s32(&coeff[8]);
+    const int32x4_t s3 = vld1q_s32(&coeff[12]);
+
+    const int32x4_t a0 = vabsq_s32(s0);
+    const int32x4_t a1 = vabsq_s32(s1);
+    const int32x4_t a2 = vabsq_s32(s2);
+    const int32x4_t a3 = vabsq_s32(s3);
+
+    return horizontal_add_s32x4(
+        vaddq_s32(vaddq_s32(a0, a1), vaddq_s32(a2, a3)));
+  }

-  int32x4_t s0 = vld1q_s32(&coeff[0]);
-  int32x4_t s1 = vld1q_s32(&coeff[4]);
-  int32x4_t s2 = vld1q_s32(&coeff[8]);
-  int32x4_t s3 = vld1q_s32(&coeff[12]);
+  const int32x4_t init0 = vld1q_s32(&coeff[0]);
+  const int32x4_t init1 = vld1q_s32(&coeff[4]);
+  const int32x4_t init2 = vld1q_s32(&coeff[8]);
+  const int32x4_t init3 = vld1q_s32(&coeff[12]);
+  const int32x4_t init4 = vld1q_s32(&coeff[16]);
+  const int32x4_t init5 = vld1q_s32(&coeff[20]);
+  const int32x4_t init6 = vld1q_s32(&coeff[24]);
+  const int32x4_t init7 = vld1q_s32(&coeff[28]);

-  int32x4_t accum0 = vabsq_s32(s0);
-  int32x4_t accum1 = vabsq_s32(s2);
-  accum0 = vabaq_s32(accum0, s1, zero);
-  accum1 = vabaq_s32(accum1, s3, zero);
+  int32x4_t acc0 = vaddq_s32(vabsq_s32(init0), vabsq_s32(init4));
+  int32x4_t acc1 = vaddq_s32(vabsq_s32(init1), vabsq_s32(init5));
+  int32x4_t acc2 = vaddq_s32(vabsq_s32(init2), vabsq_s32(init6));
+  int32x4_t acc3 = vaddq_s32(vabsq_s32(init3), vabsq_s32(init7));

-  length -= 16;
-  coeff += 16;
+  length -= 32;
+  coeff += 32;

   while (length != 0) {
-    s0 = vld1q_s32(&coeff[0]);
-    s1 = vld1q_s32(&coeff[4]);
-    s2 = vld1q_s32(&coeff[8]);
-    s3 = vld1q_s32(&coeff[12]);
-
-    accum0 = vabaq_s32(accum0, s0, zero);
-    accum1 = vabaq_s32(accum1, s1, zero);
-    accum0 = vabaq_s32(accum0, s2, zero);
-    accum1 = vabaq_s32(accum1, s3, zero);
-
-    length -= 16;
-    coeff += 16;
+    const int32x4_t s0 = vld1q_s32(&coeff[0]);
+    const int32x4_t s1 = vld1q_s32(&coeff[4]);
+    const int32x4_t s2 = vld1q_s32(&coeff[8]);
+    const int32x4_t s3 = vld1q_s32(&coeff[12]);
+    const int32x4_t s4 = vld1q_s32(&coeff[16]);
+    const int32x4_t s5 = vld1q_s32(&coeff[20]);
+    const int32x4_t s6 = vld1q_s32(&coeff[24]);
+    const int32x4_t s7 = vld1q_s32(&coeff[28]);
+
+    const int32x4_t sum0 = vaddq_s32(vabsq_s32(s0), vabsq_s32(s4));
+    const int32x4_t sum1 = vaddq_s32(vabsq_s32(s1), vabsq_s32(s5));
+    const int32x4_t sum2 = vaddq_s32(vabsq_s32(s2), vabsq_s32(s6));
+    const int32x4_t sum3 = vaddq_s32(vabsq_s32(s3), vabsq_s32(s7));
+
+    acc0 = vaddq_s32(acc0, sum0);
+    acc1 = vaddq_s32(acc1, sum1);
+    acc2 = vaddq_s32(acc2, sum2);
+    acc3 = vaddq_s32(acc3, sum3);
+
+    length -= 32;
+    coeff += 32;
   }

   // satd: 30 bits, dynamic range [-524287 * 1024, 524287 * 1024]
-  return horizontal_add_s32x4(vaddq_s32(accum0, accum1));
+  return horizontal_add_s32x4(
+      vaddq_s32(vaddq_s32(acc0, acc1), vaddq_s32(acc2, acc3)));
 }

 int aom_vector_var_neon(const int16_t *ref, const int16_t *src, int bwl) {
diff --git a/aom_dsp/arm/hadamard_neon.c b/aom_dsp/arm/hadamard_neon.c
index 4b100d23e8..83c94cfab7 100644
--- a/aom_dsp/arm/hadamard_neon.c
+++ b/aom_dsp/arm/hadamard_neon.c
@@ -187,103 +187,87 @@ void aom_hadamard_lp_16x16_neon(const int16_t *src_diff, ptrdiff_t src_stride,

 void aom_hadamard_16x16_neon(const int16_t *src_diff, ptrdiff_t src_stride,
                              tran_low_t *coeff) {
+  int16_t t_coeff[256];
   /* Rearrange 16x16 to 8x32 and remove stride.
    * Top left first. */
-  aom_hadamard_8x8_neon(src_diff + 0 + 0 * src_stride, src_stride, coeff + 0);
+  aom_hadamard_lp_8x8_neon(src_diff + 0 + 0 * src_stride, src_stride,
+                           t_coeff + 0);
   /* Top right. */
-  aom_hadamard_8x8_neon(src_diff + 8 + 0 * src_stride, src_stride, coeff + 64);
+  aom_hadamard_lp_8x8_neon(src_diff + 8 + 0 * src_stride, src_stride,
+                           t_coeff + 64);
   /* Bottom left. */
-  aom_hadamard_8x8_neon(src_diff + 0 + 8 * src_stride, src_stride, coeff + 128);
+  aom_hadamard_lp_8x8_neon(src_diff + 0 + 8 * src_stride, src_stride,
+                           t_coeff + 128);
   /* Bottom right. */
-  aom_hadamard_8x8_neon(src_diff + 8 + 8 * src_stride, src_stride, coeff + 192);
+  aom_hadamard_lp_8x8_neon(src_diff + 8 + 8 * src_stride, src_stride,
+                           t_coeff + 192);

   // Each iteration of the loop operates on entire rows (16 samples each)
   // because we need to swap the second and third quarters of every row in the
   // output to match AVX2 output (i.e., aom_hadamard_16x16_avx2). See the for
   // loop at the end of aom_hadamard_16x16_c.
-  for (int i = 0; i < 64; i += 16) {
-    const int32x4_t a00 = vld1q_s32(coeff + 0);
-    const int32x4_t a01 = vld1q_s32(coeff + 64);
-    const int32x4_t a02 = vld1q_s32(coeff + 128);
-    const int32x4_t a03 = vld1q_s32(coeff + 192);
-
-    const int32x4_t b00 = vhaddq_s32(a00, a01);
-    const int32x4_t b01 = vhsubq_s32(a00, a01);
-    const int32x4_t b02 = vhaddq_s32(a02, a03);
-    const int32x4_t b03 = vhsubq_s32(a02, a03);
-
-    const int32x4_t c00 = vaddq_s32(b00, b02);
-    const int32x4_t c01 = vaddq_s32(b01, b03);
-    const int32x4_t c02 = vsubq_s32(b00, b02);
-    const int32x4_t c03 = vsubq_s32(b01, b03);
-
-    const int32x4_t a10 = vld1q_s32(coeff + 4 + 0);
-    const int32x4_t a11 = vld1q_s32(coeff + 4 + 64);
-    const int32x4_t a12 = vld1q_s32(coeff + 4 + 128);
-    const int32x4_t a13 = vld1q_s32(coeff + 4 + 192);
-
-    const int32x4_t b10 = vhaddq_s32(a10, a11);
-    const int32x4_t b11 = vhsubq_s32(a10, a11);
-    const int32x4_t b12 = vhaddq_s32(a12, a13);
-    const int32x4_t b13 = vhsubq_s32(a12, a13);
-
-    const int32x4_t c10 = vaddq_s32(b10, b12);
-    const int32x4_t c11 = vaddq_s32(b11, b13);
-    const int32x4_t c12 = vsubq_s32(b10, b12);
-    const int32x4_t c13 = vsubq_s32(b11, b13);
-
-    const int32x4_t a20 = vld1q_s32(coeff + 8 + 0);
-    const int32x4_t a21 = vld1q_s32(coeff + 8 + 64);
-    const int32x4_t a22 = vld1q_s32(coeff + 8 + 128);
-    const int32x4_t a23 = vld1q_s32(coeff + 8 + 192);
-
-    const int32x4_t b20 = vhaddq_s32(a20, a21);
-    const int32x4_t b21 = vhsubq_s32(a20, a21);
-    const int32x4_t b22 = vhaddq_s32(a22, a23);
-    const int32x4_t b23 = vhsubq_s32(a22, a23);
-
-    const int32x4_t c20 = vaddq_s32(b20, b22);
-    const int32x4_t c21 = vaddq_s32(b21, b23);
-    const int32x4_t c22 = vsubq_s32(b20, b22);
-    const int32x4_t c23 = vsubq_s32(b21, b23);
-
-    const int32x4_t a30 = vld1q_s32(coeff + 12 + 0);
-    const int32x4_t a31 = vld1q_s32(coeff + 12 + 64);
-    const int32x4_t a32 = vld1q_s32(coeff + 12 + 128);
-    const int32x4_t a33 = vld1q_s32(coeff + 12 + 192);
-
-    const int32x4_t b30 = vhaddq_s32(a30, a31);
-    const int32x4_t b31 = vhsubq_s32(a30, a31);
-    const int32x4_t b32 = vhaddq_s32(a32, a33);
-    const int32x4_t b33 = vhsubq_s32(a32, a33);
-
-    const int32x4_t c30 = vaddq_s32(b30, b32);
-    const int32x4_t c31 = vaddq_s32(b31, b33);
-    const int32x4_t c32 = vsubq_s32(b30, b32);
-    const int32x4_t c33 = vsubq_s32(b31, b33);
-
-    vst1q_s32(coeff + 0 + 0, c00);
-    vst1q_s32(coeff + 0 + 4, c20);
-    vst1q_s32(coeff + 0 + 8, c10);
-    vst1q_s32(coeff + 0 + 12, c30);
-
-    vst1q_s32(coeff + 64 + 0, c01);
-    vst1q_s32(coeff + 64 + 4, c21);
-    vst1q_s32(coeff + 64 + 8, c11);
-    vst1q_s32(coeff + 64 + 12, c31);
-
-    vst1q_s32(coeff + 128 + 0, c02);
-    vst1q_s32(coeff + 128 + 4, c22);
-    vst1q_s32(coeff + 128 + 8, c12);
-    vst1q_s32(coeff + 128 + 12, c32);
-
-    vst1q_s32(coeff + 192 + 0, c03);
-    vst1q_s32(coeff + 192 + 4, c23);
-    vst1q_s32(coeff + 192 + 8, c13);
-    vst1q_s32(coeff + 192 + 12, c33);
-
+  // For 8-bit inputs aom_hadamard_lp_8x8_neon() outputs 15-bit values in
+  // [-16320, 16320], and for any int16_t input vhaddq_s16/vhsubq_s16 halve
+  // exactly before widening via vaddl_s16/vsubl_s16.
+  const int16_t *t_ptr = t_coeff;
+  int h = 4;
+  do {
+    const int16x8_t a0_e = vld1q_s16(t_ptr + 0);
+    const int16x8_t a0_o = vld1q_s16(t_ptr + 8);
+    const int16x8_t a1_e = vld1q_s16(t_ptr + 64);
+    const int16x8_t a1_o = vld1q_s16(t_ptr + 72);
+    const int16x8_t a2_e = vld1q_s16(t_ptr + 128);
+    const int16x8_t a2_o = vld1q_s16(t_ptr + 136);
+    const int16x8_t a3_e = vld1q_s16(t_ptr + 192);
+    const int16x8_t a3_o = vld1q_s16(t_ptr + 200);
+
+    const int16x8_t b0_e = vhaddq_s16(a0_e, a1_e);
+    const int16x8_t b1_e = vhsubq_s16(a0_e, a1_e);
+    const int16x8_t b2_e = vhaddq_s16(a2_e, a3_e);
+    const int16x8_t b3_e = vhsubq_s16(a2_e, a3_e);
+
+    const int16x8_t b0_o = vhaddq_s16(a0_o, a1_o);
+    const int16x8_t b1_o = vhsubq_s16(a0_o, a1_o);
+    const int16x8_t b2_o = vhaddq_s16(a2_o, a3_o);
+    const int16x8_t b3_o = vhsubq_s16(a2_o, a3_o);
+
+    vst1q_s32(coeff + 0 + 0, vaddl_s16(vget_low_s16(b0_e), vget_low_s16(b2_e)));
+    vst1q_s32(coeff + 0 + 4, vaddl_s16(vget_low_s16(b0_o), vget_low_s16(b2_o)));
+    vst1q_s32(coeff + 0 + 8,
+              vaddl_s16(vget_high_s16(b0_e), vget_high_s16(b2_e)));
+    vst1q_s32(coeff + 0 + 12,
+              vaddl_s16(vget_high_s16(b0_o), vget_high_s16(b2_o)));
+
+    vst1q_s32(coeff + 64 + 0,
+              vaddl_s16(vget_low_s16(b1_e), vget_low_s16(b3_e)));
+    vst1q_s32(coeff + 64 + 4,
+              vaddl_s16(vget_low_s16(b1_o), vget_low_s16(b3_o)));
+    vst1q_s32(coeff + 64 + 8,
+              vaddl_s16(vget_high_s16(b1_e), vget_high_s16(b3_e)));
+    vst1q_s32(coeff + 64 + 12,
+              vaddl_s16(vget_high_s16(b1_o), vget_high_s16(b3_o)));
+
+    vst1q_s32(coeff + 128 + 0,
+              vsubl_s16(vget_low_s16(b0_e), vget_low_s16(b2_e)));
+    vst1q_s32(coeff + 128 + 4,
+              vsubl_s16(vget_low_s16(b0_o), vget_low_s16(b2_o)));
+    vst1q_s32(coeff + 128 + 8,
+              vsubl_s16(vget_high_s16(b0_e), vget_high_s16(b2_e)));
+    vst1q_s32(coeff + 128 + 12,
+              vsubl_s16(vget_high_s16(b0_o), vget_high_s16(b2_o)));
+
+    vst1q_s32(coeff + 192 + 0,
+              vsubl_s16(vget_low_s16(b1_e), vget_low_s16(b3_e)));
+    vst1q_s32(coeff + 192 + 4,
+              vsubl_s16(vget_low_s16(b1_o), vget_low_s16(b3_o)));
+    vst1q_s32(coeff + 192 + 8,
+              vsubl_s16(vget_high_s16(b1_e), vget_high_s16(b3_e)));
+    vst1q_s32(coeff + 192 + 12,
+              vsubl_s16(vget_high_s16(b1_o), vget_high_s16(b3_o)));
+
+    t_ptr += 16;
     coeff += 16;
-  }
+  } while (--h != 0);
 }

 void aom_hadamard_32x32_neon(const int16_t *src_diff, ptrdiff_t src_stride,