Commit 827f24c7dd for aom

commit 827f24c7dd464255c4db8e378f6084c13c085268
Author: Frank Galligan <fgalligan@google.com>
Date:   Tue Nov 17 06:28:30 2026 -0800

    aom_dsp,neon: Unroll aom_subtract_block_neon

    Unroll the cols == 16 and cols == 8 row loops 4x in
    aom_subtract_block_neon() using load_u8_16x4(), load_u8_8x4(), and
    store_s16_8x4() to avoid 3 * stride GPR pressure. Convert cols == 4 to
    store_s16_4x4(), hoist the row counter, and convert cols == 32 to
    do/while (r != 0). Assert 4-row-multiple and power-of-two cols >= 4
    inputs, dropping unreachable remainder tails and cols < 4 fallback to
    eliminate ARMv7 GPR spills in cols == 4 and cols == 32.

    Kernel microbenchmark (Pixel 9 Cortex-A720 taskset 0x20, 3-round median,
    NEON/AV1SubtractBlockTest.DISABLED_Speed/* vs origin/main):
    4x4 1.23x (-18.9%), 4x8 1.12x (-10.7%), 4x16 1.15x (-13.3%),
    8x4 1.08x (-7.6%), 16x4 1.09x (-8.4%), 16x8 1.12x (-10.4%),
    16x16 1.08x (-7.7%), 16x64 1.06x (-5.3%), 32x32 1.00x (-0.1%),
    64x64 1.00x (-0.2%), 128x128 1.02x (-1.9%).

    Speedup (Pixel 9 Tensor G4 Cortex-A720, 1T mask 0x20, realtime
    768x1024 intra, bit-exact): -0.57% (speed 6), -0.61% (speed 7),
    -0.59% overall.

    TAG=agy
    CONV=bc5228f2-7fa6-4522-aad6-34935d739e28
    Change-Id: I76950d0462bff3b48101d3ae7908ae65d1db6561

diff --git a/aom_dsp/arm/subtract_neon.c b/aom_dsp/arm/subtract_neon.c
index 2da31d93f7..d655180a27 100644
--- a/aom_dsp/arm/subtract_neon.c
+++ b/aom_dsp/arm/subtract_neon.c
@@ -10,6 +10,7 @@
  */

 #include <arm_neon.h>
+#include <assert.h>

 #include "config/aom_config.h"
 #include "config/aom_dsp_rtcd.h"
@@ -22,10 +23,12 @@ void aom_subtract_block_neon(int rows, int cols, int16_t *diff,
                              ptrdiff_t diff_stride, const uint8_t *src,
                              ptrdiff_t src_stride, const uint8_t *pred,
                              ptrdiff_t pred_stride) {
+  assert(rows >= 4 && (rows & 3) == 0);
+  assert(cols >= 4 && (cols & (cols - 1)) == 0);
+  int r = rows;
   if (cols > 16) {
-    int r = rows;
     if (cols == 32) {
-      while (r >= 2) {
+      do {
         const uint8x16_t s0_0 = vld1q_u8(&src[0]);
         const uint8x16_t s0_1 = vld1q_u8(&src[16]);
         const uint8x16_t p0_0 = vld1q_u8(&pred[0]);
@@ -63,21 +66,7 @@ void aom_subtract_block_neon(int rows, int cols, int16_t *diff,
         pred += pred_stride << 1;
         src += src_stride << 1;
         r -= 2;
-      }
-      if (r != 0) {
-        const uint8x16_t s0 = vld1q_u8(&src[0]);
-        const uint8x16_t s1 = vld1q_u8(&src[16]);
-        const uint8x16_t p0 = vld1q_u8(&pred[0]);
-        const uint8x16_t p1 = vld1q_u8(&pred[16]);
-        vst1q_s16(&diff[0], vreinterpretq_s16_u16(
-                                vsubl_u8(vget_low_u8(s0), vget_low_u8(p0))));
-        vst1q_s16(&diff[8], vreinterpretq_s16_u16(
-                                vsubl_u8(vget_high_u8(s0), vget_high_u8(p0))));
-        vst1q_s16(&diff[16], vreinterpretq_s16_u16(
-                                 vsubl_u8(vget_low_u8(s1), vget_low_u8(p1))));
-        vst1q_s16(&diff[24], vreinterpretq_s16_u16(
-                                 vsubl_u8(vget_high_u8(s1), vget_high_u8(p1))));
-      }
+      } while (r != 0);
     } else {
       do {
         int c = 0;
@@ -106,116 +95,81 @@ void aom_subtract_block_neon(int rows, int cols, int16_t *diff,
       } while (--r != 0);
     }
   } else if (cols > 8) {
-    int r = rows;
-    while (r >= 2) {
-      const uint8x16_t v_src0 = vld1q_u8(&src[0]);
-      const uint8x16_t v_pred0 = vld1q_u8(&pred[0]);
-      const uint8x16_t v_src1 = vld1q_u8(&src[src_stride]);
-      const uint8x16_t v_pred1 = vld1q_u8(&pred[pred_stride]);
-      const uint16x8_t v_diff0_lo =
-          vsubl_u8(vget_low_u8(v_src0), vget_low_u8(v_pred0));
-      const uint16x8_t v_diff0_hi =
-          vsubl_u8(vget_high_u8(v_src0), vget_high_u8(v_pred0));
-      const uint16x8_t v_diff1_lo =
-          vsubl_u8(vget_low_u8(v_src1), vget_low_u8(v_pred1));
-      const uint16x8_t v_diff1_hi =
-          vsubl_u8(vget_high_u8(v_src1), vget_high_u8(v_pred1));
-      vst1q_s16(&diff[0], vreinterpretq_s16_u16(v_diff0_lo));
-      vst1q_s16(&diff[8], vreinterpretq_s16_u16(v_diff0_hi));
-      vst1q_s16(&diff[diff_stride + 0], vreinterpretq_s16_u16(v_diff1_lo));
-      vst1q_s16(&diff[diff_stride + 8], vreinterpretq_s16_u16(v_diff1_hi));
-      diff += diff_stride << 1;
-      pred += pred_stride << 1;
-      src += src_stride << 1;
-      r -= 2;
-    }
-    if (r != 0) {
-      const uint8x16_t v_src = vld1q_u8(&src[0]);
-      const uint8x16_t v_pred = vld1q_u8(&pred[0]);
-      const uint16x8_t v_diff_lo =
-          vsubl_u8(vget_low_u8(v_src), vget_low_u8(v_pred));
-      const uint16x8_t v_diff_hi =
-          vsubl_u8(vget_high_u8(v_src), vget_high_u8(v_pred));
-      vst1q_s16(&diff[0], vreinterpretq_s16_u16(v_diff_lo));
-      vst1q_s16(&diff[8], vreinterpretq_s16_u16(v_diff_hi));
-    }
+    do {
+      uint8x16_t v_src0, v_src1, v_src2, v_src3;
+      uint8x16_t v_pred0, v_pred1, v_pred2, v_pred3;
+      load_u8_16x4(src, src_stride, &v_src0, &v_src1, &v_src2, &v_src3);
+      load_u8_16x4(pred, pred_stride, &v_pred0, &v_pred1, &v_pred2, &v_pred3);
+      const int16x8_t v_diff0_lo = vreinterpretq_s16_u16(
+          vsubl_u8(vget_low_u8(v_src0), vget_low_u8(v_pred0)));
+      const int16x8_t v_diff0_hi = vreinterpretq_s16_u16(
+          vsubl_u8(vget_high_u8(v_src0), vget_high_u8(v_pred0)));
+      const int16x8_t v_diff1_lo = vreinterpretq_s16_u16(
+          vsubl_u8(vget_low_u8(v_src1), vget_low_u8(v_pred1)));
+      const int16x8_t v_diff1_hi = vreinterpretq_s16_u16(
+          vsubl_u8(vget_high_u8(v_src1), vget_high_u8(v_pred1)));
+      const int16x8_t v_diff2_lo = vreinterpretq_s16_u16(
+          vsubl_u8(vget_low_u8(v_src2), vget_low_u8(v_pred2)));
+      const int16x8_t v_diff2_hi = vreinterpretq_s16_u16(
+          vsubl_u8(vget_high_u8(v_src2), vget_high_u8(v_pred2)));
+      const int16x8_t v_diff3_lo = vreinterpretq_s16_u16(
+          vsubl_u8(vget_low_u8(v_src3), vget_low_u8(v_pred3)));
+      const int16x8_t v_diff3_hi = vreinterpretq_s16_u16(
+          vsubl_u8(vget_high_u8(v_src3), vget_high_u8(v_pred3)));
+      vst1q_s16(diff, v_diff0_lo);
+      vst1q_s16(diff + 8, v_diff0_hi);
+      diff += diff_stride;
+      vst1q_s16(diff, v_diff1_lo);
+      vst1q_s16(diff + 8, v_diff1_hi);
+      diff += diff_stride;
+      vst1q_s16(diff, v_diff2_lo);
+      vst1q_s16(diff + 8, v_diff2_hi);
+      diff += diff_stride;
+      vst1q_s16(diff, v_diff3_lo);
+      vst1q_s16(diff + 8, v_diff3_hi);
+      diff += diff_stride;
+      pred += 4 * pred_stride;
+      src += 4 * src_stride;
+      r -= 4;
+    } while (r != 0);
   } else if (cols > 4) {
-    int r = rows;
-    while (r >= 2) {
-      const uint8x8_t v_src0 = vld1_u8(&src[0]);
-      const uint8x8_t v_src1 = vld1_u8(&src[src_stride]);
-      const uint8x8_t v_pred0 = vld1_u8(&pred[0]);
-      const uint8x8_t v_pred1 = vld1_u8(&pred[pred_stride]);
-      const uint16x8_t v_diff0 = vsubl_u8(v_src0, v_pred0);
-      const uint16x8_t v_diff1 = vsubl_u8(v_src1, v_pred1);
-      vst1q_s16(&diff[0], vreinterpretq_s16_u16(v_diff0));
-      vst1q_s16(&diff[diff_stride], vreinterpretq_s16_u16(v_diff1));
-      diff += diff_stride << 1;
-      pred += pred_stride << 1;
-      src += src_stride << 1;
-      r -= 2;
-    }
-    if (r != 0) {
-      do {
-        const uint8x8_t v_src = vld1_u8(&src[0]);
-        const uint8x8_t v_pred = vld1_u8(&pred[0]);
-        const uint16x8_t v_diff = vsubl_u8(v_src, v_pred);
-        vst1q_s16(&diff[0], vreinterpretq_s16_u16(v_diff));
-        diff += diff_stride;
-        pred += pred_stride;
-        src += src_stride;
-      } while (--r != 0);
-    }
-  } else if (cols == 4) {
-    int r = rows;
-    if (r >= 4) {
-      do {
-        uint8x8_t v_src_01, v_src_23, v_pred_01, v_pred_23;
-        load_unaligned_u8_4x4(src, src_stride, &v_src_01, &v_src_23);
-        load_unaligned_u8_4x4(pred, pred_stride, &v_pred_01, &v_pred_23);
-        const int16x8_t v_diff_01 =
-            vreinterpretq_s16_u16(vsubl_u8(v_src_01, v_pred_01));
-        const int16x8_t v_diff_23 =
-            vreinterpretq_s16_u16(vsubl_u8(v_src_23, v_pred_23));
-        vst1_s16(&diff[0 * diff_stride], vget_low_s16(v_diff_01));
-        vst1_s16(&diff[1 * diff_stride], vget_high_s16(v_diff_01));
-        vst1_s16(&diff[2 * diff_stride], vget_low_s16(v_diff_23));
-        vst1_s16(&diff[3 * diff_stride], vget_high_s16(v_diff_23));
-        diff += diff_stride << 2;
-        pred += pred_stride << 2;
-        src += src_stride << 2;
-        r -= 4;
-      } while (r >= 4);
-    }
-    while (r >= 2) {
-      const uint8x8_t v_src_01 = load_unaligned_u8_4x2(src, src_stride);
-      const uint8x8_t v_pred_01 = load_unaligned_u8_4x2(pred, pred_stride);
-      const int16x8_t v_diff_01 =
-          vreinterpretq_s16_u16(vsubl_u8(v_src_01, v_pred_01));
-      vst1_s16(&diff[0 * diff_stride], vget_low_s16(v_diff_01));
-      vst1_s16(&diff[1 * diff_stride], vget_high_s16(v_diff_01));
-      diff += diff_stride << 1;
-      pred += pred_stride << 1;
-      src += src_stride << 1;
-      r -= 2;
-    }
-    if (r != 0) {
-      const uint8x8_t v_src = load_unaligned_u8_4x1(src);
-      const uint8x8_t v_pred = load_unaligned_u8_4x1(pred);
-      const int16x8_t v_diff = vreinterpretq_s16_u16(vsubl_u8(v_src, v_pred));
-      vst1_s16(diff, vget_low_s16(v_diff));
-    }
+    do {
+      uint8x8_t v_src0, v_src1, v_src2, v_src3;
+      uint8x8_t v_pred0, v_pred1, v_pred2, v_pred3;
+      load_u8_8x4(src, src_stride, &v_src0, &v_src1, &v_src2, &v_src3);
+      load_u8_8x4(pred, pred_stride, &v_pred0, &v_pred1, &v_pred2, &v_pred3);
+      const int16x8_t v_diff0 =
+          vreinterpretq_s16_u16(vsubl_u8(v_src0, v_pred0));
+      const int16x8_t v_diff1 =
+          vreinterpretq_s16_u16(vsubl_u8(v_src1, v_pred1));
+      const int16x8_t v_diff2 =
+          vreinterpretq_s16_u16(vsubl_u8(v_src2, v_pred2));
+      const int16x8_t v_diff3 =
+          vreinterpretq_s16_u16(vsubl_u8(v_src3, v_pred3));
+      store_s16_8x4(diff, diff_stride, v_diff0, v_diff1, v_diff2, v_diff3);
+      diff += 4 * diff_stride;
+      pred += 4 * pred_stride;
+      src += 4 * src_stride;
+      r -= 4;
+    } while (r != 0);
   } else {
-    int r = rows;
+    assert(cols == 4);
     do {
-      int c = 0;
-      do {
-        diff[c] = src[c] - pred[c];
-      } while (++c < cols);
-      diff += diff_stride;
-      pred += pred_stride;
-      src += src_stride;
-    } while (--r != 0);
+      uint8x8_t v_src_01, v_src_23, v_pred_01, v_pred_23;
+      load_unaligned_u8_4x4(src, src_stride, &v_src_01, &v_src_23);
+      load_unaligned_u8_4x4(pred, pred_stride, &v_pred_01, &v_pred_23);
+      const int16x8_t v_diff_01 =
+          vreinterpretq_s16_u16(vsubl_u8(v_src_01, v_pred_01));
+      const int16x8_t v_diff_23 =
+          vreinterpretq_s16_u16(vsubl_u8(v_src_23, v_pred_23));
+      store_s16_4x4(diff, diff_stride, vget_low_s16(v_diff_01),
+                    vget_high_s16(v_diff_01), vget_low_s16(v_diff_23),
+                    vget_high_s16(v_diff_23));
+      diff += diff_stride << 2;
+      pred += pred_stride << 2;
+      src += src_stride << 2;
+      r -= 4;
+    } while (r != 0);
   }
 }