Commit 12ad11ebd0 for aom

commit 12ad11ebd0c781c6b9fb95f4de1d821cbf7e7fc5
Author: Frank Galligan <fgalligan@google.com>
Date:   Sat Nov 14 11:38:19 2026 -0800

    av1,neon: Optimize LBD/HBD IntraBC 2D convolve

    Add width-specialized horizontal and vertical passes for w <= 4 and
    w == 8 in av1_convolve_2d_sr_intrabc_neon and
    av1_highbd_convolve_2d_sr_intrabc_neon, use compact im_stride = w
    intermediate buffers, and unroll vertical filtering by 2 rows with
    sliding row registers.

    Speedup: 2.52x (LBD) and 2.69x (HBD) vs C in AV1Convolve2dSrIntrabc;
    -0.28% (1T) / -1.91% (4T) on Server Arm (AmpereOne/Neoverse N2) and
    -0.40% (1T Cortex-X4) / -0.53% (1T Cortex-A720) / -0.77% (4T, min
    -2.09%) on Pixel 9 Tensor G4 avifenc (speed 6, bit-exact).

    TAG=agy
    CONV=490488d4-7005-402a-8cc4-c245227f80fe
    Change-Id: Ibf6e03909b111111111111111111111111111111

diff --git a/av1/common/arm/convolve_neon.c b/av1/common/arm/convolve_neon.c
index 68521f2114..1a751cac50 100644
--- a/av1/common/arm/convolve_neon.c
+++ b/av1/common/arm/convolve_neon.c
@@ -1792,6 +1792,9 @@ void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,
   assert(subpel_y_qn == 8);
   assert(filter_params_x->taps == 2 && filter_params_y->taps == 2);
   assert((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS);
+  assert(w >= 2 && h >= 2);
+  assert((w & 1) == 0 && (h & 1) == 0);
+  assert(w <= 8 || (w & 15) == 0);
   (void)filter_params_x;
   (void)subpel_x_qn;
   (void)filter_params_y;
@@ -1800,7 +1803,7 @@ void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,

   uint16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE];
   int im_h = h + 1;
-  int im_stride = w;
+  const int im_stride = w <= 4 ? 4 : w;
   assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE);

   uint16_t *im = im_block;
@@ -1819,6 +1822,16 @@ void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,
       // Safe to store the whole vector, the im buffer is big enough.
       vst1_u16(im, sum);

+      src += src_stride;
+      im += im_stride;
+    } while (--im_h != 0);
+  } else if (w == 8) {
+    do {
+      uint8x8_t s0 = vld1_u8(src);
+      uint8x8_t s1 = vld1_u8(src + 1);
+
+      vst1q_u16(im, vaddl_u8(s0, s1));
+
       src += src_stride;
       im += im_stride;
     } while (--im_h != 0);
@@ -1829,16 +1842,18 @@ void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,
       int width = w;

       do {
-        uint8x8_t s0 = vld1_u8(src_ptr);
-        uint8x8_t s1 = vld1_u8(src_ptr + 1);
+        uint8x16_t s0 = vld1q_u8(src_ptr);
+        uint8x16_t s1 = vld1q_u8(src_ptr + 1);

-        uint16x8_t sum = vaddl_u8(s0, s1);
+        uint16x8_t sum0 = vaddl_u8(vget_low_u8(s0), vget_low_u8(s1));
+        uint16x8_t sum1 = vaddl_u8(vget_high_u8(s0), vget_high_u8(s1));

-        vst1q_u16(im_ptr, sum);
+        vst1q_u16(im_ptr, sum0);
+        vst1q_u16(im_ptr + 8, sum1);

-        src_ptr += 8;
-        im_ptr += 8;
-        width -= 8;
+        src_ptr += 16;
+        im_ptr += 16;
+        width -= 16;
       } while (width != 0);
       src += src_stride;
       im += im_stride;
@@ -1849,8 +1864,8 @@ void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,

   // Vertical filter.
   if (w <= 4) {
+    uint16x4_t s0 = vld1_u16(im);
     do {
-      uint16x4_t s0 = vld1_u16(im);
       uint16x4_t s1 = vld1_u16(im + im_stride);
       uint16x4_t s2 = vld1_u16(im + 2 * im_stride);

@@ -1868,6 +1883,24 @@ void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,
         store_u8_4x1(dst + 1 * dst_stride, d1);
       }

+      s0 = s2;
+      im += 2 * im_stride;
+      dst += 2 * dst_stride;
+      h -= 2;
+    } while (h != 0);
+  } else if (w == 8) {
+    uint16x8_t s0 = vld1q_u16(im);
+    do {
+      uint16x8_t s1 = vld1q_u16(im + im_stride);
+      uint16x8_t s2 = vld1q_u16(im + 2 * im_stride);
+
+      uint8x8_t d0 = vqrshrn_n_u16(vaddq_u16(s0, s1), 2);
+      uint8x8_t d1 = vqrshrn_n_u16(vaddq_u16(s1, s2), 2);
+
+      vst1_u8(dst, d0);
+      vst1_u8(dst + dst_stride, d1);
+
+      s0 = s2;
       im += 2 * im_stride;
       dst += 2 * dst_stride;
       h -= 2;
@@ -1877,22 +1910,39 @@ void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,
       uint16_t *im_ptr = im;
       uint8_t *dst_ptr = dst;
       int height = h;
+      uint16x8_t s0_lo = vld1q_u16(im_ptr);
+      uint16x8_t s0_hi = vld1q_u16(im_ptr + 8);

       do {
-        uint16x8_t s0 = vld1q_u16(im_ptr);
-        uint16x8_t s1 = vld1q_u16(im_ptr + im_stride);
-
-        uint16x8_t sum = vaddq_u16(s0, s1);
-        uint8x8_t d0 = vqrshrn_n_u16(sum, 2);
+        const uint16_t *row1 = im_ptr + im_stride;
+        const uint16_t *row2 = im_ptr + 2 * im_stride;
+        uint16x8_t s1_lo = vld1q_u16(row1);
+        uint16x8_t s1_hi = vld1q_u16(row1 + 8);
+        uint16x8_t s2_lo = vld1q_u16(row2);
+        uint16x8_t s2_hi = vld1q_u16(row2 + 8);
+
+        uint16x8_t sum0_lo = vaddq_u16(s0_lo, s1_lo);
+        uint16x8_t sum0_hi = vaddq_u16(s0_hi, s1_hi);
+        uint16x8_t sum1_lo = vaddq_u16(s1_lo, s2_lo);
+        uint16x8_t sum1_hi = vaddq_u16(s1_hi, s2_hi);
+
+        uint8x16_t d0 =
+            vcombine_u8(vqrshrn_n_u16(sum0_lo, 2), vqrshrn_n_u16(sum0_hi, 2));
+        uint8x16_t d1 =
+            vcombine_u8(vqrshrn_n_u16(sum1_lo, 2), vqrshrn_n_u16(sum1_hi, 2));

-        vst1_u8(dst_ptr, d0);
+        vst1q_u8(dst_ptr, d0);
+        vst1q_u8(dst_ptr + dst_stride, d1);

-        im_ptr += im_stride;
-        dst_ptr += dst_stride;
-      } while (--height != 0);
-      im += 8;
-      dst += 8;
-      w -= 8;
+        s0_lo = s2_lo;
+        s0_hi = s2_hi;
+        im_ptr += 2 * im_stride;
+        dst_ptr += 2 * dst_stride;
+        height -= 2;
+      } while (height != 0);
+      im += 16;
+      dst += 16;
+      w -= 16;
     } while (w != 0);
   }
 }
diff --git a/av1/common/arm/highbd_convolve_neon.c b/av1/common/arm/highbd_convolve_neon.c
index 17c711bbd5..fd5c1ab823 100644
--- a/av1/common/arm/highbd_convolve_neon.c
+++ b/av1/common/arm/highbd_convolve_neon.c
@@ -1979,17 +1979,20 @@ void av1_highbd_convolve_2d_sr_intrabc_neon(
   assert(filter_params_x->taps == 2 && filter_params_y->taps == 2);
   assert((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS);
   assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE);
-  (void)filter_params_x;
+  assert(w >= 2 && h >= 2);
+  assert((w & 1) == 0 && (h & 1) == 0);
+  assert(w <= 8 || (w & 15) == 0);
   (void)subpel_x_qn;
-  (void)filter_params_y;
   (void)subpel_y_qn;
+  (void)filter_params_x;
+  (void)filter_params_y;
   (void)conv_params;
   (void)bd;

   DECLARE_ALIGNED(16, uint16_t,
                   im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]);
   int im_h = h + 1;
-  int im_stride = MAX_SB_SIZE;
+  const int im_stride = w <= 4 ? 4 : w;

   uint16x8_t vert_offset = vdupq_n_u16(1);

@@ -1998,6 +2001,9 @@ void av1_highbd_convolve_2d_sr_intrabc_neon(
   // Horizontal filter.
   if (w <= 4) {
     do {
+      // Load at most 5 pixels per row. A full 8-pixel load would read
+      // pixels to the right of the reference block, which may be in a region
+      // that is concurrently being reconstructed by another encoder thread.
       uint16x4_t s0 = vld1_u16(src);
       uint16x4_t s1 = vld1_u16(src + 1);

@@ -2006,6 +2012,16 @@ void av1_highbd_convolve_2d_sr_intrabc_neon(
       // Safe to store the whole vector, the im buffer is big enough.
       vst1_u16(im, d0);

+      src += src_stride;
+      im += im_stride;
+    } while (--im_h != 0);
+  } else if (w == 8) {
+    do {
+      uint16x8_t s0 = vld1q_u16(src);
+      uint16x8_t s1 = vld1q_u16(src + 1);
+
+      vst1q_u16(im, vaddq_u16(s0, s1));
+
       src += src_stride;
       im += im_stride;
     } while (--im_h != 0);
@@ -2018,14 +2034,15 @@ void av1_highbd_convolve_2d_sr_intrabc_neon(
       do {
         uint16x8_t s0 = vld1q_u16(src_ptr);
         uint16x8_t s1 = vld1q_u16(src_ptr + 1);
+        uint16x8_t s2 = vld1q_u16(src_ptr + 8);
+        uint16x8_t s3 = vld1q_u16(src_ptr + 9);

-        uint16x8_t d0 = vaddq_u16(s0, s1);
-
-        vst1q_u16(im_ptr, d0);
+        vst1q_u16(im_ptr, vaddq_u16(s0, s1));
+        vst1q_u16(im_ptr + 8, vaddq_u16(s2, s3));

-        src_ptr += 8;
-        im_ptr += 8;
-        width -= 8;
+        src_ptr += 16;
+        im_ptr += 16;
+        width -= 16;
       } while (width != 0);
       src += src_stride;
       im += im_stride;
@@ -2036,40 +2053,50 @@ void av1_highbd_convolve_2d_sr_intrabc_neon(

   // Vertical filter.
   if (w <= 4) {
+    const uint16x4_t v_off = vget_low_u16(vert_offset);
+    uint16x4_t s0 = vld1_u16(im);
     do {
-      uint16x4_t s0 = vld1_u16(im);
       uint16x4_t s1 = vld1_u16(im + im_stride);
+      uint16x4_t s2 = vld1_u16(im + 2 * im_stride);

-      uint16x4_t d0 = vhadd_u16(s0, s1);
-      d0 = vhadd_u16(d0, vget_low_u16(vert_offset));
+      uint16x4_t d0 = vhadd_u16(vhadd_u16(s0, s1), v_off);
+      uint16x4_t d1 = vhadd_u16(vhadd_u16(s1, s2), v_off);

       if (w == 2) {
-        store_u16_2x1(dst, d0);
+        store_u16_2x1(dst + 0 * dst_stride, d0);
+        store_u16_2x1(dst + 1 * dst_stride, d1);
       } else {
-        vst1_u16(dst, d0);
+        vst1_u16(dst + 0 * dst_stride, d0);
+        vst1_u16(dst + 1 * dst_stride, d1);
       }

-      im += im_stride;
-      dst += dst_stride;
-    } while (--h != 0);
+      s0 = s2;
+      im += 2 * im_stride;
+      dst += 2 * dst_stride;
+      h -= 2;
+    } while (h != 0);
   } else {
     do {
       uint16_t *im_ptr = im;
       uint16_t *dst_ptr = dst;
       int height = h;
+      uint16x8_t s0 = vld1q_u16(im_ptr);

       do {
-        uint16x8_t s0 = vld1q_u16(im_ptr);
         uint16x8_t s1 = vld1q_u16(im_ptr + im_stride);
+        uint16x8_t s2 = vld1q_u16(im_ptr + 2 * im_stride);

-        uint16x8_t d0 = vhaddq_u16(s0, s1);
-        d0 = vhaddq_u16(d0, vert_offset);
+        uint16x8_t d0 = vhaddq_u16(vhaddq_u16(s0, s1), vert_offset);
+        uint16x8_t d1 = vhaddq_u16(vhaddq_u16(s1, s2), vert_offset);

         vst1q_u16(dst_ptr, d0);
+        vst1q_u16(dst_ptr + dst_stride, d1);

-        im_ptr += im_stride;
-        dst_ptr += dst_stride;
-      } while (--height != 0);
+        s0 = s2;
+        im_ptr += 2 * im_stride;
+        dst_ptr += 2 * dst_stride;
+        height -= 2;
+      } while (height != 0);
       im += 8;
       dst += 8;
       w -= 8;