Commit 47d1f2ec16 for aom

commit 47d1f2ec16ba04eabb230e53a2342dec44d24f05
Author: Frank Galligan <fgalligan@google.com>
Date:   Sat Nov 14 18:28:28 2026 -0800

    aom_dsp,neon: Optimize smooth, dr_z1 & dc

    Optimize smooth_{16,32,64}xh_neon and dc_store_8..64xh_neon with 4-row
    unrolls and vaddlv/vaddvq reductions. Add unmasked interior row fast
    paths in dr_prediction_z1_neon (shared by z1 and z3).

    Speedup: -5.1% to -6.7% (dr_prediction_z1 16x16..64x64) and -3.6%
    (intra_pred 8x16) in test_libaom on Neoverse N3; -1.82% (4T AmpereOne)
    and -1.17% (1T Cortex-X4) / -2.06% (4T) on Pixel 9 avifenc
    (bit-exact).

    TAG=agy
    CONV=490488d4-7005-402a-8cc4-c245227f80fe

    Change-Id: Ibf014d94801f2b36cc0dc3f7122577781c0c56fc

diff --git a/aom_dsp/arm/intrapred_neon.c b/aom_dsp/arm/intrapred_neon.c
index be636385b2..b49d86502f 100644
--- a/aom_dsp/arm/intrapred_neon.c
+++ b/aom_dsp/arm/intrapred_neon.c
@@ -106,35 +106,66 @@ static inline uint16x8_t horizontal_add_and_broadcast_u16x8(uint16x8_t a) {

 static inline void dc_store_8xh(uint8_t *dst, ptrdiff_t stride, int h,
                                 uint8x8_t dc) {
+#if AOM_ARCH_AARCH64
+  assert(h >= 4 && (h & 3) == 0);
+  const ptrdiff_t stride2 = 2 * stride;
+  const ptrdiff_t stride3 = 3 * stride;
+  const ptrdiff_t stride4 = 4 * stride;
+  do {
+    vst1_u8(dst, dc);
+    vst1_u8(dst + stride, dc);
+    vst1_u8(dst + stride2, dc);
+    vst1_u8(dst + stride3, dc);
+    dst += stride4;
+    h -= 4;
+  } while (h != 0);
+#else
   for (int i = 0; i < h; ++i) {
     vst1_u8(dst + i * stride, dc);
   }
+#endif
 }

 void aom_dc_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
                                const uint8_t *above, const uint8_t *left) {
   const uint8x8_t sum_top = vld1_u8(above);
   const uint8x8_t sum_left = vld1_u8(left);
+#if AOM_ARCH_AARCH64
+  const uint32_t sum = vaddlv_u8(sum_top) + vaddlv_u8(sum_left);
+  const uint8x8_t dc = vdup_n_u8((uint8_t)((sum + 8) >> 4));
+  dc_store_8xh(dst, stride, 8, dc);
+#else
   uint16x8_t sum = vaddl_u8(sum_left, sum_top);
   sum = horizontal_add_and_broadcast_u16x8(sum);
   const uint8x8_t dc0 = vrshrn_n_u16(sum, 4);
   dc_store_8xh(dst, stride, 8, vdup_lane_u8(dc0, 0));
+#endif
 }

 void aom_dc_left_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
                                     const uint8_t *above, const uint8_t *left) {
+  (void)above;
+#if AOM_ARCH_AARCH64
+  const uint32_t sum = vaddlv_u8(vld1_u8(left));
+  dc_store_8xh(dst, stride, 8, vdup_n_u8((uint8_t)((sum + 4) >> 3)));
+#else
   const uint16x8_t sum_left = dc_load_sum_8(left);
   const uint8x8_t dc0 = vrshrn_n_u16(sum_left, 3);
-  (void)above;
   dc_store_8xh(dst, stride, 8, vdup_lane_u8(dc0, 0));
+#endif
 }

 void aom_dc_top_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
                                    const uint8_t *above, const uint8_t *left) {
+  (void)left;
+#if AOM_ARCH_AARCH64
+  const uint32_t sum = vaddlv_u8(vld1_u8(above));
+  dc_store_8xh(dst, stride, 8, vdup_n_u8((uint8_t)((sum + 4) >> 3)));
+#else
   const uint16x8_t sum_top = dc_load_sum_8(above);
   const uint8x8_t dc0 = vrshrn_n_u16(sum_top, 3);
-  (void)left;
   dc_store_8xh(dst, stride, 8, vdup_lane_u8(dc0, 0));
+#endif
 }

 void aom_dc_128_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
@@ -162,37 +193,67 @@ static inline uint16x8_t dc_load_sum_16(const uint8_t *in) {

 static inline void dc_store_16xh(uint8_t *dst, ptrdiff_t stride, int h,
                                  uint8x16_t dc) {
+#if AOM_ARCH_AARCH64
+  assert(h >= 4 && (h & 3) == 0);
+  const ptrdiff_t stride2 = 2 * stride;
+  const ptrdiff_t stride3 = 3 * stride;
+  const ptrdiff_t stride4 = 4 * stride;
+  do {
+    vst1q_u8(dst, dc);
+    vst1q_u8(dst + stride, dc);
+    vst1q_u8(dst + stride2, dc);
+    vst1q_u8(dst + stride3, dc);
+    dst += stride4;
+    h -= 4;
+  } while (h != 0);
+#else
   for (int i = 0; i < h; ++i) {
     vst1q_u8(dst + i * stride, dc);
   }
+#endif
 }

 void aom_dc_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
                                  const uint8_t *above, const uint8_t *left) {
+#if AOM_ARCH_AARCH64
+  const uint32_t sum = vaddlvq_u8(vld1q_u8(above)) + vaddlvq_u8(vld1q_u8(left));
+  dc_store_16xh(dst, stride, 16, vdupq_n_u8((uint8_t)((sum + 16) >> 5)));
+#else
   const uint16x8_t sum_top = dc_load_partial_sum_16(above);
   const uint16x8_t sum_left = dc_load_partial_sum_16(left);
   uint16x8_t sum = vaddq_u16(sum_left, sum_top);
   sum = horizontal_add_and_broadcast_u16x8(sum);
   const uint8x8_t dc0 = vrshrn_n_u16(sum, 5);
   dc_store_16xh(dst, stride, 16, vdupq_lane_u8(dc0, 0));
+#endif
 }

 void aom_dc_left_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
                                       const uint8_t *above,
                                       const uint8_t *left) {
+  (void)above;
+#if AOM_ARCH_AARCH64
+  const uint32_t sum = vaddlvq_u8(vld1q_u8(left));
+  dc_store_16xh(dst, stride, 16, vdupq_n_u8((uint8_t)((sum + 8) >> 4)));
+#else
   const uint16x8_t sum_left = dc_load_sum_16(left);
   const uint8x8_t dc0 = vrshrn_n_u16(sum_left, 4);
-  (void)above;
   dc_store_16xh(dst, stride, 16, vdupq_lane_u8(dc0, 0));
+#endif
 }

 void aom_dc_top_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
                                      const uint8_t *above,
                                      const uint8_t *left) {
+  (void)left;
+#if AOM_ARCH_AARCH64
+  const uint32_t sum = vaddlvq_u8(vld1q_u8(above));
+  dc_store_16xh(dst, stride, 16, vdupq_n_u8((uint8_t)((sum + 8) >> 4)));
+#else
   const uint16x8_t sum_top = dc_load_sum_16(above);
   const uint8x8_t dc0 = vrshrn_n_u16(sum_top, 4);
-  (void)left;
   dc_store_16xh(dst, stride, 16, vdupq_lane_u8(dc0, 0));
+#endif
 }

 void aom_dc_128_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
@@ -222,38 +283,72 @@ static inline uint16x8_t dc_load_sum_32(const uint8_t *in) {

 static inline void dc_store_32xh(uint8_t *dst, ptrdiff_t stride, int h,
                                  uint8x16_t dc) {
+#if AOM_ARCH_AARCH64
+  assert(h >= 4 && (h & 3) == 0);
+  const ptrdiff_t stride2 = 2 * stride;
+  const ptrdiff_t stride3 = 3 * stride;
+  const ptrdiff_t stride4 = 4 * stride;
+  do {
+    vst1q_u8(dst + 0, dc);
+    vst1q_u8(dst + 16, dc);
+    vst1q_u8(dst + stride + 0, dc);
+    vst1q_u8(dst + stride + 16, dc);
+    vst1q_u8(dst + stride2 + 0, dc);
+    vst1q_u8(dst + stride2 + 16, dc);
+    vst1q_u8(dst + stride3 + 0, dc);
+    vst1q_u8(dst + stride3 + 16, dc);
+    dst += stride4;
+    h -= 4;
+  } while (h != 0);
+#else
   for (int i = 0; i < h; ++i) {
     vst1q_u8(dst + i * stride, dc);
     vst1q_u8(dst + i * stride + 16, dc);
   }
+#endif
 }

 void aom_dc_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
                                  const uint8_t *above, const uint8_t *left) {
   const uint16x8_t sum_top = dc_load_partial_sum_32(above);
   const uint16x8_t sum_left = dc_load_partial_sum_32(left);
-  uint16x8_t sum = vaddq_u16(sum_left, sum_top);
-  sum = horizontal_add_and_broadcast_u16x8(sum);
-  const uint8x8_t dc0 = vrshrn_n_u16(sum, 6);
+  const uint16x8_t sum = vaddq_u16(sum_left, sum_top);
+#if AOM_ARCH_AARCH64
+  const uint32_t s = vaddvq_u16(sum);
+  dc_store_32xh(dst, stride, 32, vdupq_n_u8((uint8_t)((s + 32) >> 6)));
+#else
+  const uint8x8_t dc0 =
+      vrshrn_n_u16(horizontal_add_and_broadcast_u16x8(sum), 6);
   dc_store_32xh(dst, stride, 32, vdupq_lane_u8(dc0, 0));
+#endif
 }

 void aom_dc_left_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
                                       const uint8_t *above,
                                       const uint8_t *left) {
+  (void)above;
+#if AOM_ARCH_AARCH64
+  const uint32_t sum = vaddvq_u16(dc_load_partial_sum_32(left));
+  dc_store_32xh(dst, stride, 32, vdupq_n_u8((uint8_t)((sum + 16) >> 5)));
+#else
   const uint16x8_t sum_left = dc_load_sum_32(left);
   const uint8x8_t dc0 = vrshrn_n_u16(sum_left, 5);
-  (void)above;
   dc_store_32xh(dst, stride, 32, vdupq_lane_u8(dc0, 0));
+#endif
 }

 void aom_dc_top_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
                                      const uint8_t *above,
                                      const uint8_t *left) {
+  (void)left;
+#if AOM_ARCH_AARCH64
+  const uint32_t sum = vaddvq_u16(dc_load_partial_sum_32(above));
+  dc_store_32xh(dst, stride, 32, vdupq_n_u8((uint8_t)((sum + 16) >> 5)));
+#else
   const uint16x8_t sum_top = dc_load_sum_32(above);
   const uint8x8_t dc0 = vrshrn_n_u16(sum_top, 5);
-  (void)left;
   dc_store_32xh(dst, stride, 32, vdupq_lane_u8(dc0, 0));
+#endif
 }

 void aom_dc_128_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
@@ -287,40 +382,82 @@ static inline uint16x8_t dc_load_sum_64(const uint8_t *in) {

 static inline void dc_store_64xh(uint8_t *dst, ptrdiff_t stride, int h,
                                  uint8x16_t dc) {
+#if AOM_ARCH_AARCH64
+  assert(h >= 4 && (h & 3) == 0);
+  const ptrdiff_t stride2 = 2 * stride;
+  const ptrdiff_t stride3 = 3 * stride;
+  const ptrdiff_t stride4 = 4 * stride;
+  do {
+    vst1q_u8(dst + 0, dc);
+    vst1q_u8(dst + 16, dc);
+    vst1q_u8(dst + 32, dc);
+    vst1q_u8(dst + 48, dc);
+    vst1q_u8(dst + stride + 0, dc);
+    vst1q_u8(dst + stride + 16, dc);
+    vst1q_u8(dst + stride + 32, dc);
+    vst1q_u8(dst + stride + 48, dc);
+    vst1q_u8(dst + stride2 + 0, dc);
+    vst1q_u8(dst + stride2 + 16, dc);
+    vst1q_u8(dst + stride2 + 32, dc);
+    vst1q_u8(dst + stride2 + 48, dc);
+    vst1q_u8(dst + stride3 + 0, dc);
+    vst1q_u8(dst + stride3 + 16, dc);
+    vst1q_u8(dst + stride3 + 32, dc);
+    vst1q_u8(dst + stride3 + 48, dc);
+    dst += stride4;
+    h -= 4;
+  } while (h != 0);
+#else
   for (int i = 0; i < h; ++i) {
     vst1q_u8(dst + i * stride, dc);
     vst1q_u8(dst + i * stride + 16, dc);
     vst1q_u8(dst + i * stride + 32, dc);
     vst1q_u8(dst + i * stride + 48, dc);
   }
+#endif
 }

 void aom_dc_predictor_64x64_neon(uint8_t *dst, ptrdiff_t stride,
                                  const uint8_t *above, const uint8_t *left) {
   const uint16x8_t sum_top = dc_load_partial_sum_64(above);
   const uint16x8_t sum_left = dc_load_partial_sum_64(left);
-  uint16x8_t sum = vaddq_u16(sum_left, sum_top);
-  sum = horizontal_add_and_broadcast_u16x8(sum);
-  const uint8x8_t dc0 = vrshrn_n_u16(sum, 7);
+  const uint16x8_t sum = vaddq_u16(sum_left, sum_top);
+#if AOM_ARCH_AARCH64
+  const uint32_t s = vaddvq_u16(sum);
+  dc_store_64xh(dst, stride, 64, vdupq_n_u8((uint8_t)((s + 64) >> 7)));
+#else
+  const uint8x8_t dc0 =
+      vrshrn_n_u16(horizontal_add_and_broadcast_u16x8(sum), 7);
   dc_store_64xh(dst, stride, 64, vdupq_lane_u8(dc0, 0));
+#endif
 }

 void aom_dc_left_predictor_64x64_neon(uint8_t *dst, ptrdiff_t stride,
                                       const uint8_t *above,
                                       const uint8_t *left) {
+  (void)above;
+#if AOM_ARCH_AARCH64
+  const uint32_t sum = vaddvq_u16(dc_load_partial_sum_64(left));
+  dc_store_64xh(dst, stride, 64, vdupq_n_u8((uint8_t)((sum + 32) >> 6)));
+#else
   const uint16x8_t sum_left = dc_load_sum_64(left);
   const uint8x8_t dc0 = vrshrn_n_u16(sum_left, 6);
-  (void)above;
   dc_store_64xh(dst, stride, 64, vdupq_lane_u8(dc0, 0));
+#endif
 }

 void aom_dc_top_predictor_64x64_neon(uint8_t *dst, ptrdiff_t stride,
                                      const uint8_t *above,
                                      const uint8_t *left) {
+  (void)left;
+#if AOM_ARCH_AARCH64
+  const uint32_t sum = vaddvq_u16(dc_load_partial_sum_64(above));
+  dc_store_64xh(dst, stride, 64, vdupq_n_u8((uint8_t)((sum + 32) >> 6)));
+#else
   const uint16x8_t sum_top = dc_load_sum_64(above);
   const uint8x8_t dc0 = vrshrn_n_u16(sum_top, 6);
-  (void)left;
   dc_store_64xh(dst, stride, 64, vdupq_lane_u8(dc0, 0));
+#endif
 }

 void aom_dc_128_predictor_64x64_neon(uint8_t *dst, ptrdiff_t stride,
@@ -1183,7 +1320,30 @@ static AOM_FORCE_INLINE void dr_prediction_z1_WxH_internal_neon_small(
   const uint8x8_t a_mbase_x = vdup_n_u8(above[max_base_x]);

   int x = dx;
-  for (int r = 0; r < H; r++) {
+  int r = 0;
+  const int w_thresh = W << upsample_above;
+  for (; r < H; r++) {
+    const int base = x >> frac_bits;
+    if (max_base_x - base < w_thresh) break;
+
+    uint8x8x2_t a01_128;
+    uint16x8_t shift;
+    if (upsample_above) {
+      a01_128 = vld2_u8(above + base);
+      shift = vdupq_n_u16(x & 0x1f);
+    } else {
+      a01_128.val[0] = vld1_u8(above + base);
+      a01_128.val[1] = vld1_u8(above + base + 1);
+      shift = vdupq_n_u16((x & 0x3f) >> 1);
+    }
+    const uint16x8_t diff = vsubl_u8(a01_128.val[1], a01_128.val[0]);
+    const uint16x8_t a32 = vshll_n_u8(a01_128.val[0], 5);
+    const uint16x8_t res = vmlaq_u16(a32, diff, shift);
+    dst[r] = vrshrn_n_u16(res, 5);
+    x += dx;
+  }
+
+  for (; r < H; r++) {
     int base = x >> frac_bits;
     int base_max_diff = (max_base_x - base) >> upsample_above;
     if (base_max_diff <= 0) {
@@ -1410,8 +1570,31 @@ static void dr_prediction_z1_64xH_neon(int H, uint8_t *dst, ptrdiff_t stride,

   const uint8x16_t a_mbase_x = vdupq_n_u8(above[max_base_x]);

+  int r = 0;
   int x = dx;
-  for (int r = 0; r < H; r++, dst += stride) {
+  for (; r < H; r++) {
+    const int base = x >> frac_bits;
+    if (base + 64 >= max_base_x) break;
+    const uint16x8_t shift = vdupq_n_u16((x & 0x3f) >> 1);
+    for (int j = 0; j < 64; j += 16) {
+      const uint8x16_t a0_128 = vld1q_u8(above + base + j);
+      const uint8x16_t a1_128 = vld1q_u8(above + base + j + 1);
+      const uint16x8_t diff_lo =
+          vsubl_u8(vget_low_u8(a1_128), vget_low_u8(a0_128));
+      const uint16x8_t diff_hi =
+          vsubl_u8(vget_high_u8(a1_128), vget_high_u8(a0_128));
+      const uint16x8_t a32_lo = vshll_n_u8(vget_low_u8(a0_128), 5);
+      const uint16x8_t a32_hi = vshll_n_u8(vget_high_u8(a0_128), 5);
+      const uint16x8_t res_lo = vmlaq_u16(a32_lo, diff_lo, shift);
+      const uint16x8_t res_hi = vmlaq_u16(a32_hi, diff_hi, shift);
+      vst1q_u8(dst + j,
+               vcombine_u8(vrshrn_n_u16(res_lo, 5), vrshrn_n_u16(res_hi, 5)));
+    }
+    dst += stride;
+    x += dx;
+  }
+
+  for (; r < H; r++, dst += stride) {
     int base = x >> frac_bits;
     if (base >= max_base_x) {
       for (int i = r; i < H; ++i) {
@@ -2483,26 +2666,34 @@ SMOOTH_NXM(8, 32)
 #undef SMOOTH_NXM

 static inline uint8x16_t calculate_weights_and_predq(
-    const uint8x16_t top, const uint8x8_t left, const uint8x8_t top_right,
-    const uint8x8_t weights_y, const uint8x16_t weights_x,
-    const uint8x16_t scaled_weights_x, const uint16x8_t weighted_bl) {
+    const uint8x16_t top, const uint8x16_t left,
+    const uint16x8_t weighted_tr_low, const uint16x8_t weighted_tr_high,
+    const uint8x16_t weights_y, const uint8x16_t weights_x,
+    const uint16x8_t weighted_bl) {
   const uint16x8_t weighted_top_bl_low =
-      vmlal_u8(weighted_bl, weights_y, vget_low_u8(top));
-  const uint16x8_t weighted_left_low = vmull_u8(vget_low_u8(weights_x), left);
+      vmlal_u8(weighted_bl, vget_low_u8(weights_y), vget_low_u8(top));
   const uint16x8_t weighted_left_tr_low =
-      vmlal_u8(weighted_left_low, vget_low_u8(scaled_weights_x), top_right);
+      vmlal_u8(weighted_tr_low, vget_low_u8(weights_x), vget_low_u8(left));
   const uint8x8_t result_low =
       calculate_pred(weighted_top_bl_low, weighted_left_tr_low);

+#if AOM_ARCH_AARCH64
   const uint16x8_t weighted_top_bl_high =
-      vmlal_u8(weighted_bl, weights_y, vget_high_u8(top));
-  const uint16x8_t weighted_left_high = vmull_u8(vget_high_u8(weights_x), left);
+      vmlal_high_u8(weighted_bl, weights_y, top);
   const uint16x8_t weighted_left_tr_high =
-      vmlal_u8(weighted_left_high, vget_high_u8(scaled_weights_x), top_right);
+      vmlal_high_u8(weighted_tr_high, weights_x, left);
+  const uint16x8_t avg_high =
+      vhaddq_u16(weighted_top_bl_high, weighted_left_tr_high);
+  return vrshrn_high_n_u16(result_low, avg_high, SMOOTH_WEIGHT_LOG2_SCALE);
+#else
+  const uint16x8_t weighted_top_bl_high =
+      vmlal_u8(weighted_bl, vget_high_u8(weights_y), vget_high_u8(top));
+  const uint16x8_t weighted_left_tr_high =
+      vmlal_u8(weighted_tr_high, vget_high_u8(weights_x), vget_high_u8(left));
   const uint8x8_t result_high =
       calculate_pred(weighted_top_bl_high, weighted_left_tr_high);
-
   return vcombine_u8(result_low, result_high);
+#endif
 }

 // 256 - v = vneg_s8(v)
@@ -2511,77 +2702,108 @@ static inline uint8x16_t negate_s8q(const uint8x16_t v) {
 }

 // For width 16 and above.
-#define SMOOTH_PREDICTOR(W)                                                 \
-  static void smooth_##W##xh_neon(                                          \
-      uint8_t *dst, ptrdiff_t stride, const uint8_t *const top_row,         \
-      const uint8_t *const left_column, const int height) {                 \
-    const uint8_t top_right = top_row[(W) - 1];                             \
-    const uint8_t bottom_left = left_column[height - 1];                    \
-    const uint8_t *const weights_y = smooth_weights + height - 4;           \
-                                                                            \
-    uint8x16_t top_v[4];                                                    \
-    top_v[0] = vld1q_u8(top_row);                                           \
-    if ((W) > 16) {                                                         \
-      top_v[1] = vld1q_u8(top_row + 16);                                    \
-      if ((W) == 64) {                                                      \
-        top_v[2] = vld1q_u8(top_row + 32);                                  \
-        top_v[3] = vld1q_u8(top_row + 48);                                  \
-      }                                                                     \
-    }                                                                       \
-                                                                            \
-    const uint8x8_t top_right_v = vdup_n_u8(top_right);                     \
-    const uint8x8_t bottom_left_v = vdup_n_u8(bottom_left);                 \
-                                                                            \
-    uint8x16_t weights_x_v[4];                                              \
-    weights_x_v[0] = vld1q_u8(smooth_weights + (W) - 4);                    \
-    if ((W) > 16) {                                                         \
-      weights_x_v[1] = vld1q_u8(smooth_weights + (W) + 16 - 4);             \
-      if ((W) == 64) {                                                      \
-        weights_x_v[2] = vld1q_u8(smooth_weights + (W) + 32 - 4);           \
-        weights_x_v[3] = vld1q_u8(smooth_weights + (W) + 48 - 4);           \
-      }                                                                     \
-    }                                                                       \
-                                                                            \
-    uint8x16_t scaled_weights_x[4];                                         \
-    scaled_weights_x[0] = negate_s8q(weights_x_v[0]);                       \
-    if ((W) > 16) {                                                         \
-      scaled_weights_x[1] = negate_s8q(weights_x_v[1]);                     \
-      if ((W) == 64) {                                                      \
-        scaled_weights_x[2] = negate_s8q(weights_x_v[2]);                   \
-        scaled_weights_x[3] = negate_s8q(weights_x_v[3]);                   \
-      }                                                                     \
-    }                                                                       \
-                                                                            \
-    for (int y = 0; y < height; ++y) {                                      \
-      const uint8x8_t left_v = vdup_n_u8(left_column[y]);                   \
-      const uint8x8_t weights_y_v = vdup_n_u8(weights_y[y]);                \
-      const uint8x8_t scaled_weights_y = negate_s8(weights_y_v);            \
-      const uint16x8_t weighted_bl =                                        \
-          vmull_u8(scaled_weights_y, bottom_left_v);                        \
-                                                                            \
-      vst1q_u8(dst, calculate_weights_and_predq(                            \
-                        top_v[0], left_v, top_right_v, weights_y_v,         \
-                        weights_x_v[0], scaled_weights_x[0], weighted_bl)); \
-                                                                            \
-      if ((W) > 16) {                                                       \
-        vst1q_u8(dst + 16,                                                  \
-                 calculate_weights_and_predq(                               \
-                     top_v[1], left_v, top_right_v, weights_y_v,            \
-                     weights_x_v[1], scaled_weights_x[1], weighted_bl));    \
-        if ((W) == 64) {                                                    \
-          vst1q_u8(dst + 32,                                                \
-                   calculate_weights_and_predq(                             \
-                       top_v[2], left_v, top_right_v, weights_y_v,          \
-                       weights_x_v[2], scaled_weights_x[2], weighted_bl));  \
-          vst1q_u8(dst + 48,                                                \
-                   calculate_weights_and_predq(                             \
-                       top_v[3], left_v, top_right_v, weights_y_v,          \
-                       weights_x_v[3], scaled_weights_x[3], weighted_bl));  \
-        }                                                                   \
-      }                                                                     \
-                                                                            \
-      dst += stride;                                                        \
-    }                                                                       \
+#define SMOOTH_PREDICTOR(W)                                                  \
+  static void smooth_##W##xh_neon(                                           \
+      uint8_t *dst, ptrdiff_t stride, const uint8_t *const top_row,          \
+      const uint8_t *const left_column, const int height) {                  \
+    const uint8_t top_right = top_row[(W) - 1];                              \
+    const uint8_t bottom_left = left_column[height - 1];                     \
+    const uint8_t *const weights_y = smooth_weights + height - 4;            \
+                                                                             \
+    uint8x16_t top_v[4];                                                     \
+    top_v[0] = vld1q_u8(top_row);                                            \
+    if ((W) > 16) {                                                          \
+      top_v[1] = vld1q_u8(top_row + 16);                                     \
+      if ((W) == 64) {                                                       \
+        top_v[2] = vld1q_u8(top_row + 32);                                   \
+        top_v[3] = vld1q_u8(top_row + 48);                                   \
+      }                                                                      \
+    }                                                                        \
+                                                                             \
+    const uint8x8_t top_right_v = vdup_n_u8(top_right);                      \
+    const uint8x8_t bottom_left_v = vdup_n_u8(bottom_left);                  \
+                                                                             \
+    uint8x16_t weights_x_v[4];                                               \
+    weights_x_v[0] = vld1q_u8(smooth_weights + (W) - 4);                     \
+    if ((W) > 16) {                                                          \
+      weights_x_v[1] = vld1q_u8(smooth_weights + (W) + 16 - 4);              \
+      if ((W) == 64) {                                                       \
+        weights_x_v[2] = vld1q_u8(smooth_weights + (W) + 32 - 4);            \
+        weights_x_v[3] = vld1q_u8(smooth_weights + (W) + 48 - 4);            \
+      }                                                                      \
+    }                                                                        \
+                                                                             \
+    uint16x8_t weighted_tr_low[4];                                           \
+    uint16x8_t weighted_tr_high[4];                                          \
+    const uint8x16_t swx0 = negate_s8q(weights_x_v[0]);                      \
+    weighted_tr_low[0] = vmull_u8(vget_low_u8(swx0), top_right_v);           \
+    weighted_tr_high[0] = vmull_u8(vget_high_u8(swx0), top_right_v);         \
+    if ((W) > 16) {                                                          \
+      const uint8x16_t swx1 = negate_s8q(weights_x_v[1]);                    \
+      weighted_tr_low[1] = vmull_u8(vget_low_u8(swx1), top_right_v);         \
+      weighted_tr_high[1] = vmull_u8(vget_high_u8(swx1), top_right_v);       \
+      if ((W) == 64) {                                                       \
+        const uint8x16_t swx2 = negate_s8q(weights_x_v[2]);                  \
+        const uint8x16_t swx3 = negate_s8q(weights_x_v[3]);                  \
+        weighted_tr_low[2] = vmull_u8(vget_low_u8(swx2), top_right_v);       \
+        weighted_tr_high[2] = vmull_u8(vget_high_u8(swx2), top_right_v);     \
+        weighted_tr_low[3] = vmull_u8(vget_low_u8(swx3), top_right_v);       \
+        weighted_tr_high[3] = vmull_u8(vget_high_u8(swx3), top_right_v);     \
+      }                                                                      \
+    }                                                                        \
+                                                                             \
+    assert(height >= 4 && (height & 1) == 0);                                \
+    int y = 0;                                                               \
+    do {                                                                     \
+      const uint8x16_t left_0 = vdupq_n_u8(left_column[y]);                  \
+      const uint8x16_t left_1 = vdupq_n_u8(left_column[y + 1]);              \
+      const uint8x16_t wy_0 = vdupq_n_u8(weights_y[y]);                      \
+      const uint8x16_t wy_1 = vdupq_n_u8(weights_y[y + 1]);                  \
+      const uint8x8_t swy_0 = negate_s8(vget_low_u8(wy_0));                  \
+      const uint8x8_t swy_1 = negate_s8(vget_low_u8(wy_1));                  \
+      const uint16x8_t wbl_0 = vmull_u8(swy_0, bottom_left_v);               \
+      const uint16x8_t wbl_1 = vmull_u8(swy_1, bottom_left_v);               \
+                                                                             \
+      uint8_t *dst1 = dst + stride;                                          \
+      vst1q_u8(dst, calculate_weights_and_predq(                             \
+                        top_v[0], left_0, weighted_tr_low[0],                \
+                        weighted_tr_high[0], wy_0, weights_x_v[0], wbl_0));  \
+      vst1q_u8(dst1, calculate_weights_and_predq(                            \
+                         top_v[0], left_1, weighted_tr_low[0],               \
+                         weighted_tr_high[0], wy_1, weights_x_v[0], wbl_1)); \
+                                                                             \
+      if ((W) > 16) {                                                        \
+        vst1q_u8(dst + 16,                                                   \
+                 calculate_weights_and_predq(                                \
+                     top_v[1], left_0, weighted_tr_low[1],                   \
+                     weighted_tr_high[1], wy_0, weights_x_v[1], wbl_0));     \
+        vst1q_u8(dst1 + 16,                                                  \
+                 calculate_weights_and_predq(                                \
+                     top_v[1], left_1, weighted_tr_low[1],                   \
+                     weighted_tr_high[1], wy_1, weights_x_v[1], wbl_1));     \
+        if ((W) == 64) {                                                     \
+          vst1q_u8(dst + 32,                                                 \
+                   calculate_weights_and_predq(                              \
+                       top_v[2], left_0, weighted_tr_low[2],                 \
+                       weighted_tr_high[2], wy_0, weights_x_v[2], wbl_0));   \
+          vst1q_u8(dst1 + 32,                                                \
+                   calculate_weights_and_predq(                              \
+                       top_v[2], left_1, weighted_tr_low[2],                 \
+                       weighted_tr_high[2], wy_1, weights_x_v[2], wbl_1));   \
+          vst1q_u8(dst + 48,                                                 \
+                   calculate_weights_and_predq(                              \
+                       top_v[3], left_0, weighted_tr_low[3],                 \
+                       weighted_tr_high[3], wy_0, weights_x_v[3], wbl_0));   \
+          vst1q_u8(dst1 + 48,                                                \
+                   calculate_weights_and_predq(                              \
+                       top_v[3], left_1, weighted_tr_low[3],                 \
+                       weighted_tr_high[3], wy_1, weights_x_v[3], wbl_1));   \
+        }                                                                    \
+      }                                                                      \
+                                                                             \
+      dst += 2 * stride;                                                     \
+      y += 2;                                                                \
+    } while (y != height);                                                   \
   }

 SMOOTH_PREDICTOR(16)