Commit f05d742bb0 for aom

commit f05d742bb016d7fa2066f15af6e26a5f982f0fd5
Author: Frank Galligan <fgalligan@google.com>
Date:   Tue Nov 17 06:28:31 2026 -0800

    aom_dsp,neon: Optimize dr_prediction_z1/z2/z3

    Speed up NEON dr_prediction_z1 and z2 paths (and z3 via the shared
    small-block z1 helper). Eliminate dstvec stack buffers via direct
    stores in z1 4xH..32xH, split clipped/unclipped z1 loops, hoist
    loop-invariant vqtbl dispatch and base_x == -1 fast paths in z2,
    and use incremental vqtbl index stepping for AArch64 left prediction.

    Speedup (Pixel 9 Tensor G4 Cortex-A720, 1T, realtime 768x1024 intra,
    bit-exact): -3.76% (speed 6), -5.35% (speed 7) encode time (-4.52%
    paired tier1 median); test_libaom NEON/LowbdDrPredTest.DISABLED_Speed
    median (us): z1 108943 -> 91632 (-15.89%, 1.19x), z2 325905 -> 290656
    (-10.82%, 1.12x), z3 163012 -> 153753 (-5.68%, 1.06x).

    TAG=agy
    CONV=bc5228f2-7fa6-4522-aad6-34935d739e28
    Change-Id: I2fcdde47e4071ae5667854910de6e77b4ade3d30

diff --git a/aom_dsp/arm/intrapred_neon.c b/aom_dsp/arm/intrapred_neon.c
index b49d86502f..6111a81227 100644
--- a/aom_dsp/arm/intrapred_neon.c
+++ b/aom_dsp/arm/intrapred_neon.c
@@ -1307,22 +1307,126 @@ static DECLARE_ALIGNED(32, const uint8_t, BaseMask[33][32]) = {
     0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 };

-static AOM_FORCE_INLINE void dr_prediction_z1_WxH_internal_neon_small(
-    int W, int H, uint8x8_t *dst, const uint8_t *above, int upsample_above,
-    int dx) {
+static AOM_FORCE_INLINE void dr_prediction_z1_WxH_core_neon_small(
+    int W, int H, uint8x8_t *dstvec, uint8_t *dst, ptrdiff_t stride,
+    const uint8_t *above, int upsample_above, int dx, int to_vec) {
   const int frac_bits = 6 - upsample_above;
   const int max_base_x = ((W + H) - 1) << upsample_above;

+  assert(W == 4 || W == 8);
+  assert(H >= 4 && (H & 3) == 0);
+  assert(upsample_above == 0 || W + H <= 16);
   assert(dx > 0);
   //   above0 * (32 - shift) + above1 * shift
   // = (above1 - above0) * shift + above0 * 32

-  const uint8x8_t a_mbase_x = vdup_n_u8(above[max_base_x]);
-
   int x = dx;
-  int r = 0;
   const int w_thresh = W << upsample_above;
-  for (; r < H; r++) {
+  if (to_vec && H <= 8) {
+    if (max_base_x - ((H * dx) >> frac_bits) >= w_thresh) {
+      do {
+        const int base = x >> frac_bits;
+        uint8x8x2_t a01_128;
+        uint16x8_t shift;
+        if (upsample_above) {
+          a01_128 = vld2_u8(above + base);
+          shift = vdupq_n_u16(x & 0x1f);
+        } else {
+          a01_128.val[0] = vld1_u8(above + base);
+          a01_128.val[1] = vld1_u8(above + base + 1);
+          shift = vdupq_n_u16((x & 0x3f) >> 1);
+        }
+        const uint16x8_t diff = vsubl_u8(a01_128.val[1], a01_128.val[0]);
+        const uint16x8_t a32 = vshll_n_u8(a01_128.val[0], 5);
+        const uint16x8_t res = vmlaq_u16(a32, diff, shift);
+        *dstvec++ = vrshrn_n_u16(res, 5);
+        x += dx;
+      } while (--H != 0);
+      return;
+    }
+
+    const uint8x8_t a_mbase_x = vdup_n_u8(above[max_base_x]);
+    do {
+      const int base = x >> frac_bits;
+      int base_max_diff = (max_base_x - base) >> upsample_above;
+      uint8x8_t row;
+      if (base_max_diff <= 0) {
+        row = a_mbase_x;
+      } else {
+        if (base_max_diff > W) base_max_diff = W;
+
+        uint8x8x2_t a01_128;
+        uint16x8_t shift;
+        if (upsample_above) {
+          a01_128 = vld2_u8(above + base);
+          shift = vdupq_n_u16(x & 0x1f);
+        } else {
+          a01_128.val[0] = vld1_u8(above + base);
+          a01_128.val[1] = vld1_u8(above + base + 1);
+          shift = vdupq_n_u16((x & 0x3f) >> 1);
+        }
+        const uint16x8_t diff = vsubl_u8(a01_128.val[1], a01_128.val[0]);
+        const uint16x8_t a32 = vshll_n_u8(a01_128.val[0], 5);
+        const uint16x8_t res = vmlaq_u16(a32, diff, shift);
+
+        const uint8x8_t mask = vld1_u8(BaseMask[base_max_diff]);
+        row = vbsl_u8(mask, vrshrn_n_u16(res, 5), a_mbase_x);
+      }
+      *dstvec++ = row;
+      x += dx;
+    } while (--H != 0);
+    return;
+  }
+
+  // base = x >> frac_bits increases monotonically with dx > 0; last row checks
+  // full-block bounds for unclipped 2-row generation.
+  if (max_base_x - ((H * dx) >> frac_bits) >= w_thresh) {
+    do {
+      const int x0 = x;
+      const int x1 = x + dx;
+      x += 2 * dx;
+      const int b0 = x0 >> frac_bits;
+      const int b1 = x1 >> frac_bits;
+      uint8x8x2_t a01_0, a01_1;
+      uint16x8_t s0, s1;
+      if (upsample_above) {
+        a01_0 = vld2_u8(above + b0);
+        a01_1 = vld2_u8(above + b1);
+        s0 = vdupq_n_u16(x0 & 0x1f);
+        s1 = vdupq_n_u16(x1 & 0x1f);
+      } else {
+        a01_0.val[0] = vld1_u8(above + b0);
+        a01_0.val[1] = vld1_u8(above + b0 + 1);
+        a01_1.val[0] = vld1_u8(above + b1);
+        a01_1.val[1] = vld1_u8(above + b1 + 1);
+        s0 = vdupq_n_u16((x0 & 0x3f) >> 1);
+        s1 = vdupq_n_u16((x1 & 0x3f) >> 1);
+      }
+      const uint16x8_t d0 = vsubl_u8(a01_0.val[1], a01_0.val[0]);
+      const uint16x8_t d1 = vsubl_u8(a01_1.val[1], a01_1.val[0]);
+      const uint16x8_t a32_0 = vshll_n_u8(a01_0.val[0], 5);
+      const uint16x8_t a32_1 = vshll_n_u8(a01_1.val[0], 5);
+      const uint8x8_t row0 = vrshrn_n_u16(vmlaq_u16(a32_0, d0, s0), 5);
+      const uint8x8_t row1 = vrshrn_n_u16(vmlaq_u16(a32_1, d1, s1), 5);
+      if (to_vec) {
+        dstvec[0] = row0;
+        dstvec[1] = row1;
+        dstvec += 2;
+      } else if (W == 4) {
+        store_u8_4x1(dst, row0);
+        store_u8_4x1(dst + stride, row1);
+        dst += 2 * stride;
+      } else {
+        vst1_u8(dst, row0);
+        vst1_u8(dst + stride, row1);
+        dst += 2 * stride;
+      }
+      H -= 2;
+    } while (H != 0);
+    return;
+  }
+
+  for (; H != 0; --H) {
     const int base = x >> frac_bits;
     if (max_base_x - base < w_thresh) break;

@@ -1339,22 +1443,40 @@ static AOM_FORCE_INLINE void dr_prediction_z1_WxH_internal_neon_small(
     const uint16x8_t diff = vsubl_u8(a01_128.val[1], a01_128.val[0]);
     const uint16x8_t a32 = vshll_n_u8(a01_128.val[0], 5);
     const uint16x8_t res = vmlaq_u16(a32, diff, shift);
-    dst[r] = vrshrn_n_u16(res, 5);
+    const uint8x8_t row = vrshrn_n_u16(res, 5);
+    if (to_vec) {
+      *dstvec++ = row;
+    } else if (W == 4) {
+      store_u8_4x1(dst, row);
+      dst += stride;
+    } else {
+      vst1_u8(dst, row);
+      dst += stride;
+    }
     x += dx;
   }
+  if (H == 0) return;

-  for (; r < H; r++) {
-    int base = x >> frac_bits;
-    int base_max_diff = (max_base_x - base) >> upsample_above;
+  const uint8x8_t a_mbase_x = vdup_n_u8(above[max_base_x]);
+  do {
+    const int base = x >> frac_bits;
+    const int base_max_diff = (max_base_x - base) >> upsample_above;
     if (base_max_diff <= 0) {
-      for (int i = r; i < H; ++i) {
-        dst[i] = a_mbase_x;  // save 4 values
-      }
+      do {
+        if (to_vec) {
+          *dstvec++ = a_mbase_x;
+        } else if (W == 4) {
+          store_u8_4x1(dst, a_mbase_x);
+          dst += stride;
+        } else {
+          vst1_u8(dst, a_mbase_x);
+          dst += stride;
+        }
+      } while (--H != 0);
       return;
     }

-    if (base_max_diff > W) base_max_diff = W;
-
+    assert(base_max_diff < W);
     uint8x8x2_t a01_128;
     uint16x8_t shift;
     if (upsample_above) {
@@ -1365,39 +1487,60 @@ static AOM_FORCE_INLINE void dr_prediction_z1_WxH_internal_neon_small(
       a01_128.val[1] = vld1_u8(above + base + 1);
       shift = vdupq_n_u16((x & 0x3f) >> 1);
     }
-    uint16x8_t diff = vsubl_u8(a01_128.val[1], a01_128.val[0]);
-    uint16x8_t a32 = vshll_n_u8(a01_128.val[0], 5);
-    uint16x8_t res = vmlaq_u16(a32, diff, shift);
+    const uint16x8_t diff = vsubl_u8(a01_128.val[1], a01_128.val[0]);
+    const uint16x8_t a32 = vshll_n_u8(a01_128.val[0], 5);
+    const uint16x8_t res = vmlaq_u16(a32, diff, shift);

-    uint8x8_t mask = vld1_u8(BaseMask[base_max_diff]);
-    dst[r] = vbsl_u8(mask, vrshrn_n_u16(res, 5), a_mbase_x);
+    const uint8x8_t mask = vld1_u8(BaseMask[base_max_diff]);
+    const uint8x8_t row = vbsl_u8(mask, vrshrn_n_u16(res, 5), a_mbase_x);
+    if (to_vec) {
+      *dstvec++ = row;
+    } else if (W == 4) {
+      store_u8_4x1(dst, row);
+      dst += stride;
+    } else {
+      vst1_u8(dst, row);
+      dst += stride;
+    }

     x += dx;
+  } while (--H != 0);
+}
+
+static AOM_FORCE_INLINE void dr_prediction_z1_WxH_internal_neon_small(
+    int W, int H, uint8x8_t *dst, const uint8_t *above, int upsample_above,
+    int dx) {
+  assert(upsample_above == 0 || W + H <= 16);
+  if (W + H <= 16 && upsample_above) {
+    dr_prediction_z1_WxH_core_neon_small(W, H, dst, NULL, 0, above, 1, dx, 1);
+  } else {
+    dr_prediction_z1_WxH_core_neon_small(W, H, dst, NULL, 0, above, 0, dx, 1);
   }
 }

 static void dr_prediction_z1_4xH_neon(int H, uint8_t *dst, ptrdiff_t stride,
                                       const uint8_t *above, int upsample_above,
                                       int dx) {
-  uint8x8_t dstvec[16];
-
-  dr_prediction_z1_WxH_internal_neon_small(4, H, dstvec, above, upsample_above,
-                                           dx);
-  for (int i = 0; i < H; i++) {
-    vst1_lane_u32((uint32_t *)(dst + stride * i),
-                  vreinterpret_u32_u8(dstvec[i]), 0);
+  assert(upsample_above == 0 || 4 + H <= 16);
+  if (4 + H <= 16 && upsample_above) {
+    dr_prediction_z1_WxH_core_neon_small(4, H, NULL, dst, stride, above, 1, dx,
+                                         0);
+  } else {
+    dr_prediction_z1_WxH_core_neon_small(4, H, NULL, dst, stride, above, 0, dx,
+                                         0);
   }
 }

 static void dr_prediction_z1_8xH_neon(int H, uint8_t *dst, ptrdiff_t stride,
                                       const uint8_t *above, int upsample_above,
                                       int dx) {
-  uint8x8_t dstvec[32];
-
-  dr_prediction_z1_WxH_internal_neon_small(8, H, dstvec, above, upsample_above,
-                                           dx);
-  for (int i = 0; i < H; i++) {
-    vst1_u8(dst + stride * i, dstvec[i]);
+  assert(upsample_above == 0 || 8 + H <= 16);
+  if (8 + H <= 16 && upsample_above) {
+    dr_prediction_z1_WxH_core_neon_small(8, H, NULL, dst, stride, above, 1, dx,
+                                         0);
+  } else {
+    dr_prediction_z1_WxH_core_neon_small(8, H, NULL, dst, stride, above, 0, dx,
+                                         0);
   }
 }

@@ -1449,12 +1592,62 @@ static AOM_FORCE_INLINE void dr_prediction_z1_WxH_internal_neon_large(

 static void dr_prediction_z1_16xH_neon(int H, uint8_t *dst, ptrdiff_t stride,
                                        const uint8_t *above, int dx) {
-  uint8x16_t dstvec[64];
-
-  dr_prediction_z1_WxH_internal_neon_large(16, H, dstvec, above, dx);
-  for (int i = 0; i < H; i++) {
-    vst1q_u8(dst + stride * i, dstvec[i]);
+  assert(H >= 4 && (H & 3) == 0);
+  assert(dx > 0);
+  const int max_base_x = (16 + H) - 1;
+  int x = dx;
+  for (; H != 0; --H) {
+    const int base = x >> 6;
+    if (max_base_x - base < 16) break;
+    const uint8x16_t a0_128 = vld1q_u8(above + base);
+    const uint8x16_t a1_128 = vld1q_u8(above + base + 1);
+    const uint16x8_t shift = vdupq_n_u16((x & 0x3f) >> 1);
+    const uint16x8_t diff_lo =
+        vsubl_u8(vget_low_u8(a1_128), vget_low_u8(a0_128));
+    const uint16x8_t diff_hi =
+        vsubl_u8(vget_high_u8(a1_128), vget_high_u8(a0_128));
+    const uint16x8_t a32_lo = vshll_n_u8(vget_low_u8(a0_128), 5);
+    const uint16x8_t a32_hi = vshll_n_u8(vget_high_u8(a0_128), 5);
+    const uint16x8_t res_lo = vmlaq_u16(a32_lo, diff_lo, shift);
+    const uint16x8_t res_hi = vmlaq_u16(a32_hi, diff_hi, shift);
+    vst1q_u8(dst,
+             vcombine_u8(vrshrn_n_u16(res_lo, 5), vrshrn_n_u16(res_hi, 5)));
+    dst += stride;
+    x += dx;
   }
+  if (H == 0) return;
+
+  const uint8x16_t a_mbase_x = vdupq_n_u8(above[max_base_x]);
+  do {
+    const int base = x >> 6;
+    const int base_max_diff = max_base_x - base;
+    if (base_max_diff <= 0) {
+      do {
+        vst1q_u8(dst, a_mbase_x);
+        dst += stride;
+      } while (--H != 0);
+      return;
+    }
+    assert(base_max_diff < 16);
+
+    const uint8x16_t a0_128 = vld1q_u8(above + base);
+    const uint8x16_t a1_128 = vld1q_u8(above + base + 1);
+    const uint16x8_t shift = vdupq_n_u16((x & 0x3f) >> 1);
+    const uint16x8_t diff_lo =
+        vsubl_u8(vget_low_u8(a1_128), vget_low_u8(a0_128));
+    const uint16x8_t diff_hi =
+        vsubl_u8(vget_high_u8(a1_128), vget_high_u8(a0_128));
+    const uint16x8_t a32_lo = vshll_n_u8(vget_low_u8(a0_128), 5);
+    const uint16x8_t a32_hi = vshll_n_u8(vget_high_u8(a0_128), 5);
+    const uint16x8_t res_lo = vmlaq_u16(a32_lo, diff_lo, shift);
+    const uint16x8_t res_hi = vmlaq_u16(a32_hi, diff_hi, shift);
+    const uint8x16_t v_temp =
+        vcombine_u8(vrshrn_n_u16(res_lo, 5), vrshrn_n_u16(res_hi, 5));
+    const uint8x16_t mask = vld1q_u8(BaseMask[base_max_diff]);
+    vst1q_u8(dst, vbslq_u8(mask, v_temp, a_mbase_x));
+    dst += stride;
+    x += dx;
+  } while (--H != 0);
 }

 static AOM_FORCE_INLINE void dr_prediction_z1_32xH_internal_neon(
@@ -1516,13 +1709,82 @@ static AOM_FORCE_INLINE void dr_prediction_z1_32xH_internal_neon(

 static void dr_prediction_z1_32xH_neon(int H, uint8_t *dst, ptrdiff_t stride,
                                        const uint8_t *above, int dx) {
-  uint8x16x2_t dstvec[64];
-
-  dr_prediction_z1_32xH_internal_neon(H, dstvec, above, dx);
-  for (int i = 0; i < H; i++) {
-    vst1q_u8(dst + stride * i, dstvec[i].val[0]);
-    vst1q_u8(dst + stride * i + 16, dstvec[i].val[1]);
+  assert(H >= 4 && (H & 3) == 0);
+  assert(dx > 0);
+  const int max_base_x = (32 + H) - 1;
+  int x = dx;
+  for (; H != 0; --H) {
+    const int base = x >> 6;
+    if (max_base_x - base < 32) break;
+    const uint16x8_t shift = vdupq_n_u16((x & 0x3f) >> 1);
+    for (int j = 0; j < 32; j += 16) {
+      const uint8x16_t a0_128 = vld1q_u8(above + base + j);
+      const uint8x16_t a1_128 = vld1q_u8(above + base + j + 1);
+      const uint16x8_t diff_lo =
+          vsubl_u8(vget_low_u8(a1_128), vget_low_u8(a0_128));
+      const uint16x8_t diff_hi =
+          vsubl_u8(vget_high_u8(a1_128), vget_high_u8(a0_128));
+      const uint16x8_t a32_lo = vshll_n_u8(vget_low_u8(a0_128), 5);
+      const uint16x8_t a32_hi = vshll_n_u8(vget_high_u8(a0_128), 5);
+      const uint16x8_t res_lo = vmlaq_u16(a32_lo, diff_lo, shift);
+      const uint16x8_t res_hi = vmlaq_u16(a32_hi, diff_hi, shift);
+      vst1q_u8(dst + j,
+               vcombine_u8(vrshrn_n_u16(res_lo, 5), vrshrn_n_u16(res_hi, 5)));
+    }
+    dst += stride;
+    x += dx;
   }
+  if (H == 0) return;
+
+  const uint8x16_t a_mbase_x = vdupq_n_u8(above[max_base_x]);
+  do {
+    const int base = x >> 6;
+    const int base_max_diff = max_base_x - base;
+    if (base_max_diff <= 0) {
+      do {
+        vst1q_u8(dst, a_mbase_x);
+        vst1q_u8(dst + 16, a_mbase_x);
+        dst += stride;
+      } while (--H != 0);
+      return;
+    }
+    assert(base_max_diff < 32);
+
+    const uint16x8_t shift = vdupq_n_u16((x & 0x3f) >> 1);
+    const uint8x16_t a0_lo = vld1q_u8(above + base);
+    const uint8x16_t a1_lo = vld1q_u8(above + base + 1);
+    const uint16x8_t diff0_lo =
+        vsubl_u8(vget_low_u8(a1_lo), vget_low_u8(a0_lo));
+    const uint16x8_t diff0_hi =
+        vsubl_u8(vget_high_u8(a1_lo), vget_high_u8(a0_lo));
+    const uint16x8_t a32_0_lo = vshll_n_u8(vget_low_u8(a0_lo), 5);
+    const uint16x8_t a32_0_hi = vshll_n_u8(vget_high_u8(a0_lo), 5);
+    const uint8x16_t res0 =
+        vcombine_u8(vrshrn_n_u16(vmlaq_u16(a32_0_lo, diff0_lo, shift), 5),
+                    vrshrn_n_u16(vmlaq_u16(a32_0_hi, diff0_hi, shift), 5));
+    if (base_max_diff < 16) {
+      const uint8x16_t mask_lo = vld1q_u8(BaseMask[base_max_diff]);
+      vst1q_u8(dst, vbslq_u8(mask_lo, res0, a_mbase_x));
+      vst1q_u8(dst + 16, a_mbase_x);
+    } else {
+      const uint8x16_t a0_hi = vld1q_u8(above + base + 16);
+      const uint8x16_t a1_hi = vld1q_u8(above + base + 17);
+      const uint16x8_t diff1_lo =
+          vsubl_u8(vget_low_u8(a1_hi), vget_low_u8(a0_hi));
+      const uint16x8_t diff1_hi =
+          vsubl_u8(vget_high_u8(a1_hi), vget_high_u8(a0_hi));
+      const uint16x8_t a32_1_lo = vshll_n_u8(vget_low_u8(a0_hi), 5);
+      const uint16x8_t a32_1_hi = vshll_n_u8(vget_high_u8(a0_hi), 5);
+      const uint8x16_t res1 =
+          vcombine_u8(vrshrn_n_u16(vmlaq_u16(a32_1_lo, diff1_lo, shift), 5),
+                      vrshrn_n_u16(vmlaq_u16(a32_1_hi, diff1_hi, shift), 5));
+      const uint8x16_t mask_hi = vld1q_u8(BaseMask[base_max_diff - 16]);
+      vst1q_u8(dst, res0);
+      vst1q_u8(dst + 16, vbslq_u8(mask_hi, res1, a_mbase_x));
+    }
+    dst += stride;
+    x += dx;
+  } while (--H != 0);
 }

 // clang-format off
@@ -1678,10 +1940,8 @@ static DECLARE_ALIGNED(16, const uint8_t, LoadMaskz2[4][16]) = {
 #endif  // !AOM_ARCH_AARCH64

 static AOM_FORCE_INLINE void dr_prediction_z2_4xH_above_neon(
-    const uint8_t *above, int upsample_above, int dx, int base_x, int y,
+    const uint8_t *above, int upsample_above, int base_x, int x,
     uint8x8_t *a0_x, uint8x8_t *a1_x, uint16x4_t *shift0) {
-  const int x = -y * dx;
-
   if (upsample_above) {
     // Cannot use LD2 here since we only want to load eight bytes, but LD2 can
     // only load either 16 or 32.
@@ -1696,30 +1956,12 @@ static AOM_FORCE_INLINE void dr_prediction_z2_4xH_above_neon(
   }
 }

+#if !AOM_ARCH_AARCH64
 static AOM_FORCE_INLINE void dr_prediction_z2_4xH_left_neon(
-#if AOM_ARCH_AARCH64
-    uint8x16x2_t left_vals,
-#else
-    const uint8_t *left,
-#endif
-    int upsample_left, int dy, int r, int frac_bits_y, uint16x4_t *a0_y,
-    uint16x4_t *a1_y, uint16x4_t *shift1) {
-  int16x4_t dy64 = vdup_n_s16(dy);
-  int16x4_t v_1234 = vcreate_s16(0x0004000300020001);
-  int16x4_t v_frac_bits_y = vdup_n_s16(-frac_bits_y);
-  int16x4_t v_r6 = vdup_n_s16(r << 6);
-  int16x4_t y_c64 = vmls_s16(v_r6, v_1234, dy64);
+    const uint8_t *left, int upsample_left, int16x4_t y_c64,
+    int16x4_t v_frac_bits_y, uint16x4_t *a0_y, uint16x4_t *a1_y,
+    uint16x4_t *shift1) {
   int16x4_t base_y_c64 = vshl_s16(y_c64, v_frac_bits_y);
-
-#if AOM_ARCH_AARCH64
-  uint8x8_t left_idx0 =
-      vreinterpret_u8_s16(vadd_s16(base_y_c64, vdup_n_s16(2)));  // [0, 16]
-  uint8x8_t left_idx1 =
-      vreinterpret_u8_s16(vadd_s16(base_y_c64, vdup_n_s16(3)));  // [1, 17]
-
-  *a0_y = vreinterpret_u16_u8(vqtbl2_u8(left_vals, left_idx0));
-  *a1_y = vreinterpret_u16_u8(vqtbl2_u8(left_vals, left_idx1));
-#else   // !AOM_ARCH_AARCH64
   DECLARE_ALIGNED(32, int16_t, base_y_c[4]);

   vst1_s16(base_y_c, base_y_c64);
@@ -1739,7 +1981,6 @@ static AOM_FORCE_INLINE void dr_prediction_z2_4xH_left_neon(

   *a0_y = vreinterpret_u16_u8(a0_y_u8);
   *a1_y = vreinterpret_u16_u8(a1_y_u8);
-#endif  // AOM_ARCH_AARCH64

   if (upsample_left) {
     *shift1 = vand_u16(vreinterpret_u16_s16(y_c64),
@@ -1749,11 +1990,10 @@ static AOM_FORCE_INLINE void dr_prediction_z2_4xH_left_neon(
         vand_u16(vshr_n_u16(vreinterpret_u16_s16(y_c64), 1), vdup_n_u16(0x1f));
   }
 }
+#endif  // !AOM_ARCH_AARCH64

 static AOM_FORCE_INLINE uint8x8_t dr_prediction_z2_8xH_above_neon(
-    const uint8_t *above, int upsample_above, int dx, int base_x, int y) {
-  const int x = -y * dx;
-
+    const uint8_t *above, int upsample_above, int base_x, int x) {
   uint16x8_t shift0;
   uint8x8_t a0_x0;
   uint8x8_t a1_x0;
@@ -1775,36 +2015,33 @@ static AOM_FORCE_INLINE uint8x8_t dr_prediction_z2_8xH_above_neon(
   return vrshrn_n_u16(res, 5);
 }

-static AOM_FORCE_INLINE uint8x8_t dr_prediction_z2_8xH_left_neon(
 #if AOM_ARCH_AARCH64
-    uint8x16x3_t left_vals,
+static AOM_FORCE_INLINE uint8x8_t
+dr_prediction_z2_8xH_left_tbl_neon(uint8x16x3_t left_vals, const int tbl_regs,
+                                   uint8x16_t left_idx01, uint16x8_t shift1) {
+  uint8x16_t a01_x;
+  if (tbl_regs == 1) {
+    a01_x = vqtbl1q_u8(left_vals.val[0], left_idx01);
+  } else if (tbl_regs == 2) {
+    const uint8x16x2_t lv2 = { { left_vals.val[0], left_vals.val[1] } };
+    a01_x = vqtbl2q_u8(lv2, left_idx01);
+  } else {
+    a01_x = vqtbl3q_u8(left_vals, left_idx01);
+  }
+  const uint8x8_t a0_x1 = vget_low_u8(a01_x);
+  const uint8x8_t a1_x1 = vget_high_u8(a01_x);
+  const uint16x8_t diff1 = vsubl_u8(a1_x1, a0_x1);
+  const uint16x8_t a32 = vshll_n_u8(a0_x1, 5);
+  const uint16x8_t res = vmlaq_u16(a32, diff1, shift1);
+  return vrshrn_n_u16(res, 5);
+}
 #else
-    const uint8_t *left,
-#endif
-    int upsample_left, int dy, int r, int frac_bits_y) {
-  int16x8_t v_r6 = vdupq_n_s16(r << 6);
-  int16x8_t dy128 = vdupq_n_s16(dy);
-  int16x8_t v_frac_bits_y = vdupq_n_s16(-frac_bits_y);
-
-  uint16x8_t c1234 = vcombine_u16(vcreate_u16(0x0004000300020001),
-                                  vcreate_u16(0x0008000700060005));
-  int16x8_t y_c128 = vmlsq_s16(v_r6, vreinterpretq_s16_u16(c1234), dy128);
+static AOM_FORCE_INLINE uint8x8_t
+dr_prediction_z2_8xH_left_neon(const uint8_t *left, int upsample_left,
+                               int16x8_t y_c128, int16x8_t v_frac_bits_y) {
   int16x8_t base_y_c128 = vshlq_s16(y_c128, v_frac_bits_y);
-
-#if AOM_ARCH_AARCH64
-  uint8x16_t left_idx0 =
-      vreinterpretq_u8_s16(vaddq_s16(base_y_c128, vdupq_n_s16(2)));  // [0, 33]
-  uint8x16_t left_idx1 =
-      vreinterpretq_u8_s16(vaddq_s16(base_y_c128, vdupq_n_s16(3)));  // [1, 34]
-  uint8x16_t left_idx01 = vuzp1q_u8(left_idx0, left_idx1);
-
-  uint8x16_t a01_x = vqtbl3q_u8(left_vals, left_idx01);
-  uint8x8_t a0_x1 = vget_low_u8(a01_x);
-  uint8x8_t a1_x1 = vget_high_u8(a01_x);
-#else   // !AOM_ARCH_AARCH64
   uint8x8_t a0_x1 = load_u8_gather_s16_x8(left, base_y_c128);
   uint8x8_t a1_x1 = load_u8_gather_s16_x8(left + 1, base_y_c128);
-#endif  // AOM_ARCH_AARCH64

   uint16x8_t shift1;
   if (upsample_left) {
@@ -1822,13 +2059,12 @@ static AOM_FORCE_INLINE uint8x8_t dr_prediction_z2_8xH_left_neon(

   return vrshrn_n_u16(res, 5);
 }
+#endif  // AOM_ARCH_AARCH64

 static AOM_FORCE_INLINE uint8x16_t dr_prediction_z2_WxH_above_neon(
-    const uint8_t *above, int dx, int base_x, int y, int j) {
-  const int x = -y * dx;
+    const uint8_t *above, int base_x, uint16x8_t shift, int j) {
   const uint8x16_t a0_x128 = vld1q_u8(above + base_x + j);
   const uint8x16_t a1_x128 = vld1q_u8(above + base_x + j + 1);
-  const uint16x8_t shift = vdupq_n_u16((x & 0x3f) >> 1);

   // a[x+1] - a[x]
   uint16x8_t diff0 = vsubl_u8(vget_low_u8(a1_x128), vget_low_u8(a0_x128));
@@ -1849,24 +2085,15 @@ static AOM_FORCE_INLINE uint8x16_t dr_prediction_z2_WxH_left_neon(
 #else
     const uint8_t *left,
 #endif
-    int dy, int r, int j) {
-  // here upsample_above and upsample_left are 0 by design of
-  // av1_use_intra_edge_upsample
-  int16x8_t dy256 = vdupq_n_s16(dy);
-  uint16x8_t j256 = vdupq_n_u16(j);
+    int16x8_t y_base_lo, int16x8_t y_base_hi, int dy, int r, int j) {
+  // Here upsample_above and upsample_left are 0 by design of
+  // av1_use_intra_edge_upsample. Called only when base_min_diff > 0, so across
+  // valid AV1 angles (r << 6) - j * dy stays in [-48, 4032] and y_c256 in
+  // [-15024, 4029], fitting in int16_t without wrap.
+  const int16x8_t v_r6_jdy = vdupq_n_s16((int16_t)((r << 6) - j * dy));

-  uint16x8x2_t c1234 = { { vcombine_u16(vcreate_u16(0x0004000300020001),
-                                        vcreate_u16(0x0008000700060005)),
-                           vcombine_u16(vcreate_u16(0x000C000B000A0009),
-                                        vcreate_u16(0x0010000F000E000D)) } };
-
-  int16x8_t v_r6 = vdupq_n_s16(r << 6);
-
-  int16x8_t c256_0 = vreinterpretq_s16_u16(vaddq_u16(j256, c1234.val[0]));
-  int16x8_t c256_1 = vreinterpretq_s16_u16(vaddq_u16(j256, c1234.val[1]));
-
-  int16x8_t y_c256_lo = vmlsq_s16(v_r6, c256_0, dy256);
-  int16x8_t y_c256_hi = vmlsq_s16(v_r6, c256_1, dy256);
+  const int16x8_t y_c256_lo = vaddq_s16(v_r6_jdy, y_base_lo);
+  const int16x8_t y_c256_hi = vaddq_s16(v_r6_jdy, y_base_hi);

   int16x8_t base_y_c256_lo = vshrq_n_s16(y_c256_lo, 6);
   int16x8_t base_y_c256_hi = vshrq_n_s16(y_c256_hi, 6);
@@ -1960,54 +2187,217 @@ static AOM_FORCE_INLINE uint8x16_t dr_prediction_z2_WxH_left_neon(
   return vcombine_u8(vrshrn_n_u16(res0, 5), vrshrn_n_u16(res1, 5));
 }

+#if AOM_ARCH_AARCH64
+static AOM_FORCE_INLINE uint8x16_t dr_prediction_z2_4xH_tbl_neon(
+    uint8x16x2_t left_vals, int use_tbl1, uint8x16_t idx) {
+  if (use_tbl1) {
+    return vqtbl1q_u8(left_vals.val[0], idx);
+  } else {
+    return vqtbl2q_u8(left_vals, idx);
+  }
+}
+
+static AOM_FORCE_INLINE void dr_prediction_z2_4xH_left_core_aarch64(
+    int H, int r, int x, uint8_t *dst, ptrdiff_t stride, const uint8_t *above,
+    const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy,
+    int use_tbl1) {
+  const int frac_bits_x = 6 - upsample_above;
+  const uint8x16_t left_m2 = vld1q_u8(left - 2);
+  // Use ext rather than loading left + 14 directly to avoid over-read.
+  uint8x16_t left_14 = left_m2;
+  if (!use_tbl1) {
+    const uint8x16_t left_0 = vld1q_u8(left);
+    left_14 = vextq_u8(left_0, left_0, 14);
+  }
+  const uint8x16x2_t left_vals = { { left_m2, left_14 } };
+
+  const int16x4_t y_c64_base =
+      vneg_s16(vmul_n_s16(vcreate_s16(0x0004000300020001), (int16_t)dy));
+  const uint16x4_t shift1 =
+      upsample_left
+          ? vand_u16(vreinterpret_u16_s16(y_c64_base), vdup_n_u16(0x1f))
+          : vand_u16(vshr_n_u16(vreinterpret_u16_s16(y_c64_base), 1),
+                     vdup_n_u16(0x1f));
+  const int16x4_t base_y_0 = vadd_s16(
+      upsample_left ? vshr_n_s16(y_c64_base, 5) : vshr_n_s16(y_c64_base, 6),
+      vdup_n_s16((int16_t)(r << upsample_left)));
+  uint8x8_t left_idx01 =
+      vuzp1_u8(vreinterpret_u8_s16(vadd_s16(base_y_0, vdup_n_s16(2))),
+               vreinterpret_u8_s16(vadd_s16(base_y_0, vdup_n_s16(3))));
+  const uint8x8_t v_step = vdup_n_u8((uint8_t)(1 << upsample_left));
+
+  for (; r < H && x >= -256; r++, x -= dx) {
+    const int base_x = x >> frac_bits_x;
+    const int base_min_diff = (-x - 1) >> 6;
+    uint8x8_t a0_x_u8, a1_x_u8;
+    uint16x4_t shift0;
+    dr_prediction_z2_4xH_above_neon(above, upsample_above, base_x, x, &a0_x_u8,
+                                    &a1_x_u8, &shift0);
+    const uint8x8_t a01_y = use_tbl1 ? vqtbl1_u8(left_vals.val[0], left_idx01)
+                                     : vqtbl2_u8(left_vals, left_idx01);
+    const uint8x8_t a0_8 = vreinterpret_u8_u32(
+        vzip_u32(vreinterpret_u32_u8(a0_x_u8), vreinterpret_u32_u8(a01_y))
+            .val[0]);
+    const uint8x8_t a1_8 = vreinterpret_u8_u32(
+        vzip_u32(vreinterpret_u32_u8(a1_x_u8),
+                 vreinterpret_u32_u8(vext_u8(a01_y, a01_y, 4)))
+            .val[0]);
+    const uint16x8_t shift = vcombine_u16(shift0, shift1);
+    const uint16x8_t diff = vsubl_u8(a1_8, a0_8);
+    const uint16x8_t a32 = vshll_n_u8(a0_8, 5);
+    const uint16x8_t res = vmlaq_u16(a32, diff, shift);
+    const uint8x8_t resx = vrshrn_n_u16(res, 5);
+    const uint8x8_t resy = vext_u8(resx, resx, 4);
+    const uint8x8_t mask = vld1_u8(BaseMask[base_min_diff]);
+    const uint8x8_t v_resxy = vbsl_u8(mask, resy, resx);
+    store_u8_4x1(dst, v_resxy);
+    dst += stride;
+    left_idx01 = vadd_u8(left_idx01, v_step);
+  }
+
+  if (r < H) {
+    const uint16x8_t shift1_q = vcombine_u16(shift1, shift1);
+    const uint8x8_t idx_a0 = vreinterpret_u8_u32(
+        vzip_u32(vreinterpret_u32_u8(left_idx01),
+                 vreinterpret_u32_u8(vadd_u8(left_idx01, v_step)))
+            .val[0]);
+    uint8x16_t idx_a01 = vcombine_u8(idx_a0, vadd_u8(idx_a0, vdup_n_u8(1)));
+    const uint8x16_t v_step2_q = vdupq_n_u8((uint8_t)(2 << upsample_left));
+    for (; r + 2 <= H; r += 2) {
+      const uint8x16_t a01 =
+          dr_prediction_z2_4xH_tbl_neon(left_vals, use_tbl1, idx_a01);
+      const uint8x8_t a0 = vget_low_u8(a01);
+      const uint8x8_t a1 = vget_high_u8(a01);
+      const uint16x8_t diff = vsubl_u8(a1, a0);
+      const uint16x8_t a32 = vshll_n_u8(a0, 5);
+      const uint16x8_t res = vmlaq_u16(a32, diff, shift1_q);
+      const uint8x8_t resy = vrshrn_n_u16(res, 5);
+      store_u8x4_strided_x2(dst, stride, resy);
+      dst += 2 * stride;
+      idx_a01 = vaddq_u8(idx_a01, v_step2_q);
+    }
+    if (r < H) {
+      const uint8x16_t a01 =
+          dr_prediction_z2_4xH_tbl_neon(left_vals, use_tbl1, idx_a01);
+      const uint8x8_t a0 = vget_low_u8(a01);
+      const uint8x8_t a1 = vget_high_u8(a01);
+      const uint16x8_t diff = vsubl_u8(a1, a0);
+      const uint16x8_t a32 = vshll_n_u8(a0, 5);
+      const uint16x8_t res = vmlaq_u16(a32, diff, shift1_q);
+      const uint8x8_t resy = vrshrn_n_u16(res, 5);
+      store_u8_4x1(dst, resy);
+    }
+  }
+}
+
+static AOM_FORCE_INLINE void dr_prediction_z2_4xH_core_aarch64(
+    int H, uint8_t *dst, ptrdiff_t stride, const uint8_t *above,
+    const uint8_t *left, int upsample_above, int upsample_left, int dx,
+    int dy) {
+  int r = 0;
+  int x = -dx;
+
+  if (!upsample_above) {
+    // base_x = x >> 6 is constant (-1) throughout x in [-64, -1].
+    if (x >= -64) {
+      const uint8x8_t a0_8 = load_unaligned_dup_u8_4x2(above - 1);
+      const uint8x8_t a1_8 = load_unaligned_dup_u8_4x2(above);
+      const uint16x8_t diff = vsubl_u8(a1_8, a0_8);
+      const uint16x8_t a32 = vshll_n_u8(a0_8, 5);
+      for (; r + 2 <= H && x - dx >= -64; r += 2, x -= 2 * dx) {
+        const uint16x4_t s0 = vdup_n_u16((x & 0x3f) >> 1);
+        const uint16x4_t s1 = vdup_n_u16(((x - dx) & 0x3f) >> 1);
+        const uint16x8_t res = vmlaq_u16(a32, diff, vcombine_u16(s0, s1));
+        store_u8x4_strided_x2(dst, stride, vrshrn_n_u16(res, 5));
+        dst += 2 * stride;
+      }
+      if (r < H && x >= -64) {
+        const uint16x4_t s0 = vdup_n_u16((x & 0x3f) >> 1);
+        const uint16x8_t res =
+            vmlaq_u16(a32, diff, vcombine_u16(s0, vdup_n_u16(0)));
+        store_u8_4x1(dst, vrshrn_n_u16(res, 5));
+        dst += stride;
+        r++;
+        x -= dx;
+      }
+    }
+  } else {
+    for (; r < H && x >= -64; r++, x -= dx) {
+      const int base_x = x >> 5;
+      uint8x8_t a0_x, a1_x;
+      uint16x4_t shift0;
+      dr_prediction_z2_4xH_above_neon(above, 1, base_x, x, &a0_x, &a1_x,
+                                      &shift0);
+      const uint16x8_t diff = vsubl_u8(a1_x, a0_x);
+      const uint16x8_t a32 = vshll_n_u8(a0_x, 5);
+      const uint16x8_t res =
+          vmlaq_u16(a32, diff, vcombine_u16(shift0, vdup_n_u16(0)));
+      store_u8_4x1(dst, vrshrn_n_u16(res, 5));
+      dst += stride;
+    }
+  }
+  if (r == H) return;
+
+  if (((H - 1) << upsample_left) + 3 < 16) {
+    dr_prediction_z2_4xH_left_core_aarch64(H, r, x, dst, stride, above, left,
+                                           upsample_above, upsample_left, dx,
+                                           dy, 1);
+  } else {
+    dr_prediction_z2_4xH_left_core_aarch64(H, r, x, dst, stride, above, left,
+                                           upsample_above, upsample_left, dx,
+                                           dy, 0);
+  }
+}
+#endif  // AOM_ARCH_AARCH64
+
 static void dr_prediction_z2_4xH_neon(int H, uint8_t *dst, ptrdiff_t stride,
                                       const uint8_t *above, const uint8_t *left,
                                       int upsample_above, int upsample_left,
                                       int dx, int dy) {
-  const int min_base_x = -(1 << upsample_above);
-  const int frac_bits_x = 6 - upsample_above;
-  const int frac_bits_y = 6 - upsample_left;
-
+  assert(H >= 4 && (H & 3) == 0);
   assert(dx > 0);
   //   above0 * (32 - shift) + above1 * shift
   // = (above1 - above0) * shift + above0 * 32

 #if AOM_ARCH_AARCH64
-  // Use ext rather than loading left + 14 directly to avoid over-read.
-  const uint8x16_t left_m2 = vld1q_u8(left - 2);
-  const uint8x16_t left_0 = vld1q_u8(left);
-  const uint8x16_t left_14 = vextq_u8(left_0, left_0, 14);
-  const uint8x16x2_t left_vals = { { left_m2, left_14 } };
-#define LEFT left_vals
-#else  // !AOM_ARCH_AARCH64
-#define LEFT left
-#endif  // AOM_ARCH_AARCH64
+  assert(upsample_above == 0 || 4 + H <= 16);
+  if (4 + H <= 16 && upsample_above) {
+    dr_prediction_z2_4xH_core_aarch64(H, dst, stride, above, left, 1,
+                                      upsample_left, dx, dy);
+  } else {
+    dr_prediction_z2_4xH_core_aarch64(H, dst, stride, above, left, 0,
+                                      upsample_left, dx, dy);
+  }
+#else   // !AOM_ARCH_AARCH64
+  const int min_base_x = -(1 << upsample_above);
+  const int frac_bits_x = 6 - upsample_above;
+  const int frac_bits_y = 6 - upsample_left;
+  const int16x4_t y_c64_base =
+      vneg_s16(vmul_n_s16(vcreate_s16(0x0004000300020001), (int16_t)dy));
+  const int16x4_t v_frac_bits_y = vdup_n_s16(-frac_bits_y);

-  for (int r = 0; r < H; r++) {
-    int y = r + 1;
-    int base_x = (-y * dx) >> frac_bits_x;
+  int r = 0;
+  for (int x = -dx; r < H; r++, x -= dx) {
+    const int base_x = x >> frac_bits_x;
     const int base_min_diff =
-        (min_base_x - ((-y * dx) >> frac_bits_x) + upsample_above) >>
-        upsample_above;
+        (min_base_x - base_x + upsample_above) >> upsample_above;

     if (base_min_diff <= 0) {
-      uint8x8_t a0_x_u8, a1_x_u8;
+      uint8x8_t a0_x, a1_x;
       uint16x4_t shift0;
-      dr_prediction_z2_4xH_above_neon(above, upsample_above, dx, base_x, y,
-                                      &a0_x_u8, &a1_x_u8, &shift0);
-      uint8x8_t a0_x = a0_x_u8;
-      uint8x8_t a1_x = a1_x_u8;
+      dr_prediction_z2_4xH_above_neon(above, upsample_above, base_x, x, &a0_x,
+                                      &a1_x, &shift0);

-      uint16x8_t diff = vsubl_u8(a1_x, a0_x);  // a[x+1] - a[x]
-      uint16x8_t a32 = vshll_n_u8(a0_x, 5);
-      uint16x8_t res =
+      const uint16x8_t diff = vsubl_u8(a1_x, a0_x);  // a[x+1] - a[x]
+      const uint16x8_t a32 = vshll_n_u8(a0_x, 5);
+      const uint16x8_t res =
           vmlaq_u16(a32, diff, vcombine_u16(shift0, vdup_n_u16(0)));
-      uint8x8_t resx = vrshrn_n_u16(res, 5);
-      vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(resx), 0);
+      const uint8x8_t resx = vrshrn_n_u16(res, 5);
+      store_u8_4x1(dst, resx);
     } else if (base_min_diff < 4) {
       uint8x8_t a0_x_u8, a1_x_u8;
       uint16x4_t shift0;
-      dr_prediction_z2_4xH_above_neon(above, upsample_above, dx, base_x, y,
+      dr_prediction_z2_4xH_above_neon(above, upsample_above, base_x, x,
                                       &a0_x_u8, &a1_x_u8, &shift0);
       uint16x8_t a0_x = vmovl_u8(a0_x_u8);
       uint16x8_t a1_x = vmovl_u8(a1_x_u8);
@@ -2015,99 +2405,305 @@ static void dr_prediction_z2_4xH_neon(int H, uint8_t *dst, ptrdiff_t stride,
       uint16x4_t a0_y;
       uint16x4_t a1_y;
       uint16x4_t shift1;
-      dr_prediction_z2_4xH_left_neon(LEFT, upsample_left, dy, r, frac_bits_y,
+      const int16x4_t y_c64 = vadd_s16(y_c64_base, vdup_n_s16(r << 6));
+      dr_prediction_z2_4xH_left_neon(left, upsample_left, y_c64, v_frac_bits_y,
                                      &a0_y, &a1_y, &shift1);
       a0_x = vcombine_u16(vget_low_u16(a0_x), a0_y);
       a1_x = vcombine_u16(vget_low_u16(a1_x), a1_y);

-      uint16x8_t shift = vcombine_u16(shift0, shift1);
-      uint16x8_t diff = vsubq_u16(a1_x, a0_x);  // a[x+1] - a[x]
-      uint16x8_t a32 = vshlq_n_u16(a0_x, 5);
-      uint16x8_t res = vmlaq_u16(a32, diff, shift);
-      uint8x8_t resx = vrshrn_n_u16(res, 5);
-      uint8x8_t resy = vext_u8(resx, vdup_n_u8(0), 4);
+      const uint16x8_t shift = vcombine_u16(shift0, shift1);
+      const uint16x8_t diff = vsubq_u16(a1_x, a0_x);  // a[x+1] - a[x]
+      const uint16x8_t a32 = vshlq_n_u16(a0_x, 5);
+      const uint16x8_t res = vmlaq_u16(a32, diff, shift);
+      const uint8x8_t resx = vrshrn_n_u16(res, 5);
+      const uint8x8_t resy = vext_u8(resx, vdup_n_u8(0), 4);

-      uint8x8_t mask = vld1_u8(BaseMask[base_min_diff]);
-      uint8x8_t v_resxy = vbsl_u8(mask, resy, resx);
-      vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(v_resxy), 0);
+      const uint8x8_t mask = vld1_u8(BaseMask[base_min_diff]);
+      const uint8x8_t v_resxy = vbsl_u8(mask, resy, resx);
+      store_u8_4x1(dst, v_resxy);
     } else {
-      uint16x4_t a0_y, a1_y;
-      uint16x4_t shift1;
-      dr_prediction_z2_4xH_left_neon(LEFT, upsample_left, dy, r, frac_bits_y,
-                                     &a0_y, &a1_y, &shift1);
-      uint16x4_t diff = vsub_u16(a1_y, a0_y);  // a[x+1] - a[x]
-      uint16x4_t a32 = vshl_n_u16(a0_y, 5);
-      uint16x4_t res = vmla_u16(a32, diff, shift1);
-      uint8x8_t resy = vrshrn_n_u16(vcombine_u16(res, vdup_n_u16(0)), 5);
-
-      vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(resy), 0);
+      break;
     }

     dst += stride;
   }
-#undef LEFT
+
+  for (; r < H; r++) {
+    uint16x4_t a0_y, a1_y;
+    uint16x4_t shift1;
+    const int16x4_t y_c64 = vadd_s16(y_c64_base, vdup_n_s16(r << 6));
+    dr_prediction_z2_4xH_left_neon(left, upsample_left, y_c64, v_frac_bits_y,
+                                   &a0_y, &a1_y, &shift1);
+    const uint16x4_t diff = vsub_u16(a1_y, a0_y);  // a[x+1] - a[x]
+    const uint16x4_t a32 = vshl_n_u16(a0_y, 5);
+    const uint16x4_t res = vmla_u16(a32, diff, shift1);
+    const uint8x8_t resy = vrshrn_n_u16(vcombine_u16(res, vdup_n_u16(0)), 5);
+
+    store_u8_4x1(dst, resy);
+    dst += stride;
+  }
+#endif  // AOM_ARCH_AARCH64
 }

+#if AOM_ARCH_AARCH64
+static AOM_FORCE_INLINE void dr_prediction_z2_8xH_left_core_aarch64(
+    int H, int r, int x, uint8_t *dst, ptrdiff_t stride, const uint8_t *above,
+    const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy,
+    int tbl_regs) {
+  const int frac_bits_x = 6 - upsample_above;
+  const uint8x16_t left_m2 = vld1q_u8(left - 2);
+  uint8x16x3_t left_vals;
+  if (tbl_regs == 1) {
+    left_vals.val[0] = left_m2;
+    left_vals.val[1] = left_m2;
+    left_vals.val[2] = left_m2;
+  } else {
+    const uint8x16_t left_0 = vld1q_u8(left + 0);
+    const uint8x16_t left_16 = vld1q_u8(left + 16);
+    const uint8x16_t left_14 = vextq_u8(left_0, left_16, 14);
+    const uint8x16_t left_30 =
+        (tbl_regs == 2) ? left_14 : vextq_u8(left_16, left_16, 14);
+    left_vals.val[0] = left_m2;
+    left_vals.val[1] = left_14;
+    left_vals.val[2] = left_30;
+  }
+
+  const uint16x8_t c1_8 = vcombine_u16(vcreate_u16(0x0004000300020001),
+                                       vcreate_u16(0x0008000700060005));
+  const int16x8_t y_c128_base =
+      vnegq_s16(vmulq_n_s16(vreinterpretq_s16_u16(c1_8), (int16_t)dy));
+  const uint16x8_t shift1 =
+      upsample_left
+          ? vandq_u16(vreinterpretq_u16_s16(y_c128_base), vdupq_n_u16(0x1f))
+          : vshrq_n_u16(vandq_u16(vreinterpretq_u16_s16(y_c128_base),
+                                  vdupq_n_u16(0x3f)),
+                        1);
+  const int16x8_t base_y_0 = vaddq_s16(
+      upsample_left ? vshrq_n_s16(y_c128_base, 5) : vshrq_n_s16(y_c128_base, 6),
+      vdupq_n_s16((int16_t)(r << upsample_left)));
+  uint8x16_t left_idx01 =
+      vuzp1q_u8(vreinterpretq_u8_s16(vaddq_s16(base_y_0, vdupq_n_s16(2))),
+                vreinterpretq_u8_s16(vaddq_s16(base_y_0, vdupq_n_s16(3))));
+  const uint8x16_t v_step = vdupq_n_u8((uint8_t)(1 << upsample_left));
+
+  for (; r + 2 <= H && x - dx >= -512; r += 2, x -= 2 * dx) {
+    const int x0 = x;
+    const int x1 = x - dx;
+    const int base_x0 = x0 >> frac_bits_x;
+    const int base_x1 = x1 >> frac_bits_x;
+    const int bmd0 = (-x0 - 1) >> 6;
+    const int bmd1 = (-x1 - 1) >> 6;
+    const uint8x16_t idx0 = left_idx01;
+    const uint8x16_t idx1 = vaddq_u8(left_idx01, v_step);
+    left_idx01 = vaddq_u8(idx1, v_step);
+
+    const uint8x8_t resx0 =
+        dr_prediction_z2_8xH_above_neon(above, upsample_above, base_x0, x0);
+    const uint8x8_t resx1 =
+        dr_prediction_z2_8xH_above_neon(above, upsample_above, base_x1, x1);
+    const uint8x8_t resy0 =
+        dr_prediction_z2_8xH_left_tbl_neon(left_vals, tbl_regs, idx0, shift1);
+    const uint8x8_t resy1 =
+        dr_prediction_z2_8xH_left_tbl_neon(left_vals, tbl_regs, idx1, shift1);
+    const uint8x8_t mask0 = vld1_u8(BaseMask[bmd0]);
+    const uint8x8_t mask1 = vld1_u8(BaseMask[bmd1]);
+    vst1_u8(dst, vbsl_u8(mask0, resy0, resx0));
+    vst1_u8(dst + stride, vbsl_u8(mask1, resy1, resx1));
+    dst += 2 * stride;
+  }
+  if (r < H && x >= -512) {
+    const int base_x = x >> frac_bits_x;
+    const int bmd = (-x - 1) >> 6;
+    const uint8x8_t resx =
+        dr_prediction_z2_8xH_above_neon(above, upsample_above, base_x, x);
+    const uint8x8_t resy = dr_prediction_z2_8xH_left_tbl_neon(
+        left_vals, tbl_regs, left_idx01, shift1);
+    const uint8x8_t mask = vld1_u8(BaseMask[bmd]);
+    vst1_u8(dst, vbsl_u8(mask, resy, resx));
+    dst += stride;
+    left_idx01 = vaddq_u8(left_idx01, v_step);
+    x -= dx;
+    r++;
+  }
+
+  for (; r + 2 <= H; r += 2) {
+    const uint8x16_t idx0 = left_idx01;
+    const uint8x16_t idx1 = vaddq_u8(left_idx01, v_step);
+    left_idx01 = vaddq_u8(idx1, v_step);
+    const uint8x8_t resy0 =
+        dr_prediction_z2_8xH_left_tbl_neon(left_vals, tbl_regs, idx0, shift1);
+    const uint8x8_t resy1 =
+        dr_prediction_z2_8xH_left_tbl_neon(left_vals, tbl_regs, idx1, shift1);
+    vst1_u8(dst, resy0);
+    vst1_u8(dst + stride, resy1);
+    dst += 2 * stride;
+  }
+  if (r < H) {
+    const uint8x8_t resy = dr_prediction_z2_8xH_left_tbl_neon(
+        left_vals, tbl_regs, left_idx01, shift1);
+    vst1_u8(dst, resy);
+  }
+}
+
+static AOM_FORCE_INLINE void dr_prediction_z2_8xH_core_aarch64(
+    int H, uint8_t *dst, ptrdiff_t stride, const uint8_t *above,
+    const uint8_t *left, int upsample_above, int upsample_left, int dx,
+    int dy) {
+  int r = 0;
+  int x = -dx;
+
+  if (!upsample_above) {
+    // base_x = x >> 6 is constant (-1) throughout x in [-64, -1].
+    if (x >= -64) {
+      const uint8x8_t a0_x0 = vld1_u8(above - 1);
+      const uint8x8_t a1_x0 = vld1_u8(above);
+      const uint16x8_t diff0 = vsubl_u8(a1_x0, a0_x0);
+      const uint16x8_t a32 = vshll_n_u8(a0_x0, 5);
+      for (; r + 2 <= H && x - dx >= -64; r += 2, x -= 2 * dx) {
+        const uint16x8_t s0 = vdupq_n_u16((x & 0x3f) >> 1);
+        const uint16x8_t s1 = vdupq_n_u16(((x - dx) & 0x3f) >> 1);
+        vst1_u8(dst, vrshrn_n_u16(vmlaq_u16(a32, diff0, s0), 5));
+        vst1_u8(dst + stride, vrshrn_n_u16(vmlaq_u16(a32, diff0, s1), 5));
+        dst += 2 * stride;
+      }
+      if (r < H && x >= -64) {
+        const uint16x8_t s0 = vdupq_n_u16((x & 0x3f) >> 1);
+        vst1_u8(dst, vrshrn_n_u16(vmlaq_u16(a32, diff0, s0), 5));
+        dst += stride;
+        r++;
+        x -= dx;
+      }
+    }
+  } else {
+    for (; r < H && x >= -64; r++, x -= dx) {
+      const int base_x = x >> 5;
+      vst1_u8(dst, dr_prediction_z2_8xH_above_neon(above, 1, base_x, x));
+      dst += stride;
+    }
+  }
+  if (r == H) return;
+
+  const int span = ((H - 1) << upsample_left) + 3;
+  if (span < 16) {
+    dr_prediction_z2_8xH_left_core_aarch64(H, r, x, dst, stride, above, left,
+                                           upsample_above, upsample_left, dx,
+                                           dy, 1);
+  } else if (upsample_above || span < 32) {
+    assert(span < 32);
+    dr_prediction_z2_8xH_left_core_aarch64(H, r, x, dst, stride, above, left,
+                                           upsample_above, upsample_left, dx,
+                                           dy, 2);
+  } else {
+    dr_prediction_z2_8xH_left_core_aarch64(H, r, x, dst, stride, above, left,
+                                           upsample_above, upsample_left, dx,
+                                           dy, 3);
+  }
+}
+#endif  // AOM_ARCH_AARCH64
+
 static void dr_prediction_z2_8xH_neon(int H, uint8_t *dst, ptrdiff_t stride,
                                       const uint8_t *above, const uint8_t *left,
                                       int upsample_above, int upsample_left,
                                       int dx, int dy) {
-  const int min_base_x = -(1 << upsample_above);
-  const int frac_bits_x = 6 - upsample_above;
-  const int frac_bits_y = 6 - upsample_left;
-
+  assert(H >= 4 && (H & 3) == 0);
+  assert(dx > 0);
   //   above0 * (32 - shift) + above1 * shift
   // = (above1 - above0) * shift + above0 * 32

 #if AOM_ARCH_AARCH64
-  // Use ext rather than loading left + 30 directly to avoid over-read.
-  const uint8x16_t left_m2 = vld1q_u8(left - 2);
-  const uint8x16_t left_0 = vld1q_u8(left + 0);
-  const uint8x16_t left_16 = vld1q_u8(left + 16);
-  const uint8x16_t left_14 = vextq_u8(left_0, left_16, 14);
-  const uint8x16_t left_30 = vextq_u8(left_16, left_16, 14);
-  const uint8x16x3_t left_vals = { { left_m2, left_14, left_30 } };
-#define LEFT left_vals
-#else  // !AOM_ARCH_AARCH64
-#define LEFT left
-#endif  // AOM_ARCH_AARCH64
+  assert(upsample_above == 0 || 8 + H <= 16);
+  if (8 + H <= 16 && upsample_above) {
+    dr_prediction_z2_8xH_core_aarch64(H, dst, stride, above, left, 1,
+                                      upsample_left, dx, dy);
+  } else {
+    dr_prediction_z2_8xH_core_aarch64(H, dst, stride, above, left, 0,
+                                      upsample_left, dx, dy);
+  }
+#else   // !AOM_ARCH_AARCH64
+  const int min_base_x = -(1 << upsample_above);
+  const int frac_bits_x = 6 - upsample_above;
+  const int frac_bits_y = 6 - upsample_left;

-  for (int r = 0; r < H; r++) {
-    int y = r + 1;
-    int base_x = (-y * dx) >> frac_bits_x;
-    int base_min_diff =
+  const uint16x8_t c1_8 = vcombine_u16(vcreate_u16(0x0004000300020001),
+                                       vcreate_u16(0x0008000700060005));
+  const int16x8_t y_c128_base =
+      vnegq_s16(vmulq_n_s16(vreinterpretq_s16_u16(c1_8), (int16_t)dy));
+  const int16x8_t v_frac_bits_y = vdupq_n_s16(-frac_bits_y);
+
+  int r = 0;
+  for (int x = -dx; r < H; r++, x -= dx) {
+    const int base_x = x >> frac_bits_x;
+    const int base_min_diff =
         (min_base_x - base_x + upsample_above) >> upsample_above;

     if (base_min_diff <= 0) {
-      uint8x8_t resx =
-          dr_prediction_z2_8xH_above_neon(above, upsample_above, dx, base_x, y);
+      const uint8x8_t resx =
+          dr_prediction_z2_8xH_above_neon(above, upsample_above, base_x, x);
       vst1_u8(dst, resx);
     } else if (base_min_diff < 8) {
-      uint8x8_t resx =
-          dr_prediction_z2_8xH_above_neon(above, upsample_above, dx, base_x, y);
-      uint8x8_t resy = dr_prediction_z2_8xH_left_neon(LEFT, upsample_left, dy,
-                                                      r, frac_bits_y);
-      uint8x8_t mask = vld1_u8(BaseMask[base_min_diff]);
-      uint8x8_t resxy = vbsl_u8(mask, resy, resx);
+      const uint8x8_t resx =
+          dr_prediction_z2_8xH_above_neon(above, upsample_above, base_x, x);
+      const int16x8_t y_c128 = vaddq_s16(y_c128_base, vdupq_n_s16(r << 6));
+      const uint8x8_t resy = dr_prediction_z2_8xH_left_neon(
+          left, upsample_left, y_c128, v_frac_bits_y);
+      const uint8x8_t mask = vld1_u8(BaseMask[base_min_diff]);
+      const uint8x8_t resxy = vbsl_u8(mask, resy, resx);
       vst1_u8(dst, resxy);
     } else {
-      uint8x8_t resy = dr_prediction_z2_8xH_left_neon(LEFT, upsample_left, dy,
-                                                      r, frac_bits_y);
-      vst1_u8(dst, resy);
+      break;
     }

     dst += stride;
   }
-#undef LEFT
+
+  for (; r < H; r++) {
+    const int16x8_t y_c128 = vaddq_s16(y_c128_base, vdupq_n_s16(r << 6));
+    const uint8x8_t resy = dr_prediction_z2_8xH_left_neon(
+        left, upsample_left, y_c128, v_frac_bits_y);
+    vst1_u8(dst, resy);
+    dst += stride;
+  }
+#endif  // AOM_ARCH_AARCH64
 }

 static void dr_prediction_z2_WxH_neon(int W, int H, uint8_t *dst,
                                       ptrdiff_t stride, const uint8_t *above,
                                       const uint8_t *left, int dx, int dy) {
+  assert(W >= 16 && (W & 15) == 0);
+  assert(H >= 4 && (H & 3) == 0);
   // here upsample_above and upsample_left are 0 by design of
   // av1_use_intra_edge_upsample
   const int min_base_x = -1;

+  // base_x = (-y * dx) >> 6 decreases monotonically with y = r + 1, so for all
+  // r in [0, H) and j >= 0, base_min_diff <= 0 iff (-H * dx) >> 6 >= -1, i.e.
+  // H * dx <= 64 (base_x == -1 on every row; hoist above loads per 16B strip).
+  if (H * dx <= 64) {
+    int j = 0;
+    do {
+      const uint8x16_t a0 = vld1q_u8(above - 1 + j);
+      const uint8x16_t a1 = vld1q_u8(above + j);
+      const uint16x8_t diff0 = vsubl_u8(vget_low_u8(a1), vget_low_u8(a0));
+      const uint16x8_t diff1 = vsubl_u8(vget_high_u8(a1), vget_high_u8(a0));
+      const uint16x8_t a32_0 = vshll_n_u8(vget_low_u8(a0), 5);
+      const uint16x8_t a32_1 = vshll_n_u8(vget_high_u8(a0), 5);
+      uint8_t *row_dst = dst + j;
+      int r = H;
+      int x = -dx;
+      do {
+        const uint16x8_t shift = vdupq_n_u16((x & 0x3f) >> 1);
+        const uint16x8_t res0 = vmlaq_u16(a32_0, diff0, shift);
+        const uint16x8_t res1 = vmlaq_u16(a32_1, diff1, shift);
+        vst1q_u8(row_dst,
+                 vcombine_u8(vrshrn_n_u16(res0, 5), vrshrn_n_u16(res1, 5)));
+        row_dst += stride;
+        x -= dx;
+      } while (--r != 0);
+      j += 16;
+    } while (j != W);
+    return;
+  }
+
 #if AOM_ARCH_AARCH64
   const uint8x16_t left_m1 = vld1q_u8(left - 1);
   const uint8x16_t left_0 = vld1q_u8(left + 0);
@@ -2124,30 +2720,45 @@ static void dr_prediction_z2_WxH_neon(int W, int H, uint8_t *dst,
 #define LEFT left
 #endif  // AOM_ARCH_AARCH64

-  for (int r = 0; r < H; r++) {
-    int y = r + 1;
-    int base_x = (-y * dx) >> 6;
-    for (int j = 0; j < W; j += 16) {
+  const uint16x8_t c1_8 = vcombine_u16(vcreate_u16(0x0004000300020001),
+                                       vcreate_u16(0x0008000700060005));
+  const uint16x8_t c9_16 = vcombine_u16(vcreate_u16(0x000C000B000A0009),
+                                        vcreate_u16(0x0010000F000E000D));
+  const int16x8_t y_base_lo =
+      vnegq_s16(vmulq_n_s16(vreinterpretq_s16_u16(c1_8), (int16_t)dy));
+  const int16x8_t y_base_hi =
+      vnegq_s16(vmulq_n_s16(vreinterpretq_s16_u16(c9_16), (int16_t)dy));
+
+  int r = 0;
+  int x = -dx;
+  do {
+    const int base_x = x >> 6;
+    const uint16x8_t shift = vdupq_n_u16((x & 0x3f) >> 1);
+    int j = 0;
+    do {
       const int base_min_diff = min_base_x - base_x - j;

       if (base_min_diff <= 0) {
-        uint8x16_t resx =
-            dr_prediction_z2_WxH_above_neon(above, dx, base_x, y, j);
+        const uint8x16_t resx =
+            dr_prediction_z2_WxH_above_neon(above, base_x, shift, j);
         vst1q_u8(dst + j, resx);
       } else if (base_min_diff < 16) {
-        uint8x16_t resx =
-            dr_prediction_z2_WxH_above_neon(above, dx, base_x, y, j);
-        uint8x16_t resy = dr_prediction_z2_WxH_left_neon(LEFT, dy, r, j);
-        uint8x16_t mask = vld1q_u8(BaseMask[base_min_diff]);
-        uint8x16_t resxy = vbslq_u8(mask, resy, resx);
-        vst1q_u8(dst + j, resxy);
+        const uint8x16_t resx =
+            dr_prediction_z2_WxH_above_neon(above, base_x, shift, j);
+        const uint8x16_t resy = dr_prediction_z2_WxH_left_neon(
+            LEFT, y_base_lo, y_base_hi, dy, r, j);
+        const uint8x16_t mask = vld1q_u8(BaseMask[base_min_diff]);
+        vst1q_u8(dst + j, vbslq_u8(mask, resy, resx));
       } else {
-        uint8x16_t resy = dr_prediction_z2_WxH_left_neon(LEFT, dy, r, j);
+        const uint8x16_t resy = dr_prediction_z2_WxH_left_neon(
+            LEFT, y_base_lo, y_base_hi, dy, r, j);
         vst1q_u8(dst + j, resy);
       }
-    }  // for j
+      j += 16;
+    } while (j != W);
     dst += stride;
-  }
+    x -= dx;
+  } while (++r != H);
 #undef LEFT
 }