Commit d6f77c9d9f for aom

commit d6f77c9d9fe4b708a4d933cd4812634cabf0ad86
Author: Frank Galligan <fgalligan@google.com>
Date:   Sat Nov 14 10:53:13 2026 -0800

    aom_dsp,neon: Unroll highbd sad, sadxd & sse

    Unroll highbd_sad4xh_neon with 2-row vcombine_u16 and vabaq_u16. Unroll
    highbd_sad{8,32..128}xh_neon and highbd_sad{4,8}xhx{3,4}d_neon across
    2-row iterations, and highbd_sad{16..128}xhx{3,4}d_neon across 16-wide
    chunks with dual accumulators. Optimize highbd_sse_neon with unrolled
    multi-accumulator loops.

    Speedup: 1.18x (128x128 10b SAD) in test_libaom on Neoverse N3; -0.27%
    (1T) / -0.59% (4T, -0.85% min) 10b avifenc on Neoverse N2 (bit-exact).

    TAG=agy
    CONV=490488d4-7005-402a-8cc4-c245227f80fe

    Change-Id: Icc4c5cb870d65ce16d881cf20bd808f171c229bc

diff --git a/aom_dsp/arm/highbd_sad_neon.c b/aom_dsp/arm/highbd_sad_neon.c
index b8dde45805..03a1976037 100644
--- a/aom_dsp/arm/highbd_sad_neon.c
+++ b/aom_dsp/arm/highbd_sad_neon.c
@@ -11,6 +11,7 @@
  */

 #include <arm_neon.h>
+#include <assert.h>

 #include "config/aom_config.h"
 #include "config/aom_dsp_rtcd.h"
@@ -25,18 +26,38 @@ static inline uint32_t highbd_sad4xh_neon(const uint8_t *src_ptr,
                                           int ref_stride, int h) {
   const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
   const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr);
-  uint32x4_t sum = vdupq_n_u32(0);
+  const int src_stride_2 = src_stride << 1;
+  const int ref_stride_2 = ref_stride << 1;
+
+  // 'h_overflow' is the number of 4-wide rows packed in pairs into uint16x8_t
+  // before 16-bit accumulators overflow: 65535 / 4095 ~= 16 pairs = 32 rows.
+  const int h_overflow = 32;
+  const int h_limit = h < h_overflow ? h : h_overflow;
+  assert(h % 2 == 0);
+  assert(h % h_limit == 0);
+
+  uint32x4_t sum_u32 = vdupq_n_u32(0);

   do {
-    uint16x4_t s = vld1_u16(src16_ptr);
-    uint16x4_t r = vld1_u16(ref16_ptr);
-    sum = vabal_u16(sum, s, r);
+    uint16x8_t sum_u16 = vdupq_n_u16(0);
+    int i = h_limit;
+    do {
+      uint16x8_t s =
+          vcombine_u16(vld1_u16(src16_ptr), vld1_u16(src16_ptr + src_stride));
+      uint16x8_t r =
+          vcombine_u16(vld1_u16(ref16_ptr), vld1_u16(ref16_ptr + ref_stride));
+      sum_u16 = vabaq_u16(sum_u16, s, r);
+
+      src16_ptr += src_stride_2;
+      ref16_ptr += ref_stride_2;
+      i -= 2;
+    } while (i != 0);

-    src16_ptr += src_stride;
-    ref16_ptr += ref_stride;
-  } while (--h != 0);
+    sum_u32 = vpadalq_u16(sum_u32, sum_u16);
+    h -= h_limit;
+  } while (h != 0);

-  return horizontal_add_u32x4(sum);
+  return horizontal_add_u32x4(sum_u32);
 }

 static inline uint32_t highbd_sad8xh_neon(const uint8_t *src_ptr,
@@ -45,31 +66,37 @@ static inline uint32_t highbd_sad8xh_neon(const uint8_t *src_ptr,
                                           int ref_stride, int h) {
   const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
   const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr);
+  const int src_stride_2 = src_stride << 1;
+  const int ref_stride_2 = ref_stride << 1;

-  // 'h_overflow' is the number of 8-wide rows we can process before 16-bit
-  // accumulators overflow. After hitting this limit accumulate into 32-bit
-  // elements. 65535 / 4095 ~= 16, so 16 8-wide rows.
-  const int h_overflow = 16;
-  // If block height 'h' is smaller than this limit, use 'h' instead.
+  // Two 16-bit accumulators each hold up to 16 rows: 32 rows total.
+  const int h_overflow = 32;
   const int h_limit = h < h_overflow ? h : h_overflow;
+  assert(h % 2 == 0);
   assert(h % h_limit == 0);

   uint32x4_t sum_u32 = vdupq_n_u32(0);

   do {
-    uint16x8_t sum_u16 = vdupq_n_u16(0);
+    uint16x8_t sum0_u16 = vdupq_n_u16(0);
+    uint16x8_t sum1_u16 = vdupq_n_u16(0);

     int i = h_limit;
     do {
       uint16x8_t s0 = vld1q_u16(src16_ptr);
       uint16x8_t r0 = vld1q_u16(ref16_ptr);
-      sum_u16 = vabaq_u16(sum_u16, s0, r0);
+      uint16x8_t s1 = vld1q_u16(src16_ptr + src_stride);
+      uint16x8_t r1 = vld1q_u16(ref16_ptr + ref_stride);
+      sum0_u16 = vabaq_u16(sum0_u16, s0, r0);
+      sum1_u16 = vabaq_u16(sum1_u16, s1, r1);

-      src16_ptr += src_stride;
-      ref16_ptr += ref_stride;
-    } while (--i != 0);
+      src16_ptr += src_stride_2;
+      ref16_ptr += ref_stride_2;
+      i -= 2;
+    } while (i != 0);

-    sum_u32 = vpadalq_u16(sum_u32, sum_u16);
+    sum_u32 = vpadalq_u16(sum_u32, sum0_u16);
+    sum_u32 = vpadalq_u16(sum_u32, sum1_u16);

     h -= h_limit;
   } while (h != 0);
@@ -124,47 +151,75 @@ static inline uint32_t highbd_sadwxh_neon(const uint8_t *src_ptr,
                                           const int h_overflow) {
   const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
   const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr);
+  const int src_stride_2 = src_stride << 1;
+  const int ref_stride_2 = ref_stride << 1;

   const int h_limit = h < h_overflow ? h : h_overflow;
+  assert(h % 2 == 0);
   assert(h % h_limit == 0);

   uint32x4_t sum_u32 = vdupq_n_u32(0);

   do {
-    uint16x8_t sum_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
-                              vdupq_n_u16(0) };
+    uint16x8_t sum0_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+                               vdupq_n_u16(0) };
+    uint16x8_t sum1_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+                               vdupq_n_u16(0) };

     int i = h_limit;
     do {
+      const uint16_t *src16_r1 = src16_ptr + src_stride;
+      const uint16_t *ref16_r1 = ref16_ptr + ref_stride;
       int j = 0;
       do {
         uint16x8_t s0 = vld1q_u16(src16_ptr + j);
         uint16x8_t r0 = vld1q_u16(ref16_ptr + j);
-        sum_u16[0] = vabaq_u16(sum_u16[0], s0, r0);
+        sum0_u16[0] = vabaq_u16(sum0_u16[0], s0, r0);

         uint16x8_t s1 = vld1q_u16(src16_ptr + j + 8);
         uint16x8_t r1 = vld1q_u16(ref16_ptr + j + 8);
-        sum_u16[1] = vabaq_u16(sum_u16[1], s1, r1);
+        sum0_u16[1] = vabaq_u16(sum0_u16[1], s1, r1);

         uint16x8_t s2 = vld1q_u16(src16_ptr + j + 16);
         uint16x8_t r2 = vld1q_u16(ref16_ptr + j + 16);
-        sum_u16[2] = vabaq_u16(sum_u16[2], s2, r2);
+        sum0_u16[2] = vabaq_u16(sum0_u16[2], s2, r2);

         uint16x8_t s3 = vld1q_u16(src16_ptr + j + 24);
         uint16x8_t r3 = vld1q_u16(ref16_ptr + j + 24);
-        sum_u16[3] = vabaq_u16(sum_u16[3], s3, r3);
+        sum0_u16[3] = vabaq_u16(sum0_u16[3], s3, r3);
+
+        uint16x8_t t0 = vld1q_u16(src16_r1 + j);
+        uint16x8_t q0 = vld1q_u16(ref16_r1 + j);
+        sum1_u16[0] = vabaq_u16(sum1_u16[0], t0, q0);
+
+        uint16x8_t t1 = vld1q_u16(src16_r1 + j + 8);
+        uint16x8_t q1 = vld1q_u16(ref16_r1 + j + 8);
+        sum1_u16[1] = vabaq_u16(sum1_u16[1], t1, q1);
+
+        uint16x8_t t2 = vld1q_u16(src16_r1 + j + 16);
+        uint16x8_t q2 = vld1q_u16(ref16_r1 + j + 16);
+        sum1_u16[2] = vabaq_u16(sum1_u16[2], t2, q2);
+
+        uint16x8_t t3 = vld1q_u16(src16_r1 + j + 24);
+        uint16x8_t q3 = vld1q_u16(ref16_r1 + j + 24);
+        sum1_u16[3] = vabaq_u16(sum1_u16[3], t3, q3);

         j += 32;
       } while (j < w);

-      src16_ptr += src_stride;
-      ref16_ptr += ref_stride;
-    } while (--i != 0);
+      src16_ptr += src_stride_2;
+      ref16_ptr += ref_stride_2;
+      i -= 2;
+    } while (i != 0);

-    sum_u32 = vpadalq_u16(sum_u32, sum_u16[0]);
-    sum_u32 = vpadalq_u16(sum_u32, sum_u16[1]);
-    sum_u32 = vpadalq_u16(sum_u32, sum_u16[2]);
-    sum_u32 = vpadalq_u16(sum_u32, sum_u16[3]);
+    sum_u32 = vpadalq_u16(sum_u32, sum0_u16[0]);
+    sum_u32 = vpadalq_u16(sum_u32, sum0_u16[1]);
+    sum_u32 = vpadalq_u16(sum_u32, sum0_u16[2]);
+    sum_u32 = vpadalq_u16(sum_u32, sum0_u16[3]);
+    sum_u32 = vpadalq_u16(sum_u32, sum1_u16[0]);
+    sum_u32 = vpadalq_u16(sum_u32, sum1_u16[1]);
+    sum_u32 = vpadalq_u16(sum_u32, sum1_u16[2]);
+    sum_u32 = vpadalq_u16(sum_u32, sum1_u16[3]);

     h -= h_limit;
   } while (h != 0);
@@ -175,10 +230,8 @@ static inline uint32_t highbd_sad32xh_neon(const uint8_t *src_ptr,
                                            int src_stride,
                                            const uint8_t *ref_ptr,
                                            int ref_stride, int h) {
-  // 'h_overflow' is the number of 32-wide rows we can process before 16-bit
-  // accumulators overflow. After hitting this limit accumulate into 32-bit
-  // elements. 65535 / 4095 ~= 16, so 16 32-wide rows using four accumulators.
-  const int h_overflow = 16;
+  // Eight 16-bit accumulators hold up to 32 32-wide rows before overflow.
+  const int h_overflow = 32;
   return highbd_sadwxh_neon(src_ptr, src_stride, ref_ptr, ref_stride, 32, h,
                             h_overflow);
 }
@@ -187,10 +240,8 @@ static inline uint32_t highbd_sad64xh_neon(const uint8_t *src_ptr,
                                            int src_stride,
                                            const uint8_t *ref_ptr,
                                            int ref_stride, int h) {
-  // 'h_overflow' is the number of 64-wide rows we can process before 16-bit
-  // accumulators overflow. After hitting this limit accumulate into 32-bit
-  // elements. 65535 / 4095 ~= 16, so 8 64-wide rows using four accumulators.
-  const int h_overflow = 8;
+  // Eight 16-bit accumulators hold up to 16 64-wide rows before overflow.
+  const int h_overflow = 16;
   return highbd_sadwxh_neon(src_ptr, src_stride, ref_ptr, ref_stride, 64, h,
                             h_overflow);
 }
@@ -199,10 +250,8 @@ static inline uint32_t highbd_sad128xh_neon(const uint8_t *src_ptr,
                                             int src_stride,
                                             const uint8_t *ref_ptr,
                                             int ref_stride, int h) {
-  // 'h_overflow' is the number of 128-wide rows we can process before 16-bit
-  // accumulators overflow. After hitting this limit accumulate into 32-bit
-  // elements. 65535 / 4095 ~= 16, so 4 128-wide rows using four accumulators.
-  const int h_overflow = 4;
+  // Eight 16-bit accumulators hold up to 8 128-wide rows before overflow.
+  const int h_overflow = 8;
   return highbd_sadwxh_neon(src_ptr, src_stride, ref_ptr, ref_stride, 128, h,
                             h_overflow);
 }
diff --git a/aom_dsp/arm/highbd_sadxd_neon.c b/aom_dsp/arm/highbd_sadxd_neon.c
index 0b4527873a..75dd008a66 100644
--- a/aom_dsp/arm/highbd_sadxd_neon.c
+++ b/aom_dsp/arm/highbd_sadxd_neon.c
@@ -11,6 +11,7 @@
  */

 #include <arm_neon.h>
+#include <assert.h>

 #include "config/aom_config.h"
 #include "config/aom_dsp_rtcd.h"
@@ -28,26 +29,56 @@ static inline void highbd_sad4xhx4d_neon(const uint8_t *src_ptr, int src_stride,
   const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
   const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
   const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]);
+  const int src_stride_2 = src_stride << 1;
+  const int ref_stride_2 = ref_stride << 1;

-  uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
-                        vdupq_n_u32(0) };
+  // 'h_overflow' is the number of 4-wide rows packed in pairs into uint16x8_t
+  // before 16-bit accumulators overflow: 65535 / 4095 ~= 16 pairs = 32 rows.
+  const int h_overflow = 32;
+  const int h_limit = h < h_overflow ? h : h_overflow;
+  assert(h % 2 == 0);
+  assert(h % h_limit == 0);

-  int i = 0;
-  do {
-    uint16x4_t s = vld1_u16(src16_ptr + i * src_stride);
-    uint16x4_t r0 = vld1_u16(ref16_ptr0 + i * ref_stride);
-    uint16x4_t r1 = vld1_u16(ref16_ptr1 + i * ref_stride);
-    uint16x4_t r2 = vld1_u16(ref16_ptr2 + i * ref_stride);
-    uint16x4_t r3 = vld1_u16(ref16_ptr3 + i * ref_stride);
+  uint32x4_t sum_u32[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+                            vdupq_n_u32(0) };

-    sum[0] = vabal_u16(sum[0], s, r0);
-    sum[1] = vabal_u16(sum[1], s, r1);
-    sum[2] = vabal_u16(sum[2], s, r2);
-    sum[3] = vabal_u16(sum[3], s, r3);
+  do {
+    uint16x8_t sum_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+                              vdupq_n_u16(0) };
+    int i = h_limit;
+    do {
+      uint16x8_t s =
+          vcombine_u16(vld1_u16(src16_ptr), vld1_u16(src16_ptr + src_stride));
+      uint16x8_t r0 =
+          vcombine_u16(vld1_u16(ref16_ptr0), vld1_u16(ref16_ptr0 + ref_stride));
+      uint16x8_t r1 =
+          vcombine_u16(vld1_u16(ref16_ptr1), vld1_u16(ref16_ptr1 + ref_stride));
+      uint16x8_t r2 =
+          vcombine_u16(vld1_u16(ref16_ptr2), vld1_u16(ref16_ptr2 + ref_stride));
+      uint16x8_t r3 =
+          vcombine_u16(vld1_u16(ref16_ptr3), vld1_u16(ref16_ptr3 + ref_stride));
+
+      sum_u16[0] = vabaq_u16(sum_u16[0], s, r0);
+      sum_u16[1] = vabaq_u16(sum_u16[1], s, r1);
+      sum_u16[2] = vabaq_u16(sum_u16[2], s, r2);
+      sum_u16[3] = vabaq_u16(sum_u16[3], s, r3);
+
+      src16_ptr += src_stride_2;
+      ref16_ptr0 += ref_stride_2;
+      ref16_ptr1 += ref_stride_2;
+      ref16_ptr2 += ref_stride_2;
+      ref16_ptr3 += ref_stride_2;
+      i -= 2;
+    } while (i != 0);

-  } while (++i < h);
+    sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]);
+    sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]);
+    sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]);
+    sum_u32[3] = vpadalq_u16(sum_u32[3], sum_u16[3]);
+    h -= h_limit;
+  } while (h != 0);

-  vst1q_u32(res, horizontal_add_4d_u32x4(sum));
+  vst1q_u32(res, horizontal_add_4d_u32x4(sum_u32));
 }

 static inline void highbd_sad8xhx4d_neon(const uint8_t *src_ptr, int src_stride,
@@ -59,43 +90,61 @@ static inline void highbd_sad8xhx4d_neon(const uint8_t *src_ptr, int src_stride,
   const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
   const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
   const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]);
+  const int src_stride_2 = src_stride << 1;
+  const int ref_stride_2 = ref_stride << 1;

-  // 'h_overflow' is the number of 8-wide rows we can process before 16-bit
-  // accumulators overflow. After hitting this limit accumulate into 32-bit
-  // elements. 65535 / 4095 ~= 16, so 16 8-wide rows.
-  const int h_overflow = 16;
-  // If block height 'h' is smaller than this limit, use 'h' instead.
+  // Two 16-bit accumulators each hold up to 16 rows: 32 rows total.
+  const int h_overflow = 32;
   const int h_limit = h < h_overflow ? h : h_overflow;
+  assert(h % 2 == 0);
   assert(h % h_limit == 0);

   uint32x4_t sum_u32[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
                             vdupq_n_u32(0) };

-  int h_tmp = h_limit;
-  int i = 0;
   do {
-    uint16x8_t sum_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
-                              vdupq_n_u16(0) };
+    uint16x8_t sum0_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+                               vdupq_n_u16(0) };
+    uint16x8_t sum1_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+                               vdupq_n_u16(0) };

+    int i = h_limit;
     do {
-      uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride);
-
-      sum_u16[0] =
-          vabaq_u16(sum_u16[0], s0, vld1q_u16(ref16_ptr0 + i * ref_stride));
-      sum_u16[1] =
-          vabaq_u16(sum_u16[1], s0, vld1q_u16(ref16_ptr1 + i * ref_stride));
-      sum_u16[2] =
-          vabaq_u16(sum_u16[2], s0, vld1q_u16(ref16_ptr2 + i * ref_stride));
-      sum_u16[3] =
-          vabaq_u16(sum_u16[3], s0, vld1q_u16(ref16_ptr3 + i * ref_stride));
-    } while (++i < h_tmp);
-
-    sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]);
-    sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]);
-    sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]);
-    sum_u32[3] = vpadalq_u16(sum_u32[3], sum_u16[3]);
+      uint16x8_t s0 = vld1q_u16(src16_ptr);
+      uint16x8_t s1 = vld1q_u16(src16_ptr + src_stride);
+
+      sum0_u16[0] = vabaq_u16(sum0_u16[0], s0, vld1q_u16(ref16_ptr0));
+      sum0_u16[1] = vabaq_u16(sum0_u16[1], s0, vld1q_u16(ref16_ptr1));
+      sum0_u16[2] = vabaq_u16(sum0_u16[2], s0, vld1q_u16(ref16_ptr2));
+      sum0_u16[3] = vabaq_u16(sum0_u16[3], s0, vld1q_u16(ref16_ptr3));
+
+      sum1_u16[0] =
+          vabaq_u16(sum1_u16[0], s1, vld1q_u16(ref16_ptr0 + ref_stride));
+      sum1_u16[1] =
+          vabaq_u16(sum1_u16[1], s1, vld1q_u16(ref16_ptr1 + ref_stride));
+      sum1_u16[2] =
+          vabaq_u16(sum1_u16[2], s1, vld1q_u16(ref16_ptr2 + ref_stride));
+      sum1_u16[3] =
+          vabaq_u16(sum1_u16[3], s1, vld1q_u16(ref16_ptr3 + ref_stride));
+
+      src16_ptr += src_stride_2;
+      ref16_ptr0 += ref_stride_2;
+      ref16_ptr1 += ref_stride_2;
+      ref16_ptr2 += ref_stride_2;
+      ref16_ptr3 += ref_stride_2;
+      i -= 2;
+    } while (i != 0);
+
+    sum_u32[0] = vpadalq_u16(sum_u32[0], sum0_u16[0]);
+    sum_u32[1] = vpadalq_u16(sum_u32[1], sum0_u16[1]);
+    sum_u32[2] = vpadalq_u16(sum_u32[2], sum0_u16[2]);
+    sum_u32[3] = vpadalq_u16(sum_u32[3], sum0_u16[3]);
+
+    sum_u32[0] = vpadalq_u16(sum_u32[0], sum1_u16[0]);
+    sum_u32[1] = vpadalq_u16(sum_u32[1], sum1_u16[1]);
+    sum_u32[2] = vpadalq_u16(sum_u32[2], sum1_u16[2]);
+    sum_u32[3] = vpadalq_u16(sum_u32[3], sum1_u16[3]);

-    h_tmp += h_limit;
     h -= h_limit;
   } while (h != 0);

@@ -119,8 +168,10 @@ static inline void highbd_sadwxhx4d_neon(const uint8_t *src_ptr, int src_stride,
                             vdupq_n_u32(0) };

   do {
-    uint16x8_t sum_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
-                              vdupq_n_u16(0) };
+    uint16x8_t sum0_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+                               vdupq_n_u16(0) };
+    uint16x8_t sum1_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+                               vdupq_n_u16(0) };

     int i = h_limit;
     do {
@@ -128,16 +179,16 @@ static inline void highbd_sadwxhx4d_neon(const uint8_t *src_ptr, int src_stride,
       do {
         uint16x8_t s0 = vld1q_u16(src16_ptr + j);

-        sum_u16[0] = vabaq_u16(sum_u16[0], s0, vld1q_u16(ref16_ptr0 + j));
-        sum_u16[1] = vabaq_u16(sum_u16[1], s0, vld1q_u16(ref16_ptr1 + j));
-        sum_u16[2] = vabaq_u16(sum_u16[2], s0, vld1q_u16(ref16_ptr2 + j));
-        sum_u16[3] = vabaq_u16(sum_u16[3], s0, vld1q_u16(ref16_ptr3 + j));
+        sum0_u16[0] = vabaq_u16(sum0_u16[0], s0, vld1q_u16(ref16_ptr0 + j));
+        sum0_u16[1] = vabaq_u16(sum0_u16[1], s0, vld1q_u16(ref16_ptr1 + j));
+        sum0_u16[2] = vabaq_u16(sum0_u16[2], s0, vld1q_u16(ref16_ptr2 + j));
+        sum0_u16[3] = vabaq_u16(sum0_u16[3], s0, vld1q_u16(ref16_ptr3 + j));

         uint16x8_t s1 = vld1q_u16(src16_ptr + j + 8);
-        sum_u16[0] = vabaq_u16(sum_u16[0], s1, vld1q_u16(ref16_ptr0 + j + 8));
-        sum_u16[1] = vabaq_u16(sum_u16[1], s1, vld1q_u16(ref16_ptr1 + j + 8));
-        sum_u16[2] = vabaq_u16(sum_u16[2], s1, vld1q_u16(ref16_ptr2 + j + 8));
-        sum_u16[3] = vabaq_u16(sum_u16[3], s1, vld1q_u16(ref16_ptr3 + j + 8));
+        sum1_u16[0] = vabaq_u16(sum1_u16[0], s1, vld1q_u16(ref16_ptr0 + j + 8));
+        sum1_u16[1] = vabaq_u16(sum1_u16[1], s1, vld1q_u16(ref16_ptr1 + j + 8));
+        sum1_u16[2] = vabaq_u16(sum1_u16[2], s1, vld1q_u16(ref16_ptr2 + j + 8));
+        sum1_u16[3] = vabaq_u16(sum1_u16[3], s1, vld1q_u16(ref16_ptr3 + j + 8));

         j += 16;
       } while (j < w);
@@ -149,10 +200,15 @@ static inline void highbd_sadwxhx4d_neon(const uint8_t *src_ptr, int src_stride,
       ref16_ptr3 += ref_stride;
     } while (--i != 0);

-    sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]);
-    sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]);
-    sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]);
-    sum_u32[3] = vpadalq_u16(sum_u32[3], sum_u16[3]);
+    sum_u32[0] = vpadalq_u16(sum_u32[0], sum0_u16[0]);
+    sum_u32[1] = vpadalq_u16(sum_u32[1], sum0_u16[1]);
+    sum_u32[2] = vpadalq_u16(sum_u32[2], sum0_u16[2]);
+    sum_u32[3] = vpadalq_u16(sum_u32[3], sum0_u16[3]);
+
+    sum_u32[0] = vpadalq_u16(sum_u32[0], sum1_u16[0]);
+    sum_u32[1] = vpadalq_u16(sum_u32[1], sum1_u16[1]);
+    sum_u32[2] = vpadalq_u16(sum_u32[2], sum1_u16[2]);
+    sum_u32[3] = vpadalq_u16(sum_u32[3], sum1_u16[3]);

     h -= h_limit;
   } while (h != 0);
@@ -165,10 +221,8 @@ static inline void highbd_sad16xhx4d_neon(const uint8_t *src_ptr,
                                           const uint8_t *const ref_ptr[4],
                                           int ref_stride, uint32_t res[4],
                                           int h) {
-  // 'h_overflow' is the number of 16-wide rows we can process before 16-bit
-  // accumulators overflow. After hitting this limit accumulate into 32-bit
-  // elements. 65535 / 4095 ~= 16, so 8 16-wide rows.
-  const int h_overflow = 8;
+  // Two 16-bit accumulators hold up to 16 16-wide rows before overflow.
+  const int h_overflow = 16;
   highbd_sadwxhx4d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 16, h,
                         h_overflow);
 }
@@ -178,10 +232,8 @@ static inline void highbd_sad32xhx4d_neon(const uint8_t *src_ptr,
                                           const uint8_t *const ref_ptr[4],
                                           int ref_stride, uint32_t res[4],
                                           int h) {
-  // 'h_overflow' is the number of 32-wide rows we can process before 16-bit
-  // accumulators overflow. After hitting this limit accumulate into 32-bit
-  // elements. 65535 / 4095 ~= 16, so 4 32-wide rows.
-  const int h_overflow = 4;
+  // Two 16-bit accumulators hold up to 8 32-wide rows before overflow.
+  const int h_overflow = 8;
   highbd_sadwxhx4d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 32, h,
                         h_overflow);
 }
@@ -191,10 +243,8 @@ static inline void highbd_sad64xhx4d_neon(const uint8_t *src_ptr,
                                           const uint8_t *const ref_ptr[4],
                                           int ref_stride, uint32_t res[4],
                                           int h) {
-  // 'h_overflow' is the number of 64-wide rows we can process before 16-bit
-  // accumulators overflow. After hitting this limit accumulate into 32-bit
-  // elements. 65535 / 4095 ~= 16, so 2 64-wide rows.
-  const int h_overflow = 2;
+  // Two 16-bit accumulators hold up to 4 64-wide rows before overflow.
+  const int h_overflow = 4;
   highbd_sadwxhx4d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 64, h,
                         h_overflow);
 }
@@ -204,10 +254,8 @@ static inline void highbd_sad128xhx4d_neon(const uint8_t *src_ptr,
                                            const uint8_t *const ref_ptr[4],
                                            int ref_stride, uint32_t res[4],
                                            int h) {
-  // 'h_overflow' is the number of 128-wide rows we can process before 16-bit
-  // accumulators overflow. After hitting this limit accumulate into 32-bit
-  // elements. 65535 / 4095 ~= 16, so 1 128-wide rows.
-  const int h_overflow = 1;
+  // Two 16-bit accumulators hold up to 2 128-wide rows before overflow.
+  const int h_overflow = 2;
   highbd_sadwxhx4d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 128, h,
                         h_overflow);
 }
@@ -305,24 +353,51 @@ static inline void highbd_sad4xhx3d_neon(const uint8_t *src_ptr, int src_stride,
   const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
   const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
   const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
-  uint32x4_t sum[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
+  const int src_stride_2 = src_stride << 1;
+  const int ref_stride_2 = ref_stride << 1;

-  int i = 0;
-  do {
-    uint16x4_t s = vld1_u16(src16_ptr + i * src_stride);
-    uint16x4_t r0 = vld1_u16(ref16_ptr0 + i * ref_stride);
-    uint16x4_t r1 = vld1_u16(ref16_ptr1 + i * ref_stride);
-    uint16x4_t r2 = vld1_u16(ref16_ptr2 + i * ref_stride);
+  // 'h_overflow' is the number of 4-wide rows packed in pairs into uint16x8_t
+  // before 16-bit accumulators overflow: 65535 / 4095 ~= 16 pairs = 32 rows.
+  const int h_overflow = 32;
+  const int h_limit = h < h_overflow ? h : h_overflow;
+  assert(h % 2 == 0);
+  assert(h % h_limit == 0);

-    sum[0] = vabal_u16(sum[0], s, r0);
-    sum[1] = vabal_u16(sum[1], s, r1);
-    sum[2] = vabal_u16(sum[2], s, r2);
+  uint32x4_t sum_u32[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };

-  } while (++i < h);
+  do {
+    uint16x8_t sum_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };
+    int i = h_limit;
+    do {
+      uint16x8_t s =
+          vcombine_u16(vld1_u16(src16_ptr), vld1_u16(src16_ptr + src_stride));
+      uint16x8_t r0 =
+          vcombine_u16(vld1_u16(ref16_ptr0), vld1_u16(ref16_ptr0 + ref_stride));
+      uint16x8_t r1 =
+          vcombine_u16(vld1_u16(ref16_ptr1), vld1_u16(ref16_ptr1 + ref_stride));
+      uint16x8_t r2 =
+          vcombine_u16(vld1_u16(ref16_ptr2), vld1_u16(ref16_ptr2 + ref_stride));
+
+      sum_u16[0] = vabaq_u16(sum_u16[0], s, r0);
+      sum_u16[1] = vabaq_u16(sum_u16[1], s, r1);
+      sum_u16[2] = vabaq_u16(sum_u16[2], s, r2);
+
+      src16_ptr += src_stride_2;
+      ref16_ptr0 += ref_stride_2;
+      ref16_ptr1 += ref_stride_2;
+      ref16_ptr2 += ref_stride_2;
+      i -= 2;
+    } while (i != 0);

-  res[0] = horizontal_add_u32x4(sum[0]);
-  res[1] = horizontal_add_u32x4(sum[1]);
-  res[2] = horizontal_add_u32x4(sum[2]);
+    sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]);
+    sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]);
+    sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]);
+    h -= h_limit;
+  } while (h != 0);
+
+  res[0] = horizontal_add_u32x4(sum_u32[0]);
+  res[1] = horizontal_add_u32x4(sum_u32[1]);
+  res[2] = horizontal_add_u32x4(sum_u32[2]);
 }

 static inline void highbd_sad8xhx3d_neon(const uint8_t *src_ptr, int src_stride,
@@ -333,37 +408,51 @@ static inline void highbd_sad8xhx3d_neon(const uint8_t *src_ptr, int src_stride,
   const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
   const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
   const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
+  const int src_stride_2 = src_stride << 1;
+  const int ref_stride_2 = ref_stride << 1;

-  // 'h_overflow' is the number of 8-wide rows we can process before 16-bit
-  // accumulators overflow. After hitting this limit accumulate into 32-bit
-  // elements. 65535 / 4095 ~= 16, so 16 8-wide rows.
-  const int h_overflow = 16;
-  // If block height 'h' is smaller than this limit, use 'h' instead.
+  // Two 16-bit accumulators each hold up to 16 rows: 32 rows total.
+  const int h_overflow = 32;
   const int h_limit = h < h_overflow ? h : h_overflow;
+  assert(h % 2 == 0);
   assert(h % h_limit == 0);

   uint32x4_t sum_u32[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };

-  int h_tmp = h_limit;
-  int i = 0;
   do {
-    uint16x8_t sum_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };
+    uint16x8_t sum0_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };
+    uint16x8_t sum1_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };
+    int i = h_limit;
     do {
-      uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride);
-
-      sum_u16[0] =
-          vabaq_u16(sum_u16[0], s0, vld1q_u16(ref16_ptr0 + i * ref_stride));
-      sum_u16[1] =
-          vabaq_u16(sum_u16[1], s0, vld1q_u16(ref16_ptr1 + i * ref_stride));
-      sum_u16[2] =
-          vabaq_u16(sum_u16[2], s0, vld1q_u16(ref16_ptr2 + i * ref_stride));
-    } while (++i < h_tmp);
-
-    sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]);
-    sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]);
-    sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]);
+      uint16x8_t s0 = vld1q_u16(src16_ptr);
+      uint16x8_t s1 = vld1q_u16(src16_ptr + src_stride);
+
+      sum0_u16[0] = vabaq_u16(sum0_u16[0], s0, vld1q_u16(ref16_ptr0));
+      sum0_u16[1] = vabaq_u16(sum0_u16[1], s0, vld1q_u16(ref16_ptr1));
+      sum0_u16[2] = vabaq_u16(sum0_u16[2], s0, vld1q_u16(ref16_ptr2));
+
+      sum1_u16[0] =
+          vabaq_u16(sum1_u16[0], s1, vld1q_u16(ref16_ptr0 + ref_stride));
+      sum1_u16[1] =
+          vabaq_u16(sum1_u16[1], s1, vld1q_u16(ref16_ptr1 + ref_stride));
+      sum1_u16[2] =
+          vabaq_u16(sum1_u16[2], s1, vld1q_u16(ref16_ptr2 + ref_stride));
+
+      src16_ptr += src_stride_2;
+      ref16_ptr0 += ref_stride_2;
+      ref16_ptr1 += ref_stride_2;
+      ref16_ptr2 += ref_stride_2;
+      i -= 2;
+    } while (i != 0);
+
+    sum_u32[0] = vpadalq_u16(sum_u32[0], sum0_u16[0]);
+    sum_u32[1] = vpadalq_u16(sum_u32[1], sum0_u16[1]);
+    sum_u32[2] = vpadalq_u16(sum_u32[2], sum0_u16[2]);
+
+    sum_u32[0] = vpadalq_u16(sum_u32[0], sum1_u16[0]);
+    sum_u32[1] = vpadalq_u16(sum_u32[1], sum1_u16[1]);
+    sum_u32[2] = vpadalq_u16(sum_u32[2], sum1_u16[2]);

-    h_tmp += h_limit;
     h -= h_limit;
   } while (h != 0);

@@ -386,7 +475,8 @@ static inline void highbd_sadwxhx3d_neon(const uint8_t *src_ptr, int src_stride,
   assert(h % h_limit == 0);

   do {
-    uint16x8_t sum_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };
+    uint16x8_t sum0_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };
+    uint16x8_t sum1_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };

     int i = h_limit;
     do {
@@ -394,14 +484,14 @@ static inline void highbd_sadwxhx3d_neon(const uint8_t *src_ptr, int src_stride,
       do {
         uint16x8_t s0 = vld1q_u16(src16_ptr + j);

-        sum_u16[0] = vabaq_u16(sum_u16[0], s0, vld1q_u16(ref16_ptr0 + j));
-        sum_u16[1] = vabaq_u16(sum_u16[1], s0, vld1q_u16(ref16_ptr1 + j));
-        sum_u16[2] = vabaq_u16(sum_u16[2], s0, vld1q_u16(ref16_ptr2 + j));
+        sum0_u16[0] = vabaq_u16(sum0_u16[0], s0, vld1q_u16(ref16_ptr0 + j));
+        sum0_u16[1] = vabaq_u16(sum0_u16[1], s0, vld1q_u16(ref16_ptr1 + j));
+        sum0_u16[2] = vabaq_u16(sum0_u16[2], s0, vld1q_u16(ref16_ptr2 + j));

         uint16x8_t s1 = vld1q_u16(src16_ptr + j + 8);
-        sum_u16[0] = vabaq_u16(sum_u16[0], s1, vld1q_u16(ref16_ptr0 + j + 8));
-        sum_u16[1] = vabaq_u16(sum_u16[1], s1, vld1q_u16(ref16_ptr1 + j + 8));
-        sum_u16[2] = vabaq_u16(sum_u16[2], s1, vld1q_u16(ref16_ptr2 + j + 8));
+        sum1_u16[0] = vabaq_u16(sum1_u16[0], s1, vld1q_u16(ref16_ptr0 + j + 8));
+        sum1_u16[1] = vabaq_u16(sum1_u16[1], s1, vld1q_u16(ref16_ptr1 + j + 8));
+        sum1_u16[2] = vabaq_u16(sum1_u16[2], s1, vld1q_u16(ref16_ptr2 + j + 8));

         j += 16;
       } while (j < w);
@@ -412,9 +502,13 @@ static inline void highbd_sadwxhx3d_neon(const uint8_t *src_ptr, int src_stride,
       ref16_ptr2 += ref_stride;
     } while (--i != 0);

-    sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]);
-    sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]);
-    sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]);
+    sum_u32[0] = vpadalq_u16(sum_u32[0], sum0_u16[0]);
+    sum_u32[1] = vpadalq_u16(sum_u32[1], sum0_u16[1]);
+    sum_u32[2] = vpadalq_u16(sum_u32[2], sum0_u16[2]);
+
+    sum_u32[0] = vpadalq_u16(sum_u32[0], sum1_u16[0]);
+    sum_u32[1] = vpadalq_u16(sum_u32[1], sum1_u16[1]);
+    sum_u32[2] = vpadalq_u16(sum_u32[2], sum1_u16[2]);

     h -= h_limit;
   } while (h != 0);
@@ -429,10 +523,8 @@ static inline void highbd_sad16xhx3d_neon(const uint8_t *src_ptr,
                                           const uint8_t *const ref_ptr[4],
                                           int ref_stride, uint32_t res[4],
                                           int h) {
-  // 'h_overflow' is the number of 16-wide rows we can process before 16-bit
-  // accumulators overflow. After hitting this limit accumulate into 32-bit
-  // elements. 65535 / 4095 ~= 16, so 8 16-wide rows.
-  const int h_overflow = 8;
+  // Two 16-bit accumulators hold up to 16 16-wide rows before overflow.
+  const int h_overflow = 16;
   highbd_sadwxhx3d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 16, h,
                         h_overflow);
 }
@@ -442,10 +534,8 @@ static inline void highbd_sad32xhx3d_neon(const uint8_t *src_ptr,
                                           const uint8_t *const ref_ptr[4],
                                           int ref_stride, uint32_t res[4],
                                           int h) {
-  // 'h_overflow' is the number of 32-wide rows we can process before 16-bit
-  // accumulators overflow. After hitting this limit accumulate into 32-bit
-  // elements. 65535 / 4095 ~= 16, so 4 32-wide rows.
-  const int h_overflow = 4;
+  // Two 16-bit accumulators hold up to 8 32-wide rows before overflow.
+  const int h_overflow = 8;
   highbd_sadwxhx3d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 32, h,
                         h_overflow);
 }
@@ -455,10 +545,8 @@ static inline void highbd_sad64xhx3d_neon(const uint8_t *src_ptr,
                                           const uint8_t *const ref_ptr[4],
                                           int ref_stride, uint32_t res[4],
                                           int h) {
-  // 'h_overflow' is the number of 64-wide rows we can process before 16-bit
-  // accumulators overflow. After hitting this limit accumulate into 32-bit
-  // elements. 65535 / 4095 ~= 16, so 2 64-wide rows.
-  const int h_overflow = 2;
+  // Two 16-bit accumulators hold up to 4 64-wide rows before overflow.
+  const int h_overflow = 4;
   highbd_sadwxhx3d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 64, h,
                         h_overflow);
 }
@@ -468,10 +556,8 @@ static inline void highbd_sad128xhx3d_neon(const uint8_t *src_ptr,
                                            const uint8_t *const ref_ptr[4],
                                            int ref_stride, uint32_t res[4],
                                            int h) {
-  // 'h_overflow' is the number of 128-wide rows we can process before 16-bit
-  // accumulators overflow. After hitting this limit accumulate into 32-bit
-  // elements. 65535 / 4095 ~= 16, so 1 128-wide rows.
-  const int h_overflow = 1;
+  // Two 16-bit accumulators hold up to 2 128-wide rows before overflow.
+  const int h_overflow = 2;
   highbd_sadwxhx3d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 128, h,
                         h_overflow);
 }
diff --git a/aom_dsp/arm/highbd_sse_neon.c b/aom_dsp/arm/highbd_sse_neon.c
index 48d4763a07..fd26f18b1d 100644
--- a/aom_dsp/arm/highbd_sse_neon.c
+++ b/aom_dsp/arm/highbd_sse_neon.c
@@ -10,6 +10,7 @@
  */

 #include <arm_neon.h>
+#include <assert.h>

 #include "config/aom_dsp_rtcd.h"
 #include "aom_dsp/arm/sum_neon.h"
@@ -153,110 +154,128 @@ static inline int64_t highbd_sse_32xh_neon(const uint16_t *src, int src_stride,
 static inline int64_t highbd_sse_16xh_neon(const uint16_t *src, int src_stride,
                                            const uint16_t *ref, int ref_stride,
                                            int height) {
-  uint32x4_t sse[4];
+  assert(height >= 2 && (height & 1) == 0);
+  uint32x4_t sse[8];
   highbd_sse_8x1_init_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
   highbd_sse_8x1_init_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
-
+  src += src_stride;
+  ref += ref_stride;
+  highbd_sse_8x1_init_neon(src + 0 * 8, ref + 0 * 8, &sse[4], &sse[5]);
+  highbd_sse_8x1_init_neon(src + 1 * 8, ref + 1 * 8, &sse[6], &sse[7]);
   src += src_stride;
   ref += ref_stride;

-  while (--height != 0) {
+  height -= 2;
+  while (height != 0) {
     highbd_sse_8x1_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
     highbd_sse_8x1_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
-
     src += src_stride;
     ref += ref_stride;
+    highbd_sse_8x1_neon(src + 0 * 8, ref + 0 * 8, &sse[4], &sse[5]);
+    highbd_sse_8x1_neon(src + 1 * 8, ref + 1 * 8, &sse[6], &sse[7]);
+    src += src_stride;
+    ref += ref_stride;
+    height -= 2;
   }

-  return horizontal_long_add_u32x4_x4(sse);
+  return horizontal_long_add_u32x4_x8(sse);
 }

 static inline int64_t highbd_sse_8xh_neon(const uint16_t *src, int src_stride,
                                           const uint16_t *ref, int ref_stride,
                                           int height) {
-  uint32x4_t sse[2];
+  assert(height >= 2 && (height & 1) == 0);
+  uint32x4_t sse[4];
   highbd_sse_8x1_init_neon(src, ref, &sse[0], &sse[1]);
-
+  src += src_stride;
+  ref += ref_stride;
+  highbd_sse_8x1_init_neon(src, ref, &sse[2], &sse[3]);
   src += src_stride;
   ref += ref_stride;

-  while (--height != 0) {
+  height -= 2;
+  while (height != 0) {
     highbd_sse_8x1_neon(src, ref, &sse[0], &sse[1]);
-
     src += src_stride;
     ref += ref_stride;
+    highbd_sse_8x1_neon(src, ref, &sse[2], &sse[3]);
+    src += src_stride;
+    ref += ref_stride;
+    height -= 2;
   }

-  return horizontal_long_add_u32x4_x2(sse);
+  return horizontal_long_add_u32x4_x4(sse);
 }

 static inline int64_t highbd_sse_4xh_neon(const uint16_t *src, int src_stride,
                                           const uint16_t *ref, int ref_stride,
                                           int height) {
+  assert(height >= 2 && (height & 1) == 0);
   // Peel the first loop iteration.
-  uint16x4_t s = vld1_u16(src);
-  uint16x4_t r = vld1_u16(ref);
-
-  uint16x4_t abs_diff = vabd_u16(s, r);
-  uint32x4_t sse = vmull_u16(abs_diff, abs_diff);
-
-  src += src_stride;
-  ref += ref_stride;
-
-  while (--height != 0) {
-    s = vld1_u16(src);
-    r = vld1_u16(ref);
-
-    abs_diff = vabd_u16(s, r);
-    sse = vmlal_u16(sse, abs_diff, abs_diff);
-
-    src += src_stride;
-    ref += ref_stride;
+  uint32x4_t sse[2];
+  uint16x8_t s = vcombine_u16(vld1_u16(src), vld1_u16(src + src_stride));
+  uint16x8_t r = vcombine_u16(vld1_u16(ref), vld1_u16(ref + ref_stride));
+  uint16x8_t abs_diff = vabdq_u16(s, r);
+  sse[0] = vmull_u16(vget_low_u16(abs_diff), vget_low_u16(abs_diff));
+  sse[1] = vmull_u16(vget_high_u16(abs_diff), vget_high_u16(abs_diff));
+  src += 2 * src_stride;
+  ref += 2 * ref_stride;
+
+  height -= 2;
+  while (height != 0) {
+    s = vcombine_u16(vld1_u16(src), vld1_u16(src + src_stride));
+    r = vcombine_u16(vld1_u16(ref), vld1_u16(ref + ref_stride));
+    abs_diff = vabdq_u16(s, r);
+    sse[0] = vmlal_u16(sse[0], vget_low_u16(abs_diff), vget_low_u16(abs_diff));
+    sse[1] =
+        vmlal_u16(sse[1], vget_high_u16(abs_diff), vget_high_u16(abs_diff));
+    src += 2 * src_stride;
+    ref += 2 * ref_stride;
+    height -= 2;
   }

-  return horizontal_long_add_u32x4(sse);
+  return horizontal_long_add_u32x4_x2(sse);
 }

 static inline int64_t highbd_sse_wxh_neon(const uint16_t *src, int src_stride,
                                           const uint16_t *ref, int ref_stride,
                                           int width, int height) {
+  assert(width >= 8);
   // { 0, 1, 2, 3, 4, 5, 6, 7 }
-  uint16x8_t k01234567 = vmovl_u8(vcreate_u8(0x0706050403020100));
-  uint16x8_t remainder_mask = vcltq_u16(k01234567, vdupq_n_u16(width & 7));
-  uint64_t sse = 0;
+  const uint16x8_t k01234567 = vmovl_u8(vcreate_u8(0x0706050403020100));
+  const uint16x8_t remainder_mask =
+      vcltq_u16(k01234567, vdupq_n_u16(width & 7));
+  uint64x2_t sse_u64 = vdupq_n_u64(0);

   do {
-    int w = width;
-    int offset = 0;
-
-    do {
-      uint16x8_t s = vld1q_u16(src + offset);
-      uint16x8_t r = vld1q_u16(ref + offset);
-
-      if (w < 8) {
-        // Mask out-of-range elements.
-        s = vandq_u16(s, remainder_mask);
-        r = vandq_u16(r, remainder_mask);
-      }
-
-      uint16x8_t abs_diff = vabdq_u16(s, r);
-      uint16x4_t abs_diff_lo = vget_low_u16(abs_diff);
-      uint16x4_t abs_diff_hi = vget_high_u16(abs_diff);
-
-      uint32x4_t sse_u32 = vmull_u16(abs_diff_lo, abs_diff_lo);
-      sse_u32 = vmlal_u16(sse_u32, abs_diff_hi, abs_diff_hi);
-
-      sse += horizontal_long_add_u32x4(sse_u32);
+    int w = width - 8;
+    int offset = 8;
+    uint32x4_t sse_u32[2];
+    highbd_sse_8x1_init_neon(src, ref, &sse_u32[0], &sse_u32[1]);

+    while (w >= 8) {
+      highbd_sse_8x1_neon(src + offset, ref + offset, &sse_u32[0], &sse_u32[1]);
       offset += 8;
       w -= 8;
-    } while (w > 0);
-
+    }
+
+    if (w > 0) {
+      const uint16x8_t s = vandq_u16(vld1q_u16(src + offset), remainder_mask);
+      const uint16x8_t r = vandq_u16(vld1q_u16(ref + offset), remainder_mask);
+      const uint16x8_t abs_diff = vabdq_u16(s, r);
+      sse_u32[0] =
+          vmlal_u16(sse_u32[0], vget_low_u16(abs_diff), vget_low_u16(abs_diff));
+      sse_u32[1] = vmlal_u16(sse_u32[1], vget_high_u16(abs_diff),
+                             vget_high_u16(abs_diff));
+    }
+
+    sse_u64 = vpadalq_u32(sse_u64, sse_u32[0]);
+    sse_u64 = vpadalq_u32(sse_u64, sse_u32[1]);
     src += src_stride;
     ref += ref_stride;
   } while (--height != 0);

-  return sse;
+  return horizontal_add_u64x2(sse_u64);
 }

 int64_t aom_highbd_sse_neon(const uint8_t *src8, int src_stride,