Commit d6f77c9d9f for aom
commit d6f77c9d9fe4b708a4d933cd4812634cabf0ad86
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Nov 14 10:53:13 2026 -0800
aom_dsp,neon: Unroll highbd sad, sadxd & sse
Unroll highbd_sad4xh_neon with 2-row vcombine_u16 and vabaq_u16. Unroll
highbd_sad{8,32..128}xh_neon and highbd_sad{4,8}xhx{3,4}d_neon across
2-row iterations, and highbd_sad{16..128}xhx{3,4}d_neon across 16-wide
chunks with dual accumulators. Optimize highbd_sse_neon with unrolled
multi-accumulator loops.
Speedup: 1.18x (128x128 10b SAD) in test_libaom on Neoverse N3; -0.27%
(1T) / -0.59% (4T, -0.85% min) 10b avifenc on Neoverse N2 (bit-exact).
TAG=agy
CONV=490488d4-7005-402a-8cc4-c245227f80fe
Change-Id: Icc4c5cb870d65ce16d881cf20bd808f171c229bc
diff --git a/aom_dsp/arm/highbd_sad_neon.c b/aom_dsp/arm/highbd_sad_neon.c
index b8dde45805..03a1976037 100644
--- a/aom_dsp/arm/highbd_sad_neon.c
+++ b/aom_dsp/arm/highbd_sad_neon.c
@@ -11,6 +11,7 @@
*/
#include <arm_neon.h>
+#include <assert.h>
#include "config/aom_config.h"
#include "config/aom_dsp_rtcd.h"
@@ -25,18 +26,38 @@ static inline uint32_t highbd_sad4xh_neon(const uint8_t *src_ptr,
int ref_stride, int h) {
const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr);
- uint32x4_t sum = vdupq_n_u32(0);
+ const int src_stride_2 = src_stride << 1;
+ const int ref_stride_2 = ref_stride << 1;
+
+ // 'h_overflow' is the number of 4-wide rows packed in pairs into uint16x8_t
+ // before 16-bit accumulators overflow: 65535 / 4095 ~= 16 pairs = 32 rows.
+ const int h_overflow = 32;
+ const int h_limit = h < h_overflow ? h : h_overflow;
+ assert(h % 2 == 0);
+ assert(h % h_limit == 0);
+
+ uint32x4_t sum_u32 = vdupq_n_u32(0);
do {
- uint16x4_t s = vld1_u16(src16_ptr);
- uint16x4_t r = vld1_u16(ref16_ptr);
- sum = vabal_u16(sum, s, r);
+ uint16x8_t sum_u16 = vdupq_n_u16(0);
+ int i = h_limit;
+ do {
+ uint16x8_t s =
+ vcombine_u16(vld1_u16(src16_ptr), vld1_u16(src16_ptr + src_stride));
+ uint16x8_t r =
+ vcombine_u16(vld1_u16(ref16_ptr), vld1_u16(ref16_ptr + ref_stride));
+ sum_u16 = vabaq_u16(sum_u16, s, r);
+
+ src16_ptr += src_stride_2;
+ ref16_ptr += ref_stride_2;
+ i -= 2;
+ } while (i != 0);
- src16_ptr += src_stride;
- ref16_ptr += ref_stride;
- } while (--h != 0);
+ sum_u32 = vpadalq_u16(sum_u32, sum_u16);
+ h -= h_limit;
+ } while (h != 0);
- return horizontal_add_u32x4(sum);
+ return horizontal_add_u32x4(sum_u32);
}
static inline uint32_t highbd_sad8xh_neon(const uint8_t *src_ptr,
@@ -45,31 +66,37 @@ static inline uint32_t highbd_sad8xh_neon(const uint8_t *src_ptr,
int ref_stride, int h) {
const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr);
+ const int src_stride_2 = src_stride << 1;
+ const int ref_stride_2 = ref_stride << 1;
- // 'h_overflow' is the number of 8-wide rows we can process before 16-bit
- // accumulators overflow. After hitting this limit accumulate into 32-bit
- // elements. 65535 / 4095 ~= 16, so 16 8-wide rows.
- const int h_overflow = 16;
- // If block height 'h' is smaller than this limit, use 'h' instead.
+ // Two 16-bit accumulators each hold up to 16 rows: 32 rows total.
+ const int h_overflow = 32;
const int h_limit = h < h_overflow ? h : h_overflow;
+ assert(h % 2 == 0);
assert(h % h_limit == 0);
uint32x4_t sum_u32 = vdupq_n_u32(0);
do {
- uint16x8_t sum_u16 = vdupq_n_u16(0);
+ uint16x8_t sum0_u16 = vdupq_n_u16(0);
+ uint16x8_t sum1_u16 = vdupq_n_u16(0);
int i = h_limit;
do {
uint16x8_t s0 = vld1q_u16(src16_ptr);
uint16x8_t r0 = vld1q_u16(ref16_ptr);
- sum_u16 = vabaq_u16(sum_u16, s0, r0);
+ uint16x8_t s1 = vld1q_u16(src16_ptr + src_stride);
+ uint16x8_t r1 = vld1q_u16(ref16_ptr + ref_stride);
+ sum0_u16 = vabaq_u16(sum0_u16, s0, r0);
+ sum1_u16 = vabaq_u16(sum1_u16, s1, r1);
- src16_ptr += src_stride;
- ref16_ptr += ref_stride;
- } while (--i != 0);
+ src16_ptr += src_stride_2;
+ ref16_ptr += ref_stride_2;
+ i -= 2;
+ } while (i != 0);
- sum_u32 = vpadalq_u16(sum_u32, sum_u16);
+ sum_u32 = vpadalq_u16(sum_u32, sum0_u16);
+ sum_u32 = vpadalq_u16(sum_u32, sum1_u16);
h -= h_limit;
} while (h != 0);
@@ -124,47 +151,75 @@ static inline uint32_t highbd_sadwxh_neon(const uint8_t *src_ptr,
const int h_overflow) {
const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr);
+ const int src_stride_2 = src_stride << 1;
+ const int ref_stride_2 = ref_stride << 1;
const int h_limit = h < h_overflow ? h : h_overflow;
+ assert(h % 2 == 0);
assert(h % h_limit == 0);
uint32x4_t sum_u32 = vdupq_n_u32(0);
do {
- uint16x8_t sum_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
- vdupq_n_u16(0) };
+ uint16x8_t sum0_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+ vdupq_n_u16(0) };
+ uint16x8_t sum1_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+ vdupq_n_u16(0) };
int i = h_limit;
do {
+ const uint16_t *src16_r1 = src16_ptr + src_stride;
+ const uint16_t *ref16_r1 = ref16_ptr + ref_stride;
int j = 0;
do {
uint16x8_t s0 = vld1q_u16(src16_ptr + j);
uint16x8_t r0 = vld1q_u16(ref16_ptr + j);
- sum_u16[0] = vabaq_u16(sum_u16[0], s0, r0);
+ sum0_u16[0] = vabaq_u16(sum0_u16[0], s0, r0);
uint16x8_t s1 = vld1q_u16(src16_ptr + j + 8);
uint16x8_t r1 = vld1q_u16(ref16_ptr + j + 8);
- sum_u16[1] = vabaq_u16(sum_u16[1], s1, r1);
+ sum0_u16[1] = vabaq_u16(sum0_u16[1], s1, r1);
uint16x8_t s2 = vld1q_u16(src16_ptr + j + 16);
uint16x8_t r2 = vld1q_u16(ref16_ptr + j + 16);
- sum_u16[2] = vabaq_u16(sum_u16[2], s2, r2);
+ sum0_u16[2] = vabaq_u16(sum0_u16[2], s2, r2);
uint16x8_t s3 = vld1q_u16(src16_ptr + j + 24);
uint16x8_t r3 = vld1q_u16(ref16_ptr + j + 24);
- sum_u16[3] = vabaq_u16(sum_u16[3], s3, r3);
+ sum0_u16[3] = vabaq_u16(sum0_u16[3], s3, r3);
+
+ uint16x8_t t0 = vld1q_u16(src16_r1 + j);
+ uint16x8_t q0 = vld1q_u16(ref16_r1 + j);
+ sum1_u16[0] = vabaq_u16(sum1_u16[0], t0, q0);
+
+ uint16x8_t t1 = vld1q_u16(src16_r1 + j + 8);
+ uint16x8_t q1 = vld1q_u16(ref16_r1 + j + 8);
+ sum1_u16[1] = vabaq_u16(sum1_u16[1], t1, q1);
+
+ uint16x8_t t2 = vld1q_u16(src16_r1 + j + 16);
+ uint16x8_t q2 = vld1q_u16(ref16_r1 + j + 16);
+ sum1_u16[2] = vabaq_u16(sum1_u16[2], t2, q2);
+
+ uint16x8_t t3 = vld1q_u16(src16_r1 + j + 24);
+ uint16x8_t q3 = vld1q_u16(ref16_r1 + j + 24);
+ sum1_u16[3] = vabaq_u16(sum1_u16[3], t3, q3);
j += 32;
} while (j < w);
- src16_ptr += src_stride;
- ref16_ptr += ref_stride;
- } while (--i != 0);
+ src16_ptr += src_stride_2;
+ ref16_ptr += ref_stride_2;
+ i -= 2;
+ } while (i != 0);
- sum_u32 = vpadalq_u16(sum_u32, sum_u16[0]);
- sum_u32 = vpadalq_u16(sum_u32, sum_u16[1]);
- sum_u32 = vpadalq_u16(sum_u32, sum_u16[2]);
- sum_u32 = vpadalq_u16(sum_u32, sum_u16[3]);
+ sum_u32 = vpadalq_u16(sum_u32, sum0_u16[0]);
+ sum_u32 = vpadalq_u16(sum_u32, sum0_u16[1]);
+ sum_u32 = vpadalq_u16(sum_u32, sum0_u16[2]);
+ sum_u32 = vpadalq_u16(sum_u32, sum0_u16[3]);
+ sum_u32 = vpadalq_u16(sum_u32, sum1_u16[0]);
+ sum_u32 = vpadalq_u16(sum_u32, sum1_u16[1]);
+ sum_u32 = vpadalq_u16(sum_u32, sum1_u16[2]);
+ sum_u32 = vpadalq_u16(sum_u32, sum1_u16[3]);
h -= h_limit;
} while (h != 0);
@@ -175,10 +230,8 @@ static inline uint32_t highbd_sad32xh_neon(const uint8_t *src_ptr,
int src_stride,
const uint8_t *ref_ptr,
int ref_stride, int h) {
- // 'h_overflow' is the number of 32-wide rows we can process before 16-bit
- // accumulators overflow. After hitting this limit accumulate into 32-bit
- // elements. 65535 / 4095 ~= 16, so 16 32-wide rows using four accumulators.
- const int h_overflow = 16;
+ // Eight 16-bit accumulators hold up to 32 32-wide rows before overflow.
+ const int h_overflow = 32;
return highbd_sadwxh_neon(src_ptr, src_stride, ref_ptr, ref_stride, 32, h,
h_overflow);
}
@@ -187,10 +240,8 @@ static inline uint32_t highbd_sad64xh_neon(const uint8_t *src_ptr,
int src_stride,
const uint8_t *ref_ptr,
int ref_stride, int h) {
- // 'h_overflow' is the number of 64-wide rows we can process before 16-bit
- // accumulators overflow. After hitting this limit accumulate into 32-bit
- // elements. 65535 / 4095 ~= 16, so 8 64-wide rows using four accumulators.
- const int h_overflow = 8;
+ // Eight 16-bit accumulators hold up to 16 64-wide rows before overflow.
+ const int h_overflow = 16;
return highbd_sadwxh_neon(src_ptr, src_stride, ref_ptr, ref_stride, 64, h,
h_overflow);
}
@@ -199,10 +250,8 @@ static inline uint32_t highbd_sad128xh_neon(const uint8_t *src_ptr,
int src_stride,
const uint8_t *ref_ptr,
int ref_stride, int h) {
- // 'h_overflow' is the number of 128-wide rows we can process before 16-bit
- // accumulators overflow. After hitting this limit accumulate into 32-bit
- // elements. 65535 / 4095 ~= 16, so 4 128-wide rows using four accumulators.
- const int h_overflow = 4;
+ // Eight 16-bit accumulators hold up to 8 128-wide rows before overflow.
+ const int h_overflow = 8;
return highbd_sadwxh_neon(src_ptr, src_stride, ref_ptr, ref_stride, 128, h,
h_overflow);
}
diff --git a/aom_dsp/arm/highbd_sadxd_neon.c b/aom_dsp/arm/highbd_sadxd_neon.c
index 0b4527873a..75dd008a66 100644
--- a/aom_dsp/arm/highbd_sadxd_neon.c
+++ b/aom_dsp/arm/highbd_sadxd_neon.c
@@ -11,6 +11,7 @@
*/
#include <arm_neon.h>
+#include <assert.h>
#include "config/aom_config.h"
#include "config/aom_dsp_rtcd.h"
@@ -28,26 +29,56 @@ static inline void highbd_sad4xhx4d_neon(const uint8_t *src_ptr, int src_stride,
const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]);
+ const int src_stride_2 = src_stride << 1;
+ const int ref_stride_2 = ref_stride << 1;
- uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
- vdupq_n_u32(0) };
+ // 'h_overflow' is the number of 4-wide rows packed in pairs into uint16x8_t
+ // before 16-bit accumulators overflow: 65535 / 4095 ~= 16 pairs = 32 rows.
+ const int h_overflow = 32;
+ const int h_limit = h < h_overflow ? h : h_overflow;
+ assert(h % 2 == 0);
+ assert(h % h_limit == 0);
- int i = 0;
- do {
- uint16x4_t s = vld1_u16(src16_ptr + i * src_stride);
- uint16x4_t r0 = vld1_u16(ref16_ptr0 + i * ref_stride);
- uint16x4_t r1 = vld1_u16(ref16_ptr1 + i * ref_stride);
- uint16x4_t r2 = vld1_u16(ref16_ptr2 + i * ref_stride);
- uint16x4_t r3 = vld1_u16(ref16_ptr3 + i * ref_stride);
+ uint32x4_t sum_u32[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
- sum[0] = vabal_u16(sum[0], s, r0);
- sum[1] = vabal_u16(sum[1], s, r1);
- sum[2] = vabal_u16(sum[2], s, r2);
- sum[3] = vabal_u16(sum[3], s, r3);
+ do {
+ uint16x8_t sum_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+ vdupq_n_u16(0) };
+ int i = h_limit;
+ do {
+ uint16x8_t s =
+ vcombine_u16(vld1_u16(src16_ptr), vld1_u16(src16_ptr + src_stride));
+ uint16x8_t r0 =
+ vcombine_u16(vld1_u16(ref16_ptr0), vld1_u16(ref16_ptr0 + ref_stride));
+ uint16x8_t r1 =
+ vcombine_u16(vld1_u16(ref16_ptr1), vld1_u16(ref16_ptr1 + ref_stride));
+ uint16x8_t r2 =
+ vcombine_u16(vld1_u16(ref16_ptr2), vld1_u16(ref16_ptr2 + ref_stride));
+ uint16x8_t r3 =
+ vcombine_u16(vld1_u16(ref16_ptr3), vld1_u16(ref16_ptr3 + ref_stride));
+
+ sum_u16[0] = vabaq_u16(sum_u16[0], s, r0);
+ sum_u16[1] = vabaq_u16(sum_u16[1], s, r1);
+ sum_u16[2] = vabaq_u16(sum_u16[2], s, r2);
+ sum_u16[3] = vabaq_u16(sum_u16[3], s, r3);
+
+ src16_ptr += src_stride_2;
+ ref16_ptr0 += ref_stride_2;
+ ref16_ptr1 += ref_stride_2;
+ ref16_ptr2 += ref_stride_2;
+ ref16_ptr3 += ref_stride_2;
+ i -= 2;
+ } while (i != 0);
- } while (++i < h);
+ sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]);
+ sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]);
+ sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]);
+ sum_u32[3] = vpadalq_u16(sum_u32[3], sum_u16[3]);
+ h -= h_limit;
+ } while (h != 0);
- vst1q_u32(res, horizontal_add_4d_u32x4(sum));
+ vst1q_u32(res, horizontal_add_4d_u32x4(sum_u32));
}
static inline void highbd_sad8xhx4d_neon(const uint8_t *src_ptr, int src_stride,
@@ -59,43 +90,61 @@ static inline void highbd_sad8xhx4d_neon(const uint8_t *src_ptr, int src_stride,
const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]);
+ const int src_stride_2 = src_stride << 1;
+ const int ref_stride_2 = ref_stride << 1;
- // 'h_overflow' is the number of 8-wide rows we can process before 16-bit
- // accumulators overflow. After hitting this limit accumulate into 32-bit
- // elements. 65535 / 4095 ~= 16, so 16 8-wide rows.
- const int h_overflow = 16;
- // If block height 'h' is smaller than this limit, use 'h' instead.
+ // Two 16-bit accumulators each hold up to 16 rows: 32 rows total.
+ const int h_overflow = 32;
const int h_limit = h < h_overflow ? h : h_overflow;
+ assert(h % 2 == 0);
assert(h % h_limit == 0);
uint32x4_t sum_u32[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
vdupq_n_u32(0) };
- int h_tmp = h_limit;
- int i = 0;
do {
- uint16x8_t sum_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
- vdupq_n_u16(0) };
+ uint16x8_t sum0_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+ vdupq_n_u16(0) };
+ uint16x8_t sum1_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+ vdupq_n_u16(0) };
+ int i = h_limit;
do {
- uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride);
-
- sum_u16[0] =
- vabaq_u16(sum_u16[0], s0, vld1q_u16(ref16_ptr0 + i * ref_stride));
- sum_u16[1] =
- vabaq_u16(sum_u16[1], s0, vld1q_u16(ref16_ptr1 + i * ref_stride));
- sum_u16[2] =
- vabaq_u16(sum_u16[2], s0, vld1q_u16(ref16_ptr2 + i * ref_stride));
- sum_u16[3] =
- vabaq_u16(sum_u16[3], s0, vld1q_u16(ref16_ptr3 + i * ref_stride));
- } while (++i < h_tmp);
-
- sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]);
- sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]);
- sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]);
- sum_u32[3] = vpadalq_u16(sum_u32[3], sum_u16[3]);
+ uint16x8_t s0 = vld1q_u16(src16_ptr);
+ uint16x8_t s1 = vld1q_u16(src16_ptr + src_stride);
+
+ sum0_u16[0] = vabaq_u16(sum0_u16[0], s0, vld1q_u16(ref16_ptr0));
+ sum0_u16[1] = vabaq_u16(sum0_u16[1], s0, vld1q_u16(ref16_ptr1));
+ sum0_u16[2] = vabaq_u16(sum0_u16[2], s0, vld1q_u16(ref16_ptr2));
+ sum0_u16[3] = vabaq_u16(sum0_u16[3], s0, vld1q_u16(ref16_ptr3));
+
+ sum1_u16[0] =
+ vabaq_u16(sum1_u16[0], s1, vld1q_u16(ref16_ptr0 + ref_stride));
+ sum1_u16[1] =
+ vabaq_u16(sum1_u16[1], s1, vld1q_u16(ref16_ptr1 + ref_stride));
+ sum1_u16[2] =
+ vabaq_u16(sum1_u16[2], s1, vld1q_u16(ref16_ptr2 + ref_stride));
+ sum1_u16[3] =
+ vabaq_u16(sum1_u16[3], s1, vld1q_u16(ref16_ptr3 + ref_stride));
+
+ src16_ptr += src_stride_2;
+ ref16_ptr0 += ref_stride_2;
+ ref16_ptr1 += ref_stride_2;
+ ref16_ptr2 += ref_stride_2;
+ ref16_ptr3 += ref_stride_2;
+ i -= 2;
+ } while (i != 0);
+
+ sum_u32[0] = vpadalq_u16(sum_u32[0], sum0_u16[0]);
+ sum_u32[1] = vpadalq_u16(sum_u32[1], sum0_u16[1]);
+ sum_u32[2] = vpadalq_u16(sum_u32[2], sum0_u16[2]);
+ sum_u32[3] = vpadalq_u16(sum_u32[3], sum0_u16[3]);
+
+ sum_u32[0] = vpadalq_u16(sum_u32[0], sum1_u16[0]);
+ sum_u32[1] = vpadalq_u16(sum_u32[1], sum1_u16[1]);
+ sum_u32[2] = vpadalq_u16(sum_u32[2], sum1_u16[2]);
+ sum_u32[3] = vpadalq_u16(sum_u32[3], sum1_u16[3]);
- h_tmp += h_limit;
h -= h_limit;
} while (h != 0);
@@ -119,8 +168,10 @@ static inline void highbd_sadwxhx4d_neon(const uint8_t *src_ptr, int src_stride,
vdupq_n_u32(0) };
do {
- uint16x8_t sum_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
- vdupq_n_u16(0) };
+ uint16x8_t sum0_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+ vdupq_n_u16(0) };
+ uint16x8_t sum1_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+ vdupq_n_u16(0) };
int i = h_limit;
do {
@@ -128,16 +179,16 @@ static inline void highbd_sadwxhx4d_neon(const uint8_t *src_ptr, int src_stride,
do {
uint16x8_t s0 = vld1q_u16(src16_ptr + j);
- sum_u16[0] = vabaq_u16(sum_u16[0], s0, vld1q_u16(ref16_ptr0 + j));
- sum_u16[1] = vabaq_u16(sum_u16[1], s0, vld1q_u16(ref16_ptr1 + j));
- sum_u16[2] = vabaq_u16(sum_u16[2], s0, vld1q_u16(ref16_ptr2 + j));
- sum_u16[3] = vabaq_u16(sum_u16[3], s0, vld1q_u16(ref16_ptr3 + j));
+ sum0_u16[0] = vabaq_u16(sum0_u16[0], s0, vld1q_u16(ref16_ptr0 + j));
+ sum0_u16[1] = vabaq_u16(sum0_u16[1], s0, vld1q_u16(ref16_ptr1 + j));
+ sum0_u16[2] = vabaq_u16(sum0_u16[2], s0, vld1q_u16(ref16_ptr2 + j));
+ sum0_u16[3] = vabaq_u16(sum0_u16[3], s0, vld1q_u16(ref16_ptr3 + j));
uint16x8_t s1 = vld1q_u16(src16_ptr + j + 8);
- sum_u16[0] = vabaq_u16(sum_u16[0], s1, vld1q_u16(ref16_ptr0 + j + 8));
- sum_u16[1] = vabaq_u16(sum_u16[1], s1, vld1q_u16(ref16_ptr1 + j + 8));
- sum_u16[2] = vabaq_u16(sum_u16[2], s1, vld1q_u16(ref16_ptr2 + j + 8));
- sum_u16[3] = vabaq_u16(sum_u16[3], s1, vld1q_u16(ref16_ptr3 + j + 8));
+ sum1_u16[0] = vabaq_u16(sum1_u16[0], s1, vld1q_u16(ref16_ptr0 + j + 8));
+ sum1_u16[1] = vabaq_u16(sum1_u16[1], s1, vld1q_u16(ref16_ptr1 + j + 8));
+ sum1_u16[2] = vabaq_u16(sum1_u16[2], s1, vld1q_u16(ref16_ptr2 + j + 8));
+ sum1_u16[3] = vabaq_u16(sum1_u16[3], s1, vld1q_u16(ref16_ptr3 + j + 8));
j += 16;
} while (j < w);
@@ -149,10 +200,15 @@ static inline void highbd_sadwxhx4d_neon(const uint8_t *src_ptr, int src_stride,
ref16_ptr3 += ref_stride;
} while (--i != 0);
- sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]);
- sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]);
- sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]);
- sum_u32[3] = vpadalq_u16(sum_u32[3], sum_u16[3]);
+ sum_u32[0] = vpadalq_u16(sum_u32[0], sum0_u16[0]);
+ sum_u32[1] = vpadalq_u16(sum_u32[1], sum0_u16[1]);
+ sum_u32[2] = vpadalq_u16(sum_u32[2], sum0_u16[2]);
+ sum_u32[3] = vpadalq_u16(sum_u32[3], sum0_u16[3]);
+
+ sum_u32[0] = vpadalq_u16(sum_u32[0], sum1_u16[0]);
+ sum_u32[1] = vpadalq_u16(sum_u32[1], sum1_u16[1]);
+ sum_u32[2] = vpadalq_u16(sum_u32[2], sum1_u16[2]);
+ sum_u32[3] = vpadalq_u16(sum_u32[3], sum1_u16[3]);
h -= h_limit;
} while (h != 0);
@@ -165,10 +221,8 @@ static inline void highbd_sad16xhx4d_neon(const uint8_t *src_ptr,
const uint8_t *const ref_ptr[4],
int ref_stride, uint32_t res[4],
int h) {
- // 'h_overflow' is the number of 16-wide rows we can process before 16-bit
- // accumulators overflow. After hitting this limit accumulate into 32-bit
- // elements. 65535 / 4095 ~= 16, so 8 16-wide rows.
- const int h_overflow = 8;
+ // Two 16-bit accumulators hold up to 16 16-wide rows before overflow.
+ const int h_overflow = 16;
highbd_sadwxhx4d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 16, h,
h_overflow);
}
@@ -178,10 +232,8 @@ static inline void highbd_sad32xhx4d_neon(const uint8_t *src_ptr,
const uint8_t *const ref_ptr[4],
int ref_stride, uint32_t res[4],
int h) {
- // 'h_overflow' is the number of 32-wide rows we can process before 16-bit
- // accumulators overflow. After hitting this limit accumulate into 32-bit
- // elements. 65535 / 4095 ~= 16, so 4 32-wide rows.
- const int h_overflow = 4;
+ // Two 16-bit accumulators hold up to 8 32-wide rows before overflow.
+ const int h_overflow = 8;
highbd_sadwxhx4d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 32, h,
h_overflow);
}
@@ -191,10 +243,8 @@ static inline void highbd_sad64xhx4d_neon(const uint8_t *src_ptr,
const uint8_t *const ref_ptr[4],
int ref_stride, uint32_t res[4],
int h) {
- // 'h_overflow' is the number of 64-wide rows we can process before 16-bit
- // accumulators overflow. After hitting this limit accumulate into 32-bit
- // elements. 65535 / 4095 ~= 16, so 2 64-wide rows.
- const int h_overflow = 2;
+ // Two 16-bit accumulators hold up to 4 64-wide rows before overflow.
+ const int h_overflow = 4;
highbd_sadwxhx4d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 64, h,
h_overflow);
}
@@ -204,10 +254,8 @@ static inline void highbd_sad128xhx4d_neon(const uint8_t *src_ptr,
const uint8_t *const ref_ptr[4],
int ref_stride, uint32_t res[4],
int h) {
- // 'h_overflow' is the number of 128-wide rows we can process before 16-bit
- // accumulators overflow. After hitting this limit accumulate into 32-bit
- // elements. 65535 / 4095 ~= 16, so 1 128-wide rows.
- const int h_overflow = 1;
+ // Two 16-bit accumulators hold up to 2 128-wide rows before overflow.
+ const int h_overflow = 2;
highbd_sadwxhx4d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 128, h,
h_overflow);
}
@@ -305,24 +353,51 @@ static inline void highbd_sad4xhx3d_neon(const uint8_t *src_ptr, int src_stride,
const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
- uint32x4_t sum[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
+ const int src_stride_2 = src_stride << 1;
+ const int ref_stride_2 = ref_stride << 1;
- int i = 0;
- do {
- uint16x4_t s = vld1_u16(src16_ptr + i * src_stride);
- uint16x4_t r0 = vld1_u16(ref16_ptr0 + i * ref_stride);
- uint16x4_t r1 = vld1_u16(ref16_ptr1 + i * ref_stride);
- uint16x4_t r2 = vld1_u16(ref16_ptr2 + i * ref_stride);
+ // 'h_overflow' is the number of 4-wide rows packed in pairs into uint16x8_t
+ // before 16-bit accumulators overflow: 65535 / 4095 ~= 16 pairs = 32 rows.
+ const int h_overflow = 32;
+ const int h_limit = h < h_overflow ? h : h_overflow;
+ assert(h % 2 == 0);
+ assert(h % h_limit == 0);
- sum[0] = vabal_u16(sum[0], s, r0);
- sum[1] = vabal_u16(sum[1], s, r1);
- sum[2] = vabal_u16(sum[2], s, r2);
+ uint32x4_t sum_u32[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
- } while (++i < h);
+ do {
+ uint16x8_t sum_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };
+ int i = h_limit;
+ do {
+ uint16x8_t s =
+ vcombine_u16(vld1_u16(src16_ptr), vld1_u16(src16_ptr + src_stride));
+ uint16x8_t r0 =
+ vcombine_u16(vld1_u16(ref16_ptr0), vld1_u16(ref16_ptr0 + ref_stride));
+ uint16x8_t r1 =
+ vcombine_u16(vld1_u16(ref16_ptr1), vld1_u16(ref16_ptr1 + ref_stride));
+ uint16x8_t r2 =
+ vcombine_u16(vld1_u16(ref16_ptr2), vld1_u16(ref16_ptr2 + ref_stride));
+
+ sum_u16[0] = vabaq_u16(sum_u16[0], s, r0);
+ sum_u16[1] = vabaq_u16(sum_u16[1], s, r1);
+ sum_u16[2] = vabaq_u16(sum_u16[2], s, r2);
+
+ src16_ptr += src_stride_2;
+ ref16_ptr0 += ref_stride_2;
+ ref16_ptr1 += ref_stride_2;
+ ref16_ptr2 += ref_stride_2;
+ i -= 2;
+ } while (i != 0);
- res[0] = horizontal_add_u32x4(sum[0]);
- res[1] = horizontal_add_u32x4(sum[1]);
- res[2] = horizontal_add_u32x4(sum[2]);
+ sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]);
+ sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]);
+ sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]);
+ h -= h_limit;
+ } while (h != 0);
+
+ res[0] = horizontal_add_u32x4(sum_u32[0]);
+ res[1] = horizontal_add_u32x4(sum_u32[1]);
+ res[2] = horizontal_add_u32x4(sum_u32[2]);
}
static inline void highbd_sad8xhx3d_neon(const uint8_t *src_ptr, int src_stride,
@@ -333,37 +408,51 @@ static inline void highbd_sad8xhx3d_neon(const uint8_t *src_ptr, int src_stride,
const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
+ const int src_stride_2 = src_stride << 1;
+ const int ref_stride_2 = ref_stride << 1;
- // 'h_overflow' is the number of 8-wide rows we can process before 16-bit
- // accumulators overflow. After hitting this limit accumulate into 32-bit
- // elements. 65535 / 4095 ~= 16, so 16 8-wide rows.
- const int h_overflow = 16;
- // If block height 'h' is smaller than this limit, use 'h' instead.
+ // Two 16-bit accumulators each hold up to 16 rows: 32 rows total.
+ const int h_overflow = 32;
const int h_limit = h < h_overflow ? h : h_overflow;
+ assert(h % 2 == 0);
assert(h % h_limit == 0);
uint32x4_t sum_u32[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
- int h_tmp = h_limit;
- int i = 0;
do {
- uint16x8_t sum_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };
+ uint16x8_t sum0_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };
+ uint16x8_t sum1_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };
+ int i = h_limit;
do {
- uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride);
-
- sum_u16[0] =
- vabaq_u16(sum_u16[0], s0, vld1q_u16(ref16_ptr0 + i * ref_stride));
- sum_u16[1] =
- vabaq_u16(sum_u16[1], s0, vld1q_u16(ref16_ptr1 + i * ref_stride));
- sum_u16[2] =
- vabaq_u16(sum_u16[2], s0, vld1q_u16(ref16_ptr2 + i * ref_stride));
- } while (++i < h_tmp);
-
- sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]);
- sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]);
- sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]);
+ uint16x8_t s0 = vld1q_u16(src16_ptr);
+ uint16x8_t s1 = vld1q_u16(src16_ptr + src_stride);
+
+ sum0_u16[0] = vabaq_u16(sum0_u16[0], s0, vld1q_u16(ref16_ptr0));
+ sum0_u16[1] = vabaq_u16(sum0_u16[1], s0, vld1q_u16(ref16_ptr1));
+ sum0_u16[2] = vabaq_u16(sum0_u16[2], s0, vld1q_u16(ref16_ptr2));
+
+ sum1_u16[0] =
+ vabaq_u16(sum1_u16[0], s1, vld1q_u16(ref16_ptr0 + ref_stride));
+ sum1_u16[1] =
+ vabaq_u16(sum1_u16[1], s1, vld1q_u16(ref16_ptr1 + ref_stride));
+ sum1_u16[2] =
+ vabaq_u16(sum1_u16[2], s1, vld1q_u16(ref16_ptr2 + ref_stride));
+
+ src16_ptr += src_stride_2;
+ ref16_ptr0 += ref_stride_2;
+ ref16_ptr1 += ref_stride_2;
+ ref16_ptr2 += ref_stride_2;
+ i -= 2;
+ } while (i != 0);
+
+ sum_u32[0] = vpadalq_u16(sum_u32[0], sum0_u16[0]);
+ sum_u32[1] = vpadalq_u16(sum_u32[1], sum0_u16[1]);
+ sum_u32[2] = vpadalq_u16(sum_u32[2], sum0_u16[2]);
+
+ sum_u32[0] = vpadalq_u16(sum_u32[0], sum1_u16[0]);
+ sum_u32[1] = vpadalq_u16(sum_u32[1], sum1_u16[1]);
+ sum_u32[2] = vpadalq_u16(sum_u32[2], sum1_u16[2]);
- h_tmp += h_limit;
h -= h_limit;
} while (h != 0);
@@ -386,7 +475,8 @@ static inline void highbd_sadwxhx3d_neon(const uint8_t *src_ptr, int src_stride,
assert(h % h_limit == 0);
do {
- uint16x8_t sum_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };
+ uint16x8_t sum0_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };
+ uint16x8_t sum1_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };
int i = h_limit;
do {
@@ -394,14 +484,14 @@ static inline void highbd_sadwxhx3d_neon(const uint8_t *src_ptr, int src_stride,
do {
uint16x8_t s0 = vld1q_u16(src16_ptr + j);
- sum_u16[0] = vabaq_u16(sum_u16[0], s0, vld1q_u16(ref16_ptr0 + j));
- sum_u16[1] = vabaq_u16(sum_u16[1], s0, vld1q_u16(ref16_ptr1 + j));
- sum_u16[2] = vabaq_u16(sum_u16[2], s0, vld1q_u16(ref16_ptr2 + j));
+ sum0_u16[0] = vabaq_u16(sum0_u16[0], s0, vld1q_u16(ref16_ptr0 + j));
+ sum0_u16[1] = vabaq_u16(sum0_u16[1], s0, vld1q_u16(ref16_ptr1 + j));
+ sum0_u16[2] = vabaq_u16(sum0_u16[2], s0, vld1q_u16(ref16_ptr2 + j));
uint16x8_t s1 = vld1q_u16(src16_ptr + j + 8);
- sum_u16[0] = vabaq_u16(sum_u16[0], s1, vld1q_u16(ref16_ptr0 + j + 8));
- sum_u16[1] = vabaq_u16(sum_u16[1], s1, vld1q_u16(ref16_ptr1 + j + 8));
- sum_u16[2] = vabaq_u16(sum_u16[2], s1, vld1q_u16(ref16_ptr2 + j + 8));
+ sum1_u16[0] = vabaq_u16(sum1_u16[0], s1, vld1q_u16(ref16_ptr0 + j + 8));
+ sum1_u16[1] = vabaq_u16(sum1_u16[1], s1, vld1q_u16(ref16_ptr1 + j + 8));
+ sum1_u16[2] = vabaq_u16(sum1_u16[2], s1, vld1q_u16(ref16_ptr2 + j + 8));
j += 16;
} while (j < w);
@@ -412,9 +502,13 @@ static inline void highbd_sadwxhx3d_neon(const uint8_t *src_ptr, int src_stride,
ref16_ptr2 += ref_stride;
} while (--i != 0);
- sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]);
- sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]);
- sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]);
+ sum_u32[0] = vpadalq_u16(sum_u32[0], sum0_u16[0]);
+ sum_u32[1] = vpadalq_u16(sum_u32[1], sum0_u16[1]);
+ sum_u32[2] = vpadalq_u16(sum_u32[2], sum0_u16[2]);
+
+ sum_u32[0] = vpadalq_u16(sum_u32[0], sum1_u16[0]);
+ sum_u32[1] = vpadalq_u16(sum_u32[1], sum1_u16[1]);
+ sum_u32[2] = vpadalq_u16(sum_u32[2], sum1_u16[2]);
h -= h_limit;
} while (h != 0);
@@ -429,10 +523,8 @@ static inline void highbd_sad16xhx3d_neon(const uint8_t *src_ptr,
const uint8_t *const ref_ptr[4],
int ref_stride, uint32_t res[4],
int h) {
- // 'h_overflow' is the number of 16-wide rows we can process before 16-bit
- // accumulators overflow. After hitting this limit accumulate into 32-bit
- // elements. 65535 / 4095 ~= 16, so 8 16-wide rows.
- const int h_overflow = 8;
+ // Two 16-bit accumulators hold up to 16 16-wide rows before overflow.
+ const int h_overflow = 16;
highbd_sadwxhx3d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 16, h,
h_overflow);
}
@@ -442,10 +534,8 @@ static inline void highbd_sad32xhx3d_neon(const uint8_t *src_ptr,
const uint8_t *const ref_ptr[4],
int ref_stride, uint32_t res[4],
int h) {
- // 'h_overflow' is the number of 32-wide rows we can process before 16-bit
- // accumulators overflow. After hitting this limit accumulate into 32-bit
- // elements. 65535 / 4095 ~= 16, so 4 32-wide rows.
- const int h_overflow = 4;
+ // Two 16-bit accumulators hold up to 8 32-wide rows before overflow.
+ const int h_overflow = 8;
highbd_sadwxhx3d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 32, h,
h_overflow);
}
@@ -455,10 +545,8 @@ static inline void highbd_sad64xhx3d_neon(const uint8_t *src_ptr,
const uint8_t *const ref_ptr[4],
int ref_stride, uint32_t res[4],
int h) {
- // 'h_overflow' is the number of 64-wide rows we can process before 16-bit
- // accumulators overflow. After hitting this limit accumulate into 32-bit
- // elements. 65535 / 4095 ~= 16, so 2 64-wide rows.
- const int h_overflow = 2;
+ // Two 16-bit accumulators hold up to 4 64-wide rows before overflow.
+ const int h_overflow = 4;
highbd_sadwxhx3d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 64, h,
h_overflow);
}
@@ -468,10 +556,8 @@ static inline void highbd_sad128xhx3d_neon(const uint8_t *src_ptr,
const uint8_t *const ref_ptr[4],
int ref_stride, uint32_t res[4],
int h) {
- // 'h_overflow' is the number of 128-wide rows we can process before 16-bit
- // accumulators overflow. After hitting this limit accumulate into 32-bit
- // elements. 65535 / 4095 ~= 16, so 1 128-wide rows.
- const int h_overflow = 1;
+ // Two 16-bit accumulators hold up to 2 128-wide rows before overflow.
+ const int h_overflow = 2;
highbd_sadwxhx3d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 128, h,
h_overflow);
}
diff --git a/aom_dsp/arm/highbd_sse_neon.c b/aom_dsp/arm/highbd_sse_neon.c
index 48d4763a07..fd26f18b1d 100644
--- a/aom_dsp/arm/highbd_sse_neon.c
+++ b/aom_dsp/arm/highbd_sse_neon.c
@@ -10,6 +10,7 @@
*/
#include <arm_neon.h>
+#include <assert.h>
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/arm/sum_neon.h"
@@ -153,110 +154,128 @@ static inline int64_t highbd_sse_32xh_neon(const uint16_t *src, int src_stride,
static inline int64_t highbd_sse_16xh_neon(const uint16_t *src, int src_stride,
const uint16_t *ref, int ref_stride,
int height) {
- uint32x4_t sse[4];
+ assert(height >= 2 && (height & 1) == 0);
+ uint32x4_t sse[8];
highbd_sse_8x1_init_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
highbd_sse_8x1_init_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
-
+ src += src_stride;
+ ref += ref_stride;
+ highbd_sse_8x1_init_neon(src + 0 * 8, ref + 0 * 8, &sse[4], &sse[5]);
+ highbd_sse_8x1_init_neon(src + 1 * 8, ref + 1 * 8, &sse[6], &sse[7]);
src += src_stride;
ref += ref_stride;
- while (--height != 0) {
+ height -= 2;
+ while (height != 0) {
highbd_sse_8x1_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
highbd_sse_8x1_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
-
src += src_stride;
ref += ref_stride;
+ highbd_sse_8x1_neon(src + 0 * 8, ref + 0 * 8, &sse[4], &sse[5]);
+ highbd_sse_8x1_neon(src + 1 * 8, ref + 1 * 8, &sse[6], &sse[7]);
+ src += src_stride;
+ ref += ref_stride;
+ height -= 2;
}
- return horizontal_long_add_u32x4_x4(sse);
+ return horizontal_long_add_u32x4_x8(sse);
}
static inline int64_t highbd_sse_8xh_neon(const uint16_t *src, int src_stride,
const uint16_t *ref, int ref_stride,
int height) {
- uint32x4_t sse[2];
+ assert(height >= 2 && (height & 1) == 0);
+ uint32x4_t sse[4];
highbd_sse_8x1_init_neon(src, ref, &sse[0], &sse[1]);
-
+ src += src_stride;
+ ref += ref_stride;
+ highbd_sse_8x1_init_neon(src, ref, &sse[2], &sse[3]);
src += src_stride;
ref += ref_stride;
- while (--height != 0) {
+ height -= 2;
+ while (height != 0) {
highbd_sse_8x1_neon(src, ref, &sse[0], &sse[1]);
-
src += src_stride;
ref += ref_stride;
+ highbd_sse_8x1_neon(src, ref, &sse[2], &sse[3]);
+ src += src_stride;
+ ref += ref_stride;
+ height -= 2;
}
- return horizontal_long_add_u32x4_x2(sse);
+ return horizontal_long_add_u32x4_x4(sse);
}
static inline int64_t highbd_sse_4xh_neon(const uint16_t *src, int src_stride,
const uint16_t *ref, int ref_stride,
int height) {
+ assert(height >= 2 && (height & 1) == 0);
// Peel the first loop iteration.
- uint16x4_t s = vld1_u16(src);
- uint16x4_t r = vld1_u16(ref);
-
- uint16x4_t abs_diff = vabd_u16(s, r);
- uint32x4_t sse = vmull_u16(abs_diff, abs_diff);
-
- src += src_stride;
- ref += ref_stride;
-
- while (--height != 0) {
- s = vld1_u16(src);
- r = vld1_u16(ref);
-
- abs_diff = vabd_u16(s, r);
- sse = vmlal_u16(sse, abs_diff, abs_diff);
-
- src += src_stride;
- ref += ref_stride;
+ uint32x4_t sse[2];
+ uint16x8_t s = vcombine_u16(vld1_u16(src), vld1_u16(src + src_stride));
+ uint16x8_t r = vcombine_u16(vld1_u16(ref), vld1_u16(ref + ref_stride));
+ uint16x8_t abs_diff = vabdq_u16(s, r);
+ sse[0] = vmull_u16(vget_low_u16(abs_diff), vget_low_u16(abs_diff));
+ sse[1] = vmull_u16(vget_high_u16(abs_diff), vget_high_u16(abs_diff));
+ src += 2 * src_stride;
+ ref += 2 * ref_stride;
+
+ height -= 2;
+ while (height != 0) {
+ s = vcombine_u16(vld1_u16(src), vld1_u16(src + src_stride));
+ r = vcombine_u16(vld1_u16(ref), vld1_u16(ref + ref_stride));
+ abs_diff = vabdq_u16(s, r);
+ sse[0] = vmlal_u16(sse[0], vget_low_u16(abs_diff), vget_low_u16(abs_diff));
+ sse[1] =
+ vmlal_u16(sse[1], vget_high_u16(abs_diff), vget_high_u16(abs_diff));
+ src += 2 * src_stride;
+ ref += 2 * ref_stride;
+ height -= 2;
}
- return horizontal_long_add_u32x4(sse);
+ return horizontal_long_add_u32x4_x2(sse);
}
static inline int64_t highbd_sse_wxh_neon(const uint16_t *src, int src_stride,
const uint16_t *ref, int ref_stride,
int width, int height) {
+ assert(width >= 8);
// { 0, 1, 2, 3, 4, 5, 6, 7 }
- uint16x8_t k01234567 = vmovl_u8(vcreate_u8(0x0706050403020100));
- uint16x8_t remainder_mask = vcltq_u16(k01234567, vdupq_n_u16(width & 7));
- uint64_t sse = 0;
+ const uint16x8_t k01234567 = vmovl_u8(vcreate_u8(0x0706050403020100));
+ const uint16x8_t remainder_mask =
+ vcltq_u16(k01234567, vdupq_n_u16(width & 7));
+ uint64x2_t sse_u64 = vdupq_n_u64(0);
do {
- int w = width;
- int offset = 0;
-
- do {
- uint16x8_t s = vld1q_u16(src + offset);
- uint16x8_t r = vld1q_u16(ref + offset);
-
- if (w < 8) {
- // Mask out-of-range elements.
- s = vandq_u16(s, remainder_mask);
- r = vandq_u16(r, remainder_mask);
- }
-
- uint16x8_t abs_diff = vabdq_u16(s, r);
- uint16x4_t abs_diff_lo = vget_low_u16(abs_diff);
- uint16x4_t abs_diff_hi = vget_high_u16(abs_diff);
-
- uint32x4_t sse_u32 = vmull_u16(abs_diff_lo, abs_diff_lo);
- sse_u32 = vmlal_u16(sse_u32, abs_diff_hi, abs_diff_hi);
-
- sse += horizontal_long_add_u32x4(sse_u32);
+ int w = width - 8;
+ int offset = 8;
+ uint32x4_t sse_u32[2];
+ highbd_sse_8x1_init_neon(src, ref, &sse_u32[0], &sse_u32[1]);
+ while (w >= 8) {
+ highbd_sse_8x1_neon(src + offset, ref + offset, &sse_u32[0], &sse_u32[1]);
offset += 8;
w -= 8;
- } while (w > 0);
-
+ }
+
+ if (w > 0) {
+ const uint16x8_t s = vandq_u16(vld1q_u16(src + offset), remainder_mask);
+ const uint16x8_t r = vandq_u16(vld1q_u16(ref + offset), remainder_mask);
+ const uint16x8_t abs_diff = vabdq_u16(s, r);
+ sse_u32[0] =
+ vmlal_u16(sse_u32[0], vget_low_u16(abs_diff), vget_low_u16(abs_diff));
+ sse_u32[1] = vmlal_u16(sse_u32[1], vget_high_u16(abs_diff),
+ vget_high_u16(abs_diff));
+ }
+
+ sse_u64 = vpadalq_u32(sse_u64, sse_u32[0]);
+ sse_u64 = vpadalq_u32(sse_u64, sse_u32[1]);
src += src_stride;
ref += ref_stride;
} while (--height != 0);
- return sse;
+ return horizontal_add_u64x2(sse_u64);
}
int64_t aom_highbd_sse_neon(const uint8_t *src8, int src_stride,