Commit 12ad11ebd0 for aom
commit 12ad11ebd0c781c6b9fb95f4de1d821cbf7e7fc5
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Nov 14 11:38:19 2026 -0800
av1,neon: Optimize LBD/HBD IntraBC 2D convolve
Add width-specialized horizontal and vertical passes for w <= 4 and
w == 8 in av1_convolve_2d_sr_intrabc_neon and
av1_highbd_convolve_2d_sr_intrabc_neon, use compact im_stride = w
intermediate buffers, and unroll vertical filtering by 2 rows with
sliding row registers.
Speedup: 2.52x (LBD) and 2.69x (HBD) vs C in AV1Convolve2dSrIntrabc;
-0.28% (1T) / -1.91% (4T) on Server Arm (AmpereOne/Neoverse N2) and
-0.40% (1T Cortex-X4) / -0.53% (1T Cortex-A720) / -0.77% (4T, min
-2.09%) on Pixel 9 Tensor G4 avifenc (speed 6, bit-exact).
TAG=agy
CONV=490488d4-7005-402a-8cc4-c245227f80fe
Change-Id: Ibf6e03909b111111111111111111111111111111
diff --git a/av1/common/arm/convolve_neon.c b/av1/common/arm/convolve_neon.c
index 68521f2114..1a751cac50 100644
--- a/av1/common/arm/convolve_neon.c
+++ b/av1/common/arm/convolve_neon.c
@@ -1792,6 +1792,9 @@ void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,
assert(subpel_y_qn == 8);
assert(filter_params_x->taps == 2 && filter_params_y->taps == 2);
assert((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS);
+ assert(w >= 2 && h >= 2);
+ assert((w & 1) == 0 && (h & 1) == 0);
+ assert(w <= 8 || (w & 15) == 0);
(void)filter_params_x;
(void)subpel_x_qn;
(void)filter_params_y;
@@ -1800,7 +1803,7 @@ void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,
uint16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE];
int im_h = h + 1;
- int im_stride = w;
+ const int im_stride = w <= 4 ? 4 : w;
assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE);
uint16_t *im = im_block;
@@ -1819,6 +1822,16 @@ void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,
// Safe to store the whole vector, the im buffer is big enough.
vst1_u16(im, sum);
+ src += src_stride;
+ im += im_stride;
+ } while (--im_h != 0);
+ } else if (w == 8) {
+ do {
+ uint8x8_t s0 = vld1_u8(src);
+ uint8x8_t s1 = vld1_u8(src + 1);
+
+ vst1q_u16(im, vaddl_u8(s0, s1));
+
src += src_stride;
im += im_stride;
} while (--im_h != 0);
@@ -1829,16 +1842,18 @@ void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,
int width = w;
do {
- uint8x8_t s0 = vld1_u8(src_ptr);
- uint8x8_t s1 = vld1_u8(src_ptr + 1);
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+ uint8x16_t s1 = vld1q_u8(src_ptr + 1);
- uint16x8_t sum = vaddl_u8(s0, s1);
+ uint16x8_t sum0 = vaddl_u8(vget_low_u8(s0), vget_low_u8(s1));
+ uint16x8_t sum1 = vaddl_u8(vget_high_u8(s0), vget_high_u8(s1));
- vst1q_u16(im_ptr, sum);
+ vst1q_u16(im_ptr, sum0);
+ vst1q_u16(im_ptr + 8, sum1);
- src_ptr += 8;
- im_ptr += 8;
- width -= 8;
+ src_ptr += 16;
+ im_ptr += 16;
+ width -= 16;
} while (width != 0);
src += src_stride;
im += im_stride;
@@ -1849,8 +1864,8 @@ void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,
// Vertical filter.
if (w <= 4) {
+ uint16x4_t s0 = vld1_u16(im);
do {
- uint16x4_t s0 = vld1_u16(im);
uint16x4_t s1 = vld1_u16(im + im_stride);
uint16x4_t s2 = vld1_u16(im + 2 * im_stride);
@@ -1868,6 +1883,24 @@ void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,
store_u8_4x1(dst + 1 * dst_stride, d1);
}
+ s0 = s2;
+ im += 2 * im_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else if (w == 8) {
+ uint16x8_t s0 = vld1q_u16(im);
+ do {
+ uint16x8_t s1 = vld1q_u16(im + im_stride);
+ uint16x8_t s2 = vld1q_u16(im + 2 * im_stride);
+
+ uint8x8_t d0 = vqrshrn_n_u16(vaddq_u16(s0, s1), 2);
+ uint8x8_t d1 = vqrshrn_n_u16(vaddq_u16(s1, s2), 2);
+
+ vst1_u8(dst, d0);
+ vst1_u8(dst + dst_stride, d1);
+
+ s0 = s2;
im += 2 * im_stride;
dst += 2 * dst_stride;
h -= 2;
@@ -1877,22 +1910,39 @@ void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,
uint16_t *im_ptr = im;
uint8_t *dst_ptr = dst;
int height = h;
+ uint16x8_t s0_lo = vld1q_u16(im_ptr);
+ uint16x8_t s0_hi = vld1q_u16(im_ptr + 8);
do {
- uint16x8_t s0 = vld1q_u16(im_ptr);
- uint16x8_t s1 = vld1q_u16(im_ptr + im_stride);
-
- uint16x8_t sum = vaddq_u16(s0, s1);
- uint8x8_t d0 = vqrshrn_n_u16(sum, 2);
+ const uint16_t *row1 = im_ptr + im_stride;
+ const uint16_t *row2 = im_ptr + 2 * im_stride;
+ uint16x8_t s1_lo = vld1q_u16(row1);
+ uint16x8_t s1_hi = vld1q_u16(row1 + 8);
+ uint16x8_t s2_lo = vld1q_u16(row2);
+ uint16x8_t s2_hi = vld1q_u16(row2 + 8);
+
+ uint16x8_t sum0_lo = vaddq_u16(s0_lo, s1_lo);
+ uint16x8_t sum0_hi = vaddq_u16(s0_hi, s1_hi);
+ uint16x8_t sum1_lo = vaddq_u16(s1_lo, s2_lo);
+ uint16x8_t sum1_hi = vaddq_u16(s1_hi, s2_hi);
+
+ uint8x16_t d0 =
+ vcombine_u8(vqrshrn_n_u16(sum0_lo, 2), vqrshrn_n_u16(sum0_hi, 2));
+ uint8x16_t d1 =
+ vcombine_u8(vqrshrn_n_u16(sum1_lo, 2), vqrshrn_n_u16(sum1_hi, 2));
- vst1_u8(dst_ptr, d0);
+ vst1q_u8(dst_ptr, d0);
+ vst1q_u8(dst_ptr + dst_stride, d1);
- im_ptr += im_stride;
- dst_ptr += dst_stride;
- } while (--height != 0);
- im += 8;
- dst += 8;
- w -= 8;
+ s0_lo = s2_lo;
+ s0_hi = s2_hi;
+ im_ptr += 2 * im_stride;
+ dst_ptr += 2 * dst_stride;
+ height -= 2;
+ } while (height != 0);
+ im += 16;
+ dst += 16;
+ w -= 16;
} while (w != 0);
}
}
diff --git a/av1/common/arm/highbd_convolve_neon.c b/av1/common/arm/highbd_convolve_neon.c
index 17c711bbd5..fd5c1ab823 100644
--- a/av1/common/arm/highbd_convolve_neon.c
+++ b/av1/common/arm/highbd_convolve_neon.c
@@ -1979,17 +1979,20 @@ void av1_highbd_convolve_2d_sr_intrabc_neon(
assert(filter_params_x->taps == 2 && filter_params_y->taps == 2);
assert((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS);
assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE);
- (void)filter_params_x;
+ assert(w >= 2 && h >= 2);
+ assert((w & 1) == 0 && (h & 1) == 0);
+ assert(w <= 8 || (w & 15) == 0);
(void)subpel_x_qn;
- (void)filter_params_y;
(void)subpel_y_qn;
+ (void)filter_params_x;
+ (void)filter_params_y;
(void)conv_params;
(void)bd;
DECLARE_ALIGNED(16, uint16_t,
im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]);
int im_h = h + 1;
- int im_stride = MAX_SB_SIZE;
+ const int im_stride = w <= 4 ? 4 : w;
uint16x8_t vert_offset = vdupq_n_u16(1);
@@ -1998,6 +2001,9 @@ void av1_highbd_convolve_2d_sr_intrabc_neon(
// Horizontal filter.
if (w <= 4) {
do {
+ // Load at most 5 pixels per row. A full 8-pixel load would read
+ // pixels to the right of the reference block, which may be in a region
+ // that is concurrently being reconstructed by another encoder thread.
uint16x4_t s0 = vld1_u16(src);
uint16x4_t s1 = vld1_u16(src + 1);
@@ -2006,6 +2012,16 @@ void av1_highbd_convolve_2d_sr_intrabc_neon(
// Safe to store the whole vector, the im buffer is big enough.
vst1_u16(im, d0);
+ src += src_stride;
+ im += im_stride;
+ } while (--im_h != 0);
+ } else if (w == 8) {
+ do {
+ uint16x8_t s0 = vld1q_u16(src);
+ uint16x8_t s1 = vld1q_u16(src + 1);
+
+ vst1q_u16(im, vaddq_u16(s0, s1));
+
src += src_stride;
im += im_stride;
} while (--im_h != 0);
@@ -2018,14 +2034,15 @@ void av1_highbd_convolve_2d_sr_intrabc_neon(
do {
uint16x8_t s0 = vld1q_u16(src_ptr);
uint16x8_t s1 = vld1q_u16(src_ptr + 1);
+ uint16x8_t s2 = vld1q_u16(src_ptr + 8);
+ uint16x8_t s3 = vld1q_u16(src_ptr + 9);
- uint16x8_t d0 = vaddq_u16(s0, s1);
-
- vst1q_u16(im_ptr, d0);
+ vst1q_u16(im_ptr, vaddq_u16(s0, s1));
+ vst1q_u16(im_ptr + 8, vaddq_u16(s2, s3));
- src_ptr += 8;
- im_ptr += 8;
- width -= 8;
+ src_ptr += 16;
+ im_ptr += 16;
+ width -= 16;
} while (width != 0);
src += src_stride;
im += im_stride;
@@ -2036,40 +2053,50 @@ void av1_highbd_convolve_2d_sr_intrabc_neon(
// Vertical filter.
if (w <= 4) {
+ const uint16x4_t v_off = vget_low_u16(vert_offset);
+ uint16x4_t s0 = vld1_u16(im);
do {
- uint16x4_t s0 = vld1_u16(im);
uint16x4_t s1 = vld1_u16(im + im_stride);
+ uint16x4_t s2 = vld1_u16(im + 2 * im_stride);
- uint16x4_t d0 = vhadd_u16(s0, s1);
- d0 = vhadd_u16(d0, vget_low_u16(vert_offset));
+ uint16x4_t d0 = vhadd_u16(vhadd_u16(s0, s1), v_off);
+ uint16x4_t d1 = vhadd_u16(vhadd_u16(s1, s2), v_off);
if (w == 2) {
- store_u16_2x1(dst, d0);
+ store_u16_2x1(dst + 0 * dst_stride, d0);
+ store_u16_2x1(dst + 1 * dst_stride, d1);
} else {
- vst1_u16(dst, d0);
+ vst1_u16(dst + 0 * dst_stride, d0);
+ vst1_u16(dst + 1 * dst_stride, d1);
}
- im += im_stride;
- dst += dst_stride;
- } while (--h != 0);
+ s0 = s2;
+ im += 2 * im_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
} else {
do {
uint16_t *im_ptr = im;
uint16_t *dst_ptr = dst;
int height = h;
+ uint16x8_t s0 = vld1q_u16(im_ptr);
do {
- uint16x8_t s0 = vld1q_u16(im_ptr);
uint16x8_t s1 = vld1q_u16(im_ptr + im_stride);
+ uint16x8_t s2 = vld1q_u16(im_ptr + 2 * im_stride);
- uint16x8_t d0 = vhaddq_u16(s0, s1);
- d0 = vhaddq_u16(d0, vert_offset);
+ uint16x8_t d0 = vhaddq_u16(vhaddq_u16(s0, s1), vert_offset);
+ uint16x8_t d1 = vhaddq_u16(vhaddq_u16(s1, s2), vert_offset);
vst1q_u16(dst_ptr, d0);
+ vst1q_u16(dst_ptr + dst_stride, d1);
- im_ptr += im_stride;
- dst_ptr += dst_stride;
- } while (--height != 0);
+ s0 = s2;
+ im_ptr += 2 * im_stride;
+ dst_ptr += 2 * dst_stride;
+ height -= 2;
+ } while (height != 0);
im += 8;
dst += 8;
w -= 8;