Commit 515f603aa0 for aom
commit 515f603aa01a9e96485126a78319ff2adfb7dbb6
Author: Jonathan Wright <jonathan.wright@arm.com>
Date: Mon Jun 8 18:17:39 2026 +0100
Simplify av1_convolve_2d_sr_* Arm SIMD kernels
Remove some redundant constants added in the horizontal pass and
subtracted in the vertical pass of the Arm SIMD implementations of
av1_convolve_2d_sr_*.
This patch is mostly a port of the following SVT-AV1 commit:
https://gitlab.com/AOMediaCodec/SVT-AV1/-/commit/0c7d5a9f0e099c685d0a7a071bb026864782b841
Change-Id: Ib7eb9329f25a700784bdb02fa602a8a38dbc7d30
diff --git a/av1/common/arm/convolve_neon.c b/av1/common/arm/convolve_neon.c
index bc7c71128d..fc42f9e6e0 100644
--- a/av1/common/arm/convolve_neon.c
+++ b/av1/common/arm/convolve_neon.c
@@ -1188,11 +1188,9 @@ static inline void convolve_2d_sr_horiz_12tap_neon(
const uint8_t *src_ptr, int src_stride, int16_t *dst_ptr,
const int dst_stride, int w, int h, const int16x8_t x_filter_0_7,
const int16x4_t x_filter_8_11) {
- const int bd = 8;
// A shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding shifts -
// which are generally faster than rounding shifts on modern CPUs.
- const int32x4_t horiz_const =
- vdupq_n_s32((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1)));
+ const int32x4_t horiz_const = vdupq_n_s32(1 << (ROUND0_BITS - 1));
#if AOM_ARCH_AARCH64
do {
@@ -1322,7 +1320,6 @@ static inline int16x8_t convolve4_8_2d_h(const int16x8_t s0, const int16x8_t s1,
static inline void convolve_2d_sr_horiz_4tap_neon(
const uint8_t *src, ptrdiff_t src_stride, int16_t *dst,
ptrdiff_t dst_stride, int w, int h, const int16_t *filter_x) {
- const int bd = 8;
// All filter values are even, halve to reduce intermediate precision
// requirements.
const int16x4_t filter = vshr_n_s16(vld1_s16(filter_x + 2), 1);
@@ -1330,8 +1327,7 @@ static inline void convolve_2d_sr_horiz_4tap_neon(
// A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
// shifts - which are generally faster than rounding shifts on modern CPUs.
// (The extra -1 is needed because we halved the filter values.)
- const int16x8_t horiz_const = vdupq_n_s16((1 << (bd + FILTER_BITS - 2)) +
- (1 << ((ROUND0_BITS - 1) - 1)));
+ const int16x8_t horiz_const = vdupq_n_s16(1 << ((ROUND0_BITS - 1) - 1));
if (w == 4) {
do {
@@ -1451,8 +1447,6 @@ static inline int16x8_t convolve8_8_2d_h(const int16x8_t s0, const int16x8_t s1,
static inline void convolve_2d_sr_horiz_8tap_neon(
const uint8_t *src, int src_stride, int16_t *im_block, int im_stride, int w,
int im_h, const int16_t *x_filter_ptr) {
- const int bd = 8;
-
const uint8_t *src_ptr = src;
int16_t *dst_ptr = im_block;
int dst_stride = im_stride;
@@ -1461,8 +1455,7 @@ static inline void convolve_2d_sr_horiz_8tap_neon(
// A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
// shifts - which are generally faster than rounding shifts on modern CPUs.
// (The extra -1 is needed because we halved the filter values.)
- const int16x8_t horiz_const = vdupq_n_s16((1 << (bd + FILTER_BITS - 2)) +
- (1 << ((ROUND0_BITS - 1) - 1)));
+ const int16x8_t horiz_const = vdupq_n_s16(1 << ((ROUND0_BITS - 1) - 1));
// Filter values are even, so halve to reduce intermediate precision reqs.
const int16x8_t x_filter = vshrq_n_s16(vld1q_s16(x_filter_ptr), 1);
diff --git a/av1/common/arm/convolve_neon.h b/av1/common/arm/convolve_neon.h
index 206f3ba205..9d6e2fb1fa 100644
--- a/av1/common/arm/convolve_neon.h
+++ b/av1/common/arm/convolve_neon.h
@@ -50,8 +50,7 @@ static inline uint8x8_t convolve12_8_2d_v(
const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
const int16x8_t s6, const int16x8_t s7, const int16x8_t s8,
const int16x8_t s9, const int16x8_t s10, const int16x8_t s11,
- const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11,
- const int16x8_t sub_const) {
+ const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11) {
const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7);
const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7);
@@ -84,7 +83,6 @@ static inline uint8x8_t convolve12_8_2d_v(
int16x8_t res =
vcombine_s16(vqrshrn_n_s32(sum0, 2 * FILTER_BITS - ROUND0_BITS),
vqrshrn_n_s32(sum1, 2 * FILTER_BITS - ROUND0_BITS));
- res = vsubq_s16(res, sub_const);
return vqmovun_s16(res);
}
@@ -92,9 +90,6 @@ static inline uint8x8_t convolve12_8_2d_v(
static inline void convolve_2d_sr_vert_12tap_neon(
int16_t *src_ptr, int src_stride, uint8_t *dst_ptr, int dst_stride, int w,
int h, const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11) {
- const int bd = 8;
- const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1));
-
if (w <= 4) {
int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
load_s16_4x11(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7,
@@ -122,9 +117,6 @@ static inline void convolve_2d_sr_vert_12tap_neon(
vcombine_s16(vqrshrn_n_s32(d2, 2 * FILTER_BITS - ROUND0_BITS),
vqrshrn_n_s32(d3, 2 * FILTER_BITS - ROUND0_BITS));
- dd01 = vsubq_s16(dd01, sub_const);
- dd23 = vsubq_s16(dd23, sub_const);
-
uint8x8_t d01 = vqmovun_s16(dd01);
uint8x8_t d23 = vqmovun_s16(dd23);
@@ -162,18 +154,17 @@ static inline void convolve_2d_sr_vert_12tap_neon(
int16x8_t s11, s12, s13, s14;
load_s16_8x4(s, src_stride, &s11, &s12, &s13, &s14);
- uint8x8_t d0 =
- convolve12_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
- y_filter_0_7, y_filter_8_11, sub_const);
+ uint8x8_t d0 = convolve12_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9,
+ s10, s11, y_filter_0_7, y_filter_8_11);
uint8x8_t d1 =
convolve12_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
- y_filter_0_7, y_filter_8_11, sub_const);
+ y_filter_0_7, y_filter_8_11);
uint8x8_t d2 =
convolve12_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
- s13, y_filter_0_7, y_filter_8_11, sub_const);
+ s13, y_filter_0_7, y_filter_8_11);
uint8x8_t d3 =
convolve12_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13,
- s14, y_filter_0_7, y_filter_8_11, sub_const);
+ s14, y_filter_0_7, y_filter_8_11);
store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
@@ -223,8 +214,7 @@ static inline uint8x8_t convolve8_8_2d_v(const int16x8_t s0, const int16x8_t s1,
const int16x8_t s2, const int16x8_t s3,
const int16x8_t s4, const int16x8_t s5,
const int16x8_t s6, const int16x8_t s7,
- const int16x8_t y_filter,
- const int16x8_t sub_const) {
+ const int16x8_t y_filter) {
const int16x4_t y_filter_lo = vget_low_s16(y_filter);
const int16x4_t y_filter_hi = vget_high_s16(y_filter);
@@ -249,7 +239,6 @@ static inline uint8x8_t convolve8_8_2d_v(const int16x8_t s0, const int16x8_t s1,
int16x8_t res =
vcombine_s16(vqrshrn_n_s32(sum0, 2 * FILTER_BITS - ROUND0_BITS),
vqrshrn_n_s32(sum1, 2 * FILTER_BITS - ROUND0_BITS));
- res = vsubq_s16(res, sub_const);
return vqmovun_s16(res);
}
@@ -259,9 +248,6 @@ static inline void convolve_2d_sr_vert_8tap_neon(int16_t *src_ptr,
uint8_t *dst_ptr,
int dst_stride, int w, int h,
const int16x8_t y_filter) {
- const int bd = 8;
- const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1));
-
if (w <= 4) {
int16x4_t s0, s1, s2, s3, s4, s5, s6;
load_s16_4x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
@@ -278,8 +264,8 @@ static inline void convolve_2d_sr_vert_8tap_neon(int16_t *src_ptr,
int16x4_t d3 =
convolve8_4_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter);
- uint8x8_t d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), sub_const));
- uint8x8_t d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), sub_const));
+ uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1));
+ uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3));
store_u8x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, d01);
store_u8x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, d23);
@@ -297,8 +283,7 @@ static inline void convolve_2d_sr_vert_8tap_neon(int16_t *src_ptr,
#else // !AOM_ARCH_AARCH64
int16x4_t s7 = vld1_s16(src_ptr);
int16x4_t d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
- uint8x8_t d01 =
- vqmovun_s16(vsubq_s16(vcombine_s16(d0, vdup_n_s16(0)), sub_const));
+ uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, vdup_n_s16(0)));
store_u8_4x1(dst_ptr, d01);
@@ -330,14 +315,14 @@ static inline void convolve_2d_sr_vert_8tap_neon(int16_t *src_ptr,
int16x8_t s7, s8, s9, s10;
load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
- uint8x8_t d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7,
- y_filter, sub_const);
- uint8x8_t d1 = convolve8_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8,
- y_filter, sub_const);
- uint8x8_t d2 = convolve8_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9,
- y_filter, sub_const);
- uint8x8_t d3 = convolve8_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10,
- y_filter, sub_const);
+ uint8x8_t d0 =
+ convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
+ uint8x8_t d1 =
+ convolve8_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter);
+ uint8x8_t d2 =
+ convolve8_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter);
+ uint8x8_t d3 =
+ convolve8_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter);
store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
@@ -353,8 +338,8 @@ static inline void convolve_2d_sr_vert_8tap_neon(int16_t *src_ptr,
height -= 4;
#else // !AOM_ARCH_AARCH64
int16x8_t s7 = vld1q_s16(s);
- uint8x8_t d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7,
- y_filter, sub_const);
+ uint8x8_t d0 =
+ convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
vst1_u8(d, d0);
s0 = s1;
@@ -396,8 +381,7 @@ static inline int16x4_t convolve6_4_2d_v(const int16x4_t s0, const int16x4_t s1,
static inline uint8x8_t convolve6_8_2d_v(const int16x8_t s0, const int16x8_t s1,
const int16x8_t s2, const int16x8_t s3,
const int16x8_t s4, const int16x8_t s5,
- const int16x8_t y_filter,
- const int16x8_t sub_const) {
+ const int16x8_t y_filter) {
const int16x4_t y_filter_lo = vget_low_s16(y_filter);
const int16x4_t y_filter_hi = vget_high_s16(y_filter);
@@ -418,7 +402,6 @@ static inline uint8x8_t convolve6_8_2d_v(const int16x8_t s0, const int16x8_t s1,
int16x8_t res =
vcombine_s16(vqrshrn_n_s32(sum0, 2 * FILTER_BITS - ROUND0_BITS),
vqrshrn_n_s32(sum1, 2 * FILTER_BITS - ROUND0_BITS));
- res = vsubq_s16(res, sub_const);
return vqmovun_s16(res);
}
@@ -428,9 +411,6 @@ static inline void convolve_2d_sr_vert_6tap_neon(int16_t *src_ptr,
uint8_t *dst_ptr,
int dst_stride, int w, int h,
const int16x8_t y_filter) {
- const int bd = 8;
- const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1));
-
if (w <= 4) {
int16x4_t s0, s1, s2, s3, s4;
load_s16_4x5(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4);
@@ -446,8 +426,8 @@ static inline void convolve_2d_sr_vert_6tap_neon(int16_t *src_ptr,
int16x4_t d2 = convolve6_4_2d_v(s2, s3, s4, s5, s6, s7, y_filter);
int16x4_t d3 = convolve6_4_2d_v(s3, s4, s5, s6, s7, s8, y_filter);
- uint8x8_t d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), sub_const));
- uint8x8_t d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), sub_const));
+ uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1));
+ uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3));
store_u8x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, d01);
store_u8x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, d23);
@@ -463,8 +443,7 @@ static inline void convolve_2d_sr_vert_6tap_neon(int16_t *src_ptr,
#else // !AOM_ARCH_AARCH64
int16x4_t s5 = vld1_s16(src_ptr);
int16x4_t d0 = convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter);
- uint8x8_t d01 =
- vqmovun_s16(vsubq_s16(vcombine_s16(d0, vdup_n_s16(0)), sub_const));
+ uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, vdup_n_s16(0)));
store_u8_4x1(dst_ptr, d01);
@@ -494,14 +473,10 @@ static inline void convolve_2d_sr_vert_6tap_neon(int16_t *src_ptr,
int16x8_t s5, s6, s7, s8;
load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
- uint8x8_t d0 =
- convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, sub_const);
- uint8x8_t d1 =
- convolve6_8_2d_v(s1, s2, s3, s4, s5, s6, y_filter, sub_const);
- uint8x8_t d2 =
- convolve6_8_2d_v(s2, s3, s4, s5, s6, s7, y_filter, sub_const);
- uint8x8_t d3 =
- convolve6_8_2d_v(s3, s4, s5, s6, s7, s8, y_filter, sub_const);
+ uint8x8_t d0 = convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter);
+ uint8x8_t d1 = convolve6_8_2d_v(s1, s2, s3, s4, s5, s6, y_filter);
+ uint8x8_t d2 = convolve6_8_2d_v(s2, s3, s4, s5, s6, s7, y_filter);
+ uint8x8_t d3 = convolve6_8_2d_v(s3, s4, s5, s6, s7, s8, y_filter);
store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
@@ -515,8 +490,7 @@ static inline void convolve_2d_sr_vert_6tap_neon(int16_t *src_ptr,
height -= 4;
#else // !AOM_ARCH_AARCH64
int16x8_t s5 = vld1q_s16(s);
- uint8x8_t d0 =
- convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, sub_const);
+ uint8x8_t d0 = convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter);
vst1_u8(d, d0);
s0 = s1;
@@ -549,8 +523,7 @@ static inline int16x4_t convolve4_4_2d_v(const int16x4_t s0, const int16x4_t s1,
static inline uint8x8_t convolve4_8_2d_v(const int16x8_t s0, const int16x8_t s1,
const int16x8_t s2, const int16x8_t s3,
- const int16x4_t y_filter,
- const int16x8_t sub_const) {
+ const int16x4_t y_filter) {
int32x4_t sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter, 0);
sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter, 1);
sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter, 2);
@@ -564,7 +537,6 @@ static inline uint8x8_t convolve4_8_2d_v(const int16x8_t s0, const int16x8_t s1,
int16x8_t res =
vcombine_s16(vqrshrn_n_s32(sum0, 2 * FILTER_BITS - ROUND0_BITS),
vqrshrn_n_s32(sum1, 2 * FILTER_BITS - ROUND0_BITS));
- res = vsubq_s16(res, sub_const);
return vqmovun_s16(res);
}
@@ -574,9 +546,6 @@ static inline void convolve_2d_sr_vert_4tap_neon(int16_t *src_ptr,
uint8_t *dst_ptr,
int dst_stride, int w, int h,
const int16_t *y_filter) {
- const int bd = 8;
- const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1));
-
const int16x4_t filter = vld1_s16(y_filter + 2);
if (w == 4) {
@@ -593,8 +562,8 @@ static inline void convolve_2d_sr_vert_4tap_neon(int16_t *src_ptr,
int16x4_t d2 = convolve4_4_2d_v(s2, s3, s4, s5, filter);
int16x4_t d3 = convolve4_4_2d_v(s3, s4, s5, s6, filter);
- uint8x8_t d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), sub_const));
- uint8x8_t d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), sub_const));
+ uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1));
+ uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3));
store_u8x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, d01);
store_u8x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, d23);
@@ -622,10 +591,10 @@ static inline void convolve_2d_sr_vert_4tap_neon(int16_t *src_ptr,
int16x8_t s3, s4, s5, s6;
load_s16_8x4(s, src_stride, &s3, &s4, &s5, &s6);
- uint8x8_t d0 = convolve4_8_2d_v(s0, s1, s2, s3, filter, sub_const);
- uint8x8_t d1 = convolve4_8_2d_v(s1, s2, s3, s4, filter, sub_const);
- uint8x8_t d2 = convolve4_8_2d_v(s2, s3, s4, s5, filter, sub_const);
- uint8x8_t d3 = convolve4_8_2d_v(s3, s4, s5, s6, filter, sub_const);
+ uint8x8_t d0 = convolve4_8_2d_v(s0, s1, s2, s3, filter);
+ uint8x8_t d1 = convolve4_8_2d_v(s1, s2, s3, s4, filter);
+ uint8x8_t d2 = convolve4_8_2d_v(s2, s3, s4, s5, filter);
+ uint8x8_t d3 = convolve4_8_2d_v(s3, s4, s5, s6, filter);
store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
diff --git a/av1/common/arm/convolve_neon_dotprod.c b/av1/common/arm/convolve_neon_dotprod.c
index 35cdf9ed36..b105e01ab3 100644
--- a/av1/common/arm/convolve_neon_dotprod.c
+++ b/av1/common/arm/convolve_neon_dotprod.c
@@ -1065,8 +1065,6 @@ static inline void convolve_2d_sr_horiz_12tap_neon_dotprod(
// The no-op filter should never be used here.
assert(vgetq_lane_s16(x_filter_0_7, 5) != 128);
- const int bd = 8;
-
// Narrow filter values to 8-bit.
const int16x8x2_t x_filter_s16 = {
{ x_filter_0_7, vcombine_s16(x_filter_8_11, vdup_n_s16(0)) }
@@ -1076,8 +1074,7 @@ static inline void convolve_2d_sr_horiz_12tap_neon_dotprod(
// Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding
// shifts - which are generally faster than rounding shifts on modern CPUs.
- const int32_t horiz_const =
- ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1)));
+ const int32_t horiz_const = 1 << (ROUND0_BITS - 1);
// Dot product constants.
const int32x4_t correction = vdupq_n_s32((128 << FILTER_BITS) + horiz_const);
const uint8x16x3_t permute_tbl = vld1q_u8_x3(kDotProdPermuteTbl);
@@ -1203,7 +1200,6 @@ static inline int16x8_t convolve4_8_2d_h(const uint8x16_t samples,
static inline void convolve_2d_sr_horiz_4tap_neon_dotprod(
const uint8_t *src, ptrdiff_t src_stride, int16_t *dst,
ptrdiff_t dst_stride, int w, int h, const int16_t *filter_x) {
- const int bd = 8;
const int16x4_t x_filter = vld1_s16(filter_x + 2);
// All 4-tap and bilinear filter values are even, so halve them to reduce
// intermediate precision requirements.
@@ -1211,8 +1207,7 @@ static inline void convolve_2d_sr_horiz_4tap_neon_dotprod(
// Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding
// shifts - which are generally faster than rounding shifts on modern CPUs.
- const int32_t horiz_const =
- ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1)));
+ const int32_t horiz_const = 1 << (ROUND0_BITS - 1);
// Accumulate into 128 << FILTER_BITS to account for range transform.
// Halve the total because we halved the filter values.
const int32x4_t correction =
@@ -1328,11 +1323,9 @@ static inline void convolve_2d_sr_horiz_8tap_neon_dotprod(
// Filter values are even, so halve to reduce intermediate precision reqs.
const int8x8_t x_filter = vshrn_n_s16(x_filter_s16, 1);
- const int bd = 8;
// Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding
// shifts - which are generally faster than rounding shifts on modern CPUs.
- const int32_t horiz_const =
- ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1)));
+ const int32_t horiz_const = 1 << (ROUND0_BITS - 1);
// Halve the total because we halved the filter values.
const int32x4_t correction =
vdupq_n_s32(((128 << FILTER_BITS) + horiz_const) / 2);
@@ -1394,16 +1387,13 @@ static inline void convolve_2d_sr_6tap_neon_dotprod(
// Filter values are even, so halve to reduce intermediate precision reqs.
const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
- const int bd = 8;
// Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding
// shifts - which are generally faster than rounding shifts on modern CPUs.
- const int32_t horiz_const =
- ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1)));
+ const int32_t horiz_const = 1 << (ROUND0_BITS - 1);
// Accumulate into 128 << FILTER_BITS to account for range transform.
// Halve the total because we halved the filter values.
const int32x4_t correction =
vdupq_n_s32(((128 << FILTER_BITS) + horiz_const) / 2);
- const int16x8_t vert_const = vdupq_n_s16(1 << (bd - 1));
const uint8x16x3_t permute_tbl = vld1q_u8_x3(kDotProdPermuteTbl);
do {
@@ -1434,14 +1424,14 @@ static inline void convolve_2d_sr_6tap_neon_dotprod(
int16x8_t v_s8 =
convolve8_8_2d_h(h_s8, x_filter, correction, permute_tbl);
- uint8x8_t d0 = convolve6_8_2d_v(v_s0, v_s1, v_s2, v_s3, v_s4, v_s5,
- y_filter, vert_const);
- uint8x8_t d1 = convolve6_8_2d_v(v_s1, v_s2, v_s3, v_s4, v_s5, v_s6,
- y_filter, vert_const);
- uint8x8_t d2 = convolve6_8_2d_v(v_s2, v_s3, v_s4, v_s5, v_s6, v_s7,
- y_filter, vert_const);
- uint8x8_t d3 = convolve6_8_2d_v(v_s3, v_s4, v_s5, v_s6, v_s7, v_s8,
- y_filter, vert_const);
+ uint8x8_t d0 =
+ convolve6_8_2d_v(v_s0, v_s1, v_s2, v_s3, v_s4, v_s5, y_filter);
+ uint8x8_t d1 =
+ convolve6_8_2d_v(v_s1, v_s2, v_s3, v_s4, v_s5, v_s6, y_filter);
+ uint8x8_t d2 =
+ convolve6_8_2d_v(v_s2, v_s3, v_s4, v_s5, v_s6, v_s7, y_filter);
+ uint8x8_t d3 =
+ convolve6_8_2d_v(v_s3, v_s4, v_s5, v_s6, v_s7, v_s8, y_filter);
store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
@@ -1464,9 +1454,6 @@ static inline void convolve_2d_sr_6tap_neon_dotprod(
static inline void convolve_2d_sr_4tap_neon_dotprod(
const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w,
int h, const int16_t *x_filter_ptr, const int16_t *y_filter_ptr) {
- const int bd = 8;
- const int16x8_t vert_const = vdupq_n_s16(1 << (bd - 1));
-
const int16x4_t y_filter = vld1_s16(y_filter_ptr + 2);
const int16x4_t x_filter_s16 = vld1_s16(x_filter_ptr + 2);
// All 4-tap and bilinear filter values are even, so halve them to reduce
@@ -1476,8 +1463,7 @@ static inline void convolve_2d_sr_4tap_neon_dotprod(
// Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding
// shifts - which are generally faster than rounding shifts on modern CPUs.
- const int32_t horiz_const =
- ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1)));
+ const int32_t horiz_const = 1 << (ROUND0_BITS - 1);
// Accumulate into 128 << FILTER_BITS to account for range transform.
// Halve the total because we halved the filter values.
const int32x4_t correction =
@@ -1513,8 +1499,8 @@ static inline void convolve_2d_sr_4tap_neon_dotprod(
int16x4_t d2 = convolve4_4_2d_v(v_s2, v_s3, v_s4, v_s5, y_filter);
int16x4_t d3 = convolve4_4_2d_v(v_s3, v_s4, v_s5, v_s6, y_filter);
- uint8x8_t d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), vert_const));
- uint8x8_t d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), vert_const));
+ uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1));
+ uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3));
store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d01);
store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d23);
@@ -1560,14 +1546,10 @@ static inline void convolve_2d_sr_4tap_neon_dotprod(
int16x8_t v_s6 =
convolve4_8_2d_h(h_s6, x_filter, permute_tbl, correction);
- uint8x8_t d0 =
- convolve4_8_2d_v(v_s0, v_s1, v_s2, v_s3, y_filter, vert_const);
- uint8x8_t d1 =
- convolve4_8_2d_v(v_s1, v_s2, v_s3, v_s4, y_filter, vert_const);
- uint8x8_t d2 =
- convolve4_8_2d_v(v_s2, v_s3, v_s4, v_s5, y_filter, vert_const);
- uint8x8_t d3 =
- convolve4_8_2d_v(v_s3, v_s4, v_s5, v_s6, y_filter, vert_const);
+ uint8x8_t d0 = convolve4_8_2d_v(v_s0, v_s1, v_s2, v_s3, y_filter);
+ uint8x8_t d1 = convolve4_8_2d_v(v_s1, v_s2, v_s3, v_s4, y_filter);
+ uint8x8_t d2 = convolve4_8_2d_v(v_s2, v_s3, v_s4, v_s5, y_filter);
+ uint8x8_t d3 = convolve4_8_2d_v(v_s3, v_s4, v_s5, v_s6, y_filter);
store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
diff --git a/av1/common/arm/convolve_neon_i8mm.c b/av1/common/arm/convolve_neon_i8mm.c
index 1452841d4a..0e04060f86 100644
--- a/av1/common/arm/convolve_neon_i8mm.c
+++ b/av1/common/arm/convolve_neon_i8mm.c
@@ -931,12 +931,10 @@ static inline void convolve_2d_sr_horiz_8tap_neon_i8mm(
const uint8x8_t f0 = vdup_n_u8(-x_filter_ptr[0] >> 1);
const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul8PermuteTbl);
- const int bd = 8;
// This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
// shifts - which are generally faster than rounding shifts on modern CPUs.
// The outermost -1 is needed because we halved the filter values.
- const int16x8_t horiz_const = vdupq_n_s16((1 << (bd + FILTER_BITS - 2)) +
- (1 << ((ROUND0_BITS - 1) - 1)));
+ const int16x8_t horiz_const = vdupq_n_s16(1 << ((ROUND0_BITS - 1) - 1));
const uint8_t *src_ptr = src;
int16_t *dst_ptr = im_block;
@@ -1030,7 +1028,6 @@ static inline int16x8_t convolve4_8_2d_h(const uint8x16_t samples,
static inline void convolve_2d_sr_horiz_4tap_neon_i8mm(
const uint8_t *src, int src_stride, int16_t *dst, int dst_stride, int width,
int height, const int16_t *filter_x) {
- const int bd = 8;
const int16x4_t x_filter = vld1_s16(filter_x + 2);
// All 4-tap and bilinear filter values are even, so halve them to reduce
// intermediate precision requirements.
@@ -1039,8 +1036,7 @@ static inline void convolve_2d_sr_horiz_4tap_neon_i8mm(
// Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding
// shifts - which are generally faster than rounding shifts on modern CPUs.
// Halve the total because we halved the filter values.
- const int32x4_t horiz_const = vdupq_n_s32(
- (((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1))) / 2));
+ const int32x4_t horiz_const = vdupq_n_s32((1 << (ROUND0_BITS - 1)) / 2);
if (width == 4) {
const uint8x16_t perm_tbl = vld1q_u8(kDotProdPermuteTbl);
@@ -1165,14 +1161,11 @@ static inline void convolve_2d_sr_6tap_neon_i8mm(const uint8_t *src,
const int8x16_t x_filter =
vcombine_s8(vext_s8(x_filter_s8, x_filter_s8, 1), x_filter_s8);
- const int bd = 8;
// This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
// shifts in convolution kernels - which are generally faster than rounding
// shifts on modern CPUs. The outermost -1 is needed because we halved the
// filter values.
- const int32x4_t horiz_const = vdupq_n_s32((1 << (bd + FILTER_BITS - 2)) +
- (1 << ((ROUND0_BITS - 1) - 1)));
- const int16x8_t vert_const = vdupq_n_s16(1 << (bd - 1));
+ const int32x4_t horiz_const = vdupq_n_s32(1 << ((ROUND0_BITS - 1) - 1));
const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul6PermuteTbl);
do {
@@ -1203,14 +1196,14 @@ static inline void convolve_2d_sr_6tap_neon_i8mm(const uint8_t *src,
int16x8_t v_s8 =
convolve6_8_2d_h(h_s8, x_filter, permute_tbl, horiz_const);
- uint8x8_t d0 = convolve6_8_2d_v(v_s0, v_s1, v_s2, v_s3, v_s4, v_s5,
- y_filter, vert_const);
- uint8x8_t d1 = convolve6_8_2d_v(v_s1, v_s2, v_s3, v_s4, v_s5, v_s6,
- y_filter, vert_const);
- uint8x8_t d2 = convolve6_8_2d_v(v_s2, v_s3, v_s4, v_s5, v_s6, v_s7,
- y_filter, vert_const);
- uint8x8_t d3 = convolve6_8_2d_v(v_s3, v_s4, v_s5, v_s6, v_s7, v_s8,
- y_filter, vert_const);
+ uint8x8_t d0 =
+ convolve6_8_2d_v(v_s0, v_s1, v_s2, v_s3, v_s4, v_s5, y_filter);
+ uint8x8_t d1 =
+ convolve6_8_2d_v(v_s1, v_s2, v_s3, v_s4, v_s5, v_s6, y_filter);
+ uint8x8_t d2 =
+ convolve6_8_2d_v(v_s2, v_s3, v_s4, v_s5, v_s6, v_s7, y_filter);
+ uint8x8_t d3 =
+ convolve6_8_2d_v(v_s3, v_s4, v_s5, v_s6, v_s7, v_s8, y_filter);
store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
@@ -1241,13 +1234,10 @@ static inline void convolve_2d_sr_6tap_4tap_neon_i8mm(
const int8x16_t x_filter =
vcombine_s8(vext_s8(x_filter_s8, x_filter_s8, 1), x_filter_s8);
- const int bd = 8;
// Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding
// shifts - which are generally faster than rounding shifts on modern CPUs.
// Halve the total because we halved the filter values.
- const int32x4_t horiz_const = vdupq_n_s32(
- ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1))) / 2);
- const int16x8_t vert_const = vdupq_n_s16(1 << (bd - 1));
+ const int32x4_t horiz_const = vdupq_n_s32((1 << (ROUND0_BITS - 1)) / 2);
if (w == 4) {
const uint8x16_t permute_tbl = vld1q_u8(kMatMul6PermuteTbl);
@@ -1278,8 +1268,8 @@ static inline void convolve_2d_sr_6tap_4tap_neon_i8mm(
int16x4_t d2 = convolve4_4_2d_v(v_s2, v_s3, v_s4, v_s5, y_filter);
int16x4_t d3 = convolve4_4_2d_v(v_s3, v_s4, v_s5, v_s6, y_filter);
- uint8x8_t d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), vert_const));
- uint8x8_t d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), vert_const));
+ uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1));
+ uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3));
store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d01);
store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d23);
@@ -1325,14 +1315,10 @@ static inline void convolve_2d_sr_6tap_4tap_neon_i8mm(
int16x8_t v_s6 =
convolve6_8_2d_h(h_s6, x_filter, permute_tbl, horiz_const);
- uint8x8_t d0 =
- convolve4_8_2d_v(v_s0, v_s1, v_s2, v_s3, y_filter, vert_const);
- uint8x8_t d1 =
- convolve4_8_2d_v(v_s1, v_s2, v_s3, v_s4, y_filter, vert_const);
- uint8x8_t d2 =
- convolve4_8_2d_v(v_s2, v_s3, v_s4, v_s5, y_filter, vert_const);
- uint8x8_t d3 =
- convolve4_8_2d_v(v_s3, v_s4, v_s5, v_s6, y_filter, vert_const);
+ uint8x8_t d0 = convolve4_8_2d_v(v_s0, v_s1, v_s2, v_s3, y_filter);
+ uint8x8_t d1 = convolve4_8_2d_v(v_s1, v_s2, v_s3, v_s4, y_filter);
+ uint8x8_t d2 = convolve4_8_2d_v(v_s2, v_s3, v_s4, v_s5, y_filter);
+ uint8x8_t d3 = convolve4_8_2d_v(v_s3, v_s4, v_s5, v_s6, y_filter);
store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
diff --git a/av1/common/arm/convolve_neon_i8mm.h b/av1/common/arm/convolve_neon_i8mm.h
index 38e651f363..57743a58df 100644
--- a/av1/common/arm/convolve_neon_i8mm.h
+++ b/av1/common/arm/convolve_neon_i8mm.h
@@ -78,8 +78,6 @@ static inline void convolve_2d_sr_horiz_12tap_neon_i8mm(
// The no-op filter should never be used here.
assert(x_filter_ptr[5] != 128);
- const int bd = 8;
-
// Split 12-tap filter into two 6-tap filters, masking the top two elements.
// { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0 }
const int8x8_t mask = vcreate_s8(0x0000ffffffffffff);
@@ -97,8 +95,7 @@ static inline void convolve_2d_sr_horiz_12tap_neon_i8mm(
// This shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding shifts
// in convolution kernels - which are generally faster than rounding shifts on
// modern CPUs.
- const int32x4_t horiz_const =
- vdupq_n_s32((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1)));
+ const int32x4_t horiz_const = vdupq_n_s32(1 << (ROUND0_BITS - 1));
if (w <= 4) {
const uint8x16_t permute_tbl = vld1q_u8(kMatMul6PermuteTbl);
diff --git a/av1/common/arm/convolve_sve2.c b/av1/common/arm/convolve_sve2.c
index ae003867e6..9e2eb48ee5 100644
--- a/av1/common/arm/convolve_sve2.c
+++ b/av1/common/arm/convolve_sve2.c
@@ -60,9 +60,6 @@ static inline void convolve_2d_sr_vert_12tap_sve2(
// The no-op filter should never be used here.
assert(vgetq_lane_s16(y_filter_0_7, 5) != 128);
- const int bd = 8;
- const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1));
-
uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kSVEDotProdMergeBlockTbl);
// Scale indices by size of the true vector length to avoid reading from an
// 'undefined' portion of a vector on a system with SVE vectors > 128-bit.
@@ -129,9 +126,6 @@ static inline void convolve_2d_sr_vert_12tap_sve2(
vcombine_s16(vqrshrn_n_s32(d2, 2 * FILTER_BITS - ROUND0_BITS),
vqrshrn_n_s32(d3, 2 * FILTER_BITS - ROUND0_BITS));
- dd01 = vsubq_s16(dd01, sub_const);
- dd23 = vsubq_s16(dd23, sub_const);
-
uint8x8_t d01 = vqmovun_s16(dd01);
uint8x8_t d23 = vqmovun_s16(dd23);