Commit d7c788287d for aom
commit d7c788287d9cc7127d0ebb26c11c4aec4b7b8042
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Nov 14 11:33:03 2026 -0800
av1,neon: Unroll block_width == 4 CDEF filter by 4
Unroll the block_width == 4 loops in cdef_filter_8_1_neon,
cdef_filter_8_2_neon, and cdef_filter_16_1_neon across 4 rows using
load_unaligned_u16_4x4 and store_u8x4_strided_x4.
Speedup: -0.71% to -2.11% in CDEFSpeedTest on Neoverse N2; -0.19% (1T)
/ -0.47% (4T) on AmpereOne and -0.24% (1T Cortex-X4) on Pixel 9
avifenc (bit-exact).
TAG=agy
CONV=490488d4-7005-402a-8cc4-c245227f80fe
Change-Id: I91253a3a8db722116484b5192e02109480b78d62
diff --git a/av1/common/arm/cdef_block_neon.c b/av1/common/arm/cdef_block_neon.c
index 4183144691..c6d2fe7b2a 100644
--- a/av1/common/arm/cdef_block_neon.c
+++ b/av1/common/arm/cdef_block_neon.c
@@ -775,32 +775,45 @@ void cdef_filter_8_1_neon(void *dest, int dstride, const uint16_t *in,
int h = block_height;
do {
- int16x8_t sum = vdupq_n_s16(0);
- uint16x8_t s = load_unaligned_u16_4x2(in, CDEF_BSTRIDE);
+ int16x8_t sum0 = vdupq_n_s16(0);
+ int16x8_t sum1 = vdupq_n_s16(0);
+ uint16x8_t s0, s1;
+ load_unaligned_u16_4x4(in, CDEF_BSTRIDE, &s0, &s1);
- uint16x8_t pri_src[4];
+ uint16x8_t pri_src0[4], pri_src1[4];
// Primary near taps
- pri_src[0] = load_unaligned_u16_4x2(in + po1, CDEF_BSTRIDE);
- pri_src[1] = load_unaligned_u16_4x2(in - po1, CDEF_BSTRIDE);
+ load_unaligned_u16_4x4(in + po1, CDEF_BSTRIDE, &pri_src0[0],
+ &pri_src1[0]);
+ load_unaligned_u16_4x4(in - po1, CDEF_BSTRIDE, &pri_src0[1],
+ &pri_src1[1]);
// Primary far taps
- pri_src[2] = load_unaligned_u16_4x2(in + po2, CDEF_BSTRIDE);
- pri_src[3] = load_unaligned_u16_4x2(in - po2, CDEF_BSTRIDE);
+ load_unaligned_u16_4x4(in + po2, CDEF_BSTRIDE, &pri_src0[2],
+ &pri_src1[2]);
+ load_unaligned_u16_4x4(in - po2, CDEF_BSTRIDE, &pri_src0[3],
+ &pri_src1[3]);
- primary_filter(s, pri_src, pri_taps, pri_strength, pri_damping, &sum);
+ primary_filter(s0, pri_src0, pri_taps, pri_strength, pri_damping, &sum0);
+ primary_filter(s1, pri_src1, pri_taps, pri_strength, pri_damping, &sum1);
// res = s + ((sum - (sum < 0) + 8) >> 4)
- sum =
- vaddq_s16(sum, vreinterpretq_s16_u16(vcltq_s16(sum, vdupq_n_s16(0))));
- const int16x8_t res_s16 = vrsraq_n_s16(vreinterpretq_s16_u16(s), sum, 4);
-
- const uint8x8_t res_u8 = vqmovun_s16(res_s16);
- store_u8x4_strided_x2(dst8, dstride, res_u8);
-
- in += 2 * CDEF_BSTRIDE;
- dst8 += 2 * dstride;
- h -= 2;
+ sum0 = vaddq_s16(sum0,
+ vreinterpretq_s16_u16(vcltq_s16(sum0, vdupq_n_s16(0))));
+ sum1 = vaddq_s16(sum1,
+ vreinterpretq_s16_u16(vcltq_s16(sum1, vdupq_n_s16(0))));
+ const int16x8_t res0_s16 =
+ vrsraq_n_s16(vreinterpretq_s16_u16(s0), sum0, 4);
+ const int16x8_t res1_s16 =
+ vrsraq_n_s16(vreinterpretq_s16_u16(s1), sum1, 4);
+
+ const uint8x16_t res_u8 =
+ vcombine_u8(vqmovun_s16(res0_s16), vqmovun_s16(res1_s16));
+ store_u8x4_strided_x4(dst8, dstride, res_u8);
+
+ in += 4 * CDEF_BSTRIDE;
+ dst8 += 4 * dstride;
+ h -= 4;
} while (h != 0);
}
}
@@ -863,36 +876,55 @@ void cdef_filter_8_2_neon(void *dest, int dstride, const uint16_t *in,
int h = block_height;
do {
- int16x8_t sum = vdupq_n_s16(0);
- uint16x8_t s = load_unaligned_u16_4x2(in, CDEF_BSTRIDE);
+ int16x8_t sum0 = vdupq_n_s16(0);
+ int16x8_t sum1 = vdupq_n_s16(0);
+ uint16x8_t s0, s1;
+ load_unaligned_u16_4x4(in, CDEF_BSTRIDE, &s0, &s1);
- uint16x8_t sec_src[8];
+ uint16x8_t sec_src0[8], sec_src1[8];
// Secondary near taps
- sec_src[0] = load_unaligned_u16_4x2(in + s1o1, CDEF_BSTRIDE);
- sec_src[1] = load_unaligned_u16_4x2(in - s1o1, CDEF_BSTRIDE);
- sec_src[2] = load_unaligned_u16_4x2(in + s2o1, CDEF_BSTRIDE);
- sec_src[3] = load_unaligned_u16_4x2(in - s2o1, CDEF_BSTRIDE);
+ load_unaligned_u16_4x4(in + s1o1, CDEF_BSTRIDE, &sec_src0[0],
+ &sec_src1[0]);
+ load_unaligned_u16_4x4(in - s1o1, CDEF_BSTRIDE, &sec_src0[1],
+ &sec_src1[1]);
+ load_unaligned_u16_4x4(in + s2o1, CDEF_BSTRIDE, &sec_src0[2],
+ &sec_src1[2]);
+ load_unaligned_u16_4x4(in - s2o1, CDEF_BSTRIDE, &sec_src0[3],
+ &sec_src1[3]);
// Secondary far taps
- sec_src[4] = load_unaligned_u16_4x2(in + s1o2, CDEF_BSTRIDE);
- sec_src[5] = load_unaligned_u16_4x2(in - s1o2, CDEF_BSTRIDE);
- sec_src[6] = load_unaligned_u16_4x2(in + s2o2, CDEF_BSTRIDE);
- sec_src[7] = load_unaligned_u16_4x2(in - s2o2, CDEF_BSTRIDE);
-
- secondary_filter(s, sec_src, sec_taps, sec_strength, sec_damping, &sum);
+ load_unaligned_u16_4x4(in + s1o2, CDEF_BSTRIDE, &sec_src0[4],
+ &sec_src1[4]);
+ load_unaligned_u16_4x4(in - s1o2, CDEF_BSTRIDE, &sec_src0[5],
+ &sec_src1[5]);
+ load_unaligned_u16_4x4(in + s2o2, CDEF_BSTRIDE, &sec_src0[6],
+ &sec_src1[6]);
+ load_unaligned_u16_4x4(in - s2o2, CDEF_BSTRIDE, &sec_src0[7],
+ &sec_src1[7]);
+
+ secondary_filter(s0, sec_src0, sec_taps, sec_strength, sec_damping,
+ &sum0);
+ secondary_filter(s1, sec_src1, sec_taps, sec_strength, sec_damping,
+ &sum1);
// res = s + ((sum - (sum < 0) + 8) >> 4)
- sum =
- vaddq_s16(sum, vreinterpretq_s16_u16(vcltq_s16(sum, vdupq_n_s16(0))));
- const int16x8_t res_s16 = vrsraq_n_s16(vreinterpretq_s16_u16(s), sum, 4);
-
- const uint8x8_t res_u8 = vqmovun_s16(res_s16);
- store_u8x4_strided_x2(dst8, dstride, res_u8);
-
- in += 2 * CDEF_BSTRIDE;
- dst8 += 2 * dstride;
- h -= 2;
+ sum0 = vaddq_s16(sum0,
+ vreinterpretq_s16_u16(vcltq_s16(sum0, vdupq_n_s16(0))));
+ sum1 = vaddq_s16(sum1,
+ vreinterpretq_s16_u16(vcltq_s16(sum1, vdupq_n_s16(0))));
+ const int16x8_t res0_s16 =
+ vrsraq_n_s16(vreinterpretq_s16_u16(s0), sum0, 4);
+ const int16x8_t res1_s16 =
+ vrsraq_n_s16(vreinterpretq_s16_u16(s1), sum1, 4);
+
+ const uint8x16_t res_u8 =
+ vcombine_u8(vqmovun_s16(res0_s16), vqmovun_s16(res1_s16));
+ store_u8x4_strided_x4(dst8, dstride, res_u8);
+
+ in += 4 * CDEF_BSTRIDE;
+ dst8 += 4 * dstride;
+ h -= 4;
} while (h != 0);
}
}
@@ -1200,31 +1232,43 @@ void cdef_filter_16_1_neon(void *dest, int dstride, const uint16_t *in,
int h = block_height;
do {
- int16x8_t sum = vdupq_n_s16(0);
- uint16x8_t s = load_unaligned_u16_4x2(in, CDEF_BSTRIDE);
+ int16x8_t sum0 = vdupq_n_s16(0);
+ int16x8_t sum1 = vdupq_n_s16(0);
+ uint16x8_t s0, s1;
+ load_unaligned_u16_4x4(in, CDEF_BSTRIDE, &s0, &s1);
- uint16x8_t pri_src[4];
+ uint16x8_t pri_src0[4], pri_src1[4];
// Primary near taps
- pri_src[0] = load_unaligned_u16_4x2(in + po1, CDEF_BSTRIDE);
- pri_src[1] = load_unaligned_u16_4x2(in - po1, CDEF_BSTRIDE);
+ load_unaligned_u16_4x4(in + po1, CDEF_BSTRIDE, &pri_src0[0],
+ &pri_src1[0]);
+ load_unaligned_u16_4x4(in - po1, CDEF_BSTRIDE, &pri_src0[1],
+ &pri_src1[1]);
// Primary far taps
- pri_src[2] = load_unaligned_u16_4x2(in + po2, CDEF_BSTRIDE);
- pri_src[3] = load_unaligned_u16_4x2(in - po2, CDEF_BSTRIDE);
+ load_unaligned_u16_4x4(in + po2, CDEF_BSTRIDE, &pri_src0[2],
+ &pri_src1[2]);
+ load_unaligned_u16_4x4(in - po2, CDEF_BSTRIDE, &pri_src0[3],
+ &pri_src1[3]);
- primary_filter(s, pri_src, pri_taps, pri_strength, pri_damping, &sum);
+ primary_filter(s0, pri_src0, pri_taps, pri_strength, pri_damping, &sum0);
+ primary_filter(s1, pri_src1, pri_taps, pri_strength, pri_damping, &sum1);
// res = s + ((sum - (sum < 0) + 8) >> 4)
- sum =
- vaddq_s16(sum, vreinterpretq_s16_u16(vcltq_s16(sum, vdupq_n_s16(0))));
- const int16x8_t res = vrsraq_n_s16(vreinterpretq_s16_u16(s), sum, 4);
-
- store_u16x4_strided_x2(dst16, dstride, vreinterpretq_u16_s16(res));
-
- in += 2 * CDEF_BSTRIDE;
- dst16 += 2 * dstride;
- h -= 2;
+ sum0 = vaddq_s16(sum0,
+ vreinterpretq_s16_u16(vcltq_s16(sum0, vdupq_n_s16(0))));
+ sum1 = vaddq_s16(sum1,
+ vreinterpretq_s16_u16(vcltq_s16(sum1, vdupq_n_s16(0))));
+ const int16x8_t res0 = vrsraq_n_s16(vreinterpretq_s16_u16(s0), sum0, 4);
+ const int16x8_t res1 = vrsraq_n_s16(vreinterpretq_s16_u16(s1), sum1, 4);
+
+ store_u16x4_strided_x2(dst16, dstride, vreinterpretq_u16_s16(res0));
+ store_u16x4_strided_x2(dst16 + 2 * dstride, dstride,
+ vreinterpretq_u16_s16(res1));
+
+ in += 4 * CDEF_BSTRIDE;
+ dst16 += 4 * dstride;
+ h -= 4;
} while (h != 0);
}
}