Commit a592c3fa1f for aom
commit a592c3fa1f810ae9b355be55a65d24099cb122ca
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Nov 14 11:33:08 2026 -0800
av1,neon: Optimize CfL luma subsampling
Replace vld4_u8, vld2q_u16, vld4q_u16, vst2q_u16, and vst4q_u16
interleaved load/store instructions in cfl_luma_subsampling_*_neon
with contiguous vld1q/vst1q and pairwise vpaddlq/vpadalq/vpaddq.
Speedup: 1.31x (-23.5% to -24.0%, 4x4 CfL subsampling) in test_libaom
on Neoverse N2; -0.41% (1T Cortex-X4) / -1.49% (1T Cortex-A720) on
Pixel 9 avifenc (bit-exact).
TAG=agy
CONV=490488d4-7005-402a-8cc4-c245227f80fe
Change-Id: I915685d790c9f9e755248b55d1c0c4b626c0c98e
diff --git a/av1/common/arm/cfl_neon.c b/av1/common/arm/cfl_neon.c
index e7f0ff094e..6b3c8d4339 100644
--- a/av1/common/arm/cfl_neon.c
+++ b/av1/common/arm/cfl_neon.c
@@ -61,20 +61,14 @@ static void cfl_luma_subsampling_420_lbd_neon(const uint8_t *input,
const uint16x8_t sum = vpadalq_u8(top, vld1q_u8(input + input_stride));
vst1q_u16(pred_buf_q3, vshlq_n_u16(sum, 1));
} else {
- const uint8x8x4_t top = vld4_u8(input);
- const uint8x8x4_t bot = vld4_u8(input + input_stride);
- // equivalent to a vpaddlq_u8 (because vld4q interleaves)
- const uint16x8_t top_0 = vaddl_u8(top.val[0], top.val[1]);
- // equivalent to a vpaddlq_u8 (because vld4q interleaves)
- const uint16x8_t bot_0 = vaddl_u8(bot.val[0], bot.val[1]);
- // equivalent to a vpaddlq_u8 (because vld4q interleaves)
- const uint16x8_t top_1 = vaddl_u8(top.val[2], top.val[3]);
- // equivalent to a vpaddlq_u8 (because vld4q interleaves)
- const uint16x8_t bot_1 = vaddl_u8(bot.val[2], bot.val[3]);
- uint16x8x2_t sum;
- sum.val[0] = vshlq_n_u16(vaddq_u16(top_0, bot_0), 1);
- sum.val[1] = vshlq_n_u16(vaddq_u16(top_1, bot_1), 1);
- vst2q_u16(pred_buf_q3, sum);
+ const uint16x8_t top_0 = vpaddlq_u8(vld1q_u8(input));
+ const uint16x8_t top_1 = vpaddlq_u8(vld1q_u8(input + 16));
+ const uint16x8_t sum_0 =
+ vpadalq_u8(top_0, vld1q_u8(input + input_stride));
+ const uint16x8_t sum_1 =
+ vpadalq_u8(top_1, vld1q_u8(input + 16 + input_stride));
+ vst1q_u16(pred_buf_q3, vshlq_n_u16(sum_0, 1));
+ vst1q_u16(pred_buf_q3 + 8, vshlq_n_u16(sum_1, 1));
}
input += luma_stride;
} while ((pred_buf_q3 += CFL_BUF_LINE) < end);
@@ -96,12 +90,10 @@ static void cfl_luma_subsampling_422_lbd_neon(const uint8_t *input,
const uint16x8_t top = vpaddlq_u8(vld1q_u8(input));
vst1q_u16(pred_buf_q3, vshlq_n_u16(top, 2));
} else {
- const uint8x8x4_t top = vld4_u8(input);
- uint16x8x2_t sum;
- // vaddl_u8 is equivalent to a vpaddlq_u8 (because vld4q interleaves)
- sum.val[0] = vshlq_n_u16(vaddl_u8(top.val[0], top.val[1]), 2);
- sum.val[1] = vshlq_n_u16(vaddl_u8(top.val[2], top.val[3]), 2);
- vst2q_u16(pred_buf_q3, sum);
+ const uint16x8_t top_0 = vpaddlq_u8(vld1q_u8(input));
+ const uint16x8_t top_1 = vpaddlq_u8(vld1q_u8(input + 16));
+ vst1q_u16(pred_buf_q3, vshlq_n_u16(top_0, 2));
+ vst1q_u16(pred_buf_q3 + 8, vshlq_n_u16(top_1, 2));
}
input += input_stride;
} while ((pred_buf_q3 += CFL_BUF_LINE) < end);
@@ -154,35 +146,25 @@ static void cfl_luma_subsampling_420_hbd_neon(const uint16_t *input,
const uint16x4_t sum = vadd_u16(top, bot);
const uint16x4_t hsum = vpadd_u16(sum, sum);
vsth_u16(pred_buf_q3, vshl_n_u16(hsum, 1));
- } else if (width < 32) {
+ } else if (width == 8) {
const uint16x8_t top = vld1q_u16(input);
const uint16x8_t bot = vld1q_u16(input + input_stride);
const uint16x8_t sum = vaddq_u16(top, bot);
- if (width == 8) {
- const uint16x4_t hsum = vget_low_u16(vpaddq_u16(sum, sum));
- vst1_u16(pred_buf_q3, vshl_n_u16(hsum, 1));
- } else {
- const uint16x8_t top_1 = vld1q_u16(input + 8);
- const uint16x8_t bot_1 = vld1q_u16(input + 8 + input_stride);
- const uint16x8_t sum_1 = vaddq_u16(top_1, bot_1);
- const uint16x8_t hsum = vpaddq_u16(sum, sum_1);
- vst1q_u16(pred_buf_q3, vshlq_n_u16(hsum, 1));
- }
+ const uint16x4_t hsum = vget_low_u16(vpaddq_u16(sum, sum));
+ vst1_u16(pred_buf_q3, vshl_n_u16(hsum, 1));
} else {
- const uint16x8x4_t top = vld4q_u16(input);
- const uint16x8x4_t bot = vld4q_u16(input + input_stride);
- // equivalent to a vpaddq_u16 (because vld4q interleaves)
- const uint16x8_t top_0 = vaddq_u16(top.val[0], top.val[1]);
- // equivalent to a vpaddq_u16 (because vld4q interleaves)
- const uint16x8_t bot_0 = vaddq_u16(bot.val[0], bot.val[1]);
- // equivalent to a vpaddq_u16 (because vld4q interleaves)
- const uint16x8_t top_1 = vaddq_u16(top.val[2], top.val[3]);
- // equivalent to a vpaddq_u16 (because vld4q interleaves)
- const uint16x8_t bot_1 = vaddq_u16(bot.val[2], bot.val[3]);
- uint16x8x2_t sum;
- sum.val[0] = vshlq_n_u16(vaddq_u16(top_0, bot_0), 1);
- sum.val[1] = vshlq_n_u16(vaddq_u16(top_1, bot_1), 1);
- vst2q_u16(pred_buf_q3, sum);
+ const uint16x8_t sum_0 =
+ vaddq_u16(vld1q_u16(input), vld1q_u16(input + input_stride));
+ const uint16x8_t sum_1 =
+ vaddq_u16(vld1q_u16(input + 8), vld1q_u16(input + 8 + input_stride));
+ vst1q_u16(pred_buf_q3, vshlq_n_u16(vpaddq_u16(sum_0, sum_1), 1));
+ if (width == 32) {
+ const uint16x8_t sum_2 = vaddq_u16(
+ vld1q_u16(input + 16), vld1q_u16(input + 16 + input_stride));
+ const uint16x8_t sum_3 = vaddq_u16(
+ vld1q_u16(input + 24), vld1q_u16(input + 24 + input_stride));
+ vst1q_u16(pred_buf_q3 + 8, vshlq_n_u16(vpaddq_u16(sum_2, sum_3), 1));
+ }
}
input += luma_stride;
} while ((pred_buf_q3 += CFL_BUF_LINE) < end);
@@ -199,24 +181,18 @@ static void cfl_luma_subsampling_422_hbd_neon(const uint16_t *input,
const uint16x4_t hsum = vpadd_u16(top, top);
vsth_u16(pred_buf_q3, vshl_n_u16(hsum, 2));
} else if (width == 8) {
- const uint16x4x2_t top = vld2_u16(input);
- // equivalent to a vpadd_u16 (because vld2 interleaves)
- const uint16x4_t hsum = vadd_u16(top.val[0], top.val[1]);
+ const uint16x8_t top = vld1q_u16(input);
+ const uint16x4_t hsum = vget_low_u16(vpaddq_u16(top, top));
vst1_u16(pred_buf_q3, vshl_n_u16(hsum, 2));
- } else if (width == 16) {
- const uint16x8x2_t top = vld2q_u16(input);
- // equivalent to a vpaddq_u16 (because vld2q interleaves)
- const uint16x8_t hsum = vaddq_u16(top.val[0], top.val[1]);
- vst1q_u16(pred_buf_q3, vshlq_n_u16(hsum, 2));
} else {
- const uint16x8x4_t top = vld4q_u16(input);
- // equivalent to a vpaddq_u16 (because vld4q interleaves)
- const uint16x8_t hsum_0 = vaddq_u16(top.val[0], top.val[1]);
- // equivalent to a vpaddq_u16 (because vld4q interleaves)
- const uint16x8_t hsum_1 = vaddq_u16(top.val[2], top.val[3]);
- uint16x8x2_t result = { { vshlq_n_u16(hsum_0, 2),
- vshlq_n_u16(hsum_1, 2) } };
- vst2q_u16(pred_buf_q3, result);
+ const uint16x8_t hsum_0 =
+ vpaddq_u16(vld1q_u16(input), vld1q_u16(input + 8));
+ vst1q_u16(pred_buf_q3, vshlq_n_u16(hsum_0, 2));
+ if (width == 32) {
+ const uint16x8_t hsum_1 =
+ vpaddq_u16(vld1q_u16(input + 16), vld1q_u16(input + 24));
+ vst1q_u16(pred_buf_q3 + 8, vshlq_n_u16(hsum_1, 2));
+ }
}
input += input_stride;
} while ((pred_buf_q3 += CFL_BUF_LINE) < end);
@@ -234,18 +210,13 @@ static void cfl_luma_subsampling_444_hbd_neon(const uint16_t *input,
} else if (width == 8) {
const uint16x8_t top = vld1q_u16(input);
vst1q_u16(pred_buf_q3, vshlq_n_u16(top, 3));
- } else if (width == 16) {
- uint16x8x2_t top = vld2q_u16(input);
- top.val[0] = vshlq_n_u16(top.val[0], 3);
- top.val[1] = vshlq_n_u16(top.val[1], 3);
- vst2q_u16(pred_buf_q3, top);
} else {
- uint16x8x4_t top = vld4q_u16(input);
- top.val[0] = vshlq_n_u16(top.val[0], 3);
- top.val[1] = vshlq_n_u16(top.val[1], 3);
- top.val[2] = vshlq_n_u16(top.val[2], 3);
- top.val[3] = vshlq_n_u16(top.val[3], 3);
- vst4q_u16(pred_buf_q3, top);
+ vst1q_u16(pred_buf_q3, vshlq_n_u16(vld1q_u16(input), 3));
+ vst1q_u16(pred_buf_q3 + 8, vshlq_n_u16(vld1q_u16(input + 8), 3));
+ if (width == 32) {
+ vst1q_u16(pred_buf_q3 + 16, vshlq_n_u16(vld1q_u16(input + 16), 3));
+ vst1q_u16(pred_buf_q3 + 24, vshlq_n_u16(vld1q_u16(input + 24), 3));
+ }
}
input += input_stride;
} while ((pred_buf_q3 += CFL_BUF_LINE) < end);