Commit 250cc23b8f for aom
commit 250cc23b8fcb9ea606514c947fe8ebc583b0b07a
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Nov 14 11:58:58 2026 -0800
av1,neon: Add cnn_activate/add_neon & opt cnn_neon
Add NEON specializations for av1_cnn_activate_neon (RELU and SOFTSIGN)
and av1_cnn_add_neon, and add 4-output-channel batched 2x2 and
element-wise convolution kernels in cnn_neon.c.
Speedup: 6.91x (cnn_add_neon), 5.04x (RELU), 1.93x (SOFTSIGN) in
CNNElemwiseTest and 1.38x (layer 1) / 1.34x (layer 2) in
CNNConvolveTest on Arm Neoverse N3; -0.09% (1T, min -0.25%) / -0.41%
(4T, min -0.49%) on Arm Neoverse N3 and min -0.42% (1T Cortex-A720) /
-5.77% (4T) on Pixel 9 Tensor G4 avifenc (speed 6, bit-exact).
TAG=agy
CONV=490488d4-7005-402a-8cc4-c245227f80fe
Change-Id: Ibf6e03909b211111111111111111111111111111
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index 2e0dda53e9..3a72d34090 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -549,7 +549,9 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
# CNN functions
if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
add_proto qw/void av1_cnn_activate/, "float **input, int channels, int width, int height, int stride, ACTIVATION layer_activation";
+ specialize qw/av1_cnn_activate neon/;
add_proto qw/void av1_cnn_add/, "float **input, int channels, int width, int height, int stride, const float **add";
+ specialize qw/av1_cnn_add neon/;
add_proto qw/bool av1_cnn_predict/, "const float **input, int in_width, int in_height, int in_stride, const CNN_CONFIG *cnn_config, const CNN_THREAD_DATA *thread_data, CNN_MULTI_OUT *output_struct";
add_proto qw/void av1_cnn_convolve_no_maxpool_padding_valid/, "const float **input, int in_width, int in_height, int in_stride, const CNN_LAYER_CONFIG *layer_config, float **output, int out_stride, int start_idx, int cstep, int channel_step";
if (aom_config("CONFIG_EXCLUDE_SIMD_MISMATCH") ne "yes") {
diff --git a/av1/encoder/arm/cnn_neon.c b/av1/encoder/arm/cnn_neon.c
index 5e24b6916e..22baa69938 100644
--- a/av1/encoder/arm/cnn_neon.c
+++ b/av1/encoder/arm/cnn_neon.c
@@ -774,6 +774,149 @@ static inline float32x4_t add_f32x4_x4(const float32x4_t a[4]) {
return vaddq_f32(sum01, sum23);
}
+#if AOM_ARCH_AARCH64
+static inline void cnn_convolve_2x2_4out_4w_neon(
+ const float **input, int in_width, int in_height, int in_stride,
+ const float *bias, int in_channels, float **output, int out_stride,
+ const float *weights) {
+ assert(in_channels >= 2 && (in_channels & 1) == 0);
+ assert(in_width >= 8 && (in_width & 7) == 0);
+ assert(in_height >= 2 && (in_height & 1) == 0);
+ const int in_size = in_height * in_width;
+ const int w_stride = in_channels * 4;
+ const float *weight_base0 = weights;
+ const float *weight_base1 = weights + w_stride;
+ const float *weight_base2 = weights + 2 * w_stride;
+ const float *weight_base3 = weights + 3 * w_stride;
+ const float32x4_t zero = vdupq_n_f32(0);
+ const float32x4_t bias_v0 = vdupq_n_f32(bias[0]);
+ const float32x4_t bias_v1 = vdupq_n_f32(bias[1]);
+ const float32x4_t bias_v2 = vdupq_n_f32(bias[2]);
+ const float32x4_t bias_v3 = vdupq_n_f32(bias[3]);
+ const float *in_ptr0 = *input;
+ float *out_ptr0_0 = output[0];
+ float *out_ptr0_1 = output[1];
+ float *out_ptr0_2 = output[2];
+ float *out_ptr0_3 = output[3];
+ int h = in_height;
+
+ do {
+ const float *in_ptr1 = in_ptr0;
+ float *out_ptr1_0 = out_ptr0_0;
+ float *out_ptr1_1 = out_ptr0_1;
+ float *out_ptr1_2 = out_ptr0_2;
+ float *out_ptr1_3 = out_ptr0_3;
+ int w = in_width;
+
+ do {
+ const float *in_ptr2 = in_ptr1;
+ const float *wp0 = weight_base0;
+ const float *wp1 = weight_base1;
+ const float *wp2 = weight_base2;
+ const float *wp3 = weight_base3;
+ int k = in_channels;
+ float32x4_t sum0[4] = { bias_v0, zero, zero, zero };
+ float32x4_t sum1[4] = { bias_v1, zero, zero, zero };
+ float32x4_t sum2[4] = { bias_v2, zero, zero, zero };
+ float32x4_t sum3[4] = { bias_v3, zero, zero, zero };
+
+ do {
+ const float32x4x2_t in0_lo = vld2q_f32(in_ptr2);
+ const float32x4x2_t in0_hi = vld2q_f32(in_ptr2 + in_stride);
+ const float32x4x2_t in1_lo = vld2q_f32(in_ptr2 + in_size);
+ const float32x4x2_t in1_hi = vld2q_f32(in_ptr2 + in_size + in_stride);
+
+ const float32x4_t w0_0 = vld1q_f32(wp0);
+ const float32x4_t w0_1 = vld1q_f32(wp0 + 4);
+ sum0[0] = vmlaq_lane_f32(sum0[0], in0_lo.val[0], vget_low_f32(w0_0), 0);
+ sum0[0] = vmlaq_lane_f32(sum0[0], in0_lo.val[1], vget_low_f32(w0_0), 1);
+ sum0[1] =
+ vmlaq_lane_f32(sum0[1], in0_hi.val[0], vget_high_f32(w0_0), 0);
+ sum0[1] =
+ vmlaq_lane_f32(sum0[1], in0_hi.val[1], vget_high_f32(w0_0), 1);
+ sum0[2] = vmlaq_lane_f32(sum0[2], in1_lo.val[0], vget_low_f32(w0_1), 0);
+ sum0[2] = vmlaq_lane_f32(sum0[2], in1_lo.val[1], vget_low_f32(w0_1), 1);
+ sum0[3] =
+ vmlaq_lane_f32(sum0[3], in1_hi.val[0], vget_high_f32(w0_1), 0);
+ sum0[3] =
+ vmlaq_lane_f32(sum0[3], in1_hi.val[1], vget_high_f32(w0_1), 1);
+
+ const float32x4_t w1_0 = vld1q_f32(wp1);
+ const float32x4_t w1_1 = vld1q_f32(wp1 + 4);
+ sum1[0] = vmlaq_lane_f32(sum1[0], in0_lo.val[0], vget_low_f32(w1_0), 0);
+ sum1[0] = vmlaq_lane_f32(sum1[0], in0_lo.val[1], vget_low_f32(w1_0), 1);
+ sum1[1] =
+ vmlaq_lane_f32(sum1[1], in0_hi.val[0], vget_high_f32(w1_0), 0);
+ sum1[1] =
+ vmlaq_lane_f32(sum1[1], in0_hi.val[1], vget_high_f32(w1_0), 1);
+ sum1[2] = vmlaq_lane_f32(sum1[2], in1_lo.val[0], vget_low_f32(w1_1), 0);
+ sum1[2] = vmlaq_lane_f32(sum1[2], in1_lo.val[1], vget_low_f32(w1_1), 1);
+ sum1[3] =
+ vmlaq_lane_f32(sum1[3], in1_hi.val[0], vget_high_f32(w1_1), 0);
+ sum1[3] =
+ vmlaq_lane_f32(sum1[3], in1_hi.val[1], vget_high_f32(w1_1), 1);
+
+ const float32x4_t w2_0 = vld1q_f32(wp2);
+ const float32x4_t w2_1 = vld1q_f32(wp2 + 4);
+ sum2[0] = vmlaq_lane_f32(sum2[0], in0_lo.val[0], vget_low_f32(w2_0), 0);
+ sum2[0] = vmlaq_lane_f32(sum2[0], in0_lo.val[1], vget_low_f32(w2_0), 1);
+ sum2[1] =
+ vmlaq_lane_f32(sum2[1], in0_hi.val[0], vget_high_f32(w2_0), 0);
+ sum2[1] =
+ vmlaq_lane_f32(sum2[1], in0_hi.val[1], vget_high_f32(w2_0), 1);
+ sum2[2] = vmlaq_lane_f32(sum2[2], in1_lo.val[0], vget_low_f32(w2_1), 0);
+ sum2[2] = vmlaq_lane_f32(sum2[2], in1_lo.val[1], vget_low_f32(w2_1), 1);
+ sum2[3] =
+ vmlaq_lane_f32(sum2[3], in1_hi.val[0], vget_high_f32(w2_1), 0);
+ sum2[3] =
+ vmlaq_lane_f32(sum2[3], in1_hi.val[1], vget_high_f32(w2_1), 1);
+
+ const float32x4_t w3_0 = vld1q_f32(wp3);
+ const float32x4_t w3_1 = vld1q_f32(wp3 + 4);
+ sum3[0] = vmlaq_lane_f32(sum3[0], in0_lo.val[0], vget_low_f32(w3_0), 0);
+ sum3[0] = vmlaq_lane_f32(sum3[0], in0_lo.val[1], vget_low_f32(w3_0), 1);
+ sum3[1] =
+ vmlaq_lane_f32(sum3[1], in0_hi.val[0], vget_high_f32(w3_0), 0);
+ sum3[1] =
+ vmlaq_lane_f32(sum3[1], in0_hi.val[1], vget_high_f32(w3_0), 1);
+ sum3[2] = vmlaq_lane_f32(sum3[2], in1_lo.val[0], vget_low_f32(w3_1), 0);
+ sum3[2] = vmlaq_lane_f32(sum3[2], in1_lo.val[1], vget_low_f32(w3_1), 1);
+ sum3[3] =
+ vmlaq_lane_f32(sum3[3], in1_hi.val[0], vget_high_f32(w3_1), 0);
+ sum3[3] =
+ vmlaq_lane_f32(sum3[3], in1_hi.val[1], vget_high_f32(w3_1), 1);
+
+ wp0 += 8;
+ wp1 += 8;
+ wp2 += 8;
+ wp3 += 8;
+ in_ptr2 += 2 * in_size;
+ k -= 2;
+ } while (k != 0);
+
+ vst1q_f32(out_ptr1_0, add_f32x4_x4(sum0));
+ vst1q_f32(out_ptr1_1, add_f32x4_x4(sum1));
+ vst1q_f32(out_ptr1_2, add_f32x4_x4(sum2));
+ vst1q_f32(out_ptr1_3, add_f32x4_x4(sum3));
+
+ out_ptr1_0 += 4;
+ out_ptr1_1 += 4;
+ out_ptr1_2 += 4;
+ out_ptr1_3 += 4;
+ in_ptr1 += 8;
+ w -= 8;
+ } while (w != 0);
+
+ out_ptr0_0 += out_stride;
+ out_ptr0_1 += out_stride;
+ out_ptr0_2 += out_stride;
+ out_ptr0_3 += out_stride;
+ in_ptr0 += 2 * in_stride;
+ h -= 2;
+ } while (h != 0);
+}
+#endif // AOM_ARCH_AARCH64
+
static inline void av1_cnn_convolve_no_maxpool_padding_valid_2x2_large_neon(
const float **input, int in_width, int in_height, int in_stride,
const float *bias, const int skip_width, const int skip_height,
@@ -785,6 +928,18 @@ static inline void av1_cnn_convolve_no_maxpool_padding_valid_2x2_large_neon(
assert(in_width >= 16);
const int in_size = in_height * in_width;
+#if AOM_ARCH_AARCH64
+ while (start_idx + 4 <= out_channels) {
+ cnn_convolve_2x2_4out_4w_neon(input, in_width, in_height, in_stride, bias,
+ in_channels, output, out_stride, weights);
+ bias += 4;
+ output += 4;
+ weights += 4 * in_channels * filter_height * filter_width;
+ start_idx += 4;
+ }
+ if (start_idx >= out_channels) return;
+#endif // AOM_ARCH_AARCH64
+
do {
const float32x4_t bias_v = vdupq_n_f32(bias[0]);
const float *weight_ptr0 = weights;
@@ -883,6 +1038,17 @@ static inline void av1_cnn_convolve_no_maxpool_padding_valid_2x2_neon(
assert(filter_height == 2 && filter_width == 2);
assert(skip_width == 2 && skip_height == 2);
assert(in_width == 8);
+#if AOM_ARCH_AARCH64
+ while (start_idx + 4 <= out_channels) {
+ cnn_convolve_2x2_4out_4w_neon(input, in_width, in_height, in_stride, bias,
+ in_channels, output, out_stride, weights);
+ bias += 4;
+ output += 4;
+ weights += 4 * in_channels * filter_height * filter_width;
+ start_idx += 4;
+ }
+ if (start_idx >= out_channels) return;
+#endif // AOM_ARCH_AARCH64
const int in_size = in_height * in_width;
do {
const float32x4_t bias_v = vdupq_n_f32(*bias);
@@ -1142,3 +1308,127 @@ void av1_cnn_convolve_no_maxpool_padding_valid_neon(
start_idx, cstep, channel_step);
}
}
+
+static inline void add_line_neon(float *dst, const float *src, int n) {
+ int j = 0;
+ for (; j + 16 <= n; j += 16) {
+ const float32x4_t d0 = vld1q_f32(dst + j);
+ const float32x4_t d1 = vld1q_f32(dst + j + 4);
+ const float32x4_t d2 = vld1q_f32(dst + j + 8);
+ const float32x4_t d3 = vld1q_f32(dst + j + 12);
+ const float32x4_t s0 = vld1q_f32(src + j);
+ const float32x4_t s1 = vld1q_f32(src + j + 4);
+ const float32x4_t s2 = vld1q_f32(src + j + 8);
+ const float32x4_t s3 = vld1q_f32(src + j + 12);
+ vst1q_f32(dst + j, vaddq_f32(d0, s0));
+ vst1q_f32(dst + j + 4, vaddq_f32(d1, s1));
+ vst1q_f32(dst + j + 8, vaddq_f32(d2, s2));
+ vst1q_f32(dst + j + 12, vaddq_f32(d3, s3));
+ }
+ for (; j + 4 <= n; j += 4) {
+ vst1q_f32(dst + j, vaddq_f32(vld1q_f32(dst + j), vld1q_f32(src + j)));
+ }
+ for (; j < n; ++j) dst[j] += src[j];
+}
+
+void av1_cnn_add_neon(float **output, int channels, int width, int height,
+ int stride, const float **add) {
+ if (stride == width) {
+ const int n = width * height;
+ for (int c = 0; c < channels; ++c) add_line_neon(output[c], add[c], n);
+ return;
+ }
+
+ for (int c = 0; c < channels; ++c) {
+ float *dst_row = output[c];
+ const float *src_row = add[c];
+ for (int i = 0; i < height; ++i) {
+ add_line_neon(dst_row, src_row, width);
+ dst_row += stride;
+ src_row += stride;
+ }
+ }
+}
+
+static inline void relu_line_neon(float *dst, int n) {
+ const float32x4_t v_zero = vdupq_n_f32(0.0f);
+ int j = 0;
+ for (; j + 16 <= n; j += 16) {
+ const float32x4_t x0 = vld1q_f32(dst + j);
+ const float32x4_t x1 = vld1q_f32(dst + j + 4);
+ const float32x4_t x2 = vld1q_f32(dst + j + 8);
+ const float32x4_t x3 = vld1q_f32(dst + j + 12);
+ const uint32x4_t m0 = vcltq_f32(x0, v_zero);
+ const uint32x4_t m1 = vcltq_f32(x1, v_zero);
+ const uint32x4_t m2 = vcltq_f32(x2, v_zero);
+ const uint32x4_t m3 = vcltq_f32(x3, v_zero);
+ vst1q_f32(dst + j, vbslq_f32(m0, v_zero, x0));
+ vst1q_f32(dst + j + 4, vbslq_f32(m1, v_zero, x1));
+ vst1q_f32(dst + j + 8, vbslq_f32(m2, v_zero, x2));
+ vst1q_f32(dst + j + 12, vbslq_f32(m3, v_zero, x3));
+ }
+ for (; j + 4 <= n; j += 4) {
+ const float32x4_t x = vld1q_f32(dst + j);
+ vst1q_f32(dst + j, vbslq_f32(vcltq_f32(x, v_zero), v_zero, x));
+ }
+ for (; j < n; ++j) {
+ if (dst[j] < 0) dst[j] = 0;
+ }
+}
+
+static inline void softsign_line_neon(float *dst, int n) {
+ int j = 0;
+#if AOM_ARCH_AARCH64
+ const float32x4_t v_one = vdupq_n_f32(1.0f);
+ for (; j + 8 <= n; j += 8) {
+ const float32x4_t x0 = vld1q_f32(dst + j);
+ const float32x4_t x1 = vld1q_f32(dst + j + 4);
+ const float32x4_t d0 = vaddq_f32(vabsq_f32(x0), v_one);
+ const float32x4_t d1 = vaddq_f32(vabsq_f32(x1), v_one);
+ vst1q_f32(dst + j, vdivq_f32(x0, d0));
+ vst1q_f32(dst + j + 4, vdivq_f32(x1, d1));
+ }
+ for (; j + 4 <= n; j += 4) {
+ const float32x4_t x = vld1q_f32(dst + j);
+ vst1q_f32(dst + j, vdivq_f32(x, vaddq_f32(vabsq_f32(x), v_one)));
+ }
+#endif // AOM_ARCH_AARCH64
+ for (; j < n; ++j) {
+ dst[j] = dst[j] / (fabsf(dst[j]) + 1.0f);
+ }
+}
+
+void av1_cnn_activate_neon(float **output, int channels, int width, int height,
+ int stride, ACTIVATION layer_activation) {
+ if (layer_activation == RELU) {
+ if (stride == width) {
+ const int n = width * height;
+ for (int c = 0; c < channels; ++c) relu_line_neon(output[c], n);
+ } else {
+ for (int c = 0; c < channels; ++c) {
+ float *row = output[c];
+ for (int i = 0; i < height; ++i) {
+ relu_line_neon(row, width);
+ row += stride;
+ }
+ }
+ }
+ } else if (layer_activation == SOFTSIGN) {
+ if (stride == width) {
+ const int n = width * height;
+ for (int c = 0; c < channels; ++c) softsign_line_neon(output[c], n);
+ } else {
+ for (int c = 0; c < channels; ++c) {
+ float *row = output[c];
+ for (int i = 0; i < height; ++i) {
+ softsign_line_neon(row, width);
+ row += stride;
+ }
+ }
+ }
+ } else if (layer_activation == SIGMOID) {
+ assert(0 && "Sigmoid has not been supported in CNN.");
+ } else if (layer_activation != NONE) {
+ assert(0 && "Unknown activation type");
+ }
+}
diff --git a/test/cnn_test.cc b/test/cnn_test.cc
index fdf71d07e5..6565c8eca1 100644
--- a/test/cnn_test.cc
+++ b/test/cnn_test.cc
@@ -13,6 +13,8 @@
#include <math.h>
#include <stdio.h>
+#include <vector>
+
#include "gtest/gtest.h"
#include "config/av1_rtcd.h"
@@ -22,6 +24,7 @@
#include "av1/encoder/partition_cnn_weights.h"
#include "test/acm_random.h"
#include "test/function_equivalence_test.h"
+#include "test/register_state_check.h"
#include "test/util.h"
#define SQR(x) ((x) * (x))
@@ -2656,6 +2659,76 @@ INSTANTIATE_TEST_SUITE_P(NEON, CNNConvolveTest,
::testing::Values(CNNConvolveTestFuncs(
&av1_cnn_convolve_no_maxpool_padding_valid_c,
&av1_cnn_convolve_no_maxpool_padding_valid_neon)));
-#endif
+
+TEST(NEON, CNNAdd) {
+ libaom_test::ACMRandom rnd(libaom_test::ACMRandom::DeterministicSeed());
+ // { width, height, stride }
+ static constexpr int kDims[][3] = {
+ { 1, 1, 1 }, { 2, 2, 2 }, { 3, 5, 5 }, { 7, 3, 7 }, { 8, 8, 8 },
+ { 13, 3, 16 }, { 16, 16, 16 }, { 17, 5, 24 }, { 65, 65, 68 }
+ };
+ constexpr int kChannels = 4;
+ for (const auto &d : kDims) {
+ const int w = d[0], h = d[1], stride = d[2];
+ std::vector<float> buf_ref(kChannels * h * stride);
+ std::vector<float> buf_mod(kChannels * h * stride);
+ std::vector<float> buf_add(kChannels * h * stride);
+ for (size_t i = 0; i < buf_ref.size(); ++i) {
+ buf_ref[i] = ((float)rnd.Rand31() - (1 << 30)) / (1u << 28);
+ buf_mod[i] = buf_ref[i];
+ buf_add[i] = ((float)rnd.Rand31() - (1 << 30)) / (1u << 28);
+ }
+ float *out_ref[kChannels], *out_mod[kChannels];
+ const float *add_ptrs[kChannels];
+ for (int c = 0; c < kChannels; ++c) {
+ out_ref[c] = buf_ref.data() + c * h * stride;
+ out_mod[c] = buf_mod.data() + c * h * stride;
+ add_ptrs[c] = buf_add.data() + c * h * stride;
+ }
+ av1_cnn_add_c(out_ref, kChannels, w, h, stride, add_ptrs);
+ API_REGISTER_STATE_CHECK(
+ av1_cnn_add_neon(out_mod, kChannels, w, h, stride, add_ptrs));
+ for (size_t i = 0; i < buf_ref.size(); ++i) {
+ ASSERT_EQ(buf_ref[i], buf_mod[i])
+ << "w=" << w << " h=" << h << " stride=" << stride << " i=" << i;
+ }
+ }
+}
+
+TEST(NEON, CNNActivate) {
+ libaom_test::ACMRandom rnd(libaom_test::ACMRandom::DeterministicSeed());
+ // { width, height, stride }
+ static constexpr int kDims[][3] = {
+ { 1, 1, 1 }, { 2, 2, 2 }, { 3, 5, 5 }, { 7, 3, 7 }, { 8, 8, 8 },
+ { 13, 3, 16 }, { 16, 16, 16 }, { 17, 5, 24 }, { 65, 65, 68 }
+ };
+ static constexpr ACTIVATION kActs[] = { NONE, RELU, SOFTSIGN };
+ constexpr int kChannels = 4;
+ for (const auto act : kActs) {
+ for (const auto &d : kDims) {
+ const int w = d[0], h = d[1], stride = d[2];
+ std::vector<float> buf_ref(kChannels * h * stride);
+ std::vector<float> buf_mod(kChannels * h * stride);
+ for (size_t i = 0; i < buf_ref.size(); ++i) {
+ buf_ref[i] = ((float)rnd.Rand31() - (1 << 30)) / (1u << 28);
+ buf_mod[i] = buf_ref[i];
+ }
+ float *out_ref[kChannels], *out_mod[kChannels];
+ for (int c = 0; c < kChannels; ++c) {
+ out_ref[c] = buf_ref.data() + c * h * stride;
+ out_mod[c] = buf_mod.data() + c * h * stride;
+ }
+ av1_cnn_activate_c(out_ref, kChannels, w, h, stride, act);
+ API_REGISTER_STATE_CHECK(
+ av1_cnn_activate_neon(out_mod, kChannels, w, h, stride, act));
+ for (size_t i = 0; i < buf_ref.size(); ++i) {
+ ASSERT_EQ(buf_ref[i], buf_mod[i])
+ << "w=" << w << " h=" << h << " stride=" << stride << " act=" << act
+ << " i=" << i;
+ }
+ }
+ }
+}
+#endif // HAVE_NEON
} // namespace