Commit 250cc23b8f for aom

commit 250cc23b8fcb9ea606514c947fe8ebc583b0b07a
Author: Frank Galligan <fgalligan@google.com>
Date:   Sat Nov 14 11:58:58 2026 -0800

    av1,neon: Add cnn_activate/add_neon & opt cnn_neon

    Add NEON specializations for av1_cnn_activate_neon (RELU and SOFTSIGN)
    and av1_cnn_add_neon, and add 4-output-channel batched 2x2 and
    element-wise convolution kernels in cnn_neon.c.

    Speedup: 6.91x (cnn_add_neon), 5.04x (RELU), 1.93x (SOFTSIGN) in
    CNNElemwiseTest and 1.38x (layer 1) / 1.34x (layer 2) in
    CNNConvolveTest on Arm Neoverse N3; -0.09% (1T, min -0.25%) / -0.41%
    (4T, min -0.49%) on Arm Neoverse N3 and min -0.42% (1T Cortex-A720) /
    -5.77% (4T) on Pixel 9 Tensor G4 avifenc (speed 6, bit-exact).

    TAG=agy
    CONV=490488d4-7005-402a-8cc4-c245227f80fe
    Change-Id: Ibf6e03909b211111111111111111111111111111

diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index 2e0dda53e9..3a72d34090 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -549,7 +549,9 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
   # CNN functions
   if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
     add_proto qw/void av1_cnn_activate/, "float **input, int channels, int width, int height, int stride, ACTIVATION layer_activation";
+    specialize qw/av1_cnn_activate neon/;
     add_proto qw/void av1_cnn_add/, "float **input, int channels, int width, int height, int stride, const float **add";
+    specialize qw/av1_cnn_add neon/;
     add_proto qw/bool av1_cnn_predict/, "const float **input, int in_width, int in_height, int in_stride, const CNN_CONFIG *cnn_config, const CNN_THREAD_DATA *thread_data, CNN_MULTI_OUT *output_struct";
     add_proto qw/void av1_cnn_convolve_no_maxpool_padding_valid/, "const float **input, int in_width, int in_height, int in_stride, const CNN_LAYER_CONFIG *layer_config, float **output, int out_stride, int start_idx, int cstep, int channel_step";
     if (aom_config("CONFIG_EXCLUDE_SIMD_MISMATCH") ne "yes") {
diff --git a/av1/encoder/arm/cnn_neon.c b/av1/encoder/arm/cnn_neon.c
index 5e24b6916e..22baa69938 100644
--- a/av1/encoder/arm/cnn_neon.c
+++ b/av1/encoder/arm/cnn_neon.c
@@ -774,6 +774,149 @@ static inline float32x4_t add_f32x4_x4(const float32x4_t a[4]) {
   return vaddq_f32(sum01, sum23);
 }

+#if AOM_ARCH_AARCH64
+static inline void cnn_convolve_2x2_4out_4w_neon(
+    const float **input, int in_width, int in_height, int in_stride,
+    const float *bias, int in_channels, float **output, int out_stride,
+    const float *weights) {
+  assert(in_channels >= 2 && (in_channels & 1) == 0);
+  assert(in_width >= 8 && (in_width & 7) == 0);
+  assert(in_height >= 2 && (in_height & 1) == 0);
+  const int in_size = in_height * in_width;
+  const int w_stride = in_channels * 4;
+  const float *weight_base0 = weights;
+  const float *weight_base1 = weights + w_stride;
+  const float *weight_base2 = weights + 2 * w_stride;
+  const float *weight_base3 = weights + 3 * w_stride;
+  const float32x4_t zero = vdupq_n_f32(0);
+  const float32x4_t bias_v0 = vdupq_n_f32(bias[0]);
+  const float32x4_t bias_v1 = vdupq_n_f32(bias[1]);
+  const float32x4_t bias_v2 = vdupq_n_f32(bias[2]);
+  const float32x4_t bias_v3 = vdupq_n_f32(bias[3]);
+  const float *in_ptr0 = *input;
+  float *out_ptr0_0 = output[0];
+  float *out_ptr0_1 = output[1];
+  float *out_ptr0_2 = output[2];
+  float *out_ptr0_3 = output[3];
+  int h = in_height;
+
+  do {
+    const float *in_ptr1 = in_ptr0;
+    float *out_ptr1_0 = out_ptr0_0;
+    float *out_ptr1_1 = out_ptr0_1;
+    float *out_ptr1_2 = out_ptr0_2;
+    float *out_ptr1_3 = out_ptr0_3;
+    int w = in_width;
+
+    do {
+      const float *in_ptr2 = in_ptr1;
+      const float *wp0 = weight_base0;
+      const float *wp1 = weight_base1;
+      const float *wp2 = weight_base2;
+      const float *wp3 = weight_base3;
+      int k = in_channels;
+      float32x4_t sum0[4] = { bias_v0, zero, zero, zero };
+      float32x4_t sum1[4] = { bias_v1, zero, zero, zero };
+      float32x4_t sum2[4] = { bias_v2, zero, zero, zero };
+      float32x4_t sum3[4] = { bias_v3, zero, zero, zero };
+
+      do {
+        const float32x4x2_t in0_lo = vld2q_f32(in_ptr2);
+        const float32x4x2_t in0_hi = vld2q_f32(in_ptr2 + in_stride);
+        const float32x4x2_t in1_lo = vld2q_f32(in_ptr2 + in_size);
+        const float32x4x2_t in1_hi = vld2q_f32(in_ptr2 + in_size + in_stride);
+
+        const float32x4_t w0_0 = vld1q_f32(wp0);
+        const float32x4_t w0_1 = vld1q_f32(wp0 + 4);
+        sum0[0] = vmlaq_lane_f32(sum0[0], in0_lo.val[0], vget_low_f32(w0_0), 0);
+        sum0[0] = vmlaq_lane_f32(sum0[0], in0_lo.val[1], vget_low_f32(w0_0), 1);
+        sum0[1] =
+            vmlaq_lane_f32(sum0[1], in0_hi.val[0], vget_high_f32(w0_0), 0);
+        sum0[1] =
+            vmlaq_lane_f32(sum0[1], in0_hi.val[1], vget_high_f32(w0_0), 1);
+        sum0[2] = vmlaq_lane_f32(sum0[2], in1_lo.val[0], vget_low_f32(w0_1), 0);
+        sum0[2] = vmlaq_lane_f32(sum0[2], in1_lo.val[1], vget_low_f32(w0_1), 1);
+        sum0[3] =
+            vmlaq_lane_f32(sum0[3], in1_hi.val[0], vget_high_f32(w0_1), 0);
+        sum0[3] =
+            vmlaq_lane_f32(sum0[3], in1_hi.val[1], vget_high_f32(w0_1), 1);
+
+        const float32x4_t w1_0 = vld1q_f32(wp1);
+        const float32x4_t w1_1 = vld1q_f32(wp1 + 4);
+        sum1[0] = vmlaq_lane_f32(sum1[0], in0_lo.val[0], vget_low_f32(w1_0), 0);
+        sum1[0] = vmlaq_lane_f32(sum1[0], in0_lo.val[1], vget_low_f32(w1_0), 1);
+        sum1[1] =
+            vmlaq_lane_f32(sum1[1], in0_hi.val[0], vget_high_f32(w1_0), 0);
+        sum1[1] =
+            vmlaq_lane_f32(sum1[1], in0_hi.val[1], vget_high_f32(w1_0), 1);
+        sum1[2] = vmlaq_lane_f32(sum1[2], in1_lo.val[0], vget_low_f32(w1_1), 0);
+        sum1[2] = vmlaq_lane_f32(sum1[2], in1_lo.val[1], vget_low_f32(w1_1), 1);
+        sum1[3] =
+            vmlaq_lane_f32(sum1[3], in1_hi.val[0], vget_high_f32(w1_1), 0);
+        sum1[3] =
+            vmlaq_lane_f32(sum1[3], in1_hi.val[1], vget_high_f32(w1_1), 1);
+
+        const float32x4_t w2_0 = vld1q_f32(wp2);
+        const float32x4_t w2_1 = vld1q_f32(wp2 + 4);
+        sum2[0] = vmlaq_lane_f32(sum2[0], in0_lo.val[0], vget_low_f32(w2_0), 0);
+        sum2[0] = vmlaq_lane_f32(sum2[0], in0_lo.val[1], vget_low_f32(w2_0), 1);
+        sum2[1] =
+            vmlaq_lane_f32(sum2[1], in0_hi.val[0], vget_high_f32(w2_0), 0);
+        sum2[1] =
+            vmlaq_lane_f32(sum2[1], in0_hi.val[1], vget_high_f32(w2_0), 1);
+        sum2[2] = vmlaq_lane_f32(sum2[2], in1_lo.val[0], vget_low_f32(w2_1), 0);
+        sum2[2] = vmlaq_lane_f32(sum2[2], in1_lo.val[1], vget_low_f32(w2_1), 1);
+        sum2[3] =
+            vmlaq_lane_f32(sum2[3], in1_hi.val[0], vget_high_f32(w2_1), 0);
+        sum2[3] =
+            vmlaq_lane_f32(sum2[3], in1_hi.val[1], vget_high_f32(w2_1), 1);
+
+        const float32x4_t w3_0 = vld1q_f32(wp3);
+        const float32x4_t w3_1 = vld1q_f32(wp3 + 4);
+        sum3[0] = vmlaq_lane_f32(sum3[0], in0_lo.val[0], vget_low_f32(w3_0), 0);
+        sum3[0] = vmlaq_lane_f32(sum3[0], in0_lo.val[1], vget_low_f32(w3_0), 1);
+        sum3[1] =
+            vmlaq_lane_f32(sum3[1], in0_hi.val[0], vget_high_f32(w3_0), 0);
+        sum3[1] =
+            vmlaq_lane_f32(sum3[1], in0_hi.val[1], vget_high_f32(w3_0), 1);
+        sum3[2] = vmlaq_lane_f32(sum3[2], in1_lo.val[0], vget_low_f32(w3_1), 0);
+        sum3[2] = vmlaq_lane_f32(sum3[2], in1_lo.val[1], vget_low_f32(w3_1), 1);
+        sum3[3] =
+            vmlaq_lane_f32(sum3[3], in1_hi.val[0], vget_high_f32(w3_1), 0);
+        sum3[3] =
+            vmlaq_lane_f32(sum3[3], in1_hi.val[1], vget_high_f32(w3_1), 1);
+
+        wp0 += 8;
+        wp1 += 8;
+        wp2 += 8;
+        wp3 += 8;
+        in_ptr2 += 2 * in_size;
+        k -= 2;
+      } while (k != 0);
+
+      vst1q_f32(out_ptr1_0, add_f32x4_x4(sum0));
+      vst1q_f32(out_ptr1_1, add_f32x4_x4(sum1));
+      vst1q_f32(out_ptr1_2, add_f32x4_x4(sum2));
+      vst1q_f32(out_ptr1_3, add_f32x4_x4(sum3));
+
+      out_ptr1_0 += 4;
+      out_ptr1_1 += 4;
+      out_ptr1_2 += 4;
+      out_ptr1_3 += 4;
+      in_ptr1 += 8;
+      w -= 8;
+    } while (w != 0);
+
+    out_ptr0_0 += out_stride;
+    out_ptr0_1 += out_stride;
+    out_ptr0_2 += out_stride;
+    out_ptr0_3 += out_stride;
+    in_ptr0 += 2 * in_stride;
+    h -= 2;
+  } while (h != 0);
+}
+#endif  // AOM_ARCH_AARCH64
+
 static inline void av1_cnn_convolve_no_maxpool_padding_valid_2x2_large_neon(
     const float **input, int in_width, int in_height, int in_stride,
     const float *bias, const int skip_width, const int skip_height,
@@ -785,6 +928,18 @@ static inline void av1_cnn_convolve_no_maxpool_padding_valid_2x2_large_neon(
   assert(in_width >= 16);
   const int in_size = in_height * in_width;

+#if AOM_ARCH_AARCH64
+  while (start_idx + 4 <= out_channels) {
+    cnn_convolve_2x2_4out_4w_neon(input, in_width, in_height, in_stride, bias,
+                                  in_channels, output, out_stride, weights);
+    bias += 4;
+    output += 4;
+    weights += 4 * in_channels * filter_height * filter_width;
+    start_idx += 4;
+  }
+  if (start_idx >= out_channels) return;
+#endif  // AOM_ARCH_AARCH64
+
   do {
     const float32x4_t bias_v = vdupq_n_f32(bias[0]);
     const float *weight_ptr0 = weights;
@@ -883,6 +1038,17 @@ static inline void av1_cnn_convolve_no_maxpool_padding_valid_2x2_neon(
   assert(filter_height == 2 && filter_width == 2);
   assert(skip_width == 2 && skip_height == 2);
   assert(in_width == 8);
+#if AOM_ARCH_AARCH64
+  while (start_idx + 4 <= out_channels) {
+    cnn_convolve_2x2_4out_4w_neon(input, in_width, in_height, in_stride, bias,
+                                  in_channels, output, out_stride, weights);
+    bias += 4;
+    output += 4;
+    weights += 4 * in_channels * filter_height * filter_width;
+    start_idx += 4;
+  }
+  if (start_idx >= out_channels) return;
+#endif  // AOM_ARCH_AARCH64
   const int in_size = in_height * in_width;
   do {
     const float32x4_t bias_v = vdupq_n_f32(*bias);
@@ -1142,3 +1308,127 @@ void av1_cnn_convolve_no_maxpool_padding_valid_neon(
         start_idx, cstep, channel_step);
   }
 }
+
+static inline void add_line_neon(float *dst, const float *src, int n) {
+  int j = 0;
+  for (; j + 16 <= n; j += 16) {
+    const float32x4_t d0 = vld1q_f32(dst + j);
+    const float32x4_t d1 = vld1q_f32(dst + j + 4);
+    const float32x4_t d2 = vld1q_f32(dst + j + 8);
+    const float32x4_t d3 = vld1q_f32(dst + j + 12);
+    const float32x4_t s0 = vld1q_f32(src + j);
+    const float32x4_t s1 = vld1q_f32(src + j + 4);
+    const float32x4_t s2 = vld1q_f32(src + j + 8);
+    const float32x4_t s3 = vld1q_f32(src + j + 12);
+    vst1q_f32(dst + j, vaddq_f32(d0, s0));
+    vst1q_f32(dst + j + 4, vaddq_f32(d1, s1));
+    vst1q_f32(dst + j + 8, vaddq_f32(d2, s2));
+    vst1q_f32(dst + j + 12, vaddq_f32(d3, s3));
+  }
+  for (; j + 4 <= n; j += 4) {
+    vst1q_f32(dst + j, vaddq_f32(vld1q_f32(dst + j), vld1q_f32(src + j)));
+  }
+  for (; j < n; ++j) dst[j] += src[j];
+}
+
+void av1_cnn_add_neon(float **output, int channels, int width, int height,
+                      int stride, const float **add) {
+  if (stride == width) {
+    const int n = width * height;
+    for (int c = 0; c < channels; ++c) add_line_neon(output[c], add[c], n);
+    return;
+  }
+
+  for (int c = 0; c < channels; ++c) {
+    float *dst_row = output[c];
+    const float *src_row = add[c];
+    for (int i = 0; i < height; ++i) {
+      add_line_neon(dst_row, src_row, width);
+      dst_row += stride;
+      src_row += stride;
+    }
+  }
+}
+
+static inline void relu_line_neon(float *dst, int n) {
+  const float32x4_t v_zero = vdupq_n_f32(0.0f);
+  int j = 0;
+  for (; j + 16 <= n; j += 16) {
+    const float32x4_t x0 = vld1q_f32(dst + j);
+    const float32x4_t x1 = vld1q_f32(dst + j + 4);
+    const float32x4_t x2 = vld1q_f32(dst + j + 8);
+    const float32x4_t x3 = vld1q_f32(dst + j + 12);
+    const uint32x4_t m0 = vcltq_f32(x0, v_zero);
+    const uint32x4_t m1 = vcltq_f32(x1, v_zero);
+    const uint32x4_t m2 = vcltq_f32(x2, v_zero);
+    const uint32x4_t m3 = vcltq_f32(x3, v_zero);
+    vst1q_f32(dst + j, vbslq_f32(m0, v_zero, x0));
+    vst1q_f32(dst + j + 4, vbslq_f32(m1, v_zero, x1));
+    vst1q_f32(dst + j + 8, vbslq_f32(m2, v_zero, x2));
+    vst1q_f32(dst + j + 12, vbslq_f32(m3, v_zero, x3));
+  }
+  for (; j + 4 <= n; j += 4) {
+    const float32x4_t x = vld1q_f32(dst + j);
+    vst1q_f32(dst + j, vbslq_f32(vcltq_f32(x, v_zero), v_zero, x));
+  }
+  for (; j < n; ++j) {
+    if (dst[j] < 0) dst[j] = 0;
+  }
+}
+
+static inline void softsign_line_neon(float *dst, int n) {
+  int j = 0;
+#if AOM_ARCH_AARCH64
+  const float32x4_t v_one = vdupq_n_f32(1.0f);
+  for (; j + 8 <= n; j += 8) {
+    const float32x4_t x0 = vld1q_f32(dst + j);
+    const float32x4_t x1 = vld1q_f32(dst + j + 4);
+    const float32x4_t d0 = vaddq_f32(vabsq_f32(x0), v_one);
+    const float32x4_t d1 = vaddq_f32(vabsq_f32(x1), v_one);
+    vst1q_f32(dst + j, vdivq_f32(x0, d0));
+    vst1q_f32(dst + j + 4, vdivq_f32(x1, d1));
+  }
+  for (; j + 4 <= n; j += 4) {
+    const float32x4_t x = vld1q_f32(dst + j);
+    vst1q_f32(dst + j, vdivq_f32(x, vaddq_f32(vabsq_f32(x), v_one)));
+  }
+#endif  // AOM_ARCH_AARCH64
+  for (; j < n; ++j) {
+    dst[j] = dst[j] / (fabsf(dst[j]) + 1.0f);
+  }
+}
+
+void av1_cnn_activate_neon(float **output, int channels, int width, int height,
+                           int stride, ACTIVATION layer_activation) {
+  if (layer_activation == RELU) {
+    if (stride == width) {
+      const int n = width * height;
+      for (int c = 0; c < channels; ++c) relu_line_neon(output[c], n);
+    } else {
+      for (int c = 0; c < channels; ++c) {
+        float *row = output[c];
+        for (int i = 0; i < height; ++i) {
+          relu_line_neon(row, width);
+          row += stride;
+        }
+      }
+    }
+  } else if (layer_activation == SOFTSIGN) {
+    if (stride == width) {
+      const int n = width * height;
+      for (int c = 0; c < channels; ++c) softsign_line_neon(output[c], n);
+    } else {
+      for (int c = 0; c < channels; ++c) {
+        float *row = output[c];
+        for (int i = 0; i < height; ++i) {
+          softsign_line_neon(row, width);
+          row += stride;
+        }
+      }
+    }
+  } else if (layer_activation == SIGMOID) {
+    assert(0 && "Sigmoid has not been supported in CNN.");
+  } else if (layer_activation != NONE) {
+    assert(0 && "Unknown activation type");
+  }
+}
diff --git a/test/cnn_test.cc b/test/cnn_test.cc
index fdf71d07e5..6565c8eca1 100644
--- a/test/cnn_test.cc
+++ b/test/cnn_test.cc
@@ -13,6 +13,8 @@
 #include <math.h>
 #include <stdio.h>

+#include <vector>
+
 #include "gtest/gtest.h"

 #include "config/av1_rtcd.h"
@@ -22,6 +24,7 @@
 #include "av1/encoder/partition_cnn_weights.h"
 #include "test/acm_random.h"
 #include "test/function_equivalence_test.h"
+#include "test/register_state_check.h"
 #include "test/util.h"

 #define SQR(x) ((x) * (x))
@@ -2656,6 +2659,76 @@ INSTANTIATE_TEST_SUITE_P(NEON, CNNConvolveTest,
                          ::testing::Values(CNNConvolveTestFuncs(
                              &av1_cnn_convolve_no_maxpool_padding_valid_c,
                              &av1_cnn_convolve_no_maxpool_padding_valid_neon)));
-#endif
+
+TEST(NEON, CNNAdd) {
+  libaom_test::ACMRandom rnd(libaom_test::ACMRandom::DeterministicSeed());
+  // { width, height, stride }
+  static constexpr int kDims[][3] = {
+    { 1, 1, 1 },   { 2, 2, 2 },    { 3, 5, 5 },   { 7, 3, 7 },   { 8, 8, 8 },
+    { 13, 3, 16 }, { 16, 16, 16 }, { 17, 5, 24 }, { 65, 65, 68 }
+  };
+  constexpr int kChannels = 4;
+  for (const auto &d : kDims) {
+    const int w = d[0], h = d[1], stride = d[2];
+    std::vector<float> buf_ref(kChannels * h * stride);
+    std::vector<float> buf_mod(kChannels * h * stride);
+    std::vector<float> buf_add(kChannels * h * stride);
+    for (size_t i = 0; i < buf_ref.size(); ++i) {
+      buf_ref[i] = ((float)rnd.Rand31() - (1 << 30)) / (1u << 28);
+      buf_mod[i] = buf_ref[i];
+      buf_add[i] = ((float)rnd.Rand31() - (1 << 30)) / (1u << 28);
+    }
+    float *out_ref[kChannels], *out_mod[kChannels];
+    const float *add_ptrs[kChannels];
+    for (int c = 0; c < kChannels; ++c) {
+      out_ref[c] = buf_ref.data() + c * h * stride;
+      out_mod[c] = buf_mod.data() + c * h * stride;
+      add_ptrs[c] = buf_add.data() + c * h * stride;
+    }
+    av1_cnn_add_c(out_ref, kChannels, w, h, stride, add_ptrs);
+    API_REGISTER_STATE_CHECK(
+        av1_cnn_add_neon(out_mod, kChannels, w, h, stride, add_ptrs));
+    for (size_t i = 0; i < buf_ref.size(); ++i) {
+      ASSERT_EQ(buf_ref[i], buf_mod[i])
+          << "w=" << w << " h=" << h << " stride=" << stride << " i=" << i;
+    }
+  }
+}
+
+TEST(NEON, CNNActivate) {
+  libaom_test::ACMRandom rnd(libaom_test::ACMRandom::DeterministicSeed());
+  // { width, height, stride }
+  static constexpr int kDims[][3] = {
+    { 1, 1, 1 },   { 2, 2, 2 },    { 3, 5, 5 },   { 7, 3, 7 },   { 8, 8, 8 },
+    { 13, 3, 16 }, { 16, 16, 16 }, { 17, 5, 24 }, { 65, 65, 68 }
+  };
+  static constexpr ACTIVATION kActs[] = { NONE, RELU, SOFTSIGN };
+  constexpr int kChannels = 4;
+  for (const auto act : kActs) {
+    for (const auto &d : kDims) {
+      const int w = d[0], h = d[1], stride = d[2];
+      std::vector<float> buf_ref(kChannels * h * stride);
+      std::vector<float> buf_mod(kChannels * h * stride);
+      for (size_t i = 0; i < buf_ref.size(); ++i) {
+        buf_ref[i] = ((float)rnd.Rand31() - (1 << 30)) / (1u << 28);
+        buf_mod[i] = buf_ref[i];
+      }
+      float *out_ref[kChannels], *out_mod[kChannels];
+      for (int c = 0; c < kChannels; ++c) {
+        out_ref[c] = buf_ref.data() + c * h * stride;
+        out_mod[c] = buf_mod.data() + c * h * stride;
+      }
+      av1_cnn_activate_c(out_ref, kChannels, w, h, stride, act);
+      API_REGISTER_STATE_CHECK(
+          av1_cnn_activate_neon(out_mod, kChannels, w, h, stride, act));
+      for (size_t i = 0; i < buf_ref.size(); ++i) {
+        ASSERT_EQ(buf_ref[i], buf_mod[i])
+            << "w=" << w << " h=" << h << " stride=" << stride << " act=" << act
+            << " i=" << i;
+      }
+    }
+  }
+}
+#endif  // HAVE_NEON

 }  // namespace