Commit 0aa7e16ff1 for aom

commit 0aa7e16ff1a25f816742322638b4b358677a07c3
Author: Frank Galligan <fgalligan@google.com>
Date:   Sat Nov 14 11:58:40 2026 -0800

    av1,neon: Add nn_fast_softmax_16_neon & opt ml

    Add av1_nn_fast_softmax_16_neon, add multi-output propagators
    (nn_propagate_xto4, nn_propagate_xto8) in ml_neon.c, and register
    av1_nn_fast_softmax_16_neon in FastSoftmaxTest.

    Speedup: 3.52x (-71.6%, av1_nn_fast_softmax_16_neon) and 1.46x-3.18x
    (-31.5% to -68.6%, av1_nn_predict_neon) in test_libaom on Neoverse N3;
    -0.66% (1T) / -0.89% (4T) on Neoverse N3 and -0.37% (1T Cortex-X4) /
    -0.26% (1T Cortex-A720) / -1.21% (4T) 8b avifenc on Pixel 9
    (bit-exact).

    TAG=agy
    CONV=490488d4-7005-402a-8cc4-c245227f80fe

    Change-Id: I360804b89dfbf1851ea4b96c38c1aaf2a14ea2fd

diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index d1cb4427a4..b9fa941a63 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -529,7 +529,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
   add_proto qw/void av1_nn_fast_softmax_16/, "const float *input_nodes, float *output";
   if (aom_config("CONFIG_EXCLUDE_SIMD_MISMATCH") ne "yes") {
     specialize qw/av1_nn_predict sse3 avx2 neon/;
-    specialize qw/av1_nn_fast_softmax_16 sse3/;
+    specialize qw/av1_nn_fast_softmax_16 sse3 neon/;
   }

   # CNN functions
diff --git a/av1/encoder/arm/ml_neon.c b/av1/encoder/arm/ml_neon.c
index 142e465e28..b25ef0f131 100644
--- a/av1/encoder/arm/ml_neon.c
+++ b/av1/encoder/arm/ml_neon.c
@@ -15,6 +15,7 @@

 #include "config/aom_config.h"
 #include "config/av1_rtcd.h"
+#include "aom_dsp/aom_dsp_common.h"
 #include "av1/encoder/ml.h"

 static void nn_activate8(float32x4_t *out_h, float32x4_t *out_l,
@@ -62,13 +63,14 @@ static void nn_propagate_8to1(int num_inputs, const float *const inputs,
 static void nn_propagate_xto1(int num_inputs, const float *const inputs,
                               const float *const weights,
                               const float *layer_bias,
-                              float *const output_nodes) {
+                              float *const output_nodes, bool output_layer) {
+  assert(num_inputs > 8 && (num_inputs & 3) != 0);
   float32x4_t vadd = vdupq_n_f32(0);

   float total = *layer_bias;
   int j = num_inputs;
   int in = 0;
-  while (j > 7) {
+  do {
     const float32x4_t inputs_h = vld1q_f32(&inputs[in + 4]);
     const float32x4_t inputs_l = vld1q_f32(&inputs[in]);

@@ -79,7 +81,7 @@ static void nn_propagate_xto1(int num_inputs, const float *const inputs,
     vadd = vmlaq_f32(vadd, inputs_l, weights_l);
     in += 8;
     j -= 8;
-  }
+  } while (j >= 8);

 #if AOM_ARCH_AARCH64
   total += vaddvq_f32(vadd);
@@ -89,15 +91,207 @@ static void nn_propagate_xto1(int num_inputs, const float *const inputs,
   vadd_lo = vpadd_f32(vadd_lo, vadd_lo);
   total += vget_lane_f32(vadd_lo, 0);
 #endif
-  for (; in < num_inputs; in++) total += weights[in] * inputs[in];
+  do {
+    total += weights[in] * inputs[in];
+    in++;
+  } while (in != num_inputs);
+
+  if (!output_layer) CLAMP_0(total);
+  *output_nodes = total;
+}
+
+static void nn_propagate_xto4(int num_inputs, const float *const inputs,
+                              const float *const weights,
+                              const float *layer_bias,
+                              float *const output_nodes, bool output_layer) {
+  assert(num_inputs > 8 && (num_inputs & 3) != 0);
+  const float32x4_t zero = vdupq_n_f32(0);
+  float32x4_t vadd0 = zero;
+  float32x4_t vadd1 = zero;
+  float32x4_t vadd2 = zero;
+  float32x4_t vadd3 = zero;
+  const float *const w0 = weights;
+  const float *const w1 = weights + num_inputs;
+  const float *const w2 = weights + 2 * num_inputs;
+  const float *const w3 = weights + 3 * num_inputs;
+  int j = num_inputs;
+  int in = 0;
+
+  do {
+    const float32x4_t inputs_h = vld1q_f32(&inputs[in + 4]);
+    const float32x4_t inputs_l = vld1q_f32(&inputs[in]);
+    const float32x4_t w0_h = vld1q_f32(&w0[in + 4]);
+    const float32x4_t w0_l = vld1q_f32(&w0[in]);
+    const float32x4_t w1_h = vld1q_f32(&w1[in + 4]);
+    const float32x4_t w1_l = vld1q_f32(&w1[in]);
+    const float32x4_t w2_h = vld1q_f32(&w2[in + 4]);
+    const float32x4_t w2_l = vld1q_f32(&w2[in]);
+    const float32x4_t w3_h = vld1q_f32(&w3[in + 4]);
+    const float32x4_t w3_l = vld1q_f32(&w3[in]);
+
+    vadd0 = vmlaq_f32(vadd0, inputs_h, w0_h);
+    vadd1 = vmlaq_f32(vadd1, inputs_h, w1_h);
+    vadd2 = vmlaq_f32(vadd2, inputs_h, w2_h);
+    vadd3 = vmlaq_f32(vadd3, inputs_h, w3_h);
+    vadd0 = vmlaq_f32(vadd0, inputs_l, w0_l);
+    vadd1 = vmlaq_f32(vadd1, inputs_l, w1_l);
+    vadd2 = vmlaq_f32(vadd2, inputs_l, w2_l);
+    vadd3 = vmlaq_f32(vadd3, inputs_l, w3_l);
+    in += 8;
+    j -= 8;
+  } while (j >= 8);
+
+  if (j > 3) {
+    const float32x4_t in_l = vld1q_f32(&inputs[in]);
+    vadd0 = vmlaq_f32(vadd0, in_l, vld1q_f32(&w0[in]));
+    vadd1 = vmlaq_f32(vadd1, in_l, vld1q_f32(&w1[in]));
+    vadd2 = vmlaq_f32(vadd2, in_l, vld1q_f32(&w2[in]));
+    vadd3 = vmlaq_f32(vadd3, in_l, vld1q_f32(&w3[in]));
+    in += 4;
+  }
+
+#if AOM_ARCH_AARCH64
+  const float32x4_t p0 = vpaddq_f32(vadd0, vadd1);
+  const float32x4_t p1 = vpaddq_f32(vadd2, vadd3);
+  float32x4_t outputs = vaddq_f32(vld1q_f32(layer_bias), vpaddq_f32(p0, p1));
+#else
+  const float32x4_t p0 =
+      vcombine_f32(vpadd_f32(vget_low_f32(vadd0), vget_high_f32(vadd0)),
+                   vpadd_f32(vget_low_f32(vadd1), vget_high_f32(vadd1)));
+  const float32x4_t p1 =
+      vcombine_f32(vpadd_f32(vget_low_f32(vadd2), vget_high_f32(vadd2)),
+                   vpadd_f32(vget_low_f32(vadd3), vget_high_f32(vadd3)));
+  const float32x4_t sum =
+      vcombine_f32(vpadd_f32(vget_low_f32(p0), vget_high_f32(p0)),
+                   vpadd_f32(vget_low_f32(p1), vget_high_f32(p1)));
+  float32x4_t outputs = vaddq_f32(vld1q_f32(layer_bias), sum);
+#endif

-  *output_nodes = CLAMP_0(total);
+  do {
+    float32x4_t w_tail = vld1q_dup_f32(&w0[in]);
+    w_tail = vld1q_lane_f32(&w1[in], w_tail, 1);
+    w_tail = vld1q_lane_f32(&w2[in], w_tail, 2);
+    w_tail = vld1q_lane_f32(&w3[in], w_tail, 3);
+    outputs = vmlaq_n_f32(outputs, w_tail, inputs[in]);
+    in++;
+  } while (in != num_inputs);
+
+  if (!output_layer) nn_activate4(&outputs, &zero);
+  vst1q_f32(output_nodes, outputs);
+}
+
+static void nn_propagate_xto8(int num_inputs, const float *const inputs,
+                              const float *const weights,
+                              const float *layer_bias,
+                              float *const output_nodes, bool output_layer) {
+  assert(num_inputs > 8 && (num_inputs & 3) != 0);
+  const float32x4_t zero = vdupq_n_f32(0);
+  float32x4_t vadd0 = zero;
+  float32x4_t vadd1 = zero;
+  float32x4_t vadd2 = zero;
+  float32x4_t vadd3 = zero;
+  float32x4_t vadd4 = zero;
+  float32x4_t vadd5 = zero;
+  float32x4_t vadd6 = zero;
+  float32x4_t vadd7 = zero;
+  const float *const w0 = weights;
+  const float *const w2 = weights + 2 * num_inputs;
+  const float *const w4 = weights + 4 * num_inputs;
+  const float *const w6 = weights + 6 * num_inputs;
+  int j = num_inputs;
+  int in = 0;
+
+  do {
+    const float32x4_t in_l = vld1q_f32(&inputs[in]);
+    const float32x4_t in_h = vld1q_f32(&inputs[in + 4]);
+    vadd0 = vmlaq_f32(vadd0, in_l, vld1q_f32(&w0[in]));
+    vadd1 = vmlaq_f32(vadd1, in_l, vld1q_f32(&w0[in + num_inputs]));
+    vadd2 = vmlaq_f32(vadd2, in_l, vld1q_f32(&w2[in]));
+    vadd3 = vmlaq_f32(vadd3, in_l, vld1q_f32(&w2[in + num_inputs]));
+    vadd4 = vmlaq_f32(vadd4, in_l, vld1q_f32(&w4[in]));
+    vadd5 = vmlaq_f32(vadd5, in_l, vld1q_f32(&w4[in + num_inputs]));
+    vadd6 = vmlaq_f32(vadd6, in_l, vld1q_f32(&w6[in]));
+    vadd7 = vmlaq_f32(vadd7, in_l, vld1q_f32(&w6[in + num_inputs]));
+    vadd0 = vmlaq_f32(vadd0, in_h, vld1q_f32(&w0[in + 4]));
+    vadd1 = vmlaq_f32(vadd1, in_h, vld1q_f32(&w0[in + num_inputs + 4]));
+    vadd2 = vmlaq_f32(vadd2, in_h, vld1q_f32(&w2[in + 4]));
+    vadd3 = vmlaq_f32(vadd3, in_h, vld1q_f32(&w2[in + num_inputs + 4]));
+    vadd4 = vmlaq_f32(vadd4, in_h, vld1q_f32(&w4[in + 4]));
+    vadd5 = vmlaq_f32(vadd5, in_h, vld1q_f32(&w4[in + num_inputs + 4]));
+    vadd6 = vmlaq_f32(vadd6, in_h, vld1q_f32(&w6[in + 4]));
+    vadd7 = vmlaq_f32(vadd7, in_h, vld1q_f32(&w6[in + num_inputs + 4]));
+    in += 8;
+    j -= 8;
+  } while (j >= 8);
+
+  if (j > 3) {
+    const float32x4_t in_l = vld1q_f32(&inputs[in]);
+    vadd0 = vmlaq_f32(vadd0, in_l, vld1q_f32(&w0[in]));
+    vadd1 = vmlaq_f32(vadd1, in_l, vld1q_f32(&w0[in + num_inputs]));
+    vadd2 = vmlaq_f32(vadd2, in_l, vld1q_f32(&w2[in]));
+    vadd3 = vmlaq_f32(vadd3, in_l, vld1q_f32(&w2[in + num_inputs]));
+    vadd4 = vmlaq_f32(vadd4, in_l, vld1q_f32(&w4[in]));
+    vadd5 = vmlaq_f32(vadd5, in_l, vld1q_f32(&w4[in + num_inputs]));
+    vadd6 = vmlaq_f32(vadd6, in_l, vld1q_f32(&w6[in]));
+    vadd7 = vmlaq_f32(vadd7, in_l, vld1q_f32(&w6[in + num_inputs]));
+    in += 4;
+  }
+
+#if AOM_ARCH_AARCH64
+  const float32x4_t p0 = vpaddq_f32(vadd0, vadd1);
+  const float32x4_t p1 = vpaddq_f32(vadd2, vadd3);
+  const float32x4_t p2 = vpaddq_f32(vadd4, vadd5);
+  const float32x4_t p3 = vpaddq_f32(vadd6, vadd7);
+  float32x4_t out_l = vaddq_f32(vld1q_f32(layer_bias), vpaddq_f32(p0, p1));
+  float32x4_t out_h = vaddq_f32(vld1q_f32(&layer_bias[4]), vpaddq_f32(p2, p3));
+#else
+  const float32x4_t p0 =
+      vcombine_f32(vpadd_f32(vget_low_f32(vadd0), vget_high_f32(vadd0)),
+                   vpadd_f32(vget_low_f32(vadd1), vget_high_f32(vadd1)));
+  const float32x4_t p1 =
+      vcombine_f32(vpadd_f32(vget_low_f32(vadd2), vget_high_f32(vadd2)),
+                   vpadd_f32(vget_low_f32(vadd3), vget_high_f32(vadd3)));
+  const float32x4_t p2 =
+      vcombine_f32(vpadd_f32(vget_low_f32(vadd4), vget_high_f32(vadd4)),
+                   vpadd_f32(vget_low_f32(vadd5), vget_high_f32(vadd5)));
+  const float32x4_t p3 =
+      vcombine_f32(vpadd_f32(vget_low_f32(vadd6), vget_high_f32(vadd6)),
+                   vpadd_f32(vget_low_f32(vadd7), vget_high_f32(vadd7)));
+  const float32x4_t sum_l =
+      vcombine_f32(vpadd_f32(vget_low_f32(p0), vget_high_f32(p0)),
+                   vpadd_f32(vget_low_f32(p1), vget_high_f32(p1)));
+  const float32x4_t sum_h =
+      vcombine_f32(vpadd_f32(vget_low_f32(p2), vget_high_f32(p2)),
+                   vpadd_f32(vget_low_f32(p3), vget_high_f32(p3)));
+  float32x4_t out_l = vaddq_f32(vld1q_f32(layer_bias), sum_l);
+  float32x4_t out_h = vaddq_f32(vld1q_f32(&layer_bias[4]), sum_h);
+#endif
+
+  do {
+    const float inp = inputs[in];
+    float32x4_t wl = vld1q_dup_f32(&w0[in]);
+    wl = vld1q_lane_f32(&w0[in + num_inputs], wl, 1);
+    wl = vld1q_lane_f32(&w2[in], wl, 2);
+    wl = vld1q_lane_f32(&w2[in + num_inputs], wl, 3);
+    float32x4_t wh = vld1q_dup_f32(&w4[in]);
+    wh = vld1q_lane_f32(&w4[in + num_inputs], wh, 1);
+    wh = vld1q_lane_f32(&w6[in], wh, 2);
+    wh = vld1q_lane_f32(&w6[in + num_inputs], wh, 3);
+    out_l = vmlaq_n_f32(out_l, wl, inp);
+    out_h = vmlaq_n_f32(out_h, wh, inp);
+    in++;
+  } while (in != num_inputs);
+
+  if (!output_layer) nn_activate8(&out_h, &out_l, &zero);
+  vst1q_f32(output_nodes, out_l);
+  vst1q_f32(&output_nodes[4], out_h);
 }

 static void nn_propagate_xsto1(int num_inputs, const float *const inputs,
                                const float *const weights,
                                const float *layer_bias,
-                               float *const output_nodes) {
+                               float *const output_nodes, bool output_layer) {
+  assert(num_inputs > 4 && num_inputs < 8);
   float total = *layer_bias;
 #if AOM_ARCH_AARCH64
   const float32x4_t v_inputs = vld1q_f32(inputs);
@@ -108,9 +302,57 @@ static void nn_propagate_xsto1(int num_inputs, const float *const inputs,
 #else
   int in = 0;
 #endif
-  for (; in < num_inputs; in++) total += weights[in] * inputs[in];
+  do {
+    total += weights[in] * inputs[in];
+    in++;
+  } while (in != num_inputs);

-  *output_nodes = CLAMP_0(total);
+  if (!output_layer) CLAMP_0(total);
+  *output_nodes = total;
+}
+
+static void nn_propagate_xsto4(int num_inputs, const float *const inputs,
+                               const float *const weights,
+                               const float *layer_bias,
+                               float *const output_nodes, bool output_layer) {
+  assert(num_inputs > 4 && num_inputs < 8);
+  const float32x4_t zero = vdupq_n_f32(0);
+  const float32x4_t v_inputs = vld1q_f32(inputs);
+  const float *const w0 = weights;
+  const float *const w1 = weights + num_inputs;
+  const float *const w2 = weights + 2 * num_inputs;
+  const float *const w3 = weights + 3 * num_inputs;
+  const float32x4_t mul0 = vmulq_f32(v_inputs, vld1q_f32(w0));
+  const float32x4_t mul1 = vmulq_f32(v_inputs, vld1q_f32(w1));
+  const float32x4_t mul2 = vmulq_f32(v_inputs, vld1q_f32(w2));
+  const float32x4_t mul3 = vmulq_f32(v_inputs, vld1q_f32(w3));
+#if AOM_ARCH_AARCH64
+  const float32x4_t p0 = vpaddq_f32(mul0, mul1);
+  const float32x4_t p1 = vpaddq_f32(mul2, mul3);
+  float32x4_t outputs = vaddq_f32(vld1q_f32(layer_bias), vpaddq_f32(p0, p1));
+#else
+  const float32x4_t p0 =
+      vcombine_f32(vpadd_f32(vget_low_f32(mul0), vget_high_f32(mul0)),
+                   vpadd_f32(vget_low_f32(mul1), vget_high_f32(mul1)));
+  const float32x4_t p1 =
+      vcombine_f32(vpadd_f32(vget_low_f32(mul2), vget_high_f32(mul2)),
+                   vpadd_f32(vget_low_f32(mul3), vget_high_f32(mul3)));
+  const float32x4_t sum =
+      vcombine_f32(vpadd_f32(vget_low_f32(p0), vget_high_f32(p0)),
+                   vpadd_f32(vget_low_f32(p1), vget_high_f32(p1)));
+  float32x4_t outputs = vaddq_f32(vld1q_f32(layer_bias), sum);
+#endif
+  int in = 4;
+  do {
+    float32x4_t w_tail = vld1q_dup_f32(&w0[in]);
+    w_tail = vld1q_lane_f32(&w1[in], w_tail, 1);
+    w_tail = vld1q_lane_f32(&w2[in], w_tail, 2);
+    w_tail = vld1q_lane_f32(&w3[in], w_tail, 3);
+    outputs = vmlaq_n_f32(outputs, w_tail, inputs[in]);
+    in++;
+  } while (in != num_inputs);
+  if (!output_layer) nn_activate4(&outputs, &zero);
+  vst1q_f32(output_nodes, outputs);
 }

 static void nn_propagate_4to1(int num_inputs, const float *const inputs,
@@ -187,16 +429,24 @@ static void nn_propagate_4to8(const int num_inputs, const float *const inputs,
   float32x4_t mul0[4] = { zero, zero, zero, zero };
   float32x4_t mul1[4] = { zero, zero, zero, zero };

-  for (int in = 0; in < num_inputs; in += 4) {
+  assert(num_inputs >= 4 && (num_inputs & 3) == 0);
+  const float *const w0 = weights;
+  const float *const w2 = weights + 2 * num_inputs;
+  const float *const w4 = weights + 4 * num_inputs;
+  const float *const w6 = weights + 6 * num_inputs;
+  int in = 0;
+  do {
     const float32x4_t v_input = vld1q_f32(&inputs[in]);
-    for (int i = 0; i < 4; i++) {
-      const float32x4_t weight0 = vld1q_f32(&weights[in + 2 * i * num_inputs]);
-      const float32x4_t weight1 =
-          vld1q_f32(&weights[in + (2 * i + 1) * num_inputs]);
-      mul0[i] = vmlaq_f32(mul0[i], v_input, weight0);
-      mul1[i] = vmlaq_f32(mul1[i], v_input, weight1);
-    }
-  }
+    mul0[0] = vmlaq_f32(mul0[0], v_input, vld1q_f32(&w0[in]));
+    mul1[0] = vmlaq_f32(mul1[0], v_input, vld1q_f32(&w0[in + num_inputs]));
+    mul0[1] = vmlaq_f32(mul0[1], v_input, vld1q_f32(&w2[in]));
+    mul1[1] = vmlaq_f32(mul1[1], v_input, vld1q_f32(&w2[in + num_inputs]));
+    mul0[2] = vmlaq_f32(mul0[2], v_input, vld1q_f32(&w4[in]));
+    mul1[2] = vmlaq_f32(mul1[2], v_input, vld1q_f32(&w4[in + num_inputs]));
+    mul0[3] = vmlaq_f32(mul0[3], v_input, vld1q_f32(&w6[in]));
+    mul1[3] = vmlaq_f32(mul1[3], v_input, vld1q_f32(&w6[in + num_inputs]));
+    in += 4;
+  } while (in != num_inputs);
   for (int i = 0; i < 4; i++)
 #if AOM_ARCH_AARCH64
     mul0[i] = vpaddq_f32(mul0[i], mul1[i]);
@@ -261,12 +511,104 @@ static void nn_propagate_8to4(const int num_inputs, const float *const inputs,
   vst1q_f32(output_nodes, outputs);
 }

-// Calculate prediction based on the given input features and neural net config.
-// Assume there are no more than NN_MAX_NODES_PER_LAYER nodes in each hidden
-// layer.
-void av1_nn_predict_neon(const float *input_nodes,
-                         const NN_CONFIG *const nn_config, int reduce_prec,
-                         float *const output) {
+static AOM_FORCE_INLINE void nn_predict_layer_neon(
+    const float *const input_nodes, const float *const layer_weights,
+    const float *const layer_bias, int num_inputs, int num_outputs,
+    bool output_layer, float *const output_nodes) {
+  assert(num_inputs > 0 && num_outputs > 0);
+  if (num_inputs % 4 == 0 && num_outputs % 8 == 0) {
+    int out = 0;
+    do {
+      nn_propagate_4to8(num_inputs, input_nodes,
+                        &layer_weights[out * num_inputs], &layer_bias[out],
+                        &output_nodes[out], output_layer);
+      out += 8;
+    } while (out != num_outputs);
+  } else if (num_inputs % 8 == 0 && num_outputs % 4 == 0) {
+    int out = 0;
+    do {
+      nn_propagate_8to4(num_inputs, input_nodes,
+                        &layer_weights[out * num_inputs], &layer_bias[out],
+                        &output_nodes[out], output_layer);
+      out += 4;
+    } while (out != num_outputs);
+  } else if (num_inputs % 4 == 0 && num_outputs % 4 == 0) {
+    int out = 0;
+    do {
+      nn_propagate_4to4(num_inputs, input_nodes,
+                        &layer_weights[out * num_inputs], &layer_bias[out],
+                        &output_nodes[out], output_layer);
+      out += 4;
+    } while (out != num_outputs);
+  } else if (num_inputs % 8 == 0) {
+    int out = 0;
+    do {
+      nn_propagate_8to1(num_inputs, input_nodes,
+                        &layer_weights[out * num_inputs], &layer_bias[out],
+                        &output_nodes[out], output_layer);
+      out++;
+    } while (out != num_outputs);
+  } else if (num_inputs % 4 == 0) {
+    int out = 0;
+    do {
+      nn_propagate_4to1(num_inputs, input_nodes,
+                        &layer_weights[out * num_inputs], &layer_bias[out],
+                        &output_nodes[out], output_layer);
+      out++;
+    } while (out != num_outputs);
+  } else if (num_inputs > 8) {
+    int out = 0;
+    for (; out + 8 <= num_outputs; out += 8) {
+      nn_propagate_xto8(num_inputs, input_nodes,
+                        &layer_weights[out * num_inputs], &layer_bias[out],
+                        &output_nodes[out], output_layer);
+    }
+    for (; out + 4 <= num_outputs; out += 4) {
+      nn_propagate_xto4(num_inputs, input_nodes,
+                        &layer_weights[out * num_inputs], &layer_bias[out],
+                        &output_nodes[out], output_layer);
+    }
+    for (; out < num_outputs; out++) {
+      nn_propagate_xto1(num_inputs, input_nodes,
+                        &layer_weights[out * num_inputs], &layer_bias[out],
+                        &output_nodes[out], output_layer);
+    }
+  } else if (num_inputs >= 4) {
+    int out = 0;
+    for (; out + 4 <= num_outputs; out += 4) {
+      nn_propagate_xsto4(num_inputs, input_nodes,
+                         &layer_weights[out * num_inputs], &layer_bias[out],
+                         &output_nodes[out], output_layer);
+    }
+    for (; out < num_outputs; out++) {
+      nn_propagate_xsto1(num_inputs, input_nodes,
+                         &layer_weights[out * num_inputs], &layer_bias[out],
+                         &output_nodes[out], output_layer);
+    }
+  } else {
+    int node = 0;
+    do {
+      float val = layer_bias[node];
+      int i = 0;
+      do {
+        val += layer_weights[node * num_inputs + i] * input_nodes[i];
+        ++i;
+      } while (i != num_inputs);
+      if (!output_layer) {
+        // ReLU as activation function.
+        val = val > 0.0f ? val : 0.0f;  // Could use AOMMAX().
+      }
+      output_nodes[node] = val;
+      ++node;
+    } while (node != num_outputs);
+  }
+}
+
+// Avoid inlining the multi-layer hidden-buffer stack frame into the
+// 0-hidden-layer fast path of av1_nn_predict_neon.
+static AOM_NOINLINE void nn_predict_hidden_neon(
+    const float *input_nodes, const NN_CONFIG *const nn_config,
+    float *const output) {
   float buf[2][NN_MAX_NODES_PER_LAYER];
   int buf_index = 0;
   int num_inputs = nn_config->num_inputs;
@@ -279,61 +621,96 @@ void av1_nn_predict_neon(const float *input_nodes,
     const int num_outputs = output_layer ? nn_config->num_outputs
                                          : nn_config->num_hidden_nodes[layer];

-    if (num_inputs % 4 == 0 && num_outputs % 8 == 0) {
-      for (int out = 0; out < num_outputs; out += 8) {
-        nn_propagate_4to8(num_inputs, input_nodes,
-                          &layer_weights[out * num_inputs], &layer_bias[out],
-                          &output_nodes[out], output_layer);
-      }
-    } else if (num_inputs % 8 == 0 && num_outputs % 4 == 0) {
-      for (int out = 0; out < num_outputs; out += 4) {
-        nn_propagate_8to4(num_inputs, input_nodes,
-                          &layer_weights[out * num_inputs], &layer_bias[out],
-                          &output_nodes[out], output_layer);
-      }
-    } else if (num_inputs % 4 == 0 && num_outputs % 4 == 0) {
-      for (int out = 0; out < num_outputs; out += 4) {
-        nn_propagate_4to4(num_inputs, input_nodes,
-                          &layer_weights[out * num_inputs], &layer_bias[out],
-                          &output_nodes[out], output_layer);
-      }
-    } else if (num_inputs % 8 == 0) {
-      for (int out = 0; out < num_outputs; out++) {
-        nn_propagate_8to1(num_inputs, input_nodes,
-                          &layer_weights[out * num_inputs], &layer_bias[out],
-                          &output_nodes[out], output_layer);
-      }
-    } else if (num_inputs % 4 == 0) {
-      for (int out = 0; out < num_outputs; out++) {
-        nn_propagate_4to1(num_inputs, input_nodes,
-                          &layer_weights[out * num_inputs], &layer_bias[out],
-                          &output_nodes[out], output_layer);
-      }
-    } else if (num_inputs > 8) {
-      for (int out = 0; out < num_outputs; out++) {
-        nn_propagate_xto1(num_inputs, input_nodes,
-                          &layer_weights[out * num_inputs], &layer_bias[out],
-                          &output_nodes[out]);
-      }
-    } else if (num_inputs >= 4) {
-      for (int out = 0; out < num_outputs; out++) {
-        nn_propagate_xsto1(num_inputs, input_nodes,
-                           &layer_weights[out * num_inputs], &layer_bias[out],
-                           &output_nodes[out]);
-      }
-    } else {
-      for (int node = 0; node < num_outputs; ++node) {
-        float val = layer_bias[node];
-        for (int i = 0; i < num_inputs; ++i)
-          val += layer_weights[node * num_inputs + i] * input_nodes[i];
-        // ReLU as activation function.
-        val = val > 0.0f ? val : 0.0f;  // Could use AOMMAX().
-        output_nodes[node] = val;
-      }
-    }
+    nn_predict_layer_neon(input_nodes, layer_weights, layer_bias, num_inputs,
+                          num_outputs, output_layer, output_nodes);
     input_nodes = output_nodes;
     num_inputs = num_outputs;
     buf_index = 1 - buf_index;
   }
+}
+
+// Calculate prediction based on the given input features and neural net config.
+// Assume there are no more than NN_MAX_NODES_PER_LAYER nodes in each hidden
+// layer.
+void av1_nn_predict_neon(const float *input_nodes,
+                         const NN_CONFIG *const nn_config, int reduce_prec,
+                         float *const output) {
+  if (nn_config->num_hidden_layers == 0) {
+    nn_predict_layer_neon(input_nodes, nn_config->weights[0],
+                          nn_config->bias[0], nn_config->num_inputs,
+                          nn_config->num_outputs, true, output);
+  } else {
+    nn_predict_hidden_neon(input_nodes, nn_config, output);
+  }
   if (reduce_prec) av1_nn_output_prec_reduce(output, nn_config->num_outputs);
 }
+
+static inline float32x4_t approx_exp_neon(float32x4_t y) {
+  const float32x4_t a = vdupq_n_f32((1 << 23) / 0.69314718056f);
+  const int32x4_t bc = vdupq_n_s32((127 << 23) - 60801);
+  const int32x4_t exp_i32 = vaddq_s32(vcvtq_s32_f32(vmulq_f32(y, a)), bc);
+  return vreinterpretq_f32_s32(exp_i32);
+}
+
+void av1_nn_fast_softmax_16_neon(const float *input, float *output) {
+  const float32x4_t clipper = vdupq_n_f32(-10.0f);
+  const float32x4_t in0 = vld1q_f32(&input[0]);
+  const float32x4_t in1 = vld1q_f32(&input[4]);
+  const float32x4_t in2 = vld1q_f32(&input[8]);
+  const float32x4_t in3 = vld1q_f32(&input[12]);
+
+  const float32x4_t m4 = vmaxq_f32(vmaxq_f32(in0, in1), vmaxq_f32(in2, in3));
+#if AOM_ARCH_AARCH64
+  const float max_input = vmaxvq_f32(m4);
+#else
+  float32x2_t m2 = vmax_f32(vget_low_f32(m4), vget_high_f32(m4));
+  m2 = vpmax_f32(m2, m2);
+  const float max_input = vget_lane_f32(m2, 0);
+#endif
+  const float32x4_t v_max = vdupq_n_f32(max_input);
+
+  const float32x4_t exp0 =
+      approx_exp_neon(vmaxq_f32(vsubq_f32(in0, v_max), clipper));
+  const float32x4_t exp1 =
+      approx_exp_neon(vmaxq_f32(vsubq_f32(in1, v_max), clipper));
+  const float32x4_t exp2 =
+      approx_exp_neon(vmaxq_f32(vsubq_f32(in2, v_max), clipper));
+  const float32x4_t exp3 =
+      approx_exp_neon(vmaxq_f32(vsubq_f32(in3, v_max), clipper));
+
+  // Accumulate sequentially to guarantee 0-ULP parity with
+  // av1_nn_fast_softmax_16_c and 100% SHA-256 bitstream identity.
+  float sum_out = vgetq_lane_f32(exp0, 0) + vgetq_lane_f32(exp0, 1);
+  sum_out += vgetq_lane_f32(exp0, 2);
+  sum_out += vgetq_lane_f32(exp0, 3);
+  sum_out += vgetq_lane_f32(exp1, 0);
+  sum_out += vgetq_lane_f32(exp1, 1);
+  sum_out += vgetq_lane_f32(exp1, 2);
+  sum_out += vgetq_lane_f32(exp1, 3);
+  sum_out += vgetq_lane_f32(exp2, 0);
+  sum_out += vgetq_lane_f32(exp2, 1);
+  sum_out += vgetq_lane_f32(exp2, 2);
+  sum_out += vgetq_lane_f32(exp2, 3);
+  sum_out += vgetq_lane_f32(exp3, 0);
+  sum_out += vgetq_lane_f32(exp3, 1);
+  sum_out += vgetq_lane_f32(exp3, 2);
+  sum_out += vgetq_lane_f32(exp3, 3);
+
+#if AOM_ARCH_AARCH64
+  const float32x4_t v_sum = vdupq_n_f32(sum_out);
+  vst1q_f32(&output[0], vdivq_f32(exp0, v_sum));
+  vst1q_f32(&output[4], vdivq_f32(exp1, v_sum));
+  vst1q_f32(&output[8], vdivq_f32(exp2, v_sum));
+  vst1q_f32(&output[12], vdivq_f32(exp3, v_sum));
+#else
+  vst1q_f32(&output[0], exp0);
+  vst1q_f32(&output[4], exp1);
+  vst1q_f32(&output[8], exp2);
+  vst1q_f32(&output[12], exp3);
+  int i = 0;
+  do {
+    output[i] /= sum_out;
+    ++i;
+  } while (i != 16);
+#endif
+}
diff --git a/test/av1_nn_predict_test.cc b/test/av1_nn_predict_test.cc
index b05b83cbf0..7788060f92 100644
--- a/test/av1_nn_predict_test.cc
+++ b/test/av1_nn_predict_test.cc
@@ -182,7 +182,10 @@ static const NN_CONFIG kShapes[] = {
   { 8, 4, 1, { 16 }, {}, {} },      { 8, 1, 1, { 32 }, {}, {} },
   { 9, 3, 1, { 32 }, {}, {} },      { 8, 4, 0, { 0 }, {}, {} },
   { 8, 8, 0, { 0 }, {}, {} },       { 4, 4, 1, { 8 }, {}, {} },
-  { 4, 3, 0, { 64 }, {}, {} },
+  { 4, 3, 0, { 64 }, {}, {} },      { 9, 5, 0, { 0 }, {}, {} },
+  { 6, 5, 0, { 0 }, {}, {} },       { 3, 2, 0, { 0 }, {}, {} },
+  { 13, 13, 1, { 13 }, {}, {} },    { 6, 5, 1, { 5 }, {}, {} },
+  { 32, 8, 0, { 0 }, {}, {} },
 };

 void NnPredictTest::RunNnPredictTest_all(const NN_CONFIG *const shapes,
diff --git a/test/av1_softmax_test.cc b/test/av1_softmax_test.cc
index 18af945eb9..c223235f45 100644
--- a/test/av1_softmax_test.cc
+++ b/test/av1_softmax_test.cc
@@ -119,4 +119,10 @@ INSTANTIATE_TEST_SUITE_P(
     SSE3, FastSoftmaxTest,
     ::testing::Values(FastSoftmaxTestParams(av1_nn_fast_softmax_16_sse3, 16)));
 #endif
+
+#if HAVE_NEON && !CONFIG_EXCLUDE_SIMD_MISMATCH
+INSTANTIATE_TEST_SUITE_P(
+    NEON, FastSoftmaxTest,
+    ::testing::Values(FastSoftmaxTestParams(av1_nn_fast_softmax_16_neon, 16)));
+#endif
 }  // namespace