Commit 0aa7e16ff1 for aom
commit 0aa7e16ff1a25f816742322638b4b358677a07c3
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Nov 14 11:58:40 2026 -0800
av1,neon: Add nn_fast_softmax_16_neon & opt ml
Add av1_nn_fast_softmax_16_neon, add multi-output propagators
(nn_propagate_xto4, nn_propagate_xto8) in ml_neon.c, and register
av1_nn_fast_softmax_16_neon in FastSoftmaxTest.
Speedup: 3.52x (-71.6%, av1_nn_fast_softmax_16_neon) and 1.46x-3.18x
(-31.5% to -68.6%, av1_nn_predict_neon) in test_libaom on Neoverse N3;
-0.66% (1T) / -0.89% (4T) on Neoverse N3 and -0.37% (1T Cortex-X4) /
-0.26% (1T Cortex-A720) / -1.21% (4T) 8b avifenc on Pixel 9
(bit-exact).
TAG=agy
CONV=490488d4-7005-402a-8cc4-c245227f80fe
Change-Id: I360804b89dfbf1851ea4b96c38c1aaf2a14ea2fd
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index d1cb4427a4..b9fa941a63 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -529,7 +529,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/void av1_nn_fast_softmax_16/, "const float *input_nodes, float *output";
if (aom_config("CONFIG_EXCLUDE_SIMD_MISMATCH") ne "yes") {
specialize qw/av1_nn_predict sse3 avx2 neon/;
- specialize qw/av1_nn_fast_softmax_16 sse3/;
+ specialize qw/av1_nn_fast_softmax_16 sse3 neon/;
}
# CNN functions
diff --git a/av1/encoder/arm/ml_neon.c b/av1/encoder/arm/ml_neon.c
index 142e465e28..b25ef0f131 100644
--- a/av1/encoder/arm/ml_neon.c
+++ b/av1/encoder/arm/ml_neon.c
@@ -15,6 +15,7 @@
#include "config/aom_config.h"
#include "config/av1_rtcd.h"
+#include "aom_dsp/aom_dsp_common.h"
#include "av1/encoder/ml.h"
static void nn_activate8(float32x4_t *out_h, float32x4_t *out_l,
@@ -62,13 +63,14 @@ static void nn_propagate_8to1(int num_inputs, const float *const inputs,
static void nn_propagate_xto1(int num_inputs, const float *const inputs,
const float *const weights,
const float *layer_bias,
- float *const output_nodes) {
+ float *const output_nodes, bool output_layer) {
+ assert(num_inputs > 8 && (num_inputs & 3) != 0);
float32x4_t vadd = vdupq_n_f32(0);
float total = *layer_bias;
int j = num_inputs;
int in = 0;
- while (j > 7) {
+ do {
const float32x4_t inputs_h = vld1q_f32(&inputs[in + 4]);
const float32x4_t inputs_l = vld1q_f32(&inputs[in]);
@@ -79,7 +81,7 @@ static void nn_propagate_xto1(int num_inputs, const float *const inputs,
vadd = vmlaq_f32(vadd, inputs_l, weights_l);
in += 8;
j -= 8;
- }
+ } while (j >= 8);
#if AOM_ARCH_AARCH64
total += vaddvq_f32(vadd);
@@ -89,15 +91,207 @@ static void nn_propagate_xto1(int num_inputs, const float *const inputs,
vadd_lo = vpadd_f32(vadd_lo, vadd_lo);
total += vget_lane_f32(vadd_lo, 0);
#endif
- for (; in < num_inputs; in++) total += weights[in] * inputs[in];
+ do {
+ total += weights[in] * inputs[in];
+ in++;
+ } while (in != num_inputs);
+
+ if (!output_layer) CLAMP_0(total);
+ *output_nodes = total;
+}
+
+static void nn_propagate_xto4(int num_inputs, const float *const inputs,
+ const float *const weights,
+ const float *layer_bias,
+ float *const output_nodes, bool output_layer) {
+ assert(num_inputs > 8 && (num_inputs & 3) != 0);
+ const float32x4_t zero = vdupq_n_f32(0);
+ float32x4_t vadd0 = zero;
+ float32x4_t vadd1 = zero;
+ float32x4_t vadd2 = zero;
+ float32x4_t vadd3 = zero;
+ const float *const w0 = weights;
+ const float *const w1 = weights + num_inputs;
+ const float *const w2 = weights + 2 * num_inputs;
+ const float *const w3 = weights + 3 * num_inputs;
+ int j = num_inputs;
+ int in = 0;
+
+ do {
+ const float32x4_t inputs_h = vld1q_f32(&inputs[in + 4]);
+ const float32x4_t inputs_l = vld1q_f32(&inputs[in]);
+ const float32x4_t w0_h = vld1q_f32(&w0[in + 4]);
+ const float32x4_t w0_l = vld1q_f32(&w0[in]);
+ const float32x4_t w1_h = vld1q_f32(&w1[in + 4]);
+ const float32x4_t w1_l = vld1q_f32(&w1[in]);
+ const float32x4_t w2_h = vld1q_f32(&w2[in + 4]);
+ const float32x4_t w2_l = vld1q_f32(&w2[in]);
+ const float32x4_t w3_h = vld1q_f32(&w3[in + 4]);
+ const float32x4_t w3_l = vld1q_f32(&w3[in]);
+
+ vadd0 = vmlaq_f32(vadd0, inputs_h, w0_h);
+ vadd1 = vmlaq_f32(vadd1, inputs_h, w1_h);
+ vadd2 = vmlaq_f32(vadd2, inputs_h, w2_h);
+ vadd3 = vmlaq_f32(vadd3, inputs_h, w3_h);
+ vadd0 = vmlaq_f32(vadd0, inputs_l, w0_l);
+ vadd1 = vmlaq_f32(vadd1, inputs_l, w1_l);
+ vadd2 = vmlaq_f32(vadd2, inputs_l, w2_l);
+ vadd3 = vmlaq_f32(vadd3, inputs_l, w3_l);
+ in += 8;
+ j -= 8;
+ } while (j >= 8);
+
+ if (j > 3) {
+ const float32x4_t in_l = vld1q_f32(&inputs[in]);
+ vadd0 = vmlaq_f32(vadd0, in_l, vld1q_f32(&w0[in]));
+ vadd1 = vmlaq_f32(vadd1, in_l, vld1q_f32(&w1[in]));
+ vadd2 = vmlaq_f32(vadd2, in_l, vld1q_f32(&w2[in]));
+ vadd3 = vmlaq_f32(vadd3, in_l, vld1q_f32(&w3[in]));
+ in += 4;
+ }
+
+#if AOM_ARCH_AARCH64
+ const float32x4_t p0 = vpaddq_f32(vadd0, vadd1);
+ const float32x4_t p1 = vpaddq_f32(vadd2, vadd3);
+ float32x4_t outputs = vaddq_f32(vld1q_f32(layer_bias), vpaddq_f32(p0, p1));
+#else
+ const float32x4_t p0 =
+ vcombine_f32(vpadd_f32(vget_low_f32(vadd0), vget_high_f32(vadd0)),
+ vpadd_f32(vget_low_f32(vadd1), vget_high_f32(vadd1)));
+ const float32x4_t p1 =
+ vcombine_f32(vpadd_f32(vget_low_f32(vadd2), vget_high_f32(vadd2)),
+ vpadd_f32(vget_low_f32(vadd3), vget_high_f32(vadd3)));
+ const float32x4_t sum =
+ vcombine_f32(vpadd_f32(vget_low_f32(p0), vget_high_f32(p0)),
+ vpadd_f32(vget_low_f32(p1), vget_high_f32(p1)));
+ float32x4_t outputs = vaddq_f32(vld1q_f32(layer_bias), sum);
+#endif
- *output_nodes = CLAMP_0(total);
+ do {
+ float32x4_t w_tail = vld1q_dup_f32(&w0[in]);
+ w_tail = vld1q_lane_f32(&w1[in], w_tail, 1);
+ w_tail = vld1q_lane_f32(&w2[in], w_tail, 2);
+ w_tail = vld1q_lane_f32(&w3[in], w_tail, 3);
+ outputs = vmlaq_n_f32(outputs, w_tail, inputs[in]);
+ in++;
+ } while (in != num_inputs);
+
+ if (!output_layer) nn_activate4(&outputs, &zero);
+ vst1q_f32(output_nodes, outputs);
+}
+
+static void nn_propagate_xto8(int num_inputs, const float *const inputs,
+ const float *const weights,
+ const float *layer_bias,
+ float *const output_nodes, bool output_layer) {
+ assert(num_inputs > 8 && (num_inputs & 3) != 0);
+ const float32x4_t zero = vdupq_n_f32(0);
+ float32x4_t vadd0 = zero;
+ float32x4_t vadd1 = zero;
+ float32x4_t vadd2 = zero;
+ float32x4_t vadd3 = zero;
+ float32x4_t vadd4 = zero;
+ float32x4_t vadd5 = zero;
+ float32x4_t vadd6 = zero;
+ float32x4_t vadd7 = zero;
+ const float *const w0 = weights;
+ const float *const w2 = weights + 2 * num_inputs;
+ const float *const w4 = weights + 4 * num_inputs;
+ const float *const w6 = weights + 6 * num_inputs;
+ int j = num_inputs;
+ int in = 0;
+
+ do {
+ const float32x4_t in_l = vld1q_f32(&inputs[in]);
+ const float32x4_t in_h = vld1q_f32(&inputs[in + 4]);
+ vadd0 = vmlaq_f32(vadd0, in_l, vld1q_f32(&w0[in]));
+ vadd1 = vmlaq_f32(vadd1, in_l, vld1q_f32(&w0[in + num_inputs]));
+ vadd2 = vmlaq_f32(vadd2, in_l, vld1q_f32(&w2[in]));
+ vadd3 = vmlaq_f32(vadd3, in_l, vld1q_f32(&w2[in + num_inputs]));
+ vadd4 = vmlaq_f32(vadd4, in_l, vld1q_f32(&w4[in]));
+ vadd5 = vmlaq_f32(vadd5, in_l, vld1q_f32(&w4[in + num_inputs]));
+ vadd6 = vmlaq_f32(vadd6, in_l, vld1q_f32(&w6[in]));
+ vadd7 = vmlaq_f32(vadd7, in_l, vld1q_f32(&w6[in + num_inputs]));
+ vadd0 = vmlaq_f32(vadd0, in_h, vld1q_f32(&w0[in + 4]));
+ vadd1 = vmlaq_f32(vadd1, in_h, vld1q_f32(&w0[in + num_inputs + 4]));
+ vadd2 = vmlaq_f32(vadd2, in_h, vld1q_f32(&w2[in + 4]));
+ vadd3 = vmlaq_f32(vadd3, in_h, vld1q_f32(&w2[in + num_inputs + 4]));
+ vadd4 = vmlaq_f32(vadd4, in_h, vld1q_f32(&w4[in + 4]));
+ vadd5 = vmlaq_f32(vadd5, in_h, vld1q_f32(&w4[in + num_inputs + 4]));
+ vadd6 = vmlaq_f32(vadd6, in_h, vld1q_f32(&w6[in + 4]));
+ vadd7 = vmlaq_f32(vadd7, in_h, vld1q_f32(&w6[in + num_inputs + 4]));
+ in += 8;
+ j -= 8;
+ } while (j >= 8);
+
+ if (j > 3) {
+ const float32x4_t in_l = vld1q_f32(&inputs[in]);
+ vadd0 = vmlaq_f32(vadd0, in_l, vld1q_f32(&w0[in]));
+ vadd1 = vmlaq_f32(vadd1, in_l, vld1q_f32(&w0[in + num_inputs]));
+ vadd2 = vmlaq_f32(vadd2, in_l, vld1q_f32(&w2[in]));
+ vadd3 = vmlaq_f32(vadd3, in_l, vld1q_f32(&w2[in + num_inputs]));
+ vadd4 = vmlaq_f32(vadd4, in_l, vld1q_f32(&w4[in]));
+ vadd5 = vmlaq_f32(vadd5, in_l, vld1q_f32(&w4[in + num_inputs]));
+ vadd6 = vmlaq_f32(vadd6, in_l, vld1q_f32(&w6[in]));
+ vadd7 = vmlaq_f32(vadd7, in_l, vld1q_f32(&w6[in + num_inputs]));
+ in += 4;
+ }
+
+#if AOM_ARCH_AARCH64
+ const float32x4_t p0 = vpaddq_f32(vadd0, vadd1);
+ const float32x4_t p1 = vpaddq_f32(vadd2, vadd3);
+ const float32x4_t p2 = vpaddq_f32(vadd4, vadd5);
+ const float32x4_t p3 = vpaddq_f32(vadd6, vadd7);
+ float32x4_t out_l = vaddq_f32(vld1q_f32(layer_bias), vpaddq_f32(p0, p1));
+ float32x4_t out_h = vaddq_f32(vld1q_f32(&layer_bias[4]), vpaddq_f32(p2, p3));
+#else
+ const float32x4_t p0 =
+ vcombine_f32(vpadd_f32(vget_low_f32(vadd0), vget_high_f32(vadd0)),
+ vpadd_f32(vget_low_f32(vadd1), vget_high_f32(vadd1)));
+ const float32x4_t p1 =
+ vcombine_f32(vpadd_f32(vget_low_f32(vadd2), vget_high_f32(vadd2)),
+ vpadd_f32(vget_low_f32(vadd3), vget_high_f32(vadd3)));
+ const float32x4_t p2 =
+ vcombine_f32(vpadd_f32(vget_low_f32(vadd4), vget_high_f32(vadd4)),
+ vpadd_f32(vget_low_f32(vadd5), vget_high_f32(vadd5)));
+ const float32x4_t p3 =
+ vcombine_f32(vpadd_f32(vget_low_f32(vadd6), vget_high_f32(vadd6)),
+ vpadd_f32(vget_low_f32(vadd7), vget_high_f32(vadd7)));
+ const float32x4_t sum_l =
+ vcombine_f32(vpadd_f32(vget_low_f32(p0), vget_high_f32(p0)),
+ vpadd_f32(vget_low_f32(p1), vget_high_f32(p1)));
+ const float32x4_t sum_h =
+ vcombine_f32(vpadd_f32(vget_low_f32(p2), vget_high_f32(p2)),
+ vpadd_f32(vget_low_f32(p3), vget_high_f32(p3)));
+ float32x4_t out_l = vaddq_f32(vld1q_f32(layer_bias), sum_l);
+ float32x4_t out_h = vaddq_f32(vld1q_f32(&layer_bias[4]), sum_h);
+#endif
+
+ do {
+ const float inp = inputs[in];
+ float32x4_t wl = vld1q_dup_f32(&w0[in]);
+ wl = vld1q_lane_f32(&w0[in + num_inputs], wl, 1);
+ wl = vld1q_lane_f32(&w2[in], wl, 2);
+ wl = vld1q_lane_f32(&w2[in + num_inputs], wl, 3);
+ float32x4_t wh = vld1q_dup_f32(&w4[in]);
+ wh = vld1q_lane_f32(&w4[in + num_inputs], wh, 1);
+ wh = vld1q_lane_f32(&w6[in], wh, 2);
+ wh = vld1q_lane_f32(&w6[in + num_inputs], wh, 3);
+ out_l = vmlaq_n_f32(out_l, wl, inp);
+ out_h = vmlaq_n_f32(out_h, wh, inp);
+ in++;
+ } while (in != num_inputs);
+
+ if (!output_layer) nn_activate8(&out_h, &out_l, &zero);
+ vst1q_f32(output_nodes, out_l);
+ vst1q_f32(&output_nodes[4], out_h);
}
static void nn_propagate_xsto1(int num_inputs, const float *const inputs,
const float *const weights,
const float *layer_bias,
- float *const output_nodes) {
+ float *const output_nodes, bool output_layer) {
+ assert(num_inputs > 4 && num_inputs < 8);
float total = *layer_bias;
#if AOM_ARCH_AARCH64
const float32x4_t v_inputs = vld1q_f32(inputs);
@@ -108,9 +302,57 @@ static void nn_propagate_xsto1(int num_inputs, const float *const inputs,
#else
int in = 0;
#endif
- for (; in < num_inputs; in++) total += weights[in] * inputs[in];
+ do {
+ total += weights[in] * inputs[in];
+ in++;
+ } while (in != num_inputs);
- *output_nodes = CLAMP_0(total);
+ if (!output_layer) CLAMP_0(total);
+ *output_nodes = total;
+}
+
+static void nn_propagate_xsto4(int num_inputs, const float *const inputs,
+ const float *const weights,
+ const float *layer_bias,
+ float *const output_nodes, bool output_layer) {
+ assert(num_inputs > 4 && num_inputs < 8);
+ const float32x4_t zero = vdupq_n_f32(0);
+ const float32x4_t v_inputs = vld1q_f32(inputs);
+ const float *const w0 = weights;
+ const float *const w1 = weights + num_inputs;
+ const float *const w2 = weights + 2 * num_inputs;
+ const float *const w3 = weights + 3 * num_inputs;
+ const float32x4_t mul0 = vmulq_f32(v_inputs, vld1q_f32(w0));
+ const float32x4_t mul1 = vmulq_f32(v_inputs, vld1q_f32(w1));
+ const float32x4_t mul2 = vmulq_f32(v_inputs, vld1q_f32(w2));
+ const float32x4_t mul3 = vmulq_f32(v_inputs, vld1q_f32(w3));
+#if AOM_ARCH_AARCH64
+ const float32x4_t p0 = vpaddq_f32(mul0, mul1);
+ const float32x4_t p1 = vpaddq_f32(mul2, mul3);
+ float32x4_t outputs = vaddq_f32(vld1q_f32(layer_bias), vpaddq_f32(p0, p1));
+#else
+ const float32x4_t p0 =
+ vcombine_f32(vpadd_f32(vget_low_f32(mul0), vget_high_f32(mul0)),
+ vpadd_f32(vget_low_f32(mul1), vget_high_f32(mul1)));
+ const float32x4_t p1 =
+ vcombine_f32(vpadd_f32(vget_low_f32(mul2), vget_high_f32(mul2)),
+ vpadd_f32(vget_low_f32(mul3), vget_high_f32(mul3)));
+ const float32x4_t sum =
+ vcombine_f32(vpadd_f32(vget_low_f32(p0), vget_high_f32(p0)),
+ vpadd_f32(vget_low_f32(p1), vget_high_f32(p1)));
+ float32x4_t outputs = vaddq_f32(vld1q_f32(layer_bias), sum);
+#endif
+ int in = 4;
+ do {
+ float32x4_t w_tail = vld1q_dup_f32(&w0[in]);
+ w_tail = vld1q_lane_f32(&w1[in], w_tail, 1);
+ w_tail = vld1q_lane_f32(&w2[in], w_tail, 2);
+ w_tail = vld1q_lane_f32(&w3[in], w_tail, 3);
+ outputs = vmlaq_n_f32(outputs, w_tail, inputs[in]);
+ in++;
+ } while (in != num_inputs);
+ if (!output_layer) nn_activate4(&outputs, &zero);
+ vst1q_f32(output_nodes, outputs);
}
static void nn_propagate_4to1(int num_inputs, const float *const inputs,
@@ -187,16 +429,24 @@ static void nn_propagate_4to8(const int num_inputs, const float *const inputs,
float32x4_t mul0[4] = { zero, zero, zero, zero };
float32x4_t mul1[4] = { zero, zero, zero, zero };
- for (int in = 0; in < num_inputs; in += 4) {
+ assert(num_inputs >= 4 && (num_inputs & 3) == 0);
+ const float *const w0 = weights;
+ const float *const w2 = weights + 2 * num_inputs;
+ const float *const w4 = weights + 4 * num_inputs;
+ const float *const w6 = weights + 6 * num_inputs;
+ int in = 0;
+ do {
const float32x4_t v_input = vld1q_f32(&inputs[in]);
- for (int i = 0; i < 4; i++) {
- const float32x4_t weight0 = vld1q_f32(&weights[in + 2 * i * num_inputs]);
- const float32x4_t weight1 =
- vld1q_f32(&weights[in + (2 * i + 1) * num_inputs]);
- mul0[i] = vmlaq_f32(mul0[i], v_input, weight0);
- mul1[i] = vmlaq_f32(mul1[i], v_input, weight1);
- }
- }
+ mul0[0] = vmlaq_f32(mul0[0], v_input, vld1q_f32(&w0[in]));
+ mul1[0] = vmlaq_f32(mul1[0], v_input, vld1q_f32(&w0[in + num_inputs]));
+ mul0[1] = vmlaq_f32(mul0[1], v_input, vld1q_f32(&w2[in]));
+ mul1[1] = vmlaq_f32(mul1[1], v_input, vld1q_f32(&w2[in + num_inputs]));
+ mul0[2] = vmlaq_f32(mul0[2], v_input, vld1q_f32(&w4[in]));
+ mul1[2] = vmlaq_f32(mul1[2], v_input, vld1q_f32(&w4[in + num_inputs]));
+ mul0[3] = vmlaq_f32(mul0[3], v_input, vld1q_f32(&w6[in]));
+ mul1[3] = vmlaq_f32(mul1[3], v_input, vld1q_f32(&w6[in + num_inputs]));
+ in += 4;
+ } while (in != num_inputs);
for (int i = 0; i < 4; i++)
#if AOM_ARCH_AARCH64
mul0[i] = vpaddq_f32(mul0[i], mul1[i]);
@@ -261,12 +511,104 @@ static void nn_propagate_8to4(const int num_inputs, const float *const inputs,
vst1q_f32(output_nodes, outputs);
}
-// Calculate prediction based on the given input features and neural net config.
-// Assume there are no more than NN_MAX_NODES_PER_LAYER nodes in each hidden
-// layer.
-void av1_nn_predict_neon(const float *input_nodes,
- const NN_CONFIG *const nn_config, int reduce_prec,
- float *const output) {
+static AOM_FORCE_INLINE void nn_predict_layer_neon(
+ const float *const input_nodes, const float *const layer_weights,
+ const float *const layer_bias, int num_inputs, int num_outputs,
+ bool output_layer, float *const output_nodes) {
+ assert(num_inputs > 0 && num_outputs > 0);
+ if (num_inputs % 4 == 0 && num_outputs % 8 == 0) {
+ int out = 0;
+ do {
+ nn_propagate_4to8(num_inputs, input_nodes,
+ &layer_weights[out * num_inputs], &layer_bias[out],
+ &output_nodes[out], output_layer);
+ out += 8;
+ } while (out != num_outputs);
+ } else if (num_inputs % 8 == 0 && num_outputs % 4 == 0) {
+ int out = 0;
+ do {
+ nn_propagate_8to4(num_inputs, input_nodes,
+ &layer_weights[out * num_inputs], &layer_bias[out],
+ &output_nodes[out], output_layer);
+ out += 4;
+ } while (out != num_outputs);
+ } else if (num_inputs % 4 == 0 && num_outputs % 4 == 0) {
+ int out = 0;
+ do {
+ nn_propagate_4to4(num_inputs, input_nodes,
+ &layer_weights[out * num_inputs], &layer_bias[out],
+ &output_nodes[out], output_layer);
+ out += 4;
+ } while (out != num_outputs);
+ } else if (num_inputs % 8 == 0) {
+ int out = 0;
+ do {
+ nn_propagate_8to1(num_inputs, input_nodes,
+ &layer_weights[out * num_inputs], &layer_bias[out],
+ &output_nodes[out], output_layer);
+ out++;
+ } while (out != num_outputs);
+ } else if (num_inputs % 4 == 0) {
+ int out = 0;
+ do {
+ nn_propagate_4to1(num_inputs, input_nodes,
+ &layer_weights[out * num_inputs], &layer_bias[out],
+ &output_nodes[out], output_layer);
+ out++;
+ } while (out != num_outputs);
+ } else if (num_inputs > 8) {
+ int out = 0;
+ for (; out + 8 <= num_outputs; out += 8) {
+ nn_propagate_xto8(num_inputs, input_nodes,
+ &layer_weights[out * num_inputs], &layer_bias[out],
+ &output_nodes[out], output_layer);
+ }
+ for (; out + 4 <= num_outputs; out += 4) {
+ nn_propagate_xto4(num_inputs, input_nodes,
+ &layer_weights[out * num_inputs], &layer_bias[out],
+ &output_nodes[out], output_layer);
+ }
+ for (; out < num_outputs; out++) {
+ nn_propagate_xto1(num_inputs, input_nodes,
+ &layer_weights[out * num_inputs], &layer_bias[out],
+ &output_nodes[out], output_layer);
+ }
+ } else if (num_inputs >= 4) {
+ int out = 0;
+ for (; out + 4 <= num_outputs; out += 4) {
+ nn_propagate_xsto4(num_inputs, input_nodes,
+ &layer_weights[out * num_inputs], &layer_bias[out],
+ &output_nodes[out], output_layer);
+ }
+ for (; out < num_outputs; out++) {
+ nn_propagate_xsto1(num_inputs, input_nodes,
+ &layer_weights[out * num_inputs], &layer_bias[out],
+ &output_nodes[out], output_layer);
+ }
+ } else {
+ int node = 0;
+ do {
+ float val = layer_bias[node];
+ int i = 0;
+ do {
+ val += layer_weights[node * num_inputs + i] * input_nodes[i];
+ ++i;
+ } while (i != num_inputs);
+ if (!output_layer) {
+ // ReLU as activation function.
+ val = val > 0.0f ? val : 0.0f; // Could use AOMMAX().
+ }
+ output_nodes[node] = val;
+ ++node;
+ } while (node != num_outputs);
+ }
+}
+
+// Avoid inlining the multi-layer hidden-buffer stack frame into the
+// 0-hidden-layer fast path of av1_nn_predict_neon.
+static AOM_NOINLINE void nn_predict_hidden_neon(
+ const float *input_nodes, const NN_CONFIG *const nn_config,
+ float *const output) {
float buf[2][NN_MAX_NODES_PER_LAYER];
int buf_index = 0;
int num_inputs = nn_config->num_inputs;
@@ -279,61 +621,96 @@ void av1_nn_predict_neon(const float *input_nodes,
const int num_outputs = output_layer ? nn_config->num_outputs
: nn_config->num_hidden_nodes[layer];
- if (num_inputs % 4 == 0 && num_outputs % 8 == 0) {
- for (int out = 0; out < num_outputs; out += 8) {
- nn_propagate_4to8(num_inputs, input_nodes,
- &layer_weights[out * num_inputs], &layer_bias[out],
- &output_nodes[out], output_layer);
- }
- } else if (num_inputs % 8 == 0 && num_outputs % 4 == 0) {
- for (int out = 0; out < num_outputs; out += 4) {
- nn_propagate_8to4(num_inputs, input_nodes,
- &layer_weights[out * num_inputs], &layer_bias[out],
- &output_nodes[out], output_layer);
- }
- } else if (num_inputs % 4 == 0 && num_outputs % 4 == 0) {
- for (int out = 0; out < num_outputs; out += 4) {
- nn_propagate_4to4(num_inputs, input_nodes,
- &layer_weights[out * num_inputs], &layer_bias[out],
- &output_nodes[out], output_layer);
- }
- } else if (num_inputs % 8 == 0) {
- for (int out = 0; out < num_outputs; out++) {
- nn_propagate_8to1(num_inputs, input_nodes,
- &layer_weights[out * num_inputs], &layer_bias[out],
- &output_nodes[out], output_layer);
- }
- } else if (num_inputs % 4 == 0) {
- for (int out = 0; out < num_outputs; out++) {
- nn_propagate_4to1(num_inputs, input_nodes,
- &layer_weights[out * num_inputs], &layer_bias[out],
- &output_nodes[out], output_layer);
- }
- } else if (num_inputs > 8) {
- for (int out = 0; out < num_outputs; out++) {
- nn_propagate_xto1(num_inputs, input_nodes,
- &layer_weights[out * num_inputs], &layer_bias[out],
- &output_nodes[out]);
- }
- } else if (num_inputs >= 4) {
- for (int out = 0; out < num_outputs; out++) {
- nn_propagate_xsto1(num_inputs, input_nodes,
- &layer_weights[out * num_inputs], &layer_bias[out],
- &output_nodes[out]);
- }
- } else {
- for (int node = 0; node < num_outputs; ++node) {
- float val = layer_bias[node];
- for (int i = 0; i < num_inputs; ++i)
- val += layer_weights[node * num_inputs + i] * input_nodes[i];
- // ReLU as activation function.
- val = val > 0.0f ? val : 0.0f; // Could use AOMMAX().
- output_nodes[node] = val;
- }
- }
+ nn_predict_layer_neon(input_nodes, layer_weights, layer_bias, num_inputs,
+ num_outputs, output_layer, output_nodes);
input_nodes = output_nodes;
num_inputs = num_outputs;
buf_index = 1 - buf_index;
}
+}
+
+// Calculate prediction based on the given input features and neural net config.
+// Assume there are no more than NN_MAX_NODES_PER_LAYER nodes in each hidden
+// layer.
+void av1_nn_predict_neon(const float *input_nodes,
+ const NN_CONFIG *const nn_config, int reduce_prec,
+ float *const output) {
+ if (nn_config->num_hidden_layers == 0) {
+ nn_predict_layer_neon(input_nodes, nn_config->weights[0],
+ nn_config->bias[0], nn_config->num_inputs,
+ nn_config->num_outputs, true, output);
+ } else {
+ nn_predict_hidden_neon(input_nodes, nn_config, output);
+ }
if (reduce_prec) av1_nn_output_prec_reduce(output, nn_config->num_outputs);
}
+
+static inline float32x4_t approx_exp_neon(float32x4_t y) {
+ const float32x4_t a = vdupq_n_f32((1 << 23) / 0.69314718056f);
+ const int32x4_t bc = vdupq_n_s32((127 << 23) - 60801);
+ const int32x4_t exp_i32 = vaddq_s32(vcvtq_s32_f32(vmulq_f32(y, a)), bc);
+ return vreinterpretq_f32_s32(exp_i32);
+}
+
+void av1_nn_fast_softmax_16_neon(const float *input, float *output) {
+ const float32x4_t clipper = vdupq_n_f32(-10.0f);
+ const float32x4_t in0 = vld1q_f32(&input[0]);
+ const float32x4_t in1 = vld1q_f32(&input[4]);
+ const float32x4_t in2 = vld1q_f32(&input[8]);
+ const float32x4_t in3 = vld1q_f32(&input[12]);
+
+ const float32x4_t m4 = vmaxq_f32(vmaxq_f32(in0, in1), vmaxq_f32(in2, in3));
+#if AOM_ARCH_AARCH64
+ const float max_input = vmaxvq_f32(m4);
+#else
+ float32x2_t m2 = vmax_f32(vget_low_f32(m4), vget_high_f32(m4));
+ m2 = vpmax_f32(m2, m2);
+ const float max_input = vget_lane_f32(m2, 0);
+#endif
+ const float32x4_t v_max = vdupq_n_f32(max_input);
+
+ const float32x4_t exp0 =
+ approx_exp_neon(vmaxq_f32(vsubq_f32(in0, v_max), clipper));
+ const float32x4_t exp1 =
+ approx_exp_neon(vmaxq_f32(vsubq_f32(in1, v_max), clipper));
+ const float32x4_t exp2 =
+ approx_exp_neon(vmaxq_f32(vsubq_f32(in2, v_max), clipper));
+ const float32x4_t exp3 =
+ approx_exp_neon(vmaxq_f32(vsubq_f32(in3, v_max), clipper));
+
+ // Accumulate sequentially to guarantee 0-ULP parity with
+ // av1_nn_fast_softmax_16_c and 100% SHA-256 bitstream identity.
+ float sum_out = vgetq_lane_f32(exp0, 0) + vgetq_lane_f32(exp0, 1);
+ sum_out += vgetq_lane_f32(exp0, 2);
+ sum_out += vgetq_lane_f32(exp0, 3);
+ sum_out += vgetq_lane_f32(exp1, 0);
+ sum_out += vgetq_lane_f32(exp1, 1);
+ sum_out += vgetq_lane_f32(exp1, 2);
+ sum_out += vgetq_lane_f32(exp1, 3);
+ sum_out += vgetq_lane_f32(exp2, 0);
+ sum_out += vgetq_lane_f32(exp2, 1);
+ sum_out += vgetq_lane_f32(exp2, 2);
+ sum_out += vgetq_lane_f32(exp2, 3);
+ sum_out += vgetq_lane_f32(exp3, 0);
+ sum_out += vgetq_lane_f32(exp3, 1);
+ sum_out += vgetq_lane_f32(exp3, 2);
+ sum_out += vgetq_lane_f32(exp3, 3);
+
+#if AOM_ARCH_AARCH64
+ const float32x4_t v_sum = vdupq_n_f32(sum_out);
+ vst1q_f32(&output[0], vdivq_f32(exp0, v_sum));
+ vst1q_f32(&output[4], vdivq_f32(exp1, v_sum));
+ vst1q_f32(&output[8], vdivq_f32(exp2, v_sum));
+ vst1q_f32(&output[12], vdivq_f32(exp3, v_sum));
+#else
+ vst1q_f32(&output[0], exp0);
+ vst1q_f32(&output[4], exp1);
+ vst1q_f32(&output[8], exp2);
+ vst1q_f32(&output[12], exp3);
+ int i = 0;
+ do {
+ output[i] /= sum_out;
+ ++i;
+ } while (i != 16);
+#endif
+}
diff --git a/test/av1_nn_predict_test.cc b/test/av1_nn_predict_test.cc
index b05b83cbf0..7788060f92 100644
--- a/test/av1_nn_predict_test.cc
+++ b/test/av1_nn_predict_test.cc
@@ -182,7 +182,10 @@ static const NN_CONFIG kShapes[] = {
{ 8, 4, 1, { 16 }, {}, {} }, { 8, 1, 1, { 32 }, {}, {} },
{ 9, 3, 1, { 32 }, {}, {} }, { 8, 4, 0, { 0 }, {}, {} },
{ 8, 8, 0, { 0 }, {}, {} }, { 4, 4, 1, { 8 }, {}, {} },
- { 4, 3, 0, { 64 }, {}, {} },
+ { 4, 3, 0, { 64 }, {}, {} }, { 9, 5, 0, { 0 }, {}, {} },
+ { 6, 5, 0, { 0 }, {}, {} }, { 3, 2, 0, { 0 }, {}, {} },
+ { 13, 13, 1, { 13 }, {}, {} }, { 6, 5, 1, { 5 }, {}, {} },
+ { 32, 8, 0, { 0 }, {}, {} },
};
void NnPredictTest::RunNnPredictTest_all(const NN_CONFIG *const shapes,
diff --git a/test/av1_softmax_test.cc b/test/av1_softmax_test.cc
index 18af945eb9..c223235f45 100644
--- a/test/av1_softmax_test.cc
+++ b/test/av1_softmax_test.cc
@@ -119,4 +119,10 @@ INSTANTIATE_TEST_SUITE_P(
SSE3, FastSoftmaxTest,
::testing::Values(FastSoftmaxTestParams(av1_nn_fast_softmax_16_sse3, 16)));
#endif
+
+#if HAVE_NEON && !CONFIG_EXCLUDE_SIMD_MISMATCH
+INSTANTIATE_TEST_SUITE_P(
+ NEON, FastSoftmaxTest,
+ ::testing::Values(FastSoftmaxTestParams(av1_nn_fast_softmax_16_neon, 16)));
+#endif
} // namespace