Commit 3c5af087 for tesseract
commit 3c5af087f6140d2154482bbf2c2d9e0442489e81
Author: Stefan Weil <sw@weilnetz.de>
Date: Sat Nov 14 12:10:29 2026 +0100
lstm: calculate several dot products at once with FMA and AVX512F
The functions RowDotProducts and CrossDotProducts, which were added for
NEON, calculated single dot products on x86. Add implementations for
FMA (with AVX) and for AVX512F with FAST_FLOAT.
They handle four rows of the matrix per pass with two sums for each row
(matrix times vector) or a block of 4 x 2 (FMA) or 4 x 4 (AVX512F) dot
products (sum of outer products), so the vectors are loaded less often
and more independent sums are calculated at the same time. The rest of
a row is loaded with a mask instead of a scalar loop, and the four sums
of a pass are reduced together.
A CPU with AVX2 and FMA, but without AVX512F, now uses DotProductFMA
and the new functions by default instead of DotProductAVX.
dotproduct=avx2 still selects the old function with single dot
products.
Training of the default network of tesstrain with a single thread is
1.5 times faster on an Intel Core i3-1115G4 (g++ 14: 24.3 s -> 16.0 s
for 1000 lines with AVX512F, 15.7 s with dotproduct=fma), and
recognition of 12 pages with tessdata_best/eng is 1.2 times faster
(22.7 s -> 18.6 s) with identical text results.
The order of the additions is different, so the results differ slightly
from those of the old code (like those of different dot product
functions do), but they are still deterministic.
Extend the unit test for the products of WeightMatrix with more sizes
and run it for dotproduct=fma and dotproduct=generic, too.
Add the microbenchmark which was used to choose the variants.
Assisted-by: Claude Fable 5.2 (Anthropic)
Signed-off-by: Stefan Weil <sw@weilnetz.de>
diff --git a/benchmark/lstm/x86-kernel-variants.cpp b/benchmark/lstm/x86-kernel-variants.cpp
new file mode 100644
index 00000000..98f8d7b6
--- /dev/null
+++ b/benchmark/lstm/x86-kernel-variants.cpp
@@ -0,0 +1,669 @@
+///////////////////////////////////////////////////////////////////////
+// File: x86-kernel-variants.cpp
+// Description: Microbenchmark for variants of the FMA and AVX512F
+// functions which calculate several dot products at once.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+///////////////////////////////////////////////////////////////////////
+
+// This standalone program (x86_64 only) compares the time of single dot
+// products, called by a function pointer like in the generic code, with
+// variants which handle several rows per pass, with registers of 256 bits
+// (AVX and FMA) and of 512 bits (AVX512F). The sizes are those of the layers
+// of the default network of tesstrain. It was used to choose the
+// implementations of RowDotProductsFMA, CrossDotProductsFMA (src/arch/
+// dotproductfma.cpp), RowDotProductsAVX512F and CrossDotProductsAVX512F
+// (src/arch/dotproductavx512.cpp) and can be used to try other variants on
+// a new CPU or with a new compiler:
+//
+// c++ -std=c++17 -O3 -mavx2 -mfma -mavx512f -o x86-kernel-variants
+// x86-kernel-variants.cpp (one command line)
+// ./x86-kernel-variants
+//
+// It needs a CPU with AVX512F. Remove the variants for 512 bits (V16) from
+// the tables in main() to run it on a CPU without AVX512F.
+//
+// A variant RxA handles R rows per pass with A sums for each of them
+// (matrix times vector), a variant RxC calculates a block of R x C dot
+// products per pass (sum of outer products).
+
+#include <immintrin.h>
+#include <chrono>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <vector>
+
+#define NOINLINE __attribute__((noinline))
+#define INLINE inline __attribute__((always_inline))
+
+// 256 bits: AVX and FMA.
+struct V8 {
+ using R = __m256;
+ static constexpr int W = 8;
+ static INLINE R Zero() {
+ return _mm256_setzero_ps();
+ }
+ static INLINE R Load(const float *p) {
+ return _mm256_loadu_ps(p);
+ }
+ // Loads the first rem (< W) elements, the others are 0 and are not read.
+ static INLINE R LoadTail(const float *p, int rem) {
+ alignas(32) static const int32_t kMask[16] = {-1, -1, -1, -1, -1, -1, -1, -1,
+ 0, 0, 0, 0, 0, 0, 0, 0};
+ return _mm256_maskload_ps(
+ p, _mm256_loadu_si256(reinterpret_cast<const __m256i *>(kMask + 8 - rem)));
+ }
+ static INLINE R Fma(R a, R b, R c) {
+ return _mm256_fmadd_ps(a, b, c);
+ }
+ static INLINE R Add(R a, R b) {
+ return _mm256_add_ps(a, b);
+ }
+ static INLINE float Sum(R a) {
+ __m128 s = _mm_add_ps(_mm256_castps256_ps128(a), _mm256_extractf128_ps(a, 1));
+ s = _mm_add_ps(s, _mm_movehl_ps(s, s));
+ s = _mm_add_ss(s, _mm_shuffle_ps(s, s, 1));
+ return _mm_cvtss_f32(s);
+ }
+};
+
+// 512 bits: AVX512F.
+struct V16 {
+ using R = __m512;
+ static constexpr int W = 16;
+ static INLINE R Zero() {
+ return _mm512_setzero_ps();
+ }
+ static INLINE R Load(const float *p) {
+ return _mm512_loadu_ps(p);
+ }
+ static INLINE R LoadTail(const float *p, int rem) {
+ return _mm512_maskz_loadu_ps(static_cast<__mmask16>((1u << rem) - 1), p);
+ }
+ static INLINE R Fma(R a, R b, R c) {
+ return _mm512_fmadd_ps(a, b, c);
+ }
+ static INLINE R Add(R a, R b) {
+ return _mm512_add_ps(a, b);
+ }
+ static INLINE float Sum(R a) {
+ return _mm512_reduce_add_ps(a);
+ }
+};
+
+// Dot product like DotProductFMA for FAST_FLOAT.
+NOINLINE float DotFMA(const float *u, const float *v, int n) {
+ const unsigned quot = n / 16;
+ const unsigned rem = n % 16;
+ __m256 t0 = _mm256_setzero_ps();
+ __m256 t1 = _mm256_setzero_ps();
+ for (unsigned k = 0; k < quot; k++) {
+ t0 = _mm256_fmadd_ps(_mm256_loadu_ps(u), _mm256_loadu_ps(v), t0);
+ t1 = _mm256_fmadd_ps(_mm256_loadu_ps(u + 8), _mm256_loadu_ps(v + 8), t1);
+ u += 16;
+ v += 16;
+ }
+ t0 = _mm256_hadd_ps(t0, t1);
+ alignas(32) float tmp[8];
+ _mm256_store_ps(tmp, t0);
+ float result =
+ tmp[0] + tmp[1] + tmp[2] + tmp[3] + tmp[4] + tmp[5] + tmp[6] + tmp[7];
+ for (unsigned k = 0; k < rem; k++) {
+ result += *u++ * *v++;
+ }
+ return result;
+}
+
+// Dot product like DotProductAVX512F for FAST_FLOAT.
+NOINLINE float DotAVX512F(const float *u, const float *v, int n) {
+ const unsigned quot = n / 16;
+ const unsigned rem = n % 16;
+ __m512 t0 = _mm512_setzero_ps();
+ for (unsigned k = 0; k < quot; k++) {
+ t0 = _mm512_fmadd_ps(_mm512_loadu_ps(u), _mm512_loadu_ps(v), t0);
+ u += 16;
+ v += 16;
+ }
+ float result = _mm512_reduce_add_ps(t0);
+ for (unsigned k = 0; k < rem; k++) {
+ result += *u++ * *v++;
+ }
+ return result;
+}
+
+typedef float (*DotFn)(const float *, const float *, int);
+volatile DotFn g_dot;
+
+// Single dot products like the generic code.
+NOINLINE void RowsSingle(const float *w, int w_stride, int num_rows,
+ const float *u, int n, float *v) {
+ DotFn dot = g_dot;
+ for (int i = 0; i < num_rows; i++) {
+ v[i] = dot(w + static_cast<size_t>(i) * w_stride, u, n);
+ }
+}
+
+NOINLINE void CrossSingle(const float *u, int num_u, const float *v, int num_v,
+ int n, float *out, int out_stride) {
+ DotFn dot = g_dot;
+ for (int i = 0; i < num_u; i++) {
+ for (int j = 0; j < num_v; j++) {
+ out[static_cast<size_t>(i) * out_stride + j] =
+ dot(u + static_cast<size_t>(i) * n, v + static_cast<size_t>(j) * n, n);
+ }
+ }
+}
+
+// One row with two sums, for the rows which are left.
+template <class V>
+static INLINE float Dot1x2(const float *w, const float *u, int n) {
+ typename V::R a = V::Zero(), b = a;
+ int j = 0;
+ for (; j + 2 * V::W <= n; j += 2 * V::W) {
+ a = V::Fma(V::Load(w + j), V::Load(u + j), a);
+ b = V::Fma(V::Load(w + j + V::W), V::Load(u + j + V::W), b);
+ }
+ if (j + V::W <= n) {
+ a = V::Fma(V::Load(w + j), V::Load(u + j), a);
+ j += V::W;
+ }
+ if (j < n) {
+ b = V::Fma(V::LoadTail(w + j, n - j), V::LoadTail(u + j, n - j), b);
+ }
+ return V::Sum(V::Add(a, b));
+}
+
+// Four rows per pass, one sum for each row.
+template <class V>
+static INLINE void Rows4x1(const float *w, int w_stride, int num_rows,
+ const float *u, int n, float *v) {
+ int i = 0;
+ for (; i + 4 <= num_rows; i += 4) {
+ const float *w0 = w + static_cast<size_t>(i) * w_stride;
+ const float *w1 = w0 + w_stride;
+ const float *w2 = w1 + w_stride;
+ const float *w3 = w2 + w_stride;
+ typename V::R a0 = V::Zero(), a1 = a0, a2 = a0, a3 = a0;
+ int j = 0;
+ for (; j + V::W <= n; j += V::W) {
+ typename V::R x = V::Load(u + j);
+ a0 = V::Fma(V::Load(w0 + j), x, a0);
+ a1 = V::Fma(V::Load(w1 + j), x, a1);
+ a2 = V::Fma(V::Load(w2 + j), x, a2);
+ a3 = V::Fma(V::Load(w3 + j), x, a3);
+ }
+ if (j < n) {
+ int rem = n - j;
+ typename V::R x = V::LoadTail(u + j, rem);
+ a0 = V::Fma(V::LoadTail(w0 + j, rem), x, a0);
+ a1 = V::Fma(V::LoadTail(w1 + j, rem), x, a1);
+ a2 = V::Fma(V::LoadTail(w2 + j, rem), x, a2);
+ a3 = V::Fma(V::LoadTail(w3 + j, rem), x, a3);
+ }
+ v[i] = V::Sum(a0);
+ v[i + 1] = V::Sum(a1);
+ v[i + 2] = V::Sum(a2);
+ v[i + 3] = V::Sum(a3);
+ }
+ for (; i < num_rows; i++) {
+ v[i] = Dot1x2<V>(w + static_cast<size_t>(i) * w_stride, u, n);
+ }
+}
+
+// Four rows per pass, two sums for each row.
+template <class V>
+static INLINE void Rows4x2(const float *w, int w_stride, int num_rows,
+ const float *u, int n, float *v) {
+ int i = 0;
+ for (; i + 4 <= num_rows; i += 4) {
+ const float *w0 = w + static_cast<size_t>(i) * w_stride;
+ const float *w1 = w0 + w_stride;
+ const float *w2 = w1 + w_stride;
+ const float *w3 = w2 + w_stride;
+ typename V::R a0 = V::Zero(), a1 = a0, a2 = a0, a3 = a0;
+ typename V::R b0 = a0, b1 = a0, b2 = a0, b3 = a0;
+ int j = 0;
+ for (; j + 2 * V::W <= n; j += 2 * V::W) {
+ typename V::R x = V::Load(u + j);
+ typename V::R y = V::Load(u + j + V::W);
+ a0 = V::Fma(V::Load(w0 + j), x, a0);
+ b0 = V::Fma(V::Load(w0 + j + V::W), y, b0);
+ a1 = V::Fma(V::Load(w1 + j), x, a1);
+ b1 = V::Fma(V::Load(w1 + j + V::W), y, b1);
+ a2 = V::Fma(V::Load(w2 + j), x, a2);
+ b2 = V::Fma(V::Load(w2 + j + V::W), y, b2);
+ a3 = V::Fma(V::Load(w3 + j), x, a3);
+ b3 = V::Fma(V::Load(w3 + j + V::W), y, b3);
+ }
+ if (j + V::W <= n) {
+ typename V::R x = V::Load(u + j);
+ a0 = V::Fma(V::Load(w0 + j), x, a0);
+ a1 = V::Fma(V::Load(w1 + j), x, a1);
+ a2 = V::Fma(V::Load(w2 + j), x, a2);
+ a3 = V::Fma(V::Load(w3 + j), x, a3);
+ j += V::W;
+ }
+ if (j < n) {
+ int rem = n - j;
+ typename V::R x = V::LoadTail(u + j, rem);
+ b0 = V::Fma(V::LoadTail(w0 + j, rem), x, b0);
+ b1 = V::Fma(V::LoadTail(w1 + j, rem), x, b1);
+ b2 = V::Fma(V::LoadTail(w2 + j, rem), x, b2);
+ b3 = V::Fma(V::LoadTail(w3 + j, rem), x, b3);
+ }
+ v[i] = V::Sum(V::Add(a0, b0));
+ v[i + 1] = V::Sum(V::Add(a1, b1));
+ v[i + 2] = V::Sum(V::Add(a2, b2));
+ v[i + 3] = V::Sum(V::Add(a3, b3));
+ }
+ for (; i < num_rows; i++) {
+ v[i] = Dot1x2<V>(w + static_cast<size_t>(i) * w_stride, u, n);
+ }
+}
+
+// Eight rows per pass, one sum for each row.
+template <class V>
+static INLINE void Rows8x1(const float *w, int w_stride, int num_rows,
+ const float *u, int n, float *v) {
+ int i = 0;
+ for (; i + 8 <= num_rows; i += 8) {
+ const float *w0 = w + static_cast<size_t>(i) * w_stride;
+ const float *w1 = w0 + w_stride;
+ const float *w2 = w1 + w_stride;
+ const float *w3 = w2 + w_stride;
+ const float *w4 = w3 + w_stride;
+ const float *w5 = w4 + w_stride;
+ const float *w6 = w5 + w_stride;
+ const float *w7 = w6 + w_stride;
+ typename V::R a0 = V::Zero(), a1 = a0, a2 = a0, a3 = a0;
+ typename V::R a4 = a0, a5 = a0, a6 = a0, a7 = a0;
+ int j = 0;
+ for (; j + V::W <= n; j += V::W) {
+ typename V::R x = V::Load(u + j);
+ a0 = V::Fma(V::Load(w0 + j), x, a0);
+ a1 = V::Fma(V::Load(w1 + j), x, a1);
+ a2 = V::Fma(V::Load(w2 + j), x, a2);
+ a3 = V::Fma(V::Load(w3 + j), x, a3);
+ a4 = V::Fma(V::Load(w4 + j), x, a4);
+ a5 = V::Fma(V::Load(w5 + j), x, a5);
+ a6 = V::Fma(V::Load(w6 + j), x, a6);
+ a7 = V::Fma(V::Load(w7 + j), x, a7);
+ }
+ if (j < n) {
+ int rem = n - j;
+ typename V::R x = V::LoadTail(u + j, rem);
+ a0 = V::Fma(V::LoadTail(w0 + j, rem), x, a0);
+ a1 = V::Fma(V::LoadTail(w1 + j, rem), x, a1);
+ a2 = V::Fma(V::LoadTail(w2 + j, rem), x, a2);
+ a3 = V::Fma(V::LoadTail(w3 + j, rem), x, a3);
+ a4 = V::Fma(V::LoadTail(w4 + j, rem), x, a4);
+ a5 = V::Fma(V::LoadTail(w5 + j, rem), x, a5);
+ a6 = V::Fma(V::LoadTail(w6 + j, rem), x, a6);
+ a7 = V::Fma(V::LoadTail(w7 + j, rem), x, a7);
+ }
+ v[i] = V::Sum(a0);
+ v[i + 1] = V::Sum(a1);
+ v[i + 2] = V::Sum(a2);
+ v[i + 3] = V::Sum(a3);
+ v[i + 4] = V::Sum(a4);
+ v[i + 5] = V::Sum(a5);
+ v[i + 6] = V::Sum(a6);
+ v[i + 7] = V::Sum(a7);
+ }
+ Rows4x1<V>(w + static_cast<size_t>(i) * w_stride, w_stride, num_rows - i, u,
+ n, v + i);
+}
+
+// Blocks of 4 x 2 dot products.
+template <class V>
+static INLINE void Cross4x2(const float *u, int num_u, const float *v,
+ int num_v, int n, float *out, int out_stride) {
+ int i = 0;
+ for (; i + 4 <= num_u; i += 4) {
+ const float *u0 = u + static_cast<size_t>(i) * n;
+ const float *u1 = u0 + n;
+ const float *u2 = u1 + n;
+ const float *u3 = u2 + n;
+ float *out0 = out + static_cast<size_t>(i) * out_stride;
+ float *out1 = out0 + out_stride;
+ float *out2 = out1 + out_stride;
+ float *out3 = out2 + out_stride;
+ int j = 0;
+ for (; j + 2 <= num_v; j += 2) {
+ const float *v0 = v + static_cast<size_t>(j) * n;
+ const float *v1 = v0 + n;
+ typename V::R a0 = V::Zero(), a1 = a0, b0 = a0, b1 = a0;
+ typename V::R c0 = a0, c1 = a0, d0 = a0, d1 = a0;
+ int k = 0;
+ for (; k + V::W <= n; k += V::W) {
+ typename V::R y0 = V::Load(v0 + k);
+ typename V::R y1 = V::Load(v1 + k);
+ typename V::R x = V::Load(u0 + k);
+ a0 = V::Fma(x, y0, a0);
+ a1 = V::Fma(x, y1, a1);
+ x = V::Load(u1 + k);
+ b0 = V::Fma(x, y0, b0);
+ b1 = V::Fma(x, y1, b1);
+ x = V::Load(u2 + k);
+ c0 = V::Fma(x, y0, c0);
+ c1 = V::Fma(x, y1, c1);
+ x = V::Load(u3 + k);
+ d0 = V::Fma(x, y0, d0);
+ d1 = V::Fma(x, y1, d1);
+ }
+ if (k < n) {
+ int rem = n - k;
+ typename V::R y0 = V::LoadTail(v0 + k, rem);
+ typename V::R y1 = V::LoadTail(v1 + k, rem);
+ typename V::R x = V::LoadTail(u0 + k, rem);
+ a0 = V::Fma(x, y0, a0);
+ a1 = V::Fma(x, y1, a1);
+ x = V::LoadTail(u1 + k, rem);
+ b0 = V::Fma(x, y0, b0);
+ b1 = V::Fma(x, y1, b1);
+ x = V::LoadTail(u2 + k, rem);
+ c0 = V::Fma(x, y0, c0);
+ c1 = V::Fma(x, y1, c1);
+ x = V::LoadTail(u3 + k, rem);
+ d0 = V::Fma(x, y0, d0);
+ d1 = V::Fma(x, y1, d1);
+ }
+ out0[j] = V::Sum(a0);
+ out0[j + 1] = V::Sum(a1);
+ out1[j] = V::Sum(b0);
+ out1[j + 1] = V::Sum(b1);
+ out2[j] = V::Sum(c0);
+ out2[j + 1] = V::Sum(c1);
+ out3[j] = V::Sum(d0);
+ out3[j + 1] = V::Sum(d1);
+ }
+ if (j < num_v) {
+ // The last row of v is a single vector for four rows of u.
+ float last[4];
+ Rows4x1<V>(u0, n, 4, v + static_cast<size_t>(j) * n, n, last);
+ out0[j] = last[0];
+ out1[j] = last[1];
+ out2[j] = last[2];
+ out3[j] = last[3];
+ }
+ }
+ // The remaining rows of u are single vectors for the rows of v.
+ for (; i < num_u; i++) {
+ Rows4x1<V>(v, n, num_v, u + static_cast<size_t>(i) * n, n,
+ out + static_cast<size_t>(i) * out_stride);
+ }
+}
+
+// Blocks of 4 x 4 dot products.
+template <class V>
+static INLINE void Cross4x4(const float *u, int num_u, const float *v,
+ int num_v, int n, float *out, int out_stride) {
+ int i = 0;
+ for (; i + 4 <= num_u; i += 4) {
+ const float *u0 = u + static_cast<size_t>(i) * n;
+ const float *u1 = u0 + n;
+ const float *u2 = u1 + n;
+ const float *u3 = u2 + n;
+ float *out0 = out + static_cast<size_t>(i) * out_stride;
+ float *out1 = out0 + out_stride;
+ float *out2 = out1 + out_stride;
+ float *out3 = out2 + out_stride;
+ int j = 0;
+ for (; j + 4 <= num_v; j += 4) {
+ const float *v0 = v + static_cast<size_t>(j) * n;
+ const float *v1 = v0 + n;
+ const float *v2 = v1 + n;
+ const float *v3 = v2 + n;
+ typename V::R a0 = V::Zero(), a1 = a0, a2 = a0, a3 = a0;
+ typename V::R b0 = a0, b1 = a0, b2 = a0, b3 = a0;
+ typename V::R c0 = a0, c1 = a0, c2 = a0, c3 = a0;
+ typename V::R d0 = a0, d1 = a0, d2 = a0, d3 = a0;
+#define CROSS4X4_STEP(LOAD) \
+ { \
+ typename V::R y0 = LOAD(v0); \
+ typename V::R y1 = LOAD(v1); \
+ typename V::R y2 = LOAD(v2); \
+ typename V::R y3 = LOAD(v3); \
+ typename V::R x = LOAD(u0); \
+ a0 = V::Fma(x, y0, a0); \
+ a1 = V::Fma(x, y1, a1); \
+ a2 = V::Fma(x, y2, a2); \
+ a3 = V::Fma(x, y3, a3); \
+ x = LOAD(u1); \
+ b0 = V::Fma(x, y0, b0); \
+ b1 = V::Fma(x, y1, b1); \
+ b2 = V::Fma(x, y2, b2); \
+ b3 = V::Fma(x, y3, b3); \
+ x = LOAD(u2); \
+ c0 = V::Fma(x, y0, c0); \
+ c1 = V::Fma(x, y1, c1); \
+ c2 = V::Fma(x, y2, c2); \
+ c3 = V::Fma(x, y3, c3); \
+ x = LOAD(u3); \
+ d0 = V::Fma(x, y0, d0); \
+ d1 = V::Fma(x, y1, d1); \
+ d2 = V::Fma(x, y2, d2); \
+ d3 = V::Fma(x, y3, d3); \
+ }
+ int k = 0;
+#define FULL(p) V::Load(p + k)
+ for (; k + V::W <= n; k += V::W) {
+ CROSS4X4_STEP(FULL)
+ }
+ if (k < n) {
+ int rem = n - k;
+#define TAIL(p) V::LoadTail(p + k, rem)
+ CROSS4X4_STEP(TAIL)
+ }
+ out0[j] = V::Sum(a0);
+ out0[j + 1] = V::Sum(a1);
+ out0[j + 2] = V::Sum(a2);
+ out0[j + 3] = V::Sum(a3);
+ out1[j] = V::Sum(b0);
+ out1[j + 1] = V::Sum(b1);
+ out1[j + 2] = V::Sum(b2);
+ out1[j + 3] = V::Sum(b3);
+ out2[j] = V::Sum(c0);
+ out2[j + 1] = V::Sum(c1);
+ out2[j + 2] = V::Sum(c2);
+ out2[j + 3] = V::Sum(c3);
+ out3[j] = V::Sum(d0);
+ out3[j + 1] = V::Sum(d1);
+ out3[j + 2] = V::Sum(d2);
+ out3[j + 3] = V::Sum(d3);
+ }
+ for (; j < num_v; j++) {
+ float last[4];
+ Rows4x1<V>(u0, n, 4, v + static_cast<size_t>(j) * n, n, last);
+ out0[j] = last[0];
+ out1[j] = last[1];
+ out2[j] = last[2];
+ out3[j] = last[3];
+ }
+ }
+ for (; i < num_u; i++) {
+ Rows4x1<V>(v, n, num_v, u + static_cast<size_t>(i) * n, n,
+ out + static_cast<size_t>(i) * out_stride);
+ }
+}
+
+#define ROWS(NAME, IMPL, V) \
+ NOINLINE void NAME(const float *w, int w_stride, int num_rows, \
+ const float *u, int n, float *v) { \
+ IMPL<V>(w, w_stride, num_rows, u, n, v); \
+ }
+ROWS(Rows4x1_256, Rows4x1, V8)
+ROWS(Rows4x2_256, Rows4x2, V8)
+ROWS(Rows8x1_256, Rows8x1, V8)
+ROWS(Rows4x1_512, Rows4x1, V16)
+ROWS(Rows4x2_512, Rows4x2, V16)
+ROWS(Rows8x1_512, Rows8x1, V16)
+
+#define CROSS(NAME, IMPL, V) \
+ NOINLINE void NAME(const float *u, int num_u, const float *v, \
+ int num_v, int n, float *out, int out_stride) { \
+ IMPL<V>(u, num_u, v, num_v, n, out, out_stride); \
+ }
+CROSS(Cross4x2_256, Cross4x2, V8)
+CROSS(Cross4x4_256, Cross4x4, V8)
+CROSS(Cross4x2_512, Cross4x2, V16)
+CROSS(Cross4x4_512, Cross4x4, V16)
+
+// Returns the best time per call in ns of several measurements.
+template <class F>
+static double BestNs(F f, int reps) {
+ double best = 1e30;
+ for (int r = 0; r < 7; r++) {
+ auto t0 = std::chrono::steady_clock::now();
+ for (int k = 0; k < reps; k++) {
+ f();
+ }
+ double ns = std::chrono::duration<double, std::nano>(
+ std::chrono::steady_clock::now() - t0)
+ .count() /
+ reps;
+ if (ns < best) {
+ best = ns;
+ }
+ }
+ return best;
+}
+
+// Returns the largest difference of two results.
+static double MaxDiff(const std::vector<float> &a,
+ const std::vector<float> &b) {
+ double m = 0;
+ for (size_t i = 0; i < a.size(); i++) {
+ m = std::fmax(m, std::fabs(a[i] - b[i]));
+ }
+ return m;
+}
+
+// Returns deterministic pseudo-random numbers with 24 bits for the test data.
+// A linear congruential generator gives the same data on all platforms.
+static uint32_t Random() {
+ static uint32_t state = 1;
+ state = state * 1664525u + 1013904223u;
+ return state >> 8;
+}
+
+int main() {
+ auto fill = [](std::vector<float> &x) {
+ for (auto &e : x) {
+ e = Random() / (float)0x1000000 - 0.5f;
+ }
+ };
+ struct {
+ const char *name;
+ int ns, na;
+ } layers[] = {{"Conv16", 16, 9}, {"Lfys48", 48, 64}, {"Lfx96", 96, 144},
+ {"Lrx96", 96, 192}, {"Lfx192", 192, 288}, {"Out111", 111, 192},
+ {"odd", 113, 195}};
+ typedef void (*RowFn)(const float *, int, int, const float *, int, float *);
+ struct {
+ const char *name;
+ RowFn f;
+ DotFn dot;
+ } rf[] = {{"fma", RowsSingle, DotFMA}, {"4x1", Rows4x1_256, nullptr},
+ {"4x2", Rows4x2_256, nullptr}, {"8x1", Rows8x1_256, nullptr},
+ {"avx512", RowsSingle, DotAVX512F}, {"4x1", Rows4x1_512, nullptr},
+ {"4x2", Rows4x2_512, nullptr}, {"8x1", Rows8x1_512, nullptr}};
+ for (int backward = 0; backward < 2; backward++) {
+ printf(backward
+ ? "u*W backward (4 transposed gate matrices), us per timestep\n"
+ : "W*u forward (4 gate matrices in turn), us per timestep\n");
+ printf(" %-7s", "layer");
+ for (auto &f : rf) {
+ printf(" %6s", f.name);
+ }
+ printf("\n");
+ for (auto &L : layers) {
+ // Forward: ns rows with na inputs and a bias. Backward: na rows with
+ // ns elements.
+ int rows = backward ? L.na : L.ns;
+ int n = backward ? L.ns : L.na;
+ int stride = backward ? L.ns : L.na + 1;
+ std::vector<float> w(4 * (rows + 1) * stride), u(n), ref(4 * rows),
+ v(4 * rows);
+ fill(w);
+ fill(u);
+ printf(" %-7s", L.name);
+ bool first = true;
+ for (auto &f : rf) {
+ auto &dst = first ? ref : v;
+ if (f.dot) {
+ g_dot = f.dot;
+ }
+ double t = BestNs(
+ [&] {
+ for (int g = 0; g < 4; g++) {
+ f.f(w.data() + g * (rows + 1) * stride, stride, rows, u.data(),
+ n, dst.data() + g * rows);
+ }
+ },
+ 3000);
+ printf(" %6.2f", t / 1e3);
+ if (!first && MaxDiff(ref, v) > 1e-3) {
+ printf("(DIFF %g)", MaxDiff(ref, v));
+ }
+ first = false;
+ }
+ printf("\n");
+ }
+ }
+ typedef void (*CrossFn)(const float *, int, const float *, int, int, float *,
+ int);
+ struct {
+ const char *name;
+ CrossFn f;
+ DotFn dot;
+ } cf[] = {{"fma", CrossSingle, DotFMA},
+ {"4x2", Cross4x2_256, nullptr},
+ {"4x4", Cross4x4_256, nullptr},
+ {"avx512", CrossSingle, DotAVX512F},
+ {"4x2", Cross4x2_512, nullptr},
+ {"4x4", Cross4x4_512, nullptr}};
+ printf("gradient sum (one gate) for T timesteps, ms per call\n");
+ printf(" %-7s %4s", "layer", "T");
+ for (auto &f : cf) {
+ printf(" %6s", f.name);
+ }
+ printf("\n");
+ for (int T : {37, 100, 300, 1000}) {
+ for (auto &L : layers) {
+ int os = L.na + 1;
+ std::vector<float> u(L.ns * T), vv(L.na * T), ref(L.ns * os),
+ d(L.ns * os);
+ fill(u);
+ fill(vv);
+ printf(" %-7s %4d", L.name, T);
+ bool first = true;
+ for (auto &f : cf) {
+ auto &dst = first ? ref : d;
+ if (f.dot) {
+ g_dot = f.dot;
+ }
+ double t = BestNs(
+ [&] { f.f(u.data(), L.ns, vv.data(), L.na, T, dst.data(), os); },
+ 20);
+ printf(" %6.3f", t / 1e6);
+ if (!first && MaxDiff(ref, d) > 2e-3) {
+ printf("(DIFF %g)", MaxDiff(ref, d));
+ }
+ first = false;
+ }
+ printf("\n");
+ }
+ }
+ return 0;
+}
diff --git a/src/arch/dotproduct.h b/src/arch/dotproduct.h
index 79d7bc0b..1c1c5c52 100644
--- a/src/arch/dotproduct.h
+++ b/src/arch/dotproduct.h
@@ -51,6 +51,18 @@ void CrossDotProductsNEON(const float *u, int num_u, const float *v, int num_v,
int n, float *out, int out_stride);
#endif
+#if defined(FAST_FLOAT)
+// The same functions for x86 with FMA (and AVX) and with AVX512F.
+void RowDotProductsFMA(const float *w, int w_stride, int num_rows,
+ const float *u, int n, float *v);
+void CrossDotProductsFMA(const float *u, int num_u, const float *v, int num_v,
+ int n, float *out, int out_stride);
+void RowDotProductsAVX512F(const float *w, int w_stride, int num_rows,
+ const float *u, int n, float *v);
+void CrossDotProductsAVX512F(const float *u, int num_u, const float *v,
+ int num_v, int n, float *out, int out_stride);
+#endif
+
} // namespace tesseract.
#endif // TESSERACT_ARCH_DOTPRODUCT_H_
diff --git a/src/arch/dotproductavx512.cpp b/src/arch/dotproductavx512.cpp
index e1795f6b..80caf35e 100644
--- a/src/arch/dotproductavx512.cpp
+++ b/src/arch/dotproductavx512.cpp
@@ -22,6 +22,7 @@
#else
# include <immintrin.h>
+# include <cstddef> // for size_t
# include <cstdint>
# include "dotproduct.h"
@@ -47,6 +48,170 @@ float DotProductAVX512F(const float *u, const float *v, int n) {
}
return result;
}
+
+// Loads the first rem (< 16) elements of p. The other elements of the result
+// are 0, and the memory behind the first rem elements is not accessed.
+static inline __m512 LoadTail(const float *p, int rem) {
+ return _mm512_maskz_loadu_ps(static_cast<__mmask16>((1u << rem) - 1), p);
+}
+
+// Returns the sums of the elements of a, b, c and d. This is much faster
+// than four calls of _mm512_reduce_add_ps.
+static inline __m128 Sum4(__m512 a, __m512 b, __m512 c, __m512 d) {
+ auto half = [](__m512 x) {
+ __m256 y = _mm256_add_ps(
+ _mm512_castps512_ps256(x),
+ _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(x), 1)));
+ return _mm_add_ps(_mm256_castps256_ps128(y), _mm256_extractf128_ps(y, 1));
+ };
+ __m128 ab = _mm_hadd_ps(half(a), half(b));
+ __m128 cd = _mm_hadd_ps(half(c), half(d));
+ return _mm_hadd_ps(ab, cd);
+}
+
+// Computes v[i] = w[i] . u for num_rows rows of w with n used elements each.
+// Four rows are handled per pass, so u is loaded only once for them and
+// eight independent sums are calculated at the same time.
+void RowDotProductsAVX512F(const float *w, int w_stride, int num_rows,
+ const float *u, int n, float *v) {
+ int i = 0;
+ for (; i + 4 <= num_rows; i += 4) {
+ const float *w0 = w + static_cast<size_t>(i) * w_stride;
+ const float *w1 = w0 + w_stride;
+ const float *w2 = w1 + w_stride;
+ const float *w3 = w2 + w_stride;
+ __m512 a0 = _mm512_setzero_ps();
+ __m512 a1 = a0, a2 = a0, a3 = a0;
+ __m512 b0 = a0, b1 = a0, b2 = a0, b3 = a0;
+ int j = 0;
+ for (; j + 32 <= n; j += 32) {
+ __m512 x = _mm512_loadu_ps(u + j);
+ __m512 y = _mm512_loadu_ps(u + j + 16);
+ a0 = _mm512_fmadd_ps(_mm512_loadu_ps(w0 + j), x, a0);
+ b0 = _mm512_fmadd_ps(_mm512_loadu_ps(w0 + j + 16), y, b0);
+ a1 = _mm512_fmadd_ps(_mm512_loadu_ps(w1 + j), x, a1);
+ b1 = _mm512_fmadd_ps(_mm512_loadu_ps(w1 + j + 16), y, b1);
+ a2 = _mm512_fmadd_ps(_mm512_loadu_ps(w2 + j), x, a2);
+ b2 = _mm512_fmadd_ps(_mm512_loadu_ps(w2 + j + 16), y, b2);
+ a3 = _mm512_fmadd_ps(_mm512_loadu_ps(w3 + j), x, a3);
+ b3 = _mm512_fmadd_ps(_mm512_loadu_ps(w3 + j + 16), y, b3);
+ }
+ if (j + 16 <= n) {
+ __m512 x = _mm512_loadu_ps(u + j);
+ a0 = _mm512_fmadd_ps(_mm512_loadu_ps(w0 + j), x, a0);
+ a1 = _mm512_fmadd_ps(_mm512_loadu_ps(w1 + j), x, a1);
+ a2 = _mm512_fmadd_ps(_mm512_loadu_ps(w2 + j), x, a2);
+ a3 = _mm512_fmadd_ps(_mm512_loadu_ps(w3 + j), x, a3);
+ j += 16;
+ }
+ if (j < n) {
+ const int rem = n - j;
+ __m512 x = LoadTail(u + j, rem);
+ b0 = _mm512_fmadd_ps(LoadTail(w0 + j, rem), x, b0);
+ b1 = _mm512_fmadd_ps(LoadTail(w1 + j, rem), x, b1);
+ b2 = _mm512_fmadd_ps(LoadTail(w2 + j, rem), x, b2);
+ b3 = _mm512_fmadd_ps(LoadTail(w3 + j, rem), x, b3);
+ }
+ _mm_storeu_ps(v + i, Sum4(_mm512_add_ps(a0, b0), _mm512_add_ps(a1, b1),
+ _mm512_add_ps(a2, b2), _mm512_add_ps(a3, b3)));
+ }
+ for (; i < num_rows; i++) {
+ v[i] = DotProductAVX512F(w + static_cast<size_t>(i) * w_stride, u, n);
+ }
+}
+
+// Adds the products of four vectors x (loaded from four rows of u) with the
+// four vectors y0 ... y3 to 16 sums.
+# define CROSS_STEP(LOAD) \
+ { \
+ __m512 y0 = LOAD(v0); \
+ __m512 y1 = LOAD(v1); \
+ __m512 y2 = LOAD(v2); \
+ __m512 y3 = LOAD(v3); \
+ __m512 x = LOAD(u0); \
+ a0 = _mm512_fmadd_ps(x, y0, a0); \
+ a1 = _mm512_fmadd_ps(x, y1, a1); \
+ a2 = _mm512_fmadd_ps(x, y2, a2); \
+ a3 = _mm512_fmadd_ps(x, y3, a3); \
+ x = LOAD(u1); \
+ b0 = _mm512_fmadd_ps(x, y0, b0); \
+ b1 = _mm512_fmadd_ps(x, y1, b1); \
+ b2 = _mm512_fmadd_ps(x, y2, b2); \
+ b3 = _mm512_fmadd_ps(x, y3, b3); \
+ x = LOAD(u2); \
+ c0 = _mm512_fmadd_ps(x, y0, c0); \
+ c1 = _mm512_fmadd_ps(x, y1, c1); \
+ c2 = _mm512_fmadd_ps(x, y2, c2); \
+ c3 = _mm512_fmadd_ps(x, y3, c3); \
+ x = LOAD(u3); \
+ d0 = _mm512_fmadd_ps(x, y0, d0); \
+ d1 = _mm512_fmadd_ps(x, y1, d1); \
+ d2 = _mm512_fmadd_ps(x, y2, d2); \
+ d3 = _mm512_fmadd_ps(x, y3, d3); \
+ }
+# define LOAD_FULL(p) _mm512_loadu_ps((p) + k)
+# define LOAD_TAIL(p) LoadTail((p) + k, rem)
+
+// Computes out[i * out_stride + j] = u[i] . v[j] for the num_u rows of u and
+// the num_v rows of v, which all have n elements and are stored without gaps.
+// Four rows of u and four rows of v are handled per pass, so each of them is
+// loaded only once for 16 sums which are calculated at the same time.
+void CrossDotProductsAVX512F(const float *u, int num_u, const float *v,
+ int num_v, int n, float *out, int out_stride) {
+ int i = 0;
+ for (; i + 4 <= num_u; i += 4) {
+ const float *u0 = u + static_cast<size_t>(i) * n;
+ const float *u1 = u0 + n;
+ const float *u2 = u1 + n;
+ const float *u3 = u2 + n;
+ float *out0 = out + static_cast<size_t>(i) * out_stride;
+ float *out1 = out0 + out_stride;
+ float *out2 = out1 + out_stride;
+ float *out3 = out2 + out_stride;
+ int j = 0;
+ for (; j + 4 <= num_v; j += 4) {
+ const float *v0 = v + static_cast<size_t>(j) * n;
+ const float *v1 = v0 + n;
+ const float *v2 = v1 + n;
+ const float *v3 = v2 + n;
+ __m512 a0 = _mm512_setzero_ps();
+ __m512 a1 = a0, a2 = a0, a3 = a0;
+ __m512 b0 = a0, b1 = a0, b2 = a0, b3 = a0;
+ __m512 c0 = a0, c1 = a0, c2 = a0, c3 = a0;
+ __m512 d0 = a0, d1 = a0, d2 = a0, d3 = a0;
+ int k = 0;
+ for (; k + 16 <= n; k += 16) {
+ CROSS_STEP(LOAD_FULL)
+ }
+ if (k < n) {
+ const int rem = n - k;
+ CROSS_STEP(LOAD_TAIL)
+ }
+ _mm_storeu_ps(out0 + j, Sum4(a0, a1, a2, a3));
+ _mm_storeu_ps(out1 + j, Sum4(b0, b1, b2, b3));
+ _mm_storeu_ps(out2 + j, Sum4(c0, c1, c2, c3));
+ _mm_storeu_ps(out3 + j, Sum4(d0, d1, d2, d3));
+ }
+ for (; j < num_v; j++) {
+ // A remaining row of v is a single vector for the four rows of u.
+ float last[4];
+ RowDotProductsAVX512F(u0, n, 4, v + static_cast<size_t>(j) * n, n, last);
+ out0[j] = last[0];
+ out1[j] = last[1];
+ out2[j] = last[2];
+ out3[j] = last[3];
+ }
+ }
+ // The remaining rows of u are single vectors for the rows of v.
+ for (; i < num_u; i++) {
+ RowDotProductsAVX512F(v, n, num_v, u + static_cast<size_t>(i) * n, n,
+ out + static_cast<size_t>(i) * out_stride);
+ }
+}
+
+# undef CROSS_STEP
+# undef LOAD_FULL
+# undef LOAD_TAIL
# else
double DotProductAVX512F(const double *u, const double *v, int n) {
const unsigned quot = n / 8;
diff --git a/src/arch/dotproductfma.cpp b/src/arch/dotproductfma.cpp
index 01a78894..f986fcc5 100644
--- a/src/arch/dotproductfma.cpp
+++ b/src/arch/dotproductfma.cpp
@@ -22,6 +22,7 @@
#else
# include <immintrin.h>
+# include <cstddef> // for size_t
# include <cstdint>
# include "dotproduct.h"
@@ -56,6 +57,160 @@ float DotProductFMA(const float *u, const float *v, int n) {
}
return result;
}
+
+// Loads the first rem (< 8) elements of p. The other elements of the result
+// are 0, and the memory behind the first rem elements is not accessed.
+static inline __m256 LoadTail(const float *p, int rem) {
+ alignas(32) static const int32_t kMask[16] = {-1, -1, -1, -1, -1, -1, -1, -1,
+ 0, 0, 0, 0, 0, 0, 0, 0};
+ return _mm256_maskload_ps(
+ p, _mm256_loadu_si256(reinterpret_cast<const __m256i *>(kMask + 8 - rem)));
+}
+
+// Returns the sums of the elements of a, b, c and d. This is faster than
+// four single sums.
+static inline __m128 Sum4(__m256 a, __m256 b, __m256 c, __m256 d) {
+ auto half = [](__m256 x) {
+ return _mm_add_ps(_mm256_castps256_ps128(x), _mm256_extractf128_ps(x, 1));
+ };
+ __m128 ab = _mm_hadd_ps(half(a), half(b));
+ __m128 cd = _mm_hadd_ps(half(c), half(d));
+ return _mm_hadd_ps(ab, cd);
+}
+
+// Computes v[i] = w[i] . u for num_rows rows of w with n used elements each.
+// Four rows are handled per pass, so u is loaded only once for them and
+// eight independent sums are calculated at the same time.
+void RowDotProductsFMA(const float *w, int w_stride, int num_rows,
+ const float *u, int n, float *v) {
+ int i = 0;
+ for (; i + 4 <= num_rows; i += 4) {
+ const float *w0 = w + static_cast<size_t>(i) * w_stride;
+ const float *w1 = w0 + w_stride;
+ const float *w2 = w1 + w_stride;
+ const float *w3 = w2 + w_stride;
+ __m256 a0 = _mm256_setzero_ps();
+ __m256 a1 = a0, a2 = a0, a3 = a0;
+ __m256 b0 = a0, b1 = a0, b2 = a0, b3 = a0;
+ int j = 0;
+ for (; j + 16 <= n; j += 16) {
+ __m256 x = _mm256_loadu_ps(u + j);
+ __m256 y = _mm256_loadu_ps(u + j + 8);
+ a0 = _mm256_fmadd_ps(_mm256_loadu_ps(w0 + j), x, a0);
+ b0 = _mm256_fmadd_ps(_mm256_loadu_ps(w0 + j + 8), y, b0);
+ a1 = _mm256_fmadd_ps(_mm256_loadu_ps(w1 + j), x, a1);
+ b1 = _mm256_fmadd_ps(_mm256_loadu_ps(w1 + j + 8), y, b1);
+ a2 = _mm256_fmadd_ps(_mm256_loadu_ps(w2 + j), x, a2);
+ b2 = _mm256_fmadd_ps(_mm256_loadu_ps(w2 + j + 8), y, b2);
+ a3 = _mm256_fmadd_ps(_mm256_loadu_ps(w3 + j), x, a3);
+ b3 = _mm256_fmadd_ps(_mm256_loadu_ps(w3 + j + 8), y, b3);
+ }
+ if (j + 8 <= n) {
+ __m256 x = _mm256_loadu_ps(u + j);
+ a0 = _mm256_fmadd_ps(_mm256_loadu_ps(w0 + j), x, a0);
+ a1 = _mm256_fmadd_ps(_mm256_loadu_ps(w1 + j), x, a1);
+ a2 = _mm256_fmadd_ps(_mm256_loadu_ps(w2 + j), x, a2);
+ a3 = _mm256_fmadd_ps(_mm256_loadu_ps(w3 + j), x, a3);
+ j += 8;
+ }
+ if (j < n) {
+ const int rem = n - j;
+ __m256 x = LoadTail(u + j, rem);
+ b0 = _mm256_fmadd_ps(LoadTail(w0 + j, rem), x, b0);
+ b1 = _mm256_fmadd_ps(LoadTail(w1 + j, rem), x, b1);
+ b2 = _mm256_fmadd_ps(LoadTail(w2 + j, rem), x, b2);
+ b3 = _mm256_fmadd_ps(LoadTail(w3 + j, rem), x, b3);
+ }
+ _mm_storeu_ps(v + i, Sum4(_mm256_add_ps(a0, b0), _mm256_add_ps(a1, b1),
+ _mm256_add_ps(a2, b2), _mm256_add_ps(a3, b3)));
+ }
+ for (; i < num_rows; i++) {
+ v[i] = DotProductFMA(w + static_cast<size_t>(i) * w_stride, u, n);
+ }
+}
+
+// Computes out[i * out_stride + j] = u[i] . v[j] for the num_u rows of u and
+// the num_v rows of v, which all have n elements and are stored without gaps.
+// Four rows of u and two rows of v are handled per pass, so each of them is
+// loaded only once for 8 sums which are calculated at the same time. Larger
+// blocks need more than the 16 registers which exist and are slower.
+void CrossDotProductsFMA(const float *u, int num_u, const float *v, int num_v,
+ int n, float *out, int out_stride) {
+ int i = 0;
+ for (; i + 4 <= num_u; i += 4) {
+ const float *u0 = u + static_cast<size_t>(i) * n;
+ const float *u1 = u0 + n;
+ const float *u2 = u1 + n;
+ const float *u3 = u2 + n;
+ float *out0 = out + static_cast<size_t>(i) * out_stride;
+ float *out1 = out0 + out_stride;
+ float *out2 = out1 + out_stride;
+ float *out3 = out2 + out_stride;
+ int j = 0;
+ for (; j + 2 <= num_v; j += 2) {
+ const float *v0 = v + static_cast<size_t>(j) * n;
+ const float *v1 = v0 + n;
+ __m256 a0 = _mm256_setzero_ps();
+ __m256 a1 = a0, b0 = a0, b1 = a0;
+ __m256 c0 = a0, c1 = a0, d0 = a0, d1 = a0;
+ int k = 0;
+ for (; k + 8 <= n; k += 8) {
+ __m256 y0 = _mm256_loadu_ps(v0 + k);
+ __m256 y1 = _mm256_loadu_ps(v1 + k);
+ __m256 x = _mm256_loadu_ps(u0 + k);
+ a0 = _mm256_fmadd_ps(x, y0, a0);
+ a1 = _mm256_fmadd_ps(x, y1, a1);
+ x = _mm256_loadu_ps(u1 + k);
+ b0 = _mm256_fmadd_ps(x, y0, b0);
+ b1 = _mm256_fmadd_ps(x, y1, b1);
+ x = _mm256_loadu_ps(u2 + k);
+ c0 = _mm256_fmadd_ps(x, y0, c0);
+ c1 = _mm256_fmadd_ps(x, y1, c1);
+ x = _mm256_loadu_ps(u3 + k);
+ d0 = _mm256_fmadd_ps(x, y0, d0);
+ d1 = _mm256_fmadd_ps(x, y1, d1);
+ }
+ if (k < n) {
+ const int rem = n - k;
+ __m256 y0 = LoadTail(v0 + k, rem);
+ __m256 y1 = LoadTail(v1 + k, rem);
+ __m256 x = LoadTail(u0 + k, rem);
+ a0 = _mm256_fmadd_ps(x, y0, a0);
+ a1 = _mm256_fmadd_ps(x, y1, a1);
+ x = LoadTail(u1 + k, rem);
+ b0 = _mm256_fmadd_ps(x, y0, b0);
+ b1 = _mm256_fmadd_ps(x, y1, b1);
+ x = LoadTail(u2 + k, rem);
+ c0 = _mm256_fmadd_ps(x, y0, c0);
+ c1 = _mm256_fmadd_ps(x, y1, c1);
+ x = LoadTail(u3 + k, rem);
+ d0 = _mm256_fmadd_ps(x, y0, d0);
+ d1 = _mm256_fmadd_ps(x, y1, d1);
+ }
+ // The sums for two rows of u are stored with one call.
+ __m128 ab = Sum4(a0, a1, b0, b1);
+ __m128 cd = Sum4(c0, c1, d0, d1);
+ _mm_storel_pi(reinterpret_cast<__m64 *>(out0 + j), ab);
+ _mm_storeh_pi(reinterpret_cast<__m64 *>(out1 + j), ab);
+ _mm_storel_pi(reinterpret_cast<__m64 *>(out2 + j), cd);
+ _mm_storeh_pi(reinterpret_cast<__m64 *>(out3 + j), cd);
+ }
+ if (j < num_v) {
+ // The last row of v is a single vector for the four rows of u.
+ float last[4];
+ RowDotProductsFMA(u0, n, 4, v + static_cast<size_t>(j) * n, n, last);
+ out0[j] = last[0];
+ out1[j] = last[1];
+ out2[j] = last[2];
+ out3[j] = last[3];
+ }
+ }
+ // The remaining rows of u are single vectors for the rows of v.
+ for (; i < num_u; i++) {
+ RowDotProductsFMA(v, n, num_v, u + static_cast<size_t>(i) * n, n,
+ out + static_cast<size_t>(i) * out_stride);
+ }
+}
#else
double DotProductFMA(const double *u, const double *v, int n) {
const unsigned quot = n / 8;
diff --git a/src/arch/simddetect.cpp b/src/arch/simddetect.cpp
index 5d520035..5b1f6b4e 100644
--- a/src/arch/simddetect.cpp
+++ b/src/arch/simddetect.cpp
@@ -211,6 +211,18 @@ static void SetDotProduct(DotProductFunction f, const IntSimdMatrix *m = nullptr
rows = RowDotProductsNEON;
cross = CrossDotProductsNEON;
}
+#endif
+#if defined(FAST_FLOAT) && defined(HAVE_FMA)
+ if (f == DotProductFMA) {
+ rows = RowDotProductsFMA;
+ cross = CrossDotProductsFMA;
+ }
+#endif
+#if defined(FAST_FLOAT) && defined(HAVE_AVX512F)
+ if (f == DotProductAVX512F) {
+ rows = RowDotProductsAVX512F;
+ cross = CrossDotProductsAVX512F;
+ }
#endif
DotProduct = f;
RowDotProducts = rows;
@@ -348,6 +360,11 @@ SIMDDetect::SIMDDetect() {
SetDotProduct(DotProductAVX512F, &IntSimdMatrix::intSimdMatrixAVX2);
#endif
#if defined(HAVE_AVX2)
+# if defined(HAVE_FMA)
+ } else if (avx2_available_ && fma_available_) {
+ // AVX2 and FMA detected (all known CPUs with AVX2 also support FMA).
+ SetDotProduct(DotProductFMA, &IntSimdMatrix::intSimdMatrixAVX2);
+# endif
} else if (avx2_available_) {
// AVX2 detected.
SetDotProduct(DotProductAVX, &IntSimdMatrix::intSimdMatrixAVX2);
diff --git a/unittest/weightmatrix_product_test.cc b/unittest/weightmatrix_product_test.cc
index ecbfc264..f0771497 100644
--- a/unittest/weightmatrix_product_test.cc
+++ b/unittest/weightmatrix_product_test.cc
@@ -16,6 +16,8 @@
#include "include_gunit.h"
#include "helpers.h" // for TRand
+#include "params.h" // for ParamUtils, GlobalParams
+#include "simddetect.h" // for SIMDDetect
#include "weightmatrix.h" // for WeightMatrix, TransposedArray
#include <cmath>
@@ -143,33 +145,80 @@ protected:
}
}
+ // Tests the products with vectors for many sizes.
+ void TestAllVectorProducts() {
+ // Sizes around the block sizes of the SIMD code (2 or 4 rows, 4, 8 or 16
+ // elements, which are used once or twice per pass) and some sizes of
+ // real networks.
+ const int kSizes[] = {1, 2, 3, 4, 5, 7, 8, 9, 11, 12, 13,
+ 16, 17, 31, 32, 33, 47, 48, 49, 63, 111};
+ for (int no : kSizes) {
+ for (int ni : kSizes) {
+ TestVectorProducts(no, ni);
+ }
+ }
+ TestVectorProducts(192, 288);
+ }
+
+ // Tests the sum of the outer products for many sizes.
+ void TestAllSumOuter() {
+ const int kOutputs[] = {1, 2, 3, 4, 5, 7, 8, 9, 13, 48};
+ const int kInputs[] = {1, 2, 3, 4, 5, 7, 8, 9, 13, 64};
+ const int kSamples[] = {1, 2, 3, 4, 5, 7, 8, 9,
+ 15, 16, 17, 31, 32, 33, 47, 100};
+ for (int no : kOutputs) {
+ for (int ni : kInputs) {
+ for (int num_samples : kSamples) {
+ TestSumOuter(no, ni, num_samples);
+ }
+ }
+ }
+ }
+
+ // Selects the functions for the dot products like the parameter dotproduct
+ // or the environment variable DOTPRODUCT do.
+ static void SelectDotProduct(const char *name) {
+ ASSERT_TRUE(ParamUtils::SetParam("dotproduct", name,
+ SET_PARAM_CONSTRAINT_NONE, GlobalParams()));
+ SIMDDetect::Update();
+ }
+
TRand randomizer_;
};
-// Sizes around the block sizes of the SIMD code (4 rows, 4 or 8 elements)
-// and some sizes of real networks.
-const int kSizes[] = {1, 2, 3, 4, 5, 7, 8, 9, 11, 12, 13, 16, 17, 31, 48, 111};
-
+// The first tests use the functions which were selected for the CPU.
TEST_F(WeightMatrixProductTest, VectorProducts) {
- for (int no : kSizes) {
- for (int ni : kSizes) {
- TestVectorProducts(no, ni);
- }
- }
- TestVectorProducts(192, 288);
+ TestAllVectorProducts();
}
TEST_F(WeightMatrixProductTest, SumOuterTransposed) {
- const int kOutputs[] = {1, 2, 3, 4, 5, 7, 8, 9, 13, 48};
- const int kInputs[] = {1, 2, 3, 4, 5, 7, 8, 9, 13, 64};
- const int kSamples[] = {1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 100};
- for (int no : kOutputs) {
- for (int ni : kInputs) {
- for (int num_samples : kSamples) {
- TestSumOuter(no, ni, num_samples);
- }
- }
+ TestAllSumOuter();
+}
+
+// The functions for FMA are also tested on a CPU for which other functions
+// were selected (AVX512F). This changes the selection for the tests which
+// follow.
+TEST_F(WeightMatrixProductTest, FMA) {
+#if defined(HAVE_FMA)
+ if (!SIMDDetect::IsFMAAvailable()) {
+ GTEST_LOG_(INFO) << "No FMA found! Not tested!";
+ GTEST_SKIP();
}
+ SelectDotProduct("fma");
+ TestAllVectorProducts();
+ TestAllSumOuter();
+#else
+ // The code was built without the functions for FMA.
+ GTEST_LOG_(INFO) << "FMA unsupported! Not tested!";
+ GTEST_SKIP();
+#endif
+}
+
+// The generic functions calculate single dot products.
+TEST_F(WeightMatrixProductTest, Generic) {
+ SelectDotProduct("generic");
+ TestAllVectorProducts();
+ TestAllSumOuter();
}
} // namespace