Commit 66f75fbccc for openssl.org
commit 66f75fbcccf522e7ea5ead6beb1e6a84375e6326
Author: Alexander Troosh <trush@yandex.ru>
Date: Fri Oct 10 13:58:59 2025 +0300
GCM/ghash specialization for the e2kv6 (Elbrus2000) arch
Co-authored-by: Gleb Popov <arrowd@FreeBSD.org>
Reviewed-by: Andrew Dinh <andrewd@openssl.org>
Reviewed-by: Igor Ustinov <igus@openssl.foundation>
MergeDate: Fri Jul 31 16:49:56 2026
(Merged from https://github.com/openssl/openssl/pull/31269)
diff --git a/crypto/modes/asm/ghash-e2kv6.c b/crypto/modes/asm/ghash-e2kv6.c
new file mode 100644
index 0000000000..25615693e2
--- /dev/null
+++ b/crypto/modes/asm/ghash-e2kv6.c
@@ -0,0 +1,180 @@
+/*
+ * Copyright 2026 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License"). You may not use
+ * this file except in compliance with the License. You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include <stdint.h>
+#include <e2kintrin.h>
+
+#include "crypto/modes.h"
+
+static __attribute__((__always_inline__)) inline __v2di reverse_vector(const __v2di in)
+{
+ __v2di fmt = __builtin_e2k_qppackdl(0x0001020304050607LL, 0x08090a0b0c0d0e0fLL);
+ return __builtin_e2k_qppermb(in, in, fmt);
+}
+
+static __attribute__((__always_inline__)) inline __v2di gcm_reduce(__v2di B0, __v2di B1)
+{
+ __v2di X0 = __builtin_e2k_qpsrlw(B1, 31);
+ __v2di X1 = __builtin_e2k_qpsllw(B1, 1);
+ __v2di X2 = __builtin_e2k_qpsrlw(B0, 31);
+ __v2di X3 = __builtin_e2k_qpsllw(B0, 1);
+
+ X3 = X3 | __builtin_e2k_qpshufb(X0, X0, __builtin_e2k_qppackdl(0x8080808080808080LL, 0x808080800f0e0d0cLL))
+ | __builtin_e2k_qpshufb(X2, X2, __builtin_e2k_qppackdl(0x0b0a090807060504LL, 0x0302010080808080LL));
+
+ X1 = X1 | __builtin_e2k_qpshufb(X0, X0, __builtin_e2k_qppackdl(0x0b0a090807060504LL, 0x0302010080808080LL));
+
+ X0 = __builtin_e2k_qpsllw(X1, 31) ^ __builtin_e2k_qpsllw(X1, 30) ^ __builtin_e2k_qpsllw(X1, 25);
+
+ X1 ^= __builtin_e2k_qpshufb(X0, X0, __builtin_e2k_qppackdl(0x0302010080808080LL, 0x8080808080808080LL));
+
+ X0 = X1 ^ X3 ^ __builtin_e2k_qpshufb(X0, X0, __builtin_e2k_qppackdl(0x808080800f0e0d0cLL, 0x0b0a090807060504LL));
+
+ X0 ^= __builtin_e2k_qpsrlw(X1, 7) ^ __builtin_e2k_qpsrlw(X1, 2) ^ __builtin_e2k_qpsrlw(X1, 1);
+
+ return X0;
+}
+
+static __attribute__((__always_inline__)) inline __v2di gcm_multiply(__v2di H, __v2di x)
+{
+ uint64_t Hh = H[1], Hl = H[0];
+ uint64_t xh = x[1], xl = x[0];
+
+ uint64_t T0h = __builtin_e2k_clmulh(Hh, xh), T0l = __builtin_e2k_clmull(Hh, xh);
+ uint64_t T1h = __builtin_e2k_clmulh(Hh, xl), T1l = __builtin_e2k_clmull(Hh, xl);
+ uint64_t T2h = __builtin_e2k_clmulh(Hl, xh), T2l = __builtin_e2k_clmull(Hl, xh);
+ uint64_t T3h = __builtin_e2k_clmulh(Hl, xl), T3l = __builtin_e2k_clmull(Hl, xl);
+
+ T1h = __builtin_e2k_pxord(T1h, T2h);
+ T1l = __builtin_e2k_pxord(T1l, T2l);
+
+ T0l = __builtin_e2k_pxord(T0l, T1h);
+ T3h = __builtin_e2k_pxord(T3h, T1l);
+
+ return gcm_reduce(__builtin_e2k_qppackdl(T0h, T0l), __builtin_e2k_qppackdl(T3h, T3l));
+}
+
+static __attribute__((__always_inline__)) inline __v2di gcm_multiply_x4(__v2di H1, __v2di H2, __v2di H3, __v2di H4,
+ __v2di X1, __v2di X2, __v2di X3, __v2di X4)
+{
+ /*
+ * Multiply with delayed reduction, algorithm by Krzysztof Jankowski
+ * and Pierre Laurent of Intel
+ */
+
+ const uint64_t loh = (__builtin_e2k_clmulh(H1[0], X1[0]) ^ __builtin_e2k_clmulh(H2[0], X2[0])) ^ (__builtin_e2k_clmulh(H3[0], X3[0]) ^ __builtin_e2k_clmulh(H4[0], X4[0]));
+ const uint64_t lol = (__builtin_e2k_clmull(H1[0], X1[0]) ^ __builtin_e2k_clmull(H2[0], X2[0])) ^ (__builtin_e2k_clmull(H3[0], X3[0]) ^ __builtin_e2k_clmull(H4[0], X4[0]));
+
+ const uint64_t hih = (__builtin_e2k_clmulh(H1[1], X1[1]) ^ __builtin_e2k_clmulh(H2[1], X2[1])) ^ (__builtin_e2k_clmulh(H3[1], X3[1]) ^ __builtin_e2k_clmulh(H4[1], X4[1]));
+ const uint64_t hil = (__builtin_e2k_clmull(H1[1], X1[1]) ^ __builtin_e2k_clmull(H2[1], X2[1])) ^ (__builtin_e2k_clmull(H3[1], X3[1]) ^ __builtin_e2k_clmull(H4[1], X4[1]));
+ uint64_t Th, Tl;
+
+ Th = __builtin_e2k_clmulh(H1[0] ^ H1[1], X1[0] ^ X1[1]);
+ Tl = __builtin_e2k_clmull(H1[0] ^ H1[1], X1[0] ^ X1[1]);
+
+ Th ^= __builtin_e2k_clmulh(H2[0] ^ H2[1], X2[0] ^ X2[1]);
+ Tl ^= __builtin_e2k_clmull(H2[0] ^ H2[1], X2[0] ^ X2[1]);
+
+ Th ^= __builtin_e2k_clmulh(H3[0] ^ H3[1], X3[0] ^ X3[1]);
+ Tl ^= __builtin_e2k_clmull(H3[0] ^ H3[1], X3[0] ^ X3[1]);
+
+ Th ^= __builtin_e2k_clmulh(H4[0] ^ H4[1], X4[0] ^ X4[1]);
+ Tl ^= __builtin_e2k_clmull(H4[0] ^ H4[1], X4[0] ^ X4[1]);
+
+ Th ^= loh;
+ Tl ^= lol;
+ Th ^= hih;
+ Tl ^= hil;
+
+ return gcm_reduce(__builtin_e2k_qppackdl(hih, hil ^ Th),
+ __builtin_e2k_qppackdl(loh ^ Tl, lol));
+}
+
+/*##############################################################################
+# void gcm_init_e2kv6_clmul(u128 Htable[16],const uint64_t H[2]);
+#
+# input: 128-bit H - secret parameter E(K,0^128)
+# output: precomputed table filled with degrees of twisted H;
+# H is twisted to handle reverse bitness of GHASH;
+# only few of 16 slots of Htable[16] are used;
+# data is opaque to outside world (which allows to
+# optimize the code independently);
+#
+*/
+void gcm_init_e2kv6_clmul(u128 Htable[16], const uint64_t H[2])
+{
+ __v2di *Hp = (__v2di *)Htable;
+ __v2di H1 = (__v2di) { H[1], H[0] }; /* H in LE, but need swap hi/lo */
+ __v2di H2 = gcm_multiply(H1, H1);
+ __v2di H3 = gcm_multiply(H1, H2);
+ __v2di H4 = gcm_multiply(H2, H2);
+
+ Hp[0] = H1;
+ Hp[1] = H2;
+ Hp[2] = H3;
+ Hp[3] = H4;
+}
+
+/*##############################################################################
+# void gcm_gmult_e2kv6_clmul(uint64_t Xi[2],const u128 Htable[16]);
+#
+# input: Xi - current hash value;
+# Htable - table precomputed in gcm_init_e2kv6_clmul;
+# output: Xi - next hash value Xi;
+*/
+void gcm_gmult_e2kv6_clmul(uint64_t Xi[2], const u128 Htable[16])
+{
+ __v2di *Xp = (__v2di *)Xi;
+ __v2di *Hp = (__v2di *)Htable;
+ *Xp = reverse_vector(gcm_multiply(Hp[0], reverse_vector(*Xp)));
+}
+
+/*##############################################################################
+# void gcm_ghash_e2kv6_clmul(uint64_t Xi[2], const u128 Htable[16],
+# const uint8_t *inp,size_t len);
+#
+# input: table precomputed in gcm_init_e2kv6_clmul;
+# current hash value Xi;
+# pointer to input data;
+# length of input data in bytes, but divisible by block size;
+# output: next hash value Xi;
+*/
+void gcm_ghash_e2kv6_clmul(uint64_t Xi[2], const u128 Htable[16],
+ const uint8_t *inp, size_t len)
+{
+ __v2di *Hp = (__v2di *)Htable;
+ __v2di *input = (__v2di *)inp;
+ __v2di x;
+ size_t i, blocks = (len >> 4);
+
+ x = reverse_vector(*(__v2di *)Xi);
+
+ while (blocks >= 4) {
+ __v2di m0 = reverse_vector(input[0]);
+ __v2di m1 = reverse_vector(input[1]);
+ __v2di m2 = reverse_vector(input[2]);
+ __v2di m3 = reverse_vector(input[3]);
+
+ x ^= m0;
+ x = gcm_multiply_x4(Hp[0], Hp[1], Hp[2], Hp[3], m3, m2, m1, x);
+
+ input += 4;
+ blocks -= 4;
+ }
+
+#pragma loop count(3)
+ for (i = 0; i < blocks; i++) {
+ __v2di m = reverse_vector(input[i]);
+
+ x ^= m;
+ x = gcm_multiply(Hp[0], x);
+ }
+
+ *(__v2di *)Xi = reverse_vector(x);
+}
diff --git a/crypto/modes/build.info b/crypto/modes/build.info
index 9fb2d503d8..cd4e0330da 100644
--- a/crypto/modes/build.info
+++ b/crypto/modes/build.info
@@ -46,6 +46,11 @@ IF[{- !$disabled{asm} -}]
$MODESASM_riscv64=ghash-riscv64.s ghash-riscv64-zvkb-zvbc.s ghash-riscv64-zvkg.s aes-gcm-riscv64-zvkb-zvkg-zvkned.s
$MODESDEF_riscv64=GHASH_ASM
+ $MODESASM_e2kv6=asm/ghash-e2kv6.c
+ $MODESDEF_e2kv6=GHASH_ASM
+ $MODESASM_e2kv7=asm/ghash-e2kv6.c
+ $MODESDEF_e2kv7=GHASH_ASM
+
# Now that we have defined all the arch specific variables, use the
# appropriate one, and define the appropriate macros
IF[$MODESASM_{- $target{asm_arch} -}]
diff --git a/crypto/modes/gcm128.c b/crypto/modes/gcm128.c
index 1b77c2e27e..b4cab1bc00 100644
--- a/crypto/modes/gcm128.c
+++ b/crypto/modes/gcm128.c
@@ -422,6 +422,12 @@ void gcm_init_rv64i_zvkg_zvkb(u128 Htable[16], const uint64_t Xi[2]);
void gcm_gmult_rv64i_zvkg(uint64_t Xi[2], const u128 Htable[16]);
void gcm_ghash_rv64i_zvkg(uint64_t Xi[2], const u128 Htable[16],
const uint8_t *inp, size_t len);
+#elif defined(__e2k__) && (__iset__ >= 6)
+#define GHASH_ASM_E2KV6
+void gcm_init_e2kv6_clmul(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_e2kv6_clmul(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_e2kv6_clmul(uint64_t Xi[2], const u128 Htable[16],
+ const uint8_t *inp, size_t len);
#endif
#endif
@@ -552,6 +558,11 @@ static void gcm_get_funcs(struct gcm_funcs_st *ctx)
}
}
return;
+#elif defined(GHASH_ASM_E2KV6)
+ ctx->ginit = gcm_init_e2kv6_clmul;
+ ctx->gmult = gcm_gmult_e2kv6_clmul;
+ ctx->ghash = gcm_ghash_e2kv6_clmul;
+ return;
#elif defined(GHASH_ASM)
/* all other architectures use the generic names */
ctx->gmult = gcm_gmult_4bit;
diff --git a/include/crypto/modes.h b/include/crypto/modes.h
index baa0f3a6c1..7076e19647 100644
--- a/include/crypto/modes.h
+++ b/include/crypto/modes.h
@@ -25,7 +25,7 @@
#define STRICT_ALIGNMENT 1
#ifndef PEDANTIC
-#if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || defined(__aarch64__) || defined(__s390__) || defined(__s390x__)
+#if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || defined(__aarch64__) || defined(__s390__) || defined(__s390x__) || defined(__e2k__)
#undef STRICT_ALIGNMENT
#endif
#endif
@@ -72,6 +72,9 @@
#define BSWAP4(x) ({ uint32_t ret_=(x); \
asm ("rev8 %0,%0; srli %0,%0,32"\
: "+&r"(ret_)); ret_; })
+#elif defined(__e2k__)
+#define BSWAP8(x) __builtin_bswap64(x)
+#define BSWAP4(x) __builtin_bswap32(x)
#endif
#elif defined(_MSC_VER)
#if _MSC_VER >= 1300