Commit 16234b9073 for openssl.org
commit 16234b9073dd20d59e00676e631a950c09443d5d
Author: Simo Sorce <simo@redhat.com>
Date: Thu Jun 11 16:44:12 2026 -0400
Consolidate AESNI hardware implementations
Move the AES-NI specific hardware implementations for GCM, CCM, and XTS modes
from individual mode files into a single consolidated file
(`cipher_aes_hw_aesni.c`). This groups architecture-specific optimizations
together, improving code organization and maintainability.
As part of this refactoring, the unused `keybits` parameter was removed from
several AES-NI provider functions, and necessary XTS initialization and
context copy functions were exported for shared use.
Signed-off-by: Simo Sorce <simo@redhat.com>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Norbert Pocs <norbertp@openssl.org>
Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
MergeDate: Sat Jun 27 09:05:48 2026
(Merged from https://github.com/openssl/openssl/pull/31472)
diff --git a/providers/implementations/ciphers/build.info b/providers/implementations/ciphers/build.info
index 54e1a11469..720874ba78 100644
--- a/providers/implementations/ciphers/build.info
+++ b/providers/implementations/ciphers/build.info
@@ -107,11 +107,11 @@ SOURCE[$AES_GOAL]=\
cipher_aes_hw_armv8.c cipher_aes_hw_rv32i.c cipher_aes_hw_rv64i.c \
cipher_aes_hw_s390x.c cipher_aes_hw_t4.c \
cipher_aes_xts.c cipher_aes_xts_hw.c \
- cipher_aes_gcm.c cipher_aes_gcm_hw.c cipher_aes_gcm_hw_aesni.c \
+ cipher_aes_gcm.c cipher_aes_gcm_hw.c \
cipher_aes_gcm_hw_armv8.c cipher_aes_gcm_hw_ppc.c \
cipher_aes_gcm_hw_rv32i.c cipher_aes_gcm_hw_rv64i.c \
cipher_aes_gcm_hw_s390x.c cipher_aes_gcm_hw_t4.c \
- cipher_aes_ccm.c cipher_aes_ccm_hw.c cipher_aes_ccm_hw_aesni.c \
+ cipher_aes_ccm.c cipher_aes_ccm_hw.c \
cipher_aes_ccm_hw_rv32i.c cipher_aes_ccm_hw_rv64i.c \
cipher_aes_ccm_hw_s390x.c cipher_aes_ccm_hw_t4.c \
cipher_aes_wrp.c \
diff --git a/providers/implementations/ciphers/cipher_aes.h b/providers/implementations/ciphers/cipher_aes.h
index c5dfbc59e2..313c91ec9c 100644
--- a/providers/implementations/ciphers/cipher_aes.h
+++ b/providers/implementations/ciphers/cipher_aes.h
@@ -86,8 +86,7 @@ int ossl_cipher_hw_aes_initkey(PROV_CIPHER_CTX *ctx,
void ossl_cipher_aes_copyctx(PROV_CIPHER_CTX *dst, const PROV_CIPHER_CTX *src);
#if defined(AESNI_CAPABLE)
-const PROV_CIPHER_HW *ossl_prov_cipher_hw_aesni(enum aes_modes mode,
- size_t keybits);
+const PROV_CIPHER_HW *ossl_prov_cipher_hw_aesni(enum aes_modes mode);
#elif defined(ARMv8_HWAES_CAPABLE)
const PROV_CIPHER_HW *ossl_prov_cipher_hw_arm(enum aes_modes mode,
size_t keybits);
diff --git a/providers/implementations/ciphers/cipher_aes_ccm.h b/providers/implementations/ciphers/cipher_aes_ccm.h
index 91495015a8..be690dadb9 100644
--- a/providers/implementations/ciphers/cipher_aes_ccm.h
+++ b/providers/implementations/ciphers/cipher_aes_ccm.h
@@ -56,7 +56,7 @@ int ossl_cipher_set_ccm_aes_initkey(PROV_CCM_CTX *ctx,
const PROV_CCM_HW *ossl_prov_aes_hw_ccm(size_t keylen);
#if defined(AESNI_CAPABLE)
-const PROV_CCM_HW *ossl_prov_aes_hw_ccm_aesni(size_t keybits);
+const PROV_CCM_HW *ossl_prov_aes_hw_ccm_aesni(void);
#endif
#if defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 32
const PROV_CCM_HW *ossl_prov_aes_hw_ccm_rv32i(size_t keybits);
diff --git a/providers/implementations/ciphers/cipher_aes_ccm_hw.c b/providers/implementations/ciphers/cipher_aes_ccm_hw.c
index 8ee89a1bd6..fa7318d460 100644
--- a/providers/implementations/ciphers/cipher_aes_ccm_hw.c
+++ b/providers/implementations/ciphers/cipher_aes_ccm_hw.c
@@ -73,7 +73,7 @@ const PROV_CCM_HW *ossl_prov_aes_hw_ccm(size_t keybits)
{
const PROV_CCM_HW *aes_ccm_hw = NULL;
#if defined(AESNI_CAPABLE)
- aes_ccm_hw = ossl_prov_aes_hw_ccm_aesni(keybits);
+ aes_ccm_hw = ossl_prov_aes_hw_ccm_aesni();
#elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 32
aes_ccm_hw = ossl_prov_aes_hw_ccm_rv32i(keybits);
#elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 64
diff --git a/providers/implementations/ciphers/cipher_aes_ccm_hw_aesni.c b/providers/implementations/ciphers/cipher_aes_ccm_hw_aesni.c
deleted file mode 100644
index 67644b324a..0000000000
--- a/providers/implementations/ciphers/cipher_aes_ccm_hw_aesni.c
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved.
- *
- * Licensed under the Apache License 2.0 (the "License"). You may not use
- * this file except in compliance with the License. You can obtain a copy
- * in the file LICENSE in the source distribution or at
- * https://www.openssl.org/source/license.html
- */
-
-/*-
- * AES-NI support for AES CCM.
- * This file is used by cipher_aes_ccm_hw.c
- */
-
-#include "internal/deprecated.h"
-#include "cipher_aes_ccm.h"
-
-#if defined(AESNI_CAPABLE)
-
-static int ccm_aesni_initkey(PROV_CCM_CTX *ctx, const unsigned char *key,
- size_t keylen)
-{
- return ossl_cipher_set_ccm_aes_initkey(ctx, key, keylen,
- aesni_set_encrypt_key, aesni_encrypt, aesni_ccm64_encrypt_blocks,
- aesni_ccm64_decrypt_blocks);
-}
-
-static const PROV_CCM_HW aesni_ccm = {
- ccm_aesni_initkey,
- ossl_ccm_generic_setiv,
- ossl_ccm_generic_setaad,
- ossl_ccm_generic_auth_encrypt,
- ossl_ccm_generic_auth_decrypt,
- ossl_ccm_generic_gettag
-};
-
-const PROV_CCM_HW *ossl_prov_aes_hw_ccm_aesni(size_t keybits)
-{
- if (AESNI_CAPABLE)
- return &aesni_ccm;
- return NULL;
-}
-
-#endif
diff --git a/providers/implementations/ciphers/cipher_aes_gcm.h b/providers/implementations/ciphers/cipher_aes_gcm.h
index d8ce0b4563..71ee4c97eb 100644
--- a/providers/implementations/ciphers/cipher_aes_gcm.h
+++ b/providers/implementations/ciphers/cipher_aes_gcm.h
@@ -54,7 +54,7 @@ int generic_aes_gcm_cipher_update(PROV_GCM_CTX *ctx, const unsigned char *in,
const PROV_GCM_HW *ossl_prov_aes_hw_gcm(size_t keybits);
#if defined(AESNI_CAPABLE)
-const PROV_GCM_HW *ossl_prov_aes_hw_gcm_aesni(size_t keybits);
+const PROV_GCM_HW *ossl_prov_aes_hw_gcm_aesni(void);
#endif
#if defined(AES_PMULL_CAPABLE) && defined(AES_GCM_ASM)
const PROV_GCM_HW *ossl_prov_aes_hw_gcm_armv8(size_t keybits);
diff --git a/providers/implementations/ciphers/cipher_aes_gcm_hw.c b/providers/implementations/ciphers/cipher_aes_gcm_hw.c
index bcdb879400..41ff71cd21 100644
--- a/providers/implementations/ciphers/cipher_aes_gcm_hw.c
+++ b/providers/implementations/ciphers/cipher_aes_gcm_hw.c
@@ -156,7 +156,7 @@ const PROV_GCM_HW *ossl_prov_aes_hw_gcm(size_t keybits)
const PROV_GCM_HW *aes_gcm_hw = NULL;
#if defined(AESNI_CAPABLE)
- aes_gcm_hw = ossl_prov_aes_hw_gcm_aesni(keybits);
+ aes_gcm_hw = ossl_prov_aes_hw_gcm_aesni();
#elif defined(AES_PMULL_CAPABLE) && defined(AES_GCM_ASM)
aes_gcm_hw = ossl_prov_aes_hw_gcm_armv8(keybits);
#elif defined(PPC_AES_GCM_CAPABLE) && defined(_ARCH_PPC64)
diff --git a/providers/implementations/ciphers/cipher_aes_gcm_hw_aesni.c b/providers/implementations/ciphers/cipher_aes_gcm_hw_aesni.c
deleted file mode 100644
index 2b8938f27b..0000000000
--- a/providers/implementations/ciphers/cipher_aes_gcm_hw_aesni.c
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Copyright 2001-2026 The OpenSSL Project Authors. All Rights Reserved.
- *
- * Licensed under the Apache License 2.0 (the "License"). You may not use
- * this file except in compliance with the License. You can obtain a copy
- * in the file LICENSE in the source distribution or at
- * https://www.openssl.org/source/license.html
- */
-
-/*-
- * AES-NI support for AES GCM.
- * This file is used by cipher_aes_gcm_hw.c
- */
-#include "internal/deprecated.h"
-#include "cipher_aes_gcm.h"
-
-#if defined(AESNI_CAPABLE)
-
-static int aesni_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key,
- size_t keylen)
-{
- return aes_gcm_hw_initkey(ctx, key, keylen, aesni_set_encrypt_key,
- aesni_encrypt, aesni_ctr32_encrypt_blocks);
-}
-
-static const PROV_GCM_HW aesni_gcm = {
- aesni_gcm_initkey,
- ossl_gcm_setiv,
- ossl_gcm_aad_update,
- generic_aes_gcm_cipher_update,
- ossl_gcm_cipher_final,
- ossl_gcm_one_shot
-};
-
-/*-
- * AVX512 VAES + VPCLMULDQD support for AES GCM.
- */
-
-#undef VAES_GCM_ENABLED
-#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64))
-#define VAES_GCM_ENABLED
-
-/* Returns non-zero when AVX512F + VAES + VPCLMULDQD combination is available */
-int ossl_vaes_vpclmulqdq_capable(void);
-
-void ossl_aes_gcm_encrypt_avx512(const void *ks, void *gcm128ctx,
- unsigned int *pblocklen, const unsigned char *in, size_t len,
- unsigned char *out);
-void ossl_aes_gcm_decrypt_avx512(const void *ks, void *gcm128ctx,
- unsigned int *pblocklen, const unsigned char *in, size_t len,
- unsigned char *out);
-
-void ossl_aes_gcm_init_avx512(const void *ks, void *gcm128ctx);
-void ossl_aes_gcm_setiv_avx512(const void *ks, void *gcm128ctx,
- const unsigned char *iv, size_t ivlen);
-void ossl_aes_gcm_update_aad_avx512(void *gcm128ctx, const unsigned char *aad,
- size_t aadlen);
-void ossl_aes_gcm_finalize_avx512(void *gcm128ctx, unsigned int pblocklen);
-
-void ossl_gcm_gmult_avx512(uint64_t Xi[2], const void *gcm128ctx);
-
-static int vaes_gcm_setkey(PROV_GCM_CTX *ctx, const unsigned char *key,
- size_t keylen)
-{
- GCM128_CONTEXT *gcmctx = &ctx->gcm;
- PROV_AES_GCM_CTX *actx = (PROV_AES_GCM_CTX *)ctx;
- AES_KEY *ks = &actx->ks.ks;
-
- aesni_set_encrypt_key(key, (int)(keylen * 8), ks);
- memset(gcmctx, 0, sizeof(*gcmctx));
- gcmctx->key = ks;
- ctx->key_set = 1;
-
- ossl_aes_gcm_init_avx512(ks, gcmctx);
-
- return 1;
-}
-
-static int vaes_gcm_setiv(PROV_GCM_CTX *ctx, const unsigned char *iv,
- size_t ivlen)
-{
- GCM128_CONTEXT *gcmctx = &ctx->gcm;
-
- gcmctx->Yi.u[0] = 0; /* Current counter */
- gcmctx->Yi.u[1] = 0;
- gcmctx->Xi.u[0] = 0; /* AAD hash */
- gcmctx->Xi.u[1] = 0;
- gcmctx->len.u[0] = 0; /* AAD length */
- gcmctx->len.u[1] = 0; /* Message length */
- gcmctx->ares = 0;
- gcmctx->mres = 0;
-
- /* IV is limited by 2^64 bits, thus 2^61 bytes */
- if (ivlen > (U64(1) << 61))
- return 0;
-
- ossl_aes_gcm_setiv_avx512(gcmctx->key, gcmctx, iv, ivlen);
-
- return 1;
-}
-
-static int vaes_gcm_aadupdate(PROV_GCM_CTX *ctx,
- const unsigned char *aad,
- size_t aad_len)
-{
- GCM128_CONTEXT *gcmctx = &ctx->gcm;
- uint64_t alen = gcmctx->len.u[0];
- unsigned int ares;
- size_t i, lenBlks;
-
- /* Bad sequence: call of AAD update after message processing */
- if (gcmctx->len.u[1] > 0)
- return 0;
-
- alen += aad_len;
- /* AAD is limited by 2^64 bits, thus 2^61 bytes */
- if ((alen > (U64(1) << 61)) || (alen < aad_len))
- return 0;
-
- gcmctx->len.u[0] = alen;
-
- ares = gcmctx->ares;
- /* Partial AAD block left from previous AAD update calls */
- if (ares > 0) {
- /*
- * Fill partial block buffer till full block
- * (note, the hash is stored reflected)
- */
- while (ares > 0 && aad_len > 0) {
- gcmctx->Xi.c[15 - ares] ^= *(aad++);
- --aad_len;
- ares = (ares + 1) % AES_BLOCK_SIZE;
- }
- /* Full block gathered */
- if (ares == 0) {
- ossl_gcm_gmult_avx512(gcmctx->Xi.u, gcmctx);
- } else { /* no more AAD */
- gcmctx->ares = ares;
- return 1;
- }
- }
-
- /* Bulk AAD processing */
- lenBlks = aad_len & ((size_t)(-AES_BLOCK_SIZE));
- if (lenBlks > 0) {
- ossl_aes_gcm_update_aad_avx512(gcmctx, aad, lenBlks);
- aad += lenBlks;
- aad_len -= lenBlks;
- }
-
- /* Add remaining AAD to the hash (note, the hash is stored reflected) */
- if (aad_len > 0) {
- ares = (unsigned int)aad_len;
- for (i = 0; i < aad_len; i++)
- gcmctx->Xi.c[15 - i] ^= aad[i];
- }
-
- gcmctx->ares = ares;
-
- return 1;
-}
-
-static int vaes_gcm_cipherupdate(PROV_GCM_CTX *ctx, const unsigned char *in,
- size_t len, unsigned char *out)
-{
- GCM128_CONTEXT *gcmctx = &ctx->gcm;
- uint64_t mlen = gcmctx->len.u[1];
-
- mlen += len;
- if (mlen > ((U64(1) << 36) - 32) || (mlen < len))
- return 0;
-
- gcmctx->len.u[1] = mlen;
-
- /* Finalize GHASH(AAD) if AAD partial blocks left unprocessed */
- if (gcmctx->ares > 0) {
- ossl_gcm_gmult_avx512(gcmctx->Xi.u, gcmctx);
- gcmctx->ares = 0;
- }
-
- if (ctx->enc)
- ossl_aes_gcm_encrypt_avx512(gcmctx->key, gcmctx, &gcmctx->mres, in, len, out);
- else
- ossl_aes_gcm_decrypt_avx512(gcmctx->key, gcmctx, &gcmctx->mres, in, len, out);
-
- return 1;
-}
-
-static int vaes_gcm_cipherfinal(PROV_GCM_CTX *ctx, unsigned char *tag)
-{
- GCM128_CONTEXT *gcmctx = &ctx->gcm;
- unsigned int *res = &gcmctx->mres;
-
- /* Finalize AAD processing */
- if (gcmctx->ares > 0)
- res = &gcmctx->ares;
-
- ossl_aes_gcm_finalize_avx512(gcmctx, *res);
-
- if (ctx->enc) {
- ctx->taglen = GCM_TAG_MAX_SIZE;
- memcpy(tag, gcmctx->Xi.c,
- ctx->taglen <= sizeof(gcmctx->Xi.c) ? ctx->taglen : sizeof(gcmctx->Xi.c));
- *res = 0;
- } else {
- return !CRYPTO_memcmp(gcmctx->Xi.c, tag, ctx->taglen);
- }
-
- return 1;
-}
-
-static const PROV_GCM_HW vaes_gcm = {
- vaes_gcm_setkey,
- vaes_gcm_setiv,
- vaes_gcm_aadupdate,
- vaes_gcm_cipherupdate,
- vaes_gcm_cipherfinal,
- ossl_gcm_one_shot
-};
-
-#endif
-
-const PROV_GCM_HW *ossl_prov_aes_hw_gcm_aesni(size_t keybits)
-{
-#ifdef VAES_GCM_ENABLED
- if (ossl_vaes_vpclmulqdq_capable())
- return &vaes_gcm;
-#endif
- if (AESNI_CAPABLE)
- return &aesni_gcm;
-
- return NULL;
-}
-
-#endif
diff --git a/providers/implementations/ciphers/cipher_aes_hw.c b/providers/implementations/ciphers/cipher_aes_hw.c
index 5ad1d81e94..60abfd1165 100644
--- a/providers/implementations/ciphers/cipher_aes_hw.c
+++ b/providers/implementations/ciphers/cipher_aes_hw.c
@@ -217,7 +217,7 @@ static const PROV_CIPHER_HW *ossl_prov_cipher_hw_aes_mode(enum aes_modes mode,
const PROV_CIPHER_HW *aes_hw_mode = NULL;
#if defined(AESNI_CAPABLE)
- aes_hw_mode = ossl_prov_cipher_hw_aesni(mode, keybits);
+ aes_hw_mode = ossl_prov_cipher_hw_aesni(mode);
#elif defined(ARMv8_HWAES_CAPABLE)
aes_hw_mode = ossl_prov_cipher_hw_arm(mode, keybits);
#elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 32
diff --git a/providers/implementations/ciphers/cipher_aes_hw_aesni.c b/providers/implementations/ciphers/cipher_aes_hw_aesni.c
index d434874a43..6264c88426 100644
--- a/providers/implementations/ciphers/cipher_aes_hw_aesni.c
+++ b/providers/implementations/ciphers/cipher_aes_hw_aesni.c
@@ -1,5 +1,5 @@
/*
- * Copyright 2001-2025 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2001-2026 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
@@ -14,9 +14,14 @@
#include "internal/deprecated.h"
#include "cipher_aes.h"
+#include "cipher_aes_gcm.h"
+#include "cipher_aes_ccm.h"
+#include "cipher_aes_xts.h"
#if defined(AESNI_CAPABLE)
+/* MODES: ecb, cbc, cfb, ofb, ctr */
+
/* generates AES round keys for AES-NI and VAES implementations */
static int cipher_hw_aesni_initkey(PROV_CIPHER_CTX *ctx,
const unsigned char *key, size_t keylen)
@@ -135,8 +140,7 @@ static const PROV_CIPHER_HW aesni_ctr = {
ossl_cipher_aes_copyctx
};
-const PROV_CIPHER_HW *ossl_prov_cipher_hw_aesni(enum aes_modes mode,
- size_t keybits)
+const PROV_CIPHER_HW *ossl_prov_cipher_hw_aesni(enum aes_modes mode)
{
if (AESNI_CAPABLE) {
switch (mode) {
@@ -165,4 +169,297 @@ const PROV_CIPHER_HW *ossl_prov_cipher_hw_aesni(enum aes_modes mode,
return NULL;
}
+/* MODES: GCM */
+
+static int aesni_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key,
+ size_t keylen)
+{
+ return aes_gcm_hw_initkey(ctx, key, keylen, aesni_set_encrypt_key,
+ aesni_encrypt, aesni_ctr32_encrypt_blocks);
+}
+
+static const PROV_GCM_HW aesni_gcm = {
+ aesni_gcm_initkey,
+ ossl_gcm_setiv,
+ ossl_gcm_aad_update,
+ generic_aes_gcm_cipher_update,
+ ossl_gcm_cipher_final,
+ ossl_gcm_one_shot
+};
+
+/*-
+ * AVX512 VAES + VPCLMULDQD support for AES GCM.
+ */
+
+#undef VAES_GCM_ENABLED
+#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64))
+#define VAES_GCM_ENABLED
+
+/* Returns non-zero when AVX512F + VAES + VPCLMULDQD combination is available */
+int ossl_vaes_vpclmulqdq_capable(void);
+
+void ossl_aes_gcm_encrypt_avx512(const void *ks, void *gcm128ctx,
+ unsigned int *pblocklen, const unsigned char *in, size_t len,
+ unsigned char *out);
+void ossl_aes_gcm_decrypt_avx512(const void *ks, void *gcm128ctx,
+ unsigned int *pblocklen, const unsigned char *in, size_t len,
+ unsigned char *out);
+
+void ossl_aes_gcm_init_avx512(const void *ks, void *gcm128ctx);
+void ossl_aes_gcm_setiv_avx512(const void *ks, void *gcm128ctx,
+ const unsigned char *iv, size_t ivlen);
+void ossl_aes_gcm_update_aad_avx512(void *gcm128ctx, const unsigned char *aad,
+ size_t aadlen);
+void ossl_aes_gcm_finalize_avx512(void *gcm128ctx, unsigned int pblocklen);
+
+void ossl_gcm_gmult_avx512(uint64_t Xi[2], const void *gcm128ctx);
+
+static int vaes_gcm_setkey(PROV_GCM_CTX *ctx, const unsigned char *key,
+ size_t keylen)
+{
+ GCM128_CONTEXT *gcmctx = &ctx->gcm;
+ PROV_AES_GCM_CTX *actx = (PROV_AES_GCM_CTX *)ctx;
+ AES_KEY *ks = &actx->ks.ks;
+
+ aesni_set_encrypt_key(key, (int)(keylen * 8), ks);
+ memset(gcmctx, 0, sizeof(*gcmctx));
+ gcmctx->key = ks;
+ ctx->key_set = 1;
+
+ ossl_aes_gcm_init_avx512(ks, gcmctx);
+
+ return 1;
+}
+
+static int vaes_gcm_setiv(PROV_GCM_CTX *ctx, const unsigned char *iv,
+ size_t ivlen)
+{
+ GCM128_CONTEXT *gcmctx = &ctx->gcm;
+
+ gcmctx->Yi.u[0] = 0; /* Current counter */
+ gcmctx->Yi.u[1] = 0;
+ gcmctx->Xi.u[0] = 0; /* AAD hash */
+ gcmctx->Xi.u[1] = 0;
+ gcmctx->len.u[0] = 0; /* AAD length */
+ gcmctx->len.u[1] = 0; /* Message length */
+ gcmctx->ares = 0;
+ gcmctx->mres = 0;
+
+ /* IV is limited by 2^64 bits, thus 2^61 bytes */
+ if (ivlen > (U64(1) << 61))
+ return 0;
+
+ ossl_aes_gcm_setiv_avx512(gcmctx->key, gcmctx, iv, ivlen);
+
+ return 1;
+}
+
+static int vaes_gcm_aadupdate(PROV_GCM_CTX *ctx,
+ const unsigned char *aad,
+ size_t aad_len)
+{
+ GCM128_CONTEXT *gcmctx = &ctx->gcm;
+ uint64_t alen = gcmctx->len.u[0];
+ unsigned int ares;
+ size_t i, lenBlks;
+
+ /* Bad sequence: call of AAD update after message processing */
+ if (gcmctx->len.u[1] > 0)
+ return 0;
+
+ alen += aad_len;
+ /* AAD is limited by 2^64 bits, thus 2^61 bytes */
+ if ((alen > (U64(1) << 61)) || (alen < aad_len))
+ return 0;
+
+ gcmctx->len.u[0] = alen;
+
+ ares = gcmctx->ares;
+ /* Partial AAD block left from previous AAD update calls */
+ if (ares > 0) {
+ /*
+ * Fill partial block buffer till full block
+ * (note, the hash is stored reflected)
+ */
+ while (ares > 0 && aad_len > 0) {
+ gcmctx->Xi.c[15 - ares] ^= *(aad++);
+ --aad_len;
+ ares = (ares + 1) % AES_BLOCK_SIZE;
+ }
+ /* Full block gathered */
+ if (ares == 0) {
+ ossl_gcm_gmult_avx512(gcmctx->Xi.u, gcmctx);
+ } else { /* no more AAD */
+ gcmctx->ares = ares;
+ return 1;
+ }
+ }
+
+ /* Bulk AAD processing */
+ lenBlks = aad_len & ((size_t)(-AES_BLOCK_SIZE));
+ if (lenBlks > 0) {
+ ossl_aes_gcm_update_aad_avx512(gcmctx, aad, lenBlks);
+ aad += lenBlks;
+ aad_len -= lenBlks;
+ }
+
+ /* Add remaining AAD to the hash (note, the hash is stored reflected) */
+ if (aad_len > 0) {
+ ares = (unsigned int)aad_len;
+ for (i = 0; i < aad_len; i++)
+ gcmctx->Xi.c[15 - i] ^= aad[i];
+ }
+
+ gcmctx->ares = ares;
+
+ return 1;
+}
+
+static int vaes_gcm_cipherupdate(PROV_GCM_CTX *ctx, const unsigned char *in,
+ size_t len, unsigned char *out)
+{
+ GCM128_CONTEXT *gcmctx = &ctx->gcm;
+ uint64_t mlen = gcmctx->len.u[1];
+
+ mlen += len;
+ if (mlen > ((U64(1) << 36) - 32) || (mlen < len))
+ return 0;
+
+ gcmctx->len.u[1] = mlen;
+
+ /* Finalize GHASH(AAD) if AAD partial blocks left unprocessed */
+ if (gcmctx->ares > 0) {
+ ossl_gcm_gmult_avx512(gcmctx->Xi.u, gcmctx);
+ gcmctx->ares = 0;
+ }
+
+ if (ctx->enc)
+ ossl_aes_gcm_encrypt_avx512(gcmctx->key, gcmctx, &gcmctx->mres, in, len, out);
+ else
+ ossl_aes_gcm_decrypt_avx512(gcmctx->key, gcmctx, &gcmctx->mres, in, len, out);
+
+ return 1;
+}
+
+static int vaes_gcm_cipherfinal(PROV_GCM_CTX *ctx, unsigned char *tag)
+{
+ GCM128_CONTEXT *gcmctx = &ctx->gcm;
+ unsigned int *res = &gcmctx->mres;
+
+ /* Finalize AAD processing */
+ if (gcmctx->ares > 0)
+ res = &gcmctx->ares;
+
+ ossl_aes_gcm_finalize_avx512(gcmctx, *res);
+
+ if (ctx->enc) {
+ ctx->taglen = GCM_TAG_MAX_SIZE;
+ memcpy(tag, gcmctx->Xi.c,
+ ctx->taglen <= sizeof(gcmctx->Xi.c) ? ctx->taglen : sizeof(gcmctx->Xi.c));
+ *res = 0;
+ } else {
+ return !CRYPTO_memcmp(gcmctx->Xi.c, tag, ctx->taglen);
+ }
+
+ return 1;
+}
+
+static const PROV_GCM_HW vaes_gcm = {
+ vaes_gcm_setkey,
+ vaes_gcm_setiv,
+ vaes_gcm_aadupdate,
+ vaes_gcm_cipherupdate,
+ vaes_gcm_cipherfinal,
+ ossl_gcm_one_shot
+};
+
+#endif
+
+const PROV_GCM_HW *ossl_prov_aes_hw_gcm_aesni(void)
+{
+#ifdef VAES_GCM_ENABLED
+ if (ossl_vaes_vpclmulqdq_capable())
+ return &vaes_gcm;
+#endif
+ if (AESNI_CAPABLE)
+ return &aesni_gcm;
+
+ return NULL;
+}
+
+/* MODES: CCM */
+
+static int ccm_aesni_initkey(PROV_CCM_CTX *ctx, const unsigned char *key,
+ size_t keylen)
+{
+ return ossl_cipher_set_ccm_aes_initkey(ctx, key, keylen,
+ aesni_set_encrypt_key, aesni_encrypt, aesni_ccm64_encrypt_blocks,
+ aesni_ccm64_decrypt_blocks);
+}
+
+static const PROV_CCM_HW aesni_ccm = {
+ ccm_aesni_initkey,
+ ossl_ccm_generic_setiv,
+ ossl_ccm_generic_setaad,
+ ossl_ccm_generic_auth_encrypt,
+ ossl_ccm_generic_auth_decrypt,
+ ossl_ccm_generic_gettag
+};
+
+const PROV_CCM_HW *ossl_prov_aes_hw_ccm_aesni(void)
+{
+ if (AESNI_CAPABLE)
+ return &aesni_ccm;
+ return NULL;
+}
+
+/* MODES: XTS */
+
+static int cipher_hw_aesni_xts_initkey(PROV_CIPHER_CTX *ctx,
+ const unsigned char *key, size_t keylen)
+{
+ void (*aesni_xts_enc)(const unsigned char *in,
+ unsigned char *out,
+ size_t length,
+ const AES_KEY *key1, const AES_KEY *key2,
+ const unsigned char iv[16]);
+ void (*aesni_xts_dec)(const unsigned char *in,
+ unsigned char *out,
+ size_t length,
+ const AES_KEY *key1, const AES_KEY *key2,
+ const unsigned char iv[16]);
+
+ aesni_xts_enc = aesni_xts_encrypt;
+ aesni_xts_dec = aesni_xts_decrypt;
+
+#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64))
+ if (aesni_xts_avx512_eligible()) {
+ if (keylen == 64) {
+ aesni_xts_enc = aesni_xts_256_encrypt_avx512;
+ aesni_xts_dec = aesni_xts_256_decrypt_avx512;
+ } else if (keylen == 32) {
+ aesni_xts_enc = aesni_xts_128_encrypt_avx512;
+ aesni_xts_dec = aesni_xts_128_decrypt_avx512;
+ }
+ }
+#endif
+
+ return ossl_cipher_set_aes_xts_initkey(ctx, key, keylen,
+ aesni_set_encrypt_key, aesni_set_decrypt_key,
+ aesni_encrypt, aesni_decrypt, aesni_xts_enc, aesni_xts_dec);
+}
+
+static const PROV_CIPHER_HW aesni_xts = {
+ cipher_hw_aesni_xts_initkey,
+ NULL,
+ ossl_cipher_hw_aes_xts_copyctx
+};
+
+const PROV_CIPHER_HW *ossl_prov_cipher_hw_aes_xts_aesni(void)
+{
+ if (AESNI_CAPABLE)
+ return &aesni_xts;
+ return NULL;
+}
+
#endif
diff --git a/providers/implementations/ciphers/cipher_aes_xts.h b/providers/implementations/ciphers/cipher_aes_xts.h
index a2421ae9ef..f103bad0a9 100644
--- a/providers/implementations/ciphers/cipher_aes_xts.h
+++ b/providers/implementations/ciphers/cipher_aes_xts.h
@@ -60,6 +60,20 @@ typedef struct prov_aes_xts_ctx_st {
} plat;
} PROV_AES_XTS_CTX;
+int ossl_cipher_set_aes_xts_initkey(PROV_CIPHER_CTX *ctx,
+ const unsigned char *key, size_t keylen,
+ aes_set_encrypt_key_fn fn_set_enc_key,
+ aes_set_encrypt_key_fn fn_set_dec_key,
+ aes_block128_f fn_block_enc, aes_block128_f fn_block_dec,
+ OSSL_xts_stream_fn fn_stream_enc, OSSL_xts_stream_fn fn_stream_dec);
+
+void ossl_cipher_hw_aes_xts_copyctx(PROV_CIPHER_CTX *dst,
+ const PROV_CIPHER_CTX *src);
+
+#if defined(AESNI_CAPABLE)
+const PROV_CIPHER_HW *ossl_prov_cipher_hw_aes_xts_aesni(void);
+#endif
+
#ifdef AES_XTS_S390X
int s390x_aes_xts_cipher_stream(PROV_AES_XTS_CTX *xctx,
unsigned char *out, size_t *outl,
diff --git a/providers/implementations/ciphers/cipher_aes_xts_hw.c b/providers/implementations/ciphers/cipher_aes_xts_hw.c
index b9e21df4fd..215d151d69 100644
--- a/providers/implementations/ciphers/cipher_aes_xts_hw.c
+++ b/providers/implementations/ciphers/cipher_aes_xts_hw.c
@@ -15,7 +15,7 @@
#include "cipher_aes_xts.h"
-static int cipher_set_aes_xts_initkey(PROV_CIPHER_CTX *ctx,
+int ossl_cipher_set_aes_xts_initkey(PROV_CIPHER_CTX *ctx,
const unsigned char *key, size_t keylen,
aes_set_encrypt_key_fn fn_set_enc_key,
aes_set_encrypt_key_fn fn_set_dec_key,
@@ -62,7 +62,7 @@ static int cipher_hw_aes_xts_generic_initkey(PROV_CIPHER_CTX *ctx,
#ifdef HWAES_xts_decrypt
stream_dec = HWAES_xts_decrypt;
#endif /* HWAES_xts_decrypt */
- return cipher_set_aes_xts_initkey(ctx, key, keylen,
+ return ossl_cipher_set_aes_xts_initkey(ctx, key, keylen,
HWAES_set_encrypt_key, HWAES_set_decrypt_key,
HWAES_encrypt, HWAES_decrypt, stream_enc, stream_dec);
}
@@ -72,7 +72,7 @@ static int cipher_hw_aes_xts_generic_initkey(PROV_CIPHER_CTX *ctx,
if (BSAES_CAPABLE) {
stream_enc = ossl_bsaes_xts_encrypt;
stream_dec = ossl_bsaes_xts_decrypt;
- return cipher_set_aes_xts_initkey(ctx, key, keylen,
+ return ossl_cipher_set_aes_xts_initkey(ctx, key, keylen,
AES_set_encrypt_key, AES_set_decrypt_key,
AES_encrypt, AES_decrypt, stream_enc, stream_dec);
}
@@ -80,18 +80,18 @@ static int cipher_hw_aes_xts_generic_initkey(PROV_CIPHER_CTX *ctx,
#ifdef VPAES_CAPABLE
if (VPAES_CAPABLE) {
- return cipher_set_aes_xts_initkey(ctx, key, keylen,
+ return ossl_cipher_set_aes_xts_initkey(ctx, key, keylen,
vpaes_set_encrypt_key, vpaes_set_decrypt_key,
vpaes_encrypt, vpaes_decrypt, stream_enc, stream_dec);
}
#endif /* VPAES_CAPABLE */
- return cipher_set_aes_xts_initkey(ctx, key, keylen,
+ return ossl_cipher_set_aes_xts_initkey(ctx, key, keylen,
AES_set_encrypt_key, AES_set_decrypt_key,
AES_encrypt, AES_decrypt, stream_enc, stream_dec);
}
-static void cipher_hw_aes_xts_copyctx(PROV_CIPHER_CTX *dst,
+void ossl_cipher_hw_aes_xts_copyctx(PROV_CIPHER_CTX *dst,
const PROV_CIPHER_CTX *src)
{
PROV_AES_XTS_CTX *sctx = (PROV_AES_XTS_CTX *)src;
@@ -102,56 +102,7 @@ static void cipher_hw_aes_xts_copyctx(PROV_CIPHER_CTX *dst,
dctx->xts.key2 = &dctx->ks2.ks;
}
-#if defined(AESNI_CAPABLE)
-
-static int cipher_hw_aesni_xts_initkey(PROV_CIPHER_CTX *ctx,
- const unsigned char *key, size_t keylen)
-{
- void (*aesni_xts_enc)(const unsigned char *in,
- unsigned char *out,
- size_t length,
- const AES_KEY *key1, const AES_KEY *key2,
- const unsigned char iv[16]);
- void (*aesni_xts_dec)(const unsigned char *in,
- unsigned char *out,
- size_t length,
- const AES_KEY *key1, const AES_KEY *key2,
- const unsigned char iv[16]);
-
- aesni_xts_enc = aesni_xts_encrypt;
- aesni_xts_dec = aesni_xts_decrypt;
-
-#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64))
- if (aesni_xts_avx512_eligible()) {
- if (keylen == 64) {
- aesni_xts_enc = aesni_xts_256_encrypt_avx512;
- aesni_xts_dec = aesni_xts_256_decrypt_avx512;
- } else if (keylen == 32) {
- aesni_xts_enc = aesni_xts_128_encrypt_avx512;
- aesni_xts_dec = aesni_xts_128_decrypt_avx512;
- }
- }
-#endif
-
- return cipher_set_aes_xts_initkey(ctx, key, keylen,
- aesni_set_encrypt_key, aesni_set_decrypt_key,
- aesni_encrypt, aesni_decrypt, aesni_xts_enc, aesni_xts_dec);
-}
-
-static const PROV_CIPHER_HW aesni_xts = {
- cipher_hw_aesni_xts_initkey,
- NULL,
- cipher_hw_aes_xts_copyctx
-};
-
-static const PROV_CIPHER_HW *ossl_prov_cipher_hw_aes_xts_aesni()
-{
- if (AESNI_CAPABLE)
- return &aesni_xts;
- return NULL;
-}
-
-#elif defined(SPARC_AES_CAPABLE)
+#if defined(SPARC_AES_CAPABLE)
static int cipher_hw_aes_xts_t4_initkey(PROV_CIPHER_CTX *ctx,
const unsigned char *key, size_t keylen)
@@ -173,7 +124,7 @@ static int cipher_hw_aes_xts_t4_initkey(PROV_CIPHER_CTX *ctx,
return 0;
}
- return cipher_set_aes_xts_initkey(ctx, key, keylen,
+ return ossl_cipher_set_aes_xts_initkey(ctx, key, keylen,
aes_t4_set_encrypt_key, aes_t4_set_decrypt_key,
aes_t4_encrypt, aes_t4_decrypt, stream_enc, stream_dec);
}
@@ -181,7 +132,7 @@ static int cipher_hw_aes_xts_t4_initkey(PROV_CIPHER_CTX *ctx,
static const PROV_CIPHER_HW aes_xts_t4 = {
cipher_hw_aes_xts_t4_initkey,
NULL,
- cipher_hw_aes_xts_copyctx
+ ossl_cipher_hw_aes_xts_copyctx
};
static const PROV_CIPHER_HW *ossl_prov_cipher_hw_aes_xts_t4()
@@ -199,13 +150,13 @@ static int cipher_hw_aes_xts_rv64i_initkey(PROV_CIPHER_CTX *ctx,
if (RISCV_HAS_ZVBB() && RISCV_HAS_ZVKG() && RISCV_HAS_ZVKNED() && riscv_vlen() >= 128) {
/* Zvkned only supports 128 and 256 bit keys. */
if (keylen * 8 == 128 * 2 || keylen * 8 == 256 * 2)
- return cipher_set_aes_xts_initkey(ctx, key, keylen,
+ return ossl_cipher_set_aes_xts_initkey(ctx, key, keylen,
rv64i_zvkned_set_encrypt_key, rv64i_zvkned_set_decrypt_key,
rv64i_zvkned_encrypt, rv64i_zvkned_decrypt,
rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt,
rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt);
- return cipher_set_aes_xts_initkey(ctx, key, keylen,
+ return ossl_cipher_set_aes_xts_initkey(ctx, key, keylen,
AES_set_encrypt_key, AES_set_encrypt_key,
rv64i_zvkned_encrypt, rv64i_zvkned_decrypt, NULL, NULL);
}
@@ -213,17 +164,17 @@ static int cipher_hw_aes_xts_rv64i_initkey(PROV_CIPHER_CTX *ctx,
if (RISCV_HAS_ZVKNED() && riscv_vlen() >= 128) {
/* Zvkned only supports 128 and 256 bit keys. */
if (keylen * 8 == 128 * 2 || keylen * 8 == 256 * 2)
- return cipher_set_aes_xts_initkey(ctx, key, keylen,
+ return ossl_cipher_set_aes_xts_initkey(ctx, key, keylen,
rv64i_zvkned_set_encrypt_key, rv64i_zvkned_set_decrypt_key,
rv64i_zvkned_encrypt, rv64i_zvkned_decrypt, NULL, NULL);
- return cipher_set_aes_xts_initkey(ctx, key, keylen,
+ return ossl_cipher_set_aes_xts_initkey(ctx, key, keylen,
AES_set_encrypt_key, AES_set_encrypt_key,
rv64i_zvkned_encrypt, rv64i_zvkned_decrypt, NULL, NULL);
}
if (RISCV_HAS_ZKND_AND_ZKNE())
- return cipher_set_aes_xts_initkey(ctx, key, keylen,
+ return ossl_cipher_set_aes_xts_initkey(ctx, key, keylen,
rv64i_zkne_set_encrypt_key, rv64i_zknd_set_decrypt_key,
rv64i_zkne_encrypt, rv64i_zknd_decrypt, NULL, NULL);
@@ -233,7 +184,7 @@ static int cipher_hw_aes_xts_rv64i_initkey(PROV_CIPHER_CTX *ctx,
static const PROV_CIPHER_HW aes_xts_rv64i = {
cipher_hw_aes_xts_rv64i_initkey,
NULL,
- cipher_hw_aes_xts_copyctx
+ ossl_cipher_hw_aes_xts_copyctx
};
static const PROV_CIPHER_HW *ossl_prov_cipher_hw_aes_xts_rv64i()
@@ -250,13 +201,13 @@ static int cipher_hw_aes_xts_rv32i_initkey(PROV_CIPHER_CTX *ctx,
const unsigned char *key, size_t keylen)
{
if (RISCV_HAS_ZBKB_AND_ZKND_AND_ZKNE())
- return cipher_set_aes_xts_initkey(ctx, key, keylen,
+ return ossl_cipher_set_aes_xts_initkey(ctx, key, keylen,
rv32i_zbkb_zkne_set_encrypt_key,
rv32i_zbkb_zknd_zkne_set_decrypt_key,
rv32i_zkne_encrypt, rv32i_zknd_decrypt, NULL, NULL);
if (RISCV_HAS_ZKND_AND_ZKNE())
- return cipher_set_aes_xts_initkey(ctx, key, keylen,
+ return ossl_cipher_set_aes_xts_initkey(ctx, key, keylen,
rv32i_zkne_set_encrypt_key, rv32i_zknd_zkne_set_decrypt_key,
rv32i_zkne_encrypt, rv32i_zknd_decrypt, NULL, NULL);
@@ -266,7 +217,7 @@ static int cipher_hw_aes_xts_rv32i_initkey(PROV_CIPHER_CTX *ctx,
static const PROV_CIPHER_HW aes_xts_rv32i = {
cipher_hw_aes_xts_rv32i_initkey,
NULL,
- cipher_hw_aes_xts_copyctx
+ ossl_cipher_hw_aes_xts_copyctx
};
static const PROV_CIPHER_HW *ossl_prov_cipher_hw_aes_xts_rv32i()
@@ -419,7 +370,7 @@ static const PROV_CIPHER_HW *ossl_prov_cipher_hw_aes_xts_s390x(size_t keybits)
static const PROV_CIPHER_HW aes_generic_xts = {
cipher_hw_aes_xts_generic_initkey,
NULL,
- cipher_hw_aes_xts_copyctx
+ ossl_cipher_hw_aes_xts_copyctx
};
const PROV_CIPHER_HW *ossl_prov_cipher_hw_aes_xts(size_t keybits)