Commit 62fe048fbe for openssl.org
commit 62fe048fbe324faaf8e11ea9ce3c76be8a085ae7
Author: Madan mohan Manokar <madanmohan.manokar@amd.com>
Date: Wed Jul 29 18:12:22 2026 +0530
Add AVX-512/VAES optimized AES-CBC decryption
Implement an AVX-512 VAES-based bulk path for AES-CBC decryption that
processes multiple blocks in parallel, improving decryption performance
for large inputs (1024 bytes or more) by 3.5x to 3.8x.
- Add the VAES CBC decrypt intrinsic implementation
(crypto/aes/aes_cbc_vaes_intrinsic.c) and wire it into the AES-NI
provider cipher path.
- Centralize the VAES_CBC_ELIGIBLE eligibility guard in
include/crypto/aes_platform.h to keep the header and implementation in
sync, and drop the redundant AES_BLOCK_SIZE define.
- Guard the intrinsics with refined gcc, clang, and msvc compiler-version
checks, ensuring clean builds in no-asm, no-deprecated, and Windows
(shared/minimal) configurations.
- Add an EVP-level AES-CBC decrypt test exercising the VAES bulk path
across AES-128/192/256 and a range of input lengths.
Reviewed-by: Simo Sorce <simo@redhat.com>
Reviewed-by: Andrew Dinh <andrewd@openssl.org>
MergeDate: Fri Jul 31 16:33:35 2026
(Merged from https://github.com/openssl/openssl/pull/30902)
diff --git a/crypto/aes/aes_cbc_vaes_intrinsic.c b/crypto/aes/aes_cbc_vaes_intrinsic.c
new file mode 100644
index 0000000000..91862f02f9
--- /dev/null
+++ b/crypto/aes/aes_cbc_vaes_intrinsic.c
@@ -0,0 +1,389 @@
+/*
+ * Copyright 2025 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright (C) 2026, Advanced Micro Devices, all rights reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License"). You may not use
+ * this file except in compliance with the License. You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ *
+ * Implements AES-CBC128/192/256 decryption with VAES (AVX-512)
+ *
+ * CBC encryption is inherently serial (each ciphertext block depends
+ * on the previous one), so VAES provides no benefit there -- the
+ * encrypt path falls back to the aesni_cbc_encrypt assembly routine.
+ *
+ * CBC decryption IS parallel: all blocks can be independently decrypted,
+ * then XORed with the preceding ciphertext block (or IV for the first).
+ * This implementation processes 4x4=16 blocks per iteration using four
+ * ZMM registers, falling back to 8, 4, then single-block processing.
+ */
+
+#include "internal/deprecated.h"
+
+#include <openssl/opensslconf.h>
+#include "internal/cryptlib.h"
+#include <openssl/aes.h>
+#include "crypto/modes.h"
+#include "crypto/aes_platform.h"
+
+#if VAES_CBC_ELIGIBLE
+
+/* Function prototypes */
+void ossl_aes_cbc_vaes_decrypt(const unsigned char *in, unsigned char *out,
+ size_t len, const void *key,
+ unsigned char ivec[16], int enc);
+int ossl_aes_cbc_vaes_eligible(void);
+
+#include <openssl/modes.h>
+
+/* Forward declarations -- defined in aesni-x86_64.pl assembly */
+void aesni_cbc_encrypt(const unsigned char *in, unsigned char *out,
+ size_t len, const AES_KEY *key,
+ unsigned char *ivec, int enc);
+void aesni_decrypt(const unsigned char *in, unsigned char *out,
+ const AES_KEY *key);
+
+/* Portable compiler abstractions for inlining and ISA target selection */
+#define STRINGIFY_IMPL_(a) #a
+#define STRINGIFY_(a) STRINGIFY_IMPL_(a)
+
+#ifdef __clang__
+#define OPENSSL_TARGET_VAES512 \
+ _Pragma(STRINGIFY_(clang attribute push( \
+ __attribute__((target("avx512f,avx512dq,avx512bw,vaes,aes"))), \
+ apply_to = function)))
+#define OPENSSL_UNTARGET_VAES512 _Pragma("clang attribute pop")
+#elif defined(__GNUC__)
+#define OPENSSL_TARGET_VAES512 \
+ _Pragma("GCC push_options") \
+ _Pragma(STRINGIFY_(GCC target("avx512f,avx512dq,avx512bw,vaes,aes")))
+#define OPENSSL_UNTARGET_VAES512 _Pragma("GCC pop_options")
+#else
+/* MSVC: all intrinsics are always available via <immintrin.h>. */
+#define OPENSSL_TARGET_VAES512
+#define OPENSSL_UNTARGET_VAES512
+#endif
+
+#if defined(__GNUC__) || defined(__clang__)
+#define OSSL_FUNC_ALWAYS_INLINE static inline __attribute__((always_inline))
+#define OSSL_FUNC_NOINLINE __attribute__((noinline))
+#elif defined(_MSC_VER)
+#define OSSL_FUNC_ALWAYS_INLINE static __forceinline
+#define OSSL_FUNC_NOINLINE __declspec(noinline)
+#else
+#define OSSL_FUNC_ALWAYS_INLINE static inline
+#define OSSL_FUNC_NOINLINE
+#endif
+
+#include <immintrin.h>
+
+OPENSSL_TARGET_VAES512
+
+/* ------------------------------------------------------------------ */
+/* AES decryption helpers: 1x, 2x, 4x parallel 512-bit blocks */
+/* Each 512-bit register holds 4 independent 128-bit AES blocks. */
+/* always_inline guarantees the compiler keeps keys in ZMM regs. */
+/* ------------------------------------------------------------------ */
+
+#define DEFINE_AES_DECRYPT_FUNCS(ROUNDS) \
+ OSSL_FUNC_ALWAYS_INLINE \
+ void AesDec_4x512_##ROUNDS( \
+ __m512i *b1, __m512i *b2, __m512i *b3, __m512i *b4, \
+ const __m512i *rk) \
+ { \
+ *b1 = _mm512_xor_si512(*b1, rk[0]); \
+ *b2 = _mm512_xor_si512(*b2, rk[0]); \
+ *b3 = _mm512_xor_si512(*b3, rk[0]); \
+ *b4 = _mm512_xor_si512(*b4, rk[0]); \
+ for (int i = 1; i < ROUNDS; i++) { \
+ *b1 = _mm512_aesdec_epi128(*b1, rk[i]); \
+ *b2 = _mm512_aesdec_epi128(*b2, rk[i]); \
+ *b3 = _mm512_aesdec_epi128(*b3, rk[i]); \
+ *b4 = _mm512_aesdec_epi128(*b4, rk[i]); \
+ } \
+ *b1 = _mm512_aesdeclast_epi128(*b1, rk[ROUNDS]); \
+ *b2 = _mm512_aesdeclast_epi128(*b2, rk[ROUNDS]); \
+ *b3 = _mm512_aesdeclast_epi128(*b3, rk[ROUNDS]); \
+ *b4 = _mm512_aesdeclast_epi128(*b4, rk[ROUNDS]); \
+ } \
+ \
+ OSSL_FUNC_ALWAYS_INLINE \
+ void AesDec_2x512_##ROUNDS( \
+ __m512i *b1, __m512i *b2, const __m512i *rk) \
+ { \
+ *b1 = _mm512_xor_si512(*b1, rk[0]); \
+ *b2 = _mm512_xor_si512(*b2, rk[0]); \
+ for (int i = 1; i < ROUNDS; i++) { \
+ *b1 = _mm512_aesdec_epi128(*b1, rk[i]); \
+ *b2 = _mm512_aesdec_epi128(*b2, rk[i]); \
+ } \
+ *b1 = _mm512_aesdeclast_epi128(*b1, rk[ROUNDS]); \
+ *b2 = _mm512_aesdeclast_epi128(*b2, rk[ROUNDS]); \
+ } \
+ \
+ OSSL_FUNC_ALWAYS_INLINE \
+ void AesDec_1x512_##ROUNDS( \
+ __m512i *b1, const __m512i *rk) \
+ { \
+ *b1 = _mm512_xor_si512(*b1, rk[0]); \
+ for (int i = 1; i < ROUNDS; i++) \
+ *b1 = _mm512_aesdec_epi128(*b1, rk[i]); \
+ *b1 = _mm512_aesdeclast_epi128(*b1, rk[ROUNDS]); \
+ }
+
+DEFINE_AES_DECRYPT_FUNCS(10) /* AES-128 */
+DEFINE_AES_DECRYPT_FUNCS(12) /* AES-192 */
+DEFINE_AES_DECRYPT_FUNCS(14) /* AES-256 */
+
+/* ------------------------------------------------------------------ */
+/* CBC-mode decryption -- templated per round count */
+/* */
+/* Processes as many full blocks as possible: */
+/* 16 blocks at a time (4 x zmm = 4 x 4 = 16 blocks) */
+/* 8 blocks at a time (2 x zmm) */
+/* 4 blocks at a time (1 x zmm) */
+/* 1 block at a time for the remaining 0-3 blocks */
+/* */
+/* The chaining vector b1 packs [prev_ct[last] | ct[0] | ct[1] */
+/* | ct[2]] so that a single XOR after decryption applies the CBC */
+/* feedback to all four lanes simultaneously. */
+/* ------------------------------------------------------------------ */
+
+#define DEFINE_CBC_DECRYPT(NR) \
+ OSSL_FUNC_NOINLINE \
+ static void cbc_decrypt_##NR( \
+ const unsigned char *in, unsigned char *out, size_t len, \
+ const AES_KEY *key, unsigned char *iv) \
+ { \
+ const unsigned char *rk_bytes = (const unsigned char *)key->rd_key; \
+ __m512i rk[NR + 1]; \
+ for (int i = 0; i <= NR; i++) { \
+ __m128i t = _mm_loadu_si128((const __m128i *)(rk_bytes + i * 16)); \
+ rk[i] = _mm512_broadcast_i32x4(t); \
+ } \
+ \
+ __m512i a1, a2, a3, a4; \
+ __m512i b1, b2, b3, b4; \
+ \
+ const __m128i *pa = (const __m128i *)in; \
+ __m512i *po = (__m512i *)out; \
+ size_t blocks = len / AES_BLOCK_SIZE; \
+ \
+ /* Save last ciphertext block for IV update (in-place safe) */ \
+ __m128i saved_iv = _mm_setzero_si128(); \
+ int has_blocks = (blocks > 0); \
+ if (has_blocks) \
+ saved_iv = _mm_loadu_si128(pa + blocks - 1); \
+ \
+ if (blocks >= 4) { \
+ /* Build b1 = [IV | ct[0] | ct[1] | ct[2]] */ \
+ __m512i idx = _mm512_set_epi64(5, 4, 3, 2, 1, 0, 0, 0); \
+ __m512i ct0; \
+ \
+ /* CBC C[0]=IV; 0x03 loads one 128-bit block (two 64-bit lanes). */ \
+ b1 = _mm512_maskz_loadu_epi64(0x03, iv); \
+ ct0 = _mm512_loadu_si512(pa); \
+ ct0 = _mm512_permutexvar_epi64(idx, ct0); \
+ b1 = _mm512_mask_blend_epi64(0xFC, b1, ct0); \
+ \
+ /* --- 16-block (4 x zmm) main loop --- */ \
+ while (blocks >= 16) { \
+ __m128i last; \
+ \
+ a1 = _mm512_loadu_si512(pa); \
+ a2 = _mm512_loadu_si512(pa + 4); \
+ a3 = _mm512_loadu_si512(pa + 8); \
+ a4 = _mm512_loadu_si512(pa + 12); \
+ \
+ b2 = _mm512_loadu_si512(pa + 3); \
+ b3 = _mm512_loadu_si512(pa + 7); \
+ b4 = _mm512_loadu_si512(pa + 11); \
+ \
+ last = _mm_loadu_si128(pa + 15); \
+ \
+ AesDec_4x512_##NR(&a1, &a2, &a3, &a4, rk); \
+ \
+ a1 = _mm512_xor_si512(a1, b1); \
+ a2 = _mm512_xor_si512(a2, b2); \
+ a3 = _mm512_xor_si512(a3, b3); \
+ a4 = _mm512_xor_si512(a4, b4); \
+ \
+ _mm512_storeu_si512(po, a1); \
+ _mm512_storeu_si512(po + 1, a2); \
+ _mm512_storeu_si512(po + 2, a3); \
+ _mm512_storeu_si512(po + 3, a4); \
+ \
+ /* Build next b1 from last ciphertext block */ \
+ b1 = _mm512_maskz_loadu_epi64(0x03, &last); \
+ if (blocks > 16) { \
+ size_t rem = blocks - 16; \
+ /* Load only available lookahead blocks to avoid OOB read. */ \
+ /* One AES block is 16 bytes, it maps to two 64-bit lanes. */ \
+ /* 0x03 (00000011) for 1 block (2 lanes), */ \
+ /* 0x0F (00001111) for 2 blocks (4 lanes), */ \
+ /* 0x3F (00111111) for 3 or more blocks (6 lanes). */ \
+ __mmask8 nxmask = (rem >= 3) ? 0x3F : (rem == 2 ? 0x0F : 0x03); \
+ __m512i nx = _mm512_maskz_loadu_epi64(nxmask, pa + 16); \
+ nx = _mm512_permutexvar_epi64(idx, nx); \
+ b1 = _mm512_mask_blend_epi64(0xFC, b1, nx); \
+ } \
+ \
+ pa += 16; \
+ po += 4; \
+ blocks -= 16; \
+ } \
+ \
+ /* --- 8-block (2 x zmm) --- */ \
+ if (blocks >= 8) { \
+ __m128i last8; \
+ \
+ a1 = _mm512_loadu_si512(pa); \
+ a2 = _mm512_loadu_si512(pa + 4); \
+ b2 = _mm512_loadu_si512(pa + 3); \
+ last8 = _mm_loadu_si128(pa + 7); \
+ \
+ AesDec_2x512_##NR(&a1, &a2, rk); \
+ a1 = _mm512_xor_si512(a1, b1); \
+ a2 = _mm512_xor_si512(a2, b2); \
+ \
+ _mm512_storeu_si512(po, a1); \
+ _mm512_storeu_si512(po + 1, a2); \
+ \
+ b1 = _mm512_maskz_loadu_epi64(0x03, &last8); \
+ pa += 8; \
+ po += 2; \
+ blocks -= 8; \
+ \
+ if (blocks >= 4) { \
+ __m512i nx = _mm512_loadu_si512(pa); \
+ nx = _mm512_permutexvar_epi64(idx, nx); \
+ b1 = _mm512_mask_blend_epi64(0xFC, b1, nx); \
+ } \
+ } \
+ \
+ /* --- 4-block (1 x zmm) --- */ \
+ if (blocks >= 4) { \
+ __m128i last4; \
+ \
+ a1 = _mm512_loadu_si512(pa); \
+ last4 = _mm_loadu_si128(pa + 3); \
+ \
+ AesDec_1x512_##NR(&a1, rk); \
+ a1 = _mm512_xor_si512(a1, b1); \
+ _mm512_storeu_si512(po, a1); \
+ \
+ b1 = _mm512_maskz_loadu_epi64(0x03, &last4); \
+ pa += 4; \
+ po += 1; \
+ blocks -= 4; \
+ } \
+ \
+ /* --- Remaining 1-3 blocks --- */ \
+ { \
+ __m128i *po128 = (__m128i *)po; \
+ while (blocks > 0) { \
+ __m128i ct = _mm_loadu_si128(pa); \
+ a1 = _mm512_maskz_loadu_epi64(0x03, pa); \
+ AesDec_1x512_##NR(&a1, rk); \
+ a1 = _mm512_xor_si512(a1, b1); \
+ _mm512_mask_storeu_epi64(po128, 0x03, a1); \
+ b1 = _mm512_maskz_loadu_epi64(0x03, &ct); \
+ pa++; \
+ po128++; \
+ blocks--; \
+ } \
+ } \
+ } else { \
+ /* Less than 4 blocks -- process individually */ \
+ __m128i *po128 = (__m128i *)po; \
+ b1 = _mm512_maskz_loadu_epi64(0x03, iv); \
+ while (blocks > 0) { \
+ __m128i ct = _mm_loadu_si128(pa); \
+ a1 = _mm512_maskz_loadu_epi64(0x03, pa); \
+ AesDec_1x512_##NR(&a1, rk); \
+ a1 = _mm512_xor_si512(a1, b1); \
+ _mm512_mask_storeu_epi64(po128, 0x03, a1); \
+ b1 = _mm512_maskz_loadu_epi64(0x03, &ct); \
+ pa++; \
+ po128++; \
+ blocks--; \
+ } \
+ } \
+ \
+ if (has_blocks) \
+ _mm_storeu_si128((__m128i *)iv, saved_iv); \
+ \
+ /* Clear round-key material from the stack */ \
+ { \
+ /* Use of volatile prevents dead-store elimination by compilers. */ \
+ volatile __m512i *vrk = (volatile __m512i *)(volatile void *)rk; \
+ for (int i = 0; i <= NR; i++) \
+ vrk[i] = _mm512_setzero_si512(); \
+ } \
+ }
+
+DEFINE_CBC_DECRYPT(10) /* AES-128 */
+DEFINE_CBC_DECRYPT(12) /* AES-192 */
+DEFINE_CBC_DECRYPT(14) /* AES-256 */
+
+/* ------------------------------------------------------------------ */
+/* Public entry point */
+/* ------------------------------------------------------------------ */
+
+void ossl_aes_cbc_vaes_decrypt(const unsigned char *in, unsigned char *out,
+ size_t len, const void *key,
+ unsigned char ivec[16], int enc)
+{
+ size_t full_bytes;
+ int nr = ((const AES_KEY *)key)->rounds + 1;
+
+ if (len == 0)
+ return;
+
+ /* VAES path only optimises decryption; encrypt falls back to asm */
+ if (enc) {
+ aesni_cbc_encrypt(in, out, len, (const AES_KEY *)key, ivec, enc);
+ return;
+ }
+
+ full_bytes = (len / AES_BLOCK_SIZE) * AES_BLOCK_SIZE;
+ if (full_bytes > 0) {
+ switch (nr) {
+ case 10:
+ cbc_decrypt_10(in, out, full_bytes, (const AES_KEY *)key, ivec);
+ break;
+ case 12:
+ cbc_decrypt_12(in, out, full_bytes, (const AES_KEY *)key, ivec);
+ break;
+ case 14:
+ cbc_decrypt_14(in, out, full_bytes, (const AES_KEY *)key, ivec);
+ break;
+ default: /* invalid key size */
+ aesni_cbc_encrypt(in, out, len, (const AES_KEY *)key, ivec, 0);
+ break;
+ }
+ }
+}
+
+/* ------------------------------------------------------------------ */
+/* CPU feature check */
+/* ------------------------------------------------------------------ */
+
+int ossl_aes_cbc_vaes_eligible(void)
+{
+ return (OPENSSL_ia32cap_P[2] & (1 << 16)) /* AVX512F */
+ && (OPENSSL_ia32cap_P[2] & (1 << 17)) /* AVX512DQ */
+ && (OPENSSL_ia32cap_P[2] & (1 << 30)) /* AVX512BW */
+ && (OPENSSL_ia32cap_P[3] & (1 << 9)); /* AVX512VAES */
+}
+
+OPENSSL_UNTARGET_VAES512
+
+#undef OPENSSL_TARGET_VAES512
+#undef OPENSSL_UNTARGET_VAES512
+#undef STRINGIFY_IMPL_
+#undef STRINGIFY_
+#undef OSSL_FUNC_ALWAYS_INLINE
+#undef OSSL_FUNC_NOINLINE
+#endif /* VAES_CBC_ELIGIBLE */
diff --git a/crypto/aes/build.info b/crypto/aes/build.info
index cb3bb5f2de..9df6fcc790 100644
--- a/crypto/aes/build.info
+++ b/crypto/aes/build.info
@@ -74,11 +74,11 @@ IF[{- !$disabled{asm} -}]
ENDIF
$COMMON=aes_misc.c aes_ecb.c $AESASM
-SOURCE[../../libcrypto]=$COMMON aes_cfb.c aes_ofb.c aes_wrap.c
+SOURCE[../../libcrypto]=$COMMON aes_cfb.c aes_ofb.c aes_wrap.c aes_cbc_vaes_intrinsic.c
IF[{- !$disabled{'deprecated-3.0'} -}]
SOURCE[../../libcrypto]=aes_ige.c
ENDIF
-SOURCE[../../providers/libfips.a]=$COMMON
+SOURCE[../../providers/libfips.a]=$COMMON aes_cbc_vaes_intrinsic.c
# Implementations are now spread across several libraries, so the defines
# need to be applied to all affected libraries and modules.
diff --git a/include/crypto/aes_platform.h b/include/crypto/aes_platform.h
index 36ba2665ee..6c0d0258e4 100644
--- a/include/crypto/aes_platform.h
+++ b/include/crypto/aes_platform.h
@@ -184,6 +184,22 @@ void gcm_ghash_v8(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp, siz
#endif
#endif
+#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)) \
+ && !defined(OPENSSL_NO_ASM) \
+ && ((defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 8)) \
+ || (defined(__clang__) && (__clang_major__ >= 7)) \
+ || (defined(_MSC_VER) && (_MSC_VER >= 1927)))
+#define VAES_CBC_ELIGIBLE 1
+#else
+#define VAES_CBC_ELIGIBLE 0
+#endif
+
+#if VAES_CBC_ELIGIBLE
+void ossl_aes_cbc_vaes_decrypt(const unsigned char *in, unsigned char *out,
+ size_t len, const void *key, unsigned char ivec[16], int enc);
+int ossl_aes_cbc_vaes_eligible(void);
+#endif
+
#if defined(AES_ASM) && !defined(I386_ONLY) && (((defined(__i386) || defined(__i386__) || defined(_M_IX86)) && defined(OPENSSL_IA32_SSE2)) || defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64))
/* AES-NI section */
@@ -210,6 +226,7 @@ void ossl_aes_cfb128_vaes_enc(const unsigned char *in, unsigned char *out,
void ossl_aes_cfb128_vaes_dec(const unsigned char *in, unsigned char *out,
size_t len, const AES_KEY *ks,
const unsigned char ivec[16], ossl_ssize_t *num);
+
int ossl_aes_cfb128_vaes_eligible(void);
void aesni_encrypt(const unsigned char *in, unsigned char *out,
diff --git a/providers/implementations/ciphers/cipher_aes_hw_aesni.c b/providers/implementations/ciphers/cipher_aes_hw_aesni.c
index f8ae45129b..db4a9e3e77 100644
--- a/providers/implementations/ciphers/cipher_aes_hw_aesni.c
+++ b/providers/implementations/ciphers/cipher_aes_hw_aesni.c
@@ -68,6 +68,35 @@ static const PROV_CIPHER_HW aesni_cbc = {
ossl_cipher_aes_copyctx
};
+#if VAES_CBC_ELIGIBLE
+/*
+ * CBC decryption is fully parallel, so VAES accelerates it; CBC encryption is
+ * inherently serial and stays on the aesni_cbc_encrypt assembly routine. VAES
+ * is only worthwhile once the payload is large enough to amortize the per-call
+ * key broadcast, so small buffers also fall back to the assembly routine.
+ */
+static int aes_cbc_vaes_wrapper(
+ PROV_CIPHER_CTX *ctx,
+ unsigned char *out,
+ const unsigned char *in,
+ size_t len)
+{
+ if (!ctx->enc && len >= 256) {
+ ossl_aes_cbc_vaes_decrypt(in, out, len, ctx->ks, ctx->iv, ctx->enc);
+ return 1;
+ }
+
+ aesni_cbc_encrypt(in, out, len, ctx->ks, ctx->iv, ctx->enc);
+ return 1;
+}
+
+static const PROV_CIPHER_HW aesni_vaes_cbc = {
+ cipher_hw_aesni_initkey,
+ aes_cbc_vaes_wrapper,
+ ossl_cipher_aes_copyctx
+};
+#endif /* VAES_CBC_ELIGIBLE */
+
#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64))
/* active in 64-bit builds when AES-NI, AVX512F, and VAES are detected */
#define VAES_CFB128_ELIGIBLE 1
@@ -146,6 +175,10 @@ const PROV_CIPHER_HW *ossl_prov_cipher_hw_aesni(enum aes_modes mode)
case AES_MODE_ECB:
return &aesni_ecb;
case AES_MODE_CBC:
+#if VAES_CBC_ELIGIBLE
+ if (ossl_aes_cbc_vaes_eligible())
+ return &aesni_vaes_cbc;
+#endif
return &aesni_cbc;
case AES_MODE_CFB128:
#if VAES_CFB128_ELIGIBLE
diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c
index 00ed62f264..af729d13f2 100644
--- a/test/evp_extra_test.c
+++ b/test/evp_extra_test.c
@@ -6993,6 +6993,102 @@ err:
return testresult;
}
+static const char *const aes_cbc_decrypt_ciphers[] = {
+ "AES-128-CBC", "AES-192-CBC", "AES-256-CBC"
+};
+
+/*
+ * Lengths (in bytes, all block-aligned) chosen so that the block count modulo
+ * the 16-block main loop hits every tail path in the bulk CBC decrypt routine:
+ * exact multiple of 16 blocks, the 1/2/3-block lookahead variants, and the
+ * 8-block, 4-block and 1-3 block remainder paths.
+ */
+static const int aes_cbc_decrypt_lengths[] = {
+ 256, /* 16 blocks: main loop once, no lookahead */
+ 272, /* 17 blocks: lookahead rem==1, 1-block tail */
+ 288, /* 18 blocks: lookahead rem==2, 2-block tail */
+ 304, /* 19 blocks: lookahead rem==3, 3-block tail */
+ 320, /* 20 blocks: 4-block path */
+ 384, /* 24 blocks: 8-block path */
+ 448, /* 28 blocks: 8-block + nested lookahead + 4-block */
+ 496, /* 31 blocks: 8 + 4 + 3-block tail */
+ 512 /* 32 blocks: main loop twice */
+};
+
+#define AES_CBC_DECRYPT_MAXLEN 512
+
+/*
+ * For each length, decrypt the ciphertext in a single call (the >= 256 byte
+ * length exercises the bulk/VAES CBC decrypt path) and again one block at a
+ * time (keeping every call below the bulk threshold, i.e. an independent
+ * reference decrypt). The bulk output must match both the original plaintext
+ * and the reference, for AES-128/192/256 and across all length branches.
+ */
+static int test_aes_cbc_decrypt(int idx)
+{
+ const char *ciphername = aes_cbc_decrypt_ciphers[idx];
+ unsigned char key[32], iv[16], pt[AES_CBC_DECRYPT_MAXLEN];
+ unsigned char ct[AES_CBC_DECRYPT_MAXLEN];
+ unsigned char bulk_out[AES_CBC_DECRYPT_MAXLEN];
+ unsigned char ref_out[AES_CBC_DECRYPT_MAXLEN];
+ int testresult = 0, i, outl, tmpl, off;
+ size_t li;
+ EVP_CIPHER *cipher = NULL;
+ EVP_CIPHER_CTX *ctx = NULL;
+
+ for (i = 0; i < (int)sizeof(key); i++)
+ key[i] = (unsigned char)(i + 1);
+ for (i = 0; i < (int)sizeof(iv); i++)
+ iv[i] = (unsigned char)(0xf0 ^ i);
+ for (i = 0; i < AES_CBC_DECRYPT_MAXLEN; i++)
+ pt[i] = (unsigned char)(i * 7 + 3);
+
+ if (!TEST_ptr(cipher = EVP_CIPHER_fetch(testctx, ciphername, testpropq))
+ || !TEST_ptr(ctx = EVP_CIPHER_CTX_new()))
+ goto err;
+
+ for (li = 0; li < OSSL_NELEM(aes_cbc_decrypt_lengths); li++) {
+ int buflen = aes_cbc_decrypt_lengths[li];
+
+ /* Reference encrypt (block-aligned input, padding disabled). */
+ if (!TEST_true(EVP_EncryptInit_ex(ctx, cipher, NULL, key, iv))
+ || !TEST_true(EVP_CIPHER_CTX_set_padding(ctx, 0))
+ || !TEST_true(EVP_EncryptUpdate(ctx, ct, &outl, pt, buflen))
+ || !TEST_int_eq(outl, buflen))
+ goto err;
+
+ /* One-shot decrypt: exercises the bulk path for this length. */
+ if (!TEST_true(EVP_DecryptInit_ex(ctx, cipher, NULL, key, iv))
+ || !TEST_true(EVP_CIPHER_CTX_set_padding(ctx, 0))
+ || !TEST_true(EVP_DecryptUpdate(ctx, bulk_out, &outl, ct, buflen))
+ || !TEST_int_eq(outl, buflen))
+ goto err;
+
+ /* Reference decrypt: one block per call bypasses the bulk path. */
+ if (!TEST_true(EVP_DecryptInit_ex(ctx, cipher, NULL, key, iv))
+ || !TEST_true(EVP_CIPHER_CTX_set_padding(ctx, 0)))
+ goto err;
+ for (off = 0; off < buflen; off += 16) {
+ if (!TEST_true(EVP_DecryptUpdate(ctx, ref_out + off, &tmpl,
+ ct + off, 16))
+ || !TEST_int_eq(tmpl, 16))
+ goto err;
+ }
+
+ if (!TEST_mem_eq(bulk_out, buflen, pt, buflen)
+ || !TEST_mem_eq(bulk_out, buflen, ref_out, buflen)) {
+ TEST_info("%s failed at length %d", ciphername, buflen);
+ goto err;
+ }
+ }
+
+ testresult = 1;
+err:
+ EVP_CIPHER_CTX_free(ctx);
+ EVP_CIPHER_free(cipher);
+ return testresult;
+}
+
typedef struct {
const char *cipher;
int enc;
@@ -9362,6 +9458,7 @@ int setup_tests(void)
ADD_ALL_TESTS(test_evp_init_seq, OSSL_NELEM(evp_init_tests));
ADD_ALL_TESTS(test_evp_reset, OSSL_NELEM(evp_reset_tests));
+ ADD_ALL_TESTS(test_aes_cbc_decrypt, OSSL_NELEM(aes_cbc_decrypt_ciphers));
ADD_ALL_TESTS(test_evp_reinit_seq, OSSL_NELEM(evp_reinit_tests));
ADD_ALL_TESTS(test_gcm_reinit, OSSL_NELEM(gcm_reinit_tests));
ADD_ALL_TESTS(test_evp_updated_iv, OSSL_NELEM(evp_updated_iv_tests));