Commit 173480905e for openssl.org
commit 173480905e8556188884d1ca938545cabe798952
Author: Madan mohan Manokar <madanmohan.manokar@amd.com>
Date: Mon Aug 31 08:39:24 2026 +0530
aes-ctr-avx512: add VAES-512 AES-CTR encryption
Add a VAES-512 accelerated AES-CTR encryption path built on the shared
512-bit primitives. Counter blocks are generated and encrypted 4/8/16 at
a time; the low 64-bit counter word is clamped so that a 64-bit carry is
handled by a scalar step before VAES resumes, and payloads below the
provider threshold fall back to the existing aesni CTR path.
The provider gains a CTR hook that routes eligible payloads to the VAES
implementation, and aes_platform.h exposes VAES_CTR_ELIGIBLE (aliased to
VAES512_ELIGIBLE) with the ossl_aes_ctr_vaes prototypes.
The existing CTR known-answer vectors were all below the provider threshold
that dispatches to the VAES path, so that code was never exercised. Add
NIST SP800-38A 64-byte CTR KATs (AES-128/192/256) and a parametrized C test
spanning 64B-5000B across all key sizes and counter-carry boundaries,
validated against an independent ECB-based CTR reference.
Reviewed-by: Simo Sorce <simo@redhat.com>
Reviewed-by: Nikola Pajkovsky <nikolap@openssl.org>
Merge-date: Thu Sep 17 16:26:00 2026
Merged-from: https://github.com/openssl/openssl/pull/30755
diff --git a/CHANGES.md b/CHANGES.md
index dc89190dae..e2cf20579e 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -34,7 +34,11 @@ OpenSSL 4.2
### Changes between 4.1 and 4.2 [xx XXX xxxx]
- * none yet
+ * Added AVX-512 and VAES optimizations for AES-CTR mode. Performance for
+ large inputs (1024 bytes or more) improved by 2.9x to 3.9x.
+ <!-- https://github.com/openssl/openssl/pull/30755 -->
+
+ *Madan Mohan Manokar*
OpenSSL 4.1
-----------
diff --git a/crypto/aes/aes_vaes512_intrinsics.c b/crypto/aes/aes_vaes512_intrinsics.c
index 49379eed6f..15bcb2bc4e 100644
--- a/crypto/aes/aes_vaes512_intrinsics.c
+++ b/crypto/aes/aes_vaes512_intrinsics.c
@@ -16,7 +16,7 @@
* - macros to broadcast-load and securely scrub the round-key schedule,
* - a runtime CPU capability check.
*
- * Currently implemented: AES-CBC decryption.
+ * Currently implemented: AES-CTR encryption and AES-CBC decryption.
*
* CBC encryption is inherently serial (each ciphertext block depends on the
* previous one), so VAES provides no benefit there -- that path falls back to
@@ -40,12 +40,18 @@
#include <immintrin.h>
/* Function prototypes */
+void ossl_aes_ctr_vaes(const unsigned char *in, unsigned char *out,
+ size_t length, const AES_KEY *key,
+ unsigned char *counter,
+ unsigned char *ecount_buf, unsigned int *num);
+int ossl_aes_ctr_vaes_eligible(void);
void ossl_aes_cbc_vaes_decrypt(const unsigned char *in, unsigned char *out,
size_t len, const void *key,
unsigned char ivec[16], int enc);
int ossl_aes_cbc_vaes_eligible(void);
/* Forward declarations — defined in aesni-x86_64.pl assembly */
+void aesni_encrypt(const unsigned char *in, unsigned char *out, const AES_KEY *key);
void aesni_cbc_encrypt(const unsigned char *in, unsigned char *out,
size_t len, const AES_KEY *key,
unsigned char *ivec, int enc);
@@ -160,6 +166,11 @@ static ossl_inline int ossl_vaes512_cpu_capable(void)
*b1 = AESLAST(*b1, rk[ROUNDS]); \
}
+/* Emit the AES encryption round helpers (used by CTR, CFB, GCM, ...). */
+#define OSSL_VAES512_DEFINE_ENCRYPT(ROUNDS) \
+ OSSL_VAES512_DEFINE_ROUNDS(AesEnc, ROUNDS, \
+ _mm512_aesenc_epi128, _mm512_aesenclast_epi128)
+
/* Emit the AES decryption round helpers (used by CBC decrypt). */
#define OSSL_VAES512_DEFINE_DECRYPT(ROUNDS) \
OSSL_VAES512_DEFINE_ROUNDS(AesDec, ROUNDS, \
@@ -184,6 +195,298 @@ static ossl_inline int ossl_vaes512_cpu_capable(void)
OPENSSL_TARGET_VAES512
+/* AES encryption round helpers (1x/2x/4x parallel 512-bit blocks). */
+OSSL_VAES512_DEFINE_ENCRYPT(10) /* AES-128 */
+OSSL_VAES512_DEFINE_ENCRYPT(12) /* AES-192 */
+OSSL_VAES512_DEFINE_ENCRYPT(14) /* AES-256 */
+
+/*-
+ * Counter initialisation.
+ *
+ * Counters are kept in little-endian form (full 128-bit byte swap) and
+ * incremented with 64-bit arithmetic. This is safe for all practical
+ * counter values — encrypting data less than 2^68 bytes.
+ */
+static inline __m512i ctr_swap_mask(void)
+{
+ return _mm512_set_epi32(0x00010203, 0x04050607,
+ 0x08090a0b, 0x0c0d0e0f,
+ 0x00010203, 0x04050607,
+ 0x08090a0b, 0x0c0d0e0f,
+ 0x00010203, 0x04050607,
+ 0x08090a0b, 0x0c0d0e0f,
+ 0x00010203, 0x04050607,
+ 0x08090a0b, 0x0c0d0e0f);
+}
+
+static inline __m512i ctr_init4(const unsigned char *iv, __m512i swap)
+{
+ /* unaligned 128-bit load for iv and broadcast */
+ __m128i iv128 = _mm_loadu_si128((const __m128i *)iv);
+ __m512i c = _mm512_broadcast_i64x2(iv128);
+ c = _mm512_shuffle_epi8(c, swap);
+ c = _mm512_add_epi64(c, _mm512_set_epi64(0, 3, 0, 2, 0, 1, 0, 0));
+ return c;
+}
+
+/*-
+ * CTR-mode processing, templated per round count.
+ *
+ * Processes as many blocks as possible:
+ * 16 blocks at a time (4×zmm = 4×4 = 16 blocks)
+ * 8 blocks at a time (2×zmm)
+ * 4 blocks at a time (1×zmm)
+ * 0-3 tail blocks + partial residue via masked load/store
+ */
+#define DEFINE_CTR_BLOCK(NR) \
+ OSSL_FUNC_NOINLINE \
+ static void ctr_process_##NR( \
+ const unsigned char *in, unsigned char *out, \
+ size_t len, const AES_KEY *key, unsigned char *iv) \
+ { \
+ __m512i rk[NR + 1]; \
+ OSSL_VAES512_LOAD_ROUNDKEYS(rk, key, NR); \
+ \
+ const __m512i *p_in = (const __m512i *)in; \
+ __m512i *p_out = (__m512i *)out; \
+ __m512i swap = ctr_swap_mask(); \
+ __m512i c1 = ctr_init4(iv, swap); \
+ \
+ __m512i a1, a2, a3, a4; \
+ __m512i b1, b2, b3, b4; \
+ __m512i c2, c3, c4; \
+ \
+ size_t blocks = len / AES_BLOCK_SIZE; \
+ size_t res = len % AES_BLOCK_SIZE; \
+ \
+ const __m512i inc4 = _mm512_set_epi64(0, 4, 0, 4, 0, 4, 0, 4); \
+ const __m512i inc8 = _mm512_set_epi64(0, 8, 0, 8, 0, 8, 0, 8); \
+ const __m512i inc12 = _mm512_set_epi64(0, 12, 0, 12, 0, 12, 0, 12); \
+ const __m512i inc16 = _mm512_set_epi64(0, 16, 0, 16, 0, 16, 0, 16); \
+ \
+ /* --- 16-block (4×zmm) main loop --- */ \
+ while (blocks >= 16) { \
+ c2 = _mm512_add_epi64(c1, inc4); \
+ c3 = _mm512_add_epi64(c1, inc8); \
+ c4 = _mm512_add_epi64(c1, inc12); \
+ \
+ a1 = _mm512_loadu_si512(p_in); \
+ a2 = _mm512_loadu_si512(p_in + 1); \
+ a3 = _mm512_loadu_si512(p_in + 2); \
+ a4 = _mm512_loadu_si512(p_in + 3); \
+ \
+ b1 = _mm512_shuffle_epi8(c1, swap); \
+ b2 = _mm512_shuffle_epi8(c2, swap); \
+ b3 = _mm512_shuffle_epi8(c3, swap); \
+ b4 = _mm512_shuffle_epi8(c4, swap); \
+ \
+ AesEnc_4x512_##NR(&b1, &b2, &b3, &b4, rk); \
+ \
+ _mm512_storeu_si512(p_out, _mm512_xor_si512(b1, a1)); \
+ _mm512_storeu_si512(p_out + 1, _mm512_xor_si512(b2, a2)); \
+ _mm512_storeu_si512(p_out + 2, _mm512_xor_si512(b3, a3)); \
+ _mm512_storeu_si512(p_out + 3, _mm512_xor_si512(b4, a4)); \
+ \
+ c1 = _mm512_add_epi64(c1, inc16); \
+ p_in += 4; \
+ p_out += 4; \
+ blocks -= 16; \
+ } \
+ \
+ /* --- 8-block (2×zmm) --- */ \
+ if (blocks >= 8) { \
+ c2 = _mm512_add_epi64(c1, inc4); \
+ \
+ a1 = _mm512_loadu_si512(p_in); \
+ a2 = _mm512_loadu_si512(p_in + 1); \
+ \
+ b1 = _mm512_shuffle_epi8(c1, swap); \
+ b2 = _mm512_shuffle_epi8(c2, swap); \
+ \
+ AesEnc_2x512_##NR(&b1, &b2, rk); \
+ \
+ _mm512_storeu_si512(p_out, _mm512_xor_si512(b1, a1)); \
+ _mm512_storeu_si512(p_out + 1, _mm512_xor_si512(b2, a2)); \
+ \
+ c1 = _mm512_add_epi64(c1, inc8); \
+ p_in += 2; \
+ p_out += 2; \
+ blocks -= 8; \
+ } \
+ \
+ /* --- 4-block (1×zmm) --- */ \
+ if (blocks >= 4) { \
+ a1 = _mm512_loadu_si512(p_in); \
+ b1 = _mm512_shuffle_epi8(c1, swap); \
+ AesEnc_1x512_##NR(&b1, rk); \
+ _mm512_storeu_si512(p_out, _mm512_xor_si512(b1, a1)); \
+ \
+ c1 = _mm512_add_epi64(c1, inc4); \
+ p_in += 1; \
+ p_out += 1; \
+ blocks -= 4; \
+ } \
+ \
+ /* --- Tail: 0-3 full blocks + residue partial block --- */ \
+ { \
+ size_t tail_bytes = (blocks * AES_BLOCK_SIZE) + res; \
+ if (tail_bytes > 0) { \
+ __mmask64 mask = (__mmask64)((1ULL << tail_bytes) - 1ULL); \
+ a1 = _mm512_maskz_loadu_epi8(mask, p_in); \
+ b1 = _mm512_shuffle_epi8(c1, swap); \
+ AesEnc_1x512_##NR(&b1, rk); \
+ _mm512_mask_storeu_epi8(p_out, mask, _mm512_xor_si512(b1, a1)); \
+ \
+ size_t adv = blocks + (res > 0 ? 1 : 0); \
+ __m512i one_lo = _mm512_set_epi64(0, 0, 0, 0, 0, 0, 0, 1); \
+ for (size_t i = 0; i < adv; i++) \
+ c1 = _mm512_add_epi64(c1, one_lo); \
+ } \
+ } \
+ \
+ /* Write back updated counter (lane 0 only, byte-swapped to BE) */ \
+ { \
+ __m512i c_be = _mm512_shuffle_epi8(c1, swap); \
+ _mm512_mask_storeu_epi64((__m128i *)iv, 0x03, c_be); \
+ } \
+ \
+ /* Erase the broadcast schedule and the volatile vector register bank. */ \
+ ossl_aes_vaes_cleanup(rk, NR + 1); \
+ }
+
+DEFINE_CTR_BLOCK(10) /* AES-128 */
+DEFINE_CTR_BLOCK(12) /* AES-192 */
+DEFINE_CTR_BLOCK(14) /* AES-256 */
+
+/* Public entry point. */
+void ossl_aes_ctr_vaes(const unsigned char *in, unsigned char *out,
+ size_t length, const AES_KEY *key,
+ unsigned char *counter,
+ unsigned char *ecount_buf, unsigned int *num)
+{
+ size_t n = *num;
+ size_t l = length;
+ int nr = key->rounds + 1;
+
+ /* Drain leftover bytes from a previous partial block */
+ if (n != 0) {
+ while (l > 0 && n < 16) {
+ *(out++) = *(in++) ^ ecount_buf[n];
+ ++n;
+ l--;
+ }
+ *num = n % 16;
+ if (l == 0)
+ return;
+ }
+
+ /* Process full 16-byte blocks with VAES.
+ *
+ * The VAES loop uses 64-bit counter arithmetic (no carry into the
+ * upper 64 bits). If processing all requested blocks would overflow
+ * the low 64 bits of the BE counter:
+ * Phase 1 — VAES processes the safe blocks before the boundary.
+ * Phase 2 — Scalar handles 1 block to cross the 64-bit carry.
+ * Phase 3 — VAES resumes for the remaining bulk (>= 512 bytes),
+ * since the counter low-64 is now near zero and safe.
+ * Any final partial block is always handled by the scalar path.
+ */
+ {
+ size_t block_bytes = (l / 16) * 16;
+ if (block_bytes > 0) {
+ size_t total_blocks = block_bytes / 16;
+ /* Read low 64 bits of the big-endian counter (bytes 8..15) */
+ uint64_t ctr_lo = ((uint64_t)counter[8] << 56)
+ | ((uint64_t)counter[9] << 48)
+ | ((uint64_t)counter[10] << 40)
+ | ((uint64_t)counter[11] << 32)
+ | ((uint64_t)counter[12] << 24)
+ | ((uint64_t)counter[13] << 16)
+ | ((uint64_t)counter[14] << 8)
+ | ((uint64_t)counter[15]);
+
+ /* Clamp to the number of blocks safe for 64-bit arithmetic */
+ size_t safe_blocks = (ctr_lo <= UINT64_MAX - total_blocks)
+ ? total_blocks
+ : (size_t)(UINT64_MAX - ctr_lo);
+ size_t safe_bytes = safe_blocks * 16;
+
+ /* Phase 1: VAES for the safe portion before the boundary */
+ if (safe_bytes > 0) {
+ switch (nr) {
+ case 10:
+ ctr_process_10(in, out, safe_bytes, key, counter);
+ break;
+ case 12:
+ ctr_process_12(in, out, safe_bytes, key, counter);
+ break;
+ case 14:
+ ctr_process_14(in, out, safe_bytes, key, counter);
+ break;
+ default: /* invalid key size */
+ CRYPTO_ctr128_encrypt(in, out, safe_bytes, key, counter,
+ ecount_buf, num,
+ (block128_f)aesni_encrypt);
+ break;
+ }
+ in += safe_bytes;
+ out += safe_bytes;
+ l -= safe_bytes;
+ }
+
+ /* Phase 2 & 3: only entered when clamping actually occurred */
+ if (safe_blocks < total_blocks && l > 0) {
+ /* Phase 2: scalar encrypts 1 block across the carry */
+ CRYPTO_ctr128_encrypt(in, out, 16, key, counter,
+ ecount_buf, num,
+ (block128_f)aesni_encrypt);
+ in += 16;
+ out += 16;
+ l -= 16;
+
+ /* Phase 3: counter low-64 is now ~0 — VAES is safe again.
+ * Resume VAES if enough data remains (>= 512 bytes). */
+ if (l >= 512) {
+ size_t resume_bytes = (l / 16) * 16;
+ switch (nr) {
+ case 10:
+ ctr_process_10(in, out, resume_bytes, key, counter);
+ break;
+ case 12:
+ ctr_process_12(in, out, resume_bytes, key, counter);
+ break;
+ case 14:
+ ctr_process_14(in, out, resume_bytes, key, counter);
+ break;
+ default: /* invalid key size */
+ CRYPTO_ctr128_encrypt(in, out, resume_bytes, key,
+ counter, ecount_buf, num,
+ (block128_f)aesni_encrypt);
+ break;
+ }
+ in += resume_bytes;
+ out += resume_bytes;
+ l -= resume_bytes;
+ }
+ }
+ }
+ }
+
+ /* Handle any remaining bytes (partial block or small tail) */
+ if (l > 0)
+ CRYPTO_ctr128_encrypt(in, out, l, key, counter,
+ ecount_buf, num, (block128_f)aesni_encrypt);
+ else
+ *num = 0;
+}
+
+/* CPU feature check. */
+int ossl_aes_ctr_vaes_eligible(void)
+{
+ return ossl_aes_vaes_cleanup_eligible()
+ && ossl_vaes512_cpu_capable();
+}
+
/* AES decryption round helpers (1x/2x/4x parallel 512-bit blocks). */
OSSL_VAES512_DEFINE_DECRYPT(10) /* AES-128 */
OSSL_VAES512_DEFINE_DECRYPT(12) /* AES-192 */
@@ -363,7 +666,7 @@ OSSL_VAES512_DEFINE_DECRYPT(14) /* AES-256 */
_mm_storeu_si128((__m128i *)iv, saved_iv); \
\
/* Erase the broadcast schedule and the volatile vector register bank. */ \
- ossl_aes_vaes_cleanup(rk, NR + 1); \
+ ossl_aes_vaes_cleanup(rk, NR + 1); \
}
DEFINE_CBC_DECRYPT(10) /* AES-128 */
diff --git a/include/crypto/aes_platform.h b/include/crypto/aes_platform.h
index b0273e9c7d..54d8a58388 100644
--- a/include/crypto/aes_platform.h
+++ b/include/crypto/aes_platform.h
@@ -194,8 +194,17 @@ void gcm_ghash_v8(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp, siz
#define VAES512_ELIGIBLE 0
#endif
+#define VAES_CTR_ELIGIBLE VAES512_ELIGIBLE
#define VAES_CBC_ELIGIBLE VAES512_ELIGIBLE
+#if VAES_CTR_ELIGIBLE
+void ossl_aes_ctr_vaes(const unsigned char *in, unsigned char *out,
+ size_t length, const AES_KEY *key,
+ unsigned char *counter,
+ unsigned char *ecount_buf, unsigned int *num);
+int ossl_aes_ctr_vaes_eligible(void);
+#endif
+
#if VAES_CBC_ELIGIBLE
void ossl_aes_cbc_vaes_decrypt(const unsigned char *in, unsigned char *out,
size_t len, const void *key, unsigned char ivec[16], int enc);
diff --git a/providers/implementations/ciphers/cipher_aes_hw_aesni.c b/providers/implementations/ciphers/cipher_aes_hw_aesni.c
index db4a9e3e77..8a9cc264bb 100644
--- a/providers/implementations/ciphers/cipher_aes_hw_aesni.c
+++ b/providers/implementations/ciphers/cipher_aes_hw_aesni.c
@@ -162,6 +162,35 @@ static const PROV_CIPHER_HW aesni_ofb128 = {
ossl_cipher_aes_copyctx
};
+#if VAES_CTR_ELIGIBLE
+/*
+ * VAES accelerates CTR once the payload is large enough to amortize the
+ * per-call counter setup; small buffers fall back to the generic driver,
+ * which also preserves the partial-block state in ctx->buf/ctx->num.
+ */
+static int aes_ctr_vaes_wrapper(PROV_CIPHER_CTX *ctx, unsigned char *out,
+ const unsigned char *in, size_t len)
+{
+ const AES_KEY *key = (const AES_KEY *)ctx->ks;
+ unsigned int num;
+
+ if (len >= 64) {
+ num = ctx->num;
+ ossl_aes_ctr_vaes(in, out, len, key, ctx->iv, ctx->buf, &num);
+ ctx->num = num;
+ return 1;
+ }
+
+ return ossl_cipher_hw_generic_ctr(ctx, out, in, len);
+}
+
+static const PROV_CIPHER_HW aesni_vaes_ctr = {
+ cipher_hw_aesni_initkey,
+ aes_ctr_vaes_wrapper,
+ ossl_cipher_aes_copyctx
+};
+#endif /* VAES_CTR_ELIGIBLE */
+
static const PROV_CIPHER_HW aesni_ctr = {
cipher_hw_aesni_initkey,
ossl_cipher_hw_generic_ctr,
@@ -193,6 +222,10 @@ const PROV_CIPHER_HW *ossl_prov_cipher_hw_aesni(enum aes_modes mode)
case AES_MODE_OFB128:
return &aesni_ofb128;
case AES_MODE_CTR:
+#if VAES_CTR_ELIGIBLE
+ if (ossl_aes_ctr_vaes_eligible())
+ return &aesni_vaes_ctr;
+#endif
return &aesni_ctr;
default:
return NULL;
diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c
index 79add87da2..e170896d58 100644
--- a/test/evp_extra_test.c
+++ b/test/evp_extra_test.c
@@ -10031,6 +10031,146 @@ err:
#endif /* OPENSSL_NO_DH */
#endif /* OPENSSL_NO_DEPRECATED_3_0 */
+/*-
+ * AES-CTR cases for the VAES/AVX-512 path: sizes spanning the 64-byte
+ * dispatch threshold, the 16/8/4-block tiers, odd tails, and the 2^64
+ * counter carry. Each result is checked against an independent ECB-based
+ * CTR reference and a decrypt round-trip.
+ */
+static const struct {
+ int bits;
+ size_t len;
+ int ctr_carry; /* start the low 64 bits of the counter near overflow */
+} ctr_vaes_cases[] = {
+ { 128, 64, 0 },
+ { 128, 65, 0 },
+ { 128, 127, 0 },
+ { 128, 128, 0 },
+ { 128, 256, 0 },
+ { 128, 512, 0 },
+ { 128, 1024, 0 },
+ { 128, 5000, 0 },
+ { 192, 64, 0 },
+ { 192, 240, 0 },
+ { 192, 1024, 0 },
+ { 192, 4096, 0 },
+ { 256, 64, 0 },
+ { 256, 129, 0 },
+ { 256, 1024, 0 },
+ { 256, 4096, 0 },
+ { 128, 2048, 1 },
+ { 192, 2048, 1 },
+ { 256, 2048, 1 },
+};
+
+static int ctr_reference(int bits, const unsigned char *key,
+ const unsigned char *iv, const unsigned char *in,
+ unsigned char *out, size_t len)
+{
+ static const char *ecbname[] = {
+ "AES-128-ECB", "AES-192-ECB", "AES-256-ECB"
+ };
+ const char *name = ecbname[(bits - 128) / 64];
+ EVP_CIPHER *ecb = NULL;
+ EVP_CIPHER_CTX *ctx = NULL;
+ unsigned char ctr[16], ks[16];
+ size_t off = 0;
+ int outl, i, ok = 0;
+
+ if (!TEST_ptr(ecb = EVP_CIPHER_fetch(testctx, name, NULL))
+ || !TEST_ptr(ctx = EVP_CIPHER_CTX_new())
+ || !TEST_true(EVP_EncryptInit_ex2(ctx, ecb, key, NULL, NULL))
+ || !TEST_true(EVP_CIPHER_CTX_set_padding(ctx, 0)))
+ goto err;
+
+ memcpy(ctr, iv, sizeof(ctr));
+ while (off < len) {
+ size_t n = len - off < 16 ? len - off : 16;
+
+ if (!TEST_true(EVP_EncryptUpdate(ctx, ks, &outl, ctr, 16))
+ || !TEST_int_eq(outl, 16))
+ goto err;
+ for (i = 0; i < (int)n; i++)
+ out[off + i] = in[off + i] ^ ks[i];
+ off += n;
+ /* Increment the 128-bit counter as a big-endian integer. */
+ for (i = 15; i >= 0; i--)
+ if (++ctr[i] != 0)
+ break;
+ }
+ ok = 1;
+err:
+ EVP_CIPHER_CTX_free(ctx);
+ EVP_CIPHER_free(ecb);
+ return ok;
+}
+
+static int test_aes_ctr_vaes(int idx)
+{
+ static const char *ctrname[] = {
+ "AES-128-CTR", "AES-192-CTR", "AES-256-CTR"
+ };
+ int bits = ctr_vaes_cases[idx].bits;
+ size_t len = ctr_vaes_cases[idx].len;
+ const char *name = ctrname[(bits - 128) / 64];
+ EVP_CIPHER *ctr = NULL;
+ EVP_CIPHER_CTX *ctx = NULL;
+ unsigned char key[32], iv[16];
+ unsigned char *pt = NULL, *ref = NULL, *ct = NULL, *rt = NULL;
+ int outl, tmpl, ret = 0;
+
+ if (!TEST_ptr(pt = OPENSSL_malloc(len))
+ || !TEST_ptr(ref = OPENSSL_malloc(len))
+ || !TEST_ptr(ct = OPENSSL_malloc(len))
+ || !TEST_ptr(rt = OPENSSL_malloc(len)))
+ goto err;
+
+ if (!TEST_int_gt(RAND_bytes_ex(testctx, key, bits / 8, 0), 0)
+ || !TEST_int_gt(RAND_bytes_ex(testctx, iv, sizeof(iv), 0), 0)
+ || !TEST_int_gt(RAND_bytes_ex(testctx, pt, len, 0), 0))
+ goto err;
+
+ if (ctr_vaes_cases[idx].ctr_carry) {
+ /* Low 64 bits (bytes 8..15, big-endian) just below overflow. */
+ memset(iv + 8, 0xff, 8);
+ iv[15] = 0xf0;
+ }
+
+ if (!TEST_ptr(ctr = EVP_CIPHER_fetch(testctx, name, NULL))
+ || !TEST_ptr(ctx = EVP_CIPHER_CTX_new()))
+ goto err;
+
+ /* Independent reference (single-block ECB keystream). */
+ if (!ctr_reference(bits, key, iv, pt, ref, len))
+ goto err;
+
+ /* Encrypt: payloads >= 64 bytes select the VAES path on capable CPUs. */
+ if (!TEST_true(EVP_EncryptInit_ex2(ctx, ctr, key, iv, NULL))
+ || !TEST_true(EVP_EncryptUpdate(ctx, ct, &outl, pt, (int)len))
+ || !TEST_true(EVP_EncryptFinal_ex(ctx, ct + outl, &tmpl))
+ || !TEST_int_eq(outl + tmpl, (int)len)
+ || !TEST_mem_eq(ct, len, ref, len))
+ goto err;
+
+ /* Decrypt round-trip. */
+ if (!TEST_true(EVP_DecryptInit_ex2(ctx, ctr, key, iv, NULL))
+ || !TEST_true(EVP_DecryptUpdate(ctx, rt, &outl, ct, (int)len))
+ || !TEST_true(EVP_DecryptFinal_ex(ctx, rt + outl, &tmpl))
+ || !TEST_int_eq(outl + tmpl, (int)len)
+ || !TEST_mem_eq(rt, len, pt, len))
+ goto err;
+
+ ret = 1;
+err:
+ EVP_CIPHER_CTX_free(ctx);
+ EVP_CIPHER_free(ctr);
+ OPENSSL_free(pt);
+ OPENSSL_free(ref);
+ OPENSSL_free(ct);
+ OPENSSL_free(rt);
+ return ret;
+}
+
int setup_tests(void)
{
char *config_file = NULL;
@@ -10242,6 +10382,7 @@ int setup_tests(void)
ADD_ALL_TESTS(test_rsasve_degenerate_exponent, 2);
ADD_ALL_TESTS(test_rsasve_degenerate_ciphertext, 3);
+ ADD_ALL_TESTS(test_aes_ctr_vaes, OSSL_NELEM(ctr_vaes_cases));
#ifndef OPENSSL_NO_ML_KEM
ADD_ALL_TESTS(test_ml_kem_seed_only, 2);
diff --git a/test/recipes/30-test_evp_data/evpciph_aes_common.txt b/test/recipes/30-test_evp_data/evpciph_aes_common.txt
index 5dcbdd89e4..660a8b5339 100644
--- a/test/recipes/30-test_evp_data/evpciph_aes_common.txt
+++ b/test/recipes/30-test_evp_data/evpciph_aes_common.txt
@@ -654,6 +654,33 @@ Plaintext = 000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F
Ciphertext = A2D459477E6432BD74184B1B5370D2243CDC202BC43583B2A55D288CDBBD1E03
NextIV = 00000000000000008000000000000001
+# AES CTR known-answer vectors from NIST SP800-38A appendix F.5.
+# The 64-byte (4-block) payload crosses the 64-byte threshold that selects
+# the VAES/AVX-512 code path on capable CPUs, exercising it under a KAT.
+Cipher = aes-128-ctr
+Key = 2B7E151628AED2A6ABF7158809CF4F3C
+IV = F0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF
+Operation = ENCRYPT
+Plaintext = 6BC1BEE22E409F96E93D7E117393172AAE2D8A571E03AC9C9EB76FAC45AF8E5130C81C46A35CE411E5FBC1191A0A52EFF69F2445DF4F9B17AD2B417BE66C3710
+Ciphertext = 874D6191B620E3261BEF6864990DB6CE9806F66B7970FDFF8617187BB9FFFDFF5AE4DF3EDBD5D35E5B4F09020DB03EAB1E031DDA2FBE03D1792170A0F3009CEE
+NextIV = F0F1F2F3F4F5F6F7F8F9FAFBFCFDFF03
+
+Cipher = aes-192-ctr
+Key = 8E73B0F7DA0E6452C810F32B809079E562F8EAD2522C6B7B
+IV = F0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF
+Operation = ENCRYPT
+Plaintext = 6BC1BEE22E409F96E93D7E117393172AAE2D8A571E03AC9C9EB76FAC45AF8E5130C81C46A35CE411E5FBC1191A0A52EFF69F2445DF4F9B17AD2B417BE66C3710
+Ciphertext = 1ABC932417521CA24F2B0459FE7E6E0B090339EC0AA6FAEFD5CCC2C6F4CE8E941E36B26BD1EBC670D1BD1D665620ABF74F78A7F6D29809585A97DAEC58C6B050
+NextIV = F0F1F2F3F4F5F6F7F8F9FAFBFCFDFF03
+
+Cipher = aes-256-ctr
+Key = 603DEB1015CA71BE2B73AEF0857D77811F352C073B6108D72D9810A30914DFF4
+IV = F0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF
+Operation = ENCRYPT
+Plaintext = 6BC1BEE22E409F96E93D7E117393172AAE2D8A571E03AC9C9EB76FAC45AF8E5130C81C46A35CE411E5FBC1191A0A52EFF69F2445DF4F9B17AD2B417BE66C3710
+Ciphertext = 601EC313775789A5B7A7F504BBF3D228F443E3CA4D62B59ACA84E990CACAF5C52B0930DAA23DE94CE87017BA2D84988DDFC9C58DB67AADA613C2DD08457941A6
+NextIV = F0F1F2F3F4F5F6F7F8F9FAFBFCFDFF03
+
# AES CCM 256 bit key
Cipher = aes-256-ccm
Key = 1bde3251d41a8b5ea013c195ae128b218b3e0306376357077ef1c1c78548b92e