Commit 18285605f3 for openssl.org
commit 18285605f351214c0c051ba6464d1477e7e453b7
Author: Mounir IDRASSI <mounir.idrassi@idrix.fr>
Date: Sun Aug 9 23:52:24 2026 +0900
Fix VAES CBC round-key stack cleanup
The VAES CBC decryptor uses volatile stores to clear its source-level
round-key array. With GCC 13.3 at -O3, scalar replacement separates
that object from the stack slots holding broadcast round keys. The
stores remain, but the actual key spills survive return.
Route cleanup through a dedicated perlasm helper. Passing rk across
this opaque assembly boundary forces optimized callers to materialize
the consumed key array. The helper erases exactly 11, 13, or 15
64-byte entries and scrubs caller-clobbered vector and general-purpose
register state within the System V and Win64 ABI constraints. Gate
VAES-CBC eligibility on assembler support and include the helper in
regular and FIPS builds.
This is a pragmatic boundary rather than a guarantee against every
compiler-created spill or abnormal exit. In particular, the Win64 ABI
requires the low 128 bits of XMM6-XMM15 to remain intact until the
caller restores them. A complete guarantee would require moving the
decryptor behind an assembly or custom-ABI boundary.
Fixes #32254
Assisted-by: Codex:gpt-5.6-sol
Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
Merge-date: Thu Aug 27 14:00:40 2026
Merged-from: https://github.com/openssl/openssl/pull/32257
diff --git a/crypto/aes/aes_cbc_vaes_intrinsic.c b/crypto/aes/aes_cbc_vaes_intrinsic.c
index 91862f02f9..a49b0550c8 100644
--- a/crypto/aes/aes_cbc_vaes_intrinsic.c
+++ b/crypto/aes/aes_cbc_vaes_intrinsic.c
@@ -1,5 +1,5 @@
/*
- * Copyright 2025 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2025-2026 The OpenSSL Project Authors. All Rights Reserved.
* Copyright (C) 2026, Advanced Micro Devices, all rights reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
@@ -44,6 +44,14 @@ void aesni_cbc_encrypt(const unsigned char *in, unsigned char *out,
void aesni_decrypt(const unsigned char *in, unsigned char *out,
const AES_KEY *key);
+/*
+ * Keep the cleanup out of line in assembly. Its opacity to C and LTO
+ * optimizers is what prevents the cleanup call and stores from being
+ * eliminated as dead. Do not replace it with a compiler-visible C function.
+ */
+void ossl_aes_cbc_vaes_cleanup(void *key_schedule, size_t num_keys);
+int ossl_aes_cbc_vaes_cleanup_eligible(void);
+
/* Portable compiler abstractions for inlining and ISA target selection */
#define STRINGIFY_IMPL_(a) #a
#define STRINGIFY_(a) STRINGIFY_IMPL_(a)
@@ -314,13 +322,8 @@ DEFINE_AES_DECRYPT_FUNCS(14) /* AES-256 */
if (has_blocks) \
_mm_storeu_si128((__m128i *)iv, saved_iv); \
\
- /* Clear round-key material from the stack */ \
- { \
- /* Use of volatile prevents dead-store elimination by compilers. */ \
- volatile __m512i *vrk = (volatile __m512i *)(volatile void *)rk; \
- for (int i = 0; i <= NR; i++) \
- vrk[i] = _mm512_setzero_si512(); \
- } \
+ /* Erase the broadcast schedule and the volatile vector register bank. */ \
+ ossl_aes_cbc_vaes_cleanup(rk, NR + 1); \
}
DEFINE_CBC_DECRYPT(10) /* AES-128 */
@@ -372,7 +375,8 @@ void ossl_aes_cbc_vaes_decrypt(const unsigned char *in, unsigned char *out,
int ossl_aes_cbc_vaes_eligible(void)
{
- return (OPENSSL_ia32cap_P[2] & (1 << 16)) /* AVX512F */
+ return ossl_aes_cbc_vaes_cleanup_eligible()
+ && (OPENSSL_ia32cap_P[2] & (1 << 16)) /* AVX512F */
&& (OPENSSL_ia32cap_P[2] & (1 << 17)) /* AVX512DQ */
&& (OPENSSL_ia32cap_P[2] & (1 << 30)) /* AVX512BW */
&& (OPENSSL_ia32cap_P[3] & (1 << 9)); /* AVX512VAES */
diff --git a/crypto/aes/asm/aes-cbc-vaes-x86_64.pl b/crypto/aes/asm/aes-cbc-vaes-x86_64.pl
new file mode 100644
index 0000000000..8bd84d838f
--- /dev/null
+++ b/crypto/aes/asm/aes-cbc-vaes-x86_64.pl
@@ -0,0 +1,184 @@
+#! /usr/bin/env perl
+# Copyright 2026 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License"). You may not use
+# this file except in compliance with the License. You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+# Erase the temporary broadcast key schedule used by the AES-CBC VAES
+# intrinsic implementation and clear its caller-clobbered register state.
+
+$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
+$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
+
+$win64 = 0;
+$win64 = 1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
+
+$avx512vaes = 0;
+
+$0 =~ m/(.*[\/\\])[^\/\\]+$/;
+$dir = $1;
+($xlate = "${dir}x86_64-xlate.pl" and -f $xlate)
+ or ($xlate = "${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate)
+ or die "can't locate x86_64-xlate.pl";
+
+if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
+ =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
+ $avx512vaes = ($1 >= 2.30);
+}
+
+if (!$avx512vaes && $win64
+ && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/)
+ && `nasm -v 2>&1`
+ =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) {
+ $avx512vaes = ($1 == 2.13 && $2 >= 3) + ($1 >= 2.14);
+}
+
+if (!$avx512vaes && $win64
+ && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/)
+ && `ml64 2>&1` =~ /Version ([0-9]+\.[0-9]+)\./) {
+ $avx512vaes = ($1 >= 14.16);
+}
+
+if (!$avx512vaes && `$ENV{CC} -v 2>&1`
+ =~ /(Apple)?\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) {
+ $ver = $3 + $4 / 100.0 + $5 / 10000.0;
+ if ($1) {
+ $avx512vaes = ($ver >= 10.0001);
+ } else {
+ $avx512vaes = ($ver >= 7.0);
+ }
+}
+
+if (!$avx512vaes && `$ENV{CC} -x c /dev/null -dM -E 2>/dev/null`
+ =~ /#define __clang_major__\s+([0-9]+)/) {
+ $avx512vaes = ($1 >= 11); # icx started with clang 11
+}
+
+open OUT, "| \"$^X\" \"$xlate\" $flavour \"$output\""
+ or die "can't call $xlate: $!";
+*STDOUT = *OUT;
+
+$key_schedule = $win64 ? "%rcx" : "%rdi";
+$num_keys = $win64 ? "%rdx" : "%rsi";
+
+$code = <<___;
+.text
+
+.globl ossl_aes_cbc_vaes_cleanup_eligible
+.hidden ossl_aes_cbc_vaes_cleanup_eligible
+.type ossl_aes_cbc_vaes_cleanup_eligible,\@abi-omnipotent
+.align 16
+ossl_aes_cbc_vaes_cleanup_eligible:
+.cfi_startproc
+ endbranch
+___
+
+if ($avx512vaes) {
+ $code .= <<___;
+ mov \$1,%eax
+ ret
+.cfi_endproc
+.size ossl_aes_cbc_vaes_cleanup_eligible,.-ossl_aes_cbc_vaes_cleanup_eligible
+
+# void ossl_aes_cbc_vaes_cleanup(void *key_schedule, size_t num_keys);
+#
+# num_keys counts 64-byte broadcast round keys. The caller invokes this only
+# after VAES use, so AVX-512 instructions are already safe to execute.
+# On Win64, the caller's epilogue restores the ABI-preserved low 128 bits of
+# XMM6-XMM15 if it used them. Their volatile upper lanes are cleared here.
+.globl ossl_aes_cbc_vaes_cleanup
+.hidden ossl_aes_cbc_vaes_cleanup
+.type ossl_aes_cbc_vaes_cleanup,\@abi-omnipotent
+.align 32
+ossl_aes_cbc_vaes_cleanup:
+.cfi_startproc
+ endbranch
+ vpxord %zmm0,%zmm0,%zmm0
+ test $num_keys,$num_keys
+ jz .Lvaes_cbc_clear_registers
+
+.Lvaes_cbc_clear_keys:
+ vmovdqu64 %zmm0,($key_schedule)
+ add \$64,$key_schedule
+ dec $num_keys
+ jnz .Lvaes_cbc_clear_keys
+
+.Lvaes_cbc_clear_registers:
+___
+
+ if ($win64) {
+ $code .= <<___;
+ # The low 128 bits of XMM6-XMM15 are nonvolatile on Win64. Preserve
+ # them, while clearing all of ZMM0-ZMM5 here.
+___
+ for ($i = 0; $i <= 5; $i++) {
+ $code .= " vpxor %xmm$i,%xmm$i,%xmm$i\n";
+ }
+ }
+
+ for ($i = 16; $i <= 31; $i++) {
+ $code .= " vpxord %zmm$i,%zmm$i,%zmm$i\n";
+ }
+
+ if ($win64) {
+ $code .= <<___;
+ # Clear the volatile upper lanes of ZMM6-ZMM15 last.
+ vzeroupper
+
+ # Clear the Win64 volatile general-purpose registers used by the caller.
+ xor %eax,%eax
+ xor %ecx,%ecx
+ xor %edx,%edx
+ xor %r8d,%r8d
+ xor %r9d,%r9d
+ xor %r10d,%r10d
+ xor %r11d,%r11d
+___
+ } else {
+ $code .= <<___;
+ # VZEROALL clears ZMM0-ZMM15, but does not affect ZMM16-ZMM31.
+ vzeroall
+
+ # Clear the SysV volatile general-purpose registers used by the caller.
+ xor %eax,%eax
+ xor %ecx,%ecx
+ xor %edx,%edx
+ xor %esi,%esi
+ xor %edi,%edi
+ xor %r8d,%r8d
+ xor %r9d,%r9d
+ xor %r10d,%r10d
+ xor %r11d,%r11d
+___
+ }
+
+ $code .= <<___;
+ ret
+.cfi_endproc
+.size ossl_aes_cbc_vaes_cleanup,.-ossl_aes_cbc_vaes_cleanup
+___
+} else {
+ $code .= <<___;
+ xor %eax,%eax
+ ret
+.cfi_endproc
+.size ossl_aes_cbc_vaes_cleanup_eligible,.-ossl_aes_cbc_vaes_cleanup_eligible
+
+.globl ossl_aes_cbc_vaes_cleanup
+.hidden ossl_aes_cbc_vaes_cleanup
+.type ossl_aes_cbc_vaes_cleanup,\@abi-omnipotent
+ossl_aes_cbc_vaes_cleanup:
+.cfi_startproc
+ endbranch
+ .byte 0x0f,0x0b # ud2
+ ret
+.cfi_endproc
+.size ossl_aes_cbc_vaes_cleanup,.-ossl_aes_cbc_vaes_cleanup
+___
+}
+
+print $code;
+
+close STDOUT or die "error closing STDOUT: $!";
diff --git a/crypto/aes/build.info b/crypto/aes/build.info
index 9df6fcc790..47d6bcdd78 100644
--- a/crypto/aes/build.info
+++ b/crypto/aes/build.info
@@ -10,7 +10,7 @@ IF[{- !$disabled{asm} -}]
$AESASM_x86_64=\
aes-x86_64.s vpaes-x86_64.s bsaes-x86_64.s aesni-x86_64.s \
aesni-sha1-x86_64.s aesni-sha256-x86_64.s aesni-mb-x86_64.s \
- aesni-xts-avx512.s aes-cfb-avx512.s
+ aesni-xts-avx512.s aes-cfb-avx512.s aes-cbc-vaes-x86_64.s
$AESDEF_x86_64=AES_ASM VPAES_ASM BSAES_ASM
$AESASM_ia64=aes_core.c aes_cbc.c aes-ia64.s
@@ -108,6 +108,7 @@ GENERATE[vpaes-x86_64.s]=asm/vpaes-x86_64.pl
GENERATE[bsaes-x86_64.s]=asm/bsaes-x86_64.pl
GENERATE[aesni-x86_64.s]=asm/aesni-x86_64.pl
GENERATE[aes-cfb-avx512.s]=asm/aes-cfb-avx512.pl
+GENERATE[aes-cbc-vaes-x86_64.s]=asm/aes-cbc-vaes-x86_64.pl
GENERATE[aesni-sha1-x86_64.s]=asm/aesni-sha1-x86_64.pl
GENERATE[aesni-sha256-x86_64.s]=asm/aesni-sha256-x86_64.pl
GENERATE[aesni-mb-x86_64.s]=asm/aesni-mb-x86_64.pl