Commit 2c8371ddc6 for openssl.org
commit 2c8371ddc6e7b7241adf664f835d848f4e0a0790
Author: Timo Keller <tkeller@linux.ibm.com>
Date: Wed Aug 19 09:33:45 2026 +0200
ML-DSA for s390x: Fix build and change comments
Fix the way how the s390x optimized implementation is built and used.
In particular, fix the alignment. Clarify some comments.
Signed-off-by: Timo Keller <tkeller@linux.ibm.com>
Assisted-by: IBM Bob:2.0.3
Reviewed-by: Mounir Idrassi <mounir.idrassi@idrix.fr>
Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
Merge-date: Tue Sep 29 16:47:11 2026
Merged-from: https://github.com/openssl/openssl/pull/31929
diff --git a/crypto/ml_dsa/build.info b/crypto/ml_dsa/build.info
index 3c74aef2f7..c23cc9f60f 100644
--- a/crypto/ml_dsa/build.info
+++ b/crypto/ml_dsa/build.info
@@ -20,7 +20,7 @@ IF[{- !$disabled{asm} -}]
ENDIF
IF[{- !$disabled{'ml-dsa'} -}]
- IF[{- ($target{perlasm_scheme} // '') ne '31' -}]
+ IF[{- !$disabled{asm} && ($target{perlasm_scheme} // '') ne '31' && defined($config{s390x_vector_cflags}) -}]
$ML_DSA_VX_s390x=ml_dsa_ntt_vec128.c
$ML_DSA_DEF_s390x=OPENSSL_ML_DSA_S390X
ENDIF
@@ -40,3 +40,6 @@ ENDIF
# Assembly implementations
GENERATE[ml_dsa_ntt-x86_64.s]=asm/ml_dsa_ntt-x86_64.pl
+
+DEFINE[../../libcrypto]=$ML_DSA_DEF
+DEFINE[../../providers/libfips.a]=$ML_DSA_DEF
diff --git a/crypto/ml_dsa/ml_dsa_key.c b/crypto/ml_dsa/ml_dsa_key.c
index e1b4f3bb73..1d27c8feaa 100644
--- a/crypto/ml_dsa/ml_dsa_key.c
+++ b/crypto/ml_dsa/ml_dsa_key.c
@@ -338,13 +338,31 @@ static int public_from_private(const ML_DSA_KEY *key, EVP_MD_CTX *md_ctx,
const ML_DSA_PARAMS *params = key->params;
uint32_t k = (uint32_t)params->k, l = (uint32_t)params->l;
POLY *polys;
+ void *polys_freeptr;
MATRIX a_ntt;
VECTOR s1_ntt;
VECTOR t;
-
- polys = OPENSSL_malloc_array(k + l + k * l, sizeof(*polys));
+#if defined(OPENSSL_ML_DSA_S390X)
+#define POLY_ALIGN 16
+ size_t polys_bytes = (k + l + (size_t)k * l) * sizeof(*polys)
+ + sizeof(void *) + (POLY_ALIGN - 1);
+ uint8_t *raw = OPENSSL_malloc(polys_bytes);
+ uintptr_t addr;
+
+ if (raw == NULL)
+ return 0;
+ addr = ((uintptr_t)raw + sizeof(void *) + (POLY_ALIGN - 1))
+ & ~(uintptr_t)(POLY_ALIGN - 1);
+ *(void **)((uint8_t *)(void *)addr - sizeof(void *)) = raw;
+ polys = (POLY *)(void *)addr;
+ polys_freeptr = raw;
+#undef POLY_ALIGN
+#else
+ polys = OPENSSL_malloc_array(k + l + (size_t)k * l, sizeof(*polys));
+ polys_freeptr = polys;
if (polys == NULL)
return 0;
+#endif
vector_init(&t, polys, k);
vector_init(&s1_ntt, t.poly + k, l);
@@ -374,7 +392,7 @@ err:
* require any special protections.
*/
OPENSSL_cleanse(polys, (k + l) * sizeof(*polys));
- OPENSSL_free(polys);
+ OPENSSL_free(polys_freeptr);
return ret;
}
@@ -407,15 +425,37 @@ int ossl_ml_dsa_key_pairwise_check(const ML_DSA_KEY *key)
const OSSL_ML_DSA_SAMPLE_OPS *sample_ops = ossl_ml_dsa_sample_ops();
VECTOR t1, t0;
POLY *polys = NULL;
+ void *polys_freeptr = NULL;
uint32_t k = (uint32_t)key->params->k;
EVP_MD_CTX *md_ctx = NULL;
if (key->pub_encoding == NULL || key->priv_encoding == 0)
return 0;
+#if defined(OPENSSL_ML_DSA_S390X)
+#define POLY_ALIGN 16
+ {
+ size_t bytes = 2 * (size_t)k * sizeof(*polys)
+ + sizeof(void *) + (POLY_ALIGN - 1);
+ uint8_t *raw = OPENSSL_malloc(bytes);
+ uintptr_t addr;
+
+ if (raw == NULL)
+ return 0;
+ addr = ((uintptr_t)raw + sizeof(void *) + (POLY_ALIGN - 1))
+ & ~(uintptr_t)(POLY_ALIGN - 1);
+ *(void **)((uint8_t *)(void *)addr - sizeof(void *)) = raw;
+ polys = (POLY *)(void *)addr;
+ polys_freeptr = raw;
+ }
+#undef POLY_ALIGN
+#else
polys = OPENSSL_malloc_array(2 * k, sizeof(*polys));
+ polys_freeptr = polys;
if (polys == NULL)
return 0;
+#endif
+
md_ctx = EVP_MD_CTX_new();
if (md_ctx == NULL)
goto err;
@@ -428,7 +468,8 @@ int ossl_ml_dsa_key_pairwise_check(const ML_DSA_KEY *key)
ret = vector_equal(&t1, &key->t1) && vector_equal(&t0, &key->t0);
err:
EVP_MD_CTX_free(md_ctx);
- OPENSSL_clear_free(polys, 2 * k * sizeof(*polys));
+ OPENSSL_cleanse(polys, 2 * k * sizeof(*polys));
+ OPENSSL_free(polys_freeptr);
return ret;
}
diff --git a/crypto/ml_dsa/ml_dsa_local.h b/crypto/ml_dsa/ml_dsa_local.h
index a47355e7b4..aa34e1dccd 100644
--- a/crypto/ml_dsa/ml_dsa_local.h
+++ b/crypto/ml_dsa/ml_dsa_local.h
@@ -88,10 +88,17 @@ void ossl_ml_dsa_poly_ntt_inverse(POLY *s);
void ossl_ml_dsa_poly_ntt_mult(const POLY *lhs, const POLY *rhs, POLY *out);
/* Optimization for s390x */
-/* z13 supports VX, z14 supports VXE; z14 means __ARCH__ == 12 */
-#if defined(OPENSSL_ML_DSA_S390X) && defined(__s390x__) && (__ARCH__ >= 12) && defined(__VX__)
+/*
+ * The forward declarations below must be visible in every TU that includes
+ * this header while compiling for s390x with the VX object enabled —
+ * specifically in ml_dsa_ntt.c (the dispatcher) and in ml_dsa_ntt_vec128.c
+ * (the implementation). OPENSSL_ML_DSA_S390X is injected by the build
+ * system for all asm-enabled s390x targets; it is sufficient on its own —
+ * no additional __s390x__ predefined-macro check is needed because the
+ * define is never emitted for non-s390x targets.
+ */
+#if defined(OPENSSL_ML_DSA_S390X)
#include "arch/s390x_arch.h"
-#define VX_COMPILER_SUPPORT_VEC128
void ossl_ml_dsa_poly_ntt_vec128(POLY *p);
void ossl_ml_dsa_poly_ntt_inverse_vec128(POLY *p);
void ossl_poly_ntt_mult_scalar_vec128(const POLY *lhs, const POLY *rhs, POLY *out);
diff --git a/crypto/ml_dsa/ml_dsa_ntt.c b/crypto/ml_dsa/ml_dsa_ntt.c
index 17a63802f3..b204ea6896 100644
--- a/crypto/ml_dsa/ml_dsa_ntt.c
+++ b/crypto/ml_dsa/ml_dsa_ntt.c
@@ -277,7 +277,15 @@ static void ml_dsa_ntt_init(void)
}
#endif
-#ifdef VX_COMPILER_SUPPORT_VEC128
+/*
+ * OPENSSL_ML_DSA_S390X is injected by the build system for all asm-enabled
+ * s390x targets and is the only guard needed here. An additional
+ * defined(__s390x__) check is redundant — the define is never emitted for
+ * non-s390x targets — and has been observed to be absent in some clang
+ * cross-compilation environments, which would silently omit the dispatch
+ * block and break the VX fast-path.
+ */
+#if defined(OPENSSL_ML_DSA_S390X)
if (S390X_VX_CAPABLE) {
poly_ntt_impl = ossl_ml_dsa_poly_ntt_vec128;
poly_ntt_inverse_impl = ossl_ml_dsa_poly_ntt_inverse_vec128;
diff --git a/crypto/ml_dsa/ml_dsa_ntt_vec128.c b/crypto/ml_dsa/ml_dsa_ntt_vec128.c
index 54d59a9a08..c580314069 100644
--- a/crypto/ml_dsa/ml_dsa_ntt_vec128.c
+++ b/crypto/ml_dsa/ml_dsa_ntt_vec128.c
@@ -7,12 +7,29 @@
* https://www.openssl.org/source/license.html
*/
+/*
+ * Scope the VX instruction set to this translation unit only.
+ * GCC: #pragma GCC target sets the arch/feature flags for this file; the rest
+ * of libcrypto is compiled without -mvx and stays safe on pre-z13 CPUs.
+ * Clang: does not honour #pragma GCC target for <vecintrin.h> inclusion; it
+ * requires -fzvector at the command line (added globally by Configure
+ * when needed, but that flag only unlocks vecintrin.h and does NOT
+ * change the code-generation architecture).
+ */
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC target("arch=z13,vx")
+#endif
+
+/* z13 introduced VX (facility bit 129); z14 adds VXE. Only VX is needed. */
+#if defined(OPENSSL_ML_DSA_S390X) && defined(__s390x__) && defined(__VX__)
+#define VX_COMPILER_SUPPORT_VEC128
+#include <vecintrin.h>
+#endif
+
#include "ml_dsa_local.h"
#include "ml_dsa_poly.h"
-#if defined(OPENSSL_ML_DSA_S390X) && defined(__s390x__) && (__ARCH__ >= 12) && defined(__VX__)
-
-#include <vecintrin.h>
+#if defined(VX_COMPILER_SUPPORT_VEC128)
#include <stdint.h>
@@ -310,9 +327,9 @@ static const vec_int32_t vec_q = { ML_DSA_Q, ML_DSA_Q, ML_DSA_Q, ML_DSA_Q };
static const vec_int32_t vec_q_inv = { ML_DSA_Q_INV, ML_DSA_Q_INV, ML_DSA_Q_INV, ML_DSA_Q_INV };
/*
- * @brief Reduce a in (-q, q) to a mod q in [0, q).
+ * @brief Reduce a in [-q, q) to a mod q in [0, q).
*
- * @param a in (-q, q)
+ * @param a in [-q, q)
* @returns a mod q in [0, q)
*/
static ossl_inline
@@ -350,27 +367,47 @@ static ossl_inline
* @param b is the second factor.
* @returns The Montgomery product of a and b in the range
* [0, q).
+ *
+ * Implementation note: the low-word product k = a_twist * b must be computed
+ * as an *unsigned* 32-bit lane multiply. a_twist holds precomputed values
+ * from zetas_montgomery_twisted[] such as 1830765815; multiplying those by
+ * even small values of b overflows int32_t, which is undefined behaviour and
+ * caught immediately by UBSan (signed integer overflow).
+ *
+ * The fix mirrors the ML-KEM version (multiply_montgomery_unreduced in
+ * ml_kem_vec128.c): cast both operands to the unsigned __may_alias__ type
+ * (vec_uint32_t) before multiplying so that wrapping is well-defined, then
+ * reinterpret the low 32 bits back as vec_int32_t. The cast is a pure
+ * reinterpretation at the register level; the generated VX instruction
+ * (vml / vmlo) is identical for both signed and unsigned 32-bit lanes.
+ *
+ * Using the non-alias cast (vec_uint32_alias_t) for the low multiply instead
+ * of vec_uint32_t fails under Clang: the difference in __may_alias__ between
+ * the inlined call-site type and the parameter type confuses the Clang alias
+ * analyser across inlining boundaries, producing wrong code. The __may_alias__
+ * unsigned type (vec_uint32_t) is therefore required for both operands of the
+ * low multiply, matching the ML-KEM pattern exactly.
*/
-
static ossl_inline
vec_int32_t
montgomery_multiplication_vectorized(vec_int32_t a, vec_int32_t a_twist, vec_int32_t b)
{
- vec_uint32_t k = (vec_uint32_t)a_twist * (vec_uint32_t)b;
- vec_uint32_t c_u = vec_mulh((vec_uint32_alias_t)k, (vec_uint32_alias_t)vec_q);
- vec_int32_t c = (vec_int32_t)c_u;
+ vec_int32_t k = (vec_int32_t)((vec_uint32_t)a_twist * (vec_uint32_t)b);
+ vec_int32_t c = vec_mulh((vec_int32_alias_t)k, (vec_int32_alias_t)vec_q);
vec_int32_t z_high = vec_mulh((vec_int32_alias_t)a, (vec_int32_alias_t)b);
vec_int32_t r = z_high - c;
return reduce_twice_signed(r);
}
/*
- * @brief Reduce modulo q to an non-negative vector.
- * Note that the constant v_scalar equals
- * floor(2**(floor(log_2(q))-1 * 2**32/q)).
+ * @brief Reduce modulo q to a non-negative vector.
+ * See [Seiler 2018, Algorithm 5].
*
- * @param a in the range -2**31..2**31-1
- * @returns a mod q in the range 0..q-1
+ * @param a in the range [-9q, 9q]
+ * (Note that we are only calling this function twice in
+ * ossl_ml_dsa_poly_ntt_vec128 with inputs in the range [-9q, 9q],
+ * which is inside the valid range.)
+ * @returns a mod q in the range [0, q).
*/
static ossl_inline
vec_int32_t
@@ -378,10 +415,15 @@ static ossl_inline
{
const int32_t v_scalar = 1074791296;
const vec_int32_alias_t v = { v_scalar, v_scalar, v_scalar, v_scalar };
- vec_int32_t t = vec_mulh((vec_int32_alias_t)a, v) >> 21;
+ vec_int32_t t = (vec_int32_t)(vec_mulh((vec_int32_alias_t)a, v) >> 21);
t *= ML_DSA_Q;
- vec_int32_t r = a - t; /* in [0, q] */
- return reduce_once_signed(r);
+ /*
+ * For a in [-9q, 9q], the Barrett step produces r in [0, q].
+ * Hence r - q is in [-q, 0], and adding q iff it is negative
+ * produces the canonical representative in [0, q).
+ */
+ vec_int32_t r = a - t;
+ return reduce_once_signed(r - vec_q);
}
void ossl_poly_ntt_mult_scalar_vec128(const POLY *lhs, const POLY *rhs, POLY *out)
@@ -705,4 +747,4 @@ void ossl_ml_dsa_poly_ntt_inverse_vec128(POLY *p)
}
}
-#endif
+#endif /* VX_COMPILER_SUPPORT_VEC128 */
diff --git a/crypto/ml_dsa/ml_dsa_poly.h b/crypto/ml_dsa/ml_dsa_poly.h
index c45bd549cd..679ea28606 100644
--- a/crypto/ml_dsa/ml_dsa_poly.h
+++ b/crypto/ml_dsa/ml_dsa_poly.h
@@ -16,9 +16,24 @@
#define ML_DSA_NUM_POLY_COEFFICIENTS 256
-/* Polynomial object with 256 coefficients. The coefficients are unsigned 32 bits */
+/*
+ * Polynomial object with 256 coefficients. The coefficients are unsigned
+ * 32-bit integers.
+ *
+ * ALIGN16 is applied unconditionally on s390x builds that include the VX
+ * vector object (OPENSSL_ML_DSA_S390X && __s390x__). This matches the guard
+ * used in ml_dsa_local.h and ml_dsa_ntt.c so that every translation unit
+ * (including ml_dsa_matrix.c) sees _Alignof(POLY) == 16 and allocates stack
+ * objects (e.g. the local `product` in ossl_ml_dsa_matrix_mult_vector) with
+ * the correct 16-byte alignment required by ossl_poly_ntt_mult_scalar_vec128.
+ *
+ * Using VX_COMPILER_SUPPORT_VEC128 here was incorrect: that macro is only
+ * defined inside ml_dsa_ntt_vec128.c (a separate translation unit compiled
+ * with -march=z13), so baseline TUs would see _Alignof(POLY) == 4 and pass
+ * misaligned pointers to the vector implementation.
+ */
struct poly_st {
-#if defined(VX_COMPILER_SUPPORT_VEC128)
+#if defined(OPENSSL_ML_DSA_S390X) && defined(__s390x__)
ALIGN16 uint32_t coeff[ML_DSA_NUM_POLY_COEFFICIENTS];
#elif defined(_ARCH_PPC64)
ALIGN16 uint32_t coeff[ML_DSA_NUM_POLY_COEFFICIENTS];
diff --git a/crypto/ml_dsa/ml_dsa_vector.h b/crypto/ml_dsa/ml_dsa_vector.h
index feedb56eff..88db91ca7c 100644
--- a/crypto/ml_dsa/ml_dsa_vector.h
+++ b/crypto/ml_dsa/ml_dsa_vector.h
@@ -37,6 +37,87 @@ static ossl_inline ossl_unused void vector_init(VECTOR *v, POLY *polys, size_t n
v->num_poly = num_polys;
}
+/*
+ * Aligned allocation helpers for POLY arrays.
+ *
+ * On s390x with VX support the POLY type carries ALIGN16, but both
+ * OPENSSL_malloc and OPENSSL_secure_malloc may return only 8-byte-aligned
+ * storage on that platform. A self-describing header-word technique is used
+ * to guarantee 16-byte alignment without adding a freeptr field to VECTOR:
+ *
+ * - Over-allocate by sizeof(void *) + (POLY_ALIGN - 1) bytes.
+ * - Advance the base pointer to the next POLY_ALIGN boundary that is at
+ * least sizeof(void *) bytes past the raw allocation, so there is always
+ * room for a void * header even when raw is already aligned.
+ * - Store the original raw pointer in the sizeof(void *) slack bytes
+ * immediately before the aligned pointer.
+ * - To free: read back the raw pointer from that header slot.
+ *
+ * This is safe because sizeof(void *) <= 8 <= POLY_ALIGN = 16 on all
+ * supported platforms. No raw-pointer field is needed in VECTOR, so the
+ * struct layout is identical regardless of whether VX support is compiled in.
+ *
+ * On non-s390x builds POLY_ALIGN is 4 (sizeof(uint32_t)), which is always
+ * satisfied by the platform allocator, so the plain malloc/free path is used.
+ */
+#if defined(OPENSSL_ML_DSA_S390X)
+#define POLY_ALIGN 16
+
+static ossl_inline ossl_unused int vector_alloc(VECTOR *v, size_t num_polys)
+{
+ size_t bytes = num_polys * sizeof(POLY) + sizeof(void *) + (POLY_ALIGN - 1);
+ uint8_t *raw = OPENSSL_malloc(bytes);
+ uintptr_t addr;
+
+ if (raw == NULL)
+ return 0;
+ addr = ((uintptr_t)raw + sizeof(void *) + (POLY_ALIGN - 1))
+ & ~(uintptr_t)(POLY_ALIGN - 1);
+ *(void **)((uint8_t *)(void *)addr - sizeof(void *)) = raw;
+ v->poly = (POLY *)(void *)addr;
+ v->num_poly = num_polys;
+ return 1;
+}
+
+static ossl_inline ossl_unused int vector_secure_alloc(VECTOR *v, size_t num_polys)
+{
+ size_t bytes = num_polys * sizeof(POLY) + sizeof(void *) + (POLY_ALIGN - 1);
+ uint8_t *raw = OPENSSL_secure_malloc(bytes);
+ uintptr_t addr;
+
+ if (raw == NULL)
+ return 0;
+ addr = ((uintptr_t)raw + sizeof(void *) + (POLY_ALIGN - 1))
+ & ~(uintptr_t)(POLY_ALIGN - 1);
+ *(void **)((uint8_t *)(void *)addr - sizeof(void *)) = raw;
+ v->poly = (POLY *)(void *)addr;
+ v->num_poly = num_polys;
+ return 1;
+}
+
+static ossl_inline ossl_unused void vector_free(VECTOR *v)
+{
+ if (v->poly != NULL)
+ OPENSSL_free(*(void **)((uint8_t *)v->poly - sizeof(void *)));
+ v->poly = NULL;
+ v->num_poly = 0;
+}
+
+static ossl_inline ossl_unused void vector_secure_free(VECTOR *v, size_t rank)
+{
+ size_t bytes = rank * sizeof(POLY) + sizeof(void *) + (POLY_ALIGN - 1);
+
+ if (v->poly != NULL)
+ OPENSSL_secure_clear_free(*(void **)((uint8_t *)v->poly - sizeof(void *)),
+ bytes);
+ v->poly = NULL;
+ v->num_poly = 0;
+}
+
+#undef POLY_ALIGN
+
+#else /* !OPENSSL_ML_DSA_S390X */
+
static ossl_inline ossl_unused int vector_alloc(VECTOR *v, size_t num_polys)
{
v->poly = OPENSSL_malloc_array(num_polys, sizeof(POLY));
@@ -69,6 +150,8 @@ static ossl_inline ossl_unused void vector_secure_free(VECTOR *v, size_t rank)
v->num_poly = 0;
}
+#endif /* OPENSSL_ML_DSA_S390X */
+
/* @brief zeroize a vectors polynomial coefficients */
static ossl_inline ossl_unused void vector_zero(VECTOR *va)
{