Commit 1a1d088d2a for openssl.org
commit 1a1d088d2ace4fc655c5f464405ca62c9df2d118
Author: Ingo Franzki <ifranzki@linux.ibm.com>
Date: Fri Jun 12 15:30:54 2026 +0200
s390x: Don't ignore errors from s390x_mod_exp_hw() and s390x_crt()
Currently errors from s390x_mod_exp_hw() and s390x_crt() are silently
ignored and the software path is used as fallback.
Change this to only take the software path if s390x_mod_exp_hw() and
s390x_crt() returns 0 to indicate that it does not support the RSA
acceleration. In case of errors, return them to the caller. Errors could
be memory allocation failures or errors during BIGNUM calls. Those should
not be ignored, but reported as failure.
Note that it can happen that the ioctl's fail, but this is not to be
reported as error. Those are situations where for example no suitable
crypto adapter is available, or the file descriptor has been closed
by a sandbox. Those situations disable the RSA acceleration for further
RSA requests, but the current operation should still be performed via the
software fallback. For cases where the RSA key size is too large for
acceleration, the operation must also be performed via the software
fallback, and not reported as an error.
This also fixes failures of the test_rsa_pkcs1_mfail test case that found
the memory allocation failures that got ignored.
Resolves: https://github.com/openssl/openssl/issues/31480
Signed-off-by: Ingo Franzki <ifranzki@linux.ibm.com>
Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Eugene Syromiatnikov <esyr@openssl.org>
MergeDate: Sun Jun 21 14:19:54 2026
(Merged from https://github.com/openssl/openssl/pull/31482)
diff --git a/crypto/bn/bn_s390x.c b/crypto/bn/bn_s390x.c
index ef96161ced..4e7aba35f6 100644
--- a/crypto/bn/bn_s390x.c
+++ b/crypto/bn/bn_s390x.c
@@ -20,16 +20,20 @@
#include <unistd.h>
#include <errno.h>
+/*
+ * Returns 1 for success, 0 for failure, and -1 to tell the caller to use the
+ * SW-fallback.
+ */
static int s390x_mod_exp_hw(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
const BIGNUM *m)
{
struct ica_rsa_modexpo me;
unsigned char *buffer;
size_t size;
- int res = 0;
+ int res = -1;
if (OPENSSL_s390xcex == -1 || OPENSSL_s390xcex_nodev)
- return 0;
+ return -1;
size = BN_num_bytes(m);
buffer = OPENSSL_calloc(size, 4);
if (buffer == NULL)
@@ -42,11 +46,15 @@ static int s390x_mod_exp_hw(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
me.n_modulus = buffer + 3 * size;
if (BN_bn2binpad(a, me.inputdata, size) == -1
|| BN_bn2binpad(p, me.b_key, size) == -1
- || BN_bn2binpad(m, me.n_modulus, size) == -1)
+ || BN_bn2binpad(m, me.n_modulus, size) == -1) {
+ res = 0;
goto dealloc;
+ }
if (ioctl(OPENSSL_s390xcex, ICARSAMODEXPO, &me) != -1) {
if (BN_bin2bn(me.outputdata, size, r) != NULL)
res = 1;
+ else
+ res = 0;
} else if (errno == EBADF || errno == ENOTTY) {
/*
* In this cases, someone (e.g. a sandbox) closed the fd.
@@ -71,27 +79,34 @@ dealloc:
int s390x_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx)
{
- if (s390x_mod_exp_hw(r, a, p, m) == 1)
- return 1;
- return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx);
+ int rc;
+
+ rc = s390x_mod_exp_hw(r, a, p, m);
+ if (rc < 0)
+ return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx);
+ return rc;
}
+/*
+ * Returns 1 for success, 0 for failure, and -1 to tell the caller to use the
+ * SW-fallback.
+ */
int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q,
const BIGNUM *dmp, const BIGNUM *dmq, const BIGNUM *iqmp)
{
struct ica_rsa_modexpo_crt crt;
unsigned char *buffer, *part;
size_t size, plen, qlen;
- int res = 0;
+ int res = -1;
if (OPENSSL_s390xcex == -1 || OPENSSL_s390xcex_nodev)
- return 0;
+ return -1;
/*-
* Hardware-accelerated CRT can only deal with p>q. Fall back to
* software in the (hopefully rare) other cases.
*/
if (BN_ucmp(p, q) != 1)
- return 0;
+ return -1;
plen = BN_num_bytes(p);
qlen = BN_num_bytes(q);
size = (plen > qlen ? plen : qlen);
@@ -119,11 +134,15 @@ int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q,
|| BN_bn2binpad(q, crt.nq_prime, size) == -1
|| BN_bn2binpad(dmp, crt.bp_key, size + 8) == -1
|| BN_bn2binpad(dmq, crt.bq_key, size) == -1
- || BN_bn2binpad(iqmp, crt.u_mult_inv, size + 8) == -1)
+ || BN_bn2binpad(iqmp, crt.u_mult_inv, size + 8) == -1) {
+ res = 0;
goto dealloc;
+ }
if (ioctl(OPENSSL_s390xcex, ICARSACRT, &crt) != -1) {
if (BN_bin2bn(crt.outputdata, crt.outputdatalength, r) != NULL)
res = 1;
+ else
+ res = 0;
} else if (errno == EBADF || errno == ENOTTY) {
/*
* In this cases, someone (e.g. a sandbox) closed the fd.
diff --git a/crypto/rsa/rsa_ossl.c b/crypto/rsa/rsa_ossl.c
index b883baa58f..674ee4b10d 100644
--- a/crypto/rsa/rsa_ossl.c
+++ b/crypto/rsa/rsa_ossl.c
@@ -1184,9 +1184,12 @@ static int rsa_ossl_finish(RSA *rsa)
static int rsa_ossl_s390x_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa,
BN_CTX *ctx)
{
+ int rc;
+
if (rsa->version != RSA_ASN1_VERSION_MULTI) {
- if (s390x_crt(r0, i, rsa->p, rsa->q, rsa->dmp1, rsa->dmq1, rsa->iqmp) == 1)
- return 1;
+ rc = s390x_crt(r0, i, rsa->p, rsa->q, rsa->dmp1, rsa->dmq1, rsa->iqmp);
+ if (rc >= 0)
+ return rc;
}
return rsa_ossl_mod_exp(r0, i, rsa, ctx);
}