Commit e5d2eaee13 for openssl.org
commit e5d2eaee133290cce1b01bc7cead839bd7e18b8e
Author: Viktor Dukhovni <viktor@openssl.org>
Date: Thu Jul 30 23:12:26 2026 +1000
Reject clearly degenerate RSASVE parameters.
- A public exponent <= 1
- Ciphertext <= 1 or equal to n-1, where n is the public modulus.
Reviewed-by: Nikola Pajkovsky <nikolap@openssl.org>
Reviewed-by: Bob Beck <beck@openssl.org>
MergeDate: Mon Aug 10 06:27:45 2026
(Merged from https://github.com/openssl/openssl/pull/32124)
diff --git a/crypto/rsa/rsa_ossl.c b/crypto/rsa/rsa_ossl.c
index 674ee4b10d..8a3deae581 100644
--- a/crypto/rsa/rsa_ossl.c
+++ b/crypto/rsa/rsa_ossl.c
@@ -564,7 +564,6 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from,
if (BN_bin2bn(from, (int)flen, f) == NULL)
goto err;
-#ifdef FIPS_MODULE
/*
* See SP800-56Br2, section 7.1.2.1
* RSADP: 1 < f < (n – 1)
@@ -585,9 +584,7 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from,
ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
- } else
-#endif
- {
+ } else {
if (BN_ucmp(f, rsa->n) >= 0) {
ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
diff --git a/providers/implementations/kem/rsa_kem.c b/providers/implementations/kem/rsa_kem.c
index ab28a3a1a2..729c5ac6be 100644
--- a/providers/implementations/kem/rsa_kem.c
+++ b/providers/implementations/kem/rsa_kem.c
@@ -140,6 +140,7 @@ static int rsakem_init(void *vprsactx, void *vrsa,
const char *desc)
{
PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
+ const BIGNUM *e = NULL;
int protect = 0;
if (!ossl_prov_is_running())
@@ -155,6 +156,18 @@ static int rsakem_init(void *vprsactx, void *vrsa,
RSA_free(prsactx->rsa);
prsactx->rsa = vrsa;
+ /*
+ * Reject the trivial public exponent e <= 1. The FIPS module enforces the
+ * full SP 800-56B §6.4.1.1 constraints via ossl_fips_ind_rsa_key_check()
+ * below; non-FIPS callers wanting the complete §6.4.2 vetting can use
+ * EVP_PKEY_public_check().
+ */
+ RSA_get0_key(prsactx->rsa, NULL, &e, NULL);
+ if (e == NULL || BN_cmp(e, BN_value_one()) <= 0) {
+ ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY);
+ return 0;
+ }
+
OSSL_FIPS_IND_SET_APPROVED(prsactx)
if (!rsakem_set_ctx_params(prsactx, params))
return 0;
diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c
index 14eaa2a37f..177eef4b18 100644
--- a/test/evp_extra_test.c
+++ b/test/evp_extra_test.c
@@ -8833,6 +8833,135 @@ end:
return testresult;
}
+/*
+ * RSASVE (SP 800-56B 7.2) must reject mathematically degenerate inputs:
+ * a public exponent e <= 1, and a ciphertext c in {0, 1, n - 1}. Outside
+ * the FIPS module these were previously accepted; the checks now apply to
+ * every build, so exercise them in the default provider.
+ */
+
+/*
+ * With e <= 1 the RSA public operation is the identity (or worse), so
+ * encapsulation setup must reject the key with PROV_R_INVALID_KEY. idx
+ * selects the exponent: 0 or 1.
+ */
+static int test_rsasve_degenerate_exponent(int idx)
+{
+ EVP_PKEY *rsakey = NULL;
+ EVP_PKEY *pubkey = NULL;
+ EVP_PKEY_CTX *genctx = NULL;
+ EVP_PKEY_CTX *ctx = NULL;
+ OSSL_PARAM_BLD *bld = NULL;
+ OSSL_PARAM *params = NULL;
+ BIGNUM *n = NULL;
+ BIGNUM *e = NULL;
+ int testresult = 0;
+
+ /* Borrow a real modulus; only the exponent is degenerate. */
+ if (!TEST_ptr(rsakey = load_example_rsa_key())
+ || !TEST_true(EVP_PKEY_get_bn_param(rsakey, OSSL_PKEY_PARAM_RSA_N, &n)))
+ goto err;
+
+ if (!TEST_ptr(e = BN_new())
+ || !TEST_true(BN_set_word(e, (BN_ULONG)idx))) /* idx is 0 or 1 */
+ goto err;
+
+ if (!TEST_ptr(bld = OSSL_PARAM_BLD_new())
+ || !TEST_true(OSSL_PARAM_BLD_push_BN(bld, OSSL_PKEY_PARAM_RSA_N, n))
+ || !TEST_true(OSSL_PARAM_BLD_push_BN(bld, OSSL_PKEY_PARAM_RSA_E, e))
+ || !TEST_ptr(params = OSSL_PARAM_BLD_to_param(bld)))
+ goto err;
+
+ if (!TEST_ptr(genctx = EVP_PKEY_CTX_new_from_name(testctx, "RSA", NULL))
+ || !TEST_int_gt(EVP_PKEY_fromdata_init(genctx), 0)
+ || !TEST_int_gt(EVP_PKEY_fromdata(genctx, &pubkey, EVP_PKEY_PUBLIC_KEY,
+ params),
+ 0))
+ goto err;
+
+ ERR_clear_error();
+ if (!TEST_ptr(ctx = EVP_PKEY_CTX_new_from_pkey(testctx, pubkey, NULL))
+ || !TEST_int_eq(EVP_PKEY_encapsulate_init(ctx, NULL), 0)
+ || !TEST_int_eq(ERR_GET_REASON(ERR_get_error()), PROV_R_INVALID_KEY))
+ goto err;
+
+ testresult = 1;
+err:
+ EVP_PKEY_CTX_free(ctx);
+ EVP_PKEY_CTX_free(genctx);
+ EVP_PKEY_free(pubkey);
+ EVP_PKEY_free(rsakey);
+ OSSL_PARAM_free(params);
+ OSSL_PARAM_BLD_free(bld);
+ BN_free(e);
+ BN_free(n);
+ return testresult;
+}
+
+/*
+ * A ciphertext c in {0, 1, n - 1} is a fixed point or trivial case of RSADP,
+ * so RSASVE recovery must reject it. idx selects the ciphertext: 0, 1, or
+ * n - 1. The ciphertext length must equal the modulus length.
+ */
+static int test_rsasve_degenerate_ciphertext(int idx)
+{
+ EVP_PKEY *rsakey = NULL;
+ EVP_PKEY_CTX *ctx = NULL;
+ BIGNUM *n = NULL;
+ unsigned char *ct = NULL;
+ unsigned char *secret = NULL;
+ size_t ctlen = 0;
+ size_t secretlen = 0;
+ int expected_reason = 0;
+ int testresult = 0;
+
+ if (!TEST_ptr(rsakey = load_example_rsa_key())
+ || !TEST_true(EVP_PKEY_get_bn_param(rsakey, OSSL_PKEY_PARAM_RSA_N, &n)))
+ goto err;
+
+ ctlen = secretlen = (size_t)EVP_PKEY_get_size(rsakey);
+ if (!TEST_ptr(ct = OPENSSL_zalloc(ctlen))
+ || !TEST_ptr(secret = OPENSSL_malloc(secretlen)))
+ goto err;
+
+ switch (idx) {
+ case 0: /* c = 0 */
+ expected_reason = RSA_R_DATA_TOO_SMALL;
+ break;
+ case 1: /* c = 1 */
+ ct[ctlen - 1] = 1;
+ expected_reason = RSA_R_DATA_TOO_SMALL;
+ break;
+ case 2: /* c = n - 1 */
+ if (!TEST_true(BN_sub_word(n, 1))
+ || !TEST_int_eq(BN_bn2binpad(n, ct, (int)ctlen), (int)ctlen))
+ goto err;
+ expected_reason = RSA_R_DATA_TOO_LARGE_FOR_MODULUS;
+ break;
+ default:
+ goto err;
+ }
+
+ if (!TEST_ptr(ctx = EVP_PKEY_CTX_new_from_pkey(testctx, rsakey, NULL))
+ || !TEST_int_eq(EVP_PKEY_decapsulate_init(ctx, NULL), 1)
+ || !TEST_int_eq(EVP_PKEY_CTX_set_kem_op(ctx, "RSASVE"), 1))
+ goto err;
+
+ ERR_clear_error();
+ if (!TEST_int_eq(EVP_PKEY_decapsulate(ctx, secret, &secretlen, ct, ctlen), 0)
+ || !TEST_int_eq(ERR_GET_REASON(ERR_get_error()), expected_reason))
+ goto err;
+
+ testresult = 1;
+err:
+ OPENSSL_free(secret);
+ OPENSSL_free(ct);
+ EVP_PKEY_CTX_free(ctx);
+ EVP_PKEY_free(rsakey);
+ BN_free(n);
+ return testresult;
+}
+
#ifndef OPENSSL_NO_DEPRECATED_3_0
static int sign_hits = 0;
@@ -9492,6 +9621,9 @@ int setup_tests(void)
ADD_TEST(test_evp_cipher_pipeline);
+ ADD_ALL_TESTS(test_rsasve_degenerate_exponent, 2);
+ ADD_ALL_TESTS(test_rsasve_degenerate_ciphertext, 3);
+
#ifndef OPENSSL_NO_ML_KEM
ADD_ALL_TESTS(test_ml_kem_seed_only, 2);
#endif