Commit 3b6a8a1fd0 for openssl.org

commit 3b6a8a1fd014107e6d27a1dfe70f9018531aeba8
Author: Viktor Dukhovni <viktor@openssl.org>
Date:   Wed Aug 12 14:36:38 2026 +1000

    Revised RSASVE degenerate ciphertext check.

    The additional ciphertext check is now applied in rsasve_recover() where it
    belongs, and not in the underlying RSA primitives, where it remains conditional
    defined(FIPS_MODULE).

    Reviewed-by: Milan Broz <mbroz@openssl.org>
    Reviewed-by: Nikola Pajkovsky <nikolap@openssl.org>
    MergeDate: Mon Aug 17 13:56:53 2026
    (Merged from https://github.com/openssl/openssl/pull/32314)

diff --git a/crypto/rsa/rsa_ossl.c b/crypto/rsa/rsa_ossl.c
index 8a3deae581..3939d00e07 100644
--- a/crypto/rsa/rsa_ossl.c
+++ b/crypto/rsa/rsa_ossl.c
@@ -167,6 +167,14 @@ static int rsa_ossl_public_encrypt(int flen, const unsigned char *from,
      * See SP800-56Br2, section 7.1.1.1
      * RSAEP: 1 < f < (n – 1).
      * (where f is the plaintext).
+     *
+     * This bound is somewhat overkill here.  RSASVE.GENERATE (7.2.1.2)
+     * regenerates z until 1 < z < n-1, so on that path the plaintext is in
+     * range unconditionally.  On the OAEP path the leading 0x00 octet of the
+     * encoding forces m < n-1 unconditionally, while m = 0 or 1 is only
+     * cryptographically negligible, not impossible.  The check is kept to
+     * mirror the RSADP bound in rsa_ossl_private_decrypt() and to keep RSAEP
+     * faithful to 7.1.1 of the SP; nothing in the SP relies on it here.
      */
     if (padding == RSA_NO_PADDING) {
         BIGNUM *nminus1 = BN_CTX_get(ctx);
@@ -564,10 +572,17 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from,
     if (BN_bin2bn(from, (int)flen, f) == NULL)
         goto err;

+#ifdef FIPS_MODULE
     /*
      * See SP800-56Br2, section 7.1.2.1
      * RSADP: 1 < f < (n – 1)
      * (where f is the ciphertext).
+     *
+     * Kept under FIPS_MODULE because SP 800-56B KTS-OAEP (section 9.2) also
+     * decrypts through RSADP and needs this bound in a FIPS build, and there
+     * is no KTS-OAEP-specific path to attach it to.  The non-FIPS RSASVE path
+     * applies the same 1 < c < n-1 in rsasve_recover()
+     * (providers/implementations/kem/rsa_kem.c); keep the two in step.
      */
     if (padding == RSA_NO_PADDING) {
         BIGNUM *nminus1 = BN_CTX_get(ctx);
@@ -584,7 +599,9 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from,
             ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
             goto err;
         }
-    } else {
+    } else
+#endif
+    {
         if (BN_ucmp(f, rsa->n) >= 0) {
             ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
             goto err;
diff --git a/providers/implementations/kem/rsa_kem.c b/providers/implementations/kem/rsa_kem.c
index 729c5ac6be..675409ba33 100644
--- a/providers/implementations/kem/rsa_kem.c
+++ b/providers/implementations/kem/rsa_kem.c
@@ -400,6 +400,44 @@ static int rsasve_recover(PROV_RSA_CTX *prsactx,
         return 0;
     }

+#ifndef FIPS_MODULE
+    /*
+     * Reject clearly degenerate ciphertexts, c in {0, 1, n-1}.
+     *
+     * SP 800-56B Rev 2, 7.1.2.1 requires RSADP to enforce 1 < c < n-1.  In a
+     * FIPS build that bound is applied by the RSADP primitive itself (see
+     * crypto/rsa/rsa_ossl.c, guarded by FIPS_MODULE), where it is also needed
+     * for KTS-OAEP; the primitive does not apply it in a non-FIPS build, so
+     * enforce it here for RSASVE.  Raise the same errors as the primitive so
+     * the behaviour matches in both builds; keep the two sites in step.
+     */
+    {
+        const BIGNUM *n = RSA_get0_n(prsactx->rsa);
+        BIGNUM *c = BN_new();
+        BIGNUM *nminus1 = BN_new();
+        int reason = 0;
+
+        if (n == NULL || c == NULL || nminus1 == NULL
+            || BN_bin2bn(in, (int)inlen, c) == NULL
+            || BN_copy(nminus1, n) == NULL
+            || !BN_sub_word(nminus1, 1)) {
+            BN_free(c);
+            BN_free(nminus1);
+            return 0;
+        }
+        if (BN_ucmp(c, BN_value_one()) <= 0)
+            reason = RSA_R_DATA_TOO_SMALL;
+        else if (BN_ucmp(c, nminus1) >= 0)
+            reason = RSA_R_DATA_TOO_LARGE_FOR_MODULUS;
+        BN_free(c);
+        BN_free(nminus1);
+        if (reason != 0) {
+            ERR_raise(ERR_LIB_RSA, reason);
+            return 0;
+        }
+    }
+#endif
+
     /* Step (3): out = RSADP((n,d), in) */
     ret = RSA_private_decrypt((int)inlen, in, out, prsactx->rsa, RSA_NO_PADDING);
     if (ret > 0 && outlen != NULL)