Commit 35e31505d5 for openssl.org
commit 35e31505d50671b8a783a48e1fc26e5375483e51
Author: Jakub Zelenka <jakub.zelenka@openssl.foundation>
Date: Sat Jul 25 19:01:52 2026 +0200
cms: correctly fail on invalid key length in CMS_decrypt
Report an invalid content encryption key length instead of decrypting
with a random key when this cannot act as an MMA (Bleichenbacher)
oracle, that is with RSA-OAEP or when PKCS#1 v1.5 implicit rejection
is in effect. Otherwise, such as with third party providers without
implicit rejection, keep the random key masking.
The same applies to a failure of the key decryption itself. With
implicit rejection in effect the RSA decryption can only fail for a
publicly invalid ciphertext, such as a value not smaller than the
modulus, and with RSA-OAEP a padding check failure is safe to reveal,
so the error is reported instead of falling back to the random key
masking. This also makes CMS_decrypt with a wrong key deterministic
in those configurations, which the new test relies on.
Fixes #25875
Assisted-by: Claude:claude-fable-5
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Alicja Kario <hkario@redhat.com>
Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
Merge-date: Thu Sep 17 09:29:58 2026
Merged-from: https://github.com/openssl/openssl/pull/28038
diff --git a/crypto/cms/cms_enc.c b/crypto/cms/cms_enc.c
index 087a7fe638..1dfb19c3eb 100644
--- a/crypto/cms/cms_enc.c
+++ b/crypto/cms/cms_enc.c
@@ -151,25 +151,19 @@ BIO *ossl_cms_EncryptedContent_init_bio(CMS_EncryptedContentInfo *ec,
ERR_clear_error();
}
- if (ec->keylen != tkeylen) {
- /* If necessary set key length */
- if (EVP_CIPHER_CTX_set_key_length(ctx, (int)ec->keylen) <= 0) {
- /*
- * Only reveal failure if debugging so we don't leak information
- * which may be useful in MMA.
- */
- if (enc || ec->debug) {
- ERR_raise(ERR_LIB_CMS, CMS_R_INVALID_KEY_LENGTH);
- goto err;
- } else {
- /* Use random key */
- OPENSSL_clear_free(ec->key, ec->keylen);
- ec->key = tkey;
- ec->keylen = tkeylen;
- tkey = NULL;
- ERR_clear_error();
- }
+ if (ec->keylen != tkeylen
+ && EVP_CIPHER_CTX_set_key_length(ctx, (int)ec->keylen) <= 0) {
+ /* Fail only when this cannot act as an MMA oracle or debug enabled */
+ if (enc || ec->debug || ec->harderr) {
+ ERR_raise(ERR_LIB_CMS, CMS_R_INVALID_KEY_LENGTH);
+ goto err;
}
+ /* Use random key */
+ OPENSSL_clear_free(ec->key, ec->keylen);
+ ec->key = tkey;
+ ec->keylen = tkeylen;
+ tkey = NULL;
+ ERR_clear_error();
}
if (EVP_CipherInit_ex(ctx, NULL, NULL, ec->key, piv, enc) <= 0) {
diff --git a/crypto/cms/cms_env.c b/crypto/cms/cms_env.c
index d9ea9bdd7b..bd66900861 100644
--- a/crypto/cms/cms_env.c
+++ b/crypto/cms/cms_env.c
@@ -20,6 +20,8 @@
#include <openssl/err.h>
#include <openssl/cms.h>
#include <openssl/evp.h>
+#include <openssl/rsa.h>
+#include <openssl/params.h>
#include <openssl/core_names.h>
#include "internal/sizes.h"
#include "crypto/asn1.h"
@@ -591,6 +593,35 @@ err:
/* Decrypt content key from KTRI */
+/* Check whether reporting a key length mismatch cannot act as an MMA oracle */
+static int cms_ktri_harderr_ok(EVP_PKEY_CTX *pctx, EVP_PKEY *pkey)
+{
+ int pad_mode;
+ unsigned int implicit_rejection = 0;
+ OSSL_PARAM params[2];
+
+ if (!EVP_PKEY_is_a(pkey, "RSA")
+ || EVP_PKEY_CTX_get_rsa_padding(pctx, &pad_mode) <= 0)
+ return 0;
+
+ /* An RSA-OAEP decryption failure is safe to reveal */
+ if (pad_mode == RSA_PKCS1_OAEP_PADDING)
+ return 1;
+ if (pad_mode != RSA_PKCS1_PADDING)
+ return 0;
+
+ /* For PKCS#1 v1.5 it is only safe with implicit rejection in effect */
+ params[0] = OSSL_PARAM_construct_uint(
+ OSSL_ASYM_CIPHER_PARAM_IMPLICIT_REJECTION,
+ &implicit_rejection);
+ params[1] = OSSL_PARAM_construct_end();
+ if (EVP_PKEY_CTX_get_params(pctx, params) <= 0
+ || !OSSL_PARAM_modified(¶ms[0]))
+ return 0;
+
+ return implicit_rejection != 0;
+}
+
static int cms_RecipientInfo_ktri_decrypt(CMS_ContentInfo *cms,
CMS_RecipientInfo *ri)
{
@@ -649,6 +680,17 @@ static int cms_RecipientInfo_ktri_decrypt(CMS_ContentInfo *cms,
if (!ossl_cms_env_asn1_ctrl(ri, 1))
goto err;
+ /*
+ * Check whether a decryption failure or a key length mismatch can be
+ * reported without MMA risk. This must be determined before the
+ * decryption is attempted so a failure of the decryption itself (only
+ * possible for a publicly invalid ciphertext when implicit rejection
+ * is in effect, or a padding check failure with RSA-OAEP) is reported
+ * as well.
+ */
+ if (!ec->havenocert && !ec->debug)
+ ec->harderr = cms_ktri_harderr_ok(ktri->pctx, pkey);
+
if (evp_pkey_decrypt_alloc(ktri->pctx, &ek, &eklen, fixlen,
ktri->encryptedKey->data,
ktri->encryptedKey->length)
diff --git a/crypto/cms/cms_local.h b/crypto/cms/cms_local.h
index 0b1f54834f..c2dfa55be9 100644
--- a/crypto/cms/cms_local.h
+++ b/crypto/cms/cms_local.h
@@ -156,6 +156,8 @@ struct CMS_EncryptedContentInfo_st {
int debug;
/* Set to 1 if we have no cert and need extra safety measures for MMA */
int havenocert;
+ /* Set to 1 if key length mismatch can be reported without an MMA risk */
+ int harderr;
};
struct CMS_RecipientInfo_st {
diff --git a/crypto/cms/cms_smime.c b/crypto/cms/cms_smime.c
index b3737beaa9..ae99d1b27f 100644
--- a/crypto/cms/cms_smime.c
+++ b/crypto/cms/cms_smime.c
@@ -792,6 +792,7 @@ int CMS_decrypt_set1_pkey_and_peer(CMS_ContentInfo *cms, EVP_PKEY *pk,
OPENSSL_clear_free(ec->key, ec->keylen);
ec->key = NULL;
ec->keylen = 0;
+ ec->harderr = 0;
}
if (ris != NULL && ec != NULL)
@@ -835,10 +836,11 @@ int CMS_decrypt_set1_pkey_and_peer(CMS_ContentInfo *cms, EVP_PKEY *pk,
CMS_RecipientInfo_set0_pkey(ri, NULL);
if (cert != NULL) {
/*
- * If not debugging clear any error and return success to
- * avoid leaking of information useful to MMA
+ * If not debugging and a failure cannot be reported safely,
+ * clear any error and return success to avoid leaking of
+ * information useful to MMA
*/
- if (!debug) {
+ if (!debug && (ec == NULL || !ec->harderr)) {
ERR_clear_error();
return 1;
}
@@ -919,6 +921,7 @@ int CMS_decrypt_set1_password(CMS_ContentInfo *cms,
OPENSSL_clear_free(ec->key, ec->keylen);
ec->key = NULL;
ec->keylen = 0;
+ ec->harderr = 0;
}
for (i = 0; i < sk_CMS_RecipientInfo_num(ris); i++) {
diff --git a/test/cmsapitest.c b/test/cmsapitest.c
index 62bbca842e..ac34f7ef9e 100644
--- a/test/cmsapitest.c
+++ b/test/cmsapitest.c
@@ -21,6 +21,8 @@ static X509 *cert = NULL;
static EVP_PKEY *privkey = NULL;
static X509 *ed448_cert = NULL;
static EVP_PKEY *ed448_privkey = NULL;
+static X509 *cert2 = NULL;
+static EVP_PKEY *privkey2 = NULL;
static char *derin = NULL;
static char *too_long_iv_cms_in = NULL;
static char *pwri_kek_oob_der_in = NULL;
@@ -318,6 +320,50 @@ end:
return ret;
}
+static int test_decrypt_with_wrong_key(void)
+{
+ int testresult = 0;
+ STACK_OF(X509) *certstack = sk_X509_new_null();
+ const char *msg = "Hello world";
+ BIO *msgbio = BIO_new_mem_buf(msg, (int)strlen(msg));
+ BIO *outmsgbio;
+ CMS_ContentInfo *content = NULL;
+ BIO *contentbio = NULL;
+ const EVP_CIPHER *cipher = EVP_aes_128_cbc();
+
+ if (!TEST_ptr(certstack) || !TEST_ptr(msgbio))
+ goto end;
+
+ if (!TEST_int_gt(sk_X509_push(certstack, cert), 0))
+ goto end;
+
+ content = CMS_encrypt(certstack, msgbio, cipher, 0);
+ if (!TEST_ptr(content))
+ goto end;
+
+ for (int i = 0; i < 1000; ++i) {
+ outmsgbio = BIO_new(BIO_s_mem());
+ if (!TEST_false(CMS_decrypt(content, privkey2, cert, NULL, outmsgbio,
+ 0)
+ == 1)) {
+ BIO_free(outmsgbio);
+ goto end;
+ }
+ BIO_free(outmsgbio);
+ }
+
+ ERR_clear_error();
+
+ testresult = 1;
+end:
+ BIO_free(contentbio);
+ sk_X509_free(certstack);
+ BIO_free(msgbio);
+ CMS_ContentInfo_free(content);
+
+ return testresult;
+}
+
static int test_CMS_add1_cert(void)
{
CMS_ContentInfo *cms = NULL;
@@ -982,12 +1028,15 @@ end:
}
#endif
-OPT_TEST_DECLARE_USAGE("certfile privkeyfile derfile tooLongIVpem pwriKekOobDer pwriKekNoIv ecrecip [ed448certfile ed448privkeyfile]\n")
+OPT_TEST_DECLARE_USAGE("certfile privkeyfile derfile tooLongIVpem pwriKekOobDer"
+ " pwriKekNoIv ecrecip certfile2 privkeyfile2"
+ " [ed448certfile ed448privkeyfile]\n")
int setup_tests(void)
{
char *certin = NULL, *privkeyin = NULL;
char *ed448_certin = NULL, *ed448_privkeyin = NULL;
+ char *certin2 = NULL, *privkeyin2 = NULL;
if (!test_skip_common_options()) {
TEST_error("Error parsing test options\n");
@@ -1000,21 +1049,29 @@ int setup_tests(void)
|| !TEST_ptr(too_long_iv_cms_in = test_get_argument(3))
|| !TEST_ptr(pwri_kek_oob_der_in = test_get_argument(4))
|| !TEST_ptr(pwri_kek_no_iv_in = test_get_argument(5))
- || !TEST_ptr(ec_recip_in = test_get_argument(6)))
+ || !TEST_ptr(ec_recip_in = test_get_argument(6))
+ || !TEST_ptr(certin2 = test_get_argument(7))
+ || !TEST_ptr(privkeyin2 = test_get_argument(8)))
return 0;
if (!TEST_ptr(cert = load_cert_pem(certin, NULL))
- || !TEST_ptr(privkey = load_pkey_pem(privkeyin, NULL))) {
+ || !TEST_ptr(privkey = load_pkey_pem(privkeyin, NULL))
+ || !TEST_ptr(cert2 = load_cert_pem(certin2, NULL))
+ || !TEST_ptr(privkey2 = load_pkey_pem(privkeyin2, NULL))) {
X509_free(cert);
cert = NULL;
EVP_PKEY_free(privkey);
privkey = NULL;
+ X509_free(cert2);
+ cert2 = NULL;
+ EVP_PKEY_free(privkey2);
+ privkey2 = NULL;
return 0;
}
- if (test_get_argument_count() >= 9) {
- ed448_certin = test_get_argument(7);
- ed448_privkeyin = test_get_argument(8);
+ if (test_get_argument_count() >= 11) {
+ ed448_certin = test_get_argument(9);
+ ed448_privkeyin = test_get_argument(10);
if (!TEST_ptr(ed448_cert = load_cert_pem(ed448_certin, NULL))
|| !TEST_ptr(ed448_privkey = load_pkey_pem(ed448_privkeyin, NULL))) {
@@ -1034,6 +1091,7 @@ int setup_tests(void)
ADD_TEST(test_non_aead_on_auth_envelope_dec);
ADD_TEST(test_short_mac_on_auth_envelope_data);
ADD_TEST(test_CMS_add_standard_smimecap_ex);
+ ADD_TEST(test_decrypt_with_wrong_key);
ADD_TEST(test_CMS_add1_cert);
ADD_TEST(test_CMS_SignerInfo_verify_sigalg_oid);
ADD_TEST(test_d2i_CMS_bio_NULL);
@@ -1062,4 +1120,6 @@ void cleanup_tests(void)
EVP_PKEY_free(privkey);
X509_free(ed448_cert);
EVP_PKEY_free(ed448_privkey);
+ X509_free(cert2);
+ EVP_PKEY_free(privkey2);
}
diff --git a/test/recipes/80-test_cmsapi.t b/test/recipes/80-test_cmsapi.t
index 7edaca9423..adfa8d51f5 100644
--- a/test/recipes/80-test_cmsapi.t
+++ b/test/recipes/80-test_cmsapi.t
@@ -27,5 +27,7 @@ ok(run(test(["cmsapitest", srctop_file("test", "certs", "servercert.pem"),
srctop_file("test", "recipes", "80-test_cmsapi_data", "cms_pwri_kek_oob.der"),
srctop_file("test", "recipes", "80-test_cmsapi_data", "cms_pwri_kek_NoIV.der"),
srctop_file("test", "smime-certs", "smec1.pem"),
+ srctop_file("test", "certs", "alt1-cert.pem"),
+ srctop_file("test", "certs", "alt1-key.pem"),
@ed448_args])),
"running cmsapitest");