Commit ca519ad1d5 for openssl.org
commit ca519ad1d591a91138aa7d7a10686cd2f180a050
Author: Mayank Jangid <mayank.jangid.moon@gmail.com>
Date: Tue Apr 21 21:27:21 2026 +0530
rsa_sig: reject short buffers in verify_recover
The RSA PKCS#1 verify-recover provider path did not validate routsize
before passing the caller buffer to ossl_rsa_verify().
The X9.31 verify-recover path already rejects undersized output buffers,
but the PKCS#1 path could proceed with too little output space and rely
on the lower layer to write the recovered digest.
Check the expected digest size before calling ossl_rsa_verify() and
return PROV_R_OUTPUT_BUFFER_TOO_SMALL when the caller-provided buffer is
too small.
Add a regression test that covers both successful recovery with a
properly sized buffer and failure with a 1-byte output buffer, while
also checking that the short buffer is left unchanged.
Co-authored-by: Kushal <72255307+Kushalkhemka@users.noreply.github.com>
Co-authored-by: Mayank <175295782+mayank-jangid-moon@users.noreply.github.com>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
MergeDate: Wed May 27 11:46:40 2026
(Merged from https://github.com/openssl/openssl/pull/30917)
diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
index a4c5a90741..43f648e2d4 100644
--- a/providers/implementations/signature/rsa_sig.c
+++ b/providers/implementations/signature/rsa_sig.c
@@ -988,8 +988,19 @@ static int rsa_verify_recover(void *vprsactx,
break;
case RSA_PKCS1_PADDING: {
+ int mdsize = EVP_MD_get_size(prsactx->md);
size_t sltmp;
+ if (mdsize <= 0) {
+ ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_DIGEST_LENGTH);
+ return 0;
+ }
+ if (routsize < (size_t)mdsize) {
+ ERR_raise_data(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL,
+ "buffer size is %d, should be %d",
+ routsize, mdsize);
+ return 0;
+ }
ret = ossl_rsa_verify(prsactx->mdnid, NULL, 0, rout, &sltmp,
sig, siglen, prsactx->rsa);
if (ret <= 0) {
diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c
index f4bfeae2a1..0f805f6cec 100644
--- a/test/evp_extra_test.c
+++ b/test/evp_extra_test.c
@@ -4036,6 +4036,82 @@ err:
return ret;
}
+static int test_RSA_verify_recover_rejects_short_buffer(void)
+{
+ int ret = 0;
+ int recovered_cap = 0;
+ EVP_PKEY *pkey = NULL;
+ EVP_PKEY_CTX *sign_ctx = NULL, *verify_ctx = NULL;
+ unsigned char *sig = NULL, *recovered = NULL;
+ size_t sig_len = 0, recovered_len = 0;
+ unsigned long err = 0;
+ unsigned char shortbuf[] = { 0xa5, 0x5a };
+ const unsigned char shortbuf_expected[] = { 0xa5, 0x5a };
+ unsigned char digest[32];
+ size_t i;
+
+ for (i = 0; i < sizeof(digest); i++)
+ digest[i] = (unsigned char)i;
+
+ if (!TEST_ptr(pkey = load_example_rsa_key())
+ || !TEST_ptr(sign_ctx = EVP_PKEY_CTX_new_from_pkey(testctx, pkey, NULL))
+ || !TEST_int_gt(EVP_PKEY_sign_init(sign_ctx), 0)
+ || !TEST_int_gt(EVP_PKEY_CTX_set_rsa_padding(sign_ctx,
+ RSA_PKCS1_PADDING),
+ 0)
+ || !TEST_int_gt(EVP_PKEY_CTX_set_signature_md(sign_ctx, EVP_sha256()),
+ 0)
+ || !TEST_int_gt(EVP_PKEY_sign(sign_ctx, NULL, &sig_len, digest,
+ sizeof(digest)),
+ 0)
+ || !TEST_ptr(sig = OPENSSL_malloc(sig_len))
+ || !TEST_int_gt(EVP_PKEY_sign(sign_ctx, sig, &sig_len, digest,
+ sizeof(digest)),
+ 0)
+ || !TEST_int_gt(recovered_cap = EVP_PKEY_get_size(pkey), 0)
+ || !TEST_ptr(recovered = OPENSSL_malloc(recovered_cap))
+ || !TEST_ptr(verify_ctx = EVP_PKEY_CTX_new_from_pkey(testctx, pkey,
+ NULL))
+ || !TEST_int_gt(EVP_PKEY_verify_recover_init(verify_ctx), 0)
+ || !TEST_int_gt(EVP_PKEY_CTX_set_rsa_padding(verify_ctx,
+ RSA_PKCS1_PADDING),
+ 0)
+ || !TEST_int_gt(EVP_PKEY_CTX_set_signature_md(verify_ctx, EVP_sha256()),
+ 0))
+ goto done;
+
+ recovered_len = (size_t)recovered_cap;
+ if (!TEST_int_gt(EVP_PKEY_verify_recover(verify_ctx, recovered,
+ &recovered_len, sig, sig_len),
+ 0)
+ || !TEST_size_t_eq(recovered_len, sizeof(digest))
+ || !TEST_mem_eq(recovered, recovered_len, digest, sizeof(digest)))
+ goto done;
+
+ ERR_clear_error();
+ recovered_len = 1;
+ if (!TEST_int_le(EVP_PKEY_verify_recover(verify_ctx, shortbuf,
+ &recovered_len, sig, sig_len),
+ 0))
+ goto done;
+
+ err = ERR_peek_error();
+ if (!TEST_int_eq(ERR_GET_LIB(err), ERR_LIB_PROV)
+ || !TEST_int_eq(ERR_GET_REASON(err), PROV_R_OUTPUT_BUFFER_TOO_SMALL)
+ || !TEST_mem_eq(shortbuf, sizeof(shortbuf), shortbuf_expected,
+ sizeof(shortbuf_expected)))
+ goto done;
+
+ ret = 1;
+done:
+ EVP_PKEY_CTX_free(sign_ctx);
+ EVP_PKEY_CTX_free(verify_ctx);
+ EVP_PKEY_free(pkey);
+ OPENSSL_free(sig);
+ OPENSSL_free(recovered);
+ return ret;
+}
+
static int test_RSA_encrypt(void)
{
int ret = 0;
@@ -7958,6 +8034,7 @@ int setup_tests(void)
ADD_TEST(test_RSA_get_set_params);
ADD_TEST(test_RSA_OAEP_set_get_params);
ADD_TEST(test_RSA_OAEP_set_null_label);
+ ADD_TEST(test_RSA_verify_recover_rejects_short_buffer);
ADD_TEST(test_RSA_encrypt);
#ifndef OPENSSL_NO_DEPRECATED_3_0
ADD_TEST(test_RSA_legacy);