Commit f247d36074 for openssl.org
commit f247d36074353f44596cb941c0a5f929c2e46e67
Author: Viktor Dukhovni <openssl-users@dukhovni.org>
Date: Sat Dec 6 15:04:36 2025 +1100
The return value of RSA_*_{en,de}crypt() is signed
The functions RSA_(public|private)_(en|de)crypt() return a signed
result, in particular `-1` may be returned on error, so the caller
MUST treat the value as signed.
Reviewed-by: Saša NedvÄ›dický <sashan@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/29323)
diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index ecc0b1b829..9e053edb7c 100644
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -693,12 +693,12 @@ static int rsa_keygen(OSSL_LIB_CTX *libctx, RSA *rsa, int bits, int primes,
static int rsa_keygen_pairwise_test(RSA *rsa, OSSL_CALLBACK *cb, void *cbarg)
{
int ret = 0;
- unsigned int plaintxt_len;
unsigned char *plaintxt = NULL;
- unsigned int ciphertxt_len;
unsigned char *ciphertxt = NULL;
unsigned char *decoded = NULL;
- unsigned int decoded_len;
+ int plaintxt_len;
+ int ciphertxt_len;
+ int decoded_len;
int padding = RSA_NO_PADDING;
OSSL_SELF_TEST *st = NULL;