Commit 3741076a34 for openssl.org

commit 3741076a34f4953ccbb6e1e131756ac206c8f27e
Author: eclipse07077 <eclipse07077@gmail.com>
Date:   Mon Mar 9 21:35:43 2026 +0900

    Fix integer overflow in EVP_ENCODE_LENGTH and base64 encoding paths

    The EVP_ENCODE_LENGTH macro performs all arithmetic in the type of
    its argument. When the argument is int and exceeds approximately
    1.6 billion, intermediate results overflow signed int, potentially
    wrapping to a smaller positive value rather than a negative one.

    In b64_write() (crypto/evp/bio_b64.c), this causes OPENSSL_malloc
    to allocate a buffer smaller than the actual encoded output size.
    EVP_EncodeUpdate then writes past the end of the undersized buffer.

    Changes:
    - Cast macro argument to size_t in EVP_ENCODE_LENGTH to prevent
      signed integer overflow
    - Change encoded_length in b64_write() from int to size_t and add
      an explicit overflow sanity check before allocation
    - Change return type of evp_encodeblock_int() and
      encode_base64_avx2() from int to size_t so that large encoded
      output lengths are not truncated
    - Update EVP_EncodeUpdate() to use size_t for the encoder return
      value accumulator (j), consistent with the existing size_t total
    - Add explicit (int) casts in EVP_EncodeBlock() and EVP_EncodeFinal()
      where the public API requires int return values

    Reviewed-by: Saša Nedvědický <sashan@openssl.org>
    Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
    MergeDate: Fri Apr  3 14:55:29 2026
    (Merged from https://github.com/openssl/openssl/pull/30321)

diff --git a/crypto/evp/bio_b64.c b/crypto/evp/bio_b64.c
index 33ddb6334d..7f32d7a5d0 100644
--- a/crypto/evp/bio_b64.c
+++ b/crypto/evp/bio_b64.c
@@ -334,7 +334,7 @@ static int b64_write(BIO *b, const char *in, int inl)
     int i;
     BIO_B64_CTX *ctx;
     BIO *next;
-    int encoded_length;
+    size_t encoded_length;
     unsigned char *encoded;
     int n_bytes_enc;

@@ -393,7 +393,12 @@ static int b64_write(BIO *b, const char *in, int inl)

     encoded_length = EVP_ENCODE_LENGTH(inl);

-    if (ctx->encoded_buf == NULL || (size_t)encoded_length > ctx->encoded_buf_len) {
+    if (encoded_length > SIZE_MAX / 2) {
+        ERR_raise(ERR_LIB_BIO, BIO_R_LENGTH_TOO_LONG);
+        return -1;
+    }
+
+    if (ctx->encoded_buf == NULL || encoded_length > ctx->encoded_buf_len) {
         OPENSSL_free(ctx->encoded_buf);
         ctx->encoded_buf = OPENSSL_malloc(encoded_length);
         if (ctx->encoded_buf == NULL) {
diff --git a/crypto/evp/enc_b64_avx2.c b/crypto/evp/enc_b64_avx2.c
index e6f571dd8e..acf35739b5 100644
--- a/crypto/evp/enc_b64_avx2.c
+++ b/crypto/evp/enc_b64_avx2.c
@@ -477,7 +477,7 @@ static inline size_t insert_nl_str8(const __m256i v0, uint8_t *output)
 OPENSSL_UNTARGET_AVX2

 OPENSSL_TARGET_AVX2
-int encode_base64_avx2(EVP_ENCODE_CTX *ctx, unsigned char *dst,
+size_t encode_base64_avx2(EVP_ENCODE_CTX *ctx, unsigned char *dst,
     const unsigned char *src, int srclen, int ctx_length,
     int *final_wrap_cnt)
 {
@@ -665,7 +665,7 @@ int encode_base64_avx2(EVP_ENCODE_CTX *ctx, unsigned char *dst,
         *out++ = '\n';
     }

-    return (int)(out - (uint8_t *)dst) + +evp_encodeblock_int(ctx, out, src + i, srclen - i, final_wrap_cnt);
+    return (size_t)(out - (uint8_t *)dst) + evp_encodeblock_int(ctx, out, src + i, srclen - i, final_wrap_cnt);
 }
 OPENSSL_UNTARGET_AVX2
 #endif /* !defined(_M_ARM64EC) */
diff --git a/crypto/evp/enc_b64_avx2.h b/crypto/evp/enc_b64_avx2.h
index db67b5cf8a..89dba6285d 100644
--- a/crypto/evp/enc_b64_avx2.h
+++ b/crypto/evp/enc_b64_avx2.h
@@ -2,10 +2,11 @@
 #define OSSL_CRYPTO_EVP_B64_AVX2_H

 #include <openssl/evp.h>
+#include <stddef.h>

 #if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
 #if !defined(_M_ARM64EC)
-int encode_base64_avx2(EVP_ENCODE_CTX *ctx,
+size_t encode_base64_avx2(EVP_ENCODE_CTX *ctx,
     unsigned char *out, const unsigned char *src, int srclen,
     int newlines, int *wrap_cnt);
 #endif /* !defined(_M_ARM64EC) */
diff --git a/crypto/evp/enc_b64_scalar.c b/crypto/evp/enc_b64_scalar.c
index 1ca46a8357..4bdadcea0c 100644
--- a/crypto/evp/enc_b64_scalar.c
+++ b/crypto/evp/enc_b64_scalar.c
@@ -124,11 +124,11 @@ static const unsigned char base64_std_bin2ascii_2[256] = {
     '/'
 };

-int evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t,
+size_t evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t,
     const unsigned char *f, int dlen, int *wrap_cnt)
 {
     int i = 0;
-    int ret = 0;
+    size_t ret = 0;
     uint8_t t1, t2, t3;
     const unsigned char *e0, *e1, *e2;
     int srp = (ctx != NULL
diff --git a/crypto/evp/enc_b64_scalar.h b/crypto/evp/enc_b64_scalar.h
index 91d416f758..43059938cc 100644
--- a/crypto/evp/enc_b64_scalar.h
+++ b/crypto/evp/enc_b64_scalar.h
@@ -1,8 +1,9 @@
 #ifndef OSSL_CRYPTO_EVP_B64_SCALAR_H
 #define OSSL_CRYPTO_EVP_B64_SCALAR_H
 #include <openssl/evp.h>
+#include <stddef.h>

-int evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t,
+size_t evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t,
     const unsigned char *f, int dlen, int *wrap_cnt);

 #endif
diff --git a/crypto/evp/encode.c b/crypto/evp/encode.c
index eacc68bb96..faf9a3887d 100644
--- a/crypto/evp/encode.c
+++ b/crypto/evp/encode.c
@@ -26,7 +26,7 @@

 static unsigned char conv_ascii2bin(unsigned char a,
     const unsigned char *table);
-int evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t,
+size_t evp_encodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t,
     const unsigned char *f, int dlen, int *wrap_cnt);
 static int evp_decodeblock_int(EVP_ENCODE_CTX *ctx, unsigned char *t,
     const unsigned char *f, int n, int eof);
@@ -374,7 +374,8 @@ void EVP_EncodeInit(EVP_ENCODE_CTX *ctx)
 int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
     const unsigned char *in, int inl)
 {
-    int i, j;
+    int i;
+    size_t j;
     size_t total = 0;

     *outl = 0;
@@ -452,20 +453,20 @@ int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,

 void EVP_EncodeFinal(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl)
 {
-    int ret = 0;
+    size_t ret = 0;
     int wrap_cnt = 0;

     if (ctx->num != 0) {
         ret = evp_encodeblock_int(ctx, out, ctx->enc_data, ctx->num,
             &wrap_cnt);
-        if (ossl_assert(ret >= 0)) {
+        if (ret > 0) {
             if ((ctx->flags & EVP_ENCODE_CTX_NO_NEWLINES) == 0)
                 out[ret++] = '\n';
             out[ret] = '\0';
             ctx->num = 0;
         }
     }
-    *outl = ret;
+    *outl = (int)ret;
 }

 int EVP_EncodeBlock(unsigned char *t, const unsigned char *f, int dlen)
@@ -473,14 +474,14 @@ int EVP_EncodeBlock(unsigned char *t, const unsigned char *f, int dlen)
     int wrap_cnt = 0;

 #if defined(__AVX2__)
-    return encode_base64_avx2(NULL, t, f, dlen, 0, &wrap_cnt);
+    return (int)encode_base64_avx2(NULL, t, f, dlen, 0, &wrap_cnt);
 #elif defined(HAS_IA32CAP_IS_64)
     if ((OPENSSL_ia32cap_P[2] & (1u << 5)) != 0)
-        return encode_base64_avx2(NULL, t, f, dlen, 0, &wrap_cnt);
+        return (int)encode_base64_avx2(NULL, t, f, dlen, 0, &wrap_cnt);
     else
-        return evp_encodeblock_int(NULL, t, f, dlen, &wrap_cnt);
+        return (int)evp_encodeblock_int(NULL, t, f, dlen, &wrap_cnt);
 #else
-    return evp_encodeblock_int(NULL, t, f, dlen, &wrap_cnt);
+    return (int)evp_encodeblock_int(NULL, t, f, dlen, &wrap_cnt);
 #endif
 }

diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index 464a9f1efa..39a6204545 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -570,7 +570,7 @@ void *EVP_CIPHER_CTX_set_cipher_data(EVP_CIPHER_CTX *ctx, void *cipher_data);
 #define EVP_CIPHER_CTX_get_mode(c) EVP_CIPHER_get_mode(EVP_CIPHER_CTX_get0_cipher(c))
 #define EVP_CIPHER_CTX_mode EVP_CIPHER_CTX_get_mode

-#define EVP_ENCODE_LENGTH(l) ((((l) + 2) / 3 * 4) + ((l) / 48 + 1) * 2 + 80)
+#define EVP_ENCODE_LENGTH(l) (((((size_t)(l)) + 2) / 3 * 4) + (((size_t)(l)) / 48 + 1) * 2 + 80)
 #define EVP_DECODE_LENGTH(l) (((l) + 3) / 4 * 3 + 80)

 #define EVP_SignInit_ex(a, b, c) EVP_DigestInit_ex(a, b, c)