Commit 3f8eb5f51e for openssl.org
commit 3f8eb5f51e5b7539e9fe6b5214bd730697e96594
Author: Yasuaki Torimaru <yasuakitorimaru@gmail.com>
Date: Sun Oct 11 16:09:38 2026 +0900
crypto/pkcs12/p12_utl.c: fix signed integer overflow in OPENSSL_asc2uni()
OPENSSL_asc2uni() computes the output length as asclen * 2 + 2.
For asclen above (INT_MAX - 2) / 2, this overflows a signed int,
which is undefined behaviour. Use the signed safe_math helpers,
so that an input that cannot be encoded is rejected instead.
Also guard the (int)strlen(asc) conversion when asclen is -1, so that
a string whose length does not fit in an int is rejected instead
of being cast to int. Add tests for the smallest asclen that overflows
and the values above it, for asclen = 0, for outputs left untouched
on rejection, and for the asclen = -1 path.
Without a sanitizer, the previous code's behaviour depends
on the compiler and the optimisation level, because the overflow
is undefined behaviour. Measured with the function as it stood before
this change (gcc 15.3.0 and clang 21.1.8, x86-64): gcc at -O0/-O2/-O3
and clang at -O0 return NULL for all three boundary inputs. clang
at -O2/-O3 asks the allocator for a large positive size and dies
with SIGSEGV for asclen = (INT_MAX - 2) / 2 + 1 and + 2;
asclen = INT_MAX still returns NULL there, because CRYPTO_malloc(0)
does. Under UBSan the overflow is reported before any of that, which
is what the regression test relies on.
A rejected input returns NULL without pushing an error, as the existing
asclen < 0 path already did; callers such as p12_key.c raise their own
error for that case.
Assisted-by: Claude:claude-opus-4-8
Reviewed-by: Eugene Syromiatnikov <esyr@openssl.org>
Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
Merge-date: Thu Nov 19 09:28:52 2026
Merged-from: https://github.com/openssl/openssl/pull/30681
diff --git a/crypto/pkcs12/p12_utl.c b/crypto/pkcs12/p12_utl.c
index dd8e969767..3013ef81f0 100644
--- a/crypto/pkcs12/p12_utl.c
+++ b/crypto/pkcs12/p12_utl.c
@@ -8,25 +8,37 @@
*/
#include <stdio.h>
+#include <limits.h>
#include "internal/cryptlib.h"
+#include "internal/safe_math.h"
#include <openssl/pkcs12.h>
#include "p12_local.h"
#include "crypto/pkcs7/pk7_local.h"
#include <crypto/asn1.h>
+OSSL_SAFE_MATH_SIGNED(int, int)
+
/* Cheap and nasty Unicode stuff */
unsigned char *OPENSSL_asc2uni(const char *asc, int asclen,
unsigned char **uni, int *unilen)
{
- int ulen, i;
+ int ulen, i, err = 0;
unsigned char *unitmp;
- if (asclen == -1)
- asclen = (int)strlen(asc);
+ if (asclen == -1) {
+ size_t len = strlen(asc);
+
+ if (len > INT_MAX)
+ return NULL;
+ asclen = (int)len;
+ }
if (asclen < 0)
return NULL;
- ulen = asclen * 2 + 2;
+ /* If this changes, revisit test_safe_math_asc2uni_boundary() */
+ ulen = safe_add_int(safe_mul_int(asclen, 2, &err), 2, &err);
+ if (err)
+ return NULL;
if ((unitmp = OPENSSL_malloc(ulen)) == NULL)
return NULL;
for (i = 0; i < ulen - 2; i += 2) {
diff --git a/test/pkcs12_api_test.c b/test/pkcs12_api_test.c
index e9a544b61b..337fbc5474 100644
--- a/test/pkcs12_api_test.c
+++ b/test/pkcs12_api_test.c
@@ -10,8 +10,10 @@
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
+#include <limits.h>
#include "internal/nelem.h"
+#include "internal/safe_math.h"
#include <openssl/pkcs12.h>
#include <openssl/x509.h>
@@ -24,6 +26,8 @@
static OSSL_LIB_CTX *testctx = NULL;
static OSSL_PROVIDER *nullprov = NULL;
+OSSL_SAFE_MATH_SIGNED(int, int)
+
static int test_null_args(void)
{
return TEST_false(PKCS12_parse(NULL, NULL, NULL, NULL, NULL));
@@ -827,6 +831,108 @@ err:
return ret;
}
+/* (INT_MAX - 2) / 2 is the largest asclen that does not overflow asclen * 2 + 2. */
+static int test_asc2uni_overflow(void)
+{
+ unsigned char *result = NULL;
+ int unilen = 0;
+ int ret = 0;
+
+ result = OPENSSL_asc2uni("A", (INT_MAX - 2) / 2 + 1, &result, &unilen);
+ if (!TEST_ptr_null(result))
+ goto err;
+
+ result = OPENSSL_asc2uni("A", (INT_MAX - 2) / 2 + 2, &result, &unilen);
+ if (!TEST_ptr_null(result))
+ goto err;
+
+ result = OPENSSL_asc2uni("A", INT_MAX, &result, &unilen);
+ if (!TEST_ptr_null(result))
+ goto err;
+
+ result = OPENSSL_asc2uni("", 0, &result, &unilen);
+ if (!TEST_ptr(result))
+ goto err;
+ if (!TEST_int_eq(unilen, 2))
+ goto err;
+
+ ret = 1;
+err:
+ OPENSSL_free(result);
+ return ret;
+}
+
+static int test_asc2uni_outputs(void)
+{
+ unsigned char *result = NULL;
+ unsigned char *out = (unsigned char *)0x1;
+ int unilen = -12345;
+ int ret = 0;
+
+ result = OPENSSL_asc2uni("A", INT_MAX, &out, &unilen);
+ if (!TEST_ptr_null(result))
+ goto err;
+ if (!TEST_ptr_eq(out, (unsigned char *)0x1))
+ goto err;
+ if (!TEST_int_eq(unilen, -12345))
+ goto err;
+
+ result = OPENSSL_asc2uni("", 0, &result, &unilen);
+ if (!TEST_ptr(result))
+ goto err;
+ if (!TEST_int_eq(unilen, 2))
+ goto err;
+ if (!TEST_mem_eq(result, 2, "\0\0", 2))
+ goto err;
+
+ ret = 1;
+err:
+ OPENSSL_free(result);
+ return ret;
+}
+
+static int test_asc2uni_strlen(void)
+{
+ static const unsigned char expected[] = { 0x00, 0x41, 0x00, 0x42, 0x00, 0x00 };
+ unsigned char *result = NULL;
+ int unilen = 0;
+ int ret = 0;
+
+ result = OPENSSL_asc2uni("AB", -1, &result, &unilen);
+ if (!TEST_ptr(result))
+ goto err;
+ if (!TEST_int_eq(unilen, 6))
+ goto err;
+ if (!TEST_mem_eq(result, 6, expected, sizeof(expected)))
+ goto err;
+
+ ret = 1;
+err:
+ OPENSSL_free(result);
+ return ret;
+}
+
+/*
+ * A BMPString of that size (about 2 GB) is too large to allocate in a test, so
+ * this checks the arithmetic through the safe_math helpers instead.
+ */
+static int test_safe_math_asc2uni_boundary(void)
+{
+ int err = 0;
+ int ulen;
+ int max_valid = (INT_MAX - 2) / 2;
+
+ ulen = safe_add_int(safe_mul_int(max_valid, 2, &err), 2, &err);
+ if (!TEST_false(err))
+ return 0;
+ if (!TEST_int_eq(ulen, max_valid * 2 + 2))
+ return 0;
+
+ err = 0;
+ ulen = safe_add_int(safe_mul_int(max_valid + 1, 2, &err), 2, &err);
+ return TEST_true(err);
+}
+
int setup_tests(void)
{
OPTION_CHOICE o;
@@ -899,6 +1005,10 @@ int setup_tests(void)
ADD_TEST(test_parse_ex_skey);
ADD_ALL_TESTS(test_parse_ex_libctx, 2);
ADD_ALL_TESTS(test_parse_ex_skey_libctx, 4);
+ ADD_TEST(test_asc2uni_overflow);
+ ADD_TEST(test_asc2uni_outputs);
+ ADD_TEST(test_asc2uni_strlen);
+ ADD_TEST(test_safe_math_asc2uni_boundary);
}
return 1;
}