Commit 4fc10a5bfe for openssl.org
commit 4fc10a5bfe8658d080a44b08fb6229fa666949ac
Author: Darren Carreras <carrerasdarren@gmail.com>
Date: Wed Jul 22 11:55:02 2026 -0400
X509: keep names modified until canonicalization succeeds
x509_name_encode() cleared the modified flag before x509_name_canon().
If canonicalization then failed, later encodes reused the DER cache while
canonical state was absent, allowing a malformed name to compare and hash
as an empty name.
Clear modified only after both cache representations succeed. Validate both
output-pass encodes and propagate nested ASN.1 encoding failures so partial
output is not reported as success.
Add malformed-name cache regression coverage and multi-valued RDN
allocation-failure coverage.
Refs #32010
Assisted-by: OpenAI Codex:gpt-5.6-sol
Reviewed-by: Jakub Zelenka <jakub.zelenka@openssl.foundation>
Reviewed-by: Mounir Idrassi <mounir.idrassi@idrix.fr>
Merge-date: Mon Sep 14 14:13:33 2026
Merged-from: https://github.com/openssl/openssl/pull/32047
diff --git a/crypto/asn1/tasn_enc.c b/crypto/asn1/tasn_enc.c
index e489e29a0c..3307392a84 100644
--- a/crypto/asn1/tasn_enc.c
+++ b/crypto/asn1/tasn_enc.c
@@ -57,7 +57,7 @@ static int asn1_item_flags_i2d(const ASN1_VALUE *val, unsigned char **out,
{
if (out != NULL && *out == NULL) {
unsigned char *p, *buf;
- int len;
+ int len, outlen;
len = ASN1_item_ex_i2d(&val, NULL, it, -1, flags);
if (len <= 0)
@@ -65,7 +65,11 @@ static int asn1_item_flags_i2d(const ASN1_VALUE *val, unsigned char **out,
if ((buf = OPENSSL_malloc(len)) == NULL)
return -1;
p = buf;
- ASN1_item_ex_i2d(&val, &p, it, -1, flags);
+ outlen = ASN1_item_ex_i2d(&val, &p, it, -1, flags);
+ if (outlen != len || p != buf + len) {
+ OPENSSL_free(buf);
+ return -1;
+ }
*out = buf;
return len;
}
@@ -172,7 +176,7 @@ int ASN1_item_ex_i2d(const ASN1_VALUE **pval, unsigned char **out,
return 0;
pseqval = ossl_asn1_get_const_field_ptr(pval, seqtt);
tmplen = asn1_template_ex_i2d(pseqval, NULL, seqtt, -1, aclass);
- if (tmplen == -1 || (tmplen > INT_MAX - seqcontlen))
+ if (tmplen < 0 || (tmplen > INT_MAX - seqcontlen))
return -1;
seqcontlen += tmplen;
}
@@ -185,12 +189,15 @@ int ASN1_item_ex_i2d(const ASN1_VALUE **pval, unsigned char **out,
for (i = 0, tt = it->templates; i < it->tcount; tt++, i++) {
const ASN1_TEMPLATE *seqtt;
const ASN1_VALUE **pseqval;
+ int tmplen;
+
seqtt = ossl_asn1_do_adb(*pval, tt, 1);
if (!seqtt)
return 0;
pseqval = ossl_asn1_get_const_field_ptr(pval, seqtt);
- /* FIXME: check for errors in enhanced version */
- asn1_template_ex_i2d(pseqval, out, seqtt, -1, aclass);
+ tmplen = asn1_template_ex_i2d(pseqval, out, seqtt, -1, aclass);
+ if (tmplen < 0)
+ return -1;
}
if (ndef == 2)
ASN1_put_eoc(out);
@@ -210,6 +217,7 @@ static int asn1_template_ex_i2d(const ASN1_VALUE **pval, unsigned char **out,
const int flags = tt->flags;
int i, ret, ttag, tclass, ndef, len;
const ASN1_VALUE *tval;
+ unsigned char *p;
/*
* If field is embedded then val needs fixing so it is a pointer to
@@ -297,7 +305,7 @@ static int asn1_template_ex_i2d(const ASN1_VALUE **pval, unsigned char **out,
skitem = sk_const_ASN1_VALUE_value(sk, i);
len = ASN1_item_ex_i2d(&skitem, NULL, ASN1_ITEM_ptr(tt->item),
-1, iclass);
- if (len == -1 || (skcontlen > INT_MAX - len))
+ if (len < 0 || (skcontlen > INT_MAX - len))
return -1;
if (len == 0 && (tt->flags & ASN1_TFLG_OPTIONAL) == 0) {
ERR_raise(ERR_LIB_ASN1, ASN1_R_ILLEGAL_ZERO_CONTENT);
@@ -324,8 +332,9 @@ static int asn1_template_ex_i2d(const ASN1_VALUE **pval, unsigned char **out,
/* SET or SEQUENCE and IMPLICIT tag */
ASN1_put_object(out, ndef, skcontlen, sktag, skaclass);
/* And the stuff itself */
- asn1_set_seq_out(sk, out, skcontlen, ASN1_ITEM_ptr(tt->item),
- isset, iclass);
+ if (!asn1_set_seq_out(sk, out, skcontlen, ASN1_ITEM_ptr(tt->item),
+ isset, iclass))
+ return -1;
if (ndef == 2) {
ASN1_put_eoc(out);
if (flags & ASN1_TFLG_EXPTAG)
@@ -339,6 +348,8 @@ static int asn1_template_ex_i2d(const ASN1_VALUE **pval, unsigned char **out,
/* EXPLICIT tagging */
/* Find length of tagged item */
i = ASN1_item_ex_i2d(pval, NULL, ASN1_ITEM_ptr(tt->item), -1, iclass);
+ if (i < 0)
+ return -1;
if (i == 0) {
if ((tt->flags & ASN1_TFLG_OPTIONAL) == 0) {
ERR_raise(ERR_LIB_ASN1, ASN1_R_ILLEGAL_ZERO_CONTENT);
@@ -351,7 +362,11 @@ static int asn1_template_ex_i2d(const ASN1_VALUE **pval, unsigned char **out,
if (out && ret != -1) {
/* Output tag and item */
ASN1_put_object(out, ndef, i, ttag, tclass);
- ASN1_item_ex_i2d(pval, out, ASN1_ITEM_ptr(tt->item), -1, iclass);
+ p = *out;
+ len = ASN1_item_ex_i2d(pval, out, ASN1_ITEM_ptr(tt->item), -1,
+ iclass);
+ if (len != i || *out != p + i)
+ return -1;
if (ndef == 2)
ASN1_put_eoc(out);
}
@@ -417,7 +432,8 @@ static int asn1_set_seq_out(STACK_OF(const_ASN1_VALUE) *sk,
if (!do_sort) {
for (i = 0; i < sk_const_ASN1_VALUE_num(sk); i++) {
skitem = sk_const_ASN1_VALUE_value(sk, i);
- ASN1_item_ex_i2d(&skitem, out, item, -1, iclass);
+ if (ASN1_item_ex_i2d(&skitem, out, item, -1, iclass) <= 0)
+ return 0;
}
return 1;
}
@@ -428,6 +444,8 @@ static int asn1_set_seq_out(STACK_OF(const_ASN1_VALUE) *sk,
skitem = sk_const_ASN1_VALUE_value(sk, i);
tder->data = p;
tder->length = ASN1_item_ex_i2d(&skitem, &p, item, -1, iclass);
+ if (tder->length <= 0)
+ goto err;
tder->field = skitem;
}
diff --git a/crypto/x509/x_name.c b/crypto/x509/x_name.c
index 88d3ece62d..34d1f76c93 100644
--- a/crypto/x509/x_name.c
+++ b/crypto/x509/x_name.c
@@ -224,6 +224,7 @@ static int x509_name_ex_i2d(const ASN1_VALUE **val, unsigned char **out,
ret = x509_name_canon(a);
if (!ret)
return -1;
+ a->modified = 0;
}
ret = (int)a->bytes->length;
if (out != NULL) {
@@ -241,8 +242,8 @@ static int x509_name_encode(X509_NAME *a)
} intname = {
NULL
};
- int len;
- unsigned char *p;
+ int len, outlen;
+ unsigned char *p, *start;
STACK_OF(X509_NAME_ENTRY) *entries = NULL;
X509_NAME_ENTRY *entry;
int i, set = -1;
@@ -267,16 +268,20 @@ static int x509_name_encode(X509_NAME *a)
}
len = ASN1_item_ex_i2d(&intname.a, NULL,
ASN1_ITEM_rptr(X509_NAME_INTERNAL), -1, -1);
+ if (len < 0)
+ goto err;
if (!BUF_MEM_grow(a->bytes, len)) {
ERR_raise(ERR_LIB_ASN1, ERR_R_BUF_LIB);
goto err;
}
p = (unsigned char *)a->bytes->data;
- ASN1_item_ex_i2d(&intname.a,
+ start = p;
+ outlen = ASN1_item_ex_i2d(&intname.a,
&p, ASN1_ITEM_rptr(X509_NAME_INTERNAL), -1, -1);
+ if (outlen != len || p != start + len)
+ goto err;
sk_STACK_OF_X509_NAME_ENTRY_pop_free(intname.s,
local_sk_X509_NAME_ENTRY_free);
- a->modified = 0;
return len;
cerr:
ERR_raise(ERR_LIB_ASN1, ERR_R_CRYPTO_LIB);
@@ -310,14 +315,15 @@ static int x509_name_ex_print(BIO *out, const ASN1_VALUE **pval,
static int x509_name_canon(X509_NAME *a)
{
- unsigned char *p;
+ unsigned char *buf = NULL, *p, *start;
STACK_OF(STACK_OF_X509_NAME_ENTRY) *intname;
STACK_OF(X509_NAME_ENTRY) *entries = NULL;
X509_NAME_ENTRY *entry, *tmpentry = NULL;
- int i, set = -1, ret = 0, len;
+ int i, set = -1, ret = 0, len, outlen;
OPENSSL_free(a->canon_enc);
a->canon_enc = NULL;
+ a->canon_enclen = 0;
/* Special case: empty X509_NAME => null encoding */
if (sk_X509_NAME_ENTRY_num(a->entries) == 0) {
a->canon_enclen = 0;
@@ -364,19 +370,24 @@ static int x509_name_canon(X509_NAME *a)
len = i2d_name_canon(intname, NULL);
if (len < 0)
goto err;
- a->canon_enclen = len;
- p = OPENSSL_malloc(a->canon_enclen);
- if (p == NULL)
+ buf = OPENSSL_malloc(len);
+ if (buf == NULL)
goto err;
+ p = buf;
+ start = p;
- a->canon_enc = p;
-
- i2d_name_canon(intname, &p);
+ outlen = i2d_name_canon(intname, &p);
+ if (outlen != len || p != start + len)
+ goto err;
+ a->canon_enc = buf;
+ a->canon_enclen = len;
+ buf = NULL;
ret = 1;
err:
+ OPENSSL_free(buf);
X509_NAME_ENTRY_free(tmpentry);
sk_STACK_OF_X509_NAME_ENTRY_pop_free(intname,
local_sk_X509_NAME_ENTRY_pop_free);
diff --git a/test/x509_test.c b/test/x509_test.c
index 7d2eb757e5..23fb995934 100644
--- a/test/x509_test.c
+++ b/test/x509_test.c
@@ -13,6 +13,7 @@
#include <openssl/x509v3.h>
#include <openssl/asn1.h>
#include <openssl/evp.h>
+#include <openssl/pkcs7.h>
#include <openssl/rsa.h>
#include <openssl/pem.h>
#include "crypto/x509.h" /* x509_st definition */
@@ -701,6 +702,137 @@ static int test_nc_empty_dirname_permitted(void)
sizeof(nc_permitted_empty_dirname), X509_V_OK);
}
+static int test_x509_name_canon_failure_cache(void)
+{
+ static const unsigned char invalid_utf8[] = { 0xc0, 0xaf };
+ int ret = 0, hash_ok = 1;
+ X509 *cert = NULL;
+ X509_NAME *empty = NULL;
+ X509_STORE *store = NULL;
+
+ if (!TEST_ptr(cert = X509_new())
+ || !TEST_ptr(empty = X509_NAME_new())
+ || !TEST_ptr(store = X509_STORE_new())
+ || !TEST_true(X509_NAME_add_entry_by_txt(cert->cert_info.subject,
+ "CN", V_ASN1_UTF8STRING, invalid_utf8, sizeof(invalid_utf8), -1,
+ 0))
+ /*
+ * The first attempt populates DER before canonicalization fails. A
+ * retry must not reuse that partial cache.
+ */
+ || !TEST_int_lt(i2d_X509_NAME(cert->cert_info.subject, NULL), 0)
+ || !TEST_int_lt(i2d_X509_NAME(cert->cert_info.subject, NULL), 0)
+ || !TEST_int_eq(X509_NAME_cmp(cert->cert_info.subject, empty), -2)
+ || !TEST_ulong_eq(X509_NAME_hash_ex(cert->cert_info.subject, NULL,
+ NULL, &hash_ok),
+ 0)
+ || !TEST_false(hash_ok)
+ || !TEST_false(X509_STORE_add_cert(store, cert)))
+ goto end;
+
+ ret = 1;
+
+end:
+ ERR_clear_error();
+ X509_STORE_free(store);
+ X509_NAME_free(empty);
+ X509_free(cert);
+ return ret;
+}
+
+static int test_x509_name_multivalued_rdn_mfail(void)
+{
+ X509_NAME *name = NULL;
+ int ret = 0;
+
+ if (!TEST_ptr(name = X509_NAME_new())
+ || !TEST_true(X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_ASC,
+ (unsigned char *)"Alice", -1, -1, 0))
+ || !TEST_true(X509_NAME_add_entry_by_txt(name, "OU", MBSTRING_ASC,
+ (unsigned char *)"Engineering", -1, -1, -1)))
+ goto end;
+
+ MFAIL_start();
+ ret = i2d_X509_NAME(name, NULL) > 0;
+ MFAIL_end();
+
+end:
+ X509_NAME_free(name);
+ return ret;
+}
+
+static int test_x509_attribute_i2d_mfail(void)
+{
+ static const unsigned char value1[] = "first";
+ static const unsigned char value2[] = "second";
+ X509_ATTRIBUTE *attr = NULL;
+ unsigned char *der = NULL;
+ int len, ret = -1;
+
+ if (!TEST_ptr(attr = X509_ATTRIBUTE_create_by_NID(NULL, NID_commonName,
+ V_ASN1_UTF8STRING, value1, sizeof(value1) - 1))
+ || !TEST_true(X509_ATTRIBUTE_set1_data(attr, V_ASN1_UTF8STRING,
+ value2, sizeof(value2) - 1)))
+ goto end;
+
+ MFAIL_start();
+ len = i2d_X509_ATTRIBUTE(attr, &der);
+ MFAIL_end();
+
+ if (len <= 0)
+ ret = der == NULL ? 0 : -1;
+ else
+ ret = der != NULL ? 1 : -1;
+
+end:
+ OPENSSL_free(der);
+ X509_ATTRIBUTE_free(attr);
+ return ret;
+}
+
+static int test_pkcs7_digest_set_i2d_mfail(void)
+{
+ PKCS7 *p7 = NULL;
+ X509_ALGOR *alg = NULL;
+ unsigned char *der = NULL, *p;
+ int len, outlen, ret = -1;
+
+ if (!TEST_ptr(p7 = PKCS7_new())
+ || !TEST_true(PKCS7_set_type(p7, NID_pkcs7_signed))
+ || !TEST_true(PKCS7_content_new(p7, NID_pkcs7_data))
+ || !TEST_ptr(alg = X509_ALGOR_new())
+ || !TEST_true(X509_ALGOR_set_md(alg, EVP_sha256()))
+ || !TEST_int_gt(sk_X509_ALGOR_push(p7->d.sign->md_algs, alg), 0))
+ goto end;
+ alg = NULL;
+
+ if (!TEST_ptr(alg = X509_ALGOR_new())
+ || !TEST_true(X509_ALGOR_set_md(alg, EVP_sha384()))
+ || !TEST_int_gt(sk_X509_ALGOR_push(p7->d.sign->md_algs, alg), 0))
+ goto end;
+ alg = NULL;
+
+ len = i2d_PKCS7(p7, NULL);
+ if (!TEST_int_gt(len, 0) || !TEST_ptr(der = OPENSSL_malloc(len)))
+ goto end;
+ p = der;
+
+ MFAIL_start();
+ outlen = i2d_PKCS7(p7, &p);
+ MFAIL_end();
+
+ if (outlen <= 0)
+ ret = 0;
+ else
+ ret = outlen == len && p == der + len ? 1 : -1;
+
+end:
+ X509_ALGOR_free(alg);
+ OPENSSL_free(der);
+ PKCS7_free(p7);
+ return ret;
+}
+
OPT_TEST_DECLARE_USAGE("<pss-self-signed-cert.pem>\n")
int setup_tests(void)
@@ -746,6 +878,10 @@ int setup_tests(void)
ADD_TEST(test_x509_verify_with_new);
ADD_TEST(test_nc_empty_dirname_excluded);
ADD_TEST(test_nc_empty_dirname_permitted);
+ ADD_TEST(test_x509_name_canon_failure_cache);
+ ADD_MFAIL_TEST(test_x509_name_multivalued_rdn_mfail);
+ ADD_MFAIL_TEST(test_x509_attribute_i2d_mfail);
+ ADD_MFAIL_TEST(test_pkcs7_digest_set_i2d_mfail);
return 1;
}