Commit 3eab35f175 for openssl.org

commit 3eab35f1752b74a5f09eec4ac9a7d3d73f040ba8
Author: Bob Beck <beck@openssl.org>
Date:   Mon Mar 9 14:30:02 2026 -0600

    Forbid GEN_OTHERNAME SMTP UTF8 email name constraints.

    RFC 9598 States:

    Certificate Authorities that wish to issue CA certificates with email
    address name constraints MUST use rfc822Name subject alternative names
    only. These MUST be IDNA2008-conformant names with no mappings and with
    non-ASCII domains encoded in A-labels only.

    This appears to be to get around the confusion created if someone
    attempts to encode a name constraint for an email address into the
    UTF-8 version of the name

    Were someone to attempt to support this, not only would you now have
    to check two separate sets of name constraints for the same thing, but
    would now have to decide what to do if they said different things.

    So we just flag any such certficiate as invalid

    Reviewed-by: Saša Nedvědický <sashan@openssl.org>
    Reviewed-by: Neil Horman <nhorman@openssl.org>
    MergeDate: Thu May  7 16:09:44 2026
    (Merged from https://github.com/openssl/openssl/pull/30329)

diff --git a/crypto/x509/v3_purp.c b/crypto/x509/v3_purp.c
index df1f0ae2d0..462e2f12c8 100644
--- a/crypto/x509/v3_purp.c
+++ b/crypto/x509/v3_purp.c
@@ -40,6 +40,7 @@ static int no_check_purpose(const X509_PURPOSE *xp, const X509 *x,
     int non_leaf);
 static int check_purpose_ocsp_helper(const X509_PURPOSE *xp, const X509 *x,
     int non_leaf);
+static int check_name_constraints(const NAME_CONSTRAINTS *nc);

 static int xp_cmp(const X509_PURPOSE *const *a, const X509_PURPOSE *const *b);
 static void xptable_free(X509_PURPOSE *p);
@@ -728,6 +729,8 @@ int ossl_x509v3_cache_extensions(const X509 *const_x)
     tmp_nc = X509_get_ext_d2i(const_x, NID_name_constraints, &i, NULL);
     if (tmp_nc == NULL && i != -1)
         tmp_ex_flags |= EXFLAG_INVALID;
+    if (!check_name_constraints(tmp_nc))
+        tmp_ex_flags |= EXFLAG_INVALID;

     /* Handle CRL distribution point entries */
     res = setup_crldp(const_x, &tmp_crldp);
@@ -1101,6 +1104,45 @@ static int no_check_purpose(const X509_PURPOSE *xp, const X509 *x,
     return 1;
 }

+static int check_name_constraints(const NAME_CONSTRAINTS *nc)
+{
+    GENERAL_SUBTREE *sub;
+    int ret = 1;
+
+    if (nc == NULL)
+        goto done;
+
+    for (int i = 0; nc->permittedSubtrees != NULL
+        && i < sk_GENERAL_SUBTREE_num(nc->permittedSubtrees);
+        i++) {
+        sub = sk_GENERAL_SUBTREE_value(nc->permittedSubtrees, i);
+        if (sub->base->type == GEN_OTHERNAME
+            && OBJ_obj2nid(sub->base->d.otherName->type_id)
+                == NID_id_on_SmtpUTF8Mailbox) {
+            /* RFC 9598 prohibits GEN_OTHERNAME email constraints */
+            ERR_raise(ERR_LIB_X509V3, X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE);
+            ret = 0;
+            goto done;
+        }
+    }
+    for (int i = 0; nc->excludedSubtrees != NULL
+        && i < sk_GENERAL_SUBTREE_num(nc->excludedSubtrees);
+        i++) {
+        sub = sk_GENERAL_SUBTREE_value(nc->excludedSubtrees, i);
+        if (sub->base->type == GEN_OTHERNAME
+            && OBJ_obj2nid(sub->base->d.otherName->type_id)
+                == NID_id_on_SmtpUTF8Mailbox) {
+            /* RFC 9598 prohibits GEN_OTHERNAME email constraints */
+            ERR_raise(ERR_LIB_X509V3, X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE);
+            ret = 0;
+            goto done;
+        }
+    }
+
+done:
+    return ret;
+}
+
 /*-
  * Various checks to see if one certificate potentially issued the second.
  * This can be used to prune a set of possible issuer certificates which