Commit 708add2d9f for openssl.org
commit 708add2d9fb439f110fd73f943d8a64bbdf10cab
Author: Ryan Hooper <ryhooper@cisco.com>
Date: Wed Aug 5 17:56:02 2026 -0400
Fix DTLS 1.3 early data transcript hash stripping
When the client installs early write keys, negotiated_version is not
yet set to DTLS1_3_VERSION, so the RFC 9147 §5.2 stripping of
msg_seq, fragment_offset, and fragment_length was skipped in
tls13_change_cipher_state. Fix by conditioning on
SSL_CONNECTION_IS_DTLS() alone.
Fixes: openssl/project#2043
Reviewed-by: Frederik Wedel-Heinen <fwh.openssl@gmail.com>
Reviewed-by: Matt Caswell <matt@openssl.foundation>
Merge-date: Mon Aug 17 08:34:23 2026
Merged-from: https://github.com/openssl/openssl/pull/32276
diff --git a/ssl/build.info b/ssl/build.info
index 5aa12baea1..35a614e389 100644
--- a/ssl/build.info
+++ b/ssl/build.info
@@ -12,7 +12,7 @@ SOURCE[../libssl]=\
statem/statem_lib.c statem/extensions.c statem/extensions_srvr.c \
statem/extensions_clnt.c statem/extensions_cust.c s3_msg.c \
methods.c t1_lib.c t1_enc.c tls13_enc.c \
- d1_lib.c d1_msg.c \
+ d1_lib.c d1_msg.c d1_transcript.c \
statem/statem_dtls.c d1_srtp.c \
ssl_lib.c ssl_cert.c ssl_sess.c \
ssl_ciph.c ssl_stat.c ssl_rsa.c \
diff --git a/ssl/d1_transcript.c b/ssl/d1_transcript.c
new file mode 100644
index 0000000000..f14bddd267
--- /dev/null
+++ b/ssl/d1_transcript.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2026 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License"). You may not use
+ * this file except in compliance with the License. You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "ssl_local.h"
+
+/*
+ * RFC 9147 §5.2: strip msg_seq, fragment_offset, and fragment_length from
+ * each DTLS handshake message header before feeding into the transcript hash.
+ * Used by both ssl3_finish_mac and tls13_change_cipher_state.
+ */
+int dtls13_transcript_hash_update(EVP_MD_CTX *mdctx,
+ const unsigned char *buf, size_t len)
+{
+ while (len > 0) {
+ PACKET hmhdr;
+ unsigned long hmbodylen;
+ unsigned int msgtype;
+ size_t hmhdrlen;
+
+ if (!ossl_assert(len >= SSL3_HM_HEADER_LENGTH)
+ || !PACKET_buf_init(&hmhdr, buf, SSL3_HM_HEADER_LENGTH)
+ || !PACKET_get_1(&hmhdr, &msgtype)
+ || !PACKET_get_net_3(&hmhdr, &hmbodylen))
+ return 0;
+
+ hmhdrlen = (msgtype == SSL3_MT_MESSAGE_HASH)
+ ? SSL3_HM_HEADER_LENGTH
+ : DTLS1_HM_HEADER_LENGTH;
+
+ if (!ossl_assert(hmhdrlen + hmbodylen <= len)
+ || !EVP_DigestUpdate(mdctx, buf, SSL3_HM_HEADER_LENGTH)
+ || !EVP_DigestUpdate(mdctx, buf + hmhdrlen, hmbodylen))
+ return 0;
+
+ buf += hmhdrlen + hmbodylen;
+ len -= hmhdrlen + hmbodylen;
+ }
+ return 1;
+}
diff --git a/ssl/s3_enc.c b/ssl/s3_enc.c
index e6ea986898..7c6993ad01 100644
--- a/ssl/s3_enc.c
+++ b/ssl/s3_enc.c
@@ -79,51 +79,9 @@ int ssl3_finish_mac(SSL_CONNECTION *s, const unsigned char *buf, size_t len)
* point we know what the protocol version is.
*/
if (s->negotiated_version == DTLS1_3_VERSION) {
- /*
- * In DTLS 1.3 we need to parse the messages that are buffered to
- * be able to remove message_sequence, fragment_size and fragment_offset
- * from the Transcript Hash calculation.
- */
- while (len > 0) {
- PACKET hmhdr;
- unsigned long hmbodylen;
- unsigned int msgtype;
- size_t hmhdrlen;
-
- if (!ossl_assert(len >= SSL3_HM_HEADER_LENGTH)
- || !PACKET_buf_init(&hmhdr, buf, SSL3_HM_HEADER_LENGTH)
- || !PACKET_get_1(&hmhdr, &msgtype)
- || !PACKET_get_net_3(&hmhdr, &hmbodylen)) {
- SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR);
- return 0;
- }
-
- /*
- * SSL3_MT_MESSAGE_HASH is a dummy message type only used when
- * calculating the transcript hash of the synthetic message in
- * (D)TLS 1.3.
- */
- if (msgtype == SSL3_MT_MESSAGE_HASH)
- hmhdrlen = SSL3_HM_HEADER_LENGTH;
- else
- hmhdrlen = DTLS1_HM_HEADER_LENGTH;
-
- /*
- * In DTLS 1.3 the transcript hash is calculated excluding the
- * message_sequence, fragment_size and fragment_offset header
- * fields which are carried in the last
- * DTLS1_HM_HEADER_LENGTH - SSL3_HM_HEADER_LENGTH header bytes
- * of the DTLS handshake message header.
- */
- if (!ossl_assert(hmhdrlen + hmbodylen <= len)
- || !EVP_DigestUpdate(s->s3.handshake_dgst, buf, SSL3_HM_HEADER_LENGTH)
- || !EVP_DigestUpdate(s->s3.handshake_dgst, buf + hmhdrlen, hmbodylen)) {
- SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR);
- return 0;
- }
-
- buf += hmhdrlen + hmbodylen;
- len -= hmhdrlen + hmbodylen;
+ if (!dtls13_transcript_hash_update(s->s3.handshake_dgst, buf, len)) {
+ SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR);
+ return 0;
}
} else {
if (!EVP_DigestUpdate(s->s3.handshake_dgst, buf, len)) {
diff --git a/ssl/ssl_local.h b/ssl/ssl_local.h
index 96561858fd..eb1142431e 100644
--- a/ssl/ssl_local.h
+++ b/ssl/ssl_local.h
@@ -3040,6 +3040,8 @@ __owur int dtls1_do_write(SSL_CONNECTION *s, uint8_t recordtype);
int dtls1_write_app_data_bytes(SSL *s, uint8_t type, const void *buf_,
size_t len, size_t *written);
+int dtls13_transcript_hash_update(EVP_MD_CTX *mdctx,
+ const unsigned char *buf, size_t len);
__owur int dtls1_read_failed(SSL_CONNECTION *s, int code);
__owur int dtls1_buffer_sent_message(SSL_CONNECTION *s, int record_type);
diff --git a/ssl/tls13_enc.c b/ssl/tls13_enc.c
index 0abbc07fe1..661a195f94 100644
--- a/ssl/tls13_enc.c
+++ b/ssl/tls13_enc.c
@@ -653,9 +653,28 @@ int tls13_change_cipher_state(SSL_CONNECTION *s, int which)
}
md = ssl_md(sctx, sslcipher->algorithm2);
- if (md == NULL || !EVP_DigestInit_ex(mdctx, md, NULL)
- || !EVP_DigestUpdate(mdctx, hdata, handlen)
- || !EVP_DigestFinal_ex(mdctx, hashval, &hashlenui)) {
+ if (md == NULL || !EVP_DigestInit_ex(mdctx, md, NULL)) {
+ SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR);
+ EVP_MD_CTX_free(mdctx);
+ goto err;
+ }
+
+ if (SSL_CONNECTION_IS_DTLS(s)) {
+ if (!dtls13_transcript_hash_update(mdctx, hdata,
+ (size_t)handlen)) {
+ SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR);
+ EVP_MD_CTX_free(mdctx);
+ goto err;
+ }
+ } else {
+ if (!EVP_DigestUpdate(mdctx, hdata, handlen)) {
+ SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR);
+ EVP_MD_CTX_free(mdctx);
+ goto err;
+ }
+ }
+
+ if (!EVP_DigestFinal_ex(mdctx, hashval, &hashlenui)) {
SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR);
EVP_MD_CTX_free(mdctx);
goto err;
diff --git a/test/build.info b/test/build.info
index c2b4fe5a8f..0bb738bc0b 100644
--- a/test/build.info
+++ b/test/build.info
@@ -1248,7 +1248,7 @@ IF[{- !$disabled{tests} -}]
PROGRAMS{noinst}=tls13secretstest
SOURCE[tls13secretstest]=tls13secretstest.c
DEFINE[tls13secretstest]=OPENSSL_NO_KTLS
- SOURCE[tls13secretstest]= ../ssl/tls13_enc.c ../crypto/packet.c ../crypto/quic_vlint.c
+ SOURCE[tls13secretstest]= ../ssl/tls13_enc.c ../ssl/d1_transcript.c ../crypto/packet.c ../crypto/quic_vlint.c
INCLUDE[tls13secretstest]=.. ../include ../apps/include
DEPEND[tls13secretstest]=../libcrypto ../libssl libtestutil.a
ENDIF
diff --git a/test/tls13secretstest.c b/test/tls13secretstest.c
index 3aaa9e0bb6..0721464d3c 100644
--- a/test/tls13secretstest.c
+++ b/test/tls13secretstest.c
@@ -447,8 +447,94 @@ err:
return ret;
}
+/* Full DTLS 1.3 ClientHello as stored in the handshake buffer (393 bytes) */
+/* clang-format off */
+static const unsigned char dtls_raw_transcript[] = {
+ 0x01, 0x00, 0x01, 0x7d, /* type=ClientHello, length=381 (kept) */
+ 0x00, 0x00, /* msg_seq=0 (stripped) */
+ 0x00, 0x00, 0x00, /* fragment_offset=0 (stripped) */
+ 0x00, 0x01, 0x7d, /* fragment_length=381 (stripped) */
+ /* ClientHello body (381 bytes, kept) */
+ 0xfe, 0xfd, 0x12, 0x1e, 0xc1, 0x2d, 0x3a, 0xe2, 0x3d, 0xcf, 0x4e, 0x83,
+ 0x0d, 0xf4, 0x5b, 0x6a, 0x38, 0xd9, 0x4c, 0xcc, 0x8f, 0x4c, 0x53, 0xef,
+ 0xd4, 0xca, 0xaa, 0x4d, 0x5d, 0x2f, 0x03, 0x71, 0x4d, 0x50, 0x00, 0x00,
+ 0x00, 0x06, 0x13, 0x02, 0x13, 0x03, 0x13, 0x01, 0x01, 0x00, 0x01, 0x4d,
+ 0x00, 0x0a, 0x00, 0x04, 0x00, 0x02, 0x00, 0x17, 0x00, 0x23, 0x00, 0x00,
+ 0x00, 0x16, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x08,
+ 0x00, 0x06, 0x04, 0x03, 0x08, 0x04, 0x04, 0x01, 0x00, 0x2b, 0x00, 0x03,
+ 0x02, 0xfe, 0xfc, 0x00, 0x2d, 0x00, 0x02, 0x01, 0x01, 0x00, 0x33, 0x00,
+ 0x47, 0x00, 0x45, 0x00, 0x17, 0x00, 0x41, 0x04, 0x11, 0xdf, 0x6e, 0x16,
+ 0xe8, 0xb4, 0xc8, 0xf3, 0x9c, 0x74, 0x09, 0x2e, 0xe5, 0xc2, 0xc7, 0x6f,
+ 0x1b, 0x26, 0xf0, 0x9e, 0x5f, 0xb1, 0x9e, 0x2e, 0xe4, 0xcd, 0x4f, 0xbb,
+ 0xe9, 0x5e, 0x7c, 0x1e, 0x12, 0x84, 0xd7, 0xee, 0x69, 0xa8, 0x16, 0x2f,
+ 0x99, 0xdf, 0x65, 0xad, 0x2b, 0x6f, 0xed, 0x74, 0x04, 0xb3, 0xe2, 0xed,
+ 0xfa, 0x54, 0xd0, 0x62, 0xde, 0xc7, 0x20, 0xb7, 0x4d, 0x17, 0x9d, 0xf9,
+ 0x00, 0x2a, 0x00, 0x00, 0x00, 0x29, 0x00, 0xcd, 0x00, 0xa8, 0x00, 0xa2,
+ 0xa2, 0x8c, 0x83, 0x80, 0xd2, 0x1a, 0x39, 0xf1, 0x9a, 0xb9, 0xa9, 0x9e,
+ 0x00, 0x00, 0x00, 0x00, 0xb5, 0x69, 0x30, 0x63, 0xdd, 0xe8, 0x13, 0xa0,
+ 0x4a, 0x33, 0x9d, 0xbf, 0x53, 0x5c, 0x5c, 0x2e, 0x00, 0x60, 0x5b, 0xd8,
+ 0xc2, 0x31, 0xb8, 0xb7, 0x1c, 0xaf, 0x6e, 0xa5, 0x98, 0xc4, 0x84, 0xa8,
+ 0x4e, 0x75, 0xf5, 0x71, 0xa8, 0x34, 0xcb, 0x9e, 0x65, 0xe9, 0x7e, 0x3e,
+ 0x3e, 0xf2, 0xd3, 0x00, 0x69, 0x4b, 0x6a, 0x45, 0xd1, 0xb3, 0xad, 0xc3,
+ 0x7e, 0x41, 0xe5, 0x8f, 0xde, 0xae, 0x36, 0xfc, 0x38, 0x74, 0x08, 0x66,
+ 0x10, 0xfb, 0x27, 0x46, 0x0c, 0x6c, 0x2a, 0xd8, 0xc7, 0x42, 0x51, 0xce,
+ 0x0e, 0x61, 0x47, 0x7f, 0xc2, 0xeb, 0x11, 0x47, 0x9e, 0xb9, 0x04, 0xd0,
+ 0x4e, 0x57, 0xf0, 0x45, 0x1b, 0x29, 0xca, 0x9e, 0x4e, 0x12, 0x2b, 0xc4,
+ 0x02, 0x66, 0x50, 0x09, 0xd2, 0x50, 0x3c, 0x66, 0xc3, 0x15, 0x25, 0xbb,
+ 0x9b, 0x0d, 0x52, 0x1b, 0x5d, 0x6d, 0x2f, 0x2c, 0x0d, 0xde, 0xfa, 0xfd,
+ 0x9a, 0x65, 0xdd, 0xe0, 0x50, 0xfe, 0x4d, 0x9f, 0x39, 0x51, 0x70, 0x87,
+ 0x3e, 0xbb, 0x47, 0x3e, 0x26, 0xfe, 0x00, 0xfe, 0x74, 0x87, 0x00, 0x21,
+ 0x20, 0xe7, 0x09, 0x56, 0x88, 0xcd, 0xfb, 0xd4, 0xfe, 0x0f, 0x99, 0x96,
+ 0xaf, 0x24, 0x7d, 0xb1, 0xed, 0x14, 0x38, 0x25, 0x98, 0xde, 0xf8, 0x37,
+ 0x02, 0x6c, 0xe7, 0xb7, 0x6e, 0x6e, 0x56, 0xf5, 0xb5,
+};
+/* clang-format on */
+
+/*
+ * SHA-256 of the stripped transcript: 4-byte TLS header + 381-byte body,
+ * with msg_seq, fragment_offset, fragment_length omitted per RFC 9147 §5.2.
+ * Captured from a live OpenSSL-to-NSS DTLS 1.3 early data handshake.
+ */
+static const unsigned char dtls_transcript_hash[] = {
+ 0x5b, 0x8c, 0x4d, 0x67, 0x08, 0xa6, 0x82, 0xa3, 0xb9, 0x62, 0xa5, 0x3a,
+ 0x96, 0x6c, 0x13, 0x81, 0xa9, 0xbc, 0xf4, 0x77, 0xe0, 0xa5, 0x82, 0x30,
+ 0x39, 0x01, 0x13, 0x51, 0x3c, 0xf6, 0x97, 0x3c
+};
+
+/*
+ * Known-answer test for dtls13_transcript_hash_update (RFC 9147 §5.2).
+ * Wraps the helper with EVP_DigestInit/Final to verify the stripped hash
+ * matches a value captured from a live handshake.
+ */
+static int test_dtls13_transcript_hash(void)
+{
+ EVP_MD_CTX *ctx = NULL;
+ unsigned char hash[EVP_MAX_MD_SIZE];
+ unsigned int hashlen = 0;
+ int ret = 0;
+
+ ctx = EVP_MD_CTX_new();
+ if (!TEST_ptr(ctx))
+ goto err;
+ if (!TEST_true(EVP_DigestInit_ex(ctx, EVP_sha256(), NULL)))
+ goto err;
+ if (!TEST_true(dtls13_transcript_hash_update(ctx, dtls_raw_transcript,
+ sizeof(dtls_raw_transcript))))
+ goto err;
+ if (!TEST_true(EVP_DigestFinal_ex(ctx, hash, &hashlen)))
+ goto err;
+ if (!TEST_mem_eq(hash, hashlen,
+ dtls_transcript_hash, sizeof(dtls_transcript_hash)))
+ goto err;
+ ret = 1;
+err:
+ EVP_MD_CTX_free(ctx);
+ return ret;
+}
+
int setup_tests(void)
{
ADD_TEST(test_handshake_secrets);
+ ADD_TEST(test_dtls13_transcript_hash);
return 1;
}