Commit ea15c2442d for openssl.org
commit ea15c2442d3028fb0c6311e577d4b5f6726a4408
Author: Ryan Hooper <ryanh@openssl.foundation>
Date: Tue Aug 11 14:14:58 2026 -0400
DTLS 1.3 Fix DTLS_get_data_mtu() and extend MTU test to cover DTLS 1.3
DTLS_get_data_mtu() included the explicit IV overhead from
ssl_cipher_get_overhead() for AES-GCM and CCM ciphers, but DTLS 1.3 uses
implicit IVs so no IV bytes appear on the wire. Fix by subtracting
EVP_GCM_TLS_EXPLICIT_IV_LEN or EVP_CCM_TLS_EXPLICIT_IV_LEN.
Restructure the MTU test to pin DTLS 1.2 in the PSK cipher loop (previously
DTLS 1.3 was silently negotiated instead) and add a DTLS 1.3 loop using
certificate-based auth. Guard TLS_CHACHA20_POLY1305_SHA256 with both
OPENSSL_NO_CHACHA and OPENSSL_NO_POLY1305.
Assisted-by: Claude:claude-sonnet-4-6
Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
Reviewed-by: Matt Caswell <matt@openssl.foundation>
Merge-date: Thu Aug 20 09:55:39 2026
Merged-from: https://github.com/openssl/openssl/pull/32340
diff --git a/ssl/d1_lib.c b/ssl/d1_lib.c
index 5ad6b84a06..8cbc339c81 100644
--- a/ssl/d1_lib.c
+++ b/ssl/d1_lib.c
@@ -1163,8 +1163,8 @@ size_t DTLS_get_data_mtu(const SSL *ssl)
if (ciph == NULL)
return 0;
- if (!ssl_cipher_get_overhead(ciph, &mac_overhead, &int_overhead,
- &blocksize, &ext_overhead))
+ if (!ssl_cipher_get_overhead(ciph, SSL_version(ssl), &mac_overhead,
+ &int_overhead, &blocksize, &ext_overhead))
return 0;
if (SSL_READ_ETM(s))
@@ -1194,7 +1194,6 @@ size_t DTLS_get_data_mtu(const SSL *ssl)
/* Added record type at the end of the data */
int_overhead++;
-
} else {
rechdrlen = DTLS1_RT_HEADER_LENGTH;
}
diff --git a/ssl/ssl_ciph.c b/ssl/ssl_ciph.c
index 7bc9e48006..160f5e4291 100644
--- a/ssl/ssl_ciph.c
+++ b/ssl/ssl_ciph.c
@@ -2182,20 +2182,29 @@ int SSL_CIPHER_is_aead(const SSL_CIPHER *c)
return (c->algorithm_mac & SSL_AEAD) ? 1 : 0;
}
-int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead,
- size_t *int_overhead, size_t *blocksize,
- size_t *ext_overhead)
+int ssl_cipher_get_overhead(const SSL_CIPHER *c, int version,
+ size_t *mac_overhead, size_t *int_overhead,
+ size_t *blocksize, size_t *ext_overhead)
{
int mac = 0, in = 0, blk = 0, out = 0;
/* Some hard-coded numbers for the CCM/Poly1305 MAC overhead
* because there are no handy #defines for those. */
if (c->algorithm_enc & (SSL_AESGCM | SSL_ARIAGCM)) {
- out = EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN;
+ out = EVP_GCM_TLS_TAG_LEN;
+ /* DTLS 1.3 uses an implicit nonce, so no explicit IV on the wire. */
+ if (version != DTLS1_3_VERSION)
+ out += EVP_GCM_TLS_EXPLICIT_IV_LEN;
} else if (c->algorithm_enc & (SSL_AES128CCM | SSL_AES256CCM)) {
- out = EVP_CCM_TLS_EXPLICIT_IV_LEN + 16;
+ out = 16;
+ /* DTLS 1.3 uses an implicit nonce, so no explicit IV on the wire. */
+ if (version != DTLS1_3_VERSION)
+ out += EVP_CCM_TLS_EXPLICIT_IV_LEN;
} else if (c->algorithm_enc & (SSL_AES128CCM8 | SSL_AES256CCM8)) {
- out = EVP_CCM_TLS_EXPLICIT_IV_LEN + 8;
+ out = 8;
+ /* DTLS 1.3 uses an implicit nonce, so no explicit IV on the wire. */
+ if (version != DTLS1_3_VERSION)
+ out += EVP_CCM_TLS_EXPLICIT_IV_LEN;
} else if (c->algorithm_enc & SSL_CHACHA20POLY1305) {
out = 16;
} else if (c->algorithm_mac & SSL_AEAD) {
diff --git a/ssl/ssl_local.h b/ssl/ssl_local.h
index eb1142431e..9959871ecd 100644
--- a/ssl/ssl_local.h
+++ b/ssl/ssl_local.h
@@ -2893,9 +2893,9 @@ __owur int ssl_cipher_get_evp(SSL_CTX *ctx, const SSL_SESSION *s,
const EVP_MD **md,
int *mac_pkey_type, size_t *mac_secret_size,
SSL_COMP **comp, int use_etm);
-__owur int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead,
- size_t *int_overhead, size_t *blocksize,
- size_t *ext_overhead);
+__owur int ssl_cipher_get_overhead(const SSL_CIPHER *c, int version,
+ size_t *mac_overhead, size_t *int_overhead,
+ size_t *blocksize, size_t *ext_overhead);
__owur int ssl_cert_is_disabled(SSL_CTX *ctx, size_t idx);
__owur const SSL_CIPHER *ssl_get_cipher_by_char(SSL_CONNECTION *ssl,
const unsigned char *ptr,
diff --git a/test/cipher_overhead_test.c b/test/cipher_overhead_test.c
index 176d6cc877..d4b9d15437 100644
--- a/test/cipher_overhead_test.c
+++ b/test/cipher_overhead_test.c
@@ -7,6 +7,7 @@
* https://www.openssl.org/source/license.html
*/
+#include <openssl/evp.h>
#include "internal/nelem.h"
#include "testutil.h"
#include "../ssl/ssl_local.h"
@@ -30,6 +31,7 @@ static int cipher_enabled(const SSL_CIPHER *ciph)
return 1;
}
+/* The DTLS 1.2 (and earlier) ciphers live in the ssl3_ciphers[] table. */
static int cipher_overhead(void)
{
int ret = 1, i, n = ssl3_num_ciphers();
@@ -44,7 +46,8 @@ static int cipher_overhead(void)
TEST_skip("Skipping disabled cipher %s", ciph->name);
continue;
}
- if (!TEST_true(ssl_cipher_get_overhead(ciph, &mac, &in, &blk, &ex))) {
+ if (!TEST_true(ssl_cipher_get_overhead(ciph, DTLS1_2_VERSION,
+ &mac, &in, &blk, &ex))) {
TEST_info("Failed getting %s", ciph->name);
ret = 0;
} else {
@@ -55,8 +58,50 @@ static int cipher_overhead(void)
return ret;
}
+/*
+ * The DTLS 1.3 ciphers are the TLS 1.3 ciphersuites, which are not reachable
+ * through ssl3_get_cipher(), so look them up by id instead.
+ */
+static const struct {
+ uint32_t id;
+ size_t ext; /* expected external overhead in DTLS 1.3 (no explicit IV) */
+} dtls13_ciphers[] = {
+ { TLS1_3_CK_AES_128_GCM_SHA256, EVP_GCM_TLS_TAG_LEN },
+ { TLS1_3_CK_AES_256_GCM_SHA384, EVP_GCM_TLS_TAG_LEN },
+ { TLS1_3_CK_CHACHA20_POLY1305_SHA256, 16 },
+ { TLS1_3_CK_AES_128_CCM_SHA256, 16 },
+ { TLS1_3_CK_AES_128_CCM_8_SHA256, 8 },
+};
+
+static int dtls13_cipher_overhead(int idx)
+{
+ const SSL_CIPHER *ciph = ssl3_get_cipher_by_id(dtls13_ciphers[idx].id);
+ size_t mac, in, blk, ex;
+
+ if (!TEST_ptr(ciph))
+ return 0;
+ if (!cipher_enabled(ciph)) {
+ TEST_skip("Skipping disabled cipher %s", ciph->name);
+ return 1;
+ }
+
+ /* DTLS 1.3 uses an implicit nonce, so there is no explicit IV on the wire. */
+ if (!TEST_true(ssl_cipher_get_overhead(ciph, DTLS1_3_VERSION,
+ &mac, &in, &blk, &ex))) {
+ TEST_info("Failed getting %s (DTLSv1.3)", ciph->name);
+ return 0;
+ }
+ TEST_info("Cipher %s (DTLSv1.3): %zu %zu %zu %zu",
+ ciph->name, mac, in, blk, ex);
+ if (!TEST_size_t_eq(ex, dtls13_ciphers[idx].ext))
+ return 0;
+
+ return 1;
+}
+
int setup_tests(void)
{
ADD_TEST(cipher_overhead);
+ ADD_ALL_TESTS(dtls13_cipher_overhead, OSSL_NELEM(dtls13_ciphers));
return 1;
}
diff --git a/test/dtls_mtu_test.c b/test/dtls_mtu_test.c
index 9a2d8caab0..a70a6e840a 100644
--- a/test/dtls_mtu_test.c
+++ b/test/dtls_mtu_test.c
@@ -22,6 +22,8 @@
#include "internal/ssl_unwrap.h"
static int debug = 0;
+static char *cert = NULL;
+static char *privkey = NULL;
static unsigned int clnt_psk_callback(SSL *ssl, const char *hint,
char *ident, unsigned int max_ident_len,
@@ -47,7 +49,8 @@ static unsigned int srvr_psk_callback(SSL *ssl, const char *identity,
return max_psk_len;
}
-static int mtu_test(SSL_CTX *ctx, const char *cs, int no_etm)
+static int mtu_test(SSL_CTX *sctx, SSL_CTX *cctx, const char *cs, int no_etm,
+ int dtls_version)
{
SSL *srvr_ssl = NULL, *clnt_ssl = NULL;
BIO *sc_bio = NULL;
@@ -60,27 +63,55 @@ static int mtu_test(SSL_CTX *ctx, const char *cs, int no_etm)
memset(buf, 0x5a, sizeof(buf));
- if (!TEST_true(create_ssl_objects(ctx, ctx, &srvr_ssl, &clnt_ssl,
+ if (!TEST_true(create_ssl_objects(sctx, cctx, &srvr_ssl, &clnt_ssl,
NULL, NULL)))
goto end;
if (no_etm)
SSL_set_options(srvr_ssl, SSL_OP_NO_ENCRYPT_THEN_MAC);
- if (!TEST_true(SSL_set_cipher_list(srvr_ssl, cs))
- || !TEST_true(SSL_set_cipher_list(clnt_ssl, cs))
- || !TEST_ptr(sc_bio = SSL_get_rbio(srvr_ssl))
- || !TEST_true(create_ssl_connection(clnt_ssl, srvr_ssl,
+ if (dtls_version == DTLS1_3_VERSION) {
+ if (!TEST_true(SSL_set_ciphersuites(srvr_ssl, cs))
+ || !TEST_true(SSL_set_ciphersuites(clnt_ssl, cs)))
+ goto end;
+ } else {
+ if (!TEST_true(SSL_set_max_proto_version(srvr_ssl, DTLS1_2_VERSION))
+ || !TEST_true(SSL_set_max_proto_version(clnt_ssl, DTLS1_2_VERSION))
+ || !TEST_true(SSL_set_cipher_list(srvr_ssl, cs))
+ || !TEST_true(SSL_set_cipher_list(clnt_ssl, cs)))
+ goto end;
+ }
+
+ if (!TEST_ptr(sc_bio = SSL_get_rbio(srvr_ssl))
+ || !TEST_true(create_ssl_connection(srvr_ssl, clnt_ssl,
SSL_ERROR_NONE)))
goto end;
if (debug)
TEST_info("Channel established");
+ /*
+ * DTLS 1.3 sends ACKs for post-handshake messages (e.g. NewSessionTicket).
+ * Those ACKs land in sc_bio before we start measuring. Drain them so the
+ * BIO contains only the application records we write below.
+ */
+ if (dtls_version == DTLS1_3_VERSION) {
+ unsigned char tmp[1];
+ size_t nread;
+
+ while (BIO_pending(sc_bio) > 0) {
+ if (!TEST_false(SSL_read_ex(srvr_ssl, tmp, sizeof(tmp), &nread))
+ || !TEST_int_eq(SSL_get_error(srvr_ssl, 0),
+ SSL_ERROR_WANT_READ))
+ goto end;
+ }
+ }
+
/* For record MTU values between 500 and 539, call DTLS_get_data_mtu()
* to query the payload MTU which will fit. */
for (i = 0; i < 30; i++) {
- SSL_set_mtu(clnt_ssl, 500 + i);
+ if (!TEST_true(SSL_set_mtu(clnt_ssl, 500 + i)))
+ goto end;
mtus[i] = DTLS_get_data_mtu(clnt_ssl);
if (debug)
TEST_info("%s%s MTU for record mtu %d = %zu",
@@ -93,18 +124,23 @@ static int mtu_test(SSL_CTX *ctx, const char *cs, int no_etm)
}
/* Now get out of the way */
- SSL_set_mtu(clnt_ssl, 1000);
+ if (!TEST_true(SSL_set_mtu(clnt_ssl, 1000)))
+ goto end;
/*
* Now for all values in the range of payload MTUs, send a payload of
* that size and see what actual record size we end up with.
*/
for (s = mtus[0]; s <= mtus[29]; s++) {
+ int rlen;
size_t reclen;
if (!TEST_int_eq(SSL_write(clnt_ssl, buf, (int)s), (int)s))
goto end;
- reclen = BIO_read(sc_bio, buf, sizeof(buf));
+ rlen = BIO_read(sc_bio, buf, sizeof(buf));
+ if (!TEST_int_gt(rlen, 0))
+ goto end;
+ reclen = (size_t)rlen;
if (debug)
TEST_info("record %zu for payload %zu", reclen, s);
@@ -148,6 +184,17 @@ static int run_mtu_tests(void)
SSL_CTX *ctx = NULL;
STACK_OF(SSL_CIPHER) *ciphers;
int i, ret = 0;
+#ifndef OPENSSL_NO_DTLS1_3
+ static const char *const dtls13_ciphers[] = {
+ "TLS_AES_128_GCM_SHA256",
+ "TLS_AES_256_GCM_SHA384",
+#if !defined(OPENSSL_NO_CHACHA) && !defined(OPENSSL_NO_POLY1305)
+ "TLS_CHACHA20_POLY1305_SHA256",
+#endif
+ };
+ SSL_CTX *sctx13 = NULL, *cctx13 = NULL;
+ size_t j;
+#endif
if (!TEST_ptr(ctx = SSL_CTX_new(DTLS_method())))
goto end;
@@ -157,9 +204,11 @@ static int run_mtu_tests(void)
SSL_CTX_set_security_level(ctx, 0);
/*
+ * DTLS 1.2: iterate over each enc/mac variant using PSK ciphers.
* We only care about iterating over each enc/mac; we don't want to
* repeat the test for each auth/kx variant. So keep life simple and
- * only do (non-DH) PSK.
+ * only do (non-DH) PSK. Pin to DTLS 1.2 so the intended ciphers are
+ * actually negotiated rather than being overridden by DTLS 1.3.
*/
if (!TEST_true(SSL_CTX_set_cipher_list(ctx, "PSK")))
goto end;
@@ -173,20 +222,45 @@ static int run_mtu_tests(void)
if (!HAS_PREFIX(cipher_name, "PSK-"))
continue;
- if (!TEST_int_gt(ret = mtu_test(ctx, cipher_name, 0), 0))
- break;
+ if (!TEST_int_gt(ret = mtu_test(ctx, ctx, cipher_name, 0, DTLS1_2_VERSION), 0))
+ goto end;
TEST_info("%s OK", cipher_name);
if (ret == 1)
continue;
/* mtu_test() returns 2 if it used Encrypt-then-MAC */
- if (!TEST_int_gt(ret = mtu_test(ctx, cipher_name, 1), 0))
- break;
+ if (!TEST_int_gt(ret = mtu_test(ctx, ctx, cipher_name, 1, DTLS1_2_VERSION), 0))
+ goto end;
TEST_info("%s without EtM OK", cipher_name);
}
+#ifndef OPENSSL_NO_DTLS1_3
+ /*
+ * DTLS 1.3: test each ciphersuite using certificate-based auth.
+ * PSK is not needed here — auth method doesn't affect record overhead.
+ */
+ if (!TEST_true(create_ssl_ctx_pair(NULL, DTLS_server_method(),
+ DTLS_client_method(),
+ DTLS1_3_VERSION, DTLS1_3_VERSION,
+ &sctx13, &cctx13, cert, privkey)))
+ goto end;
+
+ for (j = 0; j < OSSL_NELEM(dtls13_ciphers); j++) {
+ if (!TEST_int_gt(ret = mtu_test(sctx13, cctx13, dtls13_ciphers[j],
+ 0, DTLS1_3_VERSION),
+ 0))
+ goto end;
+ TEST_info("%s OK", dtls13_ciphers[j]);
+ }
+#endif
+
+ ret = 1;
end:
SSL_CTX_free(ctx);
+#ifndef OPENSSL_NO_DTLS1_3
+ SSL_CTX_free(sctx13);
+ SSL_CTX_free(cctx13);
+#endif
return ret;
}
@@ -231,8 +305,18 @@ end:
return rv;
}
+OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
+
int setup_tests(void)
{
+ if (!test_skip_common_options()) {
+ TEST_error("Error parsing test options\n");
+ return 0;
+ }
+
+ cert = test_get_argument(0);
+ privkey = test_get_argument(1);
+
ADD_TEST(run_mtu_tests);
ADD_TEST(test_server_mtu_larger_than_max_fragment_length);
return 1;
diff --git a/test/recipes/80-test_dtls_mtu.t b/test/recipes/80-test_dtls_mtu.t
index 501c42c8a1..dccb3b654e 100644
--- a/test/recipes/80-test_dtls_mtu.t
+++ b/test/recipes/80-test_dtls_mtu.t
@@ -7,7 +7,7 @@
# https://www.openssl.org/source/license.html
-use OpenSSL::Test;
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
use OpenSSL::Test::Utils;
my $test_name = "test_dtls_mtu";
@@ -18,4 +18,7 @@ plan skip_all => "$test_name needs DTLS and PSK support enabled"
plan tests => 1;
-ok(run(test(["dtls_mtu_test"])), "running dtls_mtu_test");
+ok(run(test(["dtls_mtu_test",
+ srctop_file("test/certs/servercert.pem"),
+ srctop_file("test/certs/serverkey.pem")])),
+ "running dtls_mtu_test");