Commit 398553b5a2 for openssl.org
commit 398553b5a2f65f5c8e34d72a6f6b2ee7ea086b10
Author: Mounir IDRASSI <mounir.idrassi@idrix.fr>
Date: Sun Aug 30 18:28:36 2026 +0900
DTLS 1.3: truncate the unified header sequence number and reconstruct it
The DTLS 1.3 unified header carries only the low 8 or 16 bits of the
record sequence number. Passing the full uint64_t value to
WPACKET_put_bytes_u16() made SSL_write() fail when the counter reached
65536.
Mask only the value encoded on the wire and in the corresponding AEAD
additional data, leaving the full counter intact for the nonce. On
receipt, reconstruct the full value relative to the right edge of the
current epoch's replay window, as recommended by RFC 9147 section 4.2.2.
Add boundary vectors for both wire widths and an end-to-end test across
the 16-bit wrap.
Fixes #32584
Assisted-by: pi:Hy4-preview
Assisted-by: Codex:gpt-5.6-sol
Reviewed-by: Ryan Hooper <ryanh@openssl.foundation>
Reviewed-by: Andrew Dinh <andrewd@openssl.org>
Merge-date: Mon Sep 7 14:20:03 2026
Merged-from: https://github.com/openssl/openssl/pull/32601
diff --git a/ssl/record/methods/dtls_meth.c b/ssl/record/methods/dtls_meth.c
index ec895b9b6e..ed176773e4 100644
--- a/ssl/record/methods/dtls_meth.c
+++ b/ssl/record/methods/dtls_meth.c
@@ -402,6 +402,57 @@ int dtls_crypt_sequence_number(EVP_CIPHER_CTX *ctx, unsigned char *seq, size_t s
return 1;
}
+/*
+ * Reconstruct the full sequence number as recommended by rfc9147 section
+ * 4.2.2. Select the candidate closest to the replay window's right edge plus
+ * one, ignore candidates outside the uint64_t range, and break ties forward.
+ * An empty replay window is represented by max_seq_num == 0.
+ */
+uint64_t dtls13_reconstruct_seq_num(uint64_t max_seq_num, uint64_t truncated,
+ size_t seqlen)
+{
+ uint64_t mask, period, expected, candidate, alt, best, best_dist, dist;
+
+ mask = DTLS13_UNI_HDR_SEQ_MASK(seqlen);
+ period = mask + 1;
+
+ /* At the end of the range only candidates in the last block can be valid. */
+ if (max_seq_num == UINT64_MAX)
+ return (UINT64_MAX & ~mask) | truncated;
+
+ expected = max_seq_num + 1;
+
+ /* The candidate in the same period-sized block as |expected| */
+ candidate = (expected & ~mask) | truncated;
+ best = candidate;
+ best_dist = candidate > expected ? candidate - expected
+ : expected - candidate;
+
+ /* The candidate one period behind, if it does not underflow */
+ if (candidate >= period) {
+ alt = candidate - period;
+ dist = alt > expected ? alt - expected : expected - alt;
+ /* Strictly closer only: a tie goes to the forward candidate */
+ if (dist < best_dist) {
+ best = alt;
+ best_dist = dist;
+ }
+ }
+
+ /* The candidate one period ahead, if it does not overflow */
+ if (candidate <= UINT64_MAX - period) {
+ alt = candidate + period;
+ dist = alt > expected ? alt - expected : expected - alt;
+ /* Ties go forward */
+ if (dist <= best_dist) {
+ best = alt;
+ best_dist = dist;
+ }
+ }
+
+ return best;
+}
+
/*-
* Call this to get a new input record.
* It will return <= 0 if more data is needed, normally due to an error
@@ -675,12 +726,36 @@ again:
goto again;
}
- rl->sequence = ((uint64_t)recseqnum[0]) << 40;
- rl->sequence |= ((uint64_t)recseqnum[1]) << 32;
- rl->sequence |= ((uint64_t)recseqnum[2]) << 24;
- rl->sequence |= ((uint64_t)recseqnum[3]) << 16;
- rl->sequence |= ((uint64_t)recseqnum[4]) << 8;
- rl->sequence |= ((uint64_t)recseqnum[5]) << 0;
+ if (rl->version == DTLS1_3_VERSION && rr->epoch == rl->epoch
+ && DTLS13_UNI_HDR_FIX_BITS_IS_SET(rr->type)) {
+ /* Reconstruct current-epoch unified records using its replay window. */
+ uint64_t truncated = 0;
+ size_t i;
+
+ /* A unified header carries 8 or 16 bits of the sequence number */
+ if (!ossl_assert(recseqnumlen == 1 || recseqnumlen == 2)) {
+ RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR);
+ return OSSL_RECORD_RETURN_FATAL;
+ }
+
+ for (i = 0; i < recseqnumlen; i++)
+ truncated = (truncated << 8) | recseqnum[recseqnumoffs + i];
+
+ rl->sequence = dtls13_reconstruct_seq_num(rl->bitmap.max_seq_num,
+ truncated, recseqnumlen);
+ } else {
+ /*
+ * DTLSPlaintext carries 48 bits. A buffered next-epoch unified record
+ * is re-parsed after that epoch's record layer is installed, so this
+ * provisional value is unused.
+ */
+ rl->sequence = ((uint64_t)recseqnum[0]) << 40;
+ rl->sequence |= ((uint64_t)recseqnum[1]) << 32;
+ rl->sequence |= ((uint64_t)recseqnum[2]) << 24;
+ rl->sequence |= ((uint64_t)recseqnum[3]) << 16;
+ rl->sequence |= ((uint64_t)recseqnum[4]) << 8;
+ rl->sequence |= ((uint64_t)recseqnum[5]) << 0;
+ }
/* match epochs. NULL means the packet is dropped on the floor */
bitmap = dtls_get_bitmap(rl, rr, &is_next_epoch);
@@ -904,10 +979,14 @@ int dtls_prepare_record_header(OSSL_RECORD_LAYER *rl,
uint8_t lbit = DTLS13_UNI_HDR_LEN_BIT;
uint8_t ebits = rl->epoch & DTLS13_UNI_HDR_EPOCH_BITS_MASK;
uint8_t unifiedhdrbits = fixedbits | cbit | sbit | lbit | ebits;
+ uint64_t seqnum;
+
+ /* Truncate only the wire encoding, not the AEAD nonce counter. */
+ seqnum = rl->sequence & DTLS13_UNI_HDR_SEQ_MASK(sbit ? 2 : 1);
if (!WPACKET_put_bytes_u8(thispkt, unifiedhdrbits)
- || (sbit ? !WPACKET_put_bytes_u16(thispkt, rl->sequence)
- : !WPACKET_put_bytes_u8(thispkt, rl->sequence))
+ || (sbit ? !WPACKET_put_bytes_u16(thispkt, seqnum)
+ : !WPACKET_put_bytes_u8(thispkt, seqnum))
|| !WPACKET_start_sub_packet_u16(thispkt)
|| (rl->eivlen > 0
&& !WPACKET_allocate_bytes(thispkt, rl->eivlen, NULL))
diff --git a/ssl/record/methods/recmethod_local.h b/ssl/record/methods/recmethod_local.h
index 1b2e04e66a..aa01d4b1a8 100644
--- a/ssl/record/methods/recmethod_local.h
+++ b/ssl/record/methods/recmethod_local.h
@@ -458,6 +458,13 @@ int tls_get_more_records(OSSL_RECORD_LAYER *rl);
#define DTLS13_UNI_HDR_LEN_BIT_IS_SET(byte) \
(((byte) & DTLS13_UNI_HDR_LEN_BIT) == DTLS13_UNI_HDR_LEN_BIT)
+/* Low-order sequence number bits carried by a DTLS 1.3 unified header. */
+#define DTLS13_UNI_HDR_SEQ_MASK(len) \
+ ((((uint64_t)1) << ((len) * 8)) - 1)
+
+uint64_t dtls13_reconstruct_seq_num(uint64_t max_seq_num, uint64_t truncated,
+ size_t seqlen);
+
size_t dtls_get_rec_header_size(uint8_t hdr_first_byte);
int dtls_crypt_sequence_number(EVP_CIPHER_CTX *ctx, unsigned char *seq, size_t seqlen,
unsigned char *rec_data);
diff --git a/ssl/record/methods/tls13_meth.c b/ssl/record/methods/tls13_meth.c
index 13b46a20a7..13228d02d9 100644
--- a/ssl/record/methods/tls13_meth.c
+++ b/ssl/record/methods/tls13_meth.c
@@ -118,6 +118,7 @@ static int tls13_cipher(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *recs,
unsigned char *staticiv;
unsigned char *nonce;
unsigned char seq[SEQ_NUM_SIZE], *p_seq = seq;
+ uint64_t seqnum = 0;
int lenu, lenf;
TLS_RL_RECORD *rec = &recs[0];
WPACKET wpkt;
@@ -215,6 +216,8 @@ static int tls13_cipher(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *recs,
exphdrlen = dtls_get_rec_header_size(rec->type);
sbit = DTLS13_UNI_HDR_SEQ_BIT_IS_SET(rec->type);
addlen = DTLS13_UNI_HDR_LEN_BIT_IS_SET(rec->type);
+ /* Match the truncated value encoded in the unified header. */
+ seqnum = rl->sequence & DTLS13_UNI_HDR_SEQ_MASK(sbit ? 2 : 1);
} else {
exphdrlen = SSL3_RT_HEADER_LENGTH;
addlen = 1;
@@ -223,7 +226,9 @@ static int tls13_cipher(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *recs,
if ((isdtls && !ossl_assert(!DTLS13_UNI_HDR_CID_BIT_IS_SET(rec->type)))
|| !WPACKET_init_static_len(&wpkt, recheader, sizeof(recheader), 0)
|| !WPACKET_put_bytes_u8(&wpkt, rec->type)
- || (isdtls && (sbit ? !WPACKET_put_bytes_u16(&wpkt, rl->sequence) : !WPACKET_put_bytes_u8(&wpkt, rl->sequence)))
+ || (isdtls
+ && (sbit ? !WPACKET_put_bytes_u16(&wpkt, seqnum)
+ : !WPACKET_put_bytes_u8(&wpkt, seqnum)))
|| (!isdtls && !WPACKET_put_bytes_u16(&wpkt, rec->rec_version))
|| (addlen && !WPACKET_put_bytes_u16(&wpkt, rec->length + rl->taglen))
|| !WPACKET_get_total_written(&wpkt, &hdrlen)
diff --git a/test/build.info b/test/build.info
index c87988312b..0d4c0937a6 100644
--- a/test/build.info
+++ b/test/build.info
@@ -743,7 +743,7 @@ IF[{- !$disabled{tests} -}]
ENDIF
SOURCE[dtlstest]=dtlstest.c helpers/ssltestlib.c
- INCLUDE[dtlstest]=../include ../apps/include
+ INCLUDE[dtlstest]=.. ../include ../apps/include
DEPEND[dtlstest]=../libcrypto ../libssl libtestutil.a
IF[{- !$disabled{dtls} -}]
diff --git a/test/dtls13_internal_test.c b/test/dtls13_internal_test.c
index b164182679..8bda47fdd0 100644
--- a/test/dtls13_internal_test.c
+++ b/test/dtls13_internal_test.c
@@ -8,6 +8,7 @@
*/
#include "../ssl/record/methods/recmethod_local.h"
+#include "internal/nelem.h"
#include "testutil.h"
#include <openssl/evp.h>
@@ -72,8 +73,94 @@ err:
return 0;
}
+/* rfc9147 section 4.2.2 sequence number reconstruction vectors. */
+typedef struct seq_num_test_st {
+ /* Zero also represents the initial empty replay window. */
+ uint64_t max_seq_num;
+ uint64_t truncated;
+ size_t seqlen;
+ uint64_t seq_num;
+} SEQ_NUM_TEST;
+
+static const SEQ_NUM_TEST seq_num_tests[] = {
+ /* Empty window and first-period lower-bound cases. */
+ { 0, 0, 1, 0 },
+ { 0, 1, 1, 1 },
+ { 0, 0x7f, 1, 0x7f },
+ { 0, 0x80, 1, 0x80 },
+ { 0, 0x81, 1, 0x81 },
+ { 0, 0xff, 1, 0xff },
+ { 0, 0, 2, 0 },
+ { 0, 1, 2, 1 },
+ { 0, 0x7fff, 2, 0x7fff },
+ { 0, 0x8000, 2, 0x8000 },
+ { 0, 0x8001, 2, 0x8001 },
+ { 0, 0x8002, 2, 0x8002 },
+ { 0, 40000, 2, 40000 },
+ { 0, 0xffff, 2, 0xffff },
+
+ /* Ordinary forward progression */
+ { 5, 6, 2, 6 },
+ { 5, 6, 1, 6 },
+ { 200, 201, 2, 201 },
+ { 0x1234, 0x1235, 2, 0x1235 },
+
+ /* 8- and 16-bit wraps */
+ { 0xfe, 0xff, 1, 0xff },
+ { 0xff, 0x00, 1, 0x100 },
+ { 0x100, 0x01, 1, 0x101 },
+ { 0x1fe, 0xff, 1, 0x1ff },
+ { 0x1ff, 0x00, 1, 0x200 },
+ { 0xfffe, 0xffff, 2, 0xffff },
+ { 0xffff, 0x0000, 2, 0x10000 },
+ { 0x10000, 0x0001, 2, 0x10001 },
+ { 0x1fffe, 0xffff, 2, 0x1ffff },
+ { 0x1ffff, 0x0000, 2, 0x20000 },
+ { 0x20000, 0x0001, 2, 0x20001 },
+
+ /* Reordered records */
+ { 0x100, 0xff, 2, 0xff },
+ { 0x100, 0xfe, 2, 0xfe },
+ { 0x10010, 0x000f, 2, 0x1000f },
+ { 300, 40, 1, 296 },
+
+ /* Half-period ties select the forward candidate in either phase. */
+ { 199, 72, 1, 328 }, /* candidate 72, 128 behind: pick 328 */
+ { 299, 172, 1, 428 }, /* candidate 428, 128 ahead: keep it */
+ { 0x180ff, 0x0100, 2, 0x20100 }, /* candidate 0x10100: pick 0x20100 */
+ { 0x100ff, 0x8100, 2, 0x18100 }, /* candidate 0x18100: keep it */
+
+ /* Top-of-uint64_t fallback and overflow boundaries */
+ { UINT64_MAX, 0, 2, UINT64_MAX & ~UINT64_C(0xffff) },
+ { UINT64_MAX, 0xffff, 2, UINT64_MAX },
+ { UINT64_MAX, 0xffc0, 2, (UINT64_MAX & ~UINT64_C(0xffff)) | 0xffc0 },
+ { UINT64_MAX, 5, 1, (UINT64_MAX & ~UINT64_C(0xff)) | 5 },
+ { UINT64_MAX - 1, 0, 2, UINT64_MAX & ~UINT64_C(0xffff) },
+ { UINT64_MAX - 0x10000, 0, 2, UINT64_MAX - 0xffff },
+ { UINT64_MAX - 0x10000, 1, 2, UINT64_MAX - 0xffff + 1 },
+ { UINT64_MAX - 0x10000, 0x8000, 2,
+ (UINT64_MAX - 0xffff) | 0x8000 },
+};
+
+static int test_seq_num_reconstruction(int idx)
+{
+ const SEQ_NUM_TEST *t = &seq_num_tests[idx];
+ uint64_t seq_num = 0;
+
+ seq_num = dtls13_reconstruct_seq_num(t->max_seq_num, t->truncated,
+ t->seqlen);
+
+ if (!TEST_uint64_t_eq(seq_num, t->seq_num))
+ return 0;
+
+ /* The reconstructed value must retain the wire bits used in the AAD. */
+ return TEST_uint64_t_eq(seq_num & DTLS13_UNI_HDR_SEQ_MASK(t->seqlen),
+ t->truncated);
+}
+
int setup_tests(void)
{
ADD_ALL_TESTS(test_dtls_crypt_sequence_number, OSSL_NELEM(cipher_names));
+ ADD_ALL_TESTS(test_seq_num_reconstruction, OSSL_NELEM(seq_num_tests));
return 1;
}
diff --git a/test/dtlstest.c b/test/dtlstest.c
index f95a02ea91..96ad66bef8 100644
--- a/test/dtlstest.c
+++ b/test/dtlstest.c
@@ -13,6 +13,10 @@
#include <openssl/ssl.h>
#include <openssl/err.h>
+#include "internal/nelem.h"
+#include "internal/ssl_unwrap.h"
+#include "../ssl/ssl_local.h"
+#include "../ssl/record/methods/recmethod_local.h"
#include "helpers/ssltestlib.h"
#include "testutil.h"
@@ -974,6 +978,117 @@ end:
}
#endif /* OPENSSL_NO_DTLS */
+#ifndef OPENSSL_NO_DTLS1_3
+/* Place record state near boundaries instead of sending 65536 records. */
+typedef struct seqnum_test_st {
+ uint64_t start;
+ /* Keep the initial window for first-period cases; otherwise align it. */
+ int move_reader;
+ int num;
+} SEQNUM_TEST;
+
+static const SEQNUM_TEST seqnum_tests[] = {
+ { 0x100, 1, 2 }, /* ordinary forward progression */
+ { 0xfffe, 0, 2 }, /* end of the first period, reader at the epoch start */
+ { 40000, 0, 2 }, /* more than half a period past the reader's window */
+ { 0xffff, 1, 4 }, /* the 16 bit sequence number field wraps */
+ { 0x1ffff, 1, 4 }, /* ... and wraps again */
+ { 0xfffffffe, 1, 4 }, /* ... and past the 32 bit boundary */
+};
+
+static int test_seq_num_wrap(int idx)
+{
+ SSL_CTX *sctx = NULL, *cctx = NULL;
+ SSL *sssl = NULL, *cssl = NULL;
+ SSL_CONNECTION *ssc, *csc;
+ const SEQNUM_TEST *t = &seqnum_tests[idx];
+ unsigned char wrbuf[16], rdbuf[16];
+ int testresult = 0;
+ int i, j;
+
+ if (!TEST_true(create_ssl_ctx_pair(NULL, DTLS_server_method(),
+ DTLS_client_method(),
+ DTLS1_3_VERSION, DTLS1_3_VERSION,
+ &sctx, &cctx, cert, privkey)))
+ return 0;
+
+ if (!TEST_true(create_ssl_objects(sctx, cctx, &sssl, &cssl, NULL, NULL)))
+ goto end;
+
+ if (!TEST_true(create_ssl_connection(sssl, cssl, SSL_ERROR_NONE)))
+ goto end;
+
+ if (!TEST_ptr(ssc = SSL_CONNECTION_FROM_SSL_ONLY(sssl))
+ || !TEST_ptr(csc = SSL_CONNECTION_FROM_SSL_ONLY(cssl)))
+ goto end;
+
+ /* Application data must use a unified header. */
+ if (!TEST_uint64_t_gt(csc->rlayer.wrl->epoch, 0)
+ || !TEST_uint64_t_gt(ssc->rlayer.wrl->epoch, 0))
+ goto end;
+
+ /* Drain post-handshake ACKs before manipulating the record state. */
+ while (SSL_read(sssl, rdbuf, sizeof(rdbuf)) > 0)
+ continue;
+ while (SSL_read(cssl, rdbuf, sizeof(rdbuf)) > 0)
+ continue;
+ ERR_clear_error();
+
+ for (j = 0; j < 2; j++) {
+ SSL *wssl = j == 0 ? cssl : sssl;
+ SSL *rssl = j == 0 ? sssl : cssl;
+ SSL_CONNECTION *wsc = j == 0 ? csc : ssc;
+ SSL_CONNECTION *rsc = j == 0 ? ssc : csc;
+
+ /* The writer must not go backwards: that would reuse a nonce */
+ if (!TEST_uint64_t_ge(t->start, wsc->rlayer.wrl->sequence))
+ goto end;
+
+ wsc->rlayer.wrl->sequence = t->start;
+
+ if (t->move_reader) {
+ rsc->rlayer.rrl->bitmap.max_seq_num = t->start - 1;
+ rsc->rlayer.rrl->bitmap.map = 1;
+ }
+
+ for (i = 0; i < t->num; i++) {
+ memset(wrbuf, 0, sizeof(wrbuf));
+ wrbuf[0] = (unsigned char)(i + 1);
+
+ if (!TEST_int_eq(SSL_write(wssl, wrbuf, sizeof(wrbuf)),
+ (int)sizeof(wrbuf)))
+ goto end;
+
+ /* The full sequence number keeps increasing across the wrap */
+ if (!TEST_uint64_t_eq(wsc->rlayer.wrl->sequence,
+ t->start + i + 1))
+ goto end;
+
+ memset(rdbuf, 0, sizeof(rdbuf));
+ if (!TEST_int_eq(SSL_read(rssl, rdbuf, sizeof(rdbuf)),
+ (int)sizeof(rdbuf))
+ || !TEST_mem_eq(rdbuf, sizeof(rdbuf), wrbuf,
+ sizeof(wrbuf)))
+ goto end;
+
+ /* ... and the peer has to have reconstructed the same value */
+ if (!TEST_uint64_t_eq(rsc->rlayer.rrl->bitmap.max_seq_num,
+ t->start + i))
+ goto end;
+ }
+ }
+
+ testresult = 1;
+end:
+ SSL_free(cssl);
+ SSL_free(sssl);
+ SSL_CTX_free(cctx);
+ SSL_CTX_free(sctx);
+
+ return testresult;
+}
+#endif /* OPENSSL_NO_DTLS1_3 */
+
/* Confirm that we can create a connections using DTLSv1_listen() */
#ifndef OPENSSL_NO_DTLS1_2
static int test_listen(void)
@@ -1055,6 +1170,7 @@ int setup_tests(void)
#endif
#ifndef OPENSSL_NO_DTLS1_3
ADD_TEST(test_duplicate_app_data_dtls13);
+ ADD_ALL_TESTS(test_seq_num_wrap, OSSL_NELEM(seqnum_tests));
#endif
return 1;