Commit 497bdda39a for openssl.org
commit 497bdda39a8bbd293d44f0454c0be620c11cc8c2
Author: Mounir IDRASSI <mounir.idrassi@idrix.fr>
Date: Mon Oct 12 12:25:23 2026 +0900
DTLS listener: bound the demux free list
The demux free list caches released URXEs so a receive burst can be
served without allocating, but nothing bounds it: a transient flood
raising the per-connection receive queues also raises the free list's
retained high-water mark for the remaining lifetime of the listener.
Add a per-demux limit, ossl_dgram_demux_set_max_free_urxe(), enforced
in ossl_dgram_demux_release_urxe(): a URXE released while the free list
is at its limit is freed instead of cached, so the bound is strict
and no later trim pass is needed. Freeing there honours the documented
contract that no reference may be made to a URXE or its buffer after
it is released, and no release caller touches the entry afterwards.
The DTLS listener sets the limit to DEMUX_MAX_FREE_URXE, two receive
batches, so one full batch can always be cached; QUIC keeps the default
unbounded behaviour.
Resolves: https://github.com/openssl/openssl/issues/33173
Assisted-by: Claude Code:moonshotai/kimi-k3
Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
Reviewed-by: Ryan Hooper <ryanh@openssl.foundation>
Reviewed-by: Eugene Syromiatnikov <esyr@openssl.org>
Merge-date: Wed Oct 14 10:54:56 2026
Merged-from: https://github.com/openssl/openssl/pull/33197
diff --git a/include/internal/dgram_demux.h b/include/internal/dgram_demux.h
index 9b91b8be5c..f735c6a455 100644
--- a/include/internal/dgram_demux.h
+++ b/include/internal/dgram_demux.h
@@ -104,6 +104,17 @@ struct dgram_urxe_st {
#define URXE_DEMUX_STATE_PENDING 1 /* on urx_pending list */
#define URXE_DEMUX_STATE_ISSUED 2 /* on neither list */
+/* Maximum number of datagrams received in a single call (one batch). */
+#define DEMUX_MAX_MSGS_PER_CALL 32
+
+/*
+ * Free-list limit used by the DTLS listener: two receive batches, so one
+ * full batch can always be cached. Enforced by
+ * ossl_dgram_demux_release_urxe() once set with
+ * ossl_dgram_demux_set_max_free_urxe(); the default 0 is unbounded.
+ */
+#define DEMUX_MAX_FREE_URXE (2 * DEMUX_MAX_MSGS_PER_CALL)
+
/* List structure tracking a queue of URXEs. */
DEFINE_LIST_OF(urxe, DGRAM_URXE);
typedef OSSL_LIST(urxe) DGRAM_URXE_LIST;
@@ -168,6 +179,17 @@ void ossl_dgram_demux_set_bio(DGRAM_DEMUX *demux, BIO *net_bio);
*/
int ossl_dgram_demux_set_mtu(DGRAM_DEMUX *demux, unsigned int mtu);
+/*
+ * Sets the maximum number of URXEs cached on the free list; 0 (the
+ * default) leaves it unbounded.
+ */
+void ossl_dgram_demux_set_max_free_urxe(DGRAM_DEMUX *demux, size_t max_free);
+
+/*
+ * Returns the number of URXEs currently cached on the free list.
+ */
+size_t ossl_dgram_demux_get_free_list_len(DGRAM_DEMUX *demux);
+
/*
* Set the default packet handler. This is called for every incoming datagram.
* If a default packet handler is not set, received datagrams are silently
diff --git a/ssl/d1_lib.c b/ssl/d1_lib.c
index d42ad2e52c..cbe08c2709 100644
--- a/ssl/d1_lib.c
+++ b/ssl/d1_lib.c
@@ -1850,6 +1850,8 @@ SSL *ossl_dtls_new_listener(SSL_CTX *ctx, uint64_t flags)
goto err;
}
+ ossl_dgram_demux_set_max_free_urxe(dl->demux, DEMUX_MAX_FREE_URXE);
+
/* Set up the packet handler callback for routing datagrams to connections */
ossl_dgram_demux_set_default_handler(dl->demux, dtls_listener_packet_handler, dl);
diff --git a/ssl/dgram_demux.c b/ssl/dgram_demux.c
index a23de37a2a..7404b6299d 100644
--- a/ssl/dgram_demux.c
+++ b/ssl/dgram_demux.c
@@ -18,7 +18,6 @@
/* URXE_DEMUX_STATE_* are defined in dgram_demux.h */
-#define DEMUX_MAX_MSGS_PER_CALL 32
#define DEMUX_DEFAULT_MTU 1500
#define DEMUX_MIN_INITIAL_DGRAM_LEN 1200
@@ -64,6 +63,9 @@ struct dgram_demux_st {
/* Whether internal locking is required */
char require_mutex;
+ /* Maximum URXEs cached on urx_free; 0 means unbounded. */
+ size_t max_free_urxe;
+
/*
* Mutex protecting the URXE lists (urx_free and urx_pending).
*/
@@ -184,6 +186,27 @@ int ossl_dgram_demux_set_mtu(DGRAM_DEMUX *demux, unsigned int mtu)
return 1;
}
+void ossl_dgram_demux_set_max_free_urxe(DGRAM_DEMUX *demux, size_t max_free)
+{
+ if (demux->require_mutex)
+ ossl_crypto_mutex_lock(demux->mutex);
+ demux->max_free_urxe = max_free;
+ if (demux->require_mutex)
+ ossl_crypto_mutex_unlock(demux->mutex);
+}
+
+size_t ossl_dgram_demux_get_free_list_len(DGRAM_DEMUX *demux)
+{
+ size_t n;
+
+ if (demux->require_mutex)
+ ossl_crypto_mutex_lock(demux->mutex);
+ n = ossl_list_urxe_num(&demux->urx_free);
+ if (demux->require_mutex)
+ ossl_crypto_mutex_unlock(demux->mutex);
+ return n;
+}
+
void ossl_dgram_demux_set_default_handler(DGRAM_DEMUX *demux,
ossl_dgram_demux_cb_fn *cb,
void *cb_arg)
@@ -512,18 +535,31 @@ end:
return ret;
}
-/* Called by our user to return a URXE to the free list. */
+/*
+ * Called by our user to return a URXE to the free list. If the free list
+ * is already at its limit the URXE is freed instead of being cached;
+ * callers must not reference a URXE after releasing it, so freeing here
+ * is safe even inside the routing callback.
+ */
void ossl_dgram_demux_release_urxe(DGRAM_DEMUX *demux, DGRAM_URXE *e)
{
+ int cache;
+
assert(ossl_list_urxe_prev(e) == NULL && ossl_list_urxe_next(e) == NULL);
assert(e->demux_state == URXE_DEMUX_STATE_ISSUED);
if (demux->require_mutex)
ossl_crypto_mutex_lock(demux->mutex);
- ossl_list_urxe_insert_tail(&demux->urx_free, e);
- e->demux_state = URXE_DEMUX_STATE_FREE;
+ cache = demux->max_free_urxe == 0
+ || ossl_list_urxe_num(&demux->urx_free) < demux->max_free_urxe;
+ if (cache) {
+ ossl_list_urxe_insert_tail(&demux->urx_free, e);
+ e->demux_state = URXE_DEMUX_STATE_FREE;
+ }
if (demux->require_mutex)
ossl_crypto_mutex_unlock(demux->mutex);
+ if (!cache)
+ OPENSSL_free(e);
}
void ossl_dgram_demux_reinject_urxe(DGRAM_DEMUX *demux, DGRAM_URXE *e)
diff --git a/test/dtlsssllistenertest.c b/test/dtlsssllistenertest.c
index 71ffa8dabe..7e701a5bf7 100644
--- a/test/dtlsssllistenertest.c
+++ b/test/dtlsssllistenertest.c
@@ -6366,6 +6366,140 @@ err:
SSL_CTX_free(cctx);
return testresult;
}
+
+/*
+ * Regression test: the demux free list must be bounded, so a transient
+ * flood does not raise the listener's retained memory for its lifetime. A
+ * parked connection's queue is filled to the cap, then freed; the released
+ * URXEs must not leave more than DEMUX_MAX_FREE_URXE cached, and a fresh
+ * handshake must still complete.
+ */
+static int test_dtls_listener_flood_then_handshake_int(int version)
+{
+ SSL_CTX *sctx = NULL, *cctx = NULL;
+ SSL *listener = NULL, *clientssl = NULL, *serverssl = NULL;
+ SSL *client2 = NULL, *serverssl2 = NULL;
+ BIO_ADDR *server_addr = NULL;
+ int server_fd = -1, client_fd = -1, client2_fd = -1;
+ DTLS_LISTENER *dl;
+ SSL_CONNECTION *sc;
+ unsigned char garbage[1200];
+ size_t queued, i;
+ int testresult = 0, k;
+ const size_t cap = OSSL_DTLS_RX_DEFAULT_MAX_QUEUED;
+ /* Fill the queue, then keep flooding across many more pump batches. */
+ const size_t flood = cap + 128;
+
+ memset(garbage, 0xA5, sizeof(garbage)); /* invalid content type */
+
+ if (!TEST_true(create_ssl_ctx_pair(NULL, DTLS_server_method(),
+ DTLS_client_method(), version, version,
+ &sctx, &cctx, cert, privkey)))
+ goto end;
+
+ if (!TEST_true(create_dtls_listener(sctx,
+ SSL_LISTENER_FLAG_SINGLE_THREAD,
+ &listener, &server_addr, &server_fd)))
+ goto end;
+
+ if (!TEST_true(create_dtls_client_for_addr(cctx, server_addr,
+ &clientssl, &client_fd)))
+ goto end;
+
+ dl = (DTLS_LISTENER *)listener;
+
+ /* Complete the cookie exchange and promotion, then park the connection. */
+ if (!drive_client_to_promotion(listener, clientssl, 1))
+ goto end;
+ if (!TEST_ptr(serverssl = SSL_accept_connection(listener,
+ SSL_ACCEPT_CONNECTION_NO_BLOCK)))
+ goto end;
+
+ for (i = 0; i < flood; i++) {
+ /*
+ * Send a raw datagram from the client's own endpoint, bypassing its
+ * SSL object, so it is routed to the parked connection's queue.
+ */
+ if (!TEST_int_eq(BIO_write(SSL_get_rbio(clientssl), garbage,
+ (int)sizeof(garbage)),
+ (int)sizeof(garbage)))
+ goto end;
+ listener_tick(listener);
+ }
+ /* Drain anything still in the kernel buffer. */
+ for (k = 0; k < 50; k++) {
+ listener_tick(listener);
+ OSSL_sleep(2);
+ }
+
+ /* The receive queue is exactly at its cap, not at the flood size. */
+ if (!TEST_ptr(sc = SSL_CONNECTION_FROM_SSL_ONLY(serverssl))
+ || !TEST_ptr(sc->listener_rx))
+ goto end;
+ queued = ossl_list_urxe_num(&sc->listener_rx->urxe_pending);
+ if (!TEST_size_t_eq(queued, cap))
+ goto end;
+
+ /* Freeing the parked connection releases its URXEs to the demux. */
+ SSL_free(serverssl);
+ serverssl = NULL;
+
+ /*
+ * The flood pushed cap URXEs through the free list, but the list is
+ * capped at release time: at most DEMUX_MAX_FREE_URXE may be cached.
+ */
+ if (!TEST_size_t_le(ossl_dgram_demux_get_free_list_len(dl->demux),
+ DEMUX_MAX_FREE_URXE))
+ goto end;
+
+ /* A fresh connection over the same listener must complete a handshake. */
+ if (!TEST_true(create_dtls_client_for_addr(cctx, server_addr,
+ &client2, &client2_fd)))
+ goto end;
+ if (!drive_client_to_promotion(listener, client2, 1))
+ goto end;
+ if (!TEST_ptr(serverssl2 = SSL_accept_connection(listener,
+ SSL_ACCEPT_CONNECTION_NO_BLOCK)))
+ goto end;
+ if (!TEST_true(create_ssl_connection(serverssl2, client2,
+ SSL_ERROR_NONE)))
+ goto end;
+ if (!TEST_int_eq(SSL_version(serverssl2), version))
+ goto end;
+
+ testresult = 1;
+end:
+ SSL_free(serverssl);
+ SSL_free(serverssl2);
+ SSL_free(client2);
+ SSL_free(clientssl);
+ if (client2_fd >= 0)
+ BIO_closesocket(client2_fd);
+ if (client_fd >= 0)
+ BIO_closesocket(client_fd);
+ SSL_free(listener);
+ BIO_ADDR_free(server_addr);
+ if (server_fd >= 0)
+ BIO_closesocket(server_fd);
+ SSL_CTX_free(sctx);
+ SSL_CTX_free(cctx);
+ return testresult;
+}
+
+static int test_dtls_listener_flood_then_handshake(void)
+{
+ int ok = 1;
+
+#ifndef OPENSSL_NO_DTLS1_2
+ if (!test_dtls_listener_flood_then_handshake_int(DTLS1_2_VERSION))
+ ok = 0;
+#endif
+#ifndef OPENSSL_NO_DTLS1_3
+ if (!test_dtls_listener_flood_then_handshake_int(DTLS1_3_VERSION))
+ ok = 0;
+#endif
+ return ok;
+}
#endif /* !OPENSSL_NO_DTLS1_2 || !OPENSSL_NO_DTLS1_3 */
OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
@@ -6511,6 +6645,8 @@ int setup_tests(void)
*/
ADD_TEST(test_dtls_listener_accept_queue_capped);
ADD_TEST(test_dtls_listener_accept_queue_novalidate_capped);
+ /* Demux free-list bound regression test; runs every enabled version */
+ ADD_TEST(test_dtls_listener_flood_then_handshake);
#endif
return 1;