Commit 809b52282a for openssl.org

commit 809b52282a3ea6d56fc2e1c427f2d9bd83b06eaa
Author: Matt Caswell <matt@openssl.foundation>
Date:   Tue Jun 23 11:53:17 2026 +0100

    Avoid full read buffer allocation when buffering DTLS next-epoch records

    dtls_rlayer_buffer_record() buffers records that arrive early for the
    next epoch while a handshake is in progress. It did this by taking
    ownership of the entire live read buffer (sized for the largest
    possible record, ~16.7KB) and allocating a brand new one to carry on
    reading, regardless of how small the buffered record actually was.
    With the queue capped at 100 entries, a peer could send around 100
    tiny bogus next-epoch records (~14 bytes each on the wire) and force
    around 1.7MB of heap allocation per connection.

    Instead, copy only the record's own on-wire bytes (header and
    ciphertext) into the queue entry, and leave the live read buffer
    untouched. Memory use is now proportional to what the peer actually
    sends.

    Fixes CVE-2026-54874

    Assisted-by: Claude:claude-sonnet-4-6
    Reviewed-by: Milan Broz <mbroz@openssl.org>
    Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
    Reviewed-by: Andrew Dinh <andrewd@openssl.org>
    Merge-date: Mon Aug 24 15:37:50 2026

diff --git a/ssl/record/methods/dtls_meth.c b/ssl/record/methods/dtls_meth.c
index 434316507d..c0323d6f22 100644
--- a/ssl/record/methods/dtls_meth.c
+++ b/ssl/record/methods/dtls_meth.c
@@ -299,29 +299,26 @@ static int dtls_rlayer_buffer_record(OSSL_RECORD_LAYER *rl, struct pqueue_st *qu
         return -1;
     }

-    rdata->packet = rl->packet;
+    /*
+     * Take a copy of just this record's on-wire bytes (header + ciphertext)
+     * rather than the whole (much larger) read buffer. The live rl->rbuf is
+     * left untouched and continues to be used for subsequent reads.
+     */
     rdata->packet_length = rl->packet_length;
-    memcpy(&(rdata->rbuf), &rl->rbuf, sizeof(TLS_BUFFER));
-    memcpy(&(rdata->rrec), &rl->rrec[0], sizeof(TLS_RL_RECORD));
-
-    item->data = rdata;
-
-    rl->packet = NULL;
-    rl->packet_length = 0;
-    memset(&rl->rbuf, 0, sizeof(TLS_BUFFER));
-    memset(&rl->rrec[0], 0, sizeof(rl->rrec[0]));
-
-    if (!tls_setup_read_buffer(rl)) {
-        /* RLAYERfatal() already called */
-        OPENSSL_free(rdata->rbuf.buf);
+    rdata->packet = OPENSSL_memdup(rl->packet, rl->packet_length);
+    if (rdata->packet == NULL) {
         OPENSSL_free(rdata);
         pitem_free(item);
+        RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB);
         return -1;
     }
+    memcpy(&(rdata->rrec), &rl->rrec[0], sizeof(TLS_RL_RECORD));
+
+    item->data = rdata;

     if (pqueue_insert(queue, item) == NULL) {
         /* Must be a duplicate so ignore it */
-        OPENSSL_free(rdata->rbuf.buf);
+        OPENSSL_free(rdata->packet);
         OPENSSL_free(rdata);
         pitem_free(item);
     }
@@ -607,7 +604,7 @@ static int dtls_free(OSSL_RECORD_LAYER *rl)
             /* Push to the next record layer */
             ret &= BIO_write_ex(rl->next, rdata->packet, rdata->packet_length,
                 &written);
-            OPENSSL_free(rdata->rbuf.buf);
+            OPENSSL_free(rdata->packet);
             OPENSSL_free(item->data);
             pitem_free(item);
         }