Commit 4f5b40c8fb for openssl.org

commit 4f5b40c8fb1b12435c8a564dcbcd2abd393f425d
Author: Jakub Zelenka <jakub.zelenka@openssl.foundation>
Date:   Thu Aug 27 20:48:08 2026 +0200

    quic: collect rejected incoming streams and return their stream credit

    A stream rejected by the incoming stream policy is never placed on the accept
    queue, so it is never retired for the purposes of MAX_STREAMS RXFC and the peer
    is never granted credit for another stream. After the initial limit of 100
    streams has been reached, the peer cannot open any further stream for the
    lifetime of the connection.

    Such a stream is also never garbage collected. Rejecting a stream resets its
    send part, which frees the send stream, and the acknowledgement handler skips a
    chunk whose send stream is already gone before it confirms the STOP_SENDING and
    RESET_STREAM frames. The stream therefore never records that its STOP_SENDING
    was acknowledged, never becomes ready for garbage collection and remains in the
    stream map until the connection is torn down.

    Confirm both frames even when the send stream has already been freed, and retire
    the stream for MAX_STREAMS RXFC when it is rejected. The two changes belong
    together: returning the stream credit on its own would replace a leak bounded by
    the stream limit with an unbounded one.

    Assisted-by: Claude:claude-fable-5
    Reviewed-by: Saša NedvÄ›dický <sashan@openssl.org>
    Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
    Merge-date: Tue Sep  1 14:30:03 2026
    Merged-from: https://github.com/openssl/openssl/pull/32557

diff --git a/include/internal/quic_stream_map.h b/include/internal/quic_stream_map.h
index 36753d7196..179f2c6cc8 100644
--- a/include/internal/quic_stream_map.h
+++ b/include/internal/quic_stream_map.h
@@ -840,6 +840,15 @@ QUIC_STREAM *ossl_quic_stream_map_peek_accept_queue(QUIC_STREAM_MAP *qsm);
 QUIC_STREAM *ossl_quic_stream_map_find_in_accept_queue(QUIC_STREAM_MAP *qsm,
     int is_uni);

+/*
+ * Retires an incoming stream for the purposes of MAX_STREAMS RXFC, so that the
+ * peer is granted credit for another stream. rtt is the estimated connection
+ * RTT. Must be called at most once for a given stream.
+ */
+void ossl_quic_stream_map_retire_stream_credit(QUIC_STREAM_MAP *qsm,
+    QUIC_STREAM *s,
+    OSSL_TIME rtt);
+
 /*
  * Removes a stream from the accept queue. rtt is the estimated connection RTT.
  * The stream is retired for the purposes of MAX_STREAMS RXFC.
diff --git a/ssl/quic/quic_channel.c b/ssl/quic/quic_channel.c
index 8e7292f193..dc6b3342e2 100644
--- a/ssl/quic/quic_channel.c
+++ b/ssl/quic/quic_channel.c
@@ -4023,6 +4023,8 @@ void ossl_quic_channel_set_incoming_stream_auto_reject(QUIC_CHANNEL *ch,

 void ossl_quic_channel_reject_stream(QUIC_CHANNEL *ch, QUIC_STREAM *qs)
 {
+    OSSL_RTT_INFO rtt_info;
+
     ossl_quic_stream_map_stop_sending_recv_part(&ch->qsm, qs,
         ch->incoming_stream_auto_reject_aec);

@@ -4030,6 +4032,15 @@ void ossl_quic_channel_reject_stream(QUIC_CHANNEL *ch, QUIC_STREAM *qs)
         ch->incoming_stream_auto_reject_aec);
     qs->deleted = 1;

+    /*
+     * A rejected stream is never placed on the accept queue, so it would
+     * otherwise never be retired and would consume the peer's stream credit
+     * for the lifetime of the connection.
+     */
+    ossl_statm_get_rtt_info(ossl_quic_channel_get_statm(ch), &rtt_info);
+    ossl_quic_stream_map_retire_stream_credit(&ch->qsm, qs,
+        rtt_info.smoothed_rtt);
+
     ossl_quic_stream_map_update_state(&ch->qsm, qs);
 }

diff --git a/ssl/quic/quic_fifd.c b/ssl/quic/quic_fifd.c
index e80483b501..b8d31f4e29 100644
--- a/ssl/quic/quic_fifd.c
+++ b/ssl/quic/quic_fifd.c
@@ -78,17 +78,21 @@ static void on_acked(void *arg)
         sstream = fifd->get_sstream_by_id(chunks[i].stream_id,
             pkt->ackm_pkt.pkt_space,
             fifd->get_sstream_by_id_arg);
-        if (sstream == NULL)
-            continue;

-        if (chunks[i].end >= chunks[i].start)
-            /* coverity[check_return]: Best effort - we cannot fail here. */
-            ossl_quic_sstream_mark_acked(sstream,
-                chunks[i].start, chunks[i].end);
+        if (sstream != NULL) {
+            if (chunks[i].end >= chunks[i].start)
+                /* coverity[check_return]: Best effort - we cannot fail here. */
+                ossl_quic_sstream_mark_acked(sstream,
+                    chunks[i].start, chunks[i].end);

-        if (chunks[i].has_fin && chunks[i].stream_id != UINT64_MAX)
-            ossl_quic_sstream_mark_acked_fin(sstream);
+            if (chunks[i].has_fin && chunks[i].stream_id != UINT64_MAX)
+                ossl_quic_sstream_mark_acked_fin(sstream);
+        }

+        /*
+         * Resetting the send part frees the send stream, so these must be
+         * confirmed even when it is already gone.
+         */
         if (chunks[i].has_stop_sending && chunks[i].stream_id != UINT64_MAX)
             fifd->confirm_frame(OSSL_QUIC_FRAME_TYPE_STOP_SENDING,
                 chunks[i].stream_id, pkt,
@@ -99,7 +103,7 @@ static void on_acked(void *arg)
                 chunks[i].stream_id, pkt,
                 fifd->confirm_frame_arg);

-        if (ossl_quic_sstream_is_totally_acked(sstream))
+        if (sstream != NULL && ossl_quic_sstream_is_totally_acked(sstream))
             fifd->sstream_updated(chunks[i].stream_id, fifd->sstream_updated_arg);
     }

diff --git a/ssl/quic/quic_stream_map.c b/ssl/quic/quic_stream_map.c
index f707bf71d8..40a979b6d9 100644
--- a/ssl/quic/quic_stream_map.c
+++ b/ssl/quic/quic_stream_map.c
@@ -791,20 +791,27 @@ static QUIC_RXFC *qsm_get_max_streams_rxfc(QUIC_STREAM_MAP *qsm, QUIC_STREAM *s)
         : qsm->max_streams_uni_rxfc;
 }

-void ossl_quic_stream_map_remove_from_accept_queue(QUIC_STREAM_MAP *qsm,
+void ossl_quic_stream_map_retire_stream_credit(QUIC_STREAM_MAP *qsm,
     QUIC_STREAM *s,
     OSSL_TIME rtt)
 {
     QUIC_RXFC *max_streams_rxfc;

+    if ((max_streams_rxfc = qsm_get_max_streams_rxfc(qsm, s)) != NULL)
+        (void)ossl_quic_rxfc_on_retire(max_streams_rxfc, 1, rtt);
+}
+
+void ossl_quic_stream_map_remove_from_accept_queue(QUIC_STREAM_MAP *qsm,
+    QUIC_STREAM *s,
+    OSSL_TIME rtt)
+{
     list_remove(&qsm->accept_list, &s->accept_node);
     if (ossl_quic_stream_is_bidi(s))
         --qsm->num_accept_bidi;
     else
         --qsm->num_accept_uni;

-    if ((max_streams_rxfc = qsm_get_max_streams_rxfc(qsm, s)) != NULL)
-        (void)ossl_quic_rxfc_on_retire(max_streams_rxfc, 1, rtt);
+    ossl_quic_stream_map_retire_stream_credit(qsm, s, rtt);
 }

 size_t ossl_quic_stream_map_get_accept_queue_len(QUIC_STREAM_MAP *qsm, int is_uni)
diff --git a/test/quicapitest.c b/test/quicapitest.c
index 8dc36b8a22..9c2f27ef77 100644
--- a/test/quicapitest.c
+++ b/test/quicapitest.c
@@ -3611,6 +3611,86 @@ err:
     return testresult;
 }

+/*
+ * Streams rejected by the incoming stream policy are never placed on the accept
+ * queue. Check that they are still garbage collected and that the peer is
+ * granted credit for another stream, so that a peer which keeps opening streams
+ * neither grows the stream map without bound nor exhausts its stream limit.
+ */
+static int test_reject_stream_gc(void)
+{
+    /* Comfortably more than the default initial stream limit of 100. */
+    static const int num_streams = 250;
+    SSL_CTX *cctx = NULL, *sctx = NULL;
+    SSL *clientssl = NULL, *serverssl = NULL, *qlistener = NULL;
+    SSL *streamssl = NULL;
+    QUIC_CHANNEL *ch;
+    QUIC_STREAM_MAP *qsm;
+    size_t written = 0;
+    int testresult = 0, ret, i;
+
+    if (!TEST_ptr(sctx = create_server_ctx())
+        || !TEST_ptr(cctx = create_client_ctx())
+        || !create_quic_ssl_objects(sctx, cctx, &qlistener, &clientssl))
+        goto err;
+
+    for (i = 0; i < 2; i++) {
+        ret = SSL_connect(clientssl);
+        if (!TEST_int_le(ret, 0)
+            || !TEST_int_eq(SSL_get_error(clientssl, ret),
+                SSL_ERROR_WANT_READ))
+            goto err;
+        SSL_handle_events(qlistener);
+    }
+
+    if (!TEST_ptr(serverssl = SSL_accept_connection(qlistener, 0))
+        || !TEST_true(create_bare_ssl_connection(serverssl, clientssl,
+            SSL_ERROR_NONE, 0, 0))
+        || !TEST_true(SSL_set_incoming_stream_policy(clientssl,
+            SSL_INCOMING_STREAM_POLICY_REJECT, 42))
+        || !TEST_ptr(ch = ossl_quic_conn_get_channel(clientssl)))
+        goto err;
+
+    qsm = ossl_quic_channel_get_qsm(ch);
+
+    for (i = 0; i < num_streams; i++) {
+        if (!TEST_ptr(streamssl = SSL_new_stream(serverssl, 0))
+            || !TEST_true(SSL_write_ex(streamssl, "x", 1, &written)))
+            goto err;
+        SSL_free(streamssl);
+        streamssl = NULL;
+
+        /*
+         * Let the client reject the stream and the server pick up both the
+         * resulting frames and the MAX_STREAMS credit they release.
+         */
+        if (!TEST_int_eq(SSL_handle_events(clientssl), 1)
+            || !TEST_int_eq(SSL_handle_events(serverssl), 1)
+            || !TEST_int_eq(SSL_handle_events(clientssl), 1))
+            goto err;
+    }
+
+    /*
+     * Every rejected stream should have been collected by now, so the map must
+     * not have grown in proportion to the number of streams opened.
+     */
+    if (!TEST_size_t_lt(OPENSSL_LH_num_items((OPENSSL_LHASH *)qsm->map),
+            (size_t)num_streams / 10)
+        || !TEST_size_t_eq(SSL_get_accept_stream_queue_len(clientssl), 0))
+        goto err;
+
+    testresult = 1;
+err:
+    SSL_free(streamssl);
+    SSL_free(serverssl);
+    SSL_free(clientssl);
+    SSL_free(qlistener);
+    SSL_CTX_free(sctx);
+    SSL_CTX_free(cctx);
+
+    return testresult;
+}
+
 /*
  * When the server has a different primary group than the client, the server
  * should not fail on the client hello retry.
@@ -4405,6 +4485,7 @@ int setup_tests(void)
     ADD_TEST(test_ssl_accept_connection);
     ADD_TEST(test_ssl_set_verify);
     ADD_TEST(test_accept_stream);
+    ADD_TEST(test_reject_stream_gc);
     ADD_TEST(test_client_hello_retry);
 #if OPENSSL_USE_IPV6
     ADD_TEST(test_quic_peer_addr_v6);