Commit 09452cded for llama.cpp

commit 09452cdeddfff66d16b8ac81a6c66291215448b4
Author: Pascal <admin@serveurperso.com>
Date:   Thu Nov 12 12:16:16 2026 +0100

    server: keep the server awake while a request uses its state (#32133)

    With --sleep-idle-seconds, a handler passes wait_until_no_sleep()
    while the server is awake, then keeps using the server state, for
    example to parse the body and tokenize. If the idle timer expires in
    that window, the main loop enters sleep and the callbacks destroy the
    model: the handler reads freed vocab memory, and a task posted after
    that stays in the queue with nobody awake to process it.

    Count the requests that passed wait_until_no_sleep() and are not done
    yet, and do not enter sleep while the count is positive. The response
    generator releases its count in its destructor, so every return path
    of a handler gives it back.

    Co-authored-by: Venkata Vineel <vyalamar@users.noreply.github.com>

diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 5d9726796..effa5bbda 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -5302,12 +5302,19 @@ server_context_meta server_context::get_meta() const {
 // may have bypass_sleep = true if the task does not use ctx_server
 struct server_res_generator : server_res_spipe {
     server_response_reader rd;
+    server_queue * queue_awake = nullptr; // set while this response keeps the server awake
     server_res_generator(server_queue & queue_tasks, server_response & queue_results, int sleep_idle_seconds, bool bypass_sleep = false)
             : rd(queue_tasks, queue_results, HTTP_POLLING_SECONDS) {
         // fast path in case sleeping is disabled
         bypass_sleep |= sleep_idle_seconds < 0;
         if (!bypass_sleep) {
             queue_tasks.wait_until_no_sleep();
+            queue_awake = &queue_tasks;
+        }
+    }
+    ~server_res_generator() {
+        if (queue_awake) {
+            queue_awake->release_inflight();
         }
     }
     void ok(const json & response_data) {
diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cpp
index c4ce90721..75daebb0d 100644
--- a/tools/server/server-queue.cpp
+++ b/tools/server/server-queue.cpp
@@ -114,9 +114,7 @@ void server_queue::pop_deferred_task(int id_slot) {

 void server_queue::wait_until_no_sleep() {
     std::unique_lock<std::mutex> lock(mutex_tasks);
-    if (!sleeping) {
-        return;
-    } else {
+    if (sleeping) {
         if (!req_stop_sleeping) {
             QUE_DBG("%s", "requesting to stop sleeping\n");
             req_stop_sleeping = true;
@@ -127,6 +125,13 @@ void server_queue::wait_until_no_sleep() {
             return !sleeping;
         });
     }
+    n_inflight++;
+}
+
+void server_queue::release_inflight() {
+    std::unique_lock<std::mutex> lock(mutex_tasks);
+    GGML_ASSERT(n_inflight > 0);
+    n_inflight--;
 }

 void server_queue::terminate() {
@@ -289,7 +294,7 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
     constexpr auto max_wait_time = std::chrono::seconds(1);
     auto should_sleep = [&]() -> bool {
         // caller must hold mutex_tasks
-        if (idle_sleep_ms < 0) {
+        if (idle_sleep_ms < 0 || n_inflight > 0) {
             return false;
         }
         int64_t now = ggml_time_ms();
diff --git a/tools/server/server-queue.h b/tools/server/server-queue.h
index 692255e4f..f9d66be64 100644
--- a/tools/server/server-queue.h
+++ b/tools/server/server-queue.h
@@ -19,6 +19,7 @@ private:
     bool sleeping = false;
     bool req_stop_sleeping = false;
     int64_t time_last_task = 0;
+    int n_inflight = 0; // requests that keep the server awake, see wait_until_no_sleep()

     // queues
     std::deque<server_task> queue_tasks;
@@ -67,8 +68,12 @@ public:

     // if sleeping, request exiting sleep state and wait until it is done
     // returns immediately if not sleeping
+    // the server then stays awake until the caller calls release_inflight()
     void wait_until_no_sleep();

+    // let the server sleep again once the caller no longer uses the server state
+    void release_inflight();
+
     bool is_sleeping() {
         std::unique_lock<std::mutex> lock(mutex_tasks);
         return sleeping;