Commit b0d239164 for llama.cpp
commit b0d239164b26adcd317be1f0cffd2a1294f6ba60
Author: Pascal <admin@serveurperso.com>
Date: Thu Oct 15 15:24:01 2026 +0200
server: prefer empty slots and keep states overwritten by child tasks (#30567)
Slot selection takes free slots by id or by last use only, so child
tasks often land on a slot holding a parent state while empty slots
are idle, and overwrite it without saving it to the prompt cache.
Free slots are now ordered empty first, then least recently used, for
both the LRU fallback and the child slots, and a child task saves the
state it overwrites. With 4 grouped decision requests in flight and
12 states in rotation, prompt cache reads go from 5 to 36.
[no release]
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index d5b4a7029..c1683d9ef 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -1763,25 +1763,14 @@ private:
}
}
- // find the slot that has been least recently used
+ // find an empty slot, or the one that has been least recently used
if (ret == nullptr) {
- int64_t t_last = -1;
+ const auto free_slots = get_free_slots(1, -1);
- for (server_slot & slot : slots) {
- // skip the slot if it is not available
- if (slot.is_processing()) {
- continue;
- }
+ if (!free_slots.empty()) {
+ ret = free_slots[0];
- // select the current slot if the criteria match
- if (!ret || slot.t_last_used <= t_last) {
- t_last = slot.t_last_used;
- ret = &slot;
- }
- }
-
- if (ret != nullptr) {
- SLT_INF(*ret, "selected slot by LRU, t_last = %" PRId64 "\n", t_last);
+ SLT_INF(*ret, "selected slot by LRU, n_tokens = %d, t_last = %" PRId64 "\n", ret->prompt.n_tokens(), ret->t_last_used);
update_cache = true;
}
@@ -2480,16 +2469,24 @@ private:
return true;
}
+ // idle slots that lose the least by being reused: empty slots first, then the least recently used ones
std::vector<server_slot *> get_free_slots(size_t n_slots_needed, int exclude_id_slot) {
std::vector<server_slot *> free_slots;
for (auto & slot : slots) {
if (!slot.is_processing() && slot.id != exclude_id_slot) {
free_slots.push_back(&slot);
}
- if (free_slots.size() >= n_slots_needed) {
- break;
- }
}
+
+ std::stable_sort(free_slots.begin(), free_slots.end(), [](const server_slot * a, const server_slot * b) {
+ if ((a->prompt.n_tokens() == 0) != (b->prompt.n_tokens() == 0)) {
+ return a->prompt.n_tokens() == 0;
+ }
+ return a->t_last_used < b->t_last_used;
+ });
+
+ free_slots.resize(std::min(free_slots.size(), n_slots_needed));
+
return free_slots;
}
@@ -2519,6 +2516,11 @@ private:
size_t idx = 0;
GGML_ASSERT(child_slots.size() == parent_task.child_tasks.size());
for (auto * slot : child_slots) {
+ // keep the state that the child task is about to overwrite
+ if (prompt_cache && slot->prompt_save(*prompt_cache)) {
+ prompt_cache->update();
+ }
+
int id_child = parent_task.child_tasks[idx].id;
if (!launch_slot_with_task(*slot, std::move(parent_task.child_tasks[idx]))) {
SRV_ERR("failed to launch slot with child task, id_task = %d\n", id_child);
diff --git a/tools/server/tests/unit/test_systemone.py b/tools/server/tests/unit/test_systemone.py
index 61c2ab098..67e128e11 100644
--- a/tools/server/tests/unit/test_systemone.py
+++ b/tools/server/tests/unit/test_systemone.py
@@ -184,6 +184,44 @@ def test_systemone_prompt_cache(cache_ram, tmp_path):
assert ("found better prompt" in log) == (cache_ram is not None)
+def test_systemone_prompt_cache_child_slots(tmp_path):
+ global server
+ server = ServerPreset.tinyopenjev()
+ server.n_slots = len(TEST_QUESTIONS)
+ server.cache_ram = 100
+ server.debug = True
+ server.log_path = str(tmp_path / "server.log")
+ server.start()
+
+ states = [
+ "My parcel shows delivered but it never arrived at my door.",
+ "The app crashes every time I open the settings page.",
+ "Please close my account and delete all my personal data.",
+ ]
+ angry = {"angry": TEST_QUESTIONS["angry"]}
+
+ def decide(state, questions):
+ res = server.make_request("POST", "/v1/systemone", data={"state": state, "questions": questions})
+ assert res.status_code == 200
+
+ # one state per slot, then a request whose child tasks take every slot
+ for state in states:
+ decide(state, angry)
+ decide(TEST_STATE, TEST_QUESTIONS)
+
+ with open(server.log_path) as f:
+ n_restored = f.read().count("found better prompt with f_keep = 1.000")
+
+ # every overwritten state comes back whole from the prompt cache
+ for state in states:
+ decide(state, angry)
+
+ server.stop()
+
+ with open(server.log_path) as f:
+ assert f.read().count("found better prompt with f_keep = 1.000") == n_restored + len(states)
+
+
def test_systemone_images():
global server
server = ServerPreset.tinyopenjev()