Commit 095668452 for llama.cpp
commit 0956684529271e802c740a208f97c505d62bea26
Author: Harkanwal Bedi <bedi1887@gmail.com>
Date: Sat Nov 7 03:03:38 2026 -0800
server: visit parent slots before child slots (#30779)
* server: visit parent slots before child slots
A parent slot wakes its waiting child slots once it finishes the shared
prompt prefix, while slots are visited in id order. When a child task
lands on a lower slot id than its parent, the child is visited before
the parent wakes it and waits an extra batch to process its suffix.
Non-child slots are now visited before active child slots, so every
child woken by its parent joins the same batch.
* server: check systemone test events through test tags
The systemone tests matched free-form debug messages, including the
formatted f_keep value of the prompt cache. The server now emits
__TEST_TAG_WAIT_PARENT_SLOT__ when a child slot waits for its parent,
__TEST_TAG_PROMPT_CACHE_RESTORE__ on every prompt cache restore and
__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__ when the whole cached prompt is
a prefix of the new one, compared on token counts.
---------
Co-authored-by: Pascal <admin@serveurperso.com>
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 4596e788b..78c8ef149 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -3424,13 +3424,19 @@ private:
}
void iterate(std::vector<server_slot> & slots, std::function<void(server_slot &)> callback) {
- for (auto & slot : slots) {
- try {
- callback(slot);
- } catch (const std::exception & e) {
- SLT_ERR(slot, "got exception: %s\n", e.what());
- send_error(slot, std::string("got exception: ") + e.what(), ERROR_TYPE_SERVER);
- slot.release();
+ for (int pass = 0; pass < 2; ++pass) {
+ for (auto & slot : slots) {
+ const bool is_child = slot.is_processing() && slot.task->is_child();
+ if (is_child != (pass == 1)) {
+ continue;
+ }
+ try {
+ callback(slot);
+ } catch (const std::exception & e) {
+ SLT_ERR(slot, "got exception: %s\n", e.what());
+ send_error(slot, std::string("got exception: ") + e.what(), ERROR_TYPE_SERVER);
+ slot.release();
+ }
}
}
}
@@ -4007,6 +4013,7 @@ private:
// check if this is a child slot
if (slot.state == SLOT_STATE_WAIT_OTHER) {
SLT_DBG(slot, "%s", "waiting for parent slot to complete\n");
+ SLT_DBG(slot, "%s", "__TEST_TAG_WAIT_PARENT_SLOT__\n");
return;
}
diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
index 1592330a7..4b2a8b2f7 100644
--- a/tools/server/server-task.cpp
+++ b/tools/server/server-task.cpp
@@ -1836,7 +1836,7 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro
}
bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) {
- const int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
+ int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins
float f_sim_best = float(lcp_best) / tokens_new.size();
@@ -1862,6 +1862,7 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) {
f_keep_best = f_keep_cur;
f_sim_best = f_sim_cur;
+ lcp_best = lcp_cur;
it_best = it;
}
@@ -1904,6 +1905,13 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
}
}
+ SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_RESTORE__\n");
+
+ // the whole cached prompt is a prefix of the new one
+ if (lcp_best == (int) it_best->prompt.tokens.size()) {
+ SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__\n");
+ }
+
prompt = std::move(it_best->prompt);
states.erase(it_best);
diff --git a/tools/server/tests/unit/test_systemone.py b/tools/server/tests/unit/test_systemone.py
index 67e128e11..0c94a06d1 100644
--- a/tools/server/tests/unit/test_systemone.py
+++ b/tools/server/tests/unit/test_systemone.py
@@ -122,10 +122,13 @@ def test_systemone_invalid_request(data: dict):
assert "error" in res.body
-def test_systemone_shared_prompt():
+def test_systemone_shared_prompt(tmp_path):
global server
server = ServerPreset.tinyopenjev()
+ server.n_slots = len(TEST_QUESTIONS)
server.server_metrics = True
+ server.debug = True
+ server.log_path = str(tmp_path / "server.log")
server.start()
res = server.make_request("POST", "/v1/systemone", data={
"state": TEST_STATE,
@@ -139,8 +142,19 @@ def test_systemone_shared_prompt():
assert n_cached % 2 == 0
assert n_processed + n_cached == res.body["usage"]["input_tokens"]
- # with one slot the prompt cannot be shared, the answers must be the same
+ # the next request takes an empty slot as parent, so child slot 0 has a lower id than the parent
+ res_next = server.make_request("POST", "/v1/systemone", data={
+ "state": "Unrelated follow-up case: " * 10,
+ "questions": TEST_QUESTIONS,
+ })
+ assert res_next.status_code == 200
+
+ # in each request, each child waits for the parent during 1 batch only
server.stop()
+ with open(server.log_path) as f:
+ assert f.read().count("__TEST_TAG_WAIT_PARENT_SLOT__") == 2 * (len(TEST_QUESTIONS) - 1)
+
+ # with one slot the prompt cannot be shared, the answers must be the same
server = ServerPreset.tinyopenjev()
server.n_slots = 1
server.start()
@@ -181,7 +195,7 @@ def test_systemone_prompt_cache(cache_ram, tmp_path):
log = f.read()
# the prompt cache is opt-in for decision models, and restores their states once enabled
assert ("__TEST_TAG_CACHE_IDLE_SLOTS_ENABLED__" in log) == (cache_ram is not None)
- assert ("found better prompt" in log) == (cache_ram is not None)
+ assert ("__TEST_TAG_PROMPT_CACHE_RESTORE__" in log) == (cache_ram is not None)
def test_systemone_prompt_cache_child_slots(tmp_path):
@@ -210,7 +224,7 @@ def test_systemone_prompt_cache_child_slots(tmp_path):
decide(TEST_STATE, TEST_QUESTIONS)
with open(server.log_path) as f:
- n_restored = f.read().count("found better prompt with f_keep = 1.000")
+ n_restored = f.read().count("__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__")
# every overwritten state comes back whole from the prompt cache
for state in states:
@@ -219,7 +233,7 @@ def test_systemone_prompt_cache_child_slots(tmp_path):
server.stop()
with open(server.log_path) as f:
- assert f.read().count("found better prompt with f_keep = 1.000") == n_restored + len(states)
+ assert f.read().count("__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__") == n_restored + len(states)
def test_systemone_images():