Commit 2639a3b4e for llama.cpp
commit 2639a3b4e029d1be80ca4a472d62dfea23609075
Author: Pascal <admin@serveurperso.com>
Date: Thu Oct 15 06:30:17 2026 +0200
server: save only restorable slot states to the prompt cache (#30498)
* server: save only restorable slot states to the prompt cache
The idle-slot save loop stores every idle slot whenever a task starts,
while only completion tasks restore from the prompt cache. Rerank,
embedding, decision and infill states fill the cache up to its limit
without ever being read back, and slots already cached are offered
again at every task start.
Each slot now tracks with cache_pending whether its state is restorable
and missing from the cache: a completion release or an explicit slot
restore sets it, prompt_save consumes it, prompt_clear resets it. Both
save paths share this rule, and the LRU path restores only for
completion tasks.
* server: test that decision slots skip the prompt cache
Decision tasks never mark their slot as cacheable, so starting new
tasks must not save their idle slots to the RAM prompt cache, with or
without a unified KV cache.
* server: restore any reusing task from the prompt cache
Embedding and rerank tasks never reuse their state, so their slots are
cleared on release like child slots and the cache only holds states
that a later task can restore. Infill and decision tasks now restore
from the cache too, which replaces the cache_pending flag.
Decision models disable the prompt cache unless --cache-ram is given,
since their states seldom repeat.
diff --git a/common/arg.cpp b/common/arg.cpp
index 29f6df32f..8b302d5d7 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -1717,10 +1717,11 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
).set_env("LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT").set_examples({LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"-cram", "--cache-ram"}, "N",
- string_format("set the maximum cache size in MiB (default: %d, -1 - no limit, 0 - disable)"
+ string_format("set the maximum cache size in MiB (default: %d, 0 for decision models, -1 - no limit, 0 - disable)"
"[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)", params.cache_ram_mib),
[](common_params & params, int value) {
params.cache_ram_mib = value;
+ params.cache_ram_explicit = true;
}
).set_env("LLAMA_ARG_CACHE_RAM").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
add_opt(common_arg(
diff --git a/common/common.h b/common/common.h
index 2048d6a79..2cd6e2ec9 100644
--- a/common/common.h
+++ b/common/common.h
@@ -640,6 +640,7 @@ struct common_params {
int32_t kv_unified_per_slot = 0; // max context per parallel slot; 0 = unset
int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints
int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
+ bool cache_ram_explicit = false; // set by --cache-ram, decision models disable the cache otherwise
std::string public_path = ""; // NOLINT
std::string api_prefix = ""; // NOLINT
diff --git a/tools/cli/README.md b/tools/cli/README.md
index 163774119..be8fb09d2 100644
--- a/tools/cli/README.md
+++ b/tools/cli/README.md
@@ -148,7 +148,7 @@
| `--display-prompt, --no-display-prompt` | whether to print prompt at generation (default: true) |
| `-co, --color [on\|off\|auto]` | Colorize output to distinguish prompt and user input from generations ('on', 'off', or 'auto', default: 'auto')<br/>'auto' enables colors when output is to a terminal |
| `-ctxcp, --ctx-checkpoints, --swa-checkpoints N` | max number of context checkpoints to create per slot (default: 32)[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293)<br/>(env: LLAMA_ARG_CTX_CHECKPOINTS) |
-| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)<br/>(env: LLAMA_ARG_CACHE_RAM) |
+| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, 0 for decision models, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)<br/>(env: LLAMA_ARG_CACHE_RAM) |
| `--context-shift, --no-context-shift` | whether to use context shift on infinite text generation (default: disabled)<br/>(env: LLAMA_ARG_CONTEXT_SHIFT) |
| `-sys, --system-prompt PROMPT` | system prompt to use with model (if applicable, depending on chat template) |
| `--show-timings, --no-show-timings` | whether to show timing information after each response (default: true)<br/>(env: LLAMA_ARG_SHOW_TIMINGS) |
diff --git a/tools/server/README.md b/tools/server/README.md
index 37c41f1bb..4e4472ec2 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -165,7 +165,7 @@ For the full list of features, please refer to [server's changelog](https://gith
| `--kv-unified-per-slot N` | context limit per parallel slot (default: unset, behavior unchanged).<br/>when set without -c/--ctx-size, the shared KV pool is sized to n_parallel*N<br/>(env: LLAMA_ARG_KV_UNIFIED_PER_SLOT) |
| `-ctxcp, --ctx-checkpoints, --swa-checkpoints N` | max number of context checkpoints to create per slot (default: 32)[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293)<br/>(env: LLAMA_ARG_CTX_CHECKPOINTS) |
| `-cms, --checkpoint-min-step N` | minimum spacing between context checkpoints in tokens (default: 8192, 0 = no minimum)<br/>(env: LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT) |
-| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)<br/>(env: LLAMA_ARG_CACHE_RAM) |
+| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, 0 for decision models, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)<br/>(env: LLAMA_ARG_CACHE_RAM) |
| `-kvu, --kv-unified, -no-kvu, --no-kv-unified` | use single unified KV buffer shared across all sequences (default: enabled if number of slots is auto)<br/>(env: LLAMA_ARG_KV_UNIFIED) |
| `--cache-idle-slots, --no-cache-idle-slots` | save idle slots to the prompt cache on new task, and clear them when using unified KV (default: enabled, requires cache-ram)<br/>(env: LLAMA_ARG_CACHE_IDLE_SLOTS) |
| `--context-shift, --no-context-shift` | whether to use context shift on infinite text generation (default: disabled)<br/>(env: LLAMA_ARG_CONTEXT_SHIFT) |
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 60c563345..d5b4a7029 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -594,7 +594,8 @@ struct server_slot {
state = SLOT_STATE_IDLE;
// do not keep context of the child slots - the parent's context is enough
- if (task->is_child()) {
+ // nor of the stateless slots - no task ever reuses it
+ if (task->is_child() || task->is_stateless()) {
prompt_clear();
}
@@ -1489,6 +1490,13 @@ private:
batch.init(ctx_tgt, std::max(n_batch, params_base.n_parallel), n_embd);
}
+ // decision requests rarely repeat a state, the prompt cache is opt-in for decision models
+ if (decision.type != COMMON_DECISION_TYPE_NONE && !params_base.cache_ram_explicit) {
+ SRV_INF("%s", "decision model: prompt cache disabled by default, use `--cache-ram N` to enable it\n");
+ params_base.cache_ram_mib = 0;
+ params_base.cache_idle_slots = false;
+ }
+
if (params_base.cache_ram_mib != 0) {
if (params_base.cache_ram_mib < 0) {
SRV_TRC("prompt cache is enabled, size limit: %s\n", "no limit");
@@ -1782,8 +1790,8 @@ private:
if (ret) {
update_cache = update_cache && prompt_cache;
- // cache prompts only for completion tasks
- update_cache = update_cache && task.type == SERVER_TASK_TYPE_COMPLETION;
+ // cache prompts only for the tasks that reuse them
+ update_cache = update_cache && !task.is_stateless();
if (update_cache) {
SRV_TRC("%s", "updating prompt cache\n");
@@ -3597,9 +3605,7 @@ private:
return;
}
- const bool is_stateless_task = slot.task->type == SERVER_TASK_TYPE_EMBEDDING || slot.task->type == SERVER_TASK_TYPE_RERANK;
-
- if (slot.task->params.cache_prompt && !is_stateless_task) {
+ if (slot.task->params.cache_prompt && !slot.task->is_stateless()) {
// reuse any previously computed tokens that are common with the new prompt
n_past = slot.prompt.tokens.get_common_prefix(input_tokens);
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index 50926ec3c..e7bb04398 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -300,6 +300,11 @@ struct server_task {
bool is_child() const {
return id_parent != -1;
}
+
+ // embedding and rerank tasks always process their whole prompt, their state is never reused
+ bool is_stateless() const {
+ return type == SERVER_TASK_TYPE_EMBEDDING || type == SERVER_TASK_TYPE_RERANK;
+ }
};
struct result_prompt_progress {
diff --git a/tools/server/tests/unit/test_rerank.py b/tools/server/tests/unit/test_rerank.py
index ded826710..53f15af09 100644
--- a/tools/server/tests/unit/test_rerank.py
+++ b/tools/server/tests/unit/test_rerank.py
@@ -41,6 +41,31 @@ def test_rerank():
assert least_relevant["index"] == 3
+@pytest.mark.parametrize("kv_unified", [False, True])
+def test_rerank_skips_prompt_cache(kv_unified, tmp_path):
+ global server
+ server.n_slots = 2
+ server.cache_ram = 100
+ server.kv_unified = kv_unified
+ server.debug = True
+ server.log_path = str(tmp_path / "server.log")
+ server.start()
+
+ for i in range(4):
+ res = server.make_request("POST", "/rerank", data={
+ "query": f"query {i}",
+ "documents": TEST_DOCUMENTS[i:],
+ })
+ assert res.status_code == 200
+
+ server.stop()
+
+ with open(server.log_path) as f:
+ log = f.read()
+ assert "__TEST_TAG_CACHE_IDLE_SLOTS_ENABLED__" in log
+ assert "__TEST_TAG_CACHE_IDLE_SLOT__" not in log
+
+
def test_rerank_tei_format():
global server
server.start()
diff --git a/tools/server/tests/unit/test_systemone.py b/tools/server/tests/unit/test_systemone.py
index d9d988410..61c2ab098 100644
--- a/tools/server/tests/unit/test_systemone.py
+++ b/tools/server/tests/unit/test_systemone.py
@@ -158,6 +158,32 @@ def test_systemone_shared_prompt():
assert abs(res.body["answers"]["angry"]["noul"] - res_single.body["answers"]["angry"]["noul"]) < 0.01
+@pytest.mark.parametrize("cache_ram", [None, 100])
+def test_systemone_prompt_cache(cache_ram, tmp_path):
+ global server
+ server.n_slots = 1
+ server.cache_ram = cache_ram
+ server.debug = True
+ server.log_path = str(tmp_path / "server.log")
+ server.start()
+
+ # the third request comes back to the state of the first one
+ for state in ["Ticket A: " + TEST_STATE, "Ticket B: " + TEST_STATE * 2, "Ticket A: " + TEST_STATE]:
+ res = server.make_request("POST", "/v1/systemone", data={
+ "state": state,
+ "questions": TEST_QUESTIONS,
+ })
+ assert res.status_code == 200
+
+ server.stop()
+
+ with open(server.log_path) as f:
+ log = f.read()
+ # the prompt cache is opt-in for decision models, and restores their states once enabled
+ assert ("__TEST_TAG_CACHE_IDLE_SLOTS_ENABLED__" in log) == (cache_ram is not None)
+ assert ("found better prompt" in log) == (cache_ram is not None)
+
+
def test_systemone_images():
global server
server = ServerPreset.tinyopenjev()