Commit 52975342f for llama.cpp

commit 52975342ff6b85127ff31930d39e36725fd08809
Author: Xuan-Son Nguyen <son@huggingface.co>
Date:   Wed Nov 18 00:12:20 2026 +0100

    server: slot_sched: fix lora and recurrent (#32434)

    * server: fix lora and recurrent

    * tests: run the hybrid SWA checkpoint test with one token per ubatch

    With and without the cache, the prompt was not split into the same
    ubatches, and the generated tokens are decoded one at a time while the
    uncached run processes them in a batch. On CUDA this moves the top
    logprobs of the tiny model by up to 0.03 and swaps near ties, so the
    test failed while the cache behaved as expected.

    With one token per ubatch both runs go through the same kernels and
    the output is compared exactly.

    ---------

    Co-authored-by: Pascal <admin@serveurperso.com>

diff --git a/tests/test-server-component.cpp b/tests/test-server-component.cpp
index 2ef8b04f5..bef83e1eb 100644
--- a/tests/test-server-component.cpp
+++ b/tests/test-server-component.cpp
@@ -4,6 +4,7 @@
 #include "../tools/server/server-common.h"
 #include "../tools/server/server-decision.h"
 #include "../tools/server/server-schema.h"
+#include "../tools/server/server-slot-sched.h"

 #include <algorithm>
 #include <cmath>
@@ -330,6 +331,82 @@ static void test_json_helpers(testing & t) {
     });
 }

+//
+// server_prompt
+//
+
+static void test_server_prompt_reuse(testing & t) {
+    // 110 text tokens, the position of a token is its index
+    auto make_prompt = [](std::vector<std::array<llama_pos, 2>> checkpoints) {
+        llama_tokens toks(110);
+        for (size_t i = 0; i < toks.size(); i++) {
+            toks[i] = 100 + (llama_token) i;
+        }
+
+        server_prompt res;
+        res.tokens = server_tokens(toks, false);
+        for (const auto & range : checkpoints) {
+            auto ckpt = std::make_shared<common_prompt_checkpoint>();
+            ckpt->pos_min  = range[0];
+            ckpt->pos_max  = range[1];
+            ckpt->n_tokens = range[1] + 1;
+            res.checkpoints.push_back(std::move(ckpt));
+        }
+        return res;
+    };
+
+    t.test("recurrent", [&](testing & t) {
+        // a recurrent state holds one position: the live state is at 109, n_swa is 0
+        const auto prompt = make_prompt({{95, 95}, {99, 99}, {105, 105}});
+        const llama_pos pos_min = 109;
+
+        auto res = prompt.get_reuse(110, true, 0, pos_min);
+        t.assert_true("new tokens after the prompt, no checkpoint", res.ckpt == nullptr);
+        t.assert_equal("new tokens after the prompt, n_past", 110, res.n_past);
+
+        res = prompt.get_reuse(110, false, 0, pos_min);
+        t.assert_true("same prompt, last checkpoint", res.ckpt && res.ckpt->pos_max == 105);
+        t.assert_equal("same prompt, n_past", 106, res.n_past);
+        t.assert_equal("same prompt, pos_next", 106, res.pos_next);
+
+        // the closest checkpoint before the new tokens is used, the one after them is not
+        res = prompt.get_reuse(104, true, 0, pos_min);
+        t.assert_true("cut at 104, checkpoint", res.ckpt && res.ckpt->pos_max == 99);
+        t.assert_equal("cut at 104, n_past", 100, res.n_past);
+        t.assert_equal("cut at 104, pos_next", 100, res.pos_next);
+
+        // the state at 99 already holds the token at 99, it cannot be used to process it again
+        res = prompt.get_reuse(99, true, 0, pos_min);
+        t.assert_true("cut at 99, checkpoint", res.ckpt && res.ckpt->pos_max == 95);
+        t.assert_equal("cut at 99, n_past", 96, res.n_past);
+
+        res = prompt.get_reuse(50, true, 0, pos_min);
+        t.assert_true("cut at 50, no checkpoint", res.ckpt == nullptr);
+        t.assert_equal("cut at 50, n_past", 0, res.n_past);
+        t.assert_equal("cut at 50, pos_next", 0, res.pos_next);
+    });
+
+    t.test("swa", [&](testing & t) {
+        // n_swa is 8: the live state starts at 102, the checkpoint holds the positions 92 to 99
+        const auto prompt = make_prompt({{92, 99}});
+        const llama_pos pos_min = 102;
+
+        auto res = prompt.get_reuse(110, true, 8, pos_min);
+        t.assert_true("new tokens after the prompt, no checkpoint", res.ckpt == nullptr);
+        t.assert_equal("new tokens after the prompt, n_past", 110, res.n_past);
+
+        // the token at 104 needs the positions 97 to 103, the live state starts too late for a cut at 104
+        res = prompt.get_reuse(104, true, 8, pos_min);
+        t.assert_true("cut at 104, checkpoint", res.ckpt != nullptr);
+        t.assert_equal("cut at 104, n_past", 99, res.n_past);
+        t.assert_equal("cut at 104, pos_next", 99, res.pos_next);
+
+        res = prompt.get_reuse(95, true, 8, pos_min);
+        t.assert_true("cut at 95, no checkpoint", res.ckpt == nullptr);
+        t.assert_equal("cut at 95, n_past", 0, res.n_past);
+    });
+}
+
 static void test_lora_helpers(testing & t) {
     // only pointer identity is used, never dereferenced
     auto * p0 = reinterpret_cast<llama_adapter_lora *>(0x10);
@@ -364,6 +441,13 @@ static void test_lora_helpers(testing & t) {
         t.assert_true("diff size",    !are_lora_equal({make(p0, 1.0f)}, {}));
     });

+    t.test("lora_should_clear_cache", [&](testing & t) {
+        // the adapter pointer is only read when the configs differ
+        t.assert_true("both empty", !lora_should_clear_cache({}, {}));
+        t.assert_true("same",       !lora_should_clear_cache({make(p0, 1.0f)}, {make(p0, 1.0f)}));
+        t.assert_true("to no lora", lora_should_clear_cache({make(p0, 0.0f)}, {}));
+    });
+
     t.test("lora_get_enabled_ids", [&](testing & t) {
         auto ids = lora_get_enabled_ids({make(p0, 0.0f), make(p1, 0.5f), make(p0, -1.0f), make(p1, 2.0f)});
         t.assert_true("ids", ids == std::vector<size_t>({1, 3}));
@@ -869,6 +953,7 @@ int main(int argc, char ** argv) {
     t.test("sse",           test_sse);
     t.test("json_helpers",  test_json_helpers);
     t.test("lora_helpers",  test_lora_helpers);
+    t.test("server_prompt_reuse", test_server_prompt_reuse);
     t.test("server_pipe",   test_server_pipe);
     t.test("decision_escape", test_decision_escape);
     t.test("rerank_template", test_rerank_template);
diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp
index 82c48d2da..eb5781e74 100644
--- a/tools/server/server-common.cpp
+++ b/tools/server/server-common.cpp
@@ -306,10 +306,9 @@ bool lora_should_clear_cache(
         const std::vector<common_adapter_lora_info> & current,
         const std::vector<common_adapter_lora_info> & next) {

-    // This should always be called after determining that the two sets are
-    // _not_ equal. This assert is therefore some slightly wasted work and
-    // should be safe to remove as long as this method is called correctly.
-    GGML_ASSERT(!are_lora_equal(current, next));
+    if (are_lora_equal(current, next)) {
+        return false;
+    }

     return (
         !(lora_get_enabled_ids(current).empty() || lora_all_alora(current)) ||
diff --git a/tools/server/server-common.h b/tools/server/server-common.h
index 82c424d47..911bb31d7 100644
--- a/tools/server/server-common.h
+++ b/tools/server/server-common.h
@@ -143,7 +143,7 @@ json server_model_architecture_json(
 bool lora_all_alora(const std::vector<common_adapter_lora_info> & loras);

 // if the two sets of loras are different, they require a cache clear unless the
-// change is only from aloras to aloras.
+// change is only from aloras to aloras. equal sets never require a clear.
 bool lora_should_clear_cache(
         const std::vector<common_adapter_lora_info> & current,
         const std::vector<common_adapter_lora_info> & next);
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 0d4a9e977..0cdc02284 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -370,8 +370,8 @@ struct server_slot {
         return true;
     }

-    bool prompt_load(server_prompt_cache & prompt_cache, const server_tokens & tokens) {
-        bool res = prompt_cache.load(prompt, tokens, ctx_tgt, ctx_dft, id);
+    bool prompt_load(server_prompt_cache & prompt_cache, const server_tokens & tokens, const std::vector<common_adapter_lora_info> & lora_new) {
+        bool res = prompt_cache.load(prompt, tokens, lora_new, ctx_tgt, ctx_dft, id);
         if (!res) {
             SLT_WRN(*this, "%s", "failed to load prompt from cache\n");
         }
@@ -1108,6 +1108,9 @@ private:
     // if swa_full is enabled, this is set to 0 to simulate a non-SWA model
     int32_t n_swa;

+    // n_swa for server_prompt::get_reuse(): 0 with a recurrent state, which holds one position only
+    int32_t n_swa_reuse;
+
     // slots / clients
     std::vector<server_slot> slots;

@@ -1444,6 +1447,9 @@ private:

         n_swa = params_base.swa_full ? 0 : llama_model_n_swa(model_tgt);

+        // the min position of a hybrid memory is the position of its recurrent state, not the start of the SWA window
+        n_swa_reuse = llama_model_is_recurrent(model_tgt) || llama_model_is_hybrid(model_tgt) ? 0 : n_swa;
+
         // Necessary similarity of prompt for slot selection
         slot_sched.slot_prompt_similarity = params_base.slot_prompt_similarity;

@@ -1824,7 +1830,7 @@ private:

         slot.prompt_save(*prompt_cache);

-        if (!slot.prompt_load(*prompt_cache, task.tokens)) {
+        if (!slot.prompt_load(*prompt_cache, task.tokens, get_task_lora(task))) {
             slot.prompt_clear();
         }

@@ -1878,19 +1884,25 @@ private:
         return output;
     }

+    // per-request lora if given, otherwise the global config
+    std::vector<common_adapter_lora_info> get_task_lora(const server_task & task) const {
+        return task.params.lora.empty() ? params_base.lora_adapters : construct_lora_list(task.params.lora);
+    }
+
     bool launch_slot_with_task(server_slot & slot, server_task && task) {
-        // per-request lora if given, otherwise the global config
-        auto task_loras = task.params.lora.empty() ? params_base.lora_adapters : construct_lora_list(task.params.lora);
-        if (!are_lora_equal(task_loras, slot.lora)) {
+        auto task_loras = get_task_lora(task);
+
+        // the state can come from the prompt cache, so it is compared with its own lora config, not with the one of the last task
+        if (!are_lora_equal(task_loras, slot.prompt.lora)) {
             // if lora has changed, check to see if the cache should be cleared
-            if (lora_should_clear_cache(slot.lora, task_loras)) {
-                SLT_TRC(slot, "clearing cache for lora change. %zu loras -> %zu loras\n", slot.lora.size(), task_loras.size());
+            if (lora_should_clear_cache(slot.prompt.lora, task_loras)) {
+                SLT_TRC(slot, "clearing cache for lora change. %zu loras -> %zu loras\n", slot.prompt.lora.size(), task_loras.size());
                 slot.prompt.clear();
             } else {
                 SLT_TRC(slot, "keeping cache for alora. %zu target loras\n", task_loras.size());
             }
-            slot.lora = task_loras;
         }
+        slot.lora = task_loras;

         // if using alora, make sure it's only a single one requested and active
         size_t alora_invocation_start = task.tokens.size();
@@ -1940,6 +1952,8 @@ private:
             }
         }

+        slot.prompt.lora = slot.lora;
+
         if (!task.tokens.validate(ctx_tgt)) {
             send_error(task, "Prompt contains invalid tokens", ERROR_TYPE_INVALID_REQUEST);
             return false;
@@ -2956,7 +2970,7 @@ private:

                     const int id_task = task.id;

-                    const auto plan = slot_sched.select(get_slot_infos(), task);
+                    const auto plan = slot_sched.select(get_slot_infos(), task, get_task_lora(task));

                     server_slot * slot = plan.id_slot == -1 ? nullptr : get_slot_by_id(plan.id_slot);

@@ -4186,7 +4200,7 @@ private:
                                     SLT_WRN(slot, "%s\n", st1.str().c_str());
                                 }

-                                auto reuse = slot.prompt.get_reuse(n_past, has_new_tokens, n_swa, pos_min);
+                                auto reuse = slot.prompt.get_reuse(n_past, has_new_tokens, n_swa_reuse, pos_min);

                                 if (reuse.ckpt) {
                                     const auto & cur = *reuse.ckpt;
diff --git a/tools/server/server-slot-sched.cpp b/tools/server/server-slot-sched.cpp
index 5b33292c2..5f8aa5c33 100644
--- a/tools/server/server-slot-sched.cpp
+++ b/tools/server/server-slot-sched.cpp
@@ -86,7 +86,7 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro
     for (auto it = states.begin(); it != states.end(); ++it) {
         const int cur_lcp_len = it->prompt.tokens.get_common_prefix(prompt.tokens);

-        if (cur_lcp_len == (int) prompt.tokens.size()) {
+        if (cur_lcp_len == (int) prompt.tokens.size() && are_lora_equal(it->prompt.lora, prompt.lora)) {
             SRV_TRC("%s", " - prompt is already in the cache, skipping\n");
             return nullptr;
         }
@@ -111,7 +111,7 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro
     for (auto it = states.begin(); it != states.end();) {
         const int len = it->prompt.tokens.get_common_prefix(prompt.tokens);

-        if (len == (int) it->prompt.tokens.size()) {
+        if (len == (int) it->prompt.tokens.size() && are_lora_equal(it->prompt.lora, prompt.lora)) {
             SRV_TRC(" - removing obsolete cached prompt with length %d\n", len);

             it = states.erase(it);
@@ -153,6 +153,7 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro
         /*.prompt =*/ {
             /*.tokens      =*/ prompt.tokens.clone(),
             /*.checkpoints =*/ prompt.checkpoints,
+            /*.lora        =*/ prompt.lora,
         },
         /*.data   =*/ {
             /*.main =*/ std::move(state_data_tgt),
@@ -163,10 +164,13 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro
     return &states.back();
 }

-bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) {
-    int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
+bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, const std::vector<common_adapter_lora_info> & lora_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) {
+    // a prompt made with another lora config is cleared before the new one starts, so it counts as empty
+    const bool is_empty = prompt.tokens.empty() || lora_should_clear_cache(prompt.lora, lora_new);

-    float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins
+    int lcp_best = is_empty ? 0 : prompt.tokens.get_common_prefix(tokens_new);
+
+    float f_keep_best = is_empty ? -1.0f : float(lcp_best) / prompt.tokens.size(); // empty slot: any cache entry wins
     float f_sim_best  = float(lcp_best) / tokens_new.size();

     SRV_TRC(" - looking for better prompt, base f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
@@ -175,6 +179,11 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok

     // find the most similar cached prompt, that would also preserve the most context
     for (auto it = states.begin(); it != states.end(); ++it) {
+        if (lora_should_clear_cache(it->prompt.lora, lora_new)) {
+            SRV_TRC("   - prompt with length %7zu, skipping, other lora config\n", it->prompt.tokens.size());
+            continue;
+        }
+
         const int lcp_cur = it->prompt.tokens.get_common_prefix(tokens_new);

         const float f_keep_cur = float(lcp_cur) / it->prompt.tokens.size();
@@ -314,7 +323,7 @@ std::vector<int> server_slot_sched::get_free_slots(const std::vector<server_slot
     return res;
 }

-server_slot_plan server_slot_sched::select(const std::vector<server_slot_info> & slots, const server_task & task) const {
+server_slot_plan server_slot_sched::select(const std::vector<server_slot_info> & slots, const server_task & task, const std::vector<common_adapter_lora_info> & lora) const {
     const server_slot_info * ret = nullptr;

     bool update_cache = false;
@@ -364,6 +373,12 @@ server_slot_plan server_slot_sched::select(const std::vector<server_slot_info> &
                 continue;
             }

+            // the task would clear this slot, its state cannot be reused with another lora config
+            if (lora_should_clear_cache(slot.prompt->lora, lora)) {
+                SLT_TRC(slot, "%s", " - skipping, other lora config\n");
+                continue;
+            }
+
             // fraction of the Longest Common Prefix length with respect to the input prompt length
             const size_t lcp_len = tokens.get_common_prefix(task.tokens);
             const float f_sim_cur = float(lcp_len) / task.tokens.size();
diff --git a/tools/server/server-slot-sched.h b/tools/server/server-slot-sched.h
index 8389d05a5..fc0099dc0 100644
--- a/tools/server/server-slot-sched.h
+++ b/tools/server/server-slot-sched.h
@@ -24,6 +24,10 @@ struct server_prompt {
     // a checkpoint does not change after it is created, so the prompt cache shares it with the slot instead of a copy
     std::list<std::shared_ptr<const common_prompt_checkpoint>> checkpoints;

+    // the lora config that the tokens were processed with, it follows the state to the prompt cache and back
+    // TODO: slot files do not store it
+    std::vector<common_adapter_lora_info> lora;
+
     void clear() {
         tokens.clear();
         checkpoints.clear();
@@ -37,11 +41,13 @@ struct server_prompt {
         return server_prompt {
             tokens.clone(),
             checkpoints,
+            lora,
         };
     }

     // n_prefix: tokens shared with the new prompt, must be > 0
     // has_new_tokens: the new prompt is longer than n_prefix
+    // n_swa: 0 if a state must end right before the next token (no SWA, or recurrent)
     // pos_min: min position of the live state
     // n_past == 0 without a checkpoint means that nothing can be reused
     server_prompt_reuse get_reuse(int32_t n_prefix, bool has_new_tokens, int32_t n_swa, llama_pos pos_min) const;
@@ -91,7 +97,8 @@ struct server_prompt_cache {

     server_prompt_cache_state * alloc(const server_prompt & prompt, size_t state_size_main, size_t state_size_drft);

-    bool load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot);
+    // lora_new: the lora config of the new prompt, the states made with another config are skipped
+    bool load(server_prompt & prompt, const server_tokens & tokens_new, const std::vector<common_adapter_lora_info> & lora_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot);

     void update();
 };
@@ -122,5 +129,6 @@ struct server_slot_sched {
     // idle slots that lose the least by being reused: empty slots first, then the least recently used ones
     std::vector<int> get_free_slots(const std::vector<server_slot_info> & slots, size_t n_slots_needed, int exclude_id_slot) const;

-    server_slot_plan select(const std::vector<server_slot_info> & slots, const server_task & task) const;
+    // lora: the lora config of the task
+    server_slot_plan select(const std::vector<server_slot_info> & slots, const server_task & task, const std::vector<common_adapter_lora_info> & lora) const;
 };
diff --git a/tools/server/tests/unit/test_completion.py b/tools/server/tests/unit/test_completion.py
index 618345585..aa29454b1 100644
--- a/tools/server/tests/unit/test_completion.py
+++ b/tools/server/tests/unit/test_completion.py
@@ -811,3 +811,57 @@ def test_completion_checkpoint_kept_after_restore():
     for suffix in suffixes[1:]:
         assert len(suffix) > server.n_ubatch + 4
         assert cache_n(prefix + suffix) == len(prefix) - 1
+
+
+# hybrid model with SWA: a checkpoint or a state right before the new tokens must be reused, even if it is less than n_swa tokens away
+def test_completion_checkpoint_hybrid_swa():
+    global server
+    server = ServerPreset.tinylfm3omni()
+    server.n_slots = 1
+    server.n_batch = 16
+    # one token per ubatch, like the generated tokens: the runs with and without the cache go through the same kernels
+    server.n_ubatch = 1
+    server.cache_ram = 0
+    server.temperature = 0.0
+    server.start()
+
+    def complete(prompt, cache_prompt=True, n_predict=4):
+        res = server.make_request("POST", "/completion", data={
+            "prompt": prompt,
+            "n_predict": n_predict,
+            "n_probs": 5,
+            "cache_prompt": cache_prompt,
+            "return_tokens": True,
+        })
+        assert res.status_code == 200
+        return res.body
+
+    def top_logprobs(body):
+        return [(p["id"], p["logprob"]) for p in body["completion_probabilities"][0]["top_logprobs"]]
+
+    def assert_same_output(actual, expected):
+        assert actual["tokens"] == expected["tokens"]
+        assert top_logprobs(actual) == top_logprobs(expected)
+
+    # shorter than n_swa (256), so that every checkpoint is within the SWA window of the prompt end
+    prefix = [1000 + i for i in range(200)]
+    suffix = [5000 + i for i in range(20)]
+
+    # the new prompt drops the generated tokens: the state is rolled back to the checkpoint 4 tokens before the end
+    expected = complete(prefix + suffix, cache_prompt=False)
+    complete(prefix)
+    actual = complete(prefix + suffix)
+    assert actual["timings"]["cache_n"] == len(prefix) - 4
+    assert_same_output(actual, expected)
+
+    # the new prompt continues after the generated tokens: the state is used as it is
+    first = complete(prefix)
+    prompt = prefix + first["tokens"] + suffix
+    expected = complete(prompt, cache_prompt=False)
+    first = complete(prefix)
+    assert prefix + first["tokens"] + suffix == prompt
+    actual = complete(prompt)
+    # the last generated token was sampled, but never processed
+    assert actual["timings"]["cache_n"] == len(prefix) + len(first["tokens"]) - 1
+    assert actual["timings"]["prompt_n"] == len(suffix) + 1
+    assert_same_output(actual, expected)
diff --git a/tools/server/tests/unit/test_lora.py b/tools/server/tests/unit/test_lora.py
index c84ba52a9..dabcccbcc 100644
--- a/tools/server/tests/unit/test_lora.py
+++ b/tools/server/tests/unit/test_lora.py
@@ -122,6 +122,51 @@ def test_lora_global_change_clears_slot_cache():
     assert res["content"] == expected_one


+def test_lora_prompt_cache():
+    global server
+    server.n_slots = 1
+    server.start()
+
+    # long enough that stale kv changes the output
+    prompt = "Look in thy glass and tell the face thou viewest, now is the time that face should form another, whose fresh repair if now thou not renewest, thou dost beguile the world, unbless some mother."
+    other  = "Once upon a time there was a little girl named Lily who loved to play in the garden with her dog, and one day she found a shiny key under the big tree."
+
+    def complete(prompt: str, scale: float, cache_prompt: bool = True) -> dict:
+        res = server.make_request("POST", "/completion", data={
+            "prompt": prompt,
+            "lora": [{"id": 0, "scale": scale}],
+            "cache_prompt": cache_prompt,
+            "temperature": 0.0,
+            "seed": 42,
+            "n_predict": 16,
+        })
+        assert res.status_code == 200
+        return res.body
+
+    expected_zero = complete(prompt, 0.0, cache_prompt=False)["content"]
+    expected_one  = complete(prompt, 1.0, cache_prompt=False)["content"]
+    assert expected_zero != expected_one
+
+    # start again, so that the prompt cache has no state of the prompt with scale 0
+    server.stop()
+    server.start()
+
+    # the prompt goes to the prompt cache with scale 1 when the other prompt takes the slot
+    assert complete(prompt, 1.0)["content"] == expected_one
+    complete(other, 0.0)
+
+    # the cached state was made with another lora config, it must not be used
+    res = complete(prompt, 0.0)
+    assert res["timings"]["cache_n"] <= 1 # the BOS token
+    assert res["content"] == expected_zero
+
+    # with the same config, it is used
+    complete(other, 1.0)
+    res = complete(prompt, 1.0)
+    assert res["timings"]["cache_n"] > 1
+    assert res["content"] == expected_one
+
+
 @pytest.mark.skipif(not is_slow_test_allowed(), reason="skipping slow test")
 def test_with_big_model():
     server = ServerProcess()