Commit 225a5345b for llama.cpp

commit 225a5345bbb0e0c70859af5441ba922beb666fcf
Author: Aman Gupta <amangupta052@gmail.com>
Date:   Sun Nov 22 11:25:13 2026 +0800

    server : fit non-causal batches in one ubatch (#32713)

diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index e37661a9a..b458a17c4 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -4285,7 +4285,9 @@ private:

                     if (!slot.can_split()) {
                         // cannot fit the prompt in the current batch - will try next iter
-                        if (batch.size() + slot.task->n_tokens() > n_batch) {
+                        // with non-causal attention the whole batch must fit in one ubatch, see llama_decode
+                        const int32_t n_batch_max = llama_get_causal_attn(ctx_tgt) ? n_batch : std::min(n_batch, n_ubatch);
+                        if (batch.size() + slot.task->n_tokens() > n_batch_max) {
                             return;
                         }
                     }
diff --git a/tools/server/tests/unit/test_systemone.py b/tools/server/tests/unit/test_systemone.py
index b0b5715e9..96f70632b 100644
--- a/tools/server/tests/unit/test_systemone.py
+++ b/tools/server/tests/unit/test_systemone.py
@@ -375,6 +375,30 @@ def test_systemone_d1_parallel():
             assert_answers_close(a[qid], b[qid], 1e-6)


+def test_systemone_noncausal_parallel():
+    global server
+    server = ServerPreset.tinydecision("vllm-d3")
+    # n_batch > n_ubatch: each prompt fits in one ubatch, but several of them do not
+    server.n_batch = 2048
+    server.n_ubatch = 512
+    server.n_slots = 4
+    server.start()
+    state = " ".join(f"The order {i} arrived damaged and the customer wants a replacement." for i in range(14))
+    questions = {"angry": TEST_QUESTIONS["angry"]}
+
+    def decide():
+        res = server.make_request("POST", "/v1/systemone", data={"state": state, "questions": questions})
+        assert res.status_code == 200
+        assert res.body["usage"]["input_tokens"] > 512 // 2
+        return res.body["answers"]
+
+    # a non-causal batch is evaluated in one ubatch, the concurrent prompts are spread over several batches
+    single = decide()
+    parallel = parallel_function_calls([(decide, ()) for _ in range(4)])
+    for answers in parallel:
+        assert_answers_close(single["angry"], answers["angry"], 1e-6)
+
+
 def test_systemone_d1_special_tokens():
     global server
     server = ServerPreset.tinydecision("lfm3-d1-omni")