Commit 225a5345b for llama.cpp
commit 225a5345bbb0e0c70859af5441ba922beb666fcf
Author: Aman Gupta <amangupta052@gmail.com>
Date: Sun Nov 22 11:25:13 2026 +0800
server : fit non-causal batches in one ubatch (#32713)
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index e37661a9a..b458a17c4 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -4285,7 +4285,9 @@ private:
if (!slot.can_split()) {
// cannot fit the prompt in the current batch - will try next iter
- if (batch.size() + slot.task->n_tokens() > n_batch) {
+ // with non-causal attention the whole batch must fit in one ubatch, see llama_decode
+ const int32_t n_batch_max = llama_get_causal_attn(ctx_tgt) ? n_batch : std::min(n_batch, n_ubatch);
+ if (batch.size() + slot.task->n_tokens() > n_batch_max) {
return;
}
}
diff --git a/tools/server/tests/unit/test_systemone.py b/tools/server/tests/unit/test_systemone.py
index b0b5715e9..96f70632b 100644
--- a/tools/server/tests/unit/test_systemone.py
+++ b/tools/server/tests/unit/test_systemone.py
@@ -375,6 +375,30 @@ def test_systemone_d1_parallel():
assert_answers_close(a[qid], b[qid], 1e-6)
+def test_systemone_noncausal_parallel():
+ global server
+ server = ServerPreset.tinydecision("vllm-d3")
+ # n_batch > n_ubatch: each prompt fits in one ubatch, but several of them do not
+ server.n_batch = 2048
+ server.n_ubatch = 512
+ server.n_slots = 4
+ server.start()
+ state = " ".join(f"The order {i} arrived damaged and the customer wants a replacement." for i in range(14))
+ questions = {"angry": TEST_QUESTIONS["angry"]}
+
+ def decide():
+ res = server.make_request("POST", "/v1/systemone", data={"state": state, "questions": questions})
+ assert res.status_code == 200
+ assert res.body["usage"]["input_tokens"] > 512 // 2
+ return res.body["answers"]
+
+ # a non-causal batch is evaluated in one ubatch, the concurrent prompts are spread over several batches
+ single = decide()
+ parallel = parallel_function_calls([(decide, ()) for _ in range(4)])
+ for answers in parallel:
+ assert_answers_close(single["angry"], answers["angry"], 1e-6)
+
+
def test_systemone_d1_special_tokens():
global server
server = ServerPreset.tinydecision("lfm3-d1-omni")