Commit cbb1c1c50 for llama.cpp
commit cbb1c1c50bba81929387dea473873d1df87a6212
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Sun Nov 22 00:35:43 2026 +0100
model: fix no_realloc for clef and vllm-decision2 (#32754)
diff --git a/src/models/clef.cpp b/src/models/clef.cpp
index 86fed4e87..c41c93cb1 100644
--- a/src/models/clef.cpp
+++ b/src/models/clef.cpp
@@ -173,22 +173,35 @@ static clef_spans clef_get_spans(const llama_ubatch & ubatch) {
return res;
}
+// the head always has the max number of spans, so its graph depends only on n_tokens
+// each span has 1 token or more and each question has an option, so there is always a spare question for the padding options
+static size_t clef_n_questions(int64_t n_tokens) { return n_tokens / 2 + 1; }
+static size_t clef_n_options (int64_t n_tokens) { return std::max<int64_t>(n_tokens - 1, 1); }
+
// pooling matrices and indices computed from the spans
class llama_model_clef::input_decision : public llm_graph_input_i {
public:
- input_decision(const llama_ubatch & ubatch) : n_tokens(ubatch.n_tokens) {
- const auto spans = clef_get_spans(ubatch);
- n_questions = spans.questions.size();
- n_options = spans.options.size();
- }
+ input_decision(const llama_ubatch & ubatch) :
+ n_tokens(ubatch.n_tokens), n_questions(clef_n_questions(n_tokens)), n_options(clef_n_options(n_tokens)) {}
void set_input(const llama_ubatch * ubatch) override {
// a batch of embeddings has no token ids, and no usable spans
const std::vector<llama_token> no_tokens(ubatch->token ? 0 : n_tokens, 0);
ggml_backend_tensor_set(tokens, ubatch->token ? ubatch->token : no_tokens.data(), 0, n_tokens * sizeof(llama_token));
- const auto spans = clef_get_spans(*ubatch);
- GGML_ASSERT(spans.questions.size() == n_questions && spans.options.size() == n_options);
+ auto spans = clef_get_spans(*ubatch);
+ const int32_t n_q_used = spans.questions.size();
+ GGML_ASSERT((size_t) n_q_used < n_questions && spans.options.size() <= n_options);
+
+ // the padding options belong to the first padding question, the questions do not read the padding questions
+ spans.questions.resize(n_questions, { 0, 0, 0 });
+ spans.options.resize(n_options, { n_q_used, 0, 0 });
+
+ std::vector<float> q_mask(n_questions * n_questions, 0.0f);
+ for (size_t i = 0; i < n_questions; i++) {
+ std::fill(q_mask.begin() + i * n_questions + n_q_used, q_mask.begin() + (i + 1) * n_questions, -INFINITY);
+ }
+ ggml_backend_tensor_set(question_mask, q_mask.data(), 0, ggml_nbytes(question_mask));
// the scores are NaN if the batch has a decision order that cannot be used
const float status_data = spans.valid || ubatch->decision_order == nullptr ? 0.0f : NAN;
@@ -215,6 +228,8 @@ public:
owner[i] = o.question;
mask[o.question * n_options + i] = 0.0f;
}
+ // a padding question reads all options, a row of -inf gives NaN
+ std::fill(mask.begin() + n_q_used * n_options, mask.end(), 0.0f);
ggml_backend_tensor_set(pool_q, pool_q_data.data(), 0, ggml_nbytes(pool_q));
ggml_backend_tensor_set(pool_o, pool_o_data.data(), 0, ggml_nbytes(pool_o));
@@ -225,8 +240,7 @@ public:
bool can_reuse(const llm_graph_params & params) override {
// the values are computed again in set_input(), only the shapes must match
- const auto spans = clef_get_spans(params.ubatch);
- return spans.questions.size() == n_questions && spans.options.size() == n_options;
+ return params.ubatch.n_tokens == (uint32_t) n_tokens;
}
ggml_tensor * tokens = nullptr; // I32 [n_tokens]
@@ -235,11 +249,12 @@ public:
ggml_tensor * question_type = nullptr; // I32 [n_questions]
ggml_tensor * option_question = nullptr; // I32 [n_options]
ggml_tensor * option_mask = nullptr; // F32 [n_options, n_questions], 0 if the option belongs to the question, else -inf
+ ggml_tensor * question_mask = nullptr; // F32 [n_questions, n_questions], -inf for the padding questions
ggml_tensor * status = nullptr; // F32 [1], added to the scores: 0, or NaN on invalid input
const int64_t n_tokens;
- size_t n_questions;
- size_t n_options;
+ const size_t n_questions;
+ const size_t n_options;
};
// the backbone is copied from llama_model_qwen35::graph, without the memory module
@@ -479,8 +494,8 @@ ggml_tensor * llama_model_clef::graph::build_head_norm(ggml_tensor * cur, const
return build_norm(cur, n.w, n.b, LLM_NORM, -1);
}
-// q: [n_embd_h, n_q], kv: [n_embd_h, n_kv], no mask
-ggml_tensor * llama_model_clef::graph::build_head_attn(ggml_tensor * q, ggml_tensor * kv, const attn & a) {
+// q: [n_embd_h, n_q], kv: [n_embd_h, n_kv], kq_mask: [n_kv, n_q] or null
+ggml_tensor * llama_model_clef::graph::build_head_attn(ggml_tensor * q, ggml_tensor * kv, const attn & a, ggml_tensor * kq_mask) {
const int64_t n_embd_h = q->ne[0];
const int64_t n_head_h = model.n_head_decision;
const int64_t d_head = n_embd_h / n_head_h;
@@ -497,7 +512,7 @@ ggml_tensor * llama_model_clef::graph::build_head_attn(ggml_tensor * q, ggml_ten
Vcur = ggml_cont(ctx0, Vcur);
ggml_tensor * kq = ggml_mul_mat(ctx0, Kcur, Qcur); // [n_kv, n_q, n_head]
- kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f / sqrtf(float(d_head)), 0.0f);
+ kq = ggml_soft_max_ext(ctx0, kq, kq_mask, 1.0f / sqrtf(float(d_head)), 0.0f);
ggml_tensor * cur = ggml_mul_mat(ctx0, Vcur, kq); // [d_head, n_q, n_head]
cur = ggml_cont_2d(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3), n_embd_h, n_q);
@@ -523,8 +538,9 @@ ggml_tensor * llama_model_clef::graph::build_head(ggml_tensor * hidden, input_de
inp->question_type = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_questions);
inp->option_question = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_options);
inp->option_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_options, n_questions);
+ inp->question_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_questions, n_questions);
inp->status = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);
- for (ggml_tensor * t : { inp->tokens, inp->pool_q, inp->pool_o, inp->question_type, inp->option_question, inp->option_mask, inp->status }) {
+ for (ggml_tensor * t : { inp->tokens, inp->pool_q, inp->pool_o, inp->question_type, inp->option_question, inp->option_mask, inp->question_mask, inp->status }) {
ggml_set_input(t);
}
@@ -584,7 +600,7 @@ ggml_tensor * llama_model_clef::graph::build_head(ggml_tensor * hidden, input_de
const auto & layer = model.head_layers[il];
ggml_tensor * cur = build_head_norm(fields, layer.self_norm);
- fields = ggml_add(ctx0, fields, build_head_attn(cur, cur, layer.self_attn));
+ fields = ggml_add(ctx0, fields, build_head_attn(cur, cur, layer.self_attn, inp->question_mask));
cur = build_head_norm(fields, layer.cross_norm);
fields = ggml_add(ctx0, fields, build_head_attn(cur, memory, layer.cross_attn));
diff --git a/src/models/models.h b/src/models/models.h
index 0f49b8ae0..b12642fb9 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -2927,7 +2927,7 @@ struct llama_model_clef : public llama_model_qwen35 {
input_decision * inp);
ggml_tensor * build_head_norm(ggml_tensor * cur, const norm & n);
- ggml_tensor * build_head_attn(ggml_tensor * q, ggml_tensor * kv, const attn & a);
+ ggml_tensor * build_head_attn(ggml_tensor * q, ggml_tensor * kv, const attn & a, ggml_tensor * kq_mask = nullptr);
ggml_tensor * build_head_ffn (ggml_tensor * cur, const head_layer & layer);
const llama_model_clef & model;
diff --git a/src/models/vllm-decision2.cpp b/src/models/vllm-decision2.cpp
index 2a3b9595c..692534425 100644
--- a/src/models/vllm-decision2.cpp
+++ b/src/models/vllm-decision2.cpp
@@ -99,13 +99,19 @@ static decision2_rows decision2_get_rows(const llama_ubatch & ubatch) {
return res;
}
+// no decision order (e.g. reserve): use the max number of options
+static size_t decision2_n_options(const llama_ubatch & ubatch, uint32_t n_outputs) {
+ return ubatch.decision_order ? decision2_get_rows(ubatch).options.size() : std::max<uint32_t>(n_outputs, 1);
+}
+
class llama_model_vllm_decision2::input_decision : public llm_graph_input_i {
public:
- input_decision(const llama_ubatch & ubatch) : n_options(decision2_get_rows(ubatch).options.size()) {}
+ input_decision(const llama_ubatch & ubatch, uint32_t n_outputs) : n_options(decision2_n_options(ubatch, n_outputs)) {}
void set_input(const llama_ubatch * ubatch) override {
- const auto rows = decision2_get_rows(*ubatch);
- GGML_ASSERT(rows.options.size() == n_options);
+ auto rows = decision2_get_rows(*ubatch);
+ GGML_ASSERT(rows.options.size() <= n_options);
+ rows.options.resize(n_options, 0);
// the scores are NaN if the batch has a decision order that cannot be used
const float status_data = rows.valid || ubatch->decision_order == nullptr ? 0.0f : NAN;
@@ -117,7 +123,7 @@ public:
bool can_reuse(const llm_graph_params & params) override {
// the values are computed again in set_input(), only the shapes must match
- return decision2_get_rows(params.ubatch).options.size() == n_options;
+ return decision2_n_options(params.ubatch, params.n_outputs) == n_options;
}
ggml_tensor * options = nullptr; // I32 [n_options]
@@ -129,7 +135,7 @@ public:
llama_model_vllm_decision2::graph::graph(const llm_graph_params & params) :
llama_model_qwen3::graph(params, no_build()), model(static_cast<const llama_model_vllm_decision2 &>(*params.model)) {
- auto inp_decision_ptr = std::make_unique<input_decision>(ubatch);
+ auto inp_decision_ptr = std::make_unique<input_decision>(ubatch, n_outputs);
auto * inp_decision = inp_decision_ptr.get();
res->add_input(std::move(inp_decision_ptr));