Commit cbb1c1c50 for llama.cpp

commit cbb1c1c50bba81929387dea473873d1df87a6212
Author: Xuan-Son Nguyen <son@huggingface.co>
Date:   Sun Nov 22 00:35:43 2026 +0100

    model: fix no_realloc for clef and vllm-decision2 (#32754)

diff --git a/src/models/clef.cpp b/src/models/clef.cpp
index 86fed4e87..c41c93cb1 100644
--- a/src/models/clef.cpp
+++ b/src/models/clef.cpp
@@ -173,22 +173,35 @@ static clef_spans clef_get_spans(const llama_ubatch & ubatch) {
     return res;
 }

+// the head always has the max number of spans, so its graph depends only on n_tokens
+// each span has 1 token or more and each question has an option, so there is always a spare question for the padding options
+static size_t clef_n_questions(int64_t n_tokens) { return n_tokens / 2 + 1; }
+static size_t clef_n_options  (int64_t n_tokens) { return std::max<int64_t>(n_tokens - 1, 1); }
+
 // pooling matrices and indices computed from the spans
 class llama_model_clef::input_decision : public llm_graph_input_i {
 public:
-    input_decision(const llama_ubatch & ubatch) : n_tokens(ubatch.n_tokens) {
-        const auto spans = clef_get_spans(ubatch);
-        n_questions = spans.questions.size();
-        n_options   = spans.options.size();
-    }
+    input_decision(const llama_ubatch & ubatch) :
+        n_tokens(ubatch.n_tokens), n_questions(clef_n_questions(n_tokens)), n_options(clef_n_options(n_tokens)) {}

     void set_input(const llama_ubatch * ubatch) override {
         // a batch of embeddings has no token ids, and no usable spans
         const std::vector<llama_token> no_tokens(ubatch->token ? 0 : n_tokens, 0);
         ggml_backend_tensor_set(tokens, ubatch->token ? ubatch->token : no_tokens.data(), 0, n_tokens * sizeof(llama_token));

-        const auto spans = clef_get_spans(*ubatch);
-        GGML_ASSERT(spans.questions.size() == n_questions && spans.options.size() == n_options);
+        auto spans = clef_get_spans(*ubatch);
+        const int32_t n_q_used = spans.questions.size();
+        GGML_ASSERT((size_t) n_q_used < n_questions && spans.options.size() <= n_options);
+
+        // the padding options belong to the first padding question, the questions do not read the padding questions
+        spans.questions.resize(n_questions, { 0, 0, 0 });
+        spans.options.resize(n_options, { n_q_used, 0, 0 });
+
+        std::vector<float> q_mask(n_questions * n_questions, 0.0f);
+        for (size_t i = 0; i < n_questions; i++) {
+            std::fill(q_mask.begin() + i * n_questions + n_q_used, q_mask.begin() + (i + 1) * n_questions, -INFINITY);
+        }
+        ggml_backend_tensor_set(question_mask, q_mask.data(), 0, ggml_nbytes(question_mask));

         // the scores are NaN if the batch has a decision order that cannot be used
         const float status_data = spans.valid || ubatch->decision_order == nullptr ? 0.0f : NAN;
@@ -215,6 +228,8 @@ public:
             owner[i] = o.question;
             mask[o.question * n_options + i] = 0.0f;
         }
+        // a padding question reads all options, a row of -inf gives NaN
+        std::fill(mask.begin() + n_q_used * n_options, mask.end(), 0.0f);

         ggml_backend_tensor_set(pool_q,          pool_q_data.data(), 0, ggml_nbytes(pool_q));
         ggml_backend_tensor_set(pool_o,          pool_o_data.data(), 0, ggml_nbytes(pool_o));
@@ -225,8 +240,7 @@ public:

     bool can_reuse(const llm_graph_params & params) override {
         // the values are computed again in set_input(), only the shapes must match
-        const auto spans = clef_get_spans(params.ubatch);
-        return spans.questions.size() == n_questions && spans.options.size() == n_options;
+        return params.ubatch.n_tokens == (uint32_t) n_tokens;
     }

     ggml_tensor * tokens          = nullptr; // I32 [n_tokens]
@@ -235,11 +249,12 @@ public:
     ggml_tensor * question_type   = nullptr; // I32 [n_questions]
     ggml_tensor * option_question = nullptr; // I32 [n_options]
     ggml_tensor * option_mask     = nullptr; // F32 [n_options, n_questions], 0 if the option belongs to the question, else -inf
+    ggml_tensor * question_mask   = nullptr; // F32 [n_questions, n_questions], -inf for the padding questions
     ggml_tensor * status          = nullptr; // F32 [1], added to the scores: 0, or NaN on invalid input

     const int64_t n_tokens;
-    size_t n_questions;
-    size_t n_options;
+    const size_t  n_questions;
+    const size_t  n_options;
 };

 // the backbone is copied from llama_model_qwen35::graph, without the memory module
@@ -479,8 +494,8 @@ ggml_tensor * llama_model_clef::graph::build_head_norm(ggml_tensor * cur, const
     return build_norm(cur, n.w, n.b, LLM_NORM, -1);
 }

-// q: [n_embd_h, n_q], kv: [n_embd_h, n_kv], no mask
-ggml_tensor * llama_model_clef::graph::build_head_attn(ggml_tensor * q, ggml_tensor * kv, const attn & a) {
+// q: [n_embd_h, n_q], kv: [n_embd_h, n_kv], kq_mask: [n_kv, n_q] or null
+ggml_tensor * llama_model_clef::graph::build_head_attn(ggml_tensor * q, ggml_tensor * kv, const attn & a, ggml_tensor * kq_mask) {
     const int64_t n_embd_h = q->ne[0];
     const int64_t n_head_h = model.n_head_decision;
     const int64_t d_head   = n_embd_h / n_head_h;
@@ -497,7 +512,7 @@ ggml_tensor * llama_model_clef::graph::build_head_attn(ggml_tensor * q, ggml_ten
     Vcur = ggml_cont(ctx0, Vcur);

     ggml_tensor * kq = ggml_mul_mat(ctx0, Kcur, Qcur); // [n_kv, n_q, n_head]
-    kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f / sqrtf(float(d_head)), 0.0f);
+    kq = ggml_soft_max_ext(ctx0, kq, kq_mask, 1.0f / sqrtf(float(d_head)), 0.0f);

     ggml_tensor * cur = ggml_mul_mat(ctx0, Vcur, kq); // [d_head, n_q, n_head]
     cur = ggml_cont_2d(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3), n_embd_h, n_q);
@@ -523,8 +538,9 @@ ggml_tensor * llama_model_clef::graph::build_head(ggml_tensor * hidden, input_de
     inp->question_type   = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_questions);
     inp->option_question = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_options);
     inp->option_mask     = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_options, n_questions);
+    inp->question_mask   = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_questions, n_questions);
     inp->status          = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);
-    for (ggml_tensor * t : { inp->tokens, inp->pool_q, inp->pool_o, inp->question_type, inp->option_question, inp->option_mask, inp->status }) {
+    for (ggml_tensor * t : { inp->tokens, inp->pool_q, inp->pool_o, inp->question_type, inp->option_question, inp->option_mask, inp->question_mask, inp->status }) {
         ggml_set_input(t);
     }

@@ -584,7 +600,7 @@ ggml_tensor * llama_model_clef::graph::build_head(ggml_tensor * hidden, input_de
         const auto & layer = model.head_layers[il];

         ggml_tensor * cur = build_head_norm(fields, layer.self_norm);
-        fields = ggml_add(ctx0, fields, build_head_attn(cur, cur, layer.self_attn));
+        fields = ggml_add(ctx0, fields, build_head_attn(cur, cur, layer.self_attn, inp->question_mask));

         cur = build_head_norm(fields, layer.cross_norm);
         fields = ggml_add(ctx0, fields, build_head_attn(cur, memory, layer.cross_attn));
diff --git a/src/models/models.h b/src/models/models.h
index 0f49b8ae0..b12642fb9 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -2927,7 +2927,7 @@ struct llama_model_clef : public llama_model_qwen35 {
                  input_decision * inp);

         ggml_tensor * build_head_norm(ggml_tensor * cur, const norm & n);
-        ggml_tensor * build_head_attn(ggml_tensor * q, ggml_tensor * kv, const attn & a);
+        ggml_tensor * build_head_attn(ggml_tensor * q, ggml_tensor * kv, const attn & a, ggml_tensor * kq_mask = nullptr);
         ggml_tensor * build_head_ffn (ggml_tensor * cur, const head_layer & layer);

         const llama_model_clef & model;
diff --git a/src/models/vllm-decision2.cpp b/src/models/vllm-decision2.cpp
index 2a3b9595c..692534425 100644
--- a/src/models/vllm-decision2.cpp
+++ b/src/models/vllm-decision2.cpp
@@ -99,13 +99,19 @@ static decision2_rows decision2_get_rows(const llama_ubatch & ubatch) {
     return res;
 }

+// no decision order (e.g. reserve): use the max number of options
+static size_t decision2_n_options(const llama_ubatch & ubatch, uint32_t n_outputs) {
+    return ubatch.decision_order ? decision2_get_rows(ubatch).options.size() : std::max<uint32_t>(n_outputs, 1);
+}
+
 class llama_model_vllm_decision2::input_decision : public llm_graph_input_i {
 public:
-    input_decision(const llama_ubatch & ubatch) : n_options(decision2_get_rows(ubatch).options.size()) {}
+    input_decision(const llama_ubatch & ubatch, uint32_t n_outputs) : n_options(decision2_n_options(ubatch, n_outputs)) {}

     void set_input(const llama_ubatch * ubatch) override {
-        const auto rows = decision2_get_rows(*ubatch);
-        GGML_ASSERT(rows.options.size() == n_options);
+        auto rows = decision2_get_rows(*ubatch);
+        GGML_ASSERT(rows.options.size() <= n_options);
+        rows.options.resize(n_options, 0);

         // the scores are NaN if the batch has a decision order that cannot be used
         const float status_data = rows.valid || ubatch->decision_order == nullptr ? 0.0f : NAN;
@@ -117,7 +123,7 @@ public:

     bool can_reuse(const llm_graph_params & params) override {
         // the values are computed again in set_input(), only the shapes must match
-        return decision2_get_rows(params.ubatch).options.size() == n_options;
+        return decision2_n_options(params.ubatch, params.n_outputs) == n_options;
     }

     ggml_tensor * options = nullptr; // I32 [n_options]
@@ -129,7 +135,7 @@ public:

 llama_model_vllm_decision2::graph::graph(const llm_graph_params & params) :
     llama_model_qwen3::graph(params, no_build()), model(static_cast<const llama_model_vllm_decision2 &>(*params.model)) {
-    auto inp_decision_ptr = std::make_unique<input_decision>(ubatch);
+    auto inp_decision_ptr = std::make_unique<input_decision>(ubatch, n_outputs);
     auto * inp_decision   = inp_decision_ptr.get();
     res->add_input(std::move(inp_decision_ptr));