Commit 184862801 for llama.cpp

commit 184862801a8747ca9fc0a84501419581aa9d37aa
Author: Meet Amin <meetamin378@gmail.com>
Date:   Fri Nov 20 21:11:02 2026 +0530

    gemma4: Optimized prompt processing for multi-seq queries (#32342)

diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp
index 9c061310d..1a377fa3c 100644
--- a/src/models/gemma4.cpp
+++ b/src/models/gemma4.cpp
@@ -157,6 +157,29 @@ static ggml_tensor * gemma4_view_2d_slice(ggml_context * ctx0, ggml_tensor * x,
     return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], x->nb[1], idx * x->nb[2]);
 }

+class llm_graph_input_gemma4_prune : public llm_graph_input_i {
+    int64_t n_slices;
+    int64_t n_slice_tokens;
+    int64_t n_slice_outputs;
+public:
+    ggml_tensor * full_mask      = nullptr;
+    ggml_tensor * full_mask_swa  = nullptr;
+    ggml_tensor * out_mask       = nullptr;
+    ggml_tensor * out_mask_swa   = nullptr;
+    ggml_tensor * inp_prune_ids  = nullptr;
+    ggml_tensor * inp_prune_keep = nullptr;
+    llm_graph_input_gemma4_prune(int64_t n_slices, int64_t n_slice_tokens, int64_t n_slice_outputs)
+            : n_slices(n_slices), n_slice_tokens(n_slice_tokens), n_slice_outputs(n_slice_outputs) {}
+
+    void set_input(const llama_ubatch * ubatch) override;
+
+    bool can_reuse(const llm_graph_params & params) override;
+
+    static int64_t max_outputs_per_slice(
+        const llama_ubatch & ubatch, int64_t n_slices, int64_t n_slice_tokens
+    );
+};
+
 llama_model_gemma4::graph::graph(const llm_graph_params & params) :
         llm_graph_context(params),
         n_embd_per_layer(model.hparams.n_embd_per_layer) {
@@ -181,11 +204,8 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :
     // TODO: is causal == true correct? might need some changes
     auto * inp_attn = build_attn_inp_kv_iswa();

-    auto inp_attn_full  = *inp_attn;
     auto inp_attn_prune = *inp_attn;

-    ggml_tensor * inp_out_ids = build_inp_out_ids();
-
     if (model.per_layer_tok_embd) {
         // fold per_layer_input_scale (1/sqrt(2)) from project_per_layer_inputs() into tok_embd_scale
         const float tok_embd_scale = sqrtf((float) n_embd_per_layer) / sqrtf(2.0f);
@@ -197,34 +217,57 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :
         inp_per_layer = project_per_layer_inputs(inpL, inp_per_layer);
     }

+    const bool needs_unmasked_nextn =
+        cparams.embeddings_nextn && !cparams.embeddings_nextn_masked;
+
     // KV-shared layers do not write to the KV cache, so prune non-output token rows before the first KV-shared layer
-    bool prune_shared = crop_before_nextn(inp_out_ids) &&
+    bool prune_shared = !needs_unmasked_nextn &&
                         hparams.n_layer_kv_from_start > 0 && hparams.n_layer_kv_from_start < n_layer &&
-                        !cparams.embeddings && cparams.n_seq_max == 1;
+                        !cparams.embeddings;
     for (int il = (int) hparams.n_layer_kv_from_start; prune_shared && il <= n_layer; ++il) {
         prune_shared = !cparams.embeddings_layer_inp[il];
     }

     int64_t n_rows = n_tokens;

+    ggml_tensor * inp_prune_ids = nullptr;
+    ggml_tensor * inp_prune_keep = nullptr;
+    ggml_tensor * inp_out_ids = prune_shared ? nullptr : build_inp_out_ids();
+
     for (int il = 0; il < n_layer; ++il) {
         if (prune_shared && il == (int) hparams.n_layer_kv_from_start) {
-            inpL    = ggml_get_rows(ctx0, inpL, inp_out_ids);
-            inp_pos = ggml_reshape_1d(ctx0, ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, inp_pos, 1, n_tokens), inp_out_ids), n_outputs);
+            ggml_tensor * full     = inp_attn->self_attn_mask.raw;
+            ggml_tensor * full_swa = inp_attn->self_attn_mask_swa.raw;

-            // TODO: instead of doing get_rows here, extend llm_attn_mask to have a `out` mask that
-            //       is construted during set_input to contain just the pruned output cells
-            //       this can be used to enable pruning with more than 1 sequence, both with unified and split KV cache
-            //       ref: https://github.com/ggml-org/llama.cpp/pull/31356#discussion_r4455286628
-            inp_attn_prune.self_attn_mask.raw     = ggml_get_rows(ctx0, inp_attn_full.self_attn_mask.raw,     inp_out_ids);
-            inp_attn_prune.self_attn_mask_swa.raw = ggml_get_rows(ctx0, inp_attn_full.self_attn_mask_swa.raw, inp_out_ids);
+            const int64_t n_slices = full->ne[3], n_slice_tokens = full->ne[1];
+            const int64_t n_slice_outputs = llm_graph_input_gemma4_prune::max_outputs_per_slice(ubatch, n_slices, n_slice_tokens);

-            inp_attn_prune.self_attn_mask.raw     = ggml_cast(ctx0, inp_attn_prune.self_attn_mask.raw,     GGML_TYPE_F16);
-            inp_attn_prune.self_attn_mask_swa.raw = ggml_cast(ctx0, inp_attn_prune.self_attn_mask_swa.raw, GGML_TYPE_F16);
+            n_rows = n_slices * n_slice_outputs;

-            n_rows = n_outputs;
+            auto prune_input = std::make_unique<llm_graph_input_gemma4_prune>(n_slices, n_slice_tokens, n_slice_outputs);
+            prune_input->full_mask = full;
+            prune_input->full_mask_swa = full_swa;
+            prune_input->out_mask = ggml_new_tensor_4d(ctx0, full->type, full->ne[0], n_slice_outputs, 1, n_slices);
+            prune_input->out_mask_swa = ggml_new_tensor_4d(ctx0, full_swa->type, full_swa->ne[0], n_slice_outputs, 1, n_slices);
+            prune_input->inp_prune_ids  = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_rows);
+            prune_input->inp_prune_keep = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_outputs);

+            ggml_set_input(prune_input->out_mask);
+            ggml_set_input(prune_input->out_mask_swa);
+            ggml_set_input(prune_input->inp_prune_ids);
+            ggml_set_input(prune_input->inp_prune_keep);
+
+            inp_prune_ids  = prune_input->inp_prune_ids;
+            inp_prune_keep = prune_input->inp_prune_keep;
+
+            inpL = ggml_get_rows(ctx0, inpL, inp_prune_ids);
+            inp_pos = ggml_reshape_1d(ctx0, ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, inp_pos, 1, n_tokens), inp_prune_ids), n_rows);
+
+            inp_attn_prune.self_attn_mask.raw = prune_input->out_mask;
+            inp_attn_prune.self_attn_mask_swa.raw = prune_input->out_mask_swa;
             inp_attn = &inp_attn_prune;
+
+            res->add_input(std::move(prune_input));
         }

         const int64_t n_embd_head = hparams.n_embd_head_k(il);
@@ -406,7 +449,9 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :

             ggml_tensor * inp_this_layer = gemma4_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens]

-            if ((prune_shared && !hparams.has_kv(il)) || (il == n_layer - 1 && crop_before_nextn(inp_out_ids))) {
+            if (prune_shared && !hparams.has_kv(il)) {
+                inp_this_layer = ggml_get_rows(ctx0, inp_this_layer, inp_prune_ids);
+            } else if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {
                 inp_this_layer = ggml_get_rows(ctx0, inp_this_layer, inp_out_ids);
             }

@@ -432,6 +477,9 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :
         inpL = cur;
     }
     cur = inpL;
+    if (prune_shared) {
+        cur = ggml_get_rows(ctx0, cur, inp_prune_keep);
+    }

     cur = build_norm(cur,
             model.output_norm, nullptr,
@@ -558,3 +606,89 @@ ggml_tensor * llama_model_gemma4::graph::project_per_layer_inputs(ggml_tensor *
     // permute to shape: [n_embd_per_layer, n_tokens, n_layer]
     return ggml_permute(ctx0, inp_per_layer, 0, 2, 1, 3);
 }
+
+void llm_graph_input_gemma4_prune::set_input(const llama_ubatch * ubatch) {
+    std::vector<std::vector<int32_t>> selected(n_slices);
+
+    for (int32_t s = 0; s < n_slices; ++s) {
+        for (int32_t t = 0; t < n_slice_tokens; ++t) {
+            const int32_t i = s * n_slice_tokens + t;
+            if (ubatch->output[i]) {
+                selected[s].push_back(i);
+            }
+        }
+    }
+
+    std::vector<int32_t> take, keep;
+    for (int32_t s = 0; s < n_slices; ++s) {
+        const auto & ids = selected[s];
+
+        for (int32_t j = 0; j < int32_t(ids.size()); ++j) {
+            keep.push_back(s * n_slice_outputs + j);
+        }
+
+        take.insert(take.end(), ids.begin(), ids.end());
+        const int32_t dummy = ids.empty() ? s * n_slice_tokens : ids.back();
+        take.insert(take.end(), n_slice_outputs - ids.size(), dummy);
+    }
+
+    GGML_ASSERT(take.size() == size_t(inp_prune_ids->ne[0]));
+    if (!take.empty()) {
+        ggml_backend_tensor_set(inp_prune_ids, take.data(), 0,
+                                take.size() * sizeof(int32_t));
+    }
+
+    auto copy_mask = [&](const ggml_tensor * full, ggml_tensor * out) {
+        if (take.empty() || !out->buffer) {
+            return;
+        }
+
+        GGML_ASSERT(full->buffer && ggml_backend_buffer_is_host(full->buffer));
+        const size_t row_bytes = full->ne[0] * ggml_element_size(full);
+        const auto * src = static_cast<const uint8_t *>(full->data);
+        std::vector<uint8_t> packed(take.size() * row_bytes);
+
+        for (size_t j = 0; j < take.size(); ++j) {
+            memcpy(packed.data() + j * row_bytes,
+                        src + take[j] * row_bytes,
+                        row_bytes);
+        }
+
+        ggml_backend_tensor_set(out, packed.data(), 0, packed.size());
+    };
+
+    copy_mask(full_mask, out_mask);
+    copy_mask(full_mask_swa, out_mask_swa);
+
+    if (!keep.empty()) {
+        ggml_backend_tensor_set(
+            inp_prune_keep, keep.data(), 0,
+            keep.size() * sizeof(int32_t));
+    }
+}
+
+bool llm_graph_input_gemma4_prune::can_reuse(const llm_graph_params & params) {
+    const auto & ubatch = params.ubatch;
+
+    if (ubatch.n_tokens != n_slices * n_slice_tokens || !ubatch.output) {
+        return false;
+    }
+
+    return max_outputs_per_slice(ubatch, n_slices, n_slice_tokens) == n_slice_outputs;
+}
+
+int64_t llm_graph_input_gemma4_prune::max_outputs_per_slice(const llama_ubatch & ubatch, int64_t n_slices, int64_t n_slice_tokens) {
+    int64_t res = 0;
+
+    for (int32_t s = 0; s < n_slices; ++s) {
+        int64_t m = 0;
+        for (int32_t t = 0; t < n_slice_tokens; ++t) {
+            const int32_t i = s * n_slice_tokens + t;
+            if (ubatch.output[i]) {
+                m++;
+            }
+        }
+        res = std::max(res, m);
+    }
+    return res;
+}