Commit 6792507b7 for llama.cpp

commit 6792507b7f611a6fc68bfae3eeae85e88017fc60
Author: Aman Gupta <amangupta052@gmail.com>
Date:   Sat Nov 21 13:22:36 2026 +0800

    llama : score only the k-pools of the ubatch sequences in QSA (#32519)

diff --git a/src/llama-memory-hybrid-idx.cpp b/src/llama-memory-hybrid-idx.cpp
index 2e19576f1..0199c734e 100644
--- a/src/llama-memory-hybrid-idx.cpp
+++ b/src/llama-memory-hybrid-idx.cpp
@@ -353,6 +353,7 @@ struct llama_memory_hybrid_idx_context::kpool_state {
     uint32_t generation = 0;

     uint32_t n_pool_real = 0;
+    uint32_t n_pool_ub   = 0; // pools of the ubatch's sequences, the only ones the indexer scores
     uint32_t n_new       = 0;
     uint32_t n_new_g     = 1; // graph size of the new pool list, stable across decode steps
 };
@@ -496,6 +497,7 @@ llama_memory_hybrid_idx_context::llama_memory_hybrid_idx_context(llama_memory_hy
         const uint64_t n_pool_max = uint64_t(idx->get_size() / mem->get_kpool()) * idx->get_n_seq_max();
         GGML_ASSERT(n_pool_max <= UINT32_MAX - 64);
         st.n_pool_real = std::max(st.n_pool_real, uint32_t(n_pool_max));
+        st.n_pool_ub   = st.n_pool_real;
         st.n_new   = st.n_pool_real;
         st.n_new_g = std::max(st.n_new, 1u);
         kpool_st = std::make_unique<kpool_state>(std::move(st));
@@ -687,6 +689,12 @@ void llama_memory_hybrid_idx_context::kpool_build_state(const llama_ubatch & uba
     }
     GGML_ASSERT(ip == st.is_new.size());

+    // the indexer scores only the pools of the ubatch's sequences
+    st.n_pool_ub = 0;
+    for (uint32_t i = 0; i < ubatch.n_seqs_unq; ++i) {
+        st.n_pool_ub += (uint32_t) lay.seqs[ubatch.seq_id_unq[i]].pools.size();
+    }
+
     // in order mode a token's cell gives its rank, and the rank its pool: positions cannot, as an image shares one
     const bool by_order = mem->get_kpool_by_order();
     const auto *   sinfo = by_order ? &sinfos_kpool[i_cur] : nullptr;
@@ -731,7 +739,7 @@ const llama_memory_hybrid_idx_context::kpool_state & llama_memory_hybrid_idx_con
 }

 uint32_t llama_memory_hybrid_idx_context::get_n_kpool() const {
-    return kpool_pad(kpool_cur().n_pool_real);
+    return kpool_pad(kpool_cur().n_pool_ub);
 }

 uint32_t llama_memory_hybrid_idx_context::get_n_kpool_new() const {
@@ -761,7 +769,7 @@ void llama_memory_hybrid_idx_context::set_input_kpool(ggml_tensor * pool_cells,

     const bool by_order = mem->get_kpool_by_order();

-    GGML_ASSERT(n_pool == kpool_pad(st.n_pool_real));
+    GGML_ASSERT(n_pool == kpool_pad(st.n_pool_ub));
     GGML_ASSERT(st.is_new.size() == st.n_pool_real);
     GGML_ASSERT(pool_mask->ne[0] == (int64_t) n_pool && pool_mask->ne[1] == (int64_t) n_tokens);
     GGML_ASSERT(tail_idxs->ne[0] == (int64_t) kpool - 1 && tail_idxs->ne[1] == (int64_t) n_tokens);
@@ -783,7 +791,7 @@ void llama_memory_hybrid_idx_context::set_input_kpool(ggml_tensor * pool_cells,
         return (int64_t) sq.strm*kv_size + cell;
     };

-    // Sequences present in this ubatch, pools of absent sequences must fall on the scatter sentinel row.
+    // Sequences present in this ubatch, only their pools are scored.
     std::vector<uint8_t> seq_in_ub(LLAMA_MAX_SEQ, 0);
     for (uint32_t i = 0; i < n_tokens; ++i) {
         for (int32_t k = 0; k < ubatch->n_seq_id[i]; ++k) {
@@ -846,27 +854,26 @@ void llama_memory_hybrid_idx_context::set_input_kpool(ggml_tensor * pool_cells,
         std::fill(npos, npos + 4*n_new_g, 0);
     }

-    uint32_t i_new = 0;
+    // the reps of all pools, the padded new entries must not write them
+    const int64_t n_cells = (int64_t) kv_size*n_stream_kv;
+    std::vector<uint8_t> is_rep(n_new_g > n_new ? n_cells : 0, 0);
+
+    uint32_t i_new  = 0;
+    uint32_t ip_all = 0;
     for (llama_seq_id s = 0; s < LLAMA_MAX_SEQ; ++s) {
         const auto & sq = lay.seqs[s];
         seq_pool_start[s] = (uint32_t) pool_end.size();

-        const bool inert = n_stream_kv > 1 && !seq_in_ub[s];
-
-        for (size_t pi = 0; pi < sq.pools.size(); ++pi) {
-            const uint32_t j  = sq.pools[pi];
-            const uint32_t ip = (uint32_t) pool_end.size();
-            GGML_ASSERT(ip + 1 < n_pool);
+        for (size_t pi = 0; pi < sq.pools.size(); ++pi, ++ip_all) {
+            const uint32_t j = sq.pools[pi];

             // The pooled key lives in the last member's row.
             const uint32_t rep = sq.cells[j + kpool - 1].second;
-            pcell[ip] = (int32_t) gcell(sq, rep);
-
-            for (uint32_t k = 0; k < kpool; ++k) {
-                pidx[(size_t) ip*kpool + k] = inert ? sentinel : (int32_t) sq.cells[j + k].second;
+            if (!is_rep.empty()) {
+                is_rep[gcell(sq, rep)] = 1;
             }

-            if (st.is_new[ip] == st.generation) {
+            if (st.is_new[ip_all] == st.generation) {
                 GGML_ASSERT(i_new < n_new);
                 for (uint32_t k = 0; k < kpool; ++k) {
                     nidx[(size_t) i_new*kpool + k] = (int32_t) gcell(sq, sq.cells[j + k].second);
@@ -884,23 +891,33 @@ void llama_memory_hybrid_idx_context::set_input_kpool(ggml_tensor * pool_cells,
                 ++i_new;
             }

+            if (!seq_in_ub[s]) {
+                continue;
+            }
+
+            const uint32_t ip = (uint32_t) pool_end.size();
+            GGML_ASSERT(ip + 1 < n_pool);
+
+            pcell[ip] = (int32_t) gcell(sq, rep);
+            for (uint32_t k = 0; k < kpool; ++k) {
+                pidx[(size_t) ip*kpool + k] = (int32_t) sq.cells[j + k].second;
+            }
+
             pool_end.push_back(sq.cells[j + kpool - 1].first);
         }
     }
     GGML_ASSERT(i_new == n_new);
+    GGML_ASSERT(ip_all == st.n_pool_real && pool_end.size() == st.n_pool_ub);

     // Padded entries re-pool cells whose pooled slot is never read: only the reps of complete pools are read.
     // Each entry takes its own cell, entries sharing one would write it from several threads in the scatter.
     if (n_new_g > n_new) {
-        std::vector<int64_t> reps(pcell, pcell + pool_end.size());
-        std::sort(reps.begin(), reps.end());
-
         int64_t pad_cell = 0;
         for (uint32_t i = n_new; i < n_new_g; ++i, ++pad_cell) {
-            while (std::binary_search(reps.begin(), reps.end(), pad_cell)) {
+            while (pad_cell < n_cells && is_rep[pad_cell]) {
                 ++pad_cell;
             }
-            GGML_ASSERT(pad_cell < (int64_t) kv_size*n_stream_kv);
+            GGML_ASSERT(pad_cell < n_cells);
             for (uint32_t k = 0; k < kpool; ++k) {
                 nidx[(size_t) i*kpool + k] = (int32_t) pad_cell;
             }