Commit e412c6f46 for llama.cpp
commit e412c6f468b7b9e0f16f2540244236f8ad68e9ae
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Sat Nov 21 10:26:55 2026 +0200
llama : remove fused lightning indexer gating (#32650)
The fused lightning indexer is always used now, so drop the cparams flags, the auto probe and the non-fused fallbacks.
Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4.1-Flash-1017-GGUF (Mac Studio, M5 Ultra, 256GB)
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index ea41283f9..7490cf379 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -136,12 +136,6 @@ static const llm_fused_op_probe llm_fused_op_flash_attn_kv_rows_probe = {
/*.n_tokens_per_seq =*/ 1,
};
-static const llm_fused_op_probe llm_fused_op_lid_probe = {
- /*.op =*/ LLM_FUSED_OP_LIGHTNING_INDEXER,
- /*.name =*/ "Lightning Indexer",
- /*.n_tokens_per_seq =*/ 1,
-};
-
static const llm_fused_op_probe llm_fused_op_dsv4_hc_pre_probe = {
/*.op =*/ LLM_FUSED_OP_DSV4_HC_PRE,
/*.name =*/ "fused DeepSeek V4 HC pre",
@@ -334,9 +328,6 @@ llama_context::llama_context(
cparams.auto_fa = false;
}
- cparams.fused_lid = true;
- cparams.auto_flid = false;
-
cparams.fused_dsv4_hc_pre = true;
cparams.fused_dsv4_hc_comb = true;
cparams.fused_dsv4_hc_post = true;
@@ -725,12 +716,6 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3
cparams.auto_fkvr = false;
}
- if (cparams.auto_flid) {
- LLAMA_LOG_INFO("%s: resolving fused Lightning Indexer support:\n", func);
- resolve(llm_fused_op_lid_probe, cparams.fused_lid);
- cparams.auto_flid = false;
- }
-
if (cparams.auto_fhc) {
LLAMA_LOG_INFO("%s: resolving fused DeepSeek V4 HC support:\n", func);
resolve(llm_fused_op_dsv4_hc_pre_probe, cparams.fused_dsv4_hc_pre);
diff --git a/src/llama-cparams.h b/src/llama-cparams.h
index 1f890d6fa..c93494faf 100644
--- a/src/llama-cparams.h
+++ b/src/llama-cparams.h
@@ -39,8 +39,6 @@ struct llama_cparams {
bool offload_kqv;
bool flash_attn;
bool auto_fa;
- bool fused_lid; // use fused lightning indexer
- bool auto_flid;
bool fused_dsv4_hc_pre;
bool fused_dsv4_hc_comb;
bool fused_dsv4_hc_post;
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index 269b920fb..c5c9ddf01 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -34,13 +34,14 @@ static ggml_tensor * build_attn_inp_kq_mask(
ggml_context * ctx,
const llama_kv_cache_context * mctx,
const llama_ubatch & ubatch,
- const llama_cparams & cparams) {
+ const llama_cparams & cparams,
+ bool force_f16 = false) {
const auto n_kv = mctx->get_n_kv();
const auto n_tokens = ubatch.n_tokens;
const auto n_stream = cparams.kv_unified ? 1 : ubatch.n_seqs_unq;
- // flash attention requires an f16 mask
- const auto type = cparams.flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32;
+ // flash attention and the lightning indexer require an f16 mask
+ const auto type = (cparams.flash_attn || force_f16) ? GGML_TYPE_F16 : GGML_TYPE_F32;
ggml_tensor * res = ggml_new_tensor_4d(ctx, type, n_kv, n_tokens/n_stream, 1, n_stream);
ggml_set_input(res);
@@ -1146,7 +1147,7 @@ static void dsv4_build_comp_inputs(
GGML_ASSERT(n_stream > 0);
GGML_ASSERT(n_tokens%n_stream == 0);
- inp.kq_mask = ggml_new_tensor_4d(ctx, (strcmp(name, "lid") != 0 && cparams.flash_attn) || (strcmp(name, "lid") == 0 && cparams.fused_lid) ? GGML_TYPE_F16 : GGML_TYPE_F32, plan.n_kv, n_tokens/n_stream, 1, n_stream);
+ inp.kq_mask = ggml_new_tensor_4d(ctx, (strcmp(name, "lid") == 0 || cparams.flash_attn) ? GGML_TYPE_F16 : GGML_TYPE_F32, plan.n_kv, n_tokens/n_stream, 1, n_stream);
ggml_set_input(inp.kq_mask);
ggml_set_name(inp.kq_mask, (std::string("dsv4_") + name + "_kq_mask").c_str());
}
@@ -4034,11 +4035,7 @@ static std::unique_ptr<llm_graph_input_attn_k_dsa> build_attn_inp_k_dsa_impl(
{
inp->self_k_idxs_lid = mctx_cur->get_lid()->build_input_k_idxs(ctx0, ubatch);
- // ensure that mask type matches fused lightning indexer use (requires f16 mask)
- auto cparams_copy = cparams;
- cparams_copy.flash_attn = cparams.fused_lid;
-
- inp->self_attn_mask_lid.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_lid(), ubatch, cparams_copy);
+ inp->self_attn_mask_lid.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_lid(), ubatch, cparams, true);
inp->self_k_rot_lid = mctx_cur->get_lid()->get_k_rot();
}
diff --git a/src/llama-graph.h b/src/llama-graph.h
index b5956ead9..a9f8e5ce0 100644
--- a/src/llama-graph.h
+++ b/src/llama-graph.h
@@ -49,7 +49,6 @@ enum llm_graph_type {
enum llm_fused_op {
LLM_FUSED_OP_FLASH_ATTN,
LLM_FUSED_OP_FLASH_ATTN_KV_ROWS,
- LLM_FUSED_OP_LIGHTNING_INDEXER,
LLM_FUSED_OP_DSV4_HC_PRE,
LLM_FUSED_OP_DSV4_HC_COMB,
LLM_FUSED_OP_DSV4_HC_POST,
diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cpp
index fc6207f8b..853389b88 100644
--- a/src/models/deepseek32.cpp
+++ b/src/models/deepseek32.cpp
@@ -265,46 +265,8 @@ llama_model_deepseek32::graph::graph(const llm_graph_params & params) :
indexer_weights = ggml_scale(ctx0, indexer_weights, 1.0f / sqrtf(float(n_embd_indexer_head * n_indexer_head)));
cb(indexer_weights, "indexer_weights", il);
- ggml_tensor * indexer_score = nullptr;
- if (cparams.fused_lid) {
- indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw);
- cb(indexer_score, "indexer_score", il);
- res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il});
- } else {
- // calculate indexer kq
- indexer_q = ggml_permute(ctx0, indexer_q, 0, 2, 1, 3);
- cb(indexer_q, "indexer_q", il);
- indexer_k = ggml_permute(ctx0, indexer_k, 0, 2, 1, 3);
- cb(indexer_k, "indexer_k", il);
-
- ggml_tensor * indexer_kq = ggml_mul_mat(ctx0, indexer_k, indexer_q);
- cb(indexer_kq, "indexer_kq", il);
-
- // ReLU requires contiguous tensors
- indexer_kq = ggml_cont(ctx0, ggml_permute(ctx0, indexer_kq, 2, 1, 0, 3));
- cb(indexer_kq, "indexer_kq", il);
-
- // apply ReLU
- indexer_score = ggml_relu(ctx0, indexer_kq);
- cb(indexer_score, "indexer_score", il);
-
- // multiply scores by indexer weights
- indexer_score = ggml_mul(ctx0, indexer_score, indexer_weights);
- cb(indexer_score, "indexer_score", il);
-
- // sum by q n_indexer_head dimension
- indexer_score = ggml_sum_rows(ctx0, indexer_score);
- cb(indexer_score, "indexer_score", il);
-
- // permute result to match KQ mask
- indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3));
- cb(indexer_score, "indexer_score", il);
-
- // mask indexer scores
- ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw;
- indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask);
- cb(indexer_score, "indexer_score", il);
- }
+ ggml_tensor * indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw);
+ cb(indexer_score, "indexer_score", il);
// get indices of top k indexer scores
uint32_t n_top_k = indexer_score->ne[0] < n_indexer_top_k ? indexer_score->ne[0] : n_indexer_top_k;
diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp
index 69ebe5a60..dfc9858e7 100644
--- a/src/models/deepseek4.cpp
+++ b/src/models/deepseek4.cpp
@@ -642,32 +642,8 @@ ggml_tensor * llama_model_deepseek4::graph::build_lid_top_k(
indexer_weights->ne[0], indexer_weights->ne[1]/n_stream, indexer_weights->ne[2], n_stream,
indexer_weights->nb[1], indexer_weights->nb[2]/n_stream, indexer_weights->nb[3]/n_stream, 0);
- ggml_tensor * indexer_score = nullptr;
- if (cparams.fused_lid) {
- indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_lid.kq_mask);
- cb(indexer_score, "lid_score_masked", il);
- res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il});
- } else {
- indexer_q = ggml_permute(ctx0, indexer_q, 0, 2, 1, 3);
- cb(indexer_q, "lid_q", il);
- indexer_k = ggml_permute(ctx0, indexer_k, 0, 2, 1, 3);
- cb(indexer_k, "lid_k", il);
-
- ggml_tensor * indexer_kq = ggml_mul_mat(ctx0, indexer_k, indexer_q);
- cb(indexer_kq, "lid_kq", il);
-
- indexer_kq = ggml_cont(ctx0, ggml_permute(ctx0, indexer_kq, 2, 1, 0, 3));
- cb(indexer_kq, "lid_kq", il);
-
- indexer_score = ggml_relu(ctx0, indexer_kq);
- indexer_score = ggml_mul(ctx0, indexer_score, indexer_weights);
- indexer_score = ggml_sum_rows(ctx0, indexer_score);
- indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3));
- cb(indexer_score, "lid_score", il);
-
- indexer_score = ggml_add(ctx0, indexer_score, inp_lid.kq_mask);
- cb(indexer_score, "lid_score_masked", il);
- }
+ ggml_tensor * indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_lid.kq_mask);
+ cb(indexer_score, "lid_score_masked", il);
const uint32_t n_top_k = indexer_score->ne[0] < hparams.indexer_top_k ? indexer_score->ne[0] : hparams.indexer_top_k;
ggml_tensor * top_k = ggml_cont(ctx0, ggml_top_k(ctx0, indexer_score, n_top_k));
diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp
index 96de90412..ed93be534 100644
--- a/src/models/deepseek41.cpp
+++ b/src/models/deepseek41.cpp
@@ -533,7 +533,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41(
ggml_tensor * ik = ggml_view_4d(ctx0, comp_all, n_idx_embd, 1, n_comp, n_stream,
comp_all->nb[1], comp_all->nb[2], comp_all->nb[3], ggml_row_size(comp_all->type, n_embd_head));
// the fused indexer reads the strided view in place, the unfused path needs it contiguous; CUDA has no same-type copy of quantized views, so dequantize them
- const bool fused_idx = sparse && cparams.fused_lid && ci.kq_mask->type == GGML_TYPE_F16;
+ const bool fused_idx = sparse && ci.kq_mask->type == GGML_TYPE_F16;
if (ggml_is_quantized(ik->type)) {
ik = ggml_cast(ctx0, ik, GGML_TYPE_F32);
} else if (!fused_idx) {
@@ -572,7 +572,6 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41(
ggml_tensor * score = nullptr;
if (fused_idx) {
score = ggml_lightning_indexer(ctx0, iq, ik, iw, ci.kq_mask);
- res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il});
} else {
ggml_tensor * iqc = ggml_view_4d(ctx0, iq, iq->ne[0], iq->ne[1], nc, n_stream, iq->nb[1], iq->nb[2], iq->nb[3], t0*iq->nb[2]);
diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cpp
index a6fccd58f..da16f2480 100644
--- a/src/models/dots3note.cpp
+++ b/src/models/dots3note.cpp
@@ -257,42 +257,8 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
indexer_weights = ggml_scale(ctx0, indexer_weights, 1.0f / sqrtf(float(n_embd_indexer_head * n_indexer_head)));
cb(indexer_weights, "indexer_weights", il);
- ggml_tensor * indexer_score = nullptr;
- if (cparams.fused_lid) {
- indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn->get_dsa()->self_attn_mask_lid.raw);
- cb(indexer_score, "indexer_score", il);
- res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il});
- } else {
- indexer_q = ggml_permute(ctx0, indexer_q, 0, 2, 1, 3);
- cb(indexer_q, "indexer_q", il);
- indexer_k = ggml_permute(ctx0, indexer_k, 0, 2, 1, 3);
- cb(indexer_k, "indexer_k", il);
-
- ggml_tensor * indexer_kq = ggml_mul_mat(ctx0, indexer_k, indexer_q);
- cb(indexer_kq, "indexer_kq", il);
-
- // ReLU requires contiguous tensors
- indexer_kq = ggml_cont(ctx0, ggml_permute(ctx0, indexer_kq, 2, 1, 0, 3));
- cb(indexer_kq, "indexer_kq", il);
-
- indexer_score = ggml_relu(ctx0, indexer_kq);
- cb(indexer_score, "indexer_score", il);
-
- indexer_score = ggml_mul(ctx0, indexer_score, indexer_weights);
- cb(indexer_score, "indexer_score", il);
-
- // sum by q n_indexer_head dimension
- indexer_score = ggml_sum_rows(ctx0, indexer_score);
- cb(indexer_score, "indexer_score", il);
-
- // permute result to match KQ mask
- indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3));
- cb(indexer_score, "indexer_score", il);
-
- ggml_tensor * indexer_kq_mask = inp_attn->get_dsa()->self_attn_mask_lid.raw;
- indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask);
- cb(indexer_score, "indexer_score", il);
- }
+ ggml_tensor * indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn->get_dsa()->self_attn_mask_lid.raw);
+ cb(indexer_score, "indexer_score", il);
// get indices of top k indexer scores
uint32_t n_top_k = indexer_score->ne[0] < n_indexer_top_k ? indexer_score->ne[0] : n_indexer_top_k;
diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp
index 7007f427f..6daafd007 100644
--- a/src/models/glm-dsa.cpp
+++ b/src/models/glm-dsa.cpp
@@ -307,46 +307,8 @@ llama_model_glm_dsa::graph::graph(const llm_graph_params & params) :
indexer_weights = ggml_scale(ctx0, indexer_weights, 1.0f / sqrtf(float(n_embd_indexer_head * n_indexer_head)));
cb(indexer_weights, "indexer_weights", il);
- ggml_tensor * indexer_score = nullptr;
- if (cparams.fused_lid) {
- indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw);
- cb(indexer_score, "indexer_score", il);
- res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il});
- } else {
- // calculate indexer kq
- indexer_q = ggml_permute(ctx0, indexer_q, 0, 2, 1, 3);
- cb(indexer_q, "indexer_q", il);
- indexer_k = ggml_permute(ctx0, indexer_k, 0, 2, 1, 3);
- cb(indexer_k, "indexer_k", il);
-
- ggml_tensor * indexer_kq = ggml_mul_mat(ctx0, indexer_k, indexer_q);
- cb(indexer_kq, "indexer_kq", il);
-
- // ReLU requires contiguous tensors
- indexer_kq = ggml_cont(ctx0, ggml_permute(ctx0, indexer_kq, 2, 1, 0, 3));
- cb(indexer_kq, "indexer_kq", il);
-
- // apply ReLU
- indexer_score = ggml_relu(ctx0, indexer_kq);
- cb(indexer_score, "indexer_score", il);
-
- // multiply scores by indexer weights
- indexer_score = ggml_mul(ctx0, indexer_score, indexer_weights);
- cb(indexer_score, "indexer_score", il);
-
- // sum by q n_indexer_head dimension
- indexer_score = ggml_sum_rows(ctx0, indexer_score);
- cb(indexer_score, "indexer_score", il);
-
- // permute result to match KQ mask
- indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3));
- cb(indexer_score, "indexer_score", il);
-
- // mask indexer scores
- ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw;
- indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask);
- cb(indexer_score, "indexer_score", il);
- }
+ ggml_tensor * indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw);
+ cb(indexer_score, "indexer_score", il);
// get indices of top k indexer scores
uint32_t n_top_k = indexer_score->ne[0] < n_indexer_top_k ? indexer_score->ne[0] : n_indexer_top_k;
diff --git a/src/models/glm5-next.cpp b/src/models/glm5-next.cpp
index 844a327b3..51a26df3d 100644
--- a/src/models/glm5-next.cpp
+++ b/src/models/glm5-next.cpp
@@ -280,15 +280,12 @@ llama_model_glm5_next::llm_graph_input_kpool * llama_model_glm5_next::graph::bui
// change shape every kpool tokens
const uint32_t n_new = std::max(mctx_hyb->get_n_kpool_new(), 1u);
- // the fused lightning indexer wants an f16 mask
- const auto type_mask = cparams.fused_lid ? GGML_TYPE_F16 : GGML_TYPE_F32;
-
auto inp = std::make_unique<llm_graph_input_kpool>(mctx_hyb, kpool);
inp->k_idxs = mctx_idx->build_input_k_idxs(ctx0, ubatch);
inp->pool_cells = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pool);
inp->pool_idxs = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, kpool, n_pool);
- inp->pool_mask = ggml_new_tensor_2d(ctx0, type_mask, n_pool, n_tokens);
+ inp->pool_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F16, n_pool, n_tokens);
inp->tail_idxs = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, kpool - 1, n_tokens);
ggml_set_input(inp->pool_cells);
ggml_set_input(inp->pool_idxs);
@@ -934,22 +931,7 @@ ggml_tensor * llama_model_glm5_next::graph::build_kpool_select(
weights = ggml_scale(ctx0, weights, 1.0f / sqrtf(float(n_embd_indexer * n_indexer_head)));
cb(weights, "indexer_weights", il);
- ggml_tensor * score = nullptr;
- if (cparams.fused_lid) {
- score = ggml_lightning_indexer(ctx0, iq, pooled, weights, inp_kpool->pool_mask);
- res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il});
- } else {
- ggml_tensor * q_p = ggml_permute(ctx0, iq, 0, 2, 1, 3); // [head_dim, n_tokens, n_head]
- ggml_tensor * k_p = ggml_permute(ctx0, pooled, 0, 2, 1, 3); // [head_dim, n_pool, 1]
-
- ggml_tensor * kq = ggml_mul_mat(ctx0, k_p, q_p); // [n_pool, n_tokens, n_head]
- kq = ggml_cont(ctx0, ggml_permute(ctx0, kq, 2, 1, 0, 3)); // [n_head, n_tokens, n_pool]
- score = ggml_relu(ctx0, kq);
- score = ggml_mul(ctx0, score, weights);
- score = ggml_sum_rows(ctx0, score); // [1, n_tokens, n_pool]
- score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3)); // [n_pool, n_tokens, 1]
- score = ggml_add(ctx0, score, inp_kpool->pool_mask);
- }
+ ggml_tensor * score = ggml_lightning_indexer(ctx0, iq, pooled, weights, inp_kpool->pool_mask);
cb(score, "indexer_score", il);
const int64_t n_top_pool = std::min<int64_t>(n_pool, hparams.indexer_top_k / kpool);
diff --git a/src/models/hy-v4.cpp b/src/models/hy-v4.cpp
index 386d504bc..de7d87cd0 100644
--- a/src/models/hy-v4.cpp
+++ b/src/models/hy-v4.cpp
@@ -393,24 +393,8 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k(
// fold both reference scale factors into the weights before the big score tensor
iw = ggml_scale(ctx0, iw, 1.0f / sqrtf(float(n_embd_indexer * n_indexer_head)));
- ggml_tensor * score = nullptr;
- if (cparams.fused_lid) {
- score = ggml_lightning_indexer(ctx0, iq, ik, iw, inp_attn_dsa->self_attn_mask_lid.raw);
- cb(score, "indexer_score", il);
- res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il});
- } else {
- iq = ggml_permute(ctx0, iq, 0, 2, 1, 3);
- ik = ggml_permute(ctx0, ik, 0, 2, 1, 3);
-
- score = ggml_mul_mat(ctx0, ik, iq);
- score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3));
- score = ggml_relu(ctx0, score);
- score = ggml_mul(ctx0, score, iw);
- score = ggml_sum_rows(ctx0, score);
- score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3));
- score = ggml_add(ctx0, score, inp_attn_dsa->self_attn_mask_lid.raw);
- cb(score, "indexer_score", il);
- }
+ ggml_tensor * score = ggml_lightning_indexer(ctx0, iq, ik, iw, inp_attn_dsa->self_attn_mask_lid.raw);
+ cb(score, "indexer_score", il);
const uint32_t n_top_k = score->ne[0] < (int64_t) hparams.indexer_top_k ? score->ne[0] : hparams.indexer_top_k;
diff --git a/src/models/kimi-k31.cpp b/src/models/kimi-k31.cpp
index 0c0ca2af2..3f99eca4a 100644
--- a/src/models/kimi-k31.cpp
+++ b/src/models/kimi-k31.cpp
@@ -581,28 +581,11 @@ llama_model_kimi_k31::graph::build_dsa_layer(
iw->nb[1], iw->nb[2]/n_stream, iw->nb[2]/n_stream, 0);
ggml_tensor * kq_mask_lid = inp_attn->self_attn_mask.raw;
- if (cparams.fused_lid && kq_mask_lid->type != GGML_TYPE_F16) {
+ if (kq_mask_lid->type != GGML_TYPE_F16) {
kq_mask_lid = ggml_cast(ctx0, kq_mask_lid, GGML_TYPE_F16);
}
- ggml_tensor * score;
- if (cparams.fused_lid) {
- score = ggml_lightning_indexer(ctx0, iq, ik, iw, kq_mask_lid);
- res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il});
- } else {
- ggml_tensor * q_p = ggml_permute(ctx0, iq, 0, 2, 1, 3); // [n_idx_dim, n_tokens, n_idx_head]
- ggml_tensor * k_p = ggml_permute(ctx0, ik, 0, 2, 1, 3); // [n_idx_dim, n_kv, 1]
-
- ggml_tensor * kq = ggml_mul_mat(ctx0, k_p, q_p); // [n_kv, n_tokens, n_idx_head]
- kq = ggml_cont(ctx0, ggml_permute(ctx0, kq, 2, 1, 0, 3)); // [n_idx_head, n_tokens, n_kv]
-
- score = ggml_relu(ctx0, kq);
- score = ggml_mul(ctx0, score, iw);
- score = ggml_sum_rows(ctx0, score); // [1, n_tokens, n_kv]
- score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3)); // [n_kv, n_tokens, 1]
-
- score = ggml_add(ctx0, score, kq_mask_lid);
- }
+ ggml_tensor * score = ggml_lightning_indexer(ctx0, iq, ik, iw, kq_mask_lid);
cb(score, "indexer_score", il);
const int64_t n_top_k = std::min<int64_t>(score->ne[0], hparams.indexer_top_k);
diff --git a/src/models/mistral-large4.cpp b/src/models/mistral-large4.cpp
index a4315a991..6407d7663 100644
--- a/src/models/mistral-large4.cpp
+++ b/src/models/mistral-large4.cpp
@@ -290,46 +290,8 @@ llama_model_mistral_large4::graph::graph(const llm_graph_params & params) :
indexer_weights = ggml_scale(ctx0, indexer_weights, 1.0f / sqrtf(float(n_embd_indexer_head * n_indexer_head)));
cb(indexer_weights, "indexer_weights", il);
- ggml_tensor * indexer_score = nullptr;
- if (cparams.fused_lid) {
- indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw);
- cb(indexer_score, "indexer_score", il);
- res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il});
- } else {
- // calculate indexer kq
- indexer_q = ggml_permute(ctx0, indexer_q, 0, 2, 1, 3);
- cb(indexer_q, "indexer_q", il);
- indexer_k = ggml_permute(ctx0, indexer_k, 0, 2, 1, 3);
- cb(indexer_k, "indexer_k", il);
-
- ggml_tensor * indexer_kq = ggml_mul_mat(ctx0, indexer_k, indexer_q);
- cb(indexer_kq, "indexer_kq", il);
-
- // ReLU requires contiguous tensors
- indexer_kq = ggml_cont(ctx0, ggml_permute(ctx0, indexer_kq, 2, 1, 0, 3));
- cb(indexer_kq, "indexer_kq", il);
-
- // apply ReLU
- indexer_score = ggml_relu(ctx0, indexer_kq);
- cb(indexer_score, "indexer_score", il);
-
- // multiply scores by indexer weights
- indexer_score = ggml_mul(ctx0, indexer_score, indexer_weights);
- cb(indexer_score, "indexer_score", il);
-
- // sum by q n_indexer_head dimension
- indexer_score = ggml_sum_rows(ctx0, indexer_score);
- cb(indexer_score, "indexer_score", il);
-
- // permute result to match KQ mask
- indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3));
- cb(indexer_score, "indexer_score", il);
-
- // mask indexer scores
- ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw;
- indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask);
- cb(indexer_score, "indexer_score", il);
- }
+ ggml_tensor * indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw);
+ cb(indexer_score, "indexer_score", il);
// get indices of top k indexer scores
uint32_t n_top_k = indexer_score->ne[0] < n_indexer_top_k ? indexer_score->ne[0] : n_indexer_top_k;
diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp
index 7bf9b3fee..63c918602 100644
--- a/src/models/qwen4exp.cpp
+++ b/src/models/qwen4exp.cpp
@@ -872,7 +872,6 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel(
// which is the lightning indexer with every head weight set to that scale
ggml_tensor * weights = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_idx_h, n_tokens), 1.0f/sqrtf((float) idx_dim));
ggml_tensor * score = ggml_lightning_indexer(ctx0, q, pooled, weights, inp_kpool->pool_mask); // [n_pool, n_tokens]
- res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il});
cb(score, "indexer_score", il);
const int64_t n_top_pool = std::min<int64_t>(n_pool, hparams.indexer_top_k / kpool);