Commit ca8daad19 for llama.cpp
commit ca8daad199a9c12c5b54643b974c640939a0f7bd
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Tue Nov 3 08:11:06 2026 +0200
llama : drop the redundant cnv from llm_attn_mask (#31558)
The dspark block mask is now derived at input-fill time: llm_attn_mask::block_uniform
fills every token of a sequence with the mask row of its first token, so the
graph-level reshape/repeat in dflash is gone and cnv always aliases raw.
Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (Mac Studio, M5 Ultra, 256GB)
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index fe8bdebe5..f0463f849 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -124,8 +124,6 @@ static llm_attn_mask build_attn_inp_attn_mask(
ggml_set_name(attn_mask.raw, "attn_inp_kq_mask");
}
- attn_mask.cnv = attn_mask.raw;
-
return attn_mask;
}
@@ -3131,16 +3129,11 @@ llm_graph_input_attn_no_cache * llm_graph_context::build_attn_inp_no_cache() con
ggml_set_input(inp->self_attn_mask.raw);
cb(inp->self_attn_mask.raw, "self_kq_mask", -1);
- inp->self_attn_mask.cnv = inp->self_attn_mask.raw;
-
if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
inp->self_attn_mask_swa.raw = ggml_new_tensor_4d(ctx0, type_mask, n_kv, n_tokens, 1, 1);
ggml_set_input(inp->self_attn_mask_swa.raw);
-
- inp->self_attn_mask_swa.cnv = inp->self_attn_mask_swa.raw;
} else {
inp->self_attn_mask_swa.raw = nullptr;
- inp->self_attn_mask_swa.cnv = nullptr;
}
return (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp));
@@ -3169,7 +3162,7 @@ ggml_tensor * llm_graph_context::build_attn(
const bool is_swa = hparams.is_swa(il);
- const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.cnv : inp->self_attn_mask.cnv;
+ const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw;
const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows;
// [TAG_NO_CACHE_PAD]
@@ -3296,7 +3289,7 @@ ggml_tensor * llm_graph_context::build_attn(
v = mctx_cur->get_v(ctx0, il);
}
- const auto & kq_mask = inp->self_attn_mask.cnv;
+ const auto & kq_mask = inp->self_attn_mask.raw;
const auto & kv_rows = inp->self_attn_mask.rows;
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, 0, kq_scale, il);
@@ -3341,7 +3334,6 @@ static std::unique_ptr<llm_graph_input_attn_k> build_attn_inp_k_impl(
inp->self_k_idxs = mctx_cur->build_input_k_idxs(ctx0, ubatch);
inp->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_cur, ubatch, cparams);
- inp->self_attn_mask.cnv = inp->self_attn_mask.raw;
}
return inp;
@@ -3384,7 +3376,7 @@ ggml_tensor * llm_graph_context::build_attn(
ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il));
}
- const auto & kq_mask = inp->self_attn_mask.cnv;
+ const auto & kq_mask = inp->self_attn_mask.raw;
const auto & kv_rows = inp->self_attn_mask.rows;
ggml_tensor * q = q_cur;
@@ -3465,9 +3457,9 @@ ggml_tensor * llm_graph_context::build_attn(
// top_k indexes the cells, and the indexer mask of the same type is built the same way
// let's use it, so when no other layer reads the MLA mask is not allocated
- ggml_tensor * kq_mask = inp->self_attn_mask_mla.cnv;
- if (inp->self_attn_mask_lid.cnv->type == kq_mask->type) {
- kq_mask = inp->self_attn_mask_lid.cnv;
+ ggml_tensor * kq_mask = inp->self_attn_mask_mla.raw;
+ if (inp->self_attn_mask_lid.raw->type == kq_mask->type) {
+ kq_mask = inp->self_attn_mask_lid.raw;
}
ggml_tensor * kq_mask_top_k = nullptr;
@@ -3589,7 +3581,7 @@ ggml_tensor * llm_graph_context::build_attn(
ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));
}
- const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.cnv : inp->self_attn_mask.cnv;
+ const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw;
const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows;
ggml_tensor * q = q_cur;
@@ -3660,7 +3652,7 @@ ggml_tensor * llm_graph_context::build_attn(
ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il));
}
- const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.cnv : inp->self_attn_mask.cnv;
+ const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw;
const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows;
// MLA-style attention: the cached K is used as V
@@ -3697,8 +3689,6 @@ llm_graph_input_attn_cross * llm_graph_context::build_attn_inp_cross() const {
inp->cross_attn_mask.raw = ggml_new_tensor_4d(ctx0, type_mask, n_enc, n_tokens, 1, 1);
ggml_set_input(inp->cross_attn_mask.raw);
- inp->cross_attn_mask.cnv = inp->cross_attn_mask.raw;
-
return (llm_graph_input_attn_cross *) res->add_input(std::move(inp));
}
@@ -3721,7 +3711,7 @@ ggml_tensor * llm_graph_context::build_attn(
ggml_build_forward_expand(gf, k_cur);
ggml_build_forward_expand(gf, v_cur);
- const auto & kq_mask = inp->cross_attn_mask.cnv;
+ const auto & kq_mask = inp->cross_attn_mask.raw;
const auto & kv_rows = inp->cross_attn_mask.rows;
ggml_tensor * q = q_cur;
@@ -3759,7 +3749,6 @@ static std::unique_ptr<llm_graph_input_attn_k_dsa> build_attn_inp_k_dsa_impl(
inp->self_k_idxs_mla = mctx_cur->get_mla()->build_input_k_idxs(ctx0, ubatch);
inp->self_attn_mask_mla.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_mla(), ubatch, cparams);
- inp->self_attn_mask_mla.cnv = inp->self_attn_mask_mla.raw;
}
{
@@ -3770,7 +3759,6 @@ static std::unique_ptr<llm_graph_input_attn_k_dsa> build_attn_inp_k_dsa_impl(
cparams_copy.flash_attn = cparams.fused_lid;
inp->self_attn_mask_lid.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_lid(), ubatch, cparams_copy);
- inp->self_attn_mask_lid.cnv = inp->self_attn_mask_lid.raw;
inp->self_k_rot_lid = mctx_cur->get_lid()->get_k_rot();
}
@@ -3797,7 +3785,6 @@ llm_graph_input_attn_k_dsa_iswa * llm_graph_context::build_attn_inp_k_dsa_iswa()
inp_swa->self_k_idxs = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch);
inp_swa->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams);
- inp_swa->self_attn_mask.cnv = inp_swa->self_attn_mask.raw;
auto inp = std::make_unique<llm_graph_input_attn_k_dsa_iswa>(std::move(inp_dsa), std::move(inp_swa), mctx_cur);
@@ -3819,7 +3806,6 @@ llm_graph_input_attn_kv_msa * llm_graph_context::build_attn_inp_kv_msa(bool msa_
inp->self_v_idxs = mctx_base->build_input_v_idxs(ctx0, ubatch);
inp->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_base, ubatch, cparams);
- inp->self_attn_mask.cnv = inp->self_attn_mask.raw;
}
inp->self_k_rot = mctx_base->get_k_rot();
@@ -3874,7 +3860,6 @@ llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const {
inp->self_k_idxs = mctx_cur->get_base()->build_input_k_idxs(ctx0, ubatch);
inp->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_base(), ubatch, cparams);
- inp->self_attn_mask.cnv = inp->self_attn_mask.raw;
}
{
@@ -3883,7 +3868,6 @@ llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const {
inp->self_k_idxs_swa = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch);
inp->self_attn_mask_swa.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams);
- inp->self_attn_mask_swa.cnv = inp->self_attn_mask_swa.raw;
}
inp->self_k_rot = mctx_cur->get_base()->get_k_rot();
@@ -3905,7 +3889,6 @@ llm_graph_input_dsv4 * llm_graph_context::build_inp_dsv4() const {
inp_raw->self_k_idxs = raw_ctx->build_input_k_idxs(ctx0, ubatch);
inp_raw->self_attn_mask.raw = dsv4_build_raw_kq_mask(ctx0, raw_ctx, ubatch, cparams, n_stream);
- inp_raw->self_attn_mask.cnv = inp_raw->self_attn_mask.raw;
inp_raw->self_k_rot = raw_ctx->get_k_rot();
auto inp = std::make_unique<llm_graph_input_dsv4>(cparams, std::move(inp_raw), mctx_cur);
@@ -4089,7 +4072,6 @@ llm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa()
inp_attn->self_v_idxs = attn_ctx->get_base()->build_input_v_idxs(ctx0, ubatch);
inp_attn->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, attn_ctx->get_base(), ubatch, cparams);
- inp_attn->self_attn_mask.cnv = inp_attn->self_attn_mask.raw;
}
{
@@ -4097,7 +4079,6 @@ llm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa()
inp_attn->self_v_idxs_swa = attn_ctx->get_swa()->build_input_v_idxs(ctx0, ubatch);
inp_attn->self_attn_mask_swa.raw = build_attn_inp_kq_mask(ctx0, attn_ctx->get_swa(), ubatch, cparams);
- inp_attn->self_attn_mask_swa.cnv = inp_attn->self_attn_mask_swa.raw;
}
inp_attn->self_k_rot = attn_ctx->get_base()->get_k_rot();
diff --git a/src/llama-graph.h b/src/llama-graph.h
index 37b47acbb..ea2cf9f87 100644
--- a/src/llama-graph.h
+++ b/src/llama-graph.h
@@ -333,12 +333,13 @@ public:
// the mask has two shapes depending on whether `rows` is used:
// rows != nullptr: [n_kv_rows, n_batch/n_slices, 1, n_slices]
// rows == nullptr: [n_kv, n_batch/n_stream, 1, n_stream]
-// TODO: cnv is redundant (always aliases raw) except dflash overwrites it with a derived block mask [TAG_ATTN_MASK_CNV]
struct llm_attn_mask {
ggml_tensor * raw = nullptr;
- ggml_tensor * cnv = nullptr; // same shape
ggml_tensor * rows = nullptr; // I32 [n_kv_rows, n_slices]
uint32_t n_kv = 0;
+
+ // fill every token of a sequence with the mask row of its first token (dspark draft blocks)
+ bool block_uniform = false;
};
class llm_graph_input_attn_no_cache : public llm_graph_input_i {
diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp
index d9f46f0ad..06e218ba6 100644
--- a/src/llama-kv-cache.cpp
+++ b/src/llama-kv-cache.cpp
@@ -1683,6 +1683,9 @@ struct args_set_input_attn_mask {
// cell of each mask column per stream, nullptr if the columns are the cells
const int32_t * rows;
+
+ // every token of a sequence reuses the mask row of the first one (dspark draft blocks)
+ bool block_uniform;
};
template<typename T, bool causal, bool swa, bool is_2d, bool alibi>
@@ -1709,10 +1712,15 @@ static void set_input_attn_mask_impl(const args_set_input_attn_mask & args, T *
llama_pos seq_pos_min[LLAMA_MAX_SEQ];
std::fill(seq_pos_min, seq_pos_min + LLAMA_MAX_SEQ, INT32_MAX);
+ // first token of each sequence, used when the mask rows are shared within a sequence
+ uint32_t seq_first[LLAMA_MAX_SEQ];
+ std::fill(seq_first, seq_first + LLAMA_MAX_SEQ, ubatch->n_tokens);
+
for (uint32_t i = 0; i < ubatch->n_tokens; ++i) {
const llama_seq_id seq_id = ubatch->seq_id[i][0];
seq_pos_min[seq_id] = std::min(seq_pos_min[seq_id], ubatch->pos[i]);
+ seq_first[seq_id] = std::min(seq_first[seq_id], i);
}
for (uint32_t s = 0; s < n_stream; ++s) {
@@ -1729,12 +1737,15 @@ static void set_input_attn_mask_impl(const args_set_input_attn_mask & args, T *
const auto & cells = v_cells.at(seq_to_stream[seq_id]);
+ // shared mask rows: every query of the sequence uses the row of its first token
+ const uint32_t iq = args.block_uniform ? seq_first[seq_id] : i;
+
llama_pos p0 = -1;
- const llama_pos p1 = ubatch->pos[i];
+ const llama_pos p1 = ubatch->pos[iq];
// for M-RoPE
- const llama_pos p1_x = is_2d ? ubatch->pos[i + ubatch->n_tokens*2] : 0;
- const llama_pos p1_y = is_2d ? ubatch->pos[i + ubatch->n_tokens] : 0;
+ const llama_pos p1_x = is_2d ? ubatch->pos[iq + ubatch->n_tokens*2] : 0;
+ const llama_pos p1_y = is_2d ? ubatch->pos[iq + ubatch->n_tokens] : 0;
const uint64_t idst = n_kv*i;
@@ -1941,6 +1952,7 @@ void llama_kv_cache::set_input_attn_mask(const llm_attn_mask & attn_mask, const
/*.n_stream =*/ n_stream,
/*.n_tps =*/ n_tps,
/*.rows =*/ rows,
+ /*.block_uniform =*/ attn_mask.block_uniform,
};
if (dst->type == GGML_TYPE_F16) {
diff --git a/src/models/clef.cpp b/src/models/clef.cpp
index e9be0ec14..89d9419b8 100644
--- a/src/models/clef.cpp
+++ b/src/models/clef.cpp
@@ -352,8 +352,6 @@ llm_graph_input_attn_no_cache * llama_model_clef::graph::build_attn_inp_causal()
ggml_set_input(inp->self_attn_mask.raw);
cb(inp->self_attn_mask.raw, "self_kq_mask", -1);
- inp->self_attn_mask.cnv = inp->self_attn_mask.raw;
-
return (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp));
}
diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cpp
index 22f516a48..450022016 100644
--- a/src/models/deepseek32.cpp
+++ b/src/models/deepseek32.cpp
@@ -271,7 +271,7 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_
ggml_tensor * indexer_score = nullptr;
if (cparams.fused_lid) {
- indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.cnv);
+ indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw);
cb(indexer_score, "indexer_score", il);
res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il});
} else {
@@ -305,7 +305,7 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_
cb(indexer_score, "indexer_score", il);
// mask indexer scores
- ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.cnv;
+ ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw;
indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask);
cb(indexer_score, "indexer_score", il);
}
diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp
index 85fe141fd..1207c3bb7 100644
--- a/src/models/deepseek4.cpp
+++ b/src/models/deepseek4.cpp
@@ -759,7 +759,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_csa_lid_attention(
ggml_tensor * k_all = ggml_concat(ctx0, raw_k, csa_k, 2);
cb(k_all, "csa_k_all", il);
- ggml_tensor * raw_mask = inp_attn->self_attn_mask.cnv;
+ ggml_tensor * raw_mask = inp_attn->self_attn_mask.raw;
ggml_tensor * csa_mask = build_top_k_mask(inp_csa.kq_mask, top_k, "csa_top_k_mask", il);
ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, csa_mask, 0);
@@ -815,7 +815,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_hca_attention(
ggml_tensor * k_all = ggml_concat(ctx0, raw_k, hca_k, 2);
cb(k_all, "hca_k_all", il);
- ggml_tensor * raw_mask = inp_attn->self_attn_mask.cnv;
+ ggml_tensor * raw_mask = inp_attn->self_attn_mask.raw;
ggml_tensor * hca_mask = inp_hca.kq_mask;
ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, hca_mask, 0);
@@ -853,7 +853,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_raw_attention(
ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, kv, inp_attn->get_k_idxs(), il));
- ggml_tensor * kq_mask = inp_attn->self_attn_mask.cnv;
+ ggml_tensor * kq_mask = inp_attn->self_attn_mask.raw;
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp
index d82998970..c4ca567e7 100644
--- a/src/models/deepseek41.cpp
+++ b/src/models/deepseek41.cpp
@@ -642,7 +642,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41(
ggml_build_forward_expand(gf, inp_attn->mctx->cpy_k(ctx0, kv, inp_attn->get_k_idxs(), il));
ggml_tensor * raw_k = inp_attn->mctx->get_k(ctx0, il);
- ggml_tensor * raw_mask = inp_attn->self_attn_mask.cnv;
+ ggml_tensor * raw_mask = inp_attn->self_attn_mask.raw;
const int32_t topk_src = hparams.dsv41_topk_src[il];
const int64_t k_sel = top_k->ne[0];
diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp
index fdd67e017..0de2f38ec 100644
--- a/src/models/dflash.cpp
+++ b/src/models/dflash.cpp
@@ -1391,22 +1391,6 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
// * embd batch -> writes every stage's kv_norm(wkv(main_proj(target features))) into its window ring
// * token batch -> [anchor, noise...] block through the stages, single-pass mHC, no per-head q norm
// window of a block = window of its first query, see n_swa
-// TODO: move this to a new type of `llm_graph_input` and implement this in `set_input` [TAG_ATTN_MASK_CNV]
-static void build_dspark_block_mask(llm_graph_context & g, llm_graph_input_attn_k_iswa * inp_attn) {
- ggml_context * ctx0 = g.ctx0;
-
- ggml_tensor * m = inp_attn->self_attn_mask_swa.cnv;
- const int64_t n_tokens = g.n_tokens;
- const int64_t n_blocks = g.ubatch.n_seqs_unq;
- const int64_t bd = n_blocks > 0 ? n_tokens / n_blocks : 0;
- if (m && bd > 1 && n_tokens % n_blocks == 0 && m->ne[2] == 1 && m->ne[1] % bd == 0 && ggml_is_contiguous(m)) {
- ggml_tensor * m4 = ggml_reshape_4d(ctx0, m, m->ne[0], bd, m->ne[1]/bd, m->ne[3]);
- ggml_tensor * r0 = ggml_view_4d(ctx0, m4, m4->ne[0], 1, m4->ne[2], m4->ne[3], m4->nb[1], m4->nb[2], m4->nb[3], 0);
- m4 = ggml_repeat(ctx0, r0, m4);
- inp_attn->self_attn_mask_swa.cnv = ggml_reshape_4d(ctx0, m4, m->ne[0], m->ne[1], 1, m->ne[3]);
- g.cb(inp_attn->self_attn_mask_swa.cnv, "dspark_block_mask", -1);
- }
-}
llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const llm_graph_params & params) :
llama_model_deepseek41::graph(params) {
@@ -1496,7 +1480,8 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll
inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1);
cb(inpL, "hc_init", -1);
- build_dspark_block_mask(*this, inp_attn);
+ // the whole block shares the mask row of its first query (the anchor)
+ inp_attn->self_attn_mask_swa.block_uniform = true;
ggml_tensor * pre_prev = nullptr;
@@ -1743,7 +1728,8 @@ llama_model_dflash::graph_mistral_large4::graph_mistral_large4(const llama_model
}
cb(inpL, "inp_noise_embd", -1);
- build_dspark_block_mask(*this, inp_attn);
+ // the whole block shares the mask row of its first query (the anchor)
+ inp_attn->self_attn_mask_swa.block_uniform = true;
for (int il = 0; il < n_layer; ++il) {
const auto & layer = model.layers[il];
diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cpp
index df6fee7e4..c848b0fe2 100644
--- a/src/models/dots3note.cpp
+++ b/src/models/dots3note.cpp
@@ -256,7 +256,7 @@ llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_p
ggml_tensor * indexer_score = nullptr;
if (cparams.fused_lid) {
- indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn->get_dsa()->self_attn_mask_lid.cnv);
+ indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn->get_dsa()->self_attn_mask_lid.raw);
cb(indexer_score, "indexer_score", il);
res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il});
} else {
@@ -286,7 +286,7 @@ llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_p
indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3));
cb(indexer_score, "indexer_score", il);
- ggml_tensor * indexer_kq_mask = inp_attn->get_dsa()->self_attn_mask_lid.cnv;
+ ggml_tensor * indexer_kq_mask = inp_attn->get_dsa()->self_attn_mask_lid.raw;
indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask);
cb(indexer_score, "indexer_score", il);
}
diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp
index 0bf7a7c85..146112d36 100644
--- a/src/models/glm-dsa.cpp
+++ b/src/models/glm-dsa.cpp
@@ -305,7 +305,7 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par
ggml_tensor * indexer_score = nullptr;
if (cparams.fused_lid) {
- indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.cnv);
+ indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw);
cb(indexer_score, "indexer_score", il);
res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il});
} else {
@@ -339,7 +339,7 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par
cb(indexer_score, "indexer_score", il);
// mask indexer scores
- ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.cnv;
+ ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw;
indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask);
cb(indexer_score, "indexer_score", il);
}
diff --git a/src/models/hy-v4.cpp b/src/models/hy-v4.cpp
index 06833e20b..945e0cc37 100644
--- a/src/models/hy-v4.cpp
+++ b/src/models/hy-v4.cpp
@@ -395,7 +395,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k(
ggml_tensor * score = nullptr;
if (cparams.fused_lid) {
- score = ggml_lightning_indexer(ctx0, iq, ik, iw, inp_attn_dsa->self_attn_mask_lid.cnv);
+ score = ggml_lightning_indexer(ctx0, iq, ik, iw, inp_attn_dsa->self_attn_mask_lid.raw);
cb(score, "indexer_score", il);
res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il});
} else {
@@ -408,7 +408,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k(
score = ggml_mul(ctx0, score, iw);
score = ggml_sum_rows(ctx0, score);
score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3));
- score = ggml_add(ctx0, score, inp_attn_dsa->self_attn_mask_lid.cnv);
+ score = ggml_add(ctx0, score, inp_attn_dsa->self_attn_mask_lid.raw);
cb(score, "indexer_score", il);
}
diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cpp
index 3488b6c77..e18937faf 100644
--- a/src/models/lfm2.cpp
+++ b/src/models/lfm2.cpp
@@ -332,7 +332,6 @@ llama_model_lfm2::graph_decision::graph_decision(const llama_model & model, cons
auto inp = std::make_unique<llm_graph_input_attn_media>(hparams, cparams, is_head);
inp->self_attn_mask.raw = ggml_new_tensor_4d(ctx0, type_mask, n_tokens, n_tokens, 1, 1);
ggml_set_input(inp->self_attn_mask.raw);
- inp->self_attn_mask.cnv = inp->self_attn_mask.raw;
inp_attn[is_head] = (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp));
}
diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp
index 0da9d5a44..6d1528096 100644
--- a/src/models/minimax-m3.cpp
+++ b/src/models/minimax-m3.cpp
@@ -262,7 +262,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
if (msa_enabled) {
const auto * mctx_msa = static_cast<const llama_kv_cache_msa_context *>(mctx);
- msa_kqm = inp_attn->self_attn_mask.cnv;
+ msa_kqm = inp_attn->self_attn_mask.raw;
n_kv = msa_kqm->ne[0];
n_tps = msa_kqm->ne[1]; // tokens per stream
ns = msa_kqm->ne[3]; // streams in this ubatch
diff --git a/src/models/mistral-large4.cpp b/src/models/mistral-large4.cpp
index dbd1907a4..a196e6d5b 100644
--- a/src/models/mistral-large4.cpp
+++ b/src/models/mistral-large4.cpp
@@ -289,7 +289,7 @@ llama_model_mistral_large4::graph::graph(const llama_model & model, const llm_gr
ggml_tensor * indexer_score = nullptr;
if (cparams.fused_lid) {
- indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.cnv);
+ indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw);
cb(indexer_score, "indexer_score", il);
res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il});
} else {
@@ -323,7 +323,7 @@ llama_model_mistral_large4::graph::graph(const llama_model & model, const llm_gr
cb(indexer_score, "indexer_score", il);
// mask indexer scores
- ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.cnv;
+ ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw;
indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask);
cb(indexer_score, "indexer_score", il);
}