Commit ccd360f06 for llama.cpp
commit ccd360f060a0f17b87ee0d4c5d91ee84cfdff484
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Wed Nov 11 15:16:16 2026 +0200
llama : remove non-fused GDN and ubatch-dependent graph nodes (#32078)
- remove build_delta_net_chunking and build_delta_net_autoregressive
- always use the fused gated delta net
- make graph_max_nodes independent of the ubatch size
- drop the fused GDN probes and fused node tags
Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4.1-Flash-1017-GGUF (Mac Studio, M5 Ultra, 256GB)
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 670adcc02..89961847b 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -136,18 +136,6 @@ static const llm_fused_op_probe llm_fused_op_flash_attn_kv_rows_probe = {
/*.n_tokens_per_seq =*/ 1,
};
-static const llm_fused_op_probe llm_fused_op_gdn_ar_probe = {
- /*.op =*/ LLM_FUSED_OP_GDN_AR,
- /*.name =*/ "fused Gated Delta Net (autoregressive)",
- /*.n_tokens_per_seq =*/ 1,
-};
-
-static const llm_fused_op_probe llm_fused_op_gdn_ch_probe = {
- /*.op =*/ LLM_FUSED_OP_GDN_CH,
- /*.name =*/ "fused Gated Delta Net (chunked)",
- /*.n_tokens_per_seq =*/ 16,
-};
-
static const llm_fused_op_probe llm_fused_op_lid_probe = {
/*.op =*/ LLM_FUSED_OP_LIGHTNING_INDEXER,
/*.name =*/ "Lightning Indexer",
@@ -347,10 +335,6 @@ llama_context::llama_context(
cparams.auto_fa = false;
}
- cparams.fused_gdn_ar = true;
- cparams.fused_gdn_ch = true;
- cparams.auto_fgdn = false;
-
cparams.fused_lid = true;
cparams.auto_flid = false;
@@ -724,13 +708,6 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3
cparams.auto_fkvr = false;
}
- if (cparams.auto_fgdn) {
- LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", func);
- resolve(llm_fused_op_gdn_ar_probe, cparams.fused_gdn_ar);
- resolve(llm_fused_op_gdn_ch_probe, cparams.fused_gdn_ch);
- cparams.auto_fgdn = false;
- }
-
if (cparams.auto_flid) {
LLAMA_LOG_INFO("%s: resolving fused Lightning Indexer support:\n", func);
resolve(llm_fused_op_lid_probe, cparams.fused_lid);
@@ -816,7 +793,7 @@ void llama_context::sched_reserve() {
const uint32_t n_seqs = cparams.n_seq_max;
const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);
- const size_t max_nodes = this->graph_max_nodes(n_tokens);
+ const size_t max_nodes = this->graph_max_nodes();
LLAMA_LOG_DEBUG("%s: max_nodes = %zu\n", __func__, max_nodes);
@@ -2919,14 +2896,11 @@ void llama_context::output_reorder() {
// graph
//
-uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
+uint32_t llama_context::graph_max_nodes() const {
uint32_t res;
- if (model.arch == LLM_ARCH_KIMI_K3 || model.arch == LLM_ARCH_KIMI_K31 || model.arch == LLM_ARCH_GLM5_NEXT) {
- // the n_tokens*40 budget below is exhausted at ubatch 3840
- res = std::max<uint32_t>(n_tokens * 160, 64u * model.n_tensors());
- } else if (model.arch == LLM_ARCH_HRM_TEXT) {
- // the 128-slot looped graph needs roughly one stack per token budget
- res = std::max<uint32_t>(n_tokens * 80, 64u * model.n_tensors());
+ if (model.arch == LLM_ARCH_KIMI_K3 || model.arch == LLM_ARCH_KIMI_K31 || model.arch == LLM_ARCH_GLM5_NEXT ||
+ model.arch == LLM_ARCH_HRM_TEXT) {
+ res = std::max<uint32_t>(1024u, 64u * model.n_tensors());
} else if (model.arch == LLM_ARCH_QWEN3NEXT ||
model.arch == LLM_ARCH_KIMI_LINEAR ||
model.arch == LLM_ARCH_METIS16 ||
@@ -2944,7 +2918,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
model.arch == LLM_ARCH_MINIMAX_01 ||
model.arch == LLM_ARCH_MINIMAX_M3 ||
model.arch == LLM_ARCH_HY_V4) {
- res = std::max<uint32_t>(n_tokens * 40, 32u * model.n_tensors());
+ res = std::max<uint32_t>(1024u, 32u * model.n_tensors());
} else if (model.arch == LLM_ARCH_DFLASH && model.hparams.dflash_selector_rank > 0) {
// DFlash2's convolutions and selector are shape work rather than matmuls,
// so they cost ~8.6 nodes per tensor against ~5.9 for a plain DFlash draft
diff --git a/src/llama-context.h b/src/llama-context.h
index f4061432f..d7f1123bb 100644
--- a/src/llama-context.h
+++ b/src/llama-context.h
@@ -272,7 +272,7 @@ private:
//
public:
- uint32_t graph_max_nodes(uint32_t n_tokens) const;
+ uint32_t graph_max_nodes() const;
// can reuse the llm_graph_result instance of the context (for example to update a memory module)
llm_graph_result * get_gf_res_reserve() const;
diff --git a/src/llama-cparams.h b/src/llama-cparams.h
index 7ac1513ef..2b38e73d5 100644
--- a/src/llama-cparams.h
+++ b/src/llama-cparams.h
@@ -39,9 +39,6 @@ struct llama_cparams {
bool offload_kqv;
bool flash_attn;
bool auto_fa;
- bool fused_gdn_ar; // use fused gated delta net (autoregressive)
- bool fused_gdn_ch; // use fused gated delta net (chunked)
- bool auto_fgdn;
bool fused_lid; // use fused lightning indexer
bool auto_flid;
bool fused_dsv4_hc_pre;
diff --git a/src/llama-graph.h b/src/llama-graph.h
index 636a1c54b..8242ebb89 100644
--- a/src/llama-graph.h
+++ b/src/llama-graph.h
@@ -49,8 +49,6 @@ enum llm_graph_type {
enum llm_fused_op {
LLM_FUSED_OP_FLASH_ATTN,
LLM_FUSED_OP_FLASH_ATTN_KV_ROWS,
- LLM_FUSED_OP_GDN_AR,
- LLM_FUSED_OP_GDN_CH,
LLM_FUSED_OP_LIGHTNING_INDEXER,
LLM_FUSED_OP_DSV4_HC_PRE,
LLM_FUSED_OP_DSV4_HC_COMB,
diff --git a/src/models/clef.cpp b/src/models/clef.cpp
index 3ac5782a8..86fed4e87 100644
--- a/src/models/clef.cpp
+++ b/src/models/clef.cpp
@@ -457,16 +457,9 @@ ggml_tensor * llama_model_clef::graph::build_layer_attn_linear(
q_conv = build_gdn_l2_norm(ctx0, q_conv, hparams.f_norm_rms_eps);
k_conv = build_gdn_l2_norm(ctx0, k_conv, hparams.f_norm_rms_eps);
- // note: need explicit repeat only if we are not using the fused GDN
- if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) {
- GGML_ASSERT(num_v_heads % num_k_heads == 0);
- q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_tokens, n_seqs);
- k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_tokens, n_seqs);
- }
-
ggml_tensor * state = ggml_fill(ctx0, ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, head_v_dim, head_v_dim, num_v_heads, n_seqs), 0.0f);
- ggml_tensor * output = build_delta_net(q_conv, k_conv, v_conv, gate, beta, state, il).first;
+ ggml_tensor * output = build_delta_net(q_conv, k_conv, v_conv, gate, beta, state).first;
cb(output, "attn_output", il);
// gated normalization
diff --git a/src/models/clover-moe.cpp b/src/models/clover-moe.cpp
index 9caaa5630..8892521ee 100644
--- a/src/models/clover-moe.cpp
+++ b/src/models/clover-moe.cpp
@@ -369,11 +369,6 @@ ggml_tensor * llama_model_clover_moe::graph::build_layer_attn_linear(
q_conv = build_gdn_l2_norm(ctx0, q_conv, 1e-6f);
k_conv = build_gdn_l2_norm(ctx0, k_conv, 1e-6f);
- if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) {
- q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);
- k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);
- }
-
ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il);
ggml_tensor * z_2d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);
diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp
index 4c0c24d6c..45b2de4bb 100644
--- a/src/models/delta-net-base.cpp
+++ b/src/models/delta-net-base.cpp
@@ -3,378 +3,15 @@
#include "llama-impl.h"
#include "llama-memory-recurrent.h"
-// utility to get one slice from the third dimension
-// input dim: [x, y, c, b]
-// output dim: [x, y, 1, b]
-static ggml_tensor * get_slice_2d(ggml_context * ctx0, ggml_tensor * t, int64_t c) {
- return ggml_view_4d(ctx0, t, t->ne[0], t->ne[1], 1, t->ne[3],
- t->nb[1], t->nb[2], t->nb[3], t->nb[2] * c);
-}
-
llm_build_delta_net_base::llm_build_delta_net_base(const llm_graph_params & params) : llm_graph_context(params) {}
-std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_net_chunking(
- ggml_tensor * q,
- ggml_tensor * k,
- ggml_tensor * v,
- ggml_tensor * g,
- ggml_tensor * b,
- ggml_tensor * s,
- int il) {
- const int64_t S_k = q->ne[0];
- const int64_t H_k = q->ne[1];
- const int64_t n_tokens = q->ne[2];
- const int64_t n_seqs = q->ne[3];
-
- const int64_t S_v = v->ne[0];
- const int64_t H_v = v->ne[1];
- const bool kda = (g->ne[0] == S_k && g->ne[1] == H_k);
-
- GGML_ASSERT(S_k == S_v);
- GGML_ASSERT(H_v % H_k == 0);
-
- GGML_ASSERT(q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs);
- GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs);
- GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs);
-
- GGML_ASSERT(g->ne[0] == 1 || g->ne[0] == S_v);
- GGML_ASSERT( g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs);
- GGML_ASSERT(b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs);
- GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs);
-
- const float scale = 1.0f / sqrtf(S_k);
-
- q = ggml_scale(ctx0, q, scale);
-
- cb(q, "q_in", il);
- cb(k, "k_in", il);
- cb(v, "v_in", il);
- cb(b, "b_in", il);
- cb(g, "g_in", il);
-
- q = ggml_permute(ctx0, q, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs]
- k = ggml_permute(ctx0, k, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs]
- v = ggml_permute(ctx0, v, 0, 2, 1, 3); // [S_v, n_tokens, H_v, n_seqs]
- g = ggml_permute(ctx0, g, 0, 2, 1, 3); // [g_0, n_tokens, H_v, n_seqs]
- b = ggml_permute(ctx0, b, 0, 2, 1, 3); // [ 1, n_tokens, H_v, n_seqs]
-
- const int CS = kda ? 16 : 64; // chunk size
-
- const int pad = (CS - n_tokens % CS) % CS;
- const int n_chunks = (n_tokens + pad) / CS;
-
- q = ggml_pad(ctx0, q, 0, pad, 0, 0);
- k = ggml_pad(ctx0, k, 0, pad, 0, 0);
- v = ggml_pad(ctx0, v, 0, pad, 0, 0);
- g = ggml_pad(ctx0, g, 0, pad, 0, 0);
- b = ggml_pad(ctx0, b, 0, pad, 0, 0);
-
- ggml_tensor * v_b = ggml_mul(ctx0, v, b);
- ggml_tensor * k_b = ggml_mul(ctx0, k, b);
-
- cb(v_b, "v_b", il);
- cb(k_b, "k_b", il);
-
- q = ggml_reshape_4d(ctx0, q, S_k, CS, n_chunks, H_k * n_seqs);
- k = ggml_reshape_4d(ctx0, k, S_k, CS, n_chunks, H_k * n_seqs);
- k_b = ggml_reshape_4d(ctx0, k_b, S_k, CS, n_chunks, H_v * n_seqs);
- v = ggml_reshape_4d(ctx0, v, S_v, CS, n_chunks, H_v * n_seqs);
- v_b = ggml_reshape_4d(ctx0, v_b, S_v, CS, n_chunks, H_v * n_seqs);
-
- g = ggml_reshape_4d(ctx0, g, g->ne[0], CS, n_chunks, H_v * n_seqs);
- b = ggml_reshape_4d(ctx0, b, 1, CS, n_chunks, H_v * n_seqs);
-
- // [CS, g_0, n_chunks, H_v * n_seqs]
- // TODO: extend ggml_cumsum with axis parameter to avoid transpose
- ggml_tensor * g_cs = ggml_cumsum(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, g)));
- cb(g_cs, "g_cs", il);
-
- ggml_tensor * kb = nullptr;
- ggml_tensor * kq = nullptr;
- if (kda) {
- const int64_t CHB = n_chunks * H_k * n_seqs;
-
- ggml_tensor * g_cs_i = ggml_reshape_4d(ctx0, g_cs, CS, 1, S_k, CHB); // [chunk_size, 1, S_k, CHB]
- ggml_tensor * g_cs_j = ggml_reshape_4d(ctx0, g_cs, 1, CS, S_k, CHB); // [1, chunk_size, S_k, CHB]
-
- g_cs_j = ggml_repeat_4d(ctx0, g_cs_j, CS, CS, S_k, CHB); // [1, chunk_size, S_k, CHB] -> [chunk_size, chunk_size, S_k, CHB]
-
- // decay_mask [chunk_size,chunk_size,S_k,CHB]
- ggml_tensor * decay_mask;
- decay_mask = ggml_sub(ctx0, g_cs_j, g_cs_i);
- decay_mask = ggml_tri(ctx0, decay_mask, GGML_TRI_TYPE_LOWER_DIAG);
- decay_mask = ggml_exp(ctx0, decay_mask);
- cb(decay_mask, "decay_mask", il);
-
- // decay_mask [S_k,BT_j,BT_i,CHB] *Note* second and third chunk_sizes are switched
- decay_mask = ggml_cont_4d(ctx0, ggml_permute(ctx0, decay_mask, 2, 1, 0, 3), S_k, CS, CS, CHB);
-
- ggml_tensor * k_b_i = ggml_reshape_4d(ctx0, k_b, S_k, CS, 1, CHB);
- ggml_tensor * k_j = ggml_reshape_4d(ctx0, k, S_k, 1, CS, CHB);
- ggml_tensor * q_i = ggml_reshape_4d(ctx0, q, S_k, CS, 1, CHB);
-
- ggml_tensor * decay_k_b_i = ggml_mul(ctx0, decay_mask, k_b_i);
- ggml_tensor * decay_q_i = ggml_mul(ctx0, decay_mask, q_i);
-
- // decay_k_b_i [S,BT,BT,CHB] @ k_j [S,1,BT,CHB] = Akk [BT,1,BT,CHB]
- kb = ggml_mul_mat(ctx0, decay_k_b_i, k_j);
- kq = ggml_mul_mat(ctx0, decay_q_i, k_j);
-
- kb = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_4d(ctx0, kb, CS, CS, n_chunks, H_v * n_seqs)));
- kq = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_4d(ctx0, kq, CS, CS, n_chunks, H_v * n_seqs)));
- } else {
- ggml_tensor * g_cs_i = g_cs;
- ggml_tensor * g_cs_j = ggml_reshape_4d(ctx0, g_cs, 1, CS, n_chunks, H_v * n_seqs);
-
- g_cs_j = ggml_repeat_4d(ctx0, g_cs_j, CS, CS, n_chunks, H_v * n_seqs);
-
- // [CS, CS, n_chunks, H_v * n_seqs]
- ggml_tensor * decay_mask;
- decay_mask = ggml_sub(ctx0, g_cs_j, g_cs_i);
- decay_mask = ggml_tri(ctx0, decay_mask, GGML_TRI_TYPE_LOWER_DIAG);
- decay_mask = ggml_exp(ctx0, decay_mask);
- cb(decay_mask, "decay_mask", il);
-
- // [CS, CS, n_chunks, H_k * n_seqs]
- kb = ggml_mul_mat(ctx0, k, k_b);
- kb = ggml_mul (ctx0, kb, decay_mask);
-
- // [CS, CS, n_chunks, H_k * n_seqs]
- kq = ggml_mul_mat(ctx0, k, q);
- kq = ggml_mul(ctx0, kq, decay_mask);
- }
-
- kq = ggml_tri(ctx0, kq, GGML_TRI_TYPE_LOWER_DIAG);
- cb(kq, "kq", il);
-
- // [CS, CS, n_chunks, H_k * n_seqs]
- ggml_tensor * attn;
- attn = ggml_tri(ctx0, kb, GGML_TRI_TYPE_LOWER);
- cb(attn, "attn", il);
-
- ggml_tensor * identity;
- identity = ggml_view_1d(ctx0, attn, CS, 0);
- identity = ggml_fill (ctx0, identity, 1.0f);
- identity = ggml_diag (ctx0, identity);
-
- ggml_tensor * lhs = ggml_add(ctx0, attn, identity);
- cb(lhs, "dnet_add_ch_lhs", il);
-
- attn = ggml_neg(ctx0, attn);
- cb(attn, "attn_pre_solve", il);
-
- ggml_tensor * lin_solve = ggml_solve_tri(ctx0, lhs, attn, true, true, false);
- attn = ggml_add(ctx0, lin_solve, identity);
- cb(attn, "dnet_add_ch_attn_solved", il); // [CS, CS, n_chunks, H_k * n_seqs]
-
- // [S_v, CS, n_chunks, H_v * n_seqs]
- v = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, v_b)), attn);
-
- // [CS, 1, n_chunks, H_v * n_seqs] KDA: [CS, S_k, n_chunks, H_v * n_seqs]
- ggml_tensor * g_exp = ggml_exp(ctx0, g_cs);
-
- k_b = ggml_cont(ctx0, ggml_transpose(ctx0, k_b));
-
- // [CS, S_k, n_chunks, H_k * n_seqs]
- ggml_tensor * kbg = ggml_mul(ctx0, k_b, g_exp);
- cb(kbg, "k_beta_g_exp", il);
-
- // [S_k, CS, n_chunks, H_k * n_seqs]
- ggml_tensor * k_cd = ggml_mul_mat(ctx0, kbg, attn);
- cb(k_cd, "k_cumdecay", il);
-
- // [1, CS, n_chunks, H_k * n_seqs] KDA: [S_k, CS, n_chunks, H_k * n_seqs]
- ggml_tensor * g_exp_t = ggml_cont(ctx0, ggml_transpose(ctx0, g_exp));
- ggml_tensor * q_g_exp = ggml_mul(ctx0, q, g_exp_t);
-
- // vectorized calculation of key_gdiff
- // improved from the chunked version:
- // g_last = torch.clamp(g_cum[:, :, -1], max=50.0).exp().unsqueeze(-1).unsqueeze(-1)
- // g_diff = torch.clamp(g_cum[:, :, -1:] - g_cum, max=50.0).exp()
- // key_gdiff = key * g_diff.unsqueeze(-1)
- // kgdmulvnew = (key_gdiff).transpose(-1, -2) @ v_new
- // last_recurrent_state = last_recurrent_state * g_last + kgdmulvnew
-
- // get last element in g_cumsum along CS dimension (ne0)
- // example: [[x, y, z, ..., last], ...] -> [[last], ...]
- // [1, 1, n_chunks, H_v * n_seqs] KDA: [1, S_k, n_chunks, H_v * n_seqs]
- ggml_tensor * g_last = ggml_view_4d(ctx0, g_cs, 1, g_cs->ne[1], g_cs->ne[2], g_cs->ne[3],
- g_cs->nb[1],
- g_cs->nb[2],
- g_cs->nb[3],
- ggml_row_size(g_cs->type, g_cs->ne[0] - 1));
- cb(g_last, "g_last", il);
-
- // [1, 1, n_chunks, H_v * n_seqs] KDA: [S_k, 1, n_chunks, H_v * n_seqs]
- ggml_tensor * g_last_exp_t = ggml_transpose(ctx0, ggml_exp(ctx0, g_last));
- cb(g_last_exp_t, "g_last_exp_t", il);
-
- // [CS, 1, n_chunks, H_v * n_seqs] KDA: [CS, S_k, n_chunks, H_v * n_seqs]
- ggml_tensor * g_diff = ggml_neg(ctx0, ggml_sub(ctx0, g_cs, g_last));
- cb(g_diff, "g_diff", il);
-
- ggml_tensor * g_diff_exp_t = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_exp(ctx0, g_diff)));
-
- // [S_k, CS, n_chunks, H_v * n_seqs]
- ggml_tensor * kg = ggml_mul(ctx0, k, g_diff_exp_t);
- cb(kg, "key_gdiff", il);
-
- // [CS, S_k, n_chunks, H_v * n_seqs]
- ggml_tensor * kg_t = ggml_cont(ctx0, ggml_transpose(ctx0, kg));
- cb(kg_t, "key_gdiff_t", il);
-
- s = ggml_reshape_4d(ctx0, s, S_v, S_v, 1, H_v * n_seqs);
- cb(s, "dnet_add_ch_state", il);
-
- // [CS, S_v, n_chunks, H_v * n_seqs]
- ggml_tensor * v_t = ggml_cont(ctx0, ggml_transpose(ctx0, v));
-
- for (int64_t chunk = 0; chunk < n_chunks; chunk++) {
- ggml_tensor * ch_k_cd = get_slice_2d(ctx0, k_cd, chunk); // [S_k, CS, 1, H_k * n_seqs]
- ggml_tensor * ch_v_t = get_slice_2d(ctx0, v_t, chunk); // [ CS, S_v, 1, H_v * n_seqs]
- ggml_tensor * ch_kq = get_slice_2d(ctx0, kq, chunk); // [ CS, CS, 1, H_k * n_seqs]
- ggml_tensor * ch_q_g_exp = get_slice_2d(ctx0, q_g_exp, chunk); // [S_k, CS, 1, H_k * n_seqs]
- ggml_tensor * ch_kg_t = get_slice_2d(ctx0, kg_t, chunk); // [ CS, S_k, 1, H_v * n_seqs]
-
- // [CS, S_v, 1, H_v * n_seqs]
- ggml_tensor * v_t_p = ggml_mul_mat(ctx0, ch_k_cd, s);
- cb(v_t_p, "v_prime", il);
-
- // [CS, S_v, 1, H_v * n_seqs]
- ggml_tensor * v_t_new = ggml_sub(ctx0, ch_v_t, v_t_p);
- cb(v_t_new, "v_t_new", il);
-
- // [S_v, CS, 1, H_v * n_seqs]
- ggml_tensor * v_attn = ggml_mul_mat(ctx0, v_t_new, ch_kq);
- cb(v_attn, "v_attn", il);
-
- // [S_v, CS, 1, H_v * n_seqs]
- ggml_tensor * attn_inter = ggml_mul_mat(ctx0, s, ch_q_g_exp);
- cb(attn_inter, "attn_inter", il);
-
- // [S_v, CS, 1, H_v * n_seqs]
- ggml_tensor * o_ch = ggml_add(ctx0, attn_inter, v_attn);
- cb(o_ch, "dnet_add_ch_attn_out", il);
-
- v = ggml_set_inplace(ctx0, v, o_ch, v->nb[1], v->nb[2], v->nb[3], chunk * v->nb[2]);
-
- // kgdmulvnew = (key_gdiff).transpose(-1, -2) @ v_new
- // TODO: head broadcast might not work here - probably will need a transpose
- ggml_tensor * kgv = ggml_mul_mat(ctx0, ch_kg_t, v_t_new); // [S_k, S_v, 1, H_k * n_seqs]
-
- // last_recurrent_state = last_recurrent_state * g_last + kgdmulvnew
- ggml_tensor * ch_g_last_exp_t = get_slice_2d(ctx0, g_last_exp_t, chunk);
-
- s = ggml_mul(ctx0, s, ch_g_last_exp_t);
- s = ggml_add(ctx0, s, kgv);
- cb(s, "dnet_add_ch_state", il);
- }
-
- // truncate padded tokens
- ggml_tensor * o = ggml_view_4d(ctx0, v,
- S_v, n_tokens, H_v, n_seqs,
- ggml_row_size(v->type, S_v),
- ggml_row_size(v->type, S_v * CS * n_chunks),
- ggml_row_size(v->type, S_v * CS * n_chunks * H_v), 0);
- o = ggml_permute (ctx0, o, 0, 2, 1, 3); // [S_v, H_v, n_tokens, n_seqs]
- s = ggml_reshape_4d(ctx0, s, S_v, S_v, H_v, n_seqs);
- cb(s, "output_state", il);
-
- return {o, s};
-}
-
-std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_net_autoregressive(
- ggml_tensor * q,
- ggml_tensor * k,
- ggml_tensor * v,
- ggml_tensor * g,
- ggml_tensor * b, // beta
- ggml_tensor * s, // state
- int il) {
- const int64_t S_k = q->ne[0];
- const int64_t H_k = q->ne[1];
- const int64_t n_tokens = q->ne[2];
- const int64_t n_seqs = q->ne[3];
-
- const int64_t S_v = v->ne[0];
- const int64_t H_v = v->ne[1];
-
- GGML_ASSERT(n_tokens == 1);
-
- GGML_ASSERT(S_k == S_v);
- GGML_ASSERT(H_v % H_k == 0);
-
- GGML_ASSERT(q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs);
- GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs);
- GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs);
-
- GGML_ASSERT(g->ne[0] == 1 || g->ne[0] == S_v);
- GGML_ASSERT( g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs);
- GGML_ASSERT(b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs);
- GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs);
-
- const float scale = 1.0f / sqrtf(S_k);
-
- q = ggml_scale(ctx0, q, scale);
-
- q = ggml_permute(ctx0, q, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs]
- k = ggml_permute(ctx0, k, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs]
- v = ggml_permute(ctx0, v, 0, 2, 1, 3); // [S_v, n_tokens, H_v, n_seqs]
-
- cb(q, "q_in", il);
- cb(k, "k_in", il);
- cb(v, "v_in", il);
- cb(b, "b_in", il);
- cb(g, "g_in", il);
-
- // GDA: [1, 1, H_v, n_seqs]
- // KDA: [1, S_k, H_v, n_seqs]
- g = ggml_reshape_4d(ctx0, g, 1, g->ne[0], H_v, n_seqs);
- b = ggml_reshape_4d(ctx0, b, 1, 1, H_v, n_seqs);
-
- // [S_v, S_v, H_v, n_seqs]
- g = ggml_exp(ctx0, g);
- s = ggml_mul(ctx0, s, g);
-
- // [1, S_v, H_v, n_seqs]
- ggml_tensor * sk;
- sk = ggml_mul (ctx0, s, k);
- sk = ggml_sum_rows(ctx0, sk);
-
- // [S_v, 1, H_v, n_seqs]
- ggml_tensor * d;
- d = ggml_sub(ctx0, v, ggml_transpose(ctx0, sk));
- d = ggml_mul(ctx0, d, b);
-
- // [1, S_v, H_v, n_seqs]
- ggml_tensor * d_t;
- d_t = ggml_transpose(ctx0, d);
-
- // [S_v, S_v, H_v, n_seqs]
- ggml_tensor * kd;
- k = ggml_repeat(ctx0, k, s);
- kd = ggml_mul (ctx0, k, d_t);
-
- s = ggml_add(ctx0, s, kd);
-
- cb(s, "dnet_add_ar_state", il);
-
- ggml_tensor * s_q = ggml_mul (ctx0, s, q);
- ggml_tensor * o = ggml_sum_rows(ctx0, s_q);
-
- o = ggml_permute (ctx0, o, 2, 0, 1, 3); // [S_v, H_v, n_tokens, n_seqs]
-
- return {o, s};
-}
-
-std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_net_fused(
+std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_net(
ggml_tensor * q,
ggml_tensor * k,
ggml_tensor * v,
ggml_tensor * g,
ggml_tensor * b,
- ggml_tensor * s,
- int il) {
+ ggml_tensor * s) {
const int64_t S_k = q->ne[0];
const int64_t H_k = q->ne[1];
const int64_t n_tokens = q->ne[2];
@@ -397,12 +34,6 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne
// K=1: output carries the final state only. state s is 4D [S_v, S_v, H_v, n_seqs].
ggml_tensor * result = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, /*K=*/1);
- if (n_tokens == 1) {
- res->add_fused_node({LLM_FUSED_OP_GDN_AR, result, il});
- } else {
- res->add_fused_node({LLM_FUSED_OP_GDN_CH, result, il});
- }
-
ggml_tensor * output = ggml_view_4d(ctx0, result,
S_v, H_v, n_tokens, n_seqs,
ggml_row_size(result->type, S_v),
@@ -419,30 +50,6 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne
return {output, new_state};
}
-std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_net(
- ggml_tensor * q,
- ggml_tensor * k,
- ggml_tensor * v,
- ggml_tensor * g,
- ggml_tensor * b,
- ggml_tensor * s,
- int il) {
- const int64_t n_seq_tokens = q->ne[2];
-
- if (n_seq_tokens == 1) {
- if (cparams.fused_gdn_ar) {
- return build_delta_net_fused(q, k, v, g, b, s, il);
- }
- return build_delta_net_autoregressive(q, k, v, g, b, s, il);
- }
-
- if (cparams.fused_gdn_ch) {
- return build_delta_net_fused(q, k, v, g, b, s, il);
- }
-
- return build_delta_net_chunking(q, k, v, g, b, s, il);
-}
-
ggml_tensor * llm_build_delta_net_base::build_conv_state(
llm_graph_input_rs * inp,
ggml_tensor * conv_states_all,
@@ -491,7 +98,7 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn(
const bool keep = cparams.n_rs_seq > 0;
if (!keep) {
- auto attn_out = build_delta_net(q, k, v, g, b, s, il);
+ auto attn_out = build_delta_net(q, k, v, g, b, s);
ggml_tensor * output = attn_out.first;
ggml_tensor * new_state = attn_out.second;
cb(output, "attn_output", il);
@@ -510,12 +117,6 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn(
// state s is 4D [S_v, S_v, H_v, n_seqs]; K snapshot slots are written into the output.
ggml_tensor * gdn_out = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, K);
- if (n_seq_tokens > 1) {
- res->add_fused_node({LLM_FUSED_OP_GDN_CH, gdn_out, il});
- } else {
- res->add_fused_node({LLM_FUSED_OP_GDN_AR, gdn_out, il});
- }
-
const int64_t attn_score_elems = S_v * H_v * n_seq_tokens * n_seqs;
const int64_t state_size_per_snap = S_v * S_v * H_v * n_seqs;
diff --git a/src/models/kimi-linear.cpp b/src/models/kimi-linear.cpp
index 54d30ad47..2190df8c9 100644
--- a/src/models/kimi-linear.cpp
+++ b/src/models/kimi-linear.cpp
@@ -348,8 +348,7 @@ llama_model_kimi_linear::graph::graph(const llm_graph_params & params) :
Qcur = build_gdn_l2_norm(ctx0, Qcur, eps_norm);
Kcur = build_gdn_l2_norm(ctx0, Kcur, eps_norm);
- // Choose between build_delta_net_chunking and build_delta_net_recurrent based on n_tokens
- auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il);
+ auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state);
ggml_tensor * output = ggml_cont(ctx0, attn_out.first);
ggml_tensor * new_state = attn_out.second;
diff --git a/src/models/models.h b/src/models/models.h
index f2c43b800..cfef363b6 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -37,44 +37,13 @@ struct llm_build_delta_net_base : public llm_graph_context {
virtual ~llm_build_delta_net_base() = default;
// returns pair of output and new state
- std::pair<ggml_tensor *, ggml_tensor *> build_delta_net_chunking(
- ggml_tensor * q,
- ggml_tensor * k,
- ggml_tensor * v,
- ggml_tensor * g,
- ggml_tensor * b,
- ggml_tensor * s,
- int il);
-
- // returns pair of output and new state
- std::pair<ggml_tensor *, ggml_tensor *> build_delta_net_autoregressive(
- ggml_tensor * q,
- ggml_tensor * k,
- ggml_tensor * v,
- ggml_tensor * g,
- ggml_tensor * b,
- ggml_tensor * s,
- int il);
-
- // use the ggml_gated_delta_net fused operator (K=1; state has shape [S_v, S_v, H_v, n_seqs])
- std::pair<ggml_tensor *, ggml_tensor *> build_delta_net_fused(
- ggml_tensor * q,
- ggml_tensor * k,
- ggml_tensor * v,
- ggml_tensor * g,
- ggml_tensor * b,
- ggml_tensor * s,
- int il);
-
- // choose one of two implementations above based on the number of tokens
std::pair<ggml_tensor *, ggml_tensor *> build_delta_net(
ggml_tensor * q,
ggml_tensor * k,
ggml_tensor * v,
ggml_tensor * g,
ggml_tensor * b,
- ggml_tensor * s,
- int il);
+ ggml_tensor * s);
// read conv state from cache, concat with qkv_mixed, write back (single slot or per-token)
// qkv_mixed: (qkv_dim, n_seq_tokens, n_seqs); returns conv_input: (kernel_size + n_seq_tokens - 1, channels, n_seqs)
diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp
index 736e23f10..6ec3cff44 100644
--- a/src/models/qwen35.cpp
+++ b/src/models/qwen35.cpp
@@ -432,14 +432,6 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear(
//k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs);
//v_conv = ggml_cont_4d(ctx0, v_conv, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);
- // if head keys and value keys are different, repeat to force tensors into matching shapes
- // note: need explicit repeat only if we are not using the fused GDN.
- if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) {
- GGML_ASSERT(num_v_heads % num_k_heads == 0);
- q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);
- k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);
- }
-
cb(q_conv, "q_conv_predelta", il);
cb(k_conv, "k_conv_predelta", il);
cb(v_conv, "v_conv_predelta", il);
diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp
index c4da3e2be..c0ee6fa03 100644
--- a/src/models/qwen35moe.cpp
+++ b/src/models/qwen35moe.cpp
@@ -442,14 +442,6 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn_linear(
//k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs);
//v_conv = ggml_cont_4d(ctx0, v_conv, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);
- // if head keys and value keys are different, repeat to force tensors into matching shapes
- // note: need explicit repeat only if we are not using the fused GDN.
- if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) {
- GGML_ASSERT(num_v_heads % num_k_heads == 0);
- q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);
- k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);
- }
-
cb(q_conv, "q_conv_predelta", il);
cb(k_conv, "k_conv_predelta", il);
cb(v_conv, "v_conv_predelta", il);
diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp
index 876640e16..181e4dca4 100644
--- a/src/models/qwen4exp.cpp
+++ b/src/models/qwen4exp.cpp
@@ -1191,13 +1191,6 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear(
q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm);
k_conv = build_gdn_l2_norm(ctx0, k_conv, eps_norm);
- // repeat to match shapes when head keys != value keys; unneeded with the fused GDN
- if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) {
- GGML_ASSERT(num_v_heads % num_k_heads == 0);
- q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);
- k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);
- }
-
cb(q_conv, "q_conv_predelta", il);
cb(k_conv, "k_conv_predelta", il);
cb(v_conv, "v_conv_predelta", il);