Commit 19bc41dbb for llama.cpp
commit 19bc41dbb41e858eebc269b49ccdec1f0d28880c
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Sat Nov 21 16:30:01 2026 +0100
model: apply .scale tensors automatically + fix correctness (#32669)
* model: apply .scale tensors automatically
* rm build_moe_ffn "_s" overload
* use llama_prec_policy
* fix ci
* build_lora_mm_id: rm w_s
diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
index 595a5824a..c4e37ed46 100644
--- a/ggml/src/ggml-backend-meta.cpp
+++ b/ggml/src/ggml-backend-meta.cpp
@@ -2154,8 +2154,28 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend,
}
};
- skip_unrelated();
- if (id + 1 >= cgraph->n_nodes) {
+ // Chain of MULs with MIRRORED src[1], returns false at the end of the graph
+ auto absorb_muls = [&]() -> bool {
+ while (true) {
+ skip_unrelated();
+ if (id + 1 >= cgraph->n_nodes) {
+ return false;
+ }
+ ggml_tensor * next = cgraph->nodes[id+1];
+ if (next->op == GGML_OP_MUL && next->src[0] == node &&
+ ggml_backend_meta_get_split_state(next->src[1], false).axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) {
+ node = next;
+ id++;
+ idr = id;
+ n_used = ggml_node_get_use_count(cgraph, id);
+ } else {
+ return true;
+ }
+ }
+ };
+
+ // MULs before the PARTIAL bias (e.g. a per-expert weight scale): sum_j(s*p_j + b/n) == s*sum_j(p_j) + b
+ if (!absorb_muls()) {
return idr;
}
{
@@ -2169,22 +2189,8 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend,
n_used = ggml_node_get_use_count(cgraph, id);
}
}
- // Chain of MULs with MIRRORED src[1]
- while (true) {
- skip_unrelated();
- if (id + 1 >= cgraph->n_nodes) {
- return idr;
- }
- ggml_tensor * next = cgraph->nodes[id+1];
- if (next->op == GGML_OP_MUL && next->src[0] == node &&
- ggml_backend_meta_get_split_state(next->src[1], false).axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) {
- node = next;
- id++;
- idr = id;
- n_used = ggml_node_get_use_count(cgraph, id);
- } else {
- break;
- }
+ if (!absorb_muls()) {
+ return idr;
}
if (n_used != node->ne[1] || id + 2*n_used-1 >= cgraph->n_nodes) {
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index c5c9ddf01..1bd3f8d0f 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -1810,16 +1810,21 @@ ggml_tensor * llm_graph_context::build_lora_mm(
ggml_tensor * w,
ggml_tensor * cur,
ggml_tensor * w_s) const {
+ if (!w_s) {
+ w_s = model.get_w_scale(w);
+ }
+
// a Hadamard-folded weight reads the transformed input; a LoRA delta lives in the primal basis and reads `cur`
ggml_tensor * cur_mm = build_hadamard_input(w, cur);
ggml_tensor * res = ggml_mul_mat(ctx0, w, cur_mm);
- prec_policy.apply(res);
-
if (w->type == GGML_TYPE_NVFP4) {
ggml_prec_set_acc(res, GGML_PREC_BF16);
}
+
+ prec_policy.apply(res);
+
if (w_s) {
res = ggml_mul(ctx0, res, w_s);
}
@@ -1886,9 +1891,10 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
ggml_tensor * w, // ggml_tensor * as
ggml_tensor * cur, // ggml_tensor * b
ggml_tensor * ids,
- ggml_tensor * w_s,
ggml_tensor * slots,
bool cached) const {
+ ggml_tensor * w_s = model.get_w_scale(w);
+
// as in build_lora_mm: the folded weight reads the transformed input, the LoRA delta reads `cur`
ggml_tensor * cur_mm = build_hadamard_input(w, cur);
// the experts in the MoE cache are selected by their slots
@@ -1896,12 +1902,13 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
ggml_mul_mat_id(ctx0, w, cur_mm, ids) :
ggml_mul_mat_id(ctx0, cached ? moe_cache->get_experts(w) : w, cur_mm, slots);
- prec_policy.apply(res);
-
if (w->type == GGML_TYPE_NVFP4) {
ggml_prec_set_acc(res, GGML_PREC_BF16);
}
+ // after the NVFP4 default, so that the policy can request a higher precision
+ prec_policy.apply(res);
+
if (w_s) {
const int64_t n_expert = w_s->ne[0];
const int64_t n_tokens = cur->ne[2];
@@ -2115,34 +2122,7 @@ ggml_tensor * llm_graph_context::build_ffn(
int il) const {
const llm_arch arch = model.arch;
- // NVFP4 support is currently restricted to
- // 1) LORA absence (*_s would be applied after LORA residual, which is incorrect)
- // 2) bias absense (*_s would be applied after bias addition, which is incorrect)
- // TODO: disambiguate LLM-architectural scales (which use *_s) from NVFP4 scale_2 (which also uses *_s currently)
- auto has_lora = [this](ggml_tensor * w) {
- if (!w) {
- return false;
- }
- for (const auto & lora : *loras) {
- if (lora.first->get_weight(w) != nullptr) {
- return true;
- }
- }
- return false;
- };
-
- auto is_scaled_low_precision = [](ggml_tensor * w) {
- return w && (w->type == GGML_TYPE_NVFP4 || w->type == GGML_TYPE_F8_E4M3);
- };
-
- GGML_ASSERT(!up_s || !up_b || !is_scaled_low_precision(up));
- GGML_ASSERT(!gate_s || !gate_b || !is_scaled_low_precision(gate));
- GGML_ASSERT(!down_s || !down_b || !is_scaled_low_precision(down));
- GGML_ASSERT(!up_s || !is_scaled_low_precision(up) || !has_lora(up));
- GGML_ASSERT(!gate_s || !is_scaled_low_precision(gate) || !has_lora(gate));
- GGML_ASSERT(!down_s || !is_scaled_low_precision(down) || !has_lora(down));
-
- ggml_tensor * tmp = up ? build_lora_mm(up, cur) : cur;
+ ggml_tensor * tmp = up ? build_lora_mm(up, cur, up_s) : cur;
cb(tmp, "ffn_up", il);
if (up_b) {
@@ -2150,21 +2130,16 @@ ggml_tensor * llm_graph_context::build_ffn(
cb(tmp, "ffn_up_b", il);
}
- if (up_s) {
- tmp = ggml_mul(ctx0, tmp, up_s);
- cb(tmp, "ffn_up_s", il);
- }
-
if (gate) {
switch (type_gate) {
case LLM_FFN_SEQ:
{
- cur = build_lora_mm(gate, tmp);
+ cur = build_lora_mm(gate, tmp, gate_s);
cb(cur, "ffn_gate", il);
} break;
case LLM_FFN_PAR:
{
- cur = build_lora_mm(gate, cur);
+ cur = build_lora_mm(gate, cur, gate_s);
cb(cur, "ffn_gate", il);
} break;
}
@@ -2174,11 +2149,6 @@ ggml_tensor * llm_graph_context::build_ffn(
cb(cur, "ffn_gate_b", il);
}
- if (gate_s) {
- cur = ggml_mul(ctx0, cur, gate_s);
- cb(cur, "ffn_gate_s", il);
- }
-
} else {
cur = tmp;
}
@@ -2300,11 +2270,7 @@ ggml_tensor * llm_graph_context::build_ffn(
}
if (down) {
- cur = build_lora_mm(down, cur);
- if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
- // GLM4, GLM4_MOE, and JAIS2 have numerical issues with F16 accumulators
- ggml_prec_set_acc(cur, GGML_PREC_BF16);
- }
+ cur = build_lora_mm(down, cur, down_s);
}
if (down_b) {
@@ -2315,11 +2281,6 @@ ggml_tensor * llm_graph_context::build_ffn(
cur = ggml_add(ctx0, cur, down_b);
}
- if (down_s) {
- cur = ggml_mul(ctx0, cur, down_s);
- cb(cur, "ffn_down_s", il);
- }
-
return cur;
}
@@ -2339,9 +2300,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
int il,
ggml_tensor * probs_in,
ggml_tensor * gate_up_exps,
- ggml_tensor * up_exps_s,
- ggml_tensor * gate_exps_s,
- ggml_tensor * down_exps_s,
ggml_tensor * selected_experts_in,
ggml_tensor * post_exp_norm,
float post_exp_norm_eps) const {
@@ -2362,9 +2320,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
probs_in,
gate_up_exps,
/* gate_up_exps_b */ nullptr,
- up_exps_s,
- gate_exps_s,
- down_exps_s,
selected_experts_in,
post_exp_norm,
post_exp_norm_eps
@@ -2392,9 +2347,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
ggml_tensor * probs_in,
ggml_tensor * gate_up_exps,
ggml_tensor * gate_up_exps_b,
- ggml_tensor * up_exps_s,
- ggml_tensor * gate_exps_s,
- ggml_tensor * down_exps_s,
ggml_tensor * selected_experts_in,
ggml_tensor * post_exp_norm,
float post_exp_norm_eps) const {
@@ -2407,10 +2359,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
if (probs_in == nullptr) {
logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens]
- if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS || hparams.expert_router_f32) {
- // the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/60d8d70770c6776ff598c94bb586a859a38244f1/inference/model.py#L565
- ggml_prec_set_acc(logits, GGML_PREC_F32);
- }
cb(logits, "ffn_moe_logits", il);
} else {
logits = probs_in;
@@ -2583,13 +2531,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
if (gate_up_exps) {
// merged gate_up path: one mul_mat_id, then split into gate and up views
- ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, ids, up_exps_s, mm_ids, cached); // [n_ff*2, n_expert_used, n_tokens]
+ ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, ids, mm_ids, cached); // [n_ff*2, n_expert_used, n_tokens]
cb(gate_up, "ffn_moe_gate_up", il);
- if (up_exps_s) {
- cb(gate_up, "ffn_moe_gate_up_scaled", il);
- }
-
if (gate_up_exps_b) {
gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, ids);
cb(gate_up, "ffn_moe_gate_up_biased", il);
@@ -2602,29 +2546,21 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
cb(up, "ffn_moe_up", il);
} else {
// separate gate and up path
- up = build_lora_mm_id(up_exps, cur, ids, up_exps_s, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
+ up = build_lora_mm_id(up_exps, cur, ids, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
cb(up, "ffn_moe_up", il);
- if (up_exps_s) {
- cb(up, "ffn_moe_up_scaled", il);
- }
-
if (up_exps_b) {
up = ggml_add_id(ctx0, up, up_exps_b, ids);
cb(up, "ffn_moe_up_biased", il);
}
if (gate_exps) {
- cur = build_lora_mm_id(gate_exps, cur, ids, gate_exps_s, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
+ cur = build_lora_mm_id(gate_exps, cur, ids, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
cb(cur, "ffn_moe_gate", il);
} else {
cur = up;
}
- if (gate_exps_s) {
- cb(cur, "ffn_moe_gate_scaled", il);
- }
-
if (gate_exps_b) {
cur = ggml_add_id(ctx0, cur, gate_exps_b, ids);
cb(cur, "ffn_moe_gate_biased", il);
@@ -2727,7 +2663,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
cur = amax_src1.res;
}
- ggml_tensor * experts = build_lora_mm_id(down_exps, cur, ids, down_exps_s, mm_ids, cached); // [n_embd, n_expert_used, n_tokens]
+ ggml_tensor * experts = build_lora_mm_id(down_exps, cur, ids, mm_ids, cached); // [n_embd, n_expert_used, n_tokens]
//ggml_prec_set_src(experts, GGML_PREC_BF16, 1); // (see TODO above)
cb(experts, "ffn_moe_down", il);
@@ -2745,10 +2681,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
return (*static_cast<decltype(build_experts) *>(user_data))(cur, ids, mm_ids, cached);
}, &build_experts);
- if (down_exps_s) {
- cb(experts, "ffn_moe_down_scaled", il);
- }
-
if (down_exps_b) {
experts = ggml_add_id(ctx0, experts, down_exps_b, selected_experts);
cb(experts, "ffn_moe_down_biased", il);
@@ -3485,8 +3417,6 @@ ggml_tensor * llm_graph_context::build_attn(
ggml_tensor * v_mla, // TODO: remove
float kq_scale,
int il) const {
- const llm_arch arch = model.arch;
-
GGML_ASSERT(v_mla == nullptr);
if (inp->self_k_rot[il]) {
@@ -3543,16 +3473,7 @@ ggml_tensor * llm_graph_context::build_attn(
}
if (wo) {
- if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
- // GLM4, GLM4_MOE, and JAIS2 have numerical issues with F16 accumulators
- cur = build_lora_mm(wo, cur);
- ggml_prec_set_acc(cur, GGML_PREC_BF16);
- if (wo_s) {
- cur = ggml_mul(ctx0, cur, wo_s);
- }
- } else {
- cur = build_lora_mm(wo, cur, wo_s);
- }
+ cur = build_lora_mm(wo, cur, wo_s);
}
if (wo_b) {
@@ -3603,8 +3524,6 @@ ggml_tensor * llm_graph_context::build_attn(
ggml_tensor * v_mla,
float kq_scale,
int il) const {
- const llm_arch arch = model.arch;
-
// these nodes are added to the graph together so that they are not reordered
// by doing so, the number of splits in the graph is reduced
// expand k later to enable rope fusion which directly writes into k-v cache
@@ -3632,16 +3551,7 @@ ggml_tensor * llm_graph_context::build_attn(
cb(cur, "kqv_out", il);
if (wo) {
- if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) {
- // GLM4 and GLM4_MOE have numerical issues with F16 accumulators
- cur = build_lora_mm(wo, cur);
- ggml_prec_set_acc(cur, GGML_PREC_BF16);
- if (wo_s) {
- cur = ggml_mul(ctx0, cur, wo_s);
- }
- } else {
- cur = build_lora_mm(wo, cur, wo_s);
- }
+ cur = build_lora_mm(wo, cur, wo_s);
}
if (wo_b) {
diff --git a/src/llama-graph.h b/src/llama-graph.h
index a9f8e5ce0..116b7132a 100644
--- a/src/llama-graph.h
+++ b/src/llama-graph.h
@@ -1179,6 +1179,7 @@ struct llm_graph_context {
ggml_tensor * cur) const;
// do mat_mul, while optionally apply lora and per-tensor scale
+ // w_s defaults to the weight scale of w
ggml_tensor * build_lora_mm(
ggml_tensor * w,
ggml_tensor * cur,
@@ -1190,7 +1191,6 @@ struct llm_graph_context {
ggml_tensor * w, // ggml_tensor * as
ggml_tensor * cur, // ggml_tensor * b
ggml_tensor * ids,
- ggml_tensor * w_s = nullptr,
ggml_tensor * slots = nullptr,
bool cached = true) const;
@@ -1263,9 +1263,6 @@ struct llm_graph_context {
int il,
ggml_tensor * probs_in = nullptr,
ggml_tensor * gate_up_exps = nullptr,
- ggml_tensor * up_exps_s = nullptr,
- ggml_tensor * gate_exps_s = nullptr,
- ggml_tensor * down_exps_s = nullptr,
ggml_tensor * selected_experts_in = nullptr,
ggml_tensor * post_exp_norm = nullptr,
float post_exp_norm_eps = 1e-5f) const;
@@ -1291,9 +1288,6 @@ struct llm_graph_context {
ggml_tensor * probs_in = nullptr,
ggml_tensor * gate_up_exps = nullptr,
ggml_tensor * gate_up_exps_b = nullptr,
- ggml_tensor * up_exps_s = nullptr,
- ggml_tensor * gate_exps_s = nullptr,
- ggml_tensor * down_exps_s = nullptr,
ggml_tensor * selected_experts_in = nullptr,
ggml_tensor * post_exp_norm = nullptr,
float post_exp_norm_eps = 1e-5f) const;
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 11c456947..06eb7c929 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1410,12 +1410,19 @@ bool llama_prec_policy::apply(ggml_tensor * res) const {
return false;
}
- const auto it = prec_src1.find(res->src[0]);
- if (it == prec_src1.end()) {
- return false;
+ bool applied = false;
+
+ const auto it_acc = prec_acc.find(res->src[0]);
+ if (it_acc != prec_acc.end()) {
+ applied |= ggml_prec_set_acc(res, it_acc->second);
}
- return ggml_prec_set_src(res, it->second, 1);
+ const auto it_src1 = prec_src1.find(res->src[0]);
+ if (it_src1 != prec_src1.end()) {
+ applied |= ggml_prec_set_src(res, it_src1->second, 1);
+ }
+
+ return applied;
}
void llama_prec_policy::load(llama_model_loader & ml, const llama_model & model) {
@@ -2044,6 +2051,42 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
}
}
+ // map each weight to its "<name>.scale" tensor: shape {1} for a 2D weight, {n_expert} for a 3D weight
+ {
+ auto ends_with = [](const std::string & str, const std::string & suffix) {
+ return str.size() >= suffix.size() && str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;
+ };
+ std::unordered_map<std::string, ggml_tensor *> weights;
+ for (const auto & [name, cur] : tensors_by_name) {
+ if (ends_with(name, ".weight")) {
+ weights.emplace(name.substr(0, name.size() - 7), cur);
+ }
+ }
+ for (const auto & [name, s] : tensors_by_name) {
+ if (!ends_with(name, ".scale")) {
+ continue;
+ }
+ const auto it = weights.find(name.substr(0, name.size() - 6));
+ if (it == weights.end()) {
+ continue;
+ }
+ ggml_tensor * w = it->second;
+ const bool ok = w->ne[3] == 1 && ggml_nelements(s) == w->ne[2] && s->ne[0] == w->ne[2];
+ if (ok) {
+ w_scales.emplace(w, s);
+ } else {
+ LLAMA_LOG_DEBUG("%s: '%s' is not a weight scale for '%s', skipping\n", __func__, name.c_str(), w->name);
+ }
+ }
+ }
+
+ // the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/60d8d70770c6776ff598c94bb586a859a38244f1/inference/model.py#L565
+ if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS || hparams.expert_router_f32) {
+ for (const auto & layer : layers) {
+ prec_policy.set_acc(layer.ffn_gate_inp, GGML_PREC_F32);
+ }
+ }
+
// per-tensor activation precision policy
prec_policy.load(ml, *this);
@@ -2575,6 +2618,11 @@ size_t llama_model::size() const {
return pimpl->n_bytes;
}
+ggml_tensor * llama_model::get_w_scale(const ggml_tensor * w) const {
+ const auto it = w_scales.find(w);
+ return it == w_scales.end() ? nullptr : it->second;
+}
+
size_t llama_model::n_tensors() const {
return tensors_by_name.size();
}
diff --git a/src/llama-model.h b/src/llama-model.h
index 6473980a8..eae5c6848 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -737,8 +737,7 @@ struct llama_meta_device_get_split_state_userdata {
struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const struct ggml_tensor * tensor, void * userdata);
struct llama_prec_policy {
- // the key is the weight tensor `res->src[0]`, stores the recommended accumulation type of the op (unused for now)
- // TODO: migrate ad-hoc ggml_prec_set_acc() calls to this container + update apply() to use it
+ // the key is the weight tensor `res->src[0]`, stores the recommended accumulation type of the op
std::unordered_map<const ggml_tensor *, ggml_prec> prec_acc;
// the key is the weight tensor `res->src[0]`, stores the recommended activation precision type
@@ -746,6 +745,12 @@ struct llama_prec_policy {
bool apply(ggml_tensor * res) const;
+ void set_acc(const ggml_tensor * w, ggml_prec prec) {
+ if (w) {
+ prec_acc[w] = prec;
+ }
+ }
+
void load(llama_model_loader & ml, const llama_model & model);
};
@@ -818,6 +823,11 @@ struct llama_model {
llama_hadamard hdmd;
+ // weight -> "<name>.scale" tensor (e.g. NVFP4 scale2), applied by build_lora_mm and build_lora_mm_id
+ std::unordered_map<const ggml_tensor *, ggml_tensor *> w_scales;
+
+ ggml_tensor * get_w_scale(const ggml_tensor * w) const;
+
// see llama_model_set_attn_rot
bool attn_rot = true;
diff --git a/src/models/clover-moe.cpp b/src/models/clover-moe.cpp
index 8892521ee..837997261 100644
--- a/src/models/clover-moe.cpp
+++ b/src/models/clover-moe.cpp
@@ -398,10 +398,7 @@ ggml_tensor * llama_model_clover_moe::graph::build_layer_ffn(ggml_tensor * cur,
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
- nullptr, model.layers[il].ffn_gate_up_exps,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ nullptr, model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
if (model.layers[il].ffn_up_shexp != nullptr) {
diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cpp
index 52b959672..c4d778e50 100644
--- a/src/models/cohere2moe.cpp
+++ b/src/models/cohere2moe.cpp
@@ -226,10 +226,7 @@ llama_model_cohere2moe::graph::graph(const llm_graph_params & params) : llm_grap
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
- nullptr, layer.ffn_gate_up_exps,
- layer.ffn_up_exps_s,
- layer.ffn_gate_exps_s,
- layer.ffn_down_exps_s);
+ nullptr, layer.ffn_gate_up_exps);
cb(cur, "ffn_moe_out", il);
if (layer.ffn_up_shexp) {
@@ -371,10 +368,7 @@ llama_model_cohere2moe::graph_mtp::graph_mtp(const llm_graph_params & params) :
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
- nullptr, layer.ffn_gate_up_exps,
- layer.ffn_up_exps_s,
- layer.ffn_gate_exps_s,
- layer.ffn_down_exps_s);
+ nullptr, layer.ffn_gate_up_exps);
cb(cur, "mtp_ffn_moe_out", il);
if (layer.ffn_up_shexp) {
diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cpp
index 853389b88..8f6f34790 100644
--- a/src/models/deepseek32.cpp
+++ b/src/models/deepseek32.cpp
@@ -383,10 +383,7 @@ llama_model_deepseek32::graph::graph(const llm_graph_params & params) :
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr,
- model.layers[il].ffn_gate_up_exps,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
// FFN shared expert
@@ -624,10 +621,7 @@ llama_model_deepseek32::graph_mtp::graph_mtp(const llm_graph_params & params)
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr,
- layer.ffn_gate_up_exps,
- layer.ffn_up_exps_s,
- layer.ffn_gate_exps_s,
- layer.ffn_down_exps_s);
+ layer.ffn_gate_up_exps);
cb(moe_out, "mtp_ffn_moe_out", il);
// FFN shared expert
diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp
index dfc9858e7..f4269e043 100644
--- a/src/models/deepseek4.cpp
+++ b/src/models/deepseek4.cpp
@@ -1284,9 +1284,6 @@ llama_model_deepseek4::graph::graph(const llm_graph_params & params) :
il,
nullptr,
nullptr,
- nullptr,
- nullptr,
- nullptr,
selected_experts);
cb(moe_out, "ffn_moe_out", il);
diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp
index ed93be534..e481cbd1d 100644
--- a/src/models/deepseek41.cpp
+++ b/src/models/deepseek41.cpp
@@ -203,6 +203,8 @@ void llama_model_deepseek41::load_arch_tensors(llama_model_loader & ml) {
}
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
+ // the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017/blob/1bae0e1088993b9b82de08f59de5be5a34e42170/inference/model.py#L813
+ prec_policy.set_acc(layer.ffn_gate_inp, GGML_PREC_F32);
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);
layer.ffn_exp_probs_b_vl = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B_VL, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
@@ -874,9 +876,7 @@ llama_model_deepseek41::graph::graph(const llm_graph_params & params) :
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "ffn_norm", il);
- // the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017/blob/1bae0e1088993b9b82de08f59de5be5a34e42170/inference/model.py#L813
ggml_tensor * logits = build_lora_mm(layer.ffn_gate_inp, cur);
- ggml_prec_set_acc(logits, GGML_PREC_F32);
cb(logits, "ffn_moe_logits", il);
// image (embedding) tokens use the vision router bias, as DeepSeek-V4
diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cpp
index da16f2480..2a2d5500b 100644
--- a/src/models/dots3note.cpp
+++ b/src/models/dots3note.cpp
@@ -401,10 +401,7 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr,
- model.layers[il].ffn_gate_up_exps,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
ggml_tensor * ffn_shexp =
diff --git a/src/models/flint.cpp b/src/models/flint.cpp
index 520334ce6..173278b6f 100644
--- a/src/models/flint.cpp
+++ b/src/models/flint.cpp
@@ -158,10 +158,7 @@ llama_model_flint::graph::graph(const llm_graph_params & params) :
LLM_FFN_SILU, hparams.expert_weights_norm,
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
- il, nullptr, nullptr,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ il);
cb(moe_out, "ffn_moe_out", il);
{
diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp
index 1a377fa3c..491ef1283 100644
--- a/src/models/gemma4.cpp
+++ b/src/models/gemma4.cpp
@@ -406,10 +406,7 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :
1.0f,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
il, logits,
- model.layers[il].ffn_gate_up_exps,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ model.layers[il].ffn_gate_up_exps);
cur_moe = build_norm(cur_moe,
model.layers[il].ffn_post_norm_2, nullptr,
LLM_NORM_RMS, il);
diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp
index 6daafd007..c0ff62e4a 100644
--- a/src/models/glm-dsa.cpp
+++ b/src/models/glm-dsa.cpp
@@ -431,10 +431,7 @@ llama_model_glm_dsa::graph::graph(const llm_graph_params & params) :
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr,
- model.layers[il].ffn_gate_up_exps,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
// FFN shared expert
@@ -672,10 +669,7 @@ llama_model_glm_dsa::graph_mtp::graph_mtp(const llm_graph_params & params)
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr,
- layer.ffn_gate_up_exps,
- layer.ffn_up_exps_s,
- layer.ffn_gate_exps_s,
- layer.ffn_down_exps_s);
+ layer.ffn_gate_up_exps);
cb(moe_out, "mtp_ffn_moe_out", il);
// FFN shared expert
diff --git a/src/models/glm4-moe.cpp b/src/models/glm4-moe.cpp
index 9560fd3d0..a3e2fa46e 100644
--- a/src/models/glm4-moe.cpp
+++ b/src/models/glm4-moe.cpp
@@ -113,6 +113,11 @@ void llama_model_glm4_moe::load_arch_tensors(llama_model_loader & ml) {
layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED | flags);
layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, TENSOR_NOT_REQUIRED | flags);
}
+
+ // numerical issues with F16 accumulators
+ prec_policy.set_acc(layer.wo, GGML_PREC_BF16);
+ prec_policy.set_acc(layer.ffn_down, GGML_PREC_BF16);
+ prec_policy.set_acc(layer.ffn_down_shexp, GGML_PREC_BF16);
}
}
diff --git a/src/models/glm4.cpp b/src/models/glm4.cpp
index e57c51cbe..5c50b2f05 100644
--- a/src/models/glm4.cpp
+++ b/src/models/glm4.cpp
@@ -58,6 +58,10 @@ void llama_model_glm4::load_arch_tensors(llama_model_loader &) {
layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, flags | TENSOR_NOT_REQUIRED);
layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, flags | TENSOR_NOT_REQUIRED);
}
+
+ // numerical issues with F16 accumulators
+ prec_policy.set_acc(layer.wo, GGML_PREC_BF16);
+ prec_policy.set_acc(layer.ffn_down, GGML_PREC_BF16);
}
}
diff --git a/src/models/granite-hybrid.cpp b/src/models/granite-hybrid.cpp
index be352dfd7..d638663af 100644
--- a/src/models/granite-hybrid.cpp
+++ b/src/models/granite-hybrid.cpp
@@ -260,11 +260,7 @@ ggml_tensor * llama_model_granite_hybrid::graph::build_layer_ffn(ggml_tensor *
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
- il,
- nullptr, nullptr,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ il);
cb(moe_out, "ffn_moe_out", il);
// For Granite MoE Shared
diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp
index be9e55778..2e2609f52 100644
--- a/src/models/granite-swa.cpp
+++ b/src/models/granite-swa.cpp
@@ -277,10 +277,7 @@ ggml_tensor * llama_model_granite_swa::graph::build_layer_ffn(
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
il,
- nullptr, model.layers[il].ffn_gate_up_exps,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ nullptr, model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
// For Granite MoE Shared - gate+up kept fused in ffn_up_shexp
diff --git a/src/models/granite.cpp b/src/models/granite.cpp
index 0ab9084da..0a5dd78ea 100644
--- a/src/models/granite.cpp
+++ b/src/models/granite.cpp
@@ -269,11 +269,7 @@ ggml_tensor * llama_model_granite::graph::build_layer_ffn(
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
- il,
- nullptr, nullptr,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ il);
cb(moe_out, "ffn_moe_out", il);
// For Granite MoE Shared
diff --git a/src/models/hy-v3.cpp b/src/models/hy-v3.cpp
index c47c373f8..f53d94827 100644
--- a/src/models/hy-v3.cpp
+++ b/src/models/hy-v3.cpp
@@ -168,10 +168,7 @@ llama_model_hy_v3::graph::graph(const llm_graph_params & params) : llm_graph_con
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
- nullptr, model.layers[il].ffn_gate_up_exps,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ nullptr, model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
// shared expert (always active, no gate)
@@ -323,10 +320,7 @@ llama_model_hy_v3::graph_mtp::graph_mtp(const llm_graph_params & params)
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
- nullptr, layer.ffn_gate_up_exps,
- layer.ffn_up_exps_s,
- layer.ffn_gate_exps_s,
- layer.ffn_down_exps_s);
+ nullptr, layer.ffn_gate_up_exps);
cb(moe_out, "mtp_ffn_moe_out", il);
ggml_tensor * sh_out = build_ffn(cur,
diff --git a/src/models/inkling2.cpp b/src/models/inkling2.cpp
index 3e43ef418..45477a69e 100644
--- a/src/models/inkling2.cpp
+++ b/src/models/inkling2.cpp
@@ -97,6 +97,7 @@ void llama_model_inkling2::load_arch_tensors(llama_model_loader &) {
// output
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab_padded}, 0);
+ prec_policy.set_acc(output, GGML_PREC_F32);
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
output_res_query = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_QUERY, "weight"), {n_embd}, 0);
@@ -406,10 +407,7 @@ ggml_tensor * llama_model_inkling2::graph::build_layer(int il, int ip, ggml_tens
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
il,
nullptr,
- layer.ffn_gate_up_exps,
- layer.ffn_up_exps_s,
- layer.ffn_gate_exps_s,
- layer.ffn_down_exps_s);
+ layer.ffn_gate_up_exps);
cur = convolve(cur, layer.ffn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa + n_embd);
cb(cur, "ffn_out", il);
@@ -433,7 +431,6 @@ void llama_model_inkling2::graph::build_output(ggml_tensor * cur, const std::vec
// lm_head
cur = build_lora_mm(model.output, cur);
- ggml_prec_set_acc(cur, GGML_PREC_F32);
if (hparams.f_final_logit_softcapping != 0.0f) {
const float cap = hparams.f_final_logit_softcapping;
diff --git a/src/models/jais2.cpp b/src/models/jais2.cpp
index 8376423ea..1f077fd86 100644
--- a/src/models/jais2.cpp
+++ b/src/models/jais2.cpp
@@ -42,6 +42,10 @@ void llama_model_jais2::load_arch_tensors(llama_model_loader &) {
layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, 0);
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, 0);
+
+ // numerical issues with F16 accumulators
+ prec_policy.set_acc(layer.wo, GGML_PREC_BF16);
+ prec_policy.set_acc(layer.ffn_down, GGML_PREC_BF16);
}
}
diff --git a/src/models/kolibri1.cpp b/src/models/kolibri1.cpp
index c4d8eca1b..64182d186 100644
--- a/src/models/kolibri1.cpp
+++ b/src/models/kolibri1.cpp
@@ -142,7 +142,7 @@ llama_model_kolibri1::graph::graph(const llm_graph_params & params)
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
logits,
- nullptr, nullptr, nullptr, nullptr, selected);
+ nullptr, selected);
cb(moe_out, "ffn_moe_out", il);
ggml_tensor * shared = build_ffn(cur,
diff --git a/src/models/lfm-base.cpp b/src/models/lfm-base.cpp
index 3627c527d..03e59bd09 100644
--- a/src/models/lfm-base.cpp
+++ b/src/models/lfm-base.cpp
@@ -116,10 +116,5 @@ ggml_tensor * llm_build_lfm_base::build_lfm_moe_ffn(ggml_tensor * cur,
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
static_cast<llama_expert_gating_func_type>(hparams.expert_gating_func),
- il,
- nullptr,
- nullptr,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ il);
}
diff --git a/src/models/llama.cpp b/src/models/llama.cpp
index 26d54968c..3925699f4 100644
--- a/src/models/llama.cpp
+++ b/src/models/llama.cpp
@@ -198,11 +198,7 @@ llama_model_llama::graph<embed>::graph(const llm_graph_params & params) : llm_gr
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
- il,
- nullptr, nullptr,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ il);
cb(cur, "ffn_moe_out", il);
}
cur = ggml_add(ctx0, cur, ffn_inp);
diff --git a/src/models/mellum.cpp b/src/models/mellum.cpp
index 57dd5b79a..d0b3609b1 100644
--- a/src/models/mellum.cpp
+++ b/src/models/mellum.cpp
@@ -199,11 +199,7 @@ llama_model_mellum::graph<iswa>::graph(const llm_graph_params & params) : llm_gr
LLM_FFN_SILU, hparams.expert_weights_norm,
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
- il,
- nullptr, nullptr,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ il);
cb(moe_out, "ffn_moe_out", il);
cur = moe_out;
@@ -351,11 +347,7 @@ llama_model_mellum::graph_mtp<iswa>::graph_mtp(const llm_graph_params & params)
LLM_FFN_SILU, hparams.expert_weights_norm,
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
- il,
- nullptr, nullptr,
- layer.ffn_up_exps_s,
- layer.ffn_gate_exps_s,
- layer.ffn_down_exps_s);
+ il);
cb(cur, "mtp_ffn_moe_out", il);
cur = ggml_add(ctx0, cur, ffn_inp);
diff --git a/src/models/metis16.cpp b/src/models/metis16.cpp
index 8a433f3cc..0a57a6ab0 100644
--- a/src/models/metis16.cpp
+++ b/src/models/metis16.cpp
@@ -329,7 +329,7 @@ llama_model_metis16::graph::graph(const llm_graph_params & params) :
ggml_tensor * moe_out = build_moe_ffn(latent, nullptr, nullptr, nullptr, layer.ffn_down_exps, nullptr,
n_exp, n_used, LLM_FFN_SILU, false, 0.0f, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT, il_p,
- logits, layer.ffn_gate_up_exps, nullptr, nullptr, nullptr, selected);
+ logits, layer.ffn_gate_up_exps, selected);
ggml_tensor * shexp = build_ffn(latent,
layer.ffn_up_shexp, NULL, NULL,
layer.ffn_gate_shexp, NULL, NULL,
diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp
index 4f1184590..8560c710a 100644
--- a/src/models/minimax-m3.cpp
+++ b/src/models/minimax-m3.cpp
@@ -572,10 +572,7 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap
LLM_FFN_SWIGLU_OAI_MOE, hparams.expert_weights_norm,
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
- il, nullptr, nullptr,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ il);
cb(moe_out, "ffn_moe_out", il);
// shared expert (swigluoai)
diff --git a/src/models/mistral-large4.cpp b/src/models/mistral-large4.cpp
index 6407d7663..aab49df9d 100644
--- a/src/models/mistral-large4.cpp
+++ b/src/models/mistral-large4.cpp
@@ -420,10 +420,7 @@ llama_model_mistral_large4::graph::graph(const llm_graph_params & params) :
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr,
- layer.ffn_gate_up_exps,
- layer.ffn_up_exps_s,
- layer.ffn_gate_exps_s,
- layer.ffn_down_exps_s);
+ layer.ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
ggml_tensor * ffn_shexp = build_ffn(cur,
diff --git a/src/models/mistral3.cpp b/src/models/mistral3.cpp
index ab8121863..e1e6981bb 100644
--- a/src/models/mistral3.cpp
+++ b/src/models/mistral3.cpp
@@ -248,11 +248,7 @@ llama_model_mistral3::graph::graph(const llm_graph_params & params) : llm_graph_
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
- il,
- nullptr, nullptr,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ il);
cb(cur, "ffn_moe_out", il);
}
cur = ggml_add(ctx0, cur, ffn_inp);
diff --git a/src/models/muse-spark.cpp b/src/models/muse-spark.cpp
index ea6ab84ab..10de4676e 100644
--- a/src/models/muse-spark.cpp
+++ b/src/models/muse-spark.cpp
@@ -180,7 +180,7 @@ llama_model_muse_spark::graph::graph(const llm_graph_params & params)
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
router_logits,
- nullptr, nullptr, nullptr, nullptr, nullptr,
+ nullptr, nullptr,
model.layers[il].ffn_post_exp_norm, hparams.f_post_norm_eps);
cb(moe_out, "ffn_moe_out", il);
diff --git a/src/models/nemotron-h-moe.cpp b/src/models/nemotron-h-moe.cpp
index 475355832..3aa385379 100644
--- a/src/models/nemotron-h-moe.cpp
+++ b/src/models/nemotron-h-moe.cpp
@@ -118,10 +118,7 @@ llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llm_graph_params & params
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
il,
- router_logits, nullptr,
- layer.ffn_up_exps_s,
- nullptr, // no gate
- layer.ffn_down_exps_s);
+ router_logits);
cb(moe_out, "mtp_ffn_moe_out", il);
if (layer.ffn_latent_up) {
diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cpp
index 313f42f1c..18e45320a 100644
--- a/src/models/nemotron-h.cpp
+++ b/src/models/nemotron-h.cpp
@@ -322,10 +322,7 @@ ggml_tensor * llama_model_nemotron_h::graph::build_ffn_layer(ggml_tensor * cur,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
il,
- router_logits, nullptr,
- model.layers[il].ffn_up_exps_s,
- nullptr, // no gate
- model.layers[il].ffn_down_exps_s);
+ router_logits);
cb(moe_out, "ffn_moe_out", il);
if (model.layers[il].ffn_latent_up) {
diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp
index c0ee6fa03..c92b84d9d 100644
--- a/src/models/qwen35moe.cpp
+++ b/src/models/qwen35moe.cpp
@@ -479,10 +479,7 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_ffn(ggml_tensor * cur, c
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
- nullptr, model.layers[il].ffn_gate_up_exps,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ nullptr, model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
// Add shared experts if present - following Qwen3Next reference implementation
@@ -651,10 +648,7 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llm_graph_params & params)
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
- nullptr, layer.ffn_gate_up_exps,
- layer.ffn_up_exps_s,
- layer.ffn_gate_exps_s,
- layer.ffn_down_exps_s);
+ nullptr, layer.ffn_gate_up_exps);
cb(moe_out, "mtp_ffn_moe_out", il);
if (layer.ffn_up_shexp != nullptr) {
diff --git a/src/models/qwen3moe.cpp b/src/models/qwen3moe.cpp
index ce7144a22..fc57928f5 100644
--- a/src/models/qwen3moe.cpp
+++ b/src/models/qwen3moe.cpp
@@ -136,11 +136,7 @@ llama_model_qwen3moe::graph::graph(const llm_graph_params & params) : llm_graph_
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
- il,
- nullptr, nullptr,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ il);
cb(moe_out, "ffn_moe_out", il);
cur = moe_out;
diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cpp
index 9546fba58..208aa49d2 100644
--- a/src/models/qwen3next.cpp
+++ b/src/models/qwen3next.cpp
@@ -561,10 +561,7 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_ffn(ggml_tensor * cur, c
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
- nullptr, model.layers[il].ffn_gate_up_exps,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ nullptr, model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
// Add shared experts if present - following Qwen3Next reference implementation
@@ -748,10 +745,7 @@ llama_model_qwen3next::graph_mtp::graph_mtp(const llm_graph_params & params)
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
- nullptr, layer.ffn_gate_up_exps,
- layer.ffn_up_exps_s,
- layer.ffn_gate_exps_s,
- layer.ffn_down_exps_s);
+ nullptr, layer.ffn_gate_up_exps);
cb(moe_out, "mtp_ffn_moe_out", il);
if (layer.ffn_up_shexp != nullptr) {
diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp
index 63c918602..106530490 100644
--- a/src/models/qwen4exp.cpp
+++ b/src/models/qwen4exp.cpp
@@ -1230,10 +1230,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_ffn(ggml_tensor * cur, co
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
- nullptr, model.layers[il].ffn_gate_up_exps,
- model.layers[il].ffn_up_exps_s,
- model.layers[il].ffn_gate_exps_s,
- model.layers[il].ffn_down_exps_s);
+ nullptr, model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
// shared experts, as in the Qwen3Next reference
diff --git a/src/models/zaya1.cpp b/src/models/zaya1.cpp
index 2763500f4..013658c73 100644
--- a/src/models/zaya1.cpp
+++ b/src/models/zaya1.cpp
@@ -277,9 +277,6 @@ ggml_tensor * llama_model_zaya1::graph::build_router_moe(const llama_model & mod
il,
logits,
layer.ffn_gate_up_exps,
- nullptr,
- nullptr,
- nullptr,
expert);
return ggml_mul(ctx0, moe, keep);
diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv
index 9bd4b82bf..2a4f4b361 100644
--- a/tests/fusion/MTL.csv
+++ b/tests/fusion/MTL.csv
@@ -41,6 +41,7 @@ cohere2 ,0 ,any ,NORM+MUL , 3
cohere2moe ,1 ,any ,ADD+ADD , 2
cohere2moe ,1 ,any ,MUL+ADD , 2
cohere2moe ,1 ,any ,RMS_NORM+MUL , 3
+command-r ,0 ,any ,ADD+ADD , 2
command-r ,0 ,any ,NORM+MUL , 3
dbrx ,0 ,any ,MUL+ADD , 4
dbrx ,0 ,any ,NORM+MUL , 5
@@ -207,6 +208,7 @@ llama ,0 ,any ,MUL+ADD , 4
llama ,0 ,any ,RMS_NORM+MUL , 5
llama ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
llama4 ,0 ,any ,ADD+ADD , 2
+llama4 ,0 ,any ,MUL+ADD , 4
llama4 ,0 ,any ,RMS_NORM+MUL , 9
maincoder ,0 ,any ,RMS_NORM+MUL , 9
maion-coder ,0 ,any ,MUL+ADD , 4
@@ -257,8 +259,7 @@ moshi ,0 ,any ,RMS_NORM+MUL , 5
mpt ,0 ,any ,NORM+MUL+ADD , 5
muse-glimmer ,0 ,any ,RMS_NORM+MUL , 10
muse-glimmer ,0 ,any ,RMS_NORM+MUL+ADD , 3
-muse-spark ,0 ,prefill ,MUL+ADD , 2
-muse-spark ,0 ,decode ,MUL+ADD , 4
+muse-spark ,0 ,any ,MUL+ADD , 4
muse-spark ,0 ,any ,RMS_NORM+MUL , 11
muse-spark ,0 ,any ,RMS_NORM+MUL+ADD , 3
muse-spark ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2