Commit 19bc41dbb for llama.cpp

commit 19bc41dbb41e858eebc269b49ccdec1f0d28880c
Author: Xuan-Son Nguyen <son@huggingface.co>
Date:   Sat Nov 21 16:30:01 2026 +0100

    model: apply .scale tensors automatically + fix correctness (#32669)

    * model: apply .scale tensors automatically

    * rm build_moe_ffn "_s" overload

    * use llama_prec_policy

    * fix ci

    * build_lora_mm_id: rm w_s

diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
index 595a5824a..c4e37ed46 100644
--- a/ggml/src/ggml-backend-meta.cpp
+++ b/ggml/src/ggml-backend-meta.cpp
@@ -2154,8 +2154,28 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend,
                     }
                 };

-                skip_unrelated();
-                if (id + 1 >= cgraph->n_nodes) {
+                // Chain of MULs with MIRRORED src[1], returns false at the end of the graph
+                auto absorb_muls = [&]() -> bool {
+                    while (true) {
+                        skip_unrelated();
+                        if (id + 1 >= cgraph->n_nodes) {
+                            return false;
+                        }
+                        ggml_tensor * next = cgraph->nodes[id+1];
+                        if (next->op == GGML_OP_MUL && next->src[0] == node &&
+                                ggml_backend_meta_get_split_state(next->src[1], false).axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) {
+                            node = next;
+                            id++;
+                            idr = id;
+                            n_used = ggml_node_get_use_count(cgraph, id);
+                        } else {
+                            return true;
+                        }
+                    }
+                };
+
+                // MULs before the PARTIAL bias (e.g. a per-expert weight scale): sum_j(s*p_j + b/n) == s*sum_j(p_j) + b
+                if (!absorb_muls()) {
                     return idr;
                 }
                 {
@@ -2169,22 +2189,8 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend,
                         n_used = ggml_node_get_use_count(cgraph, id);
                     }
                 }
-                // Chain of MULs with MIRRORED src[1]
-                while (true) {
-                    skip_unrelated();
-                    if (id + 1 >= cgraph->n_nodes) {
-                        return idr;
-                    }
-                    ggml_tensor * next = cgraph->nodes[id+1];
-                    if (next->op == GGML_OP_MUL && next->src[0] == node &&
-                            ggml_backend_meta_get_split_state(next->src[1], false).axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) {
-                        node = next;
-                        id++;
-                        idr = id;
-                        n_used = ggml_node_get_use_count(cgraph, id);
-                    } else {
-                        break;
-                    }
+                if (!absorb_muls()) {
+                    return idr;
                 }

                 if (n_used != node->ne[1] || id + 2*n_used-1 >= cgraph->n_nodes) {
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index c5c9ddf01..1bd3f8d0f 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -1810,16 +1810,21 @@ ggml_tensor * llm_graph_context::build_lora_mm(
           ggml_tensor * w,
           ggml_tensor * cur,
           ggml_tensor * w_s) const {
+    if (!w_s) {
+        w_s = model.get_w_scale(w);
+    }
+
     // a Hadamard-folded weight reads the transformed input; a LoRA delta lives in the primal basis and reads `cur`
     ggml_tensor * cur_mm = build_hadamard_input(w, cur);
     ggml_tensor * res = ggml_mul_mat(ctx0, w, cur_mm);

-    prec_policy.apply(res);
-
     if (w->type == GGML_TYPE_NVFP4) {
         ggml_prec_set_acc(res, GGML_PREC_BF16);
     }

+
+    prec_policy.apply(res);
+
     if (w_s) {
         res = ggml_mul(ctx0, res, w_s);
     }
@@ -1886,9 +1891,10 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
           ggml_tensor * w,   // ggml_tensor * as
           ggml_tensor * cur, // ggml_tensor * b
           ggml_tensor * ids,
-          ggml_tensor * w_s,
           ggml_tensor * slots,
                  bool   cached) const {
+    ggml_tensor * w_s = model.get_w_scale(w);
+
     // as in build_lora_mm: the folded weight reads the transformed input, the LoRA delta reads `cur`
     ggml_tensor * cur_mm = build_hadamard_input(w, cur);
     // the experts in the MoE cache are selected by their slots
@@ -1896,12 +1902,13 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
         ggml_mul_mat_id(ctx0, w, cur_mm, ids) :
         ggml_mul_mat_id(ctx0, cached ? moe_cache->get_experts(w) : w, cur_mm, slots);

-    prec_policy.apply(res);
-
     if (w->type == GGML_TYPE_NVFP4) {
         ggml_prec_set_acc(res, GGML_PREC_BF16);
     }

+    // after the NVFP4 default, so that the policy can request a higher precision
+    prec_policy.apply(res);
+
     if (w_s) {
         const int64_t n_expert = w_s->ne[0];
         const int64_t n_tokens = cur->ne[2];
@@ -2115,34 +2122,7 @@ ggml_tensor * llm_graph_context::build_ffn(
                  int   il) const {
     const llm_arch arch = model.arch;

-    // NVFP4 support is currently restricted to
-    // 1) LORA absence (*_s would be applied after LORA residual, which is incorrect)
-    // 2) bias absense (*_s would be applied after bias addition, which is incorrect)
-    // TODO: disambiguate LLM-architectural scales (which use *_s) from NVFP4 scale_2 (which also uses *_s currently)
-    auto has_lora = [this](ggml_tensor * w) {
-        if (!w) {
-            return false;
-        }
-        for (const auto & lora : *loras) {
-            if (lora.first->get_weight(w) != nullptr) {
-                return true;
-            }
-        }
-        return false;
-    };
-
-    auto is_scaled_low_precision = [](ggml_tensor * w) {
-        return w && (w->type == GGML_TYPE_NVFP4 || w->type == GGML_TYPE_F8_E4M3);
-    };
-
-    GGML_ASSERT(!up_s   || !up_b   || !is_scaled_low_precision(up));
-    GGML_ASSERT(!gate_s || !gate_b || !is_scaled_low_precision(gate));
-    GGML_ASSERT(!down_s || !down_b || !is_scaled_low_precision(down));
-    GGML_ASSERT(!up_s   || !is_scaled_low_precision(up)   || !has_lora(up));
-    GGML_ASSERT(!gate_s || !is_scaled_low_precision(gate) || !has_lora(gate));
-    GGML_ASSERT(!down_s || !is_scaled_low_precision(down) || !has_lora(down));
-
-    ggml_tensor * tmp = up ? build_lora_mm(up, cur) : cur;
+    ggml_tensor * tmp = up ? build_lora_mm(up, cur, up_s) : cur;
     cb(tmp, "ffn_up", il);

     if (up_b) {
@@ -2150,21 +2130,16 @@ ggml_tensor * llm_graph_context::build_ffn(
         cb(tmp, "ffn_up_b", il);
     }

-    if (up_s) {
-        tmp = ggml_mul(ctx0, tmp, up_s);
-        cb(tmp, "ffn_up_s", il);
-    }
-
     if (gate) {
         switch (type_gate) {
             case LLM_FFN_SEQ:
                 {
-                    cur = build_lora_mm(gate, tmp);
+                    cur = build_lora_mm(gate, tmp, gate_s);
                     cb(cur, "ffn_gate", il);
                 } break;
             case LLM_FFN_PAR:
                 {
-                    cur = build_lora_mm(gate, cur);
+                    cur = build_lora_mm(gate, cur, gate_s);
                     cb(cur, "ffn_gate", il);
                 } break;
         }
@@ -2174,11 +2149,6 @@ ggml_tensor * llm_graph_context::build_ffn(
             cb(cur, "ffn_gate_b", il);
         }

-        if (gate_s) {
-            cur = ggml_mul(ctx0, cur, gate_s);
-            cb(cur, "ffn_gate_s", il);
-        }
-
     } else {
         cur = tmp;
     }
@@ -2300,11 +2270,7 @@ ggml_tensor * llm_graph_context::build_ffn(
     }

     if (down) {
-        cur = build_lora_mm(down, cur);
-        if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
-            // GLM4, GLM4_MOE, and JAIS2 have numerical issues with F16 accumulators
-            ggml_prec_set_acc(cur, GGML_PREC_BF16);
-        }
+        cur = build_lora_mm(down, cur, down_s);
     }

     if (down_b) {
@@ -2315,11 +2281,6 @@ ggml_tensor * llm_graph_context::build_ffn(
         cur = ggml_add(ctx0, cur, down_b);
     }

-    if (down_s) {
-        cur = ggml_mul(ctx0, cur, down_s);
-        cb(cur, "ffn_down_s", il);
-    }
-
     return cur;
 }

@@ -2339,9 +2300,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
                  int   il,
          ggml_tensor * probs_in,
          ggml_tensor * gate_up_exps,
-         ggml_tensor * up_exps_s,
-         ggml_tensor * gate_exps_s,
-         ggml_tensor * down_exps_s,
          ggml_tensor * selected_experts_in,
          ggml_tensor * post_exp_norm,
                float   post_exp_norm_eps) const {
@@ -2362,9 +2320,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
         probs_in,
         gate_up_exps,
         /* gate_up_exps_b */ nullptr,
-        up_exps_s,
-        gate_exps_s,
-        down_exps_s,
         selected_experts_in,
         post_exp_norm,
         post_exp_norm_eps
@@ -2392,9 +2347,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
          ggml_tensor * probs_in,
          ggml_tensor * gate_up_exps,
          ggml_tensor * gate_up_exps_b,
-         ggml_tensor * up_exps_s,
-         ggml_tensor * gate_exps_s,
-         ggml_tensor * down_exps_s,
          ggml_tensor * selected_experts_in,
          ggml_tensor * post_exp_norm,
                float   post_exp_norm_eps) const {
@@ -2407,10 +2359,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(

     if (probs_in == nullptr) {
         logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens]
-        if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS || hparams.expert_router_f32) {
-            // the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/60d8d70770c6776ff598c94bb586a859a38244f1/inference/model.py#L565
-            ggml_prec_set_acc(logits, GGML_PREC_F32);
-        }
         cb(logits, "ffn_moe_logits", il);
     } else {
         logits = probs_in;
@@ -2583,13 +2531,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn(

         if (gate_up_exps) {
             // merged gate_up path: one mul_mat_id, then split into gate and up views
-            ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, ids, up_exps_s, mm_ids, cached); // [n_ff*2, n_expert_used, n_tokens]
+            ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, ids, mm_ids, cached); // [n_ff*2, n_expert_used, n_tokens]
             cb(gate_up, "ffn_moe_gate_up", il);

-            if (up_exps_s) {
-                cb(gate_up, "ffn_moe_gate_up_scaled", il);
-            }
-
             if (gate_up_exps_b) {
                 gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, ids);
                 cb(gate_up, "ffn_moe_gate_up_biased", il);
@@ -2602,29 +2546,21 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
             cb(up, "ffn_moe_up", il);
         } else {
             // separate gate and up path
-            up = build_lora_mm_id(up_exps, cur, ids, up_exps_s, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
+            up = build_lora_mm_id(up_exps, cur, ids, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
             cb(up, "ffn_moe_up", il);

-            if (up_exps_s) {
-                cb(up, "ffn_moe_up_scaled", il);
-            }
-
             if (up_exps_b) {
                 up = ggml_add_id(ctx0, up, up_exps_b, ids);
                 cb(up, "ffn_moe_up_biased", il);
             }

             if (gate_exps) {
-                cur = build_lora_mm_id(gate_exps, cur, ids, gate_exps_s, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
+                cur = build_lora_mm_id(gate_exps, cur, ids, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
                 cb(cur, "ffn_moe_gate", il);
             } else {
                 cur = up;
             }

-            if (gate_exps_s) {
-                cb(cur, "ffn_moe_gate_scaled", il);
-            }
-
             if (gate_exps_b) {
                 cur = ggml_add_id(ctx0, cur, gate_exps_b, ids);
                 cb(cur, "ffn_moe_gate_biased", il);
@@ -2727,7 +2663,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
             cur = amax_src1.res;
         }

-        ggml_tensor * experts = build_lora_mm_id(down_exps, cur, ids, down_exps_s, mm_ids, cached); // [n_embd, n_expert_used, n_tokens]
+        ggml_tensor * experts = build_lora_mm_id(down_exps, cur, ids, mm_ids, cached); // [n_embd, n_expert_used, n_tokens]
         //ggml_prec_set_src(experts, GGML_PREC_BF16, 1); // (see TODO above)
         cb(experts, "ffn_moe_down", il);

@@ -2745,10 +2681,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
                     return (*static_cast<decltype(build_experts) *>(user_data))(cur, ids, mm_ids, cached);
                 }, &build_experts);

-    if (down_exps_s) {
-        cb(experts, "ffn_moe_down_scaled", il);
-    }
-
     if (down_exps_b) {
         experts = ggml_add_id(ctx0, experts, down_exps_b, selected_experts);
         cb(experts, "ffn_moe_down_biased", il);
@@ -3485,8 +3417,6 @@ ggml_tensor * llm_graph_context::build_attn(
         ggml_tensor * v_mla, // TODO: remove
             float     kq_scale,
             int       il) const {
-    const llm_arch arch = model.arch;
-
     GGML_ASSERT(v_mla == nullptr);

     if (inp->self_k_rot[il]) {
@@ -3543,16 +3473,7 @@ ggml_tensor * llm_graph_context::build_attn(
     }

     if (wo) {
-        if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
-            // GLM4, GLM4_MOE, and JAIS2 have numerical issues with F16 accumulators
-            cur = build_lora_mm(wo, cur);
-            ggml_prec_set_acc(cur, GGML_PREC_BF16);
-            if (wo_s) {
-                cur = ggml_mul(ctx0, cur, wo_s);
-            }
-        } else {
-            cur = build_lora_mm(wo, cur, wo_s);
-        }
+        cur = build_lora_mm(wo, cur, wo_s);
     }

     if (wo_b) {
@@ -3603,8 +3524,6 @@ ggml_tensor * llm_graph_context::build_attn(
         ggml_tensor * v_mla,
             float     kq_scale,
             int       il) const {
-    const llm_arch arch = model.arch;
-
     // these nodes are added to the graph together so that they are not reordered
     // by doing so, the number of splits in the graph is reduced
     // expand k later to enable rope fusion which directly writes into k-v cache
@@ -3632,16 +3551,7 @@ ggml_tensor * llm_graph_context::build_attn(
     cb(cur, "kqv_out", il);

     if (wo) {
-        if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) {
-            // GLM4 and GLM4_MOE have numerical issues with F16 accumulators
-            cur = build_lora_mm(wo, cur);
-            ggml_prec_set_acc(cur, GGML_PREC_BF16);
-            if (wo_s) {
-                cur = ggml_mul(ctx0, cur, wo_s);
-            }
-        } else {
-            cur = build_lora_mm(wo, cur, wo_s);
-        }
+        cur = build_lora_mm(wo, cur, wo_s);
     }

     if (wo_b) {
diff --git a/src/llama-graph.h b/src/llama-graph.h
index a9f8e5ce0..116b7132a 100644
--- a/src/llama-graph.h
+++ b/src/llama-graph.h
@@ -1179,6 +1179,7 @@ struct llm_graph_context {
               ggml_tensor * cur) const;

     // do mat_mul, while optionally apply lora and per-tensor scale
+    // w_s defaults to the weight scale of w
     ggml_tensor * build_lora_mm(
               ggml_tensor * w,
               ggml_tensor * cur,
@@ -1190,7 +1191,6 @@ struct llm_graph_context {
               ggml_tensor * w,   // ggml_tensor * as
               ggml_tensor * cur, // ggml_tensor * b
               ggml_tensor * ids,
-              ggml_tensor * w_s    = nullptr,
               ggml_tensor * slots  = nullptr,
                      bool   cached = true) const;

@@ -1263,9 +1263,6 @@ struct llm_graph_context {
                      int   il,
              ggml_tensor * probs_in = nullptr,
              ggml_tensor * gate_up_exps = nullptr,
-             ggml_tensor * up_exps_s = nullptr,
-             ggml_tensor * gate_exps_s = nullptr,
-             ggml_tensor * down_exps_s = nullptr,
              ggml_tensor * selected_experts_in = nullptr,
              ggml_tensor * post_exp_norm = nullptr,
                    float   post_exp_norm_eps = 1e-5f) const;
@@ -1291,9 +1288,6 @@ struct llm_graph_context {
              ggml_tensor * probs_in = nullptr,
              ggml_tensor * gate_up_exps = nullptr,
              ggml_tensor * gate_up_exps_b = nullptr,
-             ggml_tensor * up_exps_s = nullptr,
-             ggml_tensor * gate_exps_s = nullptr,
-             ggml_tensor * down_exps_s = nullptr,
              ggml_tensor * selected_experts_in = nullptr,
              ggml_tensor * post_exp_norm = nullptr,
                    float   post_exp_norm_eps = 1e-5f) const;
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 11c456947..06eb7c929 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1410,12 +1410,19 @@ bool llama_prec_policy::apply(ggml_tensor * res) const {
         return false;
     }

-    const auto it = prec_src1.find(res->src[0]);
-    if (it == prec_src1.end()) {
-        return false;
+    bool applied = false;
+
+    const auto it_acc = prec_acc.find(res->src[0]);
+    if (it_acc != prec_acc.end()) {
+        applied |= ggml_prec_set_acc(res, it_acc->second);
     }

-    return ggml_prec_set_src(res, it->second, 1);
+    const auto it_src1 = prec_src1.find(res->src[0]);
+    if (it_src1 != prec_src1.end()) {
+        applied |= ggml_prec_set_src(res, it_src1->second, 1);
+    }
+
+    return applied;
 }

 void llama_prec_policy::load(llama_model_loader & ml, const llama_model & model) {
@@ -2044,6 +2051,42 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
         }
     }

+    // map each weight to its "<name>.scale" tensor: shape {1} for a 2D weight, {n_expert} for a 3D weight
+    {
+        auto ends_with = [](const std::string & str, const std::string & suffix) {
+            return str.size() >= suffix.size() && str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;
+        };
+        std::unordered_map<std::string, ggml_tensor *> weights;
+        for (const auto & [name, cur] : tensors_by_name) {
+            if (ends_with(name, ".weight")) {
+                weights.emplace(name.substr(0, name.size() - 7), cur);
+            }
+        }
+        for (const auto & [name, s] : tensors_by_name) {
+            if (!ends_with(name, ".scale")) {
+                continue;
+            }
+            const auto it = weights.find(name.substr(0, name.size() - 6));
+            if (it == weights.end()) {
+                continue;
+            }
+            ggml_tensor * w = it->second;
+            const bool ok = w->ne[3] == 1 && ggml_nelements(s) == w->ne[2] && s->ne[0] == w->ne[2];
+            if (ok) {
+                w_scales.emplace(w, s);
+            } else {
+                LLAMA_LOG_DEBUG("%s: '%s' is not a weight scale for '%s', skipping\n", __func__, name.c_str(), w->name);
+            }
+        }
+    }
+
+    // the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/60d8d70770c6776ff598c94bb586a859a38244f1/inference/model.py#L565
+    if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS || hparams.expert_router_f32) {
+        for (const auto & layer : layers) {
+            prec_policy.set_acc(layer.ffn_gate_inp, GGML_PREC_F32);
+        }
+    }
+
     // per-tensor activation precision policy
     prec_policy.load(ml, *this);

@@ -2575,6 +2618,11 @@ size_t llama_model::size() const {
     return pimpl->n_bytes;
 }

+ggml_tensor * llama_model::get_w_scale(const ggml_tensor * w) const {
+    const auto it = w_scales.find(w);
+    return it == w_scales.end() ? nullptr : it->second;
+}
+
 size_t llama_model::n_tensors() const {
     return tensors_by_name.size();
 }
diff --git a/src/llama-model.h b/src/llama-model.h
index 6473980a8..eae5c6848 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -737,8 +737,7 @@ struct llama_meta_device_get_split_state_userdata {
 struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const struct ggml_tensor * tensor, void * userdata);

 struct llama_prec_policy {
-    // the key is the weight tensor `res->src[0]`, stores the recommended accumulation type of the op (unused for now)
-    // TODO: migrate ad-hoc ggml_prec_set_acc() calls to this container + update apply() to use it
+    // the key is the weight tensor `res->src[0]`, stores the recommended accumulation type of the op
     std::unordered_map<const ggml_tensor *, ggml_prec> prec_acc;

     // the key is the weight tensor `res->src[0]`, stores the recommended activation precision type
@@ -746,6 +745,12 @@ struct llama_prec_policy {

     bool apply(ggml_tensor * res) const;

+    void set_acc(const ggml_tensor * w, ggml_prec prec) {
+        if (w) {
+            prec_acc[w] = prec;
+        }
+    }
+
     void load(llama_model_loader & ml, const llama_model & model);
 };

@@ -818,6 +823,11 @@ struct llama_model {

     llama_hadamard hdmd;

+    // weight -> "<name>.scale" tensor (e.g. NVFP4 scale2), applied by build_lora_mm and build_lora_mm_id
+    std::unordered_map<const ggml_tensor *, ggml_tensor *> w_scales;
+
+    ggml_tensor * get_w_scale(const ggml_tensor * w) const;
+
     // see llama_model_set_attn_rot
     bool attn_rot = true;

diff --git a/src/models/clover-moe.cpp b/src/models/clover-moe.cpp
index 8892521ee..837997261 100644
--- a/src/models/clover-moe.cpp
+++ b/src/models/clover-moe.cpp
@@ -398,10 +398,7 @@ ggml_tensor * llama_model_clover_moe::graph::build_layer_ffn(ggml_tensor * cur,
             LLM_FFN_SILU, true,
             hparams.expert_weights_scale,
             LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
-            nullptr, model.layers[il].ffn_gate_up_exps,
-            model.layers[il].ffn_up_exps_s,
-            model.layers[il].ffn_gate_exps_s,
-            model.layers[il].ffn_down_exps_s);
+            nullptr, model.layers[il].ffn_gate_up_exps);
     cb(moe_out, "ffn_moe_out", il);

     if (model.layers[il].ffn_up_shexp != nullptr) {
diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cpp
index 52b959672..c4d778e50 100644
--- a/src/models/cohere2moe.cpp
+++ b/src/models/cohere2moe.cpp
@@ -226,10 +226,7 @@ llama_model_cohere2moe::graph::graph(const llm_graph_params & params) : llm_grap
                     hparams.expert_weights_scale,
                     (llama_expert_gating_func_type) hparams.expert_gating_func,
                     il,
-                    nullptr, layer.ffn_gate_up_exps,
-                    layer.ffn_up_exps_s,
-                    layer.ffn_gate_exps_s,
-                    layer.ffn_down_exps_s);
+                    nullptr, layer.ffn_gate_up_exps);
             cb(cur, "ffn_moe_out", il);

             if (layer.ffn_up_shexp) {
@@ -371,10 +368,7 @@ llama_model_cohere2moe::graph_mtp::graph_mtp(const llm_graph_params & params) :
             hparams.expert_weights_scale,
             (llama_expert_gating_func_type) hparams.expert_gating_func,
             il,
-            nullptr, layer.ffn_gate_up_exps,
-            layer.ffn_up_exps_s,
-            layer.ffn_gate_exps_s,
-            layer.ffn_down_exps_s);
+            nullptr, layer.ffn_gate_up_exps);
     cb(cur, "mtp_ffn_moe_out", il);

     if (layer.ffn_up_shexp) {
diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cpp
index 853389b88..8f6f34790 100644
--- a/src/models/deepseek32.cpp
+++ b/src/models/deepseek32.cpp
@@ -383,10 +383,7 @@ llama_model_deepseek32::graph::graph(const llm_graph_params & params) :
                 (llama_expert_gating_func_type) hparams.expert_gating_func,
                 il,
                 nullptr,
-                model.layers[il].ffn_gate_up_exps,
-                model.layers[il].ffn_up_exps_s,
-                model.layers[il].ffn_gate_exps_s,
-                model.layers[il].ffn_down_exps_s);
+                model.layers[il].ffn_gate_up_exps);
             cb(moe_out, "ffn_moe_out", il);

             // FFN shared expert
@@ -624,10 +621,7 @@ llama_model_deepseek32::graph_mtp::graph_mtp(const llm_graph_params & params)
         (llama_expert_gating_func_type) hparams.expert_gating_func,
         il,
         nullptr,
-        layer.ffn_gate_up_exps,
-        layer.ffn_up_exps_s,
-        layer.ffn_gate_exps_s,
-        layer.ffn_down_exps_s);
+        layer.ffn_gate_up_exps);
     cb(moe_out, "mtp_ffn_moe_out", il);

     // FFN shared expert
diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp
index dfc9858e7..f4269e043 100644
--- a/src/models/deepseek4.cpp
+++ b/src/models/deepseek4.cpp
@@ -1284,9 +1284,6 @@ llama_model_deepseek4::graph::graph(const llm_graph_params & params) :
                 il,
                 nullptr,
                 nullptr,
-                nullptr,
-                nullptr,
-                nullptr,
                 selected_experts);
         cb(moe_out, "ffn_moe_out", il);

diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp
index ed93be534..e481cbd1d 100644
--- a/src/models/deepseek41.cpp
+++ b/src/models/deepseek41.cpp
@@ -203,6 +203,8 @@ void llama_model_deepseek41::load_arch_tensors(llama_model_loader & ml) {
         }

         layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,    "weight", i), {n_embd, n_expert}, 0);
+        // the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017/blob/1bae0e1088993b9b82de08f59de5be5a34e42170/inference/model.py#L813
+        prec_policy.set_acc(layer.ffn_gate_inp, GGML_PREC_F32);
         layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias",   i), {n_expert}, 0);
         layer.ffn_exp_probs_b_vl = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B_VL, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
         layer.ffn_norm        = create_tensor(tn(LLM_TENSOR_FFN_NORM,        "weight", i), {n_embd}, 0);
@@ -874,9 +876,7 @@ llama_model_deepseek41::graph::graph(const llm_graph_params & params) :
         cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
         cb(cur, "ffn_norm", il);

-        // the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017/blob/1bae0e1088993b9b82de08f59de5be5a34e42170/inference/model.py#L813
         ggml_tensor * logits = build_lora_mm(layer.ffn_gate_inp, cur);
-        ggml_prec_set_acc(logits, GGML_PREC_F32);
         cb(logits, "ffn_moe_logits", il);

         // image (embedding) tokens use the vision router bias, as DeepSeek-V4
diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cpp
index da16f2480..2a2d5500b 100644
--- a/src/models/dots3note.cpp
+++ b/src/models/dots3note.cpp
@@ -401,10 +401,7 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
                 (llama_expert_gating_func_type) hparams.expert_gating_func,
                 il,
                 nullptr,
-                model.layers[il].ffn_gate_up_exps,
-                model.layers[il].ffn_up_exps_s,
-                model.layers[il].ffn_gate_exps_s,
-                model.layers[il].ffn_down_exps_s);
+                model.layers[il].ffn_gate_up_exps);
             cb(moe_out, "ffn_moe_out", il);

             ggml_tensor * ffn_shexp =
diff --git a/src/models/flint.cpp b/src/models/flint.cpp
index 520334ce6..173278b6f 100644
--- a/src/models/flint.cpp
+++ b/src/models/flint.cpp
@@ -158,10 +158,7 @@ llama_model_flint::graph::graph(const llm_graph_params & params) :
                 LLM_FFN_SILU, hparams.expert_weights_norm,
                 hparams.expert_weights_scale,
                 (llama_expert_gating_func_type) hparams.expert_gating_func,
-                il, nullptr, nullptr,
-                model.layers[il].ffn_up_exps_s,
-                model.layers[il].ffn_gate_exps_s,
-                model.layers[il].ffn_down_exps_s);
+                il);
             cb(moe_out, "ffn_moe_out", il);

             {
diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp
index 1a377fa3c..491ef1283 100644
--- a/src/models/gemma4.cpp
+++ b/src/models/gemma4.cpp
@@ -406,10 +406,7 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :
                     1.0f,
                     LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
                     il, logits,
-                    model.layers[il].ffn_gate_up_exps,
-                    model.layers[il].ffn_up_exps_s,
-                    model.layers[il].ffn_gate_exps_s,
-                    model.layers[il].ffn_down_exps_s);
+                    model.layers[il].ffn_gate_up_exps);
             cur_moe = build_norm(cur_moe,
                     model.layers[il].ffn_post_norm_2, nullptr,
                     LLM_NORM_RMS, il);
diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp
index 6daafd007..c0ff62e4a 100644
--- a/src/models/glm-dsa.cpp
+++ b/src/models/glm-dsa.cpp
@@ -431,10 +431,7 @@ llama_model_glm_dsa::graph::graph(const llm_graph_params & params) :
                 (llama_expert_gating_func_type) hparams.expert_gating_func,
                 il,
                 nullptr,
-                model.layers[il].ffn_gate_up_exps,
-                model.layers[il].ffn_up_exps_s,
-                model.layers[il].ffn_gate_exps_s,
-                model.layers[il].ffn_down_exps_s);
+                model.layers[il].ffn_gate_up_exps);
             cb(moe_out, "ffn_moe_out", il);

             // FFN shared expert
@@ -672,10 +669,7 @@ llama_model_glm_dsa::graph_mtp::graph_mtp(const llm_graph_params & params)
         (llama_expert_gating_func_type) hparams.expert_gating_func,
         il,
         nullptr,
-        layer.ffn_gate_up_exps,
-        layer.ffn_up_exps_s,
-        layer.ffn_gate_exps_s,
-        layer.ffn_down_exps_s);
+        layer.ffn_gate_up_exps);
     cb(moe_out, "mtp_ffn_moe_out", il);

     // FFN shared expert
diff --git a/src/models/glm4-moe.cpp b/src/models/glm4-moe.cpp
index 9560fd3d0..a3e2fa46e 100644
--- a/src/models/glm4-moe.cpp
+++ b/src/models/glm4-moe.cpp
@@ -113,6 +113,11 @@ void llama_model_glm4_moe::load_arch_tensors(llama_model_loader & ml) {
             layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED | flags);
             layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, TENSOR_NOT_REQUIRED | flags);
         }
+
+        // numerical issues with F16 accumulators
+        prec_policy.set_acc(layer.wo,             GGML_PREC_BF16);
+        prec_policy.set_acc(layer.ffn_down,       GGML_PREC_BF16);
+        prec_policy.set_acc(layer.ffn_down_shexp, GGML_PREC_BF16);
     }
 }

diff --git a/src/models/glm4.cpp b/src/models/glm4.cpp
index e57c51cbe..5c50b2f05 100644
--- a/src/models/glm4.cpp
+++ b/src/models/glm4.cpp
@@ -58,6 +58,10 @@ void llama_model_glm4::load_arch_tensors(llama_model_loader &) {
             layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, flags | TENSOR_NOT_REQUIRED);
             layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, flags | TENSOR_NOT_REQUIRED);
         }
+
+        // numerical issues with F16 accumulators
+        prec_policy.set_acc(layer.wo,       GGML_PREC_BF16);
+        prec_policy.set_acc(layer.ffn_down, GGML_PREC_BF16);
     }
 }

diff --git a/src/models/granite-hybrid.cpp b/src/models/granite-hybrid.cpp
index be352dfd7..d638663af 100644
--- a/src/models/granite-hybrid.cpp
+++ b/src/models/granite-hybrid.cpp
@@ -260,11 +260,7 @@ ggml_tensor * llama_model_granite_hybrid::graph::build_layer_ffn(ggml_tensor *
                 LLM_FFN_SILU, true,
                 hparams.expert_weights_scale,
                 LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
-                il,
-                nullptr, nullptr,
-                model.layers[il].ffn_up_exps_s,
-                model.layers[il].ffn_gate_exps_s,
-                model.layers[il].ffn_down_exps_s);
+                il);
         cb(moe_out, "ffn_moe_out", il);

         // For Granite MoE Shared
diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp
index be9e55778..2e2609f52 100644
--- a/src/models/granite-swa.cpp
+++ b/src/models/granite-swa.cpp
@@ -277,10 +277,7 @@ ggml_tensor * llama_model_granite_swa::graph::build_layer_ffn(
                 hparams.expert_weights_scale,
                 LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
                 il,
-                nullptr, model.layers[il].ffn_gate_up_exps,
-                model.layers[il].ffn_up_exps_s,
-                model.layers[il].ffn_gate_exps_s,
-                model.layers[il].ffn_down_exps_s);
+                nullptr, model.layers[il].ffn_gate_up_exps);
         cb(moe_out, "ffn_moe_out", il);

         // For Granite MoE Shared - gate+up kept fused in ffn_up_shexp
diff --git a/src/models/granite.cpp b/src/models/granite.cpp
index 0ab9084da..0a5dd78ea 100644
--- a/src/models/granite.cpp
+++ b/src/models/granite.cpp
@@ -269,11 +269,7 @@ ggml_tensor * llama_model_granite::graph::build_layer_ffn(
                 LLM_FFN_SILU, true,
                 hparams.expert_weights_scale,
                 LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
-                il,
-                nullptr, nullptr,
-                model.layers[il].ffn_up_exps_s,
-                model.layers[il].ffn_gate_exps_s,
-                model.layers[il].ffn_down_exps_s);
+                il);
         cb(moe_out, "ffn_moe_out", il);

         // For Granite MoE Shared
diff --git a/src/models/hy-v3.cpp b/src/models/hy-v3.cpp
index c47c373f8..f53d94827 100644
--- a/src/models/hy-v3.cpp
+++ b/src/models/hy-v3.cpp
@@ -168,10 +168,7 @@ llama_model_hy_v3::graph::graph(const llm_graph_params & params) : llm_graph_con
                     hparams.expert_weights_scale,
                     (llama_expert_gating_func_type) hparams.expert_gating_func,
                     il,
-                    nullptr, model.layers[il].ffn_gate_up_exps,
-                    model.layers[il].ffn_up_exps_s,
-                    model.layers[il].ffn_gate_exps_s,
-                    model.layers[il].ffn_down_exps_s);
+                    nullptr, model.layers[il].ffn_gate_up_exps);
             cb(moe_out, "ffn_moe_out", il);

             // shared expert (always active, no gate)
@@ -323,10 +320,7 @@ llama_model_hy_v3::graph_mtp::graph_mtp(const llm_graph_params & params)
                 hparams.expert_weights_scale,
                 (llama_expert_gating_func_type) hparams.expert_gating_func,
                 il,
-                nullptr, layer.ffn_gate_up_exps,
-                layer.ffn_up_exps_s,
-                layer.ffn_gate_exps_s,
-                layer.ffn_down_exps_s);
+                nullptr, layer.ffn_gate_up_exps);
         cb(moe_out, "mtp_ffn_moe_out", il);

         ggml_tensor * sh_out = build_ffn(cur,
diff --git a/src/models/inkling2.cpp b/src/models/inkling2.cpp
index 3e43ef418..45477a69e 100644
--- a/src/models/inkling2.cpp
+++ b/src/models/inkling2.cpp
@@ -97,6 +97,7 @@ void llama_model_inkling2::load_arch_tensors(llama_model_loader &) {

     // output
     output           = create_tensor(tn(LLM_TENSOR_OUTPUT,           "weight"), {n_embd, n_vocab_padded}, 0);
+    prec_policy.set_acc(output, GGML_PREC_F32);
     output_norm      = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM,      "weight"), {n_embd}, 0);
     output_res_query = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_QUERY, "weight"), {n_embd}, 0);

@@ -406,10 +407,7 @@ ggml_tensor * llama_model_inkling2::graph::build_layer(int il, int ip, ggml_tens
             LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
             il,
             nullptr,
-            layer.ffn_gate_up_exps,
-            layer.ffn_up_exps_s,
-            layer.ffn_gate_exps_s,
-            layer.ffn_down_exps_s);
+            layer.ffn_gate_up_exps);

     cur = convolve(cur, layer.ffn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa + n_embd);
     cb(cur, "ffn_out", il);
@@ -433,7 +431,6 @@ void llama_model_inkling2::graph::build_output(ggml_tensor * cur, const std::vec

     // lm_head
     cur = build_lora_mm(model.output, cur);
-    ggml_prec_set_acc(cur, GGML_PREC_F32);

     if (hparams.f_final_logit_softcapping != 0.0f) {
         const float cap = hparams.f_final_logit_softcapping;
diff --git a/src/models/jais2.cpp b/src/models/jais2.cpp
index 8376423ea..1f077fd86 100644
--- a/src/models/jais2.cpp
+++ b/src/models/jais2.cpp
@@ -42,6 +42,10 @@ void llama_model_jais2::load_arch_tensors(llama_model_loader &) {
         layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "bias", i),   {n_ff}, 0);
         layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
         layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i),   {n_embd}, 0);
+
+        // numerical issues with F16 accumulators
+        prec_policy.set_acc(layer.wo,       GGML_PREC_BF16);
+        prec_policy.set_acc(layer.ffn_down, GGML_PREC_BF16);
     }
 }

diff --git a/src/models/kolibri1.cpp b/src/models/kolibri1.cpp
index c4d8eca1b..64182d186 100644
--- a/src/models/kolibri1.cpp
+++ b/src/models/kolibri1.cpp
@@ -142,7 +142,7 @@ llama_model_kolibri1::graph::graph(const llm_graph_params & params)
                 (llama_expert_gating_func_type) hparams.expert_gating_func,
                 il,
                 logits,
-                nullptr, nullptr, nullptr, nullptr, selected);
+                nullptr, selected);
         cb(moe_out, "ffn_moe_out", il);

         ggml_tensor * shared = build_ffn(cur,
diff --git a/src/models/lfm-base.cpp b/src/models/lfm-base.cpp
index 3627c527d..03e59bd09 100644
--- a/src/models/lfm-base.cpp
+++ b/src/models/lfm-base.cpp
@@ -116,10 +116,5 @@ ggml_tensor * llm_build_lfm_base::build_lfm_moe_ffn(ggml_tensor * cur,
             LLM_FFN_SILU, true,
             hparams.expert_weights_scale,
             static_cast<llama_expert_gating_func_type>(hparams.expert_gating_func),
-            il,
-            nullptr,
-            nullptr,
-            model.layers[il].ffn_up_exps_s,
-            model.layers[il].ffn_gate_exps_s,
-            model.layers[il].ffn_down_exps_s);
+            il);
 }
diff --git a/src/models/llama.cpp b/src/models/llama.cpp
index 26d54968c..3925699f4 100644
--- a/src/models/llama.cpp
+++ b/src/models/llama.cpp
@@ -198,11 +198,7 @@ llama_model_llama::graph<embed>::graph(const llm_graph_params & params) : llm_gr
                     LLM_FFN_SILU, true,
                     hparams.expert_weights_scale,
                     LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
-                    il,
-                    nullptr, nullptr,
-                    model.layers[il].ffn_up_exps_s,
-                    model.layers[il].ffn_gate_exps_s,
-                    model.layers[il].ffn_down_exps_s);
+                    il);
             cb(cur, "ffn_moe_out", il);
         }
         cur = ggml_add(ctx0, cur, ffn_inp);
diff --git a/src/models/mellum.cpp b/src/models/mellum.cpp
index 57dd5b79a..d0b3609b1 100644
--- a/src/models/mellum.cpp
+++ b/src/models/mellum.cpp
@@ -199,11 +199,7 @@ llama_model_mellum::graph<iswa>::graph(const llm_graph_params & params) : llm_gr
                     LLM_FFN_SILU, hparams.expert_weights_norm,
                     hparams.expert_weights_scale,
                     (llama_expert_gating_func_type) hparams.expert_gating_func,
-                    il,
-                    nullptr, nullptr,
-                    model.layers[il].ffn_up_exps_s,
-                    model.layers[il].ffn_gate_exps_s,
-                    model.layers[il].ffn_down_exps_s);
+                    il);
         cb(moe_out, "ffn_moe_out", il);
         cur = moe_out;

@@ -351,11 +347,7 @@ llama_model_mellum::graph_mtp<iswa>::graph_mtp(const llm_graph_params & params)
             LLM_FFN_SILU, hparams.expert_weights_norm,
             hparams.expert_weights_scale,
             (llama_expert_gating_func_type) hparams.expert_gating_func,
-            il,
-            nullptr, nullptr,
-            layer.ffn_up_exps_s,
-            layer.ffn_gate_exps_s,
-            layer.ffn_down_exps_s);
+            il);
     cb(cur, "mtp_ffn_moe_out", il);

     cur = ggml_add(ctx0, cur, ffn_inp);
diff --git a/src/models/metis16.cpp b/src/models/metis16.cpp
index 8a433f3cc..0a57a6ab0 100644
--- a/src/models/metis16.cpp
+++ b/src/models/metis16.cpp
@@ -329,7 +329,7 @@ llama_model_metis16::graph::graph(const llm_graph_params & params) :

                 ggml_tensor * moe_out = build_moe_ffn(latent, nullptr, nullptr, nullptr, layer.ffn_down_exps, nullptr,
                         n_exp, n_used, LLM_FFN_SILU, false, 0.0f, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT, il_p,
-                        logits, layer.ffn_gate_up_exps, nullptr, nullptr, nullptr, selected);
+                        logits, layer.ffn_gate_up_exps, selected);
                 ggml_tensor * shexp = build_ffn(latent,
                         layer.ffn_up_shexp,   NULL, NULL,
                         layer.ffn_gate_shexp, NULL, NULL,
diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp
index 4f1184590..8560c710a 100644
--- a/src/models/minimax-m3.cpp
+++ b/src/models/minimax-m3.cpp
@@ -572,10 +572,7 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap
                     LLM_FFN_SWIGLU_OAI_MOE, hparams.expert_weights_norm,
                     hparams.expert_weights_scale,
                     (llama_expert_gating_func_type) hparams.expert_gating_func,
-                    il, nullptr, nullptr,
-                    model.layers[il].ffn_up_exps_s,
-                    model.layers[il].ffn_gate_exps_s,
-                    model.layers[il].ffn_down_exps_s);
+                    il);
             cb(moe_out, "ffn_moe_out", il);

             // shared expert (swigluoai)
diff --git a/src/models/mistral-large4.cpp b/src/models/mistral-large4.cpp
index 6407d7663..aab49df9d 100644
--- a/src/models/mistral-large4.cpp
+++ b/src/models/mistral-large4.cpp
@@ -420,10 +420,7 @@ llama_model_mistral_large4::graph::graph(const llm_graph_params & params) :
                 (llama_expert_gating_func_type) hparams.expert_gating_func,
                 il,
                 nullptr,
-                layer.ffn_gate_up_exps,
-                layer.ffn_up_exps_s,
-                layer.ffn_gate_exps_s,
-                layer.ffn_down_exps_s);
+                layer.ffn_gate_up_exps);
             cb(moe_out, "ffn_moe_out", il);

             ggml_tensor * ffn_shexp = build_ffn(cur,
diff --git a/src/models/mistral3.cpp b/src/models/mistral3.cpp
index ab8121863..e1e6981bb 100644
--- a/src/models/mistral3.cpp
+++ b/src/models/mistral3.cpp
@@ -248,11 +248,7 @@ llama_model_mistral3::graph::graph(const llm_graph_params & params) : llm_graph_
                     LLM_FFN_SILU, true,
                     hparams.expert_weights_scale,
                     LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
-                    il,
-                    nullptr, nullptr,
-                    model.layers[il].ffn_up_exps_s,
-                    model.layers[il].ffn_gate_exps_s,
-                    model.layers[il].ffn_down_exps_s);
+                    il);
             cb(cur, "ffn_moe_out", il);
         }
         cur = ggml_add(ctx0, cur, ffn_inp);
diff --git a/src/models/muse-spark.cpp b/src/models/muse-spark.cpp
index ea6ab84ab..10de4676e 100644
--- a/src/models/muse-spark.cpp
+++ b/src/models/muse-spark.cpp
@@ -180,7 +180,7 @@ llama_model_muse_spark::graph::graph(const llm_graph_params & params)
                 (llama_expert_gating_func_type) hparams.expert_gating_func,
                 il,
                 router_logits,
-                nullptr, nullptr, nullptr, nullptr, nullptr,
+                nullptr, nullptr,
                 model.layers[il].ffn_post_exp_norm, hparams.f_post_norm_eps);
         cb(moe_out, "ffn_moe_out", il);

diff --git a/src/models/nemotron-h-moe.cpp b/src/models/nemotron-h-moe.cpp
index 475355832..3aa385379 100644
--- a/src/models/nemotron-h-moe.cpp
+++ b/src/models/nemotron-h-moe.cpp
@@ -118,10 +118,7 @@ llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llm_graph_params & params
                 hparams.expert_weights_scale,
                 LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
                 il,
-                router_logits, nullptr,
-                layer.ffn_up_exps_s,
-                nullptr, // no gate
-                layer.ffn_down_exps_s);
+                router_logits);
         cb(moe_out, "mtp_ffn_moe_out", il);

         if (layer.ffn_latent_up) {
diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cpp
index 313f42f1c..18e45320a 100644
--- a/src/models/nemotron-h.cpp
+++ b/src/models/nemotron-h.cpp
@@ -322,10 +322,7 @@ ggml_tensor * llama_model_nemotron_h::graph::build_ffn_layer(ggml_tensor * cur,
                     hparams.expert_weights_scale,
                     LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
                     il,
-                    router_logits, nullptr,
-                    model.layers[il].ffn_up_exps_s,
-                    nullptr, // no gate
-                    model.layers[il].ffn_down_exps_s);
+                    router_logits);
         cb(moe_out, "ffn_moe_out", il);

         if (model.layers[il].ffn_latent_up) {
diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp
index c0ee6fa03..c92b84d9d 100644
--- a/src/models/qwen35moe.cpp
+++ b/src/models/qwen35moe.cpp
@@ -479,10 +479,7 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_ffn(ggml_tensor * cur, c
             LLM_FFN_SILU, true,
             hparams.expert_weights_scale,
             LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
-            nullptr, model.layers[il].ffn_gate_up_exps,
-            model.layers[il].ffn_up_exps_s,
-            model.layers[il].ffn_gate_exps_s,
-            model.layers[il].ffn_down_exps_s);
+            nullptr, model.layers[il].ffn_gate_up_exps);
     cb(moe_out, "ffn_moe_out", il);

     // Add shared experts if present - following Qwen3Next reference implementation
@@ -651,10 +648,7 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llm_graph_params & params)
             LLM_FFN_SILU, true,
             hparams.expert_weights_scale,
             LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
-            nullptr, layer.ffn_gate_up_exps,
-            layer.ffn_up_exps_s,
-            layer.ffn_gate_exps_s,
-            layer.ffn_down_exps_s);
+            nullptr, layer.ffn_gate_up_exps);
     cb(moe_out, "mtp_ffn_moe_out", il);

     if (layer.ffn_up_shexp != nullptr) {
diff --git a/src/models/qwen3moe.cpp b/src/models/qwen3moe.cpp
index ce7144a22..fc57928f5 100644
--- a/src/models/qwen3moe.cpp
+++ b/src/models/qwen3moe.cpp
@@ -136,11 +136,7 @@ llama_model_qwen3moe::graph::graph(const llm_graph_params & params) : llm_graph_
                     LLM_FFN_SILU, true,
                     hparams.expert_weights_scale,
                     LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
-                    il,
-                    nullptr, nullptr,
-                    model.layers[il].ffn_up_exps_s,
-                    model.layers[il].ffn_gate_exps_s,
-                    model.layers[il].ffn_down_exps_s);
+                    il);
         cb(moe_out, "ffn_moe_out", il);
         cur = moe_out;

diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cpp
index 9546fba58..208aa49d2 100644
--- a/src/models/qwen3next.cpp
+++ b/src/models/qwen3next.cpp
@@ -561,10 +561,7 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_ffn(ggml_tensor * cur, c
                 LLM_FFN_SILU, true,
                 hparams.expert_weights_scale,
                 LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
-                nullptr, model.layers[il].ffn_gate_up_exps,
-                model.layers[il].ffn_up_exps_s,
-                model.layers[il].ffn_gate_exps_s,
-                model.layers[il].ffn_down_exps_s);
+                nullptr, model.layers[il].ffn_gate_up_exps);
         cb(moe_out, "ffn_moe_out", il);

         // Add shared experts if present - following Qwen3Next reference implementation
@@ -748,10 +745,7 @@ llama_model_qwen3next::graph_mtp::graph_mtp(const llm_graph_params & params)
             LLM_FFN_SILU, true,
             hparams.expert_weights_scale,
             LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
-            nullptr, layer.ffn_gate_up_exps,
-            layer.ffn_up_exps_s,
-            layer.ffn_gate_exps_s,
-            layer.ffn_down_exps_s);
+            nullptr, layer.ffn_gate_up_exps);
     cb(moe_out, "mtp_ffn_moe_out", il);

     if (layer.ffn_up_shexp != nullptr) {
diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp
index 63c918602..106530490 100644
--- a/src/models/qwen4exp.cpp
+++ b/src/models/qwen4exp.cpp
@@ -1230,10 +1230,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_ffn(ggml_tensor * cur, co
             LLM_FFN_SILU, true,
             hparams.expert_weights_scale,
             LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
-            nullptr, model.layers[il].ffn_gate_up_exps,
-            model.layers[il].ffn_up_exps_s,
-            model.layers[il].ffn_gate_exps_s,
-            model.layers[il].ffn_down_exps_s);
+            nullptr, model.layers[il].ffn_gate_up_exps);
     cb(moe_out, "ffn_moe_out", il);

     // shared experts, as in the Qwen3Next reference
diff --git a/src/models/zaya1.cpp b/src/models/zaya1.cpp
index 2763500f4..013658c73 100644
--- a/src/models/zaya1.cpp
+++ b/src/models/zaya1.cpp
@@ -277,9 +277,6 @@ ggml_tensor * llama_model_zaya1::graph::build_router_moe(const llama_model & mod
             il,
             logits,
             layer.ffn_gate_up_exps,
-            nullptr,
-            nullptr,
-            nullptr,
             expert);

     return ggml_mul(ctx0, moe, keep);
diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv
index 9bd4b82bf..2a4f4b361 100644
--- a/tests/fusion/MTL.csv
+++ b/tests/fusion/MTL.csv
@@ -41,6 +41,7 @@ cohere2             ,0   ,any     ,NORM+MUL                    ,      3
 cohere2moe          ,1   ,any     ,ADD+ADD                     ,      2
 cohere2moe          ,1   ,any     ,MUL+ADD                     ,      2
 cohere2moe          ,1   ,any     ,RMS_NORM+MUL                ,      3
+command-r           ,0   ,any     ,ADD+ADD                     ,      2
 command-r           ,0   ,any     ,NORM+MUL                    ,      3
 dbrx                ,0   ,any     ,MUL+ADD                     ,      4
 dbrx                ,0   ,any     ,NORM+MUL                    ,      5
@@ -207,6 +208,7 @@ llama               ,0   ,any     ,MUL+ADD                     ,      4
 llama               ,0   ,any     ,RMS_NORM+MUL                ,      5
 llama               ,0   ,any     ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV,      2
 llama4              ,0   ,any     ,ADD+ADD                     ,      2
+llama4              ,0   ,any     ,MUL+ADD                     ,      4
 llama4              ,0   ,any     ,RMS_NORM+MUL                ,      9
 maincoder           ,0   ,any     ,RMS_NORM+MUL                ,      9
 maion-coder         ,0   ,any     ,MUL+ADD                     ,      4
@@ -257,8 +259,7 @@ moshi               ,0   ,any     ,RMS_NORM+MUL                ,      5
 mpt                 ,0   ,any     ,NORM+MUL+ADD                ,      5
 muse-glimmer        ,0   ,any     ,RMS_NORM+MUL                ,     10
 muse-glimmer        ,0   ,any     ,RMS_NORM+MUL+ADD            ,      3
-muse-spark          ,0   ,prefill ,MUL+ADD                     ,      2
-muse-spark          ,0   ,decode  ,MUL+ADD                     ,      4
+muse-spark          ,0   ,any     ,MUL+ADD                     ,      4
 muse-spark          ,0   ,any     ,RMS_NORM+MUL                ,     11
 muse-spark          ,0   ,any     ,RMS_NORM+MUL+ADD            ,      3
 muse-spark          ,0   ,any     ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV,      2