Commit bba840882 for llama.cpp
commit bba8408821ae88a2c4370173f60e8b24e5cfd6ae
Author: Gaurav Garg <gaugarg@nvidia.com>
Date: Wed Nov 18 17:04:40 2026 +0530
Add LLAMA_MOE_RANDOM_ROUTING=1 for benchmark-only random expert selection (#29539)
* Add LLAMA_MOE_RANDOM_ROUTING=1 for benchmark-only random expert selection
* Handle expert caching
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 969989ab5..1d8f66d0e 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -405,6 +405,15 @@ llama_context::llama_context(
}
}
+ {
+ const char * LLAMA_MOE_RANDOM_ROUTING = getenv("LLAMA_MOE_RANDOM_ROUTING");
+ cparams.moe_random_routing = LLAMA_MOE_RANDOM_ROUTING ? (atoi(LLAMA_MOE_RANDOM_ROUTING) != 0) : false;
+
+ if (cparams.moe_random_routing && hparams.n_expert > 0) {
+ LLAMA_LOG_WARN("%s: MoE random routing enabled - the output is not valid, use for benchmarks only\n", __func__);
+ }
+ }
+
// ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732
cparams.n_ctx = GGML_PAD(cparams.n_ctx, 256);
diff --git a/src/llama-cparams.h b/src/llama-cparams.h
index 52c85bdd2..1f890d6fa 100644
--- a/src/llama-cparams.h
+++ b/src/llama-cparams.h
@@ -54,6 +54,7 @@ struct llama_cparams {
bool kv_unified;
bool pipeline_parallel;
bool training; // set by llama_opt_init()
+ bool moe_random_routing; // benchmark only: random expert ids in the MoE GEMMs
size_t moe_cache_size;
float moe_cache_cpu_share;
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index 01354c36c..ab604034e 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -23,6 +23,7 @@
#include <array>
#include <cstring>
#include <numeric>
+#include <random>
#include <sstream>
#include <string>
#include <unordered_set>
@@ -398,6 +399,37 @@ bool llm_graph_input_out_ids::can_reuse(const llm_graph_params & params) {
return res;
}
+void llm_graph_input_moe_ids::set_input(const llama_ubatch * ubatch) {
+ GGML_ASSERT(ggml_backend_buffer_is_host(ids->buffer));
+
+ const int64_t n_used = ids->ne[0];
+ const int64_t n_tokens = ids->ne[1];
+
+ // seed from the position: new ids for each step, also when the graph is rebuilt
+ std::mt19937 rng(ubatch->pos ? (uint32_t) ubatch->pos[0] : 0);
+
+ std::vector<int32_t> perm(n_expert);
+ std::iota(perm.begin(), perm.end(), 0);
+
+ int32_t * data = (int32_t *) ids->data;
+
+ // partial Fisher-Yates: n_used distinct experts per token
+ for (int64_t i = 0; i < n_tokens; ++i) {
+ for (int64_t j = 0; j < n_used; ++j) {
+ std::swap(perm[j], perm[j + rng() % (n_expert - j)]);
+ data[i*n_used + j] = perm[j];
+ }
+ }
+}
+
+bool llm_graph_input_moe_ids::can_reuse(const llm_graph_params & params) {
+ bool res = true;
+
+ res &= ids->ne[1] == params.ubatch.n_tokens;
+
+ return res;
+}
+
void llm_graph_input_mean::set_input(const llama_ubatch * ubatch) {
if (cparams.embeddings &&
(cparams.pooling_type == LLAMA_POOLING_TYPE_MEAN ||
@@ -1433,6 +1465,7 @@ void llm_graph_result::reset() {
t_sampled_logits.clear();
t_candidates.clear();
t_sampling_all.clear();
+ t_moe_ids.clear();
params = {};
@@ -2424,6 +2457,20 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
//call early so that topk-moe can be used
ggml_build_forward_expand(gf, weights);
+ // benchmark only: the expert GEMMs use random ids, the router above is not changed
+ if (cparams.moe_random_routing && n_expert == hparams.n_expert && n_expert_used <= hparams.n_expert_used_max() && n_tokens <= ubatch.n_tokens) {
+ // fused top-k kernels write the router ids together with the weights, so keep the ids alive until here
+ // the empty get_rows is skipped by the backends, it only adds this dependency
+ ggml_build_forward_expand(gf, ggml_get_rows(ctx0, weights, ggml_view_2d(ctx0, selected_experts, 0, n_tokens, selected_experts->nb[1], 0)));
+
+ selected_experts = build_inp_moe_ids(n_expert_used, n_tokens);
+
+ // host experts: compute the ids on the device, as the router ids are, because the MoE cache and the expert copy read them there
+ if (down_exps->buffer && ggml_backend_buffer_is_host(down_exps->buffer)) {
+ selected_experts = ggml_cont(ctx0, selected_experts);
+ }
+ }
+
// compute the expert input before the slot lookup, so that the scheduler prefetches it for the CPU part (the router can read a different tensor)
ggml_build_forward_expand(gf, cur);
@@ -2934,6 +2981,35 @@ ggml_tensor * llm_graph_context::build_inp_cls() const {
return cur;
}
+ggml_tensor * llm_graph_context::build_inp_moe_ids(int64_t n_used, int64_t n_rows) const {
+ auto & ts = res->t_moe_ids;
+
+ if (ts.empty()) {
+ auto inp = std::make_unique<llm_graph_input_moe_ids>(hparams.n_expert);
+
+ auto & cur = inp->ids;
+
+ cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, hparams.n_expert_used_max(), n_tokens);
+ ggml_set_input(cur);
+ cb(cur, "inp_moe_ids", -1);
+
+ ts.push_back(cur);
+
+ res->add_input(std::move(inp));
+ }
+
+ // one tensor per shape: each distinct tensor is copied to the device on each ubatch
+ for (auto * t : ts) {
+ if (t->ne[0] == n_used && t->ne[1] == n_rows) {
+ return t;
+ }
+ }
+
+ ts.push_back(ggml_view_2d(ctx0, ts[0], n_used, n_rows, ts[0]->nb[1], 0));
+
+ return ts.back();
+}
+
ggml_tensor * llm_graph_context::build_inp_cross_embd() const {
auto inp = std::make_unique<llm_graph_input_cross_embd>(cross);
diff --git a/src/llama-graph.h b/src/llama-graph.h
index 2d8c65718..862e64e05 100644
--- a/src/llama-graph.h
+++ b/src/llama-graph.h
@@ -268,6 +268,21 @@ public:
const uint32_t n_outputs;
};
+// random expert ids for the MoE GEMMs, for benchmarks only (env: LLAMA_MOE_RANDOM_ROUTING)
+class llm_graph_input_moe_ids : public llm_graph_input_i {
+public:
+ llm_graph_input_moe_ids(int64_t n_expert) : n_expert(n_expert) {}
+ virtual ~llm_graph_input_moe_ids() = default;
+
+ void set_input(const llama_ubatch * ubatch) override;
+
+ bool can_reuse(const llm_graph_params & params) override;
+
+ ggml_tensor * ids = nullptr; // I32 [n_expert_used_max, n_tokens]
+
+ const int64_t n_expert;
+};
+
class llm_graph_input_mean : public llm_graph_input_i {
public:
llm_graph_input_mean(const llama_cparams & cparams) : cparams(cparams) {}
@@ -986,6 +1001,9 @@ public:
std::vector<ggml_tensor *> t_sampling_all;
+ // [0] is the LLAMA_MOE_RANDOM_ROUTING input, the rest are views of it
+ std::vector<ggml_tensor *> t_moe_ids;
+
std::vector<llm_graph_input_ptr> inputs;
std::vector<llm_graph_fused_node> fused_nodes;
@@ -1266,6 +1284,7 @@ struct llm_graph_context {
ggml_tensor * build_inp_out_ids() const;
ggml_tensor * build_inp_mean() const;
ggml_tensor * build_inp_cls() const;
+ ggml_tensor * build_inp_moe_ids(int64_t n_used, int64_t n_rows) const;
ggml_tensor * build_inp_cross_embd() const;
ggml_tensor * build_inp_pos_bucket_enc() const;