Commit 259d91709 for llama.cpp
commit 259d91709d60945911bedb9bcfb2957c6e153ea7
Author: Aman Gupta <amangupta052@gmail.com>
Date: Tue Nov 3 16:51:10 2026 +0800
meta: support GLM-5.3 next sm tensor (#30499)
* meta: support GLM-5.3 next sm tensor
* glm5-next : cont q_absorbed for -sm tensor with non-unified KV
* meta : keep GGML_OP_ARANGE in ggml.h order
* glm5-next : split MLA heads in pairs for -sm tensor
The MLA cache has a single KV head, so the heads on a device are its GQA
ratio. The HIP tile FA kernel for D=512 needs an even GQA ratio; with 3
devices the 16 heads of the test model were split 5/5/6 and it aborted.
* meta : keep the split for views inside one slab of the split axis
With a non-unified KV cache, build_attn_mha splits q into streams with a
view. For a permuted q, e.g. the absorbed query of glm5-next, that view
aborted with "view of permuted tensor not implemented". The split is now
kept if every other dim of the source and the view stays inside one slab
of the split axis. The per-device strides are only rescaled above it.
From tetherto/qvac-fabric-llm.cpp#353.
Co-authored-by: Pratik Narola <pratik.narola@tether.io>
* glm5-next : drop the q_absorbed cont
The meta backend can now split the per-stream view of the permuted q, so
the cont is not needed for -sm tensor with a non-unified KV cache. With
the cont, the q of flash attention was not contiguous, and when FA ran on
the CPU (WebGPU) the meta get_tensor aborted.
---------
Co-authored-by: Pratik Narola <pratik.narola@tether.io>
diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
index c30bdae47..aba004646 100644
--- a/ggml/src/ggml-backend-meta.cpp
+++ b/ggml/src/ggml-backend-meta.cpp
@@ -489,6 +489,32 @@ static struct ggml_tensor * ggml_backend_meta_buffer_simple_tensor(const struct
return it->second[index];
}
+// bytes spanned by all dims except skip_dim
+static size_t ggml_backend_meta_inner_extent(const struct ggml_tensor * tensor, int skip_dim) {
+ size_t extent = 0;
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
+ if (i != skip_dim && tensor->ne[i] > 1) {
+ extent += (size_t) (tensor->ne[i] - 1) * tensor->nb[i];
+ }
+ }
+ return extent;
+}
+
+// the view dim that holds the split axis of src, or -1
+// all other dims of src and view must stay inside one slab of that axis
+static int ggml_backend_meta_view_slab_dim(const struct ggml_tensor * view, const struct ggml_tensor * src, int axis) {
+ const size_t slab = src->nb[axis];
+ if (ggml_backend_meta_inner_extent(src, axis) >= slab) {
+ return -1;
+ }
+ for (int dim = 0; dim < GGML_MAX_DIMS; dim++) {
+ if (view->ne[dim] == src->ne[axis] && view->nb[dim] == slab) {
+ return ggml_backend_meta_inner_extent(view, dim) < slab ? dim : -1;
+ }
+ }
+ return -1;
+}
+
static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync);
static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
@@ -723,6 +749,16 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED || src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL) {
return src_ss[0];
}
+ // a view that only regroups dims inside one slab of the split axis keeps the split,
+ // the per-device strides are only rescaled above nb[axis] (e.g. the KV streams of a permuted q)
+ if (axis >= 0 && axis < GGML_MAX_DIMS && tensor->view_offs == tensor->src[0]->view_offs) {
+ const int dim = ggml_backend_meta_view_slab_dim(tensor, tensor->src[0], axis);
+ if (dim >= 0) {
+ ggml_backend_meta_split_state ret = src_ss[0];
+ ret.axis = ggml_backend_meta_split_axis(dim);
+ return ret;
+ }
+ }
GGML_ABORT("view of permuted tensor not implemented");
//return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1};
};
@@ -1020,8 +1056,13 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
split_state = handle_pad(src_ss);
} break;
case GGML_OP_PAD_REFLECT_1D:
- case GGML_OP_ROLL:
- case GGML_OP_ARANGE:
+ case GGML_OP_ROLL: {
+ split_state = handle_generic(src_ss, /*scalar_only =*/ true);
+ } break;
+ case GGML_OP_ARANGE: {
+ // no sources, every device computes the full range
+ split_state = {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1};
+ } break;
case GGML_OP_TIMESTEP_EMBEDDING: {
split_state = handle_generic(src_ss, /*scalar_only =*/ true);
} break;
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 40f608371..525f91031 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -1390,7 +1390,6 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
case LLM_ARCH_KIMI_LINEAR:
case LLM_ARCH_BAILINGMOE3:
case LLM_ARCH_KIMI_K3:
- case LLM_ARCH_GLM5_NEXT:
case LLM_ARCH_QWEN3TTS:
case LLM_ARCH_LIMITE:
return false;
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 7529f55c2..87893bf67 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -410,6 +410,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
const std::string tensor_name = tensor->name;
const bool is_dsv4 = ud->model->arch == LLM_ARCH_DEEPSEEK4 || ud->model->arch == LLM_ARCH_DEEPSEEK41 ||
(ud->model->arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0);
+ const bool is_glm5_next = ud->model->arch == LLM_ARCH_GLM5_NEXT;
static const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight");
static const std::regex pattern_kv_weight ("blk\\.\\d*\\.attn_(k|v).weight");
@@ -429,6 +430,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
static const std::regex pattern_attn_out_a_weight("blk\\.\\d*\\.attn_output_a\\.weight");
static const std::regex pattern_attn_out_b_weight("blk\\.\\d*\\.attn_output_b\\.weight");
static const std::regex pattern_attn_q_b_weight ("blk\\.\\d*\\.attn_q_b\\.weight");
+ static const std::regex pattern_attn_kv_b_weight("blk\\.\\d*\\.attn_(k|v)_b\\.weight");
static const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight");
static const std::regex pattern_ssm_dt ("blk\\.\\d*\\.ssm_dt.bias");
@@ -441,6 +443,8 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
static const std::regex pattern_s_cache ("cache_s_l\\d*");
static const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight");
static const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight");
+ static const std::regex pattern_kda_conv1d ("blk\\.\\d*\\.ssm_conv1d_(q|k|v)\\.weight");
+ static const std::regex pattern_kda_gate_b ("blk\\.\\d*\\.ssm_(f|g)_b\\.weight");
static const std::regex pattern_ffn_up_weight ("blk\\.\\d*\\.ffn_up(_exps)?.weight");
static const std::regex pattern_ffn_up_bias ("blk\\.\\d*\\.ffn_up(_exps)?.bias");
@@ -532,6 +536,25 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
if (std::regex_match(tensor_name, pattern_attn_out_b_weight)) {
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);
}
+ }
+ if (is_glm5_next) {
+ // split the KDA and MLA heads, the MLA latent cache and the indexer stay mirrored
+ if (std::regex_match(tensor_name, pattern_kv_cache)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
+ }
+ if (std::regex_match(tensor_name, pattern_attn_q_b_weight) || std::regex_match(tensor_name, pattern_kda_gate_b) ||
+ std::regex_match(tensor_name, pattern_ssm_beta)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight");
+ }
+ if (std::regex_match(tensor_name, pattern_attn_kv_b_weight) || std::regex_match(tensor_name, pattern_kda_conv1d)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_2, "attn_output.weight");
+ }
+ if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||
+ std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight");
+ }
+ }
+ if (is_dsv4 || is_glm5_next) {
if (std::regex_match(tensor_name, pattern_ffn_up_shexp_weight) ||
std::regex_match(tensor_name, pattern_ffn_gate_shexp_weight)) {
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down_shexp.weight");
@@ -645,6 +668,12 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
// if a model has fused tensors they need to be separated into "segments", see the comments on ggml_backend_meta_split_state struct
auto get_split_segments = [&](int axis, uint32_t il) -> std::vector<std::pair<int64_t, uint32_t>> {
+ if (is_glm5_next && std::regex_match(tensor_name, pattern_r_cache)) {
+ // the conv state holds the q, k and v windows one after the other
+ const int64_t conv_state_size = (hparams.ssm_d_conv - 1) * hparams.n_head(il) * hparams.n_embd_head_kda;
+ GGML_ASSERT(tensor->ne[axis] == 3*conv_state_size);
+ return {{conv_state_size, 3}};
+ }
// TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF || ud->model->arch == LLM_ARCH_CLOVER_MOE ||
ud->model->arch == LLM_ARCH_QWEN4EXP) {
@@ -736,7 +765,39 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
auto get_split_granularity = [&](int64_t blck_size, uint32_t il, const std::vector<std::pair<int64_t, uint32_t>> & segments) -> std::vector<int64_t> {
// for better performance it may make sense to round up blck_size to a higher power of 2 so that more efficient kernels can be used
- if (hparams.is_recr(il)) {
+ if (is_glm5_next && hparams.is_recr(il)) {
+ // KDA, each device holds whole heads
+ const int64_t head_dim = hparams.n_embd_head_kda;
+ const int64_t blck_size_perf = std::lcm(blck_size, 128);
+ const int64_t granularity_qkv = std::lcm(blck_size_perf, head_dim);
+ if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_kv_weight) ||
+ std::regex_match(tensor_name, pattern_kda_conv1d) || std::regex_match(tensor_name, pattern_kda_gate_b) ||
+ std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_attn_out_weight)) {
+ return {granularity_qkv};
+ }
+ if (std::regex_match(tensor_name, pattern_ssm_a) || std::regex_match(tensor_name, pattern_ssm_beta)) {
+ return {granularity_qkv / head_dim};
+ }
+ if (std::regex_match(tensor_name, pattern_r_cache)) {
+ return std::vector<int64_t>(segments.size(), granularity_qkv * (hparams.ssm_d_conv - 1));
+ }
+ if (std::regex_match(tensor_name, pattern_s_cache)) {
+ return {granularity_qkv * head_dim};
+ }
+ } else if (is_glm5_next) {
+ // MLA, each device holds whole heads
+ // an even number of heads per device, the HIP tile FA kernel for D=512 needs an even GQA ratio
+ const int64_t granularity_head = std::lcm(std::lcm(hparams.n_embd_head_v_mla(), blck_size) / hparams.n_embd_head_v_mla(), int64_t(2));
+ if (std::regex_match(tensor_name, pattern_attn_q_b_weight)) {
+ return {granularity_head * hparams.n_embd_head_k_mla()};
+ }
+ if (std::regex_match(tensor_name, pattern_attn_kv_b_weight)) {
+ return {granularity_head};
+ }
+ if (std::regex_match(tensor_name, pattern_attn_out_weight)) {
+ return {granularity_head * hparams.n_embd_head_v_mla()};
+ }
+ } else if (hparams.is_recr(il)) {
// linear attention
const int64_t head_dim = hparams.ssm_d_state;
const int64_t blck_size_perf = std::lcm(blck_size, 128);
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 0e61ebab5..32569dc86 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -279,7 +279,6 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|| arch == LLM_ARCH_KIMI_LINEAR
|| arch == LLM_ARCH_BAILINGMOE3
|| arch == LLM_ARCH_KIMI_K3
- || arch == LLM_ARCH_GLM5_NEXT
|| arch == LLM_ARCH_MISTRAL4
|| arch == LLM_ARCH_MISTRAL_LARGE4
|| arch == LLM_ARCH_HY_V4) {
@@ -289,6 +288,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
if (arch == LLM_ARCH_MISTRAL_LARGE4) {
n_layer = 4; // cover NoPE, shared indexer and KV sharing layers
}
+ } else if (arch == LLM_ARCH_GLM5_NEXT) {
+ // several heads so that -sm tensor can split them and the MLA flash attention has GQA > 1
+ n_embd = 128;
+ n_head = 16;
+ n_ff = 192;
} else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {
n_layer = 3;
} else if (arch == LLM_ARCH_CHAMELEON) {
@@ -357,6 +361,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
// GLM5 next KDA heads come from the uniform head count, only head_count_kv is per layer.
if (arch == LLM_ARCH_GLM5_NEXT) {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);
+ // the MLA cache holds one latent head
+ for (uint32_t & n : n_head_per_layer) {
+ n = std::min(n, uint32_t(1));
+ }
} else {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head_per_layer);
}