Commit 3ba1ce0e2 for llama.cpp
commit 3ba1ce0e28d991800e1585b7d3533f77213939d6
Author: Oliver Simons <osimons@nvidia.com>
Date: Thu Nov 12 15:54:58 2026 +0100
qwen3.5: merge tensors on convert also (#32102)
* qwen3.5: merge tensors on convert also
* Remove conversion test for qwen3.5
diff --git a/conversion/qwen.py b/conversion/qwen.py
index 83c844ce6..376535460 100644
--- a/conversion/qwen.py
+++ b/conversion/qwen.py
@@ -493,6 +493,45 @@ class _LinearAttentionVReorderBase(Qwen3NextModel):
interleaved repeat: [G0_v0, G1_v0, ..., G0_v1, G1_v1, ...].
"""
+ _SHARED_INPUT_MERGES: tuple[tuple[gguf.MODEL_TENSOR, tuple[gguf.MODEL_TENSOR, gguf.MODEL_TENSOR], bool], ...] = ()
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._merge_parts: dict[tuple[int, gguf.MODEL_TENSOR], dict[int, Tensor]] = {}
+
+ # caches one half of a merged weight until the other half arrives, then yields [a | b]
+ def _merge_shared_input(self, new_name: str, data: Tensor, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # a pre-quantized checkpoint carries per-tensor scales, which one merged weight cannot hold
+ if not self._SHARED_INPUT_MERGES or bid is None or self.hparams.get("quantization_config") or self._fp8_e4m3_preserved or self.hadamard_folded_names() or hasattr(data, "get_lora_A_B"):
+ yield (new_name, data)
+ return
+
+ layer_types = self.hparams.get("layer_types")
+ if layer_types is None:
+ is_linear = bid < self.hparams["num_hidden_layers"] and (bid + 1) % self.hparams.get("full_attention_interval", 4) != 0
+ else:
+ is_linear = bid < len(layer_types) and layer_types[bid] == "linear_attention"
+
+ match = next(((merged, parts.index(part)) for merged, parts, linear_only in self._SHARED_INPUT_MERGES
+ if merged in gguf.MODEL_TENSORS[self.model_arch]
+ for part in parts
+ if (is_linear or not linear_only) and self.match_model_tensor_name(new_name, part, bid)), None)
+ if match is None:
+ yield (new_name, data)
+ return
+
+ merged, idx = match
+ halves = self._merge_parts.setdefault((bid, merged), {})
+ halves[idx] = data
+ if len(halves) == 2:
+ del self._merge_parts[(bid, merged)]
+ yield (self.format_tensor_name(merged, bid), torch.cat([halves[0], halves[1]], dim=0))
+
+ def prepare_tensors(self):
+ super().prepare_tensors()
+ if self._merge_parts:
+ raise ValueError(f"merged weights missing a half: {sorted((bid, t.name) for bid, t in self._merge_parts)}")
+
@staticmethod
def _reorder_v_heads(tensor: Tensor, dim: int, num_k_heads: int, num_v_per_k: int, head_dim: int) -> Tensor:
"""Reorder V heads from grouped (by K head) to tiled order along the given dimension."""
@@ -672,7 +711,8 @@ class _LinearAttentionVReorderBase(Qwen3NextModel):
# Out projection weight: reorder columns (input dimension)
data_torch = self._reorder_v_heads(data_torch, 1, num_k_heads, num_v_per_k, head_v_dim)
- yield from super().modify_tensors(data_torch, name, bid)
+ for new_name, data in super().modify_tensors(data_torch, name, bid):
+ yield from self._merge_shared_input(new_name, data, bid)
class _Qwen35MRopeMixin:
@@ -696,6 +736,11 @@ class _Qwen35MRopeMixin:
class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
model_arch = gguf.MODEL_ARCH.QWEN35
+ _SHARED_INPUT_MERGES = (
+ (gguf.MODEL_TENSOR.ATTN_QKV_GATE, (gguf.MODEL_TENSOR.ATTN_QKV, gguf.MODEL_TENSOR.ATTN_GATE), True),
+ (gguf.MODEL_TENSOR.SSM_BETA_ALPHA_CAT, (gguf.MODEL_TENSOR.SSM_BETA, gguf.MODEL_TENSOR.SSM_ALPHA), True),
+ )
+
def __init__(self, dir_model, *args, **kwargs):
# Inner TextModel does not own mtp.*. Set no_mtp before mixin bumps block_count.
hparams = kwargs.pop("hparams", None)
diff --git a/conversion/qwen4exp.py b/conversion/qwen4exp.py
index 8a55d678d..9680d2937 100644
--- a/conversion/qwen4exp.py
+++ b/conversion/qwen4exp.py
@@ -41,8 +41,6 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
self._ple_shards: dict[int, str] = {}
self._ple_row_dim: int | None = None
self._mtp_fc: dict[str, Tensor] = {}
- # halves of the merged weights, keyed by (bid, merged tensor), see _merge_shared_input
- self._merge_parts: dict[tuple[int, gguf.MODEL_TENSOR], dict[int, Tensor]] = {}
@classmethod
def filter_tensors(cls, item):
@@ -171,8 +169,7 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
yield (self.map_tensor_name(name), data_torch.squeeze())
return
- for new_name, data in super().modify_tensors(data_torch, name, bid):
- yield from self._merge_shared_input(new_name, data, bid)
+ yield from super().modify_tensors(data_torch, name, bid)
# matmuls that read the same input, written as one weight [a | b] so the runtime does one matmul:
# (merged tensor, (a, b), linear-attention layers only)
@@ -183,30 +180,6 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
(gguf.MODEL_TENSOR.SSM_BETA_ALPHA_CAT, (gguf.MODEL_TENSOR.SSM_BETA, gguf.MODEL_TENSOR.SSM_ALPHA), True),
)
- # caches one half of a merged weight until the other half arrives, then yields [a | b]
- def _merge_shared_input(self, new_name: str, data: Tensor, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- # a pre-quantized checkpoint carries per-tensor scales, which one merged weight cannot hold
- if bid is None or self.hparams.get("quantization_config"):
- yield (new_name, data)
- return
-
- layer_types = self.hparams.get("layer_types") or []
- is_linear = bid < len(layer_types) and layer_types[bid] == "linear_attention"
-
- match = next(((merged, parts.index(part)) for merged, parts, linear_only in self._SHARED_INPUT_MERGES
- for part in parts
- if (is_linear or not linear_only) and self.match_model_tensor_name(new_name, part, bid)), None)
- if match is None:
- yield (new_name, data)
- return
-
- merged, idx = match
- halves = self._merge_parts.setdefault((bid, merged), {})
- halves[idx] = data
- if len(halves) == 2:
- del self._merge_parts[(bid, merged)]
- yield (self.format_tensor_name(merged, bid), torch.cat([halves[0], halves[1]], dim=0))
-
# the shards concatenate into a tensor of well over 100 GB
# use LazyChunkedTensor here, a single shard resident at a time
def _place_ple_shard(self, data_torch: Tensor, name: str) -> Iterable[tuple[str, Tensor]]:
@@ -251,8 +224,6 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
super().prepare_tensors()
if self._mtp_fc:
raise ValueError(f"MTP projection missing its other half: {sorted(self._mtp_fc)}")
- if self._merge_parts:
- raise ValueError(f"merged weights missing a half: {sorted((bid, t.name) for bid, t in self._merge_parts)}")
n_parts = self.hparams.get("split_ngram_parts", 0)
if self._ple_shards and len(self._ple_shards) != n_parts:
raise ValueError(
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 30185f02c..ad0bd8e67 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -998,8 +998,8 @@ class MODEL_TENSOR(IntEnum):
HC_FFN_INJECT = auto() # qwen4exp
HC_ATTN_DOWN_INJECT = auto() # qwen4exp, [hc_attn_down | hc_attn_inject]
HC_FFN_DOWN_INJECT = auto() # qwen4exp, [hc_ffn_down | hc_ffn_inject]
- ATTN_QKV_GATE = auto() # qwen4exp, [attn_qkv | attn_gate]
- SSM_BETA_ALPHA_CAT = auto() # qwen4exp, [ssm_beta | ssm_alpha], unlike the grouped qwen3next ssm_ba
+ ATTN_QKV_GATE = auto() # qwen35, qwen4exp, [attn_qkv | attn_gate]
+ SSM_BETA_ALPHA_CAT = auto() # qwen35, qwen4exp, [ssm_beta | ssm_alpha], unlike the grouped qwen3next ssm_ba
PLE_KEY = auto() # qwen4exp
PLE_VALUE = auto() # qwen4exp
PLE_NORM_KEY = auto() # qwen4exp
@@ -3409,6 +3409,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.ATTN_POST_NORM,
MODEL_TENSOR.ATTN_GATE,
MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_QKV_GATE,
MODEL_TENSOR.FFN_GATE,
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
@@ -3418,6 +3419,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.SSM_NORM,
MODEL_TENSOR.SSM_BETA,
MODEL_TENSOR.SSM_ALPHA,
+ MODEL_TENSOR.SSM_BETA_ALPHA_CAT,
MODEL_TENSOR.SSM_OUT,
# NextN/MTP tensors - preserved but unused
MODEL_TENSOR.NEXTN_EH_PROJ,
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index cd49ccdd0..edcf69124 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -424,6 +424,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
static const std::regex pattern_kv_weight ("blk\\.\\d*\\.attn_(k|v).weight");
static const std::regex pattern_v_exps_weight ("blk\\.\\d*\\.attn_v_exps.weight"); // K2 Horizon MoVA
static const std::regex pattern_qkv_weight ("blk\\.\\d*\\.attn_qkv.weight");
+ static const std::regex pattern_qkv_gate_weight ("blk\\.\\d*\\.attn_qkv_gate.weight");
static const std::regex pattern_q_bias ("blk\\.\\d*\\.attn_q\\.bias");
static const std::regex pattern_kv_bias ("blk\\.\\d*\\.attn_(k|v)\\.bias");
static const std::regex pattern_qkv_bias ("blk\\.\\d*\\.attn_qkv.bias");
@@ -446,6 +447,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
static const std::regex pattern_ssm_alpha ("blk\\.\\d*\\.ssm_alpha.weight");
static const std::regex pattern_ssm_beta ("blk\\.\\d*\\.ssm_beta.weight");
static const std::regex pattern_ssm_beta_alpha ("blk\\.\\d*\\.ssm_ba.weight");
+ static const std::regex pattern_ssm_beta_alpha_cat("blk\\.\\d*\\.ssm_beta_alpha.weight");
static const std::regex pattern_r_cache ("cache_r_l\\d*");
static const std::regex pattern_ple_r_cache ("cache_ple_r_l\\d*");
static const std::regex pattern_s_cache ("cache_s_l\\d*");
@@ -593,7 +595,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
if (std::regex_match(tensor_name, pattern_q_bias) || std::regex_match(tensor_name, pattern_kv_bias)) {
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight", "ssm_out.weight");
}
- if (std::regex_match(tensor_name, pattern_qkv_weight)) {
+ if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_gate_weight)) {
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");
}
if ( std::regex_match(tensor_name, pattern_qkv_bias)) {
@@ -620,7 +622,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");
}
if (std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta) ||
- std::regex_match(tensor_name, pattern_ssm_beta_alpha)) {
+ std::regex_match(tensor_name, pattern_ssm_beta_alpha) || std::regex_match(tensor_name, pattern_ssm_beta_alpha_cat)) {
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");
}
if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) {
@@ -715,6 +717,14 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
}
} else {
const int64_t head_ratio = n_v_heads / n_k_heads;
+ if (std::regex_match(tensor_name, pattern_qkv_gate_weight)) {
+ GGML_ASSERT(tensor->ne[axis] == 2*key_dim + 2*value_dim);
+ return {{key_dim, 2 + 2*head_ratio}};
+ }
+ if (std::regex_match(tensor_name, pattern_ssm_beta_alpha_cat)) {
+ GGML_ASSERT(tensor->ne[axis] == 2*n_v_heads);
+ return {{n_k_heads, 2*head_ratio}};
+ }
if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_ssm_conv1d)) {
GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim);
return {{key_dim, 2 + head_ratio}};
@@ -811,11 +821,13 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
const int64_t blck_size_perf = std::lcm(blck_size, 128);
const int64_t granularity_qkv = std::lcm(blck_size_perf, head_dim);
if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_attn_gate_weight) ||
- std::regex_match(tensor_name, pattern_ssm_conv1d) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {
+ std::regex_match(tensor_name, pattern_ssm_conv1d) || std::regex_match(tensor_name, pattern_ssm_out_weight) ||
+ std::regex_match(tensor_name, pattern_qkv_gate_weight)) {
return std::vector<int64_t>(segments.size(), granularity_qkv);
}
if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||
- std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta)) {
+ std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta) ||
+ std::regex_match(tensor_name, pattern_ssm_beta_alpha_cat)) {
return std::vector<int64_t>(segments.size(), granularity_qkv / head_dim);
}
if (std::regex_match(tensor_name, pattern_ssm_beta_alpha)) {
diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp
index 6ec3cff44..5b05fc63f 100644
--- a/src/models/qwen35.cpp
+++ b/src/models/qwen35.cpp
@@ -78,13 +78,20 @@ void llama_model_qwen35::load_arch_tensors(llama_model_loader & ml) {
} else {
// Linear attention (gated delta net) specific tensors
// Create tensors with calculated dimensions
- layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", il), { n_embd, key_dim * 2 + value_dim }, TENSOR_NOT_REQUIRED);
- layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", il), { n_embd, value_dim }, TENSOR_NOT_REQUIRED);
+ const int separate_flags = flags & TENSOR_SKIP ? flags | TENSOR_NOT_REQUIRED : flags;
+ layer.wqkv_z = create_tensor(tn(LLM_TENSOR_ATTN_QKV_GATE, "weight", il), { n_embd, key_dim * 2 + value_dim * 2 }, flags | TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
+ if (!layer.wqkv_z) {
+ layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", il), { n_embd, key_dim * 2 + value_dim }, separate_flags);
+ layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", il), { n_embd, value_dim }, separate_flags);
+ }
layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", il), { hparams.ssm_d_conv, conv_dim }, flags);
layer.ssm_dt = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", il), { hparams.ssm_dt_rank }, flags);
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, il), { hparams.ssm_dt_rank }, flags);
- layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", il), { n_embd, n_v_heads }, flags);
- layer.ssm_alpha = create_tensor(tn(LLM_TENSOR_SSM_ALPHA, "weight", il), { n_embd, n_v_heads }, flags);
+ layer.ssm_beta_alpha_cat = create_tensor(tn(LLM_TENSOR_SSM_BETA_ALPHA_CAT, "weight", il), { n_embd, n_v_heads * 2 }, flags | TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
+ if (!layer.ssm_beta_alpha_cat) {
+ layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", il), { n_embd, n_v_heads }, separate_flags);
+ layer.ssm_alpha = create_tensor(tn(LLM_TENSOR_SSM_ALPHA, "weight", il), { n_embd, n_v_heads }, separate_flags);
+ }
layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", il), { head_v_dim }, flags);
layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", il), { value_dim, n_embd }, flags);
}
@@ -239,6 +246,22 @@ std::pair<ggml_tensor *, ggml_tensor *> llama_model_qwen35::graph::build_qkvz(
const int64_t n_seqs = ubatch.n_seqs;
const int64_t n_seq_tokens = ubatch.n_seq_tokens;
+ if (model.layers[il].wqkv_z) {
+ ggml_tensor * qkvz = build_lora_mm(model.layers[il].wqkv_z, input);
+ cb(qkvz, "linear_attn_qkvz", il);
+
+ const int64_t n_z = hparams.ssm_d_state * hparams.ssm_dt_rank;
+ const int64_t n_qkv = qkvz->ne[0] - n_z;
+
+ ggml_tensor * qkv_mixed = ggml_view_3d(ctx0, qkvz, n_qkv, n_seq_tokens, n_seqs, qkvz->nb[1], qkvz->nb[1] * n_seq_tokens, 0);
+ cb(qkv_mixed, "linear_attn_qkv_mixed", il);
+
+ ggml_tensor * z = ggml_view_2d(ctx0, qkvz, n_z, n_seq_tokens * n_seqs, qkvz->nb[1], ggml_row_size(qkvz->type, n_qkv));
+ cb(z, "z", il);
+
+ return { qkv_mixed, z };
+ }
+
ggml_tensor * qkv_mixed = build_lora_mm(model.layers[il].wqkv, input, model.layers[il].wqkv_s);
qkv_mixed = ggml_reshape_3d(ctx0, qkv_mixed, qkv_mixed->ne[0], n_seq_tokens, n_seqs);
cb(qkv_mixed, "linear_attn_qkv_mixed", il);
@@ -354,17 +377,29 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear(
ggml_tensor * qkv_mixed = qkvz.first;
ggml_tensor * z = qkvz.second;
- ggml_tensor * beta = build_lora_mm(model.layers[il].ssm_beta, cur, model.layers[il].ssm_beta_s);
- beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs);
+ ggml_tensor * beta = nullptr;
+ ggml_tensor * alpha = nullptr;
+ if (model.layers[il].ssm_beta_alpha_cat) {
+ ggml_tensor * ba = build_lora_mm(model.layers[il].ssm_beta_alpha_cat, cur);
+ cb(ba, "beta_alpha", il);
+
+ beta = ggml_view_4d(ctx0, ba, 1, num_v_heads, n_seq_tokens, n_seqs,
+ ggml_element_size(ba), ba->nb[1], ba->nb[1] * n_seq_tokens, 0);
+ alpha = ggml_view_3d(ctx0, ba, num_v_heads, n_seq_tokens, n_seqs,
+ ba->nb[1], ba->nb[1] * n_seq_tokens, ggml_row_size(ba->type, num_v_heads));
+ } else {
+ beta = build_lora_mm(model.layers[il].ssm_beta, cur, model.layers[il].ssm_beta_s);
+ beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs);
+
+ alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s);
+ alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);
+ }
cb(beta, "beta", il);
+ cb(alpha, "alpha", il);
beta = ggml_sigmoid(ctx0, beta);
cb(beta, "beta_sigmoid", il);
- ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s);
- alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);
- cb(alpha, "alpha", il);
-
ggml_tensor * alpha_biased = ggml_add(ctx0, alpha, model.layers[il].ssm_dt);
ggml_tensor * alpha_softplus = ggml_softplus(ctx0, alpha_biased);
cb(alpha_softplus, "a_softplus", il);
@@ -439,7 +474,8 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear(
ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il);
// z: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * n_tokens * n_seqs, head_dim]
- ggml_tensor * z_2d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);
+ ggml_tensor * z_2d = ggml_view_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs,
+ ggml_row_size(z->type, head_v_dim), z->nb[1], z->nb[1] * n_seq_tokens, 0);
// Apply gated normalization: self.norm(core_attn_out, z)
ggml_tensor * attn_out_norm = build_norm_gated(output, model.layers[il].ssm_norm, z_2d, il);