Commit ba3c05f71 for llama.cpp
commit ba3c05f71da85cd0439e84ba42b0478b24d5db16
Author: Pascal <admin@serveurperso.com>
Date: Thu Oct 15 23:27:12 2026 +0200
model: add Muse Spark Open (text, vision and audio) (#30616)
* model: add Muse Spark Open (text, vision and audio)
Muse Spark Open is a latent space MoE: the residual stream stays at 8192
while the 256 experts live in 4096, with a projection down and up around
them, top-8 sigmoid routing with a correction bias. Attention interleaves
three sliding layers at a 2048 window with rope for one global layer
without rope, with a sigmoid output gate and a weightless QK-norm.
Residuals are norm preserving, their gate is folded at conversion time.
Vision reuses the Muse Glimmer tower. Audio is a causal convolutional
pyramid with no attention and no spectrogram, four stages folding 16, 6,
5 and 4 samples into the channel dim, one embedding per 80 ms of 24 kHz.
The quantizer keeps the latent projections and the attention gate at
Q8_0, a little over 1% of the weights, since the whole residual stream
passes through them and the low-bit types turned the output into a
repeating loop. The model saver writes the post norm epsilon, and the
arch is registered in test-llama-archs.
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
Co-authored-by: Pedro Cuenca <pedro@huggingface.co>
* Muse Spark: drop the redundant forward expand
* Muse Spark: drop the redundant forward expand and tighten the comments
* Update src/llama-quant.cpp
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
* Update src/llama-graph.cpp
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
* Update conversion/muse_spark.py
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
* Update conversion/muse_spark.py
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
* Update conversion/muse_spark.py
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
* Update conversion/muse_spark.py
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
---------
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
Co-authored-by: Pedro Cuenca <pedro@huggingface.co>
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
diff --git a/common/chat.cpp b/common/chat.cpp
index 485671308..118072b6b 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -1110,6 +1110,12 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
return common_chat_params_init_gpt_oss;
}
+ // Muse Spark shares the Muse recipient protocol and adds a commentary recipient.
+ if (src.find("to=commentary") != std::string::npos && src.find("<|eom|>") != std::string::npos) {
+ LOG_DBG("Using specialized template: Muse Spark\n");
+ return common_chat_params_init_muse_spark;
+ }
+
// Muse Glimmer format using " to=<recipient>" recipients and <|eom|>/<|eot|> message terminators.
if (src.find("<atem:function_calls>") != std::string::npos && src.find("<|eom|>") != std::string::npos) {
LOG_DBG("Using specialized template: Muse Glimmer\n");
diff --git a/common/parsers/muse-spark.cpp b/common/parsers/muse-spark.cpp
new file mode 100644
index 000000000..577090aa5
--- /dev/null
+++ b/common/parsers/muse-spark.cpp
@@ -0,0 +1,146 @@
+#include "parsers.h"
+
+// An assistant turn is rendered as one or more messages, each
+// "<|start|>assistant to=<recipient><|message|>{content}{END}" where END is
+// <|eom|> (more messages follow) or <|eot|> (end of turn):
+// - chain-of-thought: to=self, terminated by <|eom|>
+// - user facing aside: to=commentary, terminated by <|eom|>
+// - final answer: to=user, terminated by <|eot|>
+// A commentary message carries content the user sees while the assistant keeps
+// working, typically before or between tool calls.
+// The generation prompt is just "<|start|>assistant"; the model emits its own
+// " to=...<|message|>".
+common_chat_params common_chat_params_init_muse_spark(const common_chat_template & tmpl,
+ const autoparser::generation_params & inputs) {
+ common_chat_params data;
+
+ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
+ data.generation_prompt = "<|start|>assistant";
+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
+ data.supports_thinking = true;
+
+ data.preserved_tokens = {
+ "<|start|>", "<|message|>", "<|eom|>", "<|eot|>",
+ // ATEM tool-call markup emitted on " to=<tool>" turns.
+ "<atem:function_calls>", "<atem:invoke", "<atem:parameter", "</atem:parameter>",
+ "</atem:invoke>", "</atem:function_calls>",
+ };
+
+ data.message_delimiters = {
+ { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
+ { COMMON_CHAT_ROLE_USER, "<|start|>user" },
+ { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" },
+ { COMMON_CHAT_ROLE_TOOL, "<|start|>tool" },
+ };
+
+ if (inputs.has_continuation()) {
+ const auto & msg = inputs.continue_msg;
+
+ data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content;
+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+ data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content();
+ }
+
+ data.prompt.append(data.generation_prompt);
+ }
+
+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
+
+ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
+ // Constrained grammar whenever tools are offered.
+ auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
+
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ auto start = p.rule("start", p.literal("<|start|>assistant"));
+
+ if (!extract_reasoning && !include_grammar) {
+ return start + p.content(p.rest());
+ }
+
+ if (extract_reasoning) {
+ p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>"));
+ } else {
+ p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>"));
+ }
+ auto analysis = p.ref("analysis");
+
+ // Commentary is user facing content emitted mid turn, so it reads as
+ // content and always ends on <|eom|>.
+ auto preamble = p.rule("preamble", p.literal(" to=commentary<|message|>") +
+ p.content(p.until("<|eom|>")) + p.literal("<|eom|>"));
+
+ auto aside = p.rule("aside", analysis | preamble);
+
+ auto recipient = p.optional(p.literal(" to=user"));
+ auto final_msg = p.rule("final", recipient + p.literal("<|message|>") +
+ p.content(p.until_one_of({ "<|eot|>", "<|eom|>" })));
+
+ if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
+ auto string_value = p.ac(
+ p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")),
+ "</atem:parameter>");
+
+ auto tool_choice = p.choice();
+ foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
+ const auto & function = tool.at("function");
+ const std::string name = function.at("name");
+
+ std::vector<common_peg_parser> arg_rules;
+ foreach_parameter(function, [&](size_t param_index, const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) {
+ auto value_parser = p.eps();
+ if (prop.schema->may_be_string()) {
+ value_parser = string_value;
+ } else {
+ value_parser = p.tool_arg_json_value(
+ p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index) + "-schema", doc, *prop.schema))
+ + p.tool_arg_close(p.literal("</atem:parameter>"));
+ }
+
+ arg_rules.push_back(p.tool_arg(
+ p.tool_arg_open(p.literal("<atem:parameter name=\"") + p.tool_arg_name(p.literal(prop.name)) + p.literal("\">")) +
+ value_parser));
+ });
+
+ auto args = p.eps();
+ if (!arg_rules.empty()) {
+ args = p.zero_or_more(p.choice(arg_rules) + p.space());
+ }
+
+ auto tool_parser = p.tool(
+ p.tool_open(p.literal(" to=") + p.until("<|message|>") +
+ p.literal("<|message|><atem:function_calls>") + p.space() +
+ p.literal("<atem:invoke name=\"") + p.tool_name(p.literal(name)) + p.literal("\">") + p.space())
+ << p.tool_args(args)
+ << p.tool_close(p.literal("</atem:invoke>") + p.space() + p.literal("</atem:function_calls>")));
+
+ tool_choice |= p.rule("tool-" + std::to_string(tool_index), tool_parser);
+ });
+
+ auto tool_calls = inputs.parallel_tool_calls
+ ? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice))
+ : p.trigger_rule("tool-call", tool_choice);
+
+
+ if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
+ return p.zero_or_more(start + aside) + start + tool_calls;
+ }
+ auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls);
+ return p.zero_or_more(start + aside) + start + (tool_calls | (final_msg + trailing_calls));
+ }
+
+ return p.zero_or_more(start + aside) + start + final_msg;
+ });
+
+ if (include_grammar) {
+ data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {
+ data.parser.build_grammar(builder, data.grammar_lazy);
+ });
+ data.grammar_triggers = {
+ { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN,
+ "<\\|start\\|>assistant( to=(?!self<\\|message\\|>)(?!commentary<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" },
+ };
+ }
+
+ return data;
+}
diff --git a/common/parsers/parsers.h b/common/parsers/parsers.h
index 8f0a97ce3..fac69c096 100644
--- a/common/parsers/parsers.h
+++ b/common/parsers/parsers.h
@@ -79,6 +79,7 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
+common_chat_params common_chat_params_init_muse_spark(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
diff --git a/common/parsers/sources.cmake b/common/parsers/sources.cmake
index 152397074..5ee65b188 100644
--- a/common/parsers/sources.cmake
+++ b/common/parsers/sources.cmake
@@ -19,6 +19,7 @@ set(LLAMA_CHAT_PARSERS_SOURCES
${CMAKE_CURRENT_LIST_DIR}/minimax-m3.cpp
${CMAKE_CURRENT_LIST_DIR}/ministral3.cpp
${CMAKE_CURRENT_LIST_DIR}/muse-glimmer.cpp
+ ${CMAKE_CURRENT_LIST_DIR}/muse-spark.cpp
${CMAKE_CURRENT_LIST_DIR}/qwen3-coder.cpp
${CMAKE_CURRENT_LIST_DIR}/translate-gemma.cpp
)
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 8ce4cb733..6c3914fbd 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -220,6 +220,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"OlmoeForCausalLM": "olmo",
"MuseGlimmerAssistantModel": "muse_glimmer",
"MuseGlimmerForConditionalGeneration": "muse_glimmer",
+ "MuseSparkForConditionalGeneration": "muse_spark",
"OpenELMForCausalLM": "openelm",
"OpenJevModel": "qwen",
"OrionForCausalLM": "orion",
@@ -356,6 +357,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"NemotronH_Omni_Reasoning_V3": "nemotron",
"NemotronH_Nano_VL_V2": "nemotron",
"MuseGlimmerForConditionalGeneration": "muse_glimmer",
+ "MuseSparkForConditionalGeneration": "muse_spark",
"PaddleOCRVisionModel": "ernie",
"Phi4ForCausalLMV": "phi",
"Qwen2AudioForConditionalGeneration": "ultravox",
diff --git a/conversion/base.py b/conversion/base.py
index a5a80f27d..e3d265157 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -1869,6 +1869,9 @@ class TextModel(ModelBase):
if chkhsh == "66b8d4e19ab16c3bfd89bce5d785fb7e0155e8648708a1f42077cb9fe002c273":
# ref: https://huggingface.co/alvarobartt/grok-2-tokenizer
res = "grok-2"
+ if chkhsh == "f5c965e4198da6e7525072d87d1ecf4c51d981fd486c7c4ec728ef9cec7e03fc":
+ # ref: https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B
+ res = "gpt-4o"
if chkhsh == "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df":
# ref: https://huggingface.co/aari1995/German_Semantic_V3
res = "jina-v2-de"
diff --git a/conversion/muse_glimmer.py b/conversion/muse_glimmer.py
index c86b33227..9b63d3dba 100644
--- a/conversion/muse_glimmer.py
+++ b/conversion/muse_glimmer.py
@@ -94,7 +94,7 @@ class MuseGlimmerVisionModel(MmprojModel):
assert self.hparams_vision is not None
c = self.hparams_vision # enriched vision_config from get_vision_config()
- self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MUSE_GLIMMER)
+ self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.MUSE_GLIMMER)
self.gguf_writer.add_vision_attention_layernorm_eps(float(c["layer_norm_eps"]))
self.gguf_writer.add_vision_spatial_merge_size(int(c["merge_size"]))
diff --git a/conversion/muse_spark.py b/conversion/muse_spark.py
new file mode 100644
index 000000000..150f22f95
--- /dev/null
+++ b/conversion/muse_spark.py
@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+from typing import Any, Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import ModelBase, MmprojModel, TextModel, gguf
+from .muse_glimmer import MuseGlimmerVisionModel
+
+
+# The reference clamps the squared stream coefficient with this floor.
+RESIDUAL_GATE_EPS = 1e-3
+
+# Block norms hold a delta around one; the output norm holds an absolute gain.
+OFFSET_NORM_SUFFIXES = (
+ "input_layernorm.weight",
+ "pre_feedforward_layernorm.weight",
+ "post_feedforward_layernorm.weight",
+ "pre_expert_norm.weight",
+ "post_expert_norm.weight",
+)
+
+
+def _unpermute_for_rope(tensor: "Tensor", n_heads: int) -> "Tensor":
+ """Invert transformers' `_permute_for_rope`: HF stores Q/K in rotate_half layout,
+ llama.cpp consumes the interleaved (NORM) layout. Vision Q/K carry a bias too."""
+ if tensor.ndim == 1:
+ dim1, = tensor.shape
+ return tensor.view(n_heads, 2, dim1 // n_heads // 2).transpose(1, 2).reshape(dim1)
+ dim1, dim2 = tensor.shape
+ return tensor.view(n_heads, 2, dim1 // n_heads // 2, dim2).transpose(1, 2).reshape(dim1, dim2)
+
+
+@ModelBase.register("MuseSparkForConditionalGeneration")
+@ModelBase.example("meta-models/Muse-Spark-Open-816B-A42B")
+class MuseSparkModel(TextModel):
+ model_arch = gguf.MODEL_ARCH.MUSE_SPARK
+
+ def set_vocab(self):
+ self._set_vocab_gpt2()
+
+ from transformers import AutoTokenizer
+ tok = AutoTokenizer.from_pretrained(self.dir_model)
+ eot_id = tok.convert_tokens_to_ids("<|eot|>") # ty: ignore[unresolved-attribute]
+ if isinstance(eot_id, int) and eot_id >= 0:
+ self.gguf_writer.add_eot_token_id(eot_id)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ hparams = self.hparams
+
+ # MetaP resolves to two scalars
+ # The output multiplier lands here and the attention scale rides on the synthesized QK-norm weights
+ self.gguf_writer.add_logit_scale(hparams["output_multiplier"])
+ self.gguf_writer.add_final_logit_softcapping(hparams["final_logit_softcapping"])
+
+ self.gguf_writer.add_post_norm_eps(hparams["post_norm_eps"])
+
+ # Only the sliding layers carry rope, so the graph takes it from the pattern
+ sliding = [t == "sliding_attention" for t in hparams["layer_types"]]
+ theta = hparams["rope_parameters"]["rope_theta"]
+ assert sliding == [bool(t) for t in hparams["layer_rope_theta"]]
+ assert all(t in (0, theta) for t in hparams["layer_rope_theta"])
+ assert all(t == "sparse" for t in hparams["mlp_layer_types"])
+
+ self.gguf_writer.add_sliding_window(hparams["sliding_window"])
+ self.gguf_writer.add_rope_freq_base_swa(theta)
+ self.gguf_writer.add_sliding_window_pattern(sliding)
+
+ # Experts live in a latent space narrower than the residual stream.
+ self.gguf_writer.add_moe_latent_size(hparams["moe_hidden_size"])
+ self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
+ self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"])
+ self.gguf_writer.add_expert_weights_norm(True)
+ # the config carries no score function, the reference hardcodes sigmoid
+ self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name.endswith(OFFSET_NORM_SUFFIXES):
+ data_torch = data_torch + 1.0
+
+ if name.endswith(".self_attn.q_proj.weight"):
+ assert bid is not None
+ n_head = self.hparams["num_attention_heads"]
+ head_dim = self.hparams["head_dim"]
+
+ yield self.map_tensor_name(name), _unpermute_for_rope(data_torch, n_head)
+
+ # Synthesize QK-norm to carry the MetaP scale so the graph keeps the plain 1 / sqrt(head_dim)
+ ones = torch.ones(head_dim, dtype=torch.float32)
+ q_norm = ones * self.hparams["qk_scale_factor"]
+ yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q_NORM, bid), q_norm
+ yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K_NORM, bid), ones
+ return
+
+ if name.endswith(".self_attn.k_proj.weight"):
+ n_head_kv = self.hparams["num_key_value_heads"]
+ yield self.map_tensor_name(name), _unpermute_for_rope(data_torch, n_head_kv)
+ return
+
+ # gate_up_proj fuses both projections over the output dim, gate first.
+ if name.endswith(".mlp.experts.gate_up_proj"):
+ assert bid is not None
+ n_ff = data_torch.shape[1] // 2
+ yield self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), data_torch[:, :n_ff, :]
+ yield self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), data_torch[:, n_ff:, :]
+ return
+
+ if name.endswith(".mlp.experts.down_proj"):
+ assert bid is not None
+ yield self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN_EXP, bid), data_torch
+ return
+
+ # The router scales its logits by the MetaP output multiplier before the sigmoid
+ # Fold it into the weight to keep a plain matmul
+ if name.endswith(".mlp.gate.weight"):
+ data_torch = data_torch.float() * self.hparams["output_multiplier"]
+ yield from super().modify_tensors(data_torch, name, bid)
+ return
+
+ # Norm preserving residual gate: alpha scales the stream, beta the branch, with alpha^2 + beta^2 = 1
+ # Both follow from the weight, so resolve them here
+ if name.endswith("_residual_gate.gate"):
+ assert bid is not None
+ attn = "post_attention" in name
+
+ w = data_torch.float() / self.hparams["residual_gate_temperature"]
+ beta = torch.sigmoid(w)
+ alpha_sq = torch.sigmoid(-w) * (1.0 + beta)
+ alpha = torch.sqrt(torch.clamp(alpha_sq, min=RESIDUAL_GATE_EPS))
+
+ t_alpha = gguf.MODEL_TENSOR.ATTN_RESID_ALPHA if attn else gguf.MODEL_TENSOR.FFN_RESID_ALPHA
+ t_beta = gguf.MODEL_TENSOR.ATTN_RESID_BETA if attn else gguf.MODEL_TENSOR.FFN_RESID_BETA
+ yield self.format_tensor_name(t_alpha, bid), alpha
+ yield self.format_tensor_name(t_beta, bid), beta
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
+@ModelBase.register("MuseSparkForConditionalGeneration")
+@ModelBase.example("meta-models/Muse-Spark-Open-816B-A42B")
+class MuseSparkVisionModel(MuseGlimmerVisionModel):
+ has_vision_encoder = True # same vision encoder as Muse Glimmer
+ has_audio_encoder = True
+
+ def get_audio_config(self) -> dict[str, Any]:
+ c = self.global_config["audio_config"]
+ # the base class wants one width and one layer count: use the last stage width and the total number of conv blocks
+ return {
+ **c,
+ "hidden_size": c["hidden_sizes"][-1],
+ "intermediate_size": int(c["hidden_sizes"][-1] * c["mlp_ratio"]),
+ "num_attention_heads": 1,
+ "num_hidden_layers": sum(c["depths"]),
+ }
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+
+ assert self.hparams_audio
+ a = self.hparams_audio
+
+ self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.MUSE_SPARK_AUDIO)
+ # the input is the raw waveform, one channel takes the place of the mel bins
+ self.gguf_writer.add_audio_num_mel_bins(1)
+ self.gguf_writer.add_audio_attention_layernorm_eps(float(a["rms_norm_eps"]))
+ self.gguf_writer.add_audio_conv_kernel_size(int(a["conv_kernel_size"]))
+ self.gguf_writer.add_audio_hidden_sizes([int(x) for x in a["hidden_sizes"]])
+ self.gguf_writer.add_audio_patch_sizes([int(x) for x in a["patch_sizes"]])
+ self.gguf_writer.add_audio_depths([int(x) for x in a["depths"]])
+
+ @classmethod
+ def filter_tensors(cls, item):
+ name, gen = item
+ if name.startswith("model.audio_tower."):
+ return MmprojModel.filter_tensors((name, gen))
+ return super().filter_tensors((name, gen))
+
+ _AUDIO_HEAD = {
+ "model.audio_tower.norm": gguf.MODEL_TENSOR.A_PRE_NORM,
+ "model.audio_tower.proj": gguf.MODEL_TENSOR.A_MMPROJ_FC,
+ "model.audio_tower.post_proj_norm": gguf.MODEL_TENSOR.A_POST_NORM,
+ }
+
+ def _audio_tensor(self, data_torch, name, suffix):
+ assert self.hparams_audio
+
+ stem = name[: -len(suffix) - 1]
+
+ if stem in self._AUDIO_HEAD:
+ return self.format_tensor_name(self._AUDIO_HEAD[stem], suffix="." + suffix), data_torch
+
+ parts = stem.split(".")
+ stage = int(parts[3])
+
+ if parts[4] == "patch_embedder":
+ key = (gguf.MODEL_TENSOR.A_ENC_PATCH_PROJ if parts[5] == "proj"
+ else gguf.MODEL_TENSOR.A_ENC_PATCH_NORM)
+ return self.format_tensor_name(key, bid=stage, suffix="." + suffix), data_torch
+
+ # conv blocks are numbered per stage upstream and flat here
+ depths = [int(d) for d in self.hparams_audio["depths"]]
+ bid = sum(depths[:stage]) + int(parts[5])
+ tail = ".".join(parts[6:])
+ key = {
+ "conv": gguf.MODEL_TENSOR.A_ENC_CONV1D,
+ "norm": gguf.MODEL_TENSOR.A_ENC_CONV1D_NORM,
+ "mlp.fc1": gguf.MODEL_TENSOR.A_ENC_FFN_UP,
+ "mlp.fc2": gguf.MODEL_TENSOR.A_ENC_FFN_DOWN,
+ }[tail]
+ return self.format_tensor_name(key, bid=bid, suffix="." + suffix), data_torch
+
+ def modify_tensors(self, data_torch, name, bid):
+ if name.startswith("model.audio_tower."):
+ yield self._audio_tensor(data_torch, name, name.rpartition(".")[2])
+ return
+ yield from super().modify_tensors(data_torch, name, bid)
diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py
index 0afbcc7e9..d49fb8f3f 100755
--- a/convert_hf_to_gguf_update.py
+++ b/convert_hf_to_gguf_update.py
@@ -189,6 +189,8 @@ pre_computed_hashes = [
{"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B", "chkhsh": "d4540891389ea895b53b399da6ac824becc30f2fba0e9ddbb98f92e55ca0e97c"},
{"name": "qwen35", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openbmb/MiniCPM-V-4_6", "chkhsh": "1444df51289cfa8063b96f0e62b1125440111bc79a52003ea14b6eac7016fd5f"},
{"name": "grok-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/alvarobartt/grok-2-tokenizer", "chkhsh": "66b8d4e19ab16c3bfd89bce5d785fb7e0155e8648708a1f42077cb9fe002c273"},
+ # muse-spark uses the l4_202k tokenizer, whose pre-tokenizer regex is gpt-4o's
+ {"name": "gpt-4o", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "chkhsh": "f5c965e4198da6e7525072d87d1ecf4c51d981fd486c7c4ec728ef9cec7e03fc"},
# jina-v2-de variants
{"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"},
{"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"},
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index a39d046fe..a3a9c478b 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -132,6 +132,7 @@ class Keys:
EXPERTS_PER_GROUP = "{arch}.experts_per_group"
MOE_EVERY_N_LAYERS = "{arch}.moe_every_n_layers"
MOE_LATENT_SIZE = "{arch}.moe_latent_size"
+ POST_NORM_EPS = "{arch}.post_norm_epsilon"
NEXTN_PREDICT_LAYERS = "{arch}.nextn_predict_layers"
NUM_DEEPSTACK_LAYERS = "{arch}.n_deepstack_layers"
DEEPSTACK_MAPPING = "{arch}.deepstack_mapping"
@@ -466,6 +467,9 @@ class Keys:
SUBSAMPLING_FACTOR = "clip.audio.subsampling_factor"
CHUNK_SIZE = "clip.audio.chunk_size"
CONV_KERNEL_SIZE = "clip.audio.conv_kernel_size"
+ HIDDEN_SIZES = "clip.audio.hidden_sizes" # muse-spark, per stage
+ PATCH_SIZES = "clip.audio.patch_sizes" # muse-spark, per stage
+ DEPTHS = "clip.audio.depths" # muse-spark, per stage
MAX_POS_EMB = "clip.audio.max_pos_emb"
FEATURE_LAYERS = "clip.audio.feature_layer" # Granite Speech Plus
RVQ_NUM_QUANTIZERS = "clip.audio.rvq.num_quantizers"
@@ -611,6 +615,7 @@ class MODEL_ARCH(IntEnum):
OLMO2 = auto()
OLMOE = auto()
MUSE_GLIMMER = auto()
+ MUSE_SPARK = auto()
OPENELM = auto()
ARCTIC = auto()
DEEPSEEK = auto()
@@ -743,6 +748,8 @@ class MODEL_TENSOR(IntEnum):
ATTN_ROT_EMBD = auto()
ATTN_SINKS = auto()
ATTN_GATE = auto()
+ ATTN_RESID_ALPHA = auto()
+ ATTN_RESID_BETA = auto()
FFN_GATE_INP = auto()
FFN_GATE_INP_SHEXP = auto()
FFN_NORM = auto()
@@ -771,6 +778,10 @@ class MODEL_TENSOR(IntEnum):
FFN_GATE_TID2EID = auto()
MOE_LATENT_DOWN = auto() # nemotron 3 super
MOE_LATENT_UP = auto() # nemotron 3 super
+ FFN_RESID_ALPHA = auto()
+ FFN_RESID_BETA = auto()
+ FFN_PRE_EXP_NORM = auto()
+ FFN_POST_EXP_NORM = auto()
ATTN_Q_NORM = auto()
ATTN_K_NORM = auto()
LAYER_OUT_NORM = auto()
@@ -1121,6 +1132,8 @@ class MODEL_TENSOR(IntEnum):
A_ENC_EMBD_TO_LOGITS = auto() # lfm2
A_ENC_INP_PROJ = auto() # gemma4
A_ENC_CONV1D = auto()
+ A_ENC_PATCH_PROJ = auto() # muse-spark
+ A_ENC_PATCH_NORM = auto() # muse-spark
A_ENC_CONV1D_NORM = auto() # gemma3n
A_ENC_CONV2D = auto()
A_ENC_CONV_OUT = auto()
@@ -1415,6 +1428,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.OLMO2: "olmo2",
MODEL_ARCH.OLMOE: "olmoe",
MODEL_ARCH.MUSE_GLIMMER: "muse-glimmer",
+ MODEL_ARCH.MUSE_SPARK: "muse-spark",
MODEL_ARCH.OPENELM: "openelm",
MODEL_ARCH.ARCTIC: "arctic",
MODEL_ARCH.DEEPSEEK: "deepseek",
@@ -1544,6 +1558,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.ATTN_ROT_EMBD: "blk.{bid}.attn_rot_embd",
MODEL_TENSOR.ATTN_SINKS: "blk.{bid}.attn_sinks",
MODEL_TENSOR.ATTN_GATE: "blk.{bid}.attn_gate",
+ MODEL_TENSOR.ATTN_RESID_ALPHA: "blk.{bid}.attn_resid_alpha",
+ MODEL_TENSOR.ATTN_RESID_BETA: "blk.{bid}.attn_resid_beta",
MODEL_TENSOR.ATTN_Q_NORM: "blk.{bid}.attn_q_norm",
MODEL_TENSOR.ATTN_K_NORM: "blk.{bid}.attn_k_norm",
MODEL_TENSOR.ATTN_OUT_NORM: "blk.{bid}.attn_output_norm",
@@ -1576,6 +1592,10 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.FFN_GATE_TID2EID: "blk.{bid}.ffn_gate_tid2eid",
MODEL_TENSOR.MOE_LATENT_DOWN: "blk.{bid}.ffn_latent_down", # nemotron 3 super
MODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super
+ MODEL_TENSOR.FFN_RESID_ALPHA: "blk.{bid}.ffn_resid_alpha",
+ MODEL_TENSOR.FFN_RESID_BETA: "blk.{bid}.ffn_resid_beta",
+ MODEL_TENSOR.FFN_PRE_EXP_NORM: "blk.{bid}.ffn_pre_exp_norm",
+ MODEL_TENSOR.FFN_POST_EXP_NORM: "blk.{bid}.ffn_post_exp_norm",
MODEL_TENSOR.LAYER_OUT_NORM: "blk.{bid}.layer_output_norm",
MODEL_TENSOR.LAYER_OUT_SCALE: "blk.{bid}.layer_output_scale",
MODEL_TENSOR.PER_LAYER_TOKEN_EMBD: "per_layer_token_embd", # gemma3n
@@ -1921,6 +1941,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_ENC_EMBD_TO_LOGITS: "a.embd_to_logits",
MODEL_TENSOR.A_ENC_INP_PROJ: "a.input_projection",
MODEL_TENSOR.A_ENC_CONV1D: "a.conv1d.{bid}",
+ MODEL_TENSOR.A_ENC_PATCH_PROJ: "a.patch.{bid}",
+ MODEL_TENSOR.A_ENC_PATCH_NORM: "a.patch.{bid}.norm",
MODEL_TENSOR.A_ENC_CONV2D: "a.conv2d.{bid}",
MODEL_TENSOR.A_ENC_CONV_OUT: "a.conv_out",
MODEL_TENSOR.A_ENC_CONV1D_NORM: "a.conv1d.{bid}.norm",
@@ -2273,6 +2295,8 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.A_ENC_EMBD_TO_LOGITS,
MODEL_TENSOR.A_ENC_INP_PROJ,
MODEL_TENSOR.A_ENC_CONV1D,
+ MODEL_TENSOR.A_ENC_PATCH_PROJ,
+ MODEL_TENSOR.A_ENC_PATCH_NORM,
MODEL_TENSOR.A_ENC_CONV2D,
MODEL_TENSOR.A_ENC_CONV_OUT,
MODEL_TENSOR.A_ENC_CONV1D_NORM,
@@ -3979,6 +4003,34 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_PRE_NORM,
MODEL_TENSOR.FFN_POST_NORM,
],
+ MODEL_ARCH.MUSE_SPARK: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_RESID_ALPHA,
+ MODEL_TENSOR.ATTN_RESID_BETA,
+ MODEL_TENSOR.FFN_PRE_NORM,
+ MODEL_TENSOR.FFN_POST_NORM,
+ MODEL_TENSOR.FFN_RESID_ALPHA,
+ MODEL_TENSOR.FFN_RESID_BETA,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.MOE_LATENT_DOWN,
+ MODEL_TENSOR.FFN_PRE_EXP_NORM,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_POST_EXP_NORM,
+ MODEL_TENSOR.MOE_LATENT_UP,
+ ],
MODEL_ARCH.OPENELM: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
@@ -6283,6 +6335,7 @@ class VisionProjectorType:
COHERE2V = "cohere2v"
D1OMNI_V = "d1omni_v" # lfm2 vision, without separator tokens
D1OMNI_A = "d1omni_a" # lfm2a audio, with a residual block after the projector
+ MUSE_SPARK_AUDIO = "muse-spark-audio"
# Items here are (block size, type size)
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index cc2bd8e7b..a1b1f1263 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -924,6 +924,9 @@ class GGUFWriter:
def add_moe_latent_size(self, value: int) -> None:
self.add_uint32(Keys.LLM.MOE_LATENT_SIZE.format(arch=self.arch), value)
+ def add_post_norm_eps(self, value: float) -> None:
+ self.add_float32(Keys.LLM.POST_NORM_EPS.format(arch=self.arch), value)
+
def add_nextn_predict_layers(self, count: int) -> None:
self.add_uint32(Keys.LLM.NEXTN_PREDICT_LAYERS.format(arch=self.arch), count)
@@ -1591,6 +1594,15 @@ class GGUFWriter:
def add_audio_wa_pattern_mode(self, modes: Sequence[int]) -> None:
self.add_array(Keys.ClipAudio.WA_PATTERN_MODE, modes)
+ def add_audio_hidden_sizes(self, values: Sequence[int]) -> None:
+ self.add_array(Keys.ClipAudio.HIDDEN_SIZES, values)
+
+ def add_audio_patch_sizes(self, values: Sequence[int]) -> None:
+ self.add_array(Keys.ClipAudio.PATCH_SIZES, values)
+
+ def add_audio_depths(self, values: Sequence[int]) -> None:
+ self.add_array(Keys.ClipAudio.DEPTHS, values)
+
def add_audio_window_size(self, value: int) -> None:
self.add_uint32(Keys.ClipAudio.WINDOW_SIZE, value)
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index 63301cc11..ecea46463 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -530,6 +530,7 @@ class TensorNameMap:
MODEL_TENSOR.FFN_EXP_PROBS_B: (
"model.layers.{bid}.mlp.gate.e_score_correction", # deepseek-v3 dots1
+ "model.layers.{bid}.mlp.gate.e_score_correction_bias", # muse-spark
"model.layers.{bid}.mlp.moe_statics.e_score_correction", # ernie4.5-moe
"model.layers.{bid}.mlp.gate.expert_bias", # bailingmoe2
"model.layers.{bid}.mlp.expert_bias", # afmoe
@@ -674,10 +675,20 @@ class TensorNameMap:
MODEL_TENSOR.MOE_LATENT_DOWN: (
"backbone.layers.{bid}.mixer.fc1_latent_proj", # nemotron 3 super
+ "model.layers.{bid}.mlp.pre_expert_proj", # muse-spark
),
MODEL_TENSOR.MOE_LATENT_UP: (
"backbone.layers.{bid}.mixer.fc2_latent_proj", # nemotron 3 super
+ "model.layers.{bid}.mlp.post_expert_proj", # muse-spark
+ ),
+
+ MODEL_TENSOR.FFN_PRE_EXP_NORM: (
+ "model.layers.{bid}.mlp.pre_expert_norm", # muse-spark
+ ),
+
+ MODEL_TENSOR.FFN_POST_EXP_NORM: (
+ "model.layers.{bid}.mlp.experts.post_expert_norm", # muse-spark
),
# Feed-forward down
diff --git a/models/templates/muse-spark.jinja b/models/templates/muse-spark.jinja
new file mode 100644
index 000000000..8fc6c8df2
--- /dev/null
+++ b/models/templates/muse-spark.jinja
@@ -0,0 +1,228 @@
+{%- set tools = tools | default(none) -%}
+{%- set add_generation_prompt = add_generation_prompt | default(false) -%}
+{%- set reasoning_effort = reasoning_effort | default(none) -%}
+{%- set disable_reasoning = disable_reasoning | default(false) -%}
+{%- set cutoff_date = cutoff_date | default('2026-01-04') -%}
+{%- set current_date = current_date | default(strftime_now('%Y-%m-%d') if strftime_now is defined else none) -%}
+{%- set reasoning_effort_map = {'minimal': 1, 'low': 16, 'medium': 64, 'high': 256} -%}
+
+{#- Neutralise control sequences in caller-supplied text so it cannot be
+ tokenized as structure. "<|" is broken because every special token has the
+ form "<|...|>"; the bare media placeholders are broken because the
+ multimodal processor rewrites them positionally and a literal one in text
+ would desynchronise placeholders from the attached media. -#}
+{%- macro esc(value) -%}
+ {{- value | string
+ | replace('<|', '< |')
+ | replace('<image>', '< image>')
+ | replace('<video>', '< video>')
+ | replace('<audio>', '< audio>') -}}
+{%- endmacro -%}
+
+{%- macro esc_header(value) -%}
+ {{- esc(value) | replace('\r', ' ') | replace('\n', ' ') -}}
+{%- endmacro -%}
+
+{%- macro esc_attr(value) -%}
+ {{- esc_header(value)
+ | replace('&', '&')
+ | replace('"', '"')
+ | replace("'", ''')
+ | replace('<', '<')
+ | replace('>', '>') -}}
+{%- endmacro -%}
+
+{%- macro render_content(content) -%}
+ {%- if content is string -%}
+ {{- esc(content | trim) -}}
+ {%- elif content is not none -%}
+ {%- for part in content -%}
+ {%- set part_type = part.get('type', '') -%}
+ {%- if part_type in ['image', 'input_image', 'image_url'] -%}
+ {{- '<|patch|>' -}}
+ {%- elif part_type in ['video', 'input_video', 'video_url'] -%}
+ {{- '<|video|>' -}}
+ {%- elif part_type in ['audio', 'input_audio', 'audio_url'] -%}
+ {{- '<|audio|>' -}}
+ {%- elif part_type in ['text', 'input_text', 'output_text'] -%}
+ {{- esc(part.get('text', '') | trim) -}}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_atem(tc) -%}
+ {%- set args = tc.function.arguments -%}
+ {%- if args is not mapping -%}
+ {{- raise_exception('This chat template requires tool_call.function.arguments to be a dict (mapping). The jinja sandbox has no JSON parser, so deserialize the arguments string before calling apply_chat_template.') -}}
+ {%- endif -%}
+ {{- '<atem:function_calls>\n<atem:invoke name="' -}}{{- esc_attr(tc.function.name) -}}{{- '">\n' -}}
+ {%- for k, v in args.items() -%}
+ {{- '<atem:parameter name="' -}}{{- esc_attr(k) -}}{{- '">' -}}
+ {%- if v is boolean -%}
+ {%- if v -%}
+ true
+ {%- else -%}
+ false
+ {%- endif -%}
+ {%- elif v is none -%}
+ null
+ {%- elif v is mapping or (v is iterable and v is not string) -%}
+ {{- esc(v | tojson) -}}
+ {%- else -%}
+ {{- esc(v) -}}
+ {%- endif -%}
+ {{- '</atem:parameter>\n' -}}
+ {%- endfor -%}
+ {{- '</atem:invoke>\n</atem:function_calls>' -}}
+{%- endmacro -%}
+
+{%- macro render_tool_defs(tools) -%}
+ {{- 'In this environment you have access to a set of tools you can use to answer the user\'s question.\n\n' -}}
+ {{- 'You can invoke a function by writing a "<atem:function_calls>" block like the following as part of your reply to the user:\n' -}}
+ {{- '<atem:function_calls>\n<atem:invoke name="$FUNCTION_NAME">\n<atem:parameter name="$PARAMETER_NAME">$PARAMETER_VALUE</atem:parameter>\n...\n</atem:invoke>\n</atem:function_calls>\n\n' -}}
+ {{- 'String and scalar parameters should be specified as is, while lists and objects should use JSON format. Note that spaces for string values are not stripped. The output is not expected to be valid XML and is parsed with regular expressions.\n' -}}
+ {{- 'Here are the functions available in JSONSchema format:\n' -}}
+ {{- '// Tool metadata\n{"name": "", "description": ""}\n' -}}
+ {{- '// Function schemas' -}}
+ {%- for tool in tools -%}
+ {%- set fn = tool.function if tool.function is defined else tool -%}
+ {{- '\n{"name": ' -}}{{- fn.name | tojson -}}
+ {{- ', "description": ' -}}{{- (fn.description if fn.description is defined and fn.description else '') | tojson -}}
+ {{- ', "parameters": ' -}}{{- (fn.parameters if fn.parameters is defined and fn.parameters else {}) | tojson -}}{{- '}' -}}
+ {%- endfor -%}
+ {{- '\n\n' -}}
+ {{- 'Here are examples of how to call functions in the tool set. To make parallel tool calls, emit each call as its own message (one tool call per message) in consecutive messages, as shown below:\n' -}}
+ {{- '(If the tool namespace is not specified, invoke the function directly as `example_function_name_1` rather than `example_tool_name_1.example_function_name_1`)\n\n' -}}
+ {{- 'to=example_tool_name_1.example_function_name_1\n\n' -}}
+ {{- '<atem:function_calls>\n<atem:invoke name="example_tool_name_1.example_function_name_1">\n' -}}
+ {{- '<atem:parameter name="example_parameter_1">value_1</atem:parameter>\n' -}}
+ {{- '</atem:invoke>\n</atem:function_calls>\n\n' -}}
+ {{- 'to=example_tool_name_2.example_function_name_2\n\n' -}}
+ {{- '<atem:function_calls>\n<atem:invoke name="example_tool_name_2.example_function_name_2">\n' -}}
+ {{- '<atem:parameter name="example_parameter_2">This is the value for the second parameter\nthat can span\n"multiple" lines\n</atem:parameter>\n' -}}
+ {{- '<atem:parameter name="example_parameter_3">value_3</atem:parameter>\n' -}}
+ {{- '</atem:invoke>\n</atem:function_calls>' -}}
+{%- endmacro -%}
+
+{%- macro render_valid_recipients(tools) -%}
+ {%- set rns = namespace(recipients=[]) -%}
+ {{- 'Use the appropriate recipient for each message:\n\n' -}}
+ {%- if not disable_reasoning -%}
+ {%- set rns.recipients = rns.recipients + ['"self"'] -%}
+ {{- '- "self": private reasoning and tool planning.\n' -}}
+ {%- endif -%}
+ {%- set rns.recipients = rns.recipients + ['"commentary"'] -%}
+ {{- '- "commentary": user-visible intermediate updates while the assistant will continue working, including messages sent to the user before or between tool calls.\n' -}}
+ {%- if tools -%}
+ {%- set first_tool = tools | first -%}
+ {%- set first_fn = (first_tool.function if first_tool.function is defined else first_tool).name -%}
+ {%- set rns.recipients = rns.recipients + ['"[func_name] (e.g., ' ~ esc_header(first_fn) ~ ')"'] -%}
+ {{- '- "tool" or "[tool name]": the name of the tool to call.\n' -}}
+ {%- endif -%}
+ {%- set rns.recipients = rns.recipients + ['"user"'] -%}
+ {{- '- "user": messages that end the assistant turn, such as a completed response or clarification question that waits for the user\'s reply; do not use for pre-tool updates or partial responses.\n' -}}
+ {{- '\n# Valid recipients: ' ~ rns.recipients | join(', ') ~ '.' -}}
+{%- endmacro -%}
+
+{{- bos_token -}}
+{%- set ns = namespace(instructions=none, rest=messages) -%}
+{%- if messages and messages[0]['role'] in ['system', 'developer'] -%}
+ {%- set ns.instructions = render_content(messages[0]['content'])
+ | replace('Reasoning effort', 'Reasoning strength')
+ | replace('Reasoning Effort', 'Reasoning Strength')
+ | replace('reasoning effort', 'reasoning strength')
+ | replace('REASONING EFFORT', 'REASONING STRENGTH') -%}
+ {%- set ns.rest = messages[1:] -%}
+{%- endif -%}
+{%- set sys = namespace(parts=['You are a helpful AI assistant.', 'Knowledge cutoff: ' ~ cutoff_date ~ '.']) -%}
+{%- if current_date -%}
+ {%- set sys.parts = sys.parts + ['Current date: ' ~ current_date ~ '.'] -%}
+{%- endif -%}
+{%- if ns.instructions -%}
+ {%- set sys.parts = sys.parts + [ns.instructions] -%}
+{%- endif -%}
+{%- if tools -%}
+ {%- set sys.parts = sys.parts + [render_tool_defs(tools) | trim] -%}
+{%- endif -%}
+{%- if not disable_reasoning and reasoning_effort in reasoning_effort_map and 'reasoning strength:' not in ((ns.instructions if ns.instructions else '') | lower) -%}
+ {%- set sys.parts = sys.parts + ['Reasoning strength: ' ~ reasoning_effort_map[reasoning_effort] ~ '.'] -%}
+{%- endif -%}
+{%- set sys.parts = sys.parts + [render_valid_recipients(tools) | trim] -%}
+{{- '<|start|>system<|message|>' ~ sys.parts | join('\n\n') ~ '<|eot|>' -}}
+{%- for message in ns.rest -%}
+ {%- set role = message['role'] -%}
+ {%- set end_token = '<|eom|>' if (not loop.last and ns.rest[loop.index0 + 1]['role'] == role) else '<|eot|>' -%}
+ {%- if role in ['system', 'developer'] -%}
+ {{- '<|start|>' + role + '<|message|>' -}}
+ {{- render_content(message['content']) -}}
+ {{- '<|eot|>' -}}
+ {%- elif role == 'user' -%}
+ {{- '<|start|>user<|message|>' -}}
+ {%- if message.get('render_timestamp') and message.get('timestamp') -%}
+ {%- set ts = message['timestamp'] -%}
+ {%- if ts is string -%}
+ {{- '[' ~ esc_header(ts) ~ '] ' -}}
+ {%- elif ts.strftime is defined -%}
+ {{- '[' ~ (ts.strftime('%a %Y-%m-%d %H:%M %Z') | trim) ~ '] ' -}}
+ {%- endif -%}
+ {%- endif -%}
+ {{- render_content(message['content']) -}}
+ {{- '<|eot|>' -}}
+ {%- elif role == 'tool' -%}
+ {%- set tname = message.get('name') -%}
+ {%- if not tname -%}
+ {%- set tcid = message.get('tool_call_id') -%}
+ {%- set tns = namespace(name=tcid if tcid else '') -%}
+ {%- for m in messages -%}
+ {%- if m.get('tool_calls') -%}
+ {%- for tc in m['tool_calls'] -%}
+ {%- if tcid is not none and tc.id is defined and tc.id == tcid -%}
+ {%- set tns.name = tc.function.name -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- set tname = tns.name -%}
+ {%- endif -%}
+ {%- if not tname -%}
+ {{- raise_exception('tool message has no name and no matching tool_call_id') -}}
+ {%- endif -%}
+ {{- '<|start|>tool ' -}}{{- esc_header(tname) -}}{{- '<|message|><tool_output name="' -}}{{- esc_attr(tname) -}}{{- '">\n' -}}
+ {{- render_content(message['content']) -}}
+ {{- '\n</tool_output><|eot|>' -}}
+ {%- elif role == 'assistant' -%}
+ {%- set reasoning = message.get('reasoning_content') or message.get('reasoning') -%}
+ {%- if reasoning -%}
+ {{- '<|start|>assistant to=self<|message|>' -}}{{- esc(reasoning) -}}{{- '<|eom|>' -}}
+ {%- endif -%}
+ {%- set body = render_content(message.get('content')) -%}
+ {%- if message.get('tool_calls') -%}
+ {%- if body -%}
+ {{- '<|start|>assistant to=commentary<|message|>' + body + '<|eom|>' -}}
+ {%- endif -%}
+ {%- for tc in message['tool_calls'] -%}
+ {{- '<|start|>assistant to=' -}}{{- esc_header(tc.function.name) -}}{{- '<|message|>' -}}
+ {{- render_atem(tc) -}}
+ {{- (end_token if loop.last else '<|eom|>') -}}
+ {%- endfor -%}
+ {%- elif body or not reasoning -%}
+ {%- set recipient = message.get('recipient') or 'user' -%}
+ {%- if recipient == 'user' -%}
+ {%- set end_turn = true -%}
+ {%- elif recipient in ['self', 'commentary'] -%}
+ {%- set end_turn = false -%}
+ {%- elif message.get('end_turn') is not none -%}
+ {%- set end_turn = message.get('end_turn') -%}
+ {%- else -%}
+ {%- set end_turn = false -%}
+ {%- endif -%}
+ {{- '<|start|>assistant to=' -}}{{- esc_header(recipient) -}}{{- '<|message|>' -}}
+ {{- body -}}
+ {{- ('<|eot|>' if end_turn else '<|eom|>') -}}
+ {%- endif -%}
+ {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+ {{- '<|start|>assistant' -}}
+{%- endif -%}
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index d1dd4cbae..73c6d51d9 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -77,6 +77,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_OLMO2, "olmo2" },
{ LLM_ARCH_OLMOE, "olmoe" },
{ LLM_ARCH_MUSE_GLIMMER, "muse-glimmer" },
+ { LLM_ARCH_MUSE_SPARK, "muse-spark" },
{ LLM_ARCH_OPENELM, "openelm" },
{ LLM_ARCH_ARCTIC, "arctic" },
{ LLM_ARCH_DEEPSEEK, "deepseek" },
@@ -228,6 +229,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_EXPERTS_PER_GROUP, "%s.experts_per_group" },
{ LLM_KV_MOE_EVERY_N_LAYERS, "%s.moe_every_n_layers" },
{ LLM_KV_MOE_LATENT_SIZE, "%s.moe_latent_size" },
+ { LLM_KV_POST_NORM_EPS, "%s.post_norm_epsilon" },
{ LLM_KV_NEXTN_PREDICT_LAYERS, "%s.nextn_predict_layers" },
{ LLM_KV_NUM_DEEPSTACK_LAYERS, "%s.n_deepstack_layers" },
{ LLM_KV_DEEPSTACK_MAPPING, "%s.deepstack_mapping" },
@@ -500,6 +502,8 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" },
{ LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" },
{ LLM_TENSOR_ATTN_GATE, "blk.%d.attn_gate" },
+ { LLM_TENSOR_ATTN_RESID_ALPHA, "blk.%d.attn_resid_alpha" },
+ { LLM_TENSOR_ATTN_RESID_BETA, "blk.%d.attn_resid_beta" },
{ LLM_TENSOR_FFN_POST_NORM, "blk.%d.post_ffw_norm" },
{ LLM_TENSOR_FFN_POST_NORM_1, "blk.%d.post_ffw_norm_1" },
{ LLM_TENSOR_FFN_POST_NORM_2, "blk.%d.post_ffw_norm_2" },
@@ -511,6 +515,10 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_FFN_EXP_PROBS_B_VL, "blk.%d.exp_probs_b_vl" },
{ LLM_TENSOR_FFN_LATENT_DOWN, "blk.%d.ffn_latent_down" },
{ LLM_TENSOR_FFN_LATENT_UP, "blk.%d.ffn_latent_up" },
+ { LLM_TENSOR_FFN_PRE_EXP_NORM, "blk.%d.ffn_pre_exp_norm" },
+ { LLM_TENSOR_FFN_POST_EXP_NORM, "blk.%d.ffn_post_exp_norm" },
+ { LLM_TENSOR_FFN_RESID_ALPHA, "blk.%d.ffn_resid_alpha" },
+ { LLM_TENSOR_FFN_RESID_BETA, "blk.%d.ffn_resid_beta" },
{ LLM_TENSOR_ATTN_NORM_2, "blk.%d.attn_norm_2" },
{ LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
{ LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" },
@@ -839,6 +847,8 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_ATTN_QKV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_ATTN_OUT, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_ATTN_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ATTN_RESID_ALPHA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ATTN_RESID_BETA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_FFN_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_FFN_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_FFN_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
@@ -1083,6 +1093,10 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
// latent projections feed ggml_mul_mat, the buft probe must use MUL_MAT to keep them on GPU
{LLM_TENSOR_FFN_LATENT_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_FFN_LATENT_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_FFN_PRE_EXP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_FFN_POST_EXP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_FFN_RESID_ALPHA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_FFN_RESID_BETA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_MASKED_EMBD_CENTROIDS, {LLM_TENSOR_LAYER_INPUT, GGML_OP_NONE}},
{LLM_TENSOR_MASKED_EMBD_ORDERING, {LLM_TENSOR_LAYER_INPUT, GGML_OP_NONE}},
// eagle3
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 4e8e53421..4b607fec7 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -82,6 +82,7 @@ enum llm_arch {
LLM_ARCH_OLMO2,
LLM_ARCH_OLMOE,
LLM_ARCH_MUSE_GLIMMER,
+ LLM_ARCH_MUSE_SPARK,
LLM_ARCH_OPENELM,
LLM_ARCH_ARCTIC,
LLM_ARCH_DEEPSEEK,
@@ -233,6 +234,7 @@ enum llm_kv {
LLM_KV_EXPERTS_PER_GROUP,
LLM_KV_MOE_EVERY_N_LAYERS,
LLM_KV_MOE_LATENT_SIZE,
+ LLM_KV_POST_NORM_EPS,
LLM_KV_NEXTN_PREDICT_LAYERS,
LLM_KV_NUM_DEEPSTACK_LAYERS,
LLM_KV_DEEPSTACK_MAPPING,
@@ -500,6 +502,8 @@ enum llm_tensor {
LLM_TENSOR_ATTN_ROT_EMBD,
LLM_TENSOR_ATTN_SINKS,
LLM_TENSOR_ATTN_GATE,
+ LLM_TENSOR_ATTN_RESID_ALPHA,
+ LLM_TENSOR_ATTN_RESID_BETA,
LLM_TENSOR_FFN_GATE_INP,
LLM_TENSOR_FFN_GATE_INP_SHEXP,
LLM_TENSOR_FFN_NORM,
@@ -529,6 +533,10 @@ enum llm_tensor {
LLM_TENSOR_FFN_EXP_PROBS_B_VL,
LLM_TENSOR_FFN_LATENT_DOWN,
LLM_TENSOR_FFN_LATENT_UP,
+ LLM_TENSOR_FFN_PRE_EXP_NORM,
+ LLM_TENSOR_FFN_POST_EXP_NORM,
+ LLM_TENSOR_FFN_RESID_ALPHA,
+ LLM_TENSOR_FFN_RESID_BETA,
LLM_TENSOR_ATTN_Q_NORM,
LLM_TENSOR_ATTN_K_NORM,
LLM_TENSOR_LAYER_OUT_NORM,
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index 6cadda971..e8e249b21 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -2093,7 +2093,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
ggml_tensor * up_exps_s,
ggml_tensor * gate_exps_s,
ggml_tensor * down_exps_s,
- ggml_tensor * selected_experts_in) const {
+ ggml_tensor * selected_experts_in,
+ ggml_tensor * post_exp_norm,
+ float post_exp_norm_eps) const {
return build_moe_ffn(
cur,
gate_inp, /* gate_inp_b */ nullptr,
@@ -2114,7 +2116,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
up_exps_s,
gate_exps_s,
down_exps_s,
- selected_experts_in
+ selected_experts_in,
+ post_exp_norm,
+ post_exp_norm_eps
);
}
@@ -2142,7 +2146,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
ggml_tensor * up_exps_s,
ggml_tensor * gate_exps_s,
ggml_tensor * down_exps_s,
- ggml_tensor * selected_experts_in) const {
+ ggml_tensor * selected_experts_in,
+ ggml_tensor * post_exp_norm,
+ float post_exp_norm_eps) const {
const int64_t n_embd = cur->ne[0];
const int64_t n_tokens = cur->ne[1];
const bool weight_before_ffn = arch == LLM_ARCH_LLAMA4; // for llama4, we apply the sigmoid-ed weights before the FFN
@@ -2448,6 +2454,13 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
cb(experts, "ffn_moe_down_biased", il);
}
+ // an expert output can be scaled by a norm weight, then normalized, before the router weights apply
+ if (post_exp_norm) {
+ experts = ggml_mul(ctx0, experts, post_exp_norm);
+ experts = ggml_rms_norm(ctx0, experts, post_exp_norm_eps);
+ cb(experts, "ffn_moe_post_exp_norm", il);
+ }
+
if (!weight_before_ffn) {
experts = ggml_mul(ctx0, experts, weights);
cb(experts, "ffn_moe_weighted", il);
diff --git a/src/llama-graph.h b/src/llama-graph.h
index e853a0269..1a3698cad 100644
--- a/src/llama-graph.h
+++ b/src/llama-graph.h
@@ -1178,7 +1178,9 @@ struct llm_graph_context {
ggml_tensor * up_exps_s = nullptr,
ggml_tensor * gate_exps_s = nullptr,
ggml_tensor * down_exps_s = nullptr,
- ggml_tensor * selected_experts_in = nullptr) const;
+ ggml_tensor * selected_experts_in = nullptr,
+ ggml_tensor * post_exp_norm = nullptr,
+ float post_exp_norm_eps = 1e-5f) const;
ggml_tensor * build_moe_ffn(
ggml_tensor * cur,
@@ -1204,7 +1206,9 @@ struct llm_graph_context {
ggml_tensor * up_exps_s = nullptr,
ggml_tensor * gate_exps_s = nullptr,
ggml_tensor * down_exps_s = nullptr,
- ggml_tensor * selected_experts_in = nullptr) const;
+ ggml_tensor * selected_experts_in = nullptr,
+ ggml_tensor * post_exp_norm = nullptr,
+ float post_exp_norm_eps = 1e-5f) const;
// the slots of the selected experts in the MoE cache, nullptr if the experts of the layer are not read from the cache
ggml_tensor * build_moe_cache_slots(
diff --git a/src/llama-hparams.h b/src/llama-hparams.h
index 922164a54..bc0d49215 100644
--- a/src/llama-hparams.h
+++ b/src/llama-hparams.h
@@ -141,6 +141,9 @@ struct llama_hparams {
float f_norm_rms_eps;
float f_norm_group_eps;
+ // post norms use their own epsilon
+ float f_post_norm_eps = 0.0f;
+
float f_attn_logit_softcapping = 50.0f;
float f_router_logit_softcapping = 30.0f;
float f_final_logit_softcapping = 30.0f;
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index e011e24ea..46b1d2ae2 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -279,6 +279,7 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v_full);
add_kv(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ add_kv(LLM_KV_POST_NORM_EPS, hparams.f_post_norm_eps);
add_kv(LLM_KV_ATTENTION_GROUPNORM_EPS, hparams.f_norm_group_eps);
add_kv(LLM_KV_ATTENTION_GROUPNORM_GROUPS, hparams.n_norm_groups);
add_kv(LLM_KV_ATTENTION_VALUE_EXPERT_COUNT, hparams.n_value_expert);
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index ded790b2b..2660bf38a 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -188,6 +188,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_olmoe(params);
case LLM_ARCH_MUSE_GLIMMER:
return new llama_model_muse_glimmer(params);
+ case LLM_ARCH_MUSE_SPARK:
+ return new llama_model_muse_spark(params);
case LLM_ARCH_OPENELM:
return new llama_model_openelm(params);
case LLM_ARCH_GPTNEOX:
@@ -991,6 +993,7 @@ const char * llm_type_name(llm_type type) {
case LLM_TYPE_16x3_8B: return "16x3.8B";
case LLM_TYPE_10B_128x3_66B: return "10B+128x3.66B";
case LLM_TYPE_57B_A14B: return "57B.A14B";
+ case LLM_TYPE_816B_A42B: return "816B.A42B";
case LLM_TYPE_17B_16E: return "17Bx16E (Scout)";
case LLM_TYPE_17B_128E: return "17Bx128E (Maverick)";
case LLM_TYPE_A13B: return "A13B";
@@ -3492,6 +3495,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_DEEPSEEK32:
case LLM_ARCH_DEEPSEEK4:
case LLM_ARCH_MUSE_GLIMMER:
+ case LLM_ARCH_MUSE_SPARK:
case LLM_ARCH_PLM:
case LLM_ARCH_CHATGLM:
case LLM_ARCH_GRANITE:
diff --git a/src/llama-model.h b/src/llama-model.h
index bcfbf1e2c..66f9129e6 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -114,6 +114,7 @@ enum llm_type {
LLM_TYPE_16x3_8B,
LLM_TYPE_10B_128x3_66B,
LLM_TYPE_57B_A14B,
+ LLM_TYPE_816B_A42B,
LLM_TYPE_17B_16E, // llama4 Scout
LLM_TYPE_17B_128E, // llama4 Maverick
LLM_TYPE_A13B,
@@ -361,6 +362,16 @@ struct llama_layer {
struct ggml_tensor * ffn_latent_down = nullptr;
struct ggml_tensor * ffn_latent_up = nullptr;
+ // ff MoE norms around the experts, in latent space
+ struct ggml_tensor * ffn_pre_exp_norm = nullptr;
+ struct ggml_tensor * ffn_post_exp_norm = nullptr;
+
+ // norm preserving residual coefficients
+ struct ggml_tensor * attn_resid_alpha = nullptr;
+ struct ggml_tensor * attn_resid_beta = nullptr;
+ struct ggml_tensor * ffn_resid_alpha = nullptr;
+ struct ggml_tensor * ffn_resid_beta = nullptr;
+
// ff shared expert (shexp)
struct ggml_tensor * ffn_gate_inp_shexp = nullptr;
struct ggml_tensor * ffn_gate_shexp = nullptr;
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
index 9c6066796..b8f96bcef 100644
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -31,9 +31,11 @@ enum class tensor_category {
ATTENTION_QKV,
ATTENTION_KV_B,
ATTENTION_OUTPUT,
+ ATTENTION_GATE,
FFN_UP,
FFN_GATE,
FFN_DOWN,
+ FFN_LATENT,
OUTPUT,
OTHER
};
@@ -147,6 +149,12 @@ static tensor_category tensor_get_category(const std::string & tensor_name) {
if (tensor_name.find("ffn_gate") != std::string::npos) {
return tensor_category::FFN_GATE;
}
+ if (tensor_name.find("ffn_latent") != std::string::npos) {
+ return tensor_category::FFN_LATENT;
+ }
+ if (tensor_name.find("attn_gate.weight") != std::string::npos) {
+ return tensor_category::ATTENTION_GATE;
+ }
if (tensor_name.find("ffn_down") != std::string::npos) {
return tensor_category::FFN_DOWN;
}
@@ -490,6 +498,19 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type
}
}
+ // bumping to q8_0 is preferred since these tensors are not really big
+ // TODO: nemotron 3 super shares ffn_latent and afmoe, bailingmoe3, dots3note, hy-v4, kimi-k3 and laguna share attn_gate.
+ // Widening the guard is likely a win there too, but it changes their files, so it wants a perplexity comparison per architecture first.
+ if (arch == LLM_ARCH_MUSE_SPARK &&
+ (category == tensor_category::FFN_LATENT || category == tensor_category::ATTENTION_GATE)) {
+ auto bits_per_weight = [](ggml_type t) {
+ return 8.0 * ggml_type_size(t) / ggml_blck_size(t);
+ };
+ if (ggml_is_quantized(new_type) && bits_per_weight(new_type) < bits_per_weight(GGML_TYPE_Q8_0)) {
+ new_type = GGML_TYPE_Q8_0;
+ }
+ }
+
// for arches that share the same tensor between the token embeddings and the output, we quantize the token embeddings
// with the quantization of the output tensor
if (category == tensor_category::OUTPUT || (qs.has_tied_embeddings && category == tensor_category::TOKEN_EMBD)) {
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index 9b5cc2e1c..9fd5b9ef3 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -3029,6 +3029,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
if (false
|| t.first == "<|eot_id|>"
+ || t.first == "<|eot|>" // muse-spark
|| t.first == "<|im_end|>"
|| t.first == "<|end|>"
|| t.first == "<|return|>" // o200k_harmony
diff --git a/src/models/models.h b/src/models/models.h
index 88fd07d97..51fdcfbb0 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -1154,6 +1154,19 @@ struct llama_model_olmoe : public llama_model_base {
};
+struct llama_model_muse_spark : public llama_model_base {
+ llama_model_muse_spark(const struct llama_model_params & params) : llama_model_base(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ struct graph : public llm_graph_context {
+ graph(const llama_model & model, const llm_graph_params & params);
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
struct llama_model_muse_glimmer : public llama_model_base {
llama_model_muse_glimmer(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/src/models/muse-spark.cpp b/src/models/muse-spark.cpp
new file mode 100644
index 000000000..5842ecb98
--- /dev/null
+++ b/src/models/muse-spark.cpp
@@ -0,0 +1,238 @@
+#include "models.h"
+
+void llama_model_muse_spark::load_arch_hparams(llama_model_loader & ml) {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_POST_NORM_EPS, hparams.f_post_norm_eps);
+ ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+ ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa);
+ ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping);
+ ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);
+ ml.get_key(LLM_KV_MOE_LATENT_SIZE, hparams.moe_latent_size);
+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);
+ ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);
+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm);
+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);
+
+ hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
+ ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+
+ switch (hparams.n_layer()) {
+ case 62: type = LLM_TYPE_816B_A42B; break;
+ default: type = LLM_TYPE_UNKNOWN;
+ }
+}
+
+void llama_model_muse_spark::load_arch_tensors(llama_model_loader &) {
+ LLAMA_LOAD_LOCALS;
+
+ const int64_t n_embd_moe = hparams.moe_latent_size;
+ const int64_t n_ff_exp = hparams.n_ff_exp();
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0);
+
+ for (int i = 0; i < n_layer; ++i) {
+ auto & layer = layers[i];
+
+ // Pre-attention norm. All block norms hold a delta folded at conversion time.
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+
+ // Q/K/V/O projections.
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
+
+ // QK-norm. Weights are synthesized at conversion to absorb the metap scale.
+ layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
+ layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
+
+ // Attention output gate: sigmoid(gate) * attn_out before o_proj.
+ layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
+
+ // Norm preserving residual: alpha scales the stream, beta the branch.
+ layer.attn_resid_alpha = create_tensor(tn(LLM_TENSOR_ATTN_RESID_ALPHA, "weight", i), {n_embd}, 0);
+ layer.attn_resid_beta = create_tensor(tn(LLM_TENSOR_ATTN_RESID_BETA, "weight", i), {n_embd}, 0);
+
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+ layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0);
+ layer.ffn_resid_alpha = create_tensor(tn(LLM_TENSOR_FFN_RESID_ALPHA, "weight", i), {n_embd}, 0);
+ layer.ffn_resid_beta = create_tensor(tn(LLM_TENSOR_FFN_RESID_BETA, "weight", i), {n_embd}, 0);
+
+ // Every layer routes. The router reads the full stream, the experts run
+ // in a latent space entered by ffn_latent_down and left by ffn_latent_up.
+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
+ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);
+
+ layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", i), {n_embd, n_embd_moe}, 0);
+ layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", i), {n_embd_moe, n_embd}, 0);
+
+ layer.ffn_pre_exp_norm = create_tensor(tn(LLM_TENSOR_FFN_PRE_EXP_NORM, "weight", i), {n_embd_moe}, 0);
+ layer.ffn_post_exp_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_EXP_NORM, "weight", i), {n_embd_moe}, 0);
+
+ layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd_moe, n_ff_exp, n_expert}, 0);
+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd_moe, n_ff_exp, n_expert}, 0);
+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd_moe, n_expert}, 0);
+ }
+}
+
+llama_model_muse_spark::graph::graph(const llama_model & model, const llm_graph_params & params)
+ : llm_graph_context(params) {
+ const int64_t n_embd_head = hparams.n_embd_head_v();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+
+ ggml_tensor * cur;
+ ggml_tensor * inpL;
+
+ inpL = build_inp_embd(model.tok_embd);
+ inpL = build_norm(inpL, nullptr, nullptr, LLM_NORM_RMS, -1);
+ cb(inpL, "embd_norm", -1);
+
+ ggml_tensor * inp_pos = build_inp_pos();
+ auto * inp_attn = build_attn_inp_kv_iswa();
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ const float kq_scale = 1.0f / sqrtf(float(n_embd_head));
+
+ for (int il = 0; il < n_layer; ++il) {
+ res->t_layer_inp[il] = inpL;
+
+ const float freq_base_l = model.get_rope_freq_base (cparams, il);
+ const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
+
+ ggml_tensor * inpSA = inpL;
+
+ // RoPE runs on the SWA layers, NoPE on the global ones.
+ const bool use_rope = hparams.is_swa(il);
+
+ cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
+ cb(cur, "attn_norm", il);
+
+ {
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
+
+ ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, cur);
+ cb(gate, "attn_gate_proj", il);
+
+ // attn_q_norm carries the metap scale, attn_k_norm is identity.
+ Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
+ Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
+ cb(Qcur, "Qcur_normed", il);
+ cb(Kcur, "Kcur_normed", il);
+
+ if (use_rope) {
+ Qcur = ggml_rope_ext(
+ ctx0, Qcur, inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(Qcur, "Qcur_rope", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, Kcur, inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(Kcur, "Kcur_rope", il);
+ }
+
+ cur = build_attn(inp_attn,
+ NULL, NULL, NULL,
+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
+ cb(cur, "attn_out", il);
+
+ // Weightless norm on the attention output, applied per head.
+ cur = ggml_reshape_3d(ctx0, cur, n_embd_head, n_head, n_tokens);
+ cur = ggml_rms_norm(ctx0, cur, hparams.f_norm_rms_eps);
+ cur = ggml_reshape_2d(ctx0, cur, n_embd_head * n_head, n_tokens);
+ cb(cur, "attn_o_norm", il);
+
+ gate = ggml_sigmoid(ctx0, gate);
+ cb(gate, "attn_gate_sig", il);
+ cur = ggml_mul(ctx0, cur, gate);
+ cb(cur, "attn_gated", il);
+
+ cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);
+ cb(cur, "attn_o_proj", il);
+ }
+
+ // Post-attention norm holds no weight.
+ cur = ggml_rms_norm(ctx0, cur, hparams.f_post_norm_eps);
+ cb(cur, "attn_post_norm", il);
+
+ if (il == n_layer - 1 && inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ ggml_tensor * ffn_inp = ggml_add(ctx0,
+ ggml_mul(ctx0, inpSA, model.layers[il].attn_resid_alpha),
+ ggml_mul(ctx0, cur, model.layers[il].attn_resid_beta));
+ cb(ffn_inp, "ffn_inp", il);
+
+ cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);
+ cb(cur, "ffn_norm", il);
+
+ // Enter the latent space, run the experts, leave it.
+ ggml_tensor * moe_inp = build_lora_mm(model.layers[il].ffn_latent_down, cur);
+ cb(moe_inp, "ffn_latent_down", il);
+
+ moe_inp = build_norm(moe_inp, model.layers[il].ffn_pre_exp_norm, NULL, LLM_NORM_RMS, il);
+ cb(moe_inp, "ffn_pre_exp_norm", il);
+
+ // The router reads the full stream, not the latent one.
+ ggml_tensor * router_logits = build_lora_mm(model.layers[il].ffn_gate_inp, cur);
+ cb(router_logits, "ffn_moe_logits", il);
+
+ ggml_tensor * moe_out = build_moe_ffn(moe_inp,
+ model.layers[il].ffn_gate_inp,
+ model.layers[il].ffn_up_exps,
+ model.layers[il].ffn_gate_exps,
+ model.layers[il].ffn_down_exps,
+ model.layers[il].ffn_exp_probs_b,
+ n_expert, n_expert_used,
+ LLM_FFN_SILU, hparams.expert_weights_norm, hparams.expert_weights_scale,
+ (llama_expert_gating_func_type) hparams.expert_gating_func,
+ il,
+ router_logits,
+ nullptr, nullptr, nullptr, nullptr, nullptr,
+ model.layers[il].ffn_post_exp_norm, hparams.f_post_norm_eps);
+ cb(moe_out, "ffn_moe_out", il);
+
+ cur = build_lora_mm(model.layers[il].ffn_latent_up, moe_out);
+ cb(cur, "ffn_latent_up", il);
+
+ // Post-FFN norm scales the input, then normalizes.
+ cur = ggml_mul(ctx0, cur, model.layers[il].ffn_post_norm);
+ cur = ggml_rms_norm(ctx0, cur, hparams.f_post_norm_eps);
+ cb(cur, "ffn_post_norm", il);
+
+ cur = ggml_add(ctx0,
+ ggml_mul(ctx0, ffn_inp, model.layers[il].ffn_resid_alpha),
+ ggml_mul(ctx0, cur, model.layers[il].ffn_resid_beta));
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ cur = build_lora_mm(model.output, cur, model.output_s);
+ cur = ggml_scale(ctx0, cur, hparams.f_logit_scale);
+
+ if (hparams.f_final_logit_softcapping) {
+ cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping);
+ cur = ggml_tanh(ctx0, cur);
+ cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping);
+ }
+
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+}
+
+std::unique_ptr<llm_graph_context> llama_model_muse_spark::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(*this, params);
+}
diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp
index 01b2220da..49519eeca 100644
--- a/tests/test-chat.cpp
+++ b/tests/test-chat.cpp
@@ -6661,6 +6661,50 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
.run();
}
+ // Muse Spark format tests
+ {
+ auto tst = peg_tester("models/templates/muse-spark.jinja", detailed_debug);
+
+ const std::string call_markup =
+ "<atem:function_calls>\n"
+ "<atem:invoke name=\"special_function\">\n"
+ "<atem:parameter name=\"arg1\">1</atem:parameter>\n"
+ "</atem:invoke>\n"
+ "</atem:function_calls>";
+
+ // A plain answer behaves as it does for Muse Glimmer
+ tst.test(" to=user<|message|>Hello, world!\nWhat's up?<|eot|>")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .expect(message_assist)
+ .run();
+
+ // A commentary message is user facing content, not reasoning
+ tst.test(" to=commentary<|message|>Hello, world!\nWhat's up?<|eom|>"
+ "<|start|>assistant to=user<|message|>Done!<|eot|>")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .expect_content("Hello, world!\nWhat's up?Done!")
+ .run();
+
+ // Commentary announces the work, then the tool call follows in the same generation
+ tst.test(" to=commentary<|message|>Let me look that up.<|eom|>"
+ "<|start|>assistant to=special_function<|message|>" +
+ call_markup)
+ .tools({ special_function_tool })
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .expect(message_with_content_and_tool_call("Let me look that up.", "special_function",
+ "{\"arg1\":1}"))
+ .run();
+
+ // Reasoning still reaches the reasoning channel with commentary in the picture
+ tst.test(" to=self<|message|>Thinking.<|eom|>"
+ "<|start|>assistant to=commentary<|message|>Working on it.<|eom|>"
+ "<|start|>assistant to=user<|message|>Hello!<|eot|>")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .expect_reasoning("Thinking.")
+ .expect_content("Working on it.Hello!")
+ .run();
+ }
+
// GPT-OSS format tests
{
auto tst = peg_tester("models/templates/openai-gpt-oss-120b.jinja", detailed_debug);
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 3dd8f0655..2bfa1cf75 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -185,7 +185,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
uint32_t n_head_kv = n_head;
if (arch == LLM_ARCH_QWEN3) {
n_head_kv = 1; // MQA coverage
- } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER) {
+ } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER) {
n_head_kv = 2; // GQA coverage
}
const uint32_t n_embd_head = n_embd / n_head;
@@ -303,7 +303,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
// SWA pattern: every 5th layer is full attention (matches E2B layer_types)
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5));
} else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 ||
- arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE ||
+ arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE ||
arch == LLM_ARCH_MAPLE) {
std::vector<uint32_t> pattern;
pattern.reserve(n_layer);
@@ -315,6 +315,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(2));
}
+ if (arch == LLM_ARCH_MUSE_SPARK) {
+ ms.add_kv(LLM_KV_POST_NORM_EPS, 1e-5f);
+ ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);
+ ms.add_kv(LLM_KV_FINAL_LOGIT_SOFTCAPPING, 30.0f);
+ ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd/2);
+ ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
+ ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
+ }
+
// MSA requires one indexer head per GQA (KV) head, unlike the DSA archs where the
// indexer head count is independent of the main attention head count.
if (arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_GLM5_NEXT) {
@@ -715,6 +724,7 @@ static bool moe_mandatory(const llm_arch arch) {
case LLM_ARCH_MELLUM:
case LLM_ARCH_LAGUNA:
case LLM_ARCH_MAPLE:
+ case LLM_ARCH_MUSE_SPARK:
return true;
default:
return false;
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index 82f74bdc9..edb5d3d89 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -48,6 +48,7 @@ add_library(mtmd
models/nemotron-v2-vl.cpp
models/nemotron-omni.cpp
models/muse-glimmer.cpp
+ models/muse-spark-audio.cpp
models/llama4.cpp
models/llava.cpp
models/minicpmv.cpp
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 3d6c6f8c4..ad291a557 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -85,6 +85,9 @@
#define KEY_A_PROJ_STACK_FACTOR "clip.audio.projector.stack_factor"
#define KEY_A_CHUNK_SIZE "clip.audio.chunk_size"
#define KEY_A_CONV_KERNEL_SIZE "clip.audio.conv_kernel_size"
+#define KEY_A_HIDDEN_SIZES "clip.audio.hidden_sizes"
+#define KEY_A_PATCH_SIZES "clip.audio.patch_sizes"
+#define KEY_A_DEPTHS "clip.audio.depths"
#define KEY_A_MAX_POS_EMB "clip.audio.max_pos_emb"
#define KEY_A_PROJ_WINDOW_SIZE "clip.audio.projector.window_size"
#define KEY_A_PROJ_DOWNSAMPLE_RATE "clip.audio.projector.downsample_rate"
@@ -369,6 +372,8 @@
#define TN_A_INP_PROJ "a.input_projection.%s"
#define TN_A_CONV1D "a.conv1d.%d.%s"
#define TN_A_CONV1D_NORM "a.conv1d.%d.norm.%s"
+#define TN_A_PATCH "a.patch.%d.%s"
+#define TN_A_PATCH_NORM "a.patch.%d.norm.%s"
#define TN_A_OUT_PROJ "a.pre_encode.out.%s"
#define TN_A_ATTN_PRE_NORM "%s.blk.%d.attn_pre_norm.%s"
#define TN_A_ATTN_POST_NORM "%s.blk.%d.attn_post_norm.%s"
@@ -515,6 +520,7 @@ enum projector_type {
PROJECTOR_TYPE_POCKETTTS_GEN,
PROJECTOR_TYPE_MUSE_GLIMMER,
PROJECTOR_TYPE_COHERE2V,
+ PROJECTOR_TYPE_MUSE_SPARK_AUDIO,
PROJECTOR_TYPE_UNKNOWN,
};
@@ -587,6 +593,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_POCKETTTS_GEN, "pockettts_gen"},
{ PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"},
{ PROJECTOR_TYPE_COHERE2V, "cohere2v"},
+ { PROJECTOR_TYPE_MUSE_SPARK_AUDIO, "muse-spark-audio"},
};
static projector_type clip_projector_type_from_string(const std::string & str) {
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h
index 2c4ea398c..92ffe5e54 100644
--- a/tools/mtmd/clip-model.h
+++ b/tools/mtmd/clip-model.h
@@ -134,6 +134,20 @@ struct clip_hparams {
int32_t audio_proj_downsample_rate = 0;
int32_t audio_proj_head_count = 0;
+ // audio pyramid (muse-spark): widths, folded samples and blocks per stage
+ std::vector<int32_t> audio_hidden_sizes;
+ std::vector<int32_t> audio_patch_sizes;
+ std::vector<int32_t> audio_depths;
+
+ int32_t audio_samples_per_token() const {
+ int64_t n = 1;
+ for (int32_t p : audio_patch_sizes) {
+ n *= p;
+ GGML_ASSERT(n >= 1 && n <= (1 << 20));
+ }
+ return (int32_t) n;
+ }
+
// audio-to-mel preprocessor params
int32_t audio_chunk_len = -1; // in seconds
int32_t audio_sample_rate = -1;
@@ -283,6 +297,10 @@ struct clip_layer {
ggml_tensor * ff_post_norm_w = nullptr;
+ // depthwise causal conv block (muse-spark audio), conv_norm_w is declared below
+ ggml_tensor * conv_w = nullptr;
+ ggml_tensor * conv_b = nullptr;
+
// layer scale (no bias)
ggml_tensor * ls_1_w = nullptr;
ggml_tensor * ls_2_w = nullptr;
@@ -585,6 +603,14 @@ struct clip_model {
std::vector<clip_layer> layers;
+ // one patch embedder per audio pyramid stage (muse-spark)
+ struct audio_patch {
+ ggml_tensor * proj_w = nullptr;
+ ggml_tensor * proj_b = nullptr;
+ ggml_tensor * norm_w = nullptr;
+ };
+ std::vector<audio_patch> a_patches;
+
int32_t n_deepstack_layers = 0; // used by Qwen3-VL, calculated from clip_layer
ggml_tensor * post_ln_w;
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 4c1185a43..336c266b5 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -1059,6 +1059,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_whisper_enc>(ctx, img);
} break;
+ case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
+ {
+ builder = std::make_unique<clip_graph_muse_spark_audio>(ctx, img);
+ } break;
case PROJECTOR_TYPE_KIMIVL:
{
builder = std::make_unique<clip_graph_kimivl>(ctx, img);
@@ -1775,6 +1779,33 @@ struct clip_model_loader {
hparams.set_limit_image_tokens(8, 576);
hparams.set_warmup_n_tokens(16*16);
} break;
+ case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
+ {
+ hparams.audio_sample_rate = 24000; // mono waveform, no frontend
+ get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size, true);
+ get_arr_int(KEY_A_HIDDEN_SIZES, hparams.audio_hidden_sizes, true);
+ get_arr_int(KEY_A_PATCH_SIZES, hparams.audio_patch_sizes, true);
+ get_arr_int(KEY_A_DEPTHS, hparams.audio_depths, true);
+
+ // the graph indexes the three arrays by stage and model.layers by the running depth sum
+ const size_t n_stage = hparams.audio_patch_sizes.size();
+ GGML_ASSERT(n_stage > 0);
+ GGML_ASSERT(hparams.audio_hidden_sizes.size() == n_stage);
+ GGML_ASSERT(hparams.audio_depths.size() == n_stage);
+ GGML_ASSERT(hparams.n_mel_bins == 1);
+ GGML_ASSERT(hparams.audio_conv_kernel_size >= 1);
+
+ int64_t n_blocks = 0;
+ for (size_t i = 0; i < n_stage; ++i) {
+ GGML_ASSERT(hparams.audio_hidden_sizes[i] >= 1);
+ GGML_ASSERT(hparams.audio_depths[i] >= 0);
+ n_blocks += hparams.audio_depths[i];
+ }
+ if (n_blocks != hparams.n_layer) {
+ throw std::runtime_error(string_format("%s: sum of audio depths (%" PRId64 ") does not match n_layer (%d)\n", __func__, n_blocks, hparams.n_layer));
+ }
+ hparams.warmup_audio_size = hparams.audio_samples_per_token();
+ } break;
case PROJECTOR_TYPE_MUSE_GLIMMER:
{
hparams.n_merge = 2; // pixel-shuffle downsample after the ViT
@@ -2187,7 +2218,9 @@ struct clip_model_loader {
// GEMMA4UA is encoder-free: it uses n_mel_bins as a raw-waveform frame size (640) and has no FFT/filterbank, so the mel-range and FFT
// checks below do not apply to it.
// pocket-tts is encoder-free in the same sense: mimi convolves the raw waveform
+ // Muse Spark is encoder-free in the same sense: its conv pyramid reads the raw waveform, one channel takes the place of the mel bins
const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA &&
+ model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC;
// Validate audio hparams loaded from GGUF metadata
@@ -2348,7 +2381,8 @@ struct clip_model_loader {
const bool has_standard_layers = (
model.proj_type != PROJECTOR_TYPE_GEMMA3NV &&
model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC &&
- model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN);
+ model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN &&
+ model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO);
// layers
const int n_layers_to_load = has_standard_layers ? hparams.n_layer : 0;
@@ -2619,6 +2653,38 @@ struct clip_model_loader {
model.mm_merger_fc2_w = get_tensor(string_format(TN_MM_MERGER_FC2, "weight"));
model.mm_merger_fc2_b = get_tensor(string_format(TN_MM_MERGER_FC2, "bias"));
} break;
+ case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
+ {
+ model.layers.resize(hparams.n_layer);
+ model.a_patches.resize(hparams.audio_patch_sizes.size());
+ for (size_t i = 0; i < model.a_patches.size(); ++i) {
+ auto & pe = model.a_patches[i];
+ pe.proj_w = get_tensor(string_format(TN_A_PATCH, (int) i, "weight"));
+ pe.proj_b = get_tensor(string_format(TN_A_PATCH, (int) i, "bias"));
+ pe.norm_w = get_tensor(string_format(TN_A_PATCH_NORM, (int) i, "weight"));
+ if (pe.proj_w->ne[1] != hparams.audio_hidden_sizes[i]) {
+ throw std::runtime_error(string_format("%s: audio stage %zu projection has width %" PRId64 ", expected %d\n", __func__, i, pe.proj_w->ne[1], hparams.audio_hidden_sizes[i]));
+ }
+ }
+
+ for (int il = 0; il < hparams.n_layer; ++il) {
+ auto & layer = model.layers[il];
+ layer.conv_w = get_tensor(string_format(TN_CONV1D, il, "weight"));
+ layer.conv_b = get_tensor(string_format(TN_CONV1D, il, "bias"));
+ layer.conv_norm_w = get_tensor(string_format(TN_A_CONV1D_NORM, il, "weight"));
+ layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, "a", il, "weight"));
+ layer.ff_up_b = get_tensor(string_format(TN_FFN_UP, "a", il, "bias"));
+ layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, "a", il, "weight"));
+ layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, "a", il, "bias"));
+ }
+
+ // pre_ln and post_ln come from the generic path above, where they are optional
+ if (!model.pre_ln_w || !model.post_ln_w) {
+ throw std::runtime_error(string_format("%s: missing pre_ln or post_ln weight\n", __func__));
+ }
+ model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight"));
+ model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias"));
+ } break;
case PROJECTOR_TYPE_MUSE_GLIMMER:
{
// 3-linear MLP: fc -> erf-GELU -> proj -> erf-GELU -> vision_proj (into LLM residual dim)
@@ -4393,6 +4459,11 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
n_patches = n_patches_y * n_patches_x;
}
} break;
+ case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
+ {
+ const int64_t fold = ctx->model.hparams.audio_samples_per_token();
+ n_patches = (int) ((img->nx() + fold - 1) / fold);
+ } break;
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_QWEN2A:
@@ -5459,6 +5530,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
case PROJECTOR_TYPE_COGVLM:
case PROJECTOR_TYPE_YASA2:
case PROJECTOR_TYPE_GEMMA4UA:
+ case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
// do nothing
@@ -6157,6 +6229,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_GEMMA4UA:
return ctx->model.mm_input_proj_w->ne[1];
case PROJECTOR_TYPE_IDEFICS3:
+ case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
return ctx->model.mm_fc_w->ne[1];
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_VOXTRAL:
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 2f4676a54..7d7f5ac1d 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -178,6 +178,11 @@ struct clip_graph_granite_speech : clip_graph {
ggml_cgraph * build() override;
};
+struct clip_graph_muse_spark_audio : clip_graph {
+ clip_graph_muse_spark_audio(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+ ggml_cgraph * build() override;
+};
+
struct clip_graph_gemma4a : clip_graph {
clip_graph_gemma4a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/models/muse-spark-audio.cpp b/tools/mtmd/models/muse-spark-audio.cpp
new file mode 100644
index 000000000..d5340d5f9
--- /dev/null
+++ b/tools/mtmd/models/muse-spark-audio.cpp
@@ -0,0 +1,66 @@
+#include "models.h"
+
+// Muse Spark audio tower: a causal conv pyramid over the raw waveform, no attention and no spectrogram
+// each stage folds `patch` samples into the channel dim, projects, then runs `depth` residual blocks (depthwise causal conv, RMS norm, GELU MLP)
+ggml_cgraph * clip_graph_muse_spark_audio::build() {
+ const int kernel = hparams.audio_conv_kernel_size;
+ const int n_stage = (int) hparams.audio_patch_sizes.size();
+
+ // shape [1, n_samples], so the first stage folds the same way as the others
+ ggml_tensor * cur = build_inp_raw(1);
+ cur = ggml_reshape_2d(ctx0, cur, 1, img.nx());
+ cb(cur, "inp_raw", -1);
+
+ int bid = 0;
+ for (int stage = 0; stage < n_stage; ++stage) {
+ const int patch = hparams.audio_patch_sizes[stage];
+ const auto & pe = model.a_patches[stage];
+
+ // fold `patch` positions into the channel dim
+ cur = ggml_reshape_2d(ctx0, cur, cur->ne[0] * patch, cur->ne[1] / patch);
+ cb(cur, "patch_folded", stage);
+
+ GGML_ASSERT(pe.proj_w->ne[1] == hparams.audio_hidden_sizes[stage]);
+ cur = ggml_mul_mat(ctx0, pe.proj_w, cur);
+ cur = ggml_add(ctx0, cur, pe.proj_b);
+ cur = build_norm(cur, pe.norm_w, nullptr, NORM_TYPE_RMS, hparams.eps, stage);
+ cb(cur, "patch_embd", stage);
+
+ for (int i = 0; i < hparams.audio_depths[stage]; ++i, ++bid) {
+ const auto & layer = model.layers[bid];
+ ggml_tensor * residual = cur;
+
+ // depthwise conv over time, pad on the left so it is causal
+ ggml_tensor * conv = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+ conv = ggml_pad_ext(ctx0, conv, kernel - 1, 0, 0, 0, 0, 0, 0, 0);
+ conv = ggml_conv_1d_dw(ctx0, layer.conv_w, conv, 1, 0, 1);
+ conv = ggml_cont(ctx0, ggml_transpose(ctx0, conv));
+ conv = ggml_add(ctx0, conv, layer.conv_b);
+ cb(conv, "conv", bid);
+
+ cur = build_norm(conv, layer.conv_norm_w, nullptr, NORM_TYPE_RMS, hparams.eps, bid);
+
+ cur = build_ffn(cur,
+ layer.ff_up_w, layer.ff_up_b,
+ nullptr, nullptr,
+ layer.ff_down_w, layer.ff_down_b,
+ FFN_GELU_ERF, bid);
+ cb(cur, "ffn_out", bid);
+
+ cur = ggml_add(ctx0, cur, residual);
+ cb(cur, "block_out", bid);
+ }
+ }
+
+ cur = build_norm(cur, model.pre_ln_w, nullptr, NORM_TYPE_RMS, hparams.eps, -1);
+ cb(cur, "enc_norm", -1);
+
+ // straight into the text hidden size, no adapter
+ cur = ggml_mul_mat(ctx0, model.mm_fc_w, cur);
+ cur = ggml_add(ctx0, cur, model.mm_fc_b);
+ cur = build_norm(cur, model.post_ln_w, nullptr, NORM_TYPE_RMS, hparams.eps, -1);
+ cb(cur, "projected", -1);
+
+ ggml_build_forward_expand(gf, cur);
+ return gf;
+}
diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp
index 1b4ea60bf..bbeb57c80 100644
--- a/tools/mtmd/mtmd-audio.cpp
+++ b/tools/mtmd/mtmd-audio.cpp
@@ -1596,3 +1596,23 @@ bool mtmd_audio_preprocessor_pockettts::preprocess(const float *
output.push_back(std::move(out));
return true;
}
+
+// Muse Spark reads the raw waveform: pass the samples through, zero padded to a whole number of embeddings
+bool mtmd_audio_preprocessor_muse_spark_audio::preprocess(
+ const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output) const {
+ const int64_t fold = hparams.audio_samples_per_token();
+ const int64_t n_padded = ((int64_t) n_samples + fold - 1) / fold * fold;
+
+ mtmd_audio_mel mel;
+ mel.n_len = n_padded;
+ mel.n_len_org = (int64_t) n_samples;
+ mel.n_mel = 1;
+ mel.data.assign((size_t) n_padded, 0.0f);
+ std::copy(samples, samples + n_samples, mel.data.begin());
+
+ output.clear();
+ output.push_back(std::move(mel));
+ return true;
+}
diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h
index 4ca6cbd55..3b66c2758 100644
--- a/tools/mtmd/mtmd-audio.h
+++ b/tools/mtmd/mtmd-audio.h
@@ -70,6 +70,12 @@ struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
mtmd_audio_cache cache;
};
+struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor {
+ mtmd_audio_preprocessor_muse_spark_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
+ void initialize() override {}
+ bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+};
+
struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
mtmd_audio_preprocessor_conformer(const clip_hparams & hparams) : mtmd_audio_preprocessor(hparams) {}
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 2fa02f83e..343d29571 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -1009,6 +1009,12 @@ struct mtmd_context {
aud_end = "<|audio_eos|>";
audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(ctx_a);
} break;
+ case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
+ {
+ aud_beg = "<|soft_audio_start|>";
+ aud_end = "<|soft_audio_end|>";
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_muse_spark_audio>(ctx_a);
+ } break;
case PROJECTOR_TYPE_QWEN3A:
{
aud_beg = "<|audio_start|>";