Commit d92ea9bb1 for llama.cpp
commit d92ea9bb126ab957849dd0c0f1f881a64de7dc86
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Tue Nov 17 12:53:19 2026 +0100
mtmd: support Minimax-Music3 (#32435)
* mtmd: support Minimax-Music3
* fix and clean up
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 497797c06..0105099eb 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -285,6 +285,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"PocketTTSModel": "pockettts",
"Zonos3Model": "zonos3",
"YuE2ForCausalLM": "yue2",
+ "MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
"Qwen3TTSForConditionalGeneration": "qwen3tts",
"Qwen3VLForConditionalGeneration": "qwen3vl",
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
@@ -363,6 +364,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"FishOmniModel": "drama3",
"Zonos3Model": "zonos3",
"YuE2ForCausalLM": "yue2",
+ "MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
"Gemma3ForConditionalGeneration": "gemma",
"Gemma3nForConditionalGeneration": "gemma",
"Gemma4ForConditionalGeneration": "gemma",
diff --git a/conversion/minimax_music3.py b/conversion/minimax_music3.py
new file mode 100644
index 000000000..2fbba8fab
--- /dev/null
+++ b/conversion/minimax_music3.py
@@ -0,0 +1,285 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Callable, Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import ModelBase, MmprojModel, gguf
+from .drama3 import fold_snake
+from .qwen import Qwen3Model
+
+# MiniMax Music 3, diffusers layout: one checkpoint per component
+# text model: the global LLM (language_model/), samples the semantic code of each frame
+# mmproj: RVQ depth decoder, condition encoder, flow-matching DiT and DAC decoder (vocoder)
+
+LM_DIR = "language_model"
+TOK_DIR = "tokenizer"
+MMPROJ_DIRS = ("rvq_depth_decoder", "condition_encoder", "transformer", "vocoder")
+
+# token ids of the checkpoint, see encoders.py of the diffusers pipeline
+AUDIO_START = 151669
+AUDIO_END = 151670
+AUDIO_CODE_OFFSET = 151675
+N_SEMANTIC = 16384
+
+
+def _load_hparams(dir_model: Path) -> dict[str, Any]:
+ return ModelBase.load_hparams(dir_model / LM_DIR, is_mistral_format=False)
+
+
+def _index_dirs(model: ModelBase, dirs: Iterable[str], keep: Callable[[str], bool]) -> dict[str, Callable[[], Tensor]]:
+ # the names get the component dir as prefix
+ root = model.dir_model
+ tensors: dict[str, Callable[[], Tensor]] = {}
+ try:
+ for d in dirs:
+ model.dir_model = root / d
+ prefix = "model" if d == LM_DIR else "diffusion_pytorch_model"
+ for name, gen in model.index_local_tensors(prefix).items():
+ if keep(f"{d}.{name}"):
+ tensors[f"{d}.{name}"] = gen
+ finally:
+ model.dir_model = root
+ return tensors
+
+
+def _fold_wn(g: Tensor, v: Tensor) -> Tensor:
+ # torch weight norm, over every dim but 0
+ g, v = g.float(), v.float()
+ return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
+
+
+@ModelBase.register("MiniMaxMusic3ForConditionalGeneration")
+class MiniMaxMusic3TextModel(Qwen3Model):
+ model_arch = gguf.MODEL_ARCH.QWEN3
+
+ def __init__(self, dir_model: Path, *args, **kwargs):
+ kwargs.pop("hparams", None)
+ super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
+
+ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
+ tensors = _index_dirs(self, [LM_DIR], lambda _: True)
+ return {name.removeprefix(LM_DIR + "."): gen for name, gen in tensors.items()}
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ # the codes are sampled with classifier-free guidance, the uncond prompt runs as a second sequence
+ self.gguf_writer.add_classifier_free_guidance(True)
+
+ def set_vocab(self):
+ root = self.dir_model
+ self.dir_model = root / TOK_DIR
+ try:
+ tokens, toktypes, tokpre = self.get_vocab_base()
+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
+ finally:
+ self.dir_model = root
+
+ # the semantic codes are not in the tokenizer, name them for debugging
+ for i in range(N_SEMANTIC):
+ tokens[AUDIO_CODE_OFFSET + i] = f"<|audio_code_{i}|>"
+ toktypes[AUDIO_CODE_OFFSET + i] = gguf.TokenType.CONTROL
+
+ # the generation ends with <|audio_end|>, the chat template of the tokenizer is not used
+ template_path = Path(__file__).parent.parent / "models" / "templates" / "MiniMaxAI-MiniMax-Music3-tts.jinja"
+ with open(template_path, encoding="utf-8") as f:
+ special_vocab.chat_template = [{"name": "tts", "template": f.read()}]
+ special_vocab.special_token_ids["eos"] = AUDIO_END
+
+ self.gguf_writer.add_tokenizer_model("gpt2")
+ self.gguf_writer.add_tokenizer_pre(tokpre)
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_types(toktypes)
+ special_vocab.add_to_gguf(self.gguf_writer)
+
+
+@ModelBase.register("MiniMaxMusic3ForConditionalGeneration")
+class MiniMaxMusic3MmprojModel(MmprojModel):
+ has_vision_encoder = False
+ has_audio_encoder = False
+
+ def __init__(self, dir_model: Path, *args, **kwargs):
+ kwargs.pop("hparams", None)
+ hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
+ hparams["text_config"] = _load_hparams(dir_model)
+ for d in MMPROJ_DIRS:
+ hparams[d] = ModelBase.load_hparams(dir_model / d, is_mistral_format=False)
+ super().__init__(dir_model, *args, hparams=hparams, **kwargs)
+
+ def get_audio_config(self) -> dict[str, Any] | None:
+ # the generic gen layers are the depth decoder
+ return self.global_config["rvq_depth_decoder"]
+
+ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
+ # from the backbone, only the embeddings of the semantic codes
+ return _index_dirs(self, (LM_DIR, *MMPROJ_DIRS),
+ lambda name: not name.startswith(LM_DIR + ".") or name == f"{LM_DIR}.model.embed_tokens.weight")
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ return item
+
+ def set_gguf_parameters(self):
+ hp = self.hparams
+ self.gguf_writer.add_file_type(self.ftype)
+ self.gguf_writer.add_clip_has_gen_audio_encoder(True)
+ self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.MINIMAX_MUSIC3_GEN)
+ self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
+ self.gguf_writer.add_gen_audio_embedding_length(hp["hidden_size"])
+ self.gguf_writer.add_gen_audio_feed_forward_length(hp["intermediate_size"])
+ self.gguf_writer.add_gen_audio_block_count(hp["num_layers"])
+ self.gguf_writer.add_gen_audio_head_count(hp["num_attention_heads"])
+ self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-6)
+ # the DiT, the condition encoder and the vocoder hparams come from the tensor shapes or are hard-coded in tools/mtmd
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ # only the transformer layers follow --outtype, the convs and the small projections stay F32
+ if n_dims >= 2 and new_name.startswith(("a.gen.code.blk.", "a.gen.flow.blk.")):
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+ if new_name.startswith(("a.gen.code.head", "a.gen.code.embd", "a.gen.code.out_embd", "a.gen.code.proj_in")):
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+ return gguf.GGMLQuantizationType.F32
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name.startswith("a."):
+ yield (name, data_torch)
+ return
+
+ comp, key = name.split(".", 1)
+ if comp == LM_DIR:
+ yield ("a.gen.code.out_embd.weight", data_torch[AUDIO_CODE_OFFSET:AUDIO_CODE_OFFSET + N_SEMANTIC])
+ elif comp == "rvq_depth_decoder":
+ yield from self._depth(data_torch, key)
+ elif comp == "condition_encoder":
+ yield from self._cond(data_torch, key)
+ elif comp == "transformer":
+ yield from self._dit(data_torch, key)
+ # the vocoder is folded in generate_extra_tensors()
+
+ def _depth(self, t: Tensor, key: str) -> Iterable[tuple[str, Tensor]]:
+ p = "a.gen.code"
+ n_cb = self.hparams["num_codebooks"] - 1
+ if key == "audio_embeddings.weight":
+ # one table per residual codebook
+ yield (f"{p}.embd.weight", t.reshape(n_cb, -1, t.shape[-1]))
+ elif key.startswith("audio_heads."):
+ if key == "audio_heads.0.weight":
+ heads = [self._tensor(f"rvq_depth_decoder.audio_heads.{i}.weight") for i in range(n_cb)]
+ yield (f"{p}.head.weight", torch.stack(heads))
+ elif key == "projection.weight":
+ yield (f"{p}.proj_in.weight", t)
+ elif key == "pos_embedding.weight":
+ yield (f"{p}.position_embd.weight", t)
+ elif key == "norm.weight":
+ yield (f"{p}.post_ln.weight", t)
+ else:
+ _, il, rest = key.split(".", 2)
+ layer = {
+ "input_layernorm.weight": "ln1.weight",
+ "attn.to_q.weight": "attn_q.weight",
+ "attn.to_k.weight": "attn_k.weight",
+ "attn.to_v.weight": "attn_v.weight",
+ "attn.to_out.weight": "attn_out.weight",
+ "post_attention_layernorm.weight": "ln2.weight",
+ "gate_proj.weight": "ffn_gate.weight",
+ "up_proj.weight": "ffn_up.weight",
+ "down_proj.weight": "ffn_down.weight",
+ }
+ yield (f"{p}.blk.{il}.{layer[rest]}", t)
+
+ def _cond(self, t: Tensor, key: str) -> Iterable[tuple[str, Tensor]]:
+ p = "a.gen.cond"
+ if key == "layer_weight_logits":
+ # softmax mix of the 8 hidden states of a frame, with the scale folded in
+ scale = self._tensor("condition_encoder.layer_scale").float()
+ yield (f"{p}.layer_w", torch.softmax(t.float(), dim=0) * scale)
+ elif key == "proj.weight":
+ yield (f"{p}.proj.weight", t)
+ elif key == "proj.bias":
+ yield (f"{p}.proj.bias", t)
+
+ def _dit(self, t: Tensor, key: str) -> Iterable[tuple[str, Tensor]]:
+ p = "a.gen.flow"
+ if key.startswith("transformer_blocks."):
+ _, il, rest = key.split(".", 2)
+ q = f"{p}.blk.{il}"
+ if rest.startswith("ff_in."):
+ # value half first, then the silu gate half
+ suffix = rest.removeprefix("ff_in.")
+ up, gate = t.chunk(2, dim=0)
+ yield (f"{q}.ffn_up.{suffix}", up)
+ yield (f"{q}.ffn_gate.{suffix}", gate)
+ return
+ layer = {
+ "norm1.weight": "ln1.weight",
+ "norm1.bias": "ln1.bias",
+ "attn.to_q.weight": "attn_q.weight",
+ "attn.to_k.weight": "attn_k.weight",
+ "attn.to_v.weight": "attn_v.weight",
+ "attn.to_out.0.weight": "attn_out.weight",
+ "norm2.weight": "ln2.weight",
+ "norm2.bias": "ln2.bias",
+ "ff_out.weight": "ffn_down.weight",
+ "ff_out.bias": "ffn_down.bias",
+ }
+ yield (f"{q}.{layer[rest]}", t)
+ return
+ if key == "time_proj.weight":
+ # random Fourier features, 2 * pi folded in
+ yield (f"{p}.time_freq", t.float().flatten() * 2.0 * torch.pi)
+ return
+ out = {
+ "time_embed.linear_1.weight": "time_in.weight",
+ "time_embed.linear_1.bias": "time_in.bias",
+ "time_embed.linear_2.weight": "time_out.weight",
+ "time_embed.linear_2.bias": "time_out.bias",
+ "preprocess_conv.weight": "pre.weight",
+ "proj_in.weight": "proj_in.weight",
+ "proj_out.weight": "proj_out.weight",
+ "postprocess_conv.weight": "post.weight",
+ }[key]
+ yield (f"{p}.{out}", t.squeeze(-1) if key.endswith("conv.weight") else t)
+
+ def _tensor(self, name: str) -> Tensor:
+ # read directly, without the dtype handling of prepare_tensors()
+ return self.model_tensors[name]().float()
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ # DAC decoder, same layout as the yue2 VAE (a.gen.vae.*), with the snake and the weight norm folded
+ def get(key: str) -> Tensor:
+ return self._tensor(f"vocoder.{key}")
+
+ def wn(key: str) -> Tensor:
+ return _fold_wn(get(key + ".weight_g"), get(key + ".weight_v"))
+
+ def snake(dst: str, key: str) -> Iterable[tuple[str, Tensor]]:
+ a, b = fold_snake(get(key + ".alpha"))
+ yield (f"{dst}.alpha", a)
+ yield (f"{dst}.beta", b)
+
+ p = "a.gen.vae"
+ yield (f"{p}.in_proj.weight", get("dec_in_proj.weight"))
+ yield (f"{p}.in_proj.bias", get("dec_in_proj.bias"))
+ yield (f"{p}.conv_in.weight", wn("conv_in"))
+ yield (f"{p}.conv_in.bias", get("conv_in.bias"))
+ n_blk = len(self.global_config["vocoder"]["upsampling_ratios"])
+ for b in range(n_blk):
+ s = f"blocks.{b}"
+ yield from snake(f"{p}.blk.{b}.snake", f"{s}.snake1")
+ yield (f"{p}.blk.{b}.up.weight", wn(f"{s}.conv_t1"))
+ yield (f"{p}.blk.{b}.up.bias", get(f"{s}.conv_t1.bias"))
+ for r in range(3):
+ rs = f"{s}.res_unit{r + 1}"
+ yield from snake(f"{p}.blk.{b}.res.{r}.snake1", f"{rs}.snake1")
+ yield (f"{p}.blk.{b}.res.{r}.conv1.weight", wn(f"{rs}.conv1"))
+ yield (f"{p}.blk.{b}.res.{r}.conv1.bias", get(f"{rs}.conv1.bias"))
+ yield from snake(f"{p}.blk.{b}.res.{r}.snake2", f"{rs}.snake2")
+ yield (f"{p}.blk.{b}.res.{r}.conv2.weight", wn(f"{rs}.conv2"))
+ yield (f"{p}.blk.{b}.res.{r}.conv2.bias", get(f"{rs}.conv2.bias"))
+ yield from snake(f"{p}.post_snake", "snake_out")
+ yield (f"{p}.conv_out.weight", wn("conv_out"))
+ yield (f"{p}.conv_out.bias", get("conv_out.bias"))
diff --git a/docs/multimodal/tts.md b/docs/multimodal/tts.md
index a9545b5c6..b77804671 100644
--- a/docs/multimodal/tts.md
+++ b/docs/multimodal/tts.md
@@ -14,6 +14,7 @@ Supported models:
- [ZONOS3](#zonos3)
- [Fish Audio S2 Pro](#fish-audio-s2-pro)
- [YuE2](#yue2) (songs)
+- [MiniMax Music 3](#minimax-music-3) (songs)
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
@@ -306,3 +307,48 @@ python convert_hf_to_gguf.py YuE2-3B --mmproj --outfile mmproj-yue2-3b.gguf
```
The mmproj holds the NAR transformer layers (loaded with libllama as a Qwen3 text model, in the `--outtype` type), and the latent projections and the VAE decoder, always in F32.
+
+## MiniMax Music 3
+
+[MiniMaxAI/MiniMax-Music3](https://huggingface.co/MiniMaxAI/MiniMax-Music3) generates songs (vocals and accompaniment) from a music description and lyrics, as 44.1 kHz stereo. For each 40 ms frame, the backbone (a Qwen3 text model) samples the semantic code and the RVQ depth decoder the 7 other codes; the hidden states of both go to a flow-matching DiT, then a DAC decoder makes the audio. Both use the settings of the reference (the diffusers pipeline).
+
+The prompt has the same format as YuE2, the description goes under `[Tags]`:
+
+```
+[Tags]
+Genre: acoustic pop. BPM: 96. Key: C major. Vocals: soft female lead. Arrangement: fingerpicked guitar and soft piano.
+[Lyrics]
+[verse]
+Morning light filtering through the pine
+[chorus]
+Softly the world begins to breathe
+```
+
+Available params:
+- The description and the lyrics are cleaned up like the reference: markdown is removed, a line that starts with structure tags (e.g. `[Verse]`) keeps only the tags
+- The generation stops at the end-of-audio token or at the limit of the reference (9000 frames, 6 minutes); `-n` sets a lower limit in frames (25 per second)
+- `--temp` changes the temperature of the codes (default 1.0); `--top-k` and `--top-p` are not used, the codes are sampled with top-k 50 like the reference
+- The codes are sampled with classifier-free guidance, scale 1.5 like the reference; `llama-server` takes `cfg_scale` to change it (1.0 disables it). The uncond prompt runs as a second sequence that shares the context, so a song needs about 2 x (prompt tokens + frames) positions: the default context (10240) holds about 3 minutes with a short description, use a larger `-c` (e.g. 20480) for longer songs
+- The flow matching runs 30 Euler steps like the reference; `llama-server` takes `flow_steps` to change it
+- The audio is made in windows of 200 frames (8 s) every 100 frames, like the reference: a window is processed during the generation once the frame after it exists, so with streaming the audio arrives in steps of about 4 s
+
+Example usage:
+
+```sh
+llama-tts -m minimax-music3.gguf \
+ -mm mmproj-minimax-music3.gguf \
+ -f song.txt \
+ --output song.wav
+```
+
+**Note for GGUF conversion:**
+
+The conversion reads the diffusers layout of the repository, the `qwen_7B` directory and the `.pth` files are not needed:
+
+```sh
+hf download MiniMaxAI/MiniMax-Music3 --local-dir MiniMax-Music3 --exclude "qwen_7B/*" --exclude "*.pth"
+python convert_hf_to_gguf.py MiniMax-Music3 --outfile minimax-music3.gguf
+python convert_hf_to_gguf.py MiniMax-Music3 --mmproj --outfile mmproj-minimax-music3.gguf
+```
+
+The mmproj holds the depth decoder and the DiT layers (in the `--outtype` type), and the condition encoder, the small DiT projections and the DAC decoder, always in F32.
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 395cad9b3..a32811ca9 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -7291,6 +7291,7 @@ class VisionProjectorType:
ZONOS3_GEN = "zonos3_gen" # audio generation: 10 codebook heads + ZAC decoder
ZONOS3_SPKENC = "zonos3_spkenc" # audio: MECT-B2 speaker encoder
YUE2_GEN = "yue2_gen" # audio generation: flow-matching NAR + Oobleck VAE decoder
+ MINIMAX_MUSIC3_GEN = "minimax_music3_gen" # audio generation: RVQ depth decoder + flow-matching DiT + DAC decoder
FISH_S2_GEN = "fish_s2_gen" # audio generation: fast AR code predictor + Fish DAC decoder
FISH_S2_SPKENC = "fish_s2_spkenc" # audio: Fish DAC encoder + RVQ, reference clip -> prompt rows
HUNYUANVL = "hunyuanvl"
diff --git a/models/templates/MiniMaxAI-MiniMax-Music3-tts.jinja b/models/templates/MiniMaxAI-MiniMax-Music3-tts.jinja
new file mode 100644
index 000000000..2d4cc13a0
--- /dev/null
+++ b/models/templates/MiniMaxAI-MiniMax-Music3-tts.jinja
@@ -0,0 +1,52 @@
+{#- MiniMax Music 3 prompt, same text format as YuE2: "[Tags]\n<caption>\n[Lyrics]\n<lyrics>" -#}
+{#- the caption and lyrics cleanup follows _clean_caption() and _normalize_lyrics() of the diffusers pipeline (encoders.py) -#}
+{%- set text = messages[0]['content'] -%}
+{%- set parts = text.split('\n[Lyrics]\n') -%}
+{%- if parts | length < 2 -%}
+ {{- raise_exception('MiniMax Music 3 expects "[Tags]\\n<caption>\\n[Lyrics]\\n<lyrics>"') -}}
+{%- endif -%}
+{%- set caption = tts_norm_sub(parts[0], '^%[Tags%]\n', '') -%}
+{%- set lyrics = parts[1:] | join('\n[Lyrics]\n') -%}
+
+{#- caption: <|key value|> -> "key is value", then strip the markdown -#}
+{%- set ns = namespace(text='', lines=[]) -%}
+{%- for s in tts_norm_scan(caption, '<%|([^|]*)%|>') -%}
+ {%- if s.rule == 'match' -%}
+ {%- set ns.text = ns.text ~ tts_norm_sub(s.groups[0].strip(), '^(%S+)%s+(.+)', '%1 is %2') -%}
+ {%- else -%}
+ {%- set ns.text = ns.text ~ s.text -%}
+ {%- endif -%}
+{%- endfor -%}
+{%- for raw in ns.text.split('\n') -%}
+ {%- set line = tts_norm_sub(raw, '^%s{0,3}#{1,6}%s+', '') -%}
+ {%- set line = tts_norm_sub(line, '^%s*[*+%-]%s+', '') -%}
+ {%- set line = tts_norm_sub(line, '^%s*%*%s+', '') -%}
+ {%- set ln = namespace(v=line) -%}
+ {%- for _ in range(4) -%}
+ {%- set ln.v = tts_norm_sub(ln.v, '%*%*([^*]+)%*%*', '%1') -%}
+ {%- endfor -%}
+ {%- set line = tts_norm_sub(ln.v, '(?<!%*)%*([^*\n]+)%*(?!%*)', '%1').rstrip() -%}
+ {%- set line = tts_norm_sub(line, '^%s*[%-*_]{3,}%s*$', '') -%}
+ {%- set ns.lines = ns.lines + [line] -%}
+{%- endfor -%}
+{%- set caption = ns.lines | join('\n') -%}
+{%- set caption = caption.replace('• ', '').replace(' ', '') -%}
+{%- set caption = tts_norm_sub(caption, '\n{2,}', '\n') -%}
+
+{#- lyrics: a line that starts with structure tags keeps only the tags, the tags are lowercase -#}
+{%- set ns.lines = [] -%}
+{%- for line in lyrics.split('\n') -%}
+ {%- set seg = tts_norm_scan(line, '^[ \t]*((?:%[[^%]]+%][ \t]*)+)') -%}
+ {%- if seg | length > 0 and seg[0].rule == 'match' -%}
+ {%- set ns.lines = ns.lines + [seg[0].groups[0].strip()] -%}
+ {%- else -%}
+ {%- set ns.lines = ns.lines + [line] -%}
+ {%- endif -%}
+{%- endfor -%}
+{%- set lyrics = (ns.lines | join('\n')).replace('] ', ']\n').replace(' [', '\n[').replace(' ^ ', '\n') -%}
+{%- set ns.text = '' -%}
+{%- for s in tts_norm_scan(lyrics, '%[([^%]]+)%]') -%}
+ {%- set ns.text = ns.text ~ ('[' ~ (s.groups[0] | lower) ~ ']' if s.rule == 'match' else s.text) -%}
+{%- endfor -%}
+
+{{- '<|im_start|><|caption_start|>' ~ caption ~ '<|caption_end|><|lyrics_start|>[start]\n' ~ ns.text ~ '<|lyrics_end|><|im_end|><|audio_start|>' -}}
diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp
index c9b416b58..624542838 100644
--- a/tests/test-jinja.cpp
+++ b/tests/test-jinja.cpp
@@ -47,6 +47,7 @@ static void test_fuzzing(testing & t);
static void test_tts_norm_patterns(testing & t);
static void test_tts_dict(testing & t);
static void test_tts_template_zonos3(testing & t);
+static void test_tts_template_music3(testing & t);
static void test_shared_render(testing & t);
@@ -104,6 +105,7 @@ int main(int argc, char *argv[]) {
t.test("tts norm patterns", test_tts_norm_patterns);
t.test("tts dict", test_tts_dict);
t.test("tts template zonos3", test_tts_template_zonos3);
+ t.test("tts template music3", test_tts_template_music3);
if (!g_python_mode) {
t.test("shared render", test_shared_render);
t.test("hasher", test_hasher);
@@ -3232,6 +3234,37 @@ static void test_tts_template_zonos3(testing & t) {
}
}
+// the MiniMax Music 3 "tts" template: caption and lyrics cleanup of the diffusers pipeline (encoders.py), then the special-token prompt
+static void test_tts_template_music3(testing & t) {
+ std::string path = __FILE__;
+ const size_t pos = path.find_last_of("/\\");
+ path = (pos == std::string::npos ? std::string(".") : path.substr(0, pos)) + "/../models/templates/MiniMaxAI-MiniMax-Music3-tts.jinja";
+ std::ifstream fs(path, std::ios_base::binary);
+ if (!t.assert_true("template file is readable: " + path, fs.is_open())) {
+ return;
+ }
+ const std::string tmpl((std::istreambuf_iterator<char>(fs)), std::istreambuf_iterator<char>());
+
+ const std::vector<std::array<std::string, 2>> cases = {
+ { "[Tags]\nGenre: acoustic pop. BPM: 96.\nVocals: soft female lead.\n[Lyrics]\n[verse]\nMorning light\n[chorus]\nSoftly",
+ "<|im_start|><|caption_start|>Genre: acoustic pop. BPM: 96.\nVocals: soft female lead.<|caption_end|><|lyrics_start|>[start]\n[verse]\nMorning light\n[chorus]\nSoftly<|lyrics_end|><|im_end|><|audio_start|>" },
+ // markdown, <|key value|> tags, inline and lowercased section tags
+ { "[Tags]\n# Global Metadata\n- **bpm** is 92. *key* is E\n ### h3 title\n---\n\n\n\xe2\x80\xa2 dot item\n indented four\n<|genre Blues Rock|> and <|solo|>\n**a **b** c**\n"
+ "[Lyrics]\n[Verse] I'm learning how\n [Pre-Chorus] [Bass Quartet] text dropped\nline one [Chorus] line two ^ next",
+ "<|im_start|><|caption_start|>Global Metadata\nbpm is 92. key is E\nh3 title\ndot item\nindented four\ngenre is Blues Rock and solo\na b c<|caption_end|>"
+ "<|lyrics_start|>[start]\n[verse]\n[pre-chorus]\n[bass quartet]\nline one\n[chorus]\nline two\nnext<|lyrics_end|><|im_end|><|audio_start|>" },
+ };
+ for (const auto & c : cases) {
+ // same variables as common_chat_get_tts_prompt()
+ const json vars = {
+ {"messages", json::array({{{"role", "user"}, {"content", c[0]}}})},
+ {"language", ""},
+ {"voice", ""},
+ };
+ test_template(t, c[0], tmpl, vars, c[1], true);
+ }
+}
+
static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm) {
t.test(name, [&tmpl, &expect, tts_norm](testing & t) {
std::string error;
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index 57b0e098d..233ef1c62 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -88,6 +88,8 @@ add_library(mtmd
models/fish-s2-spkenc.cpp
models/zonos3-gen.cpp
models/yue2-gen.cpp
+ models/snake-vae.cpp
+ models/music3-gen.cpp
models/zonos3-spkenc.cpp
models/pockettts-seanet.cpp
models/pockettts-spkenc.cpp
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index c4044411d..c432a76b5 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -637,15 +637,26 @@ struct mtmd_graph_zonos3_gen : mtmd_graph {
std::vector<c2w_state_slot> list_zonos3_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
+// graphs that decode with a mtmd_snake_vae (yue2, minimax_music3), see snake-vae.cpp
+struct mtmd_graph_snake_vae : mtmd_graph {
+ using mtmd_graph::mtmd_graph;
+
+ // x: [T, C]
+ ggml_tensor * snake(ggml_tensor * x, const mtmd_snake_vae::snake & s) const;
+ ggml_tensor * conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const;
+ // x: [T, n_latent] -> [T * ratio, n_out]
+ ggml_tensor * build_snake_vae(ggml_tensor * x, const mtmd_snake_vae & vae) const;
+};
+
//
// YuE2 generation, the NAR transformer itself runs on a llama_context, see mtmd_gen_flow_yue2():
// LATENT_IN = latent rows -> backbone embd rows
// LATENT_OUT = backbone hidden rows -> velocity rows
// GEN_WAV = Oobleck VAE decoder, latent rows -> interleaved stereo PCM
//
-struct mtmd_graph_yue2_gen : mtmd_graph {
+struct mtmd_graph_yue2_gen : mtmd_graph_snake_vae {
mtmd_graph_yue2_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_rows)
- : mtmd_graph(ctx, img), gen_process(gen_process), n_rows(n_rows) {}
+ : mtmd_graph_snake_vae(ctx, img), gen_process(gen_process), n_rows(n_rows) {}
ggml_cgraph * build() override;
static constexpr int n_latent = 64;
@@ -659,9 +670,48 @@ struct mtmd_graph_yue2_gen : mtmd_graph {
mtmd_subctx_gen_process gen_process;
int n_rows;
+};
- ggml_tensor * snake(ggml_tensor * x, const mtmd_yue2::snake & s) const;
- ggml_tensor * conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const;
+//
+// MiniMax Music 3 generation, see mtmd_gen_wav_music3() for the flow-matching loop:
+// GEN_CODE = depth decoder of one frame: backbone hidden rows (cond, then uncond with guidance) + semantic code -> the 8 codes, the next backbone input and the condition row (mix of the 8 hidden states of the frame)
+// FLOW_COND = condition rows of a window -> DiT condition, resampled to the latent rate
+// FLOW_STEP = one DiT pass, cond and uncond rows as a batch of 2
+// GEN_WAV = DAC decoder, latent rows -> interleaved stereo PCM
+//
+struct mtmd_graph_music3_gen : mtmd_graph_snake_vae {
+ mtmd_graph_music3_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process,
+ int n_rows, int n_out, float cfg_scale, float temp)
+ : mtmd_graph_snake_vae(ctx, img), gen_process(gen_process), n_rows(n_rows), n_out(n_out), cfg_scale(cfg_scale), temp(temp) {}
+ ggml_cgraph * build() override;
+
+ static constexpr int n_latent = 128; // two channels of 64
+ static constexpr int n_channels = 2;
+ static constexpr int vae_ratio = 512; // output samples per latent frame
+ static constexpr int dit_d_head = 64;
+ static constexpr int dit_n_rot = 32; // partial rotary
+ static constexpr float dit_theta = 10000.0f;
+ static constexpr float dit_eps = 1e-5f;
+ static constexpr int top_k = 50;
+
+ // sampling and flow matching of the reference (encoders.py, denoise.py)
+ static constexpr float depth_cfg = 1.5f;
+ static constexpr float dit_cfg = 1.7f;
+ static constexpr int flow_steps = 30;
+ static constexpr int n_overlap = 172; // latent frames shared by neighboring windows
+
+ mtmd_subctx_gen_process gen_process;
+ int n_rows; // GEN_CODE: backbone rows, FLOW_COND: frames, FLOW_STEP and GEN_WAV: latent frames
+ int n_out; // FLOW_COND: latent frames
+ float cfg_scale;
+ float temp;
+
+ static int64_t latent_frames(int64_t n_frames);
+
+ ggml_cgraph * build_depth();
+ ggml_cgraph * build_cond();
+ ggml_cgraph * build_dit();
+ ggml_cgraph * build_vocoder();
};
struct mtmd_graph_soprano : mtmd_graph {
diff --git a/tools/mtmd/models/music3-gen.cpp b/tools/mtmd/models/music3-gen.cpp
new file mode 100644
index 000000000..2cdd82b13
--- /dev/null
+++ b/tools/mtmd/models/music3-gen.cpp
@@ -0,0 +1,224 @@
+#include "models.h"
+
+#include <cmath>
+
+// latent frames of a window of n_frames, see MiniMaxMusic3ConditionEncoder of the reference
+int64_t mtmd_graph_music3_gen::latent_frames(int64_t n_frames) {
+ const double n = (double) n_frames * 44100.0 / 24000.0 * 960.0 / 512.0;
+ return std::max<int64_t>(1, (int64_t) n);
+}
+
+// one frame of the depth decoder (RVQDepthDecoder): row 0 is the backbone hidden, row p > 0 the embedding of code p - 1
+// each step reruns the 4 layers over the rows so far (at most 8, no KV cache), like the reference
+ggml_cgraph * mtmd_graph_music3_gen::build_depth() {
+ const int64_t n_cb = model.gen_code_head_w->ne[2]; // residual codebooks
+ const int64_t n_voc = model.gen_code_head_w->ne[1];
+ const int64_t B = n_rows; // cond, then uncond with guidance
+
+ ggml_tensor * inp_h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, B);
+ ggml_set_name(inp_h, "inp_feats");
+ ggml_set_input(inp_h);
+
+ ggml_tensor * code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1);
+ ggml_set_name(code0, "inp_code0");
+ ggml_set_input(code0);
+
+ ggml_tensor * inp_rand = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_cb);
+ ggml_set_name(inp_rand, "inp_rand");
+ ggml_set_input(inp_rand);
+
+ mtmd_graph_qwen3tts_gen::code_gen cg(*this, top_k, 1.0f);
+
+ // [n_embd, 1, B], the codes are the same in all rows
+ auto project = [&](ggml_tensor * x) {
+ ggml_tensor * y = build_mm(model.gen_code_proj_in_w, x);
+ y = ggml_reshape_3d(ctx0, y, n_embd, 1, y->ne[1]);
+ return y->ne[2] == B ? y : ggml_repeat_4d(ctx0, y, n_embd, 1, B, 1);
+ };
+
+ ggml_tensor * e0 = ggml_get_rows(ctx0, model.gen_code_out_embd_w, code0);
+ ggml_tensor * seq = ggml_concat(ctx0, project(inp_h), project(e0), 1);
+
+ ggml_tensor * out_codes = code0;
+ ggml_tensor * hiddens = ggml_view_2d(ctx0, inp_h, n_embd, 1, inp_h->nb[1], 0); // cond row of each step
+ ggml_tensor * embd_sum = e0;
+
+ for (int64_t g = 0; g < n_cb; g++) {
+ const int64_t n_pos = g + 2;
+
+ ggml_tensor * ones = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos), 1.0f);
+ build_vit_opts opts;
+ opts.attn_mask = ggml_log(ctx0, ggml_tri(ctx0, ones, GGML_TRI_TYPE_LOWER_DIAG)); // 0 = keep, -inf = masked
+ if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) {
+ opts.attn_mask = ggml_cast(ctx0, opts.attn_mask, GGML_TYPE_F16);
+ }
+ ggml_tensor * pos = ggml_view_2d(ctx0, model.position_embeddings, n_embd, n_pos, model.position_embeddings->nb[1], 0);
+
+ ggml_tensor * out = build_vit(seq, n_pos, NORM_TYPE_RMS, FFN_SILU, pos, nullptr, opts); // [n_embd, n_pos, B]
+ ggml_tensor * last = ggml_cont(ctx0, ggml_view_2d(ctx0, out, n_embd, B, out->nb[2], (n_pos - 1) * out->nb[1]));
+ hiddens = ggml_concat(ctx0, hiddens, ggml_view_2d(ctx0, last, n_embd, 1, last->nb[1], 0), 1);
+
+ ggml_tensor * head = ggml_view_2d(ctx0, model.gen_code_head_w, n_embd, n_voc, model.gen_code_head_w->nb[1], g * model.gen_code_head_w->nb[2]);
+ ggml_tensor * logits = ggml_mul_mat(ctx0, head, last); // [n_voc, B]
+ if (B > 1) {
+ ggml_tensor * cond = ggml_view_1d(ctx0, logits, n_voc, 0);
+ ggml_tensor * uncond = ggml_view_1d(ctx0, logits, n_voc, logits->nb[1]);
+ logits = ggml_add(ctx0, uncond, ggml_scale(ctx0, ggml_sub(ctx0, cond, uncond), cfg_scale));
+ }
+
+ ggml_tensor * code;
+ if (temp > 0.0f) {
+ ggml_tensor * rnd = ggml_view_1d(ctx0, inp_rand, 1, g * inp_rand->nb[0]);
+ code = cg.do_sampling(ggml_scale(ctx0, logits, 1.0f / temp), rnd);
+ } else {
+ code = ggml_argmax(ctx0, logits);
+ }
+ code = ggml_reshape_1d(ctx0, code, 1);
+ out_codes = ggml_concat(ctx0, out_codes, code, 0);
+
+ ggml_tensor * table = ggml_view_2d(ctx0, model.gen_code_embd_w, n_embd, n_voc, model.gen_code_embd_w->nb[1], g * model.gen_code_embd_w->nb[2]);
+ ggml_tensor * e = ggml_get_rows(ctx0, table, code);
+ embd_sum = ggml_add(ctx0, embd_sum, e);
+ if (g + 1 < n_cb) {
+ seq = ggml_concat(ctx0, seq, project(e), 1);
+ }
+ }
+
+ ggml_set_name(out_codes, "out_codes");
+ ggml_set_output(out_codes);
+ ggml_build_forward_expand(gf, out_codes);
+
+ // condition row of the frame: the 8 hidden states mixed by the condition encoder weights
+ ggml_tensor * cond_row = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, hiddens)), model.music3.cond_layer_w);
+ ggml_set_name(cond_row, "out_feats");
+ ggml_set_output(cond_row);
+ ggml_build_forward_expand(gf, cond_row);
+
+ // backbone input of the next frame: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1), must be the last node
+ ggml_tensor * out_embd = ggml_scale(ctx0, embd_sum, 1.0f / sqrtf((float) (n_cb + 1)));
+ cb(out_embd, "gen_audio_out", -1);
+ ggml_build_forward_expand(gf, out_embd);
+ return gf;
+}
+
+// frame rows [n_embd, n_rows] -> conv k = 3 -> nearest resampling to n_out latent frames
+ggml_cgraph * mtmd_graph_music3_gen::build_cond() {
+ const auto & m = model.music3;
+
+ ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_rows);
+ ggml_set_name(inp, "inp_feats");
+ ggml_set_input(inp);
+
+ ggml_tensor * idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_out);
+ ggml_set_name(idx, "inp_idx");
+ ggml_set_input(idx);
+
+ ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [T, n_embd]
+ x = conv1d(x, m.cond_w, m.cond_b, 1);
+ x = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [n_cond, T]
+ x = ggml_get_rows(ctx0, x, idx);
+ ggml_set_name(x, "out_feats");
+ ggml_set_output(x);
+ ggml_build_forward_expand(gf, x);
+ return gf;
+}
+
+// one velocity pass of the DiT over a window, the cond and uncond (zero condition) inputs as a batch of 2
+// the input rows are [latent, zeros, condition], the time embedding is prepended as one extra token
+ggml_cgraph * mtmd_graph_music3_gen::build_dit() {
+ const auto & m = model.music3;
+ const int64_t n_in = m.pre_w->ne[0];
+ const int64_t n_dit = m.proj_in_w->ne[1];
+ const int64_t n_head = n_dit / dit_d_head;
+ const int64_t n_pos = n_rows + 1;
+ const int64_t B = 2;
+
+ ggml_tensor * inp = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_in, n_rows, B);
+ ggml_set_name(inp, "inp_feats");
+ ggml_set_input(inp);
+
+ ggml_tensor * inp_t = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);
+ ggml_set_name(inp_t, "inp_t");
+ ggml_set_input(inp_t);
+
+ ggml_tensor * x = ggml_add(ctx0, inp, build_mm(m.pre_w, inp));
+ x = build_mm(m.proj_in_w, x); // [n_dit, n_rows, B]
+
+ // time token: Fourier features -> linear -> silu -> linear
+ ggml_tensor * ang = ggml_mul(ctx0, m.time_freq, inp_t);
+ ggml_tensor * temb = ggml_concat(ctx0, ggml_cos(ctx0, ang), ggml_sin(ctx0, ang), 0);
+ temb = ggml_silu(ctx0, ggml_add(ctx0, build_mm(m.time_in_w, temb), m.time_in_b));
+ temb = ggml_add(ctx0, build_mm(m.time_out_w, temb), m.time_out_b);
+ temb = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, temb, n_dit, 1, 1), n_dit, 1, B, 1);
+ x = ggml_concat(ctx0, temb, x, 1); // [n_dit, n_pos, B]
+ x = ggml_reshape_2d(ctx0, x, n_dit, n_pos * B);
+
+ ggml_tensor * pos = ggml_cast(ctx0, ggml_arange(ctx0, 0.0f, (float) n_pos, 1.0f), GGML_TYPE_I32);
+ const float kq = 1.0f / sqrtf((float) dit_d_head);
+
+ for (size_t il = 0; il < m.dit.size(); il++) {
+ const auto & layer = m.dit[il];
+
+ ggml_tensor * h = build_norm(x, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, dit_eps, (int) il);
+ ggml_tensor * q = ggml_reshape_4d(ctx0, build_mm(layer.q_w, h), dit_d_head, n_head, n_pos, B);
+ ggml_tensor * k = ggml_reshape_4d(ctx0, build_mm(layer.k_w, h), dit_d_head, n_head, n_pos, B);
+ ggml_tensor * v = ggml_reshape_4d(ctx0, build_mm(layer.v_w, h), dit_d_head, n_head, n_pos, B);
+
+ // only the first dit_n_rot dims of each head rotate
+ q = ggml_rope_ext(ctx0, q, pos, nullptr, dit_n_rot, GGML_ROPE_TYPE_NEOX, 0, dit_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+ k = ggml_rope_ext(ctx0, k, pos, nullptr, dit_n_rot, GGML_ROPE_TYPE_NEOX, 0, dit_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+
+ x = ggml_add(ctx0, x, build_attn(layer.o_w, nullptr, q, k, v, nullptr, kq, (int) il));
+
+ h = build_norm(x, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, dit_eps, (int) il);
+ h = build_ffn(h, layer.ff_up_w, layer.ff_up_b, layer.ff_gate_w, layer.ff_gate_b, layer.ff_down_w, layer.ff_down_b, FFN_SILU, (int) il);
+ x = ggml_add(ctx0, x, h);
+ }
+
+ // drop the time token
+ x = ggml_reshape_3d(ctx0, x, n_dit, n_pos, B);
+ x = ggml_view_3d(ctx0, x, n_dit, n_rows, B, x->nb[1], x->nb[2], x->nb[1]);
+ x = build_mm(m.proj_out_w, ggml_cont(ctx0, x));
+ x = ggml_add(ctx0, x, build_mm(m.post_w, x)); // [n_latent, n_rows, B]
+
+ // classifier-free guidance
+ ggml_tensor * cond = ggml_view_2d(ctx0, x, n_latent, n_rows, x->nb[1], 0);
+ ggml_tensor * uncond = ggml_view_2d(ctx0, x, n_latent, n_rows, x->nb[1], x->nb[2]);
+ x = ggml_add(ctx0, uncond, ggml_scale(ctx0, ggml_sub(ctx0, cond, uncond), cfg_scale));
+ ggml_set_name(x, "out_feats");
+ ggml_set_output(x);
+ ggml_build_forward_expand(gf, x);
+ return gf;
+}
+
+// the two channels are decoded as two mono streams of n_latent / 2 channels, then tanh
+ggml_cgraph * mtmd_graph_music3_gen::build_vocoder() {
+ ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_latent, n_rows);
+ ggml_set_name(inp, "inp_feats");
+ ggml_set_input(inp);
+
+ ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [T, n_latent]
+ const int64_t n_half = n_latent / n_channels;
+ ggml_tensor * out = nullptr;
+ for (int c = 0; c < n_channels; c++) {
+ ggml_tensor * xc = ggml_cont(ctx0, ggml_view_2d(ctx0, x, n_rows, n_half, x->nb[1], c * n_half * x->nb[1]));
+ ggml_tensor * y = ggml_tanh(ctx0, build_snake_vae(xc, model.snake_vae)); // [T * ratio, 1]
+ out = out ? ggml_concat(ctx0, out, y, 1) : y;
+ }
+
+ out = ggml_cont(ctx0, ggml_transpose(ctx0, out)); // interleaved
+ ggml_set_name(out, "out_audio");
+ ggml_set_output(out);
+ ggml_build_forward_expand(gf, out);
+ return gf;
+}
+
+ggml_cgraph * mtmd_graph_music3_gen::build() {
+ switch (gen_process) {
+ case MTMD_SUBCTX_GEN_PROCESS_GEN_CODE: return build_depth();
+ case MTMD_SUBCTX_GEN_PROCESS_FLOW_COND: return build_cond();
+ case MTMD_SUBCTX_GEN_PROCESS_FLOW_STEP: return build_dit();
+ case MTMD_SUBCTX_GEN_PROCESS_GEN_WAV: return build_vocoder();
+ default: GGML_ABORT("minimax_music3: unsupported gen process");
+ }
+}
diff --git a/tools/mtmd/models/snake-vae.cpp b/tools/mtmd/models/snake-vae.cpp
new file mode 100644
index 000000000..41bfb60ae
--- /dev/null
+++ b/tools/mtmd/models/snake-vae.cpp
@@ -0,0 +1,64 @@
+#include "models.h"
+
+// SnakeBeta, alpha and beta folded at conversion: x + beta * sin(alpha * x)^2
+// x: [T, C], alpha/beta: [C]
+ggml_tensor * mtmd_graph_snake_vae::snake(ggml_tensor * x, const mtmd_snake_vae::snake & s) const {
+ ggml_tensor * a = ggml_reshape_2d(ctx0, s.alpha, 1, s.alpha->ne[0]);
+ ggml_tensor * b = ggml_reshape_2d(ctx0, s.beta, 1, s.beta->ne[0]);
+
+ // expand reshapes first so mul/sin/sqr/mul/add lands as consecutive nodes, letting backends fuse them
+ ggml_build_forward_expand(gf, a);
+ ggml_build_forward_expand(gf, b);
+
+ ggml_tensor * y = ggml_sqr(ctx0, ggml_sin(ctx0, ggml_mul(ctx0, x, a)));
+ return ggml_add(ctx0, x, ggml_mul(ctx0, y, b));
+}
+
+// same-length conv, stride 1; x: [T, IC], w: [K, IC, OC], returns [T, OC]
+// im2col in F32: the reference decoders run in FP32
+ggml_tensor * mtmd_graph_snake_vae::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const {
+ const int pad = (int) (w->ne[0] - 1) / 2 * dilation;
+ ggml_tensor * col = ggml_im2col(ctx0, w, x, 1, 0, pad, 0, dilation, 0, false, GGML_TYPE_F32); // [1, T, IC * K]
+ ggml_tensor * y = ggml_mul_mat(ctx0,
+ ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[1] * col->ne[2]),
+ ggml_reshape_2d(ctx0, w, w->ne[0] * w->ne[1], w->ne[2]));
+ if (b) {
+ y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
+ }
+ return y;
+}
+
+ggml_tensor * mtmd_graph_snake_vae::build_snake_vae(ggml_tensor * x, const mtmd_snake_vae & vae) const {
+ static constexpr int dilations[3] = { 1, 3, 9 };
+
+ if (vae.in_proj_w) {
+ x = conv1d(x, vae.in_proj_w, vae.in_proj_b, 1);
+ }
+ x = conv1d(x, vae.in_w, vae.in_b, 1);
+ for (size_t ib = 0; ib < vae.blocks.size(); ib++) {
+ const auto & blk = vae.blocks[ib];
+ const int stride = (int) (blk.up_w->ne[0] / 2);
+ const int pad = (stride + 1) / 2; // ceil(stride / 2) of the reference
+ x = snake(x, blk.act);
+ {
+ // ConvTranspose1d as GEMM + col2im, same as qwen3tts; up_w: [K, OC, IC]
+ const int64_t K = blk.up_w->ne[0];
+ const int64_t OC = blk.up_w->ne[1];
+ ggml_tensor * w2 = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, blk.up_w, K * OC, blk.up_w->ne[2])));
+ ggml_tensor * col = ggml_mul_mat(ctx0, w2, ggml_cont(ctx0, ggml_transpose(ctx0, x))); // [K * OC, T]
+ x = ggml_col2im_1d(ctx0, col, stride, (int) OC, pad);
+ }
+ x = ggml_add(ctx0, x, ggml_reshape_2d(ctx0, blk.up_b, 1, blk.up_b->ne[0]));
+ for (size_t ir = 0; ir < blk.res.size(); ir++) {
+ const auto & ru = blk.res[ir];
+ ggml_tensor * h = snake(x, ru.act1);
+ h = conv1d(h, ru.conv1_w, ru.conv1_b, dilations[ir]);
+ h = snake(h, ru.act2);
+ h = conv1d(h, ru.conv2_w, ru.conv2_b, 1);
+ x = ggml_add(ctx0, x, h);
+ }
+ cb(x, "vae_block", (int) ib);
+ }
+ x = snake(x, vae.post);
+ return conv1d(x, vae.out_w, vae.out_b, 1);
+}
diff --git a/tools/mtmd/models/yue2-gen.cpp b/tools/mtmd/models/yue2-gen.cpp
index 59cd6f12f..bcfebb89d 100644
--- a/tools/mtmd/models/yue2-gen.cpp
+++ b/tools/mtmd/models/yue2-gen.cpp
@@ -1,33 +1,5 @@
#include "models.h"
-// SnakeBeta, alpha and beta folded at conversion: x + beta * sin(alpha * x)^2
-// x: [T, C], alpha/beta: [C]
-ggml_tensor * mtmd_graph_yue2_gen::snake(ggml_tensor * x, const mtmd_yue2::snake & s) const {
- ggml_tensor * a = ggml_reshape_2d(ctx0, s.alpha, 1, s.alpha->ne[0]);
- ggml_tensor * b = ggml_reshape_2d(ctx0, s.beta, 1, s.beta->ne[0]);
-
- // expand reshapes first so mul/sin/sqr/mul/add lands as consecutive nodes, letting backends fuse them
- ggml_build_forward_expand(gf, a);
- ggml_build_forward_expand(gf, b);
-
- ggml_tensor * y = ggml_sqr(ctx0, ggml_sin(ctx0, ggml_mul(ctx0, x, a)));
- return ggml_add(ctx0, x, ggml_mul(ctx0, y, b));
-}
-
-// same-length conv, stride 1; x: [T, IC], w: [K, IC, OC], returns [T, OC]
-// im2col in F32: the reference decoder runs in FP32
-ggml_tensor * mtmd_graph_yue2_gen::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const {
- const int pad = (int) (w->ne[0] - 1) / 2 * dilation;
- ggml_tensor * col = ggml_im2col(ctx0, w, x, 1, 0, pad, 0, dilation, 0, false, GGML_TYPE_F32); // [1, T, IC * K]
- ggml_tensor * y = ggml_mul_mat(ctx0,
- ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[1] * col->ne[2]),
- ggml_reshape_2d(ctx0, w, w->ne[0] * w->ne[1], w->ne[2]));
- if (b) {
- y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
- }
- return y;
-}
-
ggml_cgraph * mtmd_graph_yue2_gen::build() {
const auto & y = model.yue2;
@@ -46,40 +18,13 @@ ggml_cgraph * mtmd_graph_yue2_gen::build() {
}
GGML_ASSERT(gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV);
- static constexpr int dilations[3] = { 1, 3, 9 };
ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_latent, n_rows);
ggml_set_name(inp, "inp_feats");
ggml_set_input(inp);
ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [T, n_latent]
- x = conv1d(x, y.vae_in_w, y.vae_in_b, 1);
- for (size_t ib = 0; ib < y.vae.size(); ib++) {
- const auto & blk = y.vae[ib];
- const int stride = (int) (blk.up_w->ne[0] / 2);
- const int pad = (stride + 1) / 2; // ceil(stride / 2) of the reference
- x = snake(x, blk.act);
- {
- // ConvTranspose1d as GEMM + col2im, same as qwen3tts; up_w: [K, OC, IC]
- const int64_t K = blk.up_w->ne[0];
- const int64_t OC = blk.up_w->ne[1];
- ggml_tensor * w2 = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, blk.up_w, K * OC, blk.up_w->ne[2])));
- ggml_tensor * col = ggml_mul_mat(ctx0, w2, ggml_cont(ctx0, ggml_transpose(ctx0, x))); // [K * OC, T]
- x = ggml_col2im_1d(ctx0, col, stride, (int) OC, pad);
- }
- x = ggml_add(ctx0, x, ggml_reshape_2d(ctx0, blk.up_b, 1, blk.up_b->ne[0]));
- for (size_t ir = 0; ir < blk.res.size(); ir++) {
- const auto & ru = blk.res[ir];
- ggml_tensor * h = snake(x, ru.act1);
- h = conv1d(h, ru.conv1_w, ru.conv1_b, dilations[ir]);
- h = snake(h, ru.act2);
- h = conv1d(h, ru.conv2_w, ru.conv2_b, 1);
- x = ggml_add(ctx0, x, h);
- }
- cb(x, "vae_block", (int) ib);
- }
- x = snake(x, y.vae_post);
- x = conv1d(x, y.vae_out_w, nullptr, 1); // [T, n_channels]
+ x = build_snake_vae(x, model.snake_vae); // [T, n_channels]
x = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // interleaved
ggml_set_name(x, "out_audio");
diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp
index 78a0eca46..350aa4c9f 100644
--- a/tools/mtmd/mtmd-graph.cpp
+++ b/tools/mtmd/mtmd-graph.cpp
@@ -1049,6 +1049,27 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
}
builder = std::make_unique<mtmd_graph_yue2_gen>(ctx, img, gen_process, n_rows);
} break;
+ case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
+ {
+ using g = mtmd_graph_music3_gen;
+ const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
+ // row size of the input of each process; the reserve pass builds GEN_CODE with one row
+ size_t n_col = (size_t) ctx->model.hparams.n_embd;
+ if (gen_process == MTMD_SUBCTX_GEN_PROCESS_FLOW_STEP) {
+ n_col = 2 * (size_t) ctx->model.music3.pre_w->ne[0];
+ } else if (gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
+ n_col = g::n_latent;
+ }
+ int n_rows = 1;
+ if (params && params->feats) {
+ GGML_ASSERT(!params->feats->empty() && params->feats->size() % n_col == 0);
+ n_rows = (int) (params->feats->size() / n_col);
+ }
+ const int n_out = gen_process == MTMD_SUBCTX_GEN_PROCESS_FLOW_COND ? (int) g::latent_frames(n_rows) : 0;
+ const float cfg_scale = params ? params->cfg_scale : 1.0f;
+ const float temp = params ? params->temp : 1.0f;
+ builder = std::make_unique<mtmd_graph_music3_gen>(ctx, img, gen_process, n_rows, n_out, cfg_scale, temp);
+ } break;
case PROJECTOR_TYPE_DRAMA3_GEN:
case PROJECTOR_TYPE_FISH_S2_GEN:
{
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 553e45f47..6a83ece40 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -3810,6 +3810,466 @@ private:
bool wav_header_sent = false;
};
+// MiniMax Music 3: the backbone samples the semantic code of each frame with guidance, the depth decoder (mtmd) the other codes
+// the condition rows of the frames go to the flow matching and the vocoder (mtmd) in windows, like the reference (diffusers pipeline)
+class music3_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
+public:
+ using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
+
+ bool supports_cfg() const override { return true; }
+ bool supports_flow_steps() const override { return true; }
+
+ // step_gen() stops at the frame limit of the reference
+ int32_t max_frames() const override { return 0; }
+
+ bool check_context() const override {
+ if (llama_vocab_n_tokens(vocab) < tok_code0 + n_semantic) {
+ LOG_ERR("mtmd_helper_gen_audio: the vocab of the backbone has no minimax_music3 audio codes\n");
+ return false;
+ }
+ return true;
+ }
+
+ void reset() override {
+ seq_id = 0;
+ seq_uncond = 0;
+ use_cfg = false;
+ cfg_scale = 1.0f;
+ temp = 1.0f;
+ flow_steps = 0;
+ pos = 0;
+ prompt.clear();
+ prompt_uncond.clear();
+ prompt_pos = 0;
+ uprompt_pos = 0;
+ logits_cond.clear();
+ logits_uncond.clear();
+ h_rows.clear();
+ n_gen = 0;
+ cond_rows.clear();
+ n_rows_dropped = 0;
+ n_win_done = 0;
+ done = false;
+ decoded = false;
+ audio_pcm.clear();
+ out_buf.clear();
+ stream = false;
+ pcm_sent = 0;
+ wav_header_sent = false;
+ gen_state.reset();
+ }
+
+ int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
+ reset();
+ seq_id = inp->seq_id;
+ out_type = inp->out_type;
+ stream = inp->stream;
+ temp = inp->temp >= 0.0f ? inp->temp : 1.0f;
+ flow_steps = inp->flow_steps;
+ cfg_scale = inp->cfg_scale > 0.0f ? inp->cfg_scale : 1.5f;
+ rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
+ if (!init_gen_state(inp->seed)) {
+ return 1;
+ }
+
+ // the "tts" template renders the whole prompt, special tokens included
+ const std::string text(inp->prompt, inp->prompt_len);
+ prompt.resize(text.size() + 16);
+ const int32_t n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), prompt.data(), (int32_t) prompt.size(), false, true);
+ if (n < 3 || prompt[0] != tok_im_start || prompt[n - 1] != tok_audio_start) {
+ LOG_ERR("mtmd_helper_gen_audio: minimax_music3 expects the prompt of its tts template\n");
+ return 1;
+ }
+ prompt.resize((size_t) n);
+ if (n > max_prompt) {
+ LOG_ERR("mtmd_helper_gen_audio: the prompt has %d tokens, the maximum is %d\n", n, max_prompt);
+ return 1;
+ }
+
+ llama_memory_t mem = llama_get_memory(lctx);
+ if (!llama_memory_seq_rm(mem, seq_id, 0, -1)) {
+ LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n");
+ return 1;
+ }
+ use_cfg = cfg_scale != 1.0f;
+ if (use_cfg) {
+ // the uncond sequence lives half a seq space away from the cond one
+ const int32_t n_seq_max = (int32_t) llama_n_seq_max(lctx);
+ seq_uncond = (llama_seq_id) (n_seq_max / 2 + seq_id);
+ if (!llama_model_classifier_free_guidance(model) || seq_id >= n_seq_max / 2) {
+ LOG_ERR("mtmd_helper_gen_audio: cfg needs a paired sequence (n_seq_max = %d, seq_id = %d), reconvert the model or use cfg_scale = 1\n", n_seq_max, seq_id);
+ return 1;
+ }
+ if (!llama_memory_seq_rm(mem, seq_uncond, 0, -1)) {
+ LOG_ERR("mtmd_helper_gen_audio: cannot reset the cfg sequence\n");
+ return 1;
+ }
+ // the uncond prompt keeps <|im_start|> and the last 2 tokens, the rest is <|audio_cfg|>
+ prompt_uncond = prompt;
+ std::fill(prompt_uncond.begin() + 1, prompt_uncond.end() - 2, tok_audio_cfg);
+ }
+ if (n_pos_needed((llama_pos) n + 1) > (int64_t) llama_n_ctx_seq(lctx)) {
+ LOG_ERR("mtmd_helper_gen_audio: the prompt does not fit in the context\n");
+ return 1;
+ }
+ return 0;
+ }
+
+ int32_t step_prompt(int32_t n_batch) override {
+ GGML_ASSERT(n_batch > 0);
+ const int32_t n_prompt = (int32_t) prompt.size();
+ if (prompt_pos < n_prompt) {
+ const int32_t n = std::min(n_batch, n_prompt - prompt_pos);
+ llama_pos p = prompt_pos;
+ if (!decode_tokens(prompt.data() + prompt_pos, n, seq_id, p)) {
+ return -1;
+ }
+ prompt_pos += n;
+ if (prompt_pos == n_prompt) {
+ save_outputs(0, -1);
+ }
+ } else if (use_cfg && uprompt_pos < n_prompt) {
+ const int32_t n = std::min(n_batch, n_prompt - uprompt_pos);
+ llama_pos p = uprompt_pos;
+ if (!decode_tokens(prompt_uncond.data() + uprompt_pos, n, seq_uncond, p)) {
+ return -1;
+ }
+ uprompt_pos += n;
+ if (uprompt_pos == n_prompt) {
+ save_outputs(1, -1);
+ }
+ }
+ pos = n_prompt;
+ const int32_t n_left = (n_prompt - prompt_pos) + (use_cfg ? n_prompt - uprompt_pos : 0);
+ return n_left;
+ }
+
+ int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
+ GGML_UNUSED(sampled);
+ GGML_UNUSED(h_state_in);
+ *h_state_out = nullptr;
+ *out_stop = false;
+
+ // the first frame only moves the backbone past <|audio_start|>, its condition row is dropped like the reference
+ const size_t n_rows = n_frames();
+ while (!done && n_frames() == n_rows) {
+ if (!gen_frame()) {
+ return 1;
+ }
+ }
+ if (n_frames() == n_rows) {
+ *out_stop = true;
+ return 0;
+ }
+ *h_state_out = h_rows.data();
+ *out_stop = done;
+
+ // a window is final once a frame past it exists, see MiniMaxMusic3PrepareChunksStep of the reference
+ while (n_frames() > window_start(n_win_done) + win_frames) {
+ if (!run_window(false)) {
+ return 1;
+ }
+ }
+ return 0;
+ }
+
+ int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
+ *out_sample_rate = info.sample_rate;
+ const uint16_t n_ch = (uint16_t) info.n_channels;
+
+ if (!stream) {
+ if (flush() != 0) {
+ return 1;
+ }
+ if (out_n_samples) {
+ *out_n_samples = (int64_t) (audio_pcm.size() / n_ch);
+ }
+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+ *out_data = (const char *) audio_pcm.data();
+ *out_data_len = audio_pcm.size() * sizeof(float);
+ return 0;
+ }
+ out_buf.clear();
+ if (!write_wav16(out_buf, audio_pcm, info.sample_rate, n_ch)) {
+ LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
+ return 1;
+ }
+ *out_data = out_buf.data();
+ *out_data_len = out_buf.size();
+ return 0;
+ }
+
+ const size_t n_new = audio_pcm.size() - pcm_sent;
+ if (out_n_samples) {
+ *out_n_samples = (int64_t) (n_new / n_ch);
+ }
+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+ *out_data = (const char *) (audio_pcm.data() + pcm_sent);
+ *out_data_len = n_new * sizeof(float);
+ pcm_sent = audio_pcm.size();
+ return 0;
+ }
+ out_buf.clear();
+ if (!wav_header_sent) {
+ write_wav16_header(out_buf, UINT32_MAX, info.sample_rate, n_ch);
+ wav_header_sent = true;
+ }
+ append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
+ pcm_sent = audio_pcm.size();
+ *out_data = out_buf.data();
+ *out_data_len = out_buf.size();
+ return 0;
+ }
+
+ int32_t flush() override {
+ if (decoded) {
+ return 0;
+ }
+ if (n_frames() == 0) {
+ LOG_ERR("mtmd_helper_gen_audio: minimax_music3 stopped before the first frame\n");
+ return 1;
+ }
+ done = true;
+ // the windows start every win_hop frames before n_frames - win_hop, the last one ends at n_frames
+ while (n_win_done == 0 || window_start(n_win_done) + win_hop < n_frames()) {
+ const bool last = window_start(n_win_done + 1) + win_hop >= n_frames();
+ if (!run_window(last)) {
+ return 1;
+ }
+ if (last) {
+ break;
+ }
+ }
+ decoded = true;
+ return 0;
+ }
+
+private:
+ // token ids of the checkpoint, see encoders.py of the reference
+ static constexpr llama_token tok_im_start = 151644;
+ static constexpr llama_token tok_audio_cfg = 151654;
+ static constexpr llama_token tok_audio_start = 151669;
+ static constexpr llama_token tok_audio_end = 151670;
+ static constexpr llama_token tok_code0 = 151675;
+ static constexpr int32_t n_semantic = 16384;
+ static constexpr int32_t max_prompt = 5000;
+ static constexpr size_t max_audio_frames = 9000;
+ static constexpr int32_t top_k = 50;
+
+ // windows of the flow matching and their crop, see before_denoise.py and decoders.py of the reference
+ static constexpr size_t win_frames = 200;
+ static constexpr size_t win_hop = 100;
+ static constexpr size_t crop_left = 86; // latent frames
+ static constexpr size_t crop_right = 344 - 86; // latent frames
+ static constexpr size_t hop_samples = 512; // samples per latent frame
+
+ // with guidance, both sequences share the unified KV cache
+ int64_t n_pos_needed(llama_pos n) const {
+ return use_cfg ? 2 * (int64_t) n : (int64_t) n;
+ }
+
+ size_t n_frames() const {
+ return n_rows_dropped + cond_rows.size() / (size_t) n_embd;
+ }
+
+ static size_t window_start(size_t k) {
+ return k * win_hop;
+ }
+
+ bool decode_tokens(const llama_token * toks, int32_t n, llama_seq_id seq, llama_pos & p) {
+ llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+ if (!batch_set_tokens(batch.get(), toks, n, p, seq)) {
+ return false;
+ }
+ llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
+ if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
+ return false;
+ }
+ p += n;
+ return true;
+ }
+
+ // the context outputs belong to the last decode of any sequence (server slots), keep ours right away
+ // only the window that sample() reads: <|audio_end|> and the semantic codes
+ void save_outputs(int row, int32_t idx) {
+ auto & lg = row == 0 ? logits_cond : logits_uncond;
+ const float * l = llama_get_logits_ith(lctx, idx);
+ lg.assign(l + tok_audio_end, l + tok_code0 + n_semantic);
+ const float * h = llama_get_embeddings_ith(lctx, idx);
+ h_rows.resize((size_t) (use_cfg ? 2 : 1) * n_embd);
+ std::copy(h, h + n_embd, h_rows.begin() + (size_t) row * n_embd);
+ }
+
+ // feeds the next backbone input row to both sequences
+ bool decode_embd(const float * embd) {
+ llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+ const int n_seq = use_cfg ? 2 : 1;
+ const llama_seq_id seqs[2] = { seq_id, seq_uncond };
+ for (int i = 0; i < n_seq; i++) {
+ const llama_embd row = { embd, 1, (size_t) n_embd };
+ const int32_t idx = llama_batch_ext_add_embd(batch.get(), seqs[i], row);
+ if (idx < 0) {
+ LOG_ERR("mtmd_helper_gen_audio: failed to add a row to the batch (error %d)\n", idx);
+ return false;
+ }
+ llama_batch_ext_set_pos(batch.get(), idx, &pos);
+ llama_batch_ext_set_output_logits(batch.get(), idx, true);
+ }
+ if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
+ return false;
+ }
+ pos++;
+ for (int i = 0; i < n_seq; i++) {
+ save_outputs(i, i);
+ }
+ return true;
+ }
+
+ // semantic code or <|audio_end|>: guidance on the top-k of the cond logits, then top-k sampling, see SemanticGenerationStep
+ llama_token sample() {
+ const size_t n = logits_cond.size();
+ std::vector<std::pair<float, size_t>> cand;
+ cand.reserve(n);
+ for (size_t i = 0; i < n; i++) {
+ const llama_token t = tok_audio_end + (llama_token) i;
+ if (t == tok_audio_end || t >= tok_code0) {
+ cand.emplace_back(logits_cond[i], i);
+ }
+ }
+ const size_t k = std::min(cand.size(), (size_t) top_k);
+ std::nth_element(cand.begin(), cand.begin() + (k - 1), cand.end(), std::greater<std::pair<float, size_t>>());
+ const float thr = cand[k - 1].first;
+
+ std::vector<std::pair<float, size_t>> guided;
+ for (const auto & c : cand) {
+ if (c.first >= thr) {
+ const float u = use_cfg ? logits_uncond[c.second] : c.first;
+ guided.emplace_back(u + (c.first - u) * cfg_scale, c.second);
+ }
+ }
+ std::sort(guided.begin(), guided.end(), std::greater<std::pair<float, size_t>>());
+ if (temp <= 0.0f) {
+ return tok_audio_end + (llama_token) guided[0].second;
+ }
+ // values below the k-th one are masked, ties stay
+ const float gthr = guided[std::min(guided.size(), (size_t) top_k) - 1].first;
+ std::vector<double> p;
+ double sum = 0.0;
+ for (const auto & g : guided) {
+ if (g.first < gthr) {
+ break;
+ }
+ p.push_back(std::exp((double) (g.first - guided[0].first) / temp));
+ sum += p.back();
+ }
+ double u = std::uniform_real_distribution<double>(0.0, sum)(rng);
+ for (size_t i = 0; i < p.size(); i++) {
+ u -= p[i];
+ if (u <= 0.0) {
+ return tok_audio_end + (llama_token) guided[i].second;
+ }
+ }
+ return tok_audio_end + (llama_token) guided[p.size() - 1].second;
+ }
+
+ // one frame: semantic code, depth decoder, condition row, then the next backbone input
+ bool gen_frame() {
+ const llama_token tok = sample();
+ if (tok == tok_audio_end) {
+ done = true;
+ return true;
+ }
+
+ mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
+ inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
+ inp.code0 = tok - tok_code0;
+ inp.feats = h_rows.data();
+ inp.n_feats = h_rows.size();
+ inp.cfg_scale = use_cfg ? cfg_scale : 1.0f;
+ inp.temp = temp;
+ mtmd_gen_out out{};
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || out.n_feats != (size_t) n_embd) {
+ LOG_ERR("mtmd_helper_gen_audio: depth decoder failed\n");
+ return false;
+ }
+ if (n_gen++ > 0) {
+ cond_rows.insert(cond_rows.end(), out.feats, out.feats + out.n_feats);
+ if (n_frames() >= max_audio_frames) {
+ done = true;
+ return true;
+ }
+ }
+ if (n_pos_needed(pos + 1) > (int64_t) llama_n_ctx_seq(lctx)) {
+ LOG_WRN("mtmd_helper_gen_audio: the context is full, the song ends at %zu frames\n", n_frames());
+ done = true;
+ return true;
+ }
+ return decode_embd(out.embd);
+ }
+
+ // flow matching and vocoder of one window, then the crop of the reference
+ bool run_window(bool last) {
+ const size_t s = window_start(n_win_done);
+ const size_t e = std::min(s + win_frames, n_frames());
+ mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
+ inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
+ inp.feats = cond_rows.data() + (s - n_rows_dropped) * n_embd;
+ inp.n_feats = (e - s) * n_embd;
+ inp.n_steps = flow_steps;
+ mtmd_gen_out out{};
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: flow matching failed\n");
+ return false;
+ }
+ const size_t n_ch = (size_t) info.n_channels;
+ const size_t n = out.n_samples / n_ch;
+ const size_t left = n_win_done == 0 ? 0 : crop_left * hop_samples;
+ const size_t right = last ? 0 : crop_right * hop_samples;
+ if (n < left + right) {
+ LOG_ERR("mtmd_helper_gen_audio: window is too short\n");
+ return false;
+ }
+ audio_pcm.insert(audio_pcm.end(), out.audio + left * n_ch, out.audio + (n - right) * n_ch);
+ n_win_done++;
+ // the rows before the next window are not needed anymore
+ const size_t n_drop = std::min(window_start(n_win_done), n_frames()) - n_rows_dropped;
+ cond_rows.erase(cond_rows.begin(), cond_rows.begin() + n_drop * n_embd);
+ n_rows_dropped += n_drop;
+ return true;
+ }
+
+ llama_seq_id seq_id = 0;
+ llama_seq_id seq_uncond = 0;
+ bool use_cfg = false;
+ float cfg_scale = 1.0f;
+ float temp = 1.0f;
+ int32_t flow_steps = 0; // 0 = the model default
+ llama_pos pos = 0;
+ std::vector<llama_token> prompt;
+ std::vector<llama_token> prompt_uncond;
+ int32_t prompt_pos = 0;
+ int32_t uprompt_pos = 0;
+ std::vector<float> logits_cond; // see save_outputs()
+ std::vector<float> logits_uncond;
+ std::vector<float> h_rows; // backbone hidden of the cond, then uncond sequence
+ size_t n_gen = 0; // frames sampled, the first one included
+ std::vector<float> cond_rows; // one condition row per frame, from frame n_rows_dropped
+ size_t n_rows_dropped = 0;
+ size_t n_win_done = 0;
+ bool done = false;
+ bool decoded = false;
+ std::mt19937 rng;
+
+ mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
+ bool stream = false;
+ std::vector<float> audio_pcm; // interleaved
+ std::vector<char> out_buf;
+ size_t pcm_sent = 0;
+ bool wav_header_sent = false;
+};
+
static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
switch (mtmd_gen_audio_get_info(mctx).type) {
case MTMD_GEN_AUDIO_TYPE_QWEN3TTS:
@@ -3828,6 +4288,8 @@ static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lc
return std::unique_ptr<mtmd_gen_audio_pipeline>(new zonos3_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_YUE2:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new yue2_gen_audio_pipeline(lctx, mctx));
+ case MTMD_GEN_AUDIO_TYPE_MINIMAX_MUSIC3:
+ return std::unique_ptr<mtmd_gen_audio_pipeline>(new music3_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_FISH_S2:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new fish_s2_gen_audio_pipeline(lctx, mctx));
default:
diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h
index eb93a1f8d..37ba390f9 100644
--- a/tools/mtmd/mtmd-impl.h
+++ b/tools/mtmd/mtmd-impl.h
@@ -614,6 +614,7 @@ enum projector_type {
PROJECTOR_TYPE_ZONOS3_GEN,
PROJECTOR_TYPE_ZONOS3_SPKENC,
PROJECTOR_TYPE_YUE2_GEN,
+ PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN,
PROJECTOR_TYPE_FISH_S2_GEN,
PROJECTOR_TYPE_FISH_S2_SPKENC,
PROJECTOR_TYPE_MUSE_GLIMMER,
@@ -709,6 +710,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_ZONOS3_GEN, "zonos3_gen"},
{ PROJECTOR_TYPE_ZONOS3_SPKENC, "zonos3_spkenc"},
{ PROJECTOR_TYPE_YUE2_GEN, "yue2_gen"},
+ { PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN, "minimax_music3_gen"},
{ PROJECTOR_TYPE_FISH_S2_GEN, "fish_s2_gen"},
{ PROJECTOR_TYPE_FISH_S2_SPKENC, "fish_s2_spkenc"},
{ PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"},
diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp
index a183a1a67..29531db2e 100644
--- a/tools/mtmd/mtmd-model.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -1097,6 +1097,10 @@ struct mtmd_model_loader {
{
hparams.audio_sample_rate = 48000;
} break;
+ case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
+ {
+ hparams.audio_sample_rate = 44100;
+ } break;
case PROJECTOR_TYPE_DRAMA3_GEN:
case PROJECTOR_TYPE_FISH_S2_GEN:
{
@@ -1759,6 +1763,44 @@ struct mtmd_model_loader {
c2w.dac_post_conv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_POST_CONV, "bias"));
};
+ // a.gen.vae.*, see mtmd_snake_vae
+ auto load_snake_vae = [&]() {
+ auto & vae = model.snake_vae;
+ const std::string p = "a.gen.vae";
+ auto load_snake = [&](const std::string & q) {
+ return mtmd_snake_vae::snake{ get_tensor(q + ".alpha"), get_tensor(q + ".beta") };
+ };
+ vae.in_proj_w = get_tensor(p + ".in_proj.weight", false);
+ vae.in_proj_b = get_tensor(p + ".in_proj.bias", false);
+ vae.in_w = get_tensor(p + ".conv_in.weight");
+ vae.in_b = get_tensor(p + ".conv_in.bias");
+ for (int b = 0; gguf_find_tensor(ctx_gguf.get(), mtmd_format("%s.blk.%d.up.weight", p.c_str(), b).c_str()) >= 0; b++) {
+ const std::string q = p + ".blk." + std::to_string(b);
+ mtmd_snake_vae::block blk;
+ blk.act = load_snake(q + ".snake");
+ blk.up_w = get_tensor(q + ".up.weight");
+ blk.up_b = get_tensor(q + ".up.bias");
+ for (int r = 0; r < 3; r++) {
+ const std::string rq = q + ".res." + std::to_string(r);
+ mtmd_snake_vae::res_unit ru;
+ ru.act1 = load_snake(rq + ".snake1");
+ ru.conv1_w = get_tensor(rq + ".conv1.weight");
+ ru.conv1_b = get_tensor(rq + ".conv1.bias");
+ ru.act2 = load_snake(rq + ".snake2");
+ ru.conv2_w = get_tensor(rq + ".conv2.weight");
+ ru.conv2_b = get_tensor(rq + ".conv2.bias");
+ blk.res.push_back(ru);
+ }
+ vae.blocks.push_back(std::move(blk));
+ }
+ vae.post = load_snake(p + ".post_snake");
+ vae.out_w = get_tensor(p + ".conv_out.weight");
+ vae.out_b = get_tensor(p + ".conv_out.bias", false);
+ if (vae.blocks.empty()) {
+ throw std::runtime_error(mtmd_format("%s: no VAE decoder blocks\n", __func__));
+ }
+ };
+
switch (model.proj_type) {
case PROJECTOR_TYPE_MLP:
case PROJECTOR_TYPE_MLP_NORM:
@@ -2926,42 +2968,81 @@ struct mtmd_model_loader {
read_host("a.gen.nar.time.2.weight", y.time2_w, n_embd, n_embd);
read_host("a.gen.nar.time.2.bias", y.time2_b, n_embd, 1);
- const std::string p = "a.gen.vae";
- auto load_snake = [&](const std::string & q) {
- return mtmd_yue2::snake{ get_tensor(q + ".alpha"), get_tensor(q + ".beta") };
- };
- y.vae_in_w = get_tensor(p + ".conv_in.weight");
- y.vae_in_b = get_tensor(p + ".conv_in.bias");
- for (int b = 0; gguf_find_tensor(ctx_gguf.get(), mtmd_format("%s.blk.%d.up.weight", p.c_str(), b).c_str()) >= 0; b++) {
- const std::string q = p + ".blk." + std::to_string(b);
- mtmd_yue2::block blk;
- blk.act = load_snake(q + ".snake");
- blk.up_w = get_tensor(q + ".up.weight");
- blk.up_b = get_tensor(q + ".up.bias");
- for (int r = 0; r < 3; r++) {
- const std::string rq = q + ".res." + std::to_string(r);
- mtmd_yue2::res_unit ru;
- ru.act1 = load_snake(rq + ".snake1");
- ru.conv1_w = get_tensor(rq + ".conv1.weight");
- ru.conv1_b = get_tensor(rq + ".conv1.bias");
- ru.act2 = load_snake(rq + ".snake2");
- ru.conv2_w = get_tensor(rq + ".conv2.weight");
- ru.conv2_b = get_tensor(rq + ".conv2.bias");
- blk.res.push_back(ru);
- }
- y.vae.push_back(std::move(blk));
+ // the tile cropping in mtmd_gen_wav_yue2() depends on the total stride
+ load_snake_vae();
+ const auto & vae = model.snake_vae;
+ if (vae.ratio() != mtmd_graph_yue2_gen::vae_ratio || vae.n_latent() != mtmd_graph_yue2_gen::n_latent ||
+ vae.out_w->ne[2] != mtmd_graph_yue2_gen::n_channels) {
+ throw std::runtime_error(mtmd_format("%s: yue2: unexpected VAE decoder layout\n", __func__));
+ }
+ } break;
+ case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
+ {
+ // generic a.gen.code.* tensors (depth decoder layers, position_embd, post_ln) are loaded above
+ using g = mtmd_graph_music3_gen;
+ model.gen_code_proj_in_w = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "weight"));
+ model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight"));
+ model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight"));
+ model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
+ // the depth decoder sees the backbone hidden, then one row per code (all but the last)
+ const int64_t n_embd = hparams.n_embd;
+ if (!model.post_ln_w || !model.position_embeddings || n_embd != hparams.projection_dim ||
+ model.gen_code_proj_in_w->ne[0] != n_embd || model.gen_code_proj_in_w->ne[1] != n_embd ||
+ model.gen_code_embd_w->ne[0] != n_embd || model.gen_code_head_w->ne[0] != n_embd ||
+ model.gen_code_out_embd_w->ne[0] != n_embd || model.position_embeddings->ne[0] != n_embd ||
+ model.gen_code_embd_w->ne[2] != model.gen_code_head_w->ne[2] ||
+ model.gen_code_embd_w->ne[1] != model.gen_code_head_w->ne[1] ||
+ model.position_embeddings->ne[1] < model.gen_code_head_w->ne[2] + 1) {
+ throw std::runtime_error(mtmd_format("%s: minimax_music3: unexpected depth decoder layout\n", __func__));
}
- y.vae_post = load_snake(p + ".post_snake");
- y.vae_out_w = get_tensor(p + ".conv_out.weight");
- // the tile cropping in mtmd_gen_wav_yue2() depends on the total stride
- int64_t ratio = 1;
- for (const auto & blk : y.vae) {
- ratio *= blk.up_w->ne[0] / 2;
+ auto & m = model.music3;
+ m.cond_layer_w = get_tensor("a.gen.cond.layer_w");
+ m.cond_w = get_tensor("a.gen.cond.proj.weight");
+ m.cond_b = get_tensor("a.gen.cond.proj.bias");
+ m.time_freq = get_tensor("a.gen.flow.time_freq");
+ m.time_in_w = get_tensor("a.gen.flow.time_in.weight");
+ m.time_in_b = get_tensor("a.gen.flow.time_in.bias");
+ m.time_out_w = get_tensor("a.gen.flow.time_out.weight");
+ m.time_out_b = get_tensor("a.gen.flow.time_out.bias");
+ m.pre_w = get_tensor("a.gen.flow.pre.weight");
+ m.proj_in_w = get_tensor("a.gen.flow.proj_in.weight");
+ m.proj_out_w = get_tensor("a.gen.flow.proj_out.weight");
+ m.post_w = get_tensor("a.gen.flow.post.weight");
+ for (int il = 0; gguf_find_tensor(ctx_gguf.get(), mtmd_format(TN_ATTN_Q, "a.gen.flow", il, "weight").c_str()) >= 0; il++) {
+ const char * q = "a.gen.flow";
+ mtmd_layer layer;
+ layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, q, il, "weight"));
+ layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, q, il, "bias"));
+ layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, q, il, "weight"));
+ layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, q, il, "weight"));
+ layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, q, il, "weight"));
+ layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, q, il, "weight"));
+ layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, q, il, "weight"));
+ layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, q, il, "bias"));
+ layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, q, il, "weight"));
+ layer.ff_up_b = get_tensor(mtmd_format(TN_FFN_UP, q, il, "bias"));
+ layer.ff_gate_w = get_tensor(mtmd_format(TN_FFN_GATE, q, il, "weight"));
+ layer.ff_gate_b = get_tensor(mtmd_format(TN_FFN_GATE, q, il, "bias"));
+ layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, q, il, "weight"));
+ layer.ff_down_b = get_tensor(mtmd_format(TN_FFN_DOWN, q, il, "bias"));
+ m.dit.push_back(layer);
}
- if (ratio != mtmd_graph_yue2_gen::vae_ratio || y.vae_in_w->ne[1] != mtmd_graph_yue2_gen::n_latent ||
- y.vae_out_w->ne[2] != mtmd_graph_yue2_gen::n_channels) {
- throw std::runtime_error(mtmd_format("%s: yue2: unexpected VAE decoder layout\n", __func__));
+
+ load_snake_vae();
+ const auto & vae = model.snake_vae;
+ const int64_t n_dit_embd = m.proj_in_w->ne[1];
+ const int64_t n_cond = m.cond_w->ne[2];
+ if (m.dit.empty() || m.cond_layer_w->ne[0] != model.gen_code_head_w->ne[2] + 1 ||
+ m.cond_w->ne[0] != 3 || m.cond_w->ne[1] != hparams.n_embd ||
+ m.pre_w->ne[0] != 2 * g::n_latent + n_cond || m.pre_w->ne[1] != m.pre_w->ne[0] ||
+ m.proj_in_w->ne[0] != m.pre_w->ne[0] || m.proj_out_w->ne[0] != n_dit_embd || m.proj_out_w->ne[1] != g::n_latent ||
+ m.post_w->ne[0] != g::n_latent || m.post_w->ne[1] != g::n_latent || m.cond_b->ne[0] != n_cond ||
+ m.time_freq->ne[0] * 2 != m.time_in_w->ne[0] || m.time_in_w->ne[1] != m.time_out_w->ne[0] ||
+ m.time_in_b->ne[0] != m.time_in_w->ne[1] || m.time_out_w->ne[1] != n_dit_embd || m.time_out_b->ne[0] != n_dit_embd ||
+ n_dit_embd % g::dit_d_head != 0 ||
+ vae.ratio() != g::vae_ratio || vae.n_latent() * 2 != g::n_latent || vae.out_w->ne[2] != 1) {
+ throw std::runtime_error(mtmd_format("%s: minimax_music3: unexpected DiT or vocoder layout\n", __func__));
}
} break;
case PROJECTOR_TYPE_DRAMA3_GEN:
@@ -4610,6 +4691,7 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
case PROJECTOR_TYPE_DRAMA3_GEN:
case PROJECTOR_TYPE_ZONOS3_GEN:
case PROJECTOR_TYPE_YUE2_GEN:
+ case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
case PROJECTOR_TYPE_FISH_S2_GEN:
{
// one hidden-state vector fed back to the talker per call
@@ -4811,6 +4893,8 @@ int mtmd_model::n_mmproj_embd() const {
return gen_code_out_embd_w->ne[0];
case PROJECTOR_TYPE_YUE2_GEN:
return yue2.inp_w->ne[1];
+ case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
+ return gen_code_out_embd_w->ne[0];
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
return spk_proj_w->ne[1];
case PROJECTOR_TYPE_POCKETTTS_GEN:
diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h
index 8a2575044..1929f9de1 100644
--- a/tools/mtmd/mtmd-model.h
+++ b/tools/mtmd/mtmd-model.h
@@ -943,12 +943,12 @@ struct mtmd_zonos3_wav {
int64_t rs_kernel_len = 0;
};
-// yue2: projections of the flow-matching NAR (a.gen.nar.*) and the Oobleck VAE decoder (a.gen.vae.*)
-// the NAR transformer layers are loaded by libllama, see mtmd_yue2_nar_load()
-struct mtmd_yue2 {
+// snake-activated conv decoder of the DAC / Oobleck family (a.gen.vae.*): yue2 VAE, minimax_music3 vocoder
+struct mtmd_snake_vae {
+ // x + beta * sin(alpha * x)^2, alpha and beta folded at conversion
struct snake {
- ggml_tensor * alpha = nullptr; // exp(alpha)
- ggml_tensor * beta = nullptr; // 1 / (exp(beta) + eps)
+ ggml_tensor * alpha = nullptr;
+ ggml_tensor * beta = nullptr;
};
struct res_unit {
snake act1;
@@ -965,6 +965,31 @@ struct mtmd_yue2 {
std::vector<res_unit> res;
};
+ ggml_tensor * in_proj_w = nullptr; // optional k = 1 conv before conv_in
+ ggml_tensor * in_proj_b = nullptr;
+ ggml_tensor * in_w = nullptr;
+ ggml_tensor * in_b = nullptr;
+ std::vector<block> blocks;
+ snake post;
+ ggml_tensor * out_w = nullptr;
+ ggml_tensor * out_b = nullptr; // optional
+
+ // output samples per latent frame
+ int64_t ratio() const {
+ int64_t r = 1;
+ for (const auto & blk : blocks) {
+ r *= blk.up_w->ne[0] / 2;
+ }
+ return r;
+ }
+ int64_t n_latent() const {
+ return (in_proj_w ? in_proj_w : in_w)->ne[1];
+ }
+};
+
+// yue2: projections of the flow-matching NAR (a.gen.nar.*), the VAE is mtmd_model::snake_vae
+// the NAR transformer layers are loaded by libllama, see mtmd_yue2_nar_load()
+struct mtmd_yue2 {
ggml_tensor * inp_w = nullptr; // latent -> backbone embd
ggml_tensor * inp_b = nullptr;
ggml_tensor * out_w = nullptr; // backbone hidden -> velocity
@@ -975,12 +1000,25 @@ struct mtmd_yue2 {
std::vector<float> time0_b;
std::vector<float> time2_w; // [n_embd, n_embd]
std::vector<float> time2_b;
+};
- ggml_tensor * vae_in_w = nullptr;
- ggml_tensor * vae_in_b = nullptr;
- std::vector<block> vae;
- snake vae_post;
- ggml_tensor * vae_out_w = nullptr;
+// minimax_music3: condition encoder (a.gen.cond.*) and flow-matching DiT (a.gen.flow.*)
+// the depth decoder is the generic a.gen.code.* layers, the vocoder is mtmd_model::snake_vae
+struct mtmd_music3 {
+ ggml_tensor * cond_layer_w = nullptr; // mix of the hidden states of a frame, softmax and scale folded in
+ ggml_tensor * cond_w = nullptr; // k = 3 conv
+ ggml_tensor * cond_b = nullptr;
+
+ ggml_tensor * time_freq = nullptr; // random Fourier features, 2 * pi folded in
+ ggml_tensor * time_in_w = nullptr;
+ ggml_tensor * time_in_b = nullptr;
+ ggml_tensor * time_out_w = nullptr;
+ ggml_tensor * time_out_b = nullptr;
+ ggml_tensor * pre_w = nullptr; // k = 1 conv over [latent, zeros, cond], residual
+ ggml_tensor * proj_in_w = nullptr;
+ ggml_tensor * proj_out_w = nullptr;
+ ggml_tensor * post_w = nullptr; // k = 1 conv, residual
+ std::vector<mtmd_layer> dit;
};
struct mtmd_vocos {
@@ -1267,6 +1305,8 @@ struct mtmd_model {
std::vector<mtmd_drama3_layer> drama3_layers;
mtmd_zonos3_wav zonos3;
mtmd_yue2 yue2;
+ mtmd_music3 music3;
+ mtmd_snake_vae snake_vae; // yue2, minimax_music3
// pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path)
mtmd_seanet seanet;
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index 4448f84f3..321b6171b 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -1510,6 +1510,40 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
{
set_input_f32("inp_feats", *params->feats);
} break;
+ case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
+ {
+ set_input_f32("inp_feats", *params->feats);
+ if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE) {
+ if (params->code0 < 0 || (int64_t) params->code0 >= model.gen_code_out_embd_w->ne[1]) {
+ LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) model.gen_code_out_embd_w->ne[1]);
+ return false;
+ }
+ std::vector<int32_t> code0 = { params->code0 };
+ set_input_i32("inp_code0", code0);
+ // greedy (temp <= 0) has no random input
+ if (ggml_graph_get_tensor(gf, "inp_rand") != nullptr) {
+ std::uniform_real_distribution<float> dist(0.0f, 1.0f);
+ std::vector<float> r(model.gen_code_head_w->ne[2]);
+ for (auto & v : r) {
+ v = dist(params->gen_state->rng);
+ }
+ set_input_f32("inp_rand", r);
+ }
+ } else if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_FLOW_COND) {
+ // F.interpolate(mode="nearest") of the reference
+ ggml_tensor * t = get_inp_tensor("inp_idx");
+ const int64_t n_in = (int64_t) params->feats->size() / model.hparams.n_embd;
+ const int64_t n_out = t->ne[0];
+ const float scale = (float) n_in / (float) n_out;
+ std::vector<int32_t> idx(n_out);
+ for (int64_t i = 0; i < n_out; i++) {
+ idx[i] = (int32_t) std::min<int64_t>((int64_t) floorf((float) i * scale), n_in - 1);
+ }
+ set_input_i32("inp_idx", idx);
+ } else if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_FLOW_STEP) {
+ set_input_f32("inp_t", { params->flow_t });
+ }
+ } break;
case PROJECTOR_TYPE_GEMMA4V:
case PROJECTOR_TYPE_GEMMA4UV:
{
@@ -2633,8 +2667,9 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
mtmd_graph_soprano::decode_spectrum(model, spectrum, out_audio);
}
- // drop the tail audio that comes from the code-0 rear padding, drama3, fish_s2 and yue2 have no padding
- if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN && model.proj_type != PROJECTOR_TYPE_FISH_S2_GEN && model.proj_type != PROJECTOR_TYPE_YUE2_GEN) {
+ // drop the tail audio that comes from the code-0 rear padding, drama3, fish_s2, yue2 and minimax_music3 have no padding
+ if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN && model.proj_type != PROJECTOR_TYPE_FISH_S2_GEN && model.proj_type != PROJECTOR_TYPE_YUE2_GEN &&
+ model.proj_type != PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN) {
const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0;
const int64_t n_frames_w = hparams.wav_tfm_swa;
const int64_t n_frames = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w;
diff --git a/tools/mtmd/mtmd-subcontext.h b/tools/mtmd/mtmd-subcontext.h
index f03fbe5ca..d43acec04 100644
--- a/tools/mtmd/mtmd-subcontext.h
+++ b/tools/mtmd/mtmd-subcontext.h
@@ -71,6 +71,8 @@ enum mtmd_subctx_gen_process {
MTMD_SUBCTX_GEN_PROCESS_TOKENIZE, // raw PCM audio to semantic speech tokens
MTMD_SUBCTX_GEN_PROCESS_LATENT_IN, // internal: flow latent rows to backbone embd rows
MTMD_SUBCTX_GEN_PROCESS_LATENT_OUT, // internal: backbone hidden rows to flow velocity rows
+ MTMD_SUBCTX_GEN_PROCESS_FLOW_COND, // internal: frame rows to flow condition rows
+ MTMD_SUBCTX_GEN_PROCESS_FLOW_STEP, // internal: one velocity pass of the flow transformer
};
// persistent state of a causal audio encoder, for streaming input (one per stream)
@@ -127,6 +129,8 @@ struct mtmd_subctx_encode_params {
std::vector<int32_t> * out_codes = nullptr; // this frame's 16 sampled codes
std::vector<float> * out_feats = nullptr; // continuous counterpart of out_codes
float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders
+ float cfg_scale = 1.0f; // classifier-free guidance, 1.0 = off
+ float flow_t = 0.0f; // FLOW_STEP: flow-matching time, 0 = noise
bool * out_is_eos = nullptr;
// TOKENIZE: speech embedding rows of the produced codes
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 8696525c5..7821cdfbd 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -2606,6 +2606,11 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
info.sample_rate = 48000;
info.n_channels = mtmd_graph_yue2_gen::n_channels;
break;
+ case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
+ info.type = MTMD_GEN_AUDIO_TYPE_MINIMAX_MUSIC3;
+ info.sample_rate = 44100;
+ info.n_channels = mtmd_graph_music3_gen::n_channels;
+ break;
case PROJECTOR_TYPE_FISH_S2_GEN:
info.type = MTMD_GEN_AUDIO_TYPE_FISH_S2;
info.sample_rate = 44100;
@@ -3028,6 +3033,12 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
case PROJECTOR_TYPE_SOPRANO:
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
break;
+ case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
+ // top-k 50 is fixed in the graph, like the reference
+ inp.top_k = 50;
+ inp.top_p = 1.0f;
+ inp.temp = 1.0f;
+ break;
case PROJECTOR_TYPE_DRAMA3_GEN:
// codebook heads, same as the reference (sglang-omni fishaudio_drama3)
inp.top_k = 30;
@@ -3076,6 +3087,10 @@ struct mtmd_gen_state {
// zonos3: the iSTFT head runs on the host
std::unique_ptr<mtmd_audio_zonos3_synth> zonos3_synth;
+ // minimax_music3: tail of the previous window, the next one starts from it, see mtmd_gen_wav_music3()
+ std::vector<float> flow_prev_latent;
+ std::vector<float> flow_prev_cond;
+
~mtmd_gen_state() {
delete gen_state;
}
@@ -3116,6 +3131,8 @@ void mtmd_gen_state_reset(mtmd_gen_state * state) {
state->istft = std::make_unique<mtmd_audio_streaming_istft>(hparams->audio_n_fft, hparams->audio_hop_len);
}
state->zonos3_synth.reset();
+ state->flow_prev_latent.clear();
+ state->flow_prev_cond.clear();
}
// zonos3 GEN_WAV: n_codes == 0 flushes and resets the state
@@ -3446,6 +3463,155 @@ static int32_t mtmd_gen_wav_yue2(mtmd_context * ctx, mtmd_gen_state * state, con
return 0;
}
+
+// minimax_music3: one pass of an mtmd graph on rows of floats
+static bool music3_encode(mtmd_context * ctx, mtmd_gen_state * state, mtmd_subctx_encode_params & params, const std::vector<float> & in) {
+ mtmd_image_f32 dummy;
+ dummy.set_size({1, 1}, false, true);
+ dummy.cpy_buf(std::vector<float>(1, 0.0f));
+ mtmd_image_f32_batch batch;
+ batch.is_audio = true;
+ batch.entries.push_back(std::move(dummy));
+
+ params.imgs = &batch;
+ params.n_threads = ctx->n_threads;
+ params.feats = ∈
+ params.gen_state = state->gen_state;
+ return ctx->ctx_gen_a->encode(¶ms);
+}
+
+// minimax_music3 GEN_CODE: the depth decoder of one frame, see _generate_depth_codes() of the reference (encoders.py)
+static int32_t mtmd_gen_code_music3(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
+ using g = mtmd_graph_music3_gen;
+ const size_t n_embd = (size_t) ctx->ctx_gen_a->model.n_mmproj_embd();
+ if (!inp->feats || (inp->n_feats != n_embd && inp->n_feats != 2 * n_embd)) {
+ LOG_ERR("%s: minimax_music3 needs 1 or 2 backbone hidden rows\n", __func__);
+ return 1;
+ }
+ const std::vector<float> rows(inp->feats, inp->feats + inp->n_feats);
+
+ std::vector<float> out_embd(n_embd);
+ mtmd_subctx_encode_params params;
+ params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
+ params.out_embd = &out_embd;
+ params.out_codes = &state->out_codes;
+ params.out_feats = &state->out_feats;
+ params.code0 = inp->code0;
+ params.cfg_scale = inp->cfg_scale > 0.0f ? inp->cfg_scale : g::depth_cfg;
+ params.temp = inp->temp;
+ if (!music3_encode(ctx, state, params, rows)) {
+ LOG_ERR("%s: encode failed (depth decoder)\n", __func__);
+ return 1;
+ }
+ state->out_embd = std::move(out_embd);
+
+ out->embd = state->out_embd.data();
+ out->codes = state->out_codes.data();
+ out->n_codes = state->out_codes.size();
+ out->feats = state->out_feats.data();
+ out->n_feats = state->out_feats.size();
+ return 0;
+}
+
+// minimax_music3 GEN_WAV: condition rows of one window -> flow matching -> vocoder, see denoise.py of the reference
+// a window starts from the tail of the previous one, kept in the state
+static int32_t mtmd_gen_wav_music3(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
+ using g = mtmd_graph_music3_gen;
+ const auto & m = ctx->ctx_gen_a->model.music3;
+ const size_t n_embd = (size_t) ctx->ctx_gen_a->model.hparams.n_embd;
+ const size_t n_cond = (size_t) m.cond_w->ne[2];
+ const size_t n_in = (size_t) m.pre_w->ne[0]; // [latent, zeros, condition]
+ const size_t n_lc = g::n_latent;
+ if (!inp->feats || inp->n_feats == 0 || inp->n_feats % n_embd != 0) {
+ LOG_ERR("%s: minimax_music3 needs whole condition rows\n", __func__);
+ return 1;
+ }
+ const std::vector<float> rows(inp->feats, inp->feats + inp->n_feats);
+
+ std::vector<float> cond;
+ {
+ mtmd_subctx_encode_params params;
+ params.gen_process = MTMD_SUBCTX_GEN_PROCESS_FLOW_COND;
+ params.out_feats = &cond;
+ if (!music3_encode(ctx, state, params, rows)) {
+ LOG_ERR("%s: encode failed (condition)\n", __func__);
+ return 1;
+ }
+ }
+ const size_t n_lat = cond.size() / n_cond;
+
+ // the overlap takes the condition of the previous window, its latents are blended toward the previous ones
+ auto & prev_lat = state->flow_prev_latent;
+ auto & prev_cond = state->flow_prev_cond;
+ const size_t n_ov = std::min(prev_lat.size() / n_lc, n_lat);
+ std::copy(prev_cond.begin(), prev_cond.begin() + n_ov * n_cond, cond.begin());
+
+ std::normal_distribution<float> dist(0.0f, 1.0f);
+ std::vector<float> x(n_lat * n_lc);
+ for (auto & v : x) {
+ v = dist(state->gen_state->rng);
+ }
+ const std::vector<float> noise_ov(x.begin(), x.begin() + n_ov * n_lc);
+
+ // batch 0 is conditioned, batch 1 has a zero condition
+ std::vector<float> dit_in(2 * n_lat * n_in, 0.0f);
+ for (size_t i = 0; i < n_lat; i++) {
+ std::copy(cond.begin() + i * n_cond, cond.begin() + (i + 1) * n_cond, dit_in.begin() + i * n_in + 2 * n_lc);
+ }
+
+ // FlowMatchEulerDiscreteScheduler with invert_sigmas: t = 1 - linspace(1, 1 / n, n), then 1
+ const int n_steps = inp->n_steps > 0 ? inp->n_steps : g::flow_steps;
+ std::vector<float> ts(n_steps + 1, 1.0f);
+ for (int i = 0; i < n_steps; i++) {
+ const double sigma = n_steps > 1 ? 1.0 + i * (1.0 / n_steps - 1.0) / (n_steps - 1) : 1.0;
+ ts[i] = 1.0f - (float) sigma;
+ }
+
+ std::vector<float> vel;
+ for (int i = 0; i < n_steps; i++) {
+ const float t = ts[i];
+ for (size_t j = 0; j < n_ov * n_lc; j++) {
+ x[j] = (1.0f - (1.0f - 1e-6f) * t) * noise_ov[j] + t * prev_lat[j];
+ }
+ for (size_t b = 0; b < 2; b++) {
+ for (size_t r = 0; r < n_lat; r++) {
+ std::copy(x.begin() + r * n_lc, x.begin() + (r + 1) * n_lc, dit_in.begin() + (b * n_lat + r) * n_in);
+ }
+ }
+ mtmd_subctx_encode_params params;
+ params.gen_process = MTMD_SUBCTX_GEN_PROCESS_FLOW_STEP;
+ params.flow_t = t;
+ params.cfg_scale = g::dit_cfg;
+ params.out_feats = &vel;
+ if (!music3_encode(ctx, state, params, dit_in) || vel.size() != x.size()) {
+ LOG_ERR("%s: encode failed (flow step %d)\n", __func__, i);
+ return 1;
+ }
+ const float dt = ts[i + 1] - ts[i];
+ for (size_t j = 0; j < x.size(); j++) {
+ x[j] += dt * vel[j];
+ }
+ }
+ std::copy(prev_lat.begin(), prev_lat.begin() + n_ov * n_lc, x.begin());
+
+ // the next window starts from latent frames [L - 2 * n_overlap, L - n_overlap) of this one
+ const size_t s0 = n_lat > 2 * (size_t) g::n_overlap ? n_lat - 2 * g::n_overlap : 0;
+ const size_t s1 = n_lat > (size_t) g::n_overlap ? n_lat - g::n_overlap : 0;
+ prev_lat.assign(x.begin() + s0 * n_lc, x.begin() + s1 * n_lc);
+ prev_cond.assign(cond.begin() + s0 * n_cond, cond.begin() + s1 * n_cond);
+
+ mtmd_subctx_encode_params params;
+ params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV;
+ params.out_audio = &state->out_audio;
+ if (!music3_encode(ctx, state, params, x)) {
+ LOG_ERR("%s: encode failed (vocoder)\n", __func__);
+ return 1;
+ }
+ out->audio = state->out_audio.data();
+ out->n_samples = state->out_audio.size();
+ return 0;
+}
+
int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) {
if (!state || state->sctx != ctx->ctx_gen_a.get()) {
LOG_ERR("%s: invalid gen state\n", __func__);
@@ -3476,6 +3642,10 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
return inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE ? mtmd_gen_flow_yue2(ctx, state, inp, out)
: mtmd_gen_wav_yue2(ctx, state, inp, out);
}
+ if (sctx->model.proj_type == PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN) {
+ return inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE ? mtmd_gen_code_music3(ctx, state, inp, out)
+ : mtmd_gen_wav_music3(ctx, state, inp, out);
+ }
if (sctx->model.proj_type == PROJECTOR_TYPE_SOPRANO) {
const size_t n_embd = (size_t) sctx->model.n_mmproj_embd();
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index 9ad2c4e2f..3829b6078 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -474,6 +474,7 @@ enum mtmd_gen_audio_type {
MTMD_GEN_AUDIO_TYPE_DRAMA3,
MTMD_GEN_AUDIO_TYPE_ZONOS3,
MTMD_GEN_AUDIO_TYPE_YUE2,
+ MTMD_GEN_AUDIO_TYPE_MINIMAX_MUSIC3,
MTMD_GEN_AUDIO_TYPE_FISH_S2,
};
@@ -504,6 +505,7 @@ enum mtmd_gen_process_type {
// for pocket-tts, this is mimi decoder
// for lfm3a, this is the codec decoder + ISTFT, n_codes == 0 flushes the tail
// for yue2, this is one VAE pass, the caller splits a long song into tiles
+ // for minimax_music3, this is one window: flow matching + vocoder
};
struct mtmd_gen_inp {
@@ -533,8 +535,12 @@ struct mtmd_gen_inp {
size_t backbone_state_size;
int32_t backbone_n_past;
- // flow-matching GEN_CODE: steps of the ODE solver, <= 0 = model default (yue2: 32)
+ // flow matching: steps of the ODE solver, <= 0 = model default (yue2 GEN_CODE: 32, minimax_music3 GEN_WAV: 30)
int32_t n_steps;
+
+ // minimax_music3 GEN_CODE: feats holds the backbone hidden rows, cond then uncond with guidance
+ // classifier-free guidance scale of the residual codes, <= 0 = model default (1.5)
+ float cfg_scale;
};
struct mtmd_gen_out {
diff --git a/tools/server/README.md b/tools/server/README.md
index 4dbb4426c..c8d25199f 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -880,9 +880,9 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For the list
`loudness`: Target loudness in LUFS, `< 0` (default `0.0`, the model's own loudness). Only for models that support it, others reject a value other than `0.0` with a 400 error.
-`cfg_scale`: Classifier-free guidance scale of the audio codes, `> 0`, `1.0` disables it (default `0.0`, the model's own). Only for models that support it (YuE2), others reject a value other than `0.0` with a 400 error.
+`cfg_scale`: Classifier-free guidance scale of the audio codes, `> 0`, `1.0` disables it (default `0.0`, the model's own). Only for models that support it (YuE2, MiniMax Music 3), others reject a value other than `0.0` with a 400 error.
-`flow_steps`: Steps of the flow-matching ODE solver, `1` to `1024` (default `0`, the model's own). Fewer steps are faster and the audio is less accurate. Only for models that support it (YuE2), others reject a value other than `0` with a 400 error.
+`flow_steps`: Steps of the flow-matching ODE solver, `1` to `1024` (default `0`, the model's own). Fewer steps are faster and the audio is less accurate. Only for models that support it (YuE2, MiniMax Music 3), others reject a value other than `0` with a 400 error.
`n_predict`: Max number of audio frames to generate. Defaults to the limit of the model (`512` for most models); generation normally stops earlier once the model emits an end-of-speech token.