Commit 92192d31e for llama.cpp
commit 92192d31e457d9e7ad68da84608f040ad72e42bb
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Mon Nov 16 20:22:23 2026 +0100
mtmd: support YuE2-3B (#32369)
* mtmd: support YuE2-3B
* improve speed
* add CFG
* fix streaming
* add cfg control from api
* check ctx size
* fix n_predict
* add flow_steps config
* nits
* fix batch api
* nits
* adapt
* mtmd: yue2 samples from its own saved logits
The context logits belong to the last decode of any slot, so with
several server slots the score and the unguided codes were sampled
from another sequence. Keep the logits of each own decode in
save_logits(), which also does the guidance mix.
* mtmd: let the yue2 snake chain fuse
The beta reshape landed between sqr and mul, so the mul, sin, sqr,
mul, add chain was not consecutive and the backends never fused it.
Expand both reshapes first, like qwen3tts does.
---------
Co-authored-by: Pascal <admin@serveurperso.com>
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 05f55fccf..497797c06 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -284,6 +284,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
"PocketTTSModel": "pockettts",
"Zonos3Model": "zonos3",
+ "YuE2ForCausalLM": "yue2",
"Qwen3TTSForConditionalGeneration": "qwen3tts",
"Qwen3VLForConditionalGeneration": "qwen3vl",
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
@@ -361,6 +362,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"Exaone4_5_ForConditionalGeneration": "exaone",
"FishOmniModel": "drama3",
"Zonos3Model": "zonos3",
+ "YuE2ForCausalLM": "yue2",
"Gemma3ForConditionalGeneration": "gemma",
"Gemma3nForConditionalGeneration": "gemma",
"Gemma4ForConditionalGeneration": "gemma",
diff --git a/conversion/yue2.py b/conversion/yue2.py
new file mode 100644
index 000000000..2242af8eb
--- /dev/null
+++ b/conversion/yue2.py
@@ -0,0 +1,234 @@
+from __future__ import annotations
+
+import base64
+from typing import Any, Callable, Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import ModelBase, MmprojModel, gguf, logger
+from .qwen import QwenModel, Qwen3Model
+
+# token ids of the yue2_infer protocol (protocol.py)
+EOD = 151643
+ABC_START, ABC_END = 151847, 151848
+MUSIC_START, MUSIC_END = 151851, 151852
+CODEC_OFFSET, CODEC_SIZE = 151853, 32768
+LATENT_START = 184621
+
+# the VAE is a separate repo (m-a-p/YuE2-Vae), downloaded into this subdir of the model dir
+VAE_DIR = "vae"
+
+INSTRUCTIONS = {
+ "off": "Generate music with codec tokens from the given conditions.",
+ "melody": "Generate a melody-only ABC transcription without chord symbols, then generate music with codec tokens from the given conditions.",
+ "full": "Generate a chord-annotated ABC transcription, then generate music with codec tokens from the given conditions.",
+}
+
+# the text is "[Tags]\n<style>\n[Lyrics]\n<lyrics>", optionally followed by "\n[ABC]\n<score>" (cover mode)
+# voice picks the planning mode: full (default), melody or off
+TTS_TEMPLATE = (
+ "{%- set text = messages[0]['content'] -%}"
+ "{%- set mode = (voice | lower) if voice is defined and voice and (voice | lower) in ['full', 'melody', 'off'] else 'full' -%}"
+ "{%- set parts = text.split('\\n[ABC]\\n') -%}"
+ "{%- set instructions = {"
+ + ", ".join(f"'{k}': '{v}'" for k, v in INSTRUCTIONS.items())
+ + "} -%}"
+ "{{- '<|endoftext|>' + instructions[mode] + '\\n' + parts[0].rstrip() + '\\n' -}}"
+ "{%- if mode == 'off' -%}"
+ "{{- '<abc></abc><|music_start|>' -}}"
+ "{%- elif parts | length > 1 -%}"
+ "{{- '<abc>' + parts[1] + '</abc><|music_start|>' -}}"
+ "{%- else -%}"
+ "{{- '<abc>' -}}"
+ "{%- endif -%}"
+)
+
+
+def _is_nar_tensor(name: str) -> bool:
+ return ".nar_" in name or name.startswith(("llm2vae.", "vae2llm.", "time_embedder.", "latent_pos_embed."))
+
+
+@ModelBase.register("YuE2ForCausalLM")
+class YuE2TextModel(Qwen3Model):
+ model_arch = gguf.MODEL_ARCH.QWEN3
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ # the NAR path goes to the mmproj
+ if _is_nar_tensor(item[0]):
+ return None
+ return super().filter_tensors(item)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ # the codes are sampled with classifier-free guidance, the uncond prompt runs as a second sequence
+ self.gguf_writer.add_classifier_free_guidance(True)
+
+ def set_vocab(self):
+ # checkpoint-native tiktoken file, see tokenization_yue2.py
+ ranks: dict[bytes, int] = {}
+ with open(self.dir_model / "qwen.tiktoken", "rb") as f:
+ for line in f.read().splitlines():
+ if line:
+ tok, rank = line.split()
+ ranks[base64.b64decode(tok)] = int(rank)
+ if len(ranks) != EOD:
+ raise ValueError(f"YuE2: expected {EOD} BPE tokens, got {len(ranks)}")
+
+ vocab_size = self.hparams["vocab_size"]
+ tokens: list[str] = [""] * vocab_size
+ toktypes: list[int] = [gguf.TokenType.UNUSED] * vocab_size
+ merges: list[str] = []
+ for tok, rank in ranks.items():
+ tokens[rank] = QwenModel.token_bytes_to_string(tok)
+ toktypes[rank] = gguf.TokenType.NORMAL
+ if len(tok) > 1:
+ merged = QwenModel.bpe(ranks, tok, max_rank=rank)
+ assert len(merged) == 2
+ merges.append(" ".join(map(QwenModel.token_bytes_to_string, merged)))
+
+ specials = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<R>", "<S>", "<X>", "<mask>", "<sep>"]
+ specials += [f"<extra_{i}>" for i in range(200)]
+ specials[204:206] = ["<abc>", "</abc>"]
+ named = {EOD + i: s for i, s in enumerate(specials)}
+ named[MUSIC_START] = "<|music_start|>"
+ named[MUSIC_END] = "<|music_end|>"
+ for i in range(CODEC_SIZE):
+ named[CODEC_OFFSET + i] = f"<|codec_{i}|>"
+ for i, s in enumerate(["<|latent_start|>", "<|latent_end|>", "<|latent_pad|>"]):
+ named[LATENT_START + i] = s
+ for i in range(EOD, vocab_size):
+ tokens[i] = named.get(i, f"[PAD{i}]")
+ toktypes[i] = gguf.TokenType.CONTROL if i in named else gguf.TokenType.UNUSED
+
+ self.gguf_writer.add_tokenizer_model("gpt2")
+ self.gguf_writer.add_tokenizer_pre("qwen2")
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_types(toktypes)
+ self.gguf_writer.add_token_merges(merges)
+ self.gguf_writer.add_bos_token_id(EOD)
+ self.gguf_writer.add_eos_token_id(MUSIC_END)
+ self.gguf_writer.add_pad_token_id(EOD)
+ self.gguf_writer.add_add_bos_token(False)
+ self.gguf_writer.add_chat_template([{"name": "tts", "template": TTS_TEMPLATE}])
+
+
+@ModelBase.register("YuE2ForCausalLM")
+class YuE2MmprojModel(MmprojModel):
+ has_vision_encoder = False
+ has_audio_encoder = False
+
+ def get_audio_config(self) -> dict[str, Any] | None:
+ # the NAR path: same shapes as the backbone layers
+ return {k: self.global_config[k] for k in ("num_hidden_layers", "hidden_size", "intermediate_size",
+ "num_attention_heads", "num_key_value_heads", "rms_norm_eps")}
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name = item[0]
+ if name == "latent_pos_embed.pe":
+ return None # sinusoidal, computed at runtime
+ if _is_nar_tensor(name) or name == "model.norm.weight":
+ return item
+ return None
+
+ def set_gguf_parameters(self):
+ hp = self.hparams
+ self.gguf_writer.add_file_type(self.ftype)
+ self.gguf_writer.add_clip_has_gen_audio_encoder(True)
+ self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.YUE2_GEN)
+ self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
+ self.gguf_writer.add_gen_audio_embedding_length(hp["hidden_size"])
+ self.gguf_writer.add_gen_audio_feed_forward_length(hp["intermediate_size"])
+ self.gguf_writer.add_gen_audio_block_count(hp["num_hidden_layers"])
+ self.gguf_writer.add_gen_audio_head_count(hp["num_attention_heads"])
+ self.gguf_writer.add_gen_audio_head_count_kv(hp["num_key_value_heads"])
+ self.gguf_writer.add_gen_audio_attention_layernorm_eps(hp["rms_norm_eps"])
+ # rope theta, the ODE solver and the VAE hparams are hard-coded in tools/mtmd
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ # read on the host, or small convs and projections of the flow / VAE graphs
+ if not new_name.startswith("a.gen.nar.blk.") or n_dims < 2:
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ @staticmethod
+ def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
+ # torch weight norm, over every dim but 0
+ g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
+ return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ from safetensors.torch import load_file
+ path = self.dir_model / VAE_DIR / "model.safetensors"
+ if not path.is_file():
+ raise FileNotFoundError(f"YuE2: VAE not found at {path}, download m-a-p/YuE2-Vae into it")
+ logger.info(f"gguf: loading VAE decoder from '{path}'")
+ sd = {k[len("decoder.layers."):]: v for k, v in load_file(str(path)).items() if k.startswith("decoder.layers.")}
+ p = "a.gen.vae"
+
+ # SnakeBeta with log-scale params: x + 1/(exp(beta) + 1e-9) * sin(exp(alpha) * x)^2
+ def snake(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
+ yield (f"{dst}.alpha", torch.exp(sd[src + ".alpha"].float()))
+ yield (f"{dst}.beta", 1.0 / (torch.exp(sd[src + ".beta"].float()) + 1e-9))
+
+ yield (f"{p}.conv_in.weight", self._wn(sd, "0"))
+ yield (f"{p}.conv_in.bias", sd["0.bias"])
+ n_blk = sum(1 for k in sd if k.endswith(".layers.1.weight_g") and k.count(".") == 3)
+ for b in range(n_blk):
+ s = f"{b + 1}.layers"
+ yield from snake(f"{p}.blk.{b}.snake", f"{s}.0")
+ yield (f"{p}.blk.{b}.up.weight", self._wn(sd, f"{s}.1"))
+ yield (f"{p}.blk.{b}.up.bias", sd[f"{s}.1.bias"])
+ for r in range(3):
+ rs = f"{s}.{r + 2}.layers"
+ yield from snake(f"{p}.blk.{b}.res.{r}.snake1", f"{rs}.0")
+ yield (f"{p}.blk.{b}.res.{r}.conv1.weight", self._wn(sd, f"{rs}.1"))
+ yield (f"{p}.blk.{b}.res.{r}.conv1.bias", sd[f"{rs}.1.bias"])
+ yield from snake(f"{p}.blk.{b}.res.{r}.snake2", f"{rs}.2")
+ yield (f"{p}.blk.{b}.res.{r}.conv2.weight", self._wn(sd, f"{rs}.3"))
+ yield (f"{p}.blk.{b}.res.{r}.conv2.bias", sd[f"{rs}.3.bias"])
+ yield from snake(f"{p}.post_snake", f"{n_blk + 1}")
+ yield (f"{p}.conv_out.weight", self._wn(sd, f"{n_blk + 2}"))
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name.startswith("a.gen.vae."):
+ yield (name, data_torch)
+ return
+
+ p = "a.gen.nar"
+ aux = {
+ "model.norm.weight": f"{p}.output_norm.weight",
+ "vae2llm.weight": f"{p}.inp.weight",
+ "vae2llm.bias": f"{p}.inp.bias",
+ "llm2vae.weight": f"{p}.out.weight",
+ "llm2vae.bias": f"{p}.out.bias",
+ "time_embedder.mlp.0.weight": f"{p}.time.0.weight",
+ "time_embedder.mlp.0.bias": f"{p}.time.0.bias",
+ "time_embedder.mlp.2.weight": f"{p}.time.2.weight",
+ "time_embedder.mlp.2.bias": f"{p}.time.2.bias",
+ }
+ if name in aux:
+ yield (aux[name], data_torch)
+ return
+
+ # the NAR layers keep the libllama names, mtmd loads them as a text model
+ assert bid is not None, name
+ layer = {
+ "nar_input_layernorm.weight": "attn_norm.weight",
+ "nar_self_attn.q_proj.weight": "attn_q.weight",
+ "nar_self_attn.k_proj.weight": "attn_k.weight",
+ "nar_self_attn.v_proj.weight": "attn_v.weight",
+ "nar_self_attn.o_proj.weight": "attn_output.weight",
+ "nar_self_attn.q_norm.weight": "attn_q_norm.weight",
+ "nar_self_attn.k_norm.weight": "attn_k_norm.weight",
+ "nar_pre_mlp_layernorm.weight": "ffn_norm.weight",
+ "nar_mlp.gate_proj.weight": "ffn_gate.weight",
+ "nar_mlp.up_proj.weight": "ffn_up.weight",
+ "nar_mlp.down_proj.weight": "ffn_down.weight",
+ }
+ key = name.split(".", 3)[3] # strip "model.layers.{bid}."
+ yield (f"{p}.blk.{bid}.{layer[key]}", data_torch)
diff --git a/docs/multimodal/tts.md b/docs/multimodal/tts.md
index 301d2394e..a9545b5c6 100644
--- a/docs/multimodal/tts.md
+++ b/docs/multimodal/tts.md
@@ -13,6 +13,7 @@ Supported models:
- [Drama 3](#drama-3)
- [ZONOS3](#zonos3)
- [Fish Audio S2 Pro](#fish-audio-s2-pro)
+- [YuE2](#yue2) (songs)
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
@@ -258,3 +259,50 @@ The mmproj holds the fast AR and `codec.pth` (the encoder half for voice cloning
python convert_hf_to_gguf.py path/to/s2-pro --outtype q8_0 --outfile s2-pro.gguf
python convert_hf_to_gguf.py path/to/s2-pro --mmproj --outtype bf16 --outfile mmproj-s2-pro.gguf
```
+
+## YuE2
+
+[m-a-p/YuE2-3B](https://huggingface.co/m-a-p/YuE2-3B) generates songs (vocals and accompaniment) from a style prompt and lyrics, as 48 kHz stereo. The backbone (a Qwen3 text model) writes an ABC score, then one semantic code per 40 ms frame; the NAR half of the checkpoint turns the codes into VAE latents by flow matching, attending to the keys and values of the backbone, and the [YuE2-Vae](https://huggingface.co/m-a-p/YuE2-Vae) decoder makes the audio.
+
+The prompt is the text of the reference request:
+
+```
+[Tags]
+City Pop, upbeat, danceable, groovy bass, electric guitar, synth
+[Lyrics]
+[Verse]
+...
+```
+
+Available params:
+- `--tts-voice` picks the planning mode of the reference (`cot`): `full` (default, a chord-annotated score), `melody` (a melody-only score) or `off` (no score)
+- Cover mode: append the score to the prompt as `\n[ABC]\n<score>`, with `--tts-voice melody` or `full`; the model then goes straight to the codes
+- The generation stops at the limits of the reference (4096 score tokens, then 9000 codes, 6 minutes); `-n` sets a lower limit on the score tokens and the codes together
+- The context must keep the default size (24576 positions, the one of the reference): a smaller `-c` is rejected when the model is loaded
+- Note: `--top-k` and `--top-p` are not used, both phases are sampled with the settings of the reference (`yue2_generation_config.json`); `--temp` changes the temperature of the codes
+- Note: the backbone context must use an F16 KV cache (default): the NAR pass loads its keys and values
+- The codes are sampled with classifier-free guidance like the reference: scale 1.01 with `--tts-voice off`, off (1.0) otherwise. `llama-server` takes `cfg_scale` to change it, the reference suggests 1.2 for a stronger text guidance. The uncond prompt runs as a second sequence, so the context must also hold it (the instruction, the score and the codes once more)
+- The NAR pass solves the flow-matching ODE in 32 midpoint steps like the reference; `llama-server` takes `flow_steps` to change it. The NAR time is about proportional to it: 16 steps are about 35% faster for the NAR and the VAE together, with a small deviation from the reference (about 0.2 dB of spectral difference); it grows with fewer steps (about 0.4 dB at 8, 0.8 dB at 4)
+- The NAR pass runs on chunks of codes like the reference: one chunk for a song that fits in 24576 positions, so the audio arrives after the last code. When streaming, each 30 s of codes is a chunk, processed during the generation, so the audio arrives in 30 s steps; the chunks are independent (each sees the prompt, the score and its own codes), which may be audible at the boundaries
+
+Example usage:
+
+```sh
+llama-tts -m yue2-3b.gguf \
+ -mm mmproj-yue2-3b.gguf \
+ -f song.txt \
+ --output song.wav
+```
+
+**Note for GGUF conversion:**
+
+The VAE is a separate repository, download it into the `vae` subdirectory of the model:
+
+```sh
+hf download m-a-p/YuE2-3B --local-dir YuE2-3B
+hf download m-a-p/YuE2-Vae --local-dir YuE2-3B/vae
+python convert_hf_to_gguf.py YuE2-3B --outfile yue2-3b.gguf
+python convert_hf_to_gguf.py YuE2-3B --mmproj --outfile mmproj-yue2-3b.gguf
+```
+
+The mmproj holds the NAR transformer layers (loaded with libllama as a Qwen3 text model, in the `--outtype` type), and the latent projections and the VAE decoder, always in F32.
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index e6c1465c2..3f13b83b2 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -7287,6 +7287,7 @@ class VisionProjectorType:
DRAMA3_GEN = "drama3_gen" # audio generation: codebook tower + Fish DAC decoder
ZONOS3_GEN = "zonos3_gen" # audio generation: 10 codebook heads + ZAC decoder
ZONOS3_SPKENC = "zonos3_spkenc" # audio: MECT-B2 speaker encoder
+ YUE2_GEN = "yue2_gen" # audio generation: flow-matching NAR + Oobleck VAE decoder
FISH_S2_GEN = "fish_s2_gen" # audio generation: fast AR code predictor + Fish DAC decoder
FISH_S2_SPKENC = "fish_s2_spkenc" # audio: Fish DAC encoder + RVQ, reference clip -> prompt rows
HUNYUANVL = "hunyuanvl"
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index d514a72cd..57b0e098d 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -87,6 +87,7 @@ add_library(mtmd
models/fish-s2-gen.cpp
models/fish-s2-spkenc.cpp
models/zonos3-gen.cpp
+ models/yue2-gen.cpp
models/zonos3-spkenc.cpp
models/pockettts-seanet.cpp
models/pockettts-spkenc.cpp
diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md
index b0d153be3..ce9c7a8be 100644
--- a/tools/mtmd/README-dev.md
+++ b/tools/mtmd/README-dev.md
@@ -84,6 +84,18 @@ Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system i
- Core API `mtmd.h`: handles main inference. Important: `mtmd_gen_state` holds only the per-sequence data that the mmproj itself writes and reads back (RNG, decoder KV / conv state, output buffers). Data that goes to or comes from the backbone (hidden states, codes, PCM) must stay explicit in `mtmd_gen_inp` / `mtmd_gen_out`; the caller drives the backbone and handles audio frame accumulation.
- Helper API `mtmd-helper.h`: provides a model-agnostic stateful API. Usage example can be found in the `tools/tts` directory.
+### Running a transformer stage on libllama (mtmd-owned `llama_context`)
+
+Some stages are a full LLM-sized transformer that attends to the keys/values of the backbone, for example the flow-matching NAR of YuE2 (same layer shapes as the backbone, its own weights, bidirectional over its rows, and it reads the backbone KV). Do not re-implement it as an mtmd graph. Instead, mtmd can own a second `llama_model` + `llama_context` (see `mtmd_yue2_nar_load()` and `mtmd_gen_flow_yue2()` in `mtmd.cpp`):
+
+- Conversion: store the layers in the mmproj under `a.gen.<stage>.` with the libllama tensor names of an existing arch (e.g. `a.gen.nar.blk.0.attn_q.weight`)
+- Loading: build a synthetic `gguf_context` with the arch hparams (`general.architecture`, `tokenizer.ggml.model = none`, `vocab_size = 1`, ...) and one tensor info per mmproj tensor (only the type is used, the shapes come from the hparams), then call `llama_model_init_from_user()`. The `set_tensor_data` callback reads the data from the mmproj file, so no copy is kept in the mtmd model
+ - libllama also creates every optional tensor of the arch (biases, `*.scale`, `*.input_scale`, `dense_2`, ...) for a model built from metadata. The callback must fill them with values that leave the graph unchanged: 0 for biases, 1 for scales, identity for `dense_2`. Zeros everywhere silently make every matmul output 0
+- Backbone keys/values: the helper passes the backbone sequence state (`llama_state_seq_get_data()`) through `mtmd_gen_inp`, mtmd loads it into its context with `llama_state_seq_set_data()`. This only checks the layer count and the K/V types and row sizes, so it works across models with the same KV layout. The two contexts must agree on the KV type and on flash attention (`v_trans`)
+- Inputs and outputs are embedding rows (`llama_batch::embd`, `llama_get_embeddings()` with `LLAMA_POOLING_TYPE_NONE`), the projections around them stay small mtmd graphs
+- Non-causal rows: `LLAMA_ATTENTION_TYPE_NON_CAUSAL`, all rows in one ubatch (`n_ubatch` >= rows), they see every cached position of the sequence. Remove them with `llama_memory_seq_rm()` after each pass
+- Size the context per call (`n_ctx`, `n_ubatch`) instead of keeping one for the whole session
+
### Checklist for porting new audio generation models to mtmd
1. Make sure to consult merged PRs about adding new TTS models, especially reviewer comments
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index d2de0273a..c4044411d 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -637,6 +637,33 @@ struct mtmd_graph_zonos3_gen : mtmd_graph {
std::vector<c2w_state_slot> list_zonos3_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
+//
+// YuE2 generation, the NAR transformer itself runs on a llama_context, see mtmd_gen_flow_yue2():
+// LATENT_IN = latent rows -> backbone embd rows
+// LATENT_OUT = backbone hidden rows -> velocity rows
+// GEN_WAV = Oobleck VAE decoder, latent rows -> interleaved stereo PCM
+//
+struct mtmd_graph_yue2_gen : mtmd_graph {
+ mtmd_graph_yue2_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_rows)
+ : mtmd_graph(ctx, img), gen_process(gen_process), n_rows(n_rows) {}
+ ggml_cgraph * build() override;
+
+ static constexpr int n_latent = 64;
+ static constexpr int n_freq = 256; // time embedder input
+ static constexpr int n_channels = 2;
+ static constexpr int vae_ratio = 1920; // output samples per latent frame
+ static constexpr int vae_trim = 64; // a decode of n frames has vae_ratio * n - vae_trim samples
+ static constexpr float rope_theta = 1000000.0f;
+ static constexpr int n_ctx_max = 24576; // the AR prefix and the NAR rows of one chunk
+ static constexpr int ode_steps = 32; // midpoint solver
+
+ mtmd_subctx_gen_process gen_process;
+ int n_rows;
+
+ ggml_tensor * snake(ggml_tensor * x, const mtmd_yue2::snake & s) const;
+ ggml_tensor * conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const;
+};
+
struct mtmd_graph_soprano : mtmd_graph {
mtmd_graph_soprano(mtmd_subcontext * ctx, const mtmd_image_f32 & img, int n_frames) : mtmd_graph(ctx, img), n_frames(n_frames) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/models/yue2-gen.cpp b/tools/mtmd/models/yue2-gen.cpp
new file mode 100644
index 000000000..59cd6f12f
--- /dev/null
+++ b/tools/mtmd/models/yue2-gen.cpp
@@ -0,0 +1,89 @@
+#include "models.h"
+
+// SnakeBeta, alpha and beta folded at conversion: x + beta * sin(alpha * x)^2
+// x: [T, C], alpha/beta: [C]
+ggml_tensor * mtmd_graph_yue2_gen::snake(ggml_tensor * x, const mtmd_yue2::snake & s) const {
+ ggml_tensor * a = ggml_reshape_2d(ctx0, s.alpha, 1, s.alpha->ne[0]);
+ ggml_tensor * b = ggml_reshape_2d(ctx0, s.beta, 1, s.beta->ne[0]);
+
+ // expand reshapes first so mul/sin/sqr/mul/add lands as consecutive nodes, letting backends fuse them
+ ggml_build_forward_expand(gf, a);
+ ggml_build_forward_expand(gf, b);
+
+ ggml_tensor * y = ggml_sqr(ctx0, ggml_sin(ctx0, ggml_mul(ctx0, x, a)));
+ return ggml_add(ctx0, x, ggml_mul(ctx0, y, b));
+}
+
+// same-length conv, stride 1; x: [T, IC], w: [K, IC, OC], returns [T, OC]
+// im2col in F32: the reference decoder runs in FP32
+ggml_tensor * mtmd_graph_yue2_gen::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const {
+ const int pad = (int) (w->ne[0] - 1) / 2 * dilation;
+ ggml_tensor * col = ggml_im2col(ctx0, w, x, 1, 0, pad, 0, dilation, 0, false, GGML_TYPE_F32); // [1, T, IC * K]
+ ggml_tensor * y = ggml_mul_mat(ctx0,
+ ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[1] * col->ne[2]),
+ ggml_reshape_2d(ctx0, w, w->ne[0] * w->ne[1], w->ne[2]));
+ if (b) {
+ y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
+ }
+ return y;
+}
+
+ggml_cgraph * mtmd_graph_yue2_gen::build() {
+ const auto & y = model.yue2;
+
+ if (gen_process == MTMD_SUBCTX_GEN_PROCESS_LATENT_IN || gen_process == MTMD_SUBCTX_GEN_PROCESS_LATENT_OUT) {
+ const bool is_in = gen_process == MTMD_SUBCTX_GEN_PROCESS_LATENT_IN;
+ ggml_tensor * w = is_in ? y.inp_w : y.out_w;
+ ggml_tensor * b = is_in ? y.inp_b : y.out_b;
+ ggml_tensor * cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, w->ne[0], n_rows);
+ ggml_set_name(cur, "inp_feats");
+ ggml_set_input(cur);
+ cur = ggml_add(ctx0, build_mm(w, cur), b);
+ ggml_set_name(cur, "out_feats");
+ ggml_set_output(cur);
+ ggml_build_forward_expand(gf, cur);
+ return gf;
+ }
+
+ GGML_ASSERT(gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV);
+ static constexpr int dilations[3] = { 1, 3, 9 };
+
+ ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_latent, n_rows);
+ ggml_set_name(inp, "inp_feats");
+ ggml_set_input(inp);
+
+ ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [T, n_latent]
+ x = conv1d(x, y.vae_in_w, y.vae_in_b, 1);
+ for (size_t ib = 0; ib < y.vae.size(); ib++) {
+ const auto & blk = y.vae[ib];
+ const int stride = (int) (blk.up_w->ne[0] / 2);
+ const int pad = (stride + 1) / 2; // ceil(stride / 2) of the reference
+ x = snake(x, blk.act);
+ {
+ // ConvTranspose1d as GEMM + col2im, same as qwen3tts; up_w: [K, OC, IC]
+ const int64_t K = blk.up_w->ne[0];
+ const int64_t OC = blk.up_w->ne[1];
+ ggml_tensor * w2 = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, blk.up_w, K * OC, blk.up_w->ne[2])));
+ ggml_tensor * col = ggml_mul_mat(ctx0, w2, ggml_cont(ctx0, ggml_transpose(ctx0, x))); // [K * OC, T]
+ x = ggml_col2im_1d(ctx0, col, stride, (int) OC, pad);
+ }
+ x = ggml_add(ctx0, x, ggml_reshape_2d(ctx0, blk.up_b, 1, blk.up_b->ne[0]));
+ for (size_t ir = 0; ir < blk.res.size(); ir++) {
+ const auto & ru = blk.res[ir];
+ ggml_tensor * h = snake(x, ru.act1);
+ h = conv1d(h, ru.conv1_w, ru.conv1_b, dilations[ir]);
+ h = snake(h, ru.act2);
+ h = conv1d(h, ru.conv2_w, ru.conv2_b, 1);
+ x = ggml_add(ctx0, x, h);
+ }
+ cb(x, "vae_block", (int) ib);
+ }
+ x = snake(x, y.vae_post);
+ x = conv1d(x, y.vae_out_w, nullptr, 1); // [T, n_channels]
+
+ x = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // interleaved
+ ggml_set_name(x, "out_audio");
+ ggml_set_output(x);
+ ggml_build_forward_expand(gf, x);
+ return gf;
+}
diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp
index cb2029685..78a0eca46 100644
--- a/tools/mtmd/mtmd-graph.cpp
+++ b/tools/mtmd/mtmd-graph.cpp
@@ -1038,6 +1038,17 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
? (int) (params->codes->size() / n_cb) : 1;
builder = std::make_unique<mtmd_graph_zonos3_gen>(ctx, img, gen_process, all_known, n_frames);
} break;
+ case PROJECTOR_TYPE_YUE2_GEN:
+ {
+ const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_LATENT_IN;
+ int n_rows = 1;
+ if (params && params->feats) {
+ const size_t n_col = gen_process == MTMD_SUBCTX_GEN_PROCESS_LATENT_OUT ? ctx->model.yue2.out_w->ne[0] : mtmd_graph_yue2_gen::n_latent;
+ GGML_ASSERT(!params->feats->empty() && params->feats->size() % n_col == 0);
+ n_rows = (int) (params->feats->size() / n_col);
+ }
+ builder = std::make_unique<mtmd_graph_yue2_gen>(ctx, img, gen_process, n_rows);
+ } break;
case PROJECTOR_TYPE_DRAMA3_GEN:
case PROJECTOR_TYPE_FISH_S2_GEN:
{
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 79541b813..553e45f47 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -72,10 +72,11 @@ static void put_bytes(std::vector<char> & buf, const void * p, size_t n) {
}
// data_sz == UINT32_MAX writes the "unknown length" sentinel (streaming), same as ffmpeg does on a pipe
-static void write_wav16_header(std::vector<char> & buf, uint32_t data_sz, int32_t rate) {
+// multi-channel PCM is interleaved
+static void write_wav16_header(std::vector<char> & buf, uint32_t data_sz, int32_t rate, uint16_t ch = 1) {
const uint32_t riff_sz = data_sz == UINT32_MAX ? UINT32_MAX : 36 + data_sz;
- const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2;
- const uint16_t fmt = 1, ch = 1, align = 2, bits = 16;
+ const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2 * ch;
+ const uint16_t fmt = 1, align = 2 * ch, bits = 16;
const uint32_t rate32 = (uint32_t) rate;
put_bytes(buf, "RIFF", 4); put_bytes(buf, &riff_sz, 4); put_bytes(buf, "WAVE", 4);
put_bytes(buf, "fmt ", 4); put_bytes(buf, &fmt_sz, 4);
@@ -91,12 +92,12 @@ static void append_wav16_pcm(std::vector<char> & buf, const float * pcm, size_t
}
}
-static bool write_wav16(std::vector<char> & buf, const std::vector<float> & pcm, int32_t rate) {
+static bool write_wav16(std::vector<char> & buf, const std::vector<float> & pcm, int32_t rate, uint16_t ch = 1) {
// RIFF chunk sizes are 32-bit; refuse to emit a file with a truncated header
if (pcm.size() > ((size_t) UINT32_MAX - 36) / 2) {
return false;
}
- write_wav16_header(buf, (uint32_t) (pcm.size() * 2), rate);
+ write_wav16_header(buf, (uint32_t) (pcm.size() * 2), rate, ch);
append_wav16_pcm(buf, pcm.data(), pcm.size());
return true;
}
@@ -108,6 +109,10 @@ public:
n_embd(llama_model_n_embd(model)), info(mtmd_gen_audio_get_info(mctx)) {}
virtual ~mtmd_gen_audio_pipeline() = default;
+ // false if the backbone context cannot run the pipeline, checked by mtmd_helper_gen_audio_init()
+ virtual bool check_context() const { return true; }
+ // see mtmd_helper_gen_audio_max_frames()
+ virtual int32_t max_frames() const { return 512; }
virtual void reset() = 0;
virtual int32_t set_input(const mtmd_helper_gen_audio_inp * inp) = 0;
// decodes at most n_batch prompt tokens; returns remaining count (0 = done), <0 on error
@@ -123,6 +128,10 @@ public:
virtual bool supports_continue() const { return false; }
// mtmd_helper_gen_audio_inp::speed and loudness other than the defaults
virtual bool supports_delivery() const { return false; }
+ // mtmd_helper_gen_audio_inp::cfg_scale other than the default
+ virtual bool supports_cfg() const { return false; }
+ // mtmd_helper_gen_audio_inp::flow_steps other than the default
+ virtual bool supports_flow_steps() const { return false; }
virtual bool next_is_text() const { return false; }
std::vector<llama_logit_bias> logit_bias; // see mtmd_helper_gen_audio_get_logit_bias()
@@ -3153,6 +3162,654 @@ private:
bool wav_header_sent = false;
};
+// YuE2: the backbone writes an ABC score (text), then the semantic codes; both are sampled here like the reference (sampling.py)
+// the NAR flow matching (in mtmd, on the backbone keys/values) and the VAE then make the stereo PCM
+class yue2_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
+public:
+ using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
+
+ bool supports_cfg() const override { return true; }
+ bool supports_flow_steps() const override { return true; }
+
+ // step_gen() stops at the score and code limits of the reference
+ int32_t max_frames() const override { return 0; }
+
+ // the limits and the NAR chunks of the reference need the training context (24576 positions)
+ bool check_context() const override {
+ if (llama_vocab_n_tokens(vocab) < tok_codec0 + n_codec) {
+ LOG_ERR("mtmd_helper_gen_audio: the vocab of the backbone has no yue2 codec tokens\n");
+ return false;
+ }
+ const uint32_t n_ctx_train = (uint32_t) llama_model_n_ctx_train(model);
+ if (llama_n_ctx_seq(lctx) < n_ctx_train) {
+ LOG_ERR("mtmd_helper_gen_audio: yue2 needs a context of %u positions per sequence, got %u (use the default context size)\n",
+ n_ctx_train, llama_n_ctx_seq(lctx));
+ return false;
+ }
+ return true;
+ }
+
+ void reset() override {
+ seq_id = 0;
+ pos = 0;
+ prompt.clear();
+ prompt_pos = 0;
+ in_codes = false;
+ mode_off = false;
+ cfg_scale = 1.0f;
+ seq_uncond = 0;
+ upos = 0;
+ neg_prefix.clear();
+ abc_tail.clear();
+ prefix_tok.clear();
+ history.clear();
+ codes.clear();
+ latents.clear();
+ n_nar_done = 0;
+ n_vae_done = 0;
+ scratch.reset();
+ scratch_n_ctx = 0;
+ done = false;
+ decoded = false;
+ audio_pcm.clear();
+ h_state_buf.clear();
+ out_buf.clear();
+ stream = false;
+ pcm_sent = 0;
+ wav_header_sent = false;
+ gen_state.reset();
+ }
+
+ int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
+ reset();
+ seq_id = inp->seq_id;
+ out_type = inp->out_type;
+ stream = inp->stream;
+ code_cfg.temp = inp->temp >= 0.0f ? inp->temp : 1.0f;
+ flow_steps = inp->flow_steps;
+ rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
+ if (!init_gen_state(inp->seed)) {
+ return 1;
+ }
+
+ // the "tts" template renders the whole prompt, special tokens included
+ const std::string text(inp->prompt, inp->prompt_len);
+ prompt.resize(text.size() + 16);
+ const int32_t n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), prompt.data(), (int32_t) prompt.size(), false, true);
+ if (n < 2 || prompt[0] != tok_eod) {
+ LOG_ERR("mtmd_helper_gen_audio: yue2 expects the prompt of its tts template\n");
+ return 1;
+ }
+ prompt.resize((size_t) n);
+ // cover mode and cot=off give the score in the prompt, the codes come next
+ auto abc = std::find(prompt.begin(), prompt.end(), tok_abc_start);
+ if (prompt.back() == tok_music_start && abc != prompt.end()) {
+ in_codes = true;
+ mode_off = prompt.end() - abc == 3; // empty score
+ } else if (prompt.back() != tok_abc_start) {
+ LOG_ERR("mtmd_helper_gen_audio: yue2 prompt must end with <abc> or <|music_start|>\n");
+ return 1;
+ }
+ abc_tail.assign(abc, prompt.end());
+ prefix_tok = prompt;
+ if (prompt.size() >= llama_n_ctx_seq(lctx)) {
+ LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n");
+ return 1;
+ }
+ llama_memory_t mem = llama_get_memory(lctx);
+ if (!llama_memory_seq_rm(mem, seq_id, 0, -1)) {
+ LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n");
+ return 1;
+ }
+
+ // classifier-free guidance of the codes, defaults of the reference (protocol.py)
+ cfg_scale = inp->cfg_scale > 0.0f ? inp->cfg_scale : mode_off ? 1.01f : 1.0f;
+ if (cfg_scale != 1.0f) {
+ // the uncond sequence lives half a seq space away from the cond one
+ const int32_t n_seq_max = (int32_t) llama_n_seq_max(lctx);
+ seq_uncond = (llama_seq_id) (n_seq_max / 2 + seq_id);
+ if (!llama_model_classifier_free_guidance(model) || seq_id >= n_seq_max / 2) {
+ if (inp->cfg_scale > 0.0f) {
+ LOG_ERR("mtmd_helper_gen_audio: cfg_scale needs a paired sequence (n_seq_max = %d, seq_id = %d), reconvert the model\n", n_seq_max, seq_id);
+ return 1;
+ }
+ LOG_WRN("mtmd_helper_gen_audio: no paired cfg sequence (n_seq_max = %d, seq_id = %d), cfg is disabled\n", n_seq_max, seq_id);
+ cfg_scale = 1.0f;
+ } else if (!llama_memory_seq_rm(mem, seq_uncond, 0, -1)) {
+ LOG_ERR("mtmd_helper_gen_audio: cannot reset the cfg sequence\n");
+ return 1;
+ }
+ }
+ if (cfg_scale != 1.0f) {
+ // the negative prompt is the bare instruction, tokenized alone like the reference
+ const size_t i0 = text.find("<|endoftext|>");
+ const size_t i1 = text.find('\n');
+ if (i0 != 0 || i1 == std::string::npos) {
+ LOG_ERR("mtmd_helper_gen_audio: yue2 expects the prompt of its tts template\n");
+ return 1;
+ }
+ const std::string instr = text.substr(13, i1 - 13);
+ neg_prefix.resize(instr.size() + 16);
+ const int32_t n_instr = llama_tokenize(vocab, instr.c_str(), (int32_t) instr.size(), neg_prefix.data() + 1, (int32_t) neg_prefix.size() - 1, false, false);
+ if (n_instr < 1) {
+ LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
+ return 1;
+ }
+ neg_prefix[0] = tok_eod;
+ neg_prefix.resize((size_t) n_instr + 1);
+ }
+ return 0;
+ }
+
+ int32_t step_prompt(int32_t n_batch) override {
+ GGML_ASSERT(n_batch > 0);
+ const int32_t n_left = (int32_t) prompt.size() - prompt_pos;
+ if (n_left <= 0) {
+ return 0;
+ }
+ const int32_t n = std::min(n_batch, n_left);
+ if (!decode(prompt.data() + prompt_pos, n)) {
+ return -1;
+ }
+ prompt_pos += n;
+ if (n == n_left) {
+ if (in_codes) {
+ if (!start_uncond()) {
+ return -1;
+ }
+ } else {
+ save_logits(llama_get_logits_ith(lctx, -1));
+ }
+ }
+ return n_left - n;
+ }
+
+ int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
+ GGML_UNUSED(sampled);
+ GGML_UNUSED(h_state_in);
+ *h_state_out = nullptr;
+ *out_stop = false;
+ if (done) {
+ *out_stop = true;
+ return 0;
+ }
+
+ const bool use_cfg = in_codes && cfg_scale != 1.0f;
+ const llama_token tok = sample(logits.data());
+ int32_t i_out = -1; // output row of the cond sequence
+ if (!in_codes) {
+ // the score ends with </abc>, or is cut at max_tokens like the reference
+ if (tok == tok_abc_end || (int32_t) history.size() + 1 >= abc_cfg.max_tokens) {
+ const llama_token next[2] = { tok_abc_end, tok_music_start };
+ if (!decode(next, 2)) {
+ return 1;
+ }
+ abc_tail.insert(abc_tail.end(), next, next + 2);
+ prefix_tok.insert(prefix_tok.end(), next, next + 2);
+ in_codes = true;
+ history.clear();
+ if (!start_uncond()) {
+ return 1;
+ }
+ } else {
+ history.push_back(tok);
+ abc_tail.push_back(tok);
+ prefix_tok.push_back(tok);
+ if (!decode(&tok, 1)) {
+ return 1;
+ }
+ save_logits(llama_get_logits_ith(lctx, -1));
+ }
+ } else {
+ // the NAR pass also attends to <|music_end|>, the uncond sequence shares the unified KV cache
+ const bool full = (int32_t) codes.size() + 1 >= code_cfg.max_tokens || pos + upos + 2 >= (llama_pos) llama_n_ctx_seq(lctx);
+ if (tok == tok_music_end || full) {
+ if (!decode(&tok_music_end, 1)) {
+ return 1;
+ }
+ done = true;
+ *out_stop = true;
+ return 0;
+ }
+ history.push_back(tok);
+ codes.push_back(tok - tok_codec0);
+ if (use_cfg) {
+ // the sampled code feeds both branches, in one decode
+ if (!decode_pair(tok)) {
+ return 1;
+ }
+ i_out = 0;
+ } else {
+ if (!decode(&tok, 1)) {
+ return 1;
+ }
+ save_logits(llama_get_logits_ith(lctx, -1));
+ }
+ }
+ const float * h = llama_get_embeddings_ith(lctx, i_out);
+ h_state_buf.assign(h, h + n_embd);
+ *h_state_out = h_state_buf.data();
+
+ // streaming: each full chunk of codes goes through the NAR pass and the VAE right away
+ if (stream && in_codes && codes.size() - n_nar_done >= chunk_size()) {
+ if (!nar_chunk(n_nar_done, n_nar_done + chunk_size(), false) || !vae_emit(false)) {
+ return 1;
+ }
+ }
+ return 0;
+ }
+
+ int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
+ *out_sample_rate = info.sample_rate;
+ const uint16_t n_ch = (uint16_t) info.n_channels;
+
+ if (!stream) {
+ if (flush() != 0) {
+ return 1;
+ }
+ if (out_n_samples) {
+ *out_n_samples = (int64_t) (audio_pcm.size() / n_ch);
+ }
+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+ *out_data = (const char *) audio_pcm.data();
+ *out_data_len = audio_pcm.size() * sizeof(float);
+ return 0;
+ }
+ out_buf.clear();
+ if (!write_wav16(out_buf, audio_pcm, info.sample_rate, n_ch)) {
+ LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
+ return 1;
+ }
+ *out_data = out_buf.data();
+ *out_data_len = out_buf.size();
+ return 0;
+ }
+
+ const size_t n_new = audio_pcm.size() - pcm_sent;
+ if (out_n_samples) {
+ *out_n_samples = (int64_t) (n_new / n_ch);
+ }
+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+ *out_data = (const char *) (audio_pcm.data() + pcm_sent);
+ *out_data_len = n_new * sizeof(float);
+ pcm_sent = audio_pcm.size();
+ return 0;
+ }
+ out_buf.clear();
+ if (!wav_header_sent) {
+ write_wav16_header(out_buf, UINT32_MAX, info.sample_rate, n_ch);
+ wav_header_sent = true;
+ }
+ append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
+ pcm_sent = audio_pcm.size();
+ *out_data = out_buf.data();
+ *out_data_len = out_buf.size();
+ return 0;
+ }
+
+ int32_t flush() override {
+ if (decoded) {
+ return 0;
+ }
+ if (codes.empty()) {
+ LOG_ERR("mtmd_helper_gen_audio: yue2 stopped before the first code\n");
+ return 1;
+ }
+ if (!done) {
+ // stopped by the caller: the NAR pass still needs <|music_end|>
+ if (!decode(&tok_music_end, 1)) {
+ return 1;
+ }
+ done = true;
+ }
+
+ // a song that fits in one chunk uses the backbone sequence itself, as it holds <|music_end|> now
+ const size_t n_codes = codes.size();
+ while (n_nar_done < n_codes) {
+ const size_t a = n_nar_done;
+ const size_t b = std::min(a + chunk_size(), n_codes);
+ if (!nar_chunk(a, b, a == 0 && b == n_codes)) {
+ return 1;
+ }
+ }
+ if (!vae_emit(true)) {
+ return 1;
+ }
+ scratch.reset();
+ decoded = true;
+ return 0;
+ }
+
+private:
+ // token ids of the yue2_infer protocol (protocol.py)
+ static constexpr llama_token tok_eod = 151643;
+ static constexpr llama_token tok_abc_start = 151847;
+ static constexpr llama_token tok_abc_end = 151848;
+ static constexpr llama_token tok_music_start = 151851;
+ static constexpr llama_token tok_music_end = 151852;
+ static constexpr llama_token tok_codec0 = 151853;
+ static constexpr int32_t n_codec = 32768;
+
+ // NAR chunks and VAE tiles, see synthesize() and decode_tiled() of the reference
+ static constexpr int64_t n_ctx_max = 24576;
+ static constexpr size_t stream_chunk = 750; // 30 s
+ static constexpr size_t n_latent = 64;
+ static constexpr size_t vae_core = 128; // frames kept per tile
+ static constexpr size_t vae_halo = 16; // frames of context on each side
+ static constexpr size_t vae_ratio = 1920;
+ static constexpr size_t vae_trim = 64;
+
+ struct sampling {
+ float temp;
+ float top_p;
+ int32_t top_k;
+ float penalty;
+ int32_t penalty_window;
+ int32_t min_tokens;
+ int32_t max_tokens;
+ };
+ sampling abc_cfg = { 0.7f, 0.9f, 30, 1.005f, 100, 32, 4096 };
+ sampling code_cfg = { 1.0f, 0.95f, 100, 1.2f, 50, 200, 9000 };
+
+ bool decode_ctx(llama_context * ctx, const llama_token * toks, int32_t n, llama_seq_id seq, llama_pos & p) {
+ llama_batch_ext_ptr batch(llama_batch_ext_init(ctx));
+ if (!batch_set_tokens(batch.get(), toks, n, p, seq)) {
+ return false;
+ }
+ llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
+ if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
+ return false;
+ }
+ p += n;
+ return true;
+ }
+
+ bool decode(const llama_token * toks, int32_t n) {
+ return decode_ctx(lctx, toks, n, seq_id, pos);
+ }
+
+ // keeps the logits of the next token, as the context logits belong to the last decode of any sequence (server slots)
+ // only the window that sample() reads: the text and </abc> for the score, <|music_end|> and the codes after it
+ // with guidance: uncond + scale * (cond - uncond)
+ void save_logits(const float * cond, const float * uncond = nullptr) {
+ const llama_token lo = in_codes ? tok_music_end : 0;
+ const llama_token hi = in_codes ? tok_codec0 + n_codec : tok_abc_end + 1;
+ logits.resize((size_t) llama_vocab_n_tokens(vocab));
+ for (llama_token t = lo; t < hi; t++) {
+ logits[t] = uncond ? uncond[t] + cfg_scale * (cond[t] - uncond[t]) : cond[t];
+ }
+ }
+
+ // the negative prompt of the reference (negative_prefix): the instruction, then the score of the cond prompt
+ // the logits of the cond prompt must be the last ones when this is called
+ bool start_uncond() {
+ if (cfg_scale == 1.0f) {
+ save_logits(llama_get_logits_ith(lctx, -1));
+ return true;
+ }
+ const float * c = llama_get_logits_ith(lctx, -1);
+ std::vector<float> cond(c, c + llama_vocab_n_tokens(vocab));
+
+ std::vector<llama_token> neg = neg_prefix;
+ if (mode_off) {
+ neg.push_back(tok_music_start);
+ } else {
+ neg.insert(neg.end(), abc_tail.begin(), abc_tail.end());
+ }
+ if (pos + (llama_pos) neg.size() + 2 >= (llama_pos) llama_n_ctx_seq(lctx)) {
+ LOG_ERR("mtmd_helper_gen_audio: the cfg prompt does not fit in the context\n");
+ return false;
+ }
+ const int32_t n_batch = (int32_t) llama_n_batch(lctx);
+ for (size_t i = 0; i < neg.size(); i += (size_t) n_batch) {
+ const int32_t n = std::min(n_batch, (int32_t) (neg.size() - i));
+ if (!decode_ctx(lctx, neg.data() + i, n, seq_uncond, upos)) {
+ return false;
+ }
+ }
+ save_logits(cond.data(), llama_get_logits_ith(lctx, -1));
+ return true;
+ }
+
+ bool decode_pair(llama_token tok) {
+ llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+ const llama_seq_id seqs[2] = { seq_id, seq_uncond };
+ const llama_pos p[2] = { pos, upos };
+ for (int i = 0; i < 2; i++) {
+ const int32_t idx = llama_batch_ext_add_token(batch.get(), seqs[i], tok);
+ if (idx < 0) {
+ LOG_ERR("mtmd_helper_gen_audio: failed to add token %d to batch (error %d)\n", tok, idx);
+ return false;
+ }
+ llama_batch_ext_set_pos(batch.get(), idx, &p[i]);
+ llama_batch_ext_set_output_logits(batch.get(), idx, true);
+ }
+ if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
+ return false;
+ }
+ pos++;
+ upos++;
+ save_logits(llama_get_logits_ith(lctx, 0), llama_get_logits_ith(lctx, 1));
+ return true;
+ }
+
+ // song_chunks() of the reference: each NAR chunk sees the prefix and its own codes
+ // the reference uses the largest chunk that fits in n_ctx_max, streaming uses smaller ones
+ size_t chunk_size() const {
+ const int64_t ref = std::max<int64_t>(1, (n_ctx_max - (int64_t) prefix_tok.size() - 3) / 2);
+ return (size_t) (stream ? std::min<int64_t>(ref, stream_chunk) : ref);
+ }
+
+ // backbone keys/values of [prefix, codes[a, b), <|music_end|>] -> latents of the chunk
+ bool nar_chunk(size_t a, size_t b, bool use_main) {
+ llama_context * ctx = lctx;
+ llama_seq_id seq = seq_id;
+ llama_pos n_past = pos;
+ if (!use_main) {
+ // a scratch context on the backbone model, the prefix is decoded once
+ const llama_pos n_prefix = (llama_pos) prefix_tok.size();
+ const uint32_t n_need = (uint32_t) (n_prefix + (b - a) + 1);
+ if (!scratch || scratch_n_ctx < n_need) {
+ llama_context_params cparams = llama_context_default_params();
+ cparams.n_ctx = n_need;
+ cparams.n_seq_max = 1;
+ cparams.n_threads = llama_n_threads(lctx);
+ cparams.n_threads_batch = llama_n_threads_batch(lctx);
+ cparams.no_perf = true;
+ scratch.reset(llama_init_from_model(const_cast<llama_model *>(model), cparams));
+ if (!scratch) {
+ LOG_ERR("mtmd_helper_gen_audio: cannot create the yue2 chunk context\n");
+ return false;
+ }
+ scratch_n_ctx = n_need;
+ llama_pos p = 0;
+ const int32_t n_batch = (int32_t) llama_n_batch(scratch.get());
+ for (size_t i = 0; i < prefix_tok.size(); i += (size_t) n_batch) {
+ const int32_t n = std::min(n_batch, (int32_t) (prefix_tok.size() - i));
+ if (!decode_ctx(scratch.get(), prefix_tok.data() + i, n, 0, p)) {
+ return false;
+ }
+ }
+ }
+ llama_memory_seq_rm(llama_get_memory(scratch.get()), 0, n_prefix, -1);
+ std::vector<llama_token> toks;
+ for (size_t i = a; i < b; i++) {
+ toks.push_back(codes[i] + tok_codec0);
+ }
+ toks.push_back(tok_music_end);
+ llama_pos p = n_prefix;
+ const int32_t n_batch = (int32_t) llama_n_batch(scratch.get());
+ for (size_t i = 0; i < toks.size(); i += (size_t) n_batch) {
+ const int32_t n = std::min(n_batch, (int32_t) (toks.size() - i));
+ if (!decode_ctx(scratch.get(), toks.data() + i, n, 0, p)) {
+ return false;
+ }
+ }
+ ctx = scratch.get();
+ seq = 0;
+ n_past = p;
+ }
+
+ std::vector<uint8_t> state(llama_state_seq_get_size(ctx, seq));
+ if (state.empty() || llama_state_seq_get_data(ctx, state.data(), state.size(), seq) != state.size()) {
+ LOG_ERR("mtmd_helper_gen_audio: cannot read the backbone state\n");
+ return false;
+ }
+ mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
+ inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
+ inp.n_codes = b - a;
+ inp.backbone_state = state.data();
+ inp.backbone_state_size = state.size();
+ inp.backbone_n_past = n_past;
+ inp.n_steps = flow_steps;
+ mtmd_gen_out out{};
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: flow matching failed\n");
+ return false;
+ }
+ latents.insert(latents.end(), out.feats, out.feats + out.n_feats);
+ n_nar_done = b;
+ return true;
+ }
+
+ // decodes the VAE tiles whose context on both sides is known, same cropping as decode_tiled() of the reference
+ bool vae_emit(bool final) {
+ const size_t n_lat = latents.size() / n_latent;
+ while (n_vae_done < n_lat) {
+ const size_t start = n_vae_done;
+ const size_t end = std::min(start + vae_core, n_lat);
+ if (!final && end + vae_halo > n_lat) {
+ break;
+ }
+ const size_t left = start >= vae_halo ? start - vae_halo : 0;
+ const size_t right = std::min(end + vae_halo, n_lat);
+ mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
+ inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
+ inp.feats = latents.data() + left * n_latent;
+ inp.n_feats = (right - left) * n_latent;
+ mtmd_gen_out out{};
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: VAE decode failed\n");
+ return false;
+ }
+ // the whole song has vae_ratio * n - vae_trim samples
+ const size_t n_ch = (size_t) info.n_channels;
+ const size_t crop0 = (start - left) * vae_ratio;
+ const size_t n_keep = (end - start) * vae_ratio - (final && end == n_lat ? vae_trim : 0);
+ if (out.n_samples < (crop0 + n_keep) * n_ch) {
+ LOG_ERR("mtmd_helper_gen_audio: VAE tile is too short\n");
+ return false;
+ }
+ audio_pcm.insert(audio_pcm.end(), out.audio + crop0 * n_ch, out.audio + (crop0 + n_keep) * n_ch);
+ n_vae_done = end;
+ }
+ return true;
+ }
+
+ // distribution() of the reference: allowed range, no end token before min_tokens, windowed penalty, top-k, top-p
+ llama_token sample(const float * logits) {
+ const sampling & c = in_codes ? code_cfg : abc_cfg;
+ const llama_token end = in_codes ? tok_music_end : tok_abc_end;
+ const llama_token lo = in_codes ? tok_codec0 : 0;
+ const llama_token hi = in_codes ? tok_codec0 + n_codec : tok_eod;
+
+ std::unordered_map<llama_token, int> freq;
+ const size_t n_win = std::min(history.size(), (size_t) c.penalty_window);
+ for (size_t i = history.size() - n_win; i < history.size(); i++) {
+ freq[history[i]]++;
+ }
+
+ std::vector<std::pair<float, llama_token>> cand;
+ cand.reserve((size_t) (hi - lo) + 1);
+ auto push = [&](llama_token t) {
+ float l = logits[t];
+ auto it = freq.find(t);
+ if (it != freq.end()) {
+ const float alpha = std::pow(c.penalty, (float) it->second);
+ l = l < 0.0f ? l * alpha : l / alpha;
+ }
+ cand.emplace_back(l, t);
+ };
+ for (llama_token t = lo; t < hi; t++) {
+ push(t);
+ }
+ if ((int32_t) history.size() >= c.min_tokens) {
+ push(end);
+ }
+
+ if (c.temp <= 0.0f) {
+ return std::max_element(cand.begin(), cand.end())->second;
+ }
+ const size_t k = std::min(cand.size(), (size_t) std::max(c.top_k, 1));
+ std::partial_sort(cand.begin(), cand.begin() + k, cand.end(), std::greater<std::pair<float, llama_token>>());
+ cand.resize(k);
+
+ std::vector<double> p(k);
+ double sum = 0.0;
+ for (size_t i = 0; i < k; i++) {
+ p[i] = std::exp((double) ((cand[i].first - cand[0].first) / c.temp));
+ sum += p[i];
+ }
+ // keep the smallest prefix whose mass reaches top_p, the first token always stays (3 for the codes of cot=off)
+ const size_t n_keep = in_codes && mode_off ? 3 : 1;
+ double before = 0.0;
+ for (size_t i = 0; i < k; i++) {
+ const double pi = p[i];
+ if (i >= n_keep && before > c.top_p * sum) {
+ p[i] = 0.0;
+ }
+ before += pi;
+ }
+ double kept = 0.0;
+ for (double v : p) {
+ kept += v;
+ }
+ double u = std::uniform_real_distribution<double>(0.0, kept)(rng);
+ for (size_t i = 0; i < k; i++) {
+ u -= p[i];
+ if (u <= 0.0 && p[i] > 0.0) {
+ return cand[i].second;
+ }
+ }
+ return cand[0].second;
+ }
+
+ llama_seq_id seq_id = 0;
+ llama_pos pos = 0;
+ std::vector<llama_token> prompt;
+ int32_t prompt_pos = 0;
+ bool in_codes = false;
+ bool mode_off = false; // cot=off: no score
+ float cfg_scale = 1.0f;
+ int32_t flow_steps = 0; // 0 = the model default
+ llama_seq_id seq_uncond = 0;
+ llama_pos upos = 0;
+ std::vector<llama_token> neg_prefix; // EOD + instruction
+ std::vector<llama_token> abc_tail; // the cond tokens from <abc> on
+ std::vector<float> logits; // see save_logits()
+ std::vector<llama_token> prefix_tok; // the cond tokens before the first code
+ std::vector<float> latents; // of the chunks done so far
+ size_t n_nar_done = 0; // codes
+ size_t n_vae_done = 0; // latent frames
+ llama_context_ptr scratch; // backbone keys/values of a chunk other than the first
+ uint32_t scratch_n_ctx = 0;
+ std::vector<llama_token> history; // tokens sampled in the current phase
+ std::vector<int32_t> codes;
+ bool done = false;
+ bool decoded = false;
+ std::mt19937 rng;
+
+ mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
+ bool stream = false;
+ std::vector<float> h_state_buf;
+ std::vector<float> audio_pcm; // interleaved
+ std::vector<char> out_buf;
+ size_t pcm_sent = 0;
+ bool wav_header_sent = false;
+};
+
static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
switch (mtmd_gen_audio_get_info(mctx).type) {
case MTMD_GEN_AUDIO_TYPE_QWEN3TTS:
@@ -3169,6 +3826,8 @@ static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lc
return std::unique_ptr<mtmd_gen_audio_pipeline>(new drama3_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_ZONOS3:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new zonos3_gen_audio_pipeline(lctx, mctx));
+ case MTMD_GEN_AUDIO_TYPE_YUE2:
+ return std::unique_ptr<mtmd_gen_audio_pipeline>(new yue2_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_FISH_S2:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new fish_s2_gen_audio_pipeline(lctx, mctx));
default:
@@ -3185,6 +3844,10 @@ mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(struct llama_context * lctx,
auto * ctx = new mtmd_helper_gen_audio();
ctx->lctx = lctx;
ctx->pipeline = make_pipeline(lctx, mctx);
+ if (ctx->pipeline && !ctx->pipeline->check_context()) {
+ delete ctx;
+ return nullptr;
+ }
return ctx;
}
@@ -3192,6 +3855,10 @@ void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx) {
delete ctx;
}
+int32_t mtmd_helper_gen_audio_max_frames(const mtmd_helper_gen_audio * ctx) {
+ return ctx && ctx->pipeline ? ctx->pipeline->max_frames() : 0;
+}
+
void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) {
if (ctx && ctx->pipeline) {
ctx->pipeline->reset();
@@ -3206,6 +3873,8 @@ struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) {
inp.seed = UINT32_MAX; // random
inp.speed = 1.0f;
inp.loudness = 0.0f;
+ inp.cfg_scale = 0.0f; // model default
+ inp.flow_steps = 0; // model default
inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
return inp;
}
@@ -3227,6 +3896,22 @@ int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_
LOG_ERR("mtmd_helper_gen_audio: the model does not support speed or loudness\n");
return 2;
}
+ if (!std::isfinite(inp->cfg_scale) || inp->cfg_scale < 0.0f) {
+ LOG_ERR("mtmd_helper_gen_audio: cfg_scale must be > 0 (0 = model default)\n");
+ return 2;
+ }
+ if (inp->cfg_scale != 0.0f && !ctx->pipeline->supports_cfg()) {
+ LOG_ERR("mtmd_helper_gen_audio: the model does not support cfg_scale\n");
+ return 2;
+ }
+ if (inp->flow_steps < 0 || inp->flow_steps > 1024) {
+ LOG_ERR("mtmd_helper_gen_audio: flow_steps must be in [1, 1024] (0 = model default)\n");
+ return 2;
+ }
+ if (inp->flow_steps != 0 && !ctx->pipeline->supports_flow_steps()) {
+ LOG_ERR("mtmd_helper_gen_audio: the model does not support flow_steps\n");
+ return 2;
+ }
ctx->pipeline->logit_bias.clear();
return ctx->pipeline->set_input(inp);
}
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index 82e86bd1f..20465cbd3 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -287,8 +287,8 @@ struct mtmd_helper_gen_audio;
typedef struct mtmd_helper_gen_audio mtmd_helper_gen_audio;
enum mtmd_helper_gen_audio_outtype {
- MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM, // raw PCM
- MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV, // WAV PCM 16-bit LE, mono
+ MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM, // raw F32 PCM, channels interleaved
+ MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV, // WAV PCM 16-bit LE
};
// what a response that continues a sequence (n_past > 0) outputs
enum mtmd_helper_gen_audio_out_mode {
@@ -322,11 +322,20 @@ struct mtmd_helper_gen_audio_inp {
float speed; // speaking speed relative to a typical pace, > 0 (default: 1.0 = the model's own pace)
float loudness; // target loudness in LUFS, < 0 (default: 0.0 = the model's own loudness)
+ // classifier-free guidance scale of the audio codes, > 0, 1.0 = no guidance (only for models that support it)
+ // default: 0.0 = the model's own; other models reject non-default values
+ float cfg_scale;
+
+ // steps of the flow-matching ODE solver, fewer is faster with less accurate audio (only for models that support it)
+ // default: 0 = the model's own; other models reject non-default values
+ int32_t flow_steps;
+
enum mtmd_helper_gen_audio_outtype out_type;
};
MTMD_API struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void);
+// returns nullptr if the context cannot run the pipeline of the model (e.g. a context too small)
MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(
struct llama_context * lctx,
struct mtmd_context * mctx);
@@ -335,6 +344,9 @@ MTMD_API void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx);
MTMD_API void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx);
+// frame limit for a caller that has none of its own (e.g. no -n); 0 = step_gen() stops by itself
+MTMD_API int32_t mtmd_helper_gen_audio_max_frames(const mtmd_helper_gen_audio * ctx);
+
// returns 0 on success, 2 if the input is invalid for the model (e.g. unsupported speed or loudness), 1 on other errors
MTMD_API int32_t mtmd_helper_gen_audio_set_input(
mtmd_helper_gen_audio * ctx,
@@ -370,7 +382,7 @@ MTMD_API int32_t mtmd_helper_audio_stream_get_logit_bias(const mtmd_context * mc
llama_logit_bias * out, int32_t n_max);
// out_data valid until next get_output() or reset() call
-// out_n_samples (optional, can be NULL) receives the number of generated PCM samples
+// out_n_samples (optional, can be NULL) receives the number of generated PCM samples per channel (see mtmd_gen_audio_info::n_channels)
// if inp->stream is true: returns only audio produced since the previous call, and
// *out_data_len == 0 whenever a full window_frames batch hasn't accumulated yet
MTMD_API int32_t mtmd_helper_gen_audio_get_output(
@@ -456,6 +468,11 @@ struct gen_audio {
bool valid() const {
return ctx.get() != nullptr;
}
+ // caller's limit if > 0, else the limit of the model
+ int32_t max_frames(int32_t n_predict) const {
+ const int32_t n_model = mtmd_helper_gen_audio_max_frames(ctx.get());
+ return n_predict > 0 ? n_predict : n_model > 0 ? n_model : INT32_MAX;
+ }
void reset() {
mtmd_helper_gen_audio_reset(ctx.get());
}
diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h
index ff2b70230..eb93a1f8d 100644
--- a/tools/mtmd/mtmd-impl.h
+++ b/tools/mtmd/mtmd-impl.h
@@ -613,6 +613,7 @@ enum projector_type {
PROJECTOR_TYPE_DRAMA3_GEN,
PROJECTOR_TYPE_ZONOS3_GEN,
PROJECTOR_TYPE_ZONOS3_SPKENC,
+ PROJECTOR_TYPE_YUE2_GEN,
PROJECTOR_TYPE_FISH_S2_GEN,
PROJECTOR_TYPE_FISH_S2_SPKENC,
PROJECTOR_TYPE_MUSE_GLIMMER,
@@ -707,6 +708,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_DRAMA3_GEN, "drama3_gen"},
{ PROJECTOR_TYPE_ZONOS3_GEN, "zonos3_gen"},
{ PROJECTOR_TYPE_ZONOS3_SPKENC, "zonos3_spkenc"},
+ { PROJECTOR_TYPE_YUE2_GEN, "yue2_gen"},
{ PROJECTOR_TYPE_FISH_S2_GEN, "fish_s2_gen"},
{ PROJECTOR_TYPE_FISH_S2_SPKENC, "fish_s2_spkenc"},
{ PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"},
diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp
index 9e2b2fec1..a183a1a67 100644
--- a/tools/mtmd/mtmd-model.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -1093,6 +1093,10 @@ struct mtmd_model_loader {
{
hparams.audio_sample_rate = 44100;
} break;
+ case PROJECTOR_TYPE_YUE2_GEN:
+ {
+ hparams.audio_sample_rate = 48000;
+ } break;
case PROJECTOR_TYPE_DRAMA3_GEN:
case PROJECTOR_TYPE_FISH_S2_GEN:
{
@@ -1565,7 +1569,8 @@ struct mtmd_model_loader {
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
model.proj_type != PROJECTOR_TYPE_CHATTERBOX &&
- model.proj_type != PROJECTOR_TYPE_CHATTERBOX_SPKENC);
+ model.proj_type != PROJECTOR_TYPE_CHATTERBOX_SPKENC &&
+ model.proj_type != PROJECTOR_TYPE_YUE2_GEN);
// layers
const int n_layers_to_load = has_standard_layers ? hparams.n_layer : 0;
@@ -2892,6 +2897,73 @@ struct mtmd_model_loader {
z.rs_new = rs->ne[1];
z.rs_kernel_len = rs->ne[0];
} break;
+ case PROJECTOR_TYPE_YUE2_GEN:
+ {
+ auto & y = model.yue2;
+ y.inp_w = get_tensor("a.gen.nar.inp.weight");
+ y.inp_b = get_tensor("a.gen.nar.inp.bias");
+ y.out_w = get_tensor("a.gen.nar.out.weight");
+ y.out_b = get_tensor("a.gen.nar.out.bias");
+ if (y.inp_w->ne[0] != mtmd_graph_yue2_gen::n_latent || y.out_w->ne[1] != mtmd_graph_yue2_gen::n_latent ||
+ y.inp_w->ne[1] != hparams.n_embd || y.out_w->ne[0] != hparams.n_embd) {
+ throw std::runtime_error(mtmd_format("%s: yue2: unexpected latent projection shape\n", __func__));
+ }
+
+ auto read_host = [&](const std::string & name, std::vector<float> & dst, int64_t ne0, int64_t ne1) {
+ const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name.c_str());
+ if (!t || t->type != GGML_TYPE_F32 || t->ne[0] != ne0 || t->ne[1] != ne1 || ggml_n_dims(t) > 2) {
+ throw std::runtime_error(mtmd_format("%s: invalid or missing tensor %s\n", __func__, name.c_str()));
+ }
+ dst.resize(ggml_nelements(t));
+ fin.seekg(tensor_offset.at(name), std::ios::beg);
+ if (!fin.read(reinterpret_cast<char *>(dst.data()), ggml_nbytes(t))) {
+ throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name.c_str()));
+ }
+ };
+ const int64_t n_embd = hparams.n_embd;
+ read_host("a.gen.nar.time.0.weight", y.time0_w, mtmd_graph_yue2_gen::n_freq, n_embd);
+ read_host("a.gen.nar.time.0.bias", y.time0_b, n_embd, 1);
+ read_host("a.gen.nar.time.2.weight", y.time2_w, n_embd, n_embd);
+ read_host("a.gen.nar.time.2.bias", y.time2_b, n_embd, 1);
+
+ const std::string p = "a.gen.vae";
+ auto load_snake = [&](const std::string & q) {
+ return mtmd_yue2::snake{ get_tensor(q + ".alpha"), get_tensor(q + ".beta") };
+ };
+ y.vae_in_w = get_tensor(p + ".conv_in.weight");
+ y.vae_in_b = get_tensor(p + ".conv_in.bias");
+ for (int b = 0; gguf_find_tensor(ctx_gguf.get(), mtmd_format("%s.blk.%d.up.weight", p.c_str(), b).c_str()) >= 0; b++) {
+ const std::string q = p + ".blk." + std::to_string(b);
+ mtmd_yue2::block blk;
+ blk.act = load_snake(q + ".snake");
+ blk.up_w = get_tensor(q + ".up.weight");
+ blk.up_b = get_tensor(q + ".up.bias");
+ for (int r = 0; r < 3; r++) {
+ const std::string rq = q + ".res." + std::to_string(r);
+ mtmd_yue2::res_unit ru;
+ ru.act1 = load_snake(rq + ".snake1");
+ ru.conv1_w = get_tensor(rq + ".conv1.weight");
+ ru.conv1_b = get_tensor(rq + ".conv1.bias");
+ ru.act2 = load_snake(rq + ".snake2");
+ ru.conv2_w = get_tensor(rq + ".conv2.weight");
+ ru.conv2_b = get_tensor(rq + ".conv2.bias");
+ blk.res.push_back(ru);
+ }
+ y.vae.push_back(std::move(blk));
+ }
+ y.vae_post = load_snake(p + ".post_snake");
+ y.vae_out_w = get_tensor(p + ".conv_out.weight");
+
+ // the tile cropping in mtmd_gen_wav_yue2() depends on the total stride
+ int64_t ratio = 1;
+ for (const auto & blk : y.vae) {
+ ratio *= blk.up_w->ne[0] / 2;
+ }
+ if (ratio != mtmd_graph_yue2_gen::vae_ratio || y.vae_in_w->ne[1] != mtmd_graph_yue2_gen::n_latent ||
+ y.vae_out_w->ne[2] != mtmd_graph_yue2_gen::n_channels) {
+ throw std::runtime_error(mtmd_format("%s: yue2: unexpected VAE decoder layout\n", __func__));
+ }
+ } break;
case PROJECTOR_TYPE_DRAMA3_GEN:
{
// generic a.gen.code.blk.* tensors are loaded by the loop above
@@ -4537,6 +4609,7 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
case PROJECTOR_TYPE_LFM3A_GEN:
case PROJECTOR_TYPE_DRAMA3_GEN:
case PROJECTOR_TYPE_ZONOS3_GEN:
+ case PROJECTOR_TYPE_YUE2_GEN:
case PROJECTOR_TYPE_FISH_S2_GEN:
{
// one hidden-state vector fed back to the talker per call
@@ -4736,6 +4809,8 @@ int mtmd_model::n_mmproj_embd() const {
case PROJECTOR_TYPE_FISH_S2_GEN:
case PROJECTOR_TYPE_FISH_S2_SPKENC:
return gen_code_out_embd_w->ne[0];
+ case PROJECTOR_TYPE_YUE2_GEN:
+ return yue2.inp_w->ne[1];
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
return spk_proj_w->ne[1];
case PROJECTOR_TYPE_POCKETTTS_GEN:
diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h
index 9af815981..8a2575044 100644
--- a/tools/mtmd/mtmd-model.h
+++ b/tools/mtmd/mtmd-model.h
@@ -943,6 +943,46 @@ struct mtmd_zonos3_wav {
int64_t rs_kernel_len = 0;
};
+// yue2: projections of the flow-matching NAR (a.gen.nar.*) and the Oobleck VAE decoder (a.gen.vae.*)
+// the NAR transformer layers are loaded by libllama, see mtmd_yue2_nar_load()
+struct mtmd_yue2 {
+ struct snake {
+ ggml_tensor * alpha = nullptr; // exp(alpha)
+ ggml_tensor * beta = nullptr; // 1 / (exp(beta) + eps)
+ };
+ struct res_unit {
+ snake act1;
+ ggml_tensor * conv1_w = nullptr; // k = 7, dilated
+ ggml_tensor * conv1_b = nullptr;
+ snake act2;
+ ggml_tensor * conv2_w = nullptr; // k = 1
+ ggml_tensor * conv2_b = nullptr;
+ };
+ struct block {
+ snake act;
+ ggml_tensor * up_w = nullptr; // ConvTranspose1d, k = 2 * stride
+ ggml_tensor * up_b = nullptr;
+ std::vector<res_unit> res;
+ };
+
+ ggml_tensor * inp_w = nullptr; // latent -> backbone embd
+ ggml_tensor * inp_b = nullptr;
+ ggml_tensor * out_w = nullptr; // backbone hidden -> velocity
+ ggml_tensor * out_b = nullptr;
+
+ // time embedder MLP, read on the host
+ std::vector<float> time0_w; // [256, n_embd]
+ std::vector<float> time0_b;
+ std::vector<float> time2_w; // [n_embd, n_embd]
+ std::vector<float> time2_b;
+
+ ggml_tensor * vae_in_w = nullptr;
+ ggml_tensor * vae_in_b = nullptr;
+ std::vector<block> vae;
+ snake vae_post;
+ ggml_tensor * vae_out_w = nullptr;
+};
+
struct mtmd_vocos {
ggml_tensor * input_w = nullptr;
ggml_tensor * input_b = nullptr;
@@ -1226,6 +1266,7 @@ struct mtmd_model {
mtmd_code2wav c2w;
std::vector<mtmd_drama3_layer> drama3_layers;
mtmd_zonos3_wav zonos3;
+ mtmd_yue2 yue2;
// pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path)
mtmd_seanet seanet;
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index db3b241d4..4448f84f3 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -1506,6 +1506,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
}
} break;
case PROJECTOR_TYPE_SOPRANO:
+ case PROJECTOR_TYPE_YUE2_GEN:
{
set_input_f32("inp_feats", *params->feats);
} break;
@@ -2632,8 +2633,8 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
mtmd_graph_soprano::decode_spectrum(model, spectrum, out_audio);
}
- // drop the tail audio that comes from the code-0 rear padding, drama3 and fish_s2 have no padding
- if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN && model.proj_type != PROJECTOR_TYPE_FISH_S2_GEN) {
+ // drop the tail audio that comes from the code-0 rear padding, drama3, fish_s2 and yue2 have no padding
+ if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN && model.proj_type != PROJECTOR_TYPE_FISH_S2_GEN && model.proj_type != PROJECTOR_TYPE_YUE2_GEN) {
const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0;
const int64_t n_frames_w = hparams.wav_tfm_swa;
const int64_t n_frames = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w;
diff --git a/tools/mtmd/mtmd-subcontext.h b/tools/mtmd/mtmd-subcontext.h
index 7e6d8232a..f03fbe5ca 100644
--- a/tools/mtmd/mtmd-subcontext.h
+++ b/tools/mtmd/mtmd-subcontext.h
@@ -69,6 +69,8 @@ enum mtmd_subctx_gen_process {
MTMD_SUBCTX_GEN_PROCESS_TTS, // full utterance of codes to raw PCM audio
MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE, // internal: mel + source stft to istft input
MTMD_SUBCTX_GEN_PROCESS_TOKENIZE, // raw PCM audio to semantic speech tokens
+ MTMD_SUBCTX_GEN_PROCESS_LATENT_IN, // internal: flow latent rows to backbone embd rows
+ MTMD_SUBCTX_GEN_PROCESS_LATENT_OUT, // internal: backbone hidden rows to flow velocity rows
};
// persistent state of a causal audio encoder, for streaming input (one per stream)
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 9669afc2a..8696525c5 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -8,6 +8,7 @@
#include "debug/mtmd-debug.h"
#include "llama.h"
+#include "llama-cpp.h"
#include "../src/llama-ext.h"
// fix problem with std::min and std::max
@@ -613,6 +614,103 @@ static bool cbx_is_mtl(const mtmd_model & model) {
return model.cbx_read_tensor("a.cenc.perceiver.pre_attention_query", nullptr, 0) > 0;
}
+// yue2: the NAR layers are a qwen3 text model, libllama loads them from a synthetic GGUF
+// the tensor data is read from the mmproj file, mtmd keeps no copy
+struct yue2_nar_source {
+ std::ifstream f;
+ std::map<std::string, std::pair<size_t, size_t>> tensors; // name in the synthetic model -> offset and size in the mmproj
+};
+
+static void yue2_nar_set_tensor_data(ggml_tensor * t, void * user_data) {
+ auto * src = (yue2_nar_source *) user_data;
+ std::vector<uint8_t> buf(ggml_nbytes(t), 0);
+ const std::string name = t->name;
+ auto it = src->tensors.find(name);
+ if (it != src->tensors.end()) {
+ if (it->second.second != buf.size()) {
+ throw std::runtime_error(mtmd_format("yue2: unexpected size of %s", t->name));
+ }
+ src->f.seekg((std::streamoff) it->second.first, std::ios::beg);
+ if (!src->f.read((char *) buf.data(), (std::streamsize) buf.size())) {
+ throw std::runtime_error(mtmd_format("yue2: failed to read %s", t->name));
+ }
+ } else {
+ // optional tensors that libllama creates by itself, give them neutral values (scale = 1, dense_2 = identity, others = 0)
+ GGML_ASSERT(t->type == GGML_TYPE_F32);
+ float * w = (float *) buf.data();
+ if (name.size() >= 5 && name.compare(name.size() - 5, 5, "scale") == 0) {
+ std::fill(w, w + ggml_nelements(t), 1.0f);
+ } else if (name == "dense_2.weight") {
+ GGML_ASSERT(t->ne[0] == t->ne[1]);
+ for (int64_t i = 0; i < t->ne[0]; i++) {
+ w[i * t->ne[0] + i] = 1.0f;
+ }
+ }
+ }
+ ggml_backend_tensor_set(t, buf.data(), 0, buf.size());
+}
+
+static llama_model * mtmd_yue2_nar_load(const char * fname, ggml_backend_dev_t device, const mtmd_hparams & hparams) {
+ ggml_context * meta = nullptr;
+ gguf_init_params gparams = { /*no_alloc =*/ true, /*ctx =*/ &meta };
+ gguf_context_ptr gsrc(gguf_init_from_file(fname, gparams));
+ ggml_context_ptr meta_ptr(meta);
+ if (!gsrc) {
+ throw std::runtime_error(mtmd_format("yue2: failed to read %s", fname));
+ }
+
+ const llm_arch arch = LLM_ARCH_QWEN3;
+ gguf_context_ptr gdst(gguf_init_empty());
+ auto key = [&](llm_kv kv) { return llama_arch_kv_name(arch, kv); };
+ gguf_set_val_str(gdst.get(), key(LLM_KV_GENERAL_ARCHITECTURE).c_str(), llama_arch_name(arch));
+ gguf_set_val_str(gdst.get(), key(LLM_KV_TOKENIZER_MODEL).c_str(), "none");
+ gguf_set_val_u32(gdst.get(), key(LLM_KV_VOCAB_SIZE).c_str(), 1);
+ gguf_set_val_u32(gdst.get(), key(LLM_KV_CONTEXT_LENGTH).c_str(), mtmd_graph_yue2_gen::n_ctx_max);
+ gguf_set_val_u32(gdst.get(), key(LLM_KV_EMBEDDING_LENGTH).c_str(), hparams.n_embd);
+ gguf_set_val_u32(gdst.get(), key(LLM_KV_FEED_FORWARD_LENGTH).c_str(), hparams.n_ff);
+ gguf_set_val_u32(gdst.get(), key(LLM_KV_BLOCK_COUNT).c_str(), hparams.n_layer);
+ gguf_set_val_u32(gdst.get(), key(LLM_KV_ATTENTION_HEAD_COUNT).c_str(), hparams.n_head);
+ gguf_set_val_u32(gdst.get(), key(LLM_KV_ATTENTION_HEAD_COUNT_KV).c_str(), hparams.n_head_kv);
+ gguf_set_val_f32(gdst.get(), key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS).c_str(), hparams.eps);
+ gguf_set_val_f32(gdst.get(), key(LLM_KV_ROPE_FREQ_BASE).c_str(), mtmd_graph_yue2_gen::rope_theta);
+
+ yue2_nar_source src;
+ const std::string prefix = "a.gen.nar.";
+ for (int64_t i = 0; i < gguf_get_n_tensors(gsrc.get()); i++) {
+ const std::string name = gguf_get_tensor_name(gsrc.get(), i);
+ if (name.rfind(prefix + "blk.", 0) != 0 && name != prefix + "output_norm.weight") {
+ continue;
+ }
+ // the synthetic model only takes the type from here, the shapes come from the hparams
+ ggml_tensor t = *ggml_get_tensor(meta, name.c_str());
+ ggml_set_name(&t, name.substr(prefix.size()).c_str());
+ gguf_add_tensor(gdst.get(), &t);
+ src.tensors[t.name] = { gguf_get_data_offset(gsrc.get()) + gguf_get_tensor_offset(gsrc.get(), i), ggml_nbytes(&t) };
+ }
+ if (src.tensors.size() != (size_t) hparams.n_layer * 11 + 1) {
+ throw std::runtime_error("yue2: unexpected NAR tensors in the mmproj");
+ }
+ src.f = open_ifstream_binary(fname);
+ if (!src.f) {
+ throw std::runtime_error(mtmd_format("yue2: failed to open %s", fname));
+ }
+
+ llama_model_params mparams = llama_model_default_params();
+ std::vector<ggml_backend_dev_t> devs;
+ if (device && ggml_backend_dev_type(device) != GGML_BACKEND_DEVICE_TYPE_CPU) {
+ devs = { device, nullptr };
+ mparams.devices = devs.data();
+ mparams.n_gpu_layers = -1;
+ } else {
+ mparams.n_gpu_layers = 0;
+ }
+ llama_model * model = llama_model_init_from_user(gdst.get(), yue2_nar_set_tensor_data, &src, mparams);
+ if (!model) {
+ throw std::runtime_error("yue2: failed to load the NAR layers");
+ }
+ return model;
+}
+
struct mtmd_context {
// must outlive the subcontexts below
mtmd_models models;
@@ -628,6 +726,10 @@ struct mtmd_context {
// generation context
std::unique_ptr<mtmd_subcontext> ctx_gen_a; // audio
+ // yue2: the NAR layers, run on a llama_context of their own, see mtmd_gen_flow_yue2()
+ llama_model_ptr yue2_nar;
+ llama_flash_attn_type yue2_flash_attn = LLAMA_FLASH_ATTN_TYPE_AUTO;
+
// gen-side reference data of the last encoded chunk (chatterbox), see mtmd_gen_state_set_ref()
mtmd_gen_ref gen_ref;
@@ -761,6 +863,10 @@ struct mtmd_context {
if (models.gen_a) {
ctx_gen_a = std::make_unique<mtmd_subcontext>(*models.gen_a, *backends, sctx_params);
}
+ if (models.gen_a && models.gen_a->proj_type == PROJECTOR_TYPE_YUE2_GEN && !no_alloc) {
+ yue2_nar.reset(mtmd_yue2_nar_load(mmproj_fname, device, models.gen_a->hparams));
+ yue2_flash_attn = ctx_params.flash_attn_type;
+ }
for (size_t i = 0; i < backends->ptrs.size(); ++i) {
const size_t size = ggml_backend_sched_get_buffer_size(backends->sched.get(), backends->ptrs[i]);
if (size > 1) {
@@ -2459,6 +2565,7 @@ float * mtmd_get_output_embd(mtmd_context * ctx) {
mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
mtmd_gen_audio_info info{};
info.model_variant = "";
+ info.n_channels = 1;
if (!ctx->ctx_gen_a) {
info.type = MTMD_GEN_AUDIO_TYPE_NONE;
return info;
@@ -2494,6 +2601,11 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
info.type = MTMD_GEN_AUDIO_TYPE_ZONOS3;
info.sample_rate = 44100;
break;
+ case PROJECTOR_TYPE_YUE2_GEN:
+ info.type = MTMD_GEN_AUDIO_TYPE_YUE2;
+ info.sample_rate = 48000;
+ info.n_channels = mtmd_graph_yue2_gen::n_channels;
+ break;
case PROJECTOR_TYPE_FISH_S2_GEN:
info.type = MTMD_GEN_AUDIO_TYPE_FISH_S2;
info.sample_rate = 44100;
@@ -3121,6 +3233,219 @@ static int32_t mtmd_gen_wav_lfm3a(mtmd_context * ctx, mtmd_gen_state * state, co
return 0;
}
+// yue2: one pass of an mtmd graph, rows in -> rows out (PCM for GEN_WAV)
+static bool yue2_encode(mtmd_context * ctx, mtmd_gen_state * state, mtmd_subctx_gen_process proc, const std::vector<float> & in, std::vector<float> & out) {
+ mtmd_image_f32 dummy;
+ dummy.set_size({1, 1}, false, true);
+ dummy.cpy_buf(std::vector<float>(1, 0.0f));
+ mtmd_image_f32_batch batch;
+ batch.is_audio = true;
+ batch.entries.push_back(std::move(dummy));
+
+ mtmd_subctx_encode_params params;
+ params.imgs = &batch;
+ params.n_threads = ctx->n_threads;
+ params.gen_process = proc;
+ params.feats = ∈
+ if (proc == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
+ params.out_audio = &out;
+ params.gen_state = state->gen_state;
+ } else {
+ params.out_feats = &out;
+ }
+ return ctx->ctx_gen_a->encode(¶ms);
+}
+
+// NAR context: the backbone keys/values, then the NAR rows of each pass
+// the state only loads into the same KV layout, try both flash attention settings
+static llama_context_ptr yue2_nar_context(mtmd_context * ctx, const mtmd_gen_inp * inp, int32_t n_rows) {
+ llama_context_params cparams = llama_context_default_params();
+ cparams.n_ctx = (uint32_t) (inp->backbone_n_past + n_rows);
+ cparams.n_batch = (uint32_t) n_rows;
+ cparams.n_ubatch = (uint32_t) n_rows; // non-causal, the rows of a pass must be in one ubatch
+ cparams.n_seq_max = 1;
+ cparams.n_threads = ctx->n_threads;
+ cparams.n_threads_batch = ctx->n_threads;
+ cparams.embeddings = true;
+ cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;
+ cparams.attention_type = LLAMA_ATTENTION_TYPE_NON_CAUSAL;
+ cparams.no_perf = true;
+
+ std::vector<llama_flash_attn_type> fa_types = { ctx->yue2_flash_attn };
+ if (ctx->yue2_flash_attn == LLAMA_FLASH_ATTN_TYPE_AUTO) {
+ fa_types = { LLAMA_FLASH_ATTN_TYPE_ENABLED, LLAMA_FLASH_ATTN_TYPE_DISABLED };
+ }
+ for (llama_flash_attn_type fa : fa_types) {
+ cparams.flash_attn_type = fa;
+ llama_context_ptr lctx(llama_init_from_model(ctx->yue2_nar.get(), cparams));
+ if (!lctx) {
+ continue;
+ }
+ if (llama_state_seq_set_data(lctx.get(), inp->backbone_state, inp->backbone_state_size, 0) == 0) {
+ continue;
+ }
+ if (llama_memory_seq_pos_max(llama_get_memory(lctx.get()), 0) != inp->backbone_n_past - 1) {
+ LOG_ERR("%s: the backbone state does not hold %d positions\n", __func__, inp->backbone_n_past);
+ return nullptr;
+ }
+ return lctx;
+ }
+ LOG_ERR("%s: cannot load the backbone state, the backbone context must use an F16 KV cache\n", __func__);
+ return nullptr;
+}
+
+// yue2 GEN_CODE: flow matching from noise to the VAE latents of one chunk, see synthesize() in the reference (nar.py)
+static int32_t mtmd_gen_flow_yue2(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
+ using g = mtmd_graph_yue2_gen;
+ const auto & y = ctx->ctx_gen_a->model.yue2;
+ const int64_t n_embd = ctx->ctx_gen_a->model.hparams.n_embd;
+
+ if (!ctx->yue2_nar || inp->n_codes == 0 || !inp->backbone_state || inp->backbone_n_past <= 0) {
+ LOG_ERR("%s: yue2 needs the frame count and the backbone state\n", __func__);
+ return 1;
+ }
+ const int64_t n_frames = (int64_t) inp->n_codes;
+ const int64_t n_rows = n_frames + 2; // latent start, frames, latent end
+ const int64_t n_past = inp->backbone_n_past;
+ if (n_past + n_rows > g::n_ctx_max) {
+ LOG_ERR("%s: song too long: %lld prompt positions + %lld frames, max %d positions\n", __func__,
+ (long long) n_past, (long long) n_frames, g::n_ctx_max);
+ return 1;
+ }
+ llama_context_ptr lctx = yue2_nar_context(ctx, inp, (int32_t) n_rows);
+ if (!lctx) {
+ return 1;
+ }
+ llama_memory_t mem = llama_get_memory(lctx.get());
+
+ // sinusoidal position of each NAR row (AudioPositionEmbedding), interleaved sin/cos
+ std::vector<float> pos_embd((size_t) (n_rows * n_embd));
+ for (int64_t i = 0; i < n_rows; i++) {
+ for (int64_t j = 0; j < n_embd / 2; j++) {
+ const double a = (double) i * std::exp((double) (2 * j) * -std::log(10000.0) / (double) n_embd);
+ pos_embd[i * n_embd + 2 * j] = (float) std::sin(a);
+ pos_embd[i * n_embd + 2 * j + 1] = (float) std::cos(a);
+ }
+ }
+
+ // TimestepEmbedder: [cos, sin] of t * freqs -> linear -> silu -> linear
+ auto time_embd = [&](double t) {
+ const double raw = std::clamp(std::log(t / (1.0 - t)), -20.0, 20.0);
+ const double ts = 1.0 / (1.0 + std::exp(-raw)); // timestep_shift = 1
+ const int64_t half = g::n_freq / 2;
+ std::vector<float> f(g::n_freq);
+ for (int64_t j = 0; j < half; j++) {
+ const double a = ts * std::exp(-std::log(10000.0) * (double) j / (double) half);
+ f[j] = (float) std::cos(a);
+ f[half + j] = (float) std::sin(a);
+ }
+ std::vector<float> h(y.time0_b), o(y.time2_b);
+ for (int64_t r = 0; r < n_embd; r++) {
+ for (int64_t c = 0; c < g::n_freq; c++) {
+ h[r] += y.time0_w[r * g::n_freq + c] * f[c];
+ }
+ h[r] = h[r] / (1.0f + std::exp(-h[r]));
+ }
+ for (int64_t r = 0; r < n_embd; r++) {
+ for (int64_t c = 0; c < n_embd; c++) {
+ o[r] += y.time2_w[r * n_embd + c] * h[c];
+ }
+ }
+ return o;
+ };
+
+ llama_batch_ext_ptr batch(llama_batch_ext_init(lctx.get()));
+ std::vector<float> x_rows((size_t) (n_rows * g::n_latent), 0.0f);
+ std::vector<float> embd, hidden, vel;
+ auto velocity = [&](const std::vector<float> & x, double t, std::vector<float> & v) -> bool {
+ std::copy(x.begin(), x.end(), x_rows.begin() + g::n_latent); // the start and end rows stay 0
+ if (!yue2_encode(ctx, state, MTMD_SUBCTX_GEN_PROCESS_LATENT_IN, x_rows, embd)) {
+ return false;
+ }
+ const std::vector<float> temb = time_embd(t);
+ llama_batch_ext_clear(batch.get());
+ for (int64_t i = 0; i < n_rows; i++) {
+ for (int64_t j = 0; j < n_embd; j++) {
+ embd[i * n_embd + j] = embd[i * n_embd + j] + temb[j] + pos_embd[i * n_embd + j];
+ }
+ const llama_embd row = { embd.data() + i * n_embd, 1, (size_t) n_embd };
+ const llama_pos p = (llama_pos) (n_past + i);
+ const int32_t idx = llama_batch_ext_add_embd(batch.get(), 0, row);
+ if (idx < 0) {
+ LOG_ERR("%s: failed to add a row to the batch (error %d)\n", __func__, idx);
+ return false;
+ }
+ llama_batch_ext_set_pos(batch.get(), idx, &p);
+ llama_batch_ext_set_output_embd(batch.get(), idx, true);
+ }
+ if (llama_process(lctx.get(), LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
+ LOG_ERR("%s: NAR decode failed\n", __func__);
+ return false;
+ }
+ const float * h = llama_get_embeddings(lctx.get());
+ hidden.assign(h, h + n_rows * n_embd);
+ llama_memory_seq_rm(mem, 0, (llama_pos) n_past, -1);
+ if (!yue2_encode(ctx, state, MTMD_SUBCTX_GEN_PROCESS_LATENT_OUT, hidden, vel)) {
+ return false;
+ }
+ v.assign(vel.begin() + g::n_latent, vel.end() - g::n_latent);
+ return true;
+ };
+
+ std::normal_distribution<float> dist(0.0f, 1.0f);
+ std::vector<float> x((size_t) (n_frames * g::n_latent));
+ for (auto & v : x) {
+ v = dist(state->gen_state->rng);
+ }
+
+ // midpoint solver from t = 1 (noise) to t = 0
+ std::vector<float> v1, v2, mid(x.size());
+ const int n_steps = inp->n_steps > 0 ? inp->n_steps : g::ode_steps;
+ const double dt = 1.0 / n_steps;
+ bool ok = true;
+ for (int step = 0; step < n_steps && ok; step++) {
+ const double t = 1.0 - step * dt;
+ ok = velocity(x, t, v1);
+ for (size_t i = 0; ok && i < x.size(); i++) {
+ mid[i] = x[i] - v1[i] * (float) (dt / 2);
+ }
+ ok = ok && velocity(mid, t - dt / 2, v2);
+ for (size_t i = 0; ok && i < x.size(); i++) {
+ x[i] -= v2[i] * (float) dt;
+ }
+ }
+ if (!ok) {
+ return 1;
+ }
+
+ state->out_feats = std::move(x);
+ out->feats = state->out_feats.data();
+ out->n_feats = state->out_feats.size();
+ return 0;
+}
+
+// yue2 GEN_WAV: latents -> interleaved stereo PCM in one VAE pass, the caller splits a long song into tiles
+static int32_t mtmd_gen_wav_yue2(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
+ using g = mtmd_graph_yue2_gen;
+ if (!inp->feats || inp->n_feats == 0 || inp->n_feats % g::n_latent != 0) {
+ LOG_ERR("%s: yue2 needs whole latent frames\n", __func__);
+ return 1;
+ }
+ const std::vector<float> feats(inp->feats, inp->feats + inp->n_feats);
+ auto & audio = state->out_audio;
+ if (!yue2_encode(ctx, state, MTMD_SUBCTX_GEN_PROCESS_GEN_WAV, feats, audio)) {
+ LOG_ERR("%s: encode failed (vae)\n", __func__);
+ return 1;
+ }
+ for (auto & v : audio) {
+ v = std::clamp(v, -1.0f, 1.0f);
+ }
+
+ out->audio = audio.data();
+ out->n_samples = audio.size();
+ return 0;
+}
+
int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) {
if (!state || state->sctx != ctx->ctx_gen_a.get()) {
LOG_ERR("%s: invalid gen state\n", __func__);
@@ -3147,6 +3472,11 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
*out = {};
+ if (sctx->model.proj_type == PROJECTOR_TYPE_YUE2_GEN) {
+ return inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE ? mtmd_gen_flow_yue2(ctx, state, inp, out)
+ : mtmd_gen_wav_yue2(ctx, state, inp, out);
+ }
+
if (sctx->model.proj_type == PROJECTOR_TYPE_SOPRANO) {
const size_t n_embd = (size_t) sctx->model.n_mmproj_embd();
if (inp->type != MTMD_GEN_PROCESS_TYPE_GEN_WAV || !inp->feats || inp->n_feats < 2 * n_embd || inp->n_feats % n_embd != 0 || inp->codes) {
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index 58c7c57a8..9ad2c4e2f 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -473,12 +473,14 @@ enum mtmd_gen_audio_type {
MTMD_GEN_AUDIO_TYPE_CHATTERBOX,
MTMD_GEN_AUDIO_TYPE_DRAMA3,
MTMD_GEN_AUDIO_TYPE_ZONOS3,
+ MTMD_GEN_AUDIO_TYPE_YUE2,
MTMD_GEN_AUDIO_TYPE_FISH_S2,
};
struct mtmd_gen_audio_info {
enum mtmd_gen_audio_type type;
int32_t sample_rate; // in Hz, for example 24000 for qwen3tts
+ int32_t n_channels; // 1 = mono, 2 = stereo (mtmd_gen_out::audio is interleaved)
const char * model_variant; // name of the weight variant, can be nullptr if not applicable
int32_t n_voices; // predefined voices, see mtmd_gen_audio_voice_name()
};
@@ -501,6 +503,7 @@ enum mtmd_gen_process_type {
// for qwen3tts, this is code2wav
// for pocket-tts, this is mimi decoder
// for lfm3a, this is the codec decoder + ISTFT, n_codes == 0 flushes the tail
+ // for yue2, this is one VAE pass, the caller splits a long song into tiles
};
struct mtmd_gen_inp {
@@ -523,6 +526,15 @@ struct mtmd_gen_inp {
size_t n_codes;
const float * feats;
size_t n_feats;
+
+ // yue2 GEN_CODE: n_codes is the frame count, codes is not read (the backbone state holds them)
+ // backbone_state: llama_state_seq_get_data() of the backbone_n_past positions (prompt, codes, end token)
+ const uint8_t * backbone_state;
+ size_t backbone_state_size;
+ int32_t backbone_n_past;
+
+ // flow-matching GEN_CODE: steps of the ODE solver, <= 0 = model default (yue2: 32)
+ int32_t n_steps;
};
struct mtmd_gen_out {
@@ -539,7 +551,7 @@ struct mtmd_gen_out {
// for MTMD_GEN_PROCESS_TYPE_GEN_WAV
const float * audio;
- size_t n_samples;
+ size_t n_samples; // total over all channels
};
struct mtmd_gen_state_params {
diff --git a/tools/server/README.md b/tools/server/README.md
index 07b530df0..4dbb4426c 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -880,9 +880,13 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For the list
`loudness`: Target loudness in LUFS, `< 0` (default `0.0`, the model's own loudness). Only for models that support it, others reject a value other than `0.0` with a 400 error.
-`n_predict`: Max number of audio frames to generate. Defaults to `512`; generation normally stops earlier once the model emits an end-of-speech token.
+`cfg_scale`: Classifier-free guidance scale of the audio codes, `> 0`, `1.0` disables it (default `0.0`, the model's own). Only for models that support it (YuE2), others reject a value other than `0.0` with a 400 error.
-`response_format`: `wav` (default), `pcm` (raw `float32` little-endian mono samples, no header; the response `Content-Type` carries the sample rate, e.g. `audio/pcm;rate=24000;encoding=float;bits=32`) or `pcm16` (same, as signed 16-bit samples, e.g. `audio/pcm;rate=24000;encoding=signed-int;bits=16`).
+`flow_steps`: Steps of the flow-matching ODE solver, `1` to `1024` (default `0`, the model's own). Fewer steps are faster and the audio is less accurate. Only for models that support it (YuE2), others reject a value other than `0` with a 400 error.
+
+`n_predict`: Max number of audio frames to generate. Defaults to the limit of the model (`512` for most models); generation normally stops earlier once the model emits an end-of-speech token.
+
+`response_format`: `wav` (default), `pcm` (raw `float32` little-endian samples, no header, channels interleaved; the response `Content-Type` carries the sample rate and, for stereo models, the channel count, e.g. `audio/pcm;rate=24000;encoding=float;bits=32`) or `pcm16` (same, as signed 16-bit samples, e.g. `audio/pcm;rate=24000;encoding=signed-int;bits=16`).
`stream`: If `true`, the response is streamed as audio becomes available instead of waiting for the full generation to finish. WAV streaming writes an RFC-noncompliant header with an unknown (`0xFFFFFFFF`) size field, since the final length isn't known up front; most players and decoders (ffmpeg, VLC, ...) handle this by reading until EOF.
@@ -1674,7 +1678,7 @@ Generates speech from text, for models that can generate audio. The request is c
`stream_format`: `audio` streams the response as it becomes available, see `stream` in `/tts` for the WAV header it writes. `sse` is rejected. The response is not streamed when omitted.
-`speed`: Speaking speed, see `speed` in `/tts`. Only models that support it accept a value other than `1.0`. `loudness` of `/tts` can be used as well.
+`speed`: Speaking speed, see `speed` in `/tts`. Only models that support it accept a value other than `1.0`. `loudness`, `cfg_scale` and `flow_steps` of `/tts` can be used as well.
`instructions`: Ignored.
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index abaaf55b6..6a57aa8ca 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -1540,6 +1540,10 @@ private:
if (has_cap_tts()) {
slot.tts.ctx.init(ctx_tgt, mctx);
+ if (!slot.tts.is_supported()) {
+ SRV_ERR("%s", "the context cannot run the audio generation of this model\n");
+ return false;
+ }
}
SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx);
@@ -1953,7 +1957,7 @@ private:
task.tts_inp.data.seq_id = slot.id;
const int32_t ret = slot.tts.ctx.set_input(task.tts_inp.get());
if (ret == 2) {
- send_error(task, "invalid TTS input: the model may not support \"speed\" or \"loudness\", see the server log", ERROR_TYPE_INVALID_REQUEST);
+ send_error(task, "invalid TTS input: the model may not support \"speed\", \"loudness\", \"cfg_scale\" or \"flow_steps\", see the server log", ERROR_TYPE_INVALID_REQUEST);
return false;
}
if (ret != 0) {
@@ -3647,7 +3651,7 @@ private:
return;
}
- const int32_t n_predict = slot.n_predict_max > 0 ? slot.n_predict_max : 512;
+ const int32_t n_predict = slot.tts.ctx.max_frames(slot.n_predict_max);
if (slot.tts.stop || slot.tts.n_decoded >= n_predict) {
int32_t sample_rate = 0;
const char * data = nullptr;
@@ -5650,12 +5654,12 @@ json server_routes::get_model_info() const {
// body of /tts and /v1/audio/speech; multipart form fields arrive as strings, the numeric and boolean params get their types back
static json parse_tts_body(const server_http_req & req) {
json body = json::parse(req.body);
- for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) {
+ for (const char * k : { "top_k", "n_predict", "max_tokens", "seed", "flow_steps" }) {
if (body.contains(k) && body[k].is_string()) {
body[k] = std::stoll(body[k].get<std::string>());
}
}
- for (const char * k : { "top_p", "temperature", "repeat_penalty", "speed", "loudness" }) {
+ for (const char * k : { "top_p", "temperature", "repeat_penalty", "speed", "loudness", "cfg_scale" }) {
if (body.contains(k) && body[k].is_string()) {
body[k] = std::stof(body[k].get<std::string>());
}
@@ -7063,8 +7067,9 @@ std::unique_ptr<server_res_generator> server_routes::handle_tts_impl(const serve
task.id = rd.get_new_id();
rd.post_task(std::move(task));
- // raw LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586)
- const std::string pcm_type = "audio/pcm;rate=" + std::to_string(info.sample_rate);
+ // raw LE samples, channels interleaved; audio/L16 would imply 16-bit big-endian (RFC 2586)
+ const std::string pcm_type = "audio/pcm;rate=" + std::to_string(info.sample_rate) +
+ (info.n_channels > 1 ? ";channels=" + std::to_string(info.n_channels) : "");
const std::string content_type = pcm16 ? pcm_type + ";encoding=signed-int;bits=16"
: response_format == "pcm" ? pcm_type + ";encoding=float;bits=32"
: "audio/wav";
diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp
index e72617d6d..ccae2458b 100644
--- a/tools/server/server-schema.cpp
+++ b/tools/server/server-schema.cpp
@@ -561,6 +561,13 @@ std::vector<std::unique_ptr<field>> make_tts_schema(server_task & task) {
add((new field_num("loudness", task.tts_inp.data.loudness))
->set_desc("Target loudness in LUFS, for models that support it (0.0 = model default)"));
+ add((new field_num("cfg_scale", task.tts_inp.data.cfg_scale))
+ ->set_desc("Classifier-free guidance scale of the audio codes, for models that support it (0.0 = model default, 1.0 = no guidance)"));
+
+ add((new field_num("flow_steps", task.tts_inp.data.flow_steps))
+ ->set_hard_limits(0, 1024)
+ ->set_desc("Steps of the flow-matching ODE solver, for models that support it (0 = model default); fewer is faster, less accurate"));
+
add((new field_num("n_predict", task.params.n_predict))
->set_hard_limits(-1, INT32_MAX)
->add_alias("max_tokens")
diff --git a/tools/tts/README.md b/tools/tts/README.md
index 9df5078bb..ce4dfec18 100644
--- a/tools/tts/README.md
+++ b/tools/tts/README.md
@@ -16,7 +16,7 @@ Note: it's highly recommended to always provide a speaker reference voice (via `
Common params:
- Sampling params such as `--top-k`, `--top-p`, `--temp`, etc. `--temp` also sets the temperature of the audio frames for models that support it, without it they use the model default
-- `-n <number_of_frames>` limits the output length, e.g. `-n 500`. Note that how many milliseconds each frame represents varies by model
+- `-n <number_of_frames>` limits the output length, e.g. `-n 500` (default: the limit of the model, 512 frames for most models). Note that how many milliseconds each frame represents varies by model
- `--tts-speaker-text` (optional) gives the transcript of `--tts-speaker-file`, some models put it in the prompt (e.g. Fish Audio S2 Pro)
- Core inference params such as `-ngl`, `-b`, `-ub`, etc.
diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp
index bbc9ac794..47f17de89 100644
--- a/tools/tts/tts.cpp
+++ b/tools/tts/tts.cpp
@@ -142,6 +142,10 @@ int main(int argc, char ** argv) {
mtmd_helper::gen_audio gen;
gen.init(lctx, mctx.get());
+ if (!gen.valid()) {
+ LOG_ERR("the context cannot run the audio generation of this model\n");
+ return 1;
+ }
mtmd_helper_gen_audio_inp inp = mtmd_helper_gen_audio_inp_default();
inp.seq_id = 0;
inp.prompt = prompt.c_str();
@@ -186,7 +190,7 @@ int main(int argc, char ** argv) {
return t;
};
- const int max_new = params.n_predict > 0 ? params.n_predict : 512;
+ const int max_new = gen.max_frames(params.n_predict);
int n_frames = 0;
llama_token sampled = sample_semantic_code();
const float * h_state = llama_get_embeddings_ith(lctx, -1);