Commit 7ae3c6b23 for llama.cpp
commit 7ae3c6b231e38057e825ff257bb734e456633d9a
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Sun Nov 15 19:44:44 2026 +0100
mtmd: add zonos3-mini TTS model (#32020)
* zonos3 init
* add audio gen pipeline
* fix string_format
* add default voice as gguf metadata
* better ref
* support voice clone
* add delivery params
* optimize speed
* nits
* nits 2
* add text normalization
* nits
diff --git a/conversion/__init__.py b/conversion/__init__.py
index fb6b8464c..e915753a8 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -282,6 +282,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"Qwen3NextForCausalLM": "qwen",
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
"PocketTTSModel": "pockettts",
+ "Zonos3Model": "zonos3",
"Qwen3TTSForConditionalGeneration": "qwen3tts",
"Qwen3VLForConditionalGeneration": "qwen3vl",
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
@@ -356,6 +357,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"EmbeddingGemma2Model": "gemma",
"Exaone4_5_ForConditionalGeneration": "exaone",
"FishOmniModel": "drama3",
+ "Zonos3Model": "zonos3",
"Gemma3ForConditionalGeneration": "gemma",
"Gemma3nForConditionalGeneration": "gemma",
"Gemma4ForConditionalGeneration": "gemma",
diff --git a/conversion/zonos3.py b/conversion/zonos3.py
new file mode 100644
index 000000000..bf1c7a625
--- /dev/null
+++ b/conversion/zonos3.py
@@ -0,0 +1,575 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Callable, Iterable, TYPE_CHECKING
+
+import numpy as np
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import ModelBase, TextModel, MmprojModel, LazyTorchTensor, gguf, logger
+
+LM_DIR = Path("lm") / "zonos3_mini"
+
+# codec decoder: quantizers and semantic head from the generator, the rest from the head checkpoint
+CODEC_GENERATOR = Path("autoencoder") / "generator.pt"
+CODEC_VARIANT = "keep2_plain"
+CODEC_HEAD = Path("autoencoder") / "heads" / CODEC_VARIANT / "step_0050000.pt"
+CODEC_STAGES = [0, 2, 4, 6, 8] # transformer stages of the decoder
+
+DEFAULT_VOICE = "Savannah_en-American_F_1"
+
+# speaker encoder (ownmect_s2), MECT_B2 blocks: (cnn depth, out channels, (stride f, stride t), expansion)
+SPK_CKPT = Path("speaker_encoders") / "ownmect_s2" / "ckpt_0037.pth"
+SPK_STAGES = [
+ [(3, 32, (1, 1), 4)] * 2,
+ [(3, 64, (2, 1), 2)] + [(3, 64, (1, 2), 2)] * 3,
+ [(3, 128, (2, 1), 2)] + [(3, 128, (1, 2), 2)] * 4,
+ [(3, 256, (2, 1), 1)] + [(2, 256, (1, 2), 1)] * 2,
+]
+VOICE_MAX_FRAMES = 900 # reference_max_frames of the reference
+
+# GDN geometry, hard-coded in the reference (zonos3.reference.lm)
+GDN_HEADS, GDN_HEAD_DIM, GDN_KERNEL = 4, 128, 4
+
+# the text table keeps the tokenizer ids shifted by this offset, ids 2 and 3 are BOS and EOS
+TEXT_OFFSET = 4
+TEXT_BOS, TEXT_EOS = 2, 3
+
+
+def _round_up(value: float, multiple: int) -> int:
+ return multiple * ((int(value) + multiple - 1) // multiple)
+
+
+def _load_params(dir_model: Path) -> dict[str, Any] | None:
+ path = dir_model / LM_DIR / "params.json"
+ if not path.is_file():
+ return None
+ with open(path, encoding="utf-8") as f:
+ params = json.load(f).get("model", {})
+ return params if params.get("model_type") == "zonos2" else None
+
+
+def _is_zonos3_checkpoint(dir_model: Path) -> bool:
+ return _load_params(dir_model) is not None
+
+
+@ModelBase.register_hparams_loader(_is_zonos3_checkpoint)
+def _load_zonos3_hparams(dir_model: Path) -> dict[str, Any]:
+ logger.info("gguf: detected ZONOS3 checkpoint")
+ params = _load_params(dir_model)
+ assert params is not None
+ for key, want in (("attention_pattern", "gdn3_global1"), ("residual_type", "full_attnres"),
+ ("moe_router_type", "linear"), ("moe_router_use_eda", False), ("use_xsa", False),
+ ("moe_end_from_layer", 0), ("speaker_enabled", True)):
+ if params.get(key) != want:
+ raise NotImplementedError(f"ZONOS3: unsupported {key} = {params.get(key)!r}, expected {want!r}")
+ text_config = dict(params, hidden_size=params["dim"])
+ return {"architectures": ["Zonos3Model"], "text_config": text_config, "audio_config": {"num_hidden_layers": 0}}
+
+
+def _index_lm_tensors(model: ModelBase) -> dict[str, Callable[[], Tensor]]:
+ path = model.dir_model / LM_DIR / "model.pth"
+ logger.info(f"gguf: indexing model part '{LM_DIR / 'model.pth'}'")
+ state = torch.load(str(path), map_location="cpu", mmap=True, weights_only=True)
+ tensors: dict[str, Callable[[], Tensor]] = {}
+ for name, data in state.items():
+ gen = (lambda data=data: LazyTorchTensor.from_eager(data)) if model.lazy else (lambda data=data: data)
+ titem = model.filter_tensors((name, gen))
+ model._source_tensor_names[name] = titem[0] if titem else None
+ if titem:
+ tensors[titem[0]] = titem[1]
+ return tensors
+
+
+@ModelBase.register("Zonos3Model")
+class Zonos3TextModel(TextModel):
+ model_arch = gguf.MODEL_ARCH.ZONOS3
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ hp = self.hparams
+ self.n_embd = hp["dim"]
+ self.n_codebooks = hp["n_codebooks"]
+ self.audio_vocab = hp["codebook_size"] + 2 # codes, EOA, pad
+ self.audio_pad_id = hp["audio_pad_id"]
+ self.text_rows = hp["text_vocab"] + 1 # + the padding row, used by the generated rows
+ self.cond_counts = [n + 1 for n in hp["condition_bucket_counts"]] # + the unknown bucket
+ self.gdn_period = int(hp["attention_pattern"][len("gdn"):].split("_")[0]) + 1
+ self.n_ff = _round_up(hp["ffn_dim_multiplier"] * hp["dim"], hp["multiple_of"])
+ self.n_ff_exp = _round_up(hp["moe_ffn_dim_multiplier"] * hp["dim"], hp["moe_multiple_of"])
+ self.n_latent = int(hp["dim"] * hp["moe_alpha"])
+ self._embd: dict[str, Tensor] = {}
+ self._router: dict[int, dict[str, Tensor]] = {}
+
+ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
+ return _index_lm_tensors(self)
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, _ = item
+ # audio heads and speaker projection go to the mmproj
+ if name.startswith(("multi_output.", "speaker_projection.")):
+ return None
+ return item
+
+ def set_vocab(self):
+ with open(self.dir_model / LM_DIR / "tokenizer" / "superbpe8192.json", encoding="utf-8") as f:
+ tok = json.load(f)
+ model = tok["model"]
+ if model["type"] != "BPE" or model.get("byte_fallback") or tok.get("normalizer") is not None:
+ raise NotImplementedError("ZONOS3: unexpected tokenizer type")
+
+ vocab: dict[str, int] = model["vocab"]
+ n_bpe = len(vocab)
+ if n_bpe + TEXT_OFFSET != self.hparams["text_vocab"]:
+ raise ValueError(f"ZONOS3: tokenizer has {n_bpe} tokens, text_vocab is {self.hparams['text_vocab']}")
+ added = {t["id"] for t in tok.get("added_tokens", [])}
+
+ tokens: list[str] = ["<|unused_0|>", "<|unused_1|>", "<|bos|>", "<|eos|>"]
+ toktypes: list[int] = [gguf.TokenType.CONTROL] * TEXT_OFFSET
+ inv = {i: t for t, i in vocab.items()}
+ for i in range(n_bpe):
+ tokens.append(inv[i])
+ toktypes.append(gguf.TokenType.CONTROL if i in added else gguf.TokenType.NORMAL)
+
+ # padding row of the text table (the text column of every generated row)
+ tokens.append("<|text_pad|>")
+ toktypes.append(gguf.TokenType.CONTROL)
+
+ # condition rows, the last bucket of each table is "unknown"
+ for cond, n in zip(self.hparams["condition_names"], self.cond_counts):
+ tokens += [f"<|{cond}_{i}|>" for i in range(n - 1)] + [f"<|{cond}_unk|>"]
+ toktypes += [gguf.TokenType.CONTROL] * n
+
+ merges = [" ".join(m) if isinstance(m, list) else m for m in model["merges"]]
+
+ self.gguf_writer.add_tokenizer_model("gpt2")
+ self.gguf_writer.add_tokenizer_pre("zonos3")
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_types(toktypes)
+ self.gguf_writer.add_token_merges(merges)
+ self.gguf_writer.add_bos_token_id(TEXT_BOS)
+ self.gguf_writer.add_eos_token_id(TEXT_EOS)
+ self.gguf_writer.add_pad_token_id(TEXT_OFFSET + 1) # <|pad|> of the tokenizer
+ # the reference wraps the text as [BOS] + ids + [EOS]
+ self.gguf_writer.add_add_bos_token(True)
+ self.gguf_writer.add_add_eos_token(True)
+ # the reference normalizes the text with NeMo, the template covers the English numbers
+ template_path = Path(__file__).parent.parent / "models" / "templates" / "Zyphra-ZONOS3-Mini-tts.jinja"
+ with open(template_path, "r", encoding="utf-8") as f:
+ self.gguf_writer.add_chat_template([{"name": "tts", "template": f.read()}])
+
+ def set_gguf_parameters(self):
+ hp = self.hparams
+ n_layer = self.block_count
+ self.gguf_writer.add_file_type(self.ftype)
+ self.gguf_writer.add_block_count(n_layer)
+ self.gguf_writer.add_context_length(hp["max_seqlen"])
+ self.gguf_writer.add_embedding_length(self.n_embd)
+ self.gguf_writer.add_feed_forward_length(self.n_ff)
+ self.gguf_writer.add_head_count(hp["n_heads"])
+ self.gguf_writer.add_head_count_kv(hp["n_kv_heads"])
+ self.gguf_writer.add_key_length(hp["head_dim"])
+ self.gguf_writer.add_value_length(hp["head_dim"])
+ self.gguf_writer.add_rope_dimension_count(hp["head_dim"])
+ self.gguf_writer.add_rope_freq_base(hp["rope_theta"])
+ self.gguf_writer.add_layer_norm_rms_eps(hp["norm_eps"])
+ self.gguf_writer.add_full_attention_interval(self.gdn_period)
+
+ # gated delta net
+ self.gguf_writer.add_ssm_conv_kernel(GDN_KERNEL)
+ self.gguf_writer.add_ssm_inner_size(GDN_HEADS * GDN_HEAD_DIM)
+ self.gguf_writer.add_ssm_state_size(GDN_HEAD_DIM)
+ self.gguf_writer.add_ssm_time_step_rank(GDN_HEADS)
+ self.gguf_writer.add_ssm_group_count(GDN_HEADS)
+
+ # latent MoE, the gating (softmax over the selected logits) is hard-coded in libllama
+ self.gguf_writer.add_leading_dense_block_count(hp["moe_start_from_layer"])
+ self.gguf_writer.add_expert_count(hp["moe_n_experts"])
+ self.gguf_writer.add_expert_used_count(hp["moe_router_topk"])
+ self.gguf_writer.add_expert_feed_forward_length(self.n_ff_exp)
+ self.gguf_writer.add_moe_latent_size(self.n_latent)
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ # router logits pick the experts, keep them exact
+ if new_name.endswith(("ffn_gate_inp.weight", "exp_probs_b.bias")):
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def _token_embd(self) -> Tensor:
+ n_cb = self.n_codebooks
+ pad = sum(self._embd[f"multi_embedder.embedders.{cb}.weight"][self.audio_pad_id].float() for cb in range(n_cb))
+ text = self._embd[f"multi_embedder.embedders.{n_cb}.weight"].float()
+ if text.shape[0] != self.text_rows:
+ raise ValueError(f"ZONOS3: text table has {text.shape[0]} rows, expected {self.text_rows}")
+ conds = [self._embd[f"condition_embeddings.{i}.weight"].float() for i in range(len(self.cond_counts))]
+ for t, n in zip(conds, self.cond_counts):
+ assert t.shape[0] == n
+ # a text row is [pad x n_codebooks, text_id], condition rows replace the summed row
+ return torch.cat([text + pad] + conds, dim=0)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ T = gguf.MODEL_TENSOR
+ hp = self.hparams
+ n_layer = self.block_count
+
+ if name.startswith(("multi_embedder.", "condition_embeddings.")):
+ self._embd[name] = data_torch
+ if len(self._embd) == self.n_codebooks + 1 + len(self.cond_counts):
+ yield (self.format_tensor_name(T.TOKEN_EMBD), self._token_embd())
+ self._embd.clear()
+ return
+
+ if name == "block_attnres_queries":
+ # one query per routing point: 2*il before the mixer, 2*il+1 before the FFN, the last one before out_norm
+ assert data_torch.shape[0] == 2 * n_layer + 1
+ for il in range(n_layer):
+ yield (self.format_tensor_name(T.ATTN_RES_SCORE, il), data_torch[2 * il])
+ yield (self.format_tensor_name(T.FFN_RES_SCORE, il), data_torch[2 * il + 1])
+ yield (self.format_tensor_name(T.OUTPUT_RES_SCORE), data_torch[2 * n_layer])
+ return
+
+ if name == "out_norm.weight":
+ yield (self.format_tensor_name(T.OUTPUT_NORM), data_torch)
+ return
+
+ assert bid is not None, name
+ key = name.split(".", 2)[2] # strip "layers.{bid}."
+
+ if key == "attention_norm.weight":
+ yield (self.format_tensor_name(T.ATTN_NORM, bid), data_torch)
+ return
+ if key == "ffn_norm.weight":
+ yield (self.format_tensor_name(T.FFN_NORM, bid), data_torch)
+ return
+
+ # gated delta net
+ if key == "attention.in_proj.weight":
+ # split the fused [qkv | z | a | b] weight into the qwen35 GDN tensors
+ # attn_gate holds z, per-head output gate in the attention layers
+ inner = GDN_HEADS * GDN_HEAD_DIM
+ qkv, z, a, b = data_torch.split([3 * inner, inner, GDN_HEADS, GDN_HEADS], dim=0)
+ yield (self.format_tensor_name(T.ATTN_QKV, bid), qkv)
+ yield (self.format_tensor_name(T.ATTN_GATE, bid), z)
+ yield (self.format_tensor_name(T.SSM_ALPHA, bid), a)
+ yield (self.format_tensor_name(T.SSM_BETA, bid), b)
+ return
+ if key == "attention.conv.weight":
+ yield (self.format_tensor_name(T.SSM_CONV1D, bid), data_torch.squeeze(1))
+ return
+ if key == "attention.A_log":
+ yield (self.format_tensor_name(T.SSM_A, bid, suffix=""), -torch.exp(data_torch.float()))
+ return
+ if key == "attention.dt_bias":
+ yield (self.format_tensor_name(T.SSM_DT, bid, suffix=".bias"), data_torch)
+ return
+ if key == "attention.out_weight":
+ yield (self.format_tensor_name(T.SSM_NORM, bid), data_torch)
+ return
+ if key == "attention.wo.weight" and (bid + 1) % self.gdn_period != 0:
+ yield (self.format_tensor_name(T.SSM_OUT, bid), data_torch)
+ return
+
+ # global attention
+ if key == "attention.wq.weight":
+ yield (self.format_tensor_name(T.ATTN_Q, bid), data_torch)
+ return
+ if key == "attention.wkv.weight":
+ yield (self.format_tensor_name(T.ATTN_K, bid), data_torch[0])
+ yield (self.format_tensor_name(T.ATTN_V, bid), data_torch[1])
+ return
+ if key == "attention.wo.weight":
+ yield (self.format_tensor_name(T.ATTN_OUT, bid), data_torch)
+ return
+ if key == "attention.gater.weight":
+ # one sigmoid gate per head, same tensor name as the GDN z gate
+ yield (self.format_tensor_name(T.ATTN_GATE, bid), data_torch)
+ return
+ if key == "attention.temp":
+ # Q = rms_norm(q) * |temp|, the per-head scale is stored as the Q norm weight
+ temp = data_torch.float().abs().reshape(hp["n_heads"], 1).expand(hp["n_heads"], hp["head_dim"])
+ yield (self.format_tensor_name(T.ATTN_Q_NORM, bid), temp.contiguous())
+ return
+
+ # dense FFN (layer 0): out = w_out(w_in[0](x) * silu(w_in[1](x)))
+ if key == "feed_forward.w_in.weight":
+ yield (self.format_tensor_name(T.FFN_UP, bid), data_torch[0])
+ yield (self.format_tensor_name(T.FFN_GATE, bid), data_torch[1])
+ return
+ if key == "feed_forward.w_out.weight":
+ yield (self.format_tensor_name(T.FFN_DOWN, bid), data_torch)
+ return
+
+ # latent MoE
+ if key in ("feed_forward.router.router_linear.weight", "feed_forward.router.rmsnorm_eda.weight"):
+ # logits = router_linear(rms_norm(x) * w), the norm weight is folded into the router columns
+ r = self._router.setdefault(bid, {})
+ r[key] = data_torch
+ if len(r) == 2:
+ w = r["feed_forward.router.router_linear.weight"].float() * r["feed_forward.router.rmsnorm_eda.weight"].float()[None, :]
+ yield (self.format_tensor_name(T.FFN_GATE_INP, bid), w)
+ del self._router[bid]
+ return
+ if key == "feed_forward.router.balancing_biases":
+ # the reference selects on logits - bias
+ yield (self.format_tensor_name(T.FFN_EXP_PROBS_B, bid, suffix=".bias"), -data_torch.float())
+ return
+ if key == "feed_forward.latent_down.weight":
+ yield (self.format_tensor_name(T.MOE_LATENT_DOWN, bid), data_torch)
+ return
+ if key == "feed_forward.latent_up.weight":
+ yield (self.format_tensor_name(T.MOE_LATENT_UP, bid), data_torch)
+ return
+ if key == "feed_forward.experts.gate_up_proj":
+ gate, up = data_torch.chunk(2, dim=1)
+ yield (self.format_tensor_name(T.FFN_GATE_EXP, bid), gate)
+ yield (self.format_tensor_name(T.FFN_UP_EXP, bid), up)
+ return
+ if key == "feed_forward.experts.down_proj":
+ yield (self.format_tensor_name(T.FFN_DOWN_EXP, bid), data_torch)
+ return
+
+ raise ValueError(f"ZONOS3: unexpected tensor {name}")
+
+
+@ModelBase.register("Zonos3Model")
+class Zonos3MmprojModel(MmprojModel):
+ has_vision_encoder = False
+ has_audio_encoder = True
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ hp = self.global_config["text_config"]
+ self.n_codebooks = hp["n_codebooks"]
+ self.text_vocab = hp["text_vocab"]
+ self._embd: dict[str, Tensor] = {}
+
+ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
+ return _index_lm_tensors(self)
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, _ = item
+ if name.startswith(("multi_output.", "multi_embedder.", "speaker_projection.")):
+ return item
+ return None
+
+ def set_gguf_parameters(self):
+ self.gguf_writer.add_file_type(self.ftype)
+
+ # speaker encoder for voice cloning, the front-end hparams are hard-coded in tools/mtmd
+ self.gguf_writer.add_clip_has_audio_encoder(True)
+ self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.ZONOS3_SPKENC)
+ self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
+ self.gguf_writer.add_audio_num_mel_bins(80)
+ self.gguf_writer.add_audio_block_count(0)
+ self.gguf_writer.add_audio_embedding_length(192)
+ self.gguf_writer.add_audio_head_count(1)
+ self.gguf_writer.add_audio_feed_forward_length(192)
+ self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
+
+ self.gguf_writer.add_clip_has_gen_audio_encoder(True)
+ self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.ZONOS3_GEN)
+ self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
+ self.gguf_writer.add_gen_audio_embedding_length(self.n_embd_text)
+ self.gguf_writer.add_gen_audio_feed_forward_length(0)
+ self.gguf_writer.add_gen_audio_block_count(0)
+ self.gguf_writer.add_gen_audio_head_count(0)
+ self.gguf_writer.add_gen_audio_attention_layernorm_eps(self.global_config["text_config"]["norm_eps"])
+ self.gguf_writer.add_gen_audio_model_variant(CODEC_VARIANT)
+ if any(self.dir_model.glob(f"gguf/voices/*/{DEFAULT_VOICE}.codes.npy")):
+ self.gguf_writer.add_gen_audio_default_voice(DEFAULT_VOICE)
+ # softcap and the codec hparams are hard-coded in tools/mtmd
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ # read on the host, or used by get_rows / ggml_ssm_conv / the conv ops of the speaker encoder
+ if new_name.startswith(("voice.", "a.spk.")) or new_name.endswith((".table", ".dw.weight", ".mixer.weight", ".resample.weight")):
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ @staticmethod
+ def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
+ # torch weight norm, over every dim but 0
+ g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
+ return (g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))).squeeze(-1)
+
+ def _speaker_encoder(self) -> Iterable[tuple[str, Tensor]]:
+ # same layout as the reference ggml runtime (tools/convert_gguf.py speaker-encoder)
+ sd = torch.load(self.dir_model / SPK_CKPT, map_location="cpu", weights_only=True)["modules"]["spk_model"]
+ sd = {k[len("model."):]: v.double() for k, v in sd.items() if not k.endswith("num_batches_tracked")}
+ p = "a.spk"
+
+ def bn(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
+ scale = sd[src + ".weight"] / torch.sqrt(sd[src + ".running_var"] + 1e-5)
+ yield (f"{p}.{dst}.scale", scale)
+ yield (f"{p}.{dst}.shift", sd[src + ".bias"] - sd[src + ".running_mean"] * scale)
+
+ def put(dst: str, t: Tensor) -> tuple[str, Tensor]:
+ return (f"{p}.{dst}", t)
+
+ fe = "feature_extractor."
+ yield put("fe.head.conv.weight", sd[fe + "head.0.weight"])
+ yield put("fe.head.conv.bias", sd[fe + "head.0.bias"])
+ yield from bn("fe.head.bn", fe + "head.1")
+ c_in, f_in = 32, 80
+ for si, stage in enumerate(SPK_STAGES):
+ for k, (depth, out_ch, (s_f, s_t), expansion) in enumerate(stage):
+ src, dst = f"{fe}mect_block{si + 1}.{k}.", f"fe.b{si + 1}.{k}."
+ for r in range(depth):
+ cs, cd = f"{src}cnn_block.{r}.", f"{dst}cnn.{r}."
+ # grouped 3x3 convs, one depthwise kernel per input slot of a group
+ for conv in ("conv1", "conv2"):
+ w = sd[cs + conv + ".weight"]
+ for j in range(w.shape[1]):
+ yield put(f"{cd}{conv}.w{j}", w[:, j:j + 1])
+ for i in (1, 2):
+ yield put(f"{cd}pw{i}.weight", sd[f"{cs}conv_pw{i}.weight"])
+ yield put(f"{cd}pw{i}.bias", sd[f"{cs}conv_pw{i}.bias"])
+ yield from bn(f"{cd}bn{i}", f"{cs}bn{i}")
+ if cs + "shortcut.0.weight" in sd:
+ yield put(cd + "sc.weight", sd[cs + "shortcut.0.weight"])
+ yield from bn(cd + "sc.bn", cs + "shortcut.1")
+ if expansion > 1:
+ yield put(dst + "reduce.weight", sd[src + "reduce_conv.0.weight"])
+ yield put(dst + "reduce.bias", sd[src + "reduce_conv.0.bias"])
+ yield from bn(dst + "reduce.bn", src + "reduce_conv.1")
+ yield put(dst + "rproj.weight", sd[src + "reduce_project.0.weight"].squeeze(-1))
+ yield put(dst + "rproj.bias", sd[src + "reduce_project.0.bias"])
+ yield put(dst + "rproj.ln.weight", sd[src + "reduce_project.1.weight"])
+ yield put(dst + "rproj.ln.bias", sd[src + "reduce_project.1.bias"])
+ yield put(dst + "pos.dw.weight", sd[src + "pos_conv.pos_conv.0.weight"].squeeze(1))
+ yield put(dst + "pos.dw.bias", sd[src + "pos_conv.pos_conv.0.bias"])
+ yield from bn(dst + "pos.bn", src + "pos_conv.pos_conv.1")
+ yield put(dst + "pos.pw.weight", sd[src + "pos_conv.pos_conv.3.weight"].squeeze(-1))
+ yield put(dst + "pos.pw.bias", sd[src + "pos_conv.pos_conv.3.bias"])
+ tr = src + "transformer."
+ for a, b in (("q", "q_proj"), ("k", "k_proj"), ("v", "v_proj"), ("o", "out_proj")):
+ yield put(f"{dst}attn.{a}.weight", sd[f"{tr}attention.{b}.weight"])
+ yield put(f"{dst}attn.{a}.bias", sd[f"{tr}attention.{b}.bias"])
+ yield put(dst + "ln1.weight", sd[tr + "layer_norm.weight"])
+ yield put(dst + "ln1.bias", sd[tr + "layer_norm.bias"])
+ moe = tr + "feed_forward.intermediate_dense.token_moe."
+ yield put(dst + "moe.gate.weight", sd[moe + "gate.weight"])
+ yield put(dst + "moe.experts.weight", sd[moe + "experts.weight"])
+ yield put(dst + "ffn_out.weight", sd[tr + "feed_forward.output_dense.weight"])
+ yield put(dst + "ffn_out.bias", sd[tr + "feed_forward.output_dense.bias"])
+ yield put(dst + "ln2.weight", sd[tr + "final_layer_norm.weight"])
+ yield put(dst + "ln2.bias", sd[tr + "final_layer_norm.bias"])
+ yield put(dst + "eproj.weight", sd[src + "expansion_project.0.weight"].squeeze(-1))
+ yield put(dst + "eproj.bias", sd[src + "expansion_project.0.bias"])
+ c_in, f_in = c_in * s_f, f_in // s_f
+ # the stage outputs are mixed by softmax weights
+ yield put("fe.dense_w", torch.softmax(sd[fe + "dense3.inputs_weights"], dim=1)[0, :, :, 0])
+ yield put("pool.l1.weight", sd["encoding_layers.linear1.weight"].squeeze(-1))
+ yield put("pool.l1.bias", sd["encoding_layers.linear1.bias"])
+ yield put("pool.l2.weight", sd["encoding_layers.linear2.weight"].squeeze(-1))
+ yield put("pool.l2.bias", sd["encoding_layers.linear2.bias"])
+ yield from bn("bn", "bn")
+ yield put("linear.weight", sd["linear.weight"])
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ yield from self._speaker_encoder()
+ logger.info(f"gguf: loading codec decoder from '{CODEC_GENERATOR}' and '{CODEC_HEAD}'")
+ gen = torch.load(self.dir_model / CODEC_GENERATOR, map_location="cpu", mmap=True, weights_only=True)["generator"]
+ head = torch.load(self.dir_model / CODEC_HEAD, map_location="cpu", mmap=True, weights_only=True)["trainable"]
+ p = "a.gen.wav"
+
+ def convnext(dst: str, src: dict[str, Tensor], prefix: str) -> Iterable[tuple[str, Tensor]]:
+ yield (f"{dst}.dw.weight", src[f"{prefix}.dwconv.weight"].squeeze(1))
+ yield (f"{dst}.dw.bias", src[f"{prefix}.dwconv.bias"])
+ yield (f"{dst}.norm.weight", src[f"{prefix}.norm.weight"])
+ yield (f"{dst}.norm.bias", src[f"{prefix}.norm.bias"])
+ yield (f"{dst}.pw1.weight", src[f"{prefix}.pwconv1.weight"])
+ yield (f"{dst}.pw1.bias", src[f"{prefix}.pwconv1.bias"])
+ yield (f"{dst}.pw2.weight", src[f"{prefix}.pwconv2.weight"])
+ yield (f"{dst}.pw2.bias", src[f"{prefix}.pwconv2.bias"])
+
+ # RVQ codebooks folded through their out_proj, the biases are added once
+ def rvq(prefix: str) -> tuple[Tensor, Tensor]:
+ return gen[prefix + ".codebook.weight"].float() @ self._wn(gen, prefix + ".out_proj").T, gen[prefix + ".out_proj.bias"].float()
+
+ t, b = rvq("quantizer.semantic_quantizer.quantizers.0")
+ yield (f"{p}.sem.table", t)
+ yield (f"{p}.sem.bias", b)
+ res = [rvq(f"quantizer.quantizer.quantizers.{i}") for i in range(self.n_codebooks - 1)]
+ yield (f"{p}.res.table", torch.cat([t for t, _ in res], dim=0))
+ # one bias per residual codebook, a frame decoded before all its codes are known leaves the missing ones out
+ yield (f"{p}.res.bias", torch.stack([b for _, b in res]))
+
+ n_sem = sum(1 for k in gen if k.startswith("semantic_head.") and k.endswith(".dwconv.weight"))
+ for i in range(n_sem):
+ yield from convnext(f"{p}.sem.{i}", gen, f"semantic_head.{i}")
+ yield (f"{p}.sem.out.weight", self._wn(gen, f"semantic_head.{n_sem}"))
+ yield (f"{p}.sem.out.bias", gen[f"semantic_head.{n_sem}.bias"])
+
+ names = {"norm1.weight": "norm1.weight", "norm1.bias": "norm1.bias", "qkv.weight": "qkv.weight",
+ "attention_out.weight": "wo.weight", "attention_scale": "attn_scale", "norm2.weight": "norm2.weight",
+ "norm2.bias": "norm2.bias", "ffn.0.weight": "ffn1.weight", "ffn.3.weight": "ffn2.weight", "ffn_scale": "ffn_scale"}
+ for si, s in enumerate(CODEC_STAGES):
+ yield (f"{p}.stage.{si}.in.weight", head[f"stages.{s}.input_proj.weight"])
+ if f"stages.{s}.output_proj.weight" in head:
+ yield (f"{p}.stage.{si}.out.weight", head[f"stages.{s}.output_proj.weight"])
+ n_layer = sum(1 for k in head if k.startswith(f"stages.{s}.layers.") and k.endswith(".qkv.weight"))
+ for il in range(n_layer):
+ for src, dst in names.items():
+ yield (f"{p}.stage.{si}.blk.{il}.{dst}", head[f"stages.{s}.layers.{il}.{src}"])
+
+ n_head = sum(1 for k in head if k.startswith("head.blocks.") and k.endswith(".dwconv.weight"))
+ for i in range(n_head):
+ yield from convnext(f"{p}.head.{i}", head, f"head.blocks.{i}")
+ yield (f"{p}.head.norm.weight", head["head.norm.weight"])
+ yield (f"{p}.head.norm.bias", head["head.norm.bias"])
+ # both iSTFT branches in one projection
+ yield (f"{p}.head.out.weight", torch.cat([head["head.outs.0.weight"], head["head.outs.1.weight"]], dim=0))
+ yield (f"{p}.head.out.bias", torch.cat([head["head.outs.0.bias"], head["head.outs.1.bias"]], dim=0))
+ # read on the host
+ yield (f"{p}.mixer.weight", head["head.mixer.weight"].squeeze(0))
+ yield (f"{p}.resample.weight", gen["to_external.kernel"].squeeze(1))
+
+ # "reference" speaker mode, the default of the reference: the voice is a prefix of delayed code rows
+ def _voices(self, tables: list[Tensor]) -> Iterable[tuple[str, Tensor]]:
+ n_cb, pad = self.n_codebooks, self.global_config["text_config"]["audio_pad_id"]
+ for path in sorted((self.dir_model / "gguf" / "voices").glob("*/*.codes.npy")):
+ codes = torch.from_numpy(np.load(path, allow_pickle=False).astype(np.int64))
+ if codes.ndim != 2 or codes.shape[1] != n_cb or codes.min() < 0 or codes.max() >= pad - 1:
+ logger.warning(f"voice {path.name}: skipped, expected [frames, {n_cb}] codes")
+ continue
+ codes = codes[:VOICE_MAX_FRAMES]
+ n_frames = codes.shape[0]
+ # codebook cb of frame t is in row t + cb, the other cells are pad
+ rows = torch.full((n_frames + n_cb - 1, n_cb), pad, dtype=torch.int64)
+ for cb in range(n_cb):
+ rows[cb:cb + n_frames, cb] = codes[:, cb]
+ embd = sum(tables[cb][rows[:, cb]] for cb in range(n_cb))
+ yield (f"voice.{path.name[:-len('.codes.npy')]}", embd)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ T = gguf.MODEL_TENSOR
+
+ if name.startswith(("a.gen.wav.", "a.spk.")):
+ yield (name, data_torch)
+ return
+
+ # the speaker encoder ends with the speaker projection of the backbone
+ if name.startswith("speaker_projection."):
+ yield ("a.spk.proj." + name.split(".")[-1], data_torch)
+ return
+
+ if name == "multi_output.weight":
+ yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), data_torch)
+ return
+
+ assert name.startswith("multi_embedder.")
+ self._embd[name] = data_torch
+ if len(self._embd) < self.n_codebooks + 1:
+ return
+ # every row built by the mmproj has text id = text_vocab, so that row is folded into codebook 0
+ tables = [self._embd[f"multi_embedder.embedders.{cb}.weight"].float() for cb in range(self.n_codebooks)]
+ tables[0] = tables[0] + self._embd[f"multi_embedder.embedders.{self.n_codebooks}.weight"][self.text_vocab].float()
+ yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), torch.cat(tables, dim=0))
+ yield from self._voices([LazyTorchTensor.to_eager(t) for t in tables])
+ self._embd.clear()
diff --git a/docs/multimodal/tts.md b/docs/multimodal/tts.md
index 3ffc6d45e..1dca6c461 100644
--- a/docs/multimodal/tts.md
+++ b/docs/multimodal/tts.md
@@ -11,6 +11,7 @@ Supported models:
- [LFM3-Audio](#lfm3-audio)
- [Chatterbox](#chatterbox)
- [Drama 3](#drama-3)
+- [ZONOS3](#zonos3)
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
@@ -180,3 +181,37 @@ The routed experts of the checkpoint are FP8, they are dequantized at conversion
python convert_hf_to_gguf.py path/to/drama-3 --outtype q8_0 --outfile drama3.gguf
python convert_hf_to_gguf.py path/to/drama-3 --mmproj --outtype bf16 --outfile mmproj-drama3.gguf
```
+
+## ZONOS3
+
+[Zyphra ZONOS3 Mini](https://huggingface.co/Zyphra/ZONOS3-Mini): a hybrid Gated DeltaNet / attention MoE backbone (800M, 100M active) predicts the 10 codebooks of each frame with an inter-codebook delay, and the ZAC codec decodes them to 44.1 kHz audio.
+
+Available params:
+- `--tts-voice` picks one of the predefined voices of the release (`Savannah_en-American_F_1` by default); `llama-server` lists them in `GET /props`. A voice is the prompt prefix of its reference codes, the default speaker mode of the reference
+- The language is the one of the text; `--tts-lang` (`lang` in `llama-server`) only selects the text normalization
+- Numbers are spelled out by the `tts` chat template of the GGUF: cardinals, decimals, money, percents, clock times, common ordinal forms (`1st`, `1er`, `1º`), English and German years. A 4-digit number is read as a year only after a word like `in`, `since`, a month name, or `im Jahr`, `seit` in German (`in 1990` -> "in nineteen ninety", `1234 people` -> "one thousand two hundred thirty-four people"). Languages: `en`, `fr`, `de`, `es`, `it`, `pt`, `ja` and `zh` (or `cmn`), the region is ignored (`fr_fr`, `pt_br`); English when the language is empty, other languages are left as written. Set the language to the one of the text: the separators differ, e.g. `1.234` is a thousand in German
+- Note: the reference uses NeMo text normalization, which also covers units, dates, abbreviations and Korean; supply normalized text for them
+- `--tts-speaker-file` clones the voice of a clip (the first 60 s are used) with the speaker encoder of the release, the "embedding" speaker mode of the reference
+- `llama-server` also takes `speed` (relative to 15 UTF-8 bytes per second) and `loudness` (LUFS), see the `/tts` endpoint; they pick the speaking-rate and loudness condition buckets of the checkpoint, the defaults leave them unknown
+- Note: `loudness` has a strong effect with a cloned voice; a predefined voice mostly keeps the loudness of its reference clip
+- Note: `--top-k` and `--top-p` are not used, frames are sampled with the settings of the reference: temperature 0.75 (`--temp` changes it), top-k 80, top-p 0.69, repetition penalty 2.4 over the last 96 frames of codebooks 0 to 5
+- Each frame is 2940 samples (about 67 ms), generation stops at the end-of-audio code
+
+Example usage:
+
+```sh
+llama-tts -m zonos3-mini.gguf \
+ -mm mmproj-zonos3-mini.gguf \
+ -p "The old lighthouse keeper climbed the spiral stairs every evening." \
+ --tts-voice Imogen_en-British_F_1 \
+ --output out.wav
+```
+
+**Note for GGUF conversion:**
+
+Convert the root of the release repository, not the `lm/` subdirectory. The mmproj holds the codebook heads, the predefined voices, the speaker encoder and the `keep2_plain` codec decoder, it is best kept in F32:
+
+```sh
+python convert_hf_to_gguf.py path/to/ZONOS3-Mini --outtype q8_0 --outfile zonos3-mini.gguf
+python convert_hf_to_gguf.py path/to/ZONOS3-Mini --mmproj --outtype f32 --outfile mmproj-zonos3-mini.gguf
+```
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index ed3ade939..c9b27a11f 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -545,6 +545,7 @@ class Keys:
PROJECTOR_TYPE = "clip.gen.audio.projector_type" # for mixed modality models
# name of the weight variant, for settings that are not in the checkpoint
MODEL_VARIANT = "clip.gen.audio.model_variant"
+ DEFAULT_VOICE = "clip.gen.audio.default_voice"
EMBEDDING_LENGTH = "clip.gen.audio.embedding_length"
FEED_FORWARD_LENGTH = "clip.gen.audio.feed_forward_length"
BLOCK_COUNT = "clip.gen.audio.block_count"
@@ -765,6 +766,7 @@ class MODEL_ARCH(IntEnum):
NANBEIGE = auto()
QWEN3TTS = auto()
POCKETTTS = auto()
+ ZONOS3 = auto()
K2HORIZON = auto()
KOLIBRI1 = auto()
LIMITE = auto()
@@ -1703,6 +1705,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.NANBEIGE: "nanbeige",
MODEL_ARCH.QWEN3TTS: "qwen3tts",
MODEL_ARCH.POCKETTTS: "pockettts",
+ MODEL_ARCH.ZONOS3: "zonos3",
MODEL_ARCH.K2HORIZON: "k2-horizon",
MODEL_ARCH.KOLIBRI1: "kolibri1",
MODEL_ARCH.LIMITE: "limite",
@@ -6722,6 +6725,39 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
],
+ MODEL_ARCH.ZONOS3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT_RES_SCORE,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_RES_SCORE,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_NORM,
+ MODEL_TENSOR.SSM_BETA,
+ MODEL_TENSOR.SSM_ALPHA,
+ MODEL_TENSOR.SSM_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_RES_SCORE,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.MOE_LATENT_DOWN,
+ MODEL_TENSOR.MOE_LATENT_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
MODEL_ARCH.K2HORIZON: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
@@ -7156,6 +7192,8 @@ class VisionProjectorType:
CHATTERBOX = "chatterbox" # audio generation: s3gen flow matching + HiFT vocoder
SOPRANO = "soprano" # audio generation: Vocos decoder
DRAMA3_GEN = "drama3_gen" # audio generation: codebook tower + Fish DAC decoder
+ ZONOS3_GEN = "zonos3_gen" # audio generation: 10 codebook heads + ZAC decoder
+ ZONOS3_SPKENC = "zonos3_spkenc" # audio: MECT-B2 speaker encoder
HUNYUANVL = "hunyuanvl"
PARAKEET = "parakeet" # audio
MINIMAXM3 = "minimax_m3"
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index 6ca757080..edbe8cd09 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1812,6 +1812,9 @@ class GGUFWriter:
def add_gen_audio_model_variant(self, value: str) -> None:
self.add_string(Keys.ClipGenAudio.MODEL_VARIANT, value)
+ def add_gen_audio_default_voice(self, value: str) -> None:
+ self.add_string(Keys.ClipGenAudio.DEFAULT_VOICE, value)
+
def add_xielu_alpha_p(self, values: Sequence[float]):
self.add_array(Keys.xIELU.ALPHA_P, values)
diff --git a/models/templates/Zyphra-ZONOS3-Mini-tts.jinja b/models/templates/Zyphra-ZONOS3-Mini-tts.jinja
new file mode 100644
index 000000000..ec1255862
--- /dev/null
+++ b/models/templates/Zyphra-ZONOS3-Mini-tts.jinja
@@ -0,0 +1,627 @@
+{#- ZONOS3 "tts" template: the text to speak (first message), normalized; the helper adds BOS and EOS itself -#}
+{#- numbers, decimals, money, percents, ordinals, years and clock times are spelled out in en, fr, de, es, it, pt, ja and zh -#}
+{#- other languages pass through; patterns: tts_norm_scan() / tts_norm_sub(), see docs/tts-norm-patterns.md -#}
+{#- the ja / zh conventions follow the NeMo grammars of the reference (e.g. zh: 两千, 二百, 三百一十六, 二零零二年) -#}
+{#- optional identifies this template, so that llama.cpp can detect it and apply quick fixes to GGUFs already converted -#}
+{%- set template_name = "zonos3-mini" -%}
+
+{#- English -#}
+{%- set ones = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve",
+ "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen"] -%}
+{%- set tens = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"] -%}
+{%- set scales = ["", " thousand", " million", " billion", " trillion"] -%}
+{%- set irregular = {"one": "first", "two": "second", "three": "third", "five": "fifth", "eight": "eighth", "nine": "ninth", "twelve": "twelfth"} -%}
+
+{%- macro under100(n) -%}
+{{- ones[n] if n < 20 else tens[(n / 10)|int] ~ ("-" ~ ones[n % 10] if n % 10 else "") -}}
+{%- endmacro -%}
+
+{%- macro under1000(n) -%}
+{{- (ones[(n / 100)|int] ~ " hundred" ~ (" " ~ under100(n % 100) if n % 100 else "")) if n >= 100 else under100(n) -}}
+{%- endmacro -%}
+
+{#- all the *_card macros take a digit string of up to 15 digits -#}
+{%- macro en_card(s) -%}
+{%- set p = ["", "00", "0"][(s|length) % 3] ~ s -%}
+{%- set ns = namespace(out="", ng=((p|length) / 3)|int) -%}
+{%- for g in range(ns.ng) -%}
+{%- set v = p[g * 3:g * 3 + 3]|int -%}
+{%- if v > 0 -%}
+{%- set ns.out = ns.out ~ (" " if ns.out else "") ~ under1000(v) ~ scales[ns.ng - 1 - g] -%}
+{%- endif -%}
+{%- endfor -%}
+{{- ns.out if ns.out else "zero" -}}
+{%- endmacro -%}
+
+{#- the last word of a cardinal becomes ordinal: twenty-one -> twenty-first -#}
+{%- macro en_ord(w) -%}
+{%- set last = (w.split(" ")|last).split("-")|last -%}
+{%- set head = w[0:(w|length) - (last|length)] -%}
+{{- head ~ (irregular[last] if last in irregular else (last[0:(last|length) - 1] ~ "ieth" if last.endswith("y") else last ~ "th")) -}}
+{%- endmacro -%}
+
+{#- French: traditional spelling, quatre-vingts / deux cents take the plural s only at the end -#}
+{%- set fr_u = ["zéro", "un", "deux", "trois", "quatre", "cinq", "six", "sept", "huit", "neuf", "dix", "onze", "douze", "treize",
+ "quatorze", "quinze", "seize"] -%}
+{%- set fr_t = ["", "", "vingt", "trente", "quarante", "cinquante", "soixante", "", "quatre-vingt", ""] -%}
+{%- set fr_big = [[], [], ["million", "millions"], ["milliard", "milliards"], ["billion", "billions"]] -%}
+
+{%- macro fr99(n, final) -%}
+{%- if n < 17 -%}
+{{- fr_u[n] -}}
+{%- elif n < 20 -%}
+{{- "dix-" ~ fr_u[n - 10] -}}
+{%- else -%}
+{%- set t = (n / 10)|int -%}
+{%- set u = n % 10 -%}
+{%- if t == 7 or t == 9 -%}
+{{- ("soixante" if t == 7 else "quatre-vingt") ~ (" et onze" if t == 7 and u == 1 else "-" ~ fr99(n - (60 if t == 7 else 80), final)) -}}
+{%- else -%}
+{{- fr_t[t] ~ ("s" if t == 8 and u == 0 and final else "") ~ (" et un" if u == 1 and t != 8 else ("-" ~ fr_u[u] if u else "")) -}}
+{%- endif -%}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro fr999(n, final) -%}
+{%- set h = (n / 100)|int -%}
+{%- set r = n % 100 -%}
+{%- set hs = "" if h == 0 else ("cent" if h == 1 else fr_u[h] ~ " cent" ~ ("s" if r == 0 and final else "")) -%}
+{{- hs ~ (" " if hs and r else "") ~ (fr99(r, final) if r else "") -}}
+{%- endmacro -%}
+
+{%- macro fr_card(s) -%}
+{%- set p = ["", "00", "0"][(s|length) % 3] ~ s -%}
+{%- set ns = namespace(out="", ng=((p|length) / 3)|int) -%}
+{%- for g in range(ns.ng) -%}
+{%- set v = p[g * 3:g * 3 + 3]|int -%}
+{%- set k = ns.ng - 1 - g -%}
+{%- if v > 0 -%}
+{%- if k == 0 -%}
+{%- set w = fr999(v, true) -%}
+{%- elif k == 1 -%}
+{%- set w = ("" if v == 1 else fr999(v, false) ~ " ") ~ "mille" -%}
+{%- else -%}
+{%- set w = fr999(v, true) ~ " " ~ fr_big[k][0 if v == 1 else 1] -%}
+{%- endif -%}
+{%- set ns.out = ns.out ~ (" " if ns.out else "") ~ w -%}
+{%- endif -%}
+{%- endfor -%}
+{{- ns.out if ns.out else "zéro" -}}
+{%- endmacro -%}
+
+{%- macro fr_ord(w) -%}
+{%- set last = (w.split(" ")|last).split("-")|last -%}
+{%- set head = w[0:(w|length) - (last|length)] -%}
+{%- set base = last[0:(last|length) - 1] if last in ["vingts", "cents", "millions", "milliards", "billions"] else last -%}
+{%- set map = {"cinq": "cinquième", "neuf": "neuvième"} -%}
+{{- head ~ (map[base] if base in map else (base[0:(base|length) - 1] ~ "ième" if base.endswith("e") else base ~ "ième")) -}}
+{%- endmacro -%}
+
+{#- German: below a million one word, eins / ein / eine -#}
+{%- set de_u = ["null", "eins", "zwei", "drei", "vier", "fünf", "sechs", "sieben", "acht", "neun", "zehn", "elf", "zwölf",
+ "dreizehn", "vierzehn", "fünfzehn", "sechzehn", "siebzehn", "achtzehn", "neunzehn"] -%}
+{%- set de_t = ["", "", "zwanzig", "dreißig", "vierzig", "fünfzig", "sechzig", "siebzig", "achtzig", "neunzig"] -%}
+{%- set de_big = [[], [], ["Million", "Millionen"], ["Milliarde", "Milliarden"], ["Billion", "Billionen"]] -%}
+
+{%- macro de99(n) -%}
+{{- de_u[n] if n < 20 else ((("ein" if n % 10 == 1 else de_u[n % 10]) ~ "und") if n % 10 else "") ~ de_t[(n / 10)|int] -}}
+{%- endmacro -%}
+
+{%- macro de999(n) -%}
+{%- set h = (n / 100)|int -%}
+{%- set r = n % 100 -%}
+{{- (((("ein" if h == 1 else de_u[h]) ~ "hundert")) if h else "") ~ (de99(r) if r else "") -}}
+{%- endmacro -%}
+
+{#- attributive form: eins -> ein -#}
+{%- macro de_attr(w) -%}
+{{- w[0:(w|length) - 1] if w.endswith("eins") else w -}}
+{%- endmacro -%}
+
+{%- macro de_card(s) -%}
+{%- set p = ["", "00", "0"][(s|length) % 3] ~ s -%}
+{%- set ns = namespace(big="", small="", ng=((p|length) / 3)|int) -%}
+{%- for g in range(ns.ng) -%}
+{%- set v = p[g * 3:g * 3 + 3]|int -%}
+{%- set k = ns.ng - 1 - g -%}
+{%- if v > 0 -%}
+{%- if k == 0 -%}
+{%- set ns.small = ns.small ~ de999(v) -%}
+{%- elif k == 1 -%}
+{%- set ns.small = ns.small ~ de_attr(de999(v)) ~ "tausend" -%}
+{%- else -%}
+{%- set ns.big = ns.big ~ (" " if ns.big else "") ~ ("eine" if v == 1 else de_attr(de999(v))) ~ " " ~ de_big[k][0 if v == 1 else 1] -%}
+{%- endif -%}
+{%- endif -%}
+{%- endfor -%}
+{%- set out = ns.big ~ (" " if ns.big and ns.small else "") ~ ns.small -%}
+{{- out if out else "null" -}}
+{%- endmacro -%}
+
+{#- join the non-empty parts with a space -#}
+{%- macro join4(a, b, c, d) -%}
+{%- set ns = namespace(out="") -%}
+{%- for x in [a, b, c, d] -%}
+{%- if x -%}
+{%- set ns.out = ns.out ~ (" " if ns.out else "") ~ x -%}
+{%- endif -%}
+{%- endfor -%}
+{{- ns.out -}}
+{%- endmacro -%}
+
+{#- Spanish: long scale (mil millones, billón), un / veintiún before a noun or mil -#}
+{%- set es_u = ["cero", "uno", "dos", "tres", "cuatro", "cinco", "seis", "siete", "ocho", "nueve", "diez", "once", "doce",
+ "trece", "catorce", "quince", "dieciséis", "diecisiete", "dieciocho", "diecinueve", "veinte", "veintiuno",
+ "veintidós", "veintitrés", "veinticuatro", "veinticinco", "veintiséis", "veintisiete", "veintiocho", "veintinueve"] -%}
+{%- set es_t = ["", "", "", "treinta", "cuarenta", "cincuenta", "sesenta", "setenta", "ochenta", "noventa"] -%}
+{%- set es_h = ["", "ciento", "doscientos", "trescientos", "cuatrocientos", "quinientos", "seiscientos", "setecientos",
+ "ochocientos", "novecientos"] -%}
+
+{%- macro es99(n) -%}
+{{- es_u[n] if n < 30 else es_t[(n / 10)|int] ~ (" y " ~ es_u[n % 10] if n % 10 else "") -}}
+{%- endmacro -%}
+
+{%- macro es999(n) -%}
+{%- if n == 100 -%}
+{{- "cien" -}}
+{%- else -%}
+{%- set h = (n / 100)|int -%}
+{%- set r = n % 100 -%}
+{{- es_h[h] ~ (" " if h and r else "") ~ (es99(r) if r else "") -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro es_apoc(w) -%}
+{{- (w[0:(w|length) - 9] ~ "veintiún") if w.endswith("veintiuno") else ((w[0:(w|length) - 3] ~ "un") if w.endswith("uno") else w) -}}
+{%- endmacro -%}
+
+{%- macro es6(n) -%}
+{%- set a = (n / 1000)|int -%}
+{%- set b = n % 1000 -%}
+{%- set hi = "" if a == 0 else ("mil" if a == 1 else es_apoc(es999(a)) ~ " mil") -%}
+{{- hi ~ (" " if hi and b else "") ~ (es999(b) if b else "") -}}
+{%- endmacro -%}
+
+{%- macro es_card(s) -%}
+{%- set p = ("000000000000000000" ~ s)[-18:] -%}
+{%- set top = p[0:6]|int -%}
+{%- set mid = p[6:12]|int -%}
+{%- set low = p[12:18]|int -%}
+{%- set a = "" if top == 0 else ("un billón" if top == 1 else es_apoc(es6(top)) ~ " billones") -%}
+{%- set b = "" if mid == 0 else ("un millón" if mid == 1 else es_apoc(es6(mid)) ~ " millones") -%}
+{%- set out = join4(a, b, es6(low) if low else "", "") -%}
+{{- out if out else "cero" -}}
+{%- endmacro -%}
+
+{#- Italian: below a million one word, ventitré, centottanta -#}
+{%- set it_u = ["zero", "uno", "due", "tre", "quattro", "cinque", "sei", "sette", "otto", "nove", "dieci", "undici", "dodici",
+ "tredici", "quattordici", "quindici", "sedici", "diciassette", "diciotto", "diciannove"] -%}
+{%- set it_t = ["", "", "venti", "trenta", "quaranta", "cinquanta", "sessanta", "settanta", "ottanta", "novanta"] -%}
+
+{%- macro it99(n) -%}
+{%- if n < 20 -%}
+{{- it_u[n] -}}
+{%- else -%}
+{%- set t = it_t[(n / 10)|int] -%}
+{%- set u = n % 10 -%}
+{{- (t[0:(t|length) - 1] if u == 1 or u == 8 else t) ~ (it_u[u] if u else "") -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro it999(n) -%}
+{%- set h = (n / 100)|int -%}
+{%- set r = n % 100 -%}
+{%- set hs = "" if h == 0 else ("cento" if h == 1 else it_u[h] ~ "cento") -%}
+{{- (hs[0:(hs|length) - 1] if (r / 10)|int == 8 else hs) ~ (it99(r) if r else "") -}}
+{%- endmacro -%}
+
+{#- without the final accent, see it_acc -#}
+{%- macro it6(n) -%}
+{%- set a = (n / 1000)|int -%}
+{%- set b = n % 1000 -%}
+{{- ("" if a == 0 else ("mille" if a == 1 else it999(a) ~ "mila")) ~ (it999(b) if b else "") -}}
+{%- endmacro -%}
+
+{%- macro it_acc(w) -%}
+{{- (w[0:(w|length) - 3] ~ "tré") if w.endswith("tre") and w != "tre" else w -}}
+{%- endmacro -%}
+
+{%- macro it_card(s) -%}
+{%- set p = ("000000000000000000" ~ s)[-18:] -%}
+{%- set bil = p[0:6]|int -%}
+{%- set mld = p[6:9]|int -%}
+{%- set mil = p[9:12]|int -%}
+{%- set low = p[12:18]|int -%}
+{%- set a = "" if bil == 0 else ("un bilione" if bil == 1 else it_acc(it6(bil)) ~ " bilioni") -%}
+{%- set b = "" if mld == 0 else ("un miliardo" if mld == 1 else it_acc(it999(mld)) ~ " miliardi") -%}
+{%- set c = "" if mil == 0 else ("un milione" if mil == 1 else it_acc(it999(mil)) ~ " milioni") -%}
+{%- set out = join4(a, b, c, it_acc(it6(low)) if low else "") -%}
+{{- out if out else "zero" -}}
+{%- endmacro -%}
+
+{#- Portuguese (Brazil): short scale, "e" before the last group when it is below 100 or a round hundred -#}
+{%- set pt_u = ["zero", "um", "dois", "três", "quatro", "cinco", "seis", "sete", "oito", "nove", "dez", "onze", "doze",
+ "treze", "catorze", "quinze", "dezesseis", "dezessete", "dezoito", "dezenove"] -%}
+{%- set pt_t = ["", "", "vinte", "trinta", "quarenta", "cinquenta", "sessenta", "setenta", "oitenta", "noventa"] -%}
+{%- set pt_h = ["", "cento", "duzentos", "trezentos", "quatrocentos", "quinhentos", "seiscentos", "setecentos", "oitocentos",
+ "novecentos"] -%}
+{%- set pt_big = [[], [], ["milhão", "milhões"], ["bilhão", "bilhões"], ["trilhão", "trilhões"]] -%}
+
+{%- macro pt99(n) -%}
+{{- pt_u[n] if n < 20 else pt_t[(n / 10)|int] ~ (" e " ~ pt_u[n % 10] if n % 10 else "") -}}
+{%- endmacro -%}
+
+{%- macro pt999(n) -%}
+{%- if n == 100 -%}
+{{- "cem" -}}
+{%- else -%}
+{%- set h = (n / 100)|int -%}
+{%- set r = n % 100 -%}
+{{- pt_h[h] ~ (" e " if h and r else "") ~ (pt99(r) if r else "") -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro pt_card(s) -%}
+{%- set p = ["", "00", "0"][(s|length) % 3] ~ s -%}
+{%- set ns = namespace(out="", ng=((p|length) / 3)|int, last=-1) -%}
+{%- for g in range(ns.ng) -%}
+{%- if p[g * 3:g * 3 + 3]|int > 0 -%}
+{%- set ns.last = g -%}
+{%- endif -%}
+{%- endfor -%}
+{%- for g in range(ns.ng) -%}
+{%- set v = p[g * 3:g * 3 + 3]|int -%}
+{%- set k = ns.ng - 1 - g -%}
+{%- if v > 0 -%}
+{%- if k == 0 -%}
+{%- set w = pt999(v) -%}
+{%- elif k == 1 -%}
+{%- set w = "mil" if v == 1 else pt999(v) ~ " mil" -%}
+{%- else -%}
+{%- set w = ("um" if v == 1 else pt999(v)) ~ " " ~ pt_big[k][0 if v == 1 else 1] -%}
+{%- endif -%}
+{%- set sep = " e " if g == ns.last and (v < 100 or v % 100 == 0) else " " -%}
+{%- set ns.out = ns.out ~ (sep if ns.out else "") ~ w -%}
+{%- endif -%}
+{%- endfor -%}
+{{- ns.out if ns.out else "zero" -}}
+{%- endmacro -%}
+
+{#- Japanese: 万 / 億 / 兆 groups, no 一 before 十 / 百 / 千 (but 一千万), zero digits are skipped -#}
+{%- set cjk_d = ["零", "一", "二", "三", "四", "五", "六", "七", "八", "九"] -%}
+
+{%- macro ja4(n, big) -%}
+{%- set th = (n / 1000)|int -%}
+{%- set h = ((n / 100)|int) % 10 -%}
+{%- set t = ((n / 10)|int) % 10 -%}
+{%- set u = n % 10 -%}
+{{- ((("一" if big else "") if th == 1 else cjk_d[th]) ~ "千" if th else "")
+ ~ (("" if h == 1 else cjk_d[h]) ~ "百" if h else "")
+ ~ (("" if t == 1 else cjk_d[t]) ~ "十" if t else "")
+ ~ (cjk_d[u] if u else "") -}}
+{%- endmacro -%}
+
+{%- macro ja_card(s) -%}
+{%- set p = ("0000000000000000" ~ s)[-16:] -%}
+{%- set ns = namespace(out="") -%}
+{%- for g in range(4) -%}
+{%- set v = p[g * 4:g * 4 + 4]|int -%}
+{%- if v > 0 -%}
+{%- set ns.out = ns.out ~ ja4(v, g < 3) ~ ["兆", "億", "万", ""][g] -%}
+{%- endif -%}
+{%- endfor -%}
+{{- ns.out if ns.out else "零" -}}
+{%- endmacro -%}
+
+{#- Chinese: 万 / 亿 groups, 两 before 千 / 万 / 亿, 二 before 百, 十 at the head but 一十 inside, one 零 per gap -#}
+{%- macro zh4(n, head) -%}
+{%- set ds = [(n / 1000)|int, ((n / 100)|int) % 10, ((n / 10)|int) % 10, n % 10] -%}
+{%- set ns = namespace(out="", zero=false) -%}
+{%- for i in range(4) -%}
+{%- set d = ds[i] -%}
+{%- if d == 0 -%}
+{%- if ns.out -%}
+{%- set ns.zero = true -%}
+{%- endif -%}
+{%- else -%}
+{%- set w = "两" if d == 2 and i == 0 else cjk_d[d] -%}
+{%- if i == 2 and d == 1 and head and not ns.out -%}
+{%- set w = "" -%}
+{%- endif -%}
+{%- set ns.out = ns.out ~ ("零" if ns.zero else "") ~ w ~ ["千", "百", "十", ""][i] -%}
+{%- set ns.zero = false -%}
+{%- endif -%}
+{%- endfor -%}
+{{- ns.out -}}
+{%- endmacro -%}
+
+{%- macro zh_card(s) -%}
+{%- set p = ("0000000000000000" ~ s)[-16:] -%}
+{%- set ns = namespace(out="", gap=false) -%}
+{%- for g in range(4) -%}
+{%- set v = p[g * 4:g * 4 + 4]|int -%}
+{%- if v == 0 -%}
+{%- if ns.out -%}
+{%- set ns.gap = true -%}
+{%- endif -%}
+{%- else -%}
+{%- set w = "两" if v == 2 and g < 3 else zh4(v, not ns.out) -%}
+{%- set ns.out = ns.out ~ ("零" if ns.out and (ns.gap or v < 1000) else "") ~ w ~ ["万亿", "亿", "万", ""][g] -%}
+{%- set ns.gap = false -%}
+{%- endif -%}
+{%- endfor -%}
+{{- ns.out if ns.out else "零" -}}
+{%- endmacro -%}
+
+{#- per-language words -#}
+{%- set digit_words = {"en": ones[0:10], "fr": fr_u[0:10], "de": de_u[0:10], "es": es_u[0:10], "it": it_u[0:10], "pt": pt_u[0:10],
+ "ja": cjk_d, "zh": cjk_d} -%}
+{%- set W = {
+ "en": {"point": " point ", "minus": "minus ", "and": " and ", "percent": " percent", "sep": " "},
+ "fr": {"point": " virgule ", "minus": "moins ", "and": " et ", "percent": " pour cent", "sep": " "},
+ "de": {"point": " Komma ", "minus": "minus ", "and": " und ", "percent": " Prozent", "sep": " "},
+ "es": {"point": " coma ", "minus": "menos ", "and": " con ", "percent": " por ciento", "sep": " "},
+ "it": {"point": " virgola ", "minus": "meno ", "and": " e ", "percent": " per cento", "sep": " "},
+ "pt": {"point": " vírgula ", "minus": "menos ", "and": " e ", "percent": " por cento", "sep": " "},
+ "ja": {"point": "点", "minus": "マイナス", "and": "", "percent": "パーセント", "sep": ""},
+ "zh": {"point": "点", "minus": "负", "and": "", "percent": "", "sep": ""},
+} -%}
+{#- currency -> [singular, plural, cent singular, cent plural]; ja / zh: the unit after the number -#}
+{%- set M = {
+ "en": {"$": ["dollar", "dollars", "cent", "cents"], "€": ["euro", "euros", "cent", "cents"],
+ "£": ["pound", "pounds", "penny", "pence"], "¥": ["yen", "yen", "sen", "sen"]},
+ "fr": {"$": ["dollar", "dollars", "cent", "cents"], "€": ["euro", "euros", "centime", "centimes"]},
+ "de": {"$": ["Dollar", "Dollar", "Cent", "Cent"], "€": ["Euro", "Euro", "Cent", "Cent"], "£": ["Pfund", "Pfund", "Penny", "Pence"]},
+ "es": {"$": ["dólar", "dólares", "centavo", "centavos"], "€": ["euro", "euros", "céntimo", "céntimos"]},
+ "it": {"$": ["dollaro", "dollari", "centesimo", "centesimi"], "€": ["euro", "euro", "centesimo", "centesimi"]},
+ "pt": {"$": ["dólar", "dólares", "centavo", "centavos"], "€": ["euro", "euros", "cêntimo", "cêntimos"],
+ "R$": ["real", "reais", "centavo", "centavos"]},
+ "ja": {"$": "ドル", "€": "ユーロ", "£": "ポンド", "¥": "円"},
+ "zh": {"$": "美元", "€": "欧元", "£": "英镑", "¥": "元"},
+} -%}
+{%- set es_ord = ["", "primero", "segundo", "tercero", "cuarto", "quinto", "sexto", "séptimo", "octavo", "noveno", "décimo"] -%}
+{%- set it_ord = ["", "primo", "secondo", "terzo", "quarto", "quinto", "sesto", "settimo", "ottavo", "nono", "decimo"] -%}
+{%- set pt_ord = ["", "primeiro", "segundo", "terceiro", "quarto", "quinto", "sexto", "sétimo", "oitavo", "nono", "décimo"] -%}
+
+{%- macro card(L, s) -%}
+{%- if L == "fr" -%}{{- fr_card(s) -}}
+{%- elif L == "de" -%}{{- de_card(s) -}}
+{%- elif L == "es" -%}{{- es_card(s) -}}
+{%- elif L == "it" -%}{{- it_card(s) -}}
+{%- elif L == "pt" -%}{{- pt_card(s) -}}
+{%- elif L == "ja" -%}{{- ja_card(s) -}}
+{%- elif L == "zh" -%}{{- zh_card(s) -}}
+{%- else -%}{{- en_card(s) -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{#- attributive form before a noun: ein Euro, un euro, veintiún euros -#}
+{%- macro attr(L, w) -%}
+{%- if L == "de" -%}{{- de_attr(w) -}}
+{%- elif L == "es" -%}{{- es_apoc(w) -}}
+{%- elif L == "it" and w == "uno" -%}{{- "un" -}}
+{%- else -%}{{- w -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro spell(L, s) -%}
+{%- set ns = namespace(out="") -%}
+{%- for i in range(s|length) -%}
+{%- set ns.out = ns.out ~ (W[L]["sep"] if i > 0 else "") ~ digit_words[L][s[i:i + 1]|int] -%}
+{%- endfor -%}
+{{- ns.out -}}
+{%- endmacro -%}
+
+{#- decimals: fr / es / it / pt read a short fraction as a number (virgule quatorze), the others digit by digit -#}
+{%- macro fraction(L, f) -%}
+{%- if L in ["fr", "es", "it", "pt"] and f|length <= 3 -%}
+{%- set z = tts_norm_sub(f, "^(0*).*", "%1") -%}
+{%- set rest = f[z|length:] -%}
+{{- spell(L, z) ~ (" " if z and rest else "") ~ (card(L, rest) if rest else "") -}}
+{%- else -%}
+{{- spell(L, f) -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{#- en: 1990 -> nineteen ninety, 1905 -> nineteen oh five, 2005 -> two thousand five; de: 1990 -> neunzehnhundertneunzig -#}
+{%- macro year(L, s) -%}
+{%- set hi = s[0:2]|int -%}
+{%- set lo = s[2:4]|int -%}
+{%- if L == "de" -%}
+{{- (de99(hi) ~ "hundert" ~ (de99(lo) if lo else "")) if hi < 20 else de_card(s) -}}
+{%- elif (hi == 20 and lo < 10) or (hi % 10 == 0 and lo == 0) -%}
+{{- en_card(s) -}}
+{%- elif lo == 0 -%}
+{{- under100(hi) ~ " hundred" -}}
+{%- else -%}
+{{- under100(hi) ~ (" oh " ~ ones[lo] if lo < 10 else " " ~ under100(lo)) -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{#- ip: digits with optional thousands separators, frac: decimal digits or empty -#}
+{%- macro number(L, ip, frac, year_ok) -%}
+{%- set d = tts_norm_sub(ip, "%D", "") -%}
+{%- if d|length > 15 or (d|length > 1 and d[0:1] == "0" and not frac) -%}
+{{- spell(L, d) ~ (W[L]["point"] ~ spell(L, frac) if frac else "") -}}
+{%- elif frac -%}
+{{- card(L, d) ~ W[L]["point"] ~ fraction(L, frac) -}}
+{%- elif year_ok and d == ip and d|length == 4 and d|int >= 1100 and d|int <= 2099 and L in ["en", "de"] -%}
+{{- year(L, d) -}}
+{%- else -%}
+{{- card(L, d) -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{#- fr / es / it / pt: million and above are nouns, a round amount takes "de" before the currency (deux millions d'euros) -#}
+{%- macro unit_sep(L, ip, unit) -%}
+{%- set d = tts_norm_sub(ip, "%D", "") -%}
+{%- if L in ["fr", "es", "it", "pt"] and d|length > 6 and d[(d|length) - 6:] == "000000" -%}
+{{- " d'" if L == "fr" and unit[0:1] in "aeiou" else " " ~ ("di" if L == "it" else "de") ~ " " -}}
+{%- else -%}
+{{- " " -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro money(L, cur, ip, cents, scale) -%}
+{%- if cur not in M[L] -%}
+{{- cur ~ W[L]["sep"] ~ number(L, ip, cents, false) -}}
+{%- elif L in ["ja", "zh"] -%}
+{{- number(L, ip, cents, false) ~ M[L][cur] -}}
+{%- else -%}
+{%- set u = M[L][cur] -%}
+{%- if scale -%}
+{{- number(L, ip, cents, false) ~ " " ~ scale ~ " " ~ u[1] -}}
+{%- elif cents|length == 2 -%}
+{%- set c = attr(L, card(L, cents)) ~ " " ~ u[2 if cents|int == 1 else 3] -%}
+{%- if tts_norm_sub(ip, "%D", "")|int == 0 -%}
+{{- c -}}
+{%- else -%}
+{{- attr(L, number(L, ip, "", false)) ~ unit_sep(L, ip, u[1]) ~ u[0 if ip == "1" else 1] ~ (W[L]["and"] ~ c if cents|int > 0 else "") -}}
+{%- endif -%}
+{%- elif cents -%}
+{{- number(L, ip, cents, false) ~ " " ~ u[1] -}}
+{%- else -%}
+{{- attr(L, number(L, ip, "", false)) ~ unit_sep(L, ip, u[1]) ~ u[0 if ip == "1" else 1] -}}
+{%- endif -%}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro clock(L, h, m) -%}
+{%- set hn = h|int -%}
+{%- set mm = m|int if m else 0 -%}
+{%- if L == "fr" -%}
+{%- set w = fr_card(h) -%}
+{{- w ~ ("e" if w.endswith("un") else "") ~ (" heure" if hn <= 1 else " heures") ~ (" " ~ fr_card(m) if mm else "") -}}
+{%- elif L == "de" -%}
+{{- de_attr(de_card(h)) ~ " Uhr" ~ (" " ~ de_card(m) if mm else "") -}}
+{%- elif L == "es" -%}
+{%- set w = es_card(h) -%}
+{{- (w[0:(w|length) - 1] ~ "a" if w.endswith("uno") else w) ~ (" y " ~ es_card(m) if mm else " en punto") -}}
+{%- elif L == "it" -%}
+{{- ("una" if hn == 1 else it_card(h)) ~ (" e " ~ it_card(m) if mm else "") -}}
+{%- elif L == "pt" -%}
+{%- set w = pt_card(h) -%}
+{%- set w = w ~ "a" if w.endswith("um") else (w[0:(w|length) - 4] ~ "duas" if w.endswith("dois") else w) -%}
+{{- w ~ (" e " ~ pt_card(m) if mm else (" hora" if hn == 1 else " horas")) -}}
+{%- elif L == "ja" -%}
+{{- ja_card(h) ~ "時" ~ (ja_card(m) ~ "分" if mm else "") -}}
+{%- elif L == "zh" -%}
+{{- ("两" if hn == 2 else zh_card(h)) ~ "点" ~ (zh_card(m) ~ "分" if mm else "") -}}
+{%- else -%}
+{{- en_card(h) ~ (" o'clock" if mm == 0 else (" oh " ~ ones[mm] if mm < 10 else " " ~ under100(mm))) -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro ordinal(L, d, suf) -%}
+{%- set n = d|int -%}
+{%- if L == "fr" -%}
+{%- set plural = "s" if suf.endswith("s") else "" -%}
+{%- if n == 1 -%}
+{{- ("première" if suf.startswith("r") or suf.startswith("è") else "premier") ~ plural -}}
+{%- elif n == 2 and suf.startswith("nd") -%}
+{{- ("seconde" if suf.startswith("nde") else "second") ~ plural -}}
+{%- else -%}
+{{- fr_ord(fr_card(d)) ~ plural -}}
+{%- endif -%}
+{%- elif L in ["es", "it", "pt"] -%}
+{%- set fem = suf == "ª" -%}
+{%- if n >= 1 and n <= 10 -%}
+{%- set w = {"es": es_ord, "it": it_ord, "pt": pt_ord}[L][n] -%}
+{{- w[0:(w|length) - 1] ~ "a" if fem else w -}}
+{%- elif L == "it" and n < 1000000 -%}
+{#- undicesimo, ventitreesimo, ventiseiesimo -#}
+{%- set w = it6(n) -%}
+{%- set w = w ~ "esimo" if w.endswith("tre") or w.endswith("sei") else w[0:(w|length) - 1] ~ "esimo" -%}
+{{- w[0:(w|length) - 1] ~ "a" if fem else w -}}
+{%- else -%}
+{{- card(L, d) -}}
+{%- endif -%}
+{%- else -%}
+{{- en_ord(en_card(d)) -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- set l = ((language | lower) if language is defined and language else "").replace("-", "_") -%}
+{%- set L = "en" if l == "" else ("zh" if l.startswith("zh") or l.startswith("cmn") else l.split("_")[0]) -%}
+{%- set text = messages[0]["content"] -%}
+{%- if L not in M -%}
+{{- text -}}
+{%- else -%}
+{%- set t = text -%}
+{%- if L not in ["ja", "zh"] -%}
+{#- a space between a number and the letters it touches (5km, mp3), except for ordinal suffixes and the French 15h30 -#}
+{%- set suffixes = {"en": "st|nd|rd|th|s", "fr": "ers?|res?|ères?|nde?s?|èmes?|es?|h", "es": "º|ª", "it": "º|ª", "pt": "º|ª"} -%}
+{%- if L in suffixes -%}
+{%- set t = tts_norm_sub(t, "(%d)(?!(?:" ~ suffixes[L] ~ ")(?!%a))(%a)", "%1 %2") -%}
+{%- else -%}
+{%- set t = tts_norm_sub(t, "(%d)(%a)", "%1 %2") -%}
+{%- endif -%}
+{%- set t = tts_norm_sub(t, "(?<!%d)(%a)(%d)", "%1 %2") -%}
+{%- endif -%}
+{#- integer part with thousands separators (groups of exactly three digits), decimal part not followed by another group -#}
+{%- set tsep = {"fr": "[%s.]", "de": "%.", "es": "%.", "it": "%.", "pt": "%."}.get(L, ",") -%}
+{%- set int = "(%d{1,3}(?:" ~ tsep ~ "%d{3})+(?!%d)|%d+)" -%}
+{%- set frac = ("(?:%.(%d+)(?![,.]%d))?" if L in ["en", "ja", "zh"] else "(?:[,.](%d+)(?![,.]%d))?") -%}
+{#- a minus sign at the start of the text, after a space or after an opening parenthesis; ja / zh: after anything but a digit -#}
+{%- set neg = "(?:(?<!%d)(-))?" if L in ["ja", "zh"] else "(?:(?<![^%s(])(-))?" -%}
+{%- set cur = "(R%$|[$€£¥])" -%}
+{#- the matcher never backtracks into a choice, so the longest hour forms come first -#}
+{%- set hour = "(?<!%d)(2[0-3]|[01]%d|%d)" -%}
+{%- set time = hour ~ "(?:h([0-5]%d)?|:([0-5]%d))(?![%d%a])" if L == "fr"
+ else hour ~ ":([0-5]%d)(?!%d)" ~ ("(?: Uhr%b)?" if L == "de" else "") -%}
+{%- set ords = {
+ "en": [["ordinal", "(?<![%d,.])(%d{1,15})(st|nd|rd|th)(?!%a)"]],
+ "fr": [["ordinal", "(?<![%d,.])(%d{1,15})(ers?|res?|ères?|nde?s?|èmes?|es?)(?!%a)"]],
+ "es": [["ordinal", "(?<![%d,.])(%d{1,15})%.?([ºª])"]],
+ "it": [["ordinal", "(?<![%d,.])(%d{1,15})%.?([ºª])"]],
+ "pt": [["ordinal", "(?<![%d,.])(%d{1,15})%.?([ºª])"]],
+}.get(L, []) -%}
+{#- en / de: a 4-digit number is read as a year only after one of these words (in 1990, im Jahr 1990), 1234 people stays a count -#}
+{%- set year_cues = {
+ "en": "(?i)%b(in|since|by|from|until|till|to|through|between|of|around|circa|before|after|during|year|early|late|mid"
+ ~ "|january|february|march|april|may|june|july|august|september|october|november|december)",
+ "de": "(?i)%b(im|seit|bis|von|ab|anno|um|vor|nach|zwischen|jahr|jahre|jahres|jahren|anfang|mitte|ende"
+ ~ "|januar|februar|märz|april|mai|juni|juli|august|september|oktober|november|dezember)",
+} -%}
+{%- set rules = [
+ ["money", cur ~ "%s?" ~ int ~ frac ~ "(?: (thousand|million|billion|trillion)%b)?"],
+ ["money2", int ~ frac ~ "%s?" ~ cur],
+ ["percent", neg ~ int ~ frac ~ "%s?[%%％]"],
+ ["date", "(?<![%d,./])(%d{1,2})[./](%d{1,2})[./](%d{4}|%d{2})(?!%d|[,./]%d)"],
+ ["time", time],
+] + ords
+ + ([["year", "(?<![%d,.])(%d{2,4})(?=年)"]] if L == "zh" else [])
+ + ([["year4", year_cues[L] ~ "(%s+)(%d{4})(?!%a|%d|[,.]%d|%s?[$€£¥%%])"]] if L in year_cues else [])
+ + ([["plural", "(?<![%d,.])(%d{1,15})s(?!%a)"]] if L == "en" else [])
+ + [["number", neg ~ int ~ frac]] -%}
+{%- for s in tts_norm_scan(t, rules) -%}
+{%- set g = s.groups -%}
+{%- if s.rule == "money" -%}
+{{- money(L, g[0], g[1], g[2], g[3]) -}}
+{%- elif s.rule == "money2" -%}
+{{- money(L, g[2], g[0], g[1], "") -}}
+{%- elif s.rule == "percent" -%}
+{%- set w = number(L, g[1], g[2], false) -%}
+{{- (W[L]["minus"] if g[0] else "") ~ ("百分之" ~ w if L == "zh" else w ~ W[L]["percent"]) -}}
+{%- elif s.rule == "date" -%}
+{#- read as three numbers, the order of day and month is the one of the text -#}
+{{- number(L, g[0], "", false) ~ W[L]["sep"] ~ number(L, g[1], "", false) ~ W[L]["sep"] ~ number(L, g[2], "", true) -}}
+{%- elif s.rule == "time" -%}
+{{- clock(L, g[0], g[1] or (g[2] if g|length > 2 else "")) -}}
+{%- elif s.rule == "ordinal" -%}
+{{- ordinal(L, g[0], g[1]) -}}
+{%- elif s.rule == "year" -%}
+{{- spell(L, g[0]) -}}
+{%- elif s.rule == "year4" -%}
+{{- g[0] ~ g[1] ~ number(L, g[2], "", true) -}}
+{%- elif s.rule == "plural" -%}
+{#- decades and plurals: 1990s -> nineteen nineties -#}
+{%- set w = number(L, g[0], "", true) -%}
+{{- (w[0:(w|length) - 1] ~ "ies") if w.endswith("y") else w ~ "s" -}}
+{%- elif s.rule == "number" -%}
+{{- (W[L]["minus"] if g[0] else "") ~ number(L, g[1], g[2], false) -}}
+{%- else -%}
+{{- s.text -}}
+{%- endif -%}
+{%- endfor -%}
+{%- endif -%}
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 22419390f..1869c1203 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -175,6 +175,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_NANBEIGE, "nanbeige" },
{ LLM_ARCH_QWEN3TTS, "qwen3tts" },
{ LLM_ARCH_POCKETTTS, "pockettts" },
+ { LLM_ARCH_ZONOS3, "zonos3" },
{ LLM_ARCH_K2_HORIZON, "k2-horizon" },
{ LLM_ARCH_BEAM, "beam" },
{ LLM_ARCH_KOLIBRI1, "kolibri1" },
@@ -1416,6 +1417,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
case LLM_ARCH_DEEPSEEK4:
case LLM_ARCH_DEEPSEEK41:
case LLM_ARCH_MINIMAX_01:
+ case LLM_ARCH_ZONOS3:
case LLM_ARCH_INKLING2:
return true;
default:
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 74bd6ce67..d015b5d5a 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -176,6 +176,7 @@ enum llm_arch {
LLM_ARCH_NANBEIGE,
LLM_ARCH_QWEN3TTS,
LLM_ARCH_POCKETTTS,
+ LLM_ARCH_ZONOS3,
LLM_ARCH_MINIMAX_01,
LLM_ARCH_HRM_TEXT,
LLM_ARCH_K2_HORIZON,
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 93edabc02..b66092b17 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -2908,6 +2908,7 @@ uint32_t llama_context::graph_max_nodes() const {
model.arch == LLM_ARCH_BAILINGMOE3 ||
model.arch == LLM_ARCH_QWEN35 ||
model.arch == LLM_ARCH_QWEN35MOE ||
+ model.arch == LLM_ARCH_ZONOS3 ||
model.arch == LLM_ARCH_CLEF ||
model.arch == LLM_ARCH_CLOVER_MOE ||
model.arch == LLM_ARCH_QWEN4EXP ||
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 194bbb92b..9801b4b0c 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -126,6 +126,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_qwen3tts(params);
case LLM_ARCH_POCKETTTS:
return new llama_model_pockettts(params);
+ case LLM_ARCH_ZONOS3:
+ return new llama_model_zonos3(params);
case LLM_ARCH_PHI2:
return new llama_model_phi2(params);
case LLM_ARCH_PHI3:
@@ -687,8 +689,9 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
return {{conv_state_size, 3}};
}
// TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]
+ // zonos3 has the qwen35 GDN layout, its attention layers have no fused qkv
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF || ud->model->arch == LLM_ARCH_CLOVER_MOE ||
- ud->model->arch == LLM_ARCH_QWEN4EXP) {
+ ud->model->arch == LLM_ARCH_QWEN4EXP || ud->model->arch == LLM_ARCH_ZONOS3) {
// fused full attention layers with Q gate tensors that need n_embd doubled:
if (!hparams.is_recr(il) && (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias))) {
@@ -731,7 +734,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim);
return {{key_dim, 2 + head_ratio}};
}
- if (std::regex_match(tensor_name, pattern_attn_gate_weight) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {
+ if ((hparams.is_recr(il) && std::regex_match(tensor_name, pattern_attn_gate_weight)) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {
return {{key_dim, head_ratio}};
}
if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||
@@ -2700,6 +2703,7 @@ void llama_model::print_info() const {
arch == LLM_ARCH_QWEN35 ||
arch == LLM_ARCH_QWEN35MOE ||
arch == LLM_ARCH_CLOVER_MOE ||
+ arch == LLM_ARCH_ZONOS3 ||
arch == LLM_ARCH_NEMOTRON_H ||
arch == LLM_ARCH_NEMOTRON_H_MOE) {
LLAMA_LOG_INFO("%s: ssm_d_conv = %u\n", __func__, hparams.ssm_d_conv);
@@ -3331,7 +3335,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
filter_recr = [&](uint32_t il) {
return hparams.is_recr(il) && hparams.n_ff(il) == 0;
};
- } else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_CLOVER_MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_MINIMAX_01) {
+ } else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_CLOVER_MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_MINIMAX_01 ||
+ arch == LLM_ARCH_ZONOS3) {
filter_attn = [&](uint32_t il) {
return il < hparams.n_layer() && !hparams.is_recr(il);
};
@@ -3745,6 +3750,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_DOTS3NOTE:
case LLM_ARCH_NANBEIGE:
case LLM_ARCH_POCKETTTS:
+ case LLM_ARCH_ZONOS3:
case LLM_ARCH_LIMITE:
return LLAMA_ROPE_TYPE_NORM;
case LLM_ARCH_BAILINGMOE3:
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index f2c1854c9..6f23a29d8 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -584,6 +584,12 @@ struct llm_tokenizer_bpe : llm_tokenizer {
regex_exprs = { "\\p{N}", "\\s+|[\\p{L}\\p{N}_]+|[^\\p{L}\\p{N}_\\s]+" };
byte_encode = false;
break;
+ case LLAMA_VOCAB_PRE_TYPE_ZONOS3:
+ // SuperBPE: no split on whitespace, tokens can span several words
+ regex_exprs = {
+ "\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]{2,}[\\r\\n/]*| +(?!\\S)",
+ };
+ break;
case LLAMA_VOCAB_PRE_TYPE_WHITESPACE:
// whitespace pre-tokenizer (jinaai/jina-embeddings-v2-base-zh)
regex_exprs = {
@@ -2774,6 +2780,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
} else if (
tokenizer_pre == "soprano") {
pre_type = LLAMA_VOCAB_PRE_TYPE_SOPRANO;
+ } else if (
+ tokenizer_pre == "zonos3") {
+ pre_type = LLAMA_VOCAB_PRE_TYPE_ZONOS3;
+ clean_spaces = false;
} else {
throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
}
diff --git a/src/llama-vocab.h b/src/llama-vocab.h
index 1ae5b65f6..9345275f5 100644
--- a/src/llama-vocab.h
+++ b/src/llama-vocab.h
@@ -75,6 +75,7 @@ enum llama_vocab_pre_type {
LLAMA_VOCAB_PRE_TYPE_NEMOTRON_3_5_EMBD = 64,
LLAMA_VOCAB_PRE_TYPE_SOPRANO = 65,
LLAMA_VOCAB_PRE_TYPE_TEKKEN_NUM3 = 66,
+ LLAMA_VOCAB_PRE_TYPE_ZONOS3 = 67,
};
// HF Metaspace pre-tokenizer prepend_scheme
diff --git a/src/models/models.h b/src/models/models.h
index 913c52ade..932887c91 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -769,6 +769,28 @@ struct llama_model_pockettts : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
+struct llama_model_zonos3 : public llama_model_base {
+ llama_model_zonos3(const struct llama_model_params & params) : llama_model_base(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ struct graph : public llm_build_delta_net_base {
+ graph(const llm_graph_params & params);
+ private:
+ // Full AttnRes bank: embedding + every sublayer output so far, [n_embd, n_src, n_tokens]
+ ggml_tensor * bank = nullptr;
+
+ void attn_res_push(ggml_tensor * cur);
+ ggml_tensor * attn_res_route(ggml_tensor * query, int il);
+
+ ggml_tensor * build_layer_attn(llm_graph_input_attn_kv * inp, ggml_tensor * cur, ggml_tensor * inp_pos, int il);
+ ggml_tensor * build_layer_gdn(llm_graph_input_rs * inp, ggml_tensor * cur, int il);
+ ggml_tensor * build_layer_ffn(ggml_tensor * cur, int il);
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
struct llama_model_codeshell : public llama_model_base {
llama_model_codeshell(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/src/models/zonos3.cpp b/src/models/zonos3.cpp
new file mode 100644
index 000000000..c5f62fe29
--- /dev/null
+++ b/src/models/zonos3.cpp
@@ -0,0 +1,351 @@
+#include "models.h"
+#include "llama-memory-recurrent.h"
+
+// ZONOS3 TTS backbone, the 10 audio heads are in the mmproj
+
+void llama_model_zonos3::load_arch_hparams(llama_model_loader & ml) {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
+ ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);
+ ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);
+ ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
+ ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
+
+ if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
+ uint32_t full_attn_interval = 0;
+ ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval);
+ if (full_attn_interval == 0) {
+ throw std::runtime_error("zonos3: full_attention_interval must be > 0");
+ }
+ for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {
+ hparams.is_recr_impl[i] = (i + 1) % full_attn_interval != 0;
+ }
+ }
+
+ ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);
+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
+ ml.get_key(LLM_KV_MOE_LATENT_SIZE, hparams.moe_latent_size);
+
+ // the router picks the top-k of logits + bias, then takes a softmax over the picked logits
+ hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT;
+ hparams.expert_weights_norm = false;
+
+ switch (hparams.n_layer()) {
+ case 11: type = LLM_TYPE_0_8B; break;
+ default: type = LLM_TYPE_UNKNOWN;
+ }
+}
+
+void llama_model_zonos3::load_arch_tensors(llama_model_loader &) {
+ LLAMA_LOAD_LOCALS;
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+ output_res_score = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_SCORE, "weight"), {n_embd}, 0);
+ // no output head, the logits are unused; reuse the embedding table so a sampler can still run
+ output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+
+ const int64_t d_inner = hparams.ssm_d_inner;
+ const int64_t head_dim = hparams.ssm_d_state;
+ const int64_t n_gdn_head = hparams.ssm_dt_rank;
+ const int64_t n_latent = hparams.moe_latent_size;
+ const int64_t n_ff_exp = hparams.n_ff_exp();
+
+ for (int i = 0; i < n_layer; ++i) {
+ auto & layer = layers[i];
+
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+ layer.attn_res_score = create_tensor(tn(LLM_TENSOR_ATTN_RES_SCORE, "weight", i), {n_embd}, 0);
+ layer.ffn_res_score = create_tensor(tn(LLM_TENSOR_FFN_RES_SCORE, "weight", i), {n_embd}, 0);
+
+ if (hparams.is_recr(i)) {
+ // split from the fused GDN in_proj at conversion, same layout as qwen35
+ layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, 3 * d_inner}, 0);
+ layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, d_inner}, 0); // z gate
+ layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", i), {hparams.ssm_d_conv, 3 * d_inner}, 0);
+ layer.ssm_dt = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {n_gdn_head}, 0);
+ layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {n_gdn_head}, 0);
+ layer.ssm_alpha = create_tensor(tn(LLM_TENSOR_SSM_ALPHA, "weight", i), {n_embd, n_gdn_head}, 0);
+ layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_gdn_head}, 0);
+ layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {head_dim}, 0);
+ layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", i), {d_inner, n_embd}, 0);
+ } else {
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
+ // per-head scale of the normalized Q, the QK norm itself has no weight
+ layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k, n_head}, 0);
+ // per-head output gate, same name as the GDN z gate
+ layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head}, 0);
+ }
+
+ if (i < (int) hparams.n_layer_dense_lead) {
+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
+ } else {
+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
+ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);
+ layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", i), {n_embd, n_latent}, 0);
+ layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", i), {n_latent, n_embd}, 0);
+ layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_latent, n_ff_exp, n_expert}, 0);
+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_latent, n_ff_exp, n_expert}, 0);
+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_latent, n_expert}, 0);
+ }
+ }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_zonos3::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(params);
+}
+
+llama_model_zonos3::graph::graph(const llm_graph_params & params) :
+ llm_build_delta_net_base(params) {
+ const float eps = hparams.f_norm_rms_eps;
+
+ ggml_tensor * cur;
+
+ ggml_tensor * inpL = build_inp_embd(model.tok_embd);
+
+ auto * inp = build_inp_mem_hybrid();
+
+ ggml_tensor * inp_pos = build_inp_pos();
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ // the summed row embedding goes through a weightless norm
+ inpL = ggml_rms_norm(ctx0, inpL, eps);
+ cb(inpL, "emb_norm", -1);
+ attn_res_push(inpL);
+
+ for (int il = 0; il < n_layer; ++il) {
+ const auto & layer = model.layers[il];
+
+ cur = attn_res_route(layer.attn_res_score, il);
+ cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "attn_norm", il);
+
+ if (hparams.is_recr(il)) {
+ cur = build_layer_gdn(inp->get_recr(), cur, il);
+ } else {
+ cur = build_layer_attn(inp->get_attn(), cur, inp_pos, il);
+ }
+ cb(cur, "attn_out", il);
+ attn_res_push(cur);
+
+ cur = attn_res_route(layer.ffn_res_score, il);
+ cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = build_layer_ffn(cur, il);
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+ attn_res_push(cur);
+ }
+
+ cur = attn_res_route(model.output_res_score, -1);
+ if (inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ }
+
+ cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ cur = build_lora_mm(model.output, cur, model.output_s);
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+}
+
+void llama_model_zonos3::graph::attn_res_push(ggml_tensor * cur) {
+ ggml_tensor * src = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
+ bank = bank ? ggml_concat(ctx0, bank, src, 1) : src;
+}
+
+// softmax over the bank sources, scored by query . rms_norm(source); the sum uses the raw sources
+ggml_tensor * llama_model_zonos3::graph::attn_res_route(ggml_tensor * query, int il) {
+ const int64_t n_src = bank->ne[1];
+
+ ggml_tensor * scores = ggml_rms_norm(ctx0, bank, hparams.f_norm_rms_eps);
+ scores = ggml_mul(ctx0, scores, query);
+ scores = ggml_sum_rows(ctx0, scores); // [1, n_src, n_tokens]
+ scores = ggml_reshape_2d(ctx0, scores, n_src, n_tokens);
+
+ ggml_tensor * probs = ggml_soft_max(ctx0, scores);
+ cb(probs, "attn_res_probs", il);
+
+ return ggml_dsv4_hc_pre(ctx0, bank, probs);
+}
+
+ggml_tensor * llama_model_zonos3::graph::build_layer_attn(
+ llm_graph_input_attn_kv * inp,
+ ggml_tensor * cur,
+ ggml_tensor * inp_pos,
+ int il) {
+ const auto & layer = model.layers[il];
+
+ const int64_t n_embd_head = hparams.n_embd_head_v();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+
+ // the reference QK norm has no weight and a fixed eps
+ const float qk_norm_eps = 1e-6f;
+
+ auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il);
+
+ ggml_tensor * gate = build_lora_mm(layer.wqkv_gate, cur);
+ gate = ggml_sigmoid(ctx0, ggml_reshape_3d(ctx0, gate, 1, n_head, n_tokens));
+ cb(gate, "attn_gate", il);
+
+ Qcur = ggml_rms_norm(ctx0, Qcur, qk_norm_eps);
+ Qcur = ggml_mul(ctx0, Qcur, layer.attn_q_norm);
+ Kcur = ggml_rms_norm(ctx0, Kcur, qk_norm_eps);
+ cb(Qcur, "Qcur_normed", il);
+ cb(Kcur, "Kcur_normed", il);
+
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ cur = build_attn(inp,
+ nullptr, nullptr, nullptr,
+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
+ cb(cur, "attn_pregate", il);
+
+ cur = ggml_reshape_3d(ctx0, cur, n_embd_head, n_head, n_tokens);
+ cur = ggml_mul(ctx0, cur, gate);
+ cur = ggml_reshape_2d(ctx0, cur, n_embd_head * n_head, n_tokens);
+ cb(cur, "attn_gated", il);
+
+ return build_lora_mm(layer.wo, cur, layer.wo_s);
+}
+
+// copied from qwen35 build_layer_attn_linear, with the reference L2 norm eps
+ggml_tensor * llama_model_zonos3::graph::build_layer_gdn(
+ llm_graph_input_rs * inp,
+ ggml_tensor * cur,
+ int il) {
+ const auto & layer = model.layers[il];
+ const auto * mctx_cur = inp->mctx;
+
+ const int64_t d_inner = hparams.ssm_d_inner;
+ const int64_t n_seqs = ubatch.n_seqs;
+ const int64_t head_k_dim = hparams.ssm_d_state;
+ const int64_t num_k_heads = hparams.ssm_n_group;
+ const int64_t num_v_heads = hparams.ssm_dt_rank;
+ const int64_t head_v_dim = d_inner / num_v_heads;
+ const int64_t n_seq_tokens = ubatch.n_seq_tokens;
+
+ GGML_ASSERT(n_seqs != 0);
+ GGML_ASSERT(ubatch.equal_seqs());
+ GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);
+ GGML_ASSERT(num_k_heads == num_v_heads);
+
+ ggml_tensor * qkv_mixed = build_lora_mm(layer.wqkv, cur, layer.wqkv_s);
+ qkv_mixed = ggml_reshape_3d(ctx0, qkv_mixed, qkv_mixed->ne[0], n_seq_tokens, n_seqs);
+ cb(qkv_mixed, "linear_attn_qkv_mixed", il);
+
+ ggml_tensor * z = build_lora_mm(layer.wqkv_gate, cur, layer.wqkv_gate_s);
+ cb(z, "z", il);
+
+ ggml_tensor * beta = build_lora_mm(layer.ssm_beta, cur, layer.ssm_beta_s);
+ beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs);
+ beta = ggml_sigmoid(ctx0, beta);
+ cb(beta, "beta_sigmoid", il);
+
+ ggml_tensor * alpha = build_lora_mm(layer.ssm_alpha, cur, layer.ssm_alpha_s);
+ alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);
+
+ ggml_tensor * alpha_softplus = ggml_softplus(ctx0, ggml_add(ctx0, alpha, layer.ssm_dt));
+ ggml_tensor * gate = ggml_mul(ctx0, alpha_softplus, layer.ssm_a); // -A_log.exp() * softplus
+ gate = ggml_reshape_4d(ctx0, gate, 1, num_v_heads, n_seq_tokens, n_seqs);
+ cb(gate, "gate", il);
+
+ ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);
+ ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il);
+
+ ggml_tensor * conv_kernel = layer.ssm_conv1d;
+ const int64_t conv_kernel_size = conv_kernel->ne[0];
+ const int64_t conv_channels = d_inner + 2 * num_k_heads * head_k_dim;
+
+ ggml_tensor * conv_input = build_conv_state(inp, conv_states_all, qkv_mixed, conv_kernel_size, conv_channels, il);
+
+ ggml_tensor * state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs);
+ state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs);
+
+ ggml_tensor * conv_out = ggml_silu(ctx0, ggml_ssm_conv(ctx0, conv_input, conv_kernel));
+ cb(conv_out, "conv_output_silu", il);
+
+ const int64_t qkv_dim = head_k_dim * num_k_heads * 2 + head_v_dim * num_v_heads;
+ const int64_t nb1_qkv = ggml_row_size(conv_out->type, qkv_dim);
+
+ ggml_tensor * q_conv = ggml_view_4d(ctx0, conv_out, head_k_dim, num_k_heads, n_seq_tokens, n_seqs,
+ ggml_row_size(conv_out->type, head_k_dim), nb1_qkv, nb1_qkv * n_seq_tokens, 0);
+ ggml_tensor * k_conv = ggml_view_4d(ctx0, conv_out, head_k_dim, num_k_heads, n_seq_tokens, n_seqs,
+ ggml_row_size(conv_out->type, head_k_dim), nb1_qkv, nb1_qkv * n_seq_tokens,
+ ggml_row_size(conv_out->type, head_k_dim * num_k_heads));
+ ggml_tensor * v_conv = ggml_view_4d(ctx0, conv_out, head_v_dim, num_v_heads, n_seq_tokens, n_seqs,
+ ggml_row_size(conv_out->type, head_v_dim), nb1_qkv, nb1_qkv * n_seq_tokens,
+ ggml_row_size(conv_out->type, 2 * head_k_dim * num_k_heads));
+
+ // the reference L2 norm is x * rsqrt(sum(x^2) + 1e-6)
+ q_conv = build_gdn_l2_norm(ctx0, q_conv, 1e-6f);
+ k_conv = build_gdn_l2_norm(ctx0, k_conv, 1e-6f);
+ cb(q_conv, "q_conv_predelta", il);
+ cb(k_conv, "k_conv_predelta", il);
+ cb(v_conv, "v_conv_predelta", il);
+
+ ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il);
+
+ // gated norm: rms_norm(output) * silu(z), per head
+ ggml_tensor * z_4d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);
+ output = build_norm(output, layer.ssm_norm, nullptr, LLM_NORM_RMS, il);
+ output = ggml_mul(ctx0, output, ggml_silu(ctx0, z_4d));
+
+ output = ggml_reshape_2d(ctx0, output, head_v_dim * num_v_heads, n_seq_tokens * n_seqs);
+ cb(output, "linear_attn_gated", il);
+
+ return build_lora_mm(layer.ssm_out, output, layer.ssm_out_s);
+}
+
+ggml_tensor * llama_model_zonos3::graph::build_layer_ffn(ggml_tensor * cur, int il) {
+ const auto & layer = model.layers[il];
+
+ if (layer.ffn_gate_inp == nullptr) {
+ cur = build_ffn(cur,
+ layer.ffn_up, nullptr, layer.ffn_up_s,
+ layer.ffn_gate, nullptr, layer.ffn_gate_s,
+ layer.ffn_down, nullptr, layer.ffn_down_s,
+ nullptr,
+ LLM_FFN_SILU, LLM_FFN_PAR, il);
+ cb(cur, "ffn_out", il);
+ return cur;
+ }
+
+ // the router reads the full-width input, the experts the latent one
+ ggml_tensor * logits = build_lora_mm(layer.ffn_gate_inp, ggml_rms_norm(ctx0, cur, hparams.f_norm_rms_eps));
+ cb(logits, "ffn_moe_logits", il);
+
+ ggml_tensor * latent = build_lora_mm(layer.ffn_latent_down, cur);
+ cb(latent, "ffn_latent_in", il);
+
+ cur = build_moe_ffn(latent,
+ nullptr,
+ layer.ffn_up_exps,
+ layer.ffn_gate_exps,
+ layer.ffn_down_exps,
+ layer.ffn_exp_probs_b,
+ n_expert, n_expert_used,
+ LLM_FFN_SILU, hparams.expert_weights_norm,
+ hparams.expert_weights_scale,
+ (llama_expert_gating_func_type) hparams.expert_gating_func,
+ il,
+ logits);
+ cb(cur, "ffn_moe_out", il);
+
+ cur = build_lora_mm(layer.ffn_latent_up, cur);
+ cb(cur, "ffn_out", il);
+ return cur;
+}
diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv
index 89cafb247..efb8718e7 100644
--- a/tests/fusion/MTL.csv
+++ b/tests/fusion/MTL.csv
@@ -348,3 +348,8 @@ xing4_0 ,0 ,any ,MUL+ADD , 2
xing4_0 ,0 ,any ,RMS_NORM+MUL , 9
xing4_0 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1
xverse ,0 ,any ,RMS_NORM+MUL , 5
+zonos3 ,0 ,any ,GATED_DELTA_NET+CPY , 1
+zonos3 ,0 ,any ,MUL+ADD , 2
+zonos3 ,0 ,any ,RMS_NORM+MUL , 12
+zonos3 ,0 ,any ,RMS_NORM+SCALE , 2
+zonos3 ,0 ,any ,SSM_CONV+UNARY , 1
diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp
index d52f1af9c..3f86d1535 100644
--- a/tests/test-jinja.cpp
+++ b/tests/test-jinja.cpp
@@ -3,6 +3,9 @@
#include <random>
#include <cstdlib>
#include <future>
+#include <array>
+#include <fstream>
+#include <iterator>
#include "chat-tts-norm.h"
#include "json.h"
@@ -42,6 +45,7 @@ static void test_caps(testing & t);
static void test_string_parts(testing & t);
static void test_fuzzing(testing & t);
static void test_tts_norm_patterns(testing & t);
+static void test_tts_template_zonos3(testing & t);
static void test_shared_render(testing & t);
@@ -81,6 +85,7 @@ int main(int argc, char *argv[]) {
t.test("array methods", test_array_methods);
t.test("object methods", test_object_methods);
t.test("tts norm patterns", test_tts_norm_patterns);
+ t.test("tts template zonos3", test_tts_template_zonos3);
if (!g_python_mode) {
t.test("shared render", test_shared_render);
t.test("hasher", test_hasher);
@@ -2979,6 +2984,72 @@ static void test_tts_norm_patterns(testing & t) {
}
}
+// the ZONOS3 "tts" template spells out numbers in en, fr, de, es, it, pt, ja and zh, other languages pass through
+static void test_tts_template_zonos3(testing & t) {
+ std::string path = __FILE__;
+ const size_t pos = path.find_last_of("/\\");
+ path = (pos == std::string::npos ? std::string(".") : path.substr(0, pos)) + "/../models/templates/Zyphra-ZONOS3-Mini-tts.jinja";
+ std::ifstream fs(path, std::ios_base::binary);
+ if (!t.assert_true("template file is readable: " + path, fs.is_open())) {
+ return;
+ }
+ const std::string tmpl((std::istreambuf_iterator<char>(fs)), std::istreambuf_iterator<char>());
+
+ const std::vector<std::array<std::string, 3>> cases = {
+ { "en", "I have 3 cats and 21 dogs.", "I have three cats and twenty-one dogs." },
+ { "en", "1,234,567 people, 1,2345 and 12345", "one million two hundred thirty-four thousand five hundred sixty-seven people, one,two thousand three hundred forty-five and twelve thousand three hundred forty-five" },
+ { "en", "$1, $0.99, $5.32 and $2.5 billion", "one dollar, ninety-nine cents, five dollars and thirty-two cents and two point five billion dollars" },
+ { "en", "\xe2\x82\xac" "20 and \xc2\xa3" "3.50", "twenty euros and three pounds and fifty pence" },
+ { "en", "3.5% then 10%", "three point five percent then ten percent" },
+ { "en", "1st, 22nd, 103rd, 12th", "first, twenty-second, one hundred third, twelfth" },
+ { "en", "in 1990, since 1905, by 2005, in 2000, from 1900, the 1990s, 1234 people",
+ "in nineteen ninety, since nineteen oh five, by two thousand five, in two thousand, from nineteen hundred, the nineteen nineties, one thousand two hundred thirty-four people" },
+ { "en", "at 3:30, 10:05 or 7:00, not 25:00", "at three thirty, ten oh five or seven o'clock, not twenty-five:zero zero" },
+ { "en", "pi is 3.14, it hit -5 today, range 10-20", "pi is three point one four, it hit minus five today, range ten-twenty" },
+ { "en", "agent 007 ran 5km with an mp3", "agent zero zero seven ran five km with an mp three" },
+ { "en", "caf\xc3\xa9 42.", "caf\xc3\xa9 forty-two." },
+ { "", "no language 42", "no language forty-two" },
+ { "en", "no digits here", "no digits here" },
+ { "ko", "3\xea\xb0\x9c", "3\xea\xb0\x9c" },
+ // other languages
+ { "fr_fr", "J'ai 21 chats, 71 chiens, 80 oiseaux et 91 souris.",
+ "J'ai vingt et un chats, soixante et onze chiens, quatre-vingts oiseaux et quatre-vingt-onze souris." },
+ { "fr", "200 ans, 2 345 678 habitants et 80 000 euros.",
+ "deux cents ans, deux millions trois cent quarante-cinq mille six cent soixante-dix-huit habitants et quatre-vingt mille euros." },
+ { "fr", "Ça coûte 5,32 €, hausse de 12,05 %.",
+ "Ça coûte cinq euros et trente-deux centimes, hausse de douze virgule zéro cinq pour cent." },
+ { "fr", "Le 1er mai, la 1re fois, le 21e siècle, à 15h30, à 1h ou à 21:05, -5 degrés.",
+ "Le premier mai, la première fois, le vingt et unième siècle, à quinze heures trente, à une heure ou à vingt et une heures cinq, moins cinq degrés." },
+ { "de", "21 Katzen, 101 Hunde, 1.234 Bücher, 1.000.000 Einwohner, im Jahr 1990.",
+ "einundzwanzig Katzen, einhunderteins Hunde, eintausendzweihundertvierunddreißig Bücher, eine Million Einwohner, im Jahr neunzehnhundertneunzig." },
+ { "fr", "2 000 000 €, 3 000 000 $ et 2 500 000 €.",
+ "deux millions d'euros, trois millions de dollars et deux millions cinq cent mille euros." },
+ { "es", "2.000.000 € y 1.000.000 $.", "dos millones de euros y un millón de dólares." },
+ { "de", "Im Jahr 1990 waren 1500 Leute da.", "Im Jahr neunzehnhundertneunzig waren eintausendfünfhundert Leute da." },
+ { "de", "5,32 € oder 1 €, plus 3,5 % um 15:30 Uhr, am 3.10.2024.",
+ "fünf Euro und zweiunddreißig Cent oder ein Euro, plus drei Komma fünf Prozent um fünfzehn Uhr dreißig, am drei zehn zweitausendvierundzwanzig." },
+ { "es", "100, 101, 21.000 personas, 2.500.000 habitantes, 21 €, a las 1:00, el 3.º y la 2ª vez.",
+ "cien, ciento uno, veintiún mil personas, dos millones quinientos mil habitantes, veintiún euros, a las una en punto, el tercero y la segunda vez." },
+ { "it", "21, 23, 28, 180, 3.003, 1.000.000 di euro, 5,32 €, all'1:00, l'11º e il 23º.",
+ "ventuno, ventitré, ventotto, centottanta, tremilatré, un milione di euro, cinque euro e trentadue centesimi, all'una, l'undicesimo e il ventitreesimo." },
+ { "pt_br", "101, 1.200, 1.234, R$ 5,32, às 2:00, o 1º e a 2ª vez.",
+ "cento e um, mil e duzentos, mil duzentos e trinta e quatro, cinco reais e trinta e dois centavos, às duas horas, o primeiro e a segunda vez." },
+ { "ja", "3個、1000円、10000人、123456789人、1.5キロ、30%、$5、午後3:30、-5度、2024年。",
+ "三個、千円、一万人、一億二千三百四十五万六千七百八十九人、一点五キロ、三十パーセント、五ドル、午後三時三十分、マイナス五度、二千二十四年。" },
+ { "cmn", "10005人，115人，2000人，20000人，316，2:30，2002年，-5度，3.5%，$23。",
+ "一万零五人，一百一十五人，两千人，两万人，三百一十六，两点三十分，二零零二年，负五度，百分之三点五，二十三美元。" },
+ };
+ for (const auto & c : cases) {
+ // same variables as common_chat_get_tts_prompt()
+ const json vars = {
+ {"messages", json::array({{{"role", "user"}, {"content", c[1]}}})},
+ {"language", c[0]},
+ {"voice", ""},
+ };
+ test_template(t, c[0] + ": " + c[1], tmpl, vars, c[2], true);
+ }
+}
+
static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm) {
t.test(name, [&tmpl, &expect, tts_norm](testing & t) {
std::string error;
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 32e77e883..147d5b536 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -785,6 +785,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const
ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, std::vector<uint32_t>{n_ff, n_ff / 2, n_ff, n_ff});
}
+ if (arch == LLM_ARCH_ZONOS3) {
+ ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd / 2);
+ ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, n_ff);
+ }
+
ms.add_kv(LLM_KV_POSNET_EMBEDDING_LENGTH, n_embd);
ms.add_kv(LLM_KV_POSNET_BLOCK_COUNT, n_layer);
ms.add_kv(LLM_KV_CONVNEXT_EMBEDDING_LENGTH, n_embd);
@@ -793,7 +798,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const
ms.add_kv(LLM_KV_XIELU_ALPHA_P, 1.0f);
ms.add_kv(LLM_KV_XIELU_BETA, 1.0f);
ms.add_kv(LLM_KV_XIELU_EPS, 1.0e-7f);
- ms.add_kv(LLM_KV_SSM_INNER_SIZE, arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_CLOVER_MOE ? 256 : 2*n_embd);
+ ms.add_kv(LLM_KV_SSM_INNER_SIZE, arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_CLOVER_MOE || arch == LLM_ARCH_ZONOS3 ? 256 : 2*n_embd);
ms.add_kv(LLM_KV_SSM_CONV_KERNEL, uint32_t(4));
ms.add_kv(LLM_KV_SSM_STATE_SIZE, uint32_t(128));
ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head);
@@ -1077,6 +1082,7 @@ static bool moe_mandatory(const llm_arch arch) {
case LLM_ARCH_MUSE_SPARK:
case LLM_ARCH_KOLIBRI1:
case LLM_ARCH_METIS16:
+ case LLM_ARCH_ZONOS3:
case LLM_ARCH_INKLING2:
return true;
default:
diff --git a/tests/test-server-component.cpp b/tests/test-server-component.cpp
index 580203be8..2ef8b04f5 100644
--- a/tests/test-server-component.cpp
+++ b/tests/test-server-component.cpp
@@ -825,9 +825,9 @@ static void test_speech_convert(testing & t) {
t.assert_true ("audio streams", out.at("stream").get<bool>());
t.assert_true ("rejects sse", rejected("stream_format", "sse"));
- // speed
- t.assert_true ("speed 1.0", !rejected("speed", 1.0));
- t.assert_true ("rejects speed 1.5", rejected("speed", 1.5));
+ // speed is forwarded, the model decides whether it is supported
+ out = convert_speech_to_tts(with("speed", 1.5));
+ t.assert_equal("speed forwarded", 1.5, out.at("speed").get<double>());
// instructions are ignored
t.assert_true ("instructions", !rejected("instructions", "whisper"));
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index fd54e2e72..32a17ef61 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -83,6 +83,8 @@ add_library(mtmd
models/qwen3tts-spkenc.cpp
models/qwen3tts-gen.cpp
models/drama3-gen.cpp
+ models/zonos3-gen.cpp
+ models/zonos3-spkenc.cpp
models/pockettts-seanet.cpp
models/pockettts-spkenc.cpp
models/pockettts-gen.cpp
diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md
index 8feb7a647..b6ed5a577 100644
--- a/tools/mtmd/README-dev.md
+++ b/tools/mtmd/README-dev.md
@@ -103,6 +103,7 @@ Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system i
- In most cases, you can hard-code model configs in the model graph class, or in `mtmd_hparams`
- If some values need to be exposed to the `mtmd_helper` layer, hard-code them in `mtmd_helper` and distinguish by pipeline and `mtmd_gen_audio_info::model_variant` if necessary
- Do NOT add new GGUF metadata or new fields to `mtmd_gen_audio_info` unless you can prove that you absolutely need them
+ - If the model requires special preprocessing of the input text prompt (prompt format, text normalization), put it in `tokenizer.chat_template.tts` as a jinja program. It receives the text as `messages[0].content`, plus `language` and `voice`. See `conversion/lfm3.py` (prompt format) and `models/templates/Zyphra-ZONOS3-Mini-tts.jinja` (text normalization) as examples. For text normalization, use `tts_norm_scan()` / `tts_norm_sub()` (see `docs/tts-norm-patterns.md`) and add test cases to `tests/test-jinja.cpp`, which also checks them against the python reference with `-py`
4. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this:
- 10-20% changes is to add new backbone (text) model and conversion
- 60% changes inside `mtmd-helper-gen.cpp`
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 58a83378a..05494c71f 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -341,6 +341,27 @@ struct mtmd_graph_mimo_audio : mtmd_graph {
ggml_cgraph * build() override;
};
+// zonos3 speaker encoder: MECT-B2 + ASTP, then the speaker projection of the backbone
+struct mtmd_graph_zonos3_spkenc : mtmd_graph {
+ mtmd_graph_zonos3_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
+ ggml_cgraph * build() override;
+
+ ggml_tensor * W(const std::string & name) const;
+ bool has(const std::string & name) const;
+ ggml_tensor * chan(ggml_tensor * v) const;
+ ggml_tensor * bn(ggml_tensor * x, const std::string & p) const;
+ ggml_tensor * conv(ggml_tensor * x, const std::string & p, int s_t, int s_f, bool bias, int pad) const;
+ ggml_tensor * grouped3x3(ggml_tensor * x, const std::string & p, int s_t, int s_f) const;
+ ggml_tensor * res_block(ggml_tensor * x, const std::string & q, int s_f, int s_t) const;
+ ggml_tensor * linear(ggml_tensor * x, const std::string & p) const;
+ ggml_tensor * layer_norm(ggml_tensor * x, const std::string & p) const;
+ ggml_tensor * flatten(ggml_tensor * x) const;
+ ggml_tensor * unflatten(ggml_tensor * x, int64_t F, int64_t C) const;
+ ggml_tensor * attention(ggml_tensor * h, const std::string & p) const;
+ ggml_tensor * moe_ffn(ggml_tensor * h, const std::string & p) const;
+ ggml_tensor * pos_conv(ggml_tensor * h, const std::string & p) const;
+};
+
struct mtmd_graph_qwen3tts_spkenc : mtmd_graph {
mtmd_graph_qwen3tts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
@@ -536,6 +557,47 @@ struct mtmd_graph_drama3_gen : mtmd_graph {
ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos);
};
+//
+// zonos3 generation:
+// GEN_CODE = 10 codebook heads, or the backbone row of a sampled frame
+// GEN_WAV = ZAC codec decoder up to the iSTFT spectra
+//
+struct mtmd_graph_zonos3_gen : mtmd_graph {
+ mtmd_graph_zonos3_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, bool all_known, int n_frames)
+ : mtmd_graph(ctx, img), gen_process(gen_process), all_known(all_known), n_frames(n_frames) {}
+ ggml_cgraph * build() override;
+
+ static constexpr int n_codebooks = 10;
+ static constexpr int n_cb_vocab = 4096;
+ static constexpr float softcap = 15.0f;
+
+ // ZAC decoder, fixed by the codec config
+ static constexpr int n_stages = 5;
+ static constexpr int stage_ctx[n_stages] = { 150, 300, 480, 480, 480 }; // attention window
+ static constexpr int head_dim = 64;
+ static constexpr float ln_eps = 1e-5f;
+ static constexpr float convnext_eps = 1e-6f;
+ static constexpr int hop = 184; // at 44160 Hz
+ static constexpr int n_fft[2] = { 736, 2208 };
+ static constexpr int rs_orig = 736; // 44160 -> 44100 Hz
+ static constexpr int rs_new = 735;
+ static constexpr int samples_per_frame = 2940;
+
+ mtmd_subctx_gen_process gen_process;
+ bool all_known; // GEN_CODE: only the backbone row
+ int n_frames; // GEN_WAV
+
+ std::map<std::string, ggml_tensor *> state_in;
+ std::vector<std::pair<std::string, ggml_tensor *>> state_out;
+
+ ggml_cgraph * build_gen_code();
+ ggml_cgraph * build_gen_wav();
+ ggml_tensor * convnext(ggml_tensor * x, const mtmd_zonos3_wav::convnext & b, const std::string & state_name);
+ ggml_tensor * moss_layer(ggml_tensor * x, const mtmd_zonos3_wav::layer & l, int s, int il, ggml_tensor * base);
+};
+
+std::vector<c2w_state_slot> list_zonos3_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
+
struct mtmd_graph_soprano : mtmd_graph {
mtmd_graph_soprano(mtmd_subcontext * ctx, const mtmd_image_f32 & img, int n_frames) : mtmd_graph(ctx, img), n_frames(n_frames) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/models/zonos3-gen.cpp b/tools/mtmd/models/zonos3-gen.cpp
new file mode 100644
index 000000000..d306f9858
--- /dev/null
+++ b/tools/mtmd/models/zonos3-gen.cpp
@@ -0,0 +1,207 @@
+#include "models.h"
+
+// GEN_CODE: hidden state -> logits of the 10 codebooks (out_feats), or codes -> backbone row (graph output)
+ggml_cgraph * mtmd_graph_zonos3_gen::build_gen_code() {
+ ggml_tensor * embd_w = model.gen_code_out_embd_w; // [n_embd, n_vocab * n_codebooks]
+ const int64_t n_vocab = embd_w->ne[1] / n_codebooks;
+
+ if (!all_known) {
+ ggml_tensor * h = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd);
+ ggml_set_name(h, "inp_raw");
+ ggml_set_input(h);
+
+ ggml_tensor * logits = ggml_mul_mat(ctx0, model.gen_code_head_w, h);
+ logits = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, logits, 1.0f / softcap)), softcap);
+ logits = ggml_reshape_2d(ctx0, logits, n_vocab, n_codebooks);
+ ggml_set_name(logits, "out_feats");
+ ggml_set_output(logits);
+ ggml_build_forward_expand(gf, logits);
+ }
+
+ ggml_tensor * inp_codes = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_codebooks);
+ ggml_set_name(inp_codes, "inp_codes");
+ ggml_set_input(inp_codes);
+
+ ggml_tensor * embd = ggml_get_rows(ctx0, embd_w, inp_codes); // [n_embd, n_codebooks]
+ embd = ggml_sum_rows(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, embd)));
+ embd = ggml_reshape_2d(ctx0, embd, n_mmproj_embd, 1);
+ cb(embd, "gen_embd", -1);
+ ggml_build_forward_expand(gf, embd);
+
+ return gf;
+}
+
+std::vector<c2w_state_slot> list_zonos3_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
+ GGML_UNUSED(hparams);
+ const auto & z = model.zonos3;
+ std::vector<c2w_state_slot> slots;
+ slots.push_back({"pos", 1, 1});
+ for (size_t i = 0; i < z.sem.size(); i++) {
+ slots.push_back({"sem_conv_" + std::to_string(i), z.sem[i].dw_w->ne[0] - 1, z.sem[i].dw_w->ne[1]});
+ }
+ for (size_t s = 0; s < z.stages.size(); s++) {
+ const int64_t d = z.stages[s].layers.empty() ? 0 : z.stages[s].layers[0].wo_w->ne[1];
+ for (size_t il = 0; il < z.stages[s].layers.size(); il++) {
+ const std::string sfx = std::to_string(s) + "_" + std::to_string(il);
+ slots.push_back({"tfm_k_" + sfx, d, mtmd_graph_zonos3_gen::stage_ctx[s] - 1});
+ slots.push_back({"tfm_v_" + sfx, d, mtmd_graph_zonos3_gen::stage_ctx[s] - 1});
+ }
+ }
+ for (size_t i = 0; i < z.head.size(); i++) {
+ slots.push_back({"head_conv_" + std::to_string(i), z.head[i].dw_w->ne[0] - 1, z.head[i].dw_w->ne[1]});
+ }
+ return slots;
+}
+
+static ggml_tensor * zonos3_layer_norm(ggml_context * ctx, ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, float eps) {
+ return ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, eps), w), b);
+}
+
+// causal ConvNeXt block, the conv history is kept in the state
+ggml_tensor * mtmd_graph_zonos3_gen::convnext(ggml_tensor * x, const mtmd_zonos3_wav::convnext & b, const std::string & state_name) {
+ const int64_t C = x->ne[0];
+ const int64_t T = x->ne[1];
+ const int64_t K = b.dw_w->ne[0];
+
+ ggml_tensor * hist = state_in.at(state_name); // [K - 1, C]
+ ggml_tensor * xt = ggml_concat(ctx0, hist, ggml_cont(ctx0, ggml_transpose(ctx0, x)), 0); // [K - 1 + T, C]
+ state_out.push_back({state_name, ggml_cont(ctx0, ggml_view_2d(ctx0, xt, K - 1, C, xt->nb[1], (size_t) T * xt->nb[0]))});
+
+ ggml_tensor * h = ggml_ssm_conv(ctx0, ggml_reshape_3d(ctx0, xt, K - 1 + T, C, 1), b.dw_w); // [C, T, 1]
+ h = ggml_add(ctx0, ggml_reshape_2d(ctx0, h, C, T), b.dw_b);
+ h = zonos3_layer_norm(ctx0, h, b.norm_w, b.norm_b, convnext_eps);
+ h = ggml_gelu_erf(ctx0, ggml_add(ctx0, ggml_mul_mat(ctx0, b.pw1_w, h), b.pw1_b));
+ h = ggml_add(ctx0, ggml_mul_mat(ctx0, b.pw2_w, h), b.pw2_b);
+ return ggml_add(ctx0, x, h);
+}
+
+// windowed causal MOSS layer, adapted from qwen3tts code2wav tfm_layer_forward()
+ggml_tensor * mtmd_graph_zonos3_gen::moss_layer(ggml_tensor * x, const mtmd_zonos3_wav::layer & l, int s, int il, ggml_tensor * base) {
+ const int64_t d = x->ne[0];
+ const int64_t N = x->ne[1];
+ const int64_t n_head = d / head_dim;
+ const int64_t W = stage_ctx[s];
+ const int64_t prefix = W - 1;
+ const int64_t n_kv = prefix + N;
+ const std::string sfx = std::to_string(s) + "_" + std::to_string(il);
+
+ ggml_tensor * h = zonos3_layer_norm(ctx0, x, l.norm1_w, l.norm1_b, ln_eps);
+ ggml_tensor * qkv = ggml_mul_mat(ctx0, l.qkv_w, h); // [3d, N]
+ auto section = [&](int i) {
+ return ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, head_dim, n_head, N, ggml_row_size(qkv->type, head_dim), qkv->nb[1],
+ ggml_row_size(qkv->type, i * d)));
+ };
+
+ ggml_tensor * pos = ggml_cast(ctx0, ggml_add(ctx0, ggml_arange(ctx0, 0.0f, (float) N, 1.0f), base), GGML_TYPE_I32);
+ ggml_tensor * q = ggml_rope_ext(ctx0, section(0), pos, nullptr, head_dim, GGML_ROPE_TYPE_NORMAL, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+ ggml_tensor * k = ggml_rope_ext(ctx0, section(1), pos, nullptr, head_dim, GGML_ROPE_TYPE_NORMAL, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+ ggml_tensor * v = section(2);
+
+ ggml_tensor * k_full = ggml_concat(ctx0, state_in.at("tfm_k_" + sfx), ggml_reshape_2d(ctx0, k, d, N), 1); // [d, n_kv]
+ ggml_tensor * v_full = ggml_concat(ctx0, state_in.at("tfm_v_" + sfx), ggml_reshape_2d(ctx0, v, d, N), 1);
+ state_out.push_back({"tfm_k_" + sfx, ggml_cont(ctx0, ggml_view_2d(ctx0, k_full, d, prefix, k_full->nb[1], (size_t) N * k_full->nb[1]))});
+ state_out.push_back({"tfm_v_" + sfx, ggml_cont(ctx0, ggml_view_2d(ctx0, v_full, d, prefix, v_full->nb[1], (size_t) N * v_full->nb[1]))});
+
+ // key j is visible to query i iff 0 <= (prefix + i) - j < W and j >= prefix - base (cold start)
+ ggml_tensor * pos_k = ggml_reshape_2d(ctx0, ggml_arange(ctx0, 0.0f, (float) n_kv, 1.0f), n_kv, 1);
+ ggml_tensor * pos_q = ggml_reshape_2d(ctx0, ggml_arange(ctx0, (float) prefix, (float) (prefix + N), 1.0f), 1, N);
+ ggml_tensor * diff = ggml_sub(ctx0, ggml_repeat_4d(ctx0, pos_q, n_kv, N, 1, 1), pos_k); // [n_kv, N]
+ ggml_tensor * keep = ggml_mul(ctx0, ggml_step(ctx0, ggml_scale_bias(ctx0, diff, 1.0f, 0.5f)),
+ ggml_step(ctx0, ggml_scale_bias(ctx0, diff, -1.0f, (float) W - 0.5f)));
+ keep = ggml_mul(ctx0, keep, ggml_step(ctx0, ggml_scale_bias(ctx0, ggml_add(ctx0, pos_k, base), 1.0f, 0.5f - (float) prefix)));
+ ggml_tensor * mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), n_kv, N, 1, 1); // 0 = keep, -inf = masked
+ if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) {
+ mask = ggml_cast(ctx0, mask, GGML_TYPE_F16);
+ }
+
+ ggml_tensor * attn = build_attn(l.wo_w, nullptr,
+ ggml_reshape_4d(ctx0, q, head_dim, n_head, N, 1),
+ ggml_reshape_4d(ctx0, k_full, head_dim, n_head, n_kv, 1),
+ ggml_reshape_4d(ctx0, v_full, head_dim, n_head, n_kv, 1),
+ mask, 1.0f / sqrtf((float) head_dim), il);
+ x = ggml_add(ctx0, x, ggml_mul(ctx0, attn, l.attn_scale));
+
+ h = zonos3_layer_norm(ctx0, x, l.norm2_w, l.norm2_b, ln_eps);
+ h = ggml_mul_mat(ctx0, l.ffn2_w, ggml_gelu_erf(ctx0, ggml_mul_mat(ctx0, l.ffn1_w, h)));
+ return ggml_add(ctx0, x, ggml_mul(ctx0, h, l.ffn_scale));
+}
+
+ggml_cgraph * mtmd_graph_zonos3_gen::build_gen_wav() {
+ const auto & z = model.zonos3;
+ const int64_t T = n_frames;
+
+ for (const auto & slot : list_zonos3_gen_state_slots(hparams, model)) {
+ ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
+ ggml_set_name(t, ("state_in_" + slot.name).c_str());
+ ggml_set_input(t);
+ state_in[slot.name] = t;
+ }
+
+ // codebook-major, residual codes are rows of the merged table
+ ggml_tensor * inp_codes = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, T * n_codebooks);
+ ggml_set_name(inp_codes, "inp_codes");
+ ggml_set_input(inp_codes);
+ auto codes_of = [&](int c) {
+ return ggml_view_1d(ctx0, inp_codes, T, (size_t) c * T * ggml_element_size(inp_codes));
+ };
+
+ // 1 for a known residual code, 0 for a missing one (early decode of a frame), codebook-major
+ ggml_tensor * inp_keep = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, T * (n_codebooks - 1));
+ ggml_set_name(inp_keep, "inp_keep");
+ ggml_set_input(inp_keep);
+
+ ggml_tensor * x = ggml_add(ctx0, ggml_get_rows(ctx0, z.sem_table, codes_of(0)), z.sem_bias); // [semantic_dim, T]
+ for (size_t i = 0; i < z.sem.size(); i++) {
+ x = convnext(x, z.sem[i], "sem_conv_" + std::to_string(i));
+ }
+ x = ggml_add(ctx0, ggml_mul_mat(ctx0, z.sem_out_w, x), z.sem_out_b); // [latent, T]
+ for (int c = 1; c < n_codebooks; c++) {
+ ggml_tensor * bias = ggml_view_1d(ctx0, z.res_bias, z.res_bias->ne[0], (size_t) (c - 1) * z.res_bias->nb[1]);
+ ggml_tensor * keep = ggml_view_2d(ctx0, inp_keep, 1, T, ggml_element_size(inp_keep), (size_t) (c - 1) * T * ggml_element_size(inp_keep));
+ x = ggml_add(ctx0, x, ggml_mul(ctx0, ggml_add(ctx0, ggml_get_rows(ctx0, z.res_table, codes_of(c)), bias), keep));
+ }
+ cb(x, "zac_latent", -1);
+
+ ggml_tensor * frames = ggml_reshape_1d(ctx0, state_in.at("pos"), 1);
+ int cpf = 1; // positions per frame
+ for (int s = 0; s < n_stages; s++) {
+ const auto & st = z.stages[s];
+ x = ggml_mul_mat(ctx0, st.in_w, x);
+ ggml_tensor * base = ggml_scale(ctx0, frames, (float) cpf);
+ for (size_t il = 0; il < st.layers.size(); il++) {
+ x = moss_layer(x, st.layers[il], s, (int) il, base);
+ }
+ if (st.out_w) {
+ x = ggml_mul_mat(ctx0, st.out_w, x);
+ }
+ if (s + 1 < n_stages) {
+ // patch up x2: out[ch, 2t + j] = in[2ch + j, t]
+ const int64_t C = x->ne[0], N = x->ne[1];
+ x = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, x, 2, C / 2, N), 1, 0, 2, 3));
+ x = ggml_reshape_2d(ctx0, x, C / 2, 2 * N);
+ cpf *= 2;
+ }
+ cb(x, "zac_stage_out", s);
+ }
+ state_out.push_back({"pos", ggml_scale_bias(ctx0, state_in.at("pos"), 1.0f, (float) T)});
+
+ for (size_t i = 0; i < z.head.size(); i++) {
+ x = convnext(x, z.head[i], "head_conv_" + std::to_string(i));
+ }
+ x = zonos3_layer_norm(ctx0, x, z.head_norm_w, z.head_norm_b, convnext_eps);
+ ggml_tensor * spec = ggml_add(ctx0, ggml_mul_mat(ctx0, z.head_out_w, x), z.head_out_b); // [n_fft0 + 2 + n_fft1 + 2, 16T]
+ ggml_set_name(spec, "out_feats");
+ ggml_set_output(spec);
+ ggml_build_forward_expand(gf, spec);
+
+ for (auto & slot : state_out) {
+ ggml_set_name(slot.second, ("state_out_" + slot.first).c_str());
+ ggml_set_output(slot.second);
+ ggml_build_forward_expand(gf, slot.second);
+ }
+ return gf;
+}
+
+ggml_cgraph * mtmd_graph_zonos3_gen::build() {
+ return gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ? build_gen_wav() : build_gen_code();
+}
diff --git a/tools/mtmd/models/zonos3-spkenc.cpp b/tools/mtmd/models/zonos3-spkenc.cpp
new file mode 100644
index 000000000..94a388cd7
--- /dev/null
+++ b/tools/mtmd/models/zonos3-spkenc.cpp
@@ -0,0 +1,221 @@
+#include "models.h"
+
+// zonos3 speaker encoder: MECT-B2 + ASTP pooling (ownmect_s2), then the speaker projection of the backbone
+// ported from the reference ggml runtime (spk-encoder.cpp)
+
+namespace {
+
+constexpr int kHidden = 64;
+constexpr int kFeat = 2560;
+constexpr int kHeads = 4;
+constexpr int kExperts = 4;
+constexpr int kPosK = 113;
+
+// per block: cnn depth, stride f, stride t
+struct mect_block { int cnn, s_f, s_t; };
+const std::vector<std::vector<mect_block>> & mect_b2() {
+ static const std::vector<std::vector<mect_block>> cfg = {
+ { {3, 1, 1}, {3, 1, 1} },
+ { {3, 2, 1}, {3, 1, 2}, {3, 1, 2}, {3, 1, 2} },
+ { {3, 2, 1}, {3, 1, 2}, {3, 1, 2}, {3, 1, 2}, {3, 1, 2} },
+ { {3, 2, 1}, {2, 1, 2}, {2, 1, 2} },
+ };
+ return cfg;
+}
+
+} // namespace
+
+ggml_tensor * mtmd_graph_zonos3_spkenc::W(const std::string & name) const {
+ return model.zonos3_spk.at("a.spk." + name);
+}
+
+bool mtmd_graph_zonos3_spkenc::has(const std::string & name) const {
+ return model.zonos3_spk.count("a.spk." + name) > 0;
+}
+
+// [C] as a per-channel broadcast over a [T, F, C] activation
+ggml_tensor * mtmd_graph_zonos3_spkenc::chan(ggml_tensor * v) const {
+ return ggml_reshape_4d(ctx0, v, 1, 1, v->ne[0], 1);
+}
+
+ggml_tensor * mtmd_graph_zonos3_spkenc::bn(ggml_tensor * x, const std::string & p) const {
+ return ggml_add(ctx0, ggml_mul(ctx0, x, chan(W(p + ".scale"))), chan(W(p + ".shift")));
+}
+
+ggml_tensor * mtmd_graph_zonos3_spkenc::conv(ggml_tensor * x, const std::string & p, int s_t, int s_f, bool bias, int pad) const {
+ ggml_tensor * y = ggml_conv_2d_direct(ctx0, W(p + ".weight"), x, s_t, s_f, pad, pad, 1, 1);
+ return bias ? ggml_add(ctx0, y, chan(W(p + ".bias"))) : y;
+}
+
+// grouped 3x3 conv as one depthwise conv per input slot of a group
+ggml_tensor * mtmd_graph_zonos3_spkenc::grouped3x3(ggml_tensor * x, const std::string & p, int s_t, int s_f) const {
+ int n_slot = 0;
+ while (has(p + ".w" + std::to_string(n_slot))) {
+ n_slot++;
+ }
+ const int64_t T = x->ne[0];
+ const int64_t F = x->ne[1];
+ const int64_t IC = x->ne[2];
+ const int64_t OC = W(p + ".w0")->ne[3];
+ const int64_t G = IC / n_slot;
+ const int64_t B = OC / G;
+ ggml_tensor * x4 = ggml_reshape_4d(ctx0, x, T, F, n_slot, G);
+ ggml_tensor * y = nullptr;
+ for (int j = 0; j < n_slot; j++) {
+ ggml_tensor * xj = x;
+ if (n_slot > 1 || B > 1) {
+ ggml_tensor * slot = ggml_view_4d(ctx0, x4, T, F, 1, G, x4->nb[1], x4->nb[2], x4->nb[3], (size_t) j * x4->nb[2]);
+ xj = ggml_reshape_3d(ctx0, ggml_repeat_4d(ctx0, slot, T, F, B, G), T, F, OC);
+ }
+ ggml_tensor * t = ggml_conv_2d_dw_direct(ctx0, W(p + ".w" + std::to_string(j)), xj, s_t, s_f, 1, 1, 1, 1);
+ y = y ? ggml_add(ctx0, y, t) : t;
+ }
+ return y;
+}
+
+ggml_tensor * mtmd_graph_zonos3_spkenc::res_block(ggml_tensor * x, const std::string & q, int s_f, int s_t) const {
+ ggml_tensor * t = grouped3x3(x, q + "conv1", s_t, s_f);
+ t = ggml_relu(ctx0, bn(conv(t, q + "pw1", 1, 1, true, 0), q + "bn1"));
+ t = grouped3x3(t, q + "conv2", 1, 1);
+ t = ggml_relu(ctx0, bn(conv(t, q + "pw2", 1, 1, true, 0), q + "bn2"));
+ ggml_tensor * sc = x;
+ if (has(q + "sc.weight")) {
+ sc = bn(conv(x, q + "sc", s_t, s_f, false, 0), q + "sc.bn");
+ }
+ return ggml_add(ctx0, t, sc);
+}
+
+ggml_tensor * mtmd_graph_zonos3_spkenc::linear(ggml_tensor * x, const std::string & p) const {
+ return ggml_add(ctx0, ggml_mul_mat(ctx0, W(p + ".weight"), x), W(p + ".bias"));
+}
+
+ggml_tensor * mtmd_graph_zonos3_spkenc::layer_norm(ggml_tensor * x, const std::string & p) const {
+ return ggml_add(ctx0, ggml_mul(ctx0, ggml_norm(ctx0, x, 1e-6f), W(p + ".weight")), W(p + ".bias"));
+}
+
+// [T, F, C] -> [F*C, T], k = f*C + c
+ggml_tensor * mtmd_graph_zonos3_spkenc::flatten(ggml_tensor * x) const {
+ const int64_t T = x->ne[0], F = x->ne[1], C = x->ne[2];
+ return ggml_reshape_2d(ctx0, ggml_cont(ctx0, ggml_permute(ctx0, x, 2, 1, 0, 3)), F * C, T);
+}
+
+ggml_tensor * mtmd_graph_zonos3_spkenc::unflatten(ggml_tensor * x, int64_t F, int64_t C) const {
+ return ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, x, C, F, x->ne[1]), 2, 1, 0, 3));
+}
+
+ggml_tensor * mtmd_graph_zonos3_spkenc::attention(ggml_tensor * h, const std::string & p) const {
+ const int64_t T = h->ne[1];
+ const int hd = kHidden / kHeads;
+ ggml_tensor * q = ggml_scale(ctx0, linear(h, p + "attn.q"), 1.0f / sqrtf((float) hd));
+ ggml_tensor * k = linear(h, p + "attn.k");
+ ggml_tensor * v = linear(h, p + "attn.v");
+ auto heads = [&](ggml_tensor * z) { // [64, T] -> [hd, T, n_head]
+ return ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, z, hd, kHeads, T), 0, 2, 1, 3));
+ };
+ ggml_tensor * qp = heads(q);
+ ggml_tensor * kp = heads(k);
+ ggml_tensor * vt = ggml_cont(ctx0, ggml_permute(ctx0, heads(v), 1, 0, 2, 3)); // [T, hd, n_head]
+ // queries in blocks, so long clips do not need a T x T score matrix per head
+ ggml_tensor * o = nullptr;
+ for (int64_t a = 0; a < T; a += 512) {
+ const int64_t nq = std::min<int64_t>(512, T - a);
+ ggml_tensor * qb = ggml_view_3d(ctx0, qp, hd, nq, kHeads, qp->nb[1], qp->nb[2], (size_t) a * qp->nb[1]);
+ ggml_tensor * score = ggml_soft_max(ctx0, ggml_mul_mat(ctx0, kp, qb)); // [T, nq, n_head]
+ ggml_tensor * ob = ggml_mul_mat(ctx0, vt, score); // [hd, nq, n_head]
+ ob = ggml_reshape_2d(ctx0, ggml_cont(ctx0, ggml_permute(ctx0, ob, 0, 2, 1, 3)), kHidden, nq);
+ o = o ? ggml_concat(ctx0, o, ob, 1) : ob;
+ }
+ return linear(o, p + "attn.o");
+}
+
+// soft MoE: all experts run, weighted by the router
+ggml_tensor * mtmd_graph_zonos3_spkenc::moe_ffn(ggml_tensor * h, const std::string & p) const {
+ ggml_tensor * r = ggml_soft_max(ctx0, ggml_mul_mat(ctx0, W(p + "moe.gate.weight"), h)); // [n_expert, T]
+ ggml_tensor * e = ggml_gelu(ctx0, ggml_mul_mat(ctx0, W(p + "moe.experts.weight"), h)); // [64 * n_expert, T]
+ ggml_tensor * mix = nullptr;
+ for (int i = 0; i < kExperts; i++) {
+ ggml_tensor * ei = ggml_view_2d(ctx0, e, kHidden, e->ne[1], e->nb[1], (size_t) i * kHidden * ggml_element_size(e));
+ ggml_tensor * ri = ggml_view_2d(ctx0, r, 1, r->ne[1], r->nb[1], (size_t) i * ggml_element_size(r));
+ ggml_tensor * t = ggml_mul(ctx0, ei, ri);
+ mix = mix ? ggml_add(ctx0, mix, t) : t;
+ }
+ return linear(ggml_gelu(ctx0, mix), p + "ffn_out");
+}
+
+ggml_tensor * mtmd_graph_zonos3_spkenc::pos_conv(ggml_tensor * h, const std::string & p) const {
+ const int64_t T = h->ne[1];
+ ggml_tensor * ht = ggml_cont(ctx0, ggml_transpose(ctx0, h)); // [T, 64]
+ ht = ggml_pad_ext(ctx0, ht, kPosK / 2, kPosK / 2, 0, 0, 0, 0, 0, 0); // [T + K - 1, 64]
+ ggml_tensor * acc = ggml_ssm_conv(ctx0, ggml_reshape_3d(ctx0, ht, ht->ne[0], kHidden, 1), W(p + "pos.dw.weight")); // [64, T, 1]
+ acc = ggml_add(ctx0, ggml_reshape_2d(ctx0, acc, kHidden, T), W(p + "pos.dw.bias"));
+ acc = ggml_add(ctx0, ggml_mul(ctx0, acc, W(p + "pos.bn.scale")), W(p + "pos.bn.shift"));
+ acc = ggml_gelu_erf(ctx0, acc);
+ return linear(acc, p + "pos.pw");
+}
+
+ggml_cgraph * mtmd_graph_zonos3_spkenc::build() {
+ const int64_t T = img.nx();
+ const int64_t n_mel = img.ny();
+
+ ggml_tensor * inp = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, T, n_mel, 1, 1);
+ ggml_set_name(inp, "inp_raw");
+ ggml_set_input(inp);
+
+ ggml_tensor * x = bn(conv(inp, "fe.head.conv", 1, 1, true, 1), "fe.head.bn"); // [T, F, C]
+ ggml_tensor * xf = flatten(x);
+ std::vector<ggml_tensor *> feats;
+ const auto & stages = mect_b2();
+ for (size_t s = 0; s < stages.size(); s++) {
+ for (size_t k = 0; k < stages[s].size(); k++) {
+ const auto & b = stages[s][k];
+ const std::string p = "fe.b" + std::to_string(s + 1) + "." + std::to_string(k) + ".";
+ ggml_tensor * y = x;
+ for (int r = 0; r < b.cnn; r++) {
+ y = res_block(y, p + "cnn." + std::to_string(r) + ".", r == 0 ? b.s_f : 1, r == 0 ? b.s_t : 1);
+ }
+ if (has(p + "reduce.weight")) {
+ y = bn(conv(y, p + "reduce", 1, 1, true, 0), p + "reduce.bn");
+ }
+ const int64_t F2 = y->ne[1], C2 = y->ne[2];
+ ggml_tensor * x2 = flatten(y); // [2560, T']
+ ggml_tensor * h = layer_norm(linear(x2, p + "rproj"), p + "rproj.ln");
+ h = ggml_add(ctx0, h, pos_conv(h, p));
+ h = layer_norm(ggml_add(ctx0, h, attention(h, p)), p + "ln1");
+ h = layer_norm(ggml_add(ctx0, h, moe_ffn(h, p)), p + "ln2");
+ ggml_tensor * e = ggml_add(ctx0, linear(h, p + "eproj"), x2);
+ if (b.s_t > 1) {
+ // nearest upsampling back to the input frames
+ const int64_t Tp = e->ne[1];
+ e = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, e, kFeat, 1, Tp), kFeat, b.s_t, Tp, 1);
+ e = ggml_view_2d(ctx0, ggml_reshape_2d(ctx0, e, kFeat, (int64_t) b.s_t * Tp), kFeat, T, e->nb[1], 0);
+ }
+ xf = ggml_add(ctx0, e, xf);
+ x = unflatten(xf, F2, C2);
+ }
+ feats.push_back(xf);
+ }
+
+ ggml_tensor * dense = W("fe.dense_w"); // [2560, n_stage]
+ ggml_tensor * fused = nullptr;
+ for (size_t s = 0; s < feats.size(); s++) {
+ ggml_tensor * t = ggml_mul(ctx0, feats[s], ggml_view_2d(ctx0, dense, kFeat, 1, dense->nb[1], s * dense->nb[1]));
+ fused = fused ? ggml_add(ctx0, fused, t) : t;
+ }
+
+ // ASTP: attention-weighted mean and std over time
+ ggml_tensor * att = linear(ggml_tanh(ctx0, linear(fused, "pool.l1")), "pool.l2"); // [2560, T]
+ ggml_tensor * alpha = ggml_soft_max(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, att))); // [T, 2560]
+ ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, fused)); // [T, 2560]
+ ggml_tensor * mean = ggml_sum_rows(ctx0, ggml_mul(ctx0, alpha, xt)); // [1, 2560]
+ ggml_tensor * sq = ggml_sum_rows(ctx0, ggml_mul(ctx0, alpha, ggml_mul(ctx0, xt, xt)));
+ ggml_tensor * var = ggml_sub(ctx0, sq, ggml_mul(ctx0, mean, mean));
+ ggml_tensor * stdv = ggml_sqrt(ctx0, ggml_clamp(ctx0, var, 1e-7f, 3.0e38f));
+ ggml_tensor * pooled = ggml_concat(ctx0, ggml_reshape_2d(ctx0, mean, kFeat, 1), ggml_reshape_2d(ctx0, stdv, kFeat, 1), 0);
+ pooled = ggml_add(ctx0, ggml_mul(ctx0, pooled, W("bn.scale")), W("bn.shift"));
+ ggml_tensor * spk = ggml_mul_mat(ctx0, W("linear.weight"), pooled); // [192, 1], not normalized
+ cb(spk, "spk_embd", -1);
+
+ ggml_tensor * out = linear(spk, "proj"); // speaker row of the prompt
+ ggml_build_forward_expand(gf, out);
+ return gf;
+}
diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp
index 33c26c31f..c6d751b41 100644
--- a/tools/mtmd/mtmd-audio.cpp
+++ b/tools/mtmd/mtmd-audio.cpp
@@ -1176,45 +1176,102 @@ static float mtmd_audio_lufs(const float * samples, size_t n_samples, int sample
// mtmd_audio_preprocessor_chatterbox_ref
//
-void mtmd_audio_preprocessor_chatterbox_ref::initialize() {
- // kaldi fbank: povey window, kaldi mel banks over the 512-point spectrum, the nyquist bin left out
- {
- const int frame_len = 400;
- const int n_fft = 512;
- const int n_bins = n_fft / 2 + 1;
- const int n_mel = hparams.n_mel_bins;
- const double sr = (double) hparams.audio_sample_rate;
-
- fbank_cache.fill_sin_cos_table(n_fft);
- fbank_cache.hann_window.resize(frame_len);
- for (int i = 0; i < frame_len; i++) {
- fbank_cache.hann_window[(size_t) i] = (float) pow(0.5 - 0.5 * cos(2.0 * M_PI * i / (frame_len - 1)), 0.85);
- }
-
- auto mel = [](double f) { return 1127.0 * log(1.0 + f / 700.0); };
- const double mel_lo = mel(20.0);
- const double mel_hi = mel(sr / 2.0);
- const double delta = (mel_hi - mel_lo) / (n_mel + 1);
- const double bin_hz = sr / n_fft;
-
- auto & filters = fbank_cache.filters;
- filters.n_mel = n_mel;
- filters.n_fft = n_fft;
- filters.data.assign((size_t) n_mel * n_bins, 0.0f);
- for (int m = 0; m < n_mel; m++) {
- const double left = mel_lo + m * delta;
- const double center = left + delta;
- const double right = center + delta;
- for (int i = 0; i < n_bins - 1; i++) {
- const double f = mel(bin_hz * i);
- if (f > left && f < right) {
- const double w = f <= center ? (f - left) / (center - left)
- : (right - f) / (right - center);
- filters.data[(size_t) m * n_bins + i] = (float) w;
- }
+// torchaudio kaldi.fbank at 16 kHz, the nyquist bin is left out of the mel banks
+static constexpr int kaldi_frame_len = 400;
+static constexpr int kaldi_hop = 160;
+static constexpr int kaldi_n_fft = 512;
+
+static void mtmd_audio_kaldi_fbank_init(mtmd_audio_cache & cache, int n_mel, double sr, bool povey) {
+ const int n_bins = kaldi_n_fft / 2 + 1;
+
+ cache.fill_sin_cos_table(kaldi_n_fft);
+ cache.hann_window.resize(kaldi_frame_len);
+ for (int i = 0; i < kaldi_frame_len; i++) {
+ const double c = cos(2.0 * M_PI * i / (kaldi_frame_len - 1));
+ cache.hann_window[(size_t) i] = (float) (povey ? pow(0.5 - 0.5 * c, 0.85) : 0.54 - 0.46 * c);
+ }
+
+ auto mel = [](double f) { return 1127.0 * log(1.0 + f / 700.0); };
+ const double mel_lo = mel(20.0);
+ const double mel_hi = mel(sr / 2.0);
+ const double delta = (mel_hi - mel_lo) / (n_mel + 1);
+ const double bin_hz = sr / kaldi_n_fft;
+
+ auto & filters = cache.filters;
+ filters.n_mel = n_mel;
+ filters.n_fft = kaldi_n_fft;
+ filters.data.assign((size_t) n_mel * n_bins, 0.0f);
+ for (int m = 0; m < n_mel; m++) {
+ const double left = mel_lo + m * delta;
+ const double center = left + delta;
+ const double right = center + delta;
+ for (int i = 0; i < n_bins - 1; i++) {
+ const double f = mel(bin_hz * i);
+ if (f > left && f < right) {
+ const double w = f <= center ? (f - left) / (center - left)
+ : (right - f) / (right - center);
+ filters.data[(size_t) m * n_bins + i] = (float) w;
}
}
}
+}
+
+// snip edges, dc removal, preemphasis 0.97, log mel energies mean subtracted over time; out is mel major
+static bool mtmd_audio_kaldi_fbank(const mtmd_audio_cache & cache, const float * pcm, size_t n_samples, int n_mel, float scale,
+ mtmd_audio_mel & out) {
+ if ((int) n_samples < kaldi_frame_len) {
+ return false;
+ }
+ const int n_frames = 1 + ((int) n_samples - kaldi_frame_len) / kaldi_hop;
+
+ out.n_len = n_frames;
+ out.n_len_org = n_frames;
+ out.n_mel = n_mel;
+ out.data.assign((size_t) n_mel * n_frames, 0.0f);
+
+ const std::vector<float> & win = cache.hann_window;
+ std::vector<float> fft_in((size_t) kaldi_n_fft * 2), fft_out((size_t) kaldi_n_fft * 8);
+ std::vector<double> e;
+ std::vector<float> x(kaldi_frame_len);
+ for (int fr = 0; fr < n_frames; fr++) {
+ for (int i = 0; i < kaldi_frame_len; i++) {
+ x[(size_t) i] = pcm[(size_t) fr * kaldi_hop + i] * scale;
+ }
+
+ double mean = 0.0;
+ for (int i = 0; i < kaldi_frame_len; i++) {
+ mean += x[i];
+ }
+ mean /= kaldi_frame_len;
+
+ std::fill(fft_in.begin(), fft_in.end(), 0.0f);
+ fft_in[0] = (float) ((x[0] - mean) * (1.0 - 0.97) * win[0]);
+ for (int i = 1; i < kaldi_frame_len; i++) {
+ fft_in[(size_t) i] = (float) (((x[i] - mean) - 0.97 * (x[i - 1] - mean)) * win[(size_t) i]);
+ }
+ mtmd_audio_mel_energies(cache, fft_in, fft_out, false, e);
+ for (int m = 0; m < n_mel; m++) {
+ out.data[(size_t) m * n_frames + fr] = (float) log(std::max(e[(size_t) m], (double) FLT_EPSILON));
+ }
+ }
+
+ for (int m = 0; m < n_mel; m++) {
+ float * row = out.data.data() + (size_t) m * n_frames;
+ double mean = 0.0;
+ for (int fr = 0; fr < n_frames; fr++) {
+ mean += row[fr];
+ }
+ mean /= n_frames;
+ for (int fr = 0; fr < n_frames; fr++) {
+ row[fr] -= (float) mean;
+ }
+ }
+ return true;
+}
+
+void mtmd_audio_preprocessor_chatterbox_ref::initialize() {
+ // kaldi fbank of CAMPPlus: povey window
+ mtmd_audio_kaldi_fbank_init(fbank_cache, hparams.n_mel_bins, (double) hparams.audio_sample_rate, true);
// s3 tokenizer: librosa slaney mel, 128 bands
s3tok_cache.fill_sin_cos_table(400);
@@ -1253,59 +1310,12 @@ bool mtmd_audio_preprocessor_chatterbox_ref::process(const float *
}
}
- // entry 0: torchaudio kaldi.fbank of CAMPPlus (snip edges, dc removal, preemphasis 0.97), mean subtracted over time
+ // entry 0: torchaudio kaldi.fbank of CAMPPlus
{
- const int frame_len = 400;
- const int hop = 160;
- const int n_fft = 512;
- const int n_mel = hparams.n_mel_bins;
-
- if ((int) pcm.size() < frame_len) {
- return false;
- }
- const int n_frames = 1 + ((int) pcm.size() - frame_len) / hop;
-
mtmd_audio_mel out;
- out.n_len = n_frames;
- out.n_len_org = n_frames;
- out.n_mel = n_mel;
- out.data.assign((size_t) n_mel * n_frames, 0.0f);
-
- const std::vector<float> & win = fbank_cache.hann_window;
- std::vector<float> fft_in((size_t) n_fft * 2), fft_out((size_t) n_fft * 8);
- std::vector<double> e;
- for (int fr = 0; fr < n_frames; fr++) {
- const float * x = pcm.data() + (size_t) fr * hop;
-
- double mean = 0.0;
- for (int i = 0; i < frame_len; i++) {
- mean += x[i];
- }
- mean /= frame_len;
-
- std::fill(fft_in.begin(), fft_in.end(), 0.0f);
- fft_in[0] = (float) ((x[0] - mean) * (1.0 - 0.97) * win[0]);
- for (int i = 1; i < frame_len; i++) {
- fft_in[(size_t) i] = (float) (((x[i] - mean) - 0.97 * (x[i - 1] - mean)) * win[(size_t) i]);
- }
- mtmd_audio_mel_energies(fbank_cache, fft_in, fft_out, false, e);
- for (int m = 0; m < n_mel; m++) {
- out.data[(size_t) m * n_frames + fr] = (float) log(std::max(e[(size_t) m], (double) FLT_EPSILON));
- }
- }
-
- for (int m = 0; m < n_mel; m++) {
- float * row = out.data.data() + (size_t) m * n_frames;
- double mean = 0.0;
- for (int fr = 0; fr < n_frames; fr++) {
- mean += row[fr];
- }
- mean /= n_frames;
- for (int fr = 0; fr < n_frames; fr++) {
- row[fr] -= (float) mean;
- }
+ if (!mtmd_audio_kaldi_fbank(fbank_cache, pcm.data(), pcm.size(), hparams.n_mel_bins, 1.0f, out)) {
+ return false;
}
-
output.push_back(std::move(out));
}
@@ -1883,6 +1893,197 @@ bool mtmd_audio_preprocessor_gemma4ua::process(const float * sam
return true;
}
+//
+// mtmd_audio_preprocessor_zonos3_spk
+//
+
+void mtmd_audio_preprocessor_zonos3_spk::initialize() {
+ mtmd_audio_kaldi_fbank_init(cache, hparams.n_mel_bins, (double) hparams.audio_sample_rate, false);
+}
+
+bool mtmd_audio_preprocessor_zonos3_spk::process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output,
+ bool /*shape_only*/) const {
+ // the reference uses the first 60 s, on the int16 sample scale
+ const size_t n = std::min(n_samples, (size_t) hparams.audio_sample_rate * 60);
+ mtmd_audio_mel out;
+ if (!mtmd_audio_kaldi_fbank(cache, samples, n, hparams.n_mel_bins, 32768.0f, out)) {
+ return false;
+ }
+ output.clear();
+ output.push_back(std::move(out));
+ return true;
+}
+
+//
+// mtmd_audio_zonos3_synth implementation
+//
+
+mtmd_audio_zonos3_synth::mtmd_audio_zonos3_synth(const std::vector<float> & mixer, const std::vector<float> & resample,
+ int n_fft0, int n_fft1, int hop, int rs_orig, int rs_new)
+ : mixer(mixer), fir_kernel((int) (mixer.size() / 2)), hop(hop), rs_kernel(resample), rs_orig(rs_orig), rs_new(rs_new) {
+ GGML_ASSERT(mixer.size() % 2 == 0 && fir_kernel % 2 == 1);
+ GGML_ASSERT(resample.size() % (size_t) rs_new == 0);
+ const int rs_len = (int) (resample.size() / rs_new);
+ GGML_ASSERT(rs_len >= rs_orig && (rs_len - rs_orig) % 2 == 0);
+ rs_width = (rs_len - rs_orig) / 2;
+
+ const int n_fft[2] = { n_fft0, n_fft1 };
+ const int h = (fir_kernel - 1) / 2;
+ for (int i = 0; i < 2; i++) {
+ branch & b = br[i];
+ b.n_fft = n_fft[i];
+ b.cache.fill_sin_cos_table(b.n_fft);
+ b.cache.fill_hann_window(b.n_fft, true);
+ b.ifft_in.assign((size_t) b.n_fft * 2 * 4, 0.0f);
+ b.ifft_out.assign((size_t) b.n_fft * 2 * 4, 0.0f);
+ b.mix_in.assign((size_t) h, 0.0f);
+ b.mix_avail = h;
+ }
+}
+
+// the first n_fft / 2 samples are the centering pad
+void mtmd_audio_zonos3_synth::ola_finalize(branch & b, int64_t pf) {
+ const int64_t pad = b.n_fft / 2;
+ const int64_t n = std::min<int64_t>(pf - b.ola_base, (int64_t) b.ola.size());
+ if (n <= 0) {
+ return;
+ }
+ for (int64_t j = 0; j < n; j++) {
+ if (b.ola_base + j < pad) {
+ continue;
+ }
+ b.mix_in.push_back(b.env[(size_t) j] > 1e-11f ? b.ola[(size_t) j] / b.env[(size_t) j] : 0.0f);
+ b.mix_avail++;
+ }
+ b.ola.erase(b.ola.begin(), b.ola.begin() + n);
+ b.env.erase(b.env.begin(), b.env.begin() + n);
+ b.ola_base += n;
+}
+
+void mtmd_audio_zonos3_synth::mix_emit(std::vector<float> & out) {
+ const int64_t avail = std::min(br[0].mix_avail, br[1].mix_avail);
+ const int64_t m1 = avail - fir_kernel + 1;
+ if (m1 <= mix_next) {
+ return;
+ }
+ const size_t n = (size_t) (m1 - mix_next);
+ std::vector<float> y(n, 0.0f);
+ for (int c = 0; c < 2; c++) {
+ const float * w = mixer.data() + (size_t) c * fir_kernel;
+ const float * s = br[c].mix_in.data() + (mix_next - mix_base);
+ for (int k = 0; k < fir_kernel; k++) {
+ for (size_t i = 0; i < n; i++) {
+ y[i] += w[k] * s[i + k];
+ }
+ }
+ }
+ mix_next = m1;
+ for (int c = 0; c < 2; c++) {
+ br[c].mix_in.erase(br[c].mix_in.begin(), br[c].mix_in.begin() + (mix_next - mix_base));
+ }
+ mix_base = mix_next;
+
+ rs_buf.insert(rs_buf.end(), y.begin(), y.end());
+ rs_in += (int64_t) n;
+ resample(false, out);
+}
+
+// on flush, the input is zero-extended and the output is ceil(new * rs_in / orig) samples
+void mtmd_audio_zonos3_synth::resample(bool flush, std::vector<float> & out) {
+ const int KL = 2 * rs_width + rs_orig;
+ const int64_t total = flush ? ((int64_t) rs_new * rs_in + rs_orig - 1) / rs_orig : -1;
+ std::vector<float> xw(KL);
+ for (;; rs_block++) {
+ const int64_t x0 = rs_block * rs_orig - rs_width;
+ if (flush ? rs_block * rs_new >= total : x0 + KL > rs_in) {
+ break;
+ }
+ for (int k = 0; k < KL; k++) {
+ const int64_t j = x0 + k;
+ xw[k] = (j < 0 || j >= rs_in) ? 0.0f : rs_buf[(size_t) (j - rs_base)];
+ }
+ const int64_t n_emit = flush ? std::min<int64_t>(rs_new, total - rs_block * rs_new) : rs_new;
+ for (int64_t i = 0; i < n_emit; i++) {
+ const float * ki = rs_kernel.data() + (size_t) i * KL;
+ float acc = 0.0f;
+ for (int k = 0; k < KL; k++) {
+ acc += ki[k] * xw[k];
+ }
+ out.push_back(acc);
+ }
+ }
+ // drop the samples no later block reads
+ const int64_t keep_from = std::min(rs_in, std::max<int64_t>(0, rs_block * rs_orig - rs_width));
+ if (keep_from > rs_base) {
+ rs_buf.erase(rs_buf.begin(), rs_buf.begin() + (keep_from - rs_base));
+ rs_base = keep_from;
+ }
+}
+
+void mtmd_audio_zonos3_synth::push(const float * spec, int64_t n_cols, std::vector<float> & out) {
+ const int64_t n_rows = (br[0].n_fft + 2) + (br[1].n_fft + 2);
+ const float max_log_mag = logf(100.0f);
+ for (int i = 0; i < 2; i++) {
+ branch & b = br[i];
+ const int n = b.n_fft;
+ const int bins = n / 2 + 1;
+ const int64_t row0 = i == 0 ? 0 : br[0].n_fft + 2;
+ for (int64_t c = 0; c < n_cols; c++) {
+ const float * logmag = spec + c * n_rows + row0;
+ const float * phase = logmag + bins;
+ // irfft ignores the imaginary parts of DC and Nyquist
+ for (int k = 0; k < bins; k++) {
+ const float a = expf(std::min(logmag[k], max_log_mag));
+ b.ifft_in[2 * k + 0] = a * cosf(phase[k]);
+ b.ifft_in[2 * k + 1] = (k == 0 || k == n / 2) ? 0.0f : a * sinf(phase[k]);
+ }
+ for (int k = 1; k < n - bins + 1; k++) {
+ b.ifft_in[2 * (n - k) + 0] = b.ifft_in[2 * k + 0];
+ b.ifft_in[2 * (n - k) + 1] = -b.ifft_in[2 * k + 1];
+ }
+ ifft(b.cache, b.ifft_in.data(), n, b.ifft_out.data());
+
+ const int64_t t = n_frames + c;
+ const size_t need = (size_t) (t * hop + n - b.ola_base);
+ if (b.ola.size() < need) {
+ b.ola.resize(need, 0.0f);
+ b.env.resize(need, 0.0f);
+ }
+ float * ola = b.ola.data() + (t * hop - b.ola_base);
+ float * env = b.env.data() + (t * hop - b.ola_base);
+ const float * win = b.cache.hann_window.data();
+ for (int j = 0; j < n; j++) {
+ ola[j] += win[j] * b.ifft_out[2 * j];
+ env[j] += win[j] * win[j];
+ }
+ }
+ }
+ n_frames += n_cols;
+ for (auto & b : br) {
+ ola_finalize(b, n_frames * hop);
+ }
+ mix_emit(out);
+}
+
+void mtmd_audio_zonos3_synth::flush(std::vector<float> & out) {
+ if (done || n_frames == 0) {
+ return;
+ }
+ done = true;
+ // the mixer has h zeros of right padding
+ const int h = (fir_kernel - 1) / 2;
+ const int64_t L = n_frames * hop;
+ for (auto & b : br) {
+ ola_finalize(b, b.ola_base + (int64_t) b.ola.size());
+ const int64_t keep = h + L - mix_base;
+ b.mix_in.resize((size_t) keep, 0.0f);
+ b.mix_in.resize((size_t) (keep + h), 0.0f);
+ b.mix_avail = mix_base + keep + h;
+ }
+ mix_emit(out);
+ resample(true, out);
+}
+
//
// mtmd_audio_streaming_istft implementation
//
diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h
index 7251f82db..937451c8a 100644
--- a/tools/mtmd/mtmd-audio.h
+++ b/tools/mtmd/mtmd-audio.h
@@ -243,6 +243,16 @@ struct mtmd_audio_preprocessor_chatterbox_ref : mtmd_audio_preprocessor {
mtmd_audio_cache ve_cache; // entry 4
};
+// zonos3 speaker encoder front-end: kaldi fbank with a hamming window [80 x n_frames] (mel major)
+struct mtmd_audio_preprocessor_zonos3_spk : mtmd_audio_preprocessor {
+ mtmd_audio_preprocessor_zonos3_spk(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
+ void initialize() override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
+
+ private:
+ mtmd_audio_cache cache;
+};
+
struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_parakeet(const mtmd_model * model) : mtmd_audio_preprocessor(model) { }
void initialize() override;
@@ -261,6 +271,51 @@ struct mtmd_audio_preprocessor_nemotron_transcribe : mtmd_audio_preprocessor_par
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
+//
+// zonos3 iSTFT head, streaming: 2 iSTFT branches, FIR mixer, 44160 -> 44100 Hz resampler
+// ported from the reference ggml runtime (zac.cpp)
+//
+struct mtmd_audio_zonos3_synth {
+ // mixer [2, fir_kernel], resample [rs_new, rs_orig + 2 * width]
+ mtmd_audio_zonos3_synth(const std::vector<float> & mixer, const std::vector<float> & resample,
+ int n_fft0, int n_fft1, int hop, int rs_orig, int rs_new);
+
+ // spec: n_cols frames of [log magnitude, phase] of each branch
+ void push(const float * spec, int64_t n_cols, std::vector<float> & out);
+ void flush(std::vector<float> & out);
+
+ private:
+ struct branch {
+ int n_fft;
+ mtmd_audio_cache cache;
+ std::vector<float> ola, env; // from ola_base on
+ int64_t ola_base = 0;
+ std::vector<float> mix_in; // from mix_base on
+ int64_t mix_avail = 0;
+ std::vector<float> ifft_in, ifft_out;
+ };
+
+ void ola_finalize(branch & b, int64_t pf);
+ void mix_emit(std::vector<float> & out);
+ void resample(bool flush, std::vector<float> & out);
+
+ branch br[2];
+ std::vector<float> mixer;
+ int fir_kernel;
+ int hop;
+ int64_t n_frames = 0;
+ int64_t mix_base = 0;
+ int64_t mix_next = 0;
+ bool done = false;
+
+ std::vector<float> rs_kernel;
+ int rs_orig, rs_new, rs_width;
+ std::vector<float> rs_buf; // from rs_base on
+ int64_t rs_base = 0;
+ int64_t rs_in = 0;
+ int64_t rs_block = 0;
+};
+
//
// streaming ISTFT - converts spectrogram frames back to audio one frame at a time
//
diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp
index 01c4c0ac6..ac656032d 100644
--- a/tools/mtmd/mtmd-graph.cpp
+++ b/tools/mtmd/mtmd-graph.cpp
@@ -960,6 +960,10 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
{
builder = std::make_unique<mtmd_graph_qwen3tts_spkenc>(ctx, img);
} break;
+ case PROJECTOR_TYPE_ZONOS3_SPKENC:
+ {
+ builder = std::make_unique<mtmd_graph_zonos3_spkenc>(ctx, img);
+ } break;
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
{
builder = std::make_unique<mtmd_graph_pockettts_spkenc>(ctx, img);
@@ -1015,6 +1019,16 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
}
builder = std::make_unique<mtmd_graph_lfm3a_gen>(ctx, img, gen_process, all_known, n_frames);
} break;
+ case PROJECTOR_TYPE_ZONOS3_GEN:
+ {
+ const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
+ const int64_t n_cb = mtmd_graph_zonos3_gen::n_codebooks;
+ // GEN_CODE with codes: only the backbone row of that frame is needed
+ const bool all_known = params && params->codes && gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
+ const int n_frames = params && params->codes && gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV
+ ? (int) (params->codes->size() / n_cb) : 1;
+ builder = std::make_unique<mtmd_graph_zonos3_gen>(ctx, img, gen_process, all_known, n_frames);
+ } break;
case PROJECTOR_TYPE_DRAMA3_GEN:
{
const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 73e7e1b27..8a5757c1e 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -107,6 +107,8 @@ public:
virtual int32_t flush() = 0;
// continuing a sequence (inp->n_past > 0) with the model's own text/audio switching
virtual bool supports_continue() const { return false; }
+ // mtmd_helper_gen_audio_inp::speed and loudness other than the defaults
+ virtual bool supports_delivery() const { return false; }
virtual bool next_is_text() const { return false; }
std::vector<llama_logit_bias> logit_bias; // see mtmd_helper_gen_audio_get_logit_bias()
@@ -2418,6 +2420,535 @@ private:
bool wav_header_sent = false;
};
+// ZONOS3: one backbone row per frame of 10 codebooks, codebook j lags j frames
+class zonos3_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
+public:
+ using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
+
+ bool supports_delivery() const override { return true; }
+
+ void reset() override {
+ seq_id = 0;
+ pos = 0;
+ segments.clear();
+ seg_idx = 0;
+ seg_pos = 0;
+ rows.clear();
+ step = 0;
+ eos_row = -1;
+ countdown = 0;
+ done = false;
+ n_decoded = 0;
+ n_codec_samples = 0;
+ flushed = false;
+ gen_state.reset();
+ pcm_sent = 0;
+ wav_header_sent = false;
+ audio_pcm.clear();
+ h_state_buf.clear();
+ out_buf.clear();
+ }
+
+ int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
+ reset();
+ seq_id = inp->seq_id;
+ out_type = inp->out_type;
+ stream = inp->stream;
+
+ // settings of the reference, only the temperature can be changed
+ const mtmd_gen_inp def = mtmd_gen_inp_default(mctx);
+ temp = inp->temp >= 0.0f ? inp->temp : def.temp;
+ top_k = def.top_k;
+ top_p = def.top_p;
+ rep_pen = def.penalty_repeat;
+ rep_win = def.penalty_last_n;
+ rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
+
+ if (!init_gen_state(inp->seed)) {
+ return 1;
+ }
+
+ // prompt: [voice rows] [condition rows] BOS text EOS [silence rows]
+ // the speaker CFG of the reference is omitted: it adds too much code and maintenance cost
+ if (inp->speaker_ref) {
+ segment seg;
+ if (!encode_speaker(inp->speaker_ref, seg.embd)) {
+ return 1;
+ }
+ segments.push_back(std::move(seg));
+ }
+
+ // condition rows: the bucket of each value over the checkpoint edges, or unknown
+ auto bucket = [](const std::vector<float> & edges, float v) {
+ return std::to_string(std::upper_bound(edges.begin(), edges.end(), v) - edges.begin());
+ };
+ const std::string loudness = inp->loudness != 0.0f ? bucket(loudness_edges, inp->loudness) : "unk";
+ const std::string rate = inp->speed != 1.0f ? bucket(rate_edges, inp->speed * typical_rate) : "unk";
+ segment text;
+ for (const std::string & name : { "<|loudness_" + loudness + "|>", std::string("<|fmax_unk|>"), "<|speaking_rate_" + rate + "|>" }) {
+ const llama_token t = find_special_token(vocab, name);
+ if (t == LLAMA_TOKEN_NULL) {
+ LOG_ERR("mtmd_helper_gen_audio: missing ZONOS3 token %s in vocab\n", name.c_str());
+ return 1;
+ }
+ text.tokens.push_back(t);
+ }
+ const std::string prompt(inp->prompt, inp->prompt_len);
+ std::vector<llama_token> ids(prompt.size() + 16);
+ const int32_t n_ids = llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(), ids.data(), (int32_t) ids.size(), true, false);
+ if (n_ids <= 2) {
+ LOG_ERR("mtmd_helper_gen_audio: empty prompt or tokenization failed\n");
+ return 1;
+ }
+ text.tokens.insert(text.tokens.end(), ids.begin(), ids.begin() + n_ids);
+ segments.push_back(std::move(text));
+
+ // codec silence, delayed
+ segment silence;
+ for (int k = 0; k < n_prefill; k++) {
+ std::vector<int32_t> row(n_cb, audio_pad);
+ for (int cb = 0; cb <= k && cb < n_cb; cb++) {
+ row[cb] = silence_frame[cb];
+ }
+ const float * embd = embed_row(row);
+ if (!embd) {
+ return 1;
+ }
+ silence.embd.insert(silence.embd.end(), embd, embd + n_embd);
+ rows.insert(rows.end(), row.begin(), row.end());
+ }
+ segments.push_back(std::move(silence));
+
+ size_t n_prompt = 0;
+ for (const auto & seg : segments) {
+ n_prompt += seg.n_rows(n_embd);
+ }
+ if (n_prompt >= llama_n_ctx_seq(lctx)) {
+ LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n");
+ return 1;
+ }
+ if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) {
+ LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n");
+ return 1;
+ }
+ return 0;
+ }
+
+ int32_t step_prompt(int32_t n_batch) override {
+ GGML_ASSERT(n_batch > 0);
+ if (seg_idx >= segments.size()) {
+ return 0;
+ }
+ const segment & seg = segments[seg_idx];
+ const int n_rows = (int) seg.n_rows(n_embd);
+ const int n = std::min(n_batch, n_rows - seg_pos);
+ const bool last = seg_idx + 1 == segments.size() && seg_pos + n == n_rows;
+
+ int ret;
+ if (!seg.tokens.empty()) {
+ llama_batch batch = llama_batch_init(n, 0, 1);
+ for (int i = 0; i < n; i++) {
+ batch.token [i] = seg.tokens[(size_t) (seg_pos + i)];
+ batch.pos [i] = pos + i;
+ batch.n_seq_id[i] = 1;
+ batch.seq_id [i][0] = seq_id;
+ batch.logits [i] = last && i == n - 1;
+ }
+ batch.n_tokens = n;
+ ret = llama_decode(lctx, batch);
+ llama_batch_free(batch);
+ } else {
+ decode_embd_batch batch(seg.embd.data() + (size_t) seg_pos * n_embd, n, 1, n_embd);
+ batch.set_position_normal(pos, seq_id);
+ batch.logits[n - 1] = last;
+ ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n));
+ }
+ if (ret != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
+ return -1;
+ }
+ pos += n;
+ seg_pos += n;
+ if (seg_pos == n_rows) {
+ seg_idx++;
+ seg_pos = 0;
+ }
+
+ int32_t n_left = 0;
+ for (size_t i = seg_idx; i < segments.size(); i++) {
+ n_left += (int32_t) segments[i].n_rows(n_embd) - (i == seg_idx ? seg_pos : 0);
+ }
+ if (n_left == 0) {
+ const float * h = llama_get_embeddings_ith(lctx, -1);
+ h_state_buf.assign(h, h + n_embd);
+ }
+ return n_left;
+ }
+
+ int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
+ GGML_UNUSED(sampled);
+ GGML_UNUSED(h_state_in);
+ *h_state_out = nullptr;
+ *out_stop = false;
+ if (done) {
+ *out_stop = true;
+ return 0;
+ }
+
+ mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
+ inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
+ inp.embd = h_state_buf.data();
+ mtmd_gen_out out{};
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || out.n_feats != (size_t) n_cb * n_vocab) {
+ LOG_ERR("mtmd_helper_gen_audio: codebook heads failed\n");
+ return 1;
+ }
+
+ std::vector<int32_t> frame(n_cb);
+ for (int cb = 0; cb < n_cb; cb++) {
+ frame[cb] = sample(out.feats + (size_t) cb * n_vocab, cb);
+ }
+ // codebooks whose delayed stream has not started yet are pad
+ const int r = n_prefill + step;
+ for (int cb = r + 1; cb < n_cb; cb++) {
+ frame[cb] = audio_pad;
+ }
+ rows.insert(rows.end(), frame.begin(), frame.end());
+ step++;
+
+ // after EOA, n_cb + 1 more rows flush the delay
+ int max_eoa = -1;
+ for (int cb = 0; cb < n_cb; cb++) {
+ if (frame[cb] == audio_eoa) {
+ max_eoa = cb;
+ }
+ }
+ if (eos_row < 0 && max_eoa >= 0) {
+ eos_row = std::max(0, r - max_eoa);
+ countdown = n_cb + 1;
+ }
+ if ((countdown > 0 && --countdown == 0) || step >= max_frames || (uint32_t) pos >= llama_n_ctx_seq(lctx)) {
+ done = true;
+ *out_stop = true;
+ return 0;
+ }
+
+ const float * next = embed_row(frame);
+ if (!next) {
+ return 1;
+ }
+ decode_embd_batch batch(next, 1, 1, n_embd);
+ batch.set_position_normal(pos, seq_id);
+ batch.logits[0] = 1;
+ if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
+ return 1;
+ }
+ pos++;
+ const float * h = llama_get_embeddings_ith(lctx, -1);
+ h_state_buf.assign(h, h + n_embd);
+ *h_state_out = h_state_buf.data();
+
+ if (stream) {
+ return decode_stream() ? 0 : 1;
+ }
+ return 0;
+ }
+
+ int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
+ *out_sample_rate = info.sample_rate;
+
+ if (!stream) {
+ if (flush() != 0) {
+ return 1;
+ }
+ if (out_n_samples) {
+ *out_n_samples = (int64_t) audio_pcm.size();
+ }
+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+ *out_data = (const char *) audio_pcm.data();
+ *out_data_len = audio_pcm.size() * sizeof(float);
+ return 0;
+ }
+ out_buf.clear();
+ if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
+ LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
+ return 1;
+ }
+ *out_data = out_buf.data();
+ *out_data_len = out_buf.size();
+ return 0;
+ }
+
+ // streaming: only the audio since the previous call
+ const size_t n_new = audio_pcm.size() - pcm_sent;
+ if (out_n_samples) {
+ *out_n_samples = (int64_t) n_new;
+ }
+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+ *out_data = (const char *) (audio_pcm.data() + pcm_sent);
+ *out_data_len = n_new * sizeof(float);
+ pcm_sent = audio_pcm.size();
+ return 0;
+ }
+ out_buf.clear();
+ if (!wav_header_sent) {
+ write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
+ wav_header_sent = true;
+ }
+ append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
+ pcm_sent = audio_pcm.size();
+ *out_data = out_buf.data();
+ *out_data_len = out_buf.size();
+ return 0;
+ }
+
+ int32_t flush() override {
+ return decode_wav(true) ? 0 : 1;
+ }
+
+private:
+ static constexpr int n_cb = 10;
+ static constexpr int n_vocab = 4098; // codes, EOA, pad
+ static constexpr int audio_eoa = 4096;
+ static constexpr int audio_pad = 4097;
+
+ // condition_edges of params.json; speed 1 is this many UTF-8 bytes per second, the middle of the rate buckets
+ const std::vector<float> loudness_edges = { -29.778f, -26.947f, -25.047f, -23.3f, -21.743f, -20.386f, -19.186f, -17.851f, -15.967f };
+ const std::vector<float> rate_edges = {
+ 6.546f, 8.182f, 9.273f, 10.31f, 11.054f, 11.667f, 12.193f, 12.656f, 13.083f, 13.476f, 13.846f, 14.196f, 14.53f, 14.887f, 15.193f,
+ 15.517f, 15.865f, 16.208f, 16.579f, 16.972f, 17.411f, 17.898f, 18.434f, 19.069f, 19.818f, 20.816f, 22.286f, 24.722f, 29.25f,
+ };
+ static constexpr float typical_rate = 15.0f;
+
+ // digital silence of the ZAC codec (registries/silence_frame_*.json)
+ static constexpr int32_t silence_frame[n_cb] = { 1918, 746, 3544, 234, 1267, 2914, 3944, 2849, 326, 1337 };
+ static constexpr int n_prefill = 3;
+
+ // -3 dB of headroom
+ static constexpr int samples_per_frame = 2940;
+ static constexpr float out_gain = 0.70794578f;
+
+ static constexpr int rep_codebooks = 6;
+ static constexpr int max_frames = 2000;
+ static constexpr size_t stream_frames = 8;
+
+ struct segment {
+ std::vector<llama_token> tokens; // either tokens or embd rows
+ std::vector<float> embd;
+ size_t n_rows(int n) const { return tokens.empty() ? embd.size() / (size_t) n : tokens.size(); }
+ };
+
+ const float * embed_row(const std::vector<int32_t> & codes) {
+ mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
+ inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
+ inp.codes = const_cast<int32_t *>(codes.data());
+ inp.n_codes = codes.size();
+ mtmd_gen_out out{};
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: codebook embedding failed\n");
+ return nullptr;
+ }
+ return out.embd;
+ }
+
+ // same as the reference sampler (servers/ggml/src/zonos3-sampler.cpp)
+ int32_t sample(const float * in, int cb) {
+ std::vector<float> l(in, in + n_vocab);
+ l[audio_pad] = -INFINITY;
+
+ if (rep_pen != 1.0f && cb < rep_codebooks) {
+ const size_t n_rows = rows.size() / n_cb - n_prefill;
+ const size_t n_win = std::min(n_rows, (size_t) std::max(rep_win, 0));
+ std::vector<bool> seen(audio_eoa, false);
+ for (size_t f = n_prefill + n_rows - n_win; f < n_prefill + n_rows; f++) {
+ const int32_t c = rows[f * n_cb + (size_t) cb];
+ if (c >= 0 && c < audio_eoa && !seen[(size_t) c]) {
+ seen[(size_t) c] = true;
+ l[c] = l[c] > 0.0f ? l[c] / rep_pen : l[c] * rep_pen;
+ }
+ }
+ }
+
+ if (temp <= 0.0f) {
+ return (int32_t) (std::max_element(l.begin(), l.end()) - l.begin());
+ }
+ for (auto & v : l) {
+ v /= temp;
+ }
+
+ float kth = -INFINITY;
+ if (top_k > 0 && top_k < n_vocab) {
+ std::vector<float> tmp(l);
+ std::nth_element(tmp.begin(), tmp.begin() + (top_k - 1), tmp.end(), std::greater<float>());
+ kth = tmp[(size_t) top_k - 1];
+ }
+ std::vector<int32_t> idx;
+ for (int32_t i = 0; i < n_vocab; i++) {
+ if (l[i] >= kth && l[i] > -INFINITY) {
+ idx.push_back(i);
+ }
+ }
+ float lmax = -INFINITY;
+ for (int32_t i : idx) {
+ lmax = std::max(lmax, l[i]);
+ }
+ std::vector<double> p(idx.size());
+ double sum = 0.0;
+ for (size_t j = 0; j < idx.size(); j++) {
+ p[j] = std::exp((double) (l[idx[j]] - lmax));
+ sum += p[j];
+ }
+ std::vector<size_t> ord(idx.size());
+ for (size_t j = 0; j < ord.size(); j++) {
+ ord[j] = j;
+ }
+ std::sort(ord.begin(), ord.end(), [&](size_t a, size_t b) { return p[a] > p[b]; });
+ if (top_p > 0.0f && top_p < 1.0f) {
+ double before = 0.0;
+ for (size_t j : ord) {
+ const double pj = p[j];
+ if (before > top_p * sum) {
+ p[j] = 0.0;
+ }
+ before += pj;
+ }
+ }
+ double kept = 0.0;
+ for (double v : p) {
+ kept += v;
+ }
+ double u = std::uniform_real_distribution<double>(0.0, kept)(rng);
+ for (size_t j = 0; j < p.size(); j++) {
+ u -= p[j];
+ if (u <= 0.0 && p[j] > 0.0) {
+ return idx[j];
+ }
+ }
+ return idx[ord[0]];
+ }
+
+ // frames with all codebooks known
+ size_t n_ready() const {
+ const size_t n_rows = rows.size() / n_cb;
+ size_t n = n_rows >= (size_t) n_cb ? n_rows - (n_cb - 1) : 0;
+ if (eos_row >= 0) {
+ n = std::min(n, (size_t) eos_row);
+ }
+ return n;
+ }
+
+ // streaming, as the reference does for the first audio: the silence frames are decoded with silence codes only,
+ // then the first generated frame from its first code alone; later chunks of complete frames grow 2, 4, 8
+ bool decode_stream() {
+ if (n_decoded == 0) {
+ const size_t n_rows = rows.size() / n_cb;
+ if (n_rows <= (size_t) n_prefill || rows[(size_t) n_prefill * n_cb] >= audio_eoa) {
+ return true;
+ }
+ std::vector<int32_t> codes;
+ for (int k = 0; k < n_prefill; k++) {
+ codes.insert(codes.end(), silence_frame, silence_frame + n_cb);
+ }
+ for (int cb = 0; cb < n_cb; cb++) {
+ const size_t r = (size_t) n_prefill + cb;
+ const int32_t c = r < n_rows ? rows[r * n_cb + cb] : -1;
+ codes.push_back(c >= audio_eoa ? -1 : c);
+ }
+ if (!run_codec(codes.data(), codes.size())) {
+ return false;
+ }
+ n_decoded = n_prefill + 1;
+ return true;
+ }
+ const size_t n_chunk = std::min(stream_frames, n_decoded - n_prefill + 1);
+ if (n_ready() >= n_decoded + n_chunk) {
+ return decode_wav(false);
+ }
+ return true;
+ }
+
+ // the silence frames are decoded too, as left context
+ bool decode_wav(bool final) {
+ const size_t n_frames = n_ready();
+ std::vector<int32_t> codes;
+ while (n_decoded < n_frames) {
+ const size_t n = std::min(n_frames - n_decoded, (size_t) 64);
+ codes.resize(n * n_cb);
+ for (size_t f = 0; f < n; f++) {
+ for (int cb = 0; cb < n_cb; cb++) {
+ // EOA and pad have no codec row
+ const int32_t c = rows[(n_decoded + f + cb) * n_cb + cb];
+ codes[f * n_cb + cb] = std::clamp(c, 0, audio_eoa - 1);
+ }
+ }
+ if (!run_codec(codes.data(), codes.size())) {
+ return false;
+ }
+ n_decoded += n;
+ }
+ if (final && done && n_decoded > 0 && !flushed) {
+ flushed = true;
+ return run_codec(nullptr, 0);
+ }
+ return true;
+ }
+
+ bool run_codec(const int32_t * codes, size_t n_codes) {
+ mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
+ inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
+ inp.codes = const_cast<int32_t *>(codes);
+ inp.n_codes = n_codes;
+ mtmd_gen_out out{};
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n");
+ return false;
+ }
+ // drop the audio of the silence frames, then the output gain and soft limiter of the reference
+ const float knee = std::pow(10.0f, -1.0f / 20.0f);
+ for (size_t i = 0; i < out.n_samples; i++, n_codec_samples++) {
+ if (n_codec_samples < (size_t) n_prefill * samples_per_frame) {
+ continue;
+ }
+ const float x = out.audio[i] * out_gain;
+ const float a = std::fabs(x);
+ audio_pcm.push_back(a <= knee ? x : std::copysign(knee + (1.0f - knee) * std::tanh((a - knee) / (1.0f - knee)), x));
+ }
+ return true;
+ }
+
+ std::vector<segment> segments;
+ size_t seg_idx = 0;
+ int seg_pos = 0;
+
+ std::vector<int32_t> rows; // delayed rows: silence rows, then the generated ones
+ int step = 0;
+ int eos_row = -1;
+ int countdown = 0;
+ bool done = false;
+ size_t n_decoded = 0;
+ size_t n_codec_samples = 0;
+ bool flushed = false;
+
+ float temp = 0.75f;
+ int32_t top_k = 80;
+ float top_p = 0.69f;
+ float rep_pen = 2.4f;
+ int32_t rep_win = 96;
+ std::mt19937 rng;
+
+ llama_seq_id seq_id = 0;
+ llama_pos pos = 0;
+ mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
+ bool stream = false;
+
+ std::vector<float> h_state_buf;
+ std::vector<float> audio_pcm;
+ std::vector<char> out_buf;
+ size_t pcm_sent = 0;
+ bool wav_header_sent = false;
+};
+
static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
switch (mtmd_gen_audio_get_info(mctx).type) {
case MTMD_GEN_AUDIO_TYPE_QWEN3TTS:
@@ -2432,6 +2963,8 @@ static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lc
return std::unique_ptr<mtmd_gen_audio_pipeline>(new chatterbox_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_DRAMA3:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new drama3_gen_audio_pipeline(lctx, mctx));
+ case MTMD_GEN_AUDIO_TYPE_ZONOS3:
+ return std::unique_ptr<mtmd_gen_audio_pipeline>(new zonos3_gen_audio_pipeline(lctx, mctx));
default:
return nullptr;
}
@@ -2463,6 +2996,8 @@ struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) {
inp.top_p = 1.0f;
inp.temp = -1.0f; // model default
inp.seed = UINT32_MAX; // random
+ inp.speed = 1.0f;
+ inp.loudness = 0.0f;
inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
return inp;
}
@@ -2476,6 +3011,14 @@ int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_
LOG_ERR("mtmd_helper_gen_audio: the pipeline cannot continue a sequence\n");
return 1;
}
+ if (!std::isfinite(inp->speed) || inp->speed <= 0.0f || !std::isfinite(inp->loudness) || inp->loudness > 0.0f) {
+ LOG_ERR("mtmd_helper_gen_audio: speed must be > 0 and loudness < 0 LUFS (0 = model default)\n");
+ return 2;
+ }
+ if ((inp->speed != 1.0f || inp->loudness != 0.0f) && !ctx->pipeline->supports_delivery()) {
+ LOG_ERR("mtmd_helper_gen_audio: the model does not support speed or loudness\n");
+ return 2;
+ }
ctx->pipeline->logit_bias.clear();
return ctx->pipeline->set_input(inp);
}
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index 1bf3ee2ba..5ca414906 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -291,6 +291,10 @@ struct mtmd_helper_gen_audio_inp {
float temp; // audio frame temperature if the model supports it, < 0 = model default, 0 = greedy
uint32_t seed; // UINT32_MAX for random (default: random)
+ // delivery (only for models that support it); other models reject non-default values
+ float speed; // speaking speed relative to a typical pace, > 0 (default: 1.0 = the model's own pace)
+ float loudness; // target loudness in LUFS, < 0 (default: 0.0 = the model's own loudness)
+
enum mtmd_helper_gen_audio_outtype out_type;
};
@@ -304,6 +308,7 @@ MTMD_API void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx);
MTMD_API void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx);
+// returns 0 on success, 2 if the input is invalid for the model (e.g. unsupported speed or loudness), 1 on other errors
MTMD_API int32_t mtmd_helper_gen_audio_set_input(
mtmd_helper_gen_audio * ctx,
const struct mtmd_helper_gen_audio_inp * inp);
diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h
index e2157f767..0d287dcdd 100644
--- a/tools/mtmd/mtmd-impl.h
+++ b/tools/mtmd/mtmd-impl.h
@@ -106,6 +106,7 @@
#define KEY_GEN_AUDIO_PROJ_TYPE "clip.gen.audio.projector_type" // for models with mixed modalities
// name of the weight variant, for settings that are not in the checkpoint
#define KEY_GEN_AUDIO_VARIANT "clip.gen.audio.model_variant"
+#define KEY_GEN_AUDIO_DEFAULT_VOICE "clip.gen.audio.default_voice"
// sampling defaults of the generated audio, like general.sampling.* for text
#define KEY_GEN_AUDIO_TEMP "clip.gen.audio.sampling.temp"
#define KEY_GEN_AUDIO_TOP_K "clip.gen.audio.sampling.top_k"
@@ -572,6 +573,8 @@ enum projector_type {
PROJECTOR_TYPE_SOPRANO,
PROJECTOR_TYPE_LFM3A_GEN,
PROJECTOR_TYPE_DRAMA3_GEN,
+ PROJECTOR_TYPE_ZONOS3_GEN,
+ PROJECTOR_TYPE_ZONOS3_SPKENC,
PROJECTOR_TYPE_MUSE_GLIMMER,
PROJECTOR_TYPE_SENSENOVA_U1,
PROJECTOR_TYPE_COHERE2V,
@@ -660,6 +663,8 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_SOPRANO, "soprano"},
{ PROJECTOR_TYPE_LFM3A_GEN, "lfm3a_gen"},
{ PROJECTOR_TYPE_DRAMA3_GEN, "drama3_gen"},
+ { PROJECTOR_TYPE_ZONOS3_GEN, "zonos3_gen"},
+ { PROJECTOR_TYPE_ZONOS3_SPKENC, "zonos3_spkenc"},
{ PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"},
{ PROJECTOR_TYPE_SENSENOVA_U1, "sensenova_u1"},
{ PROJECTOR_TYPE_COHERE2V, "cohere2v"},
diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp
index c4e5bbd54..c4c52a4d5 100644
--- a/tools/mtmd/mtmd-model.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -214,6 +214,7 @@ struct mtmd_model_loader {
hparams.image_size = 0;
hparams.patch_size = 1;
get_string(KEY_GEN_AUDIO_VARIANT, hparams.gen_model_variant, false);
+ get_string(KEY_GEN_AUDIO_DEFAULT_VOICE, hparams.gen_default_voice, false);
} else {
GGML_ASSERT(false && "unknown modality");
@@ -967,6 +968,15 @@ struct mtmd_model_loader {
"%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE));
}
} break;
+ case PROJECTOR_TYPE_ZONOS3_SPKENC:
+ {
+ // kaldi fbank front-end
+ hparams.audio_sample_rate = 16000;
+ hparams.audio_n_fft = 512;
+ hparams.audio_window_len = 400;
+ hparams.audio_hop_len = 160;
+ hparams.warmup_audio_size = 1000;
+ } break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
// ECAPA-TDNN speaker encoder, mel front-end uses the Slaney default (fmin=0, fmax=sr/2)
@@ -1032,6 +1042,10 @@ struct mtmd_model_loader {
// matches the reference decoder's sliding_window (speech_tokenizer/config.json)
hparams.wav_tfm_swa = 72;
} break;
+ case PROJECTOR_TYPE_ZONOS3_GEN:
+ {
+ hparams.audio_sample_rate = 44100;
+ } break;
case PROJECTOR_TYPE_DRAMA3_GEN:
{
// Fish DAC decoder, codec.pth has no config
@@ -1479,6 +1493,7 @@ struct mtmd_model_loader {
const bool has_standard_layers = (
model.proj_type != PROJECTOR_TYPE_GEMMA3NV &&
model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC &&
+ model.proj_type != PROJECTOR_TYPE_ZONOS3_SPKENC &&
model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN &&
model.proj_type != PROJECTOR_TYPE_SOPRANO &&
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
@@ -2574,6 +2589,118 @@ struct mtmd_model_loader {
// code2wav: RVQ codes -> raw PCM, lives in the same ctx as code_predictor
load_code2wav(true);
} break;
+ case PROJECTOR_TYPE_ZONOS3_GEN:
+ {
+ model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight"));
+ model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
+ if (model.gen_code_out_embd_w->ne[1] % mtmd_graph_zonos3_gen::n_codebooks != 0 ||
+ model.gen_code_head_w->ne[1] != model.gen_code_out_embd_w->ne[1]) {
+ throw std::runtime_error(mtmd_format("%s: zonos3: unexpected codebook table shape\n", __func__));
+ }
+
+ // voice.<name>: the voice rows of the prompt
+ for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) {
+ const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
+ if (strncmp(name, "voice.", 6) != 0) {
+ continue;
+ }
+ const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name);
+ if (t->type != GGML_TYPE_F32 || ggml_n_dims(t) > 2 || t->ne[0] != model.gen_code_out_embd_w->ne[0]) {
+ throw std::runtime_error(mtmd_format("%s: invalid voice tensor %s\n", __func__, name));
+ }
+ auto & embd = model.gen_voices[name + 6].rows;
+ embd.resize(ggml_nelements(t));
+ fin.seekg(tensor_offset.at(name), std::ios::beg);
+ if (!fin.read(reinterpret_cast<char *>(embd.data()), ggml_nbytes(t))) {
+ throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name));
+ }
+ }
+ LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size());
+
+ // ZAC codec decoder
+ auto & z = model.zonos3;
+ const std::string p = "a.gen.wav";
+ auto has = [&](const std::string & name) {
+ return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0;
+ };
+ auto load_convnext = [&](const std::string & q) {
+ mtmd_zonos3_wav::convnext b;
+ b.dw_w = get_tensor(q + ".dw.weight");
+ b.dw_b = get_tensor(q + ".dw.bias");
+ b.norm_w = get_tensor(q + ".norm.weight");
+ b.norm_b = get_tensor(q + ".norm.bias");
+ b.pw1_w = get_tensor(q + ".pw1.weight");
+ b.pw1_b = get_tensor(q + ".pw1.bias");
+ b.pw2_w = get_tensor(q + ".pw2.weight");
+ b.pw2_b = get_tensor(q + ".pw2.bias");
+ return b;
+ };
+ z.sem_table = get_tensor(p + ".sem.table");
+ z.sem_bias = get_tensor(p + ".sem.bias");
+ for (int i = 0; has(mtmd_format("%s.sem.%d.dw.weight", p.c_str(), i)); i++) {
+ z.sem.push_back(load_convnext(p + ".sem." + std::to_string(i)));
+ }
+ z.sem_out_w = get_tensor(p + ".sem.out.weight");
+ z.sem_out_b = get_tensor(p + ".sem.out.bias");
+ z.res_table = get_tensor(p + ".res.table");
+ z.res_bias = get_tensor(p + ".res.bias");
+ for (int s = 0; has(mtmd_format("%s.stage.%d.in.weight", p.c_str(), s)); s++) {
+ const std::string q = p + ".stage." + std::to_string(s);
+ mtmd_zonos3_wav::stage st;
+ st.in_w = get_tensor(q + ".in.weight");
+ st.out_w = get_tensor(q + ".out.weight", false);
+ for (int il = 0; has(mtmd_format("%s.blk.%d.qkv.weight", q.c_str(), il)); il++) {
+ const std::string b = q + ".blk." + std::to_string(il);
+ mtmd_zonos3_wav::layer l;
+ l.norm1_w = get_tensor(b + ".norm1.weight");
+ l.norm1_b = get_tensor(b + ".norm1.bias");
+ l.qkv_w = get_tensor(b + ".qkv.weight");
+ l.wo_w = get_tensor(b + ".wo.weight");
+ l.attn_scale = get_tensor(b + ".attn_scale");
+ l.norm2_w = get_tensor(b + ".norm2.weight");
+ l.norm2_b = get_tensor(b + ".norm2.bias");
+ l.ffn1_w = get_tensor(b + ".ffn1.weight");
+ l.ffn2_w = get_tensor(b + ".ffn2.weight");
+ l.ffn_scale = get_tensor(b + ".ffn_scale");
+ st.layers.push_back(l);
+ }
+ z.stages.push_back(std::move(st));
+ }
+ for (int i = 0; has(mtmd_format("%s.head.%d.dw.weight", p.c_str(), i)); i++) {
+ z.head.push_back(load_convnext(p + ".head." + std::to_string(i)));
+ }
+ z.head_norm_w = get_tensor(p + ".head.norm.weight");
+ z.head_norm_b = get_tensor(p + ".head.norm.bias");
+ z.head_out_w = get_tensor(p + ".head.out.weight");
+ z.head_out_b = get_tensor(p + ".head.out.bias");
+ if (z.stages.size() != mtmd_graph_zonos3_gen::n_stages || z.sem.empty() || z.head.empty() ||
+ z.head_out_w->ne[1] != mtmd_graph_zonos3_gen::n_fft[0] + mtmd_graph_zonos3_gen::n_fft[1] + 4) {
+ throw std::runtime_error(mtmd_format("%s: zonos3: unexpected codec decoder layout\n", __func__));
+ }
+
+ // read on the host
+ auto read_host = [&](const std::string & name, std::vector<float> & dst) -> const ggml_tensor * {
+ const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name.c_str());
+ if (!t || t->type != GGML_TYPE_F32 || ggml_n_dims(t) != 2) {
+ throw std::runtime_error(mtmd_format("%s: invalid or missing tensor %s\n", __func__, name.c_str()));
+ }
+ dst.resize(ggml_nelements(t));
+ fin.seekg(tensor_offset.at(name), std::ios::beg);
+ if (!fin.read(reinterpret_cast<char *>(dst.data()), ggml_nbytes(t))) {
+ throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name.c_str()));
+ }
+ return t;
+ };
+ const ggml_tensor * mixer = read_host(p + ".mixer.weight", z.mixer);
+ const ggml_tensor * rs = read_host(p + ".resample.weight", z.resample);
+ if (mixer->ne[1] != 2 || rs->ne[1] != mtmd_graph_zonos3_gen::rs_new ||
+ rs->ne[0] < mtmd_graph_zonos3_gen::rs_orig || (rs->ne[0] - mtmd_graph_zonos3_gen::rs_orig) % 2 != 0) {
+ throw std::runtime_error(mtmd_format("%s: zonos3: unexpected mixer or resampler shape\n", __func__));
+ }
+ z.fir_kernel = mixer->ne[0];
+ z.rs_new = rs->ne[1];
+ z.rs_kernel_len = rs->ne[0];
+ } break;
case PROJECTOR_TYPE_DRAMA3_GEN:
{
// generic a.gen.code.blk.* tensors are loaded by the loop above
@@ -2847,6 +2974,19 @@ struct mtmd_model_loader {
}
}
} break;
+ case PROJECTOR_TYPE_ZONOS3_SPKENC:
+ {
+ // the graph reads the tensors by name
+ for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) {
+ const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
+ if (strncmp(name, "a.spk.", 6) == 0) {
+ model.zonos3_spk[name] = get_tensor(name);
+ }
+ }
+ if (!model.zonos3_spk.count("a.spk.proj.weight") || !model.zonos3_spk.count("a.spk.linear.weight")) {
+ throw std::runtime_error(mtmd_format("%s: zonos3: missing speaker encoder tensors\n", __func__));
+ }
+ } break;
case PROJECTOR_TYPE_CHATTERBOX_SPKENC:
{
auto & c = model.cbx;
@@ -3756,6 +3896,10 @@ mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params
model->hparams.custom_image_max_tokens = params.image_max_tokens;
loader.load_hparams(*model, modality);
loader.load_tensors(*model, buft, params.no_alloc);
+ const std::string & def = model->hparams.gen_default_voice;
+ if (!def.empty() && model->gen_voices.find(def) == model->gen_voices.end()) {
+ throw std::runtime_error(mtmd_format("%s: default voice \"%s\" is not a predefined voice\n", __func__, def.c_str()));
+ }
return model;
};
@@ -4163,6 +4307,7 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
n_patches = ((img->nx() + ws - 1) / ws) * (ws / ds);
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
+ case PROJECTOR_TYPE_ZONOS3_SPKENC:
{
// pooling gives one speaker embedding, whatever the clip length is
n_patches = 1;
@@ -4175,6 +4320,7 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
case PROJECTOR_TYPE_QWEN3TTS_GEN:
case PROJECTOR_TYPE_LFM3A_GEN:
case PROJECTOR_TYPE_DRAMA3_GEN:
+ case PROJECTOR_TYPE_ZONOS3_GEN:
{
// one hidden-state vector fed back to the talker per call
n_patches = 1;
@@ -4356,9 +4502,12 @@ int mtmd_model::n_mmproj_embd() const {
return mm_2_w->ne[1];
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
return mm_fc_w->ne[2];
+ case PROJECTOR_TYPE_ZONOS3_SPKENC:
+ return zonos3_spk.at("a.spk.proj.weight")->ne[1];
case PROJECTOR_TYPE_QWEN3TTS_GEN:
case PROJECTOR_TYPE_LFM3A_GEN:
case PROJECTOR_TYPE_DRAMA3_GEN:
+ case PROJECTOR_TYPE_ZONOS3_GEN:
return gen_code_out_embd_w->ne[0];
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
return spk_proj_w->ne[1];
diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h
index b27b1c0d8..d04314d93 100644
--- a/tools/mtmd/mtmd-model.h
+++ b/tools/mtmd/mtmd-model.h
@@ -200,6 +200,7 @@ struct mtmd_hparams {
// name of the weight variant, some pipelines tune themselves on it
std::string gen_model_variant;
+ std::string gen_default_voice;
// sampling defaults of the generated audio: greedy, no top-k, no repetition penalty
float gen_temp = 0.0f;
@@ -869,6 +870,60 @@ struct mtmd_drama3_layer {
ggml_tensor * out_norm_w = nullptr;
};
+// zonos3 ZAC codec decoder (a.gen.wav.*)
+struct mtmd_zonos3_wav {
+ struct convnext {
+ ggml_tensor * dw_w = nullptr; // [K, C]
+ ggml_tensor * dw_b = nullptr;
+ ggml_tensor * norm_w = nullptr;
+ ggml_tensor * norm_b = nullptr;
+ ggml_tensor * pw1_w = nullptr;
+ ggml_tensor * pw1_b = nullptr;
+ ggml_tensor * pw2_w = nullptr;
+ ggml_tensor * pw2_b = nullptr;
+ };
+ struct layer {
+ ggml_tensor * norm1_w = nullptr;
+ ggml_tensor * norm1_b = nullptr;
+ ggml_tensor * qkv_w = nullptr;
+ ggml_tensor * wo_w = nullptr;
+ ggml_tensor * attn_scale = nullptr;
+ ggml_tensor * norm2_w = nullptr;
+ ggml_tensor * norm2_b = nullptr;
+ ggml_tensor * ffn1_w = nullptr;
+ ggml_tensor * ffn2_w = nullptr;
+ ggml_tensor * ffn_scale = nullptr;
+ };
+ struct stage {
+ ggml_tensor * in_w = nullptr;
+ ggml_tensor * out_w = nullptr; // nullptr in the last stage
+ std::vector<layer> layers;
+ };
+
+ ggml_tensor * sem_table = nullptr; // folded through its out_proj
+ ggml_tensor * sem_bias = nullptr;
+ std::vector<convnext> sem;
+ ggml_tensor * sem_out_w = nullptr;
+ ggml_tensor * sem_out_b = nullptr;
+ ggml_tensor * res_table = nullptr; // merged
+ ggml_tensor * res_bias = nullptr; // [latent, n_codebooks - 1]
+
+ std::vector<stage> stages;
+
+ std::vector<convnext> head;
+ ggml_tensor * head_norm_w = nullptr;
+ ggml_tensor * head_norm_b = nullptr;
+ ggml_tensor * head_out_w = nullptr; // both iSTFT branches
+ ggml_tensor * head_out_b = nullptr;
+
+ // read on the host
+ std::vector<float> mixer; // [2, fir_kernel]
+ std::vector<float> resample; // [rs_new, rs_kernel_len]
+ int64_t fir_kernel = 0;
+ int64_t rs_new = 0;
+ int64_t rs_kernel_len = 0;
+};
+
struct mtmd_vocos {
ggml_tensor * input_w = nullptr;
ggml_tensor * input_b = nullptr;
@@ -1148,6 +1203,7 @@ struct mtmd_model {
// qwen3tts code2wav: RVQ codes -> raw PCM
mtmd_code2wav c2w;
std::vector<mtmd_drama3_layer> drama3_layers;
+ mtmd_zonos3_wav zonos3;
// pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path)
mtmd_seanet seanet;
@@ -1172,6 +1228,7 @@ struct mtmd_model {
// chatterbox tensors read on the host, see cbx_read_tensor()
std::map<std::string, ggml_tensor *> cbx_tensors;
+ std::map<std::string, ggml_tensor *> zonos3_spk; // a.spk.*
// predefined voices, no rows if the voice is named in the prompt (lfm3a)
std::map<std::string, mtmd_gen_voice> gen_voices;
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index 798d875bb..ed8800fce 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -392,6 +392,7 @@ static std::vector<c2w_state_slot> list_gen_state_slots(const mtmd_hparams & hpa
case PROJECTOR_TYPE_DRAMA3_GEN: return list_drama3_state_slots(hparams, model);
case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model);
case PROJECTOR_TYPE_LFM3A_GEN: return list_lfm3a_gen_state_slots(hparams, model);
+ case PROJECTOR_TYPE_ZONOS3_GEN: return list_zonos3_gen_state_slots(hparams, model);
default: return {};
}
}
@@ -1676,6 +1677,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
case PROJECTOR_TYPE_GEMMA4UA:
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
+ case PROJECTOR_TYPE_ZONOS3_SPKENC:
{
// do nothing
} break;
@@ -1689,6 +1691,54 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
segfix[(size_t) S - 1] = 100.0f / (float) last;
set_input_f32("inp_segfix", segfix);
} break;
+ case PROJECTOR_TYPE_ZONOS3_GEN:
+ if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
+ // frame-major codes -> codebook-major rows; a residual code of -1 is missing and left out
+ const int64_t n_cb = mtmd_graph_zonos3_gen::n_codebooks;
+ const int64_t n_codes = mtmd_graph_zonos3_gen::n_cb_vocab;
+ GGML_ASSERT(params->codes != nullptr && params->codes->size() % n_cb == 0);
+ const int64_t n_frames = (int64_t) params->codes->size() / n_cb;
+ std::vector<int32_t> rows(params->codes->size());
+ std::vector<float> keep((size_t) n_frames * (n_cb - 1), 1.0f);
+ for (int64_t t = 0; t < n_frames; t++) {
+ for (int64_t c = 0; c < n_cb; c++) {
+ int32_t code = (*params->codes)[t * n_cb + c];
+ if (code == -1 && c > 0) {
+ keep[(c - 1) * n_frames + t] = 0.0f;
+ code = 0;
+ }
+ if (code < 0 || code >= n_codes) {
+ LOG_ERR("%s: code out of range (%d, codebook size %lld)\n", __func__, code, (long long) n_codes);
+ return false;
+ }
+ rows[c * n_frames + t] = code + (int32_t) (c == 0 ? 0 : (c - 1) * n_codes);
+ }
+ }
+ set_input_i32("inp_codes", rows);
+ set_input_f32("inp_keep", keep);
+ set_gen_state_in();
+ } else {
+ const int64_t n_cb = mtmd_graph_zonos3_gen::n_codebooks;
+ const int64_t n_vocab = model.gen_code_out_embd_w->ne[1] / n_cb;
+ // no codes: logits only, the row of codes 0 is ignored
+ std::vector<int32_t> codes(n_cb, 0);
+ if (params->codes) {
+ if (params->codes->size() != (size_t) n_cb) {
+ LOG_ERR("%s: expected %lld codes, got %zu\n", __func__, (long long) n_cb, params->codes->size());
+ return false;
+ }
+ codes = *params->codes;
+ }
+ for (int64_t i = 0; i < n_cb; i++) {
+ if (codes[i] < 0 || codes[i] >= n_vocab) {
+ LOG_ERR("%s: code out of range (%d, vocab %lld)\n", __func__, codes[i], (long long) n_vocab);
+ return false;
+ }
+ codes[i] += (int32_t) (i * n_vocab);
+ }
+ set_input_i32("inp_codes", codes);
+ }
+ break;
case PROJECTOR_TYPE_LFM3A_GEN:
{
const int64_t n_cb = model.gen_code_embd_w->ne[2];
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 63a8e3954..0042aeb42 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -1286,6 +1286,10 @@ struct mtmd_context {
{
audio_preproc = std::make_unique<mtmd_audio_preprocessor_qwen3tts_spk>(models.a.get());
} break;
+ case PROJECTOR_TYPE_ZONOS3_SPKENC:
+ {
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_zonos3_spk>(models.a.get());
+ } break;
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
{
audio_preproc = std::make_unique<mtmd_audio_preprocessor_pockettts>(models.a.get());
@@ -2462,6 +2466,10 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
info.type = MTMD_GEN_AUDIO_TYPE_DRAMA3;
info.sample_rate = 44100;
break;
+ case PROJECTOR_TYPE_ZONOS3_GEN:
+ info.type = MTMD_GEN_AUDIO_TYPE_ZONOS3;
+ info.sample_rate = 44100;
+ break;
default:
info.type = MTMD_GEN_AUDIO_TYPE_NONE;
break;
@@ -2822,8 +2830,23 @@ const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx) {
if (!ctx->ctx_gen_a || idx < 0) {
return nullptr;
}
- const auto & voices = ctx->ctx_gen_a->model.gen_voices;
- return idx < (int32_t) voices.size() ? std::next(voices.begin(), idx)->first.c_str() : nullptr;
+ const auto & model = ctx->ctx_gen_a->model;
+ const auto & voices = model.gen_voices;
+ if (idx >= (int32_t) voices.size()) {
+ return nullptr;
+ }
+ // the default voice goes first, the others keep their order
+ const auto def = voices.find(model.hparams.gen_default_voice);
+ if (def != voices.end() && idx == 0) {
+ return def->first.c_str();
+ }
+ int32_t i = def != voices.end() ? 1 : 0;
+ for (auto it = voices.begin(); it != voices.end(); ++it) {
+ if (it != def && i++ == idx) {
+ return it->first.c_str();
+ }
+ }
+ return nullptr;
}
mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * name) {
@@ -2871,6 +2894,14 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
inp.top_p = 0.9f;
inp.temp = 1.0f;
break;
+ case PROJECTOR_TYPE_ZONOS3_GEN:
+ // sampled in the helper, defaults of registries/generation.json
+ inp.top_k = 80;
+ inp.top_p = 0.69f;
+ inp.temp = 0.75f;
+ inp.penalty_repeat = 2.4f;
+ inp.penalty_last_n = 96;
+ break;
case PROJECTOR_TYPE_LFM3A_GEN:
{
// MaskGIT sampling runs in the helper, with the settings of the checkpoint
@@ -2902,6 +2933,9 @@ struct mtmd_gen_state {
// lfm3a: the codec graph yields STFT frames, the ISTFT runs on the host
std::unique_ptr<mtmd_audio_streaming_istft> istft;
+ // zonos3: the iSTFT head runs on the host
+ std::unique_ptr<mtmd_audio_zonos3_synth> zonos3_synth;
+
~mtmd_gen_state() {
delete gen_state;
}
@@ -2941,6 +2975,60 @@ void mtmd_gen_state_reset(mtmd_gen_state * state) {
const auto * hparams = &state->sctx->model.hparams;
state->istft = std::make_unique<mtmd_audio_streaming_istft>(hparams->audio_n_fft, hparams->audio_hop_len);
}
+ state->zonos3_synth.reset();
+}
+
+// zonos3 GEN_WAV: n_codes == 0 flushes and resets the state
+static int32_t mtmd_gen_wav_zonos3(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
+ mtmd_subcontext * sctx = ctx->ctx_gen_a.get();
+ const auto & z = sctx->model.zonos3;
+ using g = mtmd_graph_zonos3_gen;
+
+ if (!state->zonos3_synth) {
+ state->zonos3_synth = std::make_unique<mtmd_audio_zonos3_synth>(z.mixer, z.resample, g::n_fft[0], g::n_fft[1], g::hop, g::rs_orig, g::rs_new);
+ }
+ auto & audio = state->out_audio;
+ audio.clear();
+
+ if (!inp->codes || inp->n_codes == 0) {
+ state->zonos3_synth->flush(audio);
+ mtmd_gen_state_reset(state);
+ } else {
+ if (inp->n_codes % g::n_codebooks != 0) {
+ LOG_ERR("%s: expected whole frames of %d codes\n", __func__, g::n_codebooks);
+ return 1;
+ }
+ std::vector<int32_t> in_codes(inp->codes, inp->codes + inp->n_codes);
+ std::vector<float> spec;
+
+ // no hidden-state input, the batch entry is an unused placeholder
+ mtmd_image_f32 dummy;
+ dummy.set_size({1, 1}, false, true);
+ dummy.cpy_buf(std::vector<float>(1, 0.0f));
+
+ mtmd_image_f32_batch batch;
+ batch.is_audio = true;
+ batch.entries.push_back(std::move(dummy));
+
+ mtmd_subctx_encode_params params;
+ params.imgs = &batch;
+ params.n_threads = ctx->n_threads;
+ params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV;
+ params.codes = &in_codes;
+ params.out_feats = &spec;
+ params.gen_state = state->gen_state;
+ if (!sctx->encode(¶ms)) {
+ LOG_ERR("%s: encode failed (codec)\n", __func__);
+ return 1;
+ }
+ const int64_t n_rows = (g::n_fft[0] + 2) + (g::n_fft[1] + 2);
+ GGML_ASSERT(spec.size() % (size_t) n_rows == 0);
+ state->zonos3_synth->push(spec.data(), (int64_t) (spec.size() / n_rows), audio);
+ }
+
+ out->audio = audio.data();
+ out->n_samples = audio.size();
+ return 0;
}
// lfm3a GEN_WAV: ISTFT of the codec output, n_codes == 0 flushes and resets the state
@@ -3042,6 +3130,9 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_WAV && sctx->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN) {
return mtmd_gen_wav_lfm3a(ctx, state, inp, out);
}
+ if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_WAV && sctx->model.proj_type == PROJECTOR_TYPE_ZONOS3_GEN) {
+ return mtmd_gen_wav_zonos3(ctx, state, inp, out);
+ }
if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE) {
const size_t n_embd = (size_t) sctx->model.n_mmproj_embd();
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index e14f345a6..b0d673dbe 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -472,6 +472,7 @@ enum mtmd_gen_audio_type {
MTMD_GEN_AUDIO_TYPE_LFM3A,
MTMD_GEN_AUDIO_TYPE_CHATTERBOX,
MTMD_GEN_AUDIO_TYPE_DRAMA3,
+ MTMD_GEN_AUDIO_TYPE_ZONOS3,
};
struct mtmd_gen_audio_info {
@@ -485,7 +486,7 @@ MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context *
// read a named tensor of the gen audio model as F32, returns the element count (0 if not found), out may be null to query the size
MTMD_API size_t mtmd_gen_audio_read_tensor(mtmd_context * ctx, const char * name, float * out, size_t n_max);
-// name of the idx-th predefined voice, nullptr when idx is out of range
+// name of the idx-th predefined voice, nullptr when idx is out of range; idx 0 is the default voice
MTMD_API const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx);
// predefined voice with this name as a voice bitmap, to pass as speaker reference
@@ -516,6 +517,7 @@ struct mtmd_gen_inp {
// for MTMD_GEN_PROCESS_TYPE_GEN_WAV
// pass either codes (discrete) or feats (continuous), depending on the pipeline
// lfm3a GEN_CODE also takes codes: the codes revealed so far, -1 for a masked codebook
+ // zonos3 GEN_WAV: -1 for a residual code not known yet, the frame is decoded without it
int32_t * codes;
size_t n_codes;
const float * feats;
diff --git a/tools/server/README.md b/tools/server/README.md
index 753530a86..13d41d884 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -874,6 +874,10 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For the list
`seed`: RNG seed for both the backbone sampler and the codec/vocoder (`-1` = random, the default). Set to a fixed value for reproducible output.
+`speed`: Speaking speed relative to a typical pace, `> 0` (default `1.0`, the model's own pace). Only for models that support it, others reject a value other than `1.0` with a 400 error.
+
+`loudness`: Target loudness in LUFS, `< 0` (default `0.0`, the model's own loudness). Only for models that support it, others reject a value other than `0.0` with a 400 error.
+
`n_predict`: Max number of audio frames to generate. Defaults to `512`; generation normally stops earlier once the model emits an end-of-speech token.
`response_format`: `wav` (default), `pcm` (raw `float32` little-endian mono samples, no header; the response `Content-Type` carries the sample rate, e.g. `audio/pcm;rate=24000;encoding=float;bits=32`) or `pcm16` (same, as signed 16-bit samples, e.g. `audio/pcm;rate=24000;encoding=signed-int;bits=16`).
@@ -1668,7 +1672,7 @@ Generates speech from text, for models that can generate audio. The request is c
`stream_format`: `audio` streams the response as it becomes available, see `stream` in `/tts` for the WAV header it writes. `sse` is rejected. The response is not streamed when omitted.
-`speed`: Only `1.0` is accepted.
+`speed`: Speaking speed, see `speed` in `/tts`. Only models that support it accept a value other than `1.0`. `loudness` of `/tts` can be used as well.
`instructions`: Ignored.
diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp
index 4e4622baf..af1e42dbd 100644
--- a/tools/server/server-chat.cpp
+++ b/tools/server/server-chat.cpp
@@ -799,10 +799,6 @@ json convert_speech_to_tts(const json & inp_body) {
throw std::invalid_argument("Only 'audio' stream_format is supported for speech");
}
- if (json_value(inp_body, "speed", 1.0) != 1.0) {
- throw std::invalid_argument("'speed' is not supported for speech");
- }
-
if (!json_value(inp_body, "instructions", std::string()).empty()) {
SRV_WRN("%s", "'instructions' is not supported for speech and is ignored\n");
}
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 7c2c35c35..28e01c69b 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -1951,7 +1951,12 @@ private:
// mtmd_helper always re-eval the whole prompt
slot.prompt_clear();
task.tts_inp.data.seq_id = slot.id;
- if (slot.tts.ctx.set_input(task.tts_inp.get()) != 0) {
+ const int32_t ret = slot.tts.ctx.set_input(task.tts_inp.get());
+ if (ret == 2) {
+ send_error(task, "invalid TTS input: the model may not support \"speed\" or \"loudness\", see the server log", ERROR_TYPE_INVALID_REQUEST);
+ return false;
+ }
+ if (ret != 0) {
send_error(task, "failed to process TTS prompt", ERROR_TYPE_SERVER);
return false;
}
@@ -5650,7 +5655,7 @@ static json parse_tts_body(const server_http_req & req) {
body[k] = std::stoll(body[k].get<std::string>());
}
}
- for (const char * k : { "top_p", "temperature", "repeat_penalty", "speed" }) {
+ for (const char * k : { "top_p", "temperature", "repeat_penalty", "speed", "loudness" }) {
if (body.contains(k) && body[k].is_string()) {
body[k] = std::stof(body[k].get<std::string>());
}
diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp
index 84e872f21..e72617d6d 100644
--- a/tools/server/server-schema.cpp
+++ b/tools/server/server-schema.cpp
@@ -555,6 +555,12 @@ std::vector<std::unique_ptr<field>> make_tts_schema(server_task & task) {
add((new field_num("seed", task.tts_inp.data.seed))
->set_desc("RNG seed for the backbone sampler and the codec/vocoder (-1 = random)"));
+ add((new field_num("speed", task.tts_inp.data.speed))
+ ->set_desc("Speaking speed relative to a typical pace, for models that support it (1.0 = model default)"));
+
+ add((new field_num("loudness", task.tts_inp.data.loudness))
+ ->set_desc("Target loudness in LUFS, for models that support it (0.0 = model default)"));
+
add((new field_num("n_predict", task.params.n_predict))
->set_hard_limits(-1, INT32_MAX)
->add_alias("max_tokens")