Commit 307e3f155 for llama.cpp
commit 307e3f1551c2c3c6c0560c454ab46bb64baeb2df
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Tue Nov 17 22:25:44 2026 +0100
mtmd: add kokoro tts support (#32413)
* jinja: impl scope-chain key lookup
* add test for call_statement lambda
* chat: add tts_dict for kokoro G2P template
* more testing
* value_float_t range check
* no json-in-meta, add llama_model_meta_arr_str/_n
* mtmd: add kokoro tts support
* detailed guide about G2P and text norm port
* refine G2P
* clean up
* migrate batch api
* nits
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 0105099eb..9fc95f797 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -283,6 +283,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"Qwen3NextForCausalLM": "qwen",
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
"PocketTTSModel": "pockettts",
+ "KokoroModel": "kokoro",
"Zonos3Model": "zonos3",
"YuE2ForCausalLM": "yue2",
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
@@ -423,6 +424,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"Qwen3ASRForConditionalGeneration": "qwen3vl",
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
"PocketTTSModel": "pockettts",
+ "KokoroModel": "kokoro",
"SopranoModel": "soprano",
"Qwen3TTSForConditionalGeneration": "qwen3tts",
"Qwen3VLForConditionalGeneration": "qwen3vl",
diff --git a/conversion/kokoro.py b/conversion/kokoro.py
new file mode 100644
index 000000000..945eccdaa
--- /dev/null
+++ b/conversion/kokoro.py
@@ -0,0 +1,419 @@
+from __future__ import annotations
+
+import importlib
+import importlib.util
+import json
+import math
+import re
+from collections import Counter, defaultdict
+from concurrent.futures import ProcessPoolExecutor
+from pathlib import Path
+from typing import Any, Callable, Iterable
+
+import torch
+from torch import Tensor
+
+from .base import ModelBase, TextModel, MmprojModel, gguf, logger
+
+# Kokoro (StyleTTS2 with an iSTFTNet decoder), https://huggingface.co/hexgrad/Kokoro-82M
+# text model: the PL-BERT phoneme encoder (ALBERT), one token per phoneme character
+# mmproj: bert_encoder, prosody predictor, text encoder, decoder and the voices
+# the text to phonemes step is the "tts" chat template, its dictionaries come from the misaki and nltk python packages
+
+DEFAULT_VOICE = "af_heart"
+
+MODULE_PREFIX = {
+ "bert_encoder": "a.gen.bert_enc.",
+ "predictor": "a.gen.pred.",
+ "text_encoder": "a.gen.text.",
+ "decoder": "a.gen.dec.",
+}
+
+
+def _checkpoint(dir_model: Path) -> Path | None:
+ paths = sorted(dir_model.glob("kokoro-*.pth"))
+ return paths[0] if paths else None
+
+
+def _is_kokoro_checkpoint(dir_model: Path) -> bool:
+ if _checkpoint(dir_model) is None or not (dir_model / "config.json").is_file():
+ return False
+ with open(dir_model / "config.json", encoding="utf-8") as f:
+ config = json.load(f)
+ return "plbert" in config and "istftnet" in config and "vocab" in config
+
+
+@ModelBase.register_hparams_loader(_is_kokoro_checkpoint)
+def _load_kokoro_hparams(dir_model: Path) -> dict[str, Any]:
+ logger.info("gguf: detected Kokoro checkpoint")
+ with open(dir_model / "config.json", encoding="utf-8") as f:
+ config = json.load(f)
+ plbert = config["plbert"]
+ return {
+ "architectures": ["KokoroModel"],
+ "model_type": "kokoro",
+ "num_hidden_layers": plbert["num_hidden_layers"],
+ "hidden_size": plbert["hidden_size"],
+ "intermediate_size": plbert["intermediate_size"],
+ "num_attention_heads": plbert["num_attention_heads"],
+ "max_position_embeddings": plbert["max_position_embeddings"],
+ "vocab_size": config["n_token"],
+ "layer_norm_eps": 1e-12, # AlbertConfig default
+ "kokoro": config,
+ # the mmproj tensors are named explicitly, they do not go through the block tensor map
+ "audio_config": {"num_hidden_layers": 0},
+ }
+
+
+def _load_modules(dir_model: Path, modules: Iterable[str]) -> dict[str, dict[str, Tensor]]:
+ path = _checkpoint(dir_model)
+ assert path is not None
+ state = torch.load(path, map_location="cpu", mmap=True, weights_only=True)
+ # every module was saved from a DataParallel wrapper
+ return {m: {k.removeprefix("module."): v for k, v in state[m].items()} for m in modules}
+
+
+def _g2p_fallback(lexicons: list[dict[str, Any]]) -> dict[str, Any]:
+ # model for the words missing from the lexicon: n-grams over units "letter:sounds", learned from the lexicon
+ # key "<0 to 4 units before> <letter>" -> {sounds: probability}; "<s>" pads the start of a word, the letter "$" is its end
+ # the "tts" template decodes it with a beam search
+ order, min_count, stress = 5, 3, "\u02c8\u02cc"
+ vowels = set("AIOQWYaiu\u00e6\u0251\u0252\u0254\u0259\u025b\u025c\u026a\u028a\u028c\u1d7b\u1d4a")
+
+ def sounds(ps: str) -> list[str]:
+ # a stress mark goes with the sound after it, a length mark with the one before
+ out: list[str] = []
+ pre = ""
+ for c in ps:
+ if c in stress:
+ pre += c
+ elif c == "\u02d0" and out:
+ out[-1] += c
+ else:
+ out.append(pre + c)
+ pre = ""
+ return out
+
+ def penalty(letter: str, chunk: str) -> float:
+ # a consonant letter says consonants only; a vowel letter says a vowel, with at most a glide
+ flat = [c for c in chunk if c not in stress and c != "\u02d0"]
+ n_vowel = sum(c in vowels for c in flat)
+ if letter in "bcdfgjkmnpqstvxz" and n_vowel > 0:
+ return -8.0
+ if letter in "aeiou" and flat and (n_vowel == 0 or any(c not in vowels and c not in "jw" for c in flat)):
+ return -8.0
+ return 0.0
+
+ words: dict[str, list[str]] = {}
+ for lexicon in lexicons:
+ for word, v in lexicon.items():
+ ps = v["DEFAULT"] if isinstance(v, dict) else v
+ # same spelling normalization as the template: doubled consonants and "ck" say one sound
+ w = re.sub(r"([b-df-hj-np-tv-z])\1", r"\1", word.lower()).replace("ck", "k")
+ if ps and " " not in ps and len(w) >= 2 and re.fullmatch(r"[a-z']+", w) and w not in words:
+ t = sounds(ps)
+ if len(t) <= 2 * len(w):
+ words[w] = t
+
+ def align(w: str, t: list[str], cost: dict[str, dict[str, float]]) -> list[str] | None:
+ # best split of the sounds over the letters, each letter takes 0, 1 or 2 of them
+ n_l, n_t = len(w), len(t)
+ two = [t[j] + t[j + 1] for j in range(n_t - 1)]
+ best = [[-1e18] * (n_t + 1) for _ in range(n_l + 1)]
+ back = [[0] * (n_t + 1) for _ in range(n_l + 1)]
+ best[0][0] = 0.0
+ for i in range(n_l):
+ letter = w[i]
+ row = cost.setdefault(letter, {})
+ cur, nxt, bk = best[i], best[i + 1], back[i + 1]
+ # the sounds left must fit in the letters left
+ for j in range(max(0, n_t - 2 * (n_l - i)), n_t + 1):
+ base = cur[j]
+ if base < -1e17:
+ continue
+ for k, chunk in ((0, ""), (1, t[j] if j < n_t else None), (2, two[j] if j + 1 < n_t else None)):
+ if chunk is None:
+ break
+ c = row.get(chunk)
+ if c is None:
+ # not seen yet for this letter
+ c = row[chunk] = -12.0 + penalty(letter, chunk)
+ if base + c > nxt[j + k]:
+ nxt[j + k] = base + c
+ bk[j + k] = k
+ if best[n_l][n_t] < -1e17:
+ return None
+ out: list[str] = []
+ j = n_t
+ for i in range(n_l, 0, -1):
+ k = back[i][j]
+ out.append("".join(t[j - k:j]))
+ j -= k
+ return out[::-1]
+
+ # hard EM: align with the current letter -> sounds costs, count, repeat
+ # the costs are learned on a sample of the words, then every word is aligned once
+ def step(items: Iterable[tuple[str, list[str]]], cost: dict[str, dict[str, float]]) -> list[tuple[str, list[str]]]:
+ return [(w, a) for w, a in ((w, align(w, t, cost)) for w, t in items) if a is not None]
+
+ sample = list(words.items())[::max(1, len(words) // 20000)]
+ cost: dict[str, dict[str, float]] = {}
+ for _ in range(5):
+ counts: dict[str, Counter[str]] = defaultdict(Counter)
+ for w, a in step(sample, cost):
+ for c, o in zip(w, a):
+ counts[c][o] += 1
+ # small cost for a silent letter, so that a sound stays on its own letter
+ cost = {c: {o: math.log(n / sum(cnt.values())) - (0.5 if o == "" else 0.0) + penalty(c, o) for o, n in cnt.items()}
+ for c, cnt in counts.items()}
+ aligned = step(words.items(), cost)
+
+ # counts of a unit after each of its contexts, from 4 units down to none
+ ngram: dict[str, Counter[str]] = defaultdict(Counter)
+ n_ctx: Counter[str] = Counter()
+ for w, a in aligned:
+ units = ["<s>"] * (order - 1) + [c + ":" + o for c, o in zip(w, a)] + ["$:"]
+ for i in range(order - 1, len(units)):
+ letter, _, o = units[i].partition(":")
+ for n in range(order):
+ ctx = " ".join(units[i - n:i])
+ ngram[(ctx + " " if ctx else "") + letter][o] += 1
+ n_ctx[ctx] += 1
+
+ # the probability is over all the units after the context, not only the ones of this letter
+ # this tells the search how likely the letter is after the sounds it chose, so it can drop a bad choice
+ model: dict[str, Any] = {}
+ for key, cnt in ngram.items():
+ ctx, _, letter = key.rpartition(" ")
+ # the two shortest contexts keep all the readings that the letter has min_count times or more
+ kept = {o: float(f"{n / n_ctx[ctx]:.4g}") for o, n in cnt.most_common()
+ if n >= min_count or (key.count(" ") < 2 and ngram[letter][o] >= min_count)}
+ if kept:
+ model[key] = kept
+ return model
+
+
+def _g2p_dicts() -> dict[str, dict[str, Any]] | None:
+ # dictionaries of the "tts" template: the misaki lexicon and the nltk averaged perceptron tagger
+ spec = importlib.util.find_spec("misaki")
+ if spec is None or spec.origin is None:
+ logger.warning("misaki is not installed, the model will take phonemes as prompt (pip install misaki nltk)")
+ return None
+ data = Path(spec.origin).parent / "data"
+ try:
+ nltk = importlib.import_module("nltk")
+ base = Path(str(nltk.data.find("taggers/averaged_perceptron_tagger_eng"))) / "averaged_perceptron_tagger_eng"
+ except (ImportError, LookupError):
+ logger.warning("the nltk tagger is not available, the model will take phonemes as prompt "
+ "(pip install nltk && python -m nltk.downloader averaged_perceptron_tagger_eng)")
+ return None
+
+ def load(path: Path) -> Any:
+ with open(path, encoding="utf-8") as f:
+ return json.load(f)
+
+ # tagger: classes sorted by label, feature -> {"c<class index>": weight}
+ # (index keys: jinja2's map(attribute=...) splits on "." and some labels are punctuation)
+ classes = sorted(load(Path(f"{base}.classes.json")))
+ cid = {c: f"c{i}" for i, c in enumerate(classes)}
+ weights = load(Path(f"{base}.weights.json"))
+ lex = {
+ "gold": load(data / "us_gold.json"),
+ "silver": load(data / "us_silver.json"),
+ "gold_gb": load(data / "gb_gold.json"),
+ "silver_gb": load(data / "gb_silver.json"),
+ }
+ logger.info("gguf: learning the pronunciation of unknown words from the lexicon, this takes a few minutes")
+ # one process per variant
+ with ProcessPoolExecutor(max_workers=2) as pool:
+ lts, lts_gb = pool.map(_g2p_fallback, [[lex["gold"], lex["silver"]], [lex["gold_gb"], lex["silver_gb"]]])
+ return {
+ **lex,
+ "lts": lts,
+ "lts_gb": lts_gb,
+ "pos_weights": {f: {cid[t]: w for t, w in tw.items()} for f, tw in weights.items()},
+ "pos_tagdict": load(Path(f"{base}.tagdict.json")),
+ "pos_classes": {"": classes},
+ }
+
+
+@ModelBase.register("KokoroModel")
+@ModelBase.example("hexgrad/Kokoro-82M")
+class KokoroTextModel(TextModel):
+ model_arch = gguf.MODEL_ARCH.KOKORO
+
+ _LAYER = "encoder.albert_layer_groups.0.albert_layers.0."
+ _TENSOR_MAP = {
+ "embeddings.LayerNorm": gguf.MODEL_TENSOR.TOKEN_EMBD_NORM,
+ "encoder.embedding_hidden_mapping_in": gguf.MODEL_TENSOR.TOKEN_EMBD_PROJ,
+ }
+ # the layers share their weights, stored as block 0
+ _LAYER_TENSOR_MAP = {
+ "attention.query": gguf.MODEL_TENSOR.ATTN_Q,
+ "attention.key": gguf.MODEL_TENSOR.ATTN_K,
+ "attention.value": gguf.MODEL_TENSOR.ATTN_V,
+ "attention.dense": gguf.MODEL_TENSOR.ATTN_OUT,
+ "attention.LayerNorm": gguf.MODEL_TENSOR.ATTN_OUT_NORM,
+ "ffn": gguf.MODEL_TENSOR.FFN_UP,
+ "ffn_output": gguf.MODEL_TENSOR.FFN_DOWN,
+ "full_layer_layer_norm": gguf.MODEL_TENSOR.LAYER_OUT_NORM,
+ }
+
+ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
+ del remote_hf_model_id
+ bert = _load_modules(self.dir_model, ["bert"])["bert"]
+ return {name: (lambda t=t: t) for name, t in bert.items()}
+
+ def set_vocab(self):
+ config = self.hparams["kokoro"]
+ n_vocab = config["n_token"]
+ tokens: list[str] = [f"[UNUSED_{i}]" for i in range(n_vocab)]
+ toktypes = [gguf.TokenType.UNUSED] * n_vocab
+ for text, i in config["vocab"].items():
+ tokens[i] = text
+ toktypes[i] = gguf.TokenType.NORMAL
+ # id 0 pads both ends of the phoneme sequence
+ tokens[0] = "<pad>"
+ toktypes[0] = gguf.TokenType.CONTROL
+
+ # the phonemes are mapped to tokens one character at a time by the audio pipeline
+ self.gguf_writer.add_tokenizer_model("llama")
+ self.gguf_writer.add_tokenizer_pre("default")
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_scores([0.0] * n_vocab)
+ self.gguf_writer.add_token_types(toktypes)
+ self.gguf_writer.add_bos_token_id(0)
+ self.gguf_writer.add_eos_token_id(0)
+ self.gguf_writer.add_pad_token_id(0)
+ self.gguf_writer.add_add_bos_token(False)
+ self.gguf_writer.add_add_eos_token(False)
+
+ dicts = _g2p_dicts()
+ if dicts is not None:
+ template_path = Path(__file__).parent.parent / "models" / "templates" / "hexgrad-Kokoro-82M-tts.jinja"
+ with open(template_path, encoding="utf-8") as f:
+ self.gguf_writer.add_chat_template([{"name": "tts", "template": f.read()}])
+ self.gguf_writer.add_tts_dicts(dicts)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_causal_attention(False)
+ self.gguf_writer.add_pooling_type(gguf.PoolingType.NONE)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ del bid
+ n_embd = self.hparams["hidden_size"]
+
+ if name == "embeddings.word_embeddings.weight":
+ # zero-pad the rows to n_embd, llama.cpp wants token and embedding inputs of the same width
+ data_torch = torch.nn.functional.pad(data_torch, (0, n_embd - data_torch.shape[1]))
+ yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), data_torch)
+ return
+ if name == "embeddings.position_embeddings.weight":
+ # every token has type 0, fold its embedding into the positions
+ type0 = self.model_tensors["embeddings.token_type_embeddings.weight"]()[0]
+ yield (self.format_tensor_name(gguf.MODEL_TENSOR.POS_EMBD), data_torch + type0)
+ return
+ if name.startswith(("embeddings.token_type_embeddings.", "pooler.")):
+ return
+
+ key, suffix = name.rsplit(".", 1)
+ if key in self._TENSOR_MAP:
+ yield (self.format_tensor_name(self._TENSOR_MAP[key], suffix="." + suffix), data_torch)
+ return
+ if key.startswith(self._LAYER):
+ tensor = self._LAYER_TENSOR_MAP[key.removeprefix(self._LAYER)]
+ yield (self.format_tensor_name(tensor, 0, suffix="." + suffix), data_torch)
+ return
+ raise ValueError(f"unexpected tensor {name}")
+
+
+@ModelBase.register("KokoroModel")
+@ModelBase.example("hexgrad/Kokoro-82M")
+class KokoroMmprojModel(MmprojModel):
+ has_vision_encoder = False
+ has_audio_encoder = False
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._wnorm_g: dict[str, Tensor] = {}
+ self._wnorm_v: dict[str, Tensor] = {}
+ self._lstm_bias: dict[str, Tensor] = {}
+
+ def get_audio_config(self) -> dict[str, Any] | None:
+ return self.global_config.get("audio_config")
+
+ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
+ del remote_hf_model_id
+ tensors: dict[str, Callable[[], Tensor]] = {}
+ for module, state in _load_modules(self.dir_model, MODULE_PREFIX.keys()).items():
+ for name, t in state.items():
+ tensors[MODULE_PREFIX[module] + name] = (lambda t=t: t)
+ return tensors
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ # the names are final already
+ return item
+
+ def set_gguf_parameters(self):
+ config = self.global_config["kokoro"]
+ self.gguf_writer.add_file_type(self.ftype)
+ self.gguf_writer.add_clip_has_gen_audio_encoder(True)
+ self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.KOKORO)
+ self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
+ self.gguf_writer.add_gen_audio_embedding_length(config["hidden_dim"])
+ self.gguf_writer.add_gen_audio_feed_forward_length(config["hidden_dim"])
+ self.gguf_writer.add_gen_audio_block_count(config["n_layer"])
+ self.gguf_writer.add_gen_audio_head_count(1)
+ self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5)
+ if (self.dir_model / "voices" / f"{DEFAULT_VOICE}.pt").is_file():
+ self.gguf_writer.add_gen_audio_default_voice(DEFAULT_VOICE)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ del bid
+ # torch weight_norm: weight = g * v / |v| over dims 1..n
+ if name.endswith((".weight_g", ".weight_v")):
+ base = name[:-len(".weight_g")]
+ (self._wnorm_g if name.endswith("_g") else self._wnorm_v)[base] = data_torch
+ if base in self._wnorm_g and base in self._wnorm_v:
+ g = self._wnorm_g.pop(base).float()
+ v = self._wnorm_v.pop(base).float()
+ w = g * v / v.norm(dim=tuple(range(1, v.dim())), keepdim=True)
+ if base.endswith(".pool"):
+ # depthwise transposed conv [C, 1, 3], stored as one row per tap
+ w = w.squeeze(1).T.contiguous()
+ yield (base + ".weight", w)
+ return
+ # the two LSTM biases are always added
+ if ".bias_ih_l0" in name or ".bias_hh_l0" in name:
+ base = name.replace(".bias_ih_l0", ".bias_l0").replace(".bias_hh_l0", ".bias_l0")
+ if base in self._lstm_bias:
+ yield (base, self._lstm_bias.pop(base).float() + data_torch.float())
+ else:
+ self._lstm_bias[base] = data_torch
+ return
+ # snake alphas are per channel
+ if ".alpha1." in name or ".alpha2." in name:
+ data_torch = data_torch.reshape(-1)
+ yield (name, data_torch)
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ del name, bid
+ # tensors read on the host stay F32
+ if "lstm" in new_name or ".shared." in new_name or new_name.startswith(("a.gen.bert_enc.", "voice.")) \
+ or ".duration_proj." in new_name or ".m_source." in new_name or ".pool." in new_name:
+ return gguf.GGMLQuantizationType.F32
+ # CONV_TRANSPOSE_1D takes only F32 weights on some backends
+ if ".ups." in new_name:
+ return gguf.GGMLQuantizationType.F32
+ # the conv ops have no BF16 kernels
+ if n_dims >= 2:
+ return gguf.GGMLQuantizationType.F16
+ return gguf.GGMLQuantizationType.F32
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ # a voice is a table of style vectors, the row is picked by the number of phonemes
+ for path in sorted((self.dir_model / "voices").glob("*.pt")):
+ pack = torch.load(path, map_location="cpu", weights_only=True)
+ yield (f"voice.{path.stem}", pack.squeeze(1).float())
diff --git a/docs/multimodal/tts.md b/docs/multimodal/tts.md
index 5ebc10356..45e838da5 100644
--- a/docs/multimodal/tts.md
+++ b/docs/multimodal/tts.md
@@ -15,6 +15,7 @@ Supported models:
- [Fish Audio S2 Pro](#fish-audio-s2-pro)
- [YuE2](#yue2) (songs)
- [MiniMax Music 3](#minimax-music-3) (songs)
+- [Kokoro](#kokoro)
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
@@ -363,3 +364,62 @@ python convert_hf_to_gguf.py MiniMax-Music3 --mmproj --outfile mmproj-minimax-mu
```
The mmproj holds the depth decoder and the DiT layers (in the `--outtype` type), and the condition encoder, the small DiT projections and the DAC decoder, always in F32.
+
+## Kokoro
+
+Available params:
+- `--tts-voice` selects one of the voices shipped with the weights (default: `af_heart`); `llama-tts` lists them when the name is unknown. The first letter of a voice is its language: `a` American English, `b` British English, `e` Spanish, `f` French, `h` Hindi, `i` Italian, `p` Brazilian Portuguese, `j` Japanese, `z` Mandarin
+- `--tts-lang` can be `en-us` or `en-gb`; when it is not given, the voices named `b*` speak British English and those named `a*` American English
+- Note: `--tts-speaker-file` is not supported, the model has no voice cloning
+- Note: sampling params are not used, the model is not autoregressive. `--seed` only changes the noise of the vocoder source
+- `speed` (`llama-server` only) changes the speaking rate, e.g. `1.5`; `loudness` is not supported
+
+Example usage:
+
+```sh
+llama-tts -m kokoro.gguf \
+ -mm mmproj-kokoro.gguf \
+ -p "Hello world" \
+ --tts-voice am_michael \
+ --output out.wav
+```
+
+The model speaks phonemes. Only English has a text to phonemes step (the model's `tts` chat template, with the pronunciation dictionary stored in the model):
+- the pronunciation of a word missing from the dictionary (mostly rare names) is predicted from its spelling, by a model that the converter learns from the dictionary. The reference uses espeak-ng for these words; the prediction is about as accurate, but it is not the same, so such a word can sound different from the reference
+- a text longer than 510 phonemes is split at sentence ends and spoken segment by segment
+
+**Other languages, or your own phonemes:**
+
+The prompt of a voice of another language must already be phonemes, in the symbols of the model (see `vocab` in its `config.json`). Run the text to phonemes step in your own code, for example with [espeak-ng](https://github.com/espeak-ng/espeak-ng), which the reference uses for Spanish, French, Hindi, Italian and Brazilian Portuguese:
+
+```sh
+PHONEMES=$(echo "El gobierno anunció nuevas medidas." | espeak-ng -q --ipa -v es --stdin | tr '\n' ' ')
+llama-tts -m kokoro.gguf \
+ -mm mmproj-kokoro.gguf \
+ -p "$PHONEMES" \
+ --tts-voice ef_dora \
+ --output out.wav
+```
+
+Notes:
+- espeak-ng drops the punctuation, which the model uses for pauses: phonemize the text between the marks and put them back for better prosody
+- the reference also replaces a few sounds by single symbols (e.g. `tʃ` by `ʧ`), see `espeak.py` of [misaki](https://github.com/hexgrad/misaki)
+- with `llama-server`, send `"raw_prompt": true` to pass phonemes with any voice, including the English ones
+
+**Note for GGUF conversion:**
+
+Download `config.json`, `kokoro-v1_0.pth` and the `voices/*.pt` files you want from [Kokoro-82M](https://huggingface.co/hexgrad/Kokoro-82M), every voice in the directory is stored in the mmproj:
+
+```sh
+python convert_hf_to_gguf.py path/to/Kokoro-82M --outfile kokoro.gguf
+python convert_hf_to_gguf.py path/to/Kokoro-82M --mmproj --outfile mmproj-kokoro.gguf
+```
+
+The pronunciation dictionary and the part-of-speech tagger of the `tts` template come from the `misaki` and `nltk` python packages. The converter also learns the spelling-to-sound model for unknown words from that dictionary, which adds about 15 seconds:
+
+```sh
+pip install misaki nltk
+python -m nltk.downloader averaged_perceptron_tagger_eng
+```
+
+Without them the converter writes no `tts` template, and the prompt must be a phoneme string (as produced by misaki).
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index d15ca9a5f..d55e5a96f 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -770,6 +770,7 @@ class MODEL_ARCH(IntEnum):
QWEN3TTS = auto()
POCKETTTS = auto()
ZONOS3 = auto()
+ KOKORO = auto()
K2HORIZON = auto()
KOLIBRI1 = auto()
LIMITE = auto()
@@ -794,6 +795,7 @@ class VISION_PROJECTOR_TYPE(IntEnum):
class MODEL_TENSOR(IntEnum):
TOKEN_EMBD = auto()
TOKEN_EMBD_NORM = auto()
+ TOKEN_EMBD_PROJ = auto()
MASKED_EMBD_CENTROIDS= auto()
MASKED_EMBD_ORDERING = auto()
TOKEN_TYPES = auto()
@@ -1755,6 +1757,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.QWEN3TTS: "qwen3tts",
MODEL_ARCH.POCKETTTS: "pockettts",
MODEL_ARCH.ZONOS3: "zonos3",
+ MODEL_ARCH.KOKORO: "kokoro",
MODEL_ARCH.K2HORIZON: "k2-horizon",
MODEL_ARCH.KOLIBRI1: "kolibri1",
MODEL_ARCH.LIMITE: "limite",
@@ -1777,6 +1780,7 @@ VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = {
TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.TOKEN_EMBD: "token_embd",
MODEL_TENSOR.TOKEN_EMBD_NORM: "token_embd_norm",
+ MODEL_TENSOR.TOKEN_EMBD_PROJ: "token_embd_proj",
MODEL_TENSOR.TOKEN_TYPES: "token_types",
MODEL_TENSOR.MASKED_EMBD_CENTROIDS: "masked_embd_centroids",
MODEL_TENSOR.MASKED_EMBD_ORDERING: "masked_embd_ordering",
@@ -6862,6 +6866,20 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
],
+ MODEL_ARCH.KOKORO: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.TOKEN_EMBD_PROJ,
+ MODEL_TENSOR.POS_EMBD,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_OUT_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.LAYER_OUT_NORM,
+ ],
MODEL_ARCH.ZONOS3: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
@@ -7337,6 +7355,7 @@ class VisionProjectorType:
MINIMAX_MUSIC3_GEN = "minimax_music3_gen" # audio generation: RVQ depth decoder + flow-matching DiT + DAC decoder
FISH_S2_GEN = "fish_s2_gen" # audio generation: fast AR code predictor + Fish DAC decoder
FISH_S2_SPKENC = "fish_s2_spkenc" # audio: Fish DAC encoder + RVQ, reference clip -> prompt rows
+ KOKORO = "kokoro" # audio generation: StyleTTS2 prosody predictor + iSTFTNet decoder
HUNYUANVL = "hunyuanvl"
PARAKEET = "parakeet" # audio
MINIMAXM3 = "minimax_m3"
diff --git a/models/templates/hexgrad-Kokoro-82M-tts.jinja b/models/templates/hexgrad-Kokoro-82M-tts.jinja
new file mode 100644
index 000000000..7b129cbe1
--- /dev/null
+++ b/models/templates/hexgrad-Kokoro-82M-tts.jinja
@@ -0,0 +1,585 @@
+{#- Kokoro "tts" template: english text -> phonemes, a port of the english path of misaki (the reference G2P)
+ functions: tts_norm_scan(), tts_norm_sub(), tts_dict(), see docs/tts-norm-patterns.md
+ dictionaries, written by conversion/kokoro.py:
+ - gold / silver (US), gold_gb / silver_gb (british): the misaki lexicon, a value is a string or {DEFAULT, <tag>...}
+ - pos_weights / pos_tagdict / pos_classes: the nltk averaged perceptron tagger
+ - lts / lts_gb: model for the words missing from the lexicon, see lts_search()
+ a voice of another language takes phonemes, its prompt passes through -#}
+{%- set unk = unk if unk is defined else '' -%}
+{#- british english: the voices named b*, or language en-gb; it has its own lexicon and a few suffix rules -#}
+{%- set lang = (language if language is defined and language else '') | lower | replace('_', '-') -%}
+{%- set GB = lang in ['en-gb', 'british'] or (not lang and voice is defined and voice and voice[:1] == 'b') -%}
+{%- set GOLD = 'gold_gb' if GB else 'gold' -%}
+{%- set SILVER = 'silver_gb' if GB else 'silver' -%}
+{%- set LTS = 'lts_gb' if GB else 'lts' -%}
+{%- set V = 'AIOQWYaiuæɑɒɔəɛɜɪʊʌᵻ' -%}
+{%- set C = 'bdfhjklmnpstvwzðŋɡɹɾʃʒʤʧθ' -%}
+{%- set NQP = ';:,.!?—…' -%}
+{%- set TAUS = 'AIOWYiuæɑəɛɪɹʊʌ' -%}
+{%- set SYMBOLS = {'%': 'percent', '&': 'and', '+': 'plus', '@': 'at'} -%}
+{#- the lexicon has only ASCII keys: an accented letter is read as its base letter -#}
+{%- set FOLD = [['[àáâãäåāă]', 'a'], ['[ÀÁÂÃÄÅĀĂ]', 'A'], ['[çćč]', 'c'], ['[ÇĆČ]', 'C'], ['[èéêëēėęě]', 'e'], ['[ÈÉÊËĒĖĘĚ]', 'E'],
+ ['[ìíîïīı]', 'i'], ['[ÌÍÎÏĪİ]', 'I'], ['[ñńň]', 'n'], ['[ÑŃŇ]', 'N'], ['[òóôõöøōő]', 'o'], ['[ÒÓÔÕÖØŌŐ]', 'O'],
+ ['[ùúûüūůű]', 'u'], ['[ÙÚÛÜŪŮŰ]', 'U'], ['[ýÿ]', 'y'], ['[ÝŸ]', 'Y'], ['[śšș]', 's'], ['[ŚŠȘ]', 'S'], ['[źżž]', 'z'], ['[ŹŻŽ]', 'Z'],
+ ['ł', 'l'], ['Ł', 'L'], ['ß', 'ss'], ['æ', 'ae'], ['Æ', 'AE'], ['œ', 'oe'], ['Œ', 'OE']] -%}
+{%- set CURRENCIES = {'$': ['dollar', 'cent'], '£': ['pound', 'pence'], '€': ['euro', 'cent']} -%}
+
+{#- ---------- small helpers ---------- -#}
+{%- macro full(s, pat) -%}{%- set segs = tts_norm_scan(s, '^(?:' ~ pat ~ ')$') -%}{{ '1' if segs | length == 1 and segs[0].rule else '' }}{%- endmacro -%}
+{#- key of w in a lexicon, or ''; like misaki's grow_dictionary(), a lowercase entry also answers for Capitalized and vice versa -#}
+{%- macro dkey(name, w) -%}
+{%- if tts_dict(name, w) is not none -%}{{ w }}
+{%- elif w | length > 1 -%}
+ {%- set l = w | lower -%}{%- set c = l | capitalize -%}
+ {%- if w == c and w != l and tts_dict(name, l) is not none -%}{{ l }}
+ {%- elif w == l and w != c and tts_dict(name, c) is not none -%}{{ c }}
+ {%- endif -%}
+{%- endif -%}
+{%- endmacro -%}
+{%- macro has(w) -%}{{ '1' if dkey(GOLD, w) or dkey(SILVER, w) else '' }}{%- endmacro -%}
+{#- default pronunciation in gold, or '' -#}
+{%- macro gold(w) -%}
+{%- set k = dkey(GOLD, w) -%}
+{%- if k -%}{%- set e = tts_dict(GOLD, k) -%}{{ e.DEFAULT if e is mapping else e }}{%- endif -%}
+{%- endmacro -%}
+
+{#- ---------- words missing from the lexicon ---------- -#}
+{#- beam search over the letters with an n-gram model of units "letter:sounds"
+ tts_dict(LTS, "<0 to 4 units before> <letter>") -> {sounds: probability}; "<s>" pads the start, the letter "$" is the end
+ a unit takes its score from the longest context that has it, times 0.4 for each unit dropped
+ fixed is none, or the sounds of each letter without stress: then only the stress is searched, with exactly one primary stress
+ returns the sounds of each letter, joined by "|" -#}
+{%- macro lts_search(word, fixed) -%}
+{%- set bm = namespace(l=[{'p': 1.0, 'h': ['<s>', '<s>', '<s>', '<s>'], 'o': [], 's': false}]) -%}
+{%- for ch in (tts_norm_scan(word, '.') | map(attribute='text') | list) + ['$'] -%}
+ {%- set i = loop.index0 -%}
+ {%- set nx = namespace(l=[]) -%}
+ {%- for hyp in bm.l -%}
+ {%- set st = namespace(f=1.0, seen=[]) -%}
+ {%- for n in range(4, -1, -1) -%}
+ {%- set d = tts_dict(LTS, (hyp.h[4 - n:] + [ch]) | join(' ')) -%}
+ {%- if d is not none -%}
+ {%- for o, p in d | items -%}
+ {%- if o not in st.seen -%}
+ {%- set _ = st.seen.append(o) -%}
+ {%- set prim = 'ˈ' in o -%}
+ {%- set match = fixed is none or ch == '$' or (o | replace('ˈ', '') | replace('ˌ', '')) == fixed[i] -%}
+ {%- if match and not (fixed is not none and prim and hyp.s) -%}
+ {%- set _ = nx.l.append({'p': hyp.p * st.f * p, 'h': (hyp.h + [ch ~ ':' ~ o])[-4:], 'o': hyp.o + [o], 's': hyp.s or prim}) -%}
+ {%- endif -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- endif -%}
+ {%- set st.f = st.f * 0.4 -%}
+ {%- endfor -%}
+ {%- endfor -%}
+ {%- set srt = nx.l | sort(attribute='p', reverse=true) -%}
+ {%- if fixed is none -%}{%- set bm.l = srt[:16] -%}
+ {%- else -%}{%- set bm.l = (srt | selectattr('s') | list)[:8] + (srt | rejectattr('s') | list)[:8] -%}
+ {%- endif -%}
+{%- endfor -%}
+{%- set done = (bm.l | selectattr('s') | list) or bm.l -%}
+{%- if done -%}{{ (done | sort(attribute='p', reverse=true))[0].o[:-1] | join('|') }}{%- endif -%}
+{%- endmacro -%}
+
+{#- pronunciation of a word that is not in the lexicon, '' if it cannot be read -#}
+{%- macro lts(word) -%}
+{%- set w = namespace(s=word | lower | replace('’', "'")) -%}
+{#- doubled consonants and "ck" say one sound -#}
+{%- for c in ['b', 'c', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'm', 'n', 'p', 'q', 'r', 's', 't', 'v', 'w', 'x', 'y', 'z'] -%}{%- set w.s = w.s | replace(c ~ c, c) -%}{%- endfor -%}
+{%- set w.s = w.s | replace('ck', 'k') -%}
+{%- if w.s | length >= 2 and full(w.s, "[a-z']+") -%}
+ {#- first the sounds, then the stress marks on those sounds -#}
+ {%- set first = lts_search(w.s, none) -%}
+ {%- if first or w.s | length == 0 -%}
+ {%- set sounds = first.split('|') | map('replace', 'ˈ', '') | map('replace', 'ˌ', '') | list -%}
+ {%- if sounds | length == w.s | length -%}{{ lts_search(w.s, sounds) | replace('|', '') }}{%- endif -%}
+ {%- endif -%}
+{%- endif -%}
+{%- endmacro -%}
+
+{#- a word missing from the lexicon: the predicted pronunciation; a word that cannot be read (e.g. "cpp") is spelled out -#}
+{%- macro fallback(word) -%}
+{%- set ps = lts(word) if tts_norm_scan(word, '[aeiouyAEIOUY]') | selectattr('rule') | list else '' -%}
+{%- if ps and tts_norm_scan(ps, '[' ~ V ~ ']') | selectattr('rule') | list -%}{{ ps }}
+{%- else -%}{{ get_nnp(word) or unk }}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro parent_tag(tag) -%}
+{%- if tag.startswith('VB') -%}VERB
+{%- elif tag.startswith('NN') -%}NOUN
+{%- elif tag.startswith('ADV') or tag.startswith('RB') -%}ADV
+{%- elif tag.startswith('ADJ') or tag.startswith('JJ') -%}ADJ
+{%- else -%}{{ tag }}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro apply_stress(ps, stress) -%}
+{%- if stress is none or not ps -%}{{ ps }}
+{%- elif stress < -1 -%}{{ ps | replace('ˈ', '') | replace('ˌ', '') }}
+{%- elif stress == -1 or ((stress == 0 or stress == -0.5) and 'ˈ' in ps) -%}{{ ps | replace('ˌ', '') | replace('ˈ', 'ˌ') }}
+{%- elif (stress == 0 or stress == 0.5 or stress == 1) and 'ˈ' not in ps and 'ˌ' not in ps -%}{{ tts_norm_sub(ps, '^([^' ~ V ~ ']*)([' ~ V ~ '])', '%1ˌ%2') }}
+{%- elif stress >= 1 and 'ˈ' not in ps and 'ˌ' in ps -%}{{ ps | replace('ˌ', 'ˈ') }}
+{%- elif stress > 1 and 'ˈ' not in ps and 'ˌ' not in ps -%}{{ tts_norm_sub(ps, '^([^' ~ V ~ ']*)([' ~ V ~ '])', '%1ˈ%2') }}
+{%- else -%}{{ ps }}
+{%- endif -%}
+{%- endmacro -%}
+
+{#- spell out letters: "NASA" -> N A S A, "e.g." -> E G -#}
+{%- macro get_nnp(word) -%}
+{%- set ns = namespace(ps='', ok=true) -%}
+{%- for c in tts_norm_scan(word, '%a') -%}
+ {%- if c.rule -%}
+ {%- set p = gold(c.text | upper) -%}
+ {%- if not p -%}{%- set ns.ok = false -%}{%- else -%}{%- set ns.ps = ns.ps ~ p -%}{%- endif -%}
+ {%- endif -%}
+{%- endfor -%}
+{%- if ns.ok and ns.ps -%}{{ tts_norm_sub(apply_stress(ns.ps, 0), 'ˌ([^ˌ]*)$', 'ˈ%1') }}{%- endif -%}
+{%- endmacro -%}
+
+{#- tag: Penn tag or '' ; fv = future vowel context: 'none' (end / punctuation follows), 'v', 'c', or '' (no context) -#}
+{%- macro lookup(word, tag, stress, fv) -%}
+{%- set ns = namespace(w=word, nnp=false, ps=none, t='') -%}
+{%- if word == word | upper and not dkey(GOLD, word) -%}{%- set ns.w = word | lower -%}{%- set ns.nnp = tag == 'NNP' -%}{%- endif -%}
+{%- set k = dkey(GOLD, ns.w) -%}
+{%- if k -%}
+ {%- set e = tts_dict(GOLD, k) -%}
+ {%- if e is mapping -%}
+ {#- POS variants: misaki's ps.get(tag) with 'None' (no following vowel context) and parent tag fallback -#}
+ {%- if fv == 'none' and 'None' in e -%}{%- set ns.t = 'None' -%}
+ {%- elif tag and tag in e -%}{%- set ns.t = tag -%}
+ {%- else -%}{%- set ns.t = parent_tag(tag) -%}
+ {%- endif -%}
+ {#- a null variant means "no pronunciation for this tag": spell the word out -#}
+ {%- set ns.ps = (e[ns.t] or '') if ns.t and ns.t in e else e.DEFAULT -%}
+ {%- else -%}{%- set ns.ps = e -%}
+ {%- endif -%}
+{%- elif not ns.nnp -%}
+ {%- set k = dkey(SILVER, ns.w) -%}
+ {%- set ns.ps = tts_dict(SILVER, k) if k else none -%}
+{%- endif -%}
+{%- set r = get_nnp(ns.w) if (not ns.ps or (ns.nnp and 'ˈ' not in ns.ps)) else '' -%}
+{%- if r -%}{{ r }}{%- else -%}{{ apply_stress(ns.ps or '', stress) }}{%- endif -%}
+{%- endmacro -%}
+
+{%- macro is_known(word) -%}
+{%- if has(word) or word in SYMBOLS -%}1
+{%- elif not full(word, '[A-Za-z]+') -%}
+{%- elif word | length == 1 -%}1
+{%- elif word == word | upper and dkey(GOLD, word | lower) -%}1
+{%- elif word[1:] == word[1:] | upper -%}1
+{%- endif -%}
+{%- endmacro -%}
+
+{#- ---------- suffixes ---------- -#}
+{%- macro sfx_s(stem) -%}
+{%- if not stem -%}
+{%- elif stem[-1:] in 'ptkfθ' -%}{{ stem }}s
+{%- elif stem[-1:] in 'szʃʒʧʤ' -%}{{ stem }}{{ 'ɪ' if GB else 'ᵻ' }}z
+{%- else -%}{{ stem }}z
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro stem_s(word, tag, stress, fv) -%}
+{%- set n = word | length -%}
+{%- if n < 3 or not word.endswith('s') -%}
+{%- elif not word.endswith('ss') and is_known(word[:-1]) -%}{{ sfx_s(lookup(word[:-1], tag, stress, fv)) }}
+{%- elif (word.endswith("'s") or (n > 4 and word.endswith('es') and not word.endswith('ies'))) and is_known(word[:-2]) -%}{{ sfx_s(lookup(word[:-2], tag, stress, fv)) }}
+{%- elif n > 4 and word.endswith('ies') and is_known(word[:-3] ~ 'y') -%}{{ sfx_s(lookup(word[:-3] ~ 'y', tag, stress, fv)) }}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro sfx_ed(stem) -%}
+{%- if not stem -%}
+{%- elif stem[-1:] in 'pkfθʃsʧ' -%}{{ stem }}t
+{%- elif stem[-1:] == 'd' -%}{{ stem }}{{ 'ɪ' if GB else 'ᵻ' }}d
+{%- elif stem[-1:] != 't' -%}{{ stem }}d
+{%- elif GB or stem | length < 2 -%}{{ stem }}ɪd
+{%- elif stem[-2:-1] in TAUS -%}{{ stem[:-1] }}ɾᵻd
+{%- else -%}{{ stem }}ᵻd
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro stem_ed(word, tag, stress, fv) -%}
+{%- set n = word | length -%}
+{%- if n < 4 or not word.endswith('d') -%}
+{%- elif not word.endswith('dd') and is_known(word[:-1]) -%}{{ sfx_ed(lookup(word[:-1], tag, stress, fv)) }}
+{%- elif n > 4 and word.endswith('ed') and not word.endswith('eed') and is_known(word[:-2]) -%}{{ sfx_ed(lookup(word[:-2], tag, stress, fv)) }}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro sfx_ing(stem) -%}
+{%- if not stem -%}
+{%- elif GB -%}{%- if stem[-1:] not in 'əː' -%}{{ stem }}ɪŋ{%- endif -%}
+{%- elif stem | length > 1 and stem[-1:] == 't' and stem[-2:-1] in TAUS -%}{{ stem[:-1] }}ɾɪŋ
+{%- else -%}{{ stem }}ɪŋ
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro stem_ing(word, tag, stress, fv) -%}
+{%- set n = word | length -%}
+{%- if n < 5 or not word.endswith('ing') -%}
+{%- elif n > 5 and is_known(word[:-3]) -%}{{ sfx_ing(lookup(word[:-3], tag, stress, fv)) }}
+{%- elif is_known(word[:-3] ~ 'e') -%}{{ sfx_ing(lookup(word[:-3] ~ 'e', tag, stress, fv)) }}
+{%- elif n > 5 and tts_norm_scan(word, '(?:bb|cc|dd|gg|kk|ll|mm|nn|pp|rr|ss|tt|vv|xx|zz|ck)ing$') | selectattr('rule') | list and is_known(word[:-4]) -%}{{ sfx_ing(lookup(word[:-4], tag, stress, fv)) }}
+{%- endif -%}
+{%- endmacro -%}
+
+{#- ---------- words ---------- -#}
+{#- misaki Lexicon.get_special_case(); fto = the next word is "to" (for "used to") -#}
+{%- macro special(word, tag, stress, fv, fto) -%}
+{%- if word in SYMBOLS -%}{{ lookup(SYMBOLS[word], '', none, fv) }}
+{%- elif word == 'a' or word == 'A' -%}{{ 'ɐ' if tag == 'DT' else 'ˈA' }}
+{%- elif word in ['am', 'Am', 'AM'] -%}
+ {%- if tag.startswith('NN') -%}{{ get_nnp(word) }}
+ {%- elif fv == 'none' or word != 'am' or (stress is not none and stress > 0) -%}{{ gold('am') }}
+ {%- else -%}ɐm{%- endif -%}
+{%- elif word in ['an', 'An', 'AN'] -%}{{ get_nnp(word) if (word == 'AN' and tag.startswith('NN')) else 'ɐn' }}
+{%- elif word == 'I' and tag == 'PRP' -%}ˌI
+{%- elif word in ['by', 'By', 'BY'] and parent_tag(tag) == 'ADV' -%}bˈI
+{%- elif word in ['to', 'To'] or (word == 'TO' and tag in ['TO', 'IN']) -%}{{ {'v': 'tʊ', 'c': 'tə'}.get(fv, gold('to')) }}
+{%- elif word in ['in', 'In'] or (word == 'IN' and tag != 'NNP') -%}{{ 'ˈ' if (fv == 'none' or tag != 'IN') else '' }}ɪn
+{%- elif word in ['the', 'The'] or (word == 'THE' and tag == 'DT') -%}{{ 'ði' if fv == 'v' else 'ðə' }}
+{%- elif tag == 'IN' and full(word, '[Vv][Ss]%.?') -%}{{ lookup('versus', '', none, fv) }}
+{%- elif word in ['used', 'Used', 'USED'] -%}{{ tts_dict(GOLD, 'used').VBD if (tag in ['VBD', 'JJ'] and fto) else gold('used') }}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro get_word(word0, tag, stress, fv, fto) -%}
+{%- set sp = special(word0, tag, stress, fv, fto) -%}
+{%- if sp -%}{{ sp }}{%- else -%}
+{%- set ns = namespace(word=word0) -%}
+{%- set wl = word0 | lower -%}
+{%- if word0 | length > 1 and full(word0 | replace("'", ''), '%a+') and word0 != wl and (tag != 'NNP' or word0 | length > 7) and not has(word0)
+ and (word0 == word0 | upper or word0[1:] == word0[1:] | lower)
+ and (has(wl) or stem_s(wl, tag, stress, fv) or stem_ed(wl, tag, stress, fv) or stem_ing(wl, tag, stress, fv)) -%}
+ {%- set ns.word = wl -%}
+{%- endif -%}
+{%- set w = ns.word -%}
+{%- if is_known(w) -%}{{ lookup(w, tag, stress, fv) }}
+{%- elif w.endswith("s'") and is_known(w[:-2] ~ "'s") -%}{{ lookup(w[:-2] ~ "'s", tag, stress, fv) }}
+{%- elif w.endswith("'") and is_known(w[:-1]) -%}{{ lookup(w[:-1], tag, stress, fv) }}
+{%- else -%}
+ {%- set r = stem_s(w, tag, stress, fv) -%}
+ {%- if not r -%}{%- set r = stem_ed(w, tag, stress, fv) -%}{%- endif -%}
+ {%- if not r -%}{%- set r = stem_ing(w, tag, 0.5 if stress is none else stress, fv) -%}{%- endif -%}
+ {{- r -}}
+{%- endif -%}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro lex_word(word0, tag, fv, fto=false) -%}
+{%- set w = word0 | replace('‘', "'") | replace('’', "'") -%}
+{%- set stress = none if w == w | lower else (2 if w == w | upper else 0.5) -%}
+{{- get_word(w, tag, stress, fv, fto) -}}
+{%- endmacro -%}
+
+{#- ---------- numbers (num2words-compatible word lists, "and" dropped like misaki) ---------- -#}
+{%- set ONES = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'eleven', 'twelve',
+ 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'] -%}
+{%- set TENS = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'] -%}
+{%- set SCALES = [[1000000000000, 'trillion'], [1000000000, 'billion'], [1000000, 'million'], [1000, 'thousand']] -%}
+{%- macro card(n) -%}
+{%- if n < 20 -%}{{ ONES[n] }}
+{%- elif n < 100 -%}{{ TENS[n // 10] }}{{ ' ' ~ ONES[n % 10] if n % 10 else '' }}
+{%- elif n < 1000 -%}{{ ONES[n // 100] }} hundred{{ ' ' ~ card(n % 100) if n % 100 else '' }}
+{%- else -%}
+ {%- set ns = namespace(done=false) -%}
+ {%- for sc in SCALES -%}
+ {%- if not ns.done and n >= sc[0] -%}
+ {{- card(n // sc[0]) }} {{ sc[1] }}{{ ' ' ~ card(n % sc[0]) if n % sc[0] else '' -}}
+ {%- set ns.done = true -%}
+ {%- endif -%}
+ {%- endfor -%}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro ordw(n) -%}
+{%- set ws = card(n).split(' ') -%}
+{%- set last = ws[-1] -%}
+{%- set IRR = {'one': 'first', 'two': 'second', 'three': 'third', 'five': 'fifth', 'eight': 'eighth', 'nine': 'ninth', 'twelve': 'twelfth'} -%}
+{%- if last in IRR -%}{%- set last = IRR[last] -%}{%- elif last.endswith('y') -%}{%- set last = last[:-1] ~ 'ieth' -%}{%- else -%}{%- set last = last ~ 'th' -%}{%- endif -%}
+{{- (ws[:-1] + [last]) | join(' ') -}}
+{%- endmacro -%}
+
+{%- macro yearw(n) -%}
+{%- set hi = n // 100 -%}{%- set lo = n % 100 -%}
+{%- if hi == 0 or (hi % 10 == 0 and lo < 10) or hi >= 100 -%}{{ card(n) }}
+{%- else -%}{{ card(hi) }} {{ 'hundred' if lo == 0 else ('oh ' ~ card(lo) if lo < 10 else card(lo)) }}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro say(words) -%}
+{%- set ns = namespace(out=[]) -%}
+{%- for w in words.split(' ') -%}
+ {%- if w -%}{%- set ns.out = ns.out + [lookup(w, '', -2 if w == 'point' else none, '')] -%}{%- endif -%}
+{%- endfor -%}
+{{- ns.out | join(' ') -}}
+{%- endmacro -%}
+
+{%- macro digits(s) -%}{%- set ns = namespace(out=[]) -%}{%- for c in tts_norm_scan(s, '%d') -%}{%- if c.rule -%}{%- set ns.out = ns.out + [ONES[c.text | int]] -%}{%- endif -%}{%- endfor -%}{{ ns.out | join(' ') }}{%- endmacro -%}
+
+{#- num: digits with optional ,### groups and .decimals ; sfx: st/nd/rd/th/s/'s/ed/ing ; cur: currency symbol or '' -#}
+{%- macro get_number(neg, num, sfx, cur) -%}
+{%- set plain = num | replace(',', '') -%}
+{%- set ns = namespace(w='') -%}
+{%- if sfx in ['st', 'nd', 'rd', 'th'] and '.' not in plain -%}{%- set ns.w = ordw(plain | int) -%}
+{%- elif not neg and not cur and full(num, '%d%d%d%d') -%}{%- set ns.w = yearw(plain | int) -%}
+{%- elif cur and cur in CURRENCIES and full(plain, '%d*(?:%.%d%d?)?') -%}
+ {%- set parts = plain.split('.') -%}
+ {%- set units = CURRENCIES[cur] -%}
+ {%- set a = parts[0] | int if parts[0] else 0 -%}
+ {%- set b = parts[1] | int if parts | length > 1 and parts[1] else 0 -%}
+ {%- set out = namespace(l=[]) -%}
+ {%- for pair in [[a, units[0]], [b, units[1]]] -%}
+ {%- if pair[0] or (loop.first and not b) -%}
+ {%- set uw = lookup(pair[1], '', none, '') if pair[0] == 1 or pair[1] == 'pence' else stem_s(pair[1] ~ 's', '', none, '') -%}
+ {%- set out.l = out.l + [say(card(pair[0])) ~ ' ' ~ uw] -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- set ns.w = '@' ~ (out.l | join(' ' ~ lookup('and', '', none, '') ~ ' ')) -%}
+{%- elif '.' in plain -%}
+ {%- set ip, fp = plain.split('.') -%}
+ {%- set ns.w = ((card(ip | int) ~ ' ') if ip else '') ~ 'point ' ~ digits(fp) -%}
+{%- else -%}{%- set ns.w = card(plain | int) -%}
+{%- endif -%}
+{%- set r = ns.w[1:] if ns.w.startswith('@') else say(ns.w | replace('-', ' ')) -%}
+{%- if neg -%}{%- set r = lookup('minus', '', none, '') ~ ' ' ~ r -%}{%- endif -%}
+{%- if sfx in ['s', "'s"] -%}{{ sfx_s(r) }}
+{%- elif sfx in ['ed', "'d"] -%}{{ sfx_ed(r) }}
+{%- elif sfx == 'ing' -%}{{ sfx_ing(r) }}
+{%- else -%}{{ r }}
+{%- endif -%}
+{%- endmacro -%}
+
+{#- ---------- context ---------- -#}
+{%- macro next_fv(ps, fv) -%}
+{%- set ns = namespace(r=fv, done=false) -%}
+{%- for s in tts_norm_scan(ps, [['v', '[' ~ V ~ ']'], ['c', '[' ~ C ~ ']'], ['p', '[' ~ NQP ~ ']']]) -%}
+ {%- if s.rule and not ns.done -%}{%- set ns.r = 'none' if s.rule == 'p' else s.rule -%}{%- set ns.done = true -%}{%- endif -%}
+{%- endfor -%}
+{{- ns.r -}}
+{%- endmacro -%}
+
+{#- ---------- POS tagger: nltk averaged perceptron, weights in the pos_* dictionaries ---------- -#}
+{%- macro pos_norm(w) -%}
+{%- if '-' in w and w[:1] != '-' -%}!HYPHEN
+{%- elif full(w, '%d%d%d%d') -%}!YEAR
+{%- elif full(w[:1], '%d') -%}!DIGITS
+{%- else -%}{{ w | lower }}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- set POS_CLASSES = tts_dict('pos_classes', '') -%}
+
+{#- features -> best label; nltk: max(classes, key=(score, label)); classes are sorted, so >= keeps the larger label on ties -#}
+{%- macro pos_predict(feats) -%}
+{%- set ns = namespace(ws=[], best='', best_s=none) -%}
+{%- for f in feats -%}
+ {%- set w = tts_dict('pos_weights', f) -%}
+ {%- if w -%}{%- set ns.ws = ns.ws + [w] -%}{%- endif -%}
+{%- endfor -%}
+{%- for c in POS_CLASSES -%}
+ {%- set s = ns.ws | map(attribute='c' ~ loop.index0, default=0.0) | sum -%}
+ {%- if ns.best_s is none or s >= ns.best_s -%}{%- set ns.best = c -%}{%- set ns.best_s = s -%}{%- endif -%}
+{%- endfor -%}
+{{- ns.best -}}
+{%- endmacro -%}
+
+{#- tokens: list of strings (PTB-style) -> space-separated tags -#}
+{%- macro pos_tag(tokens) -%}
+{%- set cn = namespace(l=['-START-', '-START2-']) -%}
+{%- for t in tokens -%}{%- set cn.l = cn.l + [pos_norm(t)] -%}{%- endfor -%}
+{%- set c = cn.l + ['-END-', '-END2-'] -%}
+{%- set ns = namespace(prev='-START-', prev2='-START2-', out=[]) -%}
+{%- for w in tokens -%}
+ {%- set i = loop.index0 + 2 -%}
+ {%- set tag = tts_dict('pos_tagdict', w) -%}
+ {%- if not tag -%}
+ {%- set tag = pos_predict([
+ 'bias',
+ 'i suffix ' ~ w[-3:],
+ 'i pref1 ' ~ w[:1],
+ 'i-1 tag ' ~ ns.prev,
+ 'i-2 tag ' ~ ns.prev2,
+ 'i tag+i-2 tag ' ~ ns.prev ~ ' ' ~ ns.prev2,
+ 'i word ' ~ c[i],
+ 'i-1 tag+i word ' ~ ns.prev ~ ' ' ~ c[i],
+ 'i-1 word ' ~ c[i - 1],
+ 'i-1 suffix ' ~ c[i - 1][-3:],
+ 'i-2 word ' ~ c[i - 2],
+ 'i+1 word ' ~ c[i + 1],
+ 'i+1 suffix ' ~ c[i + 1][-3:],
+ 'i+2 word ' ~ c[i + 2],
+ ]) -%}
+ {%- endif -%}
+ {%- set ns.out = ns.out + [tag] -%}
+ {%- set ns.prev2 = ns.prev -%}{%- set ns.prev = tag -%}
+{%- endfor -%}
+{{- ns.out | join(' ') -}}
+{%- endmacro -%}
+
+{%- macro weight(ps) -%}{{ (ps | length) + (tts_norm_scan(ps, '[AIOQWYʤʧ]') | selectattr('rule') | list | length) }}{%- endmacro -%}
+
+{#- hyphenated compound: per-part lookup (right to left), joined without space, misaki's stress demotion -#}
+{%- macro compound(text, tag, fv) -%}
+{%- if has(text) -%}{{ lex_word(text, tag, fv) }}{%- else -%}
+{%- set ns = namespace(fv=fv, parts=[], bad=false) -%}
+{%- for s in tts_norm_scan(text, '%a+(?:[\'’]%a+)*') | reverse -%}
+ {%- if s.rule -%}
+ {%- set p = lex_word(s.text, tag, ns.fv) -%}
+ {%- if not p -%}{%- set ns.bad = true -%}{%- endif -%}
+ {%- set ns.parts = [{'t': s.text, 'p': p}] + ns.parts -%}
+ {%- set ns.fv = next_fv(p, ns.fv) -%}
+ {%- endif -%}
+{%- endfor -%}
+{%- set n = ns.parts | length -%}
+{%- set d = namespace(demote=[]) -%}
+{#- a part is missing from the lexicon: read the compound part by part -#}
+{%- if ns.bad -%}
+ {%- for x in ns.parts -%}{{ x.p or fallback(x.t) }}{%- endfor -%}
+{%- else -%}
+{%- if not ns.bad -%}
+ {%- set k = namespace(keys=[]) -%}
+ {%- for x in ns.parts -%}
+ {%- set pr = 1 if 'ˈ' in x.p else 0 -%}
+ {#- misaki: sorted((has_primary, weight, index)) -#}
+ {%- set k.keys = k.keys + [{'pr': pr, 'key': pr * 1000000 + (weight(x.p) | int) * 100 + loop.index0, 'i': loop.index0}] -%}
+ {%- endfor -%}
+ {%- set prim = k.keys | sum(attribute='pr') -%}
+ {%- if n == 2 and ns.parts[0].t | length == 1 -%}{%- set d.demote = [1] -%}
+ {%- elif n >= 2 and prim > (n + 1) // 2 -%}{%- set d.demote = (k.keys | sort(attribute='key'))[:n // 2] | map(attribute='i') | list -%}
+ {%- endif -%}
+{%- endif -%}
+{%- for x in ns.parts -%}{{ apply_stress(x.p, -0.5) if loop.index0 in d.demote else x.p }}{%- endfor -%}
+{%- endif -%}
+{%- endif -%}
+{%- endmacro -%}
+
+{#- unknown word in mixed case, read part by part: "OneAPI" -> One + API, '' if the word has one part -#}
+{%- macro camel(text, tag, fv) -%}
+{%- set parts = tts_norm_scan(text, '(?:%u(?!%l))+|%u?%l+') | selectattr('rule') | map(attribute='text') | list -%}
+{%- if parts | length > 1 and parts | join('') == text -%}
+ {%- for p in parts -%}
+ {{- get_nnp(p) if p | length == 1 else (lex_word(p, tag, fv) or fallback(p)) -}}
+ {%- endfor -%}
+{%- endif -%}
+{%- endmacro -%}
+
+{#- nxt: next token text ('' at end) -#}
+{%- macro word_tok(text, tag, fv, fto, nxt) -%}
+{%- set w = text | replace('‘', "'") | replace('’', "'") -%}
+{%- set ps = lex_word(w, tag, fv, fto) -%}
+{%- if ps -%}{{ ps }}
+{#- "etc" + "." -> dictionary entry "etc." (the dot is kept as punctuation) -#}
+{%- elif nxt[:1] == '.' and has(w ~ '.') -%}{{ lookup(w ~ '.', tag, none, fv) }}
+{#- trailing closing quote, e.g. after a plural possessive -#}
+{%- elif w.endswith("'") and lex_word(w[:-1], tag, fv) -%}{{ lex_word(w[:-1], tag, fv) }}”
+{#- unknown name + 's -#}
+{%- elif w.endswith("'s") -%}{{ (lex_word(w[:-2], tag, fv) or fallback(w[:-2])) ~ gold("'s") }}
+{%- endif -%}
+{%- endmacro -%}
+
+{#- ---------- main ---------- -#}
+{#- only english has a text to phonemes step: the prompt of the other voices is already phonemes -#}
+{%- set lc = lang[:2] if lang else ('en' if not (voice is defined and voice) or voice[:1] in 'ab' else 'other') -%}
+{%- if lc != 'en' -%}
+ {{- messages[-1].content | trim -}}
+{%- else -%}
+{%- set text = messages[-1].content | trim -%}
+{%- set t = namespace(s=tts_norm_sub(text, '%s+', ' ')) -%}
+{%- for f in FOLD -%}{%- set t.s = tts_norm_sub(t.s, f[0], f[1]) -%}{%- endfor -%}
+{%- set text = t.s -%}
+{%- set segs = tts_norm_scan(text, [
+ ['abbr', '(?<!%w)%a%a?(?:%.%a%a?)+%.?(?!%w)'],
+ ['money', '([$£€]) ?(%d+(?:,%d%d%d)*(?:%.%d+)?)(?!%d)'],
+ ['num', '((?<!%w)%-)?(%d+(?:,%d%d%d)*(?:%.%d+)?|%.%d+)(st|nd|rd|th|\'s|s|ed|ing)?(?!%w)'],
+ ['dig', '%d+(?:,%d%d%d)*(?:%.%d+)?'],
+ ['cmpd', '%a+(?:[\'’]%a+)*(?:[%-_]%a+(?:[\'’]%a+)*)+'],
+ ['wdot', '%a+%.(?= [%u%d]|$)'],
+ ['word', '%a+(?:[\'’‘]%a+)*[\'’‘]?'],
+ ['dash', '(?<= )%-(?= )|–|%-%-'],
+ ['qopen', '(?<!%S)"'],
+ ['quote', '"'],
+ ['punct', '[;:,.!?—…“”()]'],
+ ['sym', '[%%&+@]'],
+]) -%}
+{%- set nseg = segs | length -%}
+
+{#- 1. PTB-style tokens for the tagger; first[i] = index of the first token of segment i (-1 if none) -#}
+{%- set PTB_PUNCT = {'“': '``', '”': "''", '—': '--'} -%}
+{%- set tk = namespace(toks=[], first=[]) -%}
+{%- for s in segs -%}
+ {%- set n0 = tk.toks | length -%}
+ {%- if s.rule is none -%}
+ {%- elif s.rule == 'word' -%}
+ {#- split clitics like PTB: do|n't, I|'m, dog|'s -#}
+ {%- set w = s.text | replace('’', "'") | replace('‘', "'") -%}
+ {%- set m = tts_norm_scan(w, '^(%a+)(\'(?:s|m|re|ll|ve|d))$') -%}
+ {%- if (w | lower).endswith("n't") and w | length > 3 -%}{%- set tk.toks = tk.toks + [w[:-3], w[-3:]] -%}
+ {%- elif m | length == 1 and m[0].rule -%}{%- set tk.toks = tk.toks + [m[0].groups[0], m[0].groups[1]] -%}
+ {%- else -%}{%- set tk.toks = tk.toks + [w] -%}{%- endif -%}
+ {%- elif s.rule == 'wdot' -%}
+ {%- set tk.toks = tk.toks + ([s.text] if has(s.text) else [s.text[:-1], '.']) -%}
+ {%- elif s.rule == 'money' -%}{%- set tk.toks = tk.toks + [s.groups[0], s.groups[1]] -%}
+ {%- elif s.rule == 'qopen' -%}{%- set tk.toks = tk.toks + ['``'] -%}
+ {%- elif s.rule == 'quote' -%}{%- set tk.toks = tk.toks + ["''"] -%}
+ {%- elif s.rule == 'dash' -%}{%- set tk.toks = tk.toks + ['--'] -%}
+ {%- elif s.rule == 'punct' -%}{%- set tk.toks = tk.toks + [PTB_PUNCT.get(s.text, s.text)] -%}
+ {%- else -%}{%- set tk.toks = tk.toks + [s.text] -%}
+ {%- endif -%}
+ {%- set tk.first = tk.first + [n0 if (tk.toks | length) > n0 else -1] -%}
+{%- endfor -%}
+{%- set tags = pos_tag(tk.toks).split(' ') if tk.toks else [] -%}
+
+{#- 2. G2P right to left (misaki's future-vowel context) -#}
+{%- set GLUE = ['abbr', 'money', 'num', 'dig', 'cmpd', 'wdot', 'word', 'sym'] -%}
+{%- set ns = namespace(fv='none', fto=false, out=[], next='', next_text='') -%}
+{%- for s in segs | reverse -%}
+ {%- set i = nseg - 1 - loop.index0 -%}
+ {%- set tag = tags[tk.first[i]] if tk.first[i] >= 0 else '' -%}
+ {%- if s.rule == 'wdot' and has(s.text) -%}
+ {%- set ps = lex_word(s.text, tag, ns.fv) -%}
+ {%- elif s.rule == 'wdot' -%}
+ {%- set w = lex_word(s.text[:-1], tag, 'none') or fallback(s.text[:-1]) -%}
+ {%- set ps = w ~ '.' -%}
+ {%- elif s.rule is none -%}
+ {%- set ps = tts_norm_sub(s.text, '%S', '') -%}
+ {%- elif s.rule == 'abbr' -%}
+ {%- set ps = get_nnp(s.text) -%}
+ {%- elif s.rule == 'money' -%}
+ {%- set ps = get_number('', s.groups[1], '', s.groups[0]) -%}
+ {%- elif s.rule == 'num' -%}
+ {%- set ps = get_number(s.groups[0], s.groups[1], s.groups[2], '') -%}
+ {%- elif s.rule == 'dig' -%}
+ {#- digits glued to letters, e.g. "221B" -#}
+ {%- set ps = get_number('', s.text, '', '') -%}
+ {%- elif s.rule == 'cmpd' -%}
+ {%- set ps = compound(s.text, tag, ns.fv) -%}
+ {%- elif s.rule == 'word' -%}
+ {%- set ps = word_tok(s.text, tag, ns.fv, ns.fto, ns.next_text) or camel(s.text, tag, ns.fv) or fallback(s.text) -%}
+ {%- elif s.rule == 'sym' -%}
+ {%- set ps = lex_word(s.text, tag, ns.fv) or unk -%}
+ {%- elif s.rule == 'dash' -%}
+ {%- set ps = '—' -%}
+ {%- elif s.rule == 'qopen' -%}
+ {%- set ps = '“' -%}
+ {%- elif s.rule == 'quote' -%}
+ {%- set ps = '”' -%}
+ {%- else -%}
+ {%- set ps = s.text -%}
+ {%- endif -%}
+ {%- if s.rule is not none -%}
+ {%- set ns.fv = next_fv(ps, ns.fv) -%}
+ {%- set ns.fto = s.text in ['to', 'To'] or (s.text == 'TO' and tag in ['TO', 'IN']) -%}
+ {%- endif -%}
+ {#- misaki puts a space between glued tokens of mixed kinds, e.g. "20%" -#}
+ {%- if s.rule in GLUE and ns.next in GLUE and ps -%}{%- set ps = ps ~ ' ' -%}{%- endif -%}
+ {%- set ns.next = s.rule if s.rule is not none else '' -%}
+ {%- if s.rule is not none -%}{%- set ns.next_text = s.text -%}{%- endif -%}
+ {%- set ns.out = [ps] + ns.out -%}
+{%- endfor -%}
+
+{{- (ns.out | join('') | replace('ɾ', 'T') | replace('ʔ', 't')) | trim -}}
+{%- endif -%}
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 1869c1203..530fc8e12 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -176,6 +176,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_QWEN3TTS, "qwen3tts" },
{ LLM_ARCH_POCKETTTS, "pockettts" },
{ LLM_ARCH_ZONOS3, "zonos3" },
+ { LLM_ARCH_KOKORO, "kokoro" },
{ LLM_ARCH_K2_HORIZON, "k2-horizon" },
{ LLM_ARCH_BEAM, "beam" },
{ LLM_ARCH_KOLIBRI1, "kolibri1" },
@@ -629,6 +630,7 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_POS_EMBD, "position_embd" },
{ LLM_TENSOR_FFN_ACT, "blk.%d.ffn.act" },
{ LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" },
+ { LLM_TENSOR_TOKEN_EMBD_PROJ, "token_embd_proj" },
{ LLM_TENSOR_TOKEN_TYPES, "token_types" },
{ LLM_TENSOR_CLS, "cls" },
{ LLM_TENSOR_CLS_OUT, "cls.output" },
@@ -929,6 +931,7 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_TOKEN_TYPES, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},
{LLM_TENSOR_HRM_Z_L_INIT, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},
{LLM_TENSOR_TOKEN_EMBD_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, // do the norms on the first layer (not the input layer)
+ {LLM_TENSOR_TOKEN_EMBD_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_OUTPUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_CLS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_CLS_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
diff --git a/src/llama-arch.h b/src/llama-arch.h
index d015b5d5a..05a00c3ac 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -177,6 +177,7 @@ enum llm_arch {
LLM_ARCH_QWEN3TTS,
LLM_ARCH_POCKETTTS,
LLM_ARCH_ZONOS3,
+ LLM_ARCH_KOKORO,
LLM_ARCH_MINIMAX_01,
LLM_ARCH_HRM_TEXT,
LLM_ARCH_K2_HORIZON,
@@ -531,6 +532,7 @@ enum llm_kv {
enum llm_tensor {
LLM_TENSOR_TOKEN_EMBD,
LLM_TENSOR_TOKEN_EMBD_NORM,
+ LLM_TENSOR_TOKEN_EMBD_PROJ,
LLM_TENSOR_TOKEN_TYPES,
LLM_TENSOR_POS_EMBD,
LLM_TENSOR_DENSE_2_OUT,
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 15ec05408..9adb96364 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -128,6 +128,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_pockettts(params);
case LLM_ARCH_ZONOS3:
return new llama_model_zonos3(params);
+ case LLM_ARCH_KOKORO:
+ return new llama_model_kokoro(params);
case LLM_ARCH_PHI2:
return new llama_model_phi2(params);
case LLM_ARCH_PHI3:
@@ -2987,6 +2989,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
case LLM_ARCH_LLADA_MOE:
case LLM_ARCH_RND1:
case LLM_ARCH_CLEF:
+ case LLM_ARCH_KOKORO:
{
res = nullptr;
} break;
@@ -3719,6 +3722,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_GLM5_NEXT:
case LLM_ARCH_STARCODER:
case LLM_ARCH_INKLING2:
+ case LLM_ARCH_KOKORO:
return LLAMA_ROPE_TYPE_NONE;
// use what we call a normal RoPE, operating on pairs of consecutive head values
diff --git a/src/models/kokoro.cpp b/src/models/kokoro.cpp
new file mode 100644
index 000000000..c236134af
--- /dev/null
+++ b/src/models/kokoro.cpp
@@ -0,0 +1,145 @@
+#include "models.h"
+
+// backbone of the Kokoro TTS pipeline: the PL-BERT phoneme encoder (an ALBERT model)
+// - token and position embeddings have a small width, a projection brings them to n_embd
+// - one layer is stored and applied n_layer times
+// it has no lm_head, the rest of the model is inside the mmproj
+
+void llama_model_kokoro::load_arch_hparams(llama_model_loader & ml) {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+
+ type = LLM_TYPE_UNKNOWN;
+}
+
+void llama_model_kokoro::load_arch_tensors(llama_model_loader & ml) {
+ LLAMA_LOAD_LOCALS;
+
+ // the width of the token and position embeddings comes from the weights
+ const ggml_tensor * pos_meta = ml.get_tensor_meta(tn(LLM_TENSOR_POS_EMBD, "weight").str().c_str());
+ const int64_t n_embd_tok = pos_meta ? pos_meta->ne[0] : n_embd;
+
+ // the converter pads the rows with zeros from n_embd_tok to n_embd, token and embedding inputs must have the same width
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+ // the token type embedding is folded into the positions by the converter
+ pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd_tok, n_ctx_train}, 0);
+
+ tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd_tok}, 0);
+ tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd_tok}, 0);
+
+ tok_embd_proj = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_PROJ, "weight", 0), {n_embd_tok, n_embd}, 0);
+ tok_embd_proj_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_PROJ, "bias", 0), {n_embd}, 0);
+
+ // no output head, the logits are unused; reuse the embedding table so a sampler can still run
+ output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+
+ // the layers share their weights, only the first one is stored
+ {
+ auto & layer = layers[0];
+ const int i = 0;
+
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
+
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
+
+ layer.attn_out_norm = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}, 0);
+ layer.attn_out_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}, 0);
+
+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
+ layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, 0);
+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
+ layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, 0);
+
+ layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, 0);
+ layer.layer_out_norm_b = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}, 0);
+ }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_kokoro::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(params);
+}
+
+llama_model_kokoro::graph::graph(const llm_graph_params & params) : llm_graph_context(params) {
+ const auto & model = static_cast<const llama_model_kokoro &>(this->model);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v();
+ const int64_t n_embd_tok = model.pos_embd->ne[0];
+
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+
+ ggml_tensor * cur;
+ ggml_tensor * inpL;
+
+ ggml_tensor * inp_pos = build_inp_pos();
+ ggml_tensor * embd_pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);
+ ggml_build_forward_order(gf, embd_pos);
+
+ // token + position (+ type) embeddings, in the small width
+ inpL = build_inp_embd(model.tok_embd);
+ inpL = ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd_tok, n_tokens, inpL->nb[1], 0));
+ inpL = ggml_add(ctx0, inpL, embd_pos);
+ cb(inpL, "inp_embd", -1);
+
+ inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, 0);
+ cb(inpL, "inp_norm", 0);
+
+ inpL = ggml_add(ctx0, ggml_mul_mat(ctx0, model.tok_embd_proj, inpL), model.tok_embd_proj_b);
+ cb(inpL, "inp_proj", 0);
+
+ auto * inp_attn = build_attn_inp_no_cache();
+
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ const auto & layer = model.layers[0];
+
+ for (int il = 0; il < n_layer; ++il) {
+ cur = inpL;
+
+ {
+ auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur,
+ n_embd_head, n_head, n_head_kv, il);
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ cur = build_attn(inp_attn,
+ layer.wo, layer.wo_b, layer.wo_s,
+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
+ cb(cur, "kqv_out", il);
+ }
+
+ if (il == n_layer - 1 && inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+
+ cur = ggml_add(ctx0, cur, inpL);
+ cur = build_norm(cur, layer.attn_out_norm, layer.attn_out_norm_b, LLM_NORM, il);
+
+ ggml_tensor * ffn_inp = cur;
+ cb(ffn_inp, "ffn_inp", il);
+
+ cur = build_ffn(cur,
+ layer.ffn_up, layer.ffn_up_b, NULL,
+ NULL, NULL, NULL,
+ layer.ffn_down, layer.ffn_down_b, NULL, NULL,
+ LLM_FFN_GELU, LLM_FFN_SEQ, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = build_norm(cur, layer.layer_out_norm, layer.layer_out_norm_b, LLM_NORM, il);
+
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cb(cur, "result_embd", -1);
+ res->t_embd = cur;
+
+ cur = build_lora_mm(model.output, cur);
+
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+}
diff --git a/src/models/models.h b/src/models/models.h
index 932887c91..ccbd1be62 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -769,6 +769,22 @@ struct llama_model_pockettts : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
+struct llama_model_kokoro : public llama_model_base {
+ llama_model_kokoro(const struct llama_model_params & params) : llama_model_base(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ // token + position embeddings are narrower than n_embd, then projected up
+ ggml_tensor * tok_embd_proj = nullptr;
+ ggml_tensor * tok_embd_proj_b = nullptr;
+
+ struct graph : public llm_graph_context {
+ graph(const llm_graph_params & params);
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
struct llama_model_zonos3 : public llama_model_base {
llama_model_zonos3(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 36e48a0f2..0b7e8e4a9 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -399,7 +399,7 @@ endif()
# libmtmd
llama_build_and_test(test-mtmd-c-api.c LABEL behavior LINK mtmd)
-llama_build_and_test(test-mtmd-impl.cpp LABEL behavior LINK mtmd)
+llama_build_and_test(test-mtmd-impl.cpp LABEL behavior LINK mtmd llama-common)
# GGUF model data fetcher library for tests that need real model metadata
# Only compile when cpp-httplib has SSL support (CPPHTTPLIB_OPENSSL_SUPPORT)
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 8b3d4bb86..92123211a 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -1184,6 +1184,9 @@ static bool arch_supported(const llm_arch arch) {
arch == LLM_ARCH_NEO_BERT || arch == LLM_ARCH_JINA_BERT_V2 || arch == LLM_ARCH_JINA_BERT_V3 || arch == LLM_ARCH_EUROBERT) {
return false; // TODO vocab
}
+ if (arch == LLM_ARCH_KOKORO) {
+ return false; // TODO encoder without memory, a sequence must fit in one ubatch
+ }
if (arch == LLM_ARCH_PLM) {
return false; // TODO tensor shapes
}
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp
index 5dd3d9384..e24365e59 100644
--- a/tests/test-mtmd-impl.cpp
+++ b/tests/test-mtmd-impl.cpp
@@ -7,12 +7,22 @@
#include "mtmd-helper.h"
#include "mtmd-image.h"
#include "mtmd-internal.h"
+#include "models/models.h"
+
+#include "chat-tts-norm.h"
+#include "jinja/lexer.h"
+#include "jinja/parser.h"
+#include "jinja/runtime.h"
#include <algorithm>
+#include <array>
#include <cmath>
#include <cstdint>
#include <cstdlib>
+#include <fstream>
#include <iostream>
+#include <iterator>
+#include <memory>
#include <stdexcept>
#include <string>
#include <tuple>
@@ -881,6 +891,177 @@ MAKE_TEST(test_helper_resampler) {
t.assert_true("invalid rate", mtmd_helper_resampler_init(0, rate_out) == nullptr);
}
+//
+// kokoro
+//
+
+// a dictionary for tts_dict(): entries in any order, sorted here
+static common_tts_dict kokoro_dict(std::vector<std::pair<std::string, std::string>> entries) {
+ std::sort(entries.begin(), entries.end());
+ common_tts_dict dict;
+ for (auto & e : entries) {
+ dict.keys.push_back(e.first);
+ dict.values.push_back(e.second);
+ }
+ return dict;
+}
+
+// the "tts" template with small dictionaries: a few lexicon words, fixed tags, and a model for unknown words over the letters z a b i n g
+MAKE_TEST(test_kokoro_tts_template) {
+ std::string path = __FILE__;
+ const size_t pos = path.find_last_of("/\\");
+ path = (pos == std::string::npos ? std::string(".") : path.substr(0, pos)) + "/../models/templates/hexgrad-Kokoro-82M-tts.jinja";
+ std::ifstream fs(path, std::ios_base::binary);
+ if (!t.assert_true("template file is readable: " + path, fs.is_open())) {
+ return;
+ }
+ const std::string tmpl((std::istreambuf_iterator<char>(fs)), std::istreambuf_iterator<char>());
+
+ const auto dicts = std::make_shared<const common_tts_dicts>(common_tts_dicts{
+ {"gold", kokoro_dict({
+ {"hello", "\"h\xc9\x99l\xcb\x88O\""},
+ {"world", "\"w\xcb\x88\xc9\x9c\xc9\xb9ld\""},
+ {"I", "\"\xcb\x88I\""},
+ {"read", "{\"ADJ\":\"\xc9\xb9\xcb\x88\xc9\x9b" "d\",\"DEFAULT\":\"\xc9\xb9\xcb\x88id\",\"VBD\":\"\xc9\xb9\xcb\x88\xc9\x9b" "d\",\"VBN\":\"\xc9\xb9\xcb\x88\xc9\x9b" "d\",\"VBP\":\"\xc9\xb9\xcb\x88\xc9\x9b" "d\"}"},
+ {"it", "\"\xc9\xaat\""},
+ {"three", "\"\xce\xb8\xc9\xb9\xcb\x88i\""},
+ {"cat", "\"k\xcb\x88\xc3\xa6t\""},
+ {"walk", "\"w\xcb\x88\xc9\x94k\""},
+ {"the", "\"\xc3\xb0i\""},
+ {"Z", "\"z\xcb\x88i\""},
+ {"'s", "\"s\""},
+ {"a", "\"A\""},
+ {"and", "\"\xc3\xa6nd\""},
+ {"is", "\"\xc9\xaaz\""},
+ })},
+ {"silver", kokoro_dict({
+ })},
+ {"gold_gb", kokoro_dict({
+ {"hello", "\"h\xc9\x99l\xcb\x88Q\""},
+ {"world", "\"w\xcb\x88\xc9\x9c\xcb\x90ld\""},
+ {"walk", "\"w\xcb\x88\xc9\x94\xcb\x90k\""},
+ })},
+ {"silver_gb", kokoro_dict({
+ })},
+ {"pos_classes", kokoro_dict({
+ {"", "[\"NN\",\"NNP\",\"PRP\",\"VBD\"]"},
+ })},
+ {"pos_tagdict", kokoro_dict({
+ {"I", "\"PRP\""},
+ {"read", "\"VBD\""},
+ {"it", "\"PRP\""},
+ {".", "\".\""},
+ {",", "\",\""},
+ })},
+ {"pos_weights", kokoro_dict({
+ {"bias", "{\"c0\":1.0}"},
+ })},
+ {"lts", kokoro_dict({
+ {"z", "{\"z\":0.1}"},
+ {"b", "{\"b\":0.1}"},
+ {"i", "{\"\xc9\xaa\":0.1}"},
+ {"n", "{\"\xc5\x8b\":0.1}"},
+ {"g", "{\"\":0.1}"},
+ {"a", "{\"\xcb\x88\xc3\xa6\":0.1,\"\xc3\xa6\":0.06}"},
+ {"z:z a", "{\"\xcb\x88\xc3\xa6\":0.2,\"\xc3\xa6\":0.01}"},
+ {"$", "{\"\":0.2}"},
+ })},
+ {"lts_gb", kokoro_dict({
+ {"z", "{\"z\":0.1}"},
+ {"b", "{\"b\":0.1}"},
+ {"i", "{\"\xc9\xaa\":0.1}"},
+ {"n", "{\"\xc5\x8b\":0.1}"},
+ {"g", "{\"\":0.1}"},
+ {"a", "{\"\xcb\x88\xc9\x91\xcb\x90\":0.1,\"\xc9\x91\xcb\x90\":0.06}"},
+ {"z:z a", "{\"\xcb\x88\xc9\x91\xcb\x90\":0.2,\"\xc9\x91\xcb\x90\":0.01}"},
+ {"$", "{\"\":0.2}"},
+ })},
+ });
+
+ // voice, text, expected phonemes
+ const std::vector<std::array<std::string, 3>> cases = {
+ // words of the lexicon
+ { "af_heart", "Hello world.", "h\xc9\x99l\xcb\x88O w\xcb\x88\xc9\x9c\xc9\xb9ld." },
+ // the pronunciation depends on the part of speech
+ { "af_heart", "I read it.", "\xcb\x8cI \xc9\xb9\xcb\x88\xc9\x9b" "d \xc9\xaat." },
+ // a number, and the -s suffix rule
+ { "af_heart", "3 cats", "\xce\xb8\xc9\xb9\xcb\x88i k\xcb\x88\xc3\xa6ts" },
+ // the -ed and -ing suffix rules
+ { "af_heart", "walked, walking", "w\xcb\x88\xc9\x94kt, w\xcb\x88\xc9\x94k\xc9\xaa\xc5\x8b" },
+ // not in the lexicon: predicted from the spelling
+ { "af_heart", "zab", "z\xcb\x88\xc3\xa6" "b" },
+ // the free search gives two primary stresses, the stress pass keeps one
+ { "af_heart", "zaba", "z\xcb\x88\xc3\xa6" "b\xc3\xa6" },
+ // a doubled consonant says one sound
+ { "af_heart", "zabbing", "z\xcb\x88\xc3\xa6" "b\xc9\xaa\xc5\x8b" },
+ // an unknown word with 's
+ { "af_heart", "The zab's cat", "\xc3\xb0\xc9\x99 z\xcb\x88\xc3\xa6" "bs k\xcb\x88\xc3\xa6t" },
+ // no vowel: spelled out
+ { "af_heart", "zz", "z\xcb\x8ciz\xcb\x88i" },
+ // an accented letter is read as its base letter
+ { "af_heart", "H\xc3\xa9ll\xc3\xb3 world.", "h\xc9\x99l\xcb\x88O w\xcb\x88\xc9\x9c\xc9\xb9ld." },
+ // digits glued to letters are read as a number
+ { "af_heart", "3Z cats", "\xce\xb8\xc9\xb9\xcb\x88i z\xcb\x88i k\xcb\x88\xc3\xa6ts" },
+ // an unknown word in mixed case is read part by part
+ { "af_heart", "catWalk", "k\xcb\x88\xc3\xa6tw\xcb\x88\xc9\x94k" },
+ // a british voice uses the british lexicon
+ { "bf_emma", "Hello world.", "h\xc9\x99l\xcb\x88Q w\xcb\x88\xc9\x9c\xcb\x90ld." },
+ // and the british model for unknown words
+ { "bf_emma", "zaba", "z\xcb\x88\xc9\x91\xcb\x90" "b\xc9\x91\xcb\x90" },
+ // a voice of another language takes phonemes, they pass through
+ { "ef_dora", "\xcb\x88ola, m\xcb\x88undo.", "\xcb\x88ola, m\xcb\x88undo." },
+ };
+
+ jinja::lexer lexer;
+ const jinja::program ast = jinja::parse_from_tokens(lexer.tokenize(tmpl));
+ for (const auto & c : cases) {
+ std::string out;
+ try {
+ jinja::context ctx(tmpl);
+ common_chat_tts_norm_add_builtins(ctx, dicts);
+ auto msg = jinja::mk_val<jinja::value_object>();
+ msg->insert("role", jinja::mk_val<jinja::value_string>("user"));
+ msg->insert("content", jinja::mk_val<jinja::value_string>(c[1]));
+ auto msgs = jinja::mk_val<jinja::value_array>();
+ msgs->push_back(msg);
+ ctx.set_val("messages", msgs);
+ ctx.set_val("language", jinja::mk_val<jinja::value_string>(""));
+ ctx.set_val("voice", jinja::mk_val<jinja::value_string>(c[0]));
+ jinja::runtime runtime(ctx);
+ out = jinja::runtime::gather_string_parts(runtime.execute(ast))->as_string().str();
+ } catch (const std::exception & e) {
+ out = std::string("exception: ") + e.what();
+ }
+ if (!t.assert_true(c[0] + ": " + c[1], out == c[2])) {
+ t.log("Expected: " + c[2]);
+ t.log("Actual : " + out);
+ }
+ }
+}
+
+// every frame is the constant 0.25 (only the DC bin, of magnitude n_fft * 0.25)
+// the overlap-add of hann windows at a quarter hop sums to 2, their squares to 1.5: the output is 0.25 * 2 / 1.5
+MAKE_TEST(test_kokoro_istft) {
+ const int n_fft = 20, hop = 5, n_frames = 9;
+ std::vector<float> spec((size_t) n_frames * (n_fft + 2), 0.0f);
+ for (int fr = 0; fr < n_frames; fr++) {
+ float * row = spec.data() + (size_t) fr * (n_fft + 2);
+ // the first half of a row is log magnitudes, the second half phases
+ for (int k = 0; k <= n_fft / 2; k++) {
+ row[k] = k == 0 ? logf(n_fft * 0.25f) : -100.0f;
+ }
+ }
+ std::vector<float> audio;
+ mtmd_graph_kokoro::istft(spec, audio);
+ t.assert_equal("samples", (size_t) (n_frames - 1) * hop, audio.size());
+ // the first and last hop are covered by fewer frames
+ float max_err = 0.0f;
+ for (size_t i = hop; i + hop < audio.size(); i++) {
+ max_err = std::max(max_err, std::fabs(audio[i] - 1.0f / 3.0f));
+ }
+ t.assert_true("constant signal, max error " + std::to_string(max_err), max_err < 1e-4f);
+}
+
//
// main
//
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index 233ef1c62..de8c473dd 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -95,6 +95,7 @@ add_library(mtmd
models/pockettts-spkenc.cpp
models/pockettts-gen.cpp
models/soprano.cpp
+ models/kokoro.cpp
models/step3vl.cpp
models/siglip.cpp
models/whisper-enc.cpp
diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md
index 5a01e425d..55ad1c5b3 100644
--- a/tools/mtmd/README-dev.md
+++ b/tools/mtmd/README-dev.md
@@ -96,6 +96,28 @@ Some stages are a full LLM-sized transformer that attends to the keys/values of
- Non-causal rows: `LLAMA_ATTENTION_TYPE_NON_CAUSAL`, all rows in one ubatch (`n_ubatch` >= rows), they see every cached position of the sequence. Remove them with `llama_memory_seq_rm()` after each pass
- Size the context per call (`n_ctx`, `n_ubatch`) instead of keeping one for the whole session
+### Text preprocessing (the `tts` chat template)
+
+Anything a model needs done to the input text before tokenization (prompt format, text normalization, text to phonemes) goes in `tokenizer.chat_template.tts`, as a jinja program. It receives the text as `messages[0].content`, plus `language` and `voice`.
+
+Examples:
+- Prompt format: `conversion/lfm3.py`
+- Text normalization: `models/templates/Zyphra-ZONOS3-Mini-tts.jinja`
+- Text to phonemes (G2P): `models/templates/hexgrad-Kokoro-82M-tts.jinja`
+
+Functions available to the template (see `docs/tts-norm-patterns.md`):
+- `tts_norm_scan()` / `tts_norm_sub()`: pattern matching for text normalization
+- `tts_dict()`: lookup in a dictionary stored in the GGUF, such as a pronunciation lexicon
+
+Add test cases to `tests/test-jinja.cpp`, which also checks them against the python reference with `-py`.
+
+Rules for phoneme-based models:
+- C++ only holds the pieces that every such model needs (the functions above). The rules of one model (lexicon handling, stress, suffixes, number spelling, ...) live in its jinja template
+- The reason is binary size: phoneme logic compiled into C++ is shipped to every user, and most of them never run a TTS model. A template only costs the users of that model
+- Do not add a model-specific or language-specific G2P step in C++, and do not add a dependency for it (e.g. espeak-ng)
+
+G2P in jinja has limits, mostly for languages other than English (no lexicon, or rules that need a real morphological analyzer). For now the escape hatch is on the application side: the user runs the text to phonemes step in their own code and sends the phonemes with `raw_prompt` (see the `/tts` endpoint of `llama-server`), which skips the template. Document it in `docs/multimodal/tts.md` when a model relies on it.
+
### Checklist for porting new audio generation models to mtmd
1. Make sure to consult merged PRs about adding new TTS models, especially reviewer comments
@@ -116,7 +138,7 @@ Some stages are a full LLM-sized transformer that attends to the keys/values of
- In most cases, you can hard-code model configs in the model graph class, or in `mtmd_hparams`
- If some values need to be exposed to the `mtmd_helper` layer, hard-code them in `mtmd_helper` and distinguish by pipeline and `mtmd_gen_audio_info::model_variant` if necessary
- Do NOT add new GGUF metadata or new fields to `mtmd_gen_audio_info` unless you can prove that you absolutely need them
- - If the model requires special preprocessing of the input text prompt (prompt format, text normalization), put it in `tokenizer.chat_template.tts` as a jinja program. It receives the text as `messages[0].content`, plus `language` and `voice`. See `conversion/lfm3.py` (prompt format) and `models/templates/Zyphra-ZONOS3-Mini-tts.jinja` (text normalization) as examples. For text normalization, use `tts_norm_scan()` / `tts_norm_sub()`, and `tts_dict()` for dictionaries stored in the GGUF such as a pronunciation lexicon (see `docs/tts-norm-patterns.md`), and add test cases to `tests/test-jinja.cpp`, which also checks them against the python reference with `-py`
+ - If the model requires special preprocessing of the input text prompt, see [Text preprocessing](#text-preprocessing-the-tts-chat-template)
4. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this:
- 10-20% changes is to add new backbone (text) model and conversion
- 60% changes inside `mtmd-helper-gen.cpp`
diff --git a/tools/mtmd/models/kokoro.cpp b/tools/mtmd/models/kokoro.cpp
new file mode 100644
index 000000000..1379701c3
--- /dev/null
+++ b/tools/mtmd/models/kokoro.cpp
@@ -0,0 +1,515 @@
+#include "models.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <random>
+#include <stdexcept>
+
+// Kokoro: StyleTTS2 prosody predictor + iSTFTNet decoder. Tensors keep the names of the reference modules (a.gen.<module>.*)
+// One utterance runs the 3 graphs of stage_type. The LSTMs, the alignment, the harmonic source and the ISTFT run on the host in between
+// The graphs keep activations as [T, C], the layout of the conv ops
+
+static const int KOKORO_N_HIDDEN = 512;
+static const int KOKORO_N_HARM = 9; // fundamental + 8 overtones
+static const int KOKORO_UPS = 300; // samples per F0 step: upsample rates 10 * 6, times the ISTFT hop
+static const int KOKORO_N_FFT = 20;
+static const int KOKORO_HOP = 5;
+static const float KOKORO_SR = 24000.0f;
+
+ggml_tensor * mtmd_graph_kokoro::tensor(const std::string & name) const {
+ const auto it = model.cbx_tensors.find(name);
+ if (it == model.cbx_tensors.end() || !it->second) {
+ throw std::runtime_error("kokoro: missing tensor " + name);
+ }
+ return it->second;
+}
+
+bool mtmd_graph_kokoro::has_tensor(const std::string & name) const {
+ return model.cbx_tensors.find(name) != model.cbx_tensors.end();
+}
+
+// x [T, IC] -> [T_out, OC], with the bias if the module has one
+ggml_tensor * mtmd_graph_kokoro::conv(const std::string & name, ggml_tensor * x, int stride, int pad, int dil) const {
+ ggml_tensor * y = ggml_conv_1d(ctx0, tensor(name + ".weight"), x, stride, pad, dil);
+ if (has_tensor(name + ".bias")) {
+ ggml_tensor * b = tensor(name + ".bias");
+ y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
+ }
+ return y;
+}
+
+// AdaIN1d: instance norm over time, then scale and shift from the style
+ggml_tensor * mtmd_graph_kokoro::adain(const std::string & name, ggml_tensor * x, ggml_tensor * style) const {
+ const int64_t n_ch = x->ne[1];
+ ggml_tensor * h = ggml_add(ctx0, ggml_mul_mat(ctx0, tensor(name + ".fc.weight"), style), tensor(name + ".fc.bias"));
+ ggml_tensor * gamma = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, h, n_ch, 0), 1, n_ch);
+ ggml_tensor * beta = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, h, n_ch, n_ch * ggml_element_size(h)), 1, n_ch);
+ ggml_tensor * xn = ggml_norm(ctx0, x, 1e-5f);
+ return ggml_add(ctx0, ggml_add(ctx0, xn, ggml_mul(ctx0, xn, gamma)), beta);
+}
+
+// x + sin^2(alpha x) / alpha, alpha per channel
+ggml_tensor * mtmd_graph_kokoro::snake(const std::string & name, ggml_tensor * x) const {
+ ggml_tensor * alpha = tensor(name);
+ alpha = ggml_reshape_2d(ctx0, alpha, 1, alpha->ne[0]);
+ ggml_tensor * sx = ggml_sin(ctx0, ggml_mul(ctx0, x, alpha));
+ sx = ggml_div(ctx0, ggml_mul(ctx0, sx, sx), alpha);
+ return ggml_add(ctx0, x, sx);
+}
+
+// a [T, C] and b [T, C] -> [2T, C] with a on the even steps
+ggml_tensor * mtmd_graph_kokoro::interleave(ggml_tensor * a, ggml_tensor * b) const {
+ const int64_t n_t = a->ne[0], n_ch = a->ne[1];
+ ggml_tensor * ab = ggml_concat(ctx0, ggml_reshape_3d(ctx0, ggml_cont(ctx0, a), 1, n_t, n_ch),
+ ggml_reshape_3d(ctx0, ggml_cont(ctx0, b), 1, n_t, n_ch), 0);
+ return ggml_reshape_2d(ctx0, ggml_cont(ctx0, ab), 2 * n_t, n_ch);
+}
+
+// AdainResBlk1d, upsample doubles the length
+ggml_tensor * mtmd_graph_kokoro::adain_resblk(const std::string & name, ggml_tensor * x, ggml_tensor * style, bool upsample) const {
+ const int64_t n_t = x->ne[0], n_ch = x->ne[1];
+
+ ggml_tensor * r = adain(name + ".norm1", x, style);
+ r = ggml_leaky_relu(ctx0, r, 0.2f, false);
+ if (upsample) {
+ // depthwise ConvTranspose1d(kernel 3, stride 2, padding 1, output_padding 1), weight stored as [C, 3]:
+ // y[2i] = w1 x[i], y[2i+1] = w2 x[i] + w0 x[i+1]
+ ggml_tensor * w = tensor(name + ".pool.weight");
+ ggml_tensor * b = tensor(name + ".pool.bias");
+ auto tap = [&](int k) {
+ return ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, w, n_ch, (size_t) k * n_ch * ggml_element_size(w)), 1, n_ch);
+ };
+ ggml_tensor * next = ggml_cont(ctx0, ggml_view_2d(ctx0, r, n_t - 1, n_ch, r->nb[1], ggml_element_size(r)));
+ next = ggml_pad(ctx0, next, 1, 0, 0, 0);
+ ggml_tensor * even = ggml_mul(ctx0, r, tap(1));
+ ggml_tensor * odd = ggml_add(ctx0, ggml_mul(ctx0, r, tap(2)), ggml_mul(ctx0, next, tap(0)));
+ r = ggml_add(ctx0, interleave(even, odd), ggml_reshape_2d(ctx0, b, 1, n_ch));
+ }
+ r = conv(name + ".conv1", r, 1, 1, 1);
+ r = adain(name + ".norm2", r, style);
+ r = ggml_leaky_relu(ctx0, r, 0.2f, false);
+ r = conv(name + ".conv2", r, 1, 1, 1);
+
+ ggml_tensor * sc = upsample ? interleave(x, x) : x;
+ if (has_tensor(name + ".conv1x1.weight")) {
+ sc = conv(name + ".conv1x1", sc, 1, 0, 1);
+ }
+ return ggml_scale(ctx0, ggml_add(ctx0, r, sc), (float) M_SQRT1_2);
+}
+
+// AdaINResBlock1 of the generator: three units with dilations 1/3/5 on convs1
+ggml_tensor * mtmd_graph_kokoro::gen_resblock(const std::string & name, ggml_tensor * x, ggml_tensor * style) const {
+ static const int dil[3] = {1, 3, 5};
+ for (int j = 0; j < 3; j++) {
+ const std::string js = std::to_string(j);
+ const int k = (int) tensor(name + ".convs1." + js + ".weight")->ne[0];
+ ggml_tensor * xt = adain(name + ".adain1." + js, x, style);
+ xt = snake(name + ".alpha1." + js, xt);
+ xt = conv(name + ".convs1." + js, xt, 1, (k * dil[j] - dil[j]) / 2, dil[j]);
+ xt = adain(name + ".adain2." + js, xt, style);
+ xt = snake(name + ".alpha2." + js, xt);
+ xt = conv(name + ".convs2." + js, xt, 1, (k - 1) / 2, 1);
+ x = ggml_add(ctx0, xt, x);
+ }
+ return x;
+}
+
+ggml_tensor * mtmd_graph_kokoro::input(const char * name, ggml_type type, int64_t ne0, int64_t ne1) const {
+ ggml_tensor * t = ggml_new_tensor_2d(ctx0, type, ne0, ne1);
+ ggml_set_name(t, name);
+ ggml_set_input(t);
+ return t;
+}
+
+ggml_cgraph * mtmd_graph_kokoro::output(const char * name, ggml_tensor * t) const {
+ ggml_set_name(t, name);
+ ggml_set_output(t);
+ ggml_build_forward_expand(gf, t);
+ return gf;
+}
+
+// tokens -> text encoder CNN output [C, n_tok]
+ggml_cgraph * mtmd_graph_kokoro::build_text() {
+ const std::string p = "a.gen.text.";
+ ggml_tensor * tokens = input("inp_tokens", GGML_TYPE_I32, n_tok, 1);
+ tokens = ggml_reshape_1d(ctx0, tokens, n_tok);
+
+ ggml_tensor * x = ggml_get_rows(ctx0, tensor(p + "embedding.weight"), tokens); // [C, T]
+ for (int i = 0; has_tensor(p + "cnn." + std::to_string(i) + ".0.weight"); i++) {
+ const std::string q = p + "cnn." + std::to_string(i);
+ const int k = (int) tensor(q + ".0.weight")->ne[0];
+ x = conv(q + ".0", ggml_cont(ctx0, ggml_transpose(ctx0, x)), 1, (k - 1) / 2, 1);
+ // LayerNorm over the channels
+ x = ggml_norm(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, x)), 1e-5f);
+ x = ggml_add(ctx0, ggml_mul(ctx0, x, tensor(q + ".1.gamma")), tensor(q + ".1.beta"));
+ x = ggml_leaky_relu(ctx0, x, 0.2f, false);
+ }
+ return output("out_text", x);
+}
+
+// shared LSTM output [C, n_frames] -> F0 and energy curves, 2 steps per frame
+ggml_cgraph * mtmd_graph_kokoro::build_f0n() {
+ const std::string p = "a.gen.pred.";
+ ggml_tensor * inp = input("inp_x", GGML_TYPE_F32, KOKORO_N_HIDDEN, n_frames);
+ ggml_tensor * style = input("inp_style", GGML_TYPE_F32, n_style, 1);
+ style = ggml_reshape_1d(ctx0, style, n_style);
+ ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
+
+ for (const char * curve : {"F0", "N"}) {
+ ggml_tensor * cur = x;
+ for (int i = 0; i < 3; i++) {
+ const std::string q = p + curve + "." + std::to_string(i);
+ cur = adain_resblk(q, cur, style, has_tensor(q + ".pool.weight"));
+ }
+ cur = conv(p + curve + "_proj", cur, 1, 0, 1);
+ ggml_set_name(cur, curve[0] == 'F' ? "out_f0" : "out_n");
+ ggml_set_output(cur);
+ ggml_build_forward_expand(gf, cur);
+ }
+ return gf;
+}
+
+// aligned text features, F0, energy and the source STFT -> conv_post output [n_fft + 2, n_har]
+ggml_cgraph * mtmd_graph_kokoro::build_dec() {
+ const std::string p = "a.gen.dec.";
+ ggml_tensor * asr = input("inp_asr", GGML_TYPE_F32, KOKORO_N_HIDDEN, n_frames);
+ ggml_tensor * f0 = input("inp_f0", GGML_TYPE_F32, 2 * n_frames, 1);
+ ggml_tensor * en = input("inp_n", GGML_TYPE_F32, 2 * n_frames, 1);
+ ggml_tensor * har = input("inp_har", GGML_TYPE_F32, KOKORO_N_FFT + 2, n_har);
+ ggml_tensor * style = input("inp_style", GGML_TYPE_F32, n_style, 1);
+ style = ggml_reshape_1d(ctx0, style, n_style);
+ asr = ggml_cont(ctx0, ggml_transpose(ctx0, asr));
+ har = ggml_cont(ctx0, ggml_transpose(ctx0, har));
+
+ f0 = conv(p + "F0_conv", f0, 2, 1, 1);
+ en = conv(p + "N_conv", en, 2, 1, 1);
+
+ ggml_tensor * x = ggml_concat(ctx0, ggml_concat(ctx0, asr, f0, 1), en, 1);
+ x = adain_resblk(p + "encode", x, style, false);
+
+ ggml_tensor * asr_res = conv(p + "asr_res.0", asr, 1, 0, 1);
+ for (int i = 0; has_tensor(p + "decode." + std::to_string(i) + ".conv1.weight"); i++) {
+ const std::string q = p + "decode." + std::to_string(i);
+ x = ggml_concat(ctx0, ggml_concat(ctx0, ggml_concat(ctx0, x, asr_res, 1), f0, 1), en, 1);
+ x = adain_resblk(q, x, style, has_tensor(q + ".pool.weight"));
+ }
+
+ const std::string g = p + "generator.";
+ int n_ups = 0;
+ while (has_tensor(g + "ups." + std::to_string(n_ups) + ".weight")) {
+ n_ups++;
+ }
+ for (int i = 0; i < n_ups; i++) {
+ const std::string is = std::to_string(i);
+ ggml_tensor * up_w = tensor(g + "ups." + is + ".weight");
+ ggml_tensor * up_b = tensor(g + "ups." + is + ".bias");
+ // the transposed conv has kernel 2 * stride and torch padding stride / 2
+ const int stride = (int) up_w->ne[0] / 2;
+ const int trim = stride / 2;
+
+ x = ggml_leaky_relu(ctx0, x, 0.1f, false);
+
+ // the source STFT is brought to the rate of this stage
+ ggml_tensor * src_w = tensor(g + "noise_convs." + is + ".weight");
+ const int src_k = (int) src_w->ne[0];
+ const int src_s = src_k > 1 ? src_k / 2 : 1;
+ ggml_tensor * src = conv(g + "noise_convs." + is, har, src_s, src_k > 1 ? (src_s + 1) / 2 : 0, 1);
+ src = gen_resblock(g + "noise_res." + is, src, style);
+
+ x = ggml_conv_transpose_1d(ctx0, up_w, x, stride, 0, 1);
+ x = ggml_cont(ctx0, ggml_view_2d(ctx0, x, x->ne[0] - 2 * trim, x->ne[1], x->nb[1], (size_t) trim * ggml_element_size(x)));
+ x = ggml_add(ctx0, x, ggml_reshape_2d(ctx0, up_b, 1, up_b->ne[0]));
+ if (i == n_ups - 1) {
+ x = ggml_pad_reflect_1d(ctx0, x, 1, 0);
+ }
+ x = ggml_add(ctx0, x, src);
+
+ ggml_tensor * acc = nullptr;
+ for (int j = 0; j < 3; j++) {
+ ggml_tensor * r = gen_resblock(g + "resblocks." + std::to_string(i * 3 + j), x, style);
+ acc = acc ? ggml_add(ctx0, acc, r) : r;
+ }
+ x = ggml_scale(ctx0, acc, 1.0f / 3.0f);
+ }
+ x = ggml_leaky_relu(ctx0, x, 0.01f, false);
+ x = conv(g + "conv_post", x, 1, 3, 1);
+ return output("out_spec", ggml_cont(ctx0, ggml_transpose(ctx0, x)));
+}
+
+ggml_cgraph * mtmd_graph_kokoro::build() {
+ switch (stage) {
+ case STAGE_TEXT: return build_text();
+ case STAGE_F0N: return build_f0n();
+ case STAGE_DEC: return build_dec();
+ }
+ GGML_ABORT("invalid kokoro stage");
+}
+
+//
+// host side
+//
+
+static std::vector<float> kokoro_read(const mtmd_model & model, const std::string & name, size_t n_expect) {
+ std::vector<float> v(n_expect);
+ if (model.cbx_read_tensor(name.c_str(), v.data(), v.size()) != n_expect) {
+ throw std::runtime_error("kokoro: unexpected tensor " + name);
+ }
+ return v;
+}
+
+static inline float kokoro_sigmoid(float x) {
+ return 1.0f / (1.0f + expf(-x));
+}
+
+// y = W x, W row-major [n_out, n_in]
+static void kokoro_matvec(const float * w, const float * x, int n_out, int n_in, float * y) {
+ for (int o = 0; o < n_out; o++) {
+ const float * row = w + (size_t) o * n_in;
+ float acc[8] = {0.0f};
+ int i = 0;
+ for (; i + 8 <= n_in; i += 8) {
+ for (int k = 0; k < 8; k++) {
+ acc[k] += row[i + k] * x[i + k];
+ }
+ }
+ float sum = 0.0f;
+ for (; i < n_in; i++) {
+ sum += row[i] * x[i];
+ }
+ for (int k = 0; k < 8; k++) {
+ sum += acc[k];
+ }
+ y[o] = sum;
+ }
+}
+
+// bidirectional single-layer LSTM (torch layout, gates i f g o): x [n_t, n_in] -> y [n_t, 2 * n_h]
+static void kokoro_lstm(const mtmd_model & model, const std::string & name, const std::vector<float> & x, int n_t, int n_in,
+ std::vector<float> & y) {
+ const int n_h = KOKORO_N_HIDDEN / 2;
+ y.assign((size_t) n_t * 2 * n_h, 0.0f);
+ std::vector<float> gi(4 * n_h), gh(4 * n_h), h(n_h), c(n_h);
+ for (int dir = 0; dir < 2; dir++) {
+ const std::string sfx = dir == 0 ? "" : "_reverse";
+ const auto w_ih = kokoro_read(model, name + ".weight_ih_l0" + sfx, (size_t) 4 * n_h * n_in);
+ const auto w_hh = kokoro_read(model, name + ".weight_hh_l0" + sfx, (size_t) 4 * n_h * n_h);
+ const auto bias = kokoro_read(model, name + ".bias_l0" + sfx, (size_t) 4 * n_h);
+ std::fill(h.begin(), h.end(), 0.0f);
+ std::fill(c.begin(), c.end(), 0.0f);
+ for (int s = 0; s < n_t; s++) {
+ const int t = dir == 0 ? s : n_t - 1 - s;
+ kokoro_matvec(w_ih.data(), x.data() + (size_t) t * n_in, 4 * n_h, n_in, gi.data());
+ kokoro_matvec(w_hh.data(), h.data(), 4 * n_h, n_h, gh.data());
+ float * out = y.data() + (size_t) t * 2 * n_h + (size_t) dir * n_h;
+ for (int j = 0; j < n_h; j++) {
+ const float ig = kokoro_sigmoid(gi[j] + gh[j] + bias[j]);
+ const float fg = kokoro_sigmoid(gi[n_h + j] + gh[n_h + j] + bias[n_h + j]);
+ const float gg = tanhf(gi[2 * n_h + j] + gh[2 * n_h + j] + bias[2 * n_h + j]);
+ const float og = kokoro_sigmoid(gi[3 * n_h + j] + gh[3 * n_h + j] + bias[3 * n_h + j]);
+ c[j] = fg * c[j] + ig * gg;
+ h[j] = og * tanhf(c[j]);
+ out[j] = h[j];
+ }
+ }
+ }
+}
+
+// rows of x repeated by their duration: [n_tok, n_ch] -> [n_frames, n_ch]
+static std::vector<float> kokoro_align(const std::vector<float> & x, const std::vector<int32_t> & dur, int n_ch) {
+ std::vector<float> y;
+ for (size_t t = 0; t < dur.size(); t++) {
+ for (int32_t r = 0; r < dur[t]; r++) {
+ y.insert(y.end(), x.begin() + t * n_ch, x.begin() + (t + 1) * n_ch);
+ }
+ }
+ return y;
+}
+
+int mtmd_graph_kokoro::prosody(const mtmd_model & model, const float * bert, int n_embd, const float * text_cnn, int n_tok,
+ const float * style, float speed, std::vector<float> & shared, std::vector<float> & asr) {
+ const int n_h = KOKORO_N_HIDDEN;
+ const int n_cat = n_h + n_style;
+
+ // bert_encoder, then each row is followed by the style
+ const auto enc_w = kokoro_read(model, "a.gen.bert_enc.weight", (size_t) n_h * n_embd);
+ const auto enc_b = kokoro_read(model, "a.gen.bert_enc.bias", n_h);
+ std::vector<float> x((size_t) n_tok * n_cat);
+ for (int t = 0; t < n_tok; t++) {
+ float * row = x.data() + (size_t) t * n_cat;
+ kokoro_matvec(enc_w.data(), bert + (size_t) t * n_embd, n_h, n_embd, row);
+ for (int j = 0; j < n_h; j++) {
+ row[j] += enc_b[j];
+ }
+ memcpy(row + n_h, style, n_style * sizeof(float));
+ }
+
+ // DurationEncoder: LSTM, AdaLayerNorm, style again
+ std::vector<float> y;
+ std::vector<float> gamma_beta(2 * n_h);
+ for (int il = 0; model.cbx_tensors.count(mtmd_format("a.gen.pred.text_encoder.lstms.%d.weight_ih_l0", 2 * il)); il++) {
+ kokoro_lstm(model, mtmd_format("a.gen.pred.text_encoder.lstms.%d", 2 * il), x, n_tok, n_cat, y);
+ const std::string norm = mtmd_format("a.gen.pred.text_encoder.lstms.%d.fc", 2 * il + 1);
+ const auto fc_w = kokoro_read(model, norm + ".weight", (size_t) 2 * n_h * n_style);
+ const auto fc_b = kokoro_read(model, norm + ".bias", (size_t) 2 * n_h);
+ kokoro_matvec(fc_w.data(), style, 2 * n_h, n_style, gamma_beta.data());
+ for (int t = 0; t < n_tok; t++) {
+ const float * src = y.data() + (size_t) t * n_h;
+ float * dst = x.data() + (size_t) t * n_cat;
+ double mean = 0.0, var = 0.0;
+ for (int j = 0; j < n_h; j++) {
+ mean += src[j];
+ }
+ mean /= n_h;
+ for (int j = 0; j < n_h; j++) {
+ var += (src[j] - mean) * (src[j] - mean);
+ }
+ const double scale = 1.0 / sqrt(var / n_h + 1e-5);
+ for (int j = 0; j < n_h; j++) {
+ const float v = (float) ((src[j] - mean) * scale);
+ dst[j] = (1.0f + gamma_beta[j] + fc_b[j]) * v + gamma_beta[n_h + j] + fc_b[n_h + j];
+ }
+ }
+ }
+
+ // durations: frames per token
+ kokoro_lstm(model, "a.gen.pred.lstm", x, n_tok, n_cat, y);
+ const ggml_tensor * proj = model.cbx_tensors.at("a.gen.pred.duration_proj.linear_layer.weight");
+ const int n_bins = (int) proj->ne[1];
+ const auto proj_w = kokoro_read(model, "a.gen.pred.duration_proj.linear_layer.weight", (size_t) n_bins * n_h);
+ const auto proj_b = kokoro_read(model, "a.gen.pred.duration_proj.linear_layer.bias", n_bins);
+ std::vector<int32_t> dur(n_tok);
+ std::vector<float> logits(n_bins);
+ int n_frames = 0;
+ for (int t = 0; t < n_tok; t++) {
+ kokoro_matvec(proj_w.data(), y.data() + (size_t) t * n_h, n_bins, n_h, logits.data());
+ float sum = 0.0f;
+ for (int j = 0; j < n_bins; j++) {
+ sum += kokoro_sigmoid(logits[j] + proj_b[j]);
+ }
+ // round half to even, like torch.round
+ dur[t] = std::max(1, (int) nearbyintf(sum / speed));
+ n_frames += dur[t];
+ }
+
+ kokoro_lstm(model, "a.gen.pred.shared", kokoro_align(x, dur, n_cat), n_frames, n_cat, shared);
+
+ // text encoder: its LSTM runs on the CNN output
+ kokoro_lstm(model, "a.gen.text.lstm", std::vector<float>(text_cnn, text_cnn + (size_t) n_tok * n_h), n_tok, n_h, y);
+ asr = kokoro_align(y, dur, n_h);
+ return n_frames;
+}
+
+// hn-nsf source: f0 [2 * n_frames] -> harmonic source, then its STFT as [n_har, 22] rows of magnitudes and phases
+int mtmd_graph_kokoro::har_source(const mtmd_model & model, const std::vector<float> & f0, std::mt19937 & rng, std::vector<float> & har) {
+ const int n_f0 = (int) f0.size();
+ const int64_t n_wav = (int64_t) n_f0 * KOKORO_UPS;
+ const auto lin_w = kokoro_read(model, "a.gen.dec.generator.m_source.l_linear.weight", KOKORO_N_HARM);
+ const auto lin_b = kokoro_read(model, "a.gen.dec.generator.m_source.l_linear.bias", 1);
+
+ // SineGen: the phase adds up at the F0 rate, then it is interpolated to the sample rate
+ // the reference also adds a random start phase, but its own downsampling drops it
+ std::vector<float> phase((size_t) n_f0 * KOKORO_N_HARM);
+ for (int h = 0; h < KOKORO_N_HARM; h++) {
+ float cum = 0.0f;
+ for (int i = 0; i < n_f0; i++) {
+ float rad = f0[i] * (float) (h + 1) / KOKORO_SR;
+ rad -= floorf(rad);
+ cum += rad;
+ phase[(size_t) i * KOKORO_N_HARM + h] = cum * 2.0f * (float) M_PI * KOKORO_UPS;
+ }
+ }
+ std::normal_distribution<float> gauss(0.0f, 1.0f);
+ std::vector<float> src((size_t) n_wav);
+ for (int64_t t = 0; t < n_wav; t++) {
+ const float pos = std::max(0.0f, ((float) t + 0.5f) / KOKORO_UPS - 0.5f);
+ const int i0 = std::min((int) pos, n_f0 - 1);
+ const int i1 = std::min(i0 + 1, n_f0 - 1);
+ const float frac = pos - (float) i0;
+ const float uv = f0[t / KOKORO_UPS] > 10.0f ? 1.0f : 0.0f; // voiced threshold
+ const float namp = uv * 0.003f + (1.0f - uv) * 0.1f / 3.0f;
+ float merged = lin_b[0];
+ for (int h = 0; h < KOKORO_N_HARM; h++) {
+ const float ph = (1.0f - frac) * phase[(size_t) i0 * KOKORO_N_HARM + h] + frac * phase[(size_t) i1 * KOKORO_N_HARM + h];
+ merged += lin_w[h] * (0.1f * sinf(ph) * uv + namp * gauss(rng));
+ }
+ src[(size_t) t] = tanhf(merged);
+ }
+
+ // STFT: periodic hann window, centered with reflect padding
+ const int n_bins = KOKORO_N_FFT / 2 + 1;
+ const int n_har = (int) (n_wav / KOKORO_HOP) + 1;
+ float win[KOKORO_N_FFT], tab_cos[KOKORO_N_FFT], tab_sin[KOKORO_N_FFT];
+ for (int i = 0; i < KOKORO_N_FFT; i++) {
+ win[i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
+ tab_cos[i] = cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
+ tab_sin[i] = sinf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
+ }
+ har.resize((size_t) n_har * 2 * n_bins);
+ for (int fr = 0; fr < n_har; fr++) {
+ float frame[KOKORO_N_FFT];
+ for (int i = 0; i < KOKORO_N_FFT; i++) {
+ int64_t idx = (int64_t) fr * KOKORO_HOP - KOKORO_N_FFT / 2 + i;
+ if (idx < 0) {
+ idx = -idx;
+ }
+ if (idx >= n_wav) {
+ idx = 2 * (n_wav - 1) - idx;
+ }
+ frame[i] = src[(size_t) idx] * win[i];
+ }
+ float * out = har.data() + (size_t) fr * 2 * n_bins;
+ for (int k = 0; k < n_bins; k++) {
+ float re = 0.0f, im = 0.0f;
+ for (int i = 0; i < KOKORO_N_FFT; i++) {
+ const int a = (k * i) % KOKORO_N_FFT;
+ re += frame[i] * tab_cos[a];
+ im -= frame[i] * tab_sin[a];
+ }
+ out[k] = sqrtf(re * re + im * im);
+ out[n_bins + k] = atan2f(im, re);
+ }
+ }
+ return n_har;
+}
+
+// conv_post rows [n_frames, 22] -> waveform: magnitude exp(x), phase sin(x), hann overlap-add
+void mtmd_graph_kokoro::istft(const std::vector<float> & spec, std::vector<float> & audio) {
+ const int n_bins = KOKORO_N_FFT / 2 + 1;
+ const int n_frames = (int) (spec.size() / (2 * n_bins));
+ const int64_t n_out = (int64_t) (n_frames - 1) * KOKORO_HOP;
+ float win[KOKORO_N_FFT], tab_cos[KOKORO_N_FFT], tab_sin[KOKORO_N_FFT];
+ for (int i = 0; i < KOKORO_N_FFT; i++) {
+ win[i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
+ tab_cos[i] = cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
+ tab_sin[i] = sinf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
+ }
+ std::vector<float> acc((size_t) n_out + KOKORO_N_FFT, 0.0f);
+ std::vector<float> wsum((size_t) n_out + KOKORO_N_FFT, 0.0f);
+ for (int fr = 0; fr < n_frames; fr++) {
+ const float * row = spec.data() + (size_t) fr * 2 * n_bins;
+ float re[KOKORO_N_FFT / 2 + 1], im[KOKORO_N_FFT / 2 + 1];
+ for (int k = 0; k < n_bins; k++) {
+ const float mag = expf(row[k]);
+ const float ph = sinf(row[n_bins + k]);
+ re[k] = mag * cosf(ph);
+ im[k] = mag * sinf(ph);
+ }
+ for (int i = 0; i < KOKORO_N_FFT; i++) {
+ float v = 0.0f;
+ for (int k = 0; k < n_bins; k++) {
+ const int a = (k * i) % KOKORO_N_FFT;
+ const float w = (k == 0 || k == KOKORO_N_FFT / 2) ? 1.0f : 2.0f;
+ v += w * (re[k] * tab_cos[a] - im[k] * tab_sin[a]);
+ }
+ const size_t o = (size_t) fr * KOKORO_HOP + i;
+ acc[o] += v / KOKORO_N_FFT * win[i];
+ wsum[o] += win[i] * win[i];
+ }
+ }
+ audio.resize((size_t) std::max<int64_t>(n_out, 0));
+ for (size_t i = 0; i < audio.size(); i++) {
+ const size_t j = i + KOKORO_N_FFT / 2; // drop the centering pad
+ audio[i] = wsum[j] > 1e-11f ? acc[j] / wsum[j] : 0.0f;
+ }
+}
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 5e55e29bc..802561765 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -258,6 +258,47 @@ struct mtmd_graph_chatterbox : mtmd_graph_chatterbox_base {
ggml_cgraph * build_vocoder(int n_mel, int n_stft);
};
+struct mtmd_graph_kokoro : mtmd_graph {
+ enum stage_type {
+ STAGE_TEXT, // tokens -> text encoder CNN
+ STAGE_F0N, // shared LSTM output -> F0 and energy curves
+ STAGE_DEC, // aligned features + source STFT -> STFT coefficients
+ };
+ static constexpr int n_style = 128; // a voice row is the decoder style, then the predictor style
+
+ stage_type stage;
+ int n_tok = 0;
+ int n_frames = 0;
+ int n_har = 0;
+ mtmd_graph_kokoro(mtmd_subcontext * ctx, const mtmd_image_f32 & img, stage_type stage, int n_tok, int n_frames, int n_har)
+ : mtmd_graph(ctx, img), stage(stage), n_tok(n_tok), n_frames(n_frames), n_har(n_har) {}
+ ggml_cgraph * build() override;
+
+ // host side, between the graphs
+ // bert [n_tok, n_embd], text_cnn [n_tok, 512], style is the predictor style
+ // fills shared and asr as [n_frames, 512], returns n_frames
+ static int prosody(const mtmd_model & model, const float * bert, int n_embd, const float * text_cnn, int n_tok,
+ const float * style, float speed, std::vector<float> & shared, std::vector<float> & asr);
+ // fills har as [n_har, 22], returns n_har
+ static int har_source(const mtmd_model & model, const std::vector<float> & f0, std::mt19937 & rng, std::vector<float> & har);
+ static void istft(const std::vector<float> & spec, std::vector<float> & audio);
+
+ private:
+ ggml_tensor * tensor(const std::string & name) const;
+ bool has_tensor(const std::string & name) const;
+ ggml_tensor * conv(const std::string & name, ggml_tensor * x, int stride, int pad, int dil) const;
+ ggml_tensor * adain(const std::string & name, ggml_tensor * x, ggml_tensor * style) const;
+ ggml_tensor * snake(const std::string & name, ggml_tensor * x) const;
+ ggml_tensor * interleave(ggml_tensor * a, ggml_tensor * b) const;
+ ggml_tensor * adain_resblk(const std::string & name, ggml_tensor * x, ggml_tensor * style, bool upsample) const;
+ ggml_tensor * gen_resblock(const std::string & name, ggml_tensor * x, ggml_tensor * style) const;
+ ggml_tensor * input(const char * name, ggml_type type, int64_t ne0, int64_t ne1) const;
+ ggml_cgraph * output(const char * name, ggml_tensor * t) const;
+ ggml_cgraph * build_text();
+ ggml_cgraph * build_f0n();
+ ggml_cgraph * build_dec();
+};
+
struct mtmd_graph_lfm3a : mtmd_graph {
mtmd_graph_lfm3a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp
index 350aa4c9f..c57549d6e 100644
--- a/tools/mtmd/mtmd-graph.cpp
+++ b/tools/mtmd/mtmd-graph.cpp
@@ -1097,6 +1097,13 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
const int n_frames = params && params->feats ? (int) (params->feats->size() / ctx->model.hparams.projection_dim) : 2;
builder = std::make_unique<mtmd_graph_soprano>(ctx, img, n_frames);
} break;
+ case PROJECTOR_TYPE_KOKORO:
+ {
+ // without a stage (warmup), a short decoder graph: 2 frames, 600 samples each, one STFT row per 5 samples
+ const auto * io = params ? params->kokoro : nullptr;
+ builder = io ? std::make_unique<mtmd_graph_kokoro>(ctx, img, (mtmd_graph_kokoro::stage_type) io->stage, io->n_tok, io->n_frames, io->n_har)
+ : std::make_unique<mtmd_graph_kokoro>(ctx, img, mtmd_graph_kokoro::STAGE_DEC, 0, 2, 241);
+ } break;
case PROJECTOR_TYPE_YOUTUVL:
{
builder = std::make_unique<mtmd_graph_youtuvl>(ctx, img);
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index cac18b345..bd7ee1885 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -128,6 +128,8 @@ public:
virtual bool supports_continue() const { return false; }
// mtmd_helper_gen_audio_inp::speed and loudness other than the defaults
virtual bool supports_delivery() const { return false; }
+ // speed alone, for models without loudness control
+ virtual bool supports_speed() const { return supports_delivery(); }
// mtmd_helper_gen_audio_inp::cfg_scale other than the default
virtual bool supports_cfg() const { return false; }
// mtmd_helper_gen_audio_inp::flow_steps other than the default
@@ -1121,6 +1123,244 @@ private:
bool wav_header_sent = false;
};
+// Kokoro is not autoregressive: the backbone encodes the phonemes once, the mmproj turns its states into the whole utterance
+// the prompt is a phoneme string (the output of the "tts" chat template), one token per character
+class kokoro_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
+public:
+ using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
+
+ // the predicted durations are divided by the speed
+ bool supports_speed() const override { return true; }
+
+ void reset() override {
+ segments.clear();
+ audio.clear();
+ wav.clear();
+ n_encoded = 0;
+ decoded = false;
+ sent = false;
+ }
+
+ int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
+ reset();
+ if (!inp->prompt || inp->prompt_len == 0 || inp->seq_id < 0 || (uint32_t) inp->seq_id >= llama_n_seq_max(lctx)) {
+ LOG_ERR("mtmd_helper_gen_audio: kokoro requires a prompt\n");
+ return 1;
+ }
+ if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref)) {
+ LOG_ERR("mtmd_helper_gen_audio: kokoro has no voice cloning, use one of its voices\n");
+ return 1;
+ }
+ seq_id = inp->seq_id;
+ out_type = inp->out_type;
+ stream = inp->stream;
+ speed = inp->speed;
+
+ // one token per phoneme character, unknown characters are dropped like in the reference
+ if (token_of.empty()) {
+ for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); id++) {
+ if (llama_vocab_get_attr(vocab, id) & LLAMA_TOKEN_ATTR_NORMAL) {
+ token_of[llama_vocab_get_text(vocab, id)] = id;
+ }
+ }
+ }
+ std::vector<llama_token> phonemes;
+ const std::string prompt(inp->prompt, inp->prompt_len);
+ for (size_t i = 0; i < prompt.size();) {
+ const unsigned char c = (unsigned char) prompt[i];
+ const size_t n = c < 0x80 ? 1 : c < 0xE0 ? 2 : c < 0xF0 ? 3 : 4;
+ const auto it = token_of.find(prompt.substr(i, n));
+ if (it != token_of.end()) {
+ phonemes.push_back(it->second);
+ }
+ i += n;
+ }
+ if (phonemes.empty()) {
+ LOG_ERR("mtmd_helper_gen_audio: the prompt has no phoneme the model knows\n");
+ return 1;
+ }
+
+ // a voice is a table of style rows, the row is picked by the number of phonemes
+ const char * voice = inp->speaker_ref ? mtmd_bitmap_get_id(inp->speaker_ref) : mtmd_gen_audio_voice_name(mctx, 0);
+ if (!voice) {
+ LOG_ERR("mtmd_helper_gen_audio: the model has no voice\n");
+ return 1;
+ }
+ const std::string name = std::string("voice.") + voice;
+ std::vector<float> table(mtmd_gen_audio_read_tensor(mctx, name.c_str(), nullptr, 0));
+ if (table.size() < n_voice_row || table.size() % n_voice_row != 0 ||
+ mtmd_gen_audio_read_tensor(mctx, name.c_str(), table.data(), table.size()) != table.size()) {
+ LOG_ERR("mtmd_helper_gen_audio: unknown voice \"%s\"\n", voice);
+ return 1;
+ }
+
+ // the backbone takes a whole segment in one batch, padded on both ends
+ const size_t n_max = std::min({(size_t) llama_n_ctx_seq(lctx), (size_t) llama_n_ubatch(lctx), (size_t) llama_model_n_ctx_train(model)});
+ if (n_max < 3) {
+ LOG_ERR("mtmd_helper_gen_audio: the context is too small\n");
+ return 1;
+ }
+ const llama_token pad = llama_vocab_bos(vocab);
+ for (size_t pos = 0; pos < phonemes.size();) {
+ const size_t len = split_len(phonemes, pos, n_max - 2);
+ segment seg;
+ seg.tokens.push_back(pad);
+ seg.tokens.insert(seg.tokens.end(), phonemes.begin() + pos, phonemes.begin() + pos + len);
+ seg.tokens.push_back(pad);
+ const size_t row = std::min(len - 1, table.size() / n_voice_row - 1);
+ seg.feats.assign(table.begin() + row * n_voice_row, table.begin() + (row + 1) * n_voice_row);
+ segments.push_back(std::move(seg));
+ pos += len;
+ // a segment does not start with a space
+ while (pos < phonemes.size() && is_any(phonemes[pos], " ")) {
+ pos++;
+ }
+ }
+
+ if (!init_gen_state(inp->seed)) {
+ return 1;
+ }
+ return 0;
+ }
+
+ // the encoder is not causal, it takes one whole segment per call
+ int32_t step_prompt(int32_t) override {
+ if (segments.empty()) {
+ return -1;
+ }
+ if (n_encoded == segments.size()) {
+ return 0;
+ }
+ auto & seg = segments[n_encoded];
+ const int n = (int) seg.tokens.size();
+ llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+ if (!batch_set_tokens(batch.get(), seg.tokens.data(), n, 0, seq_id)) {
+ return -1;
+ }
+ for (int i = 0; i < n; ++i) {
+ llama_batch_ext_set_output_embd(batch.get(), i, true);
+ }
+ if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: kokoro backbone failed\n");
+ return -1;
+ }
+ for (int i = 0; i < n; ++i) {
+ const float * h = llama_get_embeddings_ith(lctx, i);
+ if (!h) {
+ return -1;
+ }
+ seg.feats.insert(seg.feats.end(), h, h + n_embd);
+ }
+ n_encoded++;
+ return (int32_t) (segments.size() - n_encoded);
+ }
+
+ // nothing is generated step by step
+ int32_t step_gen(llama_token, const float *, const float ** h_state_out, bool * out_stop) override {
+ *h_state_out = nullptr;
+ *out_stop = true;
+ return !segments.empty() && n_encoded == segments.size() ? 0 : 1;
+ }
+
+ int32_t get_output(int32_t * sample_rate, const char ** data, size_t * data_len, int64_t * n_samples) override {
+ *sample_rate = info.sample_rate;
+ *data_len = 0;
+ if (n_samples) { *n_samples = 0; }
+ if (stream) {
+ // the audio comes out in one chunk after flush()
+ if (!decoded || sent) {
+ return 0;
+ }
+ sent = true;
+ } else if (!decode_audio()) {
+ return 1;
+ }
+ if (n_samples) { *n_samples = audio.size(); }
+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+ *data = (const char *) audio.data();
+ *data_len = audio.size() * sizeof(float);
+ } else {
+ wav.clear();
+ if (!write_wav16(wav, audio, info.sample_rate)) {
+ return 1;
+ }
+ *data = wav.data();
+ *data_len = wav.size();
+ }
+ return 0;
+ }
+
+ int32_t flush() override {
+ return decode_audio() ? 0 : 1;
+ }
+
+private:
+ struct segment {
+ std::vector<llama_token> tokens;
+ std::vector<float> feats; // the voice row, then the backbone state of each token
+ };
+
+ bool is_any(llama_token id, const char * chars) const {
+ const char * text = llama_vocab_get_text(vocab, id);
+ return text[0] != '\0' && strstr(chars, text) != nullptr;
+ }
+
+ // length of the segment that starts at pos: a long text is cut at the last sentence end that fits, else at a weaker break
+ size_t split_len(const std::vector<llama_token> & phonemes, size_t pos, size_t n_max) const {
+ if (phonemes.size() - pos <= n_max) {
+ return phonemes.size() - pos;
+ }
+ for (const char * breaks : {"!.?\xe2\x80\xa6", ":;", ",\xe2\x80\x94", " "}) {
+ for (size_t len = n_max; len > 1; len--) {
+ if (is_any(phonemes[pos + len - 1], breaks)) {
+ return len;
+ }
+ }
+ }
+ return n_max;
+ }
+
+ bool decode_audio() {
+ if (decoded) {
+ return true;
+ }
+ if (segments.empty() || n_encoded != segments.size()) {
+ return false;
+ }
+ for (const auto & seg : segments) {
+ std::vector<int32_t> codes(seg.tokens.begin(), seg.tokens.end());
+ mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
+ inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
+ inp.codes = codes.data();
+ inp.n_codes = codes.size();
+ inp.feats = seg.feats.data();
+ inp.n_feats = seg.feats.size();
+ inp.speed = speed;
+ mtmd_gen_out out{};
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
+ return false;
+ }
+ audio.insert(audio.end(), out.audio, out.audio + out.n_samples);
+ }
+ decoded = true;
+ return true;
+ }
+
+ static constexpr size_t n_voice_row = 256; // decoder style, then predictor style
+
+ llama_seq_id seq_id = 0;
+ std::unordered_map<std::string, llama_token> token_of;
+ std::vector<segment> segments;
+ size_t n_encoded = 0;
+ float speed = 1.0f;
+ std::vector<float> audio;
+ std::vector<char> wav;
+ mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
+ bool stream = false;
+ bool decoded = false;
+ bool sent = false;
+};
+
// Soprano uses the hidden state that predicts each non-EOS token, including the prompt's last state.
class soprano_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
@@ -4322,6 +4562,8 @@ static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lc
return std::unique_ptr<mtmd_gen_audio_pipeline>(new pockettts_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_SOPRANO:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new soprano_gen_audio_pipeline(lctx, mctx));
+ case MTMD_GEN_AUDIO_TYPE_KOKORO:
+ return std::unique_ptr<mtmd_gen_audio_pipeline>(new kokoro_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_LFM3A:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new lfm3a_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_CHATTERBOX:
@@ -4398,8 +4640,12 @@ int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_
LOG_ERR("mtmd_helper_gen_audio: speed must be > 0 and loudness < 0 LUFS (0 = model default)\n");
return 2;
}
- if ((inp->speed != 1.0f || inp->loudness != 0.0f) && !ctx->pipeline->supports_delivery()) {
- LOG_ERR("mtmd_helper_gen_audio: the model does not support speed or loudness\n");
+ if (inp->speed != 1.0f && !ctx->pipeline->supports_speed()) {
+ LOG_ERR("mtmd_helper_gen_audio: the model does not support speed\n");
+ return 2;
+ }
+ if (inp->loudness != 0.0f && !ctx->pipeline->supports_delivery()) {
+ LOG_ERR("mtmd_helper_gen_audio: the model does not support loudness\n");
return 2;
}
if (!std::isfinite(inp->cfg_scale) || inp->cfg_scale < 0.0f) {
diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h
index e1dc95cee..142b56e8d 100644
--- a/tools/mtmd/mtmd-impl.h
+++ b/tools/mtmd/mtmd-impl.h
@@ -615,6 +615,7 @@ enum projector_type {
PROJECTOR_TYPE_POCKETTTS_SPKENC,
PROJECTOR_TYPE_POCKETTTS_GEN,
PROJECTOR_TYPE_SOPRANO,
+ PROJECTOR_TYPE_KOKORO,
PROJECTOR_TYPE_LFM3A_GEN,
PROJECTOR_TYPE_DRAMA3_GEN,
PROJECTOR_TYPE_ZONOS3_GEN,
@@ -711,6 +712,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_POCKETTTS_SPKENC, "pockettts_spkenc"},
{ PROJECTOR_TYPE_POCKETTTS_GEN, "pockettts_gen"},
{ PROJECTOR_TYPE_SOPRANO, "soprano"},
+ { PROJECTOR_TYPE_KOKORO, "kokoro"},
{ PROJECTOR_TYPE_LFM3A_GEN, "lfm3a_gen"},
{ PROJECTOR_TYPE_DRAMA3_GEN, "drama3_gen"},
{ PROJECTOR_TYPE_ZONOS3_GEN, "zonos3_gen"},
diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp
index c352e28d9..38992233d 100644
--- a/tools/mtmd/mtmd-model.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -1172,6 +1172,10 @@ struct mtmd_model_loader {
hparams.audio_sample_rate = 32000;
hparams.audio_hop_len = 512;
} break;
+ case PROJECTOR_TYPE_KOKORO:
+ {
+ hparams.audio_sample_rate = 24000;
+ } break;
case PROJECTOR_TYPE_PADDLEOCR:
{
hparams.n_merge = 2;
@@ -1578,6 +1582,7 @@ struct mtmd_model_loader {
model.proj_type != PROJECTOR_TYPE_ZONOS3_SPKENC &&
model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN &&
model.proj_type != PROJECTOR_TYPE_SOPRANO &&
+ model.proj_type != PROJECTOR_TYPE_KOKORO &&
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
model.proj_type != PROJECTOR_TYPE_CHATTERBOX &&
@@ -3459,6 +3464,21 @@ struct mtmd_model_loader {
}
}
} break;
+ case PROJECTOR_TYPE_KOKORO:
+ {
+ // the graphs and the host code look the tensors up by name, see models/kokoro.cpp
+ // a voice is a table of style rows, the audio pipeline reads the row it needs
+ for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) {
+ const std::string name = t->name;
+ if (name.rfind("a.gen.", 0) == 0) {
+ model.cbx_tensors[name] = get_tensor(name);
+ } else if (name.rfind("voice.", 0) == 0) {
+ model.cbx_tensors[name] = get_tensor(name);
+ model.gen_voices[name.substr(6)] = {};
+ }
+ }
+ LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size());
+ } break;
case PROJECTOR_TYPE_SOPRANO:
{
auto & v = model.vocos;
@@ -4955,6 +4975,7 @@ int mtmd_model::n_mmproj_embd() const {
// talker conditioning rows, the speaker projection carries their width
return cbx_tensors.at("a.cenc.spkr_enc.weight")->ne[1];
case PROJECTOR_TYPE_SOPRANO:
+ case PROJECTOR_TYPE_KOKORO:
return hparams.projection_dim;
case PROJECTOR_TYPE_PARAKEET:
return mm_1_w->ne[1];
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index d6ec8e862..4e4d418e4 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -521,6 +521,87 @@ mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) {
return st.release();
}
+bool mtmd_subcontext::encode_kokoro(mtmd_subctx_encode_params * params) {
+ const int n_embd = model.n_mmproj_embd();
+ const int n_style = mtmd_graph_kokoro::n_style;
+ if (params->gen_process != MTMD_SUBCTX_GEN_PROCESS_GEN_WAV || !params->codes || !params->feats || !params->gen_state || !params->out_audio) {
+ LOG_ERR("%s: kokoro takes the phoneme tokens and their backbone states\n", __func__);
+ return false;
+ }
+ const auto & tokens = *params->codes;
+ const auto & feats = *params->feats;
+ const int n_tok = (int) tokens.size();
+ if (n_tok < 2 || feats.size() != (size_t) 2 * n_style + (size_t) n_tok * n_embd) {
+ LOG_ERR("%s: expected a voice row of %d values, then %d values for each of the %d tokens\n", __func__, 2 * n_style, n_embd, n_tok);
+ return false;
+ }
+ const int64_t n_vocab = model.cbx_tensors.at("a.gen.text.embedding.weight")->ne[1];
+ for (int32_t t : tokens) {
+ if (t < 0 || t >= n_vocab) {
+ LOG_ERR("%s: phoneme token out of range (%d, vocabulary size %lld)\n", __func__, t, (long long) n_vocab);
+ return false;
+ }
+ }
+ const std::vector<float> style_dec(feats.begin(), feats.begin() + n_style);
+ const std::vector<float> style_pred(feats.begin() + n_style, feats.begin() + 2 * n_style);
+ const float * bert = feats.data() + 2 * n_style;
+
+ auto run = [&](mtmd_subctx_kokoro_io & io) {
+ mtmd_subctx_encode_params p = *params;
+ p.kokoro = &io;
+ p.out_audio = nullptr;
+ return encode(&p);
+ };
+
+ std::vector<float> text_cnn;
+ {
+ mtmd_subctx_kokoro_io io;
+ io.stage = mtmd_graph_kokoro::STAGE_TEXT;
+ io.n_tok = n_tok;
+ io.tokens = &tokens;
+ io.out0 = &text_cnn;
+ if (!run(io)) {
+ return false;
+ }
+ }
+
+ std::vector<float> shared, asr;
+ const int n_frames = mtmd_graph_kokoro::prosody(model, bert, n_embd, text_cnn.data(), n_tok, style_pred.data(), params->speed, shared, asr);
+
+ std::vector<float> f0, en;
+ {
+ mtmd_subctx_kokoro_io io;
+ io.stage = mtmd_graph_kokoro::STAGE_F0N;
+ io.n_frames = n_frames;
+ io.x = &shared;
+ io.style = &style_pred;
+ io.out0 = &f0;
+ io.out1 = &en;
+ if (!run(io)) {
+ return false;
+ }
+ }
+
+ std::vector<float> har, spec;
+ {
+ mtmd_subctx_kokoro_io io;
+ io.stage = mtmd_graph_kokoro::STAGE_DEC;
+ io.n_frames = n_frames;
+ io.n_har = mtmd_graph_kokoro::har_source(model, f0, params->gen_state->rng, har);
+ io.x = &asr;
+ io.style = &style_dec;
+ io.f0 = &f0;
+ io.en = &en;
+ io.har = &har;
+ io.out0 = &spec;
+ if (!run(io)) {
+ return false;
+ }
+ }
+ mtmd_graph_kokoro::istft(spec, *params->out_audio);
+ return true;
+}
+
bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
const mtmd_image_f32_batch & imgs = *params->imgs;
int n_batch_cur = imgs.entries.size();
@@ -536,6 +617,10 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
warmup(*params->imgs);
}
+ if (model.proj_type == PROJECTOR_TYPE_KOKORO && !params->kokoro) {
+ return encode_kokoro(params);
+ }
+
const bool needs_gen_state = params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE ||
params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ||
params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS;
@@ -1514,6 +1599,29 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
{
set_input_f32("inp_feats", *params->feats);
} break;
+ case PROJECTOR_TYPE_KOKORO:
+ {
+ const auto * io = params->kokoro;
+ GGML_ASSERT(io);
+ switch (io->stage) {
+ case mtmd_graph_kokoro::STAGE_TEXT:
+ {
+ std::vector<int32_t> tokens = *io->tokens;
+ set_input_i32("inp_tokens", tokens);
+ } break;
+ case mtmd_graph_kokoro::STAGE_F0N:
+ set_input_f32("inp_x", *io->x);
+ set_input_f32("inp_style", *io->style);
+ break;
+ case mtmd_graph_kokoro::STAGE_DEC:
+ set_input_f32("inp_asr", *io->x);
+ set_input_f32("inp_style", *io->style);
+ set_input_f32("inp_f0", *io->f0);
+ set_input_f32("inp_n", *io->en);
+ set_input_f32("inp_har", *io->har);
+ break;
+ }
+ } break;
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
{
set_input_f32("inp_feats", *params->feats);
@@ -2401,6 +2509,22 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
return false;
}
+ if (model.proj_type == PROJECTOR_TYPE_KOKORO) {
+ const auto * io = params->kokoro;
+ auto read = [&](const char * name, std::vector<float> * out) {
+ ggml_tensor * t = ggml_graph_get_tensor(gf, name);
+ GGML_ASSERT(t != nullptr && out != nullptr);
+ out->resize(ggml_nelements(t));
+ ggml_backend_tensor_get(t, out->data(), 0, ggml_nbytes(t));
+ };
+ switch (io->stage) {
+ case mtmd_graph_kokoro::STAGE_TEXT: read("out_text", io->out0); break;
+ case mtmd_graph_kokoro::STAGE_F0N: read("out_f0", io->out0); read("out_n", io->out1); break;
+ case mtmd_graph_kokoro::STAGE_DEC: read("out_spec", io->out0); break;
+ }
+ return true;
+ }
+
if (embeddings != nullptr) {
// sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one)
const int n_tokens_out = embeddings->ne[1];
diff --git a/tools/mtmd/mtmd-subcontext.h b/tools/mtmd/mtmd-subcontext.h
index d43acec04..e92ef01ee 100644
--- a/tools/mtmd/mtmd-subcontext.h
+++ b/tools/mtmd/mtmd-subcontext.h
@@ -114,6 +114,22 @@ struct mtmd_subctx_gen_state {
}
};
+// kokoro: inputs and outputs of one graph stage, see mtmd_graph_kokoro
+struct mtmd_subctx_kokoro_io {
+ int stage = 0;
+ int n_tok = 0;
+ int n_frames = 0;
+ int n_har = 0;
+ const std::vector<int32_t> * tokens = nullptr; // STAGE_TEXT
+ const std::vector<float> * x = nullptr; // STAGE_F0N: shared LSTM output, STAGE_DEC: aligned text features
+ const std::vector<float> * style = nullptr; // predictor or decoder half of the voice row
+ const std::vector<float> * f0 = nullptr; // STAGE_DEC
+ const std::vector<float> * en = nullptr;
+ const std::vector<float> * har = nullptr;
+ std::vector<float> * out0 = nullptr; // text features, F0, or STFT coefficients
+ std::vector<float> * out1 = nullptr; // STAGE_F0N: energy
+};
+
struct mtmd_subctx_encode_params {
int n_threads = 1;
const mtmd_image_f32_batch * imgs = nullptr;
@@ -148,6 +164,10 @@ struct mtmd_subctx_encode_params {
int vocode_n_stft = 0;
std::vector<float> * out_spec = nullptr;
+ // kokoro: set by encode_kokoro() for each of its graph stages
+ const mtmd_subctx_kokoro_io * kokoro = nullptr;
+ float speed = 1.0f; // kokoro: the predicted durations are divided by it
+
// audio gen: state of one sequence, encode() updates it in place
struct mtmd_subctx_gen_state * gen_state = nullptr;
@@ -205,6 +225,9 @@ private:
mtmd_image_f32_batch get_dummy_batch() const;
+ // kokoro GEN_WAV: codes are the phoneme tokens, feats the voice row followed by the backbone state of each token
+ bool encode_kokoro(mtmd_subctx_encode_params * params);
+
// only initialize backend buffers, but do not allocate them yet
support_info_graph reserve_compute_meta(const mtmd_image_f32_batch & batch);
};
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 7c22a1041..a7bfe8f49 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -2694,6 +2694,10 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
info.type = MTMD_GEN_AUDIO_TYPE_FISH_S2;
info.sample_rate = 44100;
break;
+ case PROJECTOR_TYPE_KOKORO:
+ info.type = MTMD_GEN_AUDIO_TYPE_KOKORO;
+ info.sample_rate = 24000;
+ break;
default:
info.type = MTMD_GEN_AUDIO_TYPE_NONE;
break;
@@ -3091,7 +3095,8 @@ mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * na
mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
mtmd_gen_inp inp{};
- inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
+ inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
+ inp.speed = 1.0f;
if (!ctx->ctx_gen_a) {
return inp;
}
@@ -3110,6 +3115,7 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
inp.temp = 0.3f;
break;
case PROJECTOR_TYPE_SOPRANO:
+ case PROJECTOR_TYPE_KOKORO:
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
break;
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
@@ -3738,6 +3744,46 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
}
}
+ if (sctx->model.proj_type == PROJECTOR_TYPE_KOKORO) {
+ if (inp->type != MTMD_GEN_PROCESS_TYPE_GEN_WAV || !inp->codes || inp->n_codes == 0 || !inp->feats || inp->n_feats == 0) {
+ LOG_ERR("%s: kokoro requires the phoneme tokens and their features\n", __func__);
+ return 1;
+ }
+ if (!std::isfinite(inp->speed) || inp->speed <= 0.0f) {
+ LOG_ERR("%s: speed must be > 0\n", __func__);
+ return 1;
+ }
+ const std::vector<int32_t> in_codes(inp->codes, inp->codes + inp->n_codes);
+ const std::vector<float> in_feats(inp->feats, inp->feats + inp->n_feats);
+
+ // the batch entry is unused, present to satisfy the encode interface
+ mtmd_image_f32 dummy;
+ dummy.set_size({1, 1}, false, true);
+ dummy.cpy_buf(std::vector<float>(1, 0.0f));
+ mtmd_image_f32_batch batch;
+ batch.is_audio = true;
+ batch.entries.push_back(std::move(dummy));
+
+ mtmd_subctx_encode_params params;
+ params.imgs = &batch;
+ params.n_threads = ctx->n_threads;
+ params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV;
+ params.codes = &in_codes;
+ params.feats = &in_feats;
+ params.speed = inp->speed;
+ params.out_audio = &state->out_audio;
+ params.gen_state = state->gen_state;
+
+ if (!sctx->encode(¶ms)) {
+ LOG_ERR("%s: encode failed (gen_wav)\n", __func__);
+ return 1;
+ }
+
+ out->audio = state->out_audio.data();
+ out->n_samples = state->out_audio.size();
+ return 0;
+ }
+
if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_WAV && sctx->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN) {
return mtmd_gen_wav_lfm3a(ctx, state, inp, out);
}
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index 8d5b42a9f..2ed43c38f 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -478,6 +478,7 @@ enum mtmd_gen_audio_type {
MTMD_GEN_AUDIO_TYPE_YUE2,
MTMD_GEN_AUDIO_TYPE_MINIMAX_MUSIC3,
MTMD_GEN_AUDIO_TYPE_FISH_S2,
+ MTMD_GEN_AUDIO_TYPE_KOKORO,
};
struct mtmd_gen_audio_info {
@@ -526,10 +527,12 @@ struct mtmd_gen_inp {
// pass either codes (discrete) or feats (continuous), depending on the pipeline
// lfm3a GEN_CODE also takes codes: the codes revealed so far, -1 for a masked codebook
// zonos3 GEN_WAV: -1 for a residual code not known yet, the frame is decoded without it
+ // kokoro GEN_WAV takes both: codes are the phoneme tokens, feats is the voice row (256 values) then the backbone state of each token
int32_t * codes;
size_t n_codes;
const float * feats;
size_t n_feats;
+ float speed; // kokoro: speaking speed, > 0 (default: 1.0)
// yue2 GEN_CODE: n_codes is the frame count, codes is not read (the backbone state holds them)
// backbone_state: llama_state_seq_get_data() of the backbone_n_past positions (prompt, codes, end token)
diff --git a/tools/server/README.md b/tools/server/README.md
index 6dddbc81b..4643eb7a5 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -862,6 +862,8 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For the list
`lang`: Language code for the utterance (model-dependent, e.g. `en`, `zh`). Optional.
+`raw_prompt`: If `true`, `input` is passed to the model as is, without the text preprocessing of the model's `tts` chat template (prompt format, text normalization, text to phonemes). For example, a Kokoro model then takes a phoneme string. Default: `false`
+
`voice`: Voice for the utterance, one of the `voices` listed by `GET /props` (model-dependent, e.g. `female` or `male` for LFM3-Audio). Optional: a model with predefined voices speaks with the first of them when neither `voice` nor `speaker_ref` is given. It rejects any other name, and a predefined voice cannot be combined with `speaker_ref`.
`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence. Rejected when the model has no voice encoder (e.g. Pocket TTS without voice cloning).
@@ -1678,7 +1680,7 @@ Generates speech from text, for models that can generate audio. The request is c
`stream_format`: `audio` streams the response as it becomes available, see `stream` in `/tts` for the WAV header it writes. `sse` is rejected. The response is not streamed when omitted.
-`speed`: Speaking speed, see `speed` in `/tts`. Only models that support it accept a value other than `1.0`. `loudness`, `cfg_scale` and `flow_steps` of `/tts` can be used as well.
+`speed`: Speaking speed, see `speed` in `/tts`. Only models that support it accept a value other than `1.0`. `loudness`, `cfg_scale`, `flow_steps`, `lang` and `raw_prompt` of `/tts` can be used as well.
`instructions`: Ignored.
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 6a57aa8ca..145cfa047 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -6984,7 +6984,9 @@ std::unique_ptr<server_res_generator> server_routes::handle_tts_impl(const serve
// speaker_ref in the request overrides the default voice
const bool has_speaker_ref = req.files.count("speaker_ref") > 0 || !json_value(body, "speaker_ref_b64", std::string()).empty();
const std::string voice = json_value(body, "voice", has_speaker_ref ? std::string() : params.tts_voice);
- const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice);
+ // raw_prompt: the input goes to the model as is (e.g. phonemes), the "tts" chat template is skipped
+ const bool raw_prompt = json_value(body, "raw_prompt", false);
+ const std::string tts_prompt = raw_prompt ? std::string() : common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice);
// a model with predefined voices only speaks with one of them
mtmd::bitmap_ptr voice_ref;
if (!voice.empty() && info.n_voices > 0) {
diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp
index 47f17de89..68e4d09ae 100644
--- a/tools/tts/tts.cpp
+++ b/tools/tts/tts.cpp
@@ -135,7 +135,13 @@ int main(int argc, char ** argv) {
// a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it
auto tmpls = common_chat_templates_init(model, "");
- std::string prompt = common_chat_get_tts_prompt(tmpls.get(), params.prompt, params.tts_lang, params.tts_voice);
+ std::string prompt;
+ try {
+ prompt = common_chat_get_tts_prompt(tmpls.get(), params.prompt, params.tts_lang, params.tts_voice);
+ } catch (const std::exception & e) {
+ LOG_ERR("failed to prepare the text to speak: %s\n", e.what());
+ return 1;
+ }
if (prompt.empty()) {
prompt = params.prompt;
}