Commit d02b4c0d6 for llama.cpp
commit d02b4c0d66796e0a44380611d038123fefb9fe83
Author: Marco Colombo <ing.marco.colombo@gmail.com>
Date: Mon Nov 2 18:39:51 2026 +0100
mtmd: Pocket tts - support for voices without voice cloning (#31429)
* feat: support PocketTTS voices with no voice cloning
The kyutai/pocket-tts-without-voice-cloning weights ship predefined voices as the
backbone KV state after the voice prompt. The converter turns each state back into
the prompt embeddings and stores them in the mmproj (voice.<name>), the zeroed voice
encoder is left out, and the "voice" field of /tts and --tts-voice select one of them.
A speaker reference is rejected when the model has no voice encoder.
Assisted-by: Claude Fable 5.1
* convert : pocket-tts voices never fail the conversion, log the drift of the state
The voice states shipped by Kyutai are not always the ones the pack's
weights give for the prompt: italian, german and spanish match within the
bf16 rounding, english_2026-04 is 2% off and french 20% (its layer 0 state
comes from other weights), while the reference loads them as is and they
still sound like the same voice. The converter now re-prompts the backbone
with the recovered embeddings, compares the whole state and logs the
distance per voice, warning above 5%. A state that does not belong to the
weights at all is skipped, so the packs of kyutai/pocket-tts convert too.
Assisted-by: Claude Fable 5.1
* mtmd : list the predefined voices, expose them in GET /props
mtmd_gen_audio_info gains n_voices and mtmd_gen_audio_voice_name() the
names, with the voices worded as a feature of any audio generation model.
The server lists them under "voices" in /props when the model ships some.
Assisted-by: Claude Fable 5.1
* server : an unknown voice is a bad request
A model with predefined voices answers 400 naming the voice instead of 500
"failed to process TTS prompt". A model without them ignores "voice" with a
warning, as the other audio generation models do.
Assisted-by: Claude Fable 5.1
* refactor api
* fix conversion
* small renaming
* fix lint
* tts : a model with predefined voices speaks with the first one by default
Without a voice prompt such a model only produces near silence. When neither
a voice nor a speaker reference is given, llama-tts and the server now take
the first predefined voice and say so in the log, as the LFM3-Audio template
does with its default voice.
Assisted-by: Claude Fable 5.1
* mtmd : one comment per voice accessor, converter states the rule only
The clip.h comment described clip_gen_get_voice but sat above all three
functions; the converter comment quoted the drift measured per pack, which
goes stale with the next upload of the weights.
Assisted-by: Claude Fable 5.1
* clean up redundant code and comments
* mtmd: match the LFM3-Audio voice name case-insensitively
The tts chat template lowers the voice name, so Male speaks with
the male voice; the voice lookup lowers it the same way and names the
bitmap after the canonical voice.
---------
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
Co-authored-by: Pascal <admin@serveurperso.com>
diff --git a/conversion/pockettts.py b/conversion/pockettts.py
index 1c99e58cf..1b5c84111 100644
--- a/conversion/pockettts.py
+++ b/conversion/pockettts.py
@@ -5,6 +5,7 @@ from pathlib import Path
from typing import Any, Iterable, TYPE_CHECKING
import torch
+import torch.nn.functional as F
if TYPE_CHECKING:
from torch import Tensor
@@ -45,6 +46,102 @@ def _tensor_shapes(dir_model: Path) -> dict[str, tuple[int, ...]]:
return {name: tuple(part[name].shape) for name in part.keys()}
+def _has_voice_encoder(dir_model: Path) -> bool:
+ # without voice cloning, the mimi encoder weights are all zero
+ part_names = ModelBase.get_model_part_names(dir_model, "model", ".safetensors")
+ with gguf.utility.SafetensorsLocal(dir_model / part_names[0]) as part:
+ nonzero = {bool(part[name].mmap_bytes().any()) for name in part.keys() if name.startswith("mimi.encoder")}
+ if len(nonzero) != 1:
+ raise ValueError(f"{dir_model}: the mimi encoder is partly zeroed out")
+ return nonzero.pop()
+
+
+def _load_safetensors(path: Path, prefixes: tuple[str, ...]) -> dict[str, Tensor]:
+ dtypes = {"F32": torch.float32, "BF16": torch.bfloat16, "I64": torch.int64}
+ with gguf.utility.SafetensorsLocal(path) as part:
+ return {name: torch.from_numpy(part[name].mmap_bytes()).view(dtypes[part[name].dtype]).reshape(part[name].shape)
+ for name in part.keys() if name.startswith(prefixes)}
+
+
+def _rope_pairs(x: Tensor, n_head: int, rope_theta: float) -> Tensor:
+ # ggml NORM mode rope, x: [n_pos, n_embd]
+ n_pos, n_embd = x.shape
+ head_dim = n_embd // n_head
+ angle = torch.arange(n_pos, dtype=x.dtype)[:, None, None] * \
+ torch.pow(rope_theta, -torch.arange(0, head_dim, 2, dtype=x.dtype) / head_dim)
+ x = x.view(n_pos, n_head, head_dim // 2, 2)
+ return torch.stack([x[..., 0] * angle.cos() - x[..., 1] * angle.sin(),
+ x[..., 0] * angle.sin() + x[..., 1] * angle.cos()], -1).reshape(n_pos, n_embd)
+
+
+def _voice_prompt_embd(weights: dict[str, Tensor], kv: list[Tensor], n_head: int, rope_theta: float) -> tuple[Tensor, float]:
+ # the voice state only has K/V of each layer (kv[il] is [k/v, pos, n_embd]): layer 0 V gives each frame up to its scale, layer 1 V gives the scale
+ # returns the frames without bos_before_voice, and the max relative K/V error when the frames are prompted again
+ w = {name: t.double() for name, t in weights.items()}
+ kv = [t.double() for t in kv]
+ k0, v0, v1 = kv[0][0], kv[0][1], kv[1][1]
+ n_pos, n_embd = v0.shape
+ head_dim = n_embd // n_head
+
+ def p(il: int, name: str) -> Tensor:
+ return w[f"flow_lm.transformer.layers.{il}.{name}"]
+
+ def norm(x: Tensor, il: int, name: str) -> Tensor:
+ return F.layer_norm(x, (n_embd,), p(il, f"{name}.weight"), p(il, f"{name}.bias"), 1e-5)
+
+ def heads(t: Tensor) -> Tensor:
+ return t.reshape(n_pos, n_head, head_dim).transpose(0, 1)
+
+ def rows(t: Tensor) -> Tensor:
+ return t.transpose(0, 1).reshape(n_pos, n_embd)
+
+ def attn(il: int, h: Tensor, k: Tensor, v: Tensor) -> Tensor:
+ q = _rope_pairs(h @ p(il, "self_attn.in_proj.weight").chunk(3)[0].T, n_head, rope_theta)
+ return rows(F.scaled_dot_product_attention(heads(q), heads(k), heads(v), is_causal=True)) @ p(il, "self_attn.out_proj.weight").T
+
+ def ffn(il: int, x: Tensor) -> Tensor:
+ return F.gelu(norm(x, il, "norm2") @ p(il, "linear1.weight").T) @ p(il, "linear2.weight").T
+
+ wv = p(0, "self_attn.in_proj.weight").chunk(3)[2]
+ h = torch.linalg.solve(wv, v0.T).T # norm1(x)
+ z = (h - p(0, "norm1.bias")) / p(0, "norm1.weight") # (x - mean(x)) / std(x)
+
+ # fit z on [speaker_proj, 1] to get the latent direction back
+ proj = w["flow_lm.speaker_proj_weight"]
+ fit = torch.linalg.lstsq(torch.cat([proj, torch.ones(n_embd, 1, dtype=proj.dtype)], 1), z[1:].T).solution
+ frames = (proj @ fit[:-1]).T
+
+ a0 = attn(0, h, k0, v0)[1:]
+ wv1 = p(1, "self_attn.in_proj.weight").chunk(3)[2]
+
+ def err(scale: Tensor) -> Tensor: # per frame relative error of layer 1 V
+ x = scale[:, None] * frames + a0
+ x = x + ffn(0, x)
+ return (norm(x, 1, "norm1") @ wv1.T - v1[1:]).norm(dim=1) / v1[1:].norm(dim=1)
+
+ grid = torch.logspace(-3, 3, 49, dtype=torch.float64)
+ best = torch.stack([err(s.expand(n_pos - 1)) for s in grid]).argmin(0)
+ lo, hi = grid[(best - 1).clamp(min=0)], grid[(best + 1).clamp(max=len(grid) - 1)]
+ for _ in range(40): # golden section search, all frames at once
+ m1, m2 = hi - 0.618 * (hi - lo), lo + 0.618 * (hi - lo)
+ left = err(m1) < err(m2)
+ lo, hi = torch.where(left, lo, m1), torch.where(left, m2, hi)
+ frames = (lo + hi)[:, None] / 2 * frames
+
+ # prompt the frames again and compare K/V with the shipped state
+ x = torch.cat([w["flow_lm.bos_before_voice"].reshape(1, n_embd), frames])
+ drift = 0.0
+ for il, (k_ref, v_ref) in enumerate(kv):
+ h = norm(x, il, "norm1")
+ _, wk, wv = p(il, "self_attn.in_proj.weight").chunk(3)
+ k, v = _rope_pairs(h @ wk.T, n_head, rope_theta), h @ wv.T
+ for a, b in ((k, k_ref), (v, v_ref)):
+ drift = max(drift, ((a - b).norm(dim=1) / b.norm(dim=1)).median().item())
+ x = x + attn(il, h, k, v)
+ x = x + ffn(il, x)
+ return frames, drift
+
+
@ModelBase.register_hparams_loader(lambda dir_model: "flow_lm.bos_emb" in _tensor_shapes(dir_model))
def _load_hparams(dir_model: Path) -> dict[str, Any]:
logger.info("gguf: detected pocket-tts checkpoint, deriving hparams from tensor shapes")
@@ -194,22 +291,27 @@ class PocketTTSMmprojModel(MmprojModel):
(gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V),
)
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.has_audio_encoder = _has_voice_encoder(self.dir_model)
+
def set_gguf_parameters(self):
self.gguf_writer.add_file_type(self.ftype)
assert self.hparams_audio is not None
# voice-prompt encoder: mimi encoder + speaker_proj
- self.gguf_writer.add_clip_has_audio_encoder(True)
- # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models
- self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC)
- self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
- self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"])
- self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"])
- self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"])
- self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"])
- self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
- # mimi convolves the waveform directly, it is passed around as a 1-row "mel"
- self.gguf_writer.add_audio_num_mel_bins(1)
+ if self.has_audio_encoder:
+ self.gguf_writer.add_clip_has_audio_encoder(True)
+ # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models
+ self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC)
+ self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
+ self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"])
+ self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"])
+ self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"])
+ self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"])
+ self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
+ # mimi convolves the waveform directly, it is passed around as a 1-row "mel"
+ self.gguf_writer.add_audio_num_mel_bins(1)
# generation: flow-matching decoder + mimi decoder
# the SEANet and flow net hparams are constant across the family, clip.cpp holds them
@@ -229,12 +331,57 @@ class PocketTTSMmprojModel(MmprojModel):
# conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
if ".seanet." in new_name or new_name in ("a.downsample.conv.weight", "a.gen.wav.upsample.weight"):
return gguf.GGMLQuantizationType.F16
+ # voices are read on the host
+ if new_name.startswith("voice."):
+ return gguf.GGMLQuantizationType.F32
return False
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ # embeddings/<name>.safetensors hold the K/V state after the voice prompt (see get_state_for_audio_prompt() in the reference), turn it back into prompt embeddings
+ paths = sorted((self.dir_model / "embeddings").glob("*.safetensors"))
+ if not paths:
+ return
+ weights = _load_safetensors(self.dir_model / "model.safetensors",
+ ("flow_lm.transformer.layers.", "flow_lm.speaker_proj_weight", "flow_lm.bos_before_voice"))
+ n_layer = self.global_config["num_hidden_layers"]
+ for path in paths:
+ state = _load_safetensors(path, ("transformer.layers.",))
+ kv = []
+ for il in range(n_layer):
+ prefix = f"transformer.layers.{il}.self_attn/"
+ if any(prefix + k not in state for k in ("cache", "offset")):
+ break
+ n_pos = int(state[prefix + "offset"].item())
+ # older packs have no pad entry
+ if prefix + "pad" in state and int(state[prefix + "pad"].item()) != 0:
+ break
+ kv.append(state[prefix + "cache"][:, 0, :n_pos].flatten(2)) # [k/v, pos, n_embd]
+ if len(kv) != n_layer or any(t.shape != kv[0].shape for t in kv) or kv[0].shape[1] < 2 or \
+ kv[0].shape[2] != self.global_config["hidden_size"]:
+ logger.warning(f"voice {path.stem}: skipped, not an unpadded voice state of this backbone")
+ continue
+ embd, drift = _voice_prompt_embd(weights, kv, self.global_config["num_attention_heads"],
+ self.global_config["rope_theta"])
+ # some packs ship a state made with other weights, the reference still uses it: keep up to 50% error, warn above 5%
+ if drift > 0.5:
+ logger.warning(f"voice {path.stem}: skipped, its state does not belong to these weights (error {drift:.1e})")
+ continue
+ log = logger.warning if drift > 0.05 else logger.info
+ log(f"voice {path.stem}: {embd.shape[0]} frames, prompting them again gives the shipped state within {drift:.1e}")
+ yield (f"voice.{path.stem}", embd.float())
+
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
del bid # the block index of the mimi transformers is parsed here, not by the base class
T = gguf.MODEL_TENSOR
+ if name.startswith("voice."):
+ yield (name, data_torch)
+ return
+
+ if not self.has_audio_encoder and (name.startswith(("mimi.encoder", "mimi.downsample."))
+ or name == "flow_lm.speaker_proj_weight"):
+ return # voice-prompt encoder
+
if name in ("flow_lm.bos_emb", "flow_lm.bos_before_voice", "flow_lm.conditioner.embed.weight"):
return # folded into the backbone embedding table
if name.startswith("flow_lm.transformer.") or name.startswith("flow_lm.out_norm."):
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h
index 88aa20de1..2810a5cc6 100644
--- a/tools/mtmd/clip-model.h
+++ b/tools/mtmd/clip-model.h
@@ -888,6 +888,9 @@ struct clip_model {
ggml_tensor * gen_upsample_w = nullptr; // depthwise convtr, frame rate -> encoder frame rate
std::vector<clip_layer> gen_tfm_layers; // mimi decoder_transformer
+ // predefined voices, no rows if the voice is named in the prompt (lfm3a)
+ std::map<std::string, std::vector<float>> gen_voices;
+
// soprano: Vocos decoder, hidden states -> STFT coefficients
clip_vocos vocos;
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index d723432a7..9a043d7f9 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -3502,9 +3502,33 @@ struct clip_model_loader {
layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, p, il, "weight"));
layer.ls_2_w = get_tensor(string_format(TN_LS_2, p, il, "weight"));
}
+
+ for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) {
+ const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
+ if (strncmp(name, "voice.", 6) != 0) {
+ continue;
+ }
+ const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name);
+ if (t->type != GGML_TYPE_F32 || ggml_n_dims(t) > 2 || t->ne[0] != model.gen_input_lin_w->ne[1]) {
+ throw std::runtime_error(string_format("%s: invalid voice tensor %s\n", __func__, name));
+ }
+ auto & embd = model.gen_voices[name + 6];
+ embd.resize(ggml_nelements(t));
+ fin.seekg(tensor_offset.at(name), std::ios::beg);
+ if (!fin.read(reinterpret_cast<char *>(embd.data()), ggml_nbytes(t))) {
+ throw std::runtime_error(string_format("%s: failed to read %s\n", __func__, name));
+ }
+ }
+ if (!model.gen_voices.empty()) {
+ LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size());
+ }
} break;
case PROJECTOR_TYPE_LFM3A_GEN:
{
+ // must match the "tts" chat template, see conversion/lfm3.py
+ // note for contributor: this is one-off fix, do not repeat this for future models
+ model.gen_voices = { {"female", {}}, {"male", {}} };
+
// depthformer, the generic loop above loaded the a.gen.code.blk.* tensors
for (int il = 0; il < hparams.n_layer; ++il) {
model.layers[il].attn_gate_w = get_tensor(string_format(TN_ATTN_GATE, prefix, il, "weight"));
@@ -5142,6 +5166,10 @@ void clip_gen_state_reset(clip_gen_state * state) {
}
}
+const std::map<std::string, std::vector<float>> & clip_gen_get_voices(const clip_ctx * ctx) {
+ return ctx->model.gen_voices;
+}
+
clip_enc_state * clip_enc_state_init(clip_ctx * ctx, int32_t n_frames_max) {
const auto & hparams = ctx->model.hparams;
if (ctx->proj_type() == PROJECTOR_TYPE_LFM3A) {
diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h
index cd3f434da..b2c9e0759 100644
--- a/tools/mtmd/clip.h
+++ b/tools/mtmd/clip.h
@@ -139,6 +139,8 @@ struct clip_gen_state * clip_gen_state_init(struct clip_ctx * ctx, uint32_t seed
void clip_gen_state_free(struct clip_gen_state * state);
void clip_gen_state_reset(struct clip_gen_state * state);
+const std::map<std::string, std::vector<float>> & clip_gen_get_voices(const struct clip_ctx * ctx);
+
bool clip_is_llava(const struct clip_ctx * ctx);
// note for contributor: this clip_is_(model) pattern is deprecated
// do NOT add new functions like this
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index d0342b9bd..bed8f3d8c 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -430,7 +430,7 @@ private:
// runs the reference wav through the speaker encoder, returns one x-vector embedding row
bool encode_speaker(mtmd_bitmap * bitmap, std::vector<float> & out) {
- if (!mtmd_support_audio(mctx)) {
+ if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) {
LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n");
return false;
}
@@ -998,7 +998,7 @@ private:
// runs the reference wav through the mimi encoder, returns one row per 12.5Hz frame
bool encode_speaker(mtmd_bitmap * bitmap, std::vector<float> & out) {
- if (!mtmd_support_audio(mctx)) {
+ if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) {
LOG_ERR("mtmd_helper_gen_audio: mmproj has no voice encoder\n");
return false;
}
@@ -1298,7 +1298,8 @@ public:
reset();
seq_id = inp->seq_id;
- if (inp->speaker_ref) {
+ // a voice is named in the prompt instead
+ if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref)) {
LOG_WRN("mtmd_helper_gen_audio: the model has no voice conditioning, the speaker file is ignored\n");
}
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index 99ae1738d..e562aae18 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -243,9 +243,8 @@ struct mtmd_helper_gen_audio_inp {
int32_t n_past;
enum mtmd_helper_gen_audio_out_mode out_mode; // n_past > 0 only (default: auto)
- mtmd_bitmap * speaker_ref; // optional, can be NULL
+ mtmd_bitmap * speaker_ref; // optional, can be NULL: audio to clone, or a voice bitmap
const char * lang; // optional, can be NULL
- const char * voice; // optional, can be NULL, model-dependent
int32_t top_k;
float top_p;
@@ -345,7 +344,6 @@ struct gen_audio {
mtmd_helper_gen_audio_inp data = mtmd_helper_gen_audio_inp_default();
std::string prompt_str;
std::string lang_str;
- std::string voice_str;
mtmd::bitmap_ptr speaker_ref_ptr;
inp() = default;
@@ -356,7 +354,6 @@ struct gen_audio {
void set_prompt (std::string p) { prompt_str = std::move(p); }
void set_lang (std::string l) { lang_str = std::move(l); }
- void set_voice (std::string v) { voice_str = std::move(v); }
void set_speaker_ref(mtmd::bitmap_ptr bmp) { speaker_ref_ptr = std::move(bmp); }
// pointers are only valid as long as *this is alive
@@ -364,7 +361,6 @@ struct gen_audio {
data.prompt = prompt_str.c_str();
data.prompt_len = prompt_str.size();
data.lang = lang_str.empty() ? nullptr : lang_str.c_str();
- data.voice = voice_str.empty() ? nullptr : voice_str.c_str();
data.speaker_ref = speaker_ref_ptr.get();
return &data;
}
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 410881b51..d4ea23852 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -19,6 +19,7 @@
#endif
#include <algorithm>
+#include <cctype>
#include <cerrno>
#include <cmath>
#include <cstdio>
@@ -174,6 +175,7 @@ struct mtmd_bitmap {
uint32_t ny = 0;
std::string id; // optional user-defined id, for ex: can be set to image hash, useful for KV cache tracking
bool is_audio = false; // true if the bitmap is audio
+ bool is_voice = false;
bool mergeable = false; // [QWEN_VIDEO] set only on frames of the same video
bool is_video = false; // see mtmd_bitmap_set_is_video()
@@ -199,6 +201,14 @@ struct mtmd_bitmap {
}
}
+ static mtmd_bitmap * init_voice(const std::string & name, const float * rows, uint32_t n_embd, uint32_t n_rows) {
+ auto * bitmap = new mtmd_bitmap(nullptr, n_embd, n_rows);
+ bitmap->is_voice = true;
+ bitmap->id = name;
+ bitmap->data.assign((const unsigned char *) rows, (const unsigned char *) (rows + (size_t) n_embd * n_rows));
+ return bitmap;
+ }
+
const std::vector<unsigned char> & get_ro_buf() const {
return data;
}
@@ -351,6 +361,9 @@ struct mtmd_audio_tokens {
// true if one of entries in batch_f32 is a placeholder
bool is_placeholder() const {
+ if (track && track->batch_f32.entries.empty()) {
+ return track->embd.empty(); // a voice: its rows are not serialized
+ }
for (const auto & entry : (track ? track->batch_f32 : batch_f32).entries) {
if (entry.is_placeholder()) {
return true;
@@ -660,9 +673,8 @@ struct mtmd_context {
ctx_v = res.ctx_v;
ctx_a = res.ctx_a;
ctx_gen_a = res.ctx_gen_a;
- // soprano has no speaker encoder, its mmproj only holds the decoder
- const bool gen_only = ctx_gen_a && clip_get_projector_type(ctx_gen_a) == PROJECTOR_TYPE_SOPRANO;
- if (!ctx_v && !ctx_a && !gen_only) {
+ // soprano and pocket-tts without voice cloning have no encoder
+ if (!ctx_v && !ctx_a && !ctx_gen_a) {
throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname));
}
@@ -1535,6 +1547,10 @@ struct mtmd_tokenizer {
GGML_ASSERT(!bitmaps.empty());
// note: only one type of media is supported per call, caller should enforce this
+ if (bitmaps[0]->is_voice) {
+ GGML_ASSERT(bitmaps.size() == 1);
+ return add_voice(*bitmaps[0]);
+ }
const bool is_vision = !bitmaps[0]->is_audio;
if (is_vision) {
@@ -1869,6 +1885,30 @@ struct mtmd_tokenizer {
return 0;
}
+ int32_t add_voice(const mtmd_bitmap & bitmap) {
+ if (bitmap.ny == 0) {
+ LOG_ERR("%s: error: voice \"%s\" holds no embeddings\n", __func__, bitmap.id.c_str());
+ return 2;
+ }
+ auto track = std::make_shared<mtmd_audio_track>();
+ track->n_tokens = bitmap.ny;
+ const auto & buf = bitmap.get_ro_buf();
+ track->embd.assign((const float *) buf.data(), (const float *) (buf.data() + buf.size()));
+
+ mtmd_audio_tokens_ptr audio_tokens(new mtmd_audio_tokens);
+ audio_tokens->n_tokens = bitmap.ny;
+ audio_tokens->id = bitmap.id;
+ audio_tokens->track = std::move(track);
+
+ cur.entries.emplace_back(mtmd_input_chunk{
+ MTMD_INPUT_CHUNK_TYPE_AUDIO,
+ {}, // text tokens
+ nullptr, // image tokens
+ std::move(audio_tokens),
+ });
+ return 0;
+ }
+
// a piece of a soundtrack goes in without the audio markers, it lies inside the span of its video
int32_t add_soundtrack_piece(soundtrack & st, const mtmd_bitmap & bitmap) {
if (!st.track) {
@@ -2164,10 +2204,6 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk
}
return mtmd_encode_impl(ctx, chunk->tokens_image.get(), out_embd);
} else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {
- if (!ctx->ctx_a) {
- LOG_ERR("%s: model does not support audio input\n", __func__);
- return 1;
- }
if (chunk->tokens_audio == nullptr) {
LOG_ERR("%s: audio tokens are null\n", __func__);
return 1;
@@ -2176,22 +2212,27 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk
LOG_ERR("%s: audio tokens batch is placeholder\n", __func__);
return 1;
}
- int n_mmproj_embd = ctx->n_embd_out();
const auto & track = chunk->tokens_audio->track;
+ // a filled track (e.g. a voice) needs no audio encoder
+ if (!ctx->ctx_a && (!track || track->embd.empty())) {
+ LOG_ERR("%s: model does not support audio input\n", __func__);
+ return 1;
+ }
if (track) {
if (track->embd.empty()) {
- track->embd.resize((size_t) track->n_tokens * n_mmproj_embd);
+ track->embd.resize((size_t) track->n_tokens * ctx->n_embd_out());
if (!clip_image_batch_encode(ctx->ctx_a, ctx->n_threads, &track->batch_f32, track->embd)) {
track->embd.clear();
return 1;
}
}
+ const size_t n_mmproj_embd = track->embd.size() / track->n_tokens;
const size_t off = (size_t) chunk->tokens_audio->row_begin * n_mmproj_embd;
out_embd.assign(track->embd.begin() + off,
track->embd.begin() + off + (size_t) chunk->tokens_audio->n_tokens * n_mmproj_embd);
return 0;
}
- out_embd.resize((size_t)chunk->tokens_audio->n_tokens * n_mmproj_embd);
+ out_embd.resize((size_t)chunk->tokens_audio->n_tokens * ctx->n_embd_out());
bool ok = clip_image_batch_encode(
ctx->ctx_a,
ctx->n_threads,
@@ -2239,6 +2280,7 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
return info;
}
info.model_variant = clip_get_hparams(ctx->ctx_gen_a)->gen_model_variant.c_str();
+ info.n_voices = (int32_t) clip_gen_get_voices(ctx->ctx_gen_a).size();
switch (clip_get_projector_type(ctx->ctx_gen_a)) {
case PROJECTOR_TYPE_QWEN3TTS_GEN:
info.type = MTMD_GEN_AUDIO_TYPE_QWEN3TTS;
@@ -2263,6 +2305,31 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
return info;
}
+const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx) {
+ if (!ctx->ctx_gen_a || idx < 0) {
+ return nullptr;
+ }
+ const auto & voices = clip_gen_get_voices(ctx->ctx_gen_a);
+ return idx < (int32_t) voices.size() ? std::next(voices.begin(), idx)->first.c_str() : nullptr;
+}
+
+mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * name) {
+ if (!ctx->ctx_gen_a || !name) {
+ return nullptr;
+ }
+ // the name is matched case-insensitively, as the lfm3a "tts" chat template lowers it
+ const auto & voices = clip_gen_get_voices(ctx->ctx_gen_a);
+ const auto it = std::find_if(voices.begin(), voices.end(), [name](const auto & voice) {
+ return std::equal(voice.first.begin(), voice.first.end(), name, name + strlen(name),
+ [](unsigned char a, unsigned char b) { return std::tolower(a) == std::tolower(b); });
+ });
+ if (it == voices.end()) {
+ return nullptr;
+ }
+ const uint32_t n_embd = clip_n_mmproj_embd(ctx->ctx_gen_a);
+ return mtmd_bitmap::init_voice(it->first, it->second.data(), n_embd, it->second.size() / n_embd);
+}
+
mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
mtmd_gen_inp inp{};
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
@@ -2804,6 +2871,10 @@ bool mtmd_bitmap_is_audio(const mtmd_bitmap * bitmap) {
return bitmap->is_audio;
}
+bool mtmd_bitmap_is_voice(const mtmd_bitmap * bitmap) {
+ return bitmap->is_voice;
+}
+
const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap) {
return bitmap->id.c_str();
}
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index 3e03142a4..4f69a644d 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -191,6 +191,8 @@ MTMD_API const char * mtmd_get_marker(const mtmd_context * ctx);
// if bitmap is audio:
// length of data must be n_samples * sizeof(float)
// the data is in float format (PCM F32)
+// if bitmap is voice (see mtmd_gen_audio_get_voice()):
+// nx = n_embd, ny = number of rows, data is F32 rows (can be empty), id is the voice name
//
// if data == nullptr:
// the bitmap is considered "empty", and will be treated as a placeholder for counting tokens
@@ -203,6 +205,7 @@ MTMD_API uint32_t mtmd_bitmap_get_ny (const mtmd_bitmap * bitma
MTMD_API const unsigned char * mtmd_bitmap_get_data (const mtmd_bitmap * bitmap);
MTMD_API size_t mtmd_bitmap_get_n_bytes(const mtmd_bitmap * bitmap);
MTMD_API bool mtmd_bitmap_is_audio (const mtmd_bitmap * bitmap);
+MTMD_API bool mtmd_bitmap_is_voice (const mtmd_bitmap * bitmap);
MTMD_API void mtmd_bitmap_free (mtmd_bitmap * bitmap);
// bitmap ID is optional, but useful for KV cache tracking
// these getters/setters are dedicated functions, so you can for example calculate the hash of the image based on mtmd_bitmap_get_data()
@@ -468,10 +471,18 @@ struct mtmd_gen_audio_info {
enum mtmd_gen_audio_type type;
int32_t sample_rate; // in Hz, for example 24000 for qwen3tts
const char * model_variant; // name of the weight variant, can be nullptr if not applicable
+ int32_t n_voices; // predefined voices, see mtmd_gen_audio_voice_name()
};
MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx);
+// name of the idx-th predefined voice, nullptr when idx is out of range
+MTMD_API const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx);
+
+// predefined voice with this name as a voice bitmap, to pass as speaker reference
+// returns nullptr if there is no such voice, the caller must free it
+MTMD_API mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * name);
+
enum mtmd_gen_process_type {
MTMD_GEN_PROCESS_TYPE_GEN_CODE, // h_state to semantic (codes, mel-spectrogram, etc.)
diff --git a/tools/server/README.md b/tools/server/README.md
index 66a34ff84..afa448b8a 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -857,9 +857,9 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info
`lang`: Language code for the utterance (model-dependent, e.g. `en`, `zh`). Optional.
-`voice`: Voice for the utterance (model-dependent, e.g. `female` or `male` for LFM3-Audio). Optional.
+`voice`: Voice for the utterance, one of the `voices` listed by `GET /props` (model-dependent, e.g. `female` or `male` for LFM3-Audio). Optional: a model with predefined voices speaks with the first of them when neither `voice` nor `speaker_ref` is given. It rejects any other name, and a predefined voice cannot be combined with `speaker_ref`.
-`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence.
+`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence. Rejected when the model has no voice encoder (e.g. Pocket TTS without voice cloning).
`top_k`, `top_p`: Sampling params for the acoustic code predictor. Optional, model-dependent defaults apply.
@@ -1017,6 +1017,7 @@ By default, it is read-only. To make POST request to change global properties, y
- `default_generation_settings` - the default generation settings for the `/completion` endpoint, which has the same fields as the `generation_settings` response object from the `/completion` endpoint.
- `total_slots` - the total number of slots for process requests (defined by `--parallel` option)
+- `voices` - the predefined voices of an audio generation model, to pass as `voice` to `/tts`. Only present when the model has some (e.g. LFM3-Audio, Pocket TTS without voice cloning)
- `model_path` - the path to model file (same with `-m` argument)
- `chat_template` - the model's original Jinja2 prompt template
- `chat_template_caps` - capabilities of the chat template (see `common/jinja/caps.h` for more info), `reasoning_effort_levels` lists the `reasoning_effort` values the template renders distinctly, from lowest to highest
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 9377e3c17..48f1e93a2 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -5225,6 +5225,13 @@ server_context_meta server_context::get_meta() const {
const char * ftype_name = llama_ftype_name(llama_model_ftype(impl->model_tgt));
+ std::vector<std::string> tts_voices;
+ if (impl->mctx) {
+ for (int32_t i = 0; i < mtmd_gen_audio_get_info(impl->mctx).n_voices; i++) {
+ tts_voices.push_back(mtmd_gen_audio_voice_name(impl->mctx, i));
+ }
+ }
+
return server_context_meta {
/* build_info */ std::string(llama_build_info()),
/* model_name */ impl->model_name,
@@ -5239,6 +5246,7 @@ server_context_meta server_context::get_meta() const {
/* has_cap_chat */ impl->has_cap_chat(),
/* has_cap_tts */ impl->has_cap_tts(),
/* has_audio_stream */ impl->mtmd_cap.audio_stream,
+ /* tts_voices */ std::move(tts_voices),
/* json_ui_settings */ impl->json_ui_settings,
/* slot_n_ctx */ impl->n_ctx_slot(),
/* pooling_type */ llama_pooling_type(impl->ctx_tgt),
@@ -5697,6 +5705,9 @@ static json get_res_props(const server_context_meta & meta, const common_params
if (!params.server_connect_code.empty()) {
props["connect_code"] = params.server_connect_code;
}
+ if (!meta.tts_voices.empty()) {
+ props["voices"] = meta.tts_voices;
+ }
if (params.use_jinja) {
if (!tmpl_tools.empty()) {
props["chat_template_tool_use"] = tmpl_tools;
@@ -6475,9 +6486,17 @@ void server_routes::init_routes() {
const std::string lang = json_value(body, "lang", std::string());
const std::string voice = json_value(body, "voice", std::string());
const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice);
+ // a model with predefined voices only speaks with one of them
+ mtmd::bitmap_ptr voice_ref;
+ if (!voice.empty() && info.n_voices > 0) {
+ voice_ref.reset(mtmd_gen_audio_get_voice(ctx_server.mctx, voice.c_str()));
+ if (!voice_ref) {
+ res->error(format_error_response("unknown voice \"" + voice + "\", see \"voices\" in GET /props", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+ }
task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt);
task.tts_inp.set_lang(lang);
- task.tts_inp.set_voice(voice);
task.tts_inp.data.stream = stream;
task.tts_inp.data.out_type = response_format == "pcm"
? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM
@@ -6516,12 +6535,27 @@ void server_routes::init_routes() {
}
if (speaker_ref_len > 0) {
+ if (voice_ref) {
+ res->error(format_error_response("\"voice\" and \"speaker_ref\" cannot be used together", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+ if (!mtmd_support_audio(ctx_server.mctx)) {
+ res->error(format_error_response("this model has no voice encoder, \"speaker_ref\" cannot be used", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt);
if (!wrapper.bitmap) {
res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));
return res;
}
task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap));
+ } else if (voice_ref) {
+ task.tts_inp.set_speaker_ref(std::move(voice_ref));
+ } else if (info.n_voices > 0) {
+ // without a voice prompt the model produces near silence: the first voice is the default
+ const char * first = mtmd_gen_audio_voice_name(ctx_server.mctx, 0);
+ SRV_WRN("no voice given, speaking with \"%s\"\n", first);
+ task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(mtmd_gen_audio_get_voice(ctx_server.mctx, first)));
} else {
SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n");
}
diff --git a/tools/server/server-context.h b/tools/server/server-context.h
index d283c9121..05c538ea1 100644
--- a/tools/server/server-context.h
+++ b/tools/server/server-context.h
@@ -27,6 +27,7 @@ struct server_context_meta {
bool has_cap_chat;
bool has_cap_tts;
bool has_audio_stream; // realtime transcription, see mtmd_support_audio_stream()
+ std::vector<std::string> tts_voices; // predefined voices of the audio generation model
json json_ui_settings;
int slot_n_ctx;
enum llama_pooling_type pooling_type;
diff --git a/tools/tts/README.md b/tools/tts/README.md
index 1793b7992..9bec17624 100644
--- a/tools/tts/README.md
+++ b/tools/tts/README.md
@@ -39,6 +39,7 @@ llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF \
Available params:
- `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3). It is required, the model produces almost no audio without it
+- `--tts-voice` selects one of the predefined voices shipped with the weights (e.g. `giovanni`), when the GGUF holds them. The converter reads them from `embeddings/*.safetensors` next to the weights, of both repositories, and logs how closely each one is reproduced (some packs ship voice states computed with other weights, the converter warns above 5% and skips a voice only when it does not belong to the model at all)
- Note: `lang` is not used, the language is a property of the weights
Example usage:
@@ -51,6 +52,16 @@ llama-tts -m pocket-tts.gguf \
--output out.wav
```
+The weights [without voice cloning](https://huggingface.co/kyutai/pocket-tts-without-voice-cloning) ship a set of predefined voices (`embeddings/*.safetensors`) next to each language; the converter stores them in the mmproj, as the voice prompt embeddings, and leaves out the zeroed voice encoder. With such a model `--tts-speaker-file` is rejected and a voice is selected with `--tts-voice` (the first one when it is not given):
+
+```sh
+llama-tts -m pocket-tts.gguf \
+ -mm mmproj-pocket-tts.gguf \
+ -p "Ciao a tutti" \
+ --tts-voice giovanni \
+ --output out.wav
+```
+
**Note for GGUF conversion:**
The [upstream repository](https://huggingface.co/kyutai/pocket-tts) holds one complete model per language under `languages/`, next to a set of shared files at the root. Convert one of the `languages/<name>` directories, **not** the root directory:
diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp
index 8949373cb..8578f8869 100644
--- a/tools/tts/tts.cpp
+++ b/tools/tts/tts.cpp
@@ -112,6 +112,27 @@ int main(int argc, char ** argv) {
speaker_bitmap.reset(wrapper.bitmap);
}
+ // a model with predefined voices needs one, the first by default; other models name the voice in the prompt
+ if (mtmd_gen_audio_get_info(mctx.get()).n_voices > 0 && (!params.tts_voice.empty() || !speaker_bitmap)) {
+ if (speaker_bitmap) {
+ LOG_ERR("--tts-voice and --tts-speaker-file cannot be used together\n");
+ return 1;
+ }
+ if (params.tts_voice.empty()) {
+ params.tts_voice = mtmd_gen_audio_voice_name(mctx.get(), 0);
+ LOG_WRN("no --tts-voice given, speaking with \"%s\"\n", params.tts_voice.c_str());
+ }
+ speaker_bitmap.reset(mtmd_gen_audio_get_voice(mctx.get(), params.tts_voice.c_str()));
+ if (!speaker_bitmap) {
+ LOG_ERR("unknown voice \"%s\", the model has:", params.tts_voice.c_str());
+ for (int32_t i = 0; const char * name = mtmd_gen_audio_voice_name(mctx.get(), i); i++) {
+ LOG(" %s", name);
+ }
+ LOG("\n");
+ return 1;
+ }
+ }
+
// a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it
auto tmpls = common_chat_templates_init(model, "");
std::string prompt = common_chat_get_tts_prompt(tmpls.get(), params.prompt, params.tts_lang, params.tts_voice);
@@ -127,7 +148,6 @@ int main(int argc, char ** argv) {
inp.prompt_len = prompt.size();
inp.speaker_ref = speaker_bitmap.get();
inp.lang = params.tts_lang.c_str();
- inp.voice = params.tts_voice.c_str();
inp.top_k = params.sampling.top_k;
inp.top_p = params.sampling.top_p;
inp.seed = params.sampling.seed;