Commit 8c16babba for llama.cpp
commit 8c16babba5d4c66e28a19d86e572e4b04f522d52
Author: James Prevett <jp@jamespre.dev>
Date: Tue Nov 17 11:59:11 2026 -0600
mtmd: Add support for Qwen3-TTS' ICL/reference mode (#32438)
* Added support for Qwen3-TTS ICL/reference mode
Assisted-By: Claude Opus 5.5
* fix correctness
* attach transcription to bitmap
* nits
---------
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py
index 28bb6e095..1ea14692b 100644
--- a/conversion/qwen3tts.py
+++ b/conversion/qwen3tts.py
@@ -275,6 +275,14 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight")))
):
return gguf.GGMLQuantizationType.F16
+ # the reference encoder's convs run with an F32 im2col, see f32_im2col in pockettts-seanet.cpp
+ if new_name.startswith("a.seanet.") or new_name == "a.downsample.conv.weight":
+ return gguf.GGMLQuantizationType.F32
+ # codes are picked by the nearest codebook entry: Q8_0 changes about 8% of them, F16 none
+ if new_name.startswith("a.gen.enc.tfm.blk.") and new_name.endswith(".weight") and n_dims == 2:
+ return gguf.GGMLQuantizationType.F16
+ if new_name.startswith("a.gen.enc.quant."):
+ return gguf.GGMLQuantizationType.F32
# ConvTranspose1d kernels: only F16/F32 are implemented, no BF16
if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name):
return gguf.GGMLQuantizationType.F32
@@ -298,8 +306,8 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
return super().filter_tensors((name, gen))
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- # code2wav tensors are already named by generate_extra_tensors(), pass them through
- if name.startswith("a.gen.wav."):
+ # code2wav and reference encoder tensors are already named by generate_extra_tensors(), pass them through
+ if name.startswith(("a.gen.wav.", "a.gen.enc.", "a.seanet.", "a.downsample.")):
yield (name, data_torch)
return
@@ -356,6 +364,68 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
yield from self._generate_code2wav_tensors()
+ yield from self._generate_ref_encoder_tensors()
+
+ # mimi encoder of the speech tokenizer: reference clip -> codes, for voice cloning with a transcript (ICL)
+ def _generate_ref_encoder_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ from safetensors.torch import load_file
+
+ state_dict = load_file(self.dir_model / "speech_tokenizer" / "model.safetensors")
+ if "encoder.downsample.conv.weight" not in state_dict:
+ return
+
+ with open(self.dir_model / "speech_tokenizer" / "config.json", "r", encoding="utf-8") as f:
+ enc_config = json.load(f)["encoder_config"]
+ n_stage = len(enc_config["upsampling_ratios"])
+
+ T = gguf.MODEL_TENSOR
+
+ # encoder.encoder.layers: conv_in, then per stage a residual unit, an ELU and a strided conv, then an ELU and conv_out
+ def seanet(idx: int, suffix: str) -> Tensor:
+ return state_dict[f"encoder.encoder.layers.{idx}.{suffix}"]
+
+ for suffix in ("weight", "bias"):
+ yield (self.format_tensor_name(T.A_ENC_SEANET_CONV_IN, suffix=f".{suffix}"), seanet(0, f"conv.{suffix}"))
+ yield (self.format_tensor_name(T.A_ENC_SEANET_CONV_OUT, suffix=f".{suffix}"), seanet(3 * n_stage + 2, f"conv.{suffix}"))
+ for stage in range(n_stage):
+ yield (self.format_tensor_name(T.A_ENC_SEANET_RES_CONV1, stage, suffix=f".{suffix}"), seanet(3 * stage + 1, f"block.1.conv.{suffix}"))
+ yield (self.format_tensor_name(T.A_ENC_SEANET_RES_CONV2, stage, suffix=f".{suffix}"), seanet(3 * stage + 1, f"block.3.conv.{suffix}"))
+ yield (self.format_tensor_name(T.A_ENC_SEANET_SCALE_CONV, stage, suffix=f".{suffix}"), seanet(3 * stage + 3, f"conv.{suffix}"))
+
+ yield (self.format_tensor_name(T.A_ENC_DOWNSAMPLE_CONV), state_dict["encoder.downsample.conv.weight"])
+
+ tfm_layer_map = {
+ "input_layernorm.weight": (T.A_GEN_ENC_TFM_ATTN_NORM, ".weight"),
+ "input_layernorm.bias": (T.A_GEN_ENC_TFM_ATTN_NORM, ".bias"),
+ "self_attn.q_proj.weight": (T.A_GEN_ENC_TFM_ATTN_Q, ".weight"),
+ "self_attn.k_proj.weight": (T.A_GEN_ENC_TFM_ATTN_K, ".weight"),
+ "self_attn.v_proj.weight": (T.A_GEN_ENC_TFM_ATTN_V, ".weight"),
+ "self_attn.o_proj.weight": (T.A_GEN_ENC_TFM_ATTN_OUT, ".weight"),
+ "self_attn_layer_scale.scale": (T.A_GEN_ENC_TFM_ATTN_SCALE, ".weight"),
+ "post_attention_layernorm.weight": (T.A_GEN_ENC_TFM_FFN_NORM, ".weight"),
+ "post_attention_layernorm.bias": (T.A_GEN_ENC_TFM_FFN_NORM, ".bias"),
+ "mlp.fc1.weight": (T.A_GEN_ENC_TFM_FFN_UP, ".weight"),
+ "mlp.fc2.weight": (T.A_GEN_ENC_TFM_FFN_DOWN, ".weight"),
+ "mlp_layer_scale.scale": (T.A_GEN_ENC_TFM_FFN_SCALE, ".weight"),
+ }
+ for bid in range(enc_config["num_hidden_layers"]):
+ for key, (tensor_id, suffix) in tfm_layer_map.items():
+ yield (self.format_tensor_name(tensor_id, bid, suffix=suffix), state_dict[f"encoder.encoder_transformer.layers.{bid}.{key}"])
+
+ def rvq_codebook(prefix: str, idx: int) -> Tensor:
+ # checkpoint has EMA accumulators, so codebook = embed_sum / cluster_usage
+ embed_sum = state_dict[f"{prefix}.layers.{idx}.codebook.embed_sum"]
+ cluster_usage = state_dict[f"{prefix}.layers.{idx}.codebook.cluster_usage"]
+ return embed_sum / cluster_usage.clamp_min(1e-5).unsqueeze(-1)
+
+ # the tokenizer keeps the semantic codebook and the first acoustic ones, see encoder_valid_num_quantizers
+ semantic = "encoder.quantizer.semantic_residual_vector_quantizer"
+ acoustic = "encoder.quantizer.acoustic_residual_vector_quantizer"
+ yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_FIRST_IN), state_dict[f"{semantic}.input_proj.weight"].squeeze(-1))
+ yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_FIRST_CB), rvq_codebook(semantic, 0))
+ yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_REST_IN), state_dict[f"{acoustic}.input_proj.weight"].squeeze(-1))
+ yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_REST_CB),
+ torch.stack([rvq_codebook(acoustic, i) for i in range(self._CODE_GEN_N_CODEBOOKS)], dim=0))
def _generate_code2wav_tensors(self) -> Iterable[tuple[str, Tensor]]:
# code2wav weights live in speech_tokenizer/model.safetensors, not the main safetensors
diff --git a/docs/multimodal/tts.md b/docs/multimodal/tts.md
index b77804671..5ebc10356 100644
--- a/docs/multimodal/tts.md
+++ b/docs/multimodal/tts.md
@@ -23,6 +23,7 @@ The sections below list the params supported by each model, with `llama-tts` exa
Available params:
- `--tts-lang` can be `zh`, `en`, `de`, `it`, `pt`, `es`, `ja`, `ko`, `fr`, `ru` (default: `en`)
- `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3)
+- `--tts-speaker-text` is the transcript of the speaker file. With it, the speech continues from the reference codes (the "ICL" mode of the reference), which clones the voice more closely. It needs a clip of 30 s or less and an mmproj converted with the speech tokenizer encoder, otherwise it falls back to the speaker embedding only
Example usage:
@@ -34,6 +35,16 @@ llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF \
--output out.wav
```
+Voice cloning with the transcript of the reference:
+
+```sh
+llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF \
+ -p "Hello world" \
+ --tts-speaker-file speaker.wav \
+ --tts-speaker-text "The words spoken in speaker.wav." \
+ --output out.wav
+```
+
## Pocket TTS
Available params:
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index a32811ca9..d15ca9a5f 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -1360,6 +1360,21 @@ class MODEL_TENSOR(IntEnum):
A_CODEC_VQ_FIRST_CB = auto() # semantic VQ codebook, l2-normalized
A_CODEC_VQ_REST_IN = auto() # residual VQ input projections, merged 3D
A_CODEC_VQ_REST_CB = auto() # residual VQ codebooks, l2-normalized, merged 3D
+ # qwen3tts reference encoder: raw PCM -> RVQ codes, the SEANet stack and the downsample conv use A_ENC_*
+ A_GEN_ENC_TFM_ATTN_NORM = auto()
+ A_GEN_ENC_TFM_ATTN_Q = auto()
+ A_GEN_ENC_TFM_ATTN_K = auto()
+ A_GEN_ENC_TFM_ATTN_V = auto()
+ A_GEN_ENC_TFM_ATTN_OUT = auto()
+ A_GEN_ENC_TFM_ATTN_SCALE = auto()
+ A_GEN_ENC_TFM_FFN_NORM = auto()
+ A_GEN_ENC_TFM_FFN_UP = auto()
+ A_GEN_ENC_TFM_FFN_DOWN = auto()
+ A_GEN_ENC_TFM_FFN_SCALE = auto()
+ A_GEN_ENC_QUANT_FIRST_IN = auto()
+ A_GEN_ENC_QUANT_FIRST_CB = auto()
+ A_GEN_ENC_QUANT_REST_IN = auto()
+ A_GEN_ENC_QUANT_REST_CB = auto()
# qwen3tts code2wav: RVQ codes -> raw PCM
A_GEN_WAV_QUANT_FIRST_IN = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D)
A_GEN_WAV_QUANT_FIRST_OUT = auto() # semantic RVQ, out_proj
@@ -2323,6 +2338,20 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_CODEC_VQ_FIRST_CB: "a.codec.vq.first.codebook",
MODEL_TENSOR.A_CODEC_VQ_REST_IN: "a.codec.vq.rest.in_proj",
MODEL_TENSOR.A_CODEC_VQ_REST_CB: "a.codec.vq.rest.codebook",
+ MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_NORM: "a.gen.enc.tfm.blk.{bid}.ln1",
+ MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_Q: "a.gen.enc.tfm.blk.{bid}.attn_q",
+ MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_K: "a.gen.enc.tfm.blk.{bid}.attn_k",
+ MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_V: "a.gen.enc.tfm.blk.{bid}.attn_v",
+ MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_OUT: "a.gen.enc.tfm.blk.{bid}.attn_out",
+ MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_SCALE: "a.gen.enc.tfm.blk.{bid}.ls1",
+ MODEL_TENSOR.A_GEN_ENC_TFM_FFN_NORM: "a.gen.enc.tfm.blk.{bid}.ln2",
+ MODEL_TENSOR.A_GEN_ENC_TFM_FFN_UP: "a.gen.enc.tfm.blk.{bid}.ffn_up",
+ MODEL_TENSOR.A_GEN_ENC_TFM_FFN_DOWN: "a.gen.enc.tfm.blk.{bid}.ffn_down",
+ MODEL_TENSOR.A_GEN_ENC_TFM_FFN_SCALE: "a.gen.enc.tfm.blk.{bid}.ls2",
+ MODEL_TENSOR.A_GEN_ENC_QUANT_FIRST_IN: "a.gen.enc.quant.first.in_proj",
+ MODEL_TENSOR.A_GEN_ENC_QUANT_FIRST_CB: "a.gen.enc.quant.first.codebook",
+ MODEL_TENSOR.A_GEN_ENC_QUANT_REST_IN: "a.gen.enc.quant.rest.in_proj",
+ MODEL_TENSOR.A_GEN_ENC_QUANT_REST_CB: "a.gen.enc.quant.rest.codebook",
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN: "a.gen.wav.quant.first.in_proj",
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj",
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB: "a.gen.wav.quant.first.codebook",
@@ -2778,6 +2807,20 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.A_CODEC_VQ_FIRST_CB,
MODEL_TENSOR.A_CODEC_VQ_REST_IN,
MODEL_TENSOR.A_CODEC_VQ_REST_CB,
+ MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_NORM,
+ MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_Q,
+ MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_K,
+ MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_V,
+ MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_OUT,
+ MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_SCALE,
+ MODEL_TENSOR.A_GEN_ENC_TFM_FFN_NORM,
+ MODEL_TENSOR.A_GEN_ENC_TFM_FFN_UP,
+ MODEL_TENSOR.A_GEN_ENC_TFM_FFN_DOWN,
+ MODEL_TENSOR.A_GEN_ENC_TFM_FFN_SCALE,
+ MODEL_TENSOR.A_GEN_ENC_QUANT_FIRST_IN,
+ MODEL_TENSOR.A_GEN_ENC_QUANT_FIRST_CB,
+ MODEL_TENSOR.A_GEN_ENC_QUANT_REST_IN,
+ MODEL_TENSOR.A_GEN_ENC_QUANT_REST_CB,
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN,
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT,
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB,
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index c432a76b5..5e55e29bc 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -390,6 +390,15 @@ struct mtmd_graph_qwen3tts_gen : mtmd_graph {
: mtmd_graph(ctx, img), gen_process(gen_process), top_k(top_k), top_p(top_p) {}
ggml_cgraph * build() override;
+ // MTMD_SUBCTX_GEN_PROCESS_TOKENIZE: reference clip -> codes and their talker input rows
+ ggml_cgraph * build_ref_encoder();
+ ggml_tensor * nearest_code(ggml_tensor * x, ggml_tensor * codebook) const;
+
+ // reference encoder transformer, see encoder_config in speech_tokenizer/config.json
+ static constexpr int ref_enc_n_head = 8;
+ static constexpr float ref_enc_eps = 1e-5f;
+ static constexpr float ref_enc_rope_theta = 10000.0f;
+
// which sub-graph build() constructs, fixed at graph-build time
mtmd_subctx_gen_process gen_process;
@@ -483,6 +492,11 @@ struct mtmd_graph_pockettts_seanet : mtmd_graph {
mtmd_graph_pockettts_seanet(const mtmd_graph & parent) : mtmd_graph(parent) {}
ggml_cgraph * build() override { GGML_ABORT("call encode()/decode() instead"); }
+ // qwen3tts reference codes: the output is quantized, so the im2col is F32 (the kernels must be F32 too)
+ bool f32_im2col = false;
+ // qwen3tts reference codes: pad_replicate also repeats the last frame on the right, like MimiConv1d
+ bool pad_replicate_end = false;
+
// per-call streaming state, keyed by slot name (see list_pockettts_state_slots)
std::map<std::string, ggml_tensor *> state_in;
mutable std::vector<std::pair<std::string, ggml_tensor *>> state_out;
@@ -501,11 +515,22 @@ struct mtmd_graph_pockettts_seanet : mtmd_graph {
};
// mimi encoder + speaker_proj: reference waveform -> voice conditioning rows
+// the qwen3tts reference encoder reuses encode() with its own transformer options
struct mtmd_graph_pockettts_spkenc : mtmd_graph {
mtmd_graph_pockettts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
+ mtmd_graph_pockettts_spkenc(const mtmd_graph & parent) : mtmd_graph(parent) {}
ggml_cgraph * build() override;
+ int tfm_n_head = n_head;
+ float tfm_eps = eps;
+ float tfm_rope_theta = hparams.rope_theta;
+ int tfm_rope_type = GGML_ROPE_TYPE_NORMAL;
+ ffn_op_type tfm_ffn_type = FFN_GELU;
+
ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const;
+
+ // waveform [n_samples, 1] -> [n_frames, C], at the frame rate after model.downsample_w
+ ggml_tensor * encode(const mtmd_graph_pockettts_seanet & seanet, ggml_tensor * x, const std::vector<mtmd_layer> & layers);
};
//
diff --git a/tools/mtmd/models/pockettts-seanet.cpp b/tools/mtmd/models/pockettts-seanet.cpp
index 4480aac9d..f1073c844 100644
--- a/tools/mtmd/models/pockettts-seanet.cpp
+++ b/tools/mtmd/models/pockettts-seanet.cpp
@@ -33,12 +33,26 @@ ggml_tensor * mtmd_graph_pockettts_seanet::conv1d(ggml_tensor * x, ggml_tensor *
ggml_tensor * first = ggml_view_2d(ctx0, x, 1, x->ne[1], x->nb[1], 0);
ggml_tensor * left = ggml_repeat_4d(ctx0, first, p_total, x->ne[1], 1, 1);
x = ggml_concat(ctx0, left, x, 0);
- x = ggml_pad_ext(ctx0, x, 0, p_extra, 0, 0, 0, 0, 0, 0);
+ if (pad_replicate_end && p_extra > 0) {
+ ggml_tensor * last = ggml_view_2d(ctx0, x, 1, x->ne[1], x->nb[1], (size_t) (x->ne[0] - 1) * x->nb[0]);
+ ggml_tensor * right = ggml_repeat_4d(ctx0, last, p_extra, x->ne[1], 1, 1);
+ x = ggml_concat(ctx0, x, right, 0);
+ } else {
+ x = ggml_pad_ext(ctx0, x, 0, p_extra, 0, 0, 0, 0, 0, 0);
+ }
} else {
x = ggml_pad_ext(ctx0, x, p_total, p_extra, 0, 0, 0, 0, 0, 0);
}
- ggml_tensor * y = ggml_conv_1d(ctx0, w, x, stride, 0, dilation);
+ ggml_tensor * y;
+ if (f32_im2col) {
+ // ggml_conv_1d() with an F32 im2col, the kernel must be F32
+ ggml_tensor * col = ggml_im2col(ctx0, w, x, stride, 0, 0, 0, dilation, 0, false, GGML_TYPE_F32);
+ y = ggml_mul_mat(ctx0, ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[2] * col->ne[1]),
+ ggml_reshape_2d(ctx0, w, w->ne[0] * w->ne[1], w->ne[2]));
+ } else {
+ y = ggml_conv_1d(ctx0, w, x, stride, 0, dilation);
+ }
y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[1]);
if (b) {
y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
diff --git a/tools/mtmd/models/pockettts-spkenc.cpp b/tools/mtmd/models/pockettts-spkenc.cpp
index f01727ad5..d64a0a35f 100644
--- a/tools/mtmd/models/pockettts-spkenc.cpp
+++ b/tools/mtmd/models/pockettts-spkenc.cpp
@@ -5,44 +5,41 @@
// pre-norm block with layer scale on both residual paths, see mimi_transformer.py
ggml_tensor * mtmd_graph_pockettts_spkenc::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const {
+ const int64_t n_pos = cur->ne[1];
+ const int tfm_d_head = (int) cur->ne[0] / tfm_n_head;
+
ggml_tensor * inp = cur;
- cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
+ cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, tfm_eps, il);
ggml_tensor * Qcur = build_mm(layer.q_w, cur);
ggml_tensor * Kcur = build_mm(layer.k_w, cur);
ggml_tensor * Vcur = build_mm(layer.v_w, cur);
- const int64_t n_pos = cur->ne[1];
- Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
- Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
- Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
+ Qcur = ggml_reshape_3d(ctx0, Qcur, tfm_d_head, tfm_n_head, n_pos);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, tfm_d_head, tfm_n_head, n_pos);
+ Vcur = ggml_reshape_3d(ctx0, Vcur, tfm_d_head, tfm_n_head, n_pos);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
- hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
- hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+ Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, tfm_d_head, tfm_rope_type, 0,
+ tfm_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+ Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, tfm_d_head, tfm_rope_type, 0,
+ tfm_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
- cur = build_attn(layer.o_w, nullptr, Qcur, Kcur, Vcur, kq_mask, kq_scale, il);
+ cur = build_attn(layer.o_w, nullptr, Qcur, Kcur, Vcur, kq_mask, 1.0f / sqrtf((float) tfm_d_head), il);
cur = ggml_mul(ctx0, cur, layer.ls_1_w);
cur = ggml_add(ctx0, cur, inp);
inp = cur;
- cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
- cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, FFN_GELU, il);
+ cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, tfm_eps, il);
+ cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, tfm_ffn_type, il);
cur = ggml_mul(ctx0, cur, layer.ls_2_w);
cur = ggml_add(ctx0, cur, inp);
return cur;
}
-ggml_cgraph * mtmd_graph_pockettts_spkenc::build() {
- // the preprocessor hands over the waveform as a single-row "mel", already [n_samples, 1]
- ggml_tensor * inp_raw = build_inp_raw(1);
- ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]);
-
- mtmd_graph_pockettts_seanet seanet(*this);
- cur = seanet.encode(cur);
+ggml_tensor * mtmd_graph_pockettts_spkenc::encode(const mtmd_graph_pockettts_seanet & seanet, ggml_tensor * x, const std::vector<mtmd_layer> & layers) {
+ ggml_tensor * cur = seanet.encode(x);
cb(cur, "mimi_enc", -1);
// [T, 512] -> transformer works on [512, T]
@@ -55,16 +52,27 @@ ggml_cgraph * mtmd_graph_pockettts_spkenc::build() {
// the mimi transformer is causal with a sliding window, see _build_attention_mask()
ggml_tensor * kq_mask = build_inp_attn_mask(cur->ne[1], cur->ne[1]);
- for (int il = 0; il < n_layer; il++) {
- cur = tfm_layer_forward(cur, model.layers[il], inp_pos, kq_mask, il);
+ for (size_t il = 0; il < layers.size(); il++) {
+ cur = tfm_layer_forward(cur, layers[il], inp_pos, kq_mask, (int) il);
}
cb(cur, "mimi_enc_tfm", -1);
- // downsample to the model frame rate, [512, T] -> [T, 512] -> [T / 16, 32]
+ // downsample to the model frame rate, [512, T] -> [T, 512] -> [T / mimi_downsample, C]
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
cur = seanet.conv1d(cur, model.downsample_w, nullptr, hparams.mimi_downsample, 1, true);
cb(cur, "mimi_downsample", -1);
+ return cur;
+}
+
+ggml_cgraph * mtmd_graph_pockettts_spkenc::build() {
+ // the preprocessor hands over the waveform as a single-row "mel", already [n_samples, 1]
+ ggml_tensor * inp_raw = build_inp_raw(1);
+ ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]);
+
+ mtmd_graph_pockettts_seanet seanet(*this);
+ cur = encode(seanet, cur, model.layers);
+
// voice latent -> backbone embd
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
cur = build_mm(model.spk_proj_w, cur);
diff --git a/tools/mtmd/models/qwen3tts-gen.cpp b/tools/mtmd/models/qwen3tts-gen.cpp
index 91d5a5c12..386e22db7 100644
--- a/tools/mtmd/models/qwen3tts-gen.cpp
+++ b/tools/mtmd/models/qwen3tts-gen.cpp
@@ -664,9 +664,77 @@ std::vector<c2w_state_slot> list_c2w_state_slots(const mtmd_hparams & hparams, c
return slots;
}
+// x: [dim, T], codebook: [dim, n_entries] -> [T] index of the nearest entry
+ggml_tensor * mtmd_graph_qwen3tts_gen::nearest_code(ggml_tensor * x, ggml_tensor * codebook) const {
+ // argmin |x - e|^2 = argmax 2 x.e - |e|^2
+ ggml_tensor * dots = ggml_mul_mat(ctx0, codebook, x); // [n_entries, T]
+ ggml_tensor * norms = ggml_sum_rows(ctx0, ggml_sqr(ctx0, codebook)); // [1, n_entries]
+ norms = ggml_reshape_2d(ctx0, norms, norms->ne[1], 1); // [n_entries, 1]
+ return ggml_argmax(ctx0, ggml_sub(ctx0, ggml_scale(ctx0, dots, 2.0f), norms));
+}
+
+// the mimi encoder of the speech tokenizer, then its RVQ: one semantic codebook and n_acoustic acoustic ones
+ggml_cgraph * mtmd_graph_qwen3tts_gen::build_ref_encoder() {
+ // the waveform comes in as a single-row "mel", [n_samples, 1]
+ ggml_tensor * inp_raw = build_inp_raw(1);
+ ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]);
+
+ mtmd_graph_pockettts_seanet seanet(*this);
+ seanet.f32_im2col = true;
+ seanet.pad_replicate_end = true;
+
+ // the checkpoint is in the transformers layout, so rope rotates halves (neox)
+ mtmd_graph_pockettts_spkenc mimi(*this);
+ mimi.tfm_n_head = ref_enc_n_head;
+ mimi.tfm_eps = ref_enc_eps;
+ mimi.tfm_rope_theta = ref_enc_rope_theta;
+ mimi.tfm_rope_type = GGML_ROPE_TYPE_NEOX;
+ mimi.tfm_ffn_type = FFN_GELU_ERF;
+
+ cur = mimi.encode(seanet, cur, model.ref_enc_layers);
+ cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [512, T]
+
+ std::vector<ggml_tensor *> codes;
+ codes.push_back(nearest_code(build_mm(model.ref_quant_first_in_w, cur), model.ref_quant_first_cb));
+
+ const int64_t n_acoustic = model.ref_quant_rest_cb->ne[2];
+ ggml_tensor * residual = build_mm(model.ref_quant_rest_in_w, cur);
+ for (int64_t i = 0; i < n_acoustic; i++) {
+ ggml_tensor * codebook = ggml_view_2d(ctx0, model.ref_quant_rest_cb, model.ref_quant_rest_cb->ne[0], model.ref_quant_rest_cb->ne[1],
+ model.ref_quant_rest_cb->nb[1], (size_t) i * model.ref_quant_rest_cb->nb[2]);
+ ggml_tensor * code = nearest_code(residual, codebook);
+ codes.push_back(code);
+ if (i + 1 < n_acoustic) {
+ residual = ggml_sub(ctx0, residual, ggml_get_rows(ctx0, codebook, code));
+ }
+ }
+
+ // talker input rows of the reference frames: codebook 0 from the talker table, the rest from the code predictor's
+ ggml_tensor * rows = ggml_get_rows(ctx0, model.gen_code_out_embd_w, codes[0]);
+ for (int64_t g = 1; g <= n_acoustic; g++) {
+ ggml_tensor * embd = ggml_view_2d(ctx0, model.gen_code_embd_w, model.gen_code_embd_w->ne[0], model.gen_code_embd_w->ne[1],
+ model.gen_code_embd_w->nb[1], (size_t) (g - 1) * model.gen_code_embd_w->nb[2]);
+ rows = ggml_add(ctx0, rows, ggml_get_rows(ctx0, embd, codes[(size_t) g]));
+ }
+
+ for (size_t g = 0; g < codes.size(); g++) {
+ ggml_set_name(codes[g], ("out_ref_code_" + std::to_string(g)).c_str());
+ ggml_set_output(codes[g]);
+ ggml_build_forward_expand(gf, codes[g]);
+ }
+ ggml_set_name(rows, "out_ref_rows");
+ ggml_set_output(rows);
+ ggml_build_forward_expand(gf, rows);
+ return gf;
+}
+
// both sub-graphs are always built, so the topology stays constant
// ggml_build_forward_select() then picks the one that actually runs
ggml_cgraph * mtmd_graph_qwen3tts_gen::build() {
+ if (gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
+ return build_ref_encoder();
+ }
+
GGML_ASSERT(n_batch == 1); // this module only ever processes one frame at a time
int idx;
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 6a83ece40..cac18b345 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -201,6 +201,7 @@ public:
seq_id = 0;
pos = 0;
codes_buf.clear();
+ next_window_frames = window_frames;
gen_state.reset();
audio_pcm.clear();
overlay.clear();
@@ -232,12 +233,21 @@ public:
std::vector<float> speaker_embd;
if (inp->speaker_ref) {
+ mtmd_bitmap_set_transcript(inp->speaker_ref, inp->speaker_ref_text);
if (!encode_speaker(inp->speaker_ref, speaker_embd)) {
return 1;
}
}
const int n_e = n_embd;
+
+ // cloning with the reference transcript continues the reference speech, see generate_icl_prompt() upstream
+ const bool has_ref_text = inp->speaker_ref_text && inp->speaker_ref_text[0];
+ const size_t n_ref_frames = speaker_embd.empty() ? 0 : speaker_embd.size() / (size_t) n_e - 1;
+ const bool icl = has_ref_text && n_ref_frames > 0;
+ if (has_ref_text && !icl) {
+ LOG_WRN("mtmd_helper_gen_audio: no reference codes (mmproj without reference encoder, or clip too long), cloning from the speaker embedding only\n");
+ }
auto row = [&](llama_token t) {
return std::vector<float>(tok_embd.begin() + (size_t) t * n_e,
tok_embd.begin() + (size_t) (t + 1) * n_e);
@@ -256,14 +266,31 @@ public:
// upstream chat wrap, then slices: [0:3] role, [3:-5] utterance body
const std::string full = "<|im_start|>assistant\n" + std::string(inp->prompt, inp->prompt_len) +
"<|im_end|>\n<|im_start|>assistant\n";
- std::vector<llama_token> ids(full.size() + 16);
- int n_ids = llama_tokenize(vocab, full.c_str(), (int32_t) full.size(), ids.data(), (int32_t) ids.size(),
- false, true);
+ auto tokenize = [&](const std::string & text) {
+ std::vector<llama_token> tokens(text.size() + 16);
+ const int n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), tokens.data(), (int32_t) tokens.size(),
+ false, true);
+ tokens.resize(n < 0 ? 0 : (size_t) n);
+ return tokens;
+ };
+
+ std::vector<llama_token> ids = tokenize(full);
+ const int n_ids = (int) ids.size();
if (n_ids < 8) {
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
return 1;
}
- ids.resize((size_t) n_ids);
+
+ // the reference transcript goes before the text, with the same chat wrap minus the next turn: slices [3:-2]
+ std::vector<llama_token> ref_ids;
+ if (icl) {
+ ref_ids = tokenize("<|im_start|>assistant\n" + std::string(inp->speaker_ref_text) + "<|im_end|>\n");
+ if (ref_ids.size() < 6) {
+ LOG_ERR("mtmd_helper_gen_audio: tokenization of the reference text failed\n");
+ return 1;
+ }
+ ref_ids = std::vector<llama_token>(ref_ids.begin() + 3, ref_ids.end() - 2);
+ }
std::vector<std::vector<float>> prompt;
for (int i = 0; i < 3; i++) prompt.push_back(row(ids[(size_t) i]));
@@ -273,9 +300,17 @@ public:
prompt.push_back(sum_row(tts_pad, c_think_e));
if (!speaker_embd.empty()) prompt.push_back(sum_vec(tts_pad, speaker_embd));
prompt.push_back(sum_row(tts_bos, codec_pad));
+ for (llama_token t : ref_ids) prompt.push_back(sum_row(t, codec_pad));
for (int i = 3; i < n_ids - 5; i++) prompt.push_back(sum_row(ids[(size_t) i], codec_pad));
prompt.push_back(sum_row(tts_eos, codec_pad));
prompt.push_back(sum_row(tts_pad, codec_bos));
+ // the reference frames, as if the talker had just spoken them
+ if (icl) {
+ for (size_t f = 0; f < n_ref_frames; f++) {
+ const float * frame = speaker_embd.data() + (f + 1) * (size_t) n_e;
+ prompt.push_back(sum_vec(tts_pad, std::vector<float>(frame, frame + n_e)));
+ }
+ }
n_prompt = (int) prompt.size();
@@ -303,6 +338,13 @@ public:
if (!init_gen_state(inp->seed)) {
return 1;
}
+ // the decoder continues from the reference codes of the encode above
+ if (icl) {
+ if (mtmd_gen_state_set_ref(mctx, gen_state.get()) != 0) {
+ return 1;
+ }
+ next_window_frames = window_frames - std::min(n_ref_frames, window_frames - 1);
+ }
// the prompt above holds the whole text stream up to tts_eos, so every generated
// frame adds tts_pad on top of the codes embedding
@@ -366,7 +408,7 @@ public:
}
codes_buf.insert(codes_buf.end(), out.codes, out.codes + out.n_codes);
- if (out.n_codes > 0 && codes_buf.size() / out.n_codes >= window_frames) {
+ if (out.n_codes > 0 && codes_buf.size() / out.n_codes >= next_window_frames) {
if (!flush_gen_wav()) {
return 1;
}
@@ -502,6 +544,7 @@ private:
}
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
codes_buf.clear();
+ next_window_frames = window_frames;
return true;
}
@@ -534,6 +577,7 @@ private:
int32_t top_k = 50;
float top_p = 1.0f;
std::vector<int32_t> codes_buf;
+ size_t next_window_frames = window_frames; // the first window leaves room for the reference codes
std::vector<float> audio_pcm;
std::vector<float> overlay;
std::vector<float> h_state_buf;
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index 20465cbd3..3fb75e5a3 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -310,7 +310,7 @@ struct mtmd_helper_gen_audio_inp {
enum mtmd_helper_gen_audio_out_mode out_mode; // n_past > 0 only (default: auto)
mtmd_bitmap * speaker_ref; // optional, can be NULL: audio to clone, or a voice bitmap
- const char * speaker_ref_text; // optional, can be NULL: transcript of speaker_ref, for models that put it in the prompt (e.g. Fish S2)
+ const char * speaker_ref_text; // optional, can be NULL: transcript of speaker_ref, for models that use it to clone the voice (e.g. Fish S2, Qwen3-TTS)
const char * lang; // optional, can be NULL
int32_t top_k;
diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h
index 37ba390f9..e1dc95cee 100644
--- a/tools/mtmd/mtmd-impl.h
+++ b/tools/mtmd/mtmd-impl.h
@@ -252,6 +252,12 @@
#define TN_A_ASP_ATTN "a.asp_attn.%s"
#define TN_A_ASP_TDNN "a.asp_tdnn.%s"
+// qwen3tts reference encoder, the transformer blocks use the generic names with the "a.gen.enc.tfm" prefix
+#define TN_A_GEN_ENC_QUANT_FIRST_IN "a.gen.enc.quant.first.in_proj.%s"
+#define TN_A_GEN_ENC_QUANT_FIRST_CB "a.gen.enc.quant.first.codebook.%s"
+#define TN_A_GEN_ENC_QUANT_REST_IN "a.gen.enc.quant.rest.in_proj.%s"
+#define TN_A_GEN_ENC_QUANT_REST_CB "a.gen.enc.quant.rest.codebook.%s"
+
// qwen3tts code_predictor
#define TN_A_GEN_CODE_PROJ_IN "a.gen.code.proj_in.%s"
#define TN_A_GEN_CODE_EMBD "a.gen.code.embd.%s"
diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp
index 29531db2e..c352e28d9 100644
--- a/tools/mtmd/mtmd-model.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -1075,6 +1075,14 @@ struct mtmd_model_loader {
// TODO: hardcoded for now, read from code_predictor_config instead
hparams.rope_theta = 1000000.0f;
+ // reference encoder: mimi with the stride-2 downsample of the 12.5 Hz tokenizer
+ // seanet ratios are [8,6,5,4] in the config, the encoder reverses them
+ hparams.audio_sample_rate = 24000;
+ hparams.seanet_ratios = { 4, 5, 6, 8 };
+ hparams.seanet_n_stage = (int32_t) hparams.seanet_ratios.size();
+ hparams.mimi_downsample = 2;
+ hparams.mimi_tfm_context = 250;
+
// code2wav params
hparams.wav_tfm_n_layer = 8;
hparams.wav_tfm_n_embd = 512;
@@ -2826,6 +2834,47 @@ struct mtmd_model_loader {
// code2wav: RVQ codes -> raw PCM, lives in the same ctx as code_predictor
load_code2wav(true);
+
+ // reference encoder, absent from files converted before it was added
+ model.ref_quant_rest_cb = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_REST_CB, "weight"), false);
+ if (model.ref_quant_rest_cb) {
+ load_seanet(model.seanet, false);
+ model.downsample_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight"));
+ for (int il = 0; ; il++) {
+ const char * p = "a.gen.enc.tfm";
+ ggml_tensor * ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight"), false);
+ if (!ln_1_w) {
+ break;
+ }
+ mtmd_layer layer;
+ layer.ln_1_w = ln_1_w;
+ layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, p, il, "bias"));
+ layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, p, il, "weight"));
+ layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, p, il, "weight"));
+ layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, p, il, "weight"));
+ layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight"));
+ layer.ls_1_w = get_tensor(mtmd_format(TN_LS_1, p, il, "weight"));
+ layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, p, il, "weight"));
+ layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, p, il, "bias"));
+ layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, p, il, "weight"));
+ layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, p, il, "weight"));
+ layer.ls_2_w = get_tensor(mtmd_format(TN_LS_2, p, il, "weight"));
+ model.ref_enc_layers.push_back(layer);
+ }
+ model.ref_quant_first_in_w = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_FIRST_IN, "weight"));
+ model.ref_quant_first_cb = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_FIRST_CB, "weight"));
+ model.ref_quant_rest_in_w = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_REST_IN, "weight"));
+
+ // the codes index the code predictor tables, see build_ref_encoder()
+ const int64_t n_acoustic = model.ref_quant_rest_cb->ne[2];
+ if (model.ref_enc_layers.empty() ||
+ n_acoustic != model.gen_code_head_w->ne[2] ||
+ n_acoustic > model.gen_code_embd_w->ne[2] ||
+ model.ref_quant_first_cb->ne[1] > model.gen_code_out_embd_w->ne[1] ||
+ model.ref_quant_rest_cb->ne[1] > model.gen_code_embd_w->ne[1]) {
+ throw std::runtime_error(mtmd_format("%s: qwen3tts: unexpected reference encoder shape\n", __func__));
+ }
+ }
} break;
case PROJECTOR_TYPE_ZONOS3_GEN:
{
diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h
index 1929f9de1..636bde14a 100644
--- a/tools/mtmd/mtmd-model.h
+++ b/tools/mtmd/mtmd-model.h
@@ -213,6 +213,8 @@ struct mtmd_hparams {
// pocket-tts
static constexpr int32_t pockettts_max_spk_seconds = 30;
+ // qwen3tts: the reference encoder mask is dense, so cost is quadratic in the clip
+ static constexpr int32_t qwen3tts_max_ref_seconds = 30;
// fish_s2: the codec encoder runs at 44.1 kHz, memory grows with the clip
static constexpr int32_t fish_s2_max_spk_seconds = 30;
int32_t seanet_n_stage = 0;
@@ -1300,6 +1302,17 @@ struct mtmd_model {
ggml_tensor * gen_code_head_norm_w = nullptr; // per-codebook norm before the tied head, merged 2D
ggml_tensor * gen_code_mask_embd = nullptr;
+ // qwen3tts reference encoder: reference clip -> RVQ codes, uses seanet and downsample_w too
+ std::vector<mtmd_layer> ref_enc_layers;
+ ggml_tensor * ref_quant_first_in_w = nullptr;
+ ggml_tensor * ref_quant_first_cb = nullptr;
+ ggml_tensor * ref_quant_rest_in_w = nullptr;
+ ggml_tensor * ref_quant_rest_cb = nullptr; // acoustic codebooks, merged 3D
+
+ bool has_ref_encoder() const {
+ return ref_quant_rest_cb != nullptr;
+ }
+
// qwen3tts code2wav: RVQ codes -> raw PCM
mtmd_code2wav c2w;
std::vector<mtmd_drama3_layer> drama3_layers;
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index 321b6171b..d6ec8e862 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -675,9 +675,9 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
set_input_attn_mask(mask);
};
- // rope positions and attention mask of the mimi transformers (pocket-tts).
+ // rope positions and attention mask of the mimi encoder transformer (pocket-tts, qwen3tts reference encoder).
// the mask is causal with a sliding window, see _build_attention_mask() in the reference
- auto set_pockettts_tfm_inputs = [&]() {
+ auto set_mimi_tfm_inputs = [&](int64_t max_samples) {
const int64_t n_pos = ggml_nelements(get_inp_tensor("inp_pos"));
GGML_ASSERT(n_pos > 0);
std::vector<int32_t> positions((size_t) n_pos);
@@ -686,9 +686,13 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
}
set_input_i32("inp_pos", positions);
- // the preprocessor truncates the waveform to keep this mask bounded
- const int64_t max_pos = (int64_t) mtmd_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate / 120;
- GGML_ASSERT(n_pos <= max_pos && "pocket-tts speaker reference too long for a dense mask");
+ // the caller truncates the waveform to keep this mask bounded
+ int64_t hop = 1;
+ for (int32_t ratio : hparams.seanet_ratios) {
+ hop *= ratio;
+ }
+ const int64_t max_pos = (max_samples + hop - 1) / hop;
+ GGML_ASSERT(n_pos <= max_pos && "speaker reference too long for a dense mask");
const int64_t context = hparams.mimi_tfm_context;
std::vector<float> mask((size_t) n_pos * n_pos, -INFINITY);
@@ -1483,7 +1487,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
} break;
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
{
- set_pockettts_tfm_inputs();
+ set_mimi_tfm_inputs((int64_t) mtmd_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate);
} break;
case PROJECTOR_TYPE_POCKETTTS_GEN:
{
@@ -1887,6 +1891,10 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
+ if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
+ set_mimi_tfm_inputs((int64_t) mtmd_hparams::qwen3tts_max_ref_seconds * hparams.audio_sample_rate);
+ break;
+ }
if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
GGML_ASSERT(params->codes != nullptr);
@@ -2423,6 +2431,35 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
// for audio gen models
//
+ if (model.proj_type == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
+ GGML_ASSERT(params->out_codes && params->out_code_embd);
+
+ // frame-major codes, the layout of GEN_WAV's input
+ std::vector<std::vector<int32_t>> groups;
+ for (int g = 0; ; g++) {
+ ggml_tensor * t = ggml_graph_get_tensor(gf, ("out_ref_code_" + std::to_string(g)).c_str());
+ if (!t) {
+ break;
+ }
+ groups.emplace_back((size_t) ggml_nelements(t));
+ ggml_backend_tensor_get(t, groups.back().data(), 0, ggml_nbytes(t));
+ }
+ GGML_ASSERT(!groups.empty());
+ const size_t n_frames = groups[0].size();
+ params->out_codes->resize(n_frames * groups.size());
+ for (size_t f = 0; f < n_frames; f++) {
+ for (size_t g = 0; g < groups.size(); g++) {
+ (*params->out_codes)[f * groups.size() + g] = groups[g][f];
+ }
+ }
+
+ ggml_tensor * rows = ggml_graph_get_tensor(gf, "out_ref_rows");
+ GGML_ASSERT(rows && rows->type == GGML_TYPE_F32 && (size_t) rows->ne[1] == n_frames);
+ params->out_code_embd->resize((size_t) ggml_nelements(rows));
+ ggml_backend_tensor_get(rows, params->out_code_embd->data(), 0, ggml_nbytes(rows));
+ return true;
+ }
+
if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
ggml_tensor * fsq = ggml_graph_get_tensor(gf, "out_fsq");
GGML_ASSERT(fsq != nullptr && params->out_codes);
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 7821cdfbd..7c22a1041 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -204,6 +204,7 @@ struct mtmd_bitmap {
std::shared_ptr<const mtmd_gen_ref> gen_ref; // gen-side reference of a predefined voice, if any
bool mergeable = false; // [QWEN_VIDEO] set only on frames of the same video
bool is_video = false; // see mtmd_bitmap_set_is_video()
+ std::string transcript; // see mtmd_bitmap_set_transcript()
// lazy-loaded bitmap
mtmd_bitmap_lazy_callback lazy_callback = nullptr;
@@ -2056,6 +2057,12 @@ struct mtmd_tokenizer {
// sanity check
GGML_ASSERT(ctx->audio_preproc != nullptr);
+ // qwen3tts: a clip with a transcript also gives the reference codes, see mtmd_qwen3tts_ref_encode()
+ mtmd_image_f32 ref_wav;
+ bool with_ref_codes = ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_QWEN3TTS_SPKENC &&
+ ctx->ctx_gen_a && ctx->ctx_gen_a->model.has_ref_encoder() &&
+ !bitmap->transcript.empty();
+
// preprocess audio
auto source = std::make_shared<mtmd_audio_preproc_out>();
{
@@ -2081,6 +2088,19 @@ struct mtmd_tokenizer {
LOG_ERR("%s: this model takes audio input only via mtmd_audio_stream\n", __func__);
return 2;
}
+ if (with_ref_codes) {
+ // a cut clip no longer matches its transcript, so a long clip gives no codes at all
+ const size_t sample_rate = (size_t) ctx->ctx_gen_a->model.hparams.audio_sample_rate;
+ const size_t max_samples = (size_t) mtmd_hparams::qwen3tts_max_ref_seconds * sample_rate;
+ if (n_samples > max_samples) {
+ LOG_WRN("%s: speaker reference is %.1f s, longer than %d s, the reference codes are skipped\n", __func__,
+ (double) n_samples / sample_rate, mtmd_hparams::qwen3tts_max_ref_seconds);
+ with_ref_codes = false;
+ } else {
+ ref_wav.set_size({(int) n_samples, 1}, false, true);
+ ref_wav.cpy_buf(std::vector<float>(samples, samples + n_samples));
+ }
+ }
*source = ctx->audio_preproc->preprocess(samples, n_samples);
if (source->entries.empty()) {
LOG_ERR("Unable to preprocess audio\n");
@@ -2123,6 +2143,17 @@ struct mtmd_tokenizer {
batch_f32.is_audio = true;
batch_f32.entries.push_back(std::move(mel_f32));
+ // the reference waveform is a second entry, its frames follow the speaker row
+ if (with_ref_codes && i == 0) {
+ const auto & gen_hparams = ctx->ctx_gen_a->model.hparams;
+ int64_t samples_per_frame = gen_hparams.mimi_downsample;
+ for (int32_t ratio : gen_hparams.seanet_ratios) {
+ samples_per_frame *= ratio;
+ }
+ n_tokens += (ref_wav.nx() + samples_per_frame - 1) / samples_per_frame;
+ batch_f32.entries.push_back(std::move(ref_wav));
+ }
+
mtmd_audio_tokens_ptr audio_tokens(new mtmd_audio_tokens);
audio_tokens->n_tokens = n_tokens;
audio_tokens->batch_f32 = std::move(batch_f32);
@@ -2470,6 +2501,51 @@ static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_bat
return ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd);
}
+// qwen3tts: the speaker row, then one talker input row per reference frame
+// the reference codes go to ctx->gen_ref, so GEN_WAV continues from them, see mtmd_gen_state_set_ref()
+static int32_t mtmd_qwen3tts_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd) {
+ const size_t n_embd = ctx->n_embd_out();
+ const auto & entries = audio_tokens->batch_f32.entries;
+ GGML_ASSERT(entries.size() == 2);
+
+ mtmd_image_f32_batch mel;
+ mel.is_audio = true;
+ mel.entries.push_back(entries[0]);
+
+ std::vector<float> speaker(n_embd);
+ if (!mtmd_audio_batch_encode(ctx, mel, audio_tokens->source.get(), audio_tokens->source_idx, speaker)) {
+ return 1;
+ }
+
+ mtmd_image_f32_batch wav;
+ wav.is_audio = true;
+ wav.entries.push_back(entries[1]);
+
+ std::vector<int32_t> codes;
+ std::vector<float> rows;
+
+ mtmd_subctx_encode_params params;
+ params.imgs = &wav;
+ params.n_threads = ctx->n_threads;
+ params.gen_process = MTMD_SUBCTX_GEN_PROCESS_TOKENIZE;
+ params.out_codes = &codes;
+ params.out_code_embd = &rows;
+
+ if (!ctx->ctx_gen_a->encode(¶ms)) {
+ LOG_ERR("%s: encode failed (reference codes)\n", __func__);
+ return 1;
+ }
+ if (rows.size() != (size_t) (audio_tokens->n_tokens - 1) * n_embd) {
+ LOG_ERR("%s: %zu reference rows, expected %u\n", __func__, rows.size() / n_embd, audio_tokens->n_tokens - 1);
+ return 1;
+ }
+
+ out_embd = std::move(speaker);
+ out_embd.insert(out_embd.end(), rows.begin(), rows.end());
+ ctx->gen_ref.tokens = std::move(codes);
+ return 0;
+}
+
static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk * chunk, std::vector<float> & out_embd) {
ctx->gen_ref = {}; // see mtmd_gen_state_set_ref()
if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
@@ -2524,6 +2600,9 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk
if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_CHATTERBOX_SPKENC) {
return cbx_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
}
+ if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_QWEN3TTS_SPKENC && chunk->tokens_audio->batch_f32.entries.size() == 2) {
+ return mtmd_qwen3tts_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
+ }
out_embd.resize((size_t)chunk->tokens_audio->n_tokens * ctx->n_embd_out());
bool ok = mtmd_audio_batch_encode(ctx, chunk->tokens_audio->batch_f32, chunk->tokens_audio->source.get(), chunk->tokens_audio->source_idx, out_embd);
return ok ? 0 : 1;
@@ -3079,7 +3158,9 @@ struct mtmd_gen_state {
std::vector<float> out_audio; // decoded PCM samples for the current frame (GEN_WAV)
// voice reference for GEN_WAV, from a speaker clip or a predefined voice (chatterbox)
+ // qwen3tts: the reference codes, decoded with the first GEN_WAV call so the speech continues from them
mtmd_gen_ref ref;
+ bool ref_decoded = false;
// lfm3a: the codec graph yields STFT frames, the ISTFT runs on the host
std::unique_ptr<mtmd_audio_streaming_istft> istft;
@@ -3126,6 +3207,7 @@ void mtmd_gen_state_free(mtmd_gen_state * state) {
void mtmd_gen_state_reset(mtmd_gen_state * state) {
state->gen_state->reset();
+ state->ref_decoded = false;
if (state->istft) {
const auto * hparams = &state->sctx->model.hparams;
state->istft = std::make_unique<mtmd_audio_streaming_istft>(hparams->audio_n_fft, hparams->audio_hop_len);
@@ -3622,6 +3704,7 @@ int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) {
return 1;
}
state->ref = ctx->gen_ref;
+ state->ref_decoded = false;
return 0;
}
@@ -3784,16 +3867,37 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
params.imgs = &batch;
params.n_threads = ctx->n_threads;
params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV;
- params.codes = has_codes ? &in_codes : nullptr;
- params.feats = has_feats ? &in_feats : nullptr;
params.out_audio = &state->out_audio;
params.gen_state = state->gen_state;
+ // qwen3tts: the first call puts the end of the reference before its codes, so the speech continues from it
+ // a batch shorter than the window carries a wrong state, so the reference only fills the free part of this window
+ size_t n_ref_used = 0;
+ size_t n_frames = 0;
+ if (sctx->model.proj_type == PROJECTOR_TYPE_QWEN3TTS_GEN && has_codes) {
+ const size_t n_codes = (size_t) sctx->model.gen_code_head_w->ne[2] + 1;
+ if (!state->ref.tokens.empty() && !state->ref_decoded) {
+ state->ref_decoded = true;
+ const size_t n_win = (size_t) sctx->model.hparams.wav_tfm_swa;
+ const size_t n_in = in_codes.size() / n_codes;
+ n_ref_used = std::min(state->ref.tokens.size() / n_codes, n_win > n_in ? n_win - n_in : 0);
+ in_codes.insert(in_codes.begin(), state->ref.tokens.end() - (std::ptrdiff_t) (n_ref_used * n_codes), state->ref.tokens.end());
+ }
+ n_frames = in_codes.size() / n_codes;
+ }
+
+ params.codes = has_codes ? &in_codes : nullptr;
+ params.feats = has_feats ? &in_feats : nullptr;
if (!sctx->encode(¶ms)) {
LOG_ERR("%s: encode failed (code2wav)\n", __func__);
return 1;
}
+ if (n_ref_used > 0) {
+ const size_t hop = state->out_audio.size() / n_frames;
+ state->out_audio.erase(state->out_audio.begin(), state->out_audio.begin() + (std::ptrdiff_t) (n_ref_used * hop));
+ }
+
out->audio = state->out_audio.data();
out->n_samples = state->out_audio.size();
@@ -4103,6 +4207,14 @@ void mtmd_bitmap_set_is_video(mtmd_bitmap * bitmap, bool is_video) {
bitmap->is_video = is_video;
}
+void mtmd_bitmap_set_transcript(mtmd_bitmap * bitmap, const char * transcript) {
+ if (transcript) {
+ bitmap->transcript = std::string(transcript);
+ } else {
+ bitmap->transcript.clear();
+ }
+}
+
mtmd_bitmap * mtmd_bitmap_init_lazy(const mtmd_context * ctx,
const char * id,
void * user_data,
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index 3829b6078..8d5b42a9f 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -218,6 +218,8 @@ MTMD_API void mtmd_bitmap_set_mergeable(mtmd_bitmap * bitmap, bool merge
// on a lazy bitmap: the expanded chunks are one video, wrapped by the model's video begin/end tokens
// with MTMD_VIDEO_TEMPLATE_LFM3_OMNI, its audio bitmaps are the pieces of one soundtrack, in order
MTMD_API void mtmd_bitmap_set_is_video(mtmd_bitmap * bitmap, bool is_video);
+// transcript of an audio bitmap is optional, some TTS models use it to clone the speaker voice more closely (--tts-speaker-text)
+MTMD_API void mtmd_bitmap_set_transcript(mtmd_bitmap * bitmap, const char * transcript);
// mtmd_bitmap lazy
//
diff --git a/tools/server/README.md b/tools/server/README.md
index c8d25199f..6dddbc81b 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -866,7 +866,7 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For the list
`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence. Rejected when the model has no voice encoder (e.g. Pocket TTS without voice cloning).
-`speaker_ref_text`: Transcript of the reference audio, put in the prompt by some models (e.g. Fish Audio S2 Pro). Optional.
+`speaker_ref_text`: Transcript of the reference audio, put in the prompt by some models (e.g. Fish Audio S2 Pro, Qwen3-TTS). Optional.
`top_k`, `top_p`: Sampling params for both the backbone sampler and the acoustic code predictor. Optional, `0` or absent keeps the server `--top-k`/`--top-p` for the backbone and the model defaults for the code predictor.