Commit 8c16babba for llama.cpp

commit 8c16babba5d4c66e28a19d86e572e4b04f522d52
Author: James Prevett <jp@jamespre.dev>
Date:   Tue Nov 17 11:59:11 2026 -0600

    mtmd: Add support for Qwen3-TTS' ICL/reference mode (#32438)

    * Added support for Qwen3-TTS ICL/reference mode

    Assisted-By: Claude Opus 5.5

    * fix correctness

    * attach transcription to bitmap

    * nits

    ---------

    Co-authored-by: Xuan Son Nguyen <son@huggingface.co>

diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py
index 28bb6e095..1ea14692b 100644
--- a/conversion/qwen3tts.py
+++ b/conversion/qwen3tts.py
@@ -275,6 +275,14 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
             or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight")))
         ):
             return gguf.GGMLQuantizationType.F16
+        # the reference encoder's convs run with an F32 im2col, see f32_im2col in pockettts-seanet.cpp
+        if new_name.startswith("a.seanet.") or new_name == "a.downsample.conv.weight":
+            return gguf.GGMLQuantizationType.F32
+        # codes are picked by the nearest codebook entry: Q8_0 changes about 8% of them, F16 none
+        if new_name.startswith("a.gen.enc.tfm.blk.") and new_name.endswith(".weight") and n_dims == 2:
+            return gguf.GGMLQuantizationType.F16
+        if new_name.startswith("a.gen.enc.quant."):
+            return gguf.GGMLQuantizationType.F32
         # ConvTranspose1d kernels: only F16/F32 are implemented, no BF16
         if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name):
             return gguf.GGMLQuantizationType.F32
@@ -298,8 +306,8 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
         return super().filter_tensors((name, gen))

     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # code2wav tensors are already named by generate_extra_tensors(), pass them through
-        if name.startswith("a.gen.wav."):
+        # code2wav and reference encoder tensors are already named by generate_extra_tensors(), pass them through
+        if name.startswith(("a.gen.wav.", "a.gen.enc.", "a.seanet.", "a.downsample.")):
             yield (name, data_torch)
             return

@@ -356,6 +364,68 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):

     def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
         yield from self._generate_code2wav_tensors()
+        yield from self._generate_ref_encoder_tensors()
+
+    # mimi encoder of the speech tokenizer: reference clip -> codes, for voice cloning with a transcript (ICL)
+    def _generate_ref_encoder_tensors(self) -> Iterable[tuple[str, Tensor]]:
+        from safetensors.torch import load_file
+
+        state_dict = load_file(self.dir_model / "speech_tokenizer" / "model.safetensors")
+        if "encoder.downsample.conv.weight" not in state_dict:
+            return
+
+        with open(self.dir_model / "speech_tokenizer" / "config.json", "r", encoding="utf-8") as f:
+            enc_config = json.load(f)["encoder_config"]
+        n_stage = len(enc_config["upsampling_ratios"])
+
+        T = gguf.MODEL_TENSOR
+
+        # encoder.encoder.layers: conv_in, then per stage a residual unit, an ELU and a strided conv, then an ELU and conv_out
+        def seanet(idx: int, suffix: str) -> Tensor:
+            return state_dict[f"encoder.encoder.layers.{idx}.{suffix}"]
+
+        for suffix in ("weight", "bias"):
+            yield (self.format_tensor_name(T.A_ENC_SEANET_CONV_IN, suffix=f".{suffix}"), seanet(0, f"conv.{suffix}"))
+            yield (self.format_tensor_name(T.A_ENC_SEANET_CONV_OUT, suffix=f".{suffix}"), seanet(3 * n_stage + 2, f"conv.{suffix}"))
+            for stage in range(n_stage):
+                yield (self.format_tensor_name(T.A_ENC_SEANET_RES_CONV1, stage, suffix=f".{suffix}"), seanet(3 * stage + 1, f"block.1.conv.{suffix}"))
+                yield (self.format_tensor_name(T.A_ENC_SEANET_RES_CONV2, stage, suffix=f".{suffix}"), seanet(3 * stage + 1, f"block.3.conv.{suffix}"))
+                yield (self.format_tensor_name(T.A_ENC_SEANET_SCALE_CONV, stage, suffix=f".{suffix}"), seanet(3 * stage + 3, f"conv.{suffix}"))
+
+        yield (self.format_tensor_name(T.A_ENC_DOWNSAMPLE_CONV), state_dict["encoder.downsample.conv.weight"])
+
+        tfm_layer_map = {
+            "input_layernorm.weight":          (T.A_GEN_ENC_TFM_ATTN_NORM, ".weight"),
+            "input_layernorm.bias":            (T.A_GEN_ENC_TFM_ATTN_NORM, ".bias"),
+            "self_attn.q_proj.weight":         (T.A_GEN_ENC_TFM_ATTN_Q, ".weight"),
+            "self_attn.k_proj.weight":         (T.A_GEN_ENC_TFM_ATTN_K, ".weight"),
+            "self_attn.v_proj.weight":         (T.A_GEN_ENC_TFM_ATTN_V, ".weight"),
+            "self_attn.o_proj.weight":         (T.A_GEN_ENC_TFM_ATTN_OUT, ".weight"),
+            "self_attn_layer_scale.scale":     (T.A_GEN_ENC_TFM_ATTN_SCALE, ".weight"),
+            "post_attention_layernorm.weight": (T.A_GEN_ENC_TFM_FFN_NORM, ".weight"),
+            "post_attention_layernorm.bias":   (T.A_GEN_ENC_TFM_FFN_NORM, ".bias"),
+            "mlp.fc1.weight":                  (T.A_GEN_ENC_TFM_FFN_UP, ".weight"),
+            "mlp.fc2.weight":                  (T.A_GEN_ENC_TFM_FFN_DOWN, ".weight"),
+            "mlp_layer_scale.scale":           (T.A_GEN_ENC_TFM_FFN_SCALE, ".weight"),
+        }
+        for bid in range(enc_config["num_hidden_layers"]):
+            for key, (tensor_id, suffix) in tfm_layer_map.items():
+                yield (self.format_tensor_name(tensor_id, bid, suffix=suffix), state_dict[f"encoder.encoder_transformer.layers.{bid}.{key}"])
+
+        def rvq_codebook(prefix: str, idx: int) -> Tensor:
+            # checkpoint has EMA accumulators, so codebook = embed_sum / cluster_usage
+            embed_sum = state_dict[f"{prefix}.layers.{idx}.codebook.embed_sum"]
+            cluster_usage = state_dict[f"{prefix}.layers.{idx}.codebook.cluster_usage"]
+            return embed_sum / cluster_usage.clamp_min(1e-5).unsqueeze(-1)
+
+        # the tokenizer keeps the semantic codebook and the first acoustic ones, see encoder_valid_num_quantizers
+        semantic = "encoder.quantizer.semantic_residual_vector_quantizer"
+        acoustic = "encoder.quantizer.acoustic_residual_vector_quantizer"
+        yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_FIRST_IN), state_dict[f"{semantic}.input_proj.weight"].squeeze(-1))
+        yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_FIRST_CB), rvq_codebook(semantic, 0))
+        yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_REST_IN), state_dict[f"{acoustic}.input_proj.weight"].squeeze(-1))
+        yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_REST_CB),
+               torch.stack([rvq_codebook(acoustic, i) for i in range(self._CODE_GEN_N_CODEBOOKS)], dim=0))

     def _generate_code2wav_tensors(self) -> Iterable[tuple[str, Tensor]]:
         # code2wav weights live in speech_tokenizer/model.safetensors, not the main safetensors
diff --git a/docs/multimodal/tts.md b/docs/multimodal/tts.md
index b77804671..5ebc10356 100644
--- a/docs/multimodal/tts.md
+++ b/docs/multimodal/tts.md
@@ -23,6 +23,7 @@ The sections below list the params supported by each model, with `llama-tts` exa
 Available params:
 - `--tts-lang` can be `zh`, `en`, `de`, `it`, `pt`, `es`, `ja`, `ko`, `fr`, `ru` (default: `en`)
 - `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3)
+- `--tts-speaker-text` is the transcript of the speaker file. With it, the speech continues from the reference codes (the "ICL" mode of the reference), which clones the voice more closely. It needs a clip of 30 s or less and an mmproj converted with the speech tokenizer encoder, otherwise it falls back to the speaker embedding only

 Example usage:

@@ -34,6 +35,16 @@ llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF \
     --output out.wav
 ```

+Voice cloning with the transcript of the reference:
+
+```sh
+llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF \
+    -p "Hello world" \
+    --tts-speaker-file speaker.wav \
+    --tts-speaker-text "The words spoken in speaker.wav." \
+    --output out.wav
+```
+
 ## Pocket TTS

 Available params:
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index a32811ca9..d15ca9a5f 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -1360,6 +1360,21 @@ class MODEL_TENSOR(IntEnum):
     A_CODEC_VQ_FIRST_CB    = auto() # semantic VQ codebook, l2-normalized
     A_CODEC_VQ_REST_IN     = auto() # residual VQ input projections, merged 3D
     A_CODEC_VQ_REST_CB     = auto() # residual VQ codebooks, l2-normalized, merged 3D
+    # qwen3tts reference encoder: raw PCM -> RVQ codes, the SEANet stack and the downsample conv use A_ENC_*
+    A_GEN_ENC_TFM_ATTN_NORM        = auto()
+    A_GEN_ENC_TFM_ATTN_Q           = auto()
+    A_GEN_ENC_TFM_ATTN_K           = auto()
+    A_GEN_ENC_TFM_ATTN_V           = auto()
+    A_GEN_ENC_TFM_ATTN_OUT         = auto()
+    A_GEN_ENC_TFM_ATTN_SCALE       = auto()
+    A_GEN_ENC_TFM_FFN_NORM         = auto()
+    A_GEN_ENC_TFM_FFN_UP           = auto()
+    A_GEN_ENC_TFM_FFN_DOWN         = auto()
+    A_GEN_ENC_TFM_FFN_SCALE        = auto()
+    A_GEN_ENC_QUANT_FIRST_IN       = auto()
+    A_GEN_ENC_QUANT_FIRST_CB       = auto()
+    A_GEN_ENC_QUANT_REST_IN        = auto()
+    A_GEN_ENC_QUANT_REST_CB        = auto()
     # qwen3tts code2wav: RVQ codes -> raw PCM
     A_GEN_WAV_QUANT_FIRST_IN       = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D)
     A_GEN_WAV_QUANT_FIRST_OUT      = auto() # semantic RVQ, out_proj
@@ -2323,6 +2338,20 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.A_CODEC_VQ_FIRST_CB:       "a.codec.vq.first.codebook",
     MODEL_TENSOR.A_CODEC_VQ_REST_IN:        "a.codec.vq.rest.in_proj",
     MODEL_TENSOR.A_CODEC_VQ_REST_CB:        "a.codec.vq.rest.codebook",
+    MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_NORM:   "a.gen.enc.tfm.blk.{bid}.ln1",
+    MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_Q:      "a.gen.enc.tfm.blk.{bid}.attn_q",
+    MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_K:      "a.gen.enc.tfm.blk.{bid}.attn_k",
+    MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_V:      "a.gen.enc.tfm.blk.{bid}.attn_v",
+    MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_OUT:    "a.gen.enc.tfm.blk.{bid}.attn_out",
+    MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_SCALE:  "a.gen.enc.tfm.blk.{bid}.ls1",
+    MODEL_TENSOR.A_GEN_ENC_TFM_FFN_NORM:    "a.gen.enc.tfm.blk.{bid}.ln2",
+    MODEL_TENSOR.A_GEN_ENC_TFM_FFN_UP:      "a.gen.enc.tfm.blk.{bid}.ffn_up",
+    MODEL_TENSOR.A_GEN_ENC_TFM_FFN_DOWN:    "a.gen.enc.tfm.blk.{bid}.ffn_down",
+    MODEL_TENSOR.A_GEN_ENC_TFM_FFN_SCALE:   "a.gen.enc.tfm.blk.{bid}.ls2",
+    MODEL_TENSOR.A_GEN_ENC_QUANT_FIRST_IN:  "a.gen.enc.quant.first.in_proj",
+    MODEL_TENSOR.A_GEN_ENC_QUANT_FIRST_CB:  "a.gen.enc.quant.first.codebook",
+    MODEL_TENSOR.A_GEN_ENC_QUANT_REST_IN:   "a.gen.enc.quant.rest.in_proj",
+    MODEL_TENSOR.A_GEN_ENC_QUANT_REST_CB:   "a.gen.enc.quant.rest.codebook",
     MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN:  "a.gen.wav.quant.first.in_proj",
     MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj",
     MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB:  "a.gen.wav.quant.first.codebook",
@@ -2778,6 +2807,20 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.A_CODEC_VQ_FIRST_CB,
         MODEL_TENSOR.A_CODEC_VQ_REST_IN,
         MODEL_TENSOR.A_CODEC_VQ_REST_CB,
+        MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_NORM,
+        MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_Q,
+        MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_K,
+        MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_V,
+        MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_OUT,
+        MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_SCALE,
+        MODEL_TENSOR.A_GEN_ENC_TFM_FFN_NORM,
+        MODEL_TENSOR.A_GEN_ENC_TFM_FFN_UP,
+        MODEL_TENSOR.A_GEN_ENC_TFM_FFN_DOWN,
+        MODEL_TENSOR.A_GEN_ENC_TFM_FFN_SCALE,
+        MODEL_TENSOR.A_GEN_ENC_QUANT_FIRST_IN,
+        MODEL_TENSOR.A_GEN_ENC_QUANT_FIRST_CB,
+        MODEL_TENSOR.A_GEN_ENC_QUANT_REST_IN,
+        MODEL_TENSOR.A_GEN_ENC_QUANT_REST_CB,
         MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN,
         MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT,
         MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB,
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index c432a76b5..5e55e29bc 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -390,6 +390,15 @@ struct mtmd_graph_qwen3tts_gen : mtmd_graph {
         : mtmd_graph(ctx, img), gen_process(gen_process), top_k(top_k), top_p(top_p) {}
     ggml_cgraph * build() override;

+    // MTMD_SUBCTX_GEN_PROCESS_TOKENIZE: reference clip -> codes and their talker input rows
+    ggml_cgraph * build_ref_encoder();
+    ggml_tensor * nearest_code(ggml_tensor * x, ggml_tensor * codebook) const;
+
+    // reference encoder transformer, see encoder_config in speech_tokenizer/config.json
+    static constexpr int   ref_enc_n_head     = 8;
+    static constexpr float ref_enc_eps        = 1e-5f;
+    static constexpr float ref_enc_rope_theta = 10000.0f;
+
     // which sub-graph build() constructs, fixed at graph-build time
     mtmd_subctx_gen_process gen_process;

@@ -483,6 +492,11 @@ struct mtmd_graph_pockettts_seanet : mtmd_graph {
     mtmd_graph_pockettts_seanet(const mtmd_graph & parent) : mtmd_graph(parent) {}
     ggml_cgraph * build() override { GGML_ABORT("call encode()/decode() instead"); }

+    // qwen3tts reference codes: the output is quantized, so the im2col is F32 (the kernels must be F32 too)
+    bool f32_im2col = false;
+    // qwen3tts reference codes: pad_replicate also repeats the last frame on the right, like MimiConv1d
+    bool pad_replicate_end = false;
+
     // per-call streaming state, keyed by slot name (see list_pockettts_state_slots)
     std::map<std::string, ggml_tensor *> state_in;
     mutable std::vector<std::pair<std::string, ggml_tensor *>> state_out;
@@ -501,11 +515,22 @@ struct mtmd_graph_pockettts_seanet : mtmd_graph {
 };

 // mimi encoder + speaker_proj: reference waveform -> voice conditioning rows
+// the qwen3tts reference encoder reuses encode() with its own transformer options
 struct mtmd_graph_pockettts_spkenc : mtmd_graph {
     mtmd_graph_pockettts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
+    mtmd_graph_pockettts_spkenc(const mtmd_graph & parent) : mtmd_graph(parent) {}
     ggml_cgraph * build() override;

+    int         tfm_n_head     = n_head;
+    float       tfm_eps        = eps;
+    float       tfm_rope_theta = hparams.rope_theta;
+    int         tfm_rope_type  = GGML_ROPE_TYPE_NORMAL;
+    ffn_op_type tfm_ffn_type   = FFN_GELU;
+
     ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const;
+
+    // waveform [n_samples, 1] -> [n_frames, C], at the frame rate after model.downsample_w
+    ggml_tensor * encode(const mtmd_graph_pockettts_seanet & seanet, ggml_tensor * x, const std::vector<mtmd_layer> & layers);
 };

 //
diff --git a/tools/mtmd/models/pockettts-seanet.cpp b/tools/mtmd/models/pockettts-seanet.cpp
index 4480aac9d..f1073c844 100644
--- a/tools/mtmd/models/pockettts-seanet.cpp
+++ b/tools/mtmd/models/pockettts-seanet.cpp
@@ -33,12 +33,26 @@ ggml_tensor * mtmd_graph_pockettts_seanet::conv1d(ggml_tensor * x, ggml_tensor *
         ggml_tensor * first = ggml_view_2d(ctx0, x, 1, x->ne[1], x->nb[1], 0);
         ggml_tensor * left  = ggml_repeat_4d(ctx0, first, p_total, x->ne[1], 1, 1);
         x = ggml_concat(ctx0, left, x, 0);
-        x = ggml_pad_ext(ctx0, x, 0, p_extra, 0, 0, 0, 0, 0, 0);
+        if (pad_replicate_end && p_extra > 0) {
+            ggml_tensor * last  = ggml_view_2d(ctx0, x, 1, x->ne[1], x->nb[1], (size_t) (x->ne[0] - 1) * x->nb[0]);
+            ggml_tensor * right = ggml_repeat_4d(ctx0, last, p_extra, x->ne[1], 1, 1);
+            x = ggml_concat(ctx0, x, right, 0);
+        } else {
+            x = ggml_pad_ext(ctx0, x, 0, p_extra, 0, 0, 0, 0, 0, 0);
+        }
     } else {
         x = ggml_pad_ext(ctx0, x, p_total, p_extra, 0, 0, 0, 0, 0, 0);
     }

-    ggml_tensor * y = ggml_conv_1d(ctx0, w, x, stride, 0, dilation);
+    ggml_tensor * y;
+    if (f32_im2col) {
+        // ggml_conv_1d() with an F32 im2col, the kernel must be F32
+        ggml_tensor * col = ggml_im2col(ctx0, w, x, stride, 0, 0, 0, dilation, 0, false, GGML_TYPE_F32);
+        y = ggml_mul_mat(ctx0, ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[2] * col->ne[1]),
+                         ggml_reshape_2d(ctx0, w, w->ne[0] * w->ne[1], w->ne[2]));
+    } else {
+        y = ggml_conv_1d(ctx0, w, x, stride, 0, dilation);
+    }
     y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[1]);
     if (b) {
         y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
diff --git a/tools/mtmd/models/pockettts-spkenc.cpp b/tools/mtmd/models/pockettts-spkenc.cpp
index f01727ad5..d64a0a35f 100644
--- a/tools/mtmd/models/pockettts-spkenc.cpp
+++ b/tools/mtmd/models/pockettts-spkenc.cpp
@@ -5,44 +5,41 @@

 // pre-norm block with layer scale on both residual paths, see mimi_transformer.py
 ggml_tensor * mtmd_graph_pockettts_spkenc::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const {
+    const int64_t n_pos      = cur->ne[1];
+    const int     tfm_d_head = (int) cur->ne[0] / tfm_n_head;
+
     ggml_tensor * inp = cur;

-    cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
+    cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, tfm_eps, il);

     ggml_tensor * Qcur = build_mm(layer.q_w, cur);
     ggml_tensor * Kcur = build_mm(layer.k_w, cur);
     ggml_tensor * Vcur = build_mm(layer.v_w, cur);

-    const int64_t n_pos = cur->ne[1];
-    Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
-    Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
-    Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
+    Qcur = ggml_reshape_3d(ctx0, Qcur, tfm_d_head, tfm_n_head, n_pos);
+    Kcur = ggml_reshape_3d(ctx0, Kcur, tfm_d_head, tfm_n_head, n_pos);
+    Vcur = ggml_reshape_3d(ctx0, Vcur, tfm_d_head, tfm_n_head, n_pos);

-    Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
-                         hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
-    Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
-                         hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+    Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, tfm_d_head, tfm_rope_type, 0,
+                         tfm_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+    Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, tfm_d_head, tfm_rope_type, 0,
+                         tfm_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);

-    cur = build_attn(layer.o_w, nullptr, Qcur, Kcur, Vcur, kq_mask, kq_scale, il);
+    cur = build_attn(layer.o_w, nullptr, Qcur, Kcur, Vcur, kq_mask, 1.0f / sqrtf((float) tfm_d_head), il);
     cur = ggml_mul(ctx0, cur, layer.ls_1_w);
     cur = ggml_add(ctx0, cur, inp);

     inp = cur;
-    cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
-    cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, FFN_GELU, il);
+    cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, tfm_eps, il);
+    cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, tfm_ffn_type, il);
     cur = ggml_mul(ctx0, cur, layer.ls_2_w);
     cur = ggml_add(ctx0, cur, inp);

     return cur;
 }

-ggml_cgraph * mtmd_graph_pockettts_spkenc::build() {
-    // the preprocessor hands over the waveform as a single-row "mel", already [n_samples, 1]
-    ggml_tensor * inp_raw = build_inp_raw(1);
-    ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]);
-
-    mtmd_graph_pockettts_seanet seanet(*this);
-    cur = seanet.encode(cur);
+ggml_tensor * mtmd_graph_pockettts_spkenc::encode(const mtmd_graph_pockettts_seanet & seanet, ggml_tensor * x, const std::vector<mtmd_layer> & layers) {
+    ggml_tensor * cur = seanet.encode(x);
     cb(cur, "mimi_enc", -1);

     // [T, 512] -> transformer works on [512, T]
@@ -55,16 +52,27 @@ ggml_cgraph * mtmd_graph_pockettts_spkenc::build() {
     // the mimi transformer is causal with a sliding window, see _build_attention_mask()
     ggml_tensor * kq_mask = build_inp_attn_mask(cur->ne[1], cur->ne[1]);

-    for (int il = 0; il < n_layer; il++) {
-        cur = tfm_layer_forward(cur, model.layers[il], inp_pos, kq_mask, il);
+    for (size_t il = 0; il < layers.size(); il++) {
+        cur = tfm_layer_forward(cur, layers[il], inp_pos, kq_mask, (int) il);
     }
     cb(cur, "mimi_enc_tfm", -1);

-    // downsample to the model frame rate, [512, T] -> [T, 512] -> [T / 16, 32]
+    // downsample to the model frame rate, [512, T] -> [T, 512] -> [T / mimi_downsample, C]
     cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
     cur = seanet.conv1d(cur, model.downsample_w, nullptr, hparams.mimi_downsample, 1, true);
     cb(cur, "mimi_downsample", -1);

+    return cur;
+}
+
+ggml_cgraph * mtmd_graph_pockettts_spkenc::build() {
+    // the preprocessor hands over the waveform as a single-row "mel", already [n_samples, 1]
+    ggml_tensor * inp_raw = build_inp_raw(1);
+    ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]);
+
+    mtmd_graph_pockettts_seanet seanet(*this);
+    cur = encode(seanet, cur, model.layers);
+
     // voice latent -> backbone embd
     cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
     cur = build_mm(model.spk_proj_w, cur);
diff --git a/tools/mtmd/models/qwen3tts-gen.cpp b/tools/mtmd/models/qwen3tts-gen.cpp
index 91d5a5c12..386e22db7 100644
--- a/tools/mtmd/models/qwen3tts-gen.cpp
+++ b/tools/mtmd/models/qwen3tts-gen.cpp
@@ -664,9 +664,77 @@ std::vector<c2w_state_slot> list_c2w_state_slots(const mtmd_hparams & hparams, c
     return slots;
 }

+// x: [dim, T], codebook: [dim, n_entries] -> [T] index of the nearest entry
+ggml_tensor * mtmd_graph_qwen3tts_gen::nearest_code(ggml_tensor * x, ggml_tensor * codebook) const {
+    // argmin |x - e|^2 = argmax 2 x.e - |e|^2
+    ggml_tensor * dots  = ggml_mul_mat(ctx0, codebook, x);                // [n_entries, T]
+    ggml_tensor * norms = ggml_sum_rows(ctx0, ggml_sqr(ctx0, codebook));  // [1, n_entries]
+    norms = ggml_reshape_2d(ctx0, norms, norms->ne[1], 1);                // [n_entries, 1]
+    return ggml_argmax(ctx0, ggml_sub(ctx0, ggml_scale(ctx0, dots, 2.0f), norms));
+}
+
+// the mimi encoder of the speech tokenizer, then its RVQ: one semantic codebook and n_acoustic acoustic ones
+ggml_cgraph * mtmd_graph_qwen3tts_gen::build_ref_encoder() {
+    // the waveform comes in as a single-row "mel", [n_samples, 1]
+    ggml_tensor * inp_raw = build_inp_raw(1);
+    ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]);
+
+    mtmd_graph_pockettts_seanet seanet(*this);
+    seanet.f32_im2col        = true;
+    seanet.pad_replicate_end = true;
+
+    // the checkpoint is in the transformers layout, so rope rotates halves (neox)
+    mtmd_graph_pockettts_spkenc mimi(*this);
+    mimi.tfm_n_head     = ref_enc_n_head;
+    mimi.tfm_eps        = ref_enc_eps;
+    mimi.tfm_rope_theta = ref_enc_rope_theta;
+    mimi.tfm_rope_type  = GGML_ROPE_TYPE_NEOX;
+    mimi.tfm_ffn_type   = FFN_GELU_ERF;
+
+    cur = mimi.encode(seanet, cur, model.ref_enc_layers);
+    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [512, T]
+
+    std::vector<ggml_tensor *> codes;
+    codes.push_back(nearest_code(build_mm(model.ref_quant_first_in_w, cur), model.ref_quant_first_cb));
+
+    const int64_t n_acoustic = model.ref_quant_rest_cb->ne[2];
+    ggml_tensor * residual = build_mm(model.ref_quant_rest_in_w, cur);
+    for (int64_t i = 0; i < n_acoustic; i++) {
+        ggml_tensor * codebook = ggml_view_2d(ctx0, model.ref_quant_rest_cb, model.ref_quant_rest_cb->ne[0], model.ref_quant_rest_cb->ne[1],
+                                              model.ref_quant_rest_cb->nb[1], (size_t) i * model.ref_quant_rest_cb->nb[2]);
+        ggml_tensor * code = nearest_code(residual, codebook);
+        codes.push_back(code);
+        if (i + 1 < n_acoustic) {
+            residual = ggml_sub(ctx0, residual, ggml_get_rows(ctx0, codebook, code));
+        }
+    }
+
+    // talker input rows of the reference frames: codebook 0 from the talker table, the rest from the code predictor's
+    ggml_tensor * rows = ggml_get_rows(ctx0, model.gen_code_out_embd_w, codes[0]);
+    for (int64_t g = 1; g <= n_acoustic; g++) {
+        ggml_tensor * embd = ggml_view_2d(ctx0, model.gen_code_embd_w, model.gen_code_embd_w->ne[0], model.gen_code_embd_w->ne[1],
+                                          model.gen_code_embd_w->nb[1], (size_t) (g - 1) * model.gen_code_embd_w->nb[2]);
+        rows = ggml_add(ctx0, rows, ggml_get_rows(ctx0, embd, codes[(size_t) g]));
+    }
+
+    for (size_t g = 0; g < codes.size(); g++) {
+        ggml_set_name(codes[g], ("out_ref_code_" + std::to_string(g)).c_str());
+        ggml_set_output(codes[g]);
+        ggml_build_forward_expand(gf, codes[g]);
+    }
+    ggml_set_name(rows, "out_ref_rows");
+    ggml_set_output(rows);
+    ggml_build_forward_expand(gf, rows);
+    return gf;
+}
+
 // both sub-graphs are always built, so the topology stays constant
 // ggml_build_forward_select() then picks the one that actually runs
 ggml_cgraph * mtmd_graph_qwen3tts_gen::build() {
+    if (gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
+        return build_ref_encoder();
+    }
+
     GGML_ASSERT(n_batch == 1); // this module only ever processes one frame at a time

     int idx;
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 6a83ece40..cac18b345 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -201,6 +201,7 @@ public:
         seq_id = 0;
         pos = 0;
         codes_buf.clear();
+        next_window_frames = window_frames;
         gen_state.reset();
         audio_pcm.clear();
         overlay.clear();
@@ -232,12 +233,21 @@ public:

         std::vector<float> speaker_embd;
         if (inp->speaker_ref) {
+            mtmd_bitmap_set_transcript(inp->speaker_ref, inp->speaker_ref_text);
             if (!encode_speaker(inp->speaker_ref, speaker_embd)) {
                 return 1;
             }
         }

         const int n_e = n_embd;
+
+        // cloning with the reference transcript continues the reference speech, see generate_icl_prompt() upstream
+        const bool   has_ref_text = inp->speaker_ref_text && inp->speaker_ref_text[0];
+        const size_t n_ref_frames = speaker_embd.empty() ? 0 : speaker_embd.size() / (size_t) n_e - 1;
+        const bool   icl          = has_ref_text && n_ref_frames > 0;
+        if (has_ref_text && !icl) {
+            LOG_WRN("mtmd_helper_gen_audio: no reference codes (mmproj without reference encoder, or clip too long), cloning from the speaker embedding only\n");
+        }
         auto row = [&](llama_token t) {
             return std::vector<float>(tok_embd.begin() + (size_t) t * n_e,
                                        tok_embd.begin() + (size_t) (t + 1) * n_e);
@@ -256,14 +266,31 @@ public:
         // upstream chat wrap, then slices: [0:3] role, [3:-5] utterance body
         const std::string full = "<|im_start|>assistant\n" + std::string(inp->prompt, inp->prompt_len) +
                                   "<|im_end|>\n<|im_start|>assistant\n";
-        std::vector<llama_token> ids(full.size() + 16);
-        int n_ids = llama_tokenize(vocab, full.c_str(), (int32_t) full.size(), ids.data(), (int32_t) ids.size(),
-                                   false, true);
+        auto tokenize = [&](const std::string & text) {
+            std::vector<llama_token> tokens(text.size() + 16);
+            const int n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), tokens.data(), (int32_t) tokens.size(),
+                                         false, true);
+            tokens.resize(n < 0 ? 0 : (size_t) n);
+            return tokens;
+        };
+
+        std::vector<llama_token> ids = tokenize(full);
+        const int n_ids = (int) ids.size();
         if (n_ids < 8) {
             LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
             return 1;
         }
-        ids.resize((size_t) n_ids);
+
+        // the reference transcript goes before the text, with the same chat wrap minus the next turn: slices [3:-2]
+        std::vector<llama_token> ref_ids;
+        if (icl) {
+            ref_ids = tokenize("<|im_start|>assistant\n" + std::string(inp->speaker_ref_text) + "<|im_end|>\n");
+            if (ref_ids.size() < 6) {
+                LOG_ERR("mtmd_helper_gen_audio: tokenization of the reference text failed\n");
+                return 1;
+            }
+            ref_ids = std::vector<llama_token>(ref_ids.begin() + 3, ref_ids.end() - 2);
+        }

         std::vector<std::vector<float>> prompt;
         for (int i = 0; i < 3; i++) prompt.push_back(row(ids[(size_t) i]));
@@ -273,9 +300,17 @@ public:
         prompt.push_back(sum_row(tts_pad, c_think_e));
         if (!speaker_embd.empty()) prompt.push_back(sum_vec(tts_pad, speaker_embd));
         prompt.push_back(sum_row(tts_bos, codec_pad));
+        for (llama_token t : ref_ids) prompt.push_back(sum_row(t, codec_pad));
         for (int i = 3; i < n_ids - 5; i++) prompt.push_back(sum_row(ids[(size_t) i], codec_pad));
         prompt.push_back(sum_row(tts_eos, codec_pad));
         prompt.push_back(sum_row(tts_pad, codec_bos));
+        // the reference frames, as if the talker had just spoken them
+        if (icl) {
+            for (size_t f = 0; f < n_ref_frames; f++) {
+                const float * frame = speaker_embd.data() + (f + 1) * (size_t) n_e;
+                prompt.push_back(sum_vec(tts_pad, std::vector<float>(frame, frame + n_e)));
+            }
+        }

         n_prompt = (int) prompt.size();

@@ -303,6 +338,13 @@ public:
         if (!init_gen_state(inp->seed)) {
             return 1;
         }
+        // the decoder continues from the reference codes of the encode above
+        if (icl) {
+            if (mtmd_gen_state_set_ref(mctx, gen_state.get()) != 0) {
+                return 1;
+            }
+            next_window_frames = window_frames - std::min(n_ref_frames, window_frames - 1);
+        }

         // the prompt above holds the whole text stream up to tts_eos, so every generated
         // frame adds tts_pad on top of the codes embedding
@@ -366,7 +408,7 @@ public:
         }

         codes_buf.insert(codes_buf.end(), out.codes, out.codes + out.n_codes);
-        if (out.n_codes > 0 && codes_buf.size() / out.n_codes >= window_frames) {
+        if (out.n_codes > 0 && codes_buf.size() / out.n_codes >= next_window_frames) {
             if (!flush_gen_wav()) {
                 return 1;
             }
@@ -502,6 +544,7 @@ private:
         }
         audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
         codes_buf.clear();
+        next_window_frames = window_frames;
         return true;
     }

@@ -534,6 +577,7 @@ private:
     int32_t  top_k = 50;
     float    top_p = 1.0f;
     std::vector<int32_t> codes_buf;
+    size_t next_window_frames = window_frames; // the first window leaves room for the reference codes
     std::vector<float>   audio_pcm;
     std::vector<float> overlay;
     std::vector<float> h_state_buf;
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index 20465cbd3..3fb75e5a3 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -310,7 +310,7 @@ struct mtmd_helper_gen_audio_inp {
     enum mtmd_helper_gen_audio_out_mode out_mode; // n_past > 0 only (default: auto)

     mtmd_bitmap * speaker_ref; // optional, can be NULL: audio to clone, or a voice bitmap
-    const char * speaker_ref_text; // optional, can be NULL: transcript of speaker_ref, for models that put it in the prompt (e.g. Fish S2)
+    const char * speaker_ref_text; // optional, can be NULL: transcript of speaker_ref, for models that use it to clone the voice (e.g. Fish S2, Qwen3-TTS)
     const char * lang; // optional, can be NULL

     int32_t  top_k;
diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h
index 37ba390f9..e1dc95cee 100644
--- a/tools/mtmd/mtmd-impl.h
+++ b/tools/mtmd/mtmd-impl.h
@@ -252,6 +252,12 @@
 #define TN_A_ASP_ATTN  "a.asp_attn.%s"
 #define TN_A_ASP_TDNN  "a.asp_tdnn.%s"

+// qwen3tts reference encoder, the transformer blocks use the generic names with the "a.gen.enc.tfm" prefix
+#define TN_A_GEN_ENC_QUANT_FIRST_IN  "a.gen.enc.quant.first.in_proj.%s"
+#define TN_A_GEN_ENC_QUANT_FIRST_CB  "a.gen.enc.quant.first.codebook.%s"
+#define TN_A_GEN_ENC_QUANT_REST_IN   "a.gen.enc.quant.rest.in_proj.%s"
+#define TN_A_GEN_ENC_QUANT_REST_CB   "a.gen.enc.quant.rest.codebook.%s"
+
 // qwen3tts code_predictor
 #define TN_A_GEN_CODE_PROJ_IN  "a.gen.code.proj_in.%s"
 #define TN_A_GEN_CODE_EMBD     "a.gen.code.embd.%s"
diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp
index 29531db2e..c352e28d9 100644
--- a/tools/mtmd/mtmd-model.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -1075,6 +1075,14 @@ struct mtmd_model_loader {
                         // TODO: hardcoded for now, read from code_predictor_config instead
                         hparams.rope_theta = 1000000.0f;

+                        // reference encoder: mimi with the stride-2 downsample of the 12.5 Hz tokenizer
+                        // seanet ratios are [8,6,5,4] in the config, the encoder reverses them
+                        hparams.audio_sample_rate = 24000;
+                        hparams.seanet_ratios     = { 4, 5, 6, 8 };
+                        hparams.seanet_n_stage    = (int32_t) hparams.seanet_ratios.size();
+                        hparams.mimi_downsample   = 2;
+                        hparams.mimi_tfm_context  = 250;
+
                         // code2wav params
                         hparams.wav_tfm_n_layer      = 8;
                         hparams.wav_tfm_n_embd       = 512;
@@ -2826,6 +2834,47 @@ struct mtmd_model_loader {

                     // code2wav: RVQ codes -> raw PCM, lives in the same ctx as code_predictor
                     load_code2wav(true);
+
+                    // reference encoder, absent from files converted before it was added
+                    model.ref_quant_rest_cb = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_REST_CB, "weight"), false);
+                    if (model.ref_quant_rest_cb) {
+                        load_seanet(model.seanet, false);
+                        model.downsample_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight"));
+                        for (int il = 0; ; il++) {
+                            const char * p = "a.gen.enc.tfm";
+                            ggml_tensor * ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight"), false);
+                            if (!ln_1_w) {
+                                break;
+                            }
+                            mtmd_layer layer;
+                            layer.ln_1_w    = ln_1_w;
+                            layer.ln_1_b    = get_tensor(mtmd_format(TN_LN_1,        p, il, "bias"));
+                            layer.q_w       = get_tensor(mtmd_format(TN_ATTN_Q,      p, il, "weight"));
+                            layer.k_w       = get_tensor(mtmd_format(TN_ATTN_K,      p, il, "weight"));
+                            layer.v_w       = get_tensor(mtmd_format(TN_ATTN_V,      p, il, "weight"));
+                            layer.o_w       = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight"));
+                            layer.ls_1_w    = get_tensor(mtmd_format(TN_LS_1,        p, il, "weight"));
+                            layer.ln_2_w    = get_tensor(mtmd_format(TN_LN_2,        p, il, "weight"));
+                            layer.ln_2_b    = get_tensor(mtmd_format(TN_LN_2,        p, il, "bias"));
+                            layer.ff_up_w   = get_tensor(mtmd_format(TN_FFN_UP,      p, il, "weight"));
+                            layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN,    p, il, "weight"));
+                            layer.ls_2_w    = get_tensor(mtmd_format(TN_LS_2,        p, il, "weight"));
+                            model.ref_enc_layers.push_back(layer);
+                        }
+                        model.ref_quant_first_in_w = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_FIRST_IN, "weight"));
+                        model.ref_quant_first_cb   = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_FIRST_CB, "weight"));
+                        model.ref_quant_rest_in_w  = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_REST_IN,  "weight"));
+
+                        // the codes index the code predictor tables, see build_ref_encoder()
+                        const int64_t n_acoustic = model.ref_quant_rest_cb->ne[2];
+                        if (model.ref_enc_layers.empty() ||
+                                n_acoustic != model.gen_code_head_w->ne[2] ||
+                                n_acoustic > model.gen_code_embd_w->ne[2] ||
+                                model.ref_quant_first_cb->ne[1] > model.gen_code_out_embd_w->ne[1] ||
+                                model.ref_quant_rest_cb->ne[1]  > model.gen_code_embd_w->ne[1]) {
+                            throw std::runtime_error(mtmd_format("%s: qwen3tts: unexpected reference encoder shape\n", __func__));
+                        }
+                    }
                 } break;
             case PROJECTOR_TYPE_ZONOS3_GEN:
                 {
diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h
index 1929f9de1..636bde14a 100644
--- a/tools/mtmd/mtmd-model.h
+++ b/tools/mtmd/mtmd-model.h
@@ -213,6 +213,8 @@ struct mtmd_hparams {

     // pocket-tts
     static constexpr int32_t pockettts_max_spk_seconds = 30;
+    // qwen3tts: the reference encoder mask is dense, so cost is quadratic in the clip
+    static constexpr int32_t qwen3tts_max_ref_seconds = 30;
     // fish_s2: the codec encoder runs at 44.1 kHz, memory grows with the clip
     static constexpr int32_t fish_s2_max_spk_seconds = 30;
     int32_t seanet_n_stage    = 0;
@@ -1300,6 +1302,17 @@ struct mtmd_model {
     ggml_tensor * gen_code_head_norm_w = nullptr; // per-codebook norm before the tied head, merged 2D
     ggml_tensor * gen_code_mask_embd   = nullptr;

+    // qwen3tts reference encoder: reference clip -> RVQ codes, uses seanet and downsample_w too
+    std::vector<mtmd_layer> ref_enc_layers;
+    ggml_tensor * ref_quant_first_in_w = nullptr;
+    ggml_tensor * ref_quant_first_cb   = nullptr;
+    ggml_tensor * ref_quant_rest_in_w  = nullptr;
+    ggml_tensor * ref_quant_rest_cb    = nullptr; // acoustic codebooks, merged 3D
+
+    bool has_ref_encoder() const {
+        return ref_quant_rest_cb != nullptr;
+    }
+
     // qwen3tts code2wav: RVQ codes -> raw PCM
     mtmd_code2wav c2w;
     std::vector<mtmd_drama3_layer> drama3_layers;
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index 321b6171b..d6ec8e862 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -675,9 +675,9 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
         set_input_attn_mask(mask);
     };

-    // rope positions and attention mask of the mimi transformers (pocket-tts).
+    // rope positions and attention mask of the mimi encoder transformer (pocket-tts, qwen3tts reference encoder).
     // the mask is causal with a sliding window, see _build_attention_mask() in the reference
-    auto set_pockettts_tfm_inputs = [&]() {
+    auto set_mimi_tfm_inputs = [&](int64_t max_samples) {
         const int64_t n_pos = ggml_nelements(get_inp_tensor("inp_pos"));
         GGML_ASSERT(n_pos > 0);
         std::vector<int32_t> positions((size_t) n_pos);
@@ -686,9 +686,13 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
         }
         set_input_i32("inp_pos", positions);

-        // the preprocessor truncates the waveform to keep this mask bounded
-        const int64_t max_pos = (int64_t) mtmd_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate / 120;
-        GGML_ASSERT(n_pos <= max_pos && "pocket-tts speaker reference too long for a dense mask");
+        // the caller truncates the waveform to keep this mask bounded
+        int64_t hop = 1;
+        for (int32_t ratio : hparams.seanet_ratios) {
+            hop *= ratio;
+        }
+        const int64_t max_pos = (max_samples + hop - 1) / hop;
+        GGML_ASSERT(n_pos <= max_pos && "speaker reference too long for a dense mask");

         const int64_t context = hparams.mimi_tfm_context;
         std::vector<float> mask((size_t) n_pos * n_pos, -INFINITY);
@@ -1483,7 +1487,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
             } break;
         case PROJECTOR_TYPE_POCKETTTS_SPKENC:
             {
-                set_pockettts_tfm_inputs();
+                set_mimi_tfm_inputs((int64_t) mtmd_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate);
             } break;
         case PROJECTOR_TYPE_POCKETTTS_GEN:
             {
@@ -1887,6 +1891,10 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
             } break;
         case PROJECTOR_TYPE_QWEN3TTS_GEN:
             {
+                if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
+                    set_mimi_tfm_inputs((int64_t) mtmd_hparams::qwen3tts_max_ref_seconds * hparams.audio_sample_rate);
+                    break;
+                }
                 if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
                     GGML_ASSERT(params->codes != nullptr);

@@ -2423,6 +2431,35 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
     // for audio gen models
     //

+    if (model.proj_type == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
+        GGML_ASSERT(params->out_codes && params->out_code_embd);
+
+        // frame-major codes, the layout of GEN_WAV's input
+        std::vector<std::vector<int32_t>> groups;
+        for (int g = 0; ; g++) {
+            ggml_tensor * t = ggml_graph_get_tensor(gf, ("out_ref_code_" + std::to_string(g)).c_str());
+            if (!t) {
+                break;
+            }
+            groups.emplace_back((size_t) ggml_nelements(t));
+            ggml_backend_tensor_get(t, groups.back().data(), 0, ggml_nbytes(t));
+        }
+        GGML_ASSERT(!groups.empty());
+        const size_t n_frames = groups[0].size();
+        params->out_codes->resize(n_frames * groups.size());
+        for (size_t f = 0; f < n_frames; f++) {
+            for (size_t g = 0; g < groups.size(); g++) {
+                (*params->out_codes)[f * groups.size() + g] = groups[g][f];
+            }
+        }
+
+        ggml_tensor * rows = ggml_graph_get_tensor(gf, "out_ref_rows");
+        GGML_ASSERT(rows && rows->type == GGML_TYPE_F32 && (size_t) rows->ne[1] == n_frames);
+        params->out_code_embd->resize((size_t) ggml_nelements(rows));
+        ggml_backend_tensor_get(rows, params->out_code_embd->data(), 0, ggml_nbytes(rows));
+        return true;
+    }
+
     if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
         ggml_tensor * fsq = ggml_graph_get_tensor(gf, "out_fsq");
         GGML_ASSERT(fsq != nullptr && params->out_codes);
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 7821cdfbd..7c22a1041 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -204,6 +204,7 @@ struct mtmd_bitmap {
     std::shared_ptr<const mtmd_gen_ref> gen_ref; // gen-side reference of a predefined voice, if any
     bool mergeable = false; // [QWEN_VIDEO] set only on frames of the same video
     bool is_video  = false; // see mtmd_bitmap_set_is_video()
+    std::string transcript; // see mtmd_bitmap_set_transcript()

     // lazy-loaded bitmap
     mtmd_bitmap_lazy_callback lazy_callback = nullptr;
@@ -2056,6 +2057,12 @@ struct mtmd_tokenizer {
             // sanity check
             GGML_ASSERT(ctx->audio_preproc != nullptr);

+            // qwen3tts: a clip with a transcript also gives the reference codes, see mtmd_qwen3tts_ref_encode()
+            mtmd_image_f32 ref_wav;
+            bool with_ref_codes = ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_QWEN3TTS_SPKENC &&
+                                  ctx->ctx_gen_a && ctx->ctx_gen_a->model.has_ref_encoder() &&
+                                  !bitmap->transcript.empty();
+
             // preprocess audio
             auto source = std::make_shared<mtmd_audio_preproc_out>();
             {
@@ -2081,6 +2088,19 @@ struct mtmd_tokenizer {
                     LOG_ERR("%s: this model takes audio input only via mtmd_audio_stream\n", __func__);
                     return 2;
                 }
+                if (with_ref_codes) {
+                    // a cut clip no longer matches its transcript, so a long clip gives no codes at all
+                    const size_t sample_rate = (size_t) ctx->ctx_gen_a->model.hparams.audio_sample_rate;
+                    const size_t max_samples = (size_t) mtmd_hparams::qwen3tts_max_ref_seconds * sample_rate;
+                    if (n_samples > max_samples) {
+                        LOG_WRN("%s: speaker reference is %.1f s, longer than %d s, the reference codes are skipped\n", __func__,
+                                (double) n_samples / sample_rate, mtmd_hparams::qwen3tts_max_ref_seconds);
+                        with_ref_codes = false;
+                    } else {
+                        ref_wav.set_size({(int) n_samples, 1}, false, true);
+                        ref_wav.cpy_buf(std::vector<float>(samples, samples + n_samples));
+                    }
+                }
                 *source = ctx->audio_preproc->preprocess(samples, n_samples);
                 if (source->entries.empty()) {
                     LOG_ERR("Unable to preprocess audio\n");
@@ -2123,6 +2143,17 @@ struct mtmd_tokenizer {
                 batch_f32.is_audio = true;
                 batch_f32.entries.push_back(std::move(mel_f32));

+                // the reference waveform is a second entry, its frames follow the speaker row
+                if (with_ref_codes && i == 0) {
+                    const auto & gen_hparams = ctx->ctx_gen_a->model.hparams;
+                    int64_t samples_per_frame = gen_hparams.mimi_downsample;
+                    for (int32_t ratio : gen_hparams.seanet_ratios) {
+                        samples_per_frame *= ratio;
+                    }
+                    n_tokens += (ref_wav.nx() + samples_per_frame - 1) / samples_per_frame;
+                    batch_f32.entries.push_back(std::move(ref_wav));
+                }
+
                 mtmd_audio_tokens_ptr audio_tokens(new mtmd_audio_tokens);
                 audio_tokens->n_tokens = n_tokens;
                 audio_tokens->batch_f32 = std::move(batch_f32);
@@ -2470,6 +2501,51 @@ static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_bat
     return ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd);
 }

+// qwen3tts: the speaker row, then one talker input row per reference frame
+// the reference codes go to ctx->gen_ref, so GEN_WAV continues from them, see mtmd_gen_state_set_ref()
+static int32_t mtmd_qwen3tts_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd) {
+    const size_t n_embd  = ctx->n_embd_out();
+    const auto & entries = audio_tokens->batch_f32.entries;
+    GGML_ASSERT(entries.size() == 2);
+
+    mtmd_image_f32_batch mel;
+    mel.is_audio = true;
+    mel.entries.push_back(entries[0]);
+
+    std::vector<float> speaker(n_embd);
+    if (!mtmd_audio_batch_encode(ctx, mel, audio_tokens->source.get(), audio_tokens->source_idx, speaker)) {
+        return 1;
+    }
+
+    mtmd_image_f32_batch wav;
+    wav.is_audio = true;
+    wav.entries.push_back(entries[1]);
+
+    std::vector<int32_t> codes;
+    std::vector<float>   rows;
+
+    mtmd_subctx_encode_params params;
+    params.imgs          = &wav;
+    params.n_threads     = ctx->n_threads;
+    params.gen_process   = MTMD_SUBCTX_GEN_PROCESS_TOKENIZE;
+    params.out_codes     = &codes;
+    params.out_code_embd = &rows;
+
+    if (!ctx->ctx_gen_a->encode(&params)) {
+        LOG_ERR("%s: encode failed (reference codes)\n", __func__);
+        return 1;
+    }
+    if (rows.size() != (size_t) (audio_tokens->n_tokens - 1) * n_embd) {
+        LOG_ERR("%s: %zu reference rows, expected %u\n", __func__, rows.size() / n_embd, audio_tokens->n_tokens - 1);
+        return 1;
+    }
+
+    out_embd = std::move(speaker);
+    out_embd.insert(out_embd.end(), rows.begin(), rows.end());
+    ctx->gen_ref.tokens = std::move(codes);
+    return 0;
+}
+
 static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk * chunk, std::vector<float> & out_embd) {
     ctx->gen_ref = {}; // see mtmd_gen_state_set_ref()
     if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
@@ -2524,6 +2600,9 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk
         if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_CHATTERBOX_SPKENC) {
             return cbx_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
         }
+        if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_QWEN3TTS_SPKENC && chunk->tokens_audio->batch_f32.entries.size() == 2) {
+            return mtmd_qwen3tts_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
+        }
         out_embd.resize((size_t)chunk->tokens_audio->n_tokens * ctx->n_embd_out());
         bool ok = mtmd_audio_batch_encode(ctx, chunk->tokens_audio->batch_f32, chunk->tokens_audio->source.get(), chunk->tokens_audio->source_idx, out_embd);
         return ok ? 0 : 1;
@@ -3079,7 +3158,9 @@ struct mtmd_gen_state {
     std::vector<float>   out_audio; // decoded PCM samples for the current frame (GEN_WAV)

     // voice reference for GEN_WAV, from a speaker clip or a predefined voice (chatterbox)
+    // qwen3tts: the reference codes, decoded with the first GEN_WAV call so the speech continues from them
     mtmd_gen_ref ref;
+    bool ref_decoded = false;

     // lfm3a: the codec graph yields STFT frames, the ISTFT runs on the host
     std::unique_ptr<mtmd_audio_streaming_istft> istft;
@@ -3126,6 +3207,7 @@ void mtmd_gen_state_free(mtmd_gen_state * state) {

 void mtmd_gen_state_reset(mtmd_gen_state * state) {
     state->gen_state->reset();
+    state->ref_decoded = false;
     if (state->istft) {
         const auto * hparams = &state->sctx->model.hparams;
         state->istft = std::make_unique<mtmd_audio_streaming_istft>(hparams->audio_n_fft, hparams->audio_hop_len);
@@ -3622,6 +3704,7 @@ int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) {
         return 1;
     }
     state->ref = ctx->gen_ref;
+    state->ref_decoded = false;
     return 0;
 }

@@ -3784,16 +3867,37 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
     params.imgs        = &batch;
     params.n_threads   = ctx->n_threads;
     params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV;
-    params.codes       = has_codes ? &in_codes : nullptr;
-    params.feats       = has_feats ? &in_feats : nullptr;
     params.out_audio   = &state->out_audio;
     params.gen_state   = state->gen_state;

+    // qwen3tts: the first call puts the end of the reference before its codes, so the speech continues from it
+    // a batch shorter than the window carries a wrong state, so the reference only fills the free part of this window
+    size_t n_ref_used = 0;
+    size_t n_frames   = 0;
+    if (sctx->model.proj_type == PROJECTOR_TYPE_QWEN3TTS_GEN && has_codes) {
+        const size_t n_codes = (size_t) sctx->model.gen_code_head_w->ne[2] + 1;
+        if (!state->ref.tokens.empty() && !state->ref_decoded) {
+            state->ref_decoded = true;
+            const size_t n_win = (size_t) sctx->model.hparams.wav_tfm_swa;
+            const size_t n_in  = in_codes.size() / n_codes;
+            n_ref_used = std::min(state->ref.tokens.size() / n_codes, n_win > n_in ? n_win - n_in : 0);
+            in_codes.insert(in_codes.begin(), state->ref.tokens.end() - (std::ptrdiff_t) (n_ref_used * n_codes), state->ref.tokens.end());
+        }
+        n_frames = in_codes.size() / n_codes;
+    }
+
+    params.codes = has_codes ? &in_codes : nullptr;
+    params.feats = has_feats ? &in_feats : nullptr;
     if (!sctx->encode(&params)) {
         LOG_ERR("%s: encode failed (code2wav)\n", __func__);
         return 1;
     }

+    if (n_ref_used > 0) {
+        const size_t hop = state->out_audio.size() / n_frames;
+        state->out_audio.erase(state->out_audio.begin(), state->out_audio.begin() + (std::ptrdiff_t) (n_ref_used * hop));
+    }
+
     out->audio      = state->out_audio.data();
     out->n_samples  = state->out_audio.size();

@@ -4103,6 +4207,14 @@ void mtmd_bitmap_set_is_video(mtmd_bitmap * bitmap, bool is_video) {
     bitmap->is_video = is_video;
 }

+void mtmd_bitmap_set_transcript(mtmd_bitmap * bitmap, const char * transcript) {
+    if (transcript) {
+        bitmap->transcript = std::string(transcript);
+    } else {
+        bitmap->transcript.clear();
+    }
+}
+
 mtmd_bitmap * mtmd_bitmap_init_lazy(const mtmd_context * ctx,
                                     const char * id,
                                     void * user_data,
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index 3829b6078..8d5b42a9f 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -218,6 +218,8 @@ MTMD_API void         mtmd_bitmap_set_mergeable(mtmd_bitmap * bitmap, bool merge
 // on a lazy bitmap: the expanded chunks are one video, wrapped by the model's video begin/end tokens
 //                   with MTMD_VIDEO_TEMPLATE_LFM3_OMNI, its audio bitmaps are the pieces of one soundtrack, in order
 MTMD_API void         mtmd_bitmap_set_is_video(mtmd_bitmap * bitmap, bool is_video);
+// transcript of an audio bitmap is optional, some TTS models use it to clone the speaker voice more closely (--tts-speaker-text)
+MTMD_API void         mtmd_bitmap_set_transcript(mtmd_bitmap * bitmap, const char * transcript);

 // mtmd_bitmap lazy
 //
diff --git a/tools/server/README.md b/tools/server/README.md
index c8d25199f..6dddbc81b 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -866,7 +866,7 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For the list

 `speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence. Rejected when the model has no voice encoder (e.g. Pocket TTS without voice cloning).

-`speaker_ref_text`: Transcript of the reference audio, put in the prompt by some models (e.g. Fish Audio S2 Pro). Optional.
+`speaker_ref_text`: Transcript of the reference audio, put in the prompt by some models (e.g. Fish Audio S2 Pro, Qwen3-TTS). Optional.

 `top_k`, `top_p`: Sampling params for both the backbone sampler and the acoustic code predictor. Optional, `0` or absent keeps the server `--top-k`/`--top-p` for the backbone and the model defaults for the code predictor.