Commit 34ff2e047 for llama.cpp

commit 34ff2e047ce520b1d83bd274bf3ff0bfbc72a69b
Author: Feiteng <lifeiteng0422@gmail.com>
Date:   Sat Nov 21 01:34:52 2026 +0800

    mtmd: drama3 voice cloning (#32586)

    * mtmd: drama3 voice cloning

    The Fish S2 codec encoder (fish_s2_spkenc) now also serves drama3:
    - conversion: drama3 writes the codec encoder and the audio encoder KV, shared with fish_s2
    - fish_s2_spkenc: outputs the codes of the clip (gen-side reference) and, in a drama3_gen mmproj,
      builds the drama3 backbone rows (E_lm[semantic] + E_cb[1][code 1]) / sqrt(2)
    - drama3_gen tower: teacher forcing when the caller gives the codes of the next frame
    - drama3 pipeline: reference-bracket-v1 prompt with one reference (vLLM-Omni fish_omni), the
      reference frames go through the backbone as codec rows and through the tower with their codes
    - mtmd_gen_state_get_ref_codes(): read-only access to the reference codes of a gen state

    Claude-Session: https://claude.ai/code/session_01PpX3DmaxNSmJbxGaxnUBrX

    * fix correctness

    * clean up

    * rm redundant code

    * fish_s2: rm unused import

    ---------

    Co-authored-by: Xuan Son Nguyen <son@huggingface.co>

diff --git a/conversion/drama3.py b/conversion/drama3.py
index 68bbb556b..9b490900a 100644
--- a/conversion/drama3.py
+++ b/conversion/drama3.py
@@ -42,6 +42,115 @@ def fold_snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
     return alpha, 1.0 / (alpha + 1e-9)


+# codec encoder strides (encoder_rates of the codec config), the quantizer then downsamples 2x2
+_CODEC_ENC_STRIDES = (2, 4, 8, 8)
+
+# codec encoder transformers, both 1024 wide with 16 heads of 64
+# generic audio layers a.blk.0..3: last encoder block, a.blk.4..11: quantizer pre_module
+_N_ENC_TFM_LAYERS = 4
+_N_PRE_TFM_LAYERS = 8
+_CODEC_HEAD_DIM = 64
+
+
+def _neox_rows(w: Tensor, head_dim: int) -> Tensor:
+    # checkpoint uses interleaved RoPE, reorder the rows of each head so NEOX RoPE gives the same result
+    order = torch.cat([torch.arange(0, head_dim, 2), torch.arange(1, head_dim, 2)])
+    return w.reshape(-1, head_dim, *w.shape[1:])[:, order].reshape(w.shape)
+
+
+def codec_encoder_tensors(sd: dict[str, Tensor], num_codebooks: int) -> Iterable[tuple[str, Tensor]]:
+    T = gguf.MODEL_TENSOR
+
+    def get(name: str) -> Tensor:
+        return sd[name].float()
+
+    def tn(tensor: gguf.MODEL_TENSOR, bid: int | None = None, xid: int | None = None) -> str:
+        return gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
+
+    def fmt(tensor: gguf.MODEL_TENSOR, bid: int | None = None, suffix: str = ".weight") -> str:
+        return tn(tensor, bid) + suffix
+
+    def conv_wb(tensor: gguf.MODEL_TENSOR, prefix: str, bid: int | None = None, xid: int | None = None, stride: int = 1):
+        # taps-major [K / stride, OC, stride * IC], see conv() in tools/mtmd/models/fish-s2-spkenc.cpp
+        w = fold_conv(sd, prefix)
+        n_out, n_in, k = w.shape
+        w = w.reshape(n_out, n_in, k // stride, stride).permute(2, 0, 3, 1).reshape(k // stride, n_out, stride * n_in)
+        yield (tn(tensor, bid, xid) + ".weight", w)
+        yield (tn(tensor, bid, xid) + ".bias", get(prefix + ".conv.bias"))
+
+    def snake_ab(tensor: gguf.MODEL_TENSOR, name: str, bid: int | None = None, xid: int | None = None):
+        a, b = fold_snake(get(name))
+        yield (tn(tensor, bid, xid) + ".alpha", a)
+        yield (tn(tensor, bid, xid) + ".beta", b)
+
+    def tfm_layers(prefix: str, n_layer: int, offset: int):
+        for il in range(n_layer):
+            p = f"{prefix}layers.{il}."
+            bid = offset + il
+            wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0)
+            yield (fmt(T.A_ENC_ATTN_Q,       bid), _neox_rows(wq, _CODEC_HEAD_DIM))
+            yield (fmt(T.A_ENC_ATTN_K,       bid), _neox_rows(wk, _CODEC_HEAD_DIM))
+            yield (fmt(T.A_ENC_ATTN_V,       bid), wv)
+            yield (fmt(T.A_ENC_OUTPUT,       bid), get(p + "attention.wo.weight"))
+            yield (fmt(T.A_ENC_INPUT_NORM,   bid), get(p + "attention_norm.weight"))
+            yield (fmt(T.A_ENC_ATTN_SCALE,   bid), get(p + "attention_layer_scale.gamma"))
+            yield (fmt(T.A_ENC_OUTPUT_NORM,  bid), get(p + "ffn_norm.weight"))
+            yield (fmt(T.A_ENC_FFN_GATE,     bid), get(p + "feed_forward.w1.weight"))
+            yield (fmt(T.A_ENC_FFN_UP,       bid), get(p + "feed_forward.w3.weight"))
+            yield (fmt(T.A_ENC_FFN_DOWN,     bid), get(p + "feed_forward.w2.weight"))
+            yield (fmt(T.A_ENC_FFN_SCALE_LS, bid), get(p + "ffn_layer_scale.gamma"))
+
+    # encoder: entry conv, 4x (3 residual units, Snake, strided conv), Snake, post conv
+    e = "encoder.block."
+    yield from conv_wb(T.A_CODEC_ENC_ENTRY, e + "0")
+    for bid, stride in enumerate(_CODEC_ENC_STRIDES):
+        p = f"{e}{bid + 1}.block."
+        for xid in range(3):
+            r = f"{p}{xid}.block."
+            yield from snake_ab(T.A_CODEC_ENC_RES_ACT1, r + "0.alpha", bid, xid)
+            yield from conv_wb(T.A_CODEC_ENC_RES_CONV1, r + "1", bid, xid)
+            yield from snake_ab(T.A_CODEC_ENC_RES_ACT2, r + "2.alpha", bid, xid)
+            yield from conv_wb(T.A_CODEC_ENC_RES_CONV2, r + "3", bid, xid)
+        yield from snake_ab(T.A_CODEC_ENC_SNAKE, p + "3.alpha", bid)
+        yield from conv_wb(T.A_CODEC_ENC_CONV, p + "4", bid, stride=stride)
+    # the last block ends with a windowed transformer
+    yield from tfm_layers(f"{e}4.block.5.", _N_ENC_TFM_LAYERS, 0)
+    yield (fmt(T.A_POST_NORM), get(f"{e}4.block.5.norm.weight"))
+    yield from snake_ab(T.A_CODEC_ENC_POST_SNAKE, e + "5.alpha")
+    yield from conv_wb(T.A_CODEC_ENC_POST_CONV, e + "6")
+
+    # quantizer: 2x (strided conv + ConvNeXt), windowed transformer, then semantic + residual VQ
+    q = "quantizer."
+    for bid in range(2):
+        p = f"{q}downsample.{bid}."
+        yield from conv_wb(T.A_CODEC_DOWN_CONV, p + "0", bid, stride=2)
+        # depthwise: one [C] row per tap
+        yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".weight", fold_conv(sd, p + "1.dwconv").squeeze(1).T.contiguous())
+        yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".bias", get(p + "1.dwconv.conv.bias"))
+        for key, tensor in (("norm", T.A_CODEC_DOWN_NORM), ("pwconv1", T.A_CODEC_DOWN_PW1), ("pwconv2", T.A_CODEC_DOWN_PW2)):
+            yield (tn(tensor, bid) + ".weight", get(f"{p}1.{key}.weight"))
+            yield (tn(tensor, bid) + ".bias", get(f"{p}1.{key}.bias"))
+        yield (tn(T.A_CODEC_DOWN_GAMMA, bid), get(p + "1.gamma"))
+    yield from tfm_layers(q + "pre_module.", _N_PRE_TFM_LAYERS, _N_ENC_TFM_LAYERS)
+    yield (fmt(T.A_CODEC_PRE_NORM), get(q + "pre_module.norm.weight"))
+
+    # nearest codebook entry by cosine similarity, see VectorQuantize.decode_latents()
+    def vq(prefix: str) -> tuple[Tensor, Tensor, Tensor]:
+        w = fold_wn(sd[prefix + ".in_proj.weight_g"], sd[prefix + ".in_proj.weight_v"])
+        cb = torch.nn.functional.normalize(get(prefix + ".codebook.weight"), dim=-1)
+        return w, get(prefix + ".in_proj.bias"), cb
+
+    w, b, cb = vq(q + "semantic_quantizer.quantizers.0")
+    yield (fmt(T.A_CODEC_VQ_FIRST_IN, suffix=".weight"), w)
+    yield (fmt(T.A_CODEC_VQ_FIRST_IN, suffix=".bias"), b)
+    yield (fmt(T.A_CODEC_VQ_FIRST_CB), cb)
+    n_rest = num_codebooks - 1
+    rest = [vq(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)]
+    yield (fmt(T.A_CODEC_VQ_REST_IN, suffix=".weight"), torch.stack([r[0] for r in rest]))
+    yield (fmt(T.A_CODEC_VQ_REST_IN, suffix=".bias"), torch.stack([r[1] for r in rest]))
+    yield (fmt(T.A_CODEC_VQ_REST_CB), torch.stack([r[2] for r in rest]))
+
+
 @ModelBase.register("FishOmniModel")
 @ModelBase.example("fishaudio/drama-3")
 class Drama3TextModel(Qwen3_5MoeTextModel):
@@ -59,7 +168,7 @@ class Drama3TextModel(Qwen3_5MoeTextModel):
 @ModelBase.example("fishaudio/drama-3")
 class Drama3MmprojModel(MmprojModel):
     has_vision_encoder = False
-    has_audio_encoder = False
+    has_audio_encoder = True  # codec encoder, for voice cloning
     gen_projector_type = gguf.VisionProjectorType.DRAMA3_GEN

     _LAYER_MAP = {
@@ -103,6 +212,16 @@ class Drama3MmprojModel(MmprojModel):
         self.gguf_writer.add_gen_audio_head_count_kv(cfg["n_local_heads"])
         self.gguf_writer.add_gen_audio_attention_layernorm_eps(cfg["norm_eps"])
         # other tower and codec hparams are hardcoded in tools/mtmd
+        # codec encoder: raw 44.1 kHz waveform -> backbone input rows, other hparams are hardcoded in tools/mtmd
+        self.gguf_writer.add_clip_has_audio_encoder(True)
+        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.FISH_S2_SPKENC)
+        self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
+        self.gguf_writer.add_audio_num_mel_bins(1)
+        self.gguf_writer.add_audio_block_count(_N_ENC_TFM_LAYERS + _N_PRE_TFM_LAYERS)
+        self.gguf_writer.add_audio_embedding_length(1024)
+        self.gguf_writer.add_audio_feed_forward_length(3072)
+        self.gguf_writer.add_audio_head_count(1024 // _CODEC_HEAD_DIM)
+        self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)

     @classmethod
     def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
@@ -112,6 +231,12 @@ class Drama3MmprojModel(MmprojModel):
         return None

     def tensor_force_quant(self, name, new_name, bid, n_dims):
+        # the VQ picks the nearest codebook entry, small errors in the encoder change the codes
+        if new_name.startswith("a.codec."):
+            return gguf.GGMLQuantizationType.F32
+        # codec transformers: F16 gives codes closer to the reference than BF16
+        if new_name.startswith("a.blk.") and n_dims == 2:
+            return gguf.GGMLQuantizationType.F16
         # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
         if new_name.endswith(".weight") and (
             new_name in ("a.gen.wav.dac.entry.weight", "a.gen.wav.dac.post_conv.weight")
@@ -130,7 +255,7 @@ class Drama3MmprojModel(MmprojModel):
         T = gguf.MODEL_TENSOR
         cfg = self.hparams

-        if name.startswith("a.gen.wav."):
+        if name.startswith("a."):
             yield (name, data_torch)
             return

@@ -177,7 +302,7 @@ class Drama3MmprojModel(MmprojModel):
         raise ValueError(f"unexpected tensor {name}")

     def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        # only the DAC decoder, the encoder is for voice cloning (not supported yet)
+        # Fish DAC decoder, then the shared codec encoder for voice cloning
         sd = load_codec(self.dir_model)
         T = gguf.MODEL_TENSOR

@@ -262,3 +387,5 @@ class Drama3MmprojModel(MmprojModel):
         yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b)
         yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), conv(d + "6"))
         yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".bias"), get(d + "6.conv.bias"))
+
+        yield from codec_encoder_tensors(sd, self.hparams["num_codebooks"])
diff --git a/conversion/fish_s2.py b/conversion/fish_s2.py
index 4c5aeff13..563b52b12 100644
--- a/conversion/fish_s2.py
+++ b/conversion/fish_s2.py
@@ -3,13 +3,11 @@ from __future__ import annotations
 from pathlib import Path
 from typing import Any, Callable, Iterable, TYPE_CHECKING

-import torch
-
 if TYPE_CHECKING:
     from torch import Tensor

 from .base import ModelBase, gguf
-from .drama3 import Drama3MmprojModel, fold_conv, fold_snake, fold_wn, load_codec
+from .drama3 import Drama3MmprojModel, _neox_rows
 from .qwen import Qwen3Model

 # Fish Audio S2 Pro (dual-AR)
@@ -19,15 +17,6 @@ from .qwen import Qwen3Model
 # residual codebook size of codec.pth, the fast AR head is cut to it
 _N_RESIDUAL_CODES = 1024

-# codec encoder strides (encoder_rates of the codec config), the quantizer then downsamples 2x2
-_CODEC_ENC_STRIDES = (2, 4, 8, 8)
-
-# codec encoder transformers, both 1024 wide with 16 heads of 64
-# generic audio layers a.blk.0..3: last encoder block, a.blk.4..11: quantizer pre_module
-_N_ENC_TFM_LAYERS = 4
-_N_PRE_TFM_LAYERS = 8
-_CODEC_HEAD_DIM = 64
-

 def _load_hparams(dir_model: Path) -> dict[str, Any]:
     hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
@@ -48,12 +37,6 @@ def _load_hparams(dir_model: Path) -> dict[str, Any]:
     return hparams


-def _neox_rows(w: Tensor, head_dim: int) -> Tensor:
-    # checkpoint uses interleaved RoPE, reorder the rows of each head so NEOX RoPE gives the same result
-    order = torch.cat([torch.arange(0, head_dim, 2), torch.arange(1, head_dim, 2)])
-    return w.reshape(-1, head_dim, *w.shape[1:])[:, order].reshape(w.shape)
-
-
 @ModelBase.register("fish_qwen3_omni")
 @ModelBase.example("fishaudio/s2-pro")
 class FishS2TextModel(Qwen3Model):
@@ -88,7 +71,6 @@ class FishS2TextModel(Qwen3Model):
 @ModelBase.register("fish_qwen3_omni")
 @ModelBase.example("fishaudio/s2-pro")
 class FishS2MmprojModel(Drama3MmprojModel):
-    has_audio_encoder = True  # codec encoder, for voice cloning
     gen_projector_type = gguf.VisionProjectorType.FISH_S2_GEN

     _FAST_AR_MAP = {
@@ -109,28 +91,6 @@ class FishS2MmprojModel(Drama3MmprojModel):
         cfg["num_hidden_layers"] = cfg["n_layer"]
         return cfg

-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        # codec encoder: raw 44.1 kHz waveform -> backbone input rows, other hparams are hardcoded in tools/mtmd
-        self.gguf_writer.add_clip_has_audio_encoder(True)
-        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.FISH_S2_SPKENC)
-        self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
-        self.gguf_writer.add_audio_num_mel_bins(1)
-        self.gguf_writer.add_audio_block_count(_N_ENC_TFM_LAYERS + _N_PRE_TFM_LAYERS)
-        self.gguf_writer.add_audio_embedding_length(1024)
-        self.gguf_writer.add_audio_feed_forward_length(3072)
-        self.gguf_writer.add_audio_head_count(1024 // _CODEC_HEAD_DIM)
-        self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        # the VQ picks the nearest codebook entry, small errors in the encoder change the codes
-        if new_name.startswith("a.codec."):
-            return gguf.GGMLQuantizationType.F32
-        # codec transformers: F16 gives codes closer to the reference than BF16
-        if new_name.startswith("a.blk.") and n_dims == 2:
-            return gguf.GGMLQuantizationType.F16
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
     @classmethod
     def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
         name = item[0]
@@ -182,95 +142,3 @@ class FishS2MmprojModel(Drama3MmprojModel):
             yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, bid), v)
             return
         yield (self.format_tensor_name(self._FAST_AR_MAP[key], bid), data_torch)
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        yield from super().generate_extra_tensors()
-
-        sd = load_codec(self.dir_model)
-        T = gguf.MODEL_TENSOR
-
-        def get(name: str) -> Tensor:
-            return sd[name].float()
-
-        def tn(tensor: gguf.MODEL_TENSOR, bid: int | None = None, xid: int | None = None) -> str:
-            return gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
-
-        def conv_wb(tensor: gguf.MODEL_TENSOR, prefix: str, bid: int | None = None, xid: int | None = None, stride: int = 1):
-            # taps-major [K / stride, OC, stride * IC], see conv() in tools/mtmd/models/fish-s2-spkenc.cpp
-            w = fold_conv(sd, prefix)
-            n_out, n_in, k = w.shape
-            w = w.reshape(n_out, n_in, k // stride, stride).permute(2, 0, 3, 1).reshape(k // stride, n_out, stride * n_in)
-            yield (tn(tensor, bid, xid) + ".weight", w)
-            yield (tn(tensor, bid, xid) + ".bias", get(prefix + ".conv.bias"))
-
-        def snake_ab(tensor: gguf.MODEL_TENSOR, name: str, bid: int | None = None, xid: int | None = None):
-            a, b = fold_snake(get(name))
-            yield (tn(tensor, bid, xid) + ".alpha", a)
-            yield (tn(tensor, bid, xid) + ".beta", b)
-
-        def tfm_layers(prefix: str, n_layer: int, offset: int):
-            for il in range(n_layer):
-                p = f"{prefix}layers.{il}."
-                bid = offset + il
-                wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0)
-                yield (self.format_tensor_name(T.A_ENC_ATTN_Q,       bid), _neox_rows(wq, _CODEC_HEAD_DIM))
-                yield (self.format_tensor_name(T.A_ENC_ATTN_K,       bid), _neox_rows(wk, _CODEC_HEAD_DIM))
-                yield (self.format_tensor_name(T.A_ENC_ATTN_V,       bid), wv)
-                yield (self.format_tensor_name(T.A_ENC_OUTPUT,       bid), get(p + "attention.wo.weight"))
-                yield (self.format_tensor_name(T.A_ENC_INPUT_NORM,   bid), get(p + "attention_norm.weight"))
-                yield (self.format_tensor_name(T.A_ENC_ATTN_SCALE,   bid), get(p + "attention_layer_scale.gamma"))
-                yield (self.format_tensor_name(T.A_ENC_OUTPUT_NORM,  bid), get(p + "ffn_norm.weight"))
-                yield (self.format_tensor_name(T.A_ENC_FFN_GATE,     bid), get(p + "feed_forward.w1.weight"))
-                yield (self.format_tensor_name(T.A_ENC_FFN_UP,       bid), get(p + "feed_forward.w3.weight"))
-                yield (self.format_tensor_name(T.A_ENC_FFN_DOWN,     bid), get(p + "feed_forward.w2.weight"))
-                yield (self.format_tensor_name(T.A_ENC_FFN_SCALE_LS, bid), get(p + "ffn_layer_scale.gamma"))
-
-        # encoder: entry conv, 4x (3 residual units, Snake, strided conv), Snake, post conv
-        e = "encoder.block."
-        yield from conv_wb(T.A_CODEC_ENC_ENTRY, e + "0")
-        for bid, stride in enumerate(_CODEC_ENC_STRIDES):
-            p = f"{e}{bid + 1}.block."
-            for xid in range(3):
-                r = f"{p}{xid}.block."
-                yield from snake_ab(T.A_CODEC_ENC_RES_ACT1, r + "0.alpha", bid, xid)
-                yield from conv_wb(T.A_CODEC_ENC_RES_CONV1, r + "1", bid, xid)
-                yield from snake_ab(T.A_CODEC_ENC_RES_ACT2, r + "2.alpha", bid, xid)
-                yield from conv_wb(T.A_CODEC_ENC_RES_CONV2, r + "3", bid, xid)
-            yield from snake_ab(T.A_CODEC_ENC_SNAKE, p + "3.alpha", bid)
-            yield from conv_wb(T.A_CODEC_ENC_CONV, p + "4", bid, stride=stride)
-        # the last block ends with a windowed transformer
-        yield from tfm_layers(f"{e}4.block.5.", _N_ENC_TFM_LAYERS, 0)
-        yield (self.format_tensor_name(T.A_POST_NORM), get(f"{e}4.block.5.norm.weight"))
-        yield from snake_ab(T.A_CODEC_ENC_POST_SNAKE, e + "5.alpha")
-        yield from conv_wb(T.A_CODEC_ENC_POST_CONV, e + "6")
-
-        # quantizer: 2x (strided conv + ConvNeXt), windowed transformer, then semantic + residual VQ
-        q = "quantizer."
-        for bid in range(2):
-            p = f"{q}downsample.{bid}."
-            yield from conv_wb(T.A_CODEC_DOWN_CONV, p + "0", bid, stride=2)
-            # depthwise: one [C] row per tap
-            yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".weight", fold_conv(sd, p + "1.dwconv").squeeze(1).T.contiguous())
-            yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".bias", get(p + "1.dwconv.conv.bias"))
-            for key, tensor in (("norm", T.A_CODEC_DOWN_NORM), ("pwconv1", T.A_CODEC_DOWN_PW1), ("pwconv2", T.A_CODEC_DOWN_PW2)):
-                yield (tn(tensor, bid) + ".weight", get(f"{p}1.{key}.weight"))
-                yield (tn(tensor, bid) + ".bias", get(f"{p}1.{key}.bias"))
-            yield (tn(T.A_CODEC_DOWN_GAMMA, bid), get(p + "1.gamma"))
-        yield from tfm_layers(q + "pre_module.", _N_PRE_TFM_LAYERS, _N_ENC_TFM_LAYERS)
-        yield (self.format_tensor_name(T.A_CODEC_PRE_NORM), get(q + "pre_module.norm.weight"))
-
-        # nearest codebook entry by cosine similarity, see VectorQuantize.decode_latents()
-        def vq(prefix: str) -> tuple[Tensor, Tensor, Tensor]:
-            w = fold_wn(sd[prefix + ".in_proj.weight_g"], sd[prefix + ".in_proj.weight_v"])
-            cb = torch.nn.functional.normalize(get(prefix + ".codebook.weight"), dim=-1)
-            return w, get(prefix + ".in_proj.bias"), cb
-
-        w, b, cb = vq(q + "semantic_quantizer.quantizers.0")
-        yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".weight"), w)
-        yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".bias"), b)
-        yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_CB), cb)
-        n_rest = self.hparams["num_codebooks"] - 1
-        rest = [vq(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)]
-        yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".weight"), torch.stack([r[0] for r in rest]))
-        yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".bias"), torch.stack([r[1] for r in rest]))
-        yield (self.format_tensor_name(T.A_CODEC_VQ_REST_CB), torch.stack([r[2] for r in rest]))
diff --git a/docs/multimodal/tts.md b/docs/multimodal/tts.md
index 48c6d72e5..e009566c0 100644
--- a/docs/multimodal/tts.md
+++ b/docs/multimodal/tts.md
@@ -175,7 +175,7 @@ python convert_hf_to_gguf.py path/to/chatterbox --mmproj --outfile mmproj-chatte
 Available params:
 - Speaker tags `<|speaker:0|>` to `<|speaker:15|>` in the text give each character its own voice, a text without a tag starts with `<|speaker:0|>`
 - The language is the one of the text, `--tts-lang` is not used
-- Note: `--tts-speaker-file` is not supported yet (voice cloning)
+- Voice cloning: `--tts-speaker-file` with the reference clip and `--tts-speaker-text` with its transcript (a transcript without a speaker tag is speaker 0). The clip is encoded by the codec encoder half of `codec.pth` and its codes are put in the prompt as one reference, as the reference does; the clip is truncated to the first 30 s. It needs an mmproj converted with the codec encoder
 - Note: `--top-k` and `--top-p` are not used, frames are sampled with the settings of the reference (main token: temperature 1.0, top-p 0.9, repetition-aware redraw over the last 16 frames; other codes: temperature 1.0, top-k 30, top-p 0.9)
 - Each frame is 2048 samples (about 46 ms), generation stops at the end-of-speech token
 - The codebook tower keeps the whole sequence in its KV cache, the prompt and the generated frames are limited to 4096 positions
@@ -192,7 +192,7 @@ llama-tts -m drama3.gguf \

 **Note for GGUF conversion:**

-The routed experts of the checkpoint are FP8, they are dequantized at conversion. The mmproj holds the codebook tower and the decoder half of `codec.pth`, it is best kept in BF16:
+The routed experts of the checkpoint are FP8, they are dequantized at conversion. The mmproj holds the codebook tower and `codec.pth` (the encoder half for voice cloning, the decoder half for the audio), it is best kept in BF16; the codec encoder is always stored in F16/F32:

 ```sh
 python convert_hf_to_gguf.py path/to/drama-3 --outtype q8_0 --outfile drama3.gguf
diff --git a/tools/mtmd/models/drama3-gen.cpp b/tools/mtmd/models/drama3-gen.cpp
index e77fcf540..41d8c22b1 100644
--- a/tools/mtmd/models/drama3-gen.cpp
+++ b/tools/mtmd/models/drama3-gen.cpp
@@ -130,9 +130,19 @@ ggml_cgraph * mtmd_graph_drama3_gen::build_tower() {
     ggml_set_name(inp_pos, "inp_pos");
     ggml_set_input(inp_pos);

-    ggml_tensor * inp_rand = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_heads);
-    ggml_set_name(inp_rand, "inp_rand");
-    ggml_set_input(inp_rand);
+    // teacher forcing (reference rows of a voice clone): codes 1.. of the frame are given, nothing is sampled
+    const bool forced = n_frames == 2;
+    ggml_tensor * inp_rand  = nullptr;
+    ggml_tensor * inp_force = nullptr;
+    if (forced) {
+        inp_force = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_heads);
+        ggml_set_name(inp_force, "inp_force");
+        ggml_set_input(inp_force);
+    } else {
+        inp_rand = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_heads);
+        ggml_set_name(inp_rand, "inp_rand");
+        ggml_set_input(inp_rand);
+    }

     ggml_tensor * mask     = ggml_view_1d(ctx0, inp_scale, 1, 0);
     ggml_tensor * extra_on = ggml_view_1d(ctx0, inp_scale, 1, sizeof(float));
@@ -152,6 +162,11 @@ ggml_cgraph * mtmd_graph_drama3_gen::build_tower() {
             continue;
         }
         const int ih = il - n_shared;
+        if (forced) {
+            code = ggml_view_1d(ctx0, inp_force, 1, (size_t) ih * sizeof(int32_t));
+            out_codes = cg.cache_set(out_codes, ih + 1, code);
+            continue;
+        }
         ggml_tensor * head   = ggml_view_2d(ctx0, model.gen_code_head_w, model.gen_code_head_w->ne[0], model.gen_code_head_w->ne[1],
                                             model.gen_code_head_w->nb[1], (size_t) ih * model.gen_code_head_w->nb[2]);
         ggml_tensor * logits = ggml_mul_mat(ctx0, head, x);
diff --git a/tools/mtmd/models/fish-s2-spkenc.cpp b/tools/mtmd/models/fish-s2-spkenc.cpp
index a9c51d9cd..fe2eb28ef 100644
--- a/tools/mtmd/models/fish-s2-spkenc.cpp
+++ b/tools/mtmd/models/fish-s2-spkenc.cpp
@@ -2,6 +2,7 @@

 #include <cmath>
 #include <string>
+#include <vector>

 // reference clip -> codec frames -> backbone input rows, see DAC.encode() and DownsampleResidualVectorQuantize.encode()
 // activations are [C, T]; a conv is a sum of matmuls over tap-shifted views: exact in F32, no im2col buffer
@@ -117,17 +118,35 @@ ggml_cgraph * mtmd_graph_fish_s2_spkenc::build() {
         return ggml_view_2d(ctx0, t, t->ne[0], t->ne[1], t->nb[1], (size_t) g * t->nb[2]);
     };

-    // backbone row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1), same as fish_s2_gen
     const int64_t n_cb = model.gen_code_embd_w->ne[2];
-    ggml_tensor * idx = vq(c2w.quant_first_in_w, c2w.quant_first_in_b, c2w.quant_first_cb_norm_w, c2w.quant_first_cb_w);
-    ggml_tensor * out = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, idx),
-                                       ggml_get_rows(ctx0, page(model.gen_code_embd_w, 0), idx));
+    std::vector<ggml_tensor *> idx(n_cb);
+    idx[0] = vq(c2w.quant_first_in_w, c2w.quant_first_in_b, c2w.quant_first_cb_norm_w, c2w.quant_first_cb_w);
     for (int64_t g = 1; g < n_cb; g++) {
         ggml_tensor * in_b = ggml_view_1d(ctx0, c2w.quant_rest_in_b, c2w.quant_rest_in_b->ne[0], (size_t) (g - 1) * c2w.quant_rest_in_b->nb[1]);
-        idx = vq(page(c2w.quant_rest_in_w, g - 1), in_b, page(c2w.quant_rest_cb_norm_w, g - 1), page(c2w.quant_rest_cb_w, g - 1));
-        out = ggml_add(ctx0, out, ggml_get_rows(ctx0, page(model.gen_code_embd_w, g), idx));
+        idx[g] = vq(page(c2w.quant_rest_in_w, g - 1), in_b, page(c2w.quant_rest_cb_norm_w, g - 1), page(c2w.quant_rest_cb_w, g - 1));
+    }
+    // codes of the clip, read back as the gen-side reference
+    for (int64_t g = 0; g < n_cb; g++) {
+        ggml_set_name(idx[g], ("out_ref_code_" + std::to_string(g)).c_str());
+        ggml_set_output(idx[g]);
+        ggml_build_forward_expand(gf, idx[g]);
+    }
+
+    ggml_tensor * out = nullptr;
+    if (hparams.spkenc_drama3_rows) {
+        // drama3 backbone row: (E_lm[semantic] + E_cb[1][code 1]) / sqrt(2), see mtmd_graph_drama3_gen::build_tower()
+        out = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, idx[0]),
+                             ggml_get_rows(ctx0, page(model.gen_code_embd_w, 1), idx[1]));
+        out = ggml_scale(ctx0, out, 1.0f / sqrtf(2.0f));
+    } else {
+        // backbone row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1), same as fish_s2_gen
+        out = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, idx[0]),
+                             ggml_get_rows(ctx0, page(model.gen_code_embd_w, 0), idx[0]));
+        for (int64_t g = 1; g < n_cb; g++) {
+            out = ggml_add(ctx0, out, ggml_get_rows(ctx0, page(model.gen_code_embd_w, g), idx[g]));
+        }
+        out = ggml_scale(ctx0, out, 1.0f / sqrtf((float) (n_cb + 1)));
     }
-    out = ggml_scale(ctx0, out, 1.0f / sqrtf((float) (n_cb + 1)));
     cb(out, "spk_rows", -1);
     ggml_build_forward_expand(gf, out);

diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 64f7bf70d..fb75270ad 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -625,7 +625,7 @@ struct mtmd_graph_drama3_gen : mtmd_graph {

     mtmd_subctx_gen_process gen_process;
     mtmd_subctx_gen_state * gen_state; // nullptr on buffer reserve
-    int   n_frames;             // GEN_WAV only
+    int   n_frames;             // GEN_WAV: frames to decode; GEN_CODE: 2 when the caller gives the codes of the frame (teacher forcing)
     int   top_k;
     float top_p;
     float temp;
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 32f1d39d4..3c33eff3a 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -2739,6 +2739,9 @@ public:
         seq_id = 0;
         pos = 0;
         prompt_tokens.clear();
+        ref_codes.clear();
+        ref_embd.clear();
+        ref_begin = 0;
         prompt_pos = 0;
         frame.clear();
         main_logits.clear();
@@ -2758,8 +2761,16 @@ public:
         seq_id   = inp->seq_id;
         out_type = inp->out_type;
         stream   = inp->stream;
-        if (inp->speaker_ref) {
-            LOG_WRN("mtmd_helper_gen_audio: voice cloning is not supported yet, the speaker file is ignored\n");
+
+        // voice cloning: backbone rows from the codec encoder, the codes of the clip go to the gen state
+        std::vector<float> ref_rows;
+        if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref) && !mtmd_support_audio(mctx)) {
+            LOG_WRN("mtmd_helper_gen_audio: mmproj has no speaker encoder (re-convert it for voice cloning), the speaker file is ignored\n");
+        } else if (inp->speaker_ref) {
+            if (!encode_speaker(inp->speaker_ref, ref_rows) || !init_gen_state(inp->seed) ||
+                    mtmd_gen_state_set_ref(mctx, gen_state.get()) != 0) {
+                return 1;
+            }
         }

         sem_begin = find_special_token(vocab, "<|semantic:0|>");
@@ -2770,7 +2781,12 @@ public:
             return 1;
         }

-        if (!tokenize_segments(prompt_segments(std::string(inp->prompt, inp->prompt_len)), prompt_tokens)) {
+        const std::string text(inp->prompt, inp->prompt_len);
+        if (ref_rows.empty()) {
+            if (!tokenize_segments(prompt_segments(text), prompt_tokens)) {
+                return 1;
+            }
+        } else if (!build_reference_prompt(text, inp->speaker_ref_text ? inp->speaker_ref_text : "", std::move(ref_rows))) {
             return 1;
         }
         if (prompt_tokens.size() >= llama_n_ctx_seq(lctx) || prompt_tokens.size() > max_rows()) {
@@ -2782,7 +2798,7 @@ public:
             return 1;
         }
         rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
-        return init_gen_state(inp->seed) ? 0 : 1;
+        return gen_state || init_gen_state(inp->seed) ? 0 : 1;
     }

     // run the tower on every prompt row to fill its KV
@@ -2792,27 +2808,41 @@ public:
         if (prompt_pos >= n_prompt) {
             return 0;
         }
-        // small batches: every row also outputs logits
-        const int n = std::min({ n_batch, n_prompt - prompt_pos, 64 });
-        llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
-        if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) {
-            return -1;
+        // small batches of rows of one kind (text tokens or reference frames): every row also outputs logits
+        const bool speech = is_speech_row(prompt_pos);
+        int n = std::min({ n_batch, n_prompt - prompt_pos, 64 });
+        for (int i = 1; i < n; i++) {
+            if (is_speech_row(prompt_pos + i) != speech) {
+                n = i;
+                break;
+            }
         }
-        for (int i = 0; i < n; i++) {
-            llama_batch_ext_set_output_logits(batch.get(), i, true);
+        int ret = 0;
+        if (speech) {
+            ret = decode_embd_rows(ref_embd.data() + (size_t) (prompt_pos - ref_begin) * n_embd, n);
+        } else {
+            llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+            if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) {
+                return -1;
+            }
+            for (int i = 0; i < n; i++) {
+                llama_batch_ext_set_output_logits(batch.get(), i, true);
+            }
+            ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
         }
-        if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
+        if (ret != 0) {
             LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
             return -1;
         }
         for (int i = 0; i < n; i++) {
+            const int row = prompt_pos + i;
             const float * h = llama_get_embeddings_ith(lctx, i);
-            if (prompt_pos + i == n_prompt - 1) {
+            if (row == n_prompt - 1) {
                 h_state_buf.assign(h, h + n_embd); // first frame comes from the last prompt row
                 save_main_logits(i);
                 break;
             }
-            if (!run_tower(-1, h)) {
+            if (!run_prompt_tower(row, h)) {
                 return -1;
             }
         }
@@ -2845,17 +2875,7 @@ public:
         }
         codes_buf.insert(codes_buf.end(), frame.begin(), frame.end());

-        // text/speech stream: all mrope sections use the same position
-        const bool mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE ||
-                           llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE;
-        decode_embd_batch batch(next_embd, 1, mrope ? 4 : 1, n_embd);
-        if (mrope) {
-            batch.set_position_mrope_1d(pos, seq_id);
-        } else {
-            batch.set_position_normal(pos, seq_id);
-        }
-        batch.logits[0] = 1;
-        if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) {
+        if (decode_embd_rows(next_embd, 1) != 0) {
             LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
             return 1;
         }
@@ -2938,9 +2958,24 @@ protected:
         return true;
     }

+    // offset of the leading speaker tag <|speaker:N|> after optional whitespace, npos without one
+    static size_t speaker_tag_pos(const std::string & text) {
+        const size_t lead = text.find_first_not_of(" \t\n\r\f\v");
+        static const std::string open = "<|speaker:";
+        if (lead == std::string::npos || text.compare(lead, open.size(), open) != 0) {
+            return std::string::npos;
+        }
+        size_t i = lead + open.size();
+        const size_t digits = i;
+        while (i < text.size() && text[i] >= '0' && text[i] <= '9') {
+            i++;
+        }
+        return i > digits && text.compare(i, 2, "|>") == 0 ? lead : std::string::npos;
+    }
+
     // reference-bracket-v1 prompt without references
     virtual std::vector<std::string> prompt_segments(std::string text) const {
-        if (text.rfind("<|speaker:", 0) != 0) {
+        if (speaker_tag_pos(text) == std::string::npos) {
             text = "<|speaker:0|> " + text;
         }
         return {
@@ -2952,6 +2987,86 @@ protected:
     // prompt rows but the last + generated frames
     virtual size_t max_rows() const { return n_tower_rows; }

+    // n speech rows from pos, every row outputs logits
+    // text/speech stream: all mrope sections use the same position
+    int32_t decode_embd_rows(const float * embd, int n) {
+        const bool mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE ||
+                           llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE;
+        decode_embd_batch batch(embd, n, mrope ? 4 : 1, n_embd);
+        if (mrope) {
+            batch.set_position_mrope_1d(pos, seq_id);
+        } else {
+            batch.set_position_normal(pos, seq_id);
+        }
+        for (int i = 0; i < n; i++) {
+            batch.logits[i] = 1;
+        }
+        return llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n));
+    }
+
+    bool is_speech_row(int row) const {
+        return row >= ref_begin && (size_t) (row - ref_begin) < ref_embd.size() / n_embd;
+    }
+
+    // reference-bracket-v1 with one reference, see vLLM-Omni fish_omni/prompt.py
+    // system: instruction, "\n\n[Reference 1]\nText: <transcript>\nSpeech: " then one row per frame of the clip
+    bool build_reference_prompt(std::string text, std::string ref_text, std::vector<float> && ref_rows) {
+        size_t n_codes = 0;
+        const int32_t * codes = mtmd_gen_state_get_ref_codes(gen_state.get(), &n_codes);
+        const size_t n_frames = ref_rows.size() / n_embd;
+        if (!codes || n_frames == 0 || n_codes != n_frames * n_cb || ref_rows.size() != n_frames * n_embd) {
+            LOG_ERR("mtmd_helper_gen_audio: the speaker encoder gave no codes for the clip\n");
+            return false;
+        }
+        // the speaker tag follows "Text: " directly (default speaker 0), an untagged target reuses it
+        std::string tag = "<|speaker:0|>";
+        const size_t lead = speaker_tag_pos(ref_text);
+        if (lead == std::string::npos) {
+            ref_text = tag + " " + ref_text;
+        } else {
+            ref_text = ref_text.substr(lead);
+            tag = ref_text.substr(0, ref_text.find("|>") + 2);
+        }
+        if (speaker_tag_pos(text) == std::string::npos) {
+            text = tag + " " + text;
+        }
+
+        std::vector<llama_token> head, tail;
+        if (!tokenize_segments({ "<|im_start|>system\n", "convert the provided text to speech reference to the following:",
+                                 "\n\n[Reference 1]\nText: " + ref_text + "\nSpeech: " }, head) ||
+                !tokenize_segments({ "<|im_end|>\n", "<|im_start|>user\n", text, "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>" }, tail)) {
+            return false;
+        }
+        prompt_tokens.assign(head.begin(), head.end());
+        prompt_tokens.insert(prompt_tokens.end(), n_frames, LLAMA_TOKEN_NULL);
+        prompt_tokens.insert(prompt_tokens.end(), tail.begin(), tail.end());
+        ref_codes.assign(codes, codes + n_codes);
+        ref_embd  = std::move(ref_rows);
+        ref_begin = (int) head.size();
+        return true;
+    }
+
+    // tower row of a prompt row: the input frame is the codes of this row (reference frame) or none,
+    // the codes of the next row are forced when it is a reference frame
+    bool run_prompt_tower(int row, const float * h) {
+        const bool cur  = is_speech_row(row);
+        const bool next = is_speech_row(row + 1);
+        const auto ref_frame = [&](int r) { return ref_codes.begin() + (size_t) (r - ref_begin) * n_cb; };
+        if (next) {
+            // [input frame, -1 for a text row | forced next frame]
+            frame.assign(n_cb, -1);
+            if (cur) {
+                std::copy(ref_frame(row), ref_frame(row + 1), frame.begin());
+            }
+            frame.insert(frame.end(), ref_frame(row + 1), ref_frame(row + 2));
+        } else if (cur) {
+            frame.assign(ref_frame(row), ref_frame(row + 1));
+        }
+        const bool ok = run_tower(next ? frame[n_cb] : -1, h) != nullptr;
+        frame.clear();
+        return ok;
+    }
+
     // copy right after our own decode, other sequences may decode on lctx before the next step_gen
     // layout: eos, then semantic codes 0..n_cb_vocab-1
     void save_main_logits(int32_t i) {
@@ -3072,7 +3187,10 @@ protected:

     llama_seq_id seq_id = 0;
     llama_pos    pos    = 0;
-    std::vector<llama_token> prompt_tokens;
+    std::vector<llama_token> prompt_tokens;    // one per prompt row, LLAMA_TOKEN_NULL for a reference frame
+    std::vector<int32_t>     ref_codes;        // codes of the reference frames, n_cb per frame
+    std::vector<float>       ref_embd;         // backbone rows of the reference frames
+    int ref_begin = 0;                         // prompt row of the first reference frame
     int prompt_pos = 0;

     std::mt19937 rng;
diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp
index e26824d0f..2f4f40591 100644
--- a/tools/mtmd/mtmd-model.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -1171,6 +1171,10 @@ struct mtmd_model_loader {
                         hparams.wav_tfm_n_head_kv  = 16;
                         hparams.wav_tfm_eps        = 1e-5f;
                         hparams.wav_tfm_rope_theta = 10000.0f;
+                        // drama3 shares the codec, only the backbone rows of a frame differ
+                        std::string gen_proj;
+                        get_string(KEY_GEN_AUDIO_PROJ_TYPE, gen_proj, false);
+                        hparams.spkenc_drama3_rows = projector_type_from_string(gen_proj) == PROJECTOR_TYPE_DRAMA3_GEN;
                     } break;
                 case PROJECTOR_TYPE_CHATTERBOX:
                     {
diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h
index 893578b03..10c3154e8 100644
--- a/tools/mtmd/mtmd-model.h
+++ b/tools/mtmd/mtmd-model.h
@@ -239,6 +239,9 @@ struct mtmd_hparams {
     int32_t wav_tfm_swa          = 0; // pre_transformer's KV cache size, in frames
     int32_t wav_frame_stack      = 0; // lfm3a codec: decoder positions per frame

+    // fish_s2_spkenc in a drama3_gen mmproj: backbone rows as drama3_gen builds them
+    bool spkenc_drama3_rows = false;
+
     // mimo-v2.5: LLM-side connector (input_local_transformer)
     int32_t audio_local_n_layer = 0;
     int32_t audio_local_group_size = 0;
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index f754a913b..aea0c1b92 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -2008,12 +2008,16 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
                 } else {
                     // code0 < 0: next row is text (prompt), the tower only fills its KV
                     // codes: input frame codes for a speech row, empty for a text row
+                    //        then optionally the codes of the next frame (reference rows): teacher forcing, code0 must be its code 0
                     const bool next_speech = params->code0 >= 0;
-                    const bool cur_speech  = params->codes && (int64_t) params->codes->size() == n_cb;
-                    if (params->code0 >= n_vocab) {
-                        LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) n_vocab);
+                    const size_t n_given   = params->codes ? params->codes->size() : 0;
+                    const bool forced      = n_given == (size_t) (2 * n_cb);
+                    if (params->code0 >= n_vocab || (n_given != 0 && n_given != (size_t) n_cb && !forced) ||
+                            (forced && params->code0 != (*params->codes)[n_cb])) {
+                        LOG_ERR("%s: invalid tower input (code0 %d, %zu codes)\n", __func__, params->code0, n_given);
                         return false;
                     }
+                    const bool cur_speech  = n_given != 0 && (*params->codes)[0] >= 0;
                     std::vector<int32_t> code0 = { std::max(params->code0, 0) };
                     set_input_i32("inp_code0", code0);

@@ -2030,12 +2034,20 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
                     std::vector<int32_t> inp_pos = { (int32_t) pos };
                     set_input_i32("inp_pos", inp_pos);

-                    std::uniform_real_distribution<float> dist(0.0f, 1.0f);
-                    std::vector<float> r(model.gen_code_head_w->ne[2]);
-                    for (auto & v : r) {
-                        v = dist(params->gen_state->rng);
+                    if (forced) {
+                        std::vector<int32_t> force(model.gen_code_head_w->ne[2]);
+                        for (size_t ih = 0; ih < force.size(); ih++) {
+                            force[ih] = std::clamp<int32_t>((*params->codes)[n_cb + 1 + ih], 0, (int32_t) n_vocab - 1);
+                        }
+                        set_input_i32("inp_force", force);
+                    } else {
+                        std::uniform_real_distribution<float> dist(0.0f, 1.0f);
+                        std::vector<float> r(model.gen_code_head_w->ne[2]);
+                        for (auto & v : r) {
+                            v = dist(params->gen_state->rng);
+                        }
+                        set_input_f32("inp_rand", r);
                     }
-                    set_input_f32("inp_rand", r);
                 }
             } break;
         case PROJECTOR_TYPE_QWEN3TTS_GEN:
@@ -2596,10 +2608,8 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
     // for audio gen models
     //

-    if (model.proj_type == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
-        GGML_ASSERT(params->out_codes && params->out_code_embd);
-
-        // frame-major codes, the layout of GEN_WAV's input
+    // reference clip codes, one out_ref_code_<g> tensor per codebook, read back frame-major (the layout of GEN_WAV's input)
+    auto read_ref_codes = [&gf](std::vector<int32_t> & out) -> size_t {
         std::vector<std::vector<int32_t>> groups;
         for (int g = 0; ; g++) {
             ggml_tensor * t = ggml_graph_get_tensor(gf, ("out_ref_code_" + std::to_string(g)).c_str());
@@ -2611,12 +2621,23 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
         }
         GGML_ASSERT(!groups.empty());
         const size_t n_frames = groups[0].size();
-        params->out_codes->resize(n_frames * groups.size());
+        out.resize(n_frames * groups.size());
         for (size_t f = 0; f < n_frames; f++) {
             for (size_t g = 0; g < groups.size(); g++) {
-                (*params->out_codes)[f * groups.size() + g] = groups[g][f];
+                out[f * groups.size() + g] = groups[g][f];
             }
         }
+        return n_frames;
+    };
+
+    if (model.proj_type == PROJECTOR_TYPE_FISH_S2_SPKENC && params->out_codes) {
+        read_ref_codes(*params->out_codes);
+        return true;
+    }
+
+    if (model.proj_type == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
+        GGML_ASSERT(params->out_codes && params->out_code_embd);
+        const size_t n_frames = read_ref_codes(*params->out_codes);

         ggml_tensor * rows = ggml_graph_get_tensor(gf, "out_ref_rows");
         GGML_ASSERT(rows && rows->type == GGML_TYPE_F32 && (size_t) rows->ne[1] == n_frames);
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 511e01f93..666f48eeb 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -2617,8 +2617,9 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im
 static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd);
 static int32_t voxcpm2_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd);

-static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector<float> & out_embd) {
-    mtmd_image_f32_batch batch = batch_f32.clone();
+// copy of batch_f32, the mel is computed from source when there is one
+static bool mtmd_audio_batch_prepare(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, mtmd_image_f32_batch & batch) {
+    batch = batch_f32.clone();
     if (source) {
         if (!ctx->audio_preproc->compute(*source)) {
             LOG_ERR("%s: unable to compute the mel\n", __func__);
@@ -2627,7 +2628,38 @@ static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_bat
         GGML_ASSERT(idx < source->entries.size());
         batch.entries[0] = source->entries[idx];
     }
-    return ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd);
+    return true;
+}
+
+static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector<float> & out_embd) {
+    mtmd_image_f32_batch batch;
+    return mtmd_audio_batch_prepare(ctx, batch_f32, source, idx, batch) && ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd);
+}
+
+// fish_s2_spkenc: one backbone row per reference frame, the codes go to ctx->gen_ref, see mtmd_gen_state_set_ref()
+static int32_t mtmd_fish_s2_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd) {
+    mtmd_image_f32_batch batch;
+    if (!mtmd_audio_batch_prepare(ctx, audio_tokens->batch_f32, audio_tokens->source.get(), audio_tokens->source_idx, batch)) {
+        return 1;
+    }
+    std::vector<int32_t> codes;
+    out_embd.resize((size_t) audio_tokens->n_tokens * ctx->n_embd_out());
+
+    mtmd_subctx_encode_params params;
+    params.imgs      = &batch;
+    params.n_threads = ctx->n_threads;
+    params.out_embd  = &out_embd;
+    params.out_codes = &codes;
+    if (!ctx->ctx_a->encode(&params)) {
+        return 1;
+    }
+    const size_t n_cb = ctx->ctx_a->model.gen_code_embd_w->ne[2];
+    if (codes.size() != (size_t) audio_tokens->n_tokens * n_cb) {
+        LOG_ERR("%s: %zu reference codes for %u frames\n", __func__, codes.size(), audio_tokens->n_tokens);
+        return 1;
+    }
+    ctx->gen_ref.tokens = std::move(codes);
+    return 0;
 }

 // qwen3tts: the speaker row, then one talker input row per reference frame
@@ -2732,6 +2764,9 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk
         if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_VOXCPM2_SPKENC) {
             return voxcpm2_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
         }
+        if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_FISH_S2_SPKENC) {
+            return mtmd_fish_s2_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
+        }
         if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_QWEN3TTS_SPKENC && chunk->tokens_audio->batch_f32.entries.size() == 2) {
             return mtmd_qwen3tts_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
         }
@@ -4214,6 +4249,13 @@ int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) {
     return 0;
 }

+const int32_t * mtmd_gen_state_get_ref_codes(const mtmd_gen_state * state, size_t * n_codes) {
+    if (n_codes) {
+        *n_codes = state ? state->ref.tokens.size() : 0;
+    }
+    return state && !state->ref.tokens.empty() ? state->ref.tokens.data() : nullptr;
+}
+
 static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
     mtmd_subcontext * sctx = ctx->ctx_gen_a.get();
     if (!sctx) {
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index 49fc8136e..cf2d14ce2 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -593,6 +593,10 @@ MTMD_API void             mtmd_gen_state_reset(mtmd_gen_state * state);
 // returns non-zero if the last encode produced no such data
 MTMD_API int32_t          mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state);

+// codes of the reference held by the state (frame-major, e.g. drama3: 10 per frame), NULL if it has none
+// valid until the next mtmd_gen_state_set_ref() / mtmd_gen_state_free() on this state
+MTMD_API const int32_t *  mtmd_gen_state_get_ref_codes(const mtmd_gen_state * state, size_t * n_codes);
+
 // defaults tuned for the loaded pipeline, callers override only what they care about
 MTMD_API struct mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx);