Commit 34ff2e047 for llama.cpp
commit 34ff2e047ce520b1d83bd274bf3ff0bfbc72a69b
Author: Feiteng <lifeiteng0422@gmail.com>
Date: Sat Nov 21 01:34:52 2026 +0800
mtmd: drama3 voice cloning (#32586)
* mtmd: drama3 voice cloning
The Fish S2 codec encoder (fish_s2_spkenc) now also serves drama3:
- conversion: drama3 writes the codec encoder and the audio encoder KV, shared with fish_s2
- fish_s2_spkenc: outputs the codes of the clip (gen-side reference) and, in a drama3_gen mmproj,
builds the drama3 backbone rows (E_lm[semantic] + E_cb[1][code 1]) / sqrt(2)
- drama3_gen tower: teacher forcing when the caller gives the codes of the next frame
- drama3 pipeline: reference-bracket-v1 prompt with one reference (vLLM-Omni fish_omni), the
reference frames go through the backbone as codec rows and through the tower with their codes
- mtmd_gen_state_get_ref_codes(): read-only access to the reference codes of a gen state
Claude-Session: https://claude.ai/code/session_01PpX3DmaxNSmJbxGaxnUBrX
* fix correctness
* clean up
* rm redundant code
* fish_s2: rm unused import
---------
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
diff --git a/conversion/drama3.py b/conversion/drama3.py
index 68bbb556b..9b490900a 100644
--- a/conversion/drama3.py
+++ b/conversion/drama3.py
@@ -42,6 +42,115 @@ def fold_snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
return alpha, 1.0 / (alpha + 1e-9)
+# codec encoder strides (encoder_rates of the codec config), the quantizer then downsamples 2x2
+_CODEC_ENC_STRIDES = (2, 4, 8, 8)
+
+# codec encoder transformers, both 1024 wide with 16 heads of 64
+# generic audio layers a.blk.0..3: last encoder block, a.blk.4..11: quantizer pre_module
+_N_ENC_TFM_LAYERS = 4
+_N_PRE_TFM_LAYERS = 8
+_CODEC_HEAD_DIM = 64
+
+
+def _neox_rows(w: Tensor, head_dim: int) -> Tensor:
+ # checkpoint uses interleaved RoPE, reorder the rows of each head so NEOX RoPE gives the same result
+ order = torch.cat([torch.arange(0, head_dim, 2), torch.arange(1, head_dim, 2)])
+ return w.reshape(-1, head_dim, *w.shape[1:])[:, order].reshape(w.shape)
+
+
+def codec_encoder_tensors(sd: dict[str, Tensor], num_codebooks: int) -> Iterable[tuple[str, Tensor]]:
+ T = gguf.MODEL_TENSOR
+
+ def get(name: str) -> Tensor:
+ return sd[name].float()
+
+ def tn(tensor: gguf.MODEL_TENSOR, bid: int | None = None, xid: int | None = None) -> str:
+ return gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
+
+ def fmt(tensor: gguf.MODEL_TENSOR, bid: int | None = None, suffix: str = ".weight") -> str:
+ return tn(tensor, bid) + suffix
+
+ def conv_wb(tensor: gguf.MODEL_TENSOR, prefix: str, bid: int | None = None, xid: int | None = None, stride: int = 1):
+ # taps-major [K / stride, OC, stride * IC], see conv() in tools/mtmd/models/fish-s2-spkenc.cpp
+ w = fold_conv(sd, prefix)
+ n_out, n_in, k = w.shape
+ w = w.reshape(n_out, n_in, k // stride, stride).permute(2, 0, 3, 1).reshape(k // stride, n_out, stride * n_in)
+ yield (tn(tensor, bid, xid) + ".weight", w)
+ yield (tn(tensor, bid, xid) + ".bias", get(prefix + ".conv.bias"))
+
+ def snake_ab(tensor: gguf.MODEL_TENSOR, name: str, bid: int | None = None, xid: int | None = None):
+ a, b = fold_snake(get(name))
+ yield (tn(tensor, bid, xid) + ".alpha", a)
+ yield (tn(tensor, bid, xid) + ".beta", b)
+
+ def tfm_layers(prefix: str, n_layer: int, offset: int):
+ for il in range(n_layer):
+ p = f"{prefix}layers.{il}."
+ bid = offset + il
+ wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0)
+ yield (fmt(T.A_ENC_ATTN_Q, bid), _neox_rows(wq, _CODEC_HEAD_DIM))
+ yield (fmt(T.A_ENC_ATTN_K, bid), _neox_rows(wk, _CODEC_HEAD_DIM))
+ yield (fmt(T.A_ENC_ATTN_V, bid), wv)
+ yield (fmt(T.A_ENC_OUTPUT, bid), get(p + "attention.wo.weight"))
+ yield (fmt(T.A_ENC_INPUT_NORM, bid), get(p + "attention_norm.weight"))
+ yield (fmt(T.A_ENC_ATTN_SCALE, bid), get(p + "attention_layer_scale.gamma"))
+ yield (fmt(T.A_ENC_OUTPUT_NORM, bid), get(p + "ffn_norm.weight"))
+ yield (fmt(T.A_ENC_FFN_GATE, bid), get(p + "feed_forward.w1.weight"))
+ yield (fmt(T.A_ENC_FFN_UP, bid), get(p + "feed_forward.w3.weight"))
+ yield (fmt(T.A_ENC_FFN_DOWN, bid), get(p + "feed_forward.w2.weight"))
+ yield (fmt(T.A_ENC_FFN_SCALE_LS, bid), get(p + "ffn_layer_scale.gamma"))
+
+ # encoder: entry conv, 4x (3 residual units, Snake, strided conv), Snake, post conv
+ e = "encoder.block."
+ yield from conv_wb(T.A_CODEC_ENC_ENTRY, e + "0")
+ for bid, stride in enumerate(_CODEC_ENC_STRIDES):
+ p = f"{e}{bid + 1}.block."
+ for xid in range(3):
+ r = f"{p}{xid}.block."
+ yield from snake_ab(T.A_CODEC_ENC_RES_ACT1, r + "0.alpha", bid, xid)
+ yield from conv_wb(T.A_CODEC_ENC_RES_CONV1, r + "1", bid, xid)
+ yield from snake_ab(T.A_CODEC_ENC_RES_ACT2, r + "2.alpha", bid, xid)
+ yield from conv_wb(T.A_CODEC_ENC_RES_CONV2, r + "3", bid, xid)
+ yield from snake_ab(T.A_CODEC_ENC_SNAKE, p + "3.alpha", bid)
+ yield from conv_wb(T.A_CODEC_ENC_CONV, p + "4", bid, stride=stride)
+ # the last block ends with a windowed transformer
+ yield from tfm_layers(f"{e}4.block.5.", _N_ENC_TFM_LAYERS, 0)
+ yield (fmt(T.A_POST_NORM), get(f"{e}4.block.5.norm.weight"))
+ yield from snake_ab(T.A_CODEC_ENC_POST_SNAKE, e + "5.alpha")
+ yield from conv_wb(T.A_CODEC_ENC_POST_CONV, e + "6")
+
+ # quantizer: 2x (strided conv + ConvNeXt), windowed transformer, then semantic + residual VQ
+ q = "quantizer."
+ for bid in range(2):
+ p = f"{q}downsample.{bid}."
+ yield from conv_wb(T.A_CODEC_DOWN_CONV, p + "0", bid, stride=2)
+ # depthwise: one [C] row per tap
+ yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".weight", fold_conv(sd, p + "1.dwconv").squeeze(1).T.contiguous())
+ yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".bias", get(p + "1.dwconv.conv.bias"))
+ for key, tensor in (("norm", T.A_CODEC_DOWN_NORM), ("pwconv1", T.A_CODEC_DOWN_PW1), ("pwconv2", T.A_CODEC_DOWN_PW2)):
+ yield (tn(tensor, bid) + ".weight", get(f"{p}1.{key}.weight"))
+ yield (tn(tensor, bid) + ".bias", get(f"{p}1.{key}.bias"))
+ yield (tn(T.A_CODEC_DOWN_GAMMA, bid), get(p + "1.gamma"))
+ yield from tfm_layers(q + "pre_module.", _N_PRE_TFM_LAYERS, _N_ENC_TFM_LAYERS)
+ yield (fmt(T.A_CODEC_PRE_NORM), get(q + "pre_module.norm.weight"))
+
+ # nearest codebook entry by cosine similarity, see VectorQuantize.decode_latents()
+ def vq(prefix: str) -> tuple[Tensor, Tensor, Tensor]:
+ w = fold_wn(sd[prefix + ".in_proj.weight_g"], sd[prefix + ".in_proj.weight_v"])
+ cb = torch.nn.functional.normalize(get(prefix + ".codebook.weight"), dim=-1)
+ return w, get(prefix + ".in_proj.bias"), cb
+
+ w, b, cb = vq(q + "semantic_quantizer.quantizers.0")
+ yield (fmt(T.A_CODEC_VQ_FIRST_IN, suffix=".weight"), w)
+ yield (fmt(T.A_CODEC_VQ_FIRST_IN, suffix=".bias"), b)
+ yield (fmt(T.A_CODEC_VQ_FIRST_CB), cb)
+ n_rest = num_codebooks - 1
+ rest = [vq(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)]
+ yield (fmt(T.A_CODEC_VQ_REST_IN, suffix=".weight"), torch.stack([r[0] for r in rest]))
+ yield (fmt(T.A_CODEC_VQ_REST_IN, suffix=".bias"), torch.stack([r[1] for r in rest]))
+ yield (fmt(T.A_CODEC_VQ_REST_CB), torch.stack([r[2] for r in rest]))
+
+
@ModelBase.register("FishOmniModel")
@ModelBase.example("fishaudio/drama-3")
class Drama3TextModel(Qwen3_5MoeTextModel):
@@ -59,7 +168,7 @@ class Drama3TextModel(Qwen3_5MoeTextModel):
@ModelBase.example("fishaudio/drama-3")
class Drama3MmprojModel(MmprojModel):
has_vision_encoder = False
- has_audio_encoder = False
+ has_audio_encoder = True # codec encoder, for voice cloning
gen_projector_type = gguf.VisionProjectorType.DRAMA3_GEN
_LAYER_MAP = {
@@ -103,6 +212,16 @@ class Drama3MmprojModel(MmprojModel):
self.gguf_writer.add_gen_audio_head_count_kv(cfg["n_local_heads"])
self.gguf_writer.add_gen_audio_attention_layernorm_eps(cfg["norm_eps"])
# other tower and codec hparams are hardcoded in tools/mtmd
+ # codec encoder: raw 44.1 kHz waveform -> backbone input rows, other hparams are hardcoded in tools/mtmd
+ self.gguf_writer.add_clip_has_audio_encoder(True)
+ self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.FISH_S2_SPKENC)
+ self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
+ self.gguf_writer.add_audio_num_mel_bins(1)
+ self.gguf_writer.add_audio_block_count(_N_ENC_TFM_LAYERS + _N_PRE_TFM_LAYERS)
+ self.gguf_writer.add_audio_embedding_length(1024)
+ self.gguf_writer.add_audio_feed_forward_length(3072)
+ self.gguf_writer.add_audio_head_count(1024 // _CODEC_HEAD_DIM)
+ self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
@@ -112,6 +231,12 @@ class Drama3MmprojModel(MmprojModel):
return None
def tensor_force_quant(self, name, new_name, bid, n_dims):
+ # the VQ picks the nearest codebook entry, small errors in the encoder change the codes
+ if new_name.startswith("a.codec."):
+ return gguf.GGMLQuantizationType.F32
+ # codec transformers: F16 gives codes closer to the reference than BF16
+ if new_name.startswith("a.blk.") and n_dims == 2:
+ return gguf.GGMLQuantizationType.F16
# conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
if new_name.endswith(".weight") and (
new_name in ("a.gen.wav.dac.entry.weight", "a.gen.wav.dac.post_conv.weight")
@@ -130,7 +255,7 @@ class Drama3MmprojModel(MmprojModel):
T = gguf.MODEL_TENSOR
cfg = self.hparams
- if name.startswith("a.gen.wav."):
+ if name.startswith("a."):
yield (name, data_torch)
return
@@ -177,7 +302,7 @@ class Drama3MmprojModel(MmprojModel):
raise ValueError(f"unexpected tensor {name}")
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
- # only the DAC decoder, the encoder is for voice cloning (not supported yet)
+ # Fish DAC decoder, then the shared codec encoder for voice cloning
sd = load_codec(self.dir_model)
T = gguf.MODEL_TENSOR
@@ -262,3 +387,5 @@ class Drama3MmprojModel(MmprojModel):
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b)
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), conv(d + "6"))
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".bias"), get(d + "6.conv.bias"))
+
+ yield from codec_encoder_tensors(sd, self.hparams["num_codebooks"])
diff --git a/conversion/fish_s2.py b/conversion/fish_s2.py
index 4c5aeff13..563b52b12 100644
--- a/conversion/fish_s2.py
+++ b/conversion/fish_s2.py
@@ -3,13 +3,11 @@ from __future__ import annotations
from pathlib import Path
from typing import Any, Callable, Iterable, TYPE_CHECKING
-import torch
-
if TYPE_CHECKING:
from torch import Tensor
from .base import ModelBase, gguf
-from .drama3 import Drama3MmprojModel, fold_conv, fold_snake, fold_wn, load_codec
+from .drama3 import Drama3MmprojModel, _neox_rows
from .qwen import Qwen3Model
# Fish Audio S2 Pro (dual-AR)
@@ -19,15 +17,6 @@ from .qwen import Qwen3Model
# residual codebook size of codec.pth, the fast AR head is cut to it
_N_RESIDUAL_CODES = 1024
-# codec encoder strides (encoder_rates of the codec config), the quantizer then downsamples 2x2
-_CODEC_ENC_STRIDES = (2, 4, 8, 8)
-
-# codec encoder transformers, both 1024 wide with 16 heads of 64
-# generic audio layers a.blk.0..3: last encoder block, a.blk.4..11: quantizer pre_module
-_N_ENC_TFM_LAYERS = 4
-_N_PRE_TFM_LAYERS = 8
-_CODEC_HEAD_DIM = 64
-
def _load_hparams(dir_model: Path) -> dict[str, Any]:
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
@@ -48,12 +37,6 @@ def _load_hparams(dir_model: Path) -> dict[str, Any]:
return hparams
-def _neox_rows(w: Tensor, head_dim: int) -> Tensor:
- # checkpoint uses interleaved RoPE, reorder the rows of each head so NEOX RoPE gives the same result
- order = torch.cat([torch.arange(0, head_dim, 2), torch.arange(1, head_dim, 2)])
- return w.reshape(-1, head_dim, *w.shape[1:])[:, order].reshape(w.shape)
-
-
@ModelBase.register("fish_qwen3_omni")
@ModelBase.example("fishaudio/s2-pro")
class FishS2TextModel(Qwen3Model):
@@ -88,7 +71,6 @@ class FishS2TextModel(Qwen3Model):
@ModelBase.register("fish_qwen3_omni")
@ModelBase.example("fishaudio/s2-pro")
class FishS2MmprojModel(Drama3MmprojModel):
- has_audio_encoder = True # codec encoder, for voice cloning
gen_projector_type = gguf.VisionProjectorType.FISH_S2_GEN
_FAST_AR_MAP = {
@@ -109,28 +91,6 @@ class FishS2MmprojModel(Drama3MmprojModel):
cfg["num_hidden_layers"] = cfg["n_layer"]
return cfg
- def set_gguf_parameters(self):
- super().set_gguf_parameters()
- # codec encoder: raw 44.1 kHz waveform -> backbone input rows, other hparams are hardcoded in tools/mtmd
- self.gguf_writer.add_clip_has_audio_encoder(True)
- self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.FISH_S2_SPKENC)
- self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
- self.gguf_writer.add_audio_num_mel_bins(1)
- self.gguf_writer.add_audio_block_count(_N_ENC_TFM_LAYERS + _N_PRE_TFM_LAYERS)
- self.gguf_writer.add_audio_embedding_length(1024)
- self.gguf_writer.add_audio_feed_forward_length(3072)
- self.gguf_writer.add_audio_head_count(1024 // _CODEC_HEAD_DIM)
- self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
-
- def tensor_force_quant(self, name, new_name, bid, n_dims):
- # the VQ picks the nearest codebook entry, small errors in the encoder change the codes
- if new_name.startswith("a.codec."):
- return gguf.GGMLQuantizationType.F32
- # codec transformers: F16 gives codes closer to the reference than BF16
- if new_name.startswith("a.blk.") and n_dims == 2:
- return gguf.GGMLQuantizationType.F16
- return super().tensor_force_quant(name, new_name, bid, n_dims)
-
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name = item[0]
@@ -182,95 +142,3 @@ class FishS2MmprojModel(Drama3MmprojModel):
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, bid), v)
return
yield (self.format_tensor_name(self._FAST_AR_MAP[key], bid), data_torch)
-
- def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
- yield from super().generate_extra_tensors()
-
- sd = load_codec(self.dir_model)
- T = gguf.MODEL_TENSOR
-
- def get(name: str) -> Tensor:
- return sd[name].float()
-
- def tn(tensor: gguf.MODEL_TENSOR, bid: int | None = None, xid: int | None = None) -> str:
- return gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
-
- def conv_wb(tensor: gguf.MODEL_TENSOR, prefix: str, bid: int | None = None, xid: int | None = None, stride: int = 1):
- # taps-major [K / stride, OC, stride * IC], see conv() in tools/mtmd/models/fish-s2-spkenc.cpp
- w = fold_conv(sd, prefix)
- n_out, n_in, k = w.shape
- w = w.reshape(n_out, n_in, k // stride, stride).permute(2, 0, 3, 1).reshape(k // stride, n_out, stride * n_in)
- yield (tn(tensor, bid, xid) + ".weight", w)
- yield (tn(tensor, bid, xid) + ".bias", get(prefix + ".conv.bias"))
-
- def snake_ab(tensor: gguf.MODEL_TENSOR, name: str, bid: int | None = None, xid: int | None = None):
- a, b = fold_snake(get(name))
- yield (tn(tensor, bid, xid) + ".alpha", a)
- yield (tn(tensor, bid, xid) + ".beta", b)
-
- def tfm_layers(prefix: str, n_layer: int, offset: int):
- for il in range(n_layer):
- p = f"{prefix}layers.{il}."
- bid = offset + il
- wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0)
- yield (self.format_tensor_name(T.A_ENC_ATTN_Q, bid), _neox_rows(wq, _CODEC_HEAD_DIM))
- yield (self.format_tensor_name(T.A_ENC_ATTN_K, bid), _neox_rows(wk, _CODEC_HEAD_DIM))
- yield (self.format_tensor_name(T.A_ENC_ATTN_V, bid), wv)
- yield (self.format_tensor_name(T.A_ENC_OUTPUT, bid), get(p + "attention.wo.weight"))
- yield (self.format_tensor_name(T.A_ENC_INPUT_NORM, bid), get(p + "attention_norm.weight"))
- yield (self.format_tensor_name(T.A_ENC_ATTN_SCALE, bid), get(p + "attention_layer_scale.gamma"))
- yield (self.format_tensor_name(T.A_ENC_OUTPUT_NORM, bid), get(p + "ffn_norm.weight"))
- yield (self.format_tensor_name(T.A_ENC_FFN_GATE, bid), get(p + "feed_forward.w1.weight"))
- yield (self.format_tensor_name(T.A_ENC_FFN_UP, bid), get(p + "feed_forward.w3.weight"))
- yield (self.format_tensor_name(T.A_ENC_FFN_DOWN, bid), get(p + "feed_forward.w2.weight"))
- yield (self.format_tensor_name(T.A_ENC_FFN_SCALE_LS, bid), get(p + "ffn_layer_scale.gamma"))
-
- # encoder: entry conv, 4x (3 residual units, Snake, strided conv), Snake, post conv
- e = "encoder.block."
- yield from conv_wb(T.A_CODEC_ENC_ENTRY, e + "0")
- for bid, stride in enumerate(_CODEC_ENC_STRIDES):
- p = f"{e}{bid + 1}.block."
- for xid in range(3):
- r = f"{p}{xid}.block."
- yield from snake_ab(T.A_CODEC_ENC_RES_ACT1, r + "0.alpha", bid, xid)
- yield from conv_wb(T.A_CODEC_ENC_RES_CONV1, r + "1", bid, xid)
- yield from snake_ab(T.A_CODEC_ENC_RES_ACT2, r + "2.alpha", bid, xid)
- yield from conv_wb(T.A_CODEC_ENC_RES_CONV2, r + "3", bid, xid)
- yield from snake_ab(T.A_CODEC_ENC_SNAKE, p + "3.alpha", bid)
- yield from conv_wb(T.A_CODEC_ENC_CONV, p + "4", bid, stride=stride)
- # the last block ends with a windowed transformer
- yield from tfm_layers(f"{e}4.block.5.", _N_ENC_TFM_LAYERS, 0)
- yield (self.format_tensor_name(T.A_POST_NORM), get(f"{e}4.block.5.norm.weight"))
- yield from snake_ab(T.A_CODEC_ENC_POST_SNAKE, e + "5.alpha")
- yield from conv_wb(T.A_CODEC_ENC_POST_CONV, e + "6")
-
- # quantizer: 2x (strided conv + ConvNeXt), windowed transformer, then semantic + residual VQ
- q = "quantizer."
- for bid in range(2):
- p = f"{q}downsample.{bid}."
- yield from conv_wb(T.A_CODEC_DOWN_CONV, p + "0", bid, stride=2)
- # depthwise: one [C] row per tap
- yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".weight", fold_conv(sd, p + "1.dwconv").squeeze(1).T.contiguous())
- yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".bias", get(p + "1.dwconv.conv.bias"))
- for key, tensor in (("norm", T.A_CODEC_DOWN_NORM), ("pwconv1", T.A_CODEC_DOWN_PW1), ("pwconv2", T.A_CODEC_DOWN_PW2)):
- yield (tn(tensor, bid) + ".weight", get(f"{p}1.{key}.weight"))
- yield (tn(tensor, bid) + ".bias", get(f"{p}1.{key}.bias"))
- yield (tn(T.A_CODEC_DOWN_GAMMA, bid), get(p + "1.gamma"))
- yield from tfm_layers(q + "pre_module.", _N_PRE_TFM_LAYERS, _N_ENC_TFM_LAYERS)
- yield (self.format_tensor_name(T.A_CODEC_PRE_NORM), get(q + "pre_module.norm.weight"))
-
- # nearest codebook entry by cosine similarity, see VectorQuantize.decode_latents()
- def vq(prefix: str) -> tuple[Tensor, Tensor, Tensor]:
- w = fold_wn(sd[prefix + ".in_proj.weight_g"], sd[prefix + ".in_proj.weight_v"])
- cb = torch.nn.functional.normalize(get(prefix + ".codebook.weight"), dim=-1)
- return w, get(prefix + ".in_proj.bias"), cb
-
- w, b, cb = vq(q + "semantic_quantizer.quantizers.0")
- yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".weight"), w)
- yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".bias"), b)
- yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_CB), cb)
- n_rest = self.hparams["num_codebooks"] - 1
- rest = [vq(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)]
- yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".weight"), torch.stack([r[0] for r in rest]))
- yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".bias"), torch.stack([r[1] for r in rest]))
- yield (self.format_tensor_name(T.A_CODEC_VQ_REST_CB), torch.stack([r[2] for r in rest]))
diff --git a/docs/multimodal/tts.md b/docs/multimodal/tts.md
index 48c6d72e5..e009566c0 100644
--- a/docs/multimodal/tts.md
+++ b/docs/multimodal/tts.md
@@ -175,7 +175,7 @@ python convert_hf_to_gguf.py path/to/chatterbox --mmproj --outfile mmproj-chatte
Available params:
- Speaker tags `<|speaker:0|>` to `<|speaker:15|>` in the text give each character its own voice, a text without a tag starts with `<|speaker:0|>`
- The language is the one of the text, `--tts-lang` is not used
-- Note: `--tts-speaker-file` is not supported yet (voice cloning)
+- Voice cloning: `--tts-speaker-file` with the reference clip and `--tts-speaker-text` with its transcript (a transcript without a speaker tag is speaker 0). The clip is encoded by the codec encoder half of `codec.pth` and its codes are put in the prompt as one reference, as the reference does; the clip is truncated to the first 30 s. It needs an mmproj converted with the codec encoder
- Note: `--top-k` and `--top-p` are not used, frames are sampled with the settings of the reference (main token: temperature 1.0, top-p 0.9, repetition-aware redraw over the last 16 frames; other codes: temperature 1.0, top-k 30, top-p 0.9)
- Each frame is 2048 samples (about 46 ms), generation stops at the end-of-speech token
- The codebook tower keeps the whole sequence in its KV cache, the prompt and the generated frames are limited to 4096 positions
@@ -192,7 +192,7 @@ llama-tts -m drama3.gguf \
**Note for GGUF conversion:**
-The routed experts of the checkpoint are FP8, they are dequantized at conversion. The mmproj holds the codebook tower and the decoder half of `codec.pth`, it is best kept in BF16:
+The routed experts of the checkpoint are FP8, they are dequantized at conversion. The mmproj holds the codebook tower and `codec.pth` (the encoder half for voice cloning, the decoder half for the audio), it is best kept in BF16; the codec encoder is always stored in F16/F32:
```sh
python convert_hf_to_gguf.py path/to/drama-3 --outtype q8_0 --outfile drama3.gguf
diff --git a/tools/mtmd/models/drama3-gen.cpp b/tools/mtmd/models/drama3-gen.cpp
index e77fcf540..41d8c22b1 100644
--- a/tools/mtmd/models/drama3-gen.cpp
+++ b/tools/mtmd/models/drama3-gen.cpp
@@ -130,9 +130,19 @@ ggml_cgraph * mtmd_graph_drama3_gen::build_tower() {
ggml_set_name(inp_pos, "inp_pos");
ggml_set_input(inp_pos);
- ggml_tensor * inp_rand = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_heads);
- ggml_set_name(inp_rand, "inp_rand");
- ggml_set_input(inp_rand);
+ // teacher forcing (reference rows of a voice clone): codes 1.. of the frame are given, nothing is sampled
+ const bool forced = n_frames == 2;
+ ggml_tensor * inp_rand = nullptr;
+ ggml_tensor * inp_force = nullptr;
+ if (forced) {
+ inp_force = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_heads);
+ ggml_set_name(inp_force, "inp_force");
+ ggml_set_input(inp_force);
+ } else {
+ inp_rand = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_heads);
+ ggml_set_name(inp_rand, "inp_rand");
+ ggml_set_input(inp_rand);
+ }
ggml_tensor * mask = ggml_view_1d(ctx0, inp_scale, 1, 0);
ggml_tensor * extra_on = ggml_view_1d(ctx0, inp_scale, 1, sizeof(float));
@@ -152,6 +162,11 @@ ggml_cgraph * mtmd_graph_drama3_gen::build_tower() {
continue;
}
const int ih = il - n_shared;
+ if (forced) {
+ code = ggml_view_1d(ctx0, inp_force, 1, (size_t) ih * sizeof(int32_t));
+ out_codes = cg.cache_set(out_codes, ih + 1, code);
+ continue;
+ }
ggml_tensor * head = ggml_view_2d(ctx0, model.gen_code_head_w, model.gen_code_head_w->ne[0], model.gen_code_head_w->ne[1],
model.gen_code_head_w->nb[1], (size_t) ih * model.gen_code_head_w->nb[2]);
ggml_tensor * logits = ggml_mul_mat(ctx0, head, x);
diff --git a/tools/mtmd/models/fish-s2-spkenc.cpp b/tools/mtmd/models/fish-s2-spkenc.cpp
index a9c51d9cd..fe2eb28ef 100644
--- a/tools/mtmd/models/fish-s2-spkenc.cpp
+++ b/tools/mtmd/models/fish-s2-spkenc.cpp
@@ -2,6 +2,7 @@
#include <cmath>
#include <string>
+#include <vector>
// reference clip -> codec frames -> backbone input rows, see DAC.encode() and DownsampleResidualVectorQuantize.encode()
// activations are [C, T]; a conv is a sum of matmuls over tap-shifted views: exact in F32, no im2col buffer
@@ -117,17 +118,35 @@ ggml_cgraph * mtmd_graph_fish_s2_spkenc::build() {
return ggml_view_2d(ctx0, t, t->ne[0], t->ne[1], t->nb[1], (size_t) g * t->nb[2]);
};
- // backbone row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1), same as fish_s2_gen
const int64_t n_cb = model.gen_code_embd_w->ne[2];
- ggml_tensor * idx = vq(c2w.quant_first_in_w, c2w.quant_first_in_b, c2w.quant_first_cb_norm_w, c2w.quant_first_cb_w);
- ggml_tensor * out = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, idx),
- ggml_get_rows(ctx0, page(model.gen_code_embd_w, 0), idx));
+ std::vector<ggml_tensor *> idx(n_cb);
+ idx[0] = vq(c2w.quant_first_in_w, c2w.quant_first_in_b, c2w.quant_first_cb_norm_w, c2w.quant_first_cb_w);
for (int64_t g = 1; g < n_cb; g++) {
ggml_tensor * in_b = ggml_view_1d(ctx0, c2w.quant_rest_in_b, c2w.quant_rest_in_b->ne[0], (size_t) (g - 1) * c2w.quant_rest_in_b->nb[1]);
- idx = vq(page(c2w.quant_rest_in_w, g - 1), in_b, page(c2w.quant_rest_cb_norm_w, g - 1), page(c2w.quant_rest_cb_w, g - 1));
- out = ggml_add(ctx0, out, ggml_get_rows(ctx0, page(model.gen_code_embd_w, g), idx));
+ idx[g] = vq(page(c2w.quant_rest_in_w, g - 1), in_b, page(c2w.quant_rest_cb_norm_w, g - 1), page(c2w.quant_rest_cb_w, g - 1));
+ }
+ // codes of the clip, read back as the gen-side reference
+ for (int64_t g = 0; g < n_cb; g++) {
+ ggml_set_name(idx[g], ("out_ref_code_" + std::to_string(g)).c_str());
+ ggml_set_output(idx[g]);
+ ggml_build_forward_expand(gf, idx[g]);
+ }
+
+ ggml_tensor * out = nullptr;
+ if (hparams.spkenc_drama3_rows) {
+ // drama3 backbone row: (E_lm[semantic] + E_cb[1][code 1]) / sqrt(2), see mtmd_graph_drama3_gen::build_tower()
+ out = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, idx[0]),
+ ggml_get_rows(ctx0, page(model.gen_code_embd_w, 1), idx[1]));
+ out = ggml_scale(ctx0, out, 1.0f / sqrtf(2.0f));
+ } else {
+ // backbone row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1), same as fish_s2_gen
+ out = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, idx[0]),
+ ggml_get_rows(ctx0, page(model.gen_code_embd_w, 0), idx[0]));
+ for (int64_t g = 1; g < n_cb; g++) {
+ out = ggml_add(ctx0, out, ggml_get_rows(ctx0, page(model.gen_code_embd_w, g), idx[g]));
+ }
+ out = ggml_scale(ctx0, out, 1.0f / sqrtf((float) (n_cb + 1)));
}
- out = ggml_scale(ctx0, out, 1.0f / sqrtf((float) (n_cb + 1)));
cb(out, "spk_rows", -1);
ggml_build_forward_expand(gf, out);
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 64f7bf70d..fb75270ad 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -625,7 +625,7 @@ struct mtmd_graph_drama3_gen : mtmd_graph {
mtmd_subctx_gen_process gen_process;
mtmd_subctx_gen_state * gen_state; // nullptr on buffer reserve
- int n_frames; // GEN_WAV only
+ int n_frames; // GEN_WAV: frames to decode; GEN_CODE: 2 when the caller gives the codes of the frame (teacher forcing)
int top_k;
float top_p;
float temp;
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 32f1d39d4..3c33eff3a 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -2739,6 +2739,9 @@ public:
seq_id = 0;
pos = 0;
prompt_tokens.clear();
+ ref_codes.clear();
+ ref_embd.clear();
+ ref_begin = 0;
prompt_pos = 0;
frame.clear();
main_logits.clear();
@@ -2758,8 +2761,16 @@ public:
seq_id = inp->seq_id;
out_type = inp->out_type;
stream = inp->stream;
- if (inp->speaker_ref) {
- LOG_WRN("mtmd_helper_gen_audio: voice cloning is not supported yet, the speaker file is ignored\n");
+
+ // voice cloning: backbone rows from the codec encoder, the codes of the clip go to the gen state
+ std::vector<float> ref_rows;
+ if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref) && !mtmd_support_audio(mctx)) {
+ LOG_WRN("mtmd_helper_gen_audio: mmproj has no speaker encoder (re-convert it for voice cloning), the speaker file is ignored\n");
+ } else if (inp->speaker_ref) {
+ if (!encode_speaker(inp->speaker_ref, ref_rows) || !init_gen_state(inp->seed) ||
+ mtmd_gen_state_set_ref(mctx, gen_state.get()) != 0) {
+ return 1;
+ }
}
sem_begin = find_special_token(vocab, "<|semantic:0|>");
@@ -2770,7 +2781,12 @@ public:
return 1;
}
- if (!tokenize_segments(prompt_segments(std::string(inp->prompt, inp->prompt_len)), prompt_tokens)) {
+ const std::string text(inp->prompt, inp->prompt_len);
+ if (ref_rows.empty()) {
+ if (!tokenize_segments(prompt_segments(text), prompt_tokens)) {
+ return 1;
+ }
+ } else if (!build_reference_prompt(text, inp->speaker_ref_text ? inp->speaker_ref_text : "", std::move(ref_rows))) {
return 1;
}
if (prompt_tokens.size() >= llama_n_ctx_seq(lctx) || prompt_tokens.size() > max_rows()) {
@@ -2782,7 +2798,7 @@ public:
return 1;
}
rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
- return init_gen_state(inp->seed) ? 0 : 1;
+ return gen_state || init_gen_state(inp->seed) ? 0 : 1;
}
// run the tower on every prompt row to fill its KV
@@ -2792,27 +2808,41 @@ public:
if (prompt_pos >= n_prompt) {
return 0;
}
- // small batches: every row also outputs logits
- const int n = std::min({ n_batch, n_prompt - prompt_pos, 64 });
- llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
- if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) {
- return -1;
+ // small batches of rows of one kind (text tokens or reference frames): every row also outputs logits
+ const bool speech = is_speech_row(prompt_pos);
+ int n = std::min({ n_batch, n_prompt - prompt_pos, 64 });
+ for (int i = 1; i < n; i++) {
+ if (is_speech_row(prompt_pos + i) != speech) {
+ n = i;
+ break;
+ }
}
- for (int i = 0; i < n; i++) {
- llama_batch_ext_set_output_logits(batch.get(), i, true);
+ int ret = 0;
+ if (speech) {
+ ret = decode_embd_rows(ref_embd.data() + (size_t) (prompt_pos - ref_begin) * n_embd, n);
+ } else {
+ llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+ if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) {
+ return -1;
+ }
+ for (int i = 0; i < n; i++) {
+ llama_batch_ext_set_output_logits(batch.get(), i, true);
+ }
+ ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
}
- if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
+ if (ret != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
for (int i = 0; i < n; i++) {
+ const int row = prompt_pos + i;
const float * h = llama_get_embeddings_ith(lctx, i);
- if (prompt_pos + i == n_prompt - 1) {
+ if (row == n_prompt - 1) {
h_state_buf.assign(h, h + n_embd); // first frame comes from the last prompt row
save_main_logits(i);
break;
}
- if (!run_tower(-1, h)) {
+ if (!run_prompt_tower(row, h)) {
return -1;
}
}
@@ -2845,17 +2875,7 @@ public:
}
codes_buf.insert(codes_buf.end(), frame.begin(), frame.end());
- // text/speech stream: all mrope sections use the same position
- const bool mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE ||
- llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE;
- decode_embd_batch batch(next_embd, 1, mrope ? 4 : 1, n_embd);
- if (mrope) {
- batch.set_position_mrope_1d(pos, seq_id);
- } else {
- batch.set_position_normal(pos, seq_id);
- }
- batch.logits[0] = 1;
- if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) {
+ if (decode_embd_rows(next_embd, 1) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return 1;
}
@@ -2938,9 +2958,24 @@ protected:
return true;
}
+ // offset of the leading speaker tag <|speaker:N|> after optional whitespace, npos without one
+ static size_t speaker_tag_pos(const std::string & text) {
+ const size_t lead = text.find_first_not_of(" \t\n\r\f\v");
+ static const std::string open = "<|speaker:";
+ if (lead == std::string::npos || text.compare(lead, open.size(), open) != 0) {
+ return std::string::npos;
+ }
+ size_t i = lead + open.size();
+ const size_t digits = i;
+ while (i < text.size() && text[i] >= '0' && text[i] <= '9') {
+ i++;
+ }
+ return i > digits && text.compare(i, 2, "|>") == 0 ? lead : std::string::npos;
+ }
+
// reference-bracket-v1 prompt without references
virtual std::vector<std::string> prompt_segments(std::string text) const {
- if (text.rfind("<|speaker:", 0) != 0) {
+ if (speaker_tag_pos(text) == std::string::npos) {
text = "<|speaker:0|> " + text;
}
return {
@@ -2952,6 +2987,86 @@ protected:
// prompt rows but the last + generated frames
virtual size_t max_rows() const { return n_tower_rows; }
+ // n speech rows from pos, every row outputs logits
+ // text/speech stream: all mrope sections use the same position
+ int32_t decode_embd_rows(const float * embd, int n) {
+ const bool mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE ||
+ llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE;
+ decode_embd_batch batch(embd, n, mrope ? 4 : 1, n_embd);
+ if (mrope) {
+ batch.set_position_mrope_1d(pos, seq_id);
+ } else {
+ batch.set_position_normal(pos, seq_id);
+ }
+ for (int i = 0; i < n; i++) {
+ batch.logits[i] = 1;
+ }
+ return llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n));
+ }
+
+ bool is_speech_row(int row) const {
+ return row >= ref_begin && (size_t) (row - ref_begin) < ref_embd.size() / n_embd;
+ }
+
+ // reference-bracket-v1 with one reference, see vLLM-Omni fish_omni/prompt.py
+ // system: instruction, "\n\n[Reference 1]\nText: <transcript>\nSpeech: " then one row per frame of the clip
+ bool build_reference_prompt(std::string text, std::string ref_text, std::vector<float> && ref_rows) {
+ size_t n_codes = 0;
+ const int32_t * codes = mtmd_gen_state_get_ref_codes(gen_state.get(), &n_codes);
+ const size_t n_frames = ref_rows.size() / n_embd;
+ if (!codes || n_frames == 0 || n_codes != n_frames * n_cb || ref_rows.size() != n_frames * n_embd) {
+ LOG_ERR("mtmd_helper_gen_audio: the speaker encoder gave no codes for the clip\n");
+ return false;
+ }
+ // the speaker tag follows "Text: " directly (default speaker 0), an untagged target reuses it
+ std::string tag = "<|speaker:0|>";
+ const size_t lead = speaker_tag_pos(ref_text);
+ if (lead == std::string::npos) {
+ ref_text = tag + " " + ref_text;
+ } else {
+ ref_text = ref_text.substr(lead);
+ tag = ref_text.substr(0, ref_text.find("|>") + 2);
+ }
+ if (speaker_tag_pos(text) == std::string::npos) {
+ text = tag + " " + text;
+ }
+
+ std::vector<llama_token> head, tail;
+ if (!tokenize_segments({ "<|im_start|>system\n", "convert the provided text to speech reference to the following:",
+ "\n\n[Reference 1]\nText: " + ref_text + "\nSpeech: " }, head) ||
+ !tokenize_segments({ "<|im_end|>\n", "<|im_start|>user\n", text, "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>" }, tail)) {
+ return false;
+ }
+ prompt_tokens.assign(head.begin(), head.end());
+ prompt_tokens.insert(prompt_tokens.end(), n_frames, LLAMA_TOKEN_NULL);
+ prompt_tokens.insert(prompt_tokens.end(), tail.begin(), tail.end());
+ ref_codes.assign(codes, codes + n_codes);
+ ref_embd = std::move(ref_rows);
+ ref_begin = (int) head.size();
+ return true;
+ }
+
+ // tower row of a prompt row: the input frame is the codes of this row (reference frame) or none,
+ // the codes of the next row are forced when it is a reference frame
+ bool run_prompt_tower(int row, const float * h) {
+ const bool cur = is_speech_row(row);
+ const bool next = is_speech_row(row + 1);
+ const auto ref_frame = [&](int r) { return ref_codes.begin() + (size_t) (r - ref_begin) * n_cb; };
+ if (next) {
+ // [input frame, -1 for a text row | forced next frame]
+ frame.assign(n_cb, -1);
+ if (cur) {
+ std::copy(ref_frame(row), ref_frame(row + 1), frame.begin());
+ }
+ frame.insert(frame.end(), ref_frame(row + 1), ref_frame(row + 2));
+ } else if (cur) {
+ frame.assign(ref_frame(row), ref_frame(row + 1));
+ }
+ const bool ok = run_tower(next ? frame[n_cb] : -1, h) != nullptr;
+ frame.clear();
+ return ok;
+ }
+
// copy right after our own decode, other sequences may decode on lctx before the next step_gen
// layout: eos, then semantic codes 0..n_cb_vocab-1
void save_main_logits(int32_t i) {
@@ -3072,7 +3187,10 @@ protected:
llama_seq_id seq_id = 0;
llama_pos pos = 0;
- std::vector<llama_token> prompt_tokens;
+ std::vector<llama_token> prompt_tokens; // one per prompt row, LLAMA_TOKEN_NULL for a reference frame
+ std::vector<int32_t> ref_codes; // codes of the reference frames, n_cb per frame
+ std::vector<float> ref_embd; // backbone rows of the reference frames
+ int ref_begin = 0; // prompt row of the first reference frame
int prompt_pos = 0;
std::mt19937 rng;
diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp
index e26824d0f..2f4f40591 100644
--- a/tools/mtmd/mtmd-model.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -1171,6 +1171,10 @@ struct mtmd_model_loader {
hparams.wav_tfm_n_head_kv = 16;
hparams.wav_tfm_eps = 1e-5f;
hparams.wav_tfm_rope_theta = 10000.0f;
+ // drama3 shares the codec, only the backbone rows of a frame differ
+ std::string gen_proj;
+ get_string(KEY_GEN_AUDIO_PROJ_TYPE, gen_proj, false);
+ hparams.spkenc_drama3_rows = projector_type_from_string(gen_proj) == PROJECTOR_TYPE_DRAMA3_GEN;
} break;
case PROJECTOR_TYPE_CHATTERBOX:
{
diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h
index 893578b03..10c3154e8 100644
--- a/tools/mtmd/mtmd-model.h
+++ b/tools/mtmd/mtmd-model.h
@@ -239,6 +239,9 @@ struct mtmd_hparams {
int32_t wav_tfm_swa = 0; // pre_transformer's KV cache size, in frames
int32_t wav_frame_stack = 0; // lfm3a codec: decoder positions per frame
+ // fish_s2_spkenc in a drama3_gen mmproj: backbone rows as drama3_gen builds them
+ bool spkenc_drama3_rows = false;
+
// mimo-v2.5: LLM-side connector (input_local_transformer)
int32_t audio_local_n_layer = 0;
int32_t audio_local_group_size = 0;
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index f754a913b..aea0c1b92 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -2008,12 +2008,16 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
} else {
// code0 < 0: next row is text (prompt), the tower only fills its KV
// codes: input frame codes for a speech row, empty for a text row
+ // then optionally the codes of the next frame (reference rows): teacher forcing, code0 must be its code 0
const bool next_speech = params->code0 >= 0;
- const bool cur_speech = params->codes && (int64_t) params->codes->size() == n_cb;
- if (params->code0 >= n_vocab) {
- LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) n_vocab);
+ const size_t n_given = params->codes ? params->codes->size() : 0;
+ const bool forced = n_given == (size_t) (2 * n_cb);
+ if (params->code0 >= n_vocab || (n_given != 0 && n_given != (size_t) n_cb && !forced) ||
+ (forced && params->code0 != (*params->codes)[n_cb])) {
+ LOG_ERR("%s: invalid tower input (code0 %d, %zu codes)\n", __func__, params->code0, n_given);
return false;
}
+ const bool cur_speech = n_given != 0 && (*params->codes)[0] >= 0;
std::vector<int32_t> code0 = { std::max(params->code0, 0) };
set_input_i32("inp_code0", code0);
@@ -2030,12 +2034,20 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
std::vector<int32_t> inp_pos = { (int32_t) pos };
set_input_i32("inp_pos", inp_pos);
- std::uniform_real_distribution<float> dist(0.0f, 1.0f);
- std::vector<float> r(model.gen_code_head_w->ne[2]);
- for (auto & v : r) {
- v = dist(params->gen_state->rng);
+ if (forced) {
+ std::vector<int32_t> force(model.gen_code_head_w->ne[2]);
+ for (size_t ih = 0; ih < force.size(); ih++) {
+ force[ih] = std::clamp<int32_t>((*params->codes)[n_cb + 1 + ih], 0, (int32_t) n_vocab - 1);
+ }
+ set_input_i32("inp_force", force);
+ } else {
+ std::uniform_real_distribution<float> dist(0.0f, 1.0f);
+ std::vector<float> r(model.gen_code_head_w->ne[2]);
+ for (auto & v : r) {
+ v = dist(params->gen_state->rng);
+ }
+ set_input_f32("inp_rand", r);
}
- set_input_f32("inp_rand", r);
}
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
@@ -2596,10 +2608,8 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
// for audio gen models
//
- if (model.proj_type == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
- GGML_ASSERT(params->out_codes && params->out_code_embd);
-
- // frame-major codes, the layout of GEN_WAV's input
+ // reference clip codes, one out_ref_code_<g> tensor per codebook, read back frame-major (the layout of GEN_WAV's input)
+ auto read_ref_codes = [&gf](std::vector<int32_t> & out) -> size_t {
std::vector<std::vector<int32_t>> groups;
for (int g = 0; ; g++) {
ggml_tensor * t = ggml_graph_get_tensor(gf, ("out_ref_code_" + std::to_string(g)).c_str());
@@ -2611,12 +2621,23 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
}
GGML_ASSERT(!groups.empty());
const size_t n_frames = groups[0].size();
- params->out_codes->resize(n_frames * groups.size());
+ out.resize(n_frames * groups.size());
for (size_t f = 0; f < n_frames; f++) {
for (size_t g = 0; g < groups.size(); g++) {
- (*params->out_codes)[f * groups.size() + g] = groups[g][f];
+ out[f * groups.size() + g] = groups[g][f];
}
}
+ return n_frames;
+ };
+
+ if (model.proj_type == PROJECTOR_TYPE_FISH_S2_SPKENC && params->out_codes) {
+ read_ref_codes(*params->out_codes);
+ return true;
+ }
+
+ if (model.proj_type == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
+ GGML_ASSERT(params->out_codes && params->out_code_embd);
+ const size_t n_frames = read_ref_codes(*params->out_codes);
ggml_tensor * rows = ggml_graph_get_tensor(gf, "out_ref_rows");
GGML_ASSERT(rows && rows->type == GGML_TYPE_F32 && (size_t) rows->ne[1] == n_frames);
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 511e01f93..666f48eeb 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -2617,8 +2617,9 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im
static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd);
static int32_t voxcpm2_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd);
-static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector<float> & out_embd) {
- mtmd_image_f32_batch batch = batch_f32.clone();
+// copy of batch_f32, the mel is computed from source when there is one
+static bool mtmd_audio_batch_prepare(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, mtmd_image_f32_batch & batch) {
+ batch = batch_f32.clone();
if (source) {
if (!ctx->audio_preproc->compute(*source)) {
LOG_ERR("%s: unable to compute the mel\n", __func__);
@@ -2627,7 +2628,38 @@ static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_bat
GGML_ASSERT(idx < source->entries.size());
batch.entries[0] = source->entries[idx];
}
- return ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd);
+ return true;
+}
+
+static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector<float> & out_embd) {
+ mtmd_image_f32_batch batch;
+ return mtmd_audio_batch_prepare(ctx, batch_f32, source, idx, batch) && ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd);
+}
+
+// fish_s2_spkenc: one backbone row per reference frame, the codes go to ctx->gen_ref, see mtmd_gen_state_set_ref()
+static int32_t mtmd_fish_s2_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd) {
+ mtmd_image_f32_batch batch;
+ if (!mtmd_audio_batch_prepare(ctx, audio_tokens->batch_f32, audio_tokens->source.get(), audio_tokens->source_idx, batch)) {
+ return 1;
+ }
+ std::vector<int32_t> codes;
+ out_embd.resize((size_t) audio_tokens->n_tokens * ctx->n_embd_out());
+
+ mtmd_subctx_encode_params params;
+ params.imgs = &batch;
+ params.n_threads = ctx->n_threads;
+ params.out_embd = &out_embd;
+ params.out_codes = &codes;
+ if (!ctx->ctx_a->encode(¶ms)) {
+ return 1;
+ }
+ const size_t n_cb = ctx->ctx_a->model.gen_code_embd_w->ne[2];
+ if (codes.size() != (size_t) audio_tokens->n_tokens * n_cb) {
+ LOG_ERR("%s: %zu reference codes for %u frames\n", __func__, codes.size(), audio_tokens->n_tokens);
+ return 1;
+ }
+ ctx->gen_ref.tokens = std::move(codes);
+ return 0;
}
// qwen3tts: the speaker row, then one talker input row per reference frame
@@ -2732,6 +2764,9 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk
if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_VOXCPM2_SPKENC) {
return voxcpm2_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
}
+ if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_FISH_S2_SPKENC) {
+ return mtmd_fish_s2_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
+ }
if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_QWEN3TTS_SPKENC && chunk->tokens_audio->batch_f32.entries.size() == 2) {
return mtmd_qwen3tts_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
}
@@ -4214,6 +4249,13 @@ int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) {
return 0;
}
+const int32_t * mtmd_gen_state_get_ref_codes(const mtmd_gen_state * state, size_t * n_codes) {
+ if (n_codes) {
+ *n_codes = state ? state->ref.tokens.size() : 0;
+ }
+ return state && !state->ref.tokens.empty() ? state->ref.tokens.data() : nullptr;
+}
+
static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
mtmd_subcontext * sctx = ctx->ctx_gen_a.get();
if (!sctx) {
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index 49fc8136e..cf2d14ce2 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -593,6 +593,10 @@ MTMD_API void mtmd_gen_state_reset(mtmd_gen_state * state);
// returns non-zero if the last encode produced no such data
MTMD_API int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state);
+// codes of the reference held by the state (frame-major, e.g. drama3: 10 per frame), NULL if it has none
+// valid until the next mtmd_gen_state_set_ref() / mtmd_gen_state_free() on this state
+MTMD_API const int32_t * mtmd_gen_state_get_ref_codes(const mtmd_gen_state * state, size_t * n_codes);
+
// defaults tuned for the loaded pipeline, callers override only what they care about
MTMD_API struct mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx);