Commit a68c6136b for llama.cpp
commit a68c6136bb980d35281ae68d275d6f5318accc4a
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Mon Nov 16 17:32:27 2026 +0100
mtmd: support fish audio s2 (#32330)
* add fisk-audio-s2 support
* add voice cloning
* cont merge master
* nits
* nits
* clean up
* nits
* use new batch api
diff --git a/common/arg.cpp b/common/arg.cpp
index c5fcecac9..5dc7c8d3a 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -4522,6 +4522,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.tts_speaker_file = value;
}
).set_examples({LLAMA_EXAMPLE_TTS}));
+ add_opt(common_arg(
+ {"--tts-speaker-text"}, "TEXT",
+ "transcript of --tts-speaker-file, for models that need it to clone the voice\n"
+ "see docs/multimodal/tts.md for per-model usage notes",
+ [](common_params & params, const std::string & value) {
+ params.tts_speaker_text = value;
+ }
+ ).set_examples({LLAMA_EXAMPLE_TTS}));
//
// diffusion params
diff --git a/common/common.h b/common/common.h
index 4cfc1b8f3..9b9e892d8 100644
--- a/common/common.h
+++ b/common/common.h
@@ -796,6 +796,7 @@ struct common_params {
std::string tts_lang = "";
std::string tts_voice = "";
std::string tts_speaker_file = "";
+ std::string tts_speaker_text = "";
bool is_gen_docs = false; // whether we are running inside llama-gen-docs
};
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 2ec0551e8..05f55fccf 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -336,6 +336,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"XverseForCausalLM": "xverse",
"YoutuForCausalLM": "deepseek",
"YoutuVLForConditionalGeneration": "deepseek",
+ "fish_qwen3_omni": "fish_s2",
"modeling_grove_moe.GroveMoeForCausalLM": "grovemoe",
"modeling_sarvam_moe.SarvamMoEForCausalLM": "bailingmoe",
}
@@ -438,6 +439,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"VoxtralForConditionalGeneration": "ultravox",
"VoxtralRealtimeForConditionalGeneration": "ultravox",
"YoutuVLForConditionalGeneration": "youtuvl",
+ "fish_qwen3_omni": "fish_s2",
}
diff --git a/conversion/base.py b/conversion/base.py
index b8eba85c7..a95ecc008 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -3488,6 +3488,9 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
# some models (llava_onevision2) only declare the architecture inside text_config
if arch is None and text_config.get("architectures") is not None:
arch = text_config["architectures"][0]
+ # some configs only have a model_type (fishaudio/s2-pro)
+ if arch is None:
+ arch = hparams.get("model_type")
if arch is None:
raise ValueError("Failed to detect model architecture")
return arch
diff --git a/conversion/drama3.py b/conversion/drama3.py
index ac01f481c..68bbb556b 100644
--- a/conversion/drama3.py
+++ b/conversion/drama3.py
@@ -18,6 +18,30 @@ _TOWER_SHARED = "audio_decoder_mtp_shared.layers."
_TOWER_RESIDUAL = "audio_decoder_mtp.layers."
+def load_codec(dir_model) -> dict[str, Tensor]:
+ sd = torch.load(dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True)
+ return sd.get("state_dict", sd)
+
+
+def fold_conv(sd: dict[str, Tensor], prefix: str) -> Tensor:
+ # fold weight norm
+ if prefix + ".conv.weight" in sd:
+ return sd[prefix + ".conv.weight"].float()
+ g = sd[prefix + ".conv.parametrizations.weight.original0"].float()
+ v = sd[prefix + ".conv.parametrizations.weight.original1"].float()
+ return g * v / v.norm(dim=(1, 2), keepdim=True)
+
+
+def fold_wn(g: Tensor, v: Tensor) -> Tensor:
+ # fold old-style weight norm of a 1x1 conv, returns [out, in]
+ return (g.float() * v.float() / v.float().norm(dim=(1, 2), keepdim=True)).squeeze(-1)
+
+
+def fold_snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
+ alpha = alpha.float().flatten()
+ return alpha, 1.0 / (alpha + 1e-9)
+
+
@ModelBase.register("FishOmniModel")
@ModelBase.example("fishaudio/drama-3")
class Drama3TextModel(Qwen3_5MoeTextModel):
@@ -36,6 +60,7 @@ class Drama3TextModel(Qwen3_5MoeTextModel):
class Drama3MmprojModel(MmprojModel):
has_vision_encoder = False
has_audio_encoder = False
+ gen_projector_type = gguf.VisionProjectorType.DRAMA3_GEN
_LAYER_MAP = {
"enorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ENORM, True),
@@ -69,7 +94,7 @@ class Drama3MmprojModel(MmprojModel):
cfg = self.hparams
self.gguf_writer.add_file_type(self.ftype)
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
- self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.DRAMA3_GEN)
+ self.gguf_writer.add_clip_gen_audio_projector_type(self.gen_projector_type)
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
self.gguf_writer.add_gen_audio_embedding_length(cfg["dim"])
self.gguf_writer.add_gen_audio_feed_forward_length(cfg["intermediate_size"])
@@ -153,32 +178,20 @@ class Drama3MmprojModel(MmprojModel):
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
# only the DAC decoder, the encoder is for voice cloning (not supported yet)
- sd = torch.load(self.dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True)
- sd = sd.get("state_dict", sd)
+ sd = load_codec(self.dir_model)
T = gguf.MODEL_TENSOR
def get(name: str) -> Tensor:
return sd[name].float()
def conv(prefix: str) -> Tensor:
- # fold weight norm
- if prefix + ".conv.weight" in sd:
- return get(prefix + ".conv.weight")
- g = get(prefix + ".conv.parametrizations.weight.original0")
- v = get(prefix + ".conv.parametrizations.weight.original1")
- return g * v / v.norm(dim=(1, 2), keepdim=True)
+ return fold_conv(sd, prefix)
def quant_table(prefix: str) -> Tensor:
# fold out_proj into the codebook, so decoding is a row lookup
- g = get(prefix + ".out_proj.weight_g")
- v = get(prefix + ".out_proj.weight_v")
- w = (g * v / v.norm(dim=(1, 2), keepdim=True)).squeeze(-1)
+ w = fold_wn(sd[prefix + ".out_proj.weight_g"], sd[prefix + ".out_proj.weight_v"])
return get(prefix + ".codebook.weight") @ w.T + get(prefix + ".out_proj.bias")
- def snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
- alpha = alpha.flatten()
- return alpha, 1.0 / (alpha + 1e-9)
-
q = "quantizer."
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), quant_table(q + "semantic_quantizer.quantizers.0"))
n_rest = sum(1 for k in sd if k.startswith(q + "quantizer.quantizers.") and k.endswith(".codebook.weight"))
@@ -228,7 +241,7 @@ class Drama3MmprojModel(MmprojModel):
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get(d + "0.conv.bias"))
for bid in range(4):
p = f"{d}{bid + 1}.block."
- a, b = snake(get(p + "0.alpha"))
+ a, b = fold_snake(get(p + "0.alpha"))
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a)
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b)
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), conv(p + "1"))
@@ -236,7 +249,7 @@ class Drama3MmprojModel(MmprojModel):
for xid in range(3):
r = f"{p}{xid + 2}.block."
for act, tensor in (("0", T.A_GEN_WAV_DAC_RES_ACT1), ("2", T.A_GEN_WAV_DAC_RES_ACT2)):
- a, b = snake(get(r + act + ".alpha"))
+ a, b = fold_snake(get(r + act + ".alpha"))
name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
yield (name + ".alpha", a)
yield (name + ".beta", b)
@@ -244,7 +257,7 @@ class Drama3MmprojModel(MmprojModel):
name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
yield (name + ".weight", conv(r + cv))
yield (name + ".bias", get(r + cv + ".conv.bias"))
- a, b = snake(get(d + "5.alpha"))
+ a, b = fold_snake(get(d + "5.alpha"))
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a)
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b)
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), conv(d + "6"))
diff --git a/conversion/fish_s2.py b/conversion/fish_s2.py
new file mode 100644
index 000000000..4c5aeff13
--- /dev/null
+++ b/conversion/fish_s2.py
@@ -0,0 +1,276 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Callable, Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import ModelBase, gguf
+from .drama3 import Drama3MmprojModel, fold_conv, fold_snake, fold_wn, load_codec
+from .qwen import Qwen3Model
+
+# Fish Audio S2 Pro (dual-AR)
+# text model: Qwen3 backbone (slow AR), samples the semantic code of each frame
+# mmproj: fast AR code predictor (other codebooks) and Fish DAC decoder (codec.pth, same as drama3)
+
+# residual codebook size of codec.pth, the fast AR head is cut to it
+_N_RESIDUAL_CODES = 1024
+
+# codec encoder strides (encoder_rates of the codec config), the quantizer then downsamples 2x2
+_CODEC_ENC_STRIDES = (2, 4, 8, 8)
+
+# codec encoder transformers, both 1024 wide with 16 heads of 64
+# generic audio layers a.blk.0..3: last encoder block, a.blk.4..11: quantizer pre_module
+_N_ENC_TFM_LAYERS = 4
+_N_PRE_TFM_LAYERS = 8
+_CODEC_HEAD_DIM = 64
+
+
+def _load_hparams(dir_model: Path) -> dict[str, Any]:
+ hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
+ cfg = hparams["text_config"]
+ hparams["text_config"] = {
+ "hidden_size": cfg["dim"],
+ "num_hidden_layers": cfg["n_layer"],
+ "num_attention_heads": cfg["n_head"],
+ "num_key_value_heads": cfg["n_local_heads"],
+ "head_dim": cfg["head_dim"],
+ "intermediate_size": cfg["intermediate_size"],
+ "rms_norm_eps": cfg["norm_eps"],
+ "rope_theta": cfg["rope_base"],
+ "max_position_embeddings": cfg["max_seq_len"],
+ "vocab_size": cfg["vocab_size"],
+ "tie_word_embeddings": cfg["tie_word_embeddings"],
+ }
+ return hparams
+
+
+def _neox_rows(w: Tensor, head_dim: int) -> Tensor:
+ # checkpoint uses interleaved RoPE, reorder the rows of each head so NEOX RoPE gives the same result
+ order = torch.cat([torch.arange(0, head_dim, 2), torch.arange(1, head_dim, 2)])
+ return w.reshape(-1, head_dim, *w.shape[1:])[:, order].reshape(w.shape)
+
+
+@ModelBase.register("fish_qwen3_omni")
+@ModelBase.example("fishaudio/s2-pro")
+class FishS2TextModel(Qwen3Model):
+ model_arch = gguf.MODEL_ARCH.QWEN3
+
+ def __init__(self, dir_model: Path, *args, **kwargs):
+ kwargs.pop("hparams", None)
+ super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # the rest is mapped by tensor_mapping.py
+ name = name.removeprefix("text_model.")
+ T = gguf.MODEL_TENSOR
+ head_dim = self.hparams["head_dim"]
+ if name.endswith(".attention.wqkv.weight"):
+ n_q = self.hparams["num_attention_heads"] * head_dim
+ n_kv = self.hparams["num_key_value_heads"] * head_dim
+ q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0)
+ yield (self.format_tensor_name(T.ATTN_Q, bid), _neox_rows(q, head_dim))
+ yield (self.format_tensor_name(T.ATTN_K, bid), _neox_rows(k, head_dim))
+ yield (self.format_tensor_name(T.ATTN_V, bid), v)
+ return
+ if name.endswith(".attention.q_norm.weight"):
+ yield (self.format_tensor_name(T.ATTN_Q_NORM, bid), _neox_rows(data_torch, head_dim))
+ return
+ if name.endswith(".attention.k_norm.weight"):
+ yield (self.format_tensor_name(T.ATTN_K_NORM, bid), _neox_rows(data_torch, head_dim))
+ return
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
+@ModelBase.register("fish_qwen3_omni")
+@ModelBase.example("fishaudio/s2-pro")
+class FishS2MmprojModel(Drama3MmprojModel):
+ has_audio_encoder = True # codec encoder, for voice cloning
+ gen_projector_type = gguf.VisionProjectorType.FISH_S2_GEN
+
+ _FAST_AR_MAP = {
+ "attention_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,
+ "attention.wo.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,
+ "ffn_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM,
+ "feed_forward.w1.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
+ "feed_forward.w3.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP,
+ "feed_forward.w2.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
+ }
+
+ def __init__(self, dir_model: Path, *args, **kwargs):
+ kwargs.pop("hparams", None)
+ super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
+
+ def get_audio_config(self) -> dict[str, Any] | None:
+ cfg = dict(self.global_config["audio_decoder_config"])
+ cfg["num_hidden_layers"] = cfg["n_layer"]
+ return cfg
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ # codec encoder: raw 44.1 kHz waveform -> backbone input rows, other hparams are hardcoded in tools/mtmd
+ self.gguf_writer.add_clip_has_audio_encoder(True)
+ self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.FISH_S2_SPKENC)
+ self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
+ self.gguf_writer.add_audio_num_mel_bins(1)
+ self.gguf_writer.add_audio_block_count(_N_ENC_TFM_LAYERS + _N_PRE_TFM_LAYERS)
+ self.gguf_writer.add_audio_embedding_length(1024)
+ self.gguf_writer.add_audio_feed_forward_length(3072)
+ self.gguf_writer.add_audio_head_count(1024 // _CODEC_HEAD_DIM)
+ self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ # the VQ picks the nearest codebook entry, small errors in the encoder change the codes
+ if new_name.startswith("a.codec."):
+ return gguf.GGMLQuantizationType.F32
+ # codec transformers: F16 gives codes closer to the reference than BF16
+ if new_name.startswith("a.blk.") and n_dims == 2:
+ return gguf.GGMLQuantizationType.F16
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name = item[0]
+ if name.startswith("audio_decoder.") or name == "text_model.model.embeddings.weight":
+ return item
+ return None
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ T = gguf.MODEL_TENSOR
+ cfg = self.hparams
+
+ # already named by generate_extra_tensors()
+ if name.startswith("a."):
+ yield (name, data_torch)
+ return
+
+ if name == "text_model.model.embeddings.weight":
+ # semantic band of the backbone embeddings
+ begin = self.global_config["semantic_start_token_id"]
+ yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch[begin:begin + cfg["vocab_size"]])
+ return
+
+ if name == "audio_decoder.codebook_embeddings.weight":
+ # backbone input of a speech row, one table per codebook
+ yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), data_torch.reshape(cfg["num_codebooks"], -1, data_torch.shape[-1]))
+ return
+
+ if name == "audio_decoder.embeddings.weight":
+ yield (self.format_tensor_name(T.A_GEN_CODE_TOK_EMBD), data_torch)
+ return
+
+ if name == "audio_decoder.output.weight":
+ yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), data_torch[:_N_RESIDUAL_CODES])
+ return
+
+ if name == "audio_decoder.norm.weight":
+ yield (self.format_tensor_name(T.A_GEN_CODE_OUTPUT_NORM), data_torch)
+ return
+
+ assert bid is not None and name.startswith(f"audio_decoder.layers.{bid}.")
+ key = name.removeprefix(f"audio_decoder.layers.{bid}.")
+ if key == "attention.wqkv.weight":
+ head_dim = cfg["head_dim"]
+ n_q = cfg["n_head"] * head_dim
+ n_kv = cfg["n_local_heads"] * head_dim
+ q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0)
+ yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, bid), _neox_rows(q, head_dim))
+ yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, bid), _neox_rows(k, head_dim))
+ yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, bid), v)
+ return
+ yield (self.format_tensor_name(self._FAST_AR_MAP[key], bid), data_torch)
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ yield from super().generate_extra_tensors()
+
+ sd = load_codec(self.dir_model)
+ T = gguf.MODEL_TENSOR
+
+ def get(name: str) -> Tensor:
+ return sd[name].float()
+
+ def tn(tensor: gguf.MODEL_TENSOR, bid: int | None = None, xid: int | None = None) -> str:
+ return gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
+
+ def conv_wb(tensor: gguf.MODEL_TENSOR, prefix: str, bid: int | None = None, xid: int | None = None, stride: int = 1):
+ # taps-major [K / stride, OC, stride * IC], see conv() in tools/mtmd/models/fish-s2-spkenc.cpp
+ w = fold_conv(sd, prefix)
+ n_out, n_in, k = w.shape
+ w = w.reshape(n_out, n_in, k // stride, stride).permute(2, 0, 3, 1).reshape(k // stride, n_out, stride * n_in)
+ yield (tn(tensor, bid, xid) + ".weight", w)
+ yield (tn(tensor, bid, xid) + ".bias", get(prefix + ".conv.bias"))
+
+ def snake_ab(tensor: gguf.MODEL_TENSOR, name: str, bid: int | None = None, xid: int | None = None):
+ a, b = fold_snake(get(name))
+ yield (tn(tensor, bid, xid) + ".alpha", a)
+ yield (tn(tensor, bid, xid) + ".beta", b)
+
+ def tfm_layers(prefix: str, n_layer: int, offset: int):
+ for il in range(n_layer):
+ p = f"{prefix}layers.{il}."
+ bid = offset + il
+ wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0)
+ yield (self.format_tensor_name(T.A_ENC_ATTN_Q, bid), _neox_rows(wq, _CODEC_HEAD_DIM))
+ yield (self.format_tensor_name(T.A_ENC_ATTN_K, bid), _neox_rows(wk, _CODEC_HEAD_DIM))
+ yield (self.format_tensor_name(T.A_ENC_ATTN_V, bid), wv)
+ yield (self.format_tensor_name(T.A_ENC_OUTPUT, bid), get(p + "attention.wo.weight"))
+ yield (self.format_tensor_name(T.A_ENC_INPUT_NORM, bid), get(p + "attention_norm.weight"))
+ yield (self.format_tensor_name(T.A_ENC_ATTN_SCALE, bid), get(p + "attention_layer_scale.gamma"))
+ yield (self.format_tensor_name(T.A_ENC_OUTPUT_NORM, bid), get(p + "ffn_norm.weight"))
+ yield (self.format_tensor_name(T.A_ENC_FFN_GATE, bid), get(p + "feed_forward.w1.weight"))
+ yield (self.format_tensor_name(T.A_ENC_FFN_UP, bid), get(p + "feed_forward.w3.weight"))
+ yield (self.format_tensor_name(T.A_ENC_FFN_DOWN, bid), get(p + "feed_forward.w2.weight"))
+ yield (self.format_tensor_name(T.A_ENC_FFN_SCALE_LS, bid), get(p + "ffn_layer_scale.gamma"))
+
+ # encoder: entry conv, 4x (3 residual units, Snake, strided conv), Snake, post conv
+ e = "encoder.block."
+ yield from conv_wb(T.A_CODEC_ENC_ENTRY, e + "0")
+ for bid, stride in enumerate(_CODEC_ENC_STRIDES):
+ p = f"{e}{bid + 1}.block."
+ for xid in range(3):
+ r = f"{p}{xid}.block."
+ yield from snake_ab(T.A_CODEC_ENC_RES_ACT1, r + "0.alpha", bid, xid)
+ yield from conv_wb(T.A_CODEC_ENC_RES_CONV1, r + "1", bid, xid)
+ yield from snake_ab(T.A_CODEC_ENC_RES_ACT2, r + "2.alpha", bid, xid)
+ yield from conv_wb(T.A_CODEC_ENC_RES_CONV2, r + "3", bid, xid)
+ yield from snake_ab(T.A_CODEC_ENC_SNAKE, p + "3.alpha", bid)
+ yield from conv_wb(T.A_CODEC_ENC_CONV, p + "4", bid, stride=stride)
+ # the last block ends with a windowed transformer
+ yield from tfm_layers(f"{e}4.block.5.", _N_ENC_TFM_LAYERS, 0)
+ yield (self.format_tensor_name(T.A_POST_NORM), get(f"{e}4.block.5.norm.weight"))
+ yield from snake_ab(T.A_CODEC_ENC_POST_SNAKE, e + "5.alpha")
+ yield from conv_wb(T.A_CODEC_ENC_POST_CONV, e + "6")
+
+ # quantizer: 2x (strided conv + ConvNeXt), windowed transformer, then semantic + residual VQ
+ q = "quantizer."
+ for bid in range(2):
+ p = f"{q}downsample.{bid}."
+ yield from conv_wb(T.A_CODEC_DOWN_CONV, p + "0", bid, stride=2)
+ # depthwise: one [C] row per tap
+ yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".weight", fold_conv(sd, p + "1.dwconv").squeeze(1).T.contiguous())
+ yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".bias", get(p + "1.dwconv.conv.bias"))
+ for key, tensor in (("norm", T.A_CODEC_DOWN_NORM), ("pwconv1", T.A_CODEC_DOWN_PW1), ("pwconv2", T.A_CODEC_DOWN_PW2)):
+ yield (tn(tensor, bid) + ".weight", get(f"{p}1.{key}.weight"))
+ yield (tn(tensor, bid) + ".bias", get(f"{p}1.{key}.bias"))
+ yield (tn(T.A_CODEC_DOWN_GAMMA, bid), get(p + "1.gamma"))
+ yield from tfm_layers(q + "pre_module.", _N_PRE_TFM_LAYERS, _N_ENC_TFM_LAYERS)
+ yield (self.format_tensor_name(T.A_CODEC_PRE_NORM), get(q + "pre_module.norm.weight"))
+
+ # nearest codebook entry by cosine similarity, see VectorQuantize.decode_latents()
+ def vq(prefix: str) -> tuple[Tensor, Tensor, Tensor]:
+ w = fold_wn(sd[prefix + ".in_proj.weight_g"], sd[prefix + ".in_proj.weight_v"])
+ cb = torch.nn.functional.normalize(get(prefix + ".codebook.weight"), dim=-1)
+ return w, get(prefix + ".in_proj.bias"), cb
+
+ w, b, cb = vq(q + "semantic_quantizer.quantizers.0")
+ yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".weight"), w)
+ yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".bias"), b)
+ yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_CB), cb)
+ n_rest = self.hparams["num_codebooks"] - 1
+ rest = [vq(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)]
+ yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".weight"), torch.stack([r[0] for r in rest]))
+ yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".bias"), torch.stack([r[1] for r in rest]))
+ yield (self.format_tensor_name(T.A_CODEC_VQ_REST_CB), torch.stack([r[2] for r in rest]))
diff --git a/docs/multimodal/tts.md b/docs/multimodal/tts.md
index 1dca6c461..301d2394e 100644
--- a/docs/multimodal/tts.md
+++ b/docs/multimodal/tts.md
@@ -12,6 +12,7 @@ Supported models:
- [Chatterbox](#chatterbox)
- [Drama 3](#drama-3)
- [ZONOS3](#zonos3)
+- [Fish Audio S2 Pro](#fish-audio-s2-pro)
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
@@ -215,3 +216,45 @@ Convert the root of the release repository, not the `lm/` subdirectory. The mmpr
python convert_hf_to_gguf.py path/to/ZONOS3-Mini --outtype q8_0 --outfile zonos3-mini.gguf
python convert_hf_to_gguf.py path/to/ZONOS3-Mini --mmproj --outtype f32 --outfile mmproj-zonos3-mini.gguf
```
+
+## Fish Audio S2 Pro
+
+[Fish Audio S2 Pro](https://huggingface.co/fishaudio/s2-pro): a Qwen3 backbone (4B) samples each frame's semantic token, a 4-layer fast AR predicts the 9 other codes, and the Fish DAC (same as Drama 3) decodes them to 44.1 kHz audio.
+
+Available params:
+- Speaker tags `<|speaker:0|>`, `<|speaker:1|>`, ... in the text give each character its own voice, a text without a tag starts with `<|speaker:0|>`
+- Inline instructions in brackets control the delivery, for example `[laughing]`, `[whisper]` or `[pause]`
+- The language is the one of the text, `--tts-lang` is not used
+- Voice cloning: `--tts-speaker-file` with the reference clip, optionally `--tts-speaker-text` with its transcript (it is put in the prompt as the reference does). The clip is encoded by the codec encoder half of `codec.pth`, it is truncated to the first 30 s
+- Note: `--temp`, `--top-k` and `--top-p` are not used (only `--seed`), frames are sampled with the settings of the reference (semantic token: repetition penalty 1.1 over the last 16 frames, top-k 30, top-p 0.8, temperature 0.8, or top-p 0.9 and temperature 1.0 when the last 4 frames repeat a token; other codes: greedy)
+- Each frame is 2048 samples (about 46 ms), generation stops at the end-of-speech token
+
+Example usage:
+
+```sh
+llama-tts -m s2-pro.gguf \
+ -mm mmproj-s2-pro.gguf \
+ -p "<|speaker:0|>Did you remember to buy the bread? <|speaker:1|>Of course I did [laughing] it is on the kitchen table." \
+ -n 1000 \
+ --output out.wav
+```
+
+Voice cloning (`--tts-speaker-text` can be left out):
+
+```sh
+llama-tts -m s2-pro.gguf \
+ -mm mmproj-s2-pro.gguf \
+ --tts-speaker-file reference.wav \
+ --tts-speaker-text "The words spoken in reference.wav." \
+ -p "Hello, this is a cloned voice." \
+ --output out.wav
+```
+
+**Note for GGUF conversion:**
+
+The mmproj holds the fast AR and `codec.pth` (the encoder half for voice cloning, the decoder half for the audio), it is best kept in BF16. The codec encoder is always stored in F16/F32, small errors in it change the codes picked by its VQ:
+
+```sh
+python convert_hf_to_gguf.py path/to/s2-pro --outtype q8_0 --outfile s2-pro.gguf
+python convert_hf_to_gguf.py path/to/s2-pro --mmproj --outtype bf16 --outfile mmproj-s2-pro.gguf
+```
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 037e1332a..e6c1465c2 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -1335,6 +1335,28 @@ class MODEL_TENSOR(IntEnum):
A_GEN_CODE_EH_PROJ = auto() # drama3: per-layer [code embd; hidden] -> hidden projection
A_GEN_CODE_BLK_EMBD = auto() # drama3: per-layer code embedding table
A_GEN_CODE_BLK_OUT_NORM = auto() # drama3: per-layer output norm
+ A_GEN_CODE_TOK_EMBD = auto() # fish_s2: code predictor input embedding, shared by all codebooks
+ # fish_s2 codec encoder (voice cloning): reference clip -> codes
+ A_CODEC_ENC_ENTRY = auto() # entry conv
+ A_CODEC_ENC_SNAKE = auto() # encoder block, Snake before the strided conv
+ A_CODEC_ENC_CONV = auto() # encoder block, strided conv
+ A_CODEC_ENC_RES_ACT1 = auto()
+ A_CODEC_ENC_RES_CONV1 = auto()
+ A_CODEC_ENC_RES_ACT2 = auto()
+ A_CODEC_ENC_RES_CONV2 = auto()
+ A_CODEC_ENC_POST_SNAKE = auto()
+ A_CODEC_ENC_POST_CONV = auto()
+ A_CODEC_DOWN_CONV = auto() # quantizer downsample, strided conv + ConvNeXt
+ A_CODEC_DOWN_DWCONV = auto()
+ A_CODEC_DOWN_NORM = auto()
+ A_CODEC_DOWN_PW1 = auto()
+ A_CODEC_DOWN_PW2 = auto()
+ A_CODEC_DOWN_GAMMA = auto()
+ A_CODEC_PRE_NORM = auto() # output norm of the quantizer pre_module
+ A_CODEC_VQ_FIRST_IN = auto() # semantic VQ input projection
+ A_CODEC_VQ_FIRST_CB = auto() # semantic VQ codebook, l2-normalized
+ A_CODEC_VQ_REST_IN = auto() # residual VQ input projections, merged 3D
+ A_CODEC_VQ_REST_CB = auto() # residual VQ codebooks, l2-normalized, merged 3D
# qwen3tts code2wav: RVQ codes -> raw PCM
A_GEN_WAV_QUANT_FIRST_IN = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D)
A_GEN_WAV_QUANT_FIRST_OUT = auto() # semantic RVQ, out_proj
@@ -2277,6 +2299,27 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_GEN_CODE_EH_PROJ: "a.gen.code.blk.{bid}.eh_proj",
MODEL_TENSOR.A_GEN_CODE_BLK_EMBD: "a.gen.code.blk.{bid}.embd",
MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM: "a.gen.code.blk.{bid}.out_norm",
+ MODEL_TENSOR.A_GEN_CODE_TOK_EMBD: "a.gen.code.tok_embd",
+ MODEL_TENSOR.A_CODEC_ENC_ENTRY: "a.codec.enc.entry",
+ MODEL_TENSOR.A_CODEC_ENC_SNAKE: "a.codec.enc.blk.{bid}.snake",
+ MODEL_TENSOR.A_CODEC_ENC_CONV: "a.codec.enc.blk.{bid}.conv",
+ MODEL_TENSOR.A_CODEC_ENC_RES_ACT1: "a.codec.enc.blk.{bid}.res.{xid}.act1",
+ MODEL_TENSOR.A_CODEC_ENC_RES_CONV1: "a.codec.enc.blk.{bid}.res.{xid}.conv1",
+ MODEL_TENSOR.A_CODEC_ENC_RES_ACT2: "a.codec.enc.blk.{bid}.res.{xid}.act2",
+ MODEL_TENSOR.A_CODEC_ENC_RES_CONV2: "a.codec.enc.blk.{bid}.res.{xid}.conv2",
+ MODEL_TENSOR.A_CODEC_ENC_POST_SNAKE: "a.codec.enc.post_snake",
+ MODEL_TENSOR.A_CODEC_ENC_POST_CONV: "a.codec.enc.post_conv",
+ MODEL_TENSOR.A_CODEC_DOWN_CONV: "a.codec.down.blk.{bid}.conv",
+ MODEL_TENSOR.A_CODEC_DOWN_DWCONV: "a.codec.down.blk.{bid}.dwconv",
+ MODEL_TENSOR.A_CODEC_DOWN_NORM: "a.codec.down.blk.{bid}.norm",
+ MODEL_TENSOR.A_CODEC_DOWN_PW1: "a.codec.down.blk.{bid}.pw1",
+ MODEL_TENSOR.A_CODEC_DOWN_PW2: "a.codec.down.blk.{bid}.pw2",
+ MODEL_TENSOR.A_CODEC_DOWN_GAMMA: "a.codec.down.blk.{bid}.gamma",
+ MODEL_TENSOR.A_CODEC_PRE_NORM: "a.codec.pre_norm",
+ MODEL_TENSOR.A_CODEC_VQ_FIRST_IN: "a.codec.vq.first.in_proj",
+ MODEL_TENSOR.A_CODEC_VQ_FIRST_CB: "a.codec.vq.first.codebook",
+ MODEL_TENSOR.A_CODEC_VQ_REST_IN: "a.codec.vq.rest.in_proj",
+ MODEL_TENSOR.A_CODEC_VQ_REST_CB: "a.codec.vq.rest.codebook",
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN: "a.gen.wav.quant.first.in_proj",
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj",
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB: "a.gen.wav.quant.first.codebook",
@@ -2711,6 +2754,27 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.A_GEN_CODE_EH_PROJ,
MODEL_TENSOR.A_GEN_CODE_BLK_EMBD,
MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM,
+ MODEL_TENSOR.A_GEN_CODE_TOK_EMBD,
+ MODEL_TENSOR.A_CODEC_ENC_ENTRY,
+ MODEL_TENSOR.A_CODEC_ENC_SNAKE,
+ MODEL_TENSOR.A_CODEC_ENC_CONV,
+ MODEL_TENSOR.A_CODEC_ENC_RES_ACT1,
+ MODEL_TENSOR.A_CODEC_ENC_RES_CONV1,
+ MODEL_TENSOR.A_CODEC_ENC_RES_ACT2,
+ MODEL_TENSOR.A_CODEC_ENC_RES_CONV2,
+ MODEL_TENSOR.A_CODEC_ENC_POST_SNAKE,
+ MODEL_TENSOR.A_CODEC_ENC_POST_CONV,
+ MODEL_TENSOR.A_CODEC_DOWN_CONV,
+ MODEL_TENSOR.A_CODEC_DOWN_DWCONV,
+ MODEL_TENSOR.A_CODEC_DOWN_NORM,
+ MODEL_TENSOR.A_CODEC_DOWN_PW1,
+ MODEL_TENSOR.A_CODEC_DOWN_PW2,
+ MODEL_TENSOR.A_CODEC_DOWN_GAMMA,
+ MODEL_TENSOR.A_CODEC_PRE_NORM,
+ MODEL_TENSOR.A_CODEC_VQ_FIRST_IN,
+ MODEL_TENSOR.A_CODEC_VQ_FIRST_CB,
+ MODEL_TENSOR.A_CODEC_VQ_REST_IN,
+ MODEL_TENSOR.A_CODEC_VQ_REST_CB,
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN,
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT,
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB,
@@ -7223,6 +7287,8 @@ class VisionProjectorType:
DRAMA3_GEN = "drama3_gen" # audio generation: codebook tower + Fish DAC decoder
ZONOS3_GEN = "zonos3_gen" # audio generation: 10 codebook heads + ZAC decoder
ZONOS3_SPKENC = "zonos3_spkenc" # audio: MECT-B2 speaker encoder
+ FISH_S2_GEN = "fish_s2_gen" # audio generation: fast AR code predictor + Fish DAC decoder
+ FISH_S2_SPKENC = "fish_s2_spkenc" # audio: Fish DAC encoder + RVQ, reference clip -> prompt rows
HUNYUANVL = "hunyuanvl"
PARAKEET = "parakeet" # audio
MINIMAXM3 = "minimax_m3"
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index b23fd5e6b..d514a72cd 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -84,6 +84,8 @@ add_library(mtmd
models/qwen3tts-spkenc.cpp
models/qwen3tts-gen.cpp
models/drama3-gen.cpp
+ models/fish-s2-gen.cpp
+ models/fish-s2-spkenc.cpp
models/zonos3-gen.cpp
models/zonos3-spkenc.cpp
models/pockettts-seanet.cpp
diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md
index b6ed5a577..b0d153be3 100644
--- a/tools/mtmd/README-dev.md
+++ b/tools/mtmd/README-dev.md
@@ -39,6 +39,7 @@ Rules of thumb:
- Models and backends must outlive the subcontexts, `mtmd_context` declares them first for that reason
- Models having both audio and vision encoders must have different arch names (projector types), postfixed by `a` and `v`. See Gemma 4 as an example (`gemma4a` / `gemma4v`)
- Locking happens only at the public API boundary (`mtmd.cpp`): a new public function that runs a graph must take `compute_mutex`. The internals (subcontexts, preprocessors, `mtmd_audio_log_mel()`) assume it is held and never lock
+- No magic numbers for a model's fixed shapes (layer counts, codebook sizes, window lengths...): define them once as `static constexpr` in the graph struct in `models/models.h`, and refer to them from the loader or the subcontext (e.g. `mtmd_graph_zonos3_gen::n_codebooks`, `mtmd_graph_fish_s2_spkenc::n_tfm_layer`)
## Pipeline
@@ -109,7 +110,9 @@ Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system i
- 60% changes inside `mtmd-helper-gen.cpp`
- 10% changes inside `libmtmd` core (`mtmd-model`, `mtmd-subcontext`, `models/`)
- The rest downstream code (CLI, server) should have no changes at all
-5. Update usage documentation in `docs/multimodal/tts.md`
+5. Validate with several concurrent requests on `llama-server` (e.g. `-np 2`, one plain and one streaming request at the same time), not only with `llama-tts`
+ - The server interleaves the slots on one `llama_context`, so a pipeline must copy what it needs (hidden state, logits) right after its own decode, not read it at the next step
+6. Update usage documentation in `docs/multimodal/tts.md`
IMPORTANT: If your model needs changes that don't fit the existing infrastructure, **open an issue first for discussion**.
diff --git a/tools/mtmd/models/fish-s2-gen.cpp b/tools/mtmd/models/fish-s2-gen.cpp
new file mode 100644
index 000000000..4ac93f7bb
--- /dev/null
+++ b/tools/mtmd/models/fish-s2-gen.cpp
@@ -0,0 +1,74 @@
+#include "models.h"
+
+#include <cmath>
+#include <vector>
+
+// one frame: backbone hidden + semantic code -> the codes of this frame + next backbone input
+// fast AR row 0 is the backbone hidden, row p > 0 is the embedding of code p - 1 and picks code p (greedy, as the reference)
+ggml_cgraph * mtmd_graph_fish_s2_gen::build_fast_ar() {
+ const int64_t n_cb = model.gen_code_embd_w->ne[2];
+ const int64_t n_embd_cb = model.gen_code_embd_w->ne[0];
+ const int64_t n_embd = model.gen_code_tok_embd_w->ne[0];
+ const int64_t n_kv = model.layers[0].k_w->ne[1];
+ const int n_layer = (int) model.layers.size();
+
+ ggml_tensor * h = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd);
+ ggml_set_name(h, "inp_raw");
+ ggml_set_input(h);
+
+ ggml_tensor * inp_code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1);
+ ggml_set_name(inp_code0, "inp_code0");
+ ggml_set_input(inp_code0);
+
+ mtmd_graph_qwen3tts_gen::code_gen cg(*this, top_k, top_p);
+
+ std::vector<ggml_tensor *> k_cache(n_layer), v_cache(n_layer);
+ for (int il = 0; il < n_layer; il++) {
+ k_cache[il] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, n_cb), 0.0f);
+ v_cache[il] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, n_cb), 0.0f);
+ }
+
+ ggml_tensor * out_codes = cg.cache_set(ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, 1, n_cb), 0, inp_code0);
+
+ for (int p = 0; p < n_cb; p++) {
+ ggml_tensor * cur = h;
+ if (p > 0) {
+ ggml_tensor * code = ggml_view_1d(ctx0, out_codes, 1, (size_t) (p - 1) * out_codes->nb[1]);
+ cur = ggml_reshape_1d(ctx0, ggml_get_rows(ctx0, model.gen_code_tok_embd_w, code), n_embd);
+ }
+ ggml_tensor * kq_mask = cg.causal_mask_row(n_cb, p);
+ ggml_tensor * inp_pos = cg.const_i32(k_cache[0], (float) p);
+ for (int il = 0; il < n_layer; il++) {
+ cur = cg.layer_forward(cur, model.layers[il], inp_pos, kq_mask, k_cache[il], v_cache[il], n_cb, p, il);
+ }
+ if (p == 0) {
+ continue; // only fills the KV
+ }
+ cur = ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, hparams.eps), model.gen_code_norm_w);
+ ggml_tensor * logits = ggml_mul_mat(ctx0, model.gen_code_head_w, cur);
+ out_codes = cg.cache_set(out_codes, p, ggml_argmax(ctx0, logits));
+ }
+
+ out_codes = ggml_cont(ctx0, out_codes);
+ ggml_set_name(out_codes, "out_codes");
+ ggml_set_output(out_codes);
+ ggml_build_forward_expand(gf, out_codes);
+
+ // backbone input of the next row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1)
+ ggml_tensor * out_embd = ggml_get_rows(ctx0, model.gen_code_out_embd_w, inp_code0);
+ for (int64_t g = 0; g < n_cb; g++) {
+ ggml_tensor * table = ggml_view_2d(ctx0, model.gen_code_embd_w, n_embd_cb, model.gen_code_embd_w->ne[1],
+ model.gen_code_embd_w->nb[1], (size_t) g * model.gen_code_embd_w->nb[2]);
+ ggml_tensor * code = ggml_view_1d(ctx0, out_codes, 1, (size_t) g * out_codes->nb[1]);
+ out_embd = ggml_add(ctx0, out_embd, ggml_get_rows(ctx0, table, code));
+ }
+ out_embd = ggml_scale(ctx0, out_embd, 1.0f / sqrtf((float) (n_cb + 1)));
+ cb(out_embd, "gen_audio_out", -1);
+ ggml_build_forward_expand(gf, out_embd);
+
+ return gf;
+}
+
+ggml_cgraph * mtmd_graph_fish_s2_gen::build() {
+ return gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ? build_codec() : build_fast_ar();
+}
diff --git a/tools/mtmd/models/fish-s2-spkenc.cpp b/tools/mtmd/models/fish-s2-spkenc.cpp
new file mode 100644
index 000000000..a9c51d9cd
--- /dev/null
+++ b/tools/mtmd/models/fish-s2-spkenc.cpp
@@ -0,0 +1,135 @@
+#include "models.h"
+
+#include <cmath>
+#include <string>
+
+// reference clip -> codec frames -> backbone input rows, see DAC.encode() and DownsampleResidualVectorQuantize.encode()
+// activations are [C, T]; a conv is a sum of matmuls over tap-shifted views: exact in F32, no im2col buffer
+
+// causal conv, w is taps-major [stride * IC, OC, K / stride], see conversion/fish_s2.py
+ggml_tensor * mtmd_graph_fish_s2_spkenc::conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation) const {
+ const int64_t n_tap = w->ne[2];
+ const int pad = (int) (n_tap - 1) * stride * dilation;
+ if (pad > 0) {
+ x = ggml_pad_ext(ctx0, x, 0, 0, pad, 0, 0, 0, 0, 0);
+ }
+ if (stride > 1) {
+ // stride frames per column, then each tap group is one matmul
+ x = ggml_reshape_2d(ctx0, x, x->ne[0] * stride, x->ne[1] / stride);
+ }
+ const int64_t n_out = x->ne[1] - (n_tap - 1) * dilation;
+ ggml_tensor * y = nullptr;
+ for (int64_t k = 0; k < n_tap; k++) {
+ ggml_tensor * wk = ggml_view_2d(ctx0, w, w->ne[0], w->ne[1], w->nb[1], (size_t) k * w->nb[2]);
+ ggml_tensor * xk = ggml_view_2d(ctx0, x, x->ne[0], n_out, x->nb[1], (size_t) k * dilation * x->nb[1]);
+ ggml_tensor * yk = ggml_mul_mat(ctx0, wk, xk);
+ y = y ? ggml_add(ctx0, y, yk) : yk;
+ }
+ return ggml_add(ctx0, y, b);
+}
+
+// x + sin(alpha * x)^2 / alpha, beta = 1 / alpha is folded at conversion
+// TODO @ngxson: on Metal, residual codes match the reference less often than on CPU (89-98% vs 99%), maybe the fast-math sin
+ggml_tensor * mtmd_graph_fish_s2_spkenc::snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const {
+ ggml_tensor * s = ggml_sqr(ctx0, ggml_sin(ctx0, ggml_mul(ctx0, x, alpha)));
+ return ggml_add(ctx0, x, ggml_mul(ctx0, s, beta));
+}
+
+// causal depthwise conv (w [C, K]) -> LayerNorm -> pwconv1 -> GELU -> pwconv2 -> layer scale -> residual
+ggml_tensor * mtmd_graph_fish_s2_spkenc::convnext(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk) const {
+ const int64_t n_tap = blk.dwconv_w->ne[1];
+ const int64_t n_pos = x->ne[1];
+ ggml_tensor * xp = ggml_pad_ext(ctx0, x, 0, 0, (int) n_tap - 1, 0, 0, 0, 0, 0);
+ ggml_tensor * h = nullptr;
+ for (int64_t k = 0; k < n_tap; k++) {
+ ggml_tensor * wk = ggml_view_1d(ctx0, blk.dwconv_w, blk.dwconv_w->ne[0], (size_t) k * blk.dwconv_w->nb[1]);
+ ggml_tensor * xk = ggml_view_2d(ctx0, xp, xp->ne[0], n_pos, xp->nb[1], (size_t) k * xp->nb[1]);
+ ggml_tensor * hk = ggml_mul(ctx0, xk, wk);
+ h = h ? ggml_add(ctx0, h, hk) : hk;
+ }
+ h = ggml_add(ctx0, h, blk.dwconv_b);
+ h = ggml_add(ctx0, ggml_mul(ctx0, ggml_norm(ctx0, h, 1e-6f), blk.norm_w), blk.norm_b);
+ h = ggml_gelu_erf(ctx0, ggml_add(ctx0, ggml_mul_mat(ctx0, blk.pw1_w, h), blk.pw1_b));
+ h = ggml_add(ctx0, ggml_mul_mat(ctx0, blk.pw2_w, h), blk.pw2_b);
+ return ggml_add(ctx0, x, ggml_mul(ctx0, h, blk.gamma));
+}
+
+ggml_cgraph * mtmd_graph_fish_s2_spkenc::build() {
+ const auto & c2w = model.c2w;
+
+ // the preprocessor hands over the waveform as a single-row "mel", [n_samples, 1] is also [1, n_samples]
+ ggml_tensor * inp_raw = build_inp_raw(1);
+ ggml_tensor * x = ggml_reshape_2d(ctx0, inp_raw, 1, inp_raw->ne[0]);
+
+ // windowed causal transformer on a cold start: zero state, positions from 0
+ mtmd_graph_qwen3tts_gen::code2wav dec(*this);
+ auto zero_state = [&](const std::string & name, int64_t n0, int64_t n1) {
+ dec.state_in[name] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n0, n1), 0.0f);
+ };
+ auto transformer = [&](ggml_tensor * cur, int il0, int il1, int64_t window, ggml_tensor * norm_w) {
+ const int64_t n_kv = model.layers[il0].k_w->ne[1];
+ zero_state("tfm_pos", 1, 1);
+ for (int il = il0; il < il1; il++) {
+ zero_state("tfm_k_" + std::to_string(il), n_kv, window - 1);
+ zero_state("tfm_v_" + std::to_string(il), n_kv, window - 1);
+ cur = dec.tfm_layer_forward(cur, model.layers[il], il, window);
+ }
+ return ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, hparams.wav_tfm_eps), norm_w);
+ };
+
+ // encoder: entry conv, 4x (3 residual units, Snake, strided conv), transformer, Snake, post conv
+ x = conv(x, c2w.dac_entry_w, c2w.dac_entry_b, 1, 1);
+ static constexpr int DAC_DILATIONS[3] = { 1, 3, 9 };
+ for (const auto & blk : c2w.dac) {
+ for (size_t ir = 0; ir < blk.res.size(); ir++) {
+ const auto & res = blk.res[ir];
+ ggml_tensor * h = snake(x, res.act1_alpha, res.act1_beta);
+ h = conv(h, res.conv1_w, res.conv1_b, 1, DAC_DILATIONS[ir]);
+ h = snake(h, res.act2_alpha, res.act2_beta);
+ h = conv(h, res.conv2_w, res.conv2_b, 1, 1);
+ x = ggml_add(ctx0, x, h);
+ }
+ x = snake(x, blk.snake_alpha, blk.snake_beta);
+ x = conv(x, blk.conv_w, blk.conv_b, (int) (blk.conv_w->ne[0] / x->ne[0]), 1);
+ }
+ x = transformer(x, 0, n_enc_tfm_layer, 512, model.post_ln_w);
+ x = snake(x, c2w.dac_post_snake_alpha, c2w.dac_post_snake_beta);
+ x = conv(x, c2w.dac_post_conv_w, c2w.dac_post_conv_b, 1, 1);
+ cb(x, "codec_enc", -1);
+
+ // quantizer: 2x (strided conv + ConvNeXt), transformer
+ for (const auto & down : c2w.upsample) {
+ x = conv(x, down.conv_w, down.conv_b, (int) (down.conv_w->ne[0] / x->ne[0]), 1);
+ x = convnext(x, down);
+ }
+ ggml_tensor * z = transformer(x, n_enc_tfm_layer, (int) model.layers.size(), 128, c2w.tfm_output_norm_w);
+ cb(z, "codec_latent", -1);
+
+ // nearest entry by cosine similarity, then the entry through out_proj is removed from the residual
+ auto vq = [&](ggml_tensor * in_w, ggml_tensor * in_b, ggml_tensor * cb_norm, ggml_tensor * table) {
+ ggml_tensor * e = ggml_add(ctx0, ggml_mul_mat(ctx0, in_w, z), in_b);
+ e = ggml_l2_norm(ctx0, e, 1e-12f);
+ ggml_tensor * idx = ggml_argmax(ctx0, ggml_mul_mat(ctx0, cb_norm, e));
+ z = ggml_sub(ctx0, z, ggml_get_rows(ctx0, table, idx));
+ return idx;
+ };
+ auto page = [&](ggml_tensor * t, int64_t g) {
+ return ggml_view_2d(ctx0, t, t->ne[0], t->ne[1], t->nb[1], (size_t) g * t->nb[2]);
+ };
+
+ // backbone row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1), same as fish_s2_gen
+ const int64_t n_cb = model.gen_code_embd_w->ne[2];
+ ggml_tensor * idx = vq(c2w.quant_first_in_w, c2w.quant_first_in_b, c2w.quant_first_cb_norm_w, c2w.quant_first_cb_w);
+ ggml_tensor * out = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, idx),
+ ggml_get_rows(ctx0, page(model.gen_code_embd_w, 0), idx));
+ for (int64_t g = 1; g < n_cb; g++) {
+ ggml_tensor * in_b = ggml_view_1d(ctx0, c2w.quant_rest_in_b, c2w.quant_rest_in_b->ne[0], (size_t) (g - 1) * c2w.quant_rest_in_b->nb[1]);
+ idx = vq(page(c2w.quant_rest_in_w, g - 1), in_b, page(c2w.quant_rest_cb_norm_w, g - 1), page(c2w.quant_rest_cb_w, g - 1));
+ out = ggml_add(ctx0, out, ggml_get_rows(ctx0, page(model.gen_code_embd_w, g), idx));
+ }
+ out = ggml_scale(ctx0, out, 1.0f / sqrtf((float) (n_cb + 1)));
+ cb(out, "spk_rows", -1);
+ ggml_build_forward_expand(gf, out);
+
+ return gf;
+}
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index a8e6120da..d2de0273a 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -462,7 +462,8 @@ struct mtmd_graph_qwen3tts_gen : mtmd_graph {
ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const;
ggml_tensor * quant_decode(ggml_tensor * inp_codes) const;
- ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il) const;
+ // window <= 0 uses hparams.wav_tfm_swa
+ ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il, int64_t window = 0) const;
ggml_tensor * convnext_block(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk, const std::string & state_prefix) const;
ggml_tensor * dac_res_unit(ggml_tensor * x, const mtmd_code2wav::dac_res & res, int dilation, const std::string & state_name) const;
@@ -569,6 +570,32 @@ struct mtmd_graph_drama3_gen : mtmd_graph {
ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos);
};
+//
+// fish_s2 generation:
+// GEN_CODE = fast AR, backbone hidden + semantic code -> other codes of the frame + next backbone input
+// GEN_WAV = Fish DAC decoder, same as drama3
+//
+struct mtmd_graph_fish_s2_gen : mtmd_graph_drama3_gen {
+ using mtmd_graph_drama3_gen::mtmd_graph_drama3_gen;
+ ggml_cgraph * build() override;
+
+ ggml_cgraph * build_fast_ar();
+};
+
+// fish_s2 voice cloning: raw 44.1 kHz clip -> codec encoder + RVQ -> one backbone input row per frame
+struct mtmd_graph_fish_s2_spkenc : mtmd_graph {
+ mtmd_graph_fish_s2_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
+ ggml_cgraph * build() override;
+
+ // codec encoder transformer layers, the rest of model.layers is the quantizer pre_module
+ static constexpr int n_enc_tfm_layer = 4;
+ static constexpr int n_tfm_layer = 12;
+
+ ggml_tensor * conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation) const;
+ ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const;
+ ggml_tensor * convnext(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk) const;
+};
+
//
// zonos3 generation:
// GEN_CODE = 10 codebook heads, or the backbone row of a sampled frame
diff --git a/tools/mtmd/models/qwen3tts-gen.cpp b/tools/mtmd/models/qwen3tts-gen.cpp
index 59958fc4c..91d5a5c12 100644
--- a/tools/mtmd/models/qwen3tts-gen.cpp
+++ b/tools/mtmd/models/qwen3tts-gen.cpp
@@ -156,10 +156,13 @@ ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::layer_forward(
q = ggml_reshape_3d(ctx0, q, d_head, n_head, 1);
k = ggml_reshape_3d(ctx0, k, d_head, n_head_kv, 1);
- q = ggml_rms_norm(ctx0, q, hparams.eps);
- q = ggml_mul(ctx0, q, layer.q_norm);
- k = ggml_rms_norm(ctx0, k, hparams.eps);
- k = ggml_mul(ctx0, k, layer.k_norm);
+ // fish_s2 fast AR has no qk norm
+ if (layer.q_norm) {
+ q = ggml_rms_norm(ctx0, q, hparams.eps);
+ q = ggml_mul(ctx0, q, layer.q_norm);
+ k = ggml_rms_norm(ctx0, k, hparams.eps);
+ k = ggml_mul(ctx0, k, layer.k_norm);
+ }
q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, (int) d_head, GGML_ROPE_TYPE_NEOX, 0,
hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
@@ -418,12 +421,12 @@ ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_
// one pre_transformer layer over a batch of N = sliding_window new frames
// attention runs over [(W-1)-frame prefix from the last batch] + [N new frames]
// RoPE positions come from a persisted counter, so phases line up across batches
-ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il, int64_t window) const {
const int n_head = hparams.wav_tfm_n_head;
const int n_head_kv = hparams.wav_tfm_n_head_kv;
const int64_t d_head = layer.q_w->ne[1] / n_head;
const float kq_scale = 1.0f / sqrtf((float) d_head);
- const int64_t W = hparams.wav_tfm_swa; // == N, frames per batch
+ const int64_t W = window > 0 ? window : hparams.wav_tfm_swa; // code2wav: == N, frames per batch
const int64_t N = cur->ne[1];
const int64_t prefix = W - 1;
const int64_t total_kv = prefix + N;
diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp
index dfd3b30e7..419bbede1 100644
--- a/tools/mtmd/mtmd-audio.cpp
+++ b/tools/mtmd/mtmd-audio.cpp
@@ -2242,6 +2242,16 @@ bool mtmd_audio_preprocessor_pockettts::process(const float * sa
return true;
}
+bool mtmd_audio_preprocessor_fish_s2::process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const {
+ const size_t max_samples = (size_t) mtmd_hparams::fish_s2_max_spk_seconds * hparams.audio_sample_rate;
+ if (n_samples > max_samples) {
+ LOG_WRN("%s: speaker reference is %.1f s, truncating to the first %d s\n", __func__,
+ (double) n_samples / hparams.audio_sample_rate, mtmd_hparams::fish_s2_max_spk_seconds);
+ n_samples = max_samples;
+ }
+ return mtmd_audio_preprocessor_muse_spark_audio::process(samples, n_samples, output, shape_only);
+}
+
//
// mtmd_audio_preprocessor_vibevoice_asr
//
diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h
index b4b1ce692..893739d93 100644
--- a/tools/mtmd/mtmd-audio.h
+++ b/tools/mtmd/mtmd-audio.h
@@ -129,6 +129,12 @@ struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor {
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
+// fish_s2 speaker reference: same as muse spark (raw waveform, whole frames), truncated to fish_s2_max_spk_seconds
+struct mtmd_audio_preprocessor_fish_s2 : mtmd_audio_preprocessor_muse_spark_audio {
+ using mtmd_audio_preprocessor_muse_spark_audio::mtmd_audio_preprocessor_muse_spark_audio;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
+};
+
// vibevoice: gain to -25 dBFS RMS, without clipping the peak
// the RMS is floored at -50 dBFS, so that silence or noise alone is not boosted to full scale
double mtmd_audio_vibevoice_gain(double sum_sq, int64_t n_samples, double peak);
diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp
index 100bc3c58..cb2029685 100644
--- a/tools/mtmd/mtmd-graph.cpp
+++ b/tools/mtmd/mtmd-graph.cpp
@@ -973,6 +973,10 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
{
builder = std::make_unique<mtmd_graph_pockettts_spkenc>(ctx, img);
} break;
+ case PROJECTOR_TYPE_FISH_S2_SPKENC:
+ {
+ builder = std::make_unique<mtmd_graph_fish_s2_spkenc>(ctx, img);
+ } break;
case PROJECTOR_TYPE_POCKETTTS_GEN:
{
const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
@@ -1035,13 +1039,19 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
builder = std::make_unique<mtmd_graph_zonos3_gen>(ctx, img, gen_process, all_known, n_frames);
} break;
case PROJECTOR_TYPE_DRAMA3_GEN:
+ case PROJECTOR_TYPE_FISH_S2_GEN:
{
const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
const int64_t n_cb = ctx->model.gen_code_embd_w->ne[2];
const int n_frames = params && params->codes ? (int) (params->codes->size() / n_cb) : 1;
- builder = std::make_unique<mtmd_graph_drama3_gen>(ctx, img, gen_process, params ? params->gen_state : nullptr, n_frames,
- params ? params->top_k : 30, params ? params->top_p : 0.9f,
- params ? params->temp : 1.0f);
+ const int top_k = params ? params->top_k : 30;
+ const float top_p = params ? params->top_p : 0.9f;
+ const float temp = params ? params->temp : 1.0f;
+ if (ctx->model.proj_type == PROJECTOR_TYPE_FISH_S2_GEN) {
+ builder = std::make_unique<mtmd_graph_fish_s2_gen>(ctx, img, gen_process, params ? params->gen_state : nullptr, n_frames, top_k, top_p, temp);
+ } else {
+ builder = std::make_unique<mtmd_graph_drama3_gen>(ctx, img, gen_process, params ? params->gen_state : nullptr, n_frames, top_k, top_p, temp);
+ }
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 2148ec526..79541b813 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -2088,6 +2088,7 @@ public:
prompt_tokens.clear();
prompt_pos = 0;
frame.clear();
+ main_logits.clear();
hist.clear();
codes_buf.clear();
n_decoded = 0;
@@ -2110,31 +2111,16 @@ public:
sem_begin = find_special_token(vocab, "<|semantic:0|>");
eos = find_special_token(vocab, "<|im_end|>");
- if (sem_begin == LLAMA_TOKEN_NULL || eos == LLAMA_TOKEN_NULL) {
- LOG_ERR("mtmd_helper_gen_audio: missing Drama 3 tokens in vocab\n");
+ if (sem_begin == LLAMA_TOKEN_NULL || eos == LLAMA_TOKEN_NULL ||
+ sem_begin + n_cb_vocab > llama_vocab_n_tokens(vocab)) {
+ LOG_ERR("mtmd_helper_gen_audio: missing semantic tokens in vocab\n");
return 1;
}
- // reference-bracket-v1 prompt without references, tokenize each segment alone as the reference does
- std::string text(inp->prompt, inp->prompt_len);
- if (text.rfind("<|speaker:", 0) != 0) {
- text = "<|speaker:0|> " + text;
- }
- const char * segments[] = {
- "<|im_start|>system\n", "convert the provided text to speech", "<|im_end|>\n",
- "<|im_start|>user\n", text.c_str(), "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>",
- };
- for (const char * seg : segments) {
- const int32_t len = (int32_t) strlen(seg);
- std::vector<llama_token> ids(len + 16);
- const int32_t n = llama_tokenize(vocab, seg, len, ids.data(), (int32_t) ids.size(), false, true);
- if (n < 0) {
- LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
- return 1;
- }
- prompt_tokens.insert(prompt_tokens.end(), ids.begin(), ids.begin() + n);
+ if (!tokenize_segments(prompt_segments(std::string(inp->prompt, inp->prompt_len)), prompt_tokens)) {
+ return 1;
}
- if (prompt_tokens.size() >= llama_n_ctx_seq(lctx) || prompt_tokens.size() > n_tower_rows) {
+ if (prompt_tokens.size() >= llama_n_ctx_seq(lctx) || prompt_tokens.size() > max_rows()) {
LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n");
return 1;
}
@@ -2170,6 +2156,7 @@ public:
const float * h = llama_get_embeddings_ith(lctx, i);
if (prompt_pos + i == n_prompt - 1) {
h_state_buf.assign(h, h + n_embd); // first frame comes from the last prompt row
+ save_main_logits(i);
break;
}
if (!run_tower(-1, h)) {
@@ -2188,9 +2175,9 @@ public:
*h_state_out = nullptr;
*out_stop = false;
- const llama_token main = sample_main(llama_get_logits_ith(lctx, -1));
+ const llama_token main = sample_main(main_logits.data());
const size_t n_rows = prompt_tokens.size() - 1 + codes_buf.size() / n_cb;
- if (main == eos || (uint32_t) pos >= llama_n_ctx_seq(lctx) || n_rows >= n_tower_rows) {
+ if (main == eos || (uint32_t) pos >= llama_n_ctx_seq(lctx) || n_rows >= max_rows()) {
*out_stop = true;
return 0;
}
@@ -2222,6 +2209,7 @@ public:
pos++;
const float * h = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(h, h + n_embd);
+ save_main_logits(-1);
*h_state_out = h_state_buf.data();
if (stream && codes_buf.size() / n_cb - n_decoded >= stream_frames) {
@@ -2282,7 +2270,44 @@ public:
return decode_wav() ? 0 : 1;
}
-private:
+protected:
+ // tokenize each segment alone as the reference does
+ bool tokenize_segments(const std::vector<std::string> & segments, std::vector<llama_token> & out) const {
+ for (const std::string & seg : segments) {
+ std::vector<llama_token> ids(seg.size() + 16);
+ const int32_t n = llama_tokenize(vocab, seg.c_str(), (int32_t) seg.size(), ids.data(), (int32_t) ids.size(), false, true);
+ if (n < 0) {
+ LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
+ return false;
+ }
+ out.insert(out.end(), ids.begin(), ids.begin() + n);
+ }
+ return true;
+ }
+
+ // reference-bracket-v1 prompt without references
+ virtual std::vector<std::string> prompt_segments(std::string text) const {
+ if (text.rfind("<|speaker:", 0) != 0) {
+ text = "<|speaker:0|> " + text;
+ }
+ return {
+ "<|im_start|>system\n", "convert the provided text to speech", "<|im_end|>\n",
+ "<|im_start|>user\n", text, "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>",
+ };
+ }
+
+ // prompt rows but the last + generated frames
+ virtual size_t max_rows() const { return n_tower_rows; }
+
+ // copy right after our own decode, other sequences may decode on lctx before the next step_gen
+ // layout: eos, then semantic codes 0..n_cb_vocab-1
+ void save_main_logits(int32_t i) {
+ const float * logits = llama_get_logits_ith(lctx, i);
+ main_logits.resize(n_cb_vocab + 1);
+ main_logits[0] = logits[eos];
+ std::copy(logits + sem_begin, logits + sem_begin + n_cb_vocab, main_logits.begin() + 1);
+ }
+
// repetition-aware sampling: if a semantic draw repeats one of the last ras_window main tokens, draw again with fixed settings
static constexpr size_t ras_window = 16;
static constexpr size_t n_cb = 10;
@@ -2332,11 +2357,12 @@ private:
}
// candidates: eos, then semantic codes 1.. (the reference never samples code 0)
- llama_token sample_main(const float * logits) {
+ // logits as saved by save_main_logits()
+ virtual llama_token sample_main(const float * logits) {
std::vector<float> cand(n_cb_vocab);
- cand[0] = logits[eos];
+ cand[0] = logits[0];
for (int i = 1; i < n_cb_vocab; i++) {
- cand[i] = logits[sem_begin + i];
+ cand[i] = logits[1 + i];
}
auto to_token = [&](int i) { return i == 0 ? eos : sem_begin + i; };
const llama_token first = to_token(draw(cand, 1.0f, 0, 0.9f));
@@ -2397,6 +2423,7 @@ private:
int prompt_pos = 0;
std::mt19937 rng;
+ std::vector<float> main_logits; // see save_main_logits()
std::vector<int32_t> frame; // codes of the last frame, input of the next tower row
std::vector<llama_token> hist; // last main tokens, for repetition-aware sampling
std::vector<int32_t> codes_buf; // frame-major
@@ -2411,6 +2438,195 @@ private:
bool wav_header_sent = false;
};
+// Fish Audio S2 Pro: same tokens, main-token loop and Fish DAC as drama3
+// the fast AR only runs on generated frames, so the prompt is a plain decode
+class fish_s2_gen_audio_pipeline : public drama3_gen_audio_pipeline {
+public:
+ using drama3_gen_audio_pipeline::drama3_gen_audio_pipeline;
+
+ void reset() override {
+ drama3_gen_audio_pipeline::reset();
+ parts.clear();
+ part_idx = 0;
+ part_pos = 0;
+ }
+
+ // voice cloning (sglang-omni fishaudio_s2_pro): a system message with the transcript and the codec rows of the clip
+ int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
+ std::vector<float> ref_rows;
+ if (inp->speaker_ref && !encode_speaker(inp->speaker_ref, ref_rows)) {
+ return 1;
+ }
+ mtmd_helper_gen_audio_inp text_inp = *inp;
+ text_inp.speaker_ref = nullptr;
+ if (drama3_gen_audio_pipeline::set_input(&text_inp) != 0) {
+ return 1;
+ }
+
+ size_t n_total = prompt_tokens.size();
+ if (!ref_rows.empty()) {
+ // optional, the reference leaves the text empty when it has none
+ std::string ref_text = inp->speaker_ref_text ? inp->speaker_ref_text : "";
+ if (!ref_text.empty() && ref_text.rfind("<|speaker:", 0) != 0) {
+ ref_text = "<|speaker:0|>" + ref_text;
+ }
+ prompt_part head, rows, tail;
+ if (!tokenize_segments({ "<|im_start|>system\n", "convert the provided text to speech reference to the following:\n\nText:\n",
+ ref_text, "\n\nSpeech:\n" }, head.tokens) ||
+ !tokenize_segments({ "<|im_end|>\n" }, tail.tokens)) {
+ return 1;
+ }
+ rows.rows = std::move(ref_rows);
+ for (auto * part : { &head, &rows, &tail }) {
+ n_total += part->size(n_embd);
+ parts.push_back(std::move(*part));
+ }
+ }
+ if (n_total >= llama_n_ctx_seq(lctx)) {
+ LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n");
+ return 1;
+ }
+ prompt_part text;
+ text.tokens = prompt_tokens;
+ parts.push_back(std::move(text));
+ return 0;
+ }
+
+ // the first frame comes from the last prompt row
+ int32_t step_prompt(int32_t n_batch) override {
+ GGML_ASSERT(n_batch > 0);
+ if (part_idx >= parts.size()) {
+ return 0;
+ }
+ const prompt_part & part = parts[part_idx];
+ const int n_part = part.size(n_embd);
+ const int n = std::min(n_batch, n_part - part_pos);
+ const bool last = part_idx + 1 == parts.size() && part_pos + n == n_part;
+
+ int ret = 0;
+ if (part.rows.empty()) {
+ llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
+ if (!batch_set_tokens(batch.get(), part.tokens.data() + part_pos, n, pos, seq_id)) {
+ return -1;
+ }
+ if (last) {
+ llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
+ }
+ ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
+ } else {
+ decode_embd_batch batch(part.rows.data() + (size_t) part_pos * n_embd, n, 1, n_embd);
+ batch.set_position_normal(pos, seq_id);
+ ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n));
+ }
+ if (ret != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
+ return -1;
+ }
+ if (last) {
+ const float * h = llama_get_embeddings_ith(lctx, -1);
+ h_state_buf.assign(h, h + n_embd);
+ save_main_logits(-1);
+ }
+ pos += n;
+ part_pos += n;
+ if (part_pos == n_part) {
+ part_idx++;
+ part_pos = 0;
+ }
+ int32_t n_left = -part_pos;
+ for (size_t i = part_idx; i < parts.size(); i++) {
+ n_left += parts[i].size(n_embd);
+ }
+ return n_left;
+ }
+
+protected:
+ // reference (sglang-omni fishaudio_s2_pro) has no system message
+ std::vector<std::string> prompt_segments(std::string text) const override {
+ if (text.rfind("<|speaker:", 0) != 0) {
+ text = "<|speaker:0|>" + text;
+ }
+ return {
+ "<|im_start|>user\n", text, "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>",
+ };
+ }
+
+ // no codebook tower, only the context limits the length
+ size_t max_rows() const override { return SIZE_MAX; }
+
+ // token ids, or backbone input rows of n_embd floats
+ struct prompt_part {
+ std::vector<llama_token> tokens;
+ std::vector<float> rows;
+ int size(int n_embd) const { return rows.empty() ? (int) tokens.size() : (int) (rows.size() / n_embd); }
+ };
+ std::vector<prompt_part> parts;
+ size_t part_idx = 0;
+ int part_pos = 0;
+
+ // same as the reference: repetition penalty over the last ras_window main tokens, top-k, top-p on the
+ // untempered logits (the crossing token is dropped), then temperature
+ // a repeat within the last 4 main tokens switches to the RAS settings
+ llama_token sample_main(const float * logits) override {
+ static constexpr int32_t top_k = 30;
+ static constexpr float rep_penalty = 1.1f;
+
+ std::vector<std::pair<float, llama_token>> cand;
+ cand.reserve(n_cb_vocab + 1);
+ cand.push_back({ logits[0], eos });
+ for (int i = 0; i < n_cb_vocab; i++) {
+ cand.push_back({ logits[1 + i], sem_begin + i });
+ }
+ for (auto & c : cand) {
+ if (std::find(hist.begin(), hist.end(), c.second) != hist.end()) {
+ c.first = c.first < 0.0f ? c.first * rep_penalty : c.first / rep_penalty;
+ }
+ }
+
+ bool ras = false;
+ if (hist.size() >= 4) {
+ std::vector<llama_token> last4(hist.end() - 4, hist.end());
+ std::sort(last4.begin(), last4.end());
+ ras = std::adjacent_find(last4.begin(), last4.end()) != last4.end();
+ }
+ const float temp = ras ? 1.0f : 0.8f;
+ const float top_p = ras ? 0.9f : 0.8f;
+
+ const size_t k = std::min<size_t>(top_k, cand.size());
+ std::partial_sort(cand.begin(), cand.begin() + k, cand.end(), [](const auto & a, const auto & b) { return a.first > b.first; });
+
+ std::vector<double> prob(k);
+ double sum = 0.0;
+ for (size_t i = 0; i < k; i++) {
+ prob[i] = std::exp((double) (cand[i].first - cand[0].first));
+ sum += prob[i];
+ }
+ size_t n_keep = k;
+ double cum = 0.0;
+ for (size_t i = 0; i < k; i++) {
+ cum += prob[i];
+ if (i > 0 && cum > top_p * sum) {
+ n_keep = i;
+ break;
+ }
+ }
+
+ double kept = 0.0;
+ for (size_t i = 0; i < n_keep; i++) {
+ prob[i] = std::exp((double) (cand[i].first - cand[0].first) / temp);
+ kept += prob[i];
+ }
+ double r = std::uniform_real_distribution<double>(0.0, 1.0)(rng) * kept;
+ for (size_t i = 0; i < n_keep; i++) {
+ r -= prob[i];
+ if (r <= 0.0) {
+ return cand[i].second;
+ }
+ }
+ return cand[n_keep - 1].second;
+ }
+};
+
// ZONOS3: one backbone row per frame of 10 codebooks, codebook j lags j frames
class zonos3_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
@@ -2953,17 +3169,21 @@ static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lc
return std::unique_ptr<mtmd_gen_audio_pipeline>(new drama3_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_ZONOS3:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new zonos3_gen_audio_pipeline(lctx, mctx));
+ case MTMD_GEN_AUDIO_TYPE_FISH_S2:
+ return std::unique_ptr<mtmd_gen_audio_pipeline>(new fish_s2_gen_audio_pipeline(lctx, mctx));
default:
return nullptr;
}
}
struct mtmd_helper_gen_audio {
+ llama_context * lctx = nullptr;
std::unique_ptr<mtmd_gen_audio_pipeline> pipeline;
};
mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(struct llama_context * lctx, struct mtmd_context * mctx) {
auto * ctx = new mtmd_helper_gen_audio();
+ ctx->lctx = lctx;
ctx->pipeline = make_pipeline(lctx, mctx);
return ctx;
}
@@ -3015,6 +3235,11 @@ int32_t mtmd_helper_gen_audio_step_prompt(mtmd_helper_gen_audio * ctx, int32_t n
if (!ctx->pipeline) {
return -1;
}
+ // the context may use a smaller n_batch than requested (e.g. clamped to n_ctx), llama_process() would fail above it
+ if (n_batch <= 0 || (uint32_t) n_batch > llama_n_batch(ctx->lctx)) {
+ LOG_ERR("mtmd_helper_gen_audio: n_batch must be in [1, %u], got %d\n", llama_n_batch(ctx->lctx), n_batch);
+ return -1;
+ }
return ctx->pipeline->step_prompt(n_batch);
}
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index b33ed65ec..82e86bd1f 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -310,6 +310,7 @@ struct mtmd_helper_gen_audio_inp {
enum mtmd_helper_gen_audio_out_mode out_mode; // n_past > 0 only (default: auto)
mtmd_bitmap * speaker_ref; // optional, can be NULL: audio to clone, or a voice bitmap
+ const char * speaker_ref_text; // optional, can be NULL: transcript of speaker_ref, for models that put it in the prompt (e.g. Fish S2)
const char * lang; // optional, can be NULL
int32_t top_k;
@@ -339,7 +340,7 @@ MTMD_API int32_t mtmd_helper_gen_audio_set_input(
mtmd_helper_gen_audio * ctx,
const struct mtmd_helper_gen_audio_inp * inp);
-// processes at most n_batch prompt tokens per call
+// processes at most n_batch prompt tokens per call, n_batch must not exceed llama_n_batch() of the context
// returns: >0 = number of prompt tokens remaining, 0 = done, <0 = error
MTMD_API int32_t mtmd_helper_gen_audio_step_prompt(
mtmd_helper_gen_audio * ctx,
@@ -423,6 +424,7 @@ struct gen_audio {
mtmd_helper_gen_audio_inp data = mtmd_helper_gen_audio_inp_default();
std::string prompt_str;
std::string lang_str;
+ std::string speaker_ref_text_str;
mtmd::bitmap_ptr speaker_ref_ptr;
inp() = default;
@@ -434,6 +436,7 @@ struct gen_audio {
void set_prompt (std::string p) { prompt_str = std::move(p); }
void set_lang (std::string l) { lang_str = std::move(l); }
void set_speaker_ref(mtmd::bitmap_ptr bmp) { speaker_ref_ptr = std::move(bmp); }
+ void set_speaker_ref_text(std::string t) { speaker_ref_text_str = std::move(t); }
// pointers are only valid as long as *this is alive
const mtmd_helper_gen_audio_inp * get() {
@@ -441,6 +444,7 @@ struct gen_audio {
data.prompt_len = prompt_str.size();
data.lang = lang_str.empty() ? nullptr : lang_str.c_str();
data.speaker_ref = speaker_ref_ptr.get();
+ data.speaker_ref_text = speaker_ref_text_str.empty() ? nullptr : speaker_ref_text_str.c_str();
return &data;
}
};
diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h
index 70db37654..ff2b70230 100644
--- a/tools/mtmd/mtmd-impl.h
+++ b/tools/mtmd/mtmd-impl.h
@@ -271,6 +271,31 @@
#define TN_A_GEN_CODE_BLK_EMBD "a.gen.code.blk.%d.embd.%s"
#define TN_A_GEN_CODE_BLK_OUT_NORM "a.gen.code.blk.%d.out_norm.%s"
+// fish_s2: fast AR input embedding, shared by all codebooks
+#define TN_A_GEN_CODE_TOK_EMBD "a.gen.code.tok_embd.%s"
+
+// fish_s2 codec encoder, the transformer layers use the generic a.blk.* names
+#define TN_A_CODEC_ENC_ENTRY "a.codec.enc.entry.%s"
+#define TN_A_CODEC_ENC_SNAKE "a.codec.enc.blk.%d.snake.%s"
+#define TN_A_CODEC_ENC_CONV "a.codec.enc.blk.%d.conv.%s"
+#define TN_A_CODEC_ENC_RES_ACT1 "a.codec.enc.blk.%d.res.%d.act1.%s"
+#define TN_A_CODEC_ENC_RES_CONV1 "a.codec.enc.blk.%d.res.%d.conv1.%s"
+#define TN_A_CODEC_ENC_RES_ACT2 "a.codec.enc.blk.%d.res.%d.act2.%s"
+#define TN_A_CODEC_ENC_RES_CONV2 "a.codec.enc.blk.%d.res.%d.conv2.%s"
+#define TN_A_CODEC_ENC_POST_SNAKE "a.codec.enc.post_snake.%s"
+#define TN_A_CODEC_ENC_POST_CONV "a.codec.enc.post_conv.%s"
+#define TN_A_CODEC_DOWN_CONV "a.codec.down.blk.%d.conv.%s"
+#define TN_A_CODEC_DOWN_DWCONV "a.codec.down.blk.%d.dwconv.%s"
+#define TN_A_CODEC_DOWN_NORM "a.codec.down.blk.%d.norm.%s"
+#define TN_A_CODEC_DOWN_PW1 "a.codec.down.blk.%d.pw1.%s"
+#define TN_A_CODEC_DOWN_PW2 "a.codec.down.blk.%d.pw2.%s"
+#define TN_A_CODEC_DOWN_GAMMA "a.codec.down.blk.%d.gamma"
+#define TN_A_CODEC_PRE_NORM "a.codec.pre_norm.%s"
+#define TN_A_CODEC_VQ_FIRST_IN "a.codec.vq.first.in_proj.%s"
+#define TN_A_CODEC_VQ_FIRST_CB "a.codec.vq.first.codebook.%s"
+#define TN_A_CODEC_VQ_REST_IN "a.codec.vq.rest.in_proj.%s"
+#define TN_A_CODEC_VQ_REST_CB "a.codec.vq.rest.codebook.%s"
+
// qwen3tts code2wav (RVQ codes -> raw PCM)
// pre_transformer layers use the generic TN_ATTN_*/TN_FFN_*/TN_LN_*/TN_LS_* macros, prefix "a.gen.wav.tfm"
#define TN_A_GEN_WAV_QUANT_FIRST_IN "a.gen.wav.quant.first.in_proj.%s"
@@ -588,6 +613,8 @@ enum projector_type {
PROJECTOR_TYPE_DRAMA3_GEN,
PROJECTOR_TYPE_ZONOS3_GEN,
PROJECTOR_TYPE_ZONOS3_SPKENC,
+ PROJECTOR_TYPE_FISH_S2_GEN,
+ PROJECTOR_TYPE_FISH_S2_SPKENC,
PROJECTOR_TYPE_MUSE_GLIMMER,
PROJECTOR_TYPE_SENSENOVA_U1,
PROJECTOR_TYPE_COHERE2V,
@@ -680,6 +707,8 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_DRAMA3_GEN, "drama3_gen"},
{ PROJECTOR_TYPE_ZONOS3_GEN, "zonos3_gen"},
{ PROJECTOR_TYPE_ZONOS3_SPKENC, "zonos3_spkenc"},
+ { PROJECTOR_TYPE_FISH_S2_GEN, "fish_s2_gen"},
+ { PROJECTOR_TYPE_FISH_S2_SPKENC, "fish_s2_spkenc"},
{ PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"},
{ PROJECTOR_TYPE_SENSENOVA_U1, "sensenova_u1"},
{ PROJECTOR_TYPE_COHERE2V, "cohere2v"},
diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp
index 86bb127fb..9e2b2fec1 100644
--- a/tools/mtmd/mtmd-model.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -1094,7 +1094,11 @@ struct mtmd_model_loader {
hparams.audio_sample_rate = 44100;
} break;
case PROJECTOR_TYPE_DRAMA3_GEN:
+ case PROJECTOR_TYPE_FISH_S2_GEN:
{
+ // fish_s2 fast AR, drama3 hardcodes its tower RoPE in the graph
+ hparams.rope_theta = 1000000.0f;
+
// Fish DAC decoder, codec.pth has no config
hparams.audio_sample_rate = 44100;
hparams.wav_tfm_n_layer = 8;
@@ -1125,6 +1129,19 @@ struct mtmd_model_loader {
hparams.flow_n_step = 1;
hparams.gen_eos_threshold = -4.0f;
} break;
+ case PROJECTOR_TYPE_FISH_S2_SPKENC:
+ {
+ // Fish DAC encoder, codec.pth has no config
+ // one row per 2048 samples: encoder strides, then the quantizer downsample
+ hparams.audio_sample_rate = 44100;
+ hparams.audio_patch_sizes = { 2, 4, 8, 8, 2, 2 };
+ // whole frames, about 10 s like a typical reference clip
+ hparams.warmup_audio_size = hparams.audio_samples_per_token() * 215;
+ hparams.wav_tfm_n_head = 16;
+ hparams.wav_tfm_n_head_kv = 16;
+ hparams.wav_tfm_eps = 1e-5f;
+ hparams.wav_tfm_rope_theta = 10000.0f;
+ } break;
case PROJECTOR_TYPE_CHATTERBOX:
{
// the classic cfm solver unrolls up to 10 steps x 2 cfg estimator evaluations in one graph
@@ -1372,7 +1389,8 @@ struct mtmd_model_loader {
const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA &&
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
- model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC;
+ model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC &&
+ model.proj_type != PROJECTOR_TYPE_FISH_S2_SPKENC;
// Validate audio hparams loaded from GGUF metadata
if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) {
@@ -2526,6 +2544,71 @@ struct mtmd_model_loader {
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias"));
} break;
+ case PROJECTOR_TYPE_FISH_S2_SPKENC:
+ {
+ // codec encoder in the code2wav structs: dac = encoder blocks, upsample = quantizer downsample
+ // generic a.blk.* layers: 0..3 last encoder block, 4..11 quantizer pre_module
+ auto & c2w = model.c2w;
+ c2w.dac_entry_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_ENTRY, "weight"));
+ c2w.dac_entry_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_ENTRY, "bias"));
+ c2w.dac.resize(4);
+ for (int il = 0; il < (int) c2w.dac.size(); il++) {
+ auto & blk = c2w.dac[il];
+ blk.snake_alpha = get_tensor(mtmd_format(TN_A_CODEC_ENC_SNAKE, il, "alpha"));
+ blk.snake_beta = get_tensor(mtmd_format(TN_A_CODEC_ENC_SNAKE, il, "beta"));
+ blk.conv_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_CONV, il, "weight"));
+ blk.conv_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_CONV, il, "bias"));
+ blk.res.resize(3);
+ for (int ir = 0; ir < (int) blk.res.size(); ir++) {
+ auto & res = blk.res[ir];
+ res.act1_alpha = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_ACT1, il, ir, "alpha"));
+ res.act1_beta = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_ACT1, il, ir, "beta"));
+ res.conv1_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_CONV1, il, ir, "weight"));
+ res.conv1_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_CONV1, il, ir, "bias"));
+ res.act2_alpha = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_ACT2, il, ir, "alpha"));
+ res.act2_beta = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_ACT2, il, ir, "beta"));
+ res.conv2_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_CONV2, il, ir, "weight"));
+ res.conv2_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_RES_CONV2, il, ir, "bias"));
+ }
+ }
+ c2w.dac_post_snake_alpha = get_tensor(mtmd_format(TN_A_CODEC_ENC_POST_SNAKE, "alpha"));
+ c2w.dac_post_snake_beta = get_tensor(mtmd_format(TN_A_CODEC_ENC_POST_SNAKE, "beta"));
+ c2w.dac_post_conv_w = get_tensor(mtmd_format(TN_A_CODEC_ENC_POST_CONV, "weight"));
+ c2w.dac_post_conv_b = get_tensor(mtmd_format(TN_A_CODEC_ENC_POST_CONV, "bias"));
+
+ c2w.upsample.resize(2);
+ for (int il = 0; il < (int) c2w.upsample.size(); il++) {
+ auto & down = c2w.upsample[il];
+ down.conv_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_CONV, il, "weight"));
+ down.conv_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_CONV, il, "bias"));
+ down.dwconv_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_DWCONV, il, "weight"));
+ down.dwconv_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_DWCONV, il, "bias"));
+ down.norm_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_NORM, il, "weight"));
+ down.norm_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_NORM, il, "bias"));
+ down.pw1_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_PW1, il, "weight"));
+ down.pw1_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_PW1, il, "bias"));
+ down.pw2_w = get_tensor(mtmd_format(TN_A_CODEC_DOWN_PW2, il, "weight"));
+ down.pw2_b = get_tensor(mtmd_format(TN_A_CODEC_DOWN_PW2, il, "bias"));
+ down.gamma = get_tensor(mtmd_format(TN_A_CODEC_DOWN_GAMMA, il));
+ }
+ c2w.tfm_output_norm_w = get_tensor(mtmd_format(TN_A_CODEC_PRE_NORM, "weight"));
+ if (hparams.n_layer != mtmd_graph_fish_s2_spkenc::n_tfm_layer) {
+ throw std::runtime_error(mtmd_format("%s: fish_s2_spkenc expects %d transformer layers\n", __func__, mtmd_graph_fish_s2_spkenc::n_tfm_layer));
+ }
+ // VQ: search with the normalized codebooks, subtract the codec decoder's out_proj tables
+ c2w.quant_first_in_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_FIRST_IN, "weight"));
+ c2w.quant_first_in_b = get_tensor(mtmd_format(TN_A_CODEC_VQ_FIRST_IN, "bias"));
+ c2w.quant_first_cb_norm_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_FIRST_CB, "weight"));
+ c2w.quant_rest_in_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_REST_IN, "weight"));
+ c2w.quant_rest_in_b = get_tensor(mtmd_format(TN_A_CODEC_VQ_REST_IN, "bias"));
+ c2w.quant_rest_cb_norm_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_REST_CB, "weight"));
+ c2w.quant_first_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight"));
+ c2w.quant_rest_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_CB, "weight"));
+
+ // codes -> backbone rows, same tables as fish_s2_gen
+ model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight"));
+ model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
+ } break;
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
{
load_seanet(model.seanet, false);
@@ -2827,6 +2910,20 @@ struct mtmd_model_loader {
model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); // backbone rows of the semantic band
load_code2wav(false);
} break;
+ case PROJECTOR_TYPE_FISH_S2_GEN:
+ {
+ // generic a.gen.code.blk.* tensors are loaded by the loop above
+ model.gen_code_tok_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_TOK_EMBD, "weight")); // fast AR input
+ model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight")); // fast AR head, residual codebooks
+ model.gen_code_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_NORM, "weight"));
+ model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight")); // merged 3D, backbone input tables
+ model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); // backbone rows of the semantic band
+ load_code2wav(false);
+ // the graph reads model.layers[0]
+ if (hparams.n_layer < 1) {
+ throw std::runtime_error(mtmd_format("%s: fish_s2_gen has no fast AR layer\n", __func__));
+ }
+ } break;
case PROJECTOR_TYPE_CHATTERBOX:
{
auto & c = model.cbx;
@@ -4440,10 +4537,16 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
case PROJECTOR_TYPE_LFM3A_GEN:
case PROJECTOR_TYPE_DRAMA3_GEN:
case PROJECTOR_TYPE_ZONOS3_GEN:
+ case PROJECTOR_TYPE_FISH_S2_GEN:
{
// one hidden-state vector fed back to the talker per call
n_patches = 1;
} break;
+ case PROJECTOR_TYPE_FISH_S2_SPKENC:
+ {
+ // one backbone row per codec frame, the preprocessor pads to whole frames
+ n_patches = (int) (img->nx() / hparams.audio_samples_per_token());
+ } break;
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
{
// one conditioning row per 12.5Hz frame
@@ -4630,6 +4733,8 @@ int mtmd_model::n_mmproj_embd() const {
case PROJECTOR_TYPE_LFM3A_GEN:
case PROJECTOR_TYPE_DRAMA3_GEN:
case PROJECTOR_TYPE_ZONOS3_GEN:
+ case PROJECTOR_TYPE_FISH_S2_GEN:
+ case PROJECTOR_TYPE_FISH_S2_SPKENC:
return gen_code_out_embd_w->ne[0];
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
return spk_proj_w->ne[1];
diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h
index 44863636b..9af815981 100644
--- a/tools/mtmd/mtmd-model.h
+++ b/tools/mtmd/mtmd-model.h
@@ -213,6 +213,8 @@ struct mtmd_hparams {
// pocket-tts
static constexpr int32_t pockettts_max_spk_seconds = 30;
+ // fish_s2: the codec encoder runs at 44.1 kHz, memory grows with the clip
+ static constexpr int32_t fish_s2_max_spk_seconds = 30;
int32_t seanet_n_stage = 0;
std::vector<int32_t> seanet_ratios; // encoder order (reversed compared to the config)
int32_t mimi_downsample = 0; // encoder frame rate / model frame rate
@@ -622,6 +624,12 @@ struct mtmd_code2wav {
ggml_tensor * quant_rest_out_w = nullptr;
ggml_tensor * quant_rest_cb_w = nullptr; // codebooks, merged 3D [15, vocab, dim]
+ // fish_s2 codec encoder: in_proj bias and l2-normalized codebooks for the nearest-entry search
+ ggml_tensor * quant_first_in_b = nullptr;
+ ggml_tensor * quant_rest_in_b = nullptr;
+ ggml_tensor * quant_first_cb_norm_w = nullptr;
+ ggml_tensor * quant_rest_cb_norm_w = nullptr;
+
ggml_tensor * pre_conv_w = nullptr;
ggml_tensor * pre_conv_b = nullptr;
@@ -1208,6 +1216,7 @@ struct mtmd_model {
ggml_tensor * gen_code_head_w = nullptr; // per-codebook output head, merged 3D
ggml_tensor * gen_code_out_embd_w = nullptr; // codebook-0 embedding, fed back into the talker
ggml_tensor * gen_code_norm_w = nullptr; // final norm
+ ggml_tensor * gen_code_tok_embd_w = nullptr; // fish_s2: input embedding shared by all codebooks
// lfm3a depthformer (MaskGIT)
ggml_tensor * gen_code_head_norm_w = nullptr; // per-codebook norm before the tied head, merged 2D
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index db54ec9c9..db3b241d4 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -389,6 +389,7 @@ bool mtmd_subcontext::encode_batch(int n_threads, const mtmd_image_f32_batch * i
static std::vector<c2w_state_slot> list_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
switch (model.proj_type) {
case PROJECTOR_TYPE_QWEN3TTS_GEN: return list_c2w_state_slots(hparams, model);
+ case PROJECTOR_TYPE_FISH_S2_GEN: return list_c2w_state_slots(hparams, model);
case PROJECTOR_TYPE_DRAMA3_GEN: return list_drama3_state_slots(hparams, model);
case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model);
case PROJECTOR_TYPE_LFM3A_GEN: return list_lfm3a_gen_state_slots(hparams, model);
@@ -1680,6 +1681,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
case PROJECTOR_TYPE_ZONOS3_SPKENC:
+ case PROJECTOR_TYPE_FISH_S2_SPKENC:
{
// do nothing
} break;
@@ -1789,6 +1791,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
}
} break;
case PROJECTOR_TYPE_DRAMA3_GEN:
+ case PROJECTOR_TYPE_FISH_S2_GEN:
{
const int64_t n_cb = model.gen_code_embd_w->ne[2];
const int64_t n_vocab = model.gen_code_embd_w->ne[1];
@@ -1807,6 +1810,13 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
}
set_input_i32("inp_codes", codes);
set_gen_state_in();
+ } else if (model.proj_type == PROJECTOR_TYPE_FISH_S2_GEN) {
+ if (params->code0 < 0 || params->code0 >= n_vocab) {
+ LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) n_vocab);
+ return false;
+ }
+ std::vector<int32_t> code0 = { params->code0 };
+ set_input_i32("inp_code0", code0);
} else {
// code0 < 0: next row is text (prompt), the tower only fills its KV
// codes: input frame codes for a speech row, empty for a text row
@@ -2622,8 +2632,8 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
mtmd_graph_soprano::decode_spectrum(model, spectrum, out_audio);
}
- // drop the tail audio that comes from the code-0 rear padding, drama3 has no padding
- if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN) {
+ // drop the tail audio that comes from the code-0 rear padding, drama3 and fish_s2 have no padding
+ if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN && model.proj_type != PROJECTOR_TYPE_FISH_S2_GEN) {
const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0;
const int64_t n_frames_w = hparams.wav_tfm_swa;
const int64_t n_frames = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w;
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index f8f7ea048..9669afc2a 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -1314,6 +1314,10 @@ struct mtmd_context {
{
audio_preproc = std::make_unique<mtmd_audio_preprocessor_pockettts>(models.a.get());
} break;
+ case PROJECTOR_TYPE_FISH_S2_SPKENC:
+ {
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_fish_s2>(models.a.get());
+ } break;
case PROJECTOR_TYPE_CHATTERBOX_SPKENC:
{
audio_preproc = std::make_unique<mtmd_audio_preprocessor_chatterbox_ref>(models.a.get(), cbx_is_mtl(*models.a));
@@ -2490,6 +2494,10 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
info.type = MTMD_GEN_AUDIO_TYPE_ZONOS3;
info.sample_rate = 44100;
break;
+ case PROJECTOR_TYPE_FISH_S2_GEN:
+ info.type = MTMD_GEN_AUDIO_TYPE_FISH_S2;
+ info.sample_rate = 44100;
+ break;
default:
info.type = MTMD_GEN_AUDIO_TYPE_NONE;
break;
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index b0d673dbe..58c7c57a8 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -473,6 +473,7 @@ enum mtmd_gen_audio_type {
MTMD_GEN_AUDIO_TYPE_CHATTERBOX,
MTMD_GEN_AUDIO_TYPE_DRAMA3,
MTMD_GEN_AUDIO_TYPE_ZONOS3,
+ MTMD_GEN_AUDIO_TYPE_FISH_S2,
};
struct mtmd_gen_audio_info {
diff --git a/tools/server/README.md b/tools/server/README.md
index 9d1028d25..07b530df0 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -866,6 +866,8 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For the list
`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence. Rejected when the model has no voice encoder (e.g. Pocket TTS without voice cloning).
+`speaker_ref_text`: Transcript of the reference audio, put in the prompt by some models (e.g. Fish Audio S2 Pro). Optional.
+
`top_k`, `top_p`: Sampling params for both the backbone sampler and the acoustic code predictor. Optional, `0` or absent keeps the server `--top-k`/`--top-p` for the backbone and the model defaults for the code predictor.
`temperature`: Temperature for the backbone sampler and the audio frames, for models that support it. `0` is greedy sampling. Optional, absent keeps the server `--temp` for the backbone and the model defaults for the audio.
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 28e01c69b..abaaf55b6 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -6992,6 +6992,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_tts_impl(const serve
}
task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt);
task.tts_inp.set_lang(lang);
+ task.tts_inp.set_speaker_ref_text(json_value(body, "speaker_ref_text", std::string()));
task.tts_inp.data.stream = stream;
task.tts_inp.data.out_type = response_format == "pcm" || pcm16
? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM
diff --git a/tools/tts/README.md b/tools/tts/README.md
index 9608db569..9df5078bb 100644
--- a/tools/tts/README.md
+++ b/tools/tts/README.md
@@ -17,6 +17,7 @@ Note: it's highly recommended to always provide a speaker reference voice (via `
Common params:
- Sampling params such as `--top-k`, `--top-p`, `--temp`, etc. `--temp` also sets the temperature of the audio frames for models that support it, without it they use the model default
- `-n <number_of_frames>` limits the output length, e.g. `-n 500`. Note that how many milliseconds each frame represents varies by model
+- `--tts-speaker-text` (optional) gives the transcript of `--tts-speaker-file`, some models put it in the prompt (e.g. Fish Audio S2 Pro)
- Core inference params such as `-ngl`, `-b`, `-ub`, etc.
## Supported models
diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp
index 7edf605b8..bbc9ac794 100644
--- a/tools/tts/tts.cpp
+++ b/tools/tts/tts.cpp
@@ -147,6 +147,7 @@ int main(int argc, char ** argv) {
inp.prompt = prompt.c_str();
inp.prompt_len = prompt.size();
inp.speaker_ref = speaker_bitmap.get();
+ inp.speaker_ref_text = params.tts_speaker_text.empty() ? nullptr : params.tts_speaker_text.c_str();
inp.lang = params.tts_lang.c_str();
inp.top_k = params.sampling.top_k;
inp.top_p = params.sampling.top_p;
@@ -168,7 +169,7 @@ int main(int argc, char ** argv) {
const int64_t t_prompt_start_us = ggml_time_us();
for (;;) {
- int32_t ret = gen.step_prompt(params.n_batch);
+ int32_t ret = gen.step_prompt(llama_n_batch(lctx));
if (ret < 0) {
LOG_ERR("prompt processing failed\n");
return 1;