Commit 9ae73e68f for llama.cpp
commit 9ae73e68f5a78933c0d988bbd17bf4647ad6a0b2
Author: Hans <me@hans00.me>
Date: Sat Oct 24 07:09:50 2026 +0800
mtmd : add Soprano TTS support (#28607)
Add Soprano-80M and Soprano-1.1-80M (https://huggingface.co/ekwek/Soprano-1.1-80M)
as an audio generation model: a Qwen3 backbone plus a Vocos decoder that runs
as a gen-only mmproj.
- convert: detect Soprano checkpoints and export the backbone as qwen3 and the
decoder.pth Vocos weights as the mmproj
- vocab: add the soprano pre-tokenizer (lowercase, whitespace collapse,
per-digit split, [UNK] fallback)
- mtmd: add the Vocos graph and ISTFT, allow loading a gen-only mmproj for
Soprano and include the gen-audio context in mtmd_get_memory_usage
- mtmd-helper: add the Soprano pipeline; the decoder runs over the whole
utterance, so stream mode sends the audio in one chunk after flush()
- tts: document Soprano usage
Assisted-by: Codex
Assisted-by: Claude Opus 5.5
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 5f252b490..35e73b41c 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -286,6 +286,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"SeedOssForCausalLM": "olmo",
"SmallThinkerForCausalLM": "smallthinker",
"SmolLM3ForCausalLM": "llama",
+ "SopranoTextModel": "soprano",
"Spark2_5ForCausalLM": "spark2_5",
"SolarOpenForCausalLM": "glm",
"StableLMEpochForCausalLM": "stablelm",
@@ -380,6 +381,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"Qwen3ASRForConditionalGeneration": "qwen3vl",
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
"PocketTTSModel": "pockettts",
+ "SopranoModel": "soprano",
"Qwen3TTSForConditionalGeneration": "qwen3tts",
"Qwen3VLForConditionalGeneration": "qwen3vl",
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
diff --git a/conversion/soprano.py b/conversion/soprano.py
new file mode 100644
index 000000000..cf4c959c6
--- /dev/null
+++ b/conversion/soprano.py
@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import torch
+
+from .base import ModelBase, MmprojModel, gguf, logger
+from .qwen import Qwen3Model
+
+
+def _is_soprano_checkpoint(dir_model: Path) -> bool:
+ if not (dir_model / "decoder.pth").is_file() or not (dir_model / "config.json").is_file():
+ return False
+ with open(dir_model / "config.json", encoding="utf-8") as f:
+ config = json.load(f)
+ if config.get("model_type") != "qwen3" or config.get("bos_token_id") != 3 or config.get("eos_token_id") != 3:
+ return False
+ if not (dir_model / "tokenizer.json").is_file():
+ return False
+ with open(dir_model / "tokenizer.json", encoding="utf-8") as f:
+ added = {t["id"]: t["content"] for t in json.load(f).get("added_tokens", [])}
+ return [added.get(i) for i in range(4)] == ["[UNK]", "[TEXT]", "[START]", "[STOP]"]
+
+
+@ModelBase.register_hparams_loader(_is_soprano_checkpoint)
+def _load_soprano_hparams(dir_model: Path) -> dict[str, Any]:
+ logger.info("gguf: detected Soprano checkpoint")
+ hparams = ModelBase.load_hparams(dir_model, False, guess=False)
+ # the text model stays Qwen3, the mmproj holds the decoder
+ hparams["text_config"] = {"architectures": ["SopranoTextModel"]}
+ hparams["architectures"] = ["SopranoModel"]
+ return hparams
+
+
+@ModelBase.register("SopranoTextModel")
+class SopranoTextModel(Qwen3Model):
+ model_arch = gguf.MODEL_ARCH.QWEN3
+
+ def get_vocab_base_pre(self, tokenizer) -> str:
+ return "soprano"
+
+ def set_vocab(self):
+ super().set_vocab()
+ self.gguf_writer.add_unk_token_id(0)
+
+
+@ModelBase.register("SopranoModel")
+@ModelBase.example("ekwek/Soprano-80M", "ekwek/Soprano-1.1-80M")
+class SopranoModel(MmprojModel):
+ has_vision_encoder = False
+ has_audio_encoder = False
+
+ def _load_decoder(self) -> dict[str, torch.Tensor]:
+ state = torch.load(self.dir_model / "decoder.pth", map_location="cpu", mmap=True, weights_only=True)
+ return state.get("state_dict", state)
+
+ def get_audio_config(self):
+ state = self._load_decoder()
+ n_embd, n_inp, _ = state["decoder.embed.weight"].shape
+ n_ff = state["decoder.convnext.0.pwconv1.weight"].shape[0]
+ n_layer = sum(1 for name in state if name.endswith(".pwconv1.weight"))
+ if n_inp != self.n_embd_text:
+ raise ValueError(f"Soprano decoder input dim {n_inp} does not match the backbone dim {self.n_embd_text}")
+ return {"num_hidden_layers": n_layer, "hidden_size": n_embd, "intermediate_size": n_ff}
+
+ def set_gguf_parameters(self):
+ self.gguf_writer.add_file_type(self.ftype)
+ self.gguf_writer.add_clip_has_gen_audio_encoder(True)
+ self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.SOPRANO)
+ self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
+ self.gguf_writer.add_gen_audio_embedding_length(self.hparams["hidden_size"])
+ self.gguf_writer.add_gen_audio_feed_forward_length(self.hparams["intermediate_size"])
+ self.gguf_writer.add_gen_audio_block_count(self.hparams["num_hidden_layers"])
+ self.gguf_writer.add_gen_audio_head_count(1)
+ self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-6)
+
+ def get_tensors(self):
+ yield from self._load_decoder().items()
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ if "dwconv.weight" in new_name or (name == "decoder.embed.weight" and n_dims == 3):
+ return gguf.GGMLQuantizationType.F16
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def modify_tensors(self, data_torch, name, bid):
+ if name == "head.istft.window":
+ # the ISTFT in mtmd builds its own window
+ if not torch.equal(data_torch.float(), torch.hann_window(data_torch.shape[0])):
+ raise ValueError("Soprano requires a periodic Hann window")
+ return
+ if name == "decoder.embed.weight" and data_torch.shape[-1] == 1:
+ data_torch = data_torch.squeeze(-1)
+
+ T = gguf.MODEL_TENSOR
+ if name.startswith("decoder.convnext."):
+ _, _, blk, key, *suffix = name.split(".")
+ block_map = {
+ "dwconv": T.A_GEN_WAV_UP_DWCONV,
+ "norm": T.A_GEN_WAV_UP_NORM,
+ "pwconv1": T.A_GEN_WAV_UP_PW1,
+ "pwconv2": T.A_GEN_WAV_UP_PW2,
+ "gamma": T.A_GEN_WAV_UP_GAMMA,
+ }
+ yield self.format_tensor_name(block_map[key], int(blk), suffix="".join("." + s for s in suffix)), data_torch
+ return
+
+ tensor_map = {
+ "decoder.embed": T.A_GEN_WAV_INPUT,
+ "decoder.norm": T.A_GEN_WAV_NORM,
+ "decoder.final_layer_norm": T.A_GEN_WAV_OUTPUT_NORM,
+ "head.out": T.A_GEN_WAV_OUTPUT,
+ }
+ key, suffix = name.rsplit(".", 1)
+ yield self.format_tensor_name(tensor_map[key], suffix="." + suffix), data_torch
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index b8cfa3a85..f941106f2 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -1272,6 +1272,11 @@ class MODEL_TENSOR(IntEnum):
A_GEN_WAV_SEANET_RES_CONV1 = auto()
A_GEN_WAV_SEANET_RES_CONV2 = auto()
A_GEN_WAV_SEANET_SCALE_CONV = auto() # strided upsample convtr
+ # soprano: Vocos decoder, the ConvNeXt blocks reuse A_GEN_WAV_UP_*
+ A_GEN_WAV_INPUT = auto()
+ A_GEN_WAV_NORM = auto()
+ A_GEN_WAV_OUTPUT_NORM = auto()
+ A_GEN_WAV_OUTPUT = auto()
A_MMPROJ = auto()
A_MMPROJ_FC = auto()
A_MM_NORM_PRE = auto()
@@ -2084,6 +2089,10 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV1: "a.gen.wav.seanet.blk.{bid}.res_conv1",
MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV2: "a.gen.wav.seanet.blk.{bid}.res_conv2",
MODEL_TENSOR.A_GEN_WAV_SEANET_SCALE_CONV: "a.gen.wav.seanet.blk.{bid}.scale_conv",
+ MODEL_TENSOR.A_GEN_WAV_INPUT: "a.gen.wav.input",
+ MODEL_TENSOR.A_GEN_WAV_NORM: "a.gen.wav.norm",
+ MODEL_TENSOR.A_GEN_WAV_OUTPUT_NORM: "a.gen.wav.output_norm",
+ MODEL_TENSOR.A_GEN_WAV_OUTPUT: "a.gen.wav.output",
MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}",
MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc",
MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre",
@@ -2462,6 +2471,10 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV1,
MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV2,
MODEL_TENSOR.A_GEN_WAV_SEANET_SCALE_CONV,
+ MODEL_TENSOR.A_GEN_WAV_INPUT,
+ MODEL_TENSOR.A_GEN_WAV_NORM,
+ MODEL_TENSOR.A_GEN_WAV_OUTPUT_NORM,
+ MODEL_TENSOR.A_GEN_WAV_OUTPUT,
MODEL_TENSOR.A_ENC_CONV_NORM_MEAN,
MODEL_TENSOR.A_ENC_CONV_NORM_VAR,
MODEL_TENSOR.A_ENC_MEL_FILTERS,
@@ -6447,6 +6460,7 @@ class VisionProjectorType:
QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor
POCKETTTS_SPKENC = "pockettts_spkenc" # audio: mimi encoder as voice-prompt encoder
POCKETTTS_GEN = "pockettts_gen" # audio generation: flow-matching decoder + mimi decoder
+ SOPRANO = "soprano" # audio generation: Vocos decoder
HUNYUANVL = "hunyuanvl"
PARAKEET = "parakeet" # audio
MINIMAXM3 = "minimax_m3"
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index f72fbd221..05334e1fc 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -572,6 +572,10 @@ struct llm_tokenizer_bpe : llm_tokenizer {
"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
};
break;
+ case LLAMA_VOCAB_PRE_TYPE_SOPRANO:
+ regex_exprs = { "\\p{N}", "\\s+|[\\p{L}\\p{N}_]+|[^\\p{L}\\p{N}_\\s]+" };
+ byte_encode = false;
+ break;
case LLAMA_VOCAB_PRE_TYPE_WHITESPACE:
// whitespace pre-tokenizer (jinaai/jina-embeddings-v2-base-zh)
regex_exprs = {
@@ -647,6 +651,22 @@ struct llm_tokenizer_bpe_session {
replace_all(normalized_text, "[" + zwnj, "[");
replace_all(normalized_text, "<" + zwnj, "<");
word_collection = unicode_regex_split(normalized_text, tokenizer.regex_exprs, tokenizer.byte_encode);
+ } else if (tok_pre == LLAMA_VOCAB_PRE_TYPE_SOPRANO) {
+ std::string normalized_text;
+ bool space = false;
+ for (uint32_t cpt : unicode_cpts_from_utf8(text)) {
+ if (unicode_cpt_flags_from_cpt(cpt).is_whitespace) {
+ if (!space) {
+ normalized_text += ' ';
+ }
+ space = true;
+ } else {
+ // Unicode full lowercase expands LATIN CAPITAL LETTER I WITH DOT ABOVE.
+ normalized_text += cpt == 0x0130 ? "i\xcc\x87" : unicode_cpt_to_utf8(unicode_tolower(cpt));
+ space = false;
+ }
+ }
+ word_collection = unicode_regex_split(normalized_text, tokenizer.regex_exprs, tokenizer.byte_encode);
} else {
word_collection = unicode_regex_split(text, tokenizer.regex_exprs, tokenizer.byte_encode);
}
@@ -769,7 +789,9 @@ struct llm_tokenizer_bpe_session {
const std::string str = std::string(symbol.text, symbol.n);
const auto token = vocab.text_to_token(str);
- if (token == LLAMA_TOKEN_NULL) {
+ if (token == LLAMA_TOKEN_NULL && tok_pre == LLAMA_VOCAB_PRE_TYPE_SOPRANO) {
+ output.push_back(vocab.token_unk());
+ } else if (token == LLAMA_TOKEN_NULL) {
for (auto j = str.begin(); j != str.end(); ++j) {
llama_token token_multibyte = LLAMA_TOKEN_NULL;
if (tokenizer.byte_encode) {
@@ -2730,6 +2752,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
pre_type = LLAMA_VOCAB_PRE_TYPE_NEMOTRON_3_5_EMBD;
ignore_merges = true;
add_bos = true;
+ } else if (
+ tokenizer_pre == "soprano") {
+ pre_type = LLAMA_VOCAB_PRE_TYPE_SOPRANO;
} else {
throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
}
@@ -3283,6 +3308,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
}
}
+ if (pre_type == LLAMA_VOCAB_PRE_TYPE_SOPRANO) {
+ special_eog_ids.insert(special_eos_id);
+ }
+
// sanity checks
if (special_eos_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eos_id) == 0) {
special_eog_ids.insert(special_eos_id);
@@ -4096,6 +4125,9 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t
return _try_copy(token_text.data(), token_text.size());
}
if (attr & LLAMA_TOKEN_ATTR_NORMAL) {
+ if (pre_type == LLAMA_VOCAB_PRE_TYPE_SOPRANO) {
+ return _try_copy(token_text.data(), token_text.size());
+ }
if (escape_whitespaces) {
// SPM-style BPE: tokens contain ▁ for spaces
std::string result = token_text;
diff --git a/src/llama-vocab.h b/src/llama-vocab.h
index c6f3ac06d..14a90fbdb 100644
--- a/src/llama-vocab.h
+++ b/src/llama-vocab.h
@@ -73,6 +73,7 @@ enum llama_vocab_pre_type {
LLAMA_VOCAB_PRE_TYPE_FLINT = 62,
LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL = 63,
LLAMA_VOCAB_PRE_TYPE_NEMOTRON_3_5_EMBD = 64,
+ LLAMA_VOCAB_PRE_TYPE_SOPRANO = 65,
};
// HF Metaspace pre-tokenizer prepend_scheme
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index bd7fb7356..8ca4ae3a4 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -67,6 +67,7 @@ add_library(mtmd
models/pockettts-seanet.cpp
models/pockettts-spkenc.cpp
models/pockettts-gen.cpp
+ models/soprano.cpp
models/step3vl.cpp
models/siglip.cpp
models/whisper-enc.cpp
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 5583a7c57..4ffb7056a 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -300,6 +300,12 @@
#define TN_A_GEN_WAV_SEANET_RES_CONV2 "a.gen.wav.seanet.blk.%d.res_conv2.%s"
#define TN_A_GEN_WAV_SEANET_SCALE_CONV "a.gen.wav.seanet.blk.%d.scale_conv.%s"
+// soprano: Vocos decoder, the ConvNeXt blocks use TN_A_GEN_WAV_UP_*
+#define TN_A_GEN_WAV_INPUT "a.gen.wav.input.%s"
+#define TN_A_GEN_WAV_NORM "a.gen.wav.norm.%s"
+#define TN_A_GEN_WAV_OUTPUT_NORM "a.gen.wav.output_norm.%s"
+#define TN_A_GEN_WAV_OUTPUT "a.gen.wav.output.%s"
+
// cogvlm
#define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s"
#define TN_MM_H_TO_4H "mm.up.%s"
@@ -547,6 +553,7 @@ enum projector_type {
PROJECTOR_TYPE_QWEN3TTS_GEN,
PROJECTOR_TYPE_POCKETTTS_SPKENC,
PROJECTOR_TYPE_POCKETTTS_GEN,
+ PROJECTOR_TYPE_SOPRANO,
PROJECTOR_TYPE_MUSE_GLIMMER,
PROJECTOR_TYPE_SENSENOVA_U1,
PROJECTOR_TYPE_COHERE2V,
@@ -623,6 +630,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_QWEN3TTS_GEN, "qwen3tts_gen"},
{ PROJECTOR_TYPE_POCKETTTS_SPKENC, "pockettts_spkenc"},
{ PROJECTOR_TYPE_POCKETTTS_GEN, "pockettts_gen"},
+ { PROJECTOR_TYPE_SOPRANO, "soprano"},
{ PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"},
{ PROJECTOR_TYPE_SENSENOVA_U1, "sensenova_u1"},
{ PROJECTOR_TYPE_COHERE2V, "cohere2v"},
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h
index 5caa93b42..350c8fc9d 100644
--- a/tools/mtmd/clip-model.h
+++ b/tools/mtmd/clip-model.h
@@ -579,6 +579,18 @@ struct clip_code2wav {
ggml_tensor * dac_post_conv_b = nullptr;
};
+struct clip_vocos {
+ ggml_tensor * input_w = nullptr;
+ ggml_tensor * input_b = nullptr;
+ ggml_tensor * norm_w = nullptr;
+ ggml_tensor * norm_b = nullptr;
+ ggml_tensor * output_norm_w = nullptr;
+ ggml_tensor * output_norm_b = nullptr;
+ ggml_tensor * output_w = nullptr;
+ ggml_tensor * output_b = nullptr;
+ std::vector<clip_code2wav::upsample_block> blocks;
+};
+
struct clip_model {
clip_modality modality = CLIP_MODALITY_VISION;
projector_type proj_type = PROJECTOR_TYPE_MLP;
@@ -845,6 +857,9 @@ struct clip_model {
ggml_tensor * gen_upsample_w = nullptr; // depthwise convtr, frame rate -> encoder frame rate
std::vector<clip_layer> gen_tfm_layers; // mimi decoder_transformer
+ // soprano: Vocos decoder, hidden states -> STFT coefficients
+ clip_vocos vocos;
+
// cogvlm
ggml_tensor * mm_post_fc_norm_w = nullptr;
ggml_tensor * mm_post_fc_norm_b = nullptr;
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 5737400df..950d8fde0 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -1168,6 +1168,11 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
const float top_p = params ? params->top_p : 1.0f;
builder = std::make_unique<clip_graph_qwen3tts_gen>(ctx, img, gen_process, top_k, top_p);
} break;
+ case PROJECTOR_TYPE_SOPRANO:
+ {
+ const int n_frames = params && params->feats ? (int) (params->feats->size() / ctx->model.hparams.projection_dim) : 2;
+ builder = std::make_unique<clip_graph_soprano>(ctx, img, n_frames);
+ } break;
case PROJECTOR_TYPE_YOUTUVL:
{
builder = std::make_unique<clip_graph_youtuvl>(ctx, img);
@@ -2073,6 +2078,12 @@ struct clip_model_loader {
hparams.flow_n_step = 1;
hparams.gen_eos_threshold = -4.0f;
} break;
+ case PROJECTOR_TYPE_SOPRANO:
+ {
+ // SopranoDecoder defaults, n_fft comes from the head tensor
+ hparams.audio_sample_rate = 32000;
+ hparams.audio_hop_len = 512;
+ } break;
case PROJECTOR_TYPE_PADDLEOCR:
{
hparams.n_merge = 2;
@@ -2452,6 +2463,7 @@ struct clip_model_loader {
model.proj_type != PROJECTOR_TYPE_GEMMA3NV &&
model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC &&
model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN &&
+ model.proj_type != PROJECTOR_TYPE_SOPRANO &&
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO);
// layers
@@ -3391,6 +3403,31 @@ struct clip_model_loader {
c2w.dac_post_conv_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV, "bias"));
}
} break;
+ case PROJECTOR_TYPE_SOPRANO:
+ {
+ auto & v = model.vocos;
+ v.input_w = get_tensor(string_format(TN_A_GEN_WAV_INPUT, "weight"));
+ v.input_b = get_tensor(string_format(TN_A_GEN_WAV_INPUT, "bias"));
+ v.norm_w = get_tensor(string_format(TN_A_GEN_WAV_NORM, "weight"));
+ v.norm_b = get_tensor(string_format(TN_A_GEN_WAV_NORM, "bias"));
+ v.output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT_NORM, "weight"));
+ v.output_norm_b = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT_NORM, "bias"));
+ v.output_w = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT, "weight"));
+ v.output_b = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT, "bias"));
+ v.blocks.resize(hparams.n_layer);
+ for (int il = 0; il < hparams.n_layer; ++il) {
+ auto & b = v.blocks[il];
+ b.dwconv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight"));
+ b.dwconv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias"));
+ b.norm_w = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "weight"));
+ b.norm_b = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "bias"));
+ b.pw1_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "weight"));
+ b.pw1_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "bias"));
+ b.pw2_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "weight"));
+ b.pw2_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "bias"));
+ b.gamma = get_tensor(string_format(TN_A_GEN_WAV_UP_GAMMA, il));
+ }
+ } break;
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_VOXTRAL_RT:
{
@@ -5679,6 +5716,10 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
set_input_f32("inp_noise", noise);
}
} break;
+ case PROJECTOR_TYPE_SOPRANO:
+ {
+ set_input_f32("inp_feats", *params->feats);
+ } break;
case PROJECTOR_TYPE_GEMMA4V:
case PROJECTOR_TYPE_GEMMA4UV:
{
@@ -6329,6 +6370,11 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
out_audio.resize(ggml_nelements(audio));
ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio));
+ if (model.proj_type == PROJECTOR_TYPE_SOPRANO) {
+ std::vector<float> spectrum = std::move(out_audio);
+ clip_graph_soprano::decode_spectrum(model, spectrum, out_audio);
+ }
+
// drop the tail audio that comes from the code-0 rear padding
const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0;
const int64_t n_frames_w = hparams.wav_tfm_swa;
@@ -6536,6 +6582,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.spk_proj_w->ne[1];
case PROJECTOR_TYPE_POCKETTTS_GEN:
return ctx->model.gen_input_lin_w->ne[1];
+ case PROJECTOR_TYPE_SOPRANO:
+ return ctx->model.hparams.projection_dim;
case PROJECTOR_TYPE_PARAKEET:
return ctx->model.mm_1_w->ne[1];
default:
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 55e525cdc..8ae2a5b25 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -420,6 +420,15 @@ std::vector<c2w_state_slot> list_c2w_state_slots(const clip_hparams & hparams, c
// same, for the streaming mimi decoder (pocket-tts GEN_WAV)
std::vector<c2w_state_slot> list_pockettts_state_slots(const clip_hparams & hparams, const clip_model & model);
+struct clip_graph_soprano : clip_graph {
+ clip_graph_soprano(clip_ctx * ctx, const clip_image_f32 & img, int n_frames) : clip_graph(ctx, img), n_frames(n_frames) {}
+ ggml_cgraph * build() override;
+ static void decode_spectrum(const clip_model & model, const std::vector<float> & spectrum, std::vector<float> & pcm);
+ // SopranoDecoder default
+ static constexpr int upscale = 4;
+ int n_frames;
+};
+
struct clip_graph_kimik25 : clip_graph {
clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/models/soprano.cpp b/tools/mtmd/models/soprano.cpp
new file mode 100644
index 000000000..bd16c6912
--- /dev/null
+++ b/tools/mtmd/models/soprano.cpp
@@ -0,0 +1,78 @@
+#include "models.h"
+#include "../mtmd-audio.h"
+
+// Soprano Vocos: normalized Qwen3 hidden states -> STFT coefficients.
+ggml_cgraph * clip_graph_soprano::build() {
+ const auto & v = model.vocos;
+ const int n_inp = hparams.projection_dim;
+ const int n_up = upscale * (n_frames - 1) + 1;
+ ggml_tensor * cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_inp, n_frames);
+ ggml_set_name(cur, "inp_feats");
+ ggml_set_input(cur);
+
+ cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+ cur = ggml_interpolate(ctx0, cur, n_up, n_inp, 1, 1, GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS);
+ if (ggml_n_dims(v.input_w) == 3) {
+ const int k = v.input_w->ne[0];
+ cur = ggml_conv_1d(ctx0, v.input_w, cur, 1, k / 2, 1);
+ cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+ } else {
+ cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+ cur = build_mm(v.input_w, cur);
+ }
+ cur = ggml_add(ctx0, cur, v.input_b);
+ cur = build_norm(cur, v.norm_w, v.norm_b, NORM_TYPE_NORMAL, eps, -1);
+
+ for (size_t il = 0; il < v.blocks.size(); ++il) {
+ const auto & b = v.blocks[il];
+ const int k = b.dwconv_w->ne[0];
+ ggml_tensor * residual = cur;
+ cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+ cur = ggml_conv_1d_dw(ctx0, b.dwconv_w, cur, 1, k / 2, 1);
+ cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+ cur = ggml_add(ctx0, cur, b.dwconv_b);
+ cur = build_norm(cur, b.norm_w, b.norm_b, NORM_TYPE_NORMAL, eps, il);
+ cur = ggml_add(ctx0, build_mm(b.pw1_w, cur), b.pw1_b);
+ cur = ggml_gelu_erf(ctx0, cur);
+ cur = ggml_add(ctx0, build_mm(b.pw2_w, cur), b.pw2_b);
+ cur = ggml_add(ctx0, residual, ggml_mul(ctx0, cur, b.gamma));
+ cb(cur, "vocos_block", il);
+ }
+ cur = build_norm(cur, v.output_norm_w, v.output_norm_b, NORM_TYPE_NORMAL, eps, -1);
+ cur = ggml_add(ctx0, build_mm(v.output_w, cur), v.output_b);
+ cb(cur, "vocos_head", -1);
+
+ // the head holds n_fft/2+1 log-magnitudes followed by n_fft/2+1 phases
+ const int n_bins = v.output_w->ne[1] / 2;
+ ggml_tensor * mag = ggml_view_2d(ctx0, cur, n_bins, n_up, cur->nb[1], 0);
+ ggml_tensor * phase = ggml_view_2d(ctx0, cur, n_bins, n_up, cur->nb[1], n_bins * sizeof(float));
+ mag = ggml_clamp(ctx0, ggml_exp(ctx0, ggml_cont(ctx0, mag)), 0.0f, 100.0f);
+ phase = ggml_cont(ctx0, phase);
+ ggml_tensor * re = ggml_mul(ctx0, mag, ggml_cos(ctx0, phase));
+ ggml_tensor * im = ggml_mul(ctx0, mag, ggml_sin(ctx0, phase));
+ re = ggml_reshape_3d(ctx0, re, 1, n_bins, n_up);
+ im = ggml_reshape_3d(ctx0, im, 1, n_bins, n_up);
+ cur = ggml_concat(ctx0, re, im, 0);
+ ggml_set_name(cur, "out_audio");
+ ggml_set_output(cur);
+ ggml_build_forward_expand(gf, cur);
+ return gf;
+}
+
+void clip_graph_soprano::decode_spectrum(const clip_model & model, const std::vector<float> & spectrum, std::vector<float> & pcm) {
+ const int n_frame_vals = model.vocos.output_w->ne[1];
+ const int n_fft = n_frame_vals - 2;
+ const int hop_length = model.hparams.audio_hop_len;
+ const size_t n_frames = spectrum.size() / n_frame_vals;
+ mtmd_audio_streaming_istft istft(n_fft, hop_length);
+ pcm.clear();
+ for (size_t i = 0; i < n_frames; ++i) {
+ auto frame = istft.process_frame(spectrum.data() + i * n_frame_vals);
+ pcm.insert(pcm.end(), frame.begin(), frame.end());
+ }
+ auto tail = istft.flush();
+ pcm.insert(pcm.end(), tail.begin(), tail.end());
+ // the shared ISTFT removes (n_fft - hop) / 2 samples, torch.istft(center=True) removes n_fft / 2
+ pcm.erase(pcm.begin(), pcm.begin() + hop_length / 2);
+ pcm.resize((n_frames - 1) * hop_length);
+}
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index d8299ba44..44bee60bd 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -1076,12 +1076,199 @@ private:
bool wav_header_sent = false;
};
+// Soprano uses the hidden state that predicts each non-EOS token, including the prompt's last state.
+class soprano_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
+public:
+ using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
+
+ void reset() override {
+ prompt.clear();
+ features.clear();
+ h_next.clear();
+ audio.clear();
+ wav.clear();
+ pos = 0;
+ prompt_pos = 0;
+ decoded = false;
+ sent = false;
+ }
+
+ int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
+ reset();
+ if (inp->speaker_ref || !inp->prompt || inp->prompt_len == 0 || inp->prompt_len > INT32_MAX ||
+ inp->seq_id < 0 || (uint32_t) inp->seq_id >= llama_n_seq_max(lctx)) {
+ LOG_ERR("mtmd_helper_gen_audio: soprano requires text, without a speaker reference\n");
+ return 1;
+ }
+ seq_id = inp->seq_id;
+ out_type = inp->out_type;
+ stream = inp->stream;
+ const llama_token text = find_special_token(vocab, "[TEXT]");
+ const llama_token start = find_special_token(vocab, "[START]");
+ const llama_token stop = find_special_token(vocab, "[STOP]");
+ if (text == LLAMA_TOKEN_NULL || start == LLAMA_TOKEN_NULL || stop == LLAMA_TOKEN_NULL) {
+ LOG_ERR("mtmd_helper_gen_audio: incompatible soprano vocabulary\n");
+ return 1;
+ }
+ const int required = llama_tokenize(vocab, inp->prompt, inp->prompt_len, nullptr, 0, false, false);
+ if (required >= 0 || (uint32_t) (-required + 3) >= llama_n_ctx_seq(lctx)) {
+ LOG_ERR("mtmd_helper_gen_audio: soprano prompt does not fit in the context\n");
+ return 1;
+ }
+ const int n = -required;
+ prompt.resize(n + 3);
+ prompt[0] = stop;
+ prompt[1] = text;
+ if (llama_tokenize(vocab, inp->prompt, inp->prompt_len, prompt.data() + 2, n, false, false) != n) {
+ return 1;
+ }
+ prompt.back() = start;
+ if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) {
+ LOG_ERR("mtmd_helper_gen_audio: cannot reset soprano sequence\n");
+ return 1;
+ }
+ if (!init_gen_state(inp->seed)) {
+ return 1;
+ }
+ return 0;
+ }
+
+ int32_t step_prompt(int32_t n_batch) override {
+ if (n_batch <= 0 || prompt.empty()) {
+ return -1;
+ }
+ const int n = std::min(n_batch, (int) prompt.size() - prompt_pos);
+ if (n == 0) {
+ return 0;
+ }
+ if (!decode(prompt.data() + prompt_pos, n)) {
+ return -1;
+ }
+ prompt_pos += n;
+ return (int) prompt.size() - prompt_pos;
+ }
+
+ int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
+ *h_state_out = nullptr;
+ *out_stop = false;
+ if (sampled < 0 || sampled >= llama_vocab_n_tokens(vocab) || !h_state_in || prompt_pos != (int) prompt.size() || prompt.empty()) {
+ return 1;
+ }
+ if (llama_vocab_is_eog(vocab, sampled) || (uint32_t) pos >= llama_n_ctx_seq(lctx)) {
+ *out_stop = true;
+ return 0;
+ }
+ features.insert(features.end(), h_state_in, h_state_in + n_embd);
+ if (!decode(&sampled, 1)) {
+ features.resize(features.size() - n_embd);
+ return 1;
+ }
+ const float * h = llama_get_embeddings_ith(lctx, -1);
+ if (!h) {
+ return 1;
+ }
+ h_next.assign(h, h + n_embd);
+ *h_state_out = h_next.data();
+ return 0;
+ }
+
+ int32_t get_output(int32_t * sample_rate, const char ** data, size_t * data_len, int64_t * n_samples) override {
+ *sample_rate = info.sample_rate;
+ *data_len = 0;
+ if (n_samples) { *n_samples = 0; }
+ if (stream) {
+ // the decoder needs the whole utterance, so the audio comes out in one chunk after flush()
+ if (!decoded || sent) {
+ return 0;
+ }
+ sent = true;
+ } else if (!decode_audio()) {
+ return 1;
+ }
+ if (n_samples) { *n_samples = audio.size(); }
+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+ *data = (const char *) audio.data();
+ *data_len = audio.size() * sizeof(float);
+ } else {
+ wav.clear();
+ if (!write_wav16(wav, audio, info.sample_rate)) {
+ return 1;
+ }
+ *data = wav.data();
+ *data_len = wav.size();
+ }
+ return 0;
+ }
+
+ // the whole utterance is decoded at once
+ int32_t flush() override {
+ return decode_audio() ? 0 : 1;
+ }
+
+private:
+ bool decode_audio() {
+ if (decoded) {
+ return true;
+ }
+ if (features.size() < 2 * (size_t) n_embd) {
+ LOG_ERR("mtmd_helper_gen_audio: soprano needs at least two generated frames\n");
+ return false;
+ }
+ mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
+ inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
+ inp.feats = features.data();
+ inp.n_feats = features.size();
+ mtmd_gen_out out{};
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
+ return false;
+ }
+ audio.assign(out.audio, out.audio + out.n_samples);
+ decoded = true;
+ return true;
+ }
+
+ bool decode(llama_token * tokens, int n) {
+ llama_batch batch = llama_batch_init(n, 0, 1);
+ batch.n_tokens = n;
+ for (int i = 0; i < n; ++i) {
+ batch.token[i] = tokens[i];
+ batch.pos[i] = pos + i;
+ batch.n_seq_id[i] = 1;
+ batch.seq_id[i][0] = seq_id;
+ batch.logits[i] = i == n - 1;
+ }
+ const int ret = llama_decode(lctx, batch);
+ llama_batch_free(batch);
+ if (ret != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: soprano backbone decode failed\n");
+ return false;
+ }
+ pos += n;
+ return true;
+ }
+
+ llama_seq_id seq_id = 0;
+ llama_pos pos = 0;
+ int prompt_pos = 0;
+ std::vector<llama_token> prompt;
+ std::vector<float> features;
+ std::vector<float> h_next;
+ std::vector<float> audio;
+ std::vector<char> wav;
+ mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
+ bool stream = false;
+ bool decoded = false;
+ bool sent = false;
+};
+
static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
switch (mtmd_gen_audio_get_info(mctx).type) {
case MTMD_GEN_AUDIO_TYPE_QWEN3TTS:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new qwen3tts_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_POCKETTTS:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new pockettts_gen_audio_pipeline(lctx, mctx));
+ case MTMD_GEN_AUDIO_TYPE_SOPRANO:
+ return std::unique_ptr<mtmd_gen_audio_pipeline>(new soprano_gen_audio_pipeline(lctx, mctx));
default:
return nullptr;
}
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 7c825c0dc..e18f639de 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -624,7 +624,9 @@ struct mtmd_context {
ctx_v = res.ctx_v;
ctx_a = res.ctx_a;
ctx_gen_a = res.ctx_gen_a;
- if (!ctx_v && !ctx_a) {
+ // soprano has no speaker encoder, its mmproj only holds the decoder
+ const bool gen_only = ctx_gen_a && clip_get_projector_type(ctx_gen_a) == PROJECTOR_TYPE_SOPRANO;
+ if (!ctx_v && !ctx_a && !gen_only) {
throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname));
}
@@ -641,7 +643,8 @@ struct mtmd_context {
// since we already validate n_embd of vision and audio mmproj,
// we can safely assume that they are the same
- int n_embd_clip = clip_n_mmproj_embd(ctx_v ? ctx_v : ctx_a);
+ // a gen-only mmproj is validated below
+ int n_embd_clip = ctx_v || ctx_a ? clip_n_mmproj_embd(ctx_v ? ctx_v : ctx_a) : n_embd_text;
if (n_embd_text > 0 && n_embd_text != n_embd_clip) {
throw std::runtime_error(string_format(
"mismatch between text model (n_embd = %d) and mmproj (n_embd = %d)\n"
@@ -2082,6 +2085,10 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
info.type = MTMD_GEN_AUDIO_TYPE_POCKETTTS;
info.sample_rate = 24000;
break;
+ case PROJECTOR_TYPE_SOPRANO:
+ info.type = MTMD_GEN_AUDIO_TYPE_SOPRANO;
+ info.sample_rate = 32000;
+ break;
default:
info.type = MTMD_GEN_AUDIO_TYPE_NONE;
break;
@@ -2109,6 +2116,9 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
inp.top_p = 1.0f;
inp.temp = 0.7f;
break;
+ case PROJECTOR_TYPE_SOPRANO:
+ inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
+ break;
default:
break;
}
@@ -2170,6 +2180,14 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
*out = {};
+ if (clip_get_projector_type(ctx_clip) == PROJECTOR_TYPE_SOPRANO) {
+ const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip);
+ if (inp->type != MTMD_GEN_PROCESS_TYPE_GEN_WAV || !inp->feats || inp->n_feats < 2 * n_embd || inp->n_feats % n_embd != 0 || inp->codes) {
+ LOG_ERR("%s: soprano requires at least 2 frames of features, without codes\n", __func__);
+ return 1;
+ }
+ }
+
if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE) {
const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip);
@@ -3046,6 +3064,9 @@ mtmd_memory_usage mtmd_get_memory_usage(const char * mmproj_fname,
if (ctx->ctx_a) {
merge(ctx->ctx_a);
}
+ if (ctx->ctx_gen_a) {
+ merge(ctx->ctx_gen_a);
+ }
mtmd_memory_usage res;
res.backend_mem_usage = std::move(total_mem);
res.image_max_tokens = ctx->ctx_v ? clip_get_image_max_tokens(ctx->ctx_v) : -1;
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index 5af73a0f4..0a6bf4c4c 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -427,6 +427,7 @@ enum mtmd_gen_audio_type {
MTMD_GEN_AUDIO_TYPE_NONE, // not supported
MTMD_GEN_AUDIO_TYPE_QWEN3TTS,
MTMD_GEN_AUDIO_TYPE_POCKETTTS,
+ MTMD_GEN_AUDIO_TYPE_SOPRANO,
};
struct mtmd_gen_audio_info {
diff --git a/tools/tts/README.md b/tools/tts/README.md
index 55b99932c..41397b82e 100644
--- a/tools/tts/README.md
+++ b/tools/tts/README.md
@@ -59,3 +59,28 @@ The [upstream repository](https://huggingface.co/kyutai/pocket-tts) holds one co
python convert_hf_to_gguf.py path/to/pocket-tts/languages/english --outfile pocket-tts.gguf
python convert_hf_to_gguf.py path/to/pocket-tts/languages/english --mmproj --outfile mmproj-pocket-tts.gguf
```
+
+## Soprano
+
+Available params:
+- `--tts-speaker-file` is not supported; the model uses a fixed voice
+- Note: `lang` is not used. Supply normalized English text
+
+Example usage:
+
+```sh
+llama-tts -m soprano.gguf \
+ -mm mmproj-soprano.gguf \
+ -p "Hello world" \
+ --temp 0 \
+ --output out.wav
+```
+
+**Note for GGUF conversion:**
+
+Both [Soprano-80M](https://huggingface.co/ekwek/Soprano-80M) and [Soprano-1.1-80M](https://huggingface.co/ekwek/Soprano-1.1-80M) are supported. Convert both files from the same model directory, including `decoder.pth`:
+
+```sh
+python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --outtype f32 --outfile soprano.gguf
+python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --mmproj --outtype f16 --outfile mmproj-soprano.gguf
+```