Commit 5851a8e50 for llama.cpp
commit 5851a8e505f4d5fbb183f2847698eb6155d176b7
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Wed Nov 4 15:15:09 2026 +0100
model, mtmd: drama-3 tts support (#31613)
* model, mtmd: drama-3 tts support
* clean up
* mtmd: drama3 helper stops at the codebook tower limit
---------
Co-authored-by: Pascal <admin@serveurperso.com>
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 1370dd3b1..0c6204b87 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -95,6 +95,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"FalconForCausalLM": "falcon",
"FalconH1ForCausalLM": "falcon_h1",
"FalconMambaForCausalLM": "mamba",
+ "FishOmniModel": "drama3",
"GPT2LMHeadModel": "gpt2",
"GPTBigCodeForCausalLM": "starcoder",
"GPTNeoXForCausalLM": "gptneox",
@@ -346,6 +347,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"DotsOCRForCausalLM": "dotsocr",
"EmbeddingGemma2Model": "gemma",
"Exaone4_5_ForConditionalGeneration": "exaone",
+ "FishOmniModel": "drama3",
"Gemma3ForConditionalGeneration": "gemma",
"Gemma3nForConditionalGeneration": "gemma",
"Gemma4ForConditionalGeneration": "gemma",
diff --git a/conversion/base.py b/conversion/base.py
index 28f2bf241..7028d633b 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -494,7 +494,7 @@ class ModelBase:
s = self.model_tensors[name]
self.model_tensors[weight_name] = lambda w=w, s=s: dequant_bitnet(w(), s())
tensors_to_remove.append(name)
- elif quant_method == "fp8":
+ elif quant_method in ("fp8", "bsgemm-moe"): # bsgemm-moe: drama-3, same block layout
block_size = quant_config.get("weight_block_size")
for name in self.model_tensors.keys():
if name.endswith("_scale_inv"):
diff --git a/conversion/drama3.py b/conversion/drama3.py
new file mode 100644
index 000000000..7c590a014
--- /dev/null
+++ b/conversion/drama3.py
@@ -0,0 +1,251 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import ModelBase, MmprojModel, gguf
+from .qwen import Qwen3_5MoeTextModel
+
+# Fish Audio Drama 3 (FishOmniModel)
+# text model: Qwen3.5-MoE backbone, samples the main token of each frame
+# mmproj: codebook tower and Fish DAC decoder (codec.pth)
+
+_TOWER_SHARED = "audio_decoder_mtp_shared.layers."
+_TOWER_RESIDUAL = "audio_decoder_mtp.layers."
+
+
+@ModelBase.register("FishOmniModel")
+@ModelBase.example("fishaudio/drama-3")
+class Drama3TextModel(Qwen3_5MoeTextModel):
+ model_arch = gguf.MODEL_ARCH.QWEN35MOE
+ no_mtp = True # the checkpoint has no MTP head
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ if item[0].startswith("audio_decoder"):
+ return None
+ return super().filter_tensors(item)
+
+
+@ModelBase.register("FishOmniModel")
+@ModelBase.example("fishaudio/drama-3")
+class Drama3MmprojModel(MmprojModel):
+ has_vision_encoder = False
+ has_audio_encoder = False
+
+ _LAYER_MAP = {
+ "enorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ENORM, True),
+ "hnorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_HNORM, True),
+ "eh_proj.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_EH_PROJ, False),
+ "semantic_embedding.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_EMBD, False),
+ "codebook_embedding.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_EMBD, False),
+ "final_layernorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM, True),
+ "transformer_layer.attention_norm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM, True),
+ "transformer_layer.ffn_norm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM, True),
+ "transformer_layer.attention.q_norm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM, True),
+ "transformer_layer.attention.k_norm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM, True),
+ "transformer_layer.attention.wo.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT, False),
+ "transformer_layer.feed_forward.w1.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE, False),
+ "transformer_layer.feed_forward.w3.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP, False),
+ "transformer_layer.feed_forward.w2.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN, False),
+ }
+
+ _heads: dict[int, Tensor]
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._heads = {}
+
+ def get_audio_config(self) -> dict[str, Any] | None:
+ cfg = dict(self.global_config["audio_decoder_config"])
+ cfg["num_hidden_layers"] = cfg["shared_enrichment_layers"] + cfg["num_codebooks"] - 1
+ return cfg
+
+ def set_gguf_parameters(self):
+ cfg = self.hparams
+ self.gguf_writer.add_file_type(self.ftype)
+ self.gguf_writer.add_clip_has_gen_audio_encoder(True)
+ self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.DRAMA3_GEN)
+ self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
+ self.gguf_writer.add_gen_audio_embedding_length(cfg["dim"])
+ self.gguf_writer.add_gen_audio_feed_forward_length(cfg["intermediate_size"])
+ self.gguf_writer.add_gen_audio_block_count(cfg["num_hidden_layers"])
+ self.gguf_writer.add_gen_audio_head_count(cfg["n_head"])
+ self.gguf_writer.add_gen_audio_head_count_kv(cfg["n_local_heads"])
+ self.gguf_writer.add_gen_audio_attention_layernorm_eps(cfg["norm_eps"])
+ # other tower and codec hparams are hardcoded in clip.cpp
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name = item[0]
+ if name.startswith("audio_decoder") or name == "model.language_model.embed_tokens.weight":
+ return item
+ return None
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
+ if new_name.endswith(".weight") and (
+ new_name in ("a.gen.wav.dac.entry.weight", "a.gen.wav.dac.post_conv.weight")
+ or (".up.blk." in new_name and new_name.endswith(".dwconv.weight"))
+ or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight")))
+ ):
+ return gguf.GGMLQuantizationType.F16
+ # keep ConvTranspose1d kernels and folded codebook tables in F32
+ if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name):
+ return gguf.GGMLQuantizationType.F32
+ if new_name.startswith("a.gen.wav.quant."):
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ T = gguf.MODEL_TENSOR
+ cfg = self.hparams
+
+ if name.startswith("a.gen.wav."):
+ yield (name, data_torch)
+ return
+
+ if name == "model.language_model.embed_tokens.weight":
+ # semantic band of the backbone embeddings
+ begin = self.global_config["semantic_start_token_id"]
+ yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch[begin:begin + cfg["vocab_size"]])
+ return
+
+ if name == "audio_decoder.codebook_embeddings.weight":
+ n_cb = cfg["num_codebooks"]
+ yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), data_torch.reshape(n_cb, -1, data_torch.shape[-1]))
+ return
+
+ if name.startswith("audio_decoder_mtp.heads."):
+ n_heads = cfg["num_codebooks"] - 1
+ self._heads[int(name.split(".")[2])] = data_torch
+ if len(self._heads) < n_heads:
+ return
+ yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), torch.stack([self._heads.pop(i) for i in range(n_heads)]))
+ return
+
+ for prefix, offset in ((_TOWER_SHARED, 0), (_TOWER_RESIDUAL, cfg["shared_enrichment_layers"])):
+ if not name.startswith(prefix):
+ continue
+ idx, key = name[len(prefix):].split(".", 1)
+ il = int(idx) + offset
+ if key == "transformer_layer.attention.wqkv.weight":
+ # rows are [query | gate | key | value]
+ n_q = cfg["n_head"] * cfg["head_dim"]
+ n_kv = cfg["n_local_heads"] * cfg["head_dim"]
+ q, gate, k, v = data_torch.split([n_q, n_q, n_kv, n_kv], dim=0)
+ yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, il), q)
+ yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_GATE, il), gate)
+ yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, il), k)
+ yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, il), v)
+ return
+ tensor, zero_centered = self._LAYER_MAP[key]
+ if zero_centered:
+ data_torch = data_torch.float() + 1
+ yield (self.format_tensor_name(tensor, il), data_torch)
+ return
+
+ raise ValueError(f"unexpected tensor {name}")
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ # only the DAC decoder, the encoder is for voice cloning (not supported yet)
+ sd = torch.load(self.dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True)
+ sd = sd.get("state_dict", sd)
+ T = gguf.MODEL_TENSOR
+
+ def get(name: str) -> Tensor:
+ return sd[name].float()
+
+ def conv(prefix: str) -> Tensor:
+ # fold weight norm
+ if prefix + ".conv.weight" in sd:
+ return get(prefix + ".conv.weight")
+ g = get(prefix + ".conv.parametrizations.weight.original0")
+ v = get(prefix + ".conv.parametrizations.weight.original1")
+ return g * v / v.norm(dim=(1, 2), keepdim=True)
+
+ def quant_table(prefix: str) -> Tensor:
+ # fold out_proj into the codebook, so decoding is a row lookup
+ g = get(prefix + ".out_proj.weight_g")
+ v = get(prefix + ".out_proj.weight_v")
+ w = (g * v / v.norm(dim=(1, 2), keepdim=True)).squeeze(-1)
+ return get(prefix + ".codebook.weight") @ w.T + get(prefix + ".out_proj.bias")
+
+ def snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
+ alpha = alpha.flatten()
+ return alpha, 1.0 / (alpha + 1e-9)
+
+ q = "quantizer."
+ yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), quant_table(q + "semantic_quantizer.quantizers.0"))
+ n_rest = sum(1 for k in sd if k.startswith(q + "quantizer.quantizers.") and k.endswith(".codebook.weight"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_CB),
+ torch.stack([quant_table(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)]))
+
+ # post_module uses interleaved RoPE, permute q/k rows so NEOX RoPE gives the same result
+ tfm = q + "post_module."
+ n_head, d_head = 16, 64
+ order = torch.cat([torch.arange(0, d_head, 2), torch.arange(1, d_head, 2)])
+
+ def neox_rows(w: Tensor) -> Tensor:
+ return w.reshape(n_head, d_head, -1)[:, order].reshape(w.shape)
+
+ n_tfm = sum(1 for k in sd if k.startswith(tfm + "layers.") and k.endswith(".ffn_norm.weight"))
+ for il in range(n_tfm):
+ p = f"{tfm}layers.{il}."
+ wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0)
+ yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_Q, il), neox_rows(wq))
+ yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_K, il), neox_rows(wk))
+ yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_V, il), wv)
+ yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_OUT, il), get(p + "attention.wo.weight"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_NORM, il), get(p + "attention_norm.weight"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_SCALE, il), get(p + "attention_layer_scale.gamma"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_NORM, il), get(p + "ffn_norm.weight"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_GATE, il), get(p + "feed_forward.w1.weight"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_UP, il), get(p + "feed_forward.w3.weight"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_DOWN, il), get(p + "feed_forward.w2.weight"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_SCALE, il), get(p + "ffn_layer_scale.gamma"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUTPUT_NORM), get(tfm + "norm.weight"))
+
+ # upsample: 2x (ConvTranspose1d + ConvNeXt)
+ for il in range(2):
+ p = f"{q}upsample.{il}."
+ yield (self.format_tensor_name(T.A_GEN_WAV_UP_CONV, il, suffix=".weight"), conv(p + "0"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_UP_CONV, il, suffix=".bias"), get(p + "0.conv.bias"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_UP_DWCONV, il, suffix=".weight"), conv(p + "1.dwconv"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_UP_DWCONV, il, suffix=".bias"), get(p + "1.dwconv.conv.bias"))
+ for key, tensor in (("norm", T.A_GEN_WAV_UP_NORM), ("pwconv1", T.A_GEN_WAV_UP_PW1), ("pwconv2", T.A_GEN_WAV_UP_PW2)):
+ yield (self.format_tensor_name(tensor, il, suffix=".weight"), get(f"{p}1.{key}.weight"))
+ yield (self.format_tensor_name(tensor, il, suffix=".bias"), get(f"{p}1.{key}.bias"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_UP_GAMMA, il, suffix=""), get(p + "1.gamma"))
+
+ # DAC: entry conv, 4x (Snake, ConvTranspose1d, 3 residual units), Snake, post conv
+ d = "decoder.model."
+ yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".weight"), conv(d + "0"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get(d + "0.conv.bias"))
+ for bid in range(4):
+ p = f"{d}{bid + 1}.block."
+ a, b = snake(get(p + "0.alpha"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a)
+ yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b)
+ yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), conv(p + "1"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".bias"), get(p + "1.conv.bias"))
+ for xid in range(3):
+ r = f"{p}{xid + 2}.block."
+ for act, tensor in (("0", T.A_GEN_WAV_DAC_RES_ACT1), ("2", T.A_GEN_WAV_DAC_RES_ACT2)):
+ a, b = snake(get(r + act + ".alpha"))
+ name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
+ yield (name + ".alpha", a)
+ yield (name + ".beta", b)
+ for cv, tensor in (("1", T.A_GEN_WAV_DAC_RES_CONV1), ("3", T.A_GEN_WAV_DAC_RES_CONV2)):
+ name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
+ yield (name + ".weight", conv(r + cv))
+ yield (name + ".bias", get(r + cv + ".conv.bias"))
+ a, b = snake(get(d + "5.alpha"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a)
+ yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b)
+ yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), conv(d + "6"))
+ yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".bias"), get(d + "6.conv.bias"))
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index fcd17f4b5..930714f30 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -1295,6 +1295,11 @@ class MODEL_TENSOR(IntEnum):
A_GEN_CODE_ATTN_GATE = auto() # lfm3a: per-head sigmoid gate on the attention output
A_GEN_CODE_HEAD_NORM = auto() # lfm3a: per-codebook norm before the output head, merged 2D [n_codebooks, dim]
A_GEN_CODE_MASK_EMBD = auto() # lfm3a: MaskGIT mask embedding
+ A_GEN_CODE_ENORM = auto() # drama3: per-layer norm of the code embedding
+ A_GEN_CODE_HNORM = auto() # drama3: per-layer norm of the input hidden state
+ A_GEN_CODE_EH_PROJ = auto() # drama3: per-layer [code embd; hidden] -> hidden projection
+ A_GEN_CODE_BLK_EMBD = auto() # drama3: per-layer code embedding table
+ A_GEN_CODE_BLK_OUT_NORM = auto() # drama3: per-layer output norm
# qwen3tts code2wav: RVQ codes -> raw PCM
A_GEN_WAV_QUANT_FIRST_IN = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D)
A_GEN_WAV_QUANT_FIRST_OUT = auto() # semantic RVQ, out_proj
@@ -2198,6 +2203,11 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_GEN_CODE_ATTN_GATE: "a.gen.code.blk.{bid}.attn_gate",
MODEL_TENSOR.A_GEN_CODE_HEAD_NORM: "a.gen.code.head_norm",
MODEL_TENSOR.A_GEN_CODE_MASK_EMBD: "a.gen.code.mask_embd",
+ MODEL_TENSOR.A_GEN_CODE_ENORM: "a.gen.code.blk.{bid}.enorm",
+ MODEL_TENSOR.A_GEN_CODE_HNORM: "a.gen.code.blk.{bid}.hnorm",
+ MODEL_TENSOR.A_GEN_CODE_EH_PROJ: "a.gen.code.blk.{bid}.eh_proj",
+ MODEL_TENSOR.A_GEN_CODE_BLK_EMBD: "a.gen.code.blk.{bid}.embd",
+ MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM: "a.gen.code.blk.{bid}.out_norm",
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN: "a.gen.wav.quant.first.in_proj",
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj",
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB: "a.gen.wav.quant.first.codebook",
@@ -2615,6 +2625,11 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.A_GEN_CODE_ATTN_GATE,
MODEL_TENSOR.A_GEN_CODE_HEAD_NORM,
MODEL_TENSOR.A_GEN_CODE_MASK_EMBD,
+ MODEL_TENSOR.A_GEN_CODE_ENORM,
+ MODEL_TENSOR.A_GEN_CODE_HNORM,
+ MODEL_TENSOR.A_GEN_CODE_EH_PROJ,
+ MODEL_TENSOR.A_GEN_CODE_BLK_EMBD,
+ MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM,
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN,
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT,
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB,
@@ -6919,6 +6934,7 @@ class VisionProjectorType:
CHATTERBOX_SPKENC = "chatterbox_spkenc" # audio: CAMPPlus speaker encoder
CHATTERBOX = "chatterbox" # audio generation: s3gen flow matching + HiFT vocoder
SOPRANO = "soprano" # audio generation: Vocos decoder
+ DRAMA3_GEN = "drama3_gen" # audio generation: codebook tower + Fish DAC decoder
HUNYUANVL = "hunyuanvl"
PARAKEET = "parakeet" # audio
MINIMAXM3 = "minimax_m3"
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index 1b178111d..fc9f0bb34 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -70,6 +70,7 @@ add_library(mtmd
models/mimo-audio.cpp
models/qwen3tts-spkenc.cpp
models/qwen3tts-gen.cpp
+ models/drama3-gen.cpp
models/pockettts-seanet.cpp
models/pockettts-spkenc.cpp
models/pockettts-gen.cpp
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index dcd6ec216..cd7359e1d 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -261,6 +261,13 @@
#define TN_A_GEN_CODE_MASK_EMBD "a.gen.code.mask_embd"
#define TN_ATTN_GATE "%s.blk.%d.attn_gate.%s"
+// drama3 tower, the rest of the block uses the generic names with prefix "a.gen.code"
+#define TN_A_GEN_CODE_ENORM "a.gen.code.blk.%d.enorm.%s"
+#define TN_A_GEN_CODE_HNORM "a.gen.code.blk.%d.hnorm.%s"
+#define TN_A_GEN_CODE_EH_PROJ "a.gen.code.blk.%d.eh_proj.%s"
+#define TN_A_GEN_CODE_BLK_EMBD "a.gen.code.blk.%d.embd.%s"
+#define TN_A_GEN_CODE_BLK_OUT_NORM "a.gen.code.blk.%d.out_norm.%s"
+
// qwen3tts code2wav (RVQ codes -> raw PCM)
// pre_transformer layers use the generic TN_ATTN_*/TN_FFN_*/TN_LN_*/TN_LS_* macros, prefix "a.gen.wav.tfm"
#define TN_A_GEN_WAV_QUANT_FIRST_IN "a.gen.wav.quant.first.in_proj.%s"
@@ -595,6 +602,7 @@ enum projector_type {
PROJECTOR_TYPE_POCKETTTS_GEN,
PROJECTOR_TYPE_SOPRANO,
PROJECTOR_TYPE_LFM3A_GEN,
+ PROJECTOR_TYPE_DRAMA3_GEN,
PROJECTOR_TYPE_MUSE_GLIMMER,
PROJECTOR_TYPE_SENSENOVA_U1,
PROJECTOR_TYPE_COHERE2V,
@@ -679,6 +687,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_POCKETTTS_GEN, "pockettts_gen"},
{ PROJECTOR_TYPE_SOPRANO, "soprano"},
{ PROJECTOR_TYPE_LFM3A_GEN, "lfm3a_gen"},
+ { PROJECTOR_TYPE_DRAMA3_GEN, "drama3_gen"},
{ PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"},
{ PROJECTOR_TYPE_SENSENOVA_U1, "sensenova_u1"},
{ PROJECTOR_TYPE_COHERE2V, "cohere2v"},
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h
index b5a4a1225..13ac5f02b 100644
--- a/tools/mtmd/clip-model.h
+++ b/tools/mtmd/clip-model.h
@@ -841,6 +841,15 @@ struct clip_chatterbox {
ggml_tensor * spk_affine_b = nullptr;
};
+// drama3 tower: extra tensors around the generic block
+struct clip_drama3_layer {
+ ggml_tensor * enorm_w = nullptr;
+ ggml_tensor * hnorm_w = nullptr;
+ ggml_tensor * eh_proj_w = nullptr; // [code embd; hidden] -> hidden
+ ggml_tensor * embd_w = nullptr;
+ ggml_tensor * out_norm_w = nullptr;
+};
+
struct clip_vocos {
ggml_tensor * input_w = nullptr;
ggml_tensor * input_b = nullptr;
@@ -1107,6 +1116,7 @@ struct clip_model {
// qwen3tts code2wav: RVQ codes -> raw PCM
clip_code2wav c2w;
+ std::vector<clip_drama3_layer> drama3_layers;
// pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path)
clip_seanet seanet;
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index e6d931e69..a5ab432bb 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -1237,6 +1237,15 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
}
builder = std::make_unique<clip_graph_lfm3a_gen>(ctx, img, gen_process, all_known, n_frames);
} break;
+ case PROJECTOR_TYPE_DRAMA3_GEN:
+ {
+ const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE;
+ const int64_t n_cb = ctx->model.gen_code_embd_w->ne[2];
+ const int n_frames = params && params->codes ? (int) (params->codes->size() / n_cb) : 1;
+ builder = std::make_unique<clip_graph_drama3_gen>(ctx, img, gen_process, params ? params->gen_state : nullptr, n_frames,
+ params ? params->top_k : 30, params ? params->top_p : 0.9f,
+ params ? params->temp : 1.0f);
+ } break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE;
@@ -2257,6 +2266,22 @@ struct clip_model_loader {
// matches the reference decoder's sliding_window (speech_tokenizer/config.json)
hparams.wav_tfm_swa = 72;
} break;
+ case PROJECTOR_TYPE_DRAMA3_GEN:
+ {
+ // Fish DAC decoder, codec.pth has no config
+ hparams.audio_sample_rate = 44100;
+ hparams.wav_tfm_n_layer = 8;
+ hparams.wav_tfm_n_embd = 1024;
+ hparams.wav_tfm_n_ff = 3072;
+ hparams.wav_tfm_n_head = 16;
+ hparams.wav_tfm_n_head_kv = 16;
+ hparams.wav_tfm_eps = 1e-5f;
+ hparams.wav_tfm_rope_theta = 10000.0f;
+ hparams.wav_tfm_swa = 128;
+ hparams.wav_upsample_n_block = 2;
+ hparams.wav_dac_n_block = 4;
+ hparams.wav_dac_n_res = 3;
+ } break;
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
case PROJECTOR_TYPE_POCKETTTS_GEN:
{
@@ -2785,6 +2810,92 @@ struct clip_model_loader {
}
}
+ // code2wav: RVQ codes -> raw PCM, drama3 has no quant/pre_conv/tfm projections
+ auto load_code2wav = [&](bool with_proj) {
+ auto & c2w = model.c2w;
+
+ c2w.quant_first_in_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_IN, "weight"), with_proj);
+ c2w.quant_first_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight"), with_proj);
+ c2w.quant_first_cb_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight"));
+ c2w.quant_rest_in_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_IN, "weight"), with_proj);
+ c2w.quant_rest_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_OUT, "weight"), with_proj);
+ c2w.quant_rest_cb_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_CB, "weight"));
+
+ c2w.pre_conv_w = get_tensor(string_format(TN_A_GEN_WAV_PRE_CONV, "weight"), with_proj);
+ c2w.pre_conv_b = get_tensor(string_format(TN_A_GEN_WAV_PRE_CONV, "bias"), with_proj);
+
+ c2w.tfm_in_proj_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ, "weight"), with_proj);
+ c2w.tfm_in_proj_b = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ, "bias"), with_proj);
+ c2w.tfm_out_proj_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "weight"), with_proj);
+ c2w.tfm_out_proj_b = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias"), with_proj);
+ c2w.tfm_output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight"));
+
+ // loaded manually, the generic model.layers loop is taken by code_predictor
+ c2w.tfm_layers.resize(hparams.wav_tfm_n_layer);
+ for (int il = 0; il < hparams.wav_tfm_n_layer; il++) {
+ auto & layer = c2w.tfm_layers[il];
+ const char * p = "a.gen.wav.tfm";
+ layer.q_w = get_tensor(string_format(TN_ATTN_Q, p, il, "weight"));
+ layer.k_w = get_tensor(string_format(TN_ATTN_K, p, il, "weight"));
+ layer.v_w = get_tensor(string_format(TN_ATTN_V, p, il, "weight"));
+ layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, p, il, "weight"));
+ layer.ln_1_w = get_tensor(string_format(TN_LN_1, p, il, "weight"));
+ layer.ln_2_w = get_tensor(string_format(TN_LN_2, p, il, "weight"));
+ layer.ls_1_w = get_tensor(string_format(TN_LS_1, p, il, "weight"));
+ layer.ls_2_w = get_tensor(string_format(TN_LS_2, p, il, "weight"));
+ layer.ff_gate_w = get_tensor(string_format(TN_FFN_GATE, p, il, "weight"));
+ layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, p, il, "weight"));
+ layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, p, il, "weight"));
+ }
+
+ // upsample: 2x (causal ConvTranspose1d + ConvNeXt block)
+ c2w.upsample.resize(hparams.wav_upsample_n_block);
+ for (int il = 0; il < hparams.wav_upsample_n_block; il++) {
+ auto & up = c2w.upsample[il];
+ up.conv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_CONV, il, "weight"));
+ up.conv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_CONV, il, "bias"));
+ up.dwconv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight"));
+ up.dwconv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias"));
+ up.norm_w = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "weight"));
+ up.norm_b = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "bias"));
+ up.pw1_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "weight"));
+ up.pw1_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "bias"));
+ up.pw2_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "weight"));
+ up.pw2_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "bias"));
+ up.gamma = get_tensor(string_format(TN_A_GEN_WAV_UP_GAMMA, il));
+ }
+
+ // DAC decoder: conv_pre + n upsample blocks (each with n_res residual units) + conv_post
+ c2w.dac_entry_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_ENTRY, "weight"));
+ c2w.dac_entry_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_ENTRY, "bias"));
+
+ c2w.dac.resize(hparams.wav_dac_n_block);
+ for (int il = 0; il < hparams.wav_dac_n_block; il++) {
+ auto & blk = c2w.dac[il];
+ blk.snake_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_SNAKE, il, "alpha"));
+ blk.snake_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_SNAKE, il, "beta"));
+ blk.conv_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_CONV, il, "weight"));
+ blk.conv_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_CONV, il, "bias"));
+
+ blk.res.resize(hparams.wav_dac_n_res);
+ for (int ir = 0; ir < hparams.wav_dac_n_res; ir++) {
+ auto & res = blk.res[ir];
+ res.act1_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "alpha"));
+ res.act1_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "beta"));
+ res.conv1_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "weight"));
+ res.conv1_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "bias"));
+ res.act2_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "alpha"));
+ res.act2_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "beta"));
+ res.conv2_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "weight"));
+ res.conv2_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "bias"));
+ }
+ }
+
+ c2w.dac_post_snake_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "alpha"));
+ c2w.dac_post_snake_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "beta"));
+ c2w.dac_post_conv_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV, "weight"));
+ c2w.dac_post_conv_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV, "bias"));
+ };
switch (model.proj_type) {
case PROJECTOR_TYPE_MLP:
@@ -3656,91 +3767,25 @@ struct clip_model_loader {
model.gen_code_norm_w = get_tensor(string_format(TN_A_GEN_CODE_NORM, "weight"));
// code2wav: RVQ codes -> raw PCM, lives in the same ctx as code_predictor
- {
- auto & c2w = model.c2w;
-
- c2w.quant_first_in_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_IN, "weight"));
- c2w.quant_first_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight"));
- c2w.quant_first_cb_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight"));
- c2w.quant_rest_in_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_IN, "weight"));
- c2w.quant_rest_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_OUT, "weight"));
- c2w.quant_rest_cb_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_CB, "weight"));
-
- c2w.pre_conv_w = get_tensor(string_format(TN_A_GEN_WAV_PRE_CONV, "weight"));
- c2w.pre_conv_b = get_tensor(string_format(TN_A_GEN_WAV_PRE_CONV, "bias"));
-
- c2w.tfm_in_proj_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ, "weight"));
- c2w.tfm_in_proj_b = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ, "bias"));
- c2w.tfm_out_proj_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "weight"));
- c2w.tfm_out_proj_b = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias"));
- c2w.tfm_output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight"));
-
- // loaded manually, the generic model.layers loop is taken by code_predictor
- c2w.tfm_layers.resize(hparams.wav_tfm_n_layer);
- for (int il = 0; il < hparams.wav_tfm_n_layer; il++) {
- auto & layer = c2w.tfm_layers[il];
- const char * p = "a.gen.wav.tfm";
- layer.q_w = get_tensor(string_format(TN_ATTN_Q, p, il, "weight"));
- layer.k_w = get_tensor(string_format(TN_ATTN_K, p, il, "weight"));
- layer.v_w = get_tensor(string_format(TN_ATTN_V, p, il, "weight"));
- layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, p, il, "weight"));
- layer.ln_1_w = get_tensor(string_format(TN_LN_1, p, il, "weight"));
- layer.ln_2_w = get_tensor(string_format(TN_LN_2, p, il, "weight"));
- layer.ls_1_w = get_tensor(string_format(TN_LS_1, p, il, "weight"));
- layer.ls_2_w = get_tensor(string_format(TN_LS_2, p, il, "weight"));
- layer.ff_gate_w = get_tensor(string_format(TN_FFN_GATE, p, il, "weight"));
- layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, p, il, "weight"));
- layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, p, il, "weight"));
- }
-
- // upsample: 2x (causal ConvTranspose1d + ConvNeXt block)
- c2w.upsample.resize(hparams.wav_upsample_n_block);
- for (int il = 0; il < hparams.wav_upsample_n_block; il++) {
- auto & up = c2w.upsample[il];
- up.conv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_CONV, il, "weight"));
- up.conv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_CONV, il, "bias"));
- up.dwconv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight"));
- up.dwconv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias"));
- up.norm_w = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "weight"));
- up.norm_b = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "bias"));
- up.pw1_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "weight"));
- up.pw1_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "bias"));
- up.pw2_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "weight"));
- up.pw2_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "bias"));
- up.gamma = get_tensor(string_format(TN_A_GEN_WAV_UP_GAMMA, il));
- }
-
- // DAC decoder: conv_pre + n upsample blocks (each with n_res residual units) + conv_post
- c2w.dac_entry_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_ENTRY, "weight"));
- c2w.dac_entry_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_ENTRY, "bias"));
-
- c2w.dac.resize(hparams.wav_dac_n_block);
- for (int il = 0; il < hparams.wav_dac_n_block; il++) {
- auto & blk = c2w.dac[il];
- blk.snake_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_SNAKE, il, "alpha"));
- blk.snake_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_SNAKE, il, "beta"));
- blk.conv_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_CONV, il, "weight"));
- blk.conv_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_CONV, il, "bias"));
-
- blk.res.resize(hparams.wav_dac_n_res);
- for (int ir = 0; ir < hparams.wav_dac_n_res; ir++) {
- auto & res = blk.res[ir];
- res.act1_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "alpha"));
- res.act1_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "beta"));
- res.conv1_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "weight"));
- res.conv1_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "bias"));
- res.act2_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "alpha"));
- res.act2_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "beta"));
- res.conv2_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "weight"));
- res.conv2_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "bias"));
- }
- }
-
- c2w.dac_post_snake_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "alpha"));
- c2w.dac_post_snake_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "beta"));
- c2w.dac_post_conv_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV, "weight"));
- c2w.dac_post_conv_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV, "bias"));
+ load_code2wav(true);
+ } break;
+ case PROJECTOR_TYPE_DRAMA3_GEN:
+ {
+ // generic a.gen.code.blk.* tensors are loaded by the loop above
+ model.drama3_layers.resize(hparams.n_layer);
+ for (int il = 0; il < hparams.n_layer; ++il) {
+ auto & tl = model.drama3_layers[il];
+ model.layers[il].attn_gate_w = get_tensor(string_format(TN_ATTN_GATE, prefix, il, "weight"));
+ tl.enorm_w = get_tensor(string_format(TN_A_GEN_CODE_ENORM, il, "weight"));
+ tl.hnorm_w = get_tensor(string_format(TN_A_GEN_CODE_HNORM, il, "weight"));
+ tl.eh_proj_w = get_tensor(string_format(TN_A_GEN_CODE_EH_PROJ, il, "weight"));
+ tl.embd_w = get_tensor(string_format(TN_A_GEN_CODE_BLK_EMBD, il, "weight"));
+ tl.out_norm_w = get_tensor(string_format(TN_A_GEN_CODE_BLK_OUT_NORM, il, "weight"));
}
+ model.gen_code_embd_w = get_tensor(string_format(TN_A_GEN_CODE_EMBD, "weight")); // merged 3D, one table per codebook
+ model.gen_code_head_w = get_tensor(string_format(TN_A_GEN_CODE_HEAD, "weight")); // merged 3D, codebooks 1..
+ model.gen_code_out_embd_w = get_tensor(string_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); // backbone rows of the semantic band
+ load_code2wav(false);
} break;
case PROJECTOR_TYPE_CHATTERBOX:
{
@@ -5520,6 +5565,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
case PROJECTOR_TYPE_LFM3A_GEN:
+ case PROJECTOR_TYPE_DRAMA3_GEN:
{
// one hidden-state vector fed back to the talker per call
n_patches = 1;
@@ -5587,6 +5633,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
static std::vector<c2w_state_slot> list_gen_state_slots(const clip_hparams & hparams, const clip_model & model) {
switch (model.proj_type) {
case PROJECTOR_TYPE_QWEN3TTS_GEN: return list_c2w_state_slots(hparams, model);
+ case PROJECTOR_TYPE_DRAMA3_GEN: return list_drama3_state_slots(hparams, model);
case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model);
case PROJECTOR_TYPE_LFM3A_GEN: return list_lfm3a_gen_state_slots(hparams, model);
default: return {};
@@ -5772,6 +5819,14 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
LOG_ERR("%s: audio gen requires a gen state\n", __func__);
return false;
}
+ if (ctx->proj_type() == PROJECTOR_TYPE_DRAMA3_GEN && params->gen_process == CLIP_GEN_PROCESS_GEN_CODE) {
+ float pos = 0.0f;
+ ggml_backend_tensor_get(params->gen_state->slots[0], &pos, 0, sizeof(float));
+ if ((int) pos >= clip_graph_drama3_gen::n_pos_max) {
+ LOG_ERR("%s: the codebook tower holds at most %d rows\n", __func__, clip_graph_drama3_gen::n_pos_max);
+ return false;
+ }
+ }
// build the inference graph
ggml_backend_sched_reset(ctx->sched.get());
@@ -5845,6 +5900,10 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
size_t i = 0;
for (const auto & slot : list_gen_state_slots(hparams, model)) {
GGML_ASSERT(i < slots.size());
+ if (slot.in_place) {
+ i++;
+ continue;
+ }
ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str()));
}
};
@@ -6900,6 +6959,58 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
}
} break;
+ case PROJECTOR_TYPE_DRAMA3_GEN:
+ {
+ const int64_t n_cb = model.gen_code_embd_w->ne[2];
+ const int64_t n_vocab = model.gen_code_embd_w->ne[1];
+ if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
+ // frame-major to group-major, clamp: codec residual codebooks are smaller than the tower vocab
+ GGML_ASSERT(params->codes != nullptr && params->codes->size() % n_cb == 0);
+ const int64_t n_frames = (int64_t) params->codes->size() / n_cb;
+ const int64_t n_first = model.c2w.quant_first_cb_w->ne[1];
+ const int64_t n_rest = model.c2w.quant_rest_cb_w->ne[1];
+ std::vector<int32_t> codes(params->codes->size());
+ for (int64_t f = 0; f < n_frames; f++) {
+ for (int64_t g = 0; g < n_cb; g++) {
+ const int64_t n = g == 0 ? n_first : n_rest;
+ codes[g * n_frames + f] = std::clamp<int32_t>((*params->codes)[f * n_cb + g], 0, (int32_t) n - 1);
+ }
+ }
+ set_input_i32("inp_codes", codes);
+ set_gen_state_in();
+ } else {
+ // code0 < 0: next row is text (prompt), the tower only fills its KV
+ // codes: input frame codes for a speech row, empty for a text row
+ const bool next_speech = params->code0 >= 0;
+ const bool cur_speech = params->codes && (int64_t) params->codes->size() == n_cb;
+ if (params->code0 >= n_vocab) {
+ LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) n_vocab);
+ return false;
+ }
+ std::vector<int32_t> code0 = { std::max(params->code0, 0) };
+ set_input_i32("inp_code0", code0);
+
+ std::vector<int32_t> extra(n_cb - 2);
+ for (int64_t g = 2; g < n_cb; g++) {
+ const int32_t c = cur_speech ? std::clamp<int32_t>((*params->codes)[g], 0, (int32_t) n_vocab - 1) : 0;
+ extra[g - 2] = c + (int32_t) (g * n_vocab);
+ }
+ set_input_i32("inp_extra", extra);
+ set_input_f32("inp_scale", { next_speech ? 1.0f : 0.0f, cur_speech ? 1.0f : 0.0f });
+
+ float pos = 0.0f;
+ ggml_backend_tensor_get(params->gen_state->slots[0], &pos, 0, sizeof(float));
+ std::vector<int32_t> inp_pos = { (int32_t) pos };
+ set_input_i32("inp_pos", inp_pos);
+
+ std::uniform_real_distribution<float> dist(0.0f, 1.0f);
+ std::vector<float> r(model.gen_code_head_w->ne[2]);
+ for (auto & v : r) {
+ v = dist(params->gen_state->rng);
+ }
+ set_input_f32("inp_rand", r);
+ }
+ } break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
@@ -7700,13 +7811,15 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
clip_graph_soprano::decode_spectrum(model, spectrum, out_audio);
}
- // drop the tail audio that comes from the code-0 rear padding
- const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0;
- const int64_t n_frames_w = hparams.wav_tfm_swa;
- const int64_t n_frames = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w;
- if (n_frames < n_frames_w) {
- const size_t hop = out_audio.size() / n_frames_w;
- out_audio.resize((size_t) n_frames * hop);
+ // drop the tail audio that comes from the code-0 rear padding, drama3 has no padding
+ if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN) {
+ const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0;
+ const int64_t n_frames_w = hparams.wav_tfm_swa;
+ const int64_t n_frames = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w;
+ if (n_frames < n_frames_w) {
+ const size_t hop = out_audio.size() / n_frames_w;
+ out_audio.resize((size_t) n_frames * hop);
+ }
}
}
if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_LFM3A) {
@@ -7744,6 +7857,10 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
const auto & slots = params->gen_state->slots;
size_t i = 0;
for (const auto & slot : list_gen_state_slots(hparams, model)) {
+ if (slot.in_place) {
+ i++;
+ continue;
+ }
ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str());
if (t == nullptr) {
GGML_ABORT("graph has no \"state_out_%s\" tensor", slot.name.c_str());
@@ -7924,6 +8041,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_fc_w->ne[2];
case PROJECTOR_TYPE_QWEN3TTS_GEN:
case PROJECTOR_TYPE_LFM3A_GEN:
+ case PROJECTOR_TYPE_DRAMA3_GEN:
return ctx->model.gen_code_out_embd_w->ne[0];
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
return ctx->model.spk_proj_w->ne[1];
diff --git a/tools/mtmd/models/drama3-gen.cpp b/tools/mtmd/models/drama3-gen.cpp
new file mode 100644
index 000000000..de7a4f9e8
--- /dev/null
+++ b/tools/mtmd/models/drama3-gen.cpp
@@ -0,0 +1,256 @@
+#include "models.h"
+
+#include <cmath>
+#include <string>
+
+// tower RoPE: interleaved, first 64 dims of each head
+static constexpr int DRAMA3_N_ROT = 64;
+static constexpr float DRAMA3_ROPE_THETA = 1e7f;
+
+std::vector<c2w_state_slot> list_drama3_state_slots(const clip_hparams & hparams, const clip_model & model) {
+ std::vector<c2w_state_slot> slots;
+ slots.push_back({"tower_pos", 1, 1, true});
+ for (size_t il = 0; il < model.layers.size(); il++) {
+ const int64_t n_kv = model.layers[il].k_w->ne[1];
+ slots.push_back({"tower_k_" + std::to_string(il), n_kv, clip_graph_drama3_gen::n_pos_max, true});
+ slots.push_back({"tower_v_" + std::to_string(il), n_kv, clip_graph_drama3_gen::n_pos_max, true});
+ }
+ for (auto & slot : list_c2w_state_slots(hparams, model)) {
+ slots.push_back(slot);
+ }
+ return slots;
+}
+
+// h_out = out_norm(Block(eh_proj([enorm(mask * E[code] + extra); hnorm(h_in)])))
+ggml_tensor * clip_graph_drama3_gen::tower_layer(ggml_tensor * x, int il, ggml_tensor * code, ggml_tensor * mask, ggml_tensor * extra,
+ ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos) {
+ const auto & layer = model.layers[il];
+ const auto & tl = model.drama3_layers[il];
+ const int n_head = hparams.n_head;
+ const int n_head_kv = hparams.n_head_kv;
+ const int64_t d_head = layer.q_w->ne[1] / n_head;
+ const int64_t n_kv = d_head * n_head_kv;
+ const float eps = hparams.eps;
+
+ ggml_tensor * e = ggml_mul(ctx0, ggml_get_rows(ctx0, tl.embd_w, code), mask);
+ if (extra) {
+ e = ggml_add(ctx0, e, extra);
+ }
+ e = ggml_mul(ctx0, ggml_rms_norm(ctx0, e, eps), tl.enorm_w);
+ ggml_tensor * h = ggml_mul(ctx0, ggml_rms_norm(ctx0, x, eps), tl.hnorm_w);
+ ggml_tensor * mixed = ggml_mul_mat(ctx0, tl.eh_proj_w, ggml_concat(ctx0, e, h, 0));
+
+ ggml_tensor * cur = ggml_mul(ctx0, ggml_rms_norm(ctx0, mixed, eps), layer.ln_1_w);
+ ggml_tensor * q = ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, layer.q_w, cur), d_head, n_head, 1);
+ ggml_tensor * k = ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, layer.k_w, cur), d_head, n_head_kv, 1);
+ ggml_tensor * v = ggml_mul_mat(ctx0, layer.v_w, cur);
+ ggml_tensor * gate = ggml_mul_mat(ctx0, layer.attn_gate_w, cur);
+
+ q = ggml_mul(ctx0, ggml_rms_norm(ctx0, q, eps), layer.q_norm);
+ k = ggml_mul(ctx0, ggml_rms_norm(ctx0, k, eps), layer.k_norm);
+ q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, DRAMA3_N_ROT, GGML_ROPE_TYPE_NORMAL, 0, DRAMA3_ROPE_THETA, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+ k = ggml_rope_ext(ctx0, k, inp_pos, nullptr, DRAMA3_N_ROT, GGML_ROPE_TYPE_NORMAL, 0, DRAMA3_ROPE_THETA, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+
+ // write K/V of this row to the slot, attention uses the cached rows + this row
+ ggml_tensor * k_new = ggml_reshape_2d(ctx0, k, n_kv, 1);
+ ggml_build_forward_expand(gf, ggml_cpy(ctx0, k_new, ggml_view_2d(ctx0, k_slot, n_kv, 1, k_slot->nb[1], (size_t) pos * k_slot->nb[1])));
+ ggml_build_forward_expand(gf, ggml_cpy(ctx0, v, ggml_view_2d(ctx0, v_slot, n_kv, 1, v_slot->nb[1], (size_t) pos * v_slot->nb[1])));
+ ggml_tensor * k_all = k_new;
+ ggml_tensor * v_all = v;
+ if (pos > 0) {
+ k_all = ggml_concat(ctx0, ggml_view_2d(ctx0, k_slot, n_kv, pos, k_slot->nb[1], 0), k_new, 1);
+ v_all = ggml_concat(ctx0, ggml_view_2d(ctx0, v_slot, n_kv, pos, v_slot->nb[1], 0), v, 1);
+ }
+
+ ggml_tensor * attn = build_attn(nullptr, nullptr,
+ ggml_reshape_4d(ctx0, q, d_head, n_head, 1, 1),
+ ggml_reshape_4d(ctx0, k_all, d_head, n_head_kv, pos + 1, 1),
+ ggml_reshape_4d(ctx0, v_all, d_head, n_head_kv, pos + 1, 1),
+ nullptr, 1.0f / sqrtf((float) d_head), il);
+ attn = ggml_mul(ctx0, attn, ggml_sigmoid(ctx0, gate));
+ cur = ggml_add(ctx0, mixed, ggml_mul_mat(ctx0, layer.o_w, attn));
+
+ ggml_tensor * ff = ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, eps), layer.ln_2_w);
+ ff = ggml_swiglu_split(ctx0, ggml_mul_mat(ctx0, layer.ff_gate_w, ff), ggml_mul_mat(ctx0, layer.ff_up_w, ff));
+ cur = ggml_add(ctx0, cur, ggml_mul_mat(ctx0, layer.ff_down_w, ff));
+
+ cur = ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, eps), tl.out_norm_w);
+ cb(cur, "tower_out", il);
+ return cur;
+}
+
+// one row: backbone hidden + next main code -> other codes of that frame + next backbone input
+ggml_cgraph * clip_graph_drama3_gen::build_tower() {
+ const int n_layer = (int) model.layers.size();
+ const int n_heads = (int) model.gen_code_head_w->ne[2]; // one per residual codebook
+ const int n_shared = n_layer - n_heads;
+ const int64_t n_cb = model.gen_code_embd_w->ne[2];
+ const int64_t n_embd_cb = model.gen_code_embd_w->ne[0];
+
+ int pos = 0;
+ std::vector<ggml_tensor *> slots;
+ const auto slot_list = list_drama3_state_slots(hparams, model);
+ if (gen_state) {
+ float pos_f = 0.0f;
+ ggml_backend_tensor_get(gen_state->slots[0], &pos_f, 0, sizeof(float));
+ pos = (int) pos_f;
+ slots = gen_state->slots;
+ } else {
+ // buffer reserve: no gen state, use stand-in slots
+ for (const auto & slot : slot_list) {
+ ggml_tensor * t = nullptr;
+ if (slot.in_place) {
+ t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
+ ggml_set_input(t);
+ }
+ slots.push_back(t);
+ }
+ }
+ GGML_ASSERT(pos < n_pos_max);
+
+ ggml_tensor * h = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd);
+ ggml_set_name(h, "inp_raw");
+ ggml_set_input(h);
+
+ ggml_tensor * inp_code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1);
+ ggml_set_name(inp_code0, "inp_code0");
+ ggml_set_input(inp_code0);
+
+ // codes 2.. of the input frame, as row ids of the merged codebook table
+ ggml_tensor * inp_extra = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_cb - 2);
+ ggml_set_name(inp_extra, "inp_extra");
+ ggml_set_input(inp_extra);
+
+ // [next frame is speech, input frame is speech]
+ ggml_tensor * inp_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 2);
+ ggml_set_name(inp_scale, "inp_scale");
+ ggml_set_input(inp_scale);
+
+ ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1);
+ ggml_set_name(inp_pos, "inp_pos");
+ ggml_set_input(inp_pos);
+
+ ggml_tensor * inp_rand = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_heads);
+ ggml_set_name(inp_rand, "inp_rand");
+ ggml_set_input(inp_rand);
+
+ ggml_tensor * mask = ggml_view_1d(ctx0, inp_scale, 1, 0);
+ ggml_tensor * extra_on = ggml_view_1d(ctx0, inp_scale, 1, sizeof(float));
+
+ ggml_tensor * cb_all = ggml_reshape_2d(ctx0, model.gen_code_embd_w, n_embd_cb, model.gen_code_embd_w->ne[1] * n_cb);
+ ggml_tensor * extra = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_get_rows(ctx0, cb_all, inp_extra)));
+ extra = ggml_mul(ctx0, ggml_reshape_2d(ctx0, ggml_sum_rows(ctx0, extra), n_embd_cb, 1), extra_on);
+
+ clip_graph_qwen3tts_gen::code_gen cg(*this, top_k, top_p);
+ ggml_tensor * out_codes = cg.cache_set(ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, 1, n_cb), 0, inp_code0);
+
+ ggml_tensor * code = inp_code0;
+ ggml_tensor * x = ggml_reshape_2d(ctx0, h, n_mmproj_embd, 1);
+ for (int il = 0; il < n_layer; il++) {
+ x = tower_layer(x, il, code, mask, il == 0 ? extra : nullptr, inp_pos, slots[1 + 2 * il], slots[2 + 2 * il], pos);
+ if (il < n_shared) {
+ continue;
+ }
+ const int ih = il - n_shared;
+ ggml_tensor * head = ggml_view_2d(ctx0, model.gen_code_head_w, model.gen_code_head_w->ne[0], model.gen_code_head_w->ne[1],
+ model.gen_code_head_w->nb[1], (size_t) ih * model.gen_code_head_w->nb[2]);
+ ggml_tensor * logits = ggml_mul_mat(ctx0, head, x);
+ if (temp > 0.0f && temp != 1.0f) {
+ logits = ggml_scale(ctx0, logits, 1.0f / temp);
+ }
+ code = cg.do_sampling(logits, ggml_view_1d(ctx0, inp_rand, 1, (size_t) ih * sizeof(float)));
+ out_codes = cg.cache_set(out_codes, ih + 1, code);
+ }
+
+ ggml_tensor * pos_slot = slots[0];
+ ggml_build_forward_expand(gf, ggml_cpy(ctx0, ggml_scale_bias(ctx0, pos_slot, 1.0f, 1.0f), pos_slot));
+
+ out_codes = ggml_cont(ctx0, out_codes);
+ ggml_set_name(out_codes, "out_codes");
+ ggml_set_output(out_codes);
+ ggml_build_forward_expand(gf, out_codes);
+
+ // backbone input of the next row: (E_lm[main] + E_cb1[code 1]) / sqrt(2)
+ ggml_tensor * code1 = ggml_view_1d(ctx0, out_codes, 1, out_codes->nb[1]);
+ ggml_tensor * cb1 = ggml_view_2d(ctx0, model.gen_code_embd_w, n_embd_cb, model.gen_code_embd_w->ne[1],
+ model.gen_code_embd_w->nb[1], model.gen_code_embd_w->nb[2]);
+ ggml_tensor * out_embd = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, inp_code0), ggml_get_rows(ctx0, cb1, code1));
+ out_embd = ggml_scale(ctx0, out_embd, 1.0f / sqrtf(2.0f));
+ cb(out_embd, "gen_audio_out", -1);
+ ggml_build_forward_expand(gf, out_embd);
+
+ return gf;
+}
+
+// frames -> PCM, qwen3tts code2wav blocks without pre_conv and tfm projections
+ggml_cgraph * clip_graph_drama3_gen::build_codec() {
+ const auto & c2w = model.c2w;
+ const int64_t n_cb = model.gen_code_embd_w->ne[2];
+
+ ggml_tensor * inp_codes = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_frames, n_cb);
+ ggml_set_name(inp_codes, "inp_codes");
+ ggml_set_input(inp_codes);
+
+ clip_graph_qwen3tts_gen::code2wav dec(*this);
+ dec.gelu_erf = true;
+ for (const auto & slot : list_c2w_state_slots(hparams, model)) {
+ ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
+ ggml_set_name(t, ("state_in_" + slot.name).c_str());
+ ggml_set_input(t);
+ dec.state_in[slot.name] = t;
+ }
+
+ // out_proj is folded into the codebook tables at conversion
+ auto group_ids = [&](int g) {
+ return ggml_view_1d(ctx0, inp_codes, n_frames, (size_t) g * inp_codes->nb[1]);
+ };
+ ggml_tensor * cur = ggml_get_rows(ctx0, c2w.quant_first_cb_w, group_ids(0)); // [dim, T]
+ for (int g = 1; g < n_cb; g++) {
+ ggml_tensor * table = ggml_view_2d(ctx0, c2w.quant_rest_cb_w, c2w.quant_rest_cb_w->ne[0], c2w.quant_rest_cb_w->ne[1],
+ c2w.quant_rest_cb_w->nb[1], (size_t) (g - 1) * c2w.quant_rest_cb_w->nb[2]);
+ cur = ggml_add(ctx0, cur, ggml_get_rows(ctx0, table, group_ids(g)));
+ }
+
+ for (int il = 0; il < hparams.wav_tfm_n_layer; il++) {
+ cur = dec.tfm_layer_forward(cur, c2w.tfm_layers[il], il);
+ }
+ cur = ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, hparams.wav_tfm_eps), c2w.tfm_output_norm_w);
+ cb(cur, "wav_tfm_out", -1);
+
+ ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [T, dim]
+ for (size_t il = 0; il < c2w.upsample.size(); il++) {
+ const auto & up = c2w.upsample[il];
+ x = dec.causal_conv_transpose1d(x, up.conv_w, up.conv_b, 2, "");
+ x = dec.convnext_block(x, up, "up" + std::to_string(il));
+ }
+
+ static constexpr int DAC_DILATIONS[3] = { 1, 3, 9 };
+ x = dec.causal_conv1d(x, c2w.dac_entry_w, c2w.dac_entry_b, 1, "dac_entry");
+ for (size_t il = 0; il < c2w.dac.size(); il++) {
+ const auto & blk = c2w.dac[il];
+ const int stride = (int) (blk.conv_w->ne[0] / 2);
+ const std::string blk_name = "dac" + std::to_string(il);
+ x = dec.snake(x, blk.snake_alpha, blk.snake_beta);
+ x = dec.causal_conv_transpose1d(x, blk.conv_w, blk.conv_b, stride, blk_name + "_tail");
+ for (size_t ir = 0; ir < blk.res.size(); ir++) {
+ x = dec.dac_res_unit(x, blk.res[ir], DAC_DILATIONS[ir], blk_name + "_res" + std::to_string(ir));
+ }
+ }
+ x = dec.snake(x, c2w.dac_post_snake_alpha, c2w.dac_post_snake_beta);
+ x = dec.causal_conv1d(x, c2w.dac_post_conv_w, c2w.dac_post_conv_b, 1, "dac_post_conv");
+ x = ggml_tanh(ctx0, ggml_reshape_1d(ctx0, x, x->ne[0]));
+ ggml_set_name(x, "out_audio");
+ ggml_set_output(x);
+ ggml_build_forward_expand(gf, x);
+
+ for (auto & slot : dec.state_out) {
+ ggml_set_name(slot.second, ("state_out_" + slot.first).c_str());
+ ggml_set_output(slot.second);
+ ggml_build_forward_expand(gf, slot.second);
+ }
+ return gf;
+}
+
+ggml_cgraph * clip_graph_drama3_gen::build() {
+ return gen_process == CLIP_GEN_PROCESS_GEN_WAV ? build_codec() : build_tower();
+}
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index c7520468f..3323f0e9d 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -424,6 +424,9 @@ struct clip_graph_qwen3tts_gen : clip_graph {
ggml_tensor * convnext_block(ggml_tensor * x, const clip_code2wav::upsample_block & blk, const std::string & state_prefix) const;
ggml_tensor * dac_res_unit(ggml_tensor * x, const clip_code2wav::dac_res & res, int dilation, const std::string & state_name) const;
+ // drama3: erf GELU in ConvNeXt, not the tanh approximation
+ bool gelu_erf = false;
+
// inp_codes [1, n_codes] I32 -> this frame's audio samples [n_samples] F32, clamped to [-1, 1]
ggml_tensor * decode(ggml_tensor * inp_codes) const;
};
@@ -487,12 +490,43 @@ struct c2w_state_slot {
std::string name;
int64_t ne0;
int64_t ne1;
+ bool in_place = false; // graph writes the slot directly, no state_in/state_out copy
};
std::vector<c2w_state_slot> list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model);
+// drama3: tower KV slots, then list_c2w_state_slots()
+std::vector<c2w_state_slot> list_drama3_state_slots(const clip_hparams & hparams, const clip_model & model);
+
// same, for the streaming mimi decoder (pocket-tts GEN_WAV)
std::vector<c2w_state_slot> list_pockettts_state_slots(const clip_hparams & hparams, const clip_model & model);
+//
+// drama3 generation:
+// GEN_CODE = one codebook tower row, each layer attends over the whole sequence (KV in gen state)
+// GEN_WAV = Fish DAC decoder, frames -> PCM
+//
+struct clip_graph_drama3_gen : clip_graph {
+ clip_graph_drama3_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, clip_gen_state * gen_state,
+ int n_frames, int top_k, float top_p, float temp)
+ : clip_graph(ctx, img), gen_process(gen_process), gen_state(gen_state), n_frames(n_frames), top_k(top_k), top_p(top_p), temp(temp) {}
+ ggml_cgraph * build() override;
+
+ // tower KV rows, shared by the prompt and the generated frames
+ static constexpr int n_pos_max = 4096;
+
+ clip_gen_process_type gen_process;
+ clip_gen_state * gen_state; // nullptr on buffer reserve
+ int n_frames; // GEN_WAV only
+ int top_k;
+ float top_p;
+ float temp;
+
+ ggml_cgraph * build_tower();
+ ggml_cgraph * build_codec();
+ ggml_tensor * tower_layer(ggml_tensor * x, int il, ggml_tensor * code, ggml_tensor * mask, ggml_tensor * extra,
+ ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos);
+};
+
struct clip_graph_soprano : clip_graph {
clip_graph_soprano(clip_ctx * ctx, const clip_image_f32 & img, int n_frames) : clip_graph(ctx, img), n_frames(n_frames) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/models/qwen3tts-gen.cpp b/tools/mtmd/models/qwen3tts-gen.cpp
index 9826190e6..ab619b237 100644
--- a/tools/mtmd/models/qwen3tts-gen.cpp
+++ b/tools/mtmd/models/qwen3tts-gen.cpp
@@ -527,7 +527,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::convnext_block(ggml_tensor * x,
ggml_tensor * g = ggml_mul_mat(ctx0, blk.pw1_w, hc);
g = ggml_add(ctx0, g, blk.pw1_b);
- g = ggml_gelu(ctx0, g);
+ g = gelu_erf ? ggml_gelu_erf(ctx0, g) : ggml_gelu(ctx0, g);
g = ggml_mul_mat(ctx0, blk.pw2_w, g);
g = ggml_add(ctx0, g, blk.pw2_b);
g = ggml_mul(ctx0, g, blk.gamma);
@@ -617,7 +617,7 @@ std::vector<c2w_state_slot> list_c2w_state_slots(const clip_hparams & hparams, c
const auto & c2w = model.c2w;
std::vector<c2w_state_slot> slots;
- if (c2w.pre_conv_w == nullptr) {
+ if (c2w.dac_entry_w == nullptr) {
return slots; // not a code2wav model, it keeps no state between calls
}
@@ -632,7 +632,9 @@ std::vector<c2w_state_slot> list_c2w_state_slots(const clip_hparams & hparams, c
slots.push_back({"tfm_v_" + std::to_string(il), kv_ch, prefix});
}
- slots.push_back({"pre_conv", c2w.pre_conv_w->ne[0] - 1, c2w.pre_conv_w->ne[1]});
+ if (c2w.pre_conv_w) {
+ slots.push_back({"pre_conv", c2w.pre_conv_w->ne[0] - 1, c2w.pre_conv_w->ne[1]});
+ }
for (size_t il = 0; il < c2w.upsample.size(); il++) {
const auto & up = c2w.upsample[il];
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 0d7901575..ab5cbd424 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -2077,6 +2077,345 @@ private:
bool wav_header_sent = false;
};
+// Drama 3: backbone samples the main token of each frame, codebook tower samples the other codes, Fish DAC makes the PCM
+// the tower attends over the whole sequence, so it also runs on every prompt row
+class drama3_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
+public:
+ using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
+
+ void reset() override {
+ seq_id = 0;
+ pos = 0;
+ prompt_tokens.clear();
+ prompt_pos = 0;
+ frame.clear();
+ hist.clear();
+ codes_buf.clear();
+ n_decoded = 0;
+ gen_state.reset();
+ pcm_sent = 0;
+ wav_header_sent = false;
+ audio_pcm.clear();
+ h_state_buf.clear();
+ out_buf.clear();
+ }
+
+ int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
+ reset();
+ seq_id = inp->seq_id;
+ out_type = inp->out_type;
+ stream = inp->stream;
+ if (inp->speaker_ref) {
+ LOG_WRN("mtmd_helper_gen_audio: voice cloning is not supported yet, the speaker file is ignored\n");
+ }
+
+ sem_begin = find_special_token(vocab, "<|semantic:0|>");
+ eos = find_special_token(vocab, "<|im_end|>");
+ if (sem_begin == LLAMA_TOKEN_NULL || eos == LLAMA_TOKEN_NULL) {
+ LOG_ERR("mtmd_helper_gen_audio: missing Drama 3 tokens in vocab\n");
+ return 1;
+ }
+
+ // reference-bracket-v1 prompt without references, tokenize each segment alone as the reference does
+ std::string text(inp->prompt, inp->prompt_len);
+ if (text.rfind("<|speaker:", 0) != 0) {
+ text = "<|speaker:0|> " + text;
+ }
+ const char * segments[] = {
+ "<|im_start|>system\n", "convert the provided text to speech", "<|im_end|>\n",
+ "<|im_start|>user\n", text.c_str(), "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>",
+ };
+ for (const char * seg : segments) {
+ const int32_t len = (int32_t) strlen(seg);
+ std::vector<llama_token> ids(len + 16);
+ const int32_t n = llama_tokenize(vocab, seg, len, ids.data(), (int32_t) ids.size(), false, true);
+ if (n < 0) {
+ LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
+ return 1;
+ }
+ prompt_tokens.insert(prompt_tokens.end(), ids.begin(), ids.begin() + n);
+ }
+ if (prompt_tokens.size() >= llama_n_ctx_seq(lctx) || prompt_tokens.size() > n_tower_rows) {
+ LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n");
+ return 1;
+ }
+ if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) {
+ LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n");
+ return 1;
+ }
+ rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
+ return init_gen_state(inp->seed) ? 0 : 1;
+ }
+
+ // run the tower on every prompt row to fill its KV
+ int32_t step_prompt(int32_t n_batch) override {
+ GGML_ASSERT(n_batch > 0);
+ const int n_prompt = (int) prompt_tokens.size();
+ if (prompt_pos >= n_prompt) {
+ return 0;
+ }
+ // small batches: every row also outputs logits
+ const int n = std::min({ n_batch, n_prompt - prompt_pos, 64 });
+ llama_batch batch = llama_batch_init(n, 0, 1);
+ for (int i = 0; i < n; i++) {
+ batch.token [i] = prompt_tokens[(size_t) (prompt_pos + i)];
+ batch.pos [i] = pos + i;
+ batch.n_seq_id[i] = 1;
+ batch.seq_id [i][0] = seq_id;
+ batch.logits [i] = 1;
+ }
+ batch.n_tokens = n;
+ const int ret = llama_decode(lctx, batch);
+ llama_batch_free(batch);
+ if (ret != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
+ return -1;
+ }
+ for (int i = 0; i < n; i++) {
+ const float * h = llama_get_embeddings_ith(lctx, i);
+ if (prompt_pos + i == n_prompt - 1) {
+ h_state_buf.assign(h, h + n_embd); // first frame comes from the last prompt row
+ break;
+ }
+ if (!run_tower(-1, h)) {
+ return -1;
+ }
+ }
+ pos += n;
+ prompt_pos += n;
+ return n_prompt - prompt_pos;
+ }
+
+ // sample the main token here, ignore the token of the caller
+ int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
+ GGML_UNUSED(sampled);
+ GGML_UNUSED(h_state_in);
+ *h_state_out = nullptr;
+ *out_stop = false;
+
+ const llama_token main = sample_main(llama_get_logits_ith(lctx, -1));
+ const size_t n_rows = prompt_tokens.size() - 1 + codes_buf.size() / n_cb;
+ if (main == eos || (uint32_t) pos >= llama_n_ctx_seq(lctx) || n_rows >= n_tower_rows) {
+ *out_stop = true;
+ return 0;
+ }
+ hist.push_back(main);
+ if (hist.size() > ras_window) {
+ hist.erase(hist.begin());
+ }
+
+ const float * next_embd = run_tower(main - sem_begin, h_state_buf.data());
+ if (!next_embd) {
+ return 1;
+ }
+ codes_buf.insert(codes_buf.end(), frame.begin(), frame.end());
+
+ // text/speech stream: all mrope sections use the same position
+ const bool mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE ||
+ llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE;
+ decode_embd_batch batch(next_embd, 1, mrope ? 4 : 1, n_embd);
+ if (mrope) {
+ batch.set_position_mrope_1d(pos, seq_id);
+ } else {
+ batch.set_position_normal(pos, seq_id);
+ }
+ batch.logits[0] = 1;
+ if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
+ return 1;
+ }
+ pos++;
+ const float * h = llama_get_embeddings_ith(lctx, -1);
+ h_state_buf.assign(h, h + n_embd);
+ *h_state_out = h_state_buf.data();
+
+ if (stream && codes_buf.size() / n_cb - n_decoded >= stream_frames) {
+ return decode_wav() ? 0 : 1;
+ }
+ return 0;
+ }
+
+ int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
+ *out_sample_rate = info.sample_rate;
+
+ if (!stream) {
+ if (flush() != 0) {
+ return 1;
+ }
+ if (out_n_samples) {
+ *out_n_samples = (int64_t) audio_pcm.size();
+ }
+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+ *out_data = (const char *) audio_pcm.data();
+ *out_data_len = audio_pcm.size() * sizeof(float);
+ return 0;
+ }
+ out_buf.clear();
+ if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
+ LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
+ return 1;
+ }
+ *out_data = out_buf.data();
+ *out_data_len = out_buf.size();
+ return 0;
+ }
+
+ // streaming: only the audio since the previous call
+ const size_t n_new = audio_pcm.size() - pcm_sent;
+ if (out_n_samples) {
+ *out_n_samples = (int64_t) n_new;
+ }
+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+ *out_data = (const char *) (audio_pcm.data() + pcm_sent);
+ *out_data_len = n_new * sizeof(float);
+ pcm_sent = audio_pcm.size();
+ return 0;
+ }
+ out_buf.clear();
+ if (!wav_header_sent) {
+ write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
+ wav_header_sent = true;
+ }
+ append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
+ pcm_sent = audio_pcm.size();
+ *out_data = out_buf.data();
+ *out_data_len = out_buf.size();
+ return 0;
+ }
+
+ int32_t flush() override {
+ return decode_wav() ? 0 : 1;
+ }
+
+private:
+ // repetition-aware sampling: if a semantic draw repeats one of the last ras_window main tokens, draw again with fixed settings
+ static constexpr size_t ras_window = 16;
+ static constexpr size_t n_cb = 10;
+ // codebook tower KV rows (clip_graph_drama3_gen::n_pos_max): one per prompt row but the last, one per frame
+ static constexpr size_t n_tower_rows = 4096;
+ // frames per codec call when streaming (0.33 s)
+ static constexpr size_t stream_frames = 7;
+
+ // top-p keeps the token that crosses the threshold, as the reference does
+ llama_token draw(const std::vector<float> & cand_logits, float temp, int32_t top_k, float top_p) {
+ std::vector<std::pair<float, int>> cand(cand_logits.size());
+ for (size_t i = 0; i < cand.size(); i++) {
+ cand[i] = { cand_logits[i] / temp, (int) i };
+ }
+ std::sort(cand.begin(), cand.end(), [](const auto & a, const auto & b) { return a.first > b.first; });
+ const float lmax = cand[0].first;
+ double sum = 0.0;
+ std::vector<double> prob(cand.size());
+ for (size_t i = 0; i < cand.size(); i++) {
+ prob[i] = std::exp((double) (cand[i].first - lmax));
+ sum += prob[i];
+ }
+ size_t n_keep = cand.size();
+ if (top_k > 0) {
+ n_keep = std::min(n_keep, (size_t) top_k);
+ }
+ double cum = 0.0;
+ for (size_t i = 0; i < n_keep; i++) {
+ if (cum > top_p * sum) {
+ n_keep = i;
+ break;
+ }
+ cum += prob[i];
+ }
+ double kept = 0.0;
+ for (size_t i = 0; i < n_keep; i++) {
+ kept += prob[i];
+ }
+ double r = std::uniform_real_distribution<double>(0.0, 1.0)(rng) * kept;
+ for (size_t i = 0; i < n_keep; i++) {
+ r -= prob[i];
+ if (r <= 0.0) {
+ return cand[i].second;
+ }
+ }
+ return cand[n_keep - 1].second;
+ }
+
+ // candidates: eos, then semantic codes 1.. (the reference never samples code 0)
+ llama_token sample_main(const float * logits) {
+ std::vector<float> cand(n_cb_vocab);
+ cand[0] = logits[eos];
+ for (int i = 1; i < n_cb_vocab; i++) {
+ cand[i] = logits[sem_begin + i];
+ }
+ auto to_token = [&](int i) { return i == 0 ? eos : sem_begin + i; };
+ const llama_token first = to_token(draw(cand, 1.0f, 0, 0.9f));
+ if (first != eos && std::find(hist.begin(), hist.end(), first) != hist.end()) {
+ return to_token(draw(cand, 1.0f, 30, 0.95f));
+ }
+ return first;
+ }
+
+ // code0 < 0 for a prompt row, else the main code of the next frame
+ // returns the next backbone input, the codes of the next frame go to frame
+ const float * run_tower(int32_t code0, const float * h) {
+ mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
+ inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
+ inp.code0 = code0;
+ inp.embd = const_cast<float *>(h);
+ inp.codes = frame.empty() ? nullptr : frame.data();
+ inp.n_codes = frame.size();
+ mtmd_gen_out out{};
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || (code0 >= 0 && out.n_codes != n_cb)) {
+ LOG_ERR("mtmd_helper_gen_audio: codebook tower failed\n");
+ return nullptr;
+ }
+ if (code0 >= 0) {
+ frame.assign(out.codes, out.codes + out.n_codes);
+ }
+ return out.embd;
+ }
+
+ // the codec is stateful, so frames can be decoded in pieces
+ bool decode_wav() {
+ const size_t n_frames = codes_buf.size() / n_cb;
+ while (n_decoded < n_frames) {
+ const size_t n = std::min(n_frames - n_decoded, (size_t) 64);
+ mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
+ inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
+ inp.codes = codes_buf.data() + n_decoded * n_cb;
+ inp.n_codes = n * n_cb;
+ mtmd_gen_out out{};
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
+ LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n");
+ return false;
+ }
+ audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
+ n_decoded += n;
+ }
+ return true;
+ }
+
+ static constexpr int n_cb_vocab = 4096; // <|semantic:0..4095|>
+
+ llama_token sem_begin = LLAMA_TOKEN_NULL;
+ llama_token eos = LLAMA_TOKEN_NULL;
+
+ llama_seq_id seq_id = 0;
+ llama_pos pos = 0;
+ std::vector<llama_token> prompt_tokens;
+ int prompt_pos = 0;
+
+ std::mt19937 rng;
+ std::vector<int32_t> frame; // codes of the last frame, input of the next tower row
+ std::vector<llama_token> hist; // last main tokens, for repetition-aware sampling
+ std::vector<int32_t> codes_buf; // frame-major
+ size_t n_decoded = 0; // frames already through the codec
+
+ std::vector<float> audio_pcm;
+ std::vector<float> h_state_buf;
+ mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
+ std::vector<char> out_buf;
+ bool stream = false;
+ size_t pcm_sent = 0; // samples already returned by get_output()
+ bool wav_header_sent = false;
+};
+
static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
switch (mtmd_gen_audio_get_info(mctx).type) {
case MTMD_GEN_AUDIO_TYPE_QWEN3TTS:
@@ -2089,6 +2428,8 @@ static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lc
return std::unique_ptr<mtmd_gen_audio_pipeline>(new lfm3a_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_CHATTERBOX:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new chatterbox_gen_audio_pipeline(lctx, mctx));
+ case MTMD_GEN_AUDIO_TYPE_DRAMA3:
+ return std::unique_ptr<mtmd_gen_audio_pipeline>(new drama3_gen_audio_pipeline(lctx, mctx));
default:
return nullptr;
}
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 128e0ebd6..990ca042c 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -2373,6 +2373,10 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
info.type = MTMD_GEN_AUDIO_TYPE_CHATTERBOX;
info.sample_rate = 24000;
break;
+ case PROJECTOR_TYPE_DRAMA3_GEN:
+ info.type = MTMD_GEN_AUDIO_TYPE_DRAMA3;
+ info.sample_rate = 44100;
+ break;
default:
info.type = MTMD_GEN_AUDIO_TYPE_NONE;
break;
@@ -2776,6 +2780,12 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
case PROJECTOR_TYPE_SOPRANO:
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
break;
+ case PROJECTOR_TYPE_DRAMA3_GEN:
+ // codebook heads, same as the reference (sglang-omni fishaudio_drama3)
+ inp.top_k = 30;
+ inp.top_p = 0.9f;
+ inp.temp = 1.0f;
+ break;
case PROJECTOR_TYPE_LFM3A_GEN:
{
// MaskGIT sampling runs in the helper, with the settings of the checkpoint
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index a593e7889..27763563c 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -467,6 +467,7 @@ enum mtmd_gen_audio_type {
MTMD_GEN_AUDIO_TYPE_SOPRANO,
MTMD_GEN_AUDIO_TYPE_LFM3A,
MTMD_GEN_AUDIO_TYPE_CHATTERBOX,
+ MTMD_GEN_AUDIO_TYPE_DRAMA3,
};
struct mtmd_gen_audio_info {
diff --git a/tools/tts/README.md b/tools/tts/README.md
index 82c23473c..66c7af496 100644
--- a/tools/tts/README.md
+++ b/tools/tts/README.md
@@ -154,3 +154,34 @@ The upstream repositories ship raw checkpoints without `config.json`. Download t
python convert_hf_to_gguf.py path/to/chatterbox --outfile chatterbox.gguf
python convert_hf_to_gguf.py path/to/chatterbox --mmproj --outfile mmproj-chatterbox.gguf
```
+
+## Drama 3
+
+[Fish Audio Drama 3](https://huggingface.co/fishaudio/drama-3): a Qwen3.5-MoE backbone (35B, 3B active) samples each frame's main token, a codebook tower samples the 9 other codes, and the Fish DAC decodes them to 44.1 kHz audio.
+
+Available params:
+- Speaker tags `<|speaker:0|>` to `<|speaker:15|>` in the text give each character its own voice, a text without a tag starts with `<|speaker:0|>`
+- The language is the one of the text, `--tts-lang` is not used
+- Note: `--tts-speaker-file` is not supported yet (voice cloning)
+- Note: `--top-k` and `--top-p` are not used, frames are sampled with the settings of the reference (main token: temperature 1.0, top-p 0.9, repetition-aware redraw over the last 16 frames; other codes: temperature 1.0, top-k 30, top-p 0.9)
+- Each frame is 2048 samples (about 46 ms), generation stops at the end-of-speech token
+- The codebook tower keeps the whole sequence in its KV cache, the prompt and the generated frames are limited to 4096 positions
+
+Example usage:
+
+```sh
+llama-tts -m drama3.gguf \
+ -mm mmproj-drama3.gguf \
+ -p "<|speaker:0|> Did you remember to buy the bread? <|speaker:1|> Of course I did, it is on the kitchen table." \
+ -n 1000 \
+ --output out.wav
+```
+
+**Note for GGUF conversion:**
+
+The routed experts of the checkpoint are FP8, they are dequantized at conversion. The mmproj holds the codebook tower and the decoder half of `codec.pth`, it is best kept in BF16:
+
+```sh
+python convert_hf_to_gguf.py path/to/drama-3 --outtype q8_0 --outfile drama3.gguf
+python convert_hf_to_gguf.py path/to/drama-3 --mmproj --outtype bf16 --outfile mmproj-drama3.gguf
+```