Commit fc504f91d for llama.cpp
commit fc504f91d8cfdfcd9b0fedcf14600fd2b0e63420
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Sun Nov 15 21:25:34 2026 +0100
mtmd: support higgs v3 STT and vibevoice-ASR (#32329)
* mtmd: support higgs v3 and vibevoice-ASR
* use proper mapping
* address review comment
* nits
* rm hacky get_audio_length, add mtmd_helper_get_media_meta
diff --git a/common/chat.cpp b/common/chat.cpp
index e060362d2..b3c930ec4 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -827,6 +827,11 @@ static std::string string_lower(std::string s) {
return s;
}
+// VibeVoice-ASR: default system prompt of the template written by the converter
+static bool is_vibevoice_asr_template(const std::string & src) {
+ return src.find("transcribes audio input into text output in JSON format") != std::string::npos;
+}
+
// LFM3-Audio: language code -> name
static std::string lfm3_language_name(const std::string & language) {
static const std::map<std::string, std::string> names = {
@@ -847,7 +852,8 @@ static std::string lfm3_asr_system_prompt(const std::string & language) {
return "Perform ASR in " + lfm3_language_name(language) + ".";
}
-common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) {
+common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language,
+ double audio_length, const std::string & context) {
common_chat_prompt_preset asr_preset;
asr_preset.system = "";
asr_preset.user = "Transcribe audio to text";
@@ -861,6 +867,17 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates
asr_preset.system = lfm3_asr_system_prompt(language);
asr_preset.user = "";
asr_preset.has_language = true;
+ } else if (is_vibevoice_asr_template(src)) {
+ // without the audio length the model does not know where the audio ends and loops
+ const char * keys = "transcribe it with these keys: Start time, End time, Speaker ID, Content";
+ if (context.empty()) {
+ asr_preset.user = string_format("\nThis is a %.2f seconds audio, please %s", audio_length, keys);
+ } else {
+ asr_preset.user = string_format("\nThis is a %.2f seconds audio, with extra info: %s\n\nPlease %s", audio_length, context.c_str(), keys);
+ asr_preset.has_context = true;
+ }
+ asr_preset.media_first = true;
+ asr_preset.needs_audio_length = true;
}
}
diff --git a/common/chat.h b/common/chat.h
index 304ccf9ec..d66afaef7 100644
--- a/common/chat.h
+++ b/common/chat.h
@@ -491,10 +491,16 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
struct common_chat_prompt_preset {
std::string system;
std::string user;
- bool has_language = false; // the preset already names the requested language
+ bool has_language = false; // the preset already names the requested language
+ bool has_context = false; // the user text already holds the context
+ bool media_first = false; // the media goes before the user text
+ bool needs_audio_length = false; // the user text depends on the audio length, call again with it
};
-common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "");
+// audio_length: in seconds, 0 if unknown
+// context: extra info from the user (e.g. names, hotwords), only used by some presets, see has_context
+common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "",
+ double audio_length = 0.0, const std::string & context = "");
// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
// language and voice are optional, the template defines their defaults
diff --git a/conversion/__init__.py b/conversion/__init__.py
index e915753a8..2ec0551e8 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -138,6 +138,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"Grok1ForCausalLM": "grok",
"GrokForCausalLM": "grok",
"GroveMoeForCausalLM": "grovemoe",
+ "HiggsAudio3Model": "higgs_audio",
"HunYuanDenseV1ForCausalLM": "hunyuan",
"HunYuanMoEV1ForCausalLM": "hunyuan",
"HunYuanVLForConditionalGeneration": "hunyuan",
@@ -325,6 +326,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"UMT5Model": "t5",
"UltravoxModel": "ultravox",
"UnlimitedOCRForCausalLM": "deepseek",
+ "VibeVoiceAsrForConditionalGeneration": "vibevoice_asr",
"VLlama3ForCausalLM": "llama",
"VllmDecision2Model": "vllm_decision2",
"VoxtralForConditionalGeneration": "llama",
@@ -372,6 +374,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"Granite4VisionForConditionalGeneration": "granite",
"GraniteSpeechForConditionalGeneration": "granite",
"GraniteSpeechPlusForConditionalGeneration": "granite",
+ "HiggsAudio3Model": "higgs_audio",
"HunYuanVLForConditionalGeneration": "hunyuan",
"Idefics3ForConditionalGeneration": "smolvlm",
"Inkling2ForConditionalGeneration": "inkling2",
@@ -431,6 +434,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"Step3p7ForConditionalGeneration": "step3",
"UltravoxModel": "ultravox",
"UnlimitedOCRForCausalLM": "deepseek",
+ "VibeVoiceAsrForConditionalGeneration": "vibevoice_asr",
"VoxtralForConditionalGeneration": "ultravox",
"VoxtralRealtimeForConditionalGeneration": "ultravox",
"YoutuVLForConditionalGeneration": "youtuvl",
diff --git a/conversion/base.py b/conversion/base.py
index 7558fdcb4..2abd15c91 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -3475,7 +3475,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
# Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older
# Kimi-Linear-48B architecture and cannot load K3 (no attention residuals,
# latent MoE, situ, ...). Route on the top-level architecture instead.
- if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration"):
+ if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration", "HiggsAudio3Model"):
return arch
# if "architectures" is found in the sub-config, use that instead
diff --git a/conversion/higgs_audio.py b/conversion/higgs_audio.py
new file mode 100644
index 000000000..9e4d8b2b8
--- /dev/null
+++ b/conversion/higgs_audio.py
@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from typing import Any, Callable, TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import ModelBase, gguf
+from .qwen import Qwen3Model
+from .ultravox import WhisperEncoderModel
+
+
+# plain ChatML as built by transcribe.py upstream, the Qwen3 template renders past assistant turns differently from the last one
+HIGGS_AUDIO_CHAT_TEMPLATE = (
+ "{%- for message in messages -%}"
+ "{{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' -}}"
+ "{%- endfor -%}"
+ "{%- if add_generation_prompt -%}"
+ "{{- '<|im_start|>assistant\\n' -}}"
+ "{%- if enable_thinking is defined and enable_thinking is false -%}"
+ "{{- '<think>\\n\\n</think>\\n\\n' -}}"
+ "{%- endif -%}"
+ "{%- endif -%}"
+)
+
+
+@ModelBase.register("HiggsAudio3Model")
+@ModelBase.example("bosonai/higgs-audio-v3-stt")
+class HiggsAudioModel(Qwen3Model):
+ model_arch = gguf.MODEL_ARCH.QWEN3
+
+ def set_vocab(self):
+ super().set_vocab()
+ self.gguf_writer.add_chat_template(HIGGS_AUDIO_CHAT_TEMPLATE)
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, gen = item
+
+ # the audio_* tensors are dropped by TextModel, rename the text head before that
+ if name == "audio_decoder_proj.text_lm_head.weight":
+ name = "lm_head.weight"
+ else:
+ name = "model." + name
+
+ return super().filter_tensors((name, gen))
+
+
+@ModelBase.register("HiggsAudio3Model")
+class HiggsAudioWhisperEncoderModel(WhisperEncoderModel):
+ has_vision_encoder = False
+ has_audio_encoder = True
+
+ def get_audio_config(self) -> dict[str, Any] | None:
+ return self.global_config["audio_encoder_config"]
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ # depthwise conv kernel, same as the whisper convs
+ if name == "audio_encoder_proj.temporal.weight":
+ return gguf.GGMLQuantizationType.F16
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def set_gguf_parameters(self):
+ assert self.global_config["projector_type"] == "mlp" and self.global_config["projector_temporal_downsample"] == 2
+ assert self.global_config["audio_adapter_type"] == "stack" and not self.global_config.get("duplicate_audio_triplet", False)
+ super().set_gguf_parameters()
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.HIGGS_AUDIO)
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, _ = item
+ if not name.startswith(("audio_tower.", "audio_encoder_proj.")):
+ return None
+ return super().filter_tensors(item)
diff --git a/conversion/vibevoice_asr.py b/conversion/vibevoice_asr.py
new file mode 100644
index 000000000..f160c0e01
--- /dev/null
+++ b/conversion/vibevoice_asr.py
@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Iterable, TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import MmprojModel, ModelBase, gguf
+from .qwen import Qwen2Model
+
+
+# the upstream chat_template.jinja is a processor template (list content, audio parts), use plain ChatML with its default system prompt
+VIBEVOICE_ASR_CHAT_TEMPLATE = (
+ "{%- if messages[0]['role'] == 'system' -%}"
+ "{{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' -}}"
+ "{%- set loop_messages = messages[1:] -%}"
+ "{%- else -%}"
+ "{{- '<|im_start|>system\\nYou are a helpful assistant that transcribes audio input into text output in JSON format.<|im_end|>\\n' -}}"
+ "{%- set loop_messages = messages -%}"
+ "{%- endif -%}"
+ "{%- for message in loop_messages -%}"
+ "{{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' -}}"
+ "{%- endfor -%}"
+ "{%- if add_generation_prompt -%}"
+ "{{- '<|im_start|>assistant\\n' -}}"
+ "{%- endif -%}"
+)
+
+
+@ModelBase.register("VibeVoiceAsrForConditionalGeneration")
+@ModelBase.example("microsoft/VibeVoice-ASR-HF")
+class VibeVoiceAsrModel(Qwen2Model):
+ model_arch = gguf.MODEL_ARCH.QWEN2
+
+ def set_vocab(self):
+ super().set_vocab()
+ self.gguf_writer.add_chat_template(VIBEVOICE_ASR_CHAT_TEMPLATE)
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, _ = item
+ if not name.startswith("language_model."):
+ return None
+ return super().filter_tensors(item)
+
+
+@ModelBase.register("VibeVoiceAsrForConditionalGeneration")
+class VibeVoiceAsrAudioModel(MmprojModel):
+ has_vision_encoder = False
+ has_audio_encoder = True
+
+ # values hard-coded in tools/mtmd/models/vibevoice-asr.cpp
+ _encoder_consts = {"channels": 1, "kernel_size": 7, "num_filters": 32, "ffn_expansion": 4, "hidden_act": "gelu", "rms_norm_eps": 1e-5}
+ _encoders = ("acoustic_tokenizer_encoder", "semantic_tokenizer_encoder")
+
+ def get_audio_config(self) -> dict[str, Any] | None:
+ c = self.global_config["acoustic_tokenizer_encoder_config"]
+ return {
+ **c,
+ "intermediate_size": c["ffn_expansion"] * c["hidden_size"],
+ "num_attention_heads": 1,
+ "num_hidden_layers": sum(c["depths"]),
+ }
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ for enc in self._encoders:
+ c = self.global_config[f"{enc}_config"]
+ for k, v in self._encoder_consts.items():
+ assert c[k] == v, f"{enc}.{k} = {c[k]}, expected {v}"
+ assert c["depths"] == self.hparams["depths"] and c["downsampling_ratios"] == self.hparams["downsampling_ratios"]
+
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.VIBEVOICE_ASR)
+ # the input is the raw waveform, one channel takes the place of the mel bins
+ self.gguf_writer.add_audio_num_mel_bins(1)
+ self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams["rms_norm_eps"])
+ self.gguf_writer.add_audio_conv_kernel_size(self.hparams["kernel_size"])
+ # stride of each stage, the stem has none
+ self.gguf_writer.add_audio_patch_sizes([1] + [int(r) for r in self.hparams["downsampling_ratios"]])
+ self.gguf_writer.add_audio_depths([int(d) for d in self.hparams["depths"]])
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ if ".dw." in new_name:
+ return gguf.GGMLQuantizationType.F32 # ggml_ssm_conv
+ if ".conv." in new_name and new_name.endswith(".weight"):
+ return gguf.GGMLQuantizationType.F16
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, _ = item
+ if not name.startswith(cls._encoders + ("multi_modal_projector.",)):
+ return None
+ return super().filter_tensors(item)
+
+ _block_tensors = {
+ "norm.weight": gguf.MODEL_TENSOR.A_ENC_CNX_NORM,
+ "mixer.conv.weight": gguf.MODEL_TENSOR.A_ENC_CNX_DW,
+ "mixer.conv.bias": gguf.MODEL_TENSOR.A_ENC_CNX_DW,
+ "gamma": gguf.MODEL_TENSOR.A_ENC_CNX_LS1,
+ "ffn_norm.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_NORM,
+ "ffn.linear1.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_UP,
+ "ffn.linear1.bias": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_UP,
+ "ffn.linear2.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
+ "ffn.linear2.bias": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
+ "ffn_gamma": gguf.MODEL_TENSOR.A_ENC_CNX_LS2,
+ }
+
+ # mm.a.mlp.{0,1} + mm.a.mlp_norm.0 for the acoustic connector, {2,3} + 2 for the semantic one
+ _proj_tensors = {
+ "acoustic_linear_1": (gguf.MODEL_TENSOR.A_MMPROJ, 0),
+ "acoustic_norm": (gguf.MODEL_TENSOR.A_MMPROJ_NORM, 0),
+ "acoustic_linear_2": (gguf.MODEL_TENSOR.A_MMPROJ, 1),
+ "semantic_linear_1": (gguf.MODEL_TENSOR.A_MMPROJ, 2),
+ "semantic_norm": (gguf.MODEL_TENSOR.A_MMPROJ_NORM, 2),
+ "semantic_linear_2": (gguf.MODEL_TENSOR.A_MMPROJ, 3),
+ }
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # the layer scales (gamma) have no suffix upstream
+ suffix = "." + name.rsplit(".", 1)[-1] if name.endswith((".weight", ".bias")) else ".weight"
+
+ if name.startswith("multi_modal_projector."):
+ key, idx = self._proj_tensors[name.split(".")[1]]
+ yield (self.format_tensor_name(key, idx, suffix=suffix), data_torch)
+ return
+
+ enc, _, rest = name.partition(".")
+ enc_id = self._encoders.index(enc)
+ parts = rest.split(".")
+ depths = [int(d) for d in self.hparams["depths"]]
+
+ # causal convs of the stages: 0 = stem, 1..6 = downsample, 7 = head
+ if rest.startswith("stem.conv."):
+ conv_id = 0
+ elif parts[0] == "conv_layers" and parts[2] == "conv":
+ conv_id = int(parts[1]) + 1
+ elif parts[0] == "head":
+ conv_id = len(depths)
+ else:
+ conv_id = None
+ if conv_id is not None:
+ yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_CNX_CONV].format(bid=enc_id, xid=conv_id) + suffix, data_torch)
+ return
+
+ # ConvNeXt blocks are numbered per stage upstream and flat here
+ if parts[0] == "stem":
+ stage, blk, tail = 0, int(parts[2]), ".".join(parts[3:])
+ else:
+ assert parts[0] == "conv_layers" and parts[2] == "stage", name
+ stage, blk, tail = int(parts[1]) + 1, int(parts[3]), ".".join(parts[4:])
+ xid = sum(depths[:stage]) + blk
+
+ key = self._block_tensors[tail]
+ if key == gguf.MODEL_TENSOR.A_ENC_CNX_DW and tail.endswith(".weight"):
+ data_torch = data_torch.squeeze(1) # [C, 1, K] -> [C, K] for ggml_ssm_conv
+ yield (gguf.TENSOR_NAMES[key].format(bid=enc_id, xid=xid) + suffix, data_torch)
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index c9b27a11f..037e1332a 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -1386,6 +1386,14 @@ class MODEL_TENSOR(IntEnum):
A_ENC_SEANET_RES_CONV1 = auto() # residual unit, dilated conv
A_ENC_SEANET_RES_CONV2 = auto() # residual unit, pointwise conv
A_ENC_SEANET_SCALE_CONV = auto() # strided downsample conv
+ A_ENC_CNX_CONV = auto() # vibevoice: causal conv of a stage (stem, downsample, head), bid = encoder
+ A_ENC_CNX_NORM = auto() # vibevoice: ConvNeXt block, bid = encoder, xid = block
+ A_ENC_CNX_DW = auto()
+ A_ENC_CNX_LS1 = auto()
+ A_ENC_CNX_FFN_NORM = auto()
+ A_ENC_CNX_FFN_UP = auto()
+ A_ENC_CNX_FFN_DOWN = auto()
+ A_ENC_CNX_LS2 = auto()
A_ENC_ATTN_SCALE = auto() # layer scale (gamma) on the attn output
A_ENC_FFN_SCALE_LS = auto() # layer scale (gamma) on the FFN output
A_ENC_SPEAKER_PROJ = auto() # voice latent -> backbone embd
@@ -1419,6 +1427,7 @@ class MODEL_TENSOR(IntEnum):
A_GEN_WAV_OUTPUT = auto()
A_MMPROJ = auto()
A_MMPROJ_FC = auto()
+ A_MMPROJ_NORM = auto() # vibevoice: norm after mm.a.mlp.{bid}
A_MM_NORM_PRE = auto()
A_MM_NORM_MID = auto()
A_MM_EMBEDDING = auto() # gemma3n
@@ -2317,6 +2326,14 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_ENC_SEANET_RES_CONV1: "a.seanet.blk.{bid}.res_conv1",
MODEL_TENSOR.A_ENC_SEANET_RES_CONV2: "a.seanet.blk.{bid}.res_conv2",
MODEL_TENSOR.A_ENC_SEANET_SCALE_CONV: "a.seanet.blk.{bid}.scale_conv",
+ MODEL_TENSOR.A_ENC_CNX_CONV: "a.cnx.{bid}.conv.{xid}",
+ MODEL_TENSOR.A_ENC_CNX_NORM: "a.cnx.{bid}.blk.{xid}.norm",
+ MODEL_TENSOR.A_ENC_CNX_DW: "a.cnx.{bid}.blk.{xid}.dw",
+ MODEL_TENSOR.A_ENC_CNX_LS1: "a.cnx.{bid}.blk.{xid}.ls1",
+ MODEL_TENSOR.A_ENC_CNX_FFN_NORM: "a.cnx.{bid}.blk.{xid}.ffn_norm",
+ MODEL_TENSOR.A_ENC_CNX_FFN_UP: "a.cnx.{bid}.blk.{xid}.ffn_up",
+ MODEL_TENSOR.A_ENC_CNX_FFN_DOWN: "a.cnx.{bid}.blk.{xid}.ffn_down",
+ MODEL_TENSOR.A_ENC_CNX_LS2: "a.cnx.{bid}.blk.{xid}.ls2",
MODEL_TENSOR.A_ENC_ATTN_SCALE: "a.blk.{bid}.ls1",
MODEL_TENSOR.A_ENC_FFN_SCALE_LS: "a.blk.{bid}.ls2",
MODEL_TENSOR.A_ENC_SPEAKER_PROJ: "a.speaker_proj",
@@ -2349,6 +2366,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_GEN_WAV_OUTPUT: "a.gen.wav.output",
MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}",
MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc",
+ MODEL_TENSOR.A_MMPROJ_NORM: "mm.a.mlp_norm.{bid}",
MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre",
MODEL_TENSOR.A_MM_NORM_MID: "mm.a.norm_mid",
MODEL_TENSOR.A_MM_INP_PROJ: "mm.a.input_projection", # gemma3n
@@ -2641,6 +2659,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.A_ENC_RVQ_CODEBOOK,
MODEL_TENSOR.A_MMPROJ,
MODEL_TENSOR.A_MMPROJ_FC,
+ MODEL_TENSOR.A_MMPROJ_NORM,
MODEL_TENSOR.A_MM_NORM_PRE,
MODEL_TENSOR.A_MM_NORM_MID,
MODEL_TENSOR.A_MM_CODE_EMBD,
@@ -2741,6 +2760,14 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.A_ENC_SEANET_RES_CONV1,
MODEL_TENSOR.A_ENC_SEANET_RES_CONV2,
MODEL_TENSOR.A_ENC_SEANET_SCALE_CONV,
+ MODEL_TENSOR.A_ENC_CNX_CONV,
+ MODEL_TENSOR.A_ENC_CNX_NORM,
+ MODEL_TENSOR.A_ENC_CNX_DW,
+ MODEL_TENSOR.A_ENC_CNX_LS1,
+ MODEL_TENSOR.A_ENC_CNX_FFN_NORM,
+ MODEL_TENSOR.A_ENC_CNX_FFN_UP,
+ MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
+ MODEL_TENSOR.A_ENC_CNX_LS2,
MODEL_TENSOR.A_ENC_ATTN_SCALE,
MODEL_TENSOR.A_ENC_FFN_SCALE_LS,
MODEL_TENSOR.A_ENC_SPEAKER_PROJ,
@@ -7155,6 +7182,8 @@ class VisionProjectorType:
QWEN2A = "qwen2a" # audio
QWEN3A = "qwen3a" # audio
GLMA = "glma" # audio
+ HIGGS_AUDIO = "higgs_audio" # audio
+ VIBEVOICE_ASR = "vibevoice_asr" # audio
QWEN25O = "qwen2.5o" # omni
VOXTRAL = "voxtral"
VOXTRAL_RT = "voxtral_rt"
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index 4ee516afa..61e4e7ddc 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -2973,6 +2973,7 @@ class TensorNameMap:
MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: (
"encoder.down_sample_layer.0", # mimo-audio-tokenizer
+ "audio_encoder_proj.temporal", # higgs-audio
),
MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: (
@@ -3026,6 +3027,7 @@ class TensorNameMap:
"mlp.model.{bid}", # lfm3a
"audio_tower.proj{bid}", # qwen3omni
"sound_projection.linear{bid}", # parakeet (linear1, linear2)
+ "audio_encoder_proj.linear{bid}", # higgs-audio (linear1, linear2)
),
MODEL_TENSOR.A_MMPROJ_FC: (
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index 32a17ef61..b23fd5e6b 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -63,6 +63,7 @@ add_library(mtmd
models/nemotron-transcribe.cpp
models/muse-glimmer.cpp
models/muse-spark-audio.cpp
+ models/vibevoice-asr.cpp
models/llama4.cpp
models/llava.cpp
models/minicpmv.cpp
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 05494c71f..a8e6120da 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -268,6 +268,18 @@ struct mtmd_graph_granite_speech : mtmd_graph {
ggml_cgraph * build() override;
};
+struct mtmd_graph_vibevoice_asr : mtmd_graph {
+ mtmd_graph_vibevoice_asr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
+ ggml_cgraph * build() override;
+
+ std::map<std::string, ggml_tensor *> state_in;
+ mutable std::vector<std::pair<std::string, ggml_tensor *>> state_out;
+
+ ggml_tensor * with_left_ctx(ggml_tensor * x, const std::string & name) const;
+ ggml_tensor * causal_conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & name) const;
+ ggml_tensor * encode(int e, ggml_tensor * x) const;
+};
+
struct mtmd_graph_muse_spark_audio : mtmd_graph {
mtmd_graph_muse_spark_audio(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
@@ -643,6 +655,9 @@ std::vector<c2w_state_slot> list_lfm3a_state_slots(const mtmd_hparams & hparams,
// state slots of the streaming Gemma 4 audio encoder (gemma4a)
std::vector<c2w_state_slot> list_gemma4a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
+// vibevoice-asr: the left context of every causal conv, a zero state is the causal zero padding
+std::vector<c2w_state_slot> list_vibevoice_asr_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
+
struct mtmd_graph_kimik25 : mtmd_graph {
mtmd_graph_kimik25(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/models/vibevoice-asr.cpp b/tools/mtmd/models/vibevoice-asr.cpp
new file mode 100644
index 000000000..1158988cb
--- /dev/null
+++ b/tools/mtmd/models/vibevoice-asr.cpp
@@ -0,0 +1,138 @@
+#include "models.h"
+
+// VibeVoice-ASR audio tower: two causal ConvNeXt pyramids (acoustic and semantic) over the raw 24 kHz waveform
+// each one is a VAE tokenizer encoder, its latents go through their own connector and the two are summed
+// see modeling_vibevoice_acoustic_tokenizer.py and modeling_vibevoice_asr.py in transformers
+//
+// tensors are T-first [T, C] around the convs and C-first [C, T] in the blocks
+
+static std::string cnx_slot(int e, const char * kind, int i) {
+ return "e" + std::to_string(e) + "_" + kind + "_" + std::to_string(i);
+}
+
+std::vector<c2w_state_slot> list_vibevoice_asr_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
+ std::vector<c2w_state_slot> slots;
+ slots.push_back({"pos", 1, 1}); // unused, set_enc_state_in() expects it
+ const int n_stage = (int) hparams.audio_patch_sizes.size();
+ for (int e = 0; e < 2; ++e) {
+ const auto & enc = model.cnx_enc[e];
+ for (int i = 0; i <= n_stage; ++i) {
+ const int64_t stride = i < n_stage ? hparams.audio_patch_sizes[i] : 1;
+ slots.push_back({cnx_slot(e, "conv", i), enc.conv_w[i]->ne[0] - stride, enc.conv_w[i]->ne[1]});
+ }
+ for (int il = 0; il < (int) enc.blocks.size(); ++il) {
+ const auto * w = enc.blocks[il].conv_w;
+ slots.push_back({cnx_slot(e, "dw", il), w->ne[0] - 1, w->ne[1]});
+ }
+ }
+ return slots;
+}
+
+// x: [T, C] -> [left + T, C], the left context comes from the state, its new value is the tail
+ggml_tensor * mtmd_graph_vibevoice_asr::with_left_ctx(ggml_tensor * x, const std::string & name) const {
+ ggml_tensor * left = state_in.at(name);
+ x = ggml_concat(ctx0, left, x, 0);
+ state_out.push_back({name, ggml_cont(ctx0, ggml_view_2d(ctx0, x, left->ne[0], x->ne[1], x->nb[1], (x->ne[0] - left->ne[0]) * x->nb[0]))});
+ return x;
+}
+
+// x: [T, IC], w: [K, IC, OC] -> [T / stride, OC]
+ggml_tensor * mtmd_graph_vibevoice_asr::causal_conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & name) const {
+ x = with_left_ctx(x, name);
+ ggml_tensor * y = ggml_conv_1d(ctx0, w, x, stride, 0, 1);
+ y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[1]);
+ return ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
+}
+
+// x: [T, 1] -> [C_latent, T / hop]
+ggml_tensor * mtmd_graph_vibevoice_asr::encode(int e, ggml_tensor * x) const {
+ const auto & enc = model.cnx_enc[e];
+ const int n_stage = (int) hparams.audio_patch_sizes.size();
+
+ ggml_tensor * cur = x;
+ int bid = 0;
+ for (int stage = 0; stage < n_stage; ++stage) {
+ cur = causal_conv(cur, enc.conv_w[stage], enc.conv_b[stage], hparams.audio_patch_sizes[stage], cnx_slot(e, "conv", stage));
+ cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+ cb(cur, "stage_in", stage);
+
+ const int64_t n_pos = cur->ne[1];
+ for (int i = 0; i < hparams.audio_depths[stage]; ++i, ++bid) {
+ const auto & layer = enc.blocks[bid];
+
+ // causal depthwise conv mixer
+ ggml_tensor * h = build_norm(cur, layer.ln_1_w, nullptr, NORM_TYPE_RMS, hparams.eps, bid);
+ h = ggml_cont(ctx0, ggml_transpose(ctx0, h));
+ h = with_left_ctx(h, cnx_slot(e, "dw", bid));
+ h = ggml_reshape_3d(ctx0, h, h->ne[0], h->ne[1], 1);
+ h = ggml_ssm_conv(ctx0, h, layer.conv_w);
+ h = ggml_reshape_2d(ctx0, h, h->ne[0], n_pos);
+ h = ggml_add(ctx0, h, layer.conv_b);
+ cur = ggml_add(ctx0, cur, ggml_mul(ctx0, h, layer.ls_1_w));
+
+ h = build_norm(cur, layer.ln_2_w, nullptr, NORM_TYPE_RMS, hparams.eps, bid);
+ h = build_ffn(h,
+ layer.ff_up_w, layer.ff_up_b,
+ nullptr, nullptr,
+ layer.ff_down_w, layer.ff_down_b,
+ FFN_GELU_ERF, bid);
+ cur = ggml_add(ctx0, cur, ggml_mul(ctx0, h, layer.ls_2_w));
+ cb(cur, "block_out", bid);
+ }
+
+ cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+ }
+
+ cur = causal_conv(cur, enc.conv_w[n_stage], enc.conv_b[n_stage], 1, cnx_slot(e, "conv", n_stage));
+ return ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+}
+
+ggml_cgraph * mtmd_graph_vibevoice_asr::build() {
+ const int64_t hop = hparams.audio_samples_per_token();
+ const int64_t n_frames = img.nx() / hop;
+ const int64_t n_ctx = img.audio_n_ctx / hop;
+ GGML_ASSERT(img.nx() % hop == 0 && img.audio_n_ctx % hop == 0 && n_ctx < n_frames);
+
+ for (const auto & slot : list_vibevoice_asr_state_slots(hparams, model)) {
+ ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
+ ggml_set_name(t, ("state_in_" + slot.name).c_str());
+ ggml_set_input(t);
+ state_in[slot.name] = t;
+ }
+ state_out.push_back({"pos", ggml_scale_bias(ctx0, state_in.at("pos"), 1.0f, (float) n_frames)});
+
+ ggml_tensor * inp = build_inp_raw(1);
+ inp = ggml_reshape_2d(ctx0, inp, img.nx(), 1);
+
+ // the leading context frames only warm up the causal convs
+ auto drop_ctx = [&](ggml_tensor * t) {
+ return ggml_view_2d(ctx0, t, t->ne[0], n_frames - n_ctx, t->nb[1], n_ctx * t->nb[1]);
+ };
+
+ // the reference adds VAE noise to the acoustic latents, skipped here to keep the output deterministic
+ ggml_tensor * acoustic = drop_ctx(encode(0, inp));
+ ggml_tensor * semantic = drop_ctx(encode(1, inp));
+ cb(acoustic, "acoustic_latents", -1);
+ cb(semantic, "semantic_latents", -1);
+
+ acoustic = ggml_add(ctx0, build_mm(model.mm_0_w, acoustic), model.mm_0_b);
+ acoustic = build_norm(acoustic, model.mm_norm_pre_w, nullptr, NORM_TYPE_RMS, 1e-6f, -1);
+ acoustic = ggml_add(ctx0, build_mm(model.mm_1_w, acoustic), model.mm_1_b);
+
+ semantic = ggml_add(ctx0, build_mm(model.mm_2_w, semantic), model.mm_2_b);
+ semantic = build_norm(semantic, model.mm_norm_mid_w, nullptr, NORM_TYPE_RMS, 1e-6f, -1);
+ semantic = ggml_add(ctx0, build_mm(model.mm_3_w, semantic), model.mm_3_b);
+
+ ggml_tensor * cur = ggml_add(ctx0, acoustic, semantic);
+ cb(cur, "projected", -1);
+
+ // the embeddings must stay the last node
+ for (const auto & s : state_out) {
+ ggml_set_name(s.second, ("state_out_" + s.first).c_str());
+ ggml_set_output(s.second);
+ ggml_build_forward_expand(gf, s.second);
+ }
+
+ ggml_build_forward_expand(gf, cur);
+ return gf;
+}
diff --git a/tools/mtmd/models/whisper-enc.cpp b/tools/mtmd/models/whisper-enc.cpp
index 6225b707c..e755990d5 100644
--- a/tools/mtmd/models/whisper-enc.cpp
+++ b/tools/mtmd/models/whisper-enc.cpp
@@ -2,7 +2,7 @@
ggml_cgraph * mtmd_graph_whisper_enc::build() {
const int n_frames = img.nx();
- const int n_pos = n_frames / 2;
+ const int n_pos = (n_frames + 1) / 2; // stride-2 conv1d with padding 1
GGML_ASSERT(model.position_embeddings->ne[1] >= n_pos);
ggml_tensor * inp = build_inp_raw(1);
@@ -77,6 +77,19 @@ ggml_cgraph * mtmd_graph_whisper_enc::build() {
cur = build_mm(model.mm_fc_w, cur);
cur = ggml_add(ctx0, cur, model.mm_fc_b);
+ } else if (proj_type == PROJECTOR_TYPE_HIGGS_AUDIO) {
+ // depthwise conv (k=3, s=2, p=1) over time, then linear-relu-linear
+ cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+ cur = ggml_conv_1d_dw(ctx0, model.downsample_conv_w, cur, 2, 1, 1);
+ cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+ cur = ggml_add(ctx0, cur, model.downsample_conv_b);
+
+ cur = build_mm(model.mm_1_w, cur);
+ cur = ggml_add(ctx0, cur, model.mm_1_b);
+ cur = ggml_relu(ctx0, cur);
+ cur = build_mm(model.mm_2_w, cur);
+ cur = ggml_add(ctx0, cur, model.mm_2_b);
+
} else if (proj_type == PROJECTOR_TYPE_VOXTRAL) {
// projector
cur = build_ffn(cur,
diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp
index c6d751b41..dfd3b30e7 100644
--- a/tools/mtmd/mtmd-audio.cpp
+++ b/tools/mtmd/mtmd-audio.cpp
@@ -277,6 +277,7 @@ static void mtmd_audio_fill_entries(std::vector<mtmd_audio_mel> & mels, bool sha
GGML_ASSERT(mel.n_mel <= INT32_MAX && mel.n_mel >= 0);
mtmd_image_f32 entry;
entry.set_size({(int) mel.n_len, (int) mel.n_mel}, shape_only || mel.data.empty(), /* is_audio */ true);
+ entry.audio_n_ctx = (int32_t) mel.n_ctx;
if (!entry.is_placeholder()) {
entry.cpy_buf(mel.data);
}
@@ -688,7 +689,7 @@ bool mtmd_audio_preprocessor_qwen3a::process(const float * sampl
//
// mtmd_audio_preprocessor_dots3note
//
-// Matches Dots3NoteFeatureExtractor: the waveform is split into 60s chunks and each chunk gets
+// Matches Dots3NoteFeatureExtractor: the waveform is split into audio_chunk_len chunks and each chunk gets
// its own whisper-style log-mel (center=True, log10 + (max-8)/4). Only sample_length//hop frames
// per chunk are valid; the reference masks everything beyond them, so we emit exactly that many.
//
@@ -733,7 +734,7 @@ bool mtmd_audio_preprocessor_dots3note::process(const float * sa
continue; // sub-hop tail, contributes no frames
}
- // reflect-pad the start; the reference zero-pads partial chunks to 60s before the STFT,
+ // reflect-pad the start; the reference zero-pads partial chunks to the chunk length before the STFT,
// so a partial chunk sees zeros past its end while a full chunk reflects its own tail
std::vector<float> padded(n_chunk + 2 * pad, 0.0f);
for (int i = 0; i < pad; i++) {
@@ -780,6 +781,20 @@ bool mtmd_audio_preprocessor_dots3note::process(const float * sa
return !output.empty();
}
+bool mtmd_audio_preprocessor_higgs_audio::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool shape_only) const {
+ if (!mtmd_audio_preprocessor_dots3note::process(samples, n_samples, output, shape_only)) {
+ return false;
+ }
+ // a tail under 3 frames gives no token, the reference drops it too
+ if (output.size() > 1 && output.back().n_len < 3) {
+ output.pop_back();
+ }
+ return !output.empty() && output.back().n_len >= 3;
+}
+
//
// mtmd_audio_preprocessor_mimo_audio
//
@@ -2227,6 +2242,65 @@ bool mtmd_audio_preprocessor_pockettts::process(const float * sa
return true;
}
+//
+// mtmd_audio_preprocessor_vibevoice_asr
+//
+// Matches VibeVoiceAcousticTokenizerFeatureExtractor: the clip is scaled to -25 dBFS RMS, then zero-padded to a whole number of frames.
+// The reference encodes it with a streaming conv cache, here each chunk re-reads audio_n_ctx samples before its start instead, which gives the same output.
+//
+
+double mtmd_audio_vibevoice_gain(double sum_sq, int64_t n_samples, double peak) {
+ const double rms = std::max(std::sqrt(sum_sq / std::max<int64_t>(n_samples, 1)), std::pow(10.0, -50.0 / 20.0));
+ double gain = std::pow(10.0, -25.0 / 20.0) / (rms + 1e-6);
+ if (peak * gain > 1.0) {
+ gain /= peak * gain + 1e-6;
+ }
+ return gain;
+}
+
+bool mtmd_audio_preprocessor_vibevoice_asr::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool /*shape_only*/) const {
+ if (n_samples == 0) {
+ return false;
+ }
+
+ const int64_t hop = hparams.audio_samples_per_token();
+ const int64_t n_ctx = hparams.audio_n_ctx;
+ const int64_t n_chunk = (int64_t) hparams.audio_chunk_len * hparams.audio_sample_rate / hop * hop;
+ const int64_t n_padded = ((int64_t) n_samples + hop - 1) / hop * hop;
+ GGML_ASSERT(n_chunk > 0 && n_ctx % hop == 0);
+
+ double sum_sq = 0.0;
+ double peak = 0.0;
+ for (size_t i = 0; i < n_samples; i++) {
+ sum_sq += (double) samples[i] * samples[i];
+ peak = std::max(peak, (double) std::fabs(samples[i]));
+ }
+ const double scale = mtmd_audio_vibevoice_gain(sum_sq, (int64_t) n_samples, peak);
+
+ std::vector<float> audio((size_t) n_padded, 0.0f);
+ for (size_t i = 0; i < n_samples; i++) {
+ audio[i] = (float) (samples[i] * scale);
+ }
+
+ output.clear();
+ for (int64_t start = 0; start < n_padded; start += n_chunk) {
+ const int64_t beg = std::max<int64_t>(0, start - n_ctx);
+ const int64_t end = std::min(n_padded, start + n_chunk);
+
+ mtmd_audio_mel mel;
+ mel.n_len = end - beg;
+ mel.n_len_org = end - beg;
+ mel.n_mel = 1;
+ mel.n_ctx = start - beg;
+ mel.data.assign(audio.begin() + beg, audio.begin() + end);
+ output.push_back(std::move(mel));
+ }
+ return true;
+}
+
// Muse Spark reads the raw waveform: pass the samples through, zero padded to a whole number of embeddings
bool mtmd_audio_preprocessor_muse_spark_audio::process(
const float * samples,
diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h
index 937451c8a..b4b1ce692 100644
--- a/tools/mtmd/mtmd-audio.h
+++ b/tools/mtmd/mtmd-audio.h
@@ -14,6 +14,7 @@ struct mtmd_audio_mel {
int64_t n_len;
int64_t n_len_org;
int64_t n_mel;
+ int64_t n_ctx = 0; // see mtmd_image_f32::audio_n_ctx
std::vector<float> data;
};
@@ -128,6 +129,17 @@ struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor {
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
+// vibevoice: gain to -25 dBFS RMS, without clipping the peak
+// the RMS is floored at -50 dBFS, so that silence or noise alone is not boosted to full scale
+double mtmd_audio_vibevoice_gain(double sum_sq, int64_t n_samples, double peak);
+
+// raw 24 kHz waveform, cut in chunks that carry their causal context, see mtmd_hparams::audio_n_ctx
+struct mtmd_audio_preprocessor_vibevoice_asr : mtmd_audio_preprocessor {
+ mtmd_audio_preprocessor_vibevoice_asr(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
+ void initialize() override {}
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
+};
+
struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_conformer(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
mtmd_audio_preprocessor_conformer(const mtmd_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {}
@@ -198,6 +210,12 @@ struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
mtmd_audio_cache cache;
};
+// same per-chunk log-mel as dots3note, with 4s chunks
+struct mtmd_audio_preprocessor_higgs_audio : mtmd_audio_preprocessor_dots3note {
+ mtmd_audio_preprocessor_higgs_audio(const mtmd_model * model) : mtmd_audio_preprocessor_dots3note(model) {}
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
+};
+
struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_mimo_audio(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp
index ac656032d..100bc3c58 100644
--- a/tools/mtmd/mtmd-graph.cpp
+++ b/tools/mtmd/mtmd-graph.cpp
@@ -870,6 +870,7 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_QWEN2A:
+ case PROJECTOR_TYPE_HIGGS_AUDIO:
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_MERALION:
case PROJECTOR_TYPE_MUSIC_FLAMINGO:
@@ -880,6 +881,10 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
{
builder = std::make_unique<mtmd_graph_muse_spark_audio>(ctx, img);
} break;
+ case PROJECTOR_TYPE_VIBEVOICE_ASR:
+ {
+ builder = std::make_unique<mtmd_graph_vibevoice_asr>(ctx, img);
+ } break;
case PROJECTOR_TYPE_KIMIVL:
{
builder = std::make_unique<mtmd_graph_kimivl>(ctx, img);
diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp
index e14476788..bce73a74e 100644
--- a/tools/mtmd/mtmd-helper.cpp
+++ b/tools/mtmd/mtmd-helper.cpp
@@ -18,6 +18,7 @@
#include <algorithm>
#include <cmath>
#include <cinttypes>
+#include <climits>
#include <vector>
//#define MTMD_AUDIO_DEBUG
@@ -939,6 +940,54 @@ static void mtmd_helper_video_set_id(mtmd_helper_video * vctx, const std::string
vctx->id = id;
}
+int32_t mtmd_helper_get_media_meta(const unsigned char * buf, size_t len, mtmd_helper_init_opt opt, mtmd_helper_media_meta * out) {
+ *out = {MTMD_HELPER_MEDIA_TYPE_UNKNOWN, 0, 0, 0, -1.0f, -1};
+
+ if (audio_helpers::is_audio_file((const char *) buf, len)) {
+ // zero channels and rate: the source format, so the length comes from the header (or an mp3 frame scan)
+ ma_decoder_config cfg = ma_decoder_config_init(ma_format_f32, 0, 0);
+ ma_decoder dec;
+ if (ma_decoder_init_memory(buf, len, &cfg, &dec) != MA_SUCCESS) {
+ return 1;
+ }
+ ma_uint64 n = 0;
+ if (ma_decoder_get_length_in_pcm_frames(&dec, &n) == MA_SUCCESS && dec.outputSampleRate > 0) {
+ out->duration = (float) ((double) n / dec.outputSampleRate);
+ }
+ out->type = MTMD_HELPER_MEDIA_TYPE_AUDIO;
+ out->sample_rate = (int32_t) dec.outputSampleRate;
+ ma_decoder_uninit(&dec);
+ return 0;
+ }
+
+ int nx, ny, nc;
+ if (len <= INT_MAX && stbi_info_from_memory(buf, (int) len, &nx, &ny, &nc)) {
+ out->type = MTMD_HELPER_MEDIA_TYPE_IMAGE;
+ out->width = (uint32_t) nx;
+ out->height = (uint32_t) ny;
+ return 0;
+ }
+
+ // webp (not read by stb_image) and video go through the video decoder
+ mtmd_helper_video_init_params params = opt.video_params;
+ const bool is_webp = is_webp_file(buf, len);
+ if (is_webp && video_builtin_decoder()) {
+ params.decoder = nullptr; // see decode_still_image()
+ }
+ mtmd_helper_video_stream_info src;
+ if (!video_open(nullptr, buf, len, params, src)) {
+ return 1;
+ }
+ out->type = is_webp ? MTMD_HELPER_MEDIA_TYPE_IMAGE : MTMD_HELPER_MEDIA_TYPE_VIDEO;
+ out->width = src.width;
+ out->height = src.height;
+ if (!is_webp) {
+ out->duration = src.duration;
+ out->n_frames = src.n_frames;
+ }
+ return 0;
+}
+
mtmd_helper_video_init_params mtmd_helper_video_init_params_default() {
return {
/* fps_target */ 4.0f,
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index 5ca414906..b33ed65ec 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -107,6 +107,32 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(
bool placeholder,
struct mtmd_helper_init_opt opt);
+enum mtmd_helper_media_type {
+ MTMD_HELPER_MEDIA_TYPE_UNKNOWN,
+ MTMD_HELPER_MEDIA_TYPE_IMAGE,
+ MTMD_HELPER_MEDIA_TYPE_AUDIO,
+ MTMD_HELPER_MEDIA_TYPE_VIDEO,
+};
+
+struct mtmd_helper_media_meta {
+ enum mtmd_helper_media_type type;
+ uint32_t width; // image, video (frame size)
+ uint32_t height;
+ int32_t sample_rate; // audio, at the source rate
+ float duration; // audio, video: in seconds, -1 if unknown
+ int32_t n_frames; // video: -1 if unknown
+};
+typedef struct mtmd_helper_media_meta mtmd_helper_media_meta;
+
+// metadata of a media file in a buffer, the same formats as mtmd_helper_bitmap_init_from_buf()
+// only the headers are read (ffprobe or opt.video_params.decoder for video), nothing is decoded
+// returns 0 on success, 1 if the format is not recognized
+// this function is thread-safe
+MTMD_API int32_t mtmd_helper_get_media_meta(
+ const unsigned char * buf, size_t len,
+ struct mtmd_helper_init_opt opt,
+ struct mtmd_helper_media_meta * out);
+
// streaming resampler for raw mono F32 PCM (e.g. realtime input), to match mtmd_get_audio_sample_rate()
// it keeps state between calls, use one per stream
struct mtmd_helper_resampler;
diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h
index 0d287dcdd..70db37654 100644
--- a/tools/mtmd/mtmd-impl.h
+++ b/tools/mtmd/mtmd-impl.h
@@ -415,6 +415,17 @@
#define TN_A_CONV1D_NORM "a.conv1d.%d.norm.%s"
#define TN_A_PATCH "a.patch.%d.%s"
#define TN_A_PATCH_NORM "a.patch.%d.norm.%s"
+
+// vibevoice-asr: causal ConvNeXt encoders, first index = encoder (0 = acoustic, 1 = semantic)
+#define TN_A_CNX_CONV "a.cnx.%d.conv.%d.%s"
+#define TN_A_CNX_NORM "a.cnx.%d.blk.%d.norm.%s"
+#define TN_A_CNX_DW "a.cnx.%d.blk.%d.dw.%s"
+#define TN_A_CNX_LS1 "a.cnx.%d.blk.%d.ls1.%s"
+#define TN_A_CNX_FFN_NORM "a.cnx.%d.blk.%d.ffn_norm.%s"
+#define TN_A_CNX_FFN_UP "a.cnx.%d.blk.%d.ffn_up.%s"
+#define TN_A_CNX_FFN_DOWN "a.cnx.%d.blk.%d.ffn_down.%s"
+#define TN_A_CNX_LS2 "a.cnx.%d.blk.%d.ls2.%s"
+#define TN_MM_AUDIO_MLP_NORM "mm.a.mlp_norm.%d.%s"
#define TN_A_OUT_PROJ "a.pre_encode.out.%s"
#define TN_A_ATTN_PRE_NORM "%s.blk.%d.attn_pre_norm.%s"
#define TN_A_ATTN_POST_NORM "%s.blk.%d.attn_post_norm.%s"
@@ -522,6 +533,8 @@ enum projector_type {
PROJECTOR_TYPE_QWEN2A,
PROJECTOR_TYPE_QWEN3A,
PROJECTOR_TYPE_GLMA,
+ PROJECTOR_TYPE_HIGGS_AUDIO,
+ PROJECTOR_TYPE_VIBEVOICE_ASR,
PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on mtmd_subcontext
PROJECTOR_TYPE_VOXTRAL,
PROJECTOR_TYPE_VOXTRAL_RT,
@@ -612,6 +625,8 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_QWEN2A, "qwen2a"},
{ PROJECTOR_TYPE_QWEN3A, "qwen3a"},
{ PROJECTOR_TYPE_GLMA, "glma"},
+ { PROJECTOR_TYPE_HIGGS_AUDIO, "higgs_audio"},
+ { PROJECTOR_TYPE_VIBEVOICE_ASR, "vibevoice_asr"},
{ PROJECTOR_TYPE_QWEN25O, "qwen2.5o"},
{ PROJECTOR_TYPE_VOXTRAL, "voxtral"},
{ PROJECTOR_TYPE_VOXTRAL_RT, "voxtral_rt"},
@@ -815,6 +830,8 @@ struct mtmd_image_f32 {
// deepseek4v: number of leading IMAGE_PAD embeddings, aligns IMAGE_START to the LLM compressor ratio
// depends on the chunk position, set at tokenize time (see mtmd_tokenizer::add_media)
int32_t lead_pad = 0;
+ // audio (vibevoice): leading samples that only feed the causal convs, they produce no embedding
+ int32_t audio_n_ctx = 0;
// separators appended after the image tokens
mtmd_suffix_type suffix_type = MTMD_SUFFIX_NONE;
diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp
index c4c52a4d5..86bb127fb 100644
--- a/tools/mtmd/mtmd-model.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -785,6 +785,46 @@ struct mtmd_model_loader {
}
hparams.warmup_audio_size = hparams.audio_samples_per_token();
} break;
+ case PROJECTOR_TYPE_VIBEVOICE_ASR:
+ {
+ hparams.audio_sample_rate = 24000; // mono waveform, no frontend
+ hparams.audio_chunk_len = 30; // in seconds, any length works as the convs are causal
+ get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size, true);
+ get_arr_int(KEY_A_PATCH_SIZES, hparams.audio_patch_sizes, true);
+ get_arr_int(KEY_A_DEPTHS, hparams.audio_depths, true);
+
+ // bounds keep the receptive field below from overflowing
+ const size_t n_stage = hparams.audio_patch_sizes.size();
+ if (n_stage == 0 || n_stage > 16 || hparams.audio_depths.size() != n_stage || hparams.n_mel_bins != 1 ||
+ hparams.audio_conv_kernel_size < 1 || hparams.audio_conv_kernel_size > 64) {
+ throw std::runtime_error(mtmd_format("%s: invalid vibevoice audio hparams\n", __func__));
+ }
+
+ // receptive field: stem conv (kernel), downsample convs (2 * stride), blocks (kernel), head conv (kernel)
+ const int64_t k = hparams.audio_conv_kernel_size;
+ int64_t hop = 1;
+ int64_t rf = 0;
+ int64_t n_blocks = 0;
+ for (size_t i = 0; i < n_stage; ++i) {
+ const int64_t stride = hparams.audio_patch_sizes[i];
+ if (stride < 1 || stride > 64 || hop * stride > (1 << 20) || hparams.audio_depths[i] < 0 || hparams.audio_depths[i] > 64) {
+ throw std::runtime_error(mtmd_format("%s: invalid vibevoice audio hparams\n", __func__));
+ }
+ rf += ((i == 0 ? k : 2 * stride) - 1) * hop;
+ hop *= stride;
+ rf += hparams.audio_depths[i] * (k - 1) * hop;
+ n_blocks += hparams.audio_depths[i];
+ }
+ rf += (k - 1) * hop;
+ if (rf > 60 * hparams.audio_sample_rate) {
+ throw std::runtime_error(mtmd_format("%s: vibevoice receptive field of %" PRId64 " samples is too long\n", __func__, rf));
+ }
+ if (n_blocks != hparams.n_layer) {
+ throw std::runtime_error(mtmd_format("%s: sum of audio depths (%" PRId64 ") does not match n_layer (%d)\n", __func__, n_blocks, hparams.n_layer));
+ }
+ hparams.audio_n_ctx = (int32_t) ((rf + hop - 1) / hop * hop);
+ hparams.warmup_audio_size = (int32_t) ((int64_t) hparams.audio_chunk_len * hparams.audio_sample_rate / hop * hop) + hparams.audio_n_ctx;
+ } break;
case PROJECTOR_TYPE_INKLING2V:
{
hparams.ffn_op = FFN_GELU_ERF;
@@ -892,6 +932,7 @@ struct mtmd_model_loader {
} break;
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_QWEN2A:
+ case PROJECTOR_TYPE_HIGGS_AUDIO:
case PROJECTOR_TYPE_QWEN3A:
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_VOXTRAL:
@@ -912,6 +953,12 @@ struct mtmd_model_loader {
hparams.audio_n_fft = 400;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
+
+ if (model.proj_type == PROJECTOR_TYPE_HIGGS_AUDIO) {
+ // each 4s chunk is encoded on its own
+ hparams.audio_chunk_len = 4;
+ hparams.warmup_audio_size = 400;
+ }
} break;
case PROJECTOR_TYPE_VOXTRAL_RT:
{
@@ -1324,6 +1371,7 @@ struct mtmd_model_loader {
// Muse Spark is encoder-free in the same sense: its conv pyramid reads the raw waveform, one channel takes the place of the mel bins
const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA &&
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
+ model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC;
// Validate audio hparams loaded from GGUF metadata
@@ -1497,6 +1545,7 @@ struct mtmd_model_loader {
model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN &&
model.proj_type != PROJECTOR_TYPE_SOPRANO &&
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
+ model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
model.proj_type != PROJECTOR_TYPE_CHATTERBOX &&
model.proj_type != PROJECTOR_TYPE_CHATTERBOX_SPKENC);
@@ -1928,6 +1977,52 @@ struct mtmd_model_loader {
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias"));
} break;
+ case PROJECTOR_TYPE_VIBEVOICE_ASR:
+ {
+ const int n_stage = (int) hparams.audio_patch_sizes.size();
+ for (int e = 0; e < 2; ++e) {
+ auto & enc = model.cnx_enc[e];
+ enc.conv_w.resize(n_stage + 1);
+ enc.conv_b.resize(n_stage + 1);
+ for (int i = 0; i <= n_stage; ++i) {
+ enc.conv_w[i] = get_tensor(mtmd_format(TN_A_CNX_CONV, e, i, "weight"));
+ enc.conv_b[i] = get_tensor(mtmd_format(TN_A_CNX_CONV, e, i, "bias"));
+ // the receptive field in load_hparams assumes these kernel sizes
+ const int64_t k_expected = (i == 0 || i == n_stage) ? hparams.audio_conv_kernel_size : 2 * hparams.audio_patch_sizes[i];
+ if (enc.conv_w[i]->ne[0] != k_expected) {
+ throw std::runtime_error(mtmd_format("%s: %s has kernel %" PRId64 ", expected %" PRId64 "\n", __func__, ggml_get_name(enc.conv_w[i]), enc.conv_w[i]->ne[0], k_expected));
+ }
+ }
+ enc.blocks.resize(hparams.n_layer);
+ for (int il = 0; il < hparams.n_layer; ++il) {
+ auto & layer = enc.blocks[il];
+ layer.ln_1_w = get_tensor(mtmd_format(TN_A_CNX_NORM, e, il, "weight"));
+ layer.conv_w = get_tensor(mtmd_format(TN_A_CNX_DW, e, il, "weight"));
+ layer.conv_b = get_tensor(mtmd_format(TN_A_CNX_DW, e, il, "bias"));
+ layer.ls_1_w = get_tensor(mtmd_format(TN_A_CNX_LS1, e, il, "weight"));
+ layer.ln_2_w = get_tensor(mtmd_format(TN_A_CNX_FFN_NORM, e, il, "weight"));
+ layer.ff_up_w = get_tensor(mtmd_format(TN_A_CNX_FFN_UP, e, il, "weight"));
+ layer.ff_up_b = get_tensor(mtmd_format(TN_A_CNX_FFN_UP, e, il, "bias"));
+ layer.ff_down_w = get_tensor(mtmd_format(TN_A_CNX_FFN_DOWN, e, il, "weight"));
+ layer.ff_down_b = get_tensor(mtmd_format(TN_A_CNX_FFN_DOWN, e, il, "bias"));
+ layer.ls_2_w = get_tensor(mtmd_format(TN_A_CNX_LS2, e, il, "weight"));
+ if (layer.conv_w->ne[0] != hparams.audio_conv_kernel_size || layer.conv_w->type != GGML_TYPE_F32) {
+ throw std::runtime_error(mtmd_format("%s: %s must be F32 with kernel %d\n", __func__, ggml_get_name(layer.conv_w), hparams.audio_conv_kernel_size));
+ }
+ }
+ }
+ // acoustic connector: mlp 0 -> norm -> mlp 1, semantic connector: mlp 2 -> norm -> mlp 3
+ model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight"));
+ model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias"));
+ model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP_NORM, 0, "weight"));
+ model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
+ model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
+ model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
+ model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias"));
+ model.mm_norm_mid_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP_NORM, 2, "weight"));
+ model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight"));
+ model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias"));
+ } break;
case PROJECTOR_TYPE_INKLING2V:
{
// these come from the generic path above, where they are optional
@@ -2315,6 +2410,19 @@ struct mtmd_model_loader {
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias"));
} break;
+ case PROJECTOR_TYPE_HIGGS_AUDIO:
+ {
+ model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight"));
+ model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias"));
+ model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight"));
+ model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias"));
+ model.downsample_conv_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight"));
+ model.downsample_conv_b = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "bias"));
+ model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
+ model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
+ model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
+ model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias"));
+ } break;
case PROJECTOR_TYPE_QWEN3A:
{
model.conv2d_1_w = get_tensor(mtmd_format(TN_CONV2D, 1, "weight"));
@@ -4187,6 +4295,10 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
const int64_t fold = hparams.audio_samples_per_token();
n_patches = (int) ((img->nx() + fold - 1) / fold);
} break;
+ case PROJECTOR_TYPE_VIBEVOICE_ASR:
+ {
+ n_patches = (img->nx() - img->audio_n_ctx) / hparams.audio_samples_per_token();
+ } break;
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_QWEN2A:
@@ -4210,6 +4322,13 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
n_patches /= 2;
}
} break;
+ case PROJECTOR_TYPE_HIGGS_AUDIO:
+ {
+ // stride-2 conv1d, avgpool(2), stride-2 depthwise conv1d
+ n_patches = (img->nx() + 1) / 2;
+ n_patches = n_patches / 2;
+ n_patches = (n_patches + 1) / 2;
+ } break;
case PROJECTOR_TYPE_QWEN3A:
{
// chunk_size=100 frames --> 3x stride-2 conv2d --> 13 tokens per chunk
@@ -4449,6 +4568,8 @@ int mtmd_model::n_mmproj_embd() const {
case PROJECTOR_TYPE_IDEFICS3:
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
return mm_fc_w->ne[1];
+ case PROJECTOR_TYPE_VIBEVOICE_ASR:
+ return mm_1_w->ne[1];
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_VOXTRAL_RT:
@@ -4466,6 +4587,7 @@ int mtmd_model::n_mmproj_embd() const {
case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
return mm_fc_w->ne[1];
case PROJECTOR_TYPE_QWEN3A:
+ case PROJECTOR_TYPE_HIGGS_AUDIO:
return mm_2_w->ne[1];
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_LFM2:
diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h
index d04314d93..44863636b 100644
--- a/tools/mtmd/mtmd-model.h
+++ b/tools/mtmd/mtmd-model.h
@@ -160,7 +160,7 @@ struct mtmd_hparams {
int32_t audio_proj_downsample_rate = 0;
int32_t audio_proj_head_count = 0;
- // audio pyramid (muse-spark): widths, folded samples and blocks per stage
+ // audio pyramid (muse-spark, vibevoice): widths, folded samples (vibevoice: conv strides) and blocks per stage
std::vector<int32_t> audio_hidden_sizes;
std::vector<int32_t> audio_patch_sizes;
std::vector<int32_t> audio_depths;
@@ -174,6 +174,9 @@ struct mtmd_hparams {
return (int32_t) n;
}
+ // vibevoice: leading samples of a chunk that only feed the causal convs, at least their receptive field
+ int32_t audio_n_ctx = 0;
+
// audio-to-mel preprocessor params
int32_t audio_chunk_len = -1; // in seconds
int32_t audio_sample_rate = -1;
@@ -528,6 +531,14 @@ struct mtmd_seanet {
std::vector<stage> stages;
};
+// vibevoice-asr: causal conv pyramid over the raw waveform
+// stage = strided conv -> ConvNeXt blocks (RMS norm, depthwise conv, layer scale, then RMS norm, MLP, layer scale)
+struct mtmd_cnx_encoder {
+ std::vector<ggml_tensor *> conv_w; // stem, one downsample conv per stage, head
+ std::vector<ggml_tensor *> conv_b;
+ std::vector<mtmd_layer> blocks; // ln_1, conv (depthwise), ls_1, ln_2, ffn_up, ffn_down, ls_2
+};
+
// pocket-tts flow-matching decoder (SimpleMLPAdaLN)
struct mtmd_flow_net {
// AdaLN res block: in_ln -> modulate -> Linear -> SiLU -> Linear, gated residual
@@ -1160,7 +1171,9 @@ struct mtmd_model {
ggml_tensor * mm_norm_mid_w = nullptr;
// mimo-audio-tokenizer: post-transformer downsample + RVQ codebook
- ggml_tensor * downsample_conv_w = nullptr; // no bias
+ // higgs-audio: depthwise downsample conv of the projector
+ ggml_tensor * downsample_conv_w = nullptr;
+ ggml_tensor * downsample_conv_b = nullptr; // higgs-audio only
ggml_tensor * downsample_norm_w = nullptr;
ggml_tensor * downsample_norm_b = nullptr;
ggml_tensor * rvq_codebook = nullptr; // merged 3D [n_q, max_bins, dim]
@@ -1208,6 +1221,9 @@ struct mtmd_model {
// pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path)
mtmd_seanet seanet;
+ // vibevoice-asr: acoustic and semantic encoders
+ mtmd_cnx_encoder cnx_enc[2];
+
// pocket-tts: voice latent -> backbone embd (speaker path)
ggml_tensor * spk_proj_w = nullptr;
ggml_tensor * downsample_w = nullptr;
@@ -1332,6 +1348,7 @@ struct mtmd_model {
bool audio_has_avgpool() const {
return proj_type == PROJECTOR_TYPE_QWEN2A
+ || proj_type == PROJECTOR_TYPE_HIGGS_AUDIO
|| proj_type == PROJECTOR_TYPE_VOXTRAL
|| proj_type == PROJECTOR_TYPE_MUSIC_FLAMINGO;
}
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index ed8800fce..db54ec9c9 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -402,6 +402,7 @@ static std::vector<c2w_state_slot> list_enc_state_slots(const mtmd_hparams & hpa
switch (model.proj_type) {
case PROJECTOR_TYPE_LFM3A: return list_lfm3a_state_slots(hparams, model);
case PROJECTOR_TYPE_GEMMA4A: return list_gemma4a_state_slots(hparams, model);
+ case PROJECTOR_TYPE_VIBEVOICE_ASR: return list_vibevoice_asr_state_slots(hparams, model);
default: return {};
}
}
@@ -451,7 +452,7 @@ mtmd_subctx_gen_state * mtmd_subcontext::gen_state_init(uint32_t seed) {
mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) {
const auto & hparams = model.hparams;
- if (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A) {
+ if (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A || model.proj_type == PROJECTOR_TYPE_VIBEVOICE_ASR) {
// see list_enc_state_slots()
const auto slots = list_enc_state_slots(hparams, model);
auto st = std::make_unique<mtmd_subctx_enc_state>();
@@ -1662,6 +1663,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
case PROJECTOR_TYPE_NEMOTRON_OMNI:
case PROJECTOR_TYPE_QWEN2A:
+ case PROJECTOR_TYPE_HIGGS_AUDIO:
case PROJECTOR_TYPE_QWEN3A:
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_ULTRAVOX:
@@ -1982,6 +1984,10 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
}
set_input_i32("positions", positions);
} break;
+ case PROJECTOR_TYPE_VIBEVOICE_ASR:
+ {
+ set_enc_state_in();
+ } break;
case PROJECTOR_TYPE_GEMMA4A:
{
GGML_ASSERT(imgs.entries.size() == 1);
@@ -2627,7 +2633,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
}
}
}
- if (params->enc_state != nullptr && (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A)) {
+ if (params->enc_state != nullptr && (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A || model.proj_type == PROJECTOR_TYPE_VIBEVOICE_ASR)) {
const auto & slots = params->enc_state->slots;
size_t i = 0;
for (const auto & slot : list_enc_state_slots(hparams, model)) {
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 0042aeb42..f8f7ea048 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -48,7 +48,7 @@ std::string mtmd_format(const char * fmt, ...) {
}
// remember to bump this if the serialization format changes
-#define MTMD_SERIALIZATION_VERSION 4
+#define MTMD_SERIALIZATION_VERSION 5
// oldest compat version that can be loaded
#define MTMD_SERIALIZATION_VERSION_MIN 2
@@ -137,6 +137,7 @@ void mtmd_image_f32::serialize(mtmd_serialization & ser) const {
ser.write((int32_t)anyres.orig_ny);
ser.write((int32_t)nx_);
ser.write((int32_t)ny_);
+ ser.write(audio_n_ctx);
}
void mtmd_image_f32::deserialize(mtmd_serialization & ser) {
add_viewsep = ser.read<bool>();
@@ -155,6 +156,12 @@ void mtmd_image_f32::deserialize(mtmd_serialization & ser) {
}
nx_ = ser.read<int32_t>();
ny_ = ser.read<int32_t>();
+ if (ser.version >= 5) {
+ audio_n_ctx = ser.read<int32_t>();
+ if (audio_n_ctx < 0 || audio_n_ctx > nx_) {
+ throw std::runtime_error("invalid audio_n_ctx");
+ }
+ }
buf.clear(); // always a placeholder after loading
}
@@ -1194,6 +1201,19 @@ struct mtmd_context {
aud_end = "<|audio_eos|>";
audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(models.a.get());
} break;
+ case PROJECTOR_TYPE_HIGGS_AUDIO:
+ {
+ // the 4s chunks are joined in one <|audio_bos|> ... <|audio_eos|> span
+ aud_beg = "<|audio_bos|>";
+ aud_end = "<|audio_eos|>";
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_higgs_audio>(models.a.get());
+ } break;
+ case PROJECTOR_TYPE_VIBEVOICE_ASR:
+ {
+ aud_beg = "<|object_ref_start|>";
+ aud_end = "<|object_ref_end|>";
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_vibevoice_asr>(models.a.get());
+ } break;
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
{
aud_beg = "<|soft_audio_start|>";
@@ -4270,12 +4290,58 @@ struct mtmd_audio_stream_gemma4a : mtmd_audio_stream {
bool encode(int n);
};
+// Higgs Audio: each 4 s chunk is encoded on its own, so a chunk goes out once it is complete
+// a flush encodes the partial chunk as a chunk of its own, the reference also cuts chunks at VAD boundaries
+struct mtmd_audio_stream_higgs_audio : mtmd_audio_stream {
+ int64_t chunk_len = 0; // samples per complete chunk
+
+ std::vector<float> beg_rows;
+ std::vector<float> end_rows;
+ bool beg_done = false;
+ bool end_done = false;
+
+ bool init(const mtmd_audio_stream_params & params) override;
+ int32_t flush(bool end) override;
+ int32_t step(llama_token last_token) override;
+
+ int64_t n_chunk_rows(int64_t n) const;
+ bool encode(int64_t n);
+};
+
+// VibeVoice-ASR: the encoder is causal, its convs keep their left context in the state
+// the reference scales the whole clip to -25 dBFS, here the gain comes from the audio pushed so far (see mtmd_audio_vibevoice_gain)
+struct mtmd_audio_stream_vibevoice_asr : mtmd_audio_stream {
+ mtmd_subctx_enc_state * enc_state = nullptr;
+
+ int64_t n_step = 0; // samples per encoder call
+ int64_t n_step_max = 0; // largest call, the warmup size
+ int64_t n_stat = 0; // pushed samples counted in sum_sq and peak
+ double sum_sq = 0.0;
+ double peak = 0.0;
+
+ std::vector<float> beg_rows;
+ std::vector<float> end_rows;
+ bool beg_done = false;
+ bool end_done = false;
+
+ ~mtmd_audio_stream_vibevoice_asr() override {
+ delete enc_state;
+ }
+
+ bool init(const mtmd_audio_stream_params & params) override;
+ int32_t flush(bool end) override;
+ int32_t step(llama_token last_token) override;
+
+ bool encode(int64_t n);
+};
+
bool mtmd_support_audio_stream(const mtmd_context * ctx) {
if (!ctx->ctx_a || !ctx->text_model) {
return false;
}
const projector_type proj = ctx->ctx_a->model.proj_type;
- return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A || proj == PROJECTOR_TYPE_GEMMA4A;
+ return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A || proj == PROJECTOR_TYPE_GEMMA4A ||
+ proj == PROJECTOR_TYPE_HIGGS_AUDIO || proj == PROJECTOR_TYPE_VIBEVOICE_ASR;
}
mtmd_audio_stream_params mtmd_audio_stream_params_default(void) {
@@ -4298,6 +4364,8 @@ mtmd_audio_stream * mtmd_audio_stream_init(mtmd_context * ctx, mtmd_audio_stream
case PROJECTOR_TYPE_VOXTRAL_RT: s = std::make_unique<mtmd_audio_stream_voxtral_rt>(); break;
case PROJECTOR_TYPE_LFM3A: s = std::make_unique<mtmd_audio_stream_lfm3a>(); break;
case PROJECTOR_TYPE_GEMMA4A: s = std::make_unique<mtmd_audio_stream_gemma4a>(); break;
+ case PROJECTOR_TYPE_HIGGS_AUDIO: s = std::make_unique<mtmd_audio_stream_higgs_audio>(); break;
+ case PROJECTOR_TYPE_VIBEVOICE_ASR: s = std::make_unique<mtmd_audio_stream_vibevoice_asr>(); break;
default: GGML_ABORT("unsupported projector");
}
s->ctx = ctx;
@@ -5036,6 +5104,210 @@ int32_t mtmd_audio_stream_gemma4a::step(llama_token /* last_token */) {
return (int32_t) n_out;
}
+// encodes one audio entry, the rows are appended to embd
+static bool mtmd_audio_stream_encode(mtmd_audio_stream & s, mtmd_image_f32 && img, mtmd_subctx_enc_state * enc_state) {
+ const int n_out = s.sctx->model.n_output_tokens(&img);
+
+ mtmd_image_f32_batch batch;
+ batch.is_audio = true;
+ batch.entries.push_back(std::move(img));
+
+ std::vector<float> res((size_t) s.n_embd * n_out);
+ mtmd_subctx_encode_params params;
+ params.n_threads = s.ctx->n_threads;
+ params.imgs = &batch;
+ params.out_embd = &res;
+ params.enc_state = enc_state;
+ if (!s.sctx->encode(¶ms)) {
+ LOG_ERR("%s: failed to encode audio\n", __func__);
+ return false;
+ }
+ s.embd.insert(s.embd.end(), res.begin(), res.end());
+ return true;
+}
+
+// rows carry no text: all of them go out at once between the audio markers, last_token is not used
+static int32_t mtmd_audio_stream_emit_rows(mtmd_audio_stream & s, const std::vector<float> & beg_rows, const std::vector<float> & end_rows,
+ bool & beg_done, bool & end_done) {
+ const int64_t n_avail = (int64_t) (s.embd.size() / s.n_embd);
+ s.out.clear();
+ if (n_avail > 0 || s.ended) {
+ if (!beg_done) {
+ beg_done = true;
+ s.out = beg_rows;
+ }
+ s.out.insert(s.out.end(), s.embd.begin(), s.embd.end());
+ s.embd.clear();
+ if (s.ended && !end_done) {
+ end_done = true;
+ s.out.insert(s.out.end(), end_rows.begin(), end_rows.end());
+ }
+ }
+ const int64_t n_out = (int64_t) (s.out.size() / s.n_embd);
+ if (n_out == 0) {
+ return s.ended ? -1 : 0;
+ }
+ s.n_rows += n_out;
+ s.n_rows_done += n_avail; // the markers are not audio rows
+ return (int32_t) n_out;
+}
+
+//
+// Higgs Audio
+//
+
+bool mtmd_audio_stream_higgs_audio::init(const mtmd_audio_stream_params & params) {
+ const mtmd_hparams * hparams = &sctx->model.hparams;
+ if (params.n_prompt > 0) {
+ LOG_ERR("%s: Higgs Audio takes no prompt, its audio is a chat turn\n", __func__);
+ return false;
+ }
+ if (!ctx->audio_preproc) {
+ LOG_ERR("%s: the audio preprocessor is not initialized\n", __func__);
+ return false;
+ }
+ chunk_len = (int64_t) hparams->audio_chunk_len * hparams->audio_sample_rate;
+ frames_per_tok = 8; // stride-2 conv, avgpool, stride-2 projector conv
+ samples_per_tok = frames_per_tok * hop;
+ return mtmd_audio_stream_marker_rows(ctx, ctx->aud_beg, beg_rows) && mtmd_audio_stream_marker_rows(ctx, ctx->aud_end, end_rows);
+}
+
+// rows of a chunk with n samples, see mtmd_audio_preprocessor_higgs_audio
+int64_t mtmd_audio_stream_higgs_audio::n_chunk_rows(int64_t n) const {
+ const int64_t n_frames = n / hop;
+ if (n_frames < 3) {
+ return 0;
+ }
+ return ((n_frames + 1) / 2 / 2 + 1) / 2;
+}
+
+bool mtmd_audio_stream_higgs_audio::encode(int64_t n) {
+ mtmd_audio_preproc_out pre = ctx->audio_preproc->preprocess(buf.data(), (size_t) n);
+ if (pre.entries.empty() || !ctx->audio_preproc->compute(pre)) {
+ LOG_ERR("%s: unable to preprocess the audio\n", __func__);
+ return false;
+ }
+ for (auto & entry : pre.entries) {
+ if (!mtmd_audio_stream_encode(*this, std::move(entry), nullptr)) {
+ return false;
+ }
+ }
+ buf.erase(buf.begin(), buf.begin() + n);
+ buf_pos += n;
+ return true;
+}
+
+int32_t mtmd_audio_stream_higgs_audio::flush(bool end) {
+ const int64_t n_pending = n_samples - buf_pos;
+ n_rows_flushed = n_rows_done + (int64_t) (embd.size() / n_embd) + (n_pending / chunk_len) * n_chunk_rows(chunk_len) + n_chunk_rows(n_pending % chunk_len);
+ pending_flush = true;
+ ended = end;
+ return 0;
+}
+
+int32_t mtmd_audio_stream_higgs_audio::step(llama_token /* last_token */) {
+ const bool flush = pending_flush;
+ pending_flush = false;
+
+ while ((int64_t) buf.size() >= chunk_len) {
+ if (!encode(chunk_len)) {
+ return -2;
+ }
+ }
+ // a tail too short for a row is kept until the next flush, or dropped at the end
+ if ((flush || ended) && !buf.empty()) {
+ if (n_chunk_rows((int64_t) buf.size()) > 0) {
+ if (!encode((int64_t) buf.size())) {
+ return -2;
+ }
+ } else if (ended) {
+ buf_pos += (int64_t) buf.size();
+ buf.clear();
+ }
+ }
+ return mtmd_audio_stream_emit_rows(*this, beg_rows, end_rows, beg_done, end_done);
+}
+
+//
+// VibeVoice-ASR
+//
+
+bool mtmd_audio_stream_vibevoice_asr::init(const mtmd_audio_stream_params & params) {
+ const mtmd_hparams * hparams = &sctx->model.hparams;
+ if (params.n_prompt > 0) {
+ LOG_ERR("%s: VibeVoice-ASR takes no prompt, its audio is a chat turn\n", __func__);
+ return false;
+ }
+ hop = hparams->audio_samples_per_token();
+ frames_per_tok = 1;
+ samples_per_tok = hop;
+ n_step = std::max<int64_t>(1, ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 + hop - 1) / hop) * hop;
+ n_step_max = std::max(n_step, (int64_t) hparams->audio_chunk_len * hparams->audio_sample_rate / hop * hop);
+
+ enc_state = sctx->enc_state_init(0);
+ if (!enc_state) {
+ return false;
+ }
+ return mtmd_audio_stream_marker_rows(ctx, ctx->aud_beg, beg_rows) && mtmd_audio_stream_marker_rows(ctx, ctx->aud_end, end_rows);
+}
+
+// n is a multiple of hop, see mtmd_audio_preprocessor_vibevoice_asr for the gain
+bool mtmd_audio_stream_vibevoice_asr::encode(int64_t n) {
+ const double gain = mtmd_audio_vibevoice_gain(sum_sq, n_stat, peak);
+
+ std::vector<float> data((size_t) n);
+ for (int64_t i = 0; i < n; i++) {
+ data[i] = (float) (buf[i] * gain);
+ }
+ mtmd_image_f32 img;
+ img.set_size({(int) n, 1}, false, /* is_audio */ true);
+ img.cpy_buf(data);
+ if (!mtmd_audio_stream_encode(*this, std::move(img), enc_state)) {
+ return false;
+ }
+ buf.erase(buf.begin(), buf.begin() + n);
+ buf_pos += n;
+ return true;
+}
+
+int32_t mtmd_audio_stream_vibevoice_asr::flush(bool end) {
+ n_rows_flushed = end ? (n_samples + hop - 1) / hop : n_samples / hop;
+ pending_flush = true;
+ ended = end;
+ return 0;
+}
+
+int32_t mtmd_audio_stream_vibevoice_asr::step(llama_token /* last_token */) {
+ const bool flush = pending_flush;
+ pending_flush = false;
+
+ // the padding added at the end does not count in the gain
+ for (int64_t i = n_stat - buf_pos; i < (int64_t) buf.size(); i++) {
+ sum_sq += (double) buf[i] * buf[i];
+ peak = std::max(peak, (double) std::fabs(buf[i]));
+ }
+ n_stat = n_samples;
+ if (ended && n_samples % hop != 0) {
+ add_silence(hop - n_samples % hop);
+ n_stat = n_samples;
+ }
+
+ // whole rows only: a partial row would put its zero padding into the state
+ int64_t n = (int64_t) buf.size();
+ n -= n % hop;
+ if (!flush && !ended) {
+ n -= n % n_step;
+ }
+ while (n > 0) {
+ const int64_t m = std::min(n, n_step_max);
+ if (!encode(m)) {
+ return -2;
+ }
+ n -= m;
+ }
+ return mtmd_audio_stream_emit_rows(*this, beg_rows, end_rows, beg_done, end_done);
+}
+
int64_t mtmd_audio_stream_get_n_rows(const mtmd_audio_stream * s) {
return s->n_rows_done;
}
diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp
index af1e42dbd..1724a6702 100644
--- a/tools/server/server-chat.cpp
+++ b/tools/server/server-chat.cpp
@@ -739,14 +739,22 @@ json convert_transcriptions_to_chatcmpl(
if (response_format != "json") {
throw std::invalid_argument("Only 'json' response_format is supported for transcription");
}
- const common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language);
- if (prompt.empty()) {
+ common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language, 0.0, prompt);
+ if (preset.needs_audio_length) {
+ mtmd_helper_media_meta meta;
+ if (mtmd_helper_get_media_meta(out_files[0].data(), out_files[0].size(), mtmd_helper_init_opt_default(), &meta) != 0 ||
+ meta.type != MTMD_HELPER_MEDIA_TYPE_AUDIO || meta.duration < 0) {
+ throw std::invalid_argument("the input file is not a supported audio file");
+ }
+ preset = common_chat_get_asr_prompt(tmpls, language, meta.duration, prompt);
+ }
+ if (prompt.empty() || preset.has_context) {
prompt = preset.user;
}
if (!language.empty() && !preset.has_language) {
prompt += string_format(" (language: %s)", language.c_str());
}
- prompt += get_media_marker();
+ prompt = preset.media_first ? get_media_marker() + prompt : prompt + get_media_marker();
json messages = json::array();
if (!preset.system.empty()) {
diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp
index 8ed3dd535..e747d32d7 100644
--- a/tools/server/server-realtime.cpp
+++ b/tools/server/server-realtime.cpp
@@ -155,6 +155,7 @@ struct realtime_conversation {
bool started = false; // something went into the slot, the session config is fixed
bool turn_open = false; // a user turn is being filled
bool has_audio = false; // the open turn has audio not committed yet
+ int64_t n_turn_samples = 0; // audio of the open turn, at the model rate
bool tool_results = false; // tool results went in after the last response, the next one answers them
int n_turns = 0;
std::string next_header; // text that goes in before the next user turn
@@ -376,8 +377,10 @@ struct realtime_conversation {
if (!resample(samples) || samples.empty() || !ensure_turn()) {
return;
}
+ const int64_t n = (int64_t) samples.size();
if (control(SERVER_CONTROL_ACTION_AUDIO_APPEND, {}, std::move(samples))) {
has_audio = true;
+ n_turn_samples += n;
}
}
@@ -410,7 +413,8 @@ struct realtime_conversation {
if (!control(SERVER_CONTROL_ACTION_AUDIO_END)) {
return false;
}
- has_audio = false;
+ has_audio = false;
+ n_turn_samples = 0;
std::lock_guard<std::mutex> lock(mutex);
const json prev = last_item_id.empty() ? json(nullptr) : json(last_item_id);
@@ -424,6 +428,7 @@ struct realtime_conversation {
}
void on_commit() {
+ const int64_t n_audio = n_turn_samples;
if (!commit() || type != "transcription") {
return;
}
@@ -433,11 +438,19 @@ struct realtime_conversation {
transcribe_item_id = last_item_id;
transcript.clear();
}
+ // some models (VibeVoice-ASR) take an instruction after the audio, with its length
+ const auto * tmpls = deps.meta.chat_params.tmpls.get();
+ common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language);
+ if (preset.needs_audio_length) {
+ preset = common_chat_get_asr_prompt(tmpls, language, (double) n_audio / mtmd_get_audio_sample_rate(deps.mctx));
+ }
+ const std::string after_audio = preset.media_first ? preset.user : "";
+
server_control_response response;
response.n_predict = max_output_tokens;
response.out_mode = MTMD_HELPER_GEN_AUDIO_OUT_TEXT;
response.sampling = response_sampling(nullptr);
- if (!control(SERVER_CONTROL_ACTION_RESPONSE_CREATE, user_footer + assistant_header, {}, response)) {
+ if (!control(SERVER_CONTROL_ACTION_RESPONSE_CREATE, after_audio + user_footer + assistant_header, {}, response)) {
return;
}
control(SERVER_CONTROL_ACTION_CLEAR);