Commit 28b812d6c for llama.cpp
commit 28b812d6cb9cfe569d368c2774c3886ad4fe10e8
Author: Prabhsimran Singh <prabhsimrans@nvidia.com>
Date: Tue Nov 3 18:57:05 2026 +0530
mtmd : add Nemotron 3.5 Transcribe support (#31471)
* mtmd : add Nemotron 3.5 Transcribe support
Assisted-by: Claude
* convert : fix vocab of draft models for Nemotron-H targets
Assisted-by: Claude
* convert : reject non-BF16 Transcribe checkpoints
Assisted-by: Claude
* mtmd : make speaker FIFO predictions local
Assisted-by: Claude
* mtmd : normalize the mel in encode_windows
Assisted-by: Claude
* convert : map the generated speaker kernel
Assisted-by: Claude
* mtmd : validate Nemotron Transcribe mmproj shapes
Assisted-by: Claude
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 6a34da41f..dcbb97c1f 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -223,6 +223,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"ModernBertForSequenceClassification": "bert",
"ModernBertModel": "bert",
"NanbeigeForCausalLM": "nanbeige",
+ "Nemotron3_5TranscribeForConditionalGeneration": "nemotron",
"NemotronForCausalLM": "nemotron",
"NemotronHForCausalLM": "nemotron",
"NemotronHPuzzleForCausalLM": "nemotron",
@@ -380,6 +381,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"MiniCPMOcrForConditionalGeneration": "minicpm",
"Mistral3ForConditionalGeneration": "llava",
"Mistral3Model": "llava",
+ "Nemotron3_5TranscribeForConditionalGeneration": "nemotron",
"NemotronH_Omni_Reasoning_V3": "nemotron",
"NemotronH_Nano_VL_V2": "nemotron",
"MuseGlimmerForConditionalGeneration": "muse_glimmer",
diff --git a/conversion/base.py b/conversion/base.py
index 1c7aecec8..fd880bbab 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -3386,7 +3386,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
# Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older
# Kimi-Linear-48B architecture and cannot load K3 (no attention residuals,
# latent MoE, situ, ...). Route on the top-level architecture instead.
- if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration"):
+ if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration"):
return arch
# if "architectures" is found in the sub-config, use that instead
diff --git a/conversion/nemotron.py b/conversion/nemotron.py
index 371f5b616..066149744 100644
--- a/conversion/nemotron.py
+++ b/conversion/nemotron.py
@@ -428,6 +428,9 @@ class NemotronHModel(GraniteHybridModel):
is_moe: bool = False
supports_mtp_export = True
_experts: list[dict[str, Tensor]] | None = None
+ # tokens past the embedding rows (left out of the vocab), and special tokens that are printed as text
+ _vocab_skip: tuple[str, ...] = ()
+ _user_defined_tokens: re.Pattern[str] | None = None
_SSM_LAYER_TYPES = {"mamba", "linear_attention"}
_ATTN_LAYER_TYPES = {"attention", "full_attention"}
@@ -616,7 +619,11 @@ class NemotronHModel(GraniteHybridModel):
vocab_size = -(vocab_size // -pad_vocab) * pad_vocab
self.hparams["vocab_size"] = vocab_size
- assert max(tokenizer.vocab.values()) < vocab_size # ty: ignore[unresolved-attribute]
+ # draft-model converters (DFlash, LiLiCorr) call this with their own instance, which has neither hook
+ vocab_skip = getattr(self, "_vocab_skip", ())
+ user_defined_tokens = getattr(self, "_user_defined_tokens", None)
+
+ assert max(id_ for tok, id_ in tokenizer.vocab.items() if tok not in vocab_skip) < vocab_size # ty: ignore[unresolved-attribute]
tokpre = self.get_vocab_base_pre(tokenizer)
@@ -638,7 +645,8 @@ class NemotronHModel(GraniteHybridModel):
if previous_token != token:
logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")
- if added_tokens_decoder[i].special or self.does_token_look_special(token):
+ printed = user_defined_tokens is not None and user_defined_tokens.fullmatch(token)
+ if (added_tokens_decoder[i].special or self.does_token_look_special(token)) and not printed:
toktypes.append(gguf.TokenType.CONTROL)
else:
token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ") # pre-normalize user-defined spaces
@@ -834,3 +842,140 @@ class NemotronHPuzzleModel(NemotronHModel):
if item[0].startswith("mtp."):
return None
return super().filter_tensors(item)
+
+
+# NeMo speechlm2 SALM checkpoint: a Nemotron-H decoder under llm.*, the speech encoder under perception.*
+@ModelBase.register("Nemotron3_5TranscribeForConditionalGeneration")
+@ModelBase.example("nvidia/NVIDIA-Nemotron-3.5-Transcribe-30B-A4B")
+class NemotronTranscribeModel(NemotronHModel):
+ model_arch = gguf.MODEL_ARCH.NEMOTRON_H
+ # the audio placeholder (id vocab_size) has no embedding row; speaker, event and style tags print with the transcript
+ _vocab_skip = ("<|audio|>",)
+ _user_defined_tokens = re.compile(r"<spk:\d+>|<ev:[a-z_]+>|<sty:(?:stt|end):[a-z_]+>")
+ _renames = {
+ "llm.model.embed_tokens.weight": "backbone.embeddings.weight",
+ "llm.model.norm.weight": "backbone.norm_f.weight",
+ "llm.lm_head.weight": "lm_head.weight",
+ }
+ # grouped experts: [n_expert, n_in, n_out] -> [n_expert, n_out, n_in]
+ _grouped_experts = {"gate_and_up_projs": "up_proj", "down_projs": "down_proj"}
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, gen = item
+ if name.startswith(("perception.", "ctc_timestamp.")) or "_extra_state" in name:
+ return None
+ if not name.startswith("llm."):
+ # the FP8 and NVFP4 releases store the decoder as backbone.*
+ raise ValueError(f"unexpected tensor {name}: only the BF16 checkpoint layout (llm.*) is supported")
+ if name in cls._renames:
+ name = cls._renames[name]
+ elif name.startswith("llm.model.layers."):
+ name = "backbone.layers." + name[len("llm.model.layers."):]
+ elif name.startswith("llm.mtp.layers."):
+ name = "mtp.layers." + name[len("llm.mtp.layers."):]
+ else:
+ raise ValueError(f"unexpected decoder tensor {name}")
+ return super().filter_tensors((name.replace("._fp32_params.", "."), gen))
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ leaf = name.rsplit(".", 1)[-1]
+ if ".mixer.experts." in name and leaf in self._grouped_experts:
+ if name.startswith("mtp."):
+ assert self._mtp_bid is not None, f"{name}: MTP head is not exported"
+ bid = self._mtp_bid
+ assert bid is not None
+ data_torch = data_torch.permute(0, 2, 1).contiguous()
+ name = f"model.layers.{bid}.mlp.experts.{self._grouped_experts[leaf]}.weight"
+ yield from ModelBase.modify_tensors(self, data_torch, name, bid)
+ return
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
+@ModelBase.register("Nemotron3_5TranscribeForConditionalGeneration")
+class NemotronTranscribeAudioModel(MmprojModel):
+ # NeMo ParallelExpertEncoder: ASR and speaker transformers, speaker activity fused into the ASR states
+ has_vision_encoder = False
+ has_audio_encoder = True
+
+ def get_audio_config(self) -> dict[str, Any] | None:
+ asr = self.global_config["pe_encoder_config"]["asr_encoder_cfg"]
+ return {
+ "hidden_size": asr["d_model"],
+ "intermediate_size": int(float(asr.get("ff_expansion", 4)) * asr["d_model"]),
+ "num_hidden_layers": asr["n_layers"],
+ "num_attention_heads": asr["n_heads"],
+ }
+
+ @staticmethod
+ def _rope_dims(cfg: dict[str, Any]) -> int:
+ assert cfg.get("self_attention_model") == "rope" and cfg.get("subsampling", "feature_stacking") == "feature_stacking"
+ assert cfg.get("attn_mode", "full") == "full" and not cfg.get("xscaling", False) and cfg.get("pre_block_norm", True)
+ assert float(cfg.get("rope_base", 10000.0)) == 10000.0
+ n_rot = int(cfg["d_model"] // cfg["n_heads"] * float(cfg.get("rotary_fraction", 1.0)))
+ assert n_rot >= 2 and n_rot % 2 == 0
+ return n_rot
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ pe = self.global_config["pe_encoder_config"]
+ asr = pe["asr_encoder_cfg"]
+ diar = pe["diarization_model_cfg"]
+ spk = diar["encoder"]
+ assert pe.get("asr_encoder_type") == "transformer" and pe.get("speaker_feature_mode", "thresholded") == "thresholded"
+ assert pe.get("asr_normalize_type") == "per_feature" and pe.get("diar_normalize_type") in (None, "NA")
+ assert asr["subsampling_factor"] == spk["subsampling_factor"] == 8 and asr["feat_in"] == spk["feat_in"]
+ assert self._rope_dims(spk) == spk["d_model"] // spk["n_heads"] and not spk.get("qkv_bias", False)
+ assert diar.get("high_resolution", False)
+ # constants in clip: the ParallelExpertEncoder online settings and the Sortformer streaming config
+ assert (pe.get("online_inference_length", 500), pe.get("chunk_left_context", 50), pe.get("chunk_right_context", 50),
+ pe.get("diar_spkcache_len", 188), pe.get("diar_fifo_len", 40), pe.get("diar_spkcache_update_period", 300),
+ pe.get("speaker_activity_threshold", 0.5)) == (500, 50, 50, 188, 40, 300, 0.5), \
+ "streaming settings differ from the constants in tools/mtmd/models/nemotron-transcribe.cpp"
+ sm = diar["sortformer_modules"]
+ assert (sm["spkcache_sil_frames_per_spk"], sm["pred_score_threshold"], sm["scores_boost_latest"], sm["strong_boost_rate"],
+ sm["weak_boost_rate"], sm["min_pos_scores_rate"], sm["use_learnable_sil_emb"]) == (1, 0.25, 0.05, 0.75, 1.5, 0.5, True), \
+ "speaker cache settings differ from the constants in tools/mtmd/models/nemotron-transcribe.cpp"
+
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.NEMOTRON_TRANSCRIBE)
+ self.gguf_writer.add_audio_num_mel_bins(asr["feat_in"])
+ self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
+ self.gguf_writer.add_audio_subsampling_factor(asr["subsampling_factor"])
+ self.gguf_writer.add_audio_rope_dims(self._rope_dims(asr))
+ self.gguf_writer.add_audio_spk_block_count(spk["n_layers"])
+ self.gguf_writer.add_audio_spk_head_count(spk["n_heads"])
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ # ParallelExpertEncoder._build_sinusoid_position_encoding
+ pe = self.global_config["pe_encoder_config"]
+ n_spk = int(pe["diarization_model_cfg"]["sortformer_modules"]["num_spks"])
+ d = int(pe["asr_encoder_cfg"]["d_model"])
+ scale = float(pe.get("spk_kernel_scale", 1.0))
+ pos = torch.arange(n_spk, dtype=torch.float32)[:, None]
+ div = torch.exp(torch.arange(0, d, 2, dtype=torch.float32) * -(np.log(10000.0) / d))
+ kernel = torch.zeros(n_spk, d, dtype=torch.float32)
+ kernel[:, 0::2] = torch.sin(pos * div)
+ kernel[:, 1::2] = torch.cos(pos * div)
+ # stored as [d, n_spk]: the graph multiplies it with the speaker activity directly
+ yield self.format_tensor_name(gguf.MODEL_TENSOR.A_FUSION_SPK_KERNEL, suffix=""), (kernel * scale).T.contiguous()
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.A_SPK_HEAD_UPSAMPLE, None):
+ return gguf.GGMLQuantizationType.F16 # ggml_conv_1d kernel
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, gen = item
+ if not name.startswith("perception.") or "_extra_state" in name:
+ return None
+ # training-only head; the speaker model's unused front end
+ if name.startswith(("perception.encoder.diarization_model.sortformer_modules.hidden_to_spks.",
+ "perception.encoder.diarization_model.preprocessor.")):
+ return None
+ return name, gen
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name == "perception.preprocessor.featurizer.fb":
+ data_torch = data_torch.squeeze(0)
+ yield from super().modify_tensors(data_torch, name, bid)
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 4d057295c..ad4e675f6 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -506,6 +506,11 @@ class Keys:
WINDOW_SIZE = "clip.audio.window_size"
LOCAL_BLOCK_COUNT = "clip.audio.local_block_count" # mimo-v2.5: input_local_transformer layer count
LOCAL_GROUP_SIZE = "clip.audio.local_group_size" # mimo-v2.5: input_local_transformer grouping size
+ ROPE_DIMS = "clip.audio.rope_dims" # nemotron-transcribe
+
+ class Speaker: # nemotron-transcribe
+ BLOCK_COUNT = "clip.audio.spk.block_count"
+ HEAD_COUNT = "clip.audio.spk.head_count"
class Attention:
HEAD_COUNT = "clip.audio.attention.head_count"
@@ -1411,6 +1416,24 @@ class MODEL_TENSOR(IntEnum):
A_ENC_CONV_NORM_VAR = auto() # parakeet
A_ENC_MEL_FILTERS = auto() # parakeet
A_ENC_WINDOW = auto() # parakeet
+ A_ENC_ATTN_QKV = auto() # nemotron-transcribe
+ A_SPK_INP_PROJ = auto() # nemotron-transcribe
+ A_SPK_PRE_NORM = auto() # nemotron-transcribe
+ A_SPK_POST_NORM = auto() # nemotron-transcribe
+ A_SPK_ATTN_NORM = auto() # nemotron-transcribe
+ A_SPK_ATTN_QKV = auto() # nemotron-transcribe
+ A_SPK_ATTN_OUT = auto() # nemotron-transcribe
+ A_SPK_FFN_NORM = auto() # nemotron-transcribe
+ A_SPK_FFN_UP = auto() # nemotron-transcribe
+ A_SPK_FFN_DOWN = auto() # nemotron-transcribe
+ A_SPK_HEAD_PROJ = auto() # nemotron-transcribe
+ A_SPK_HEAD_UPSAMPLE = auto() # nemotron-transcribe
+ A_SPK_HEAD_FC1 = auto() # nemotron-transcribe
+ A_SPK_HEAD_FC2 = auto() # nemotron-transcribe
+ A_SPK_SIL_EMB = auto() # nemotron-transcribe
+ A_FUSION_ASR_NORM = auto() # nemotron-transcribe
+ A_FUSION_SPK_NORM = auto() # nemotron-transcribe
+ A_FUSION_SPK_KERNEL = auto() # nemotron-transcribe
A_CTC_OUT = auto()
A_CTC_OUT_MID = auto()
A_ENC_ATTN_REL_POS_EMB = auto()
@@ -2219,6 +2242,24 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_ENC_CONV_NORM_VAR: "a.blk.{bid}.conv_norm_var",
MODEL_TENSOR.A_ENC_MEL_FILTERS: "a.mel_filters",
MODEL_TENSOR.A_ENC_WINDOW: "a.window",
+ MODEL_TENSOR.A_ENC_ATTN_QKV: "a.blk.{bid}.attn_qkv",
+ MODEL_TENSOR.A_SPK_INP_PROJ: "a.spk.input_projection",
+ MODEL_TENSOR.A_SPK_PRE_NORM: "a.spk.pre_ln",
+ MODEL_TENSOR.A_SPK_POST_NORM: "a.spk.post_ln",
+ MODEL_TENSOR.A_SPK_ATTN_NORM: "a.spk.blk.{bid}.ln1",
+ MODEL_TENSOR.A_SPK_ATTN_QKV: "a.spk.blk.{bid}.attn_qkv",
+ MODEL_TENSOR.A_SPK_ATTN_OUT: "a.spk.blk.{bid}.attn_out",
+ MODEL_TENSOR.A_SPK_FFN_NORM: "a.spk.blk.{bid}.ln2",
+ MODEL_TENSOR.A_SPK_FFN_UP: "a.spk.blk.{bid}.ffn_up",
+ MODEL_TENSOR.A_SPK_FFN_DOWN: "a.spk.blk.{bid}.ffn_down",
+ MODEL_TENSOR.A_SPK_HEAD_PROJ: "a.spk.head.proj",
+ MODEL_TENSOR.A_SPK_HEAD_UPSAMPLE: "a.spk.head.upsample",
+ MODEL_TENSOR.A_SPK_HEAD_FC1: "a.spk.head.fc1",
+ MODEL_TENSOR.A_SPK_HEAD_FC2: "a.spk.head.fc2",
+ MODEL_TENSOR.A_SPK_SIL_EMB: "a.spk.head.sil_emb",
+ MODEL_TENSOR.A_FUSION_ASR_NORM: "a.fusion.asr_norm",
+ MODEL_TENSOR.A_FUSION_SPK_NORM: "a.fusion.spk_norm",
+ MODEL_TENSOR.A_FUSION_SPK_KERNEL: "a.fusion.spk_kernel",
MODEL_TENSOR.A_CTC_OUT: "a.enc_ctc_out",
MODEL_TENSOR.A_CTC_OUT_MID: "a.enc_ctc_out_mid",
MODEL_TENSOR.A_ENC_ATTN_REL_POS_EMB: "a.blk.{bid}.attn_rel_pos_emb",
@@ -2587,6 +2628,24 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.A_ENC_CONV_NORM_VAR,
MODEL_TENSOR.A_ENC_MEL_FILTERS,
MODEL_TENSOR.A_ENC_WINDOW,
+ MODEL_TENSOR.A_ENC_ATTN_QKV,
+ MODEL_TENSOR.A_SPK_INP_PROJ,
+ MODEL_TENSOR.A_SPK_PRE_NORM,
+ MODEL_TENSOR.A_SPK_POST_NORM,
+ MODEL_TENSOR.A_SPK_ATTN_NORM,
+ MODEL_TENSOR.A_SPK_ATTN_QKV,
+ MODEL_TENSOR.A_SPK_ATTN_OUT,
+ MODEL_TENSOR.A_SPK_FFN_NORM,
+ MODEL_TENSOR.A_SPK_FFN_UP,
+ MODEL_TENSOR.A_SPK_FFN_DOWN,
+ MODEL_TENSOR.A_SPK_HEAD_PROJ,
+ MODEL_TENSOR.A_SPK_HEAD_UPSAMPLE,
+ MODEL_TENSOR.A_SPK_HEAD_FC1,
+ MODEL_TENSOR.A_SPK_HEAD_FC2,
+ MODEL_TENSOR.A_SPK_SIL_EMB,
+ MODEL_TENSOR.A_FUSION_ASR_NORM,
+ MODEL_TENSOR.A_FUSION_SPK_NORM,
+ MODEL_TENSOR.A_FUSION_SPK_KERNEL,
MODEL_TENSOR.A_MM_INP_PROJ,
MODEL_TENSOR.A_MM_SOFT_EMB_NORM,
MODEL_TENSOR.A_MM_EMBEDDING,
@@ -6740,6 +6799,7 @@ class VisionProjectorType:
MINICPMV4_7 = "minicpmv4_7"
MINICPM_OCR_1_0 = "minicpm_ocr_1_0"
GRANITE_SPEECH = "granite_speech" # audio
+ NEMOTRON_TRANSCRIBE = "nemotron_transcribe" # audio
MIMOVL = "mimovl"
MIMO_AUDIO = "mimo_audio"
GRANITE4_VISION = "granite4_vision"
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index a0800e24c..56e8938c4 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1712,6 +1712,15 @@ class GGUFWriter:
def add_audio_projector_head_count(self, value: int) -> None:
self.add_uint32(Keys.ClipAudio.Projector.HEAD_COUNT, value)
+ def add_audio_rope_dims(self, value: int) -> None:
+ self.add_uint32(Keys.ClipAudio.ROPE_DIMS, value)
+
+ def add_audio_spk_block_count(self, value: int) -> None:
+ self.add_uint32(Keys.ClipAudio.Speaker.BLOCK_COUNT, value)
+
+ def add_audio_spk_head_count(self, value: int) -> None:
+ self.add_uint32(Keys.ClipAudio.Speaker.HEAD_COUNT, value)
+
# audio generation (mmproj)
def add_clip_gen_audio_projector_type(self, value: str) -> None:
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index d06dc4ec5..2602bfb61 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -2532,6 +2532,7 @@ class TensorNameMap:
MODEL_TENSOR.A_ENC_INP_PROJ: (
"conformer.subsample_conv_projection.input_proj_linear", # gemma4
+ "perception.encoder.asr_encoder.pre_encode.proj", # nemotron-transcribe (feature stacking)
"encoder.input_linear",
),
@@ -2546,13 +2547,16 @@ class TensorNameMap:
"speaker_encoder.mfa.conv", # qwen3tts speaker encoder: multi-layer feature aggregation
),
- MODEL_TENSOR.A_PRE_NORM: (),
+ MODEL_TENSOR.A_PRE_NORM: (
+ "perception.encoder.asr_encoder.embed_norm", # nemotron-transcribe
+ ),
MODEL_TENSOR.A_POST_NORM: (
"audio_tower.layer_norm", # ultravox
"audio_tower.norm", # voxtral realtime
"audio_encoder.dots_encoder.speech_encoder.layer_norm", # dots3note
"audio_tower.ln_post", # qwen2omni
+ "perception.encoder.asr_encoder.final_norm", # nemotron-transcribe
"encoder.layer_norm", # mimo-audio-tokenizer
"encoder.lfm.lfm.embedding_norm", # lfm3a
),
@@ -2595,10 +2599,12 @@ class TensorNameMap:
MODEL_TENSOR.A_ENC_ATTN_Q_NORM: (
"encoder.lfm.lfm.layers.{bid}.self_attn.q_layernorm", # lfm3a
+ "perception.encoder.asr_encoder.layers.{bid}.attn.q_norm", # nemotron-transcribe
),
MODEL_TENSOR.A_ENC_ATTN_K_NORM: (
"encoder.lfm.lfm.layers.{bid}.self_attn.k_layernorm", # lfm3a
+ "perception.encoder.asr_encoder.layers.{bid}.attn.k_norm", # nemotron-transcribe
),
MODEL_TENSOR.A_ENC_ATTN_K_REL: (
@@ -2629,6 +2635,7 @@ class TensorNameMap:
"encoder.lfm.lfm.layers.{bid}.operator_norm", # lfm3a
"conformer.layers.{bid}.attention.pre_attn_norm", # gemma3n
"sound_encoder.encoder.layers.{bid}.norm_self_att", # parakeet
+ "perception.encoder.asr_encoder.layers.{bid}.norm1", # nemotron-transcribe
"encoder.layers.{bid}.attn.pre_norm", # granite_speech
"encoder.layers.{bid}.self_attn_layer_norm", # mimo-audio-tokenizer
),
@@ -2642,6 +2649,7 @@ class TensorNameMap:
"conformer.layers.{bid}.attention.post", # gemma3n
"conformer.layers.{bid}.self_attn.post", # gemma4
"sound_encoder.encoder.layers.{bid}.self_attn.o_proj", # parakeet
+ "perception.encoder.asr_encoder.layers.{bid}.attn.out_proj", # nemotron-transcribe
"encoder.layers.{bid}.attn.to_out", # granite_speech
"encoder.layers.{bid}.self_attn.out_proj", # mimo-audio-tokenizer
),
@@ -2653,6 +2661,7 @@ class TensorNameMap:
"encoder.lfm.lfm.layers.{bid}.ffn_norm", # lfm3a
"conformer.layers.{bid}.attention.post_norm", # gemma3n
"sound_encoder.encoder.layers.{bid}.norm_out", # parakeet
+ "perception.encoder.asr_encoder.layers.{bid}.norm2", # nemotron-transcribe
"encoder.layers.{bid}.post_norm", # granite_speech
"encoder.layers.{bid}.final_layer_norm", # mimo-audio-tokenizer
),
@@ -2683,6 +2692,7 @@ class TensorNameMap:
"conformer.layers.{bid}.ffw_layer_start.ffw_layer_1", # gemma3n
"conformer.layers.{bid}.feed_forward1.ffw_layer_1", # gemma4
"sound_encoder.encoder.layers.{bid}.feed_forward1.linear1", # parakeet
+ "perception.encoder.asr_encoder.layers.{bid}.ffn.net.0", # nemotron-transcribe
"encoder.layers.{bid}.ff1.up_proj", # granite_speech
"encoder.layers.{bid}.fc1", # mimo-audio-tokenizer
),
@@ -2702,6 +2712,7 @@ class TensorNameMap:
"conformer.layers.{bid}.ffw_layer_start.ffw_layer_2", # gemma3n
"conformer.layers.{bid}.feed_forward1.ffw_layer_2", # gemma4
"sound_encoder.encoder.layers.{bid}.feed_forward1.linear2", # parakeet
+ "perception.encoder.asr_encoder.layers.{bid}.ffn.net.3", # nemotron-transcribe
"encoder.layers.{bid}.ff1.down_proj", # granite_speech
"encoder.layers.{bid}.fc2", # mimo-audio-tokenizer
),
@@ -2792,6 +2803,7 @@ class TensorNameMap:
"audio_tower.proj", # qwen2omni
"model.audio_tower.output_proj", # gemma4
"speaker_encoder.fc", # qwen3tts speaker encoder: final speaker embedding projection
+ "perception.proj", # nemotron-transcribe
),
MODEL_TENSOR.A_MM_NORM_PRE: (
@@ -2914,12 +2926,85 @@ class TensorNameMap:
MODEL_TENSOR.A_ENC_MEL_FILTERS: (
"sound_encoder.encoder.feature_extractor.featurizer.fb", # parakeet
+ "perception.preprocessor.featurizer.fb", # nemotron-transcribe
),
MODEL_TENSOR.A_ENC_WINDOW: (
"sound_encoder.encoder.feature_extractor.featurizer.window", # parakeet
+ "perception.preprocessor.featurizer.window", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_ENC_ATTN_QKV: (
+ "perception.encoder.asr_encoder.layers.{bid}.attn.w_qkv", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_SPK_INP_PROJ: (
+ "perception.encoder.diarization_model.encoder.pre_encode.proj", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_SPK_PRE_NORM: (
+ "perception.encoder.diarization_model.encoder.embed_norm", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_SPK_POST_NORM: (
+ "perception.encoder.diarization_model.encoder.final_norm", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_SPK_ATTN_NORM: (
+ "perception.encoder.diarization_model.encoder.layers.{bid}.norm1", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_SPK_ATTN_QKV: (
+ "perception.encoder.diarization_model.encoder.layers.{bid}.attn.w_qkv", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_SPK_ATTN_OUT: (
+ "perception.encoder.diarization_model.encoder.layers.{bid}.attn.out_proj", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_SPK_FFN_NORM: (
+ "perception.encoder.diarization_model.encoder.layers.{bid}.norm2", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_SPK_FFN_UP: (
+ "perception.encoder.diarization_model.encoder.layers.{bid}.ffn.net.0", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_SPK_FFN_DOWN: (
+ "perception.encoder.diarization_model.encoder.layers.{bid}.ffn.net.3", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_SPK_HEAD_PROJ: (
+ "perception.encoder.diarization_model.sortformer_modules.encoder_proj", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_SPK_HEAD_UPSAMPLE: (
+ "perception.encoder.diarization_model.sortformer_modules.subpixel_upsample", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_SPK_HEAD_FC1: (
+ "perception.encoder.diarization_model.sortformer_modules.first_hidden_to_hidden", # nemotron-transcribe
),
+ MODEL_TENSOR.A_SPK_HEAD_FC2: (
+ "perception.encoder.diarization_model.sortformer_modules.single_hidden_to_spks", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_SPK_SIL_EMB: (
+ "perception.encoder.diarization_model.sortformer_modules.learnable_sil_emb", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_FUSION_ASR_NORM: (
+ "perception.encoder.asr_norm", # nemotron-transcribe
+ ),
+
+ MODEL_TENSOR.A_FUSION_SPK_NORM: (
+ "perception.encoder.diar_norm", # nemotron-transcribe
+ ),
+
+ # generated in conversion code (nemotron-transcribe)
+ MODEL_TENSOR.A_FUSION_SPK_KERNEL: (),
+
MODEL_TENSOR.A_MM_EMBEDDING: (
"model.embed_audio.embedding", # gemma3n
),
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index f1ec3c456..e514de60b 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -47,6 +47,7 @@ add_library(mtmd
models/kimik25.cpp
models/nemotron-v2-vl.cpp
models/nemotron-omni.cpp
+ models/nemotron-transcribe.cpp
models/muse-glimmer.cpp
models/muse-spark-audio.cpp
models/llama4.cpp
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 0504e8fdc..52d5fffe3 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -113,6 +113,11 @@
#define KEY_GEN_AUDIO_PENALTY_LAST_N "clip.gen.audio.sampling.penalty_last_n"
#define KEY_AUDIO_SUBSMPL_FACTOR "clip.audio.subsampling_factor"
+// nemotron transcribe
+#define KEY_A_ROPE_DIMS "clip.audio.rope_dims"
+#define KEY_A_SPK_N_BLOCK "clip.audio.spk.block_count"
+#define KEY_A_SPK_N_HEAD "clip.audio.spk.head_count"
+
//
// tensor name constants
//
@@ -469,6 +474,13 @@
#define TN_CONV_NORM_MEAN "%s.blk.%d.conv_norm_mean"
#define TN_CONV_NORM_VAR "%s.blk.%d.conv_norm_var"
+// nemotron transcribe
+#define TN_A_SPK_INP_PROJ "a.spk.input_projection.%s"
+#define TN_A_SPK_HEAD "a.spk.head.%s.%s" // proj, upsample, fc1, fc2
+#define TN_A_SPK_SIL_EMB "a.spk.head.sil_emb"
+#define TN_A_FUSION_NORM "a.fusion.%s.%s" // asr_norm, spk_norm
+#define TN_A_FUSION_KERNEL "a.fusion.spk_kernel"
+
// align x to upper multiple of n
#define CLIP_ALIGN(x, n) ((((x) + (n) - 1) / (n)) * (n))
@@ -487,6 +499,11 @@ struct clip_enc_state {
// lfm3a: same order as list_lfm3a_state_slots(), F32 [ne0, ne1], updated in place
std::vector<ggml_tensor *> slots;
+
+ // nemotron transcribe: the window's normalized log-mel, cache and FIFO rows in; its speaker rows and predictions out
+ std::vector<float> mel_norm;
+ std::vector<float> spk_prev;
+ std::vector<float> spk_chunk, spk_preds;
};
// per-sequence state of audio generation, see clip_gen_state_init()
@@ -567,6 +584,7 @@ enum projector_type {
PROJECTOR_TYPE_MINICPMV4_7,
PROJECTOR_TYPE_MINICPM_OCR_1_0,
PROJECTOR_TYPE_GRANITE_SPEECH,
+ PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE,
PROJECTOR_TYPE_MIMOVL,
PROJECTOR_TYPE_MINIMAX_M3,
PROJECTOR_TYPE_GRANITE4_VISION,
@@ -647,6 +665,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_MINICPMV4_7, "minicpmv4_7"},
{ PROJECTOR_TYPE_MINICPM_OCR_1_0, "minicpm_ocr_1_0"},
{ PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"},
+ { PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE, "nemotron_transcribe"},
{ PROJECTOR_TYPE_MIMOVL, "mimovl"},
{ PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"},
{ PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"},
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h
index 2810a5cc6..c42f417c8 100644
--- a/tools/mtmd/clip-model.h
+++ b/tools/mtmd/clip-model.h
@@ -165,6 +165,12 @@ struct clip_hparams {
std::vector<float> mel_filters;
std::vector<float> window;
+ // nemotron transcribe
+ int32_t audio_rope_dims = 0;
+ int32_t spk_n_embd = 0;
+ int32_t spk_n_layer = 0;
+ int32_t spk_n_head = 0;
+
// mimo-audio-tokenizer: residual vector quantizer
int32_t rvq_num_quantizers = 0;
std::vector<int32_t> rvq_codebook_size; // per-quantizer bin count (ragged, e.g. 1024/1024/256/128x17)
@@ -292,6 +298,8 @@ struct clip_layer {
ggml_tensor * k_norm = nullptr;
ggml_tensor * q_norm = nullptr;
+ ggml_tensor * k_norm_b = nullptr;
+ ggml_tensor * q_norm_b = nullptr;
// lfm3: per-head sigmoid gate on the attention output
ggml_tensor * attn_gate_w = nullptr;
@@ -926,6 +934,28 @@ struct clip_model {
std::vector<clip_layer> sam_layers;
+ // nemotron transcribe: speaker expert, Sortformer head, fusion
+ std::vector<clip_layer> spk_layers;
+ ggml_tensor * spk_inp_proj_w = nullptr;
+ ggml_tensor * spk_pre_ln_w = nullptr;
+ ggml_tensor * spk_pre_ln_b = nullptr;
+ ggml_tensor * spk_post_ln_w = nullptr;
+ ggml_tensor * spk_post_ln_b = nullptr;
+ ggml_tensor * spk_head_proj_w = nullptr;
+ ggml_tensor * spk_head_proj_b = nullptr;
+ ggml_tensor * spk_head_up_w = nullptr;
+ ggml_tensor * spk_head_up_b = nullptr;
+ ggml_tensor * spk_head_fc1_w = nullptr;
+ ggml_tensor * spk_head_fc1_b = nullptr;
+ ggml_tensor * spk_head_fc2_w = nullptr;
+ ggml_tensor * spk_head_fc2_b = nullptr;
+ ggml_tensor * fusion_asr_norm_w = nullptr;
+ ggml_tensor * fusion_asr_norm_b = nullptr;
+ ggml_tensor * fusion_spk_norm_w = nullptr;
+ ggml_tensor * fusion_spk_norm_b = nullptr;
+ ggml_tensor * fusion_spk_kernel = nullptr;
+ std::vector<float> spk_sil_emb;
+
// deepseek-ocr-2
ggml_tensor * resample_query_768 = nullptr;
ggml_tensor * resample_query_1024 = nullptr;
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 9a043d7f9..f2c5c31b8 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -405,13 +405,13 @@ ggml_tensor * clip_graph::build_vit(
if (layer.q_norm) {
GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]);
- Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);
+ Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il);
cb(Qcur, "Qcur_norm", il);
}
if (layer.k_norm) {
GGML_ASSERT(layer.k_norm->ne[0] == Kcur->ne[0]);
- Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);
+ Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il);
cb(Kcur, "Kcur_norm", il);
}
@@ -437,11 +437,11 @@ ggml_tensor * clip_graph::build_vit(
if (!norm_per_head) {
if (layer.q_norm) {
- Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);
+ Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il);
cb(Qcur, "Qcur_norm", il);
}
if (layer.k_norm) {
- Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);
+ Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il);
cb(Kcur, "Kcur_norm", il);
}
}
@@ -452,11 +452,11 @@ ggml_tensor * clip_graph::build_vit(
if (norm_per_head) {
if (layer.q_norm) {
- Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);
+ Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il);
cb(Qcur, "Qcur_norm_per_head", il);
}
if (layer.k_norm) {
- Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);
+ Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il);
cb(Kcur, "Kcur_norm_per_head", il);
}
}
@@ -1242,6 +1242,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_parakeet>(ctx, img);
} break;
+ case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
+ {
+ builder = std::make_unique<clip_graph_nemotron_transcribe>(ctx, img, params ? params->enc_state : nullptr);
+ } break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
builder = std::make_unique<clip_graph_granite4_vision>(ctx, img);
@@ -1623,6 +1627,27 @@ struct clip_model_loader {
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
} break;
+ case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
+ {
+ get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor);
+ get_u32(KEY_A_ROPE_DIMS, hparams.audio_rope_dims);
+ get_u32(KEY_A_SPK_N_BLOCK, hparams.spk_n_layer);
+ get_u32(KEY_A_SPK_N_HEAD, hparams.spk_n_head);
+ if (hparams.subsampling_factor != 8) {
+ throw std::runtime_error(string_format("%s: unsupported subsampling factor %d\n", __func__, hparams.subsampling_factor));
+ }
+ if (hparams.spk_n_layer <= 0 || hparams.spk_n_layer > (int) gguf_get_n_tensors(ctx_gguf.get())) {
+ throw std::runtime_error(string_format("%s: invalid speaker block count %d\n", __func__, hparams.spk_n_layer));
+ }
+ hparams.audio_chunk_len = 0;
+ hparams.audio_sample_rate = 16000;
+ hparams.audio_n_fft = 512;
+ hparams.audio_window_len = 400;
+ hparams.audio_hop_len = 160;
+ // one full window
+ using nt = clip_graph_nemotron_transcribe;
+ hparams.warmup_audio_size = (nt::window_left + nt::window_core + nt::window_right) * hparams.subsampling_factor;
+ } break;
case PROJECTOR_TYPE_IDEFICS3:
{
// use default llava-uhd preprocessing params
@@ -4073,6 +4098,87 @@ struct clip_model_loader {
model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
model.mm_model_mlp_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
} break;
+ case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
+ {
+ hparams.mel_filters = get_vector(TN_MEL_FILTERS);
+ hparams.window = get_vector(TN_WINDOW);
+
+ // ASR expert: blocks, pre_ln and post_ln come from the generic loader
+ model.inp_proj_w = get_tensor(string_format(TN_INP_PROJ, "weight"));
+ for (int il = 0; il < hparams.n_layer; ++il) {
+ auto & layer = model.layers[il];
+ layer.q_norm_b = get_tensor(string_format(TN_ATTN_Q_NORM, prefix, il, "bias"));
+ layer.k_norm_b = get_tensor(string_format(TN_ATTN_K_NORM, prefix, il, "bias"));
+ }
+
+ // speaker expert
+ const char * sp = "a.spk";
+ model.spk_inp_proj_w = get_tensor(string_format(TN_A_SPK_INP_PROJ, "weight"));
+ model.spk_pre_ln_w = get_tensor(string_format(TN_LN_PRE, sp, "weight"));
+ model.spk_pre_ln_b = get_tensor(string_format(TN_LN_PRE, sp, "bias"));
+ model.spk_post_ln_w = get_tensor(string_format(TN_LN_POST, sp, "weight"));
+ model.spk_post_ln_b = get_tensor(string_format(TN_LN_POST, sp, "bias"));
+ model.spk_layers.resize(hparams.spk_n_layer);
+ for (int il = 0; il < hparams.spk_n_layer; ++il) {
+ auto & layer = model.spk_layers[il];
+ layer.ln_1_w = get_tensor(string_format(TN_LN_1, sp, il, "weight"));
+ layer.ln_1_b = get_tensor(string_format(TN_LN_1, sp, il, "bias"));
+ layer.qkv_w = get_tensor(string_format(TN_ATTN_QKV, sp, il, "weight"));
+ layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, sp, il, "weight"));
+ layer.o_b = get_tensor(string_format(TN_ATTN_OUTPUT, sp, il, "bias"));
+ layer.ln_2_w = get_tensor(string_format(TN_LN_2, sp, il, "weight"));
+ layer.ln_2_b = get_tensor(string_format(TN_LN_2, sp, il, "bias"));
+ layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, sp, il, "weight"));
+ layer.ff_up_b = get_tensor(string_format(TN_FFN_UP, sp, il, "bias"));
+ layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, sp, il, "weight"));
+ layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, sp, il, "bias"));
+ }
+
+ // Sortformer head, fusion
+ model.spk_head_proj_w = get_tensor(string_format(TN_A_SPK_HEAD, "proj", "weight"));
+ model.spk_head_proj_b = get_tensor(string_format(TN_A_SPK_HEAD, "proj", "bias"));
+ model.spk_head_up_w = get_tensor(string_format(TN_A_SPK_HEAD, "upsample", "weight"));
+ model.spk_head_up_b = get_tensor(string_format(TN_A_SPK_HEAD, "upsample", "bias"));
+ model.spk_head_fc1_w = get_tensor(string_format(TN_A_SPK_HEAD, "fc1", "weight"));
+ model.spk_head_fc1_b = get_tensor(string_format(TN_A_SPK_HEAD, "fc1", "bias"));
+ model.spk_head_fc2_w = get_tensor(string_format(TN_A_SPK_HEAD, "fc2", "weight"));
+ model.spk_head_fc2_b = get_tensor(string_format(TN_A_SPK_HEAD, "fc2", "bias"));
+ model.fusion_asr_norm_w = get_tensor(string_format(TN_A_FUSION_NORM, "asr_norm", "weight"));
+ model.fusion_asr_norm_b = get_tensor(string_format(TN_A_FUSION_NORM, "asr_norm", "bias"));
+ model.fusion_spk_norm_w = get_tensor(string_format(TN_A_FUSION_NORM, "spk_norm", "weight"));
+ model.fusion_spk_norm_b = get_tensor(string_format(TN_A_FUSION_NORM, "spk_norm", "bias"));
+ model.fusion_spk_kernel = get_tensor(TN_A_FUSION_KERNEL);
+
+ model.spk_sil_emb = get_vector(TN_A_SPK_SIL_EMB);
+
+ // the head layouts and the widths the graph and the speaker state use must match the tensors
+ hparams.spk_n_embd = model.spk_inp_proj_w->ne[1];
+ const int64_t d_head = hparams.n_head > 0 ? hparams.n_embd / hparams.n_head : 0;
+ const int64_t d_head_spk = hparams.spk_n_head > 0 ? hparams.spk_n_embd / hparams.spk_n_head : 0;
+ const int64_t n_spk = model.spk_head_fc2_w->ne[1];
+ // the graph splits each fused QKV into three views of the layer width
+ bool qkv_ok = true;
+ for (const auto & layer : model.layers) {
+ qkv_ok = qkv_ok && layer.qkv_w && layer.qkv_w->ne[0] == hparams.n_embd && layer.qkv_w->ne[1] == 3 * hparams.n_embd;
+ }
+ for (const auto & layer : model.spk_layers) {
+ qkv_ok = qkv_ok && layer.qkv_w->ne[0] == hparams.spk_n_embd && layer.qkv_w->ne[1] == 3 * hparams.spk_n_embd;
+ }
+ if (!qkv_ok || model.inp_proj_w->ne[1] != hparams.n_embd || d_head <= 0 || d_head * hparams.n_head != hparams.n_embd ||
+ (hparams.n_embd_head > 0 && hparams.n_embd_head != d_head) ||
+ hparams.audio_rope_dims <= 0 || hparams.audio_rope_dims % 2 != 0 || hparams.audio_rope_dims > d_head ||
+ d_head_spk <= 0 || d_head_spk % 2 != 0 || d_head_spk * hparams.spk_n_head != hparams.spk_n_embd ||
+ (int64_t) model.spk_sil_emb.size() != hparams.spk_n_embd || n_spk <= 0 ||
+ model.fusion_spk_kernel->ne[0] != n_spk || model.fusion_spk_kernel->ne[1] != hparams.n_embd ||
+ hparams.mel_filters.size() != (size_t) hparams.n_mel_bins * (hparams.audio_n_fft / 2 + 1) ||
+ hparams.window.size() != (size_t) hparams.audio_window_len) {
+ throw std::runtime_error(string_format("%s: hparams do not match the tensor shapes\n", __func__));
+ }
+
+ // projection to the text model
+ model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight"));
+ model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias"));
+ } break;
case PROJECTOR_TYPE_GRANITE_SPEECH:
{
model.inp_proj_w = get_tensor(string_format(TN_INP_PROJ, "weight"));
@@ -5013,6 +5119,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
}
n_patches = n;
} break;
+ case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
case PROJECTOR_TYPE_PARAKEET:
{
n_patches = (img->nx() + (params.subsampling_factor - 1)) / params.subsampling_factor;
@@ -5100,6 +5207,9 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
params.n_threads = n_threads;
params.out_embd = &out_batch_embd;
+ if (ctx->proj_type() == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) {
+ return clip_graph_nemotron_transcribe::encode_windows(ctx, ctx->model, ¶ms);
+ }
return clip_encode(ctx, ¶ms);
}
@@ -6593,6 +6703,20 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
set_lfm3a_window_inputs(base, n_pos, hparams.attn_window_size);
} break;
+ case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
+ {
+ GGML_ASSERT(imgs.entries.size() == 1);
+ std::vector<int32_t> positions(ggml_nelements(get_inp_tensor("positions")));
+ for (size_t i = 0; i < positions.size(); i++) {
+ positions[i] = (int32_t) i;
+ }
+ set_input_i32("positions", positions);
+ GGML_ASSERT(params->enc_state != nullptr);
+ set_input_f32("inp_norm", params->enc_state->mel_norm);
+ if (ggml_graph_get_tensor(gf, "spk_prev") != nullptr) {
+ set_input_f32("spk_prev", params->enc_state->spk_prev);
+ }
+ } break;
case PROJECTOR_TYPE_PARAKEET:
{
GGML_ASSERT(imgs.entries.size() == 1);
@@ -6899,6 +7023,15 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
GGML_ASSERT(t != nullptr);
copy_gen_state_out(t, slots[i++]);
}
+ } else if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) {
+ clip_enc_state * st = params->enc_state;
+ ggml_tensor * chunk = ggml_graph_get_tensor(gf, "spk_chunk");
+ ggml_tensor * preds = ggml_graph_get_tensor(gf, "spk_preds");
+ GGML_ASSERT(chunk != nullptr && preds != nullptr);
+ st->spk_chunk.resize(ggml_nelements(chunk));
+ st->spk_preds.resize(ggml_nelements(preds));
+ ggml_backend_tensor_get(chunk, st->spk_chunk.data(), 0, ggml_nbytes(chunk));
+ ggml_backend_tensor_get(preds, st->spk_preds.data(), 0, ggml_nbytes(preds));
} else if (params->enc_state != nullptr) {
clip_enc_state * st = params->enc_state;
ggml_tensor * tail = ggml_graph_get_tensor(gf, "out_stem_tail");
@@ -7057,6 +7190,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_LLAMA4:
return ctx->model.mm_model_proj->ne[1];
case PROJECTOR_TYPE_QWEN2A:
+ case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
return ctx->model.mm_fc_w->ne[1];
case PROJECTOR_TYPE_QWEN3A:
return ctx->model.mm_2_w->ne[1];
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index c75004d43..dc816fa87 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -30,6 +30,22 @@ struct clip_graph_gemma4uv : clip_graph {
ggml_cgraph * build() override;
};
+struct clip_graph_nemotron_transcribe : clip_graph {
+ const clip_enc_state * state; // nullptr during warmup
+ clip_graph_nemotron_transcribe(clip_ctx * ctx, const clip_image_f32 & img, const clip_enc_state * state)
+ : clip_graph(ctx, img), state(state) {}
+ ggml_cgraph * build() override;
+ ggml_tensor * build_spk_encoder(ggml_tensor * cur, ggml_tensor * pos);
+
+ // one recording: encodes its windows in turn, carrying the speaker state between them
+ static bool encode_windows(clip_ctx * ctx, const clip_model & model, clip_encode_params * params);
+
+ // NeMo ParallelExpertEncoder online defaults, in output frames: 40 s windows with 4 s of context on each side
+ static constexpr int window_core = 500;
+ static constexpr int window_left = 50;
+ static constexpr int window_right = 50;
+};
+
struct clip_graph_pixtral : clip_graph {
clip_graph_pixtral(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/models/nemotron-transcribe.cpp b/tools/mtmd/models/nemotron-transcribe.cpp
new file mode 100644
index 000000000..2ce20c81f
--- /dev/null
+++ b/tools/mtmd/models/nemotron-transcribe.cpp
@@ -0,0 +1,343 @@
+#include "models.h"
+
+#include <algorithm>
+#include <cmath>
+#include <limits>
+#include <numeric>
+
+// NeMo ParallelExpertEncoder: ASR and speaker (Sortformer) transformers, speaker activity fused into the ASR states
+
+// a speaker is active in an 80 ms frame where its prediction exceeds this (speaker_activity_threshold)
+static constexpr float SPK_ACTIVITY_THR = 0.5f;
+
+// NeMo streaming Sortformer: PEE online defaults and the checkpoint's sortformer_modules config
+static constexpr int SPK_FIFO_LEN = 40;
+static constexpr int SPK_CACHE_LEN = 188;
+static constexpr int SPK_SIL_FRAMES = 1;
+static constexpr float SPK_PRED_SCORE_THR = 0.25f;
+static constexpr float SPK_BOOST_LATEST = 0.05f;
+static constexpr float SPK_STRONG_BOOST_RATE = 0.75f;
+static constexpr float SPK_WEAK_BOOST_RATE = 1.5f;
+static constexpr float SPK_MIN_POS_RATE = 0.5f;
+
+// _compress_spkcache runs here on exp(score): its additive log terms become factors and -inf becomes 0
+static constexpr float SPK_SCORE_OFFSET = 2.0f; // -log(0.5) in _get_log_pred_scores
+static constexpr float SPK_STRONG_BOOST = 4.0f; // _boost_topk_scores, scale_factor 2
+static constexpr float SPK_WEAK_BOOST = 2.0f; // _boost_topk_scores, scale_factor 1
+static constexpr float SPK_SPEECH_THR = 0.5f; // _disable_low_scores: a speaker's frames at or below it are not cached
+
+// the speaker expert has its own layers, width and head count, which build_vit does not take; full rotary
+ggml_tensor * clip_graph_nemotron_transcribe::build_spk_encoder(ggml_tensor * cur, ggml_tensor * pos) {
+ const int n_head = hparams.spk_n_head;
+ const int64_t n_embd = cur->ne[0];
+ const int64_t n_pos = cur->ne[1];
+ const int64_t d_head = n_embd / n_head;
+
+ cur = build_norm(cur, model.spk_pre_ln_w, model.spk_pre_ln_b, NORM_TYPE_NORMAL, eps, -1);
+ for (int il = 0; il < (int) model.spk_layers.size(); il++) {
+ const auto & layer = model.spk_layers[il];
+ ggml_tensor * x = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
+
+ ggml_tensor * qkv = build_mm(layer.qkv_w, x);
+ const size_t es = ggml_element_size(qkv);
+ ggml_tensor * q = ggml_view_3d(ctx0, qkv, d_head, n_head, n_pos, d_head * es, qkv->nb[1], 0);
+ ggml_tensor * k = ggml_view_3d(ctx0, qkv, d_head, n_head, n_pos, d_head * es, qkv->nb[1], n_embd * es);
+ ggml_tensor * v = ggml_view_3d(ctx0, qkv, d_head, n_head, n_pos, d_head * es, qkv->nb[1], 2 * n_embd * es);
+ q = ggml_rope_ext(ctx0, q, pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+ k = ggml_rope_ext(ctx0, k, pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+
+ x = build_attn(layer.o_w, layer.o_b, q, k, v, nullptr, 1.0f / sqrtf((float) d_head), il);
+ cur = ggml_add(ctx0, cur, x);
+
+ x = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
+ x = build_ffn(x, layer.ff_up_w, layer.ff_up_b, nullptr, nullptr, layer.ff_down_w, layer.ff_down_b, FFN_GELU_ERF, il);
+ cur = ggml_add(ctx0, cur, x);
+ }
+
+ return build_norm(cur, model.spk_post_ln_w, model.spk_post_ln_b, NORM_TYPE_NORMAL, eps, -1);
+}
+
+ggml_cgraph * clip_graph_nemotron_transcribe::build() {
+ const int sub = hparams.subsampling_factor;
+ const int64_t n_mel = hparams.n_mel_bins;
+ const int64_t n_mel_frames = img.nx();
+ const int64_t n_pos = (n_mel_frames + sub - 1) / sub;
+ GGML_ASSERT(img.ny() == n_mel);
+
+ // warmup: the largest graph, with a full speaker cache and FIFO
+ const int64_t n_prev = state == nullptr ? SPK_CACHE_LEN + SPK_FIFO_LEN
+ : (int64_t) state->spk_prev.size() / hparams.spk_n_embd;
+
+ // [n_frames, n_mel] log-mel: raw for the speaker expert, normalized over the recording (encode_windows) for the ASR one
+ ggml_tensor * inp = build_inp_raw(1);
+ ggml_tensor * inp_norm = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_mel_frames, n_mel);
+ ggml_set_name(inp_norm, "inp_norm");
+ ggml_set_input(inp_norm);
+ auto stack_mel = [&](ggml_tensor * mel) {
+ return build_stack(ggml_cont(ctx0, ggml_transpose(ctx0, mel)), sub, n_mel);
+ };
+
+ // positions over [cache | FIFO | window]; the ASR expert takes the first n_pos
+ ggml_tensor * pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_prev + n_pos);
+ ggml_set_name(pos, "positions");
+ ggml_set_input(pos);
+
+ ggml_tensor * spk_prev = nullptr;
+ if (n_prev > 0) {
+ spk_prev = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.spk_n_embd, n_prev);
+ ggml_set_name(spk_prev, "spk_prev");
+ ggml_set_input(spk_prev);
+ }
+
+ ggml_tensor * asr_pos = ggml_view_1d(ctx0, pos, n_pos, 0);
+ auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ return ggml_rope_ext(ctx0, cur, asr_pos, nullptr, hparams.audio_rope_dims, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f,
+ 0.0f, 1.0f, 0.0f, 0.0f);
+ };
+ ggml_tensor * asr = build_vit(build_mm(model.inp_proj_w, stack_mel(inp_norm)), n_pos, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr,
+ add_pos);
+ cb(asr, "asr_out", -1);
+
+ // speaker expert over [cache | FIFO | window]; the window's rows and predictions feed the next window's state
+ ggml_tensor * chunk = build_mm(model.spk_inp_proj_w, stack_mel(inp));
+ ggml_set_name(chunk, "spk_chunk");
+ ggml_set_output(chunk);
+ ggml_tensor * spk = spk_prev ? ggml_concat(ctx0, spk_prev, chunk, 1) : chunk;
+ const int64_t n_spk_pos = n_prev + n_pos;
+ spk = build_spk_encoder(spk, pos);
+ cb(spk, "spk_out", -1);
+
+ // Sortformer head, 10 ms frames
+ ggml_tensor * cur = ggml_add(ctx0, build_mm(model.spk_head_proj_w, spk), model.spk_head_proj_b);
+ const int64_t d_tf = cur->ne[0];
+ cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+ cur = ggml_conv_1d(ctx0, model.spk_head_up_w, cur, 1, 1, 1); // [n_spk_pos, sub*d_tf]
+ cur = ggml_add(ctx0, cur, ggml_reshape_2d(ctx0, model.spk_head_up_b, 1, sub * d_tf));
+ cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+ cur = ggml_reshape_2d(ctx0, cur, d_tf, sub * n_spk_pos);
+ cur = ggml_add(ctx0, build_mm(model.spk_head_fc1_w, ggml_relu(ctx0, cur)), model.spk_head_fc1_b);
+ cur = ggml_add(ctx0, build_mm(model.spk_head_fc2_w, ggml_relu(ctx0, cur)), model.spk_head_fc2_b);
+ ggml_tensor * probs = ggml_sigmoid(ctx0, cur); // [n_spk, sub*n_spk_pos]
+ cb(probs, "spk_probs", -1);
+
+ // 80 ms predictions
+ ggml_tensor * preds = ggml_pool_2d(ctx0, probs, GGML_OP_POOL_AVG, 1, sub, 1, sub, 0, 0); // [n_spk, n_spk_pos]
+ ggml_set_name(preds, "spk_preds");
+ ggml_set_output(preds);
+
+ // fusion, on the window's rows
+ preds = ggml_view_2d(ctx0, preds, preds->ne[0], n_pos, preds->nb[1], n_prev * preds->nb[1]);
+ ggml_tensor * act = ggml_step(ctx0, ggml_scale_bias(ctx0, preds, 1.0f, -SPK_ACTIVITY_THR));
+ act = build_norm(act, model.fusion_spk_norm_w, model.fusion_spk_norm_b, NORM_TYPE_NORMAL, eps, -1);
+ cur = build_norm(asr, model.fusion_asr_norm_w, model.fusion_asr_norm_b, NORM_TYPE_NORMAL, eps, -1);
+ cur = ggml_add(ctx0, cur, build_mm(model.fusion_spk_kernel, act));
+ cb(cur, "fused", -1);
+
+ cur = ggml_add(ctx0, build_mm(model.mm_fc_w, cur), model.mm_fc_b);
+ cb(cur, "projected", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+}
+
+// indices of the k largest of v[0, n), ties to the lower index
+static std::vector<int> spk_topk(const float * v, int n, int k) {
+ std::vector<int> idx(n);
+ std::iota(idx.begin(), idx.end(), 0);
+ k = std::clamp(k, 0, n);
+ std::partial_sort(idx.begin(), idx.begin() + k, idx.end(), [&](int a, int b) {
+ return v[a] > v[b] || (v[a] == v[b] && a < b);
+ });
+ idx.resize(k);
+ return idx;
+}
+
+// NeMo SortformerModules._compress_spkcache: keep the SPK_CACHE_LEN highest-scoring rows, speaker by speaker
+static void spk_compress_cache(std::vector<float> & emb, std::vector<float> & preds, int n, int d, int n_spk,
+ const std::vector<float> & sil_emb) {
+ const int per_spk = std::max(0, SPK_CACHE_LEN / n_spk - SPK_SIL_FRAMES);
+ const int strong = (int) std::floor(per_spk * SPK_STRONG_BOOST_RATE);
+ const int weak = (int) std::floor(per_spk * SPK_WEAK_BOOST_RATE);
+ const int min_pos = (int) std::floor(per_spk * SPK_MIN_POS_RATE);
+ const float disabled = 0.0f;
+ const float latest = std::exp(SPK_BOOST_LATEST);
+
+ // speaker-major: each speaker's n rows, then its silence rows (+inf)
+ const int nt = n + SPK_SIL_FRAMES;
+ std::vector<float> scores((size_t) n_spk * nt, std::numeric_limits<float>::infinity());
+ for (int s = 0; s < n_spk; s++) {
+ float * sc = scores.data() + (size_t) s * nt;
+ int n_pos = 0;
+ for (int t = 0; t < n; t++) {
+ const float * pt = preds.data() + (size_t) t * n_spk;
+ float prod = 1.0f;
+ for (int j = 0; j < n_spk; j++) {
+ if (j != s) {
+ prod *= std::max(1.0f - pt[j], SPK_PRED_SCORE_THR);
+ }
+ }
+ prod *= std::max(pt[s], SPK_PRED_SCORE_THR);
+ sc[t] = pt[s] > SPK_SPEECH_THR ? prod * SPK_SCORE_OFFSET : disabled;
+ n_pos += sc[t] > 1.0f;
+ }
+ if (n_pos >= min_pos) {
+ for (int t = 0; t < n; t++) {
+ if (!(sc[t] > 1.0f)) {
+ sc[t] = disabled;
+ }
+ }
+ }
+ for (int t = SPK_CACHE_LEN; t < n; t++) {
+ sc[t] *= latest;
+ }
+ for (int t : spk_topk(sc, n, strong)) {
+ sc[t] *= SPK_STRONG_BOOST;
+ }
+ for (int t : spk_topk(sc, n, weak)) {
+ sc[t] *= SPK_WEAK_BOOST;
+ }
+ }
+
+ std::vector<int> pick = spk_topk(scores.data(), (int) scores.size(), SPK_CACHE_LEN);
+ const int placeholder = (int) scores.size();
+ for (int & i : pick) {
+ if (scores[i] == disabled) {
+ i = placeholder;
+ }
+ }
+ std::sort(pick.begin(), pick.end());
+
+ std::vector<float> out_emb((size_t) SPK_CACHE_LEN * d);
+ std::vector<float> out_preds((size_t) SPK_CACHE_LEN * n_spk, 0.0f);
+ for (int i = 0; i < SPK_CACHE_LEN; i++) {
+ const int t = pick[i] == placeholder ? -1 : pick[i] % nt;
+ float * e = out_emb.data() + (size_t) i * d;
+ if (t < 0 || t >= n) {
+ std::copy(sil_emb.begin(), sil_emb.end(), e);
+ } else {
+ std::copy_n(emb.data() + (size_t) t * d, d, e);
+ std::copy_n(preds.data() + (size_t) t * n_spk, n_spk, out_preds.data() + (size_t) i * n_spk);
+ }
+ }
+ emb.swap(out_emb);
+ preds.swap(out_preds);
+}
+
+// speaker cache, with its predictions, and FIFO of one recording
+struct spk_stream {
+ std::vector<float> cache, cache_preds;
+ std::vector<float> fifo;
+};
+
+// every update has a full window, which overflows the FIFO and the cache (NeMo's update period of 300 never applies)
+static_assert(clip_graph_nemotron_transcribe::window_core - SPK_FIFO_LEN > SPK_CACHE_LEN);
+
+// NeMo SortformerModules.streaming_update: core rows join the FIFO, its overflow moves to the cache, which is compressed
+static void update_spk_state(spk_stream & ss, const clip_enc_state & st, const clip_model & model, int64_t core_begin,
+ int64_t n_core) {
+ const int64_t d = model.hparams.spk_n_embd;
+ const int64_t n_spk = model.spk_head_fc2_w->ne[1];
+ const int64_t n_cache = (int64_t) ss.cache.size() / d;
+ const int64_t n_fifo = (int64_t) ss.fifo.size() / d;
+ const int64_t n_rows = (int64_t) st.spk_chunk.size() / d;
+ GGML_ASSERT((n_cache + n_fifo + n_rows) * n_spk == (int64_t) st.spk_preds.size() && core_begin + n_core <= n_rows);
+ const float * preds = st.spk_preds.data();
+ const float * core_preds = preds + (n_cache + n_fifo + core_begin) * n_spk;
+
+ std::vector<float> fifo_preds(preds + n_cache * n_spk, preds + (n_cache + n_fifo) * n_spk);
+ ss.fifo.insert(ss.fifo.end(), st.spk_chunk.begin() + core_begin * d, st.spk_chunk.begin() + (core_begin + n_core) * d);
+ fifo_preds.insert(fifo_preds.end(), core_preds, core_preds + n_core * n_spk);
+
+ const int64_t n_pop = n_fifo + n_core - SPK_FIFO_LEN;
+ ss.cache.insert(ss.cache.end(), ss.fifo.begin(), ss.fifo.begin() + n_pop * d);
+ ss.cache_preds.insert(ss.cache_preds.end(), fifo_preds.begin(), fifo_preds.begin() + n_pop * n_spk);
+ ss.fifo.erase(ss.fifo.begin(), ss.fifo.begin() + n_pop * d);
+ spk_compress_cache(ss.cache, ss.cache_preds, (int) (ss.cache.size() / d), (int) d, (int) n_spk, model.spk_sil_emb);
+}
+
+// NeMo ParallelExpertEncoder._forward_online: windows with left/right context, one graph each, the speaker state carried
+bool clip_graph_nemotron_transcribe::encode_windows(clip_ctx * ctx, const clip_model & model, clip_encode_params * params) {
+ GGML_ASSERT(params->imgs->entries.size() == 1 && params->out_embd != nullptr);
+ const clip_image_f32 & rec = params->imgs->entries[0];
+ const auto & buf = rec.get_ro_buf();
+ const int64_t n_frames = rec.nx();
+ const int64_t n_rows = rec.ny();
+ const int64_t n_embd = clip_n_mmproj_embd(ctx);
+ const int sub = model.hparams.subsampling_factor;
+ const int64_t core = (int64_t) window_core * sub;
+ const int64_t left = (int64_t) window_left * sub;
+ const int64_t right = (int64_t) window_right * sub;
+ const int64_t n_win = std::max<int64_t>(1, (n_frames + core - 1) / core);
+ GGML_ASSERT((int64_t) buf.size() == n_frames * n_rows);
+
+ auto & out = *params->out_embd;
+ GGML_ASSERT(out.size() == (size_t) n_embd * clip_n_output_tokens(ctx, &rec));
+
+ // the ASR expert's input: per-feature normalization over the whole recording (NeMo normalize_batch, per_feature)
+ std::vector<double> mean(n_rows), denom(n_rows);
+ for (int64_t r = 0; r < n_rows; r++) {
+ const float * row = buf.data() + r * n_frames;
+ double sum = 0.0;
+ double sq_diff_sum = 0.0;
+ for (int64_t t = 0; t < n_frames; t++) {
+ sum += (double) row[t];
+ }
+ mean[r] = sum / n_frames;
+ for (int64_t t = 0; t < n_frames; t++) {
+ const double diff = (double) row[t] - mean[r];
+ sq_diff_sum += diff * diff;
+ }
+ denom[r] = std::sqrt(sq_diff_sum / (n_frames - 1.0)) + 1e-5;
+ }
+
+ spk_stream ss;
+ clip_enc_state st;
+ size_t n_out = 0;
+ for (int64_t w = 0; w < n_win; w++) {
+ const int64_t core_begin = w * core;
+ const int64_t core_end = std::min(core_begin + core, n_frames);
+ const int64_t win_begin = std::max<int64_t>(core_begin - left, 0);
+ const int64_t win_end = std::min(core_end + right, n_frames);
+ const int64_t win_len = win_end - win_begin;
+
+ std::vector<float> data((size_t) win_len * n_rows);
+ st.mel_norm.resize(data.size());
+ for (int64_t r = 0; r < n_rows; r++) {
+ const float * src = buf.data() + r * n_frames + win_begin;
+ std::copy_n(src, win_len, data.data() + r * win_len);
+ for (int64_t t = 0; t < win_len; t++) {
+ st.mel_norm[r * win_len + t] = (float) ((src[t] - mean[r]) / denom[r]);
+ }
+ }
+ clip_image_f32_batch batch;
+ batch.is_audio = true;
+ batch.entries.emplace_back();
+ batch.entries[0].set_size({(int) win_len, (int) n_rows}, false, true);
+ batch.entries[0].cpy_buf(data);
+
+ std::vector<float> embd((size_t) n_embd * clip_n_output_tokens(ctx, &batch.entries[0]));
+ clip_encode_params wp = *params;
+ wp.imgs = &batch;
+ wp.out_embd = &embd;
+ wp.enc_state = &st;
+ if (!clip_encode(ctx, &wp)) {
+ return false;
+ }
+
+ // output frames of the core, rounded up as NeMo's _asr_output_frame_boundary
+ const int64_t n_left = (core_begin - win_begin) / sub;
+ const int64_t n_core = (core_end - core_begin + sub - 1) / sub;
+ GGML_ASSERT((n_out + n_core) * n_embd <= out.size());
+ std::copy_n(embd.data() + n_left * n_embd, n_core * n_embd, out.data() + n_out * n_embd);
+ n_out += n_core;
+
+ if (w + 1 < n_win) {
+ update_spk_state(ss, st, model, n_left, n_core);
+ st.spk_prev = ss.cache;
+ st.spk_prev.insert(st.spk_prev.end(), ss.fifo.begin(), ss.fifo.end());
+ }
+ }
+ GGML_ASSERT(n_out * n_embd == out.size());
+ return true;
+}
diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp
index 14b5a7213..6e1ee2324 100644
--- a/tools/mtmd/mtmd-audio.cpp
+++ b/tools/mtmd/mtmd-audio.cpp
@@ -1385,13 +1385,8 @@ void mtmd_audio_preprocessor_parakeet::initialize() {
cache.hann_window = hparams.window;
}
-bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples,
- size_t n_samples_in,
- std::vector<mtmd_audio_mel> & output) const {
- if (n_samples_in == 0) {
- return false;
- }
-
+// log-mel without normalization
+void mtmd_audio_preprocessor_parakeet::log_mel(const float * samples, size_t n_samples_in, mtmd_audio_mel & out_full) const {
filter_params params;
params.n_mel = hparams.n_mel_bins;
params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
@@ -1422,7 +1417,6 @@ bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples,
std::vector<float> samples_padded(n_samples_in + 2 * pad, 0.0f);
std::copy(samples_preprocessed.begin(), samples_preprocessed.end(), samples_padded.begin() + pad);
- mtmd_audio_mel out_full;
out_full.n_mel = params.n_mel;
out_full.n_len = (samples_padded.size() - frame_size) / frame_step + 1;
out_full.n_len_org = out_full.n_len;
@@ -1461,11 +1455,22 @@ bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples,
for (int iw = 0; iw < n_threads - 1; ++iw) {
workers[iw].join();
}
+}
+
+bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples,
+ size_t n_samples_in,
+ std::vector<mtmd_audio_mel> & output) const {
+ if (n_samples_in == 0) {
+ return false;
+ }
+
+ mtmd_audio_mel out_full;
+ log_mel(samples, n_samples_in, out_full);
// Per-feature normalization (only on valid frames)
{
const double eps = 1e-5;
- int valid_frames = n_samples_in / frame_step;
+ int valid_frames = n_samples_in / hparams.audio_hop_len;
for (int j = 0; j < out_full.n_mel; j++) {
double sum = 0.0;
@@ -1497,6 +1502,32 @@ bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples,
return true;
}
+//
+// mtmd_audio_preprocessor_nemotron_transcribe implementation
+//
+
+bool mtmd_audio_preprocessor_nemotron_transcribe::preprocess(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output) const {
+ if (n_samples < (size_t) hparams.audio_n_fft) {
+ return false;
+ }
+
+ mtmd_audio_mel mel;
+ log_mel(samples, n_samples, mel);
+
+ // frames past n_samples / hop are padding (NeMo get_seq_len); kept raw, encode_windows normalizes them for ASR
+ const int64_t n_frames = n_samples / hparams.audio_hop_len;
+ for (int64_t j = 1; j < mel.n_mel; j++) {
+ std::copy_n(mel.data.begin() + j * mel.n_len, n_frames, mel.data.begin() + j * n_frames);
+ }
+ mel.n_len = n_frames;
+ mel.n_len_org = n_frames;
+ mel.data.resize(mel.n_mel * n_frames);
+ output.push_back(std::move(mel));
+ return true;
+}
+
// mtmd_audio_preprocessor_gemma4ua
//
diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h
index e9194ad5e..087e4b8c0 100644
--- a/tools/mtmd/mtmd-audio.h
+++ b/tools/mtmd/mtmd-audio.h
@@ -176,6 +176,9 @@ struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor {
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ protected:
+ void log_mel(const float * samples, size_t n_samples, mtmd_audio_mel & out) const;
+
private:
mtmd_audio_cache cache;
@@ -186,6 +189,12 @@ struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor {
const mtmd_audio_cache & cache, mtmd_audio_mel & mel);
};
+// one chunk per recording: the raw log-mel of its valid frames
+struct mtmd_audio_preprocessor_nemotron_transcribe : mtmd_audio_preprocessor_parakeet {
+ mtmd_audio_preprocessor_nemotron_transcribe(clip_ctx * ctx) : mtmd_audio_preprocessor_parakeet(ctx) { }
+ bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+};
+
//
// streaming ISTFT - converts spectrogram frames back to audio one frame at a time
//
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index d4ea23852..c1b024a88 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -1153,6 +1153,10 @@ struct mtmd_context {
{
audio_preproc = std::make_unique<mtmd_audio_preprocessor_parakeet>(ctx_a);
} break;
+ case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
+ {
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_nemotron_transcribe>(ctx_a);
+ } break;
case PROJECTOR_TYPE_GEMMA4UA:
{
aud_beg = "<|audio>";