Commit cd0a5b6a1 for llama.cpp
commit cd0a5b6a180dff0048e5fd993098b21c0b2c91b8
Author: Kartik Gulia <kgulia@nvidia.com>
Date: Thu Oct 15 19:04:56 2026 +0530
models : add Nemotron Super 3.5 VL and Nemotron Bolt 74B A9B support (#30520)
* add support
* remove unneeded bos fix
* add video flag
* fix flake error
* remove video-specific budgeting
* convert : fix lilicorr-dflash bid clash
* spec : enforce lilicorr n_tokens condition
* spec : fix lilicorr walk
[no release]
diff --git a/common/speculative.cpp b/common/speculative.cpp
index 701093308..d8afd4095 100644
--- a/common/speculative.cpp
+++ b/common/speculative.cpp
@@ -1338,7 +1338,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
GGML_ASSERT(lattice && "DFlash2 selector produced no lattice");
int32_t predecessor = 0;
- for (int32_t i = 1; i < n_block_tokens; ++i) {
+ for (int32_t i = 1; i < (is_dflash2 ? n_block_tokens : params.n_max + 1); ++i) {
const float * row = lattice + (size_t) (beg + i) * n_embd_dec;
const int32_t top_k = is_dflash2 ? selector_top_k : lilicorr_top_k;
const float * scores = row + top_k + (size_t) predecessor * top_k;
diff --git a/conversion/__init__.py b/conversion/__init__.py
index d5b480a9b..8ce4cb733 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -353,6 +353,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"MiniCPMV4_6ForConditionalGeneration": "minicpm",
"MiniCPMV4_7ForConditionalGeneration": "minicpm",
"Mistral3ForConditionalGeneration": "llava",
+ "NemotronH_Omni_Reasoning_V3": "nemotron",
"NemotronH_Nano_VL_V2": "nemotron",
"MuseGlimmerForConditionalGeneration": "muse_glimmer",
"PaddleOCRVisionModel": "ernie",
diff --git a/conversion/base.py b/conversion/base.py
index efb7ea72e..a5a80f27d 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -568,7 +568,7 @@ class ModelBase:
tensors_to_remove.append(base_name + "_zero_point")
else:
raise NotImplementedError(f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported")
- elif quant_method == "modelopt":
+ elif quant_method in ("modelopt", "modelopt_mixed"):
dequant_fp8()
elif quant_method is not None:
raise NotImplementedError(f"Quant method is not yet supported: {quant_method!r}")
diff --git a/conversion/nemotron.py b/conversion/nemotron.py
index 581f41a02..c90048194 100644
--- a/conversion/nemotron.py
+++ b/conversion/nemotron.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-from typing import Any, Callable, Iterable, TYPE_CHECKING
+from typing import Any, Callable, Iterable, Sequence, TYPE_CHECKING
import numpy as np
import torch
@@ -89,19 +89,22 @@ class LiLiCorrDraftModel(DFlashModel):
self.gguf_writer.add_lilicorr_vector_eps(float(dflash_config["lilicorr_vector_eps"]))
self.gguf_writer.add_lilicorr_logit_scale(float(dflash_config["lilicorr_logit_scale"]))
+ def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", ".bias")) -> str:
+ if name.startswith("model.lilicorr.layers."):
+ parts = name.split(".", 4)
+ if not parts[4].startswith("attn.in_proj."):
+ parts[3] = str(self.block_count + int(parts[3]))
+ name = ".".join(parts)
+ elif name.startswith("model.lilicorr.feature_norm."):
+ feature_bid = self.block_count + self.lilicorr_geometry["num_layers"]
+ name = name.replace("model.lilicorr.feature_norm.", f"model.lilicorr.feature_mlp.{feature_bid}.norm.")
+ elif name.startswith("model.lilicorr.feature_mlp."):
+ feature_bid = self.block_count + self.lilicorr_geometry["num_layers"]
+ name = name.replace("model.lilicorr.feature_mlp.", f"model.lilicorr.feature_mlp.{feature_bid}.")
+ return super().map_tensor_name(name, try_suffixes)
+
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if name.startswith("model.lilicorr."):
- if name.startswith("model.lilicorr.layers."):
- parts = name.split(".", 4)
- if not parts[4].startswith("attn.in_proj."):
- parts[3] = str(self.block_count + int(parts[3]))
- name = ".".join(parts)
- elif name.startswith("model.lilicorr.feature_norm."):
- feature_bid = self.block_count + self.lilicorr_geometry["num_layers"]
- name = name.replace("model.lilicorr.feature_norm.", f"model.lilicorr.feature_mlp.{feature_bid}.norm.")
- elif name.startswith("model.lilicorr.feature_mlp."):
- feature_bid = self.block_count + self.lilicorr_geometry["num_layers"]
- name = name.replace("model.lilicorr.feature_mlp.", f"model.lilicorr.feature_mlp.{feature_bid}.")
yield self.map_tensor_name(name), data_torch
return
yield from super().modify_tensors(data_torch, name, bid)
@@ -251,6 +254,113 @@ class NemotronNanoV2VLModel(MmprojModel):
yield mapped_name, tensor
+@ModelBase.register("NemotronH_Omni_Reasoning_V3")
+class NemotronOmniVLModel(NemotronNanoV2VLModel):
+ _omni_tensor_prefixes = (
+ "vision_model.embeddings.",
+ "vision_model.encoder.layer.",
+ "vision_projector.mlp1.",
+ )
+
+ def get_vision_config(self) -> dict[str, Any] | None:
+ vision_config = dict(self.global_config["vision_config"])
+ vision_config["intermediate_size"] = int(vision_config["hidden_size"] * vision_config["mlp_ratio"])
+ vision_config["image_size"] = 512 # warmup only; the full position table is resized at runtime
+ return vision_config
+
+ def set_gguf_parameters(self):
+ preprocessor_config = self.preprocessor_config
+
+ preprocessor_config["image_mean"] = preprocessor_config["norm_mean"]
+ preprocessor_config["image_std"] = preprocessor_config["norm_std"]
+
+ MmprojModel.set_gguf_parameters(self)
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.NEMOTRON_OMNI)
+ self.gguf_writer.add_vision_attention_layernorm_eps(1e-6)
+ self.gguf_writer.add_vision_use_gelu(True)
+ downsample_ratio = self.global_config.get("downsample_ratio", 0.5)
+ self.gguf_writer.add_vision_projector_scale_factor(int(1.0 / downsample_ratio))
+ patch_area = self.hparams["patch_size"] ** 2
+ self.gguf_writer.add_vision_min_pixels(preprocessor_config["min_num_patches"] * patch_area)
+ self.gguf_writer.add_vision_max_pixels(preprocessor_config["max_num_patches"] * patch_area)
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, gen = item
+
+ # Transformers can rename RADIO and projector tensors in Omni checkpoints.
+ # Accept both checkpoint layouts.
+ if name == "vision_model.summary_idxs":
+ # This buffer is only used by RadioModel.summary. The multimodal
+ # projector consumes RadioModel.features for both images and video.
+ return None
+ if name.startswith(cls._omni_tensor_prefixes):
+ return name, gen
+
+ if "radio_model.model.patch_generator.video_embedder" in name or name.startswith("vision_projector.vision_final_layernorm."):
+ return MmprojModel.filter_tensors((name, gen))
+
+ return super().filter_tensors(item)
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ if new_name in ("v.position_embd.weight", "v.class_embd"):
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name.startswith(self._omni_tensor_prefixes):
+ if name == "vision_model.embeddings.video_patch_projection.weight":
+ data_torch = data_torch.reshape(
+ self.hparams["hidden_size"],
+ 6,
+ self.hparams["patch_size"],
+ self.hparams["patch_size"],
+ )
+ yield gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch
+ return
+
+ if name.startswith("vision_projector.mlp1."):
+ projector_mapping = {
+ "vision_projector.mlp1.norm.weight": "mlp1.0.weight",
+ "vision_projector.mlp1.linear1.weight": "mlp1.1.weight",
+ "vision_projector.mlp1.linear2.weight": "mlp1.3.weight",
+ }
+ name = projector_mapping[name]
+ elif name == "vision_model.embeddings.patch_projection.weight":
+ data_torch = data_torch.reshape(
+ self.hparams["hidden_size"],
+ 3,
+ self.hparams["patch_size"],
+ self.hparams["patch_size"],
+ )
+ elif name == "vision_model.embeddings.position_embedding":
+ data_torch = data_torch.squeeze(0)
+ name += ".weight"
+ elif ".layer_scale" in name and name.endswith(".lambda1"):
+ name = name.removesuffix("lambda1") + "weight"
+
+ yield from MmprojModel.modify_tensors(self, data_torch, name, bid)
+ return
+
+ if "patch_generator.pos_embed" in name:
+ # Keep the full table for runtime interpolation.
+ yield from MmprojModel.modify_tensors(self, data_torch, name, bid)
+ return
+
+ if "patch_generator.video_embedder" in name:
+ patch_size = self.hparams["patch_size"]
+ n_embd = self.hparams["hidden_size"]
+ temporal_patch_size = self.global_config["video_temporal_patch_size"]
+ if temporal_patch_size != 2:
+ raise ValueError("Current implementation only supports a temporal patch size of 2")
+ data_torch = data_torch.reshape(n_embd, temporal_patch_size * 3, patch_size, patch_size)
+ new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1"
+ yield new_name, data_torch
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
@ModelBase.register("NemotronForCausalLM")
@ModelBase.example("nvidia/Minitron-4B-Base")
class NemotronModel(TextModel):
@@ -382,6 +492,8 @@ class NemotronHModel(GraniteHybridModel):
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, gen = item
+ if name.startswith("language_model."):
+ name = name[len("language_model."):]
if name.startswith("mtp."):
# --no-mtp: drop the MTP head entirely
if cls.no_mtp:
@@ -531,6 +643,14 @@ class NemotronHModel(GraniteHybridModel):
"mtp.layers.1.final_layernorm.weight": "model.layers.{bid}.shared_head.norm.weight",
}
+ def _generate_nvfp4_tensors(self):
+ if self._mtp_bid is not None:
+ for name in list(self.model_tensors):
+ if name.startswith("mtp.layers.1.mixer"):
+ new_name = f"backbone.layers.{self._mtp_bid}.{name.split('.', 3)[3]}"
+ self.model_tensors[new_name] = self.model_tensors.pop(name)
+ super()._generate_nvfp4_tensors()
+
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# mtp.layers.0: NextN input fusion + attention
# mtp.layers.1: MoE + final head norm
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 0c9872228..a39d046fe 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -6265,6 +6265,7 @@ class VisionProjectorType:
GLM5V = "glm5v"
YOUTUVL = "youtuvl"
NEMOTRON_V2_VL = "nemotron_v2_vl"
+ NEMOTRON_OMNI = "nemotron_omni"
QWEN3TTS_SPKENC = "qwen3tts_spkenc" # audio: ECAPA-TDNN speaker encoder
QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor
POCKETTTS_SPKENC = "pockettts_spkenc" # audio: mimi encoder as voice-prompt encoder
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index 5eab5b1fd..63301cc11 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -1735,6 +1735,7 @@ class TensorNameMap:
"vision_model.class_embedding", # llama 4
"model.vision.patch_embedding.cls_embedding", # cogvlm
"vision_model.radio_model.model.patch_generator.cls_token.token", # Nemotron Nano v2 VL
+ "vision_model.embeddings.cls_register_token", # Nemotron Omni
"model.vision_model.embeddings.class_embedding", # Deepseek-OCR
),
@@ -1756,6 +1757,7 @@ class TensorNameMap:
"model.vision_model.embeddings.patch_embedding", # Deepseek-OCR CLIP
"siglip2.vision_model.embeddings.patch_embedding",
"vision_model.radio_model.model.patch_generator.embedder", # Nemotron Nano v2 VL
+ "vision_model.embeddings.patch_projection", # Nemotron Omni
"model.vision_tower.patch_embedder.input_proj", # gemma4
"vision_tower.patch_embed.patchifier.proj", # dots.ocr
"vision_encoder.patch_embed.proj", # dots3note
@@ -1788,6 +1790,7 @@ class TensorNameMap:
"model.vision.patch_embedding.position_embedding", # cogvlm
"visual.embeddings.position_embedding", # glm4v
"vision_model.radio_model.model.patch_generator.pos_embed", # Nemotron Nano v2 VL
+ "vision_model.embeddings.position_embedding", # Nemotron Omni
"model.vision_tower.patch_embedder.position_embedding_table", # gemma4
"vision_model.positional_embedding", # Step3-VL
"model.vision_embedder.pos_embedding", # gemma4 unified
@@ -1818,6 +1821,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_ATTN_Q: (
+ "vision_model.encoder.layer.{bid}.attention.attention.query", # Nemotron Omni
"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.q_proj", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.self_attn.q_proj",
"model.vision_tower.encoder.layers.{bid}.self_attn.q_proj", # minicpmv4_6
@@ -1846,6 +1850,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_ATTN_K: (
+ "vision_model.encoder.layer.{bid}.attention.attention.key", # Nemotron Omni
"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.k_proj", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.self_attn.k_proj",
"model.vision_tower.encoder.layers.{bid}.self_attn.k_proj", # minicpmv4_6
@@ -1874,6 +1879,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_ATTN_V: (
+ "vision_model.encoder.layer.{bid}.attention.attention.value", # Nemotron Omni
"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.v_proj", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.self_attn.v_proj",
"model.vision_tower.encoder.layers.{bid}.self_attn.v_proj", # minicpmv4_6
@@ -1894,6 +1900,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_INPUT_NORM: (
+ "vision_model.encoder.layer.{bid}.norm1", # Nemotron Omni
"vision.blocks.{bid}.norm1", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.layer_norm1",
@@ -1920,6 +1927,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_ATTN_O: (
+ "vision_model.encoder.layer.{bid}.attention.output.dense", # Nemotron Omni
"vision.blocks.{bid}.attn.wo", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj",
@@ -1952,6 +1960,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_POST_ATTN_NORM: (
+ "vision_model.encoder.layer.{bid}.norm2", # Nemotron Omni
"vision.blocks.{bid}.norm2", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.layer_norm2",
@@ -1979,6 +1988,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_FFN_UP: (
+ "vision_model.encoder.layer.{bid}.mlp.fc1", # Nemotron Omni
"vision.blocks.{bid}.mlp.w1_up", # deepseek4v (split from fused w1)
"vision_encoder.blocks.{bid}.mlp.fc3", # dots3note
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision
@@ -2016,6 +2026,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_FFN_DOWN: (
+ "vision_model.encoder.layer.{bid}.mlp.fc2", # Nemotron Omni
"vision.blocks.{bid}.mlp.w2", # deepseek4v
"vision_encoder.blocks.{bid}.mlp.fc2", # dots3note
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision
@@ -2074,12 +2085,14 @@ class TensorNameMap:
),
MODEL_TENSOR.V_LAYER_SCALE_1: (
+ "vision_model.encoder.layer.{bid}.layer_scale1", # Nemotron Omni
"vision_tower.vision_model.encoder.layers.{bid}.ls1", # InternVL
"model.vision_tower.encoder.layer.{bid}.lambda_1", # Intern-S1
"vision_model.transformer.resblocks.{bid}.ls_1", # Step3-VL
),
MODEL_TENSOR.V_LAYER_SCALE_2: (
+ "vision_model.encoder.layer.{bid}.layer_scale2", # Nemotron Omni
"vision_tower.vision_model.encoder.layers.{bid}.ls2", # InternVL
"model.vision_tower.encoder.layer.{bid}.lambda_2", # Intern-S1
"vision_model.transformer.resblocks.{bid}.ls_2", # Step3-VL
@@ -2114,6 +2127,7 @@ class TensorNameMap:
"siglip2.vision_model.post_layernorm",
"model.qwen2_model.model.model.norm", # Deepseek-OCR-2 qwen2
"model.vision_tower.ln_post", # muse-glimmer
+ "vision_projector.vision_final_layernorm", # Nemotron Omni Reasoning V3
),
MODEL_TENSOR.V_MM_POST_NORM: (
diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp
index 982351dc8..6f28d2f1d 100644
--- a/src/models/dflash.cpp
+++ b/src/models/dflash.cpp
@@ -930,7 +930,7 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra
res->t_inp_tokens = inp_tokens;
ggml_tensor * inp_anchor = nullptr;
- if (model.lilicorr_pass_hidden_proj && ubatch.embd_state) {
+ if (model.lilicorr_pass_hidden_proj && ubatch.embd_state && n_tokens == (int64_t) hparams.dflash_block_size * ubatch.n_seqs_unq) {
// LiliCorr sets inp_anchor to pass the context features to the lattice scorer
auto inp = std::make_unique<llm_graph_input_embd_h>(n_embd, n_embd);
inp->tokens = inp_tokens;
@@ -1100,7 +1100,7 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra
if (model.dflash_selector_hidden) {
build_dflash2_selector(*this, model, inp_tokens);
- } else if (model.lilicorr_pass_hidden_proj && ubatch.embd_state) {
+ } else if (model.lilicorr_pass_hidden_proj && ubatch.embd_state && n_tokens == (int64_t) hparams.dflash_block_size * ubatch.n_seqs_unq) {
build_lilicorr_scores(*this, model, tok_embd, inp_anchor);
}
}
diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cpp
index b14913f2a..e9c61308f 100644
--- a/src/models/nemotron-h.cpp
+++ b/src/models/nemotron-h.cpp
@@ -109,21 +109,26 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) {
layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
} else {
if (n_expert != 0) {
+ const auto * router_meta = ml.get_tensor_meta(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i).str().c_str());
+ const int64_t n_expert_layer = router_meta ? router_meta->ne[1] : n_expert;
+ if (n_expert_layer < hparams.n_expert_used(i) || n_expert_layer > n_expert) {
+ throw std::runtime_error("Invalid per-layer Nemotron expert count");
+ }
// Use per-layer n_ff_exp; fall back to n_ff/n_expert_used if absent (existing GGUFs).
const int64_t n_ff_exp_i = hparams.n_ff_exp(i)
? (int64_t)hparams.n_ff_exp(i)
: hparams.n_ff(i) / (int64_t)hparams.n_expert_used(i);
const int64_t n_ff_shexp = hparams.n_ff_shexp;
- layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert}, trunk_flags);
- layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert }, trunk_flags);
+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert_layer}, trunk_flags);
+ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert_layer}, trunk_flags);
// MoE branch
layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", i), {n_embd, moe_n_embd}, TENSOR_NOT_REQUIRED);
layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", i), {moe_n_embd, n_embd}, TENSOR_NOT_REQUIRED);
- layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp_i, moe_n_embd, n_expert}, trunk_flags);
- layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp_i, n_expert}, trunk_flags);
+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp_i, moe_n_embd, n_expert_layer}, trunk_flags);
+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp_i, n_expert_layer}, trunk_flags);
// Shared expert branch
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, trunk_flags);
@@ -178,6 +183,15 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) {
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp, n_expert}, mtp_flags);
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, mtp_flags);
layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, mtp_flags);
+
+ layer.ffn_down_exps_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED);
+ layer.ffn_up_exps_s = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED);
+ layer.ffn_down_exps_in_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "input_scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED);
+ layer.ffn_up_exps_in_s = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "input_scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED);
+ layer.ffn_down_shexp_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED);
+ layer.ffn_up_shexp_s = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED);
+ layer.ffn_down_shexp_in_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "input_scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED);
+ layer.ffn_up_shexp_in_s = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "input_scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED);
}
}
@@ -304,7 +318,7 @@ ggml_tensor * llama_model_nemotron_h::graph::build_ffn_layer(ggml_tensor * cur,
nullptr, // no gate
model.layers[il].ffn_down_exps,
model.layers[il].ffn_exp_probs_b,
- n_expert, (int64_t)hparams.n_expert_used(il),
+ model.layers[il].ffn_up_exps->ne[2], (int64_t)hparams.n_expert_used(il),
LLM_FFN_RELU_SQR, hparams.expert_weights_norm,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index 20098937c..82f74bdc9 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -46,6 +46,7 @@ add_library(mtmd
models/kimivl.cpp
models/kimik25.cpp
models/nemotron-v2-vl.cpp
+ models/nemotron-omni.cpp
models/muse-glimmer.cpp
models/llama4.cpp
models/llava.cpp
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 30cb30789..3d6c6f8c4 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -498,6 +498,7 @@ enum projector_type {
PROJECTOR_TYPE_YASA2,
PROJECTOR_TYPE_KIMIK25,
PROJECTOR_TYPE_NEMOTRON_V2_VL,
+ PROJECTOR_TYPE_NEMOTRON_OMNI,
PROJECTOR_TYPE_HUNYUANVL,
PROJECTOR_TYPE_PARAKEET,
PROJECTOR_TYPE_EXAONE4_5,
@@ -569,6 +570,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_YASA2, "yasa2"},
{ PROJECTOR_TYPE_KIMIK25, "kimik25"},
{ PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"},
+ { PROJECTOR_TYPE_NEMOTRON_OMNI, "nemotron_omni"},
{ PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"},
{ PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"},
{ PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"},
@@ -904,11 +906,13 @@ static void clip_log_internal(enum ggml_log_level level, const char * format, ..
struct clip_image_f32_batch {
std::vector<clip_image_f32> entries;
bool is_audio = false;
+ bool is_video = false;
clip_image_f32_batch clone() const {
clip_image_f32_batch new_batch{
/* entries */ {},
/* is_audio */ is_audio,
+ /* is_video */ is_video,
};
new_batch.entries.reserve(entries.size());
for (const auto & entry : entries) {
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h
index 657671ded..2c4ea398c 100644
--- a/tools/mtmd/clip-model.h
+++ b/tools/mtmd/clip-model.h
@@ -566,7 +566,7 @@ struct clip_model {
// embeddings
ggml_tensor * class_embedding = nullptr;
ggml_tensor * patch_embeddings_0 = nullptr;
- ggml_tensor * patch_embeddings_1 = nullptr; // second Conv2D kernel when we decouple Conv3D along temporal dimension (Qwen2VL)
+ ggml_tensor * patch_embeddings_1 = nullptr; // additional patch kernel; interpretation is model-specific (Qwen2VL, Nemotron Omni)
ggml_tensor * patch_bias = nullptr;
ggml_tensor * position_embeddings = nullptr;
ggml_tensor * norm_embd_w = nullptr;
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index fccb00c3a..4c1185a43 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -1042,6 +1042,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_nemotron_v2_vl>(ctx, img);
} break;
+ case PROJECTOR_TYPE_NEMOTRON_OMNI:
+ {
+ builder = std::make_unique<clip_graph_nemotron_omni>(ctx, img);
+ } break;
case PROJECTOR_TYPE_LLAMA4:
{
builder = std::make_unique<clip_graph_llama4>(ctx, img);
@@ -1523,6 +1527,16 @@ struct clip_model_loader {
{
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
} break;
+ case PROJECTOR_TYPE_NEMOTRON_OMNI:
+ {
+ hparams.ffn_op = FFN_GELU_ERF;
+ log_ffn_op = "gelu_erf";
+ hparams.image_resize_pad = PAD_NONE;
+ get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
+ get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
+ get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
+ hparams.warmup_image_size = hparams.image_size;
+ } break;
case PROJECTOR_TYPE_PARAKEET:
{
get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor);
@@ -3269,6 +3283,7 @@ struct clip_model_loader {
model.mm_3_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "bias"));
} break;
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
+ case PROJECTOR_TYPE_NEMOTRON_OMNI:
{
model.mm_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
model.mm_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
@@ -4320,6 +4335,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_COHERE2V:
case PROJECTOR_TYPE_INTERNVL:
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
+ case PROJECTOR_TYPE_NEMOTRON_OMNI:
case PROJECTOR_TYPE_LLAMA4:
{
// both X and Y are downscaled by the scale factor
@@ -5428,6 +5444,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
case PROJECTOR_TYPE_COHERE2V:
case PROJECTOR_TYPE_INTERNVL:
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
+ case PROJECTOR_TYPE_NEMOTRON_OMNI:
case PROJECTOR_TYPE_QWEN2A:
case PROJECTOR_TYPE_QWEN3A:
case PROJECTOR_TYPE_GLMA:
@@ -6149,6 +6166,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_3_w->ne[1]; // out_proj output dim
case PROJECTOR_TYPE_INTERNVL:
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
+ case PROJECTOR_TYPE_NEMOTRON_OMNI:
return ctx->model.mm_3_w->ne[1];
case PROJECTOR_TYPE_LLAMA4:
return ctx->model.mm_model_proj->ne[1];
@@ -6225,6 +6243,7 @@ int clip_model_n_temporal_merge(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
+ case PROJECTOR_TYPE_NEMOTRON_OMNI:
case PROJECTOR_TYPE_LING3VL:
return 2;
default:
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index adb5ede30..2f4676a54 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -109,6 +109,12 @@ struct clip_graph_nemotron_v2_vl : clip_graph {
ggml_cgraph * build() override;
};
+struct clip_graph_nemotron_omni : clip_graph {
+ clip_graph_nemotron_omni(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+ ggml_cgraph * build() override;
+ ggml_tensor * build_inp_with_temporal_merge();
+};
+
struct clip_graph_llama4 : clip_graph {
clip_graph_llama4(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/models/nemotron-omni.cpp b/tools/mtmd/models/nemotron-omni.cpp
new file mode 100644
index 000000000..c4b1bff40
--- /dev/null
+++ b/tools/mtmd/models/nemotron-omni.cpp
@@ -0,0 +1,56 @@
+#include "models.h"
+
+ggml_tensor * clip_graph_nemotron_omni::build_inp_with_temporal_merge() {
+ if (n_batch == 1 && !img_batch->is_video) {
+ return build_inp();
+ }
+
+ GGML_ASSERT(n_batch == 1 || n_batch == 2);
+ GGML_ASSERT(model.patch_embeddings_1 != nullptr);
+ GGML_ASSERT(model.patch_embeddings_1->ne[2] == 6);
+
+ ggml_tensor * inp_raw = build_inp_raw();
+ if (n_batch == 1) {
+ // pad the last video frame inside the graph
+ inp_raw = ggml_concat(ctx0, inp_raw, inp_raw, 2);
+ } else {
+ inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.ny(), 6, 1);
+ }
+
+ ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
+ inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, 1);
+ return ggml_cont(ctx0, ggml_transpose(ctx0, inp));
+}
+
+ggml_cgraph * clip_graph_nemotron_omni::build() {
+ GGML_ASSERT(model.class_embedding != nullptr);
+ GGML_ASSERT(model.position_embeddings != nullptr);
+
+ const int n_registers = model.class_embedding->ne[1];
+ const int n_pos = n_patches + n_registers;
+
+ ggml_tensor * inp = build_inp_with_temporal_merge();
+
+ inp = ggml_add(ctx0, inp, resize_position_embeddings(GGML_SCALE_MODE_BILINEAR));
+ cb(inp, "inp_pos", -1);
+
+ inp = ggml_concat(ctx0, model.class_embedding, inp, 1);
+
+ ggml_tensor * cur = build_vit(inp, n_pos, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, nullptr);
+
+ cur = ggml_view_2d(ctx0, cur,
+ n_embd, n_patches,
+ ggml_row_size(cur->type, n_embd),
+ n_registers * ggml_row_size(cur->type, n_embd));
+
+ cur = build_patch_merge_permute(cur, model.hparams.n_merge);
+
+ {
+ cur = build_norm(cur, model.mm_0_w, nullptr, NORM_TYPE_RMS, 1e-5, -1);
+ cur = build_ffn(cur, model.mm_1_w, nullptr, nullptr, nullptr, model.mm_3_w, nullptr, FFN_RELU_SQR, -1);
+ }
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+}
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 2c2b06936..2fa02f83e 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -898,6 +898,12 @@ struct mtmd_context {
{
image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v);
} break;
+ case PROJECTOR_TYPE_NEMOTRON_OMNI:
+ {
+ img_beg = "<img>";
+ img_end = "</img>";
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ } break;
case PROJECTOR_TYPE_LFM2:
{
// multi-tile:
@@ -1607,6 +1613,7 @@ struct mtmd_tokenizer {
clip_image_f32_batch batch_f32;
batch_f32.is_audio = false;
+ batch_f32.is_video = bitmaps[0]->mergeable;
batch_f32.entries = std::move(preproc_out.entries);
// do NOT use preproc_out from this point on, it's moved