Commit 7775ffb7e for llama.cpp
commit 7775ffb7e48fa0c7dcffd5815743b3d2fb3b6761
Author: Maphist0 <28743569+Maphist0@users.noreply.github.com>
Date: Thu Oct 22 03:54:16 2026 +0800
model: Add support for SenseNova U1 model (#28919)
* model : add SenseNova U1 understanding support
Add support for the dense SenseNova U1 understanding branch, including
the U1.5-8B-MoT model.
- Add HF-to-GGUF conversion for NEOChatModel, excluding MoT generation
tensors and concatenating temporal and spatial Q/K normalization
weights.
- Register the sensenova_u1 GGUF architecture and custom spatial RoPE
metadata.
- Implement temporal/spatial Q/K normalization with 4-axis RoPE.
- Keep U1-specific text position handling in the model implementation.
- Add synthetic architecture coverage for sensenova_u1.
Only the dense understanding branch is supported; generation
branches are intentionally rejected during conversion.
* conversion : treat one-expert SenseNova U1 configs as dense
* mtmd : add SenseNova U1 vision support
Add conversion support for the SenseNova U1 vision encoder and implement
its CPU mtmd projector graph.
The graph applies patch embedding, GELU, adjacent-pair 2D RoPE, spatial
downsampling, and dense projection for dynamic-resolution image inputs.
* mtmd : fix SenseNova U1 vision token layout
* model : clean up SenseNova U1 RoPE metadata
Replace the hard-coded spatial RoPE key with shared GGUF and LLM_KV
definitions used by conversion, model loading, and architecture tests.
Clarify why SenseNova U1 retains its custom text-position input: U1 uses
[t, 0, 0, 0], while the generic M-RoPE input uses [t, t, t, 0].
* mtmd : simplify SenseNova U1 projector integration
Map the SenseNova U1 dense projection to the standard mm.0 weight and
bias tensors and reuse the shared projector fields in the loader and graph.
Read vision parameters from the checkpoint configuration and remove the
U1-specific tensor names and model fields while preserving its strided
projection, bias, and row-major token layout.
* mtmd : fix SenseNova U1 image positions and attention
* fix spacing
* mtmd : restore generic test input chunks
* mtmd : restore SenseNova U1 integration after upstream merge
* Update conversion/sensenova_u1.py
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
* Update src/models/sensenova-u1.cpp
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
* model : remove SenseNova U1 BF16 activation casts
Use standard backend precision handling for QKV, FFN, and output
projections instead of casting inputs based on the output tensor type.
Assisted-by: Codex
* Update src/models/sensenova-u1.cpp
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
---------
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
diff --git a/conversion/__init__.py b/conversion/__init__.py
index bddfe3c1f..369ce7afd 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -17,6 +17,7 @@ __all__ = [
TEXT_MODEL_MAP: dict[str, str] = {
+ "NEOChatModel": "sensenova_u1",
"AfmoeForCausalLM": "afmoe",
"LagunaForCausalLM": "laguna",
"ApertusForCausalLM": "llama",
@@ -367,6 +368,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"MuseSparkForConditionalGeneration": "muse_spark",
"PaddleOCRVisionModel": "ernie",
"Phi4ForCausalLMV": "phi",
+ "NEOVisionModel": "sensenova_u1",
"Qwen2AudioForConditionalGeneration": "ultravox",
"Qwen2VLForConditionalGeneration": "qwenvl",
"Qwen2VLModel": "qwenvl",
diff --git a/conversion/base.py b/conversion/base.py
index dec1d4322..2cbaed700 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -3143,7 +3143,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
# Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older
# Kimi-Linear-48B architecture and cannot load K3 (no attention residuals,
# latent MoE, situ, ...). Route on the top-level architecture instead.
- if model_type == ModelType.TEXT and arch in ("StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration"):
+ if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration"):
return arch
# if "architectures" is found in the sub-config, use that instead
diff --git a/conversion/sensenova_u1.py b/conversion/sensenova_u1.py
new file mode 100644
index 000000000..bde5f9d9b
--- /dev/null
+++ b/conversion/sensenova_u1.py
@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import torch
+
+from .base import MmprojModel, ModelBase, gguf
+from .qwen import Qwen2Model
+
+
+@ModelBase.example("sensenova/SenseNova-U1.5-8B-MoT")
+@ModelBase.register("NEOChatModel")
+class SenseNovaU1Model(Qwen2Model):
+ model_arch = gguf.MODEL_ARCH.SENSENOVA_U1
+
+ @classmethod
+ def filter_tensors(cls, item):
+ name, _ = item
+ if not name.startswith("language_model.") or "_mot_gen" in name:
+ return None
+ return super().filter_tensors(item)
+
+ def set_gguf_parameters(self):
+ if int(self.hparams.get("num_experts") or 0) > 1:
+ raise ValueError("SenseNova U1 conversion currently supports the dense understanding branch")
+ if self.rope_parameters.get("rope_type", "default") not in ("default", None):
+ raise ValueError("SenseNova U1 scaled RoPE is not supported")
+ if self.hparams.get("attention_bias", False):
+ raise ValueError("SenseNova U1 attention bias is not supported")
+ if self.hparams.get("use_sliding_window", False):
+ raise ValueError("SenseNova U1 sliding attention is not supported")
+ super().set_gguf_parameters()
+ self.gguf_writer.add_rope_freq_base_spatial(self.hparams["rope_theta_hw"])
+
+ def get_tensors(self):
+ for name, gen in self.model_tensors.items():
+ if name.endswith((".q_norm_hw.weight", ".k_norm_hw.weight")):
+ continue
+ data = gen()
+ if name.endswith((".q_norm.weight", ".k_norm.weight")):
+ spatial = self.model_tensors[name.replace("_norm.weight", "_norm_hw.weight")]()
+ # Keep one weight vector; the graph normalizes its two halves independently.
+ data = torch.cat((data, spatial), dim=0)
+ yield name, data
+
+
+@ModelBase.register("NEOVisionModel")
+@ModelBase.example("sensenova/SenseNova-U1.5-8B-MoT")
+class SenseNovaU1VisionModel(MmprojModel):
+ def __init__(self, dir_model: Path, *args, **kwargs):
+ hparams = kwargs.pop("hparams", None)
+ if hparams is None:
+ hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
+ else:
+ hparams = dict(hparams)
+
+ vision_config = dict(hparams["vision_config"])
+ # NEOVisionModel has no transformer blocks
+ # these defaults satisfy the generic mmproj metadata contract without replacing checkpoint values
+ vision_config.setdefault("image_size", 0)
+ vision_config.setdefault("intermediate_size", 0)
+ vision_config.setdefault("num_hidden_layers", 0)
+ vision_config.setdefault("num_attention_heads", 1)
+ hparams["vision_config"] = vision_config
+ super().__init__(dir_model, *args, hparams=hparams, **kwargs)
+
+ self.preprocessor_config.setdefault("image_mean", [0.485, 0.456, 0.406])
+ self.preprocessor_config.setdefault("image_std", [0.229, 0.224, 0.225])
+
+ @classmethod
+ def filter_tensors(cls, item):
+ name, gen = item
+ if not name.startswith("vision_model."):
+ return None
+ return super().filter_tensors((name, gen))
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_clip_projector_type("sensenova_u1")
+ vision_config = self.hparams["vision_config"]
+ # These defaults match the known U1.5 checkpoint when older configs omit fields.
+ hidden_size = int(vision_config.get("hidden_size", 1024))
+ min_pixels = int(vision_config.get("min_pixels", 65536))
+ max_pixels = int(vision_config.get("max_pixels", 16777216))
+ downsample_ratio = float(vision_config.get("downsample_ratio", 0.5))
+ if downsample_ratio <= 0:
+ raise ValueError(f"SenseNova U1 downsample_ratio must be positive, got {downsample_ratio}")
+ merge_size = round(1.0 / downsample_ratio)
+ if abs((1.0 / downsample_ratio) - merge_size) > 1e-6:
+ raise ValueError(f"SenseNova U1 downsample_ratio must have an integer reciprocal, got {downsample_ratio}")
+ norm_eps = float(vision_config.get("layer_norm_eps", vision_config.get("rms_norm_eps", 1e-6)))
+ self.gguf_writer.add_vision_head_dim(hidden_size)
+ self.gguf_writer.add_vision_attention_layernorm_eps(norm_eps)
+ self.gguf_writer.add_vision_min_pixels(min_pixels)
+ self.gguf_writer.add_vision_max_pixels(max_pixels)
+ self.gguf_writer.add_vision_spatial_merge_size(merge_size)
+
+ def modify_tensors(self, data_torch, name, bid):
+ tensor_names = {
+ "vision_model.embeddings.patch_embedding.weight": "v.patch_embd.weight",
+ "vision_model.embeddings.patch_embedding.bias": "v.patch_embd.bias",
+ "vision_model.embeddings.dense_embedding.weight": self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, 0),
+ "vision_model.embeddings.dense_embedding.bias": self.format_tensor_name(
+ gguf.MODEL_TENSOR.V_MMPROJ, 0, suffix=".bias"
+ ),
+ }
+ if name not in tensor_names:
+ raise ValueError(f"Unexpected SenseNova U1 vision tensor: {name!r}")
+ return [(tensor_names[name], data_torch)]
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ if new_name == self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, 0):
+ return gguf.GGMLQuantizationType.F16 if self.ftype == gguf.LlamaFileType.MOSTLY_F16 else gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 2840348cb..619a15fc9 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -263,6 +263,7 @@ class Keys:
DIMENSION_SECTIONS = "{arch}.rope.dimension_sections"
SECTION_ORDER = "{arch}.rope.section_order"
FREQ_BASE = "{arch}.rope.freq_base"
+ FREQ_BASE_SPATIAL = "{arch}.rope.freq_base_spatial"
FREQ_BASE_SWA = "{arch}.rope.freq_base_swa"
SCALING_TYPE = "{arch}.rope.scaling.type"
SCALING_FACTOR = "{arch}.rope.scaling.factor"
@@ -573,6 +574,7 @@ class MODEL_ARCH(IntEnum):
QWEN2 = auto()
QWEN2MOE = auto()
QWEN2VL = auto()
+ SENSENOVA_U1 = auto()
QWEN3 = auto()
QWEN3MOE = auto()
QWEN3NEXT = auto()
@@ -1390,6 +1392,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.QWEN2: "qwen2",
MODEL_ARCH.QWEN2MOE: "qwen2moe",
MODEL_ARCH.QWEN2VL: "qwen2vl",
+ MODEL_ARCH.SENSENOVA_U1: "sensenova_u1",
MODEL_ARCH.QWEN3: "qwen3",
MODEL_ARCH.QWEN3MOE: "qwen3moe",
MODEL_ARCH.QWEN3NEXT: "qwen3next",
@@ -2916,6 +2919,24 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_DOWN_SHEXP,
MODEL_TENSOR.FFN_UP_SHEXP,
],
+ MODEL_ARCH.SENSENOVA_U1: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
MODEL_ARCH.QWEN3: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
@@ -6485,6 +6506,7 @@ KEY_ATTENTION_LAYERNORM_RMS_EPS = Keys.Attention.LAYERNORM_RMS_EPS
# RoPE
KEY_ROPE_DIMENSION_COUNT = Keys.Rope.DIMENSION_COUNT
KEY_ROPE_FREQ_BASE = Keys.Rope.FREQ_BASE
+KEY_ROPE_FREQ_BASE_SPATIAL = Keys.Rope.FREQ_BASE_SPATIAL
KEY_ROPE_SCALING_TYPE = Keys.Rope.SCALING_TYPE
KEY_ROPE_SCALING_FACTOR = Keys.Rope.SCALING_FACTOR
KEY_ROPE_SCALING_ORIG_CTX_LEN = Keys.Rope.SCALING_ORIG_CTX_LEN
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index e8a96e571..d00392211 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1167,6 +1167,9 @@ class GGUFWriter:
def add_rope_freq_base(self, value: float) -> None:
self.add_float32(Keys.Rope.FREQ_BASE.format(arch=self.arch), value)
+ def add_rope_freq_base_spatial(self, value: float) -> None:
+ self.add_float32(Keys.Rope.FREQ_BASE_SPATIAL.format(arch=self.arch), value)
+
def add_rope_scaling_type(self, value: RopeScalingType) -> None:
self.add_string(Keys.Rope.SCALING_TYPE.format(arch=self.arch), value.value)
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 4534be22f..1c11536b5 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -33,6 +33,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_QWEN2, "qwen2" },
{ LLM_ARCH_QWEN2MOE, "qwen2moe" },
{ LLM_ARCH_QWEN2VL, "qwen2vl" },
+ { LLM_ARCH_SENSENOVA_U1, "sensenova_u1" },
{ LLM_ARCH_QWEN3, "qwen3" },
{ LLM_ARCH_QWEN3MOE, "qwen3moe" },
{ LLM_ARCH_QWEN3NEXT, "qwen3next" },
@@ -339,6 +340,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_ROPE_DIMENSION_SECTIONS, "%s.rope.dimension_sections" },
{ LLM_KV_ROPE_SECTION_ORDER, "%s.rope.section_order" },
{ LLM_KV_ROPE_FREQ_BASE, "%s.rope.freq_base" },
+ { LLM_KV_ROPE_FREQ_BASE_SPATIAL, "%s.rope.freq_base_spatial" },
{ LLM_KV_ROPE_FREQ_BASE_SWA, "%s.rope.freq_base_swa" },
{ LLM_KV_ROPE_SCALE_LINEAR, "%s.rope.scale_linear" },
{ LLM_KV_ROPE_SCALING_TYPE, "%s.rope.scaling.type" },
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 5a8ca4250..afceb8ea6 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -38,6 +38,7 @@ enum llm_arch {
LLM_ARCH_QWEN2,
LLM_ARCH_QWEN2MOE,
LLM_ARCH_QWEN2VL,
+ LLM_ARCH_SENSENOVA_U1,
LLM_ARCH_QWEN3,
LLM_ARCH_QWEN3MOE,
LLM_ARCH_QWEN3NEXT,
@@ -344,6 +345,7 @@ enum llm_kv {
LLM_KV_ROPE_DIMENSION_SECTIONS,
LLM_KV_ROPE_SECTION_ORDER,
LLM_KV_ROPE_FREQ_BASE,
+ LLM_KV_ROPE_FREQ_BASE_SPATIAL,
LLM_KV_ROPE_FREQ_BASE_SWA,
LLM_KV_ROPE_SCALE_LINEAR,
LLM_KV_ROPE_SCALING_TYPE,
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index 46b1d2ae2..d620fcd9c 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -15,6 +15,7 @@
bool llama_model_saver_supports_arch(llm_arch arch) {
switch (arch) {
+ case LLM_ARCH_SENSENOVA_U1:
case LLM_ARCH_GEMMA3N:
case LLM_ARCH_BITNET:
case LLM_ARCH_T5:
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 6712f0f22..3797c567d 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -110,6 +110,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_qwen2vl(params);
case LLM_ARCH_QWEN2MOE:
return new llama_model_qwen2moe(params);
+ case LLM_ARCH_SENSENOVA_U1:
+ return new llama_model_sensenova_u1(params);
case LLM_ARCH_QWEN3:
return new llama_model_qwen3(params);
case LLM_ARCH_QWEN3MOE:
@@ -3640,6 +3642,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
// DSV4 DSpark drafters use DeepSeek-V4's normal RoPE; legacy DFlash backbones are NeoX
return model->hparams.dsv4_hc_mult > 0 ? LLAMA_ROPE_TYPE_NORM : LLAMA_ROPE_TYPE_NEOX;
+ case LLM_ARCH_SENSENOVA_U1:
case LLM_ARCH_QWEN2VL:
case LLM_ARCH_PADDLEOCR:
return LLAMA_ROPE_TYPE_MROPE;
diff --git a/src/models/models.h b/src/models/models.h
index 9aa450dea..47b0203da 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -614,6 +614,21 @@ struct llama_model_qwen2moe : public llama_model_base {
};
+struct llama_model_sensenova_u1 : public llama_model_base {
+ llama_model_sensenova_u1(const struct llama_model_params & params) : llama_model_base(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ float rope_freq_base_spatial = 10000.0f;
+
+ struct graph : public llm_graph_context {
+ graph(const llama_model_sensenova_u1 & model, const llm_graph_params & params);
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
struct llama_model_qwen3 : public llama_model_base {
llama_model_qwen3(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/src/models/sensenova-u1.cpp b/src/models/sensenova-u1.cpp
new file mode 100644
index 000000000..c42c604e8
--- /dev/null
+++ b/src/models/sensenova-u1.cpp
@@ -0,0 +1,190 @@
+#include "models.h"
+
+namespace {
+class llm_graph_input_pos_sensenova_u1 : public llm_graph_input_pos {
+public:
+ llm_graph_input_pos_sensenova_u1() : llm_graph_input_pos(4) {}
+
+ void set_input(const llama_ubatch * ubatch) override {
+ if (ubatch->pos && pos) {
+ const int64_t n_tokens = ubatch->n_tokens;
+
+ // U1 expects text-token positions in the temporal plane only: [t, 0, 0, 0].
+ // The generic M-RoPE input uses [t, t, t, 0], so U1 needs a custom input class.
+ if (ubatch->token) {
+ std::vector<llama_pos> pos_data(n_tokens*n_pos_per_embd, 0);
+ for (int i = 0; i < n_tokens; ++i) {
+ pos_data[i] = ubatch->pos[i];
+ }
+ ggml_backend_tensor_set(pos, pos_data.data(), 0, pos_data.size()*ggml_element_size(pos));
+ } else {
+ ggml_backend_tensor_set(pos, ubatch->pos, 0, n_tokens*n_pos_per_embd*ggml_element_size(pos));
+ }
+ }
+ }
+};
+}
+
+void llama_model_sensenova_u1::load_arch_hparams(llama_model_loader & ml) {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ ml.get_key(LLM_KV_ROPE_FREQ_BASE_SPATIAL, rope_freq_base_spatial);
+ if (hparams.n_embd_head_k() % 4 != 0 || hparams.n_embd_head_v() != hparams.n_embd_head_k()) {
+ throw std::runtime_error("SenseNova U1 requires equal Q/K/V head sizes divisible by four");
+ }
+ switch (hparams.n_layer()) {
+ case 42: type = LLM_TYPE_8B; break;
+ default: type = LLM_TYPE_UNKNOWN;
+ }
+}
+
+void llama_model_sensenova_u1::load_arch_tensors(llama_model_loader &) {
+ LLAMA_LOAD_LOCALS;
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+ // output
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+ // if output is NULL, init from the input tok embed
+ if (output == NULL) {
+ output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ auto & layer = layers[i];
+
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, 0);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
+
+ layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
+ layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
+
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0);
+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
+ }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_sensenova_u1::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(*this, params);
+}
+
+llama_model_sensenova_u1::graph::graph(const llama_model_sensenova_u1 & model, const llm_graph_params & params) : llm_graph_context(params) {
+ const int64_t n_embd_head = hparams.n_embd_head_v();
+
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+
+ ggml_tensor * cur;
+ ggml_tensor * inpL;
+
+ auto normalize = [&](ggml_tensor * x, ggml_tensor * weight) {
+ x = ggml_rms_norm(ctx0, x, hparams.f_norm_rms_eps);
+
+ return ggml_mul(ctx0, x, weight);
+ };
+
+ inpL = build_inp_embd(model.tok_embd);
+
+ ggml_tensor * pos[3]{};
+ auto inp = std::make_unique<llm_graph_input_pos_sensenova_u1>();
+ auto & inp_pos = inp->pos;
+ inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, (int64_t)n_tokens*hparams.n_pos_per_embd());
+ ggml_set_input(inp_pos);
+ res->add_input(std::move(inp));
+
+ for (int axis = 0; axis < 3; ++axis) {
+ pos[axis] = ggml_view_1d(ctx0, inp_pos, n_tokens, axis*n_tokens*sizeof(int32_t));
+ }
+
+ auto normalize_and_rotate = [&](ggml_tensor * x, ggml_tensor * weight) {
+ const int64_t heads = x->ne[1];
+ x = ggml_reshape_4d(ctx0, x, n_embd_head/2, 2, heads, n_tokens);
+ x = normalize(x, ggml_reshape_2d(ctx0, weight, n_embd_head/2, 2));
+ x = ggml_reshape_3d(ctx0, x, n_embd_head, heads, n_tokens);
+
+ // Normalize temporal and spatial halves separately; rotate time, height, then width.
+ for (int axis = 0; axis < 3; ++axis) {
+ const int dims = axis == 0 ? n_embd_head/2 : n_embd_head/4;
+ const int offset = axis == 0 ? 0 : n_embd_head/2 + (axis - 1)*n_embd_head/4;
+ const float base = axis == 0 ? freq_base : model.rope_freq_base_spatial;
+ x = ggml_rope_ext(ctx0, x, pos[axis], nullptr, dims, GGML_ROPE_TYPE_NEOX,
+ n_ctx_orig, base, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f);
+ ggml_rope_set_offset(x, offset);
+ }
+ return x;
+ };
+
+ auto * inp_attn = build_attn_inp_kv();
+
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ for (int il = 0; il < n_layer; ++il) {
+ res->t_layer_inp[il] = inpL;
+
+ ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = normalize(inpL, model.layers[il].attn_norm);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
+
+ Qcur = normalize_and_rotate(Qcur, model.layers[il].attn_q_norm);
+ Kcur = normalize_and_rotate(Kcur, model.layers[il].attn_k_norm);
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ cur = build_attn(inp_attn,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
+ }
+ if (il == n_layer - 1 && inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+ ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ cur = normalize(ffn_inp, model.layers[il].ffn_norm);
+ cb(cur, "ffn_norm", il);
+
+ cur = build_ffn(cur,
+ model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s,
+ model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s,
+ model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+ cur = inpL;
+
+ cur = normalize(cur, model.output_norm);
+
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ // lm_head
+ cur = build_lora_mm(model.output, cur, model.output_s);
+
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+}
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 21d27e17d..c86b1315a 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -329,6 +329,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
// partial rotary: n_rot must not exceed the indexer key length (64)
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, uint32_t(64));
}
+ if (arch == LLM_ARCH_SENSENOVA_U1) {
+ ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SPATIAL, 10000.0f);
+ }
ms.add_kv(LLM_KV_ATTENTION_CLAMP_KQV, 1.0f);
ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_EPS, 1e-5f);
ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, 1e-5f);
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index 398b73904..bd7fb7356 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -55,6 +55,7 @@ add_library(mtmd
models/paddleocr.cpp
models/pixtral.cpp
models/qwen2vl.cpp
+ models/sensenova-u1.cpp
models/minimax-m3.cpp
models/qwen3vl.cpp
models/ling3vl.cpp
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 71d349cc5..6ca465280 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -547,6 +547,7 @@ enum projector_type {
PROJECTOR_TYPE_POCKETTTS_SPKENC,
PROJECTOR_TYPE_POCKETTTS_GEN,
PROJECTOR_TYPE_MUSE_GLIMMER,
+ PROJECTOR_TYPE_SENSENOVA_U1,
PROJECTOR_TYPE_COHERE2V,
PROJECTOR_TYPE_MUSE_SPARK_AUDIO,
PROJECTOR_TYPE_UNKNOWN,
@@ -621,6 +622,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_POCKETTTS_SPKENC, "pockettts_spkenc"},
{ PROJECTOR_TYPE_POCKETTTS_GEN, "pockettts_gen"},
{ PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"},
+ { PROJECTOR_TYPE_SENSENOVA_U1, "sensenova_u1"},
{ PROJECTOR_TYPE_COHERE2V, "cohere2v"},
{ PROJECTOR_TYPE_MUSE_SPARK_AUDIO, "muse-spark-audio"},
};
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 359ec5ac3..869e3fff8 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -997,6 +997,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_qwen3vl>(ctx, img);
} break;
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ {
+ builder = std::make_unique<clip_graph_sensenova_u1>(ctx, img);
+ } break;
case PROJECTOR_TYPE_LING3VL:
{
builder = std::make_unique<clip_graph_ling3vl>(ctx, img);
@@ -1795,6 +1799,20 @@ struct clip_model_loader {
LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__);
}
} break;
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ {
+ hparams.n_merge = 2;
+ hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
+ get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
+ get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
+ get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
+ hparams.rope_theta = 10000.0f;
+ hparams.set_limit_image_tokens(16, 4096);
+ if (hparams.n_merge != 2) {
+ throw std::runtime_error(string_format("%s: SenseNova U1 requires spatial_merge_size=2, got %u", __func__, hparams.n_merge));
+ }
+ hparams.set_warmup_n_tokens(16*16);
+ } break;
case PROJECTOR_TYPE_MINIMAX_M3:
{
hparams.n_merge = 2; // spatial_merge_size
@@ -3837,6 +3855,11 @@ struct clip_model_loader {
}
} break;
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ {
+ model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
+ model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
+ } break;
default:
GGML_ASSERT(false && "unknown projector type");
}
@@ -4311,6 +4334,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
+ case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_LING3VL:
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_MIMOVL:
@@ -4341,6 +4365,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
+ case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_LING3VL:
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_MIMOVL:
@@ -4426,6 +4451,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
+ case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_LING3VL:
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_MIMOVL:
@@ -5525,6 +5551,19 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
set_input_i32("pos_w", pos_data);
} break;
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ {
+ const int n_patches_per_row = image_size_width / patch_size;
+ std::vector<int> pos_data(n_pos);
+ for (int i = 0; i < n_pos; i++) {
+ pos_data[i] = i / n_patches_per_row;
+ }
+ set_input_i32("pos_y", pos_data);
+ for (int i = 0; i < n_pos; i++) {
+ pos_data[i] = i % n_patches_per_row;
+ }
+ set_input_i32("pos_x", pos_data);
+ } break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
// set the 2D positions (mrope layout, only the first 2 channels are used)
@@ -6411,6 +6450,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_JANUS_PRO:
case PROJECTOR_TYPE_YOUTUVL:
return ctx->model.mm_1_b->ne[0];
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ return ctx->model.hparams.projection_dim;
case PROJECTOR_TYPE_QWEN3VL:
// main path + deepstack paths
return ctx->model.mm_1_b->ne[0] * (1 + ctx->model.n_deepstack_layers);
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 294448605..55e525cdc 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -50,6 +50,11 @@ struct clip_graph_qwen3vl : clip_graph_qwen2vl {
ggml_cgraph * build() override;
};
+struct clip_graph_sensenova_u1 : clip_graph {
+ clip_graph_sensenova_u1(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+ ggml_cgraph * build() override;
+};
+
struct clip_graph_ling3vl : clip_graph_qwen3vl {
clip_graph_ling3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen3vl(ctx, img) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/models/sensenova-u1.cpp b/tools/mtmd/models/sensenova-u1.cpp
new file mode 100644
index 000000000..57875c6ed
--- /dev/null
+++ b/tools/mtmd/models/sensenova-u1.cpp
@@ -0,0 +1,40 @@
+#include "models.h"
+
+ggml_cgraph * clip_graph_sensenova_u1::build() {
+ GGML_ASSERT(model.patch_embeddings_0 != nullptr);
+ GGML_ASSERT(model.patch_bias != nullptr);
+ GGML_ASSERT(model.mm_0_w != nullptr);
+ GGML_ASSERT(model.mm_0_b != nullptr);
+ GGML_ASSERT(hparams.n_merge == 2);
+
+ ggml_tensor * pos_x = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
+ ggml_set_name(pos_x, "pos_x");
+ ggml_set_input(pos_x);
+
+ ggml_tensor * pos_y = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
+ ggml_set_name(pos_y, "pos_y");
+ ggml_set_input(pos_y);
+
+ ggml_tensor * inp = build_inp();
+ inp = ggml_gelu(ctx0, inp);
+
+ // Apply adjacent-pair 2D RoPE to patch embeddings before downsampling.
+ inp = ggml_reshape_4d(ctx0, inp, n_embd, 1, n_patches, n_batch);
+ inp = build_rope_2d(ctx0, inp, pos_x, pos_y, hparams.rope_theta, false);
+ inp = ggml_reshape_3d(ctx0, inp, n_embd, n_patches, n_batch);
+ cb(inp, "patch_embd_rope", -1);
+
+ inp = ggml_reshape_4d(ctx0, inp, n_embd, n_patches_x, n_patches_y, n_batch);
+ inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3));
+ inp = ggml_conv_2d(ctx0, model.mm_0_w, inp,
+ hparams.n_merge, hparams.n_merge, 0, 0, 1, 1);
+
+ const int n_output_patches = n_patches / (hparams.n_merge * hparams.n_merge);
+ inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 2, 0, 3));
+ inp = ggml_reshape_3d(ctx0, inp, n_mmproj_embd, n_output_patches, n_batch);
+ inp = ggml_add(ctx0, inp, model.mm_0_b);
+ cb(inp, "proj_out", -1);
+
+ ggml_build_forward_expand(gf, inp);
+ return gf;
+}
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 9e2c5f3c9..9460f15fe 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -225,6 +225,7 @@ enum mtmd_pos_type {
MTMD_POS_TYPE_NORMAL, // number of positions equals to number of tokens
MTMD_POS_TYPE_MROPE, // qwen-vl mrope style, each image takes max(t,h,w) position indexes
MTMD_POS_TYPE_HUNYUANVL, // HunyuanVL mrope + BOI/EOI/newline layout with XD-RoPE dim-3
+ MTMD_POS_TYPE_SENSENOVA_U1, // U1 uses zero-based H/W and one temporal position per image
MTMD_POS_TYPE_CANVAS, // MiniCPM-V 4.7: overview + slices in one chunk, sharing one 2D canvas (see mtmd_image_tokens::canvas_tile_grid)
MTMD_POS_TYPE_COUNT, // for validation
};
@@ -751,6 +752,12 @@ struct mtmd_context {
img_end = "]<]end of image[>[";
image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
} break;
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ {
+ img_beg = "<img>";
+ img_end = "</img>";
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ } break;
case PROJECTOR_TYPE_MUSE_GLIMMER:
{
img_beg = "<|image_start|>";
@@ -1615,10 +1622,15 @@ struct mtmd_tokenizer {
image_tokens->ny = 1;
}
image_tokens->pos = ctx->pos_type;
+ // U1 uses M-RoPE, but unlike Qwen its image H/W coordinates are
+ // zero-based and the whole image occupies one temporal position.
+ if (ctx->proj_type_v() == PROJECTOR_TYPE_SENSENOVA_U1) {
+ image_tokens->pos = MTMD_POS_TYPE_SENSENOVA_U1;
+ }
// HunyuanVL wraps the image grid with BOI/EOI and adds one newline per row,
// and uses XD-RoPE (dim-3 = image index). Override the position type so that
// n_tokens() and mtmd_image_tokens_get_decoder_pos pick the HunyuanVL layout.
- if (ctx->proj_type_v() == PROJECTOR_TYPE_HUNYUANVL) {
+ else if (ctx->proj_type_v() == PROJECTOR_TYPE_HUNYUANVL) {
image_tokens->pos = MTMD_POS_TYPE_HUNYUANVL;
image_tokens->image_idx = n_images_added;
GGML_ASSERT(n_tokens == (size_t)image_tokens->n_tokens());
@@ -2398,6 +2410,7 @@ bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk
}
case PROJECTOR_TYPE_GEMMA4UV:
case PROJECTOR_TYPE_GEMMA3:
+ case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_DEEPSEEK4V:
return true;
default:
@@ -2761,6 +2774,13 @@ mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * ima
pos.y = pos_0 + i;
pos.z = pos_0 + i;
} break;
+ case MTMD_POS_TYPE_SENSENOVA_U1:
+ {
+ pos.t = pos_0;
+ pos.x = i % image_tokens->nx;
+ pos.y = i / image_tokens->nx;
+ pos.z = 0;
+ } break;
case MTMD_POS_TYPE_HUNYUANVL:
{
// HunyuanVL layout: [BOI] [row0 tokens + newline] ... [row(ny-1) tokens + newline] [EOI]
@@ -2813,6 +2833,8 @@ llama_pos mtmd_image_tokens_get_n_pos(const mtmd_image_tokens * image_tokens) {
return std::max(image_tokens->nx, image_tokens->ny);
case MTMD_POS_TYPE_NORMAL:
return image_tokens->n_tokens();
+ case MTMD_POS_TYPE_SENSENOVA_U1:
+ return 1;
case MTMD_POS_TYPE_HUNYUANVL:
// HunyuanVL: the sequential (dim-0) position advances by the full token count
// (includes BOI/EOI and row newline tokens), not by max(nx, ny)