Commit 9f0e5098a for llama.cpp
commit 9f0e5098a5d5d8adb8dd5c59036ae2a38ab07db9
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Sat Nov 7 19:36:44 2026 +0100
mtmd: support MageVL-Cowork model (#31819)
* mtmd: support MageVL-Cowork model
* mtmd: keep magevl rope_freqs at model level and check its shape
The rope factors are shared by every layer, so they live in clip_model
instead of a pointer copied into each clip_layer. The loader now checks
that v.rope_freqs is F32 with d_head elements and that d_head splits
into the 4:6:6 pairs the graph expects.
---------
Co-authored-by: Pascal <admin@serveurperso.com>
diff --git a/conversion/__init__.py b/conversion/__init__.py
index de568f58d..f9dabf4f2 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -193,6 +193,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"Eagle3LlamaForCausalLM": "llama",
"LlamaForCausalLMEagle3": "llama",
"LlavaForConditionalGeneration": "llama",
+ "LlavaOnevision2ForConditionalGeneration": "qwen",
"LlavaStableLMEpochForCausalLM": "stablelm",
"MPTForCausalLM": "mpt",
"MT5ForConditionalGeneration": "t5",
@@ -380,6 +381,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"LightOnOCRForConditionalGeneration": "lighton_ocr",
"Llama4ForConditionalGeneration": "llama4",
"LlavaForConditionalGeneration": "llava",
+ "LlavaOnevision2ForConditionalGeneration": "magevl",
"MERaLiON2ForConditionalGeneration": "ultravox",
"MiMoV2ForCausalLM": "mimo",
"MiniMaxM3SparseForConditionalGeneration": "minimax",
diff --git a/conversion/base.py b/conversion/base.py
index ecce23b9c..f31297459 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -3412,6 +3412,9 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
arch = text_config["architectures"][0]
elif model_type == ModelType.MMPROJ and vision_config.get("architectures") is not None:
arch = vision_config["architectures"][0]
+ # some models (llava_onevision2) only declare the architecture inside text_config
+ if arch is None and text_config.get("architectures") is not None:
+ arch = text_config["architectures"][0]
if arch is None:
raise ValueError("Failed to detect model architecture")
return arch
diff --git a/conversion/magevl.py b/conversion/magevl.py
new file mode 100644
index 000000000..910aeecf1
--- /dev/null
+++ b/conversion/magevl.py
@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from typing import Callable, Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import MmprojModel, ModelBase, gguf
+
+
+@ModelBase.register("LlavaOnevision2ForConditionalGeneration")
+@ModelBase.example("microsoft/Mage-VL-Cowork")
+class MageVLVisionModel(MmprojModel):
+ # a rope pair uses 2 different angles: the graph runs 2 ropes on heads permuted to [evens | odds]
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ assert self.hparams_vision is not None
+
+ self.n_embd = self.find_vparam(["hidden_size"])
+ self.n_head = self.find_vparam(["num_attention_heads"])
+ self.d_head = self.n_embd // self.n_head
+
+ d_half = self.d_head // 2
+ assert d_half % 16 == 0, "the 4:6:6 split of the rope requires d_head to be a multiple of 32"
+
+ # 4:6:6 split for t:h:w
+ n_t = 4 * (d_half // 16)
+ n_s = 6 * (d_half // 16)
+ theta = float(self.find_vparam(["rope_theta"]))
+ freq_t = 1.0 / theta ** (torch.arange(n_t, dtype=torch.float64) / n_t)
+ freq_s = 1.0 / theta ** (torch.arange(n_s, dtype=torch.float64) / n_s)
+ freq = torch.cat([freq_t, freq_s, freq_s])
+ self.rope_freq = torch.cat([freq, freq])
+
+ def pair_axis(i: int) -> int:
+ dim = (2 * i) % d_half
+ return 0 if dim < n_t else 1 if dim < n_t + n_s else 2
+
+ self.rope_pairs = sorted(range(d_half), key=pair_axis)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MAGEVL)
+ self.gguf_writer.add_vision_attention_layernorm_eps(self.find_vparam(["layer_norm_eps"]))
+ self.gguf_writer.add_vision_spatial_merge_size(self.find_vparam(["spatial_merge_size"]))
+ self.gguf_writer.add_vision_min_pixels(int(self.preprocessor_config["min_pixels"]))
+ self.gguf_writer.add_vision_max_pixels(int(self.preprocessor_config["max_pixels"]))
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, _ = item
+
+ if not name.startswith("visual."):
+ return None
+
+ return super().filter_tensors(item)
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ # ggml uses theta = pos / rope_freqs
+ pairs = torch.tensor(self.rope_pairs)
+ freq_a = self.rope_freq[2 * pairs]
+ freq_b = self.rope_freq[2 * pairs + 1]
+ yield ("visual.rope_freqs.weight", (1.0 / torch.cat([freq_a, freq_b])).to(torch.float32))
+
+ def permute_qk(self, data_torch: Tensor) -> Tensor:
+ rows = [2 * i for i in self.rope_pairs] + [2 * i + 1 for i in self.rope_pairs]
+ return data_torch.reshape(self.n_head, self.d_head, -1)[:, rows].reshape(data_torch.shape)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if ".self_attn.qkv." in name:
+ q, k, v = data_torch.split(self.n_embd, dim=0)
+ data_torch = torch.cat([self.permute_qk(q), self.permute_qk(k), v], dim=0)
+
+ yield from super().modify_tensors(data_torch, name, bid)
diff --git a/conversion/qwen.py b/conversion/qwen.py
index e6456e5a5..7c9744f51 100644
--- a/conversion/qwen.py
+++ b/conversion/qwen.py
@@ -170,7 +170,7 @@ class Qwen2MoeModel(TextModel):
raise ValueError(f"Unprocessed experts: {experts}")
-@ModelBase.register("Qwen3ForCausalLM", "Qwen3Model")
+@ModelBase.register("Qwen3ForCausalLM", "Qwen3Model", "LlavaOnevision2ForConditionalGeneration")
@ModelBase.example("Qwen/Qwen3-8B")
class Qwen3Model(Qwen2Model):
model_arch = gguf.MODEL_ARCH.QWEN3
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index cdcccd4be..61c51c8fd 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -1112,6 +1112,7 @@ class MODEL_TENSOR(IntEnum):
V_ENC_EMBD_NORM = auto()
V_ENC_EMBD_PATCH_NORM = auto() # allow multiple norms in the same embd, e.g. for gemma4u
V_ENC_EMBD_POS = auto()
+ V_ENC_ROPE_FREQS = auto()
V_ENC_INPUT_NORM = auto()
V_ENC_ATTN_QKV = auto()
V_ENC_ATTN_Q = auto()
@@ -2026,6 +2027,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.V_ENC_EMBD_NORM: "v.norm_embd",
MODEL_TENSOR.V_ENC_EMBD_PATCH_NORM: "v.patch_norm.{bid}",
MODEL_TENSOR.V_ENC_EMBD_POS: "v.position_embd",
+ MODEL_TENSOR.V_ENC_ROPE_FREQS: "v.rope_freqs",
MODEL_TENSOR.V_ENC_ATTN_QKV: "v.blk.{bid}.attn_qkv",
MODEL_TENSOR.V_ENC_ATTN_Q: "v.blk.{bid}.attn_q",
MODEL_TENSOR.V_ENC_ATTN_Q_NORM: "v.blk.{bid}.attn_q_norm",
@@ -2426,6 +2428,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.V_ENC_EMBD_NORM,
MODEL_TENSOR.V_ENC_EMBD_PATCH_NORM,
MODEL_TENSOR.V_ENC_EMBD_POS,
+ MODEL_TENSOR.V_ENC_ROPE_FREQS,
MODEL_TENSOR.V_ENC_EMBD_IMGNL,
MODEL_TENSOR.V_ENC_EMBD_VSEP,
MODEL_TENSOR.V_TOK_EMBD_SEP,
@@ -7077,6 +7080,7 @@ class VisionProjectorType:
GRANITE_SPEECH = "granite_speech" # audio
NEMOTRON_TRANSCRIBE = "nemotron_transcribe" # audio
MIMOVL = "mimovl"
+ MAGEVL = "magevl"
MIMO_AUDIO = "mimo_audio"
GRANITE4_VISION = "granite4_vision"
MUSE_GLIMMER = "muse-glimmer"
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index eeadd6a9a..8888dc76e 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -1863,7 +1863,12 @@ class TensorNameMap:
"model.vision_model.embeddings.class_embedding", # Deepseek-OCR
),
+ MODEL_TENSOR.V_ENC_ROPE_FREQS: (
+ "visual.rope_freqs",
+ ),
+
MODEL_TENSOR.V_ENC_EMBD_PATCH: (
+ "visual.embeddings.patch_embedding", # mage-vl
"vision.patch_embed.proj", # deepseek4v
"model.vision_tower.vision_model.embeddings.patch_embedding", # Granite4Vision
"vision_tower.vision_model.embeddings.patch_embedding",
@@ -1933,6 +1938,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_ATTN_QKV: (
+ "visual.encoder.layers.{bid}.self_attn.qkv", # mage-vl
"vision.blocks.{bid}.attn.wqkv", # deepseek4v
"visual.blocks.{bid}.attn.qkv", # qwen3vl
"vision_tower.blocks.{bid}.attn.qkv", # dots.ocr
@@ -2029,6 +2035,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_INPUT_NORM: (
+ "visual.encoder.layers.{bid}.layer_norm1", # mage-vl
"vision_model.encoder.layer.{bid}.norm1", # Nemotron Omni
"vision.blocks.{bid}.norm1", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", # Granite4Vision
@@ -2057,6 +2064,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_ATTN_O: (
+ "visual.encoder.layers.{bid}.self_attn.proj", # mage-vl
"vision_model.encoder.layer.{bid}.attention.output.dense", # Nemotron Omni
"vision.blocks.{bid}.attn.wo", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", # Granite4Vision
@@ -2091,6 +2099,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_POST_ATTN_NORM: (
+ "visual.encoder.layers.{bid}.layer_norm2", # mage-vl
"vision_model.encoder.layer.{bid}.norm2", # Nemotron Omni
"vision.blocks.{bid}.norm2", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", # Granite4Vision
@@ -2120,6 +2129,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_FFN_UP: (
+ "visual.encoder.layers.{bid}.mlp.fc1", # mage-vl
"vision_model.encoder.layer.{bid}.mlp.fc1", # Nemotron Omni
"vision.blocks.{bid}.mlp.w1_up", # deepseek4v (split from fused w1)
"vision_encoder.blocks.{bid}.mlp.fc3", # dots3note
@@ -2160,6 +2170,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_FFN_DOWN: (
+ "visual.encoder.layers.{bid}.mlp.fc2", # mage-vl
"vision_model.encoder.layer.{bid}.mlp.fc2", # Nemotron Omni
"vision.blocks.{bid}.mlp.w2", # deepseek4v
"vision_encoder.blocks.{bid}.mlp.fc2", # dots3note
@@ -2238,6 +2249,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_PRE_NORM: (
+ "visual.layernorm_pre", # mage-vl
"vision_tower.vision_model.pre_layrnorm",
"vision_tower.ln_pre", # pixtral-hf
"vision_encoder.ln_pre", # pixtral
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index d92340f6b..fd8b24efd 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -65,6 +65,7 @@ add_library(mtmd
models/sensenova-u1.cpp
models/minimax-m3.cpp
models/qwen3vl.cpp
+ models/magevl.cpp
models/lfm3vl.cpp
models/lfm3a.cpp
models/lfm3a-gen.cpp
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index cd7359e1d..8a4098da8 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -127,6 +127,7 @@
#define TN_PATCH_EMBD "v.patch_embd.weight" // not rename tensor with ".0" postfix for backward compat
#define TN_PATCH_EMBD_1 "v.patch_embd.weight.1"
#define TN_PATCH_BIAS "v.patch_embd.bias"
+#define TN_ROPE_FREQS "v.rope_freqs.weight"
#define TN_NORM_EMBD "v.norm_embd.%s"
#define TN_PATCH_NORM "v.patch_norm.%d.%s"
#define TN_ATTN_QKV "%s.blk.%d.attn_qkv.%s"
@@ -593,6 +594,7 @@ enum projector_type {
PROJECTOR_TYPE_GRANITE_SPEECH,
PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE,
PROJECTOR_TYPE_MIMOVL,
+ PROJECTOR_TYPE_MAGEVL,
PROJECTOR_TYPE_MINIMAX_M3,
PROJECTOR_TYPE_GRANITE4_VISION,
PROJECTOR_TYPE_MIMO_AUDIO,
@@ -677,6 +679,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"},
{ PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE, "nemotron_transcribe"},
{ PROJECTOR_TYPE_MIMOVL, "mimovl"},
+ { PROJECTOR_TYPE_MAGEVL, "magevl"},
{ PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"},
{ PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"},
{ PROJECTOR_TYPE_MIMO_AUDIO, "mimo_audio"},
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h
index 13ac5f02b..f50fc40b9 100644
--- a/tools/mtmd/clip-model.h
+++ b/tools/mtmd/clip-model.h
@@ -873,6 +873,7 @@ struct clip_model {
ggml_tensor * patch_embeddings_1 = nullptr; // additional patch kernel; interpretation is model-specific (Qwen2VL, Nemotron Omni)
ggml_tensor * patch_bias = nullptr;
ggml_tensor * position_embeddings = nullptr;
+ ggml_tensor * rope_freqs = nullptr;
ggml_tensor * norm_embd_w = nullptr;
ggml_tensor * norm_embd_b = nullptr;
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index d4e0fc901..7ab36a9fc 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -1027,6 +1027,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_qwen3vl>(ctx, img);
} break;
+ case PROJECTOR_TYPE_MAGEVL:
+ {
+ builder = std::make_unique<clip_graph_magevl>(ctx, img);
+ } break;
case PROJECTOR_TYPE_SENSENOVA_U1:
{
builder = std::make_unique<clip_graph_sensenova_u1>(ctx, img);
@@ -1941,6 +1945,18 @@ struct clip_model_loader {
LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__);
}
} break;
+ case PROJECTOR_TYPE_MAGEVL:
+ {
+ hparams.ffn_op = FFN_GELU_ERF;
+ log_ffn_op = "gelu_erf";
+ hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
+ hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding
+ hparams.image_round_half_even = true; // smart_resize uses Python round()
+ get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge);
+ get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
+ get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
+ hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
+ } break;
case PROJECTOR_TYPE_SENSENOVA_U1:
{
hparams.n_merge = 2;
@@ -3038,6 +3054,20 @@ struct clip_model_loader {
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
+ case PROJECTOR_TYPE_MAGEVL:
+ {
+ // d_head rope factors: the first half drives the even outputs, the second half the odd ones
+ model.rope_freqs = get_tensor(TN_ROPE_FREQS);
+ const int d_head = hparams.n_head > 0 ? hparams.n_embd / hparams.n_head : 0;
+ if (d_head <= 0 || d_head * hparams.n_head != hparams.n_embd || d_head % 32 != 0 ||
+ model.rope_freqs->type != GGML_TYPE_F32 || ggml_nelements(model.rope_freqs) != d_head) {
+ throw std::runtime_error(string_format("%s: hparams do not match the tensor shapes\n", __func__));
+ }
+ model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
+ model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
+ model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
+ model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
+ } break;
case PROJECTOR_TYPE_QWEN3VL:
{
model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
@@ -5207,6 +5237,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
+ case PROJECTOR_TYPE_MAGEVL:
case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_LING3VL:
case PROJECTOR_TYPE_EXAONE4_5:
@@ -5240,6 +5271,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
+ case PROJECTOR_TYPE_MAGEVL:
case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_LING3VL:
case PROJECTOR_TYPE_EXAONE4_5:
@@ -5329,6 +5361,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
+ case PROJECTOR_TYPE_MAGEVL:
case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_LING3VL:
case PROJECTOR_TYPE_EXAONE4_5:
@@ -6277,6 +6310,27 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
set_input_i32("positions", positions);
} break;
+ case PROJECTOR_TYPE_MAGEVL:
+ {
+ // t, h, w in 2x2 block order, t is always 0 for an image
+ const int merge_ratio = hparams.n_merge;
+ const int pw = image_size_width / patch_size / merge_ratio;
+ const int ph = image_size_height / patch_size / merge_ratio;
+ std::vector<int> positions(n_pos * 4, 0);
+ int ptr = 0;
+ for (int ty = 0; ty < ph; ty++) {
+ for (int tx = 0; tx < pw; tx++) {
+ for (int dy = 0; dy < merge_ratio; dy++) {
+ for (int dx = 0; dx < merge_ratio; dx++) {
+ positions[ num_patches + ptr] = ty * merge_ratio + dy;
+ positions[2 * num_patches + ptr] = tx * merge_ratio + dx;
+ ptr++;
+ }
+ }
+ }
+ }
+ set_input_i32("positions", positions);
+ } break;
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_LING3VL:
@@ -7959,6 +8013,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
+ case PROJECTOR_TYPE_MAGEVL:
case PROJECTOR_TYPE_LFM3VL:
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_JANUS_PRO:
diff --git a/tools/mtmd/models/magevl.cpp b/tools/mtmd/models/magevl.cpp
new file mode 100644
index 000000000..f5370ea9f
--- /dev/null
+++ b/tools/mtmd/models/magevl.cpp
@@ -0,0 +1,61 @@
+#include "models.h"
+
+ggml_cgraph * clip_graph_magevl::build() {
+ GGML_ASSERT(hparams.n_merge == 2);
+
+ const int n_pair = d_head / 2;
+ const int unit = n_pair / 16;
+
+ // pairs are grouped as t, h, w at conversion
+ int sections[4] = {4 * unit, 6 * unit, 6 * unit, 0};
+
+ ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4);
+ ggml_set_name(positions, "positions");
+ ggml_set_input(positions);
+
+ // a pair uses 2 angles: even outputs come from one rope, odd outputs from the other
+ auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ ggml_tensor * freqs_a = ggml_view_1d(ctx0, model.rope_freqs, n_pair, 0);
+ ggml_tensor * freqs_b = ggml_view_1d(ctx0, model.rope_freqs, n_pair, ggml_row_size(model.rope_freqs->type, n_pair));
+
+ ggml_tensor * rope_a = ggml_rope_multi(ctx0, cur, positions, freqs_a, d_head, sections,
+ GGML_ROPE_TYPE_MROPE, 32768, 1.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f);
+ ggml_tensor * rope_b = ggml_rope_multi(ctx0, cur, positions, freqs_b, d_head, sections,
+ GGML_ROPE_TYPE_MROPE, 32768, 1.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f);
+
+ ggml_tensor * even = ggml_view_3d(ctx0, rope_a, n_pair, n_head, n_patches,
+ rope_a->nb[1], rope_a->nb[2], 0);
+ ggml_tensor * odd = ggml_view_3d(ctx0, rope_b, n_pair, n_head, n_patches,
+ rope_b->nb[1], rope_b->nb[2], ggml_row_size(rope_b->type, n_pair));
+
+ return ggml_concat(ctx0, even, odd, 0);
+ };
+
+ // ggml_conv_2d rounds patches to F16
+ ggml_tensor * inp_raw = build_inp_raw();
+ ggml_tensor * patches = ggml_im2col(ctx0, model.patch_embeddings_0, inp_raw,
+ patch_size, patch_size, 0, 0, 1, 1, true, GGML_TYPE_F32);
+ const int64_t n_patch_dim = patches->ne[0];
+ ggml_tensor * inp = ggml_mul_mat(ctx0,
+ ggml_reshape_2d(ctx0, model.patch_embeddings_0, n_patch_dim, n_embd),
+ ggml_reshape_2d(ctx0, patches, n_patch_dim, n_patches));
+
+ // reorder into 2x2 blocks
+ inp = ggml_reshape_4d(ctx0, inp, n_embd * 2, n_patches_x / 2, 2, n_patches_y / 2);
+ inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
+ inp = ggml_cont_3d(ctx0, inp, n_embd, n_patches, 1);
+
+ ggml_tensor * cur = build_vit(inp, n_patches, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, add_pos);
+
+ // multimodal projection
+ cur = ggml_reshape_3d(ctx0, cur, n_embd * 4, n_patches / 4, 1);
+ cur = build_ffn(cur,
+ model.mm_0_w, model.mm_0_b,
+ nullptr, nullptr,
+ model.mm_1_w, model.mm_1_b,
+ hparams.ffn_op, -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+}
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 3323f0e9d..8487a0042 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -66,6 +66,11 @@ struct clip_graph_qwen3vl : clip_graph_qwen2vl {
ggml_cgraph * build() override;
};
+struct clip_graph_magevl : clip_graph {
+ clip_graph_magevl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+ ggml_cgraph * build() override;
+};
+
struct clip_graph_sensenova_u1 : clip_graph {
clip_graph_sensenova_u1(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 990ca042c..72c92179f 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -816,6 +816,7 @@ struct mtmd_context {
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_LING3VL:
case PROJECTOR_TYPE_MIMOVL:
+ case PROJECTOR_TYPE_MAGEVL:
{
// <|vision_start|> ... (image embeddings) ... <|vision_end|>
img_beg = "<|vision_start|>";