Commit 7d7f02875 for llama.cpp
commit 7d7f0287530bc8981690f361d8c1627dbae1c99a
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Sun Nov 8 15:49:06 2026 +0100
mtmd: refactoring: remove clip.h, consolidate code into mtmd (#31865)
diff --git a/conversion/chatterbox.py b/conversion/chatterbox.py
index bca0a2633..d438228e5 100644
--- a/conversion/chatterbox.py
+++ b/conversion/chatterbox.py
@@ -355,7 +355,7 @@ class ChatterboxMmprojModel(MmprojModel):
def set_gguf_parameters(self):
self.gguf_writer.add_file_type(self.ftype)
- # speaker encoder, the front-end hparams are hard-coded in clip.cpp
+ # speaker encoder, the front-end hparams are hard-coded in tools/mtmd
self.gguf_writer.add_clip_has_audio_encoder(True)
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.CHATTERBOX_SPKENC)
self.gguf_writer.add_audio_projection_dim(80)
diff --git a/conversion/deepseek.py b/conversion/deepseek.py
index 581111da5..d559b604e 100644
--- a/conversion/deepseek.py
+++ b/conversion/deepseek.py
@@ -1078,7 +1078,7 @@ class DeepseekV4FlashVisionModel(MmprojModel):
self.gguf_writer.add_vision_use_silu(True) # SwiGLU MLP
self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"])
self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"])
- # hardcoded on the C++ side (see PROJECTOR_TYPE_DEEPSEEK4V in clip.cpp)
+ # hardcoded on the C++ side (see PROJECTOR_TYPE_DEEPSEEK4V in tools/mtmd)
# if future models use different values, add GGUF keys for those
assert self.global_config["vision_max_n_token"] == 384
assert self.global_config["vision_max_wh_ratio"] == 8
diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py
index 53980c1d0..6b098a037 100644
--- a/conversion/deepseek41.py
+++ b/conversion/deepseek41.py
@@ -458,7 +458,7 @@ class DeepseekV41VisionModel(DeepseekV4FlashVisionModel):
self.gguf_writer.add_vision_use_silu(True)
self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"])
self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"])
- # hardcoded in clip.cpp (PROJECTOR_TYPE_DEEPSEEK41V); add GGUF keys if a later model differs
+ # hardcoded in tools/mtmd (PROJECTOR_TYPE_DEEPSEEK41V); add GGUF keys if a later model differs
assert self.hparams_vision["max_image_tokens"] == 1024, self.hparams_vision["max_image_tokens"]
assert self.hparams_vision["max_wh_ratio"] is None, self.hparams_vision["max_wh_ratio"]
assert self.hparams_vision["rope_theta"] == 10000, self.hparams_vision["rope_theta"]
diff --git a/conversion/drama3.py b/conversion/drama3.py
index 7c590a014..ac01f481c 100644
--- a/conversion/drama3.py
+++ b/conversion/drama3.py
@@ -77,7 +77,7 @@ class Drama3MmprojModel(MmprojModel):
self.gguf_writer.add_gen_audio_head_count(cfg["n_head"])
self.gguf_writer.add_gen_audio_head_count_kv(cfg["n_local_heads"])
self.gguf_writer.add_gen_audio_attention_layernorm_eps(cfg["norm_eps"])
- # other tower and codec hparams are hardcoded in clip.cpp
+ # other tower and codec hparams are hardcoded in tools/mtmd
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
diff --git a/conversion/lfm3.py b/conversion/lfm3.py
index a0287d4bb..38fbb5a31 100644
--- a/conversion/lfm3.py
+++ b/conversion/lfm3.py
@@ -181,11 +181,11 @@ class LFM3VLVisionMixin(MmprojModel):
def _set_lfm3vl_gguf_parameters(self):
assert self.hparams_vision is not None
- # clip.cpp hardcodes the GELU projector and its input norm eps
+ # mtmd hardcodes the GELU projector and its input norm eps
if (self.global_config["projector_hidden_act"] != "gelu" or not self.global_config["projector_bias"]
or self.global_config["projector_layer_norm_eps"] != 1e-6):
raise ValueError("unsupported projector_hidden_act / projector_bias / projector_layer_norm_eps")
- # clip.cpp hardcodes the vision rope base, the RGB patchifier and the 3x3 spatial conv
+ # mtmd hardcodes the vision rope base, the RGB patchifier and the 3x3 spatial conv
if (self.hparams_vision["rope_parameters"]["rope_theta"] != 1e6 or self.hparams_vision["in_channels"] != 3
or self.hparams_vision["conv_kernel_size"] != 3):
raise ValueError("unsupported vision rope_theta / in_channels / conv_kernel_size")
@@ -347,7 +347,7 @@ class LFM3AudioMmprojModel(MmprojModel):
"feed_forward.w3": gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP,
"feed_forward.w2": gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN,
}
- # the codec graph in clip.cpp hardcodes the liquid-codec FSQ front end and decoder shape, conversion fails on a mismatch
+ # the codec graph in mtmd hardcodes the liquid-codec FSQ front end and decoder shape, conversion fails on a mismatch
_codec = {
"model_type": "liquid_codec", "fsq_levels": [8, 7, 6, 6], "n_q": 12, "quantizer_dim": 256,
"encoder_dim": 512, "stft_n_fft": 1280, "stft_hop_length": 320, "stft_win_length": 1280, "stft_frame_stack": 6,
@@ -405,7 +405,7 @@ class LFM3AudioMmprojModel(MmprojModel):
if fe[k] != v:
raise ValueError(f"unsupported feature_extractor.{k}={fe[k]!r}, expected {v!r}")
- # clip.cpp hardcodes the encoder rope base
+ # mtmd hardcodes the encoder rope base
if self.hparams_audio["rope_parameters"]["rope_theta"] != 1e6:
raise ValueError("unsupported audio encoder rope_theta")
@@ -420,7 +420,7 @@ class LFM3AudioMmprojModel(MmprojModel):
if not self._audio_output:
return
- # audio generation: depthformer hparams, the codec ones are constant and live in clip.cpp
+ # audio generation: depthformer hparams, the codec ones are constant and live in mtmd
gen = self.get_generation_config()
for k, v in self._generation.items():
if gen[k] != v:
diff --git a/conversion/minicpm.py b/conversion/minicpm.py
index d8fceb0c0..8c7be8a9a 100644
--- a/conversion/minicpm.py
+++ b/conversion/minicpm.py
@@ -161,7 +161,7 @@ class MiniCPMV4_6VisionModel(MmprojModel):
# In MiniCPM-V 4.6 `vision_config.image_size` (980) describes the SigLIP
# positional embedding bucket grid (70 x 70), while the per-slice processing
# resolution is the preprocessor's `scale_resolution` (typically 448).
- # The CLIP loader in tools/mtmd/clip.cpp consumes `clip.vision.image_size`
+ # The mmproj loader in tools/mtmd/mtmd-model.cpp consumes `clip.vision.image_size`
# as the slice size and warmup resolution, so report `scale_resolution` there
# to match the upstream MiniCPMV4_6ImageProcessorPil slicing rules.
scale_resolution = self.preprocessor_config.get(
@@ -173,7 +173,7 @@ class MiniCPMV4_6VisionModel(MmprojModel):
super().set_gguf_parameters()
assert self.hparams_vision is not None
- # projector type string is consumed by clip_projector_type_from_string() in clip.cpp
+ # projector type string is consumed by projector_type_from_string() in tools/mtmd/mtmd-impl.h
self.gguf_writer.add_clip_projector_type(self.projector_type)
self.gguf_writer.add_vision_projector_scale_factor(
@@ -243,7 +243,7 @@ class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel):
projector_type = gguf.VisionProjectorType.MINICPMV4_7
# MiniCPMV4_7ImageProcessorPil default
default_scale_resolution = 448
- # rows of v.tok_embd_sep, the order must match clip_suffix_rows() in clip-impl.h
+ # rows of v.tok_embd_sep, the order must match mtmd_suffix_rows() in tools/mtmd/mtmd-impl.h
tok_embd_sep = ["</image>", "<slice>", "</slice>", "\n"]
def get_downsample_mode(self) -> str:
diff --git a/conversion/nemotron.py b/conversion/nemotron.py
index 066149744..c818a73e1 100644
--- a/conversion/nemotron.py
+++ b/conversion/nemotron.py
@@ -208,7 +208,7 @@ class NemotronNanoV2VLModel(MmprojModel):
return name, gen
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- # RADIO's pos_embed doesn't have .weight suffix, but clip.cpp expects it
+ # RADIO's pos_embed doesn't have .weight suffix, but mtmd expects it
if "patch_generator.pos_embed" in name:
# Downsample position embeddings for fixed 512x512 image size
import torch.nn.functional as F
diff --git a/conversion/pockettts.py b/conversion/pockettts.py
index 1b5c84111..c9198a221 100644
--- a/conversion/pockettts.py
+++ b/conversion/pockettts.py
@@ -302,7 +302,7 @@ class PocketTTSMmprojModel(MmprojModel):
# voice-prompt encoder: mimi encoder + speaker_proj
if self.has_audio_encoder:
self.gguf_writer.add_clip_has_audio_encoder(True)
- # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models
+ # note: the 24kHz sample rate is hardcoded on the mtmd side, like the other audio models
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC)
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"])
@@ -314,7 +314,7 @@ class PocketTTSMmprojModel(MmprojModel):
self.gguf_writer.add_audio_num_mel_bins(1)
# generation: flow-matching decoder + mimi decoder
- # the SEANet and flow net hparams are constant across the family, clip.cpp holds them
+ # the SEANet and flow net hparams are constant across the family, mtmd holds them
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_GEN)
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py
index cdb1e0c9b..28bb6e095 100644
--- a/conversion/qwen3tts.py
+++ b/conversion/qwen3tts.py
@@ -235,11 +235,11 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
# handle speaker encoder config
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
- # mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the clip.cpp default)
+ # mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the mtmd default)
self.gguf_writer.add_audio_num_mel_bins(128)
# 3 SE-Res2Net stages; the stem conv, mfa, asp and fc are not counted here
self.gguf_writer.add_audio_block_count(3)
- # ECAPA-TDNN has no attention/FFN, these are dummy to allow clip.cpp to load it
+ # ECAPA-TDNN has no attention/FFN, these are dummy to allow mtmd to load it
self.gguf_writer.add_audio_embedding_length(1536)
self.gguf_writer.add_audio_head_count(1)
self.gguf_writer.add_audio_feed_forward_length(1536)
@@ -256,7 +256,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
self.gguf_writer.add_gen_audio_head_count(code_predictor_config["num_attention_heads"])
self.gguf_writer.add_gen_audio_head_count_kv(code_predictor_config["num_key_value_heads"])
self.gguf_writer.add_gen_audio_attention_layernorm_eps(code_predictor_config["rms_norm_eps"])
- # note: code2wav hparams are hardcoded on the mtmd/clip.cpp side for now, not written here
+ # note: code2wav hparams are hardcoded on the mtmd side for now, not written here
def _wav_decoder_config(self) -> dict[str, Any] | None:
# code2wav has its own config.json, inside the speech_tokenizer dir
diff --git a/docs/development/HOWTO-add-model.md b/docs/development/HOWTO-add-model.md
index abf6503f2..558363f4e 100644
--- a/docs/development/HOWTO-add-model.md
+++ b/docs/development/HOWTO-add-model.md
@@ -128,7 +128,7 @@ Note: to debug the inference graph: you can use [llama-eval-callback](/examples/
If the new model supports multimodal inputs, you will need to add a new encoder definition in `libmtmd`. You can find more information about llama.cpp's multimodal support in [the docs](../multimodal.md) and in the `tools/mtmd` source directory.
1. In the conversion script, make sure you add a subclass that extends `MmprojModel` or another class that inherits from the same base class.
-2. Add the encoder definition in `clip.cpp`.
+2. Add the encoder definition: hparams and tensor loading in `mtmd-model.cpp`, graph inputs in `mtmd-subcontext.cpp`.
3. Implement the preprocessor in `mtmd.cpp`. In most cases, you can reuse an existing preprocessor.
4. Implement the encoder GGML graph, either in a dedicated file if the model is truly different from existing ones, or by reusing an existing implementation (for example: siglip, pixtral, or qwen) and adding a model-specific projector.
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 61c51c8fd..5f9cbf1e3 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -589,7 +589,7 @@ class GGUFType:
class MODEL_ARCH(IntEnum):
- MMPROJ = auto() # dummy arch for clip.cpp
+ MMPROJ = auto() # dummy arch for mmproj
LLAMA = auto()
LLAMA4 = auto()
DECI = auto()
@@ -1508,7 +1508,7 @@ class MODEL_TENSOR(IntEnum):
MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
- MODEL_ARCH.MMPROJ: "clip", # dummy arch for clip.cpp
+ MODEL_ARCH.MMPROJ: "clip", # dummy arch for mmproj files (tools/mtmd)
MODEL_ARCH.LLAMA: "llama",
MODEL_ARCH.LLAMA4: "llama4",
MODEL_ARCH.DECI: "deci",
@@ -2200,14 +2200,14 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_GEN_CODE_EMBD: "a.gen.code.embd",
MODEL_TENSOR.A_GEN_CODE_HEAD: "a.gen.code.head",
MODEL_TENSOR.A_GEN_CODE_OUT_EMBD: "a.gen.code.out_embd",
- MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.ln1", # reuses the generic clip.cpp block loader (TN_LN_1)
+ MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.ln1", # reuses the generic mtmd block loader (TN_LN_1)
MODEL_TENSOR.A_GEN_CODE_ATTN_Q: "a.gen.code.blk.{bid}.attn_q",
MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM: "a.gen.code.blk.{bid}.attn_q_norm",
MODEL_TENSOR.A_GEN_CODE_ATTN_K: "a.gen.code.blk.{bid}.attn_k",
MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM: "a.gen.code.blk.{bid}.attn_k_norm",
MODEL_TENSOR.A_GEN_CODE_ATTN_V: "a.gen.code.blk.{bid}.attn_v",
MODEL_TENSOR.A_GEN_CODE_ATTN_OUT: "a.gen.code.blk.{bid}.attn_out",
- MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ln2", # reuses the generic clip.cpp block loader (TN_LN_2)
+ MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ln2", # reuses the generic mtmd block loader (TN_LN_2)
MODEL_TENSOR.A_GEN_CODE_FFN_GATE: "a.gen.code.blk.{bid}.ffn_gate",
MODEL_TENSOR.A_GEN_CODE_FFN_UP: "a.gen.code.blk.{bid}.ffn_up",
MODEL_TENSOR.A_GEN_CODE_FFN_DOWN: "a.gen.code.blk.{bid}.ffn_down",
diff --git a/skills/add-new-model/SKILL.md b/skills/add-new-model/SKILL.md
index 710a1ebb4..3e4d178eb 100644
--- a/skills/add-new-model/SKILL.md
+++ b/skills/add-new-model/SKILL.md
@@ -50,7 +50,7 @@ Skill-specific addition: before writing `src/models/<name>.cpp`, read at least 1
## Step 4 - Optional: multimodal encoder
-Only do this if the contributor flagged a vision/audio encoder in Step 0. Follow HOWTO-add-model.md section 4 and `docs/multimodal.md` for the actual touch points (`MmprojModel` subclass, `clip.cpp`, `mtmd.cpp`, encoder graph in `tools/mtmd/models`, etc.).
+Only do this if the contributor flagged a vision/audio encoder in Step 0. Follow HOWTO-add-model.md section 4 and `docs/multimodal.md` for the actual touch points (`MmprojModel` subclass, `mtmd-model.cpp`, `mtmd-subcontext.cpp`, `mtmd.cpp`, encoder graph in `tools/mtmd/models`, etc.).
Skill-specific addition, and read this carefully: **whether the multimodal encoder can be bundled into the same PR as the base text-model support depends on how conventional the change is.** It's OK to bundle it if the encoder support is conventional - i.e. no new infra or logic is needed, it's just a new cgraph reusing existing preprocessing/projector machinery (e.g. siglip/pixtral/qwen with just a new projector). If it requires anything beyond that - a new preprocessor, non-standard projector logic, or changes to shared `libmtmd` infra/logic - STOP, tell the contributor this is non-conventional, and have them land the text model first with the encoder as a dedicated follow-up PR. Do not let this decision pass silently - call it out explicitly to the contributor before writing any `clip.cpp`/`mtmd.cpp` code.
diff --git a/src/models/clip.cpp b/src/models/clip.cpp
index 537766aeb..3bf7ac3b8 100644
--- a/src/models/clip.cpp
+++ b/src/models/clip.cpp
@@ -14,5 +14,5 @@ void llama_model_clip::load_arch_tensors(llama_model_loader &) {
[[noreturn]]
std::unique_ptr<llm_graph_context> llama_model_clip::build_arch_graph(const llm_graph_params &) const {
- GGML_ABORT("CLIP has no inference graph via llama_model dispatch; runtime lives in tools/mtmd/clip.cpp");
+ GGML_ABORT("CLIP has no inference graph via llama_model dispatch; runtime lives in tools/mtmd");
}
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 016fca3c3..a17f6b43e 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -6561,7 +6561,7 @@ struct test_rope : public test_case {
a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0);
ggml_set_name(a, "view_of_a");
} else if (v == 2) {
- // second-half slice along dim 0 (mimics build_rope_2d in clip.cpp).
+ // second-half slice along dim 0 (mimics build_rope_2d in tools/mtmd/mtmd-graph.cpp).
// The non-zero view offset (ne_a[0] * elem_size) often produces a
// non-aligned buffer offset, which exercises backends' alignment paths.
auto ne = ne_a; ne[0] *= 2;
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp
index e9c6ac856..a0201bdde 100644
--- a/tests/test-mtmd-impl.cpp
+++ b/tests/test-mtmd-impl.cpp
@@ -3,6 +3,7 @@
#include "ggml-backend.h"
#include "ggml-cpp.h"
#include "mtmd-audio.h"
+#include "mtmd-subcontext.h"
#include "mtmd-helper.h"
#include "mtmd-image.h"
#include "mtmd-internal.h"
@@ -52,12 +53,12 @@ struct test_registry {
//
MAKE_TEST(test_image_preprocessor_longest_edge) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.image_longest_edge = 1540;
hparams.patch_size = 14;
hparams.n_merge = 2;
- const std::vector<std::pair<clip_image_size, clip_image_size>> cases = {
+ const std::vector<std::pair<mtmd_image_size, mtmd_image_size>> cases = {
{ { 463, 463 }, { 476, 476 } },
{ { 463, 300 }, { 476, 308 } },
{ { 448, 280 }, { 448, 280 } },
@@ -65,7 +66,7 @@ MAKE_TEST(test_image_preprocessor_longest_edge) {
};
for (const auto & [input, expected] : cases) {
- const clip_image_size actual = mtmd_image_preprocessor_longest_edge::get_target_size(hparams, input);
+ const mtmd_image_size actual = mtmd_image_preprocessor_longest_edge::get_target_size(hparams, input);
const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height);
t.assert_equal(name + " width", expected.width, actual.width);
t.assert_equal(name + " height", expected.height, actual.height);
@@ -75,7 +76,7 @@ MAKE_TEST(test_image_preprocessor_longest_edge) {
MAKE_TEST(test_image_preprocessor_dyn_size) {
// { patch_size, image size, expected size }
// a side that is an even number plus .5 merged patches rounds down, like Python round() in smart_resize
- const std::vector<std::tuple<int, clip_image_size, clip_image_size>> cases = {
+ const std::vector<std::tuple<int, mtmd_image_size, mtmd_image_size>> cases = {
{ 16, { 768, 400 }, { 768, 384 } },
{ 16, { 800, 336 }, { 800, 320 } },
{ 16, { 400, 400 }, { 384, 384 } },
@@ -84,13 +85,13 @@ MAKE_TEST(test_image_preprocessor_dyn_size) {
};
for (const auto & [patch_size, input, expected] : cases) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.patch_size = patch_size;
hparams.n_merge = 2;
hparams.image_round_half_even = true;
hparams.set_limit_image_tokens(8, 4096);
- const clip_image_size actual = mtmd_image_preprocessor_dyn_size::get_target_size(hparams, input);
+ const mtmd_image_size actual = mtmd_image_preprocessor_dyn_size::get_target_size(hparams, input);
const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height);
t.assert_equal(name + " width", expected.width, actual.width);
t.assert_equal(name + " height", expected.height, actual.height);
@@ -98,13 +99,13 @@ MAKE_TEST(test_image_preprocessor_dyn_size) {
}
MAKE_TEST(test_image_preprocessor_qwen3vl_video) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.patch_size = 16;
hparams.n_merge = 2;
const mtmd_image_preprocessor_qwen3vl preproc(hparams);
// { n_frames, frame size, expected size from Qwen3VLVideoProcessor smart_resize }
- const std::vector<std::tuple<int, clip_image_size, clip_image_size>> cases = {
+ const std::vector<std::tuple<int, mtmd_image_size, mtmd_image_size>> cases = {
{ 20, { 720, 358 }, { 704, 352 } }, // 720 / 32 = 22.5 rounds half to even
{ 768, { 1920, 1080 }, { 224, 128 } }, // over the longest_edge budget
{ 4, { 16, 40 }, { 32, 64 } }, // edge below the factor is upscaled first
@@ -114,7 +115,7 @@ MAKE_TEST(test_image_preprocessor_qwen3vl_video) {
};
for (const auto & [n_frames, input, expected] : cases) {
- clip_image_size actual;
+ mtmd_image_size actual;
const std::string name = std::to_string(n_frames) + " frames of " + std::to_string(input.width) + "x" + std::to_string(input.height);
if (!t.assert_true(name + " has a frame size", preproc.get_video_frame_size(n_frames, input.width, input.height, actual))) {
continue;
@@ -124,7 +125,7 @@ MAKE_TEST(test_image_preprocessor_qwen3vl_video) {
}
}
-static std::string image_size_to_str(const clip_image_size & size) {
+static std::string image_size_to_str(const mtmd_image_size & size) {
return std::to_string(size.width) + "x" + std::to_string(size.height);
}
@@ -143,8 +144,8 @@ static std::string resize_algo_to_str(resize_algo algo) {
}
// pseudo-random pixels: s = s * 1103515245 + 12345 from s = 1, one byte (s >> 16) & 255 per channel
-static clip_image_u8 make_test_image(const clip_image_size & size) {
- clip_image_u8 img;
+static mtmd_image_u8 make_test_image(const mtmd_image_size & size) {
+ mtmd_image_u8 img;
img.set_size(size, false);
uint32_t s = 1;
auto next = [&s]() {
@@ -160,7 +161,7 @@ static clip_image_u8 make_test_image(const clip_image_size & size) {
}
// FNV-1a of the pixels
-static uint32_t hash_pixels(const clip_image_u8 & img, uint32_t hash = 2166136261u) {
+static uint32_t hash_pixels(const mtmd_image_u8 & img, uint32_t hash = 2166136261u) {
for (uint8_t v : img.get_ro_buf()) {
hash = (hash ^ v) * 16777619u;
}
@@ -168,7 +169,7 @@ static uint32_t hash_pixels(const clip_image_u8 & img, uint32_t hash = 216613626
}
MAKE_TEST(test_image_preprocessor_lfm2) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.patch_size = 16;
hparams.n_merge = 2;
hparams.set_limit_image_tokens(64, 256);
@@ -178,7 +179,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) {
hparams.preproc_max_pixels_tolerance = 2.0f;
// { image size, expected tiling }
- const std::vector<std::pair<clip_image_size, bool>> cases = {
+ const std::vector<std::pair<mtmd_image_size, bool>> cases = {
{ { 704, 704 }, false },
// 720 / (patch_size * n_merge) is exactly 22.5, so this only matches HF
// if round_by_factor rounds half to even (22) instead of away from zero (23)
@@ -198,7 +199,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) {
}
// { image size, expected size from HF smart_resize }
- const std::vector<std::pair<clip_image_size, clip_image_size>> resize_cases = {
+ const std::vector<std::pair<mtmd_image_size, mtmd_image_size>> resize_cases = {
{ { 400, 600 }, { 384, 608 } }, // 400 / 32 = 12.5 rounds half to even
{ { 480, 336 }, { 480, 320 } },
{ { 64, 169 }, { 160, 416 } }, // 169 * beta / 32 is exactly 13 in double
@@ -211,7 +212,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) {
}
// { image size, expected grid from HF }
- const std::vector<std::pair<clip_image_size, clip_image_size>> grid_cases = {
+ const std::vector<std::pair<mtmd_image_size, mtmd_image_size>> grid_cases = {
{ { 720, 1600 }, { 2, 4 } }, // 0.45 is an exact tie between 2/4 and 2/5 in double, not in float
{ { 420, 1440 }, { 1, 3 } },
{ { 255, 2160 }, { 1, 8 } },
@@ -231,7 +232,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) {
}
MAKE_TEST(test_image_preprocessor_lfm2_torch_resize) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.patch_size = 16;
hparams.n_merge = 2;
hparams.set_limit_image_tokens(64, 256);
@@ -243,7 +244,7 @@ MAKE_TEST(test_image_preprocessor_lfm2_torch_resize) {
hparams.image_pad_rf = PAD_NONE;
// { resize algo, image size, expected hash of the tiles and the thumbnail from Lfm2VlImageProcessor (torchvision 0.29, native uint8 resize: arm64, or x86 with AVX2/AVX512) }
- const std::vector<std::tuple<resize_algo, clip_image_size, uint32_t>> cases = {
+ const std::vector<std::tuple<resize_algo, mtmd_image_size, uint32_t>> cases = {
{ RESIZE_ALGO_BILINEAR, { 620, 410 }, 0xf5e9c941 }, // 608x416
{ RESIZE_ALGO_BILINEAR, { 2000, 1500 }, 0x3331f56c }, // 3x2 tiles, 576x416 thumbnail
{ RESIZE_ALGO_BICUBIC, { 620, 410 }, 0x2b34eac9 },
@@ -338,14 +339,14 @@ MAKE_TEST(test_temporal_merge_grouping) {
//
MAKE_TEST(test_video_config) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.patch_size = 16;
hparams.n_merge = 2;
const mtmd_image_preprocessor_qwen3vl preproc(hparams);
// { template, n_frames and fps of the source, expected fps, n_frames and frame size }
// the 1920x1080 sizes are from Qwen3VLVideoProcessor smart_resize
- const std::vector<std::tuple<mtmd_video_template, int32_t, float, float, int32_t, clip_image_size>> cases = {
+ const std::vector<std::tuple<mtmd_video_template, int32_t, float, float, int32_t, mtmd_image_size>> cases = {
{ MTMD_VIDEO_TEMPLATE_QWEN3VL, 36000, 30.0f, 2.0f, 768, { 224, 128 } },
{ MTMD_VIDEO_TEMPLATE_LFM3, 36000, 30.0f, 2.0f, 768, { 224, 128 } },
{ MTMD_VIDEO_TEMPLATE_LFM3_OMNI, 36000, 30.0f, 2.0f, 384, { 320, 192 } },
@@ -405,13 +406,13 @@ static std::string video_sequence_to_str(testing & t, mtmd_helper_video_seq & vi
}
MAKE_TEST(test_video_sequence_qwen3vl) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.patch_size = 16;
hparams.n_merge = 2;
const mtmd_image_preprocessor_qwen3vl preproc(hparams);
// { source video, expected frame size and sequence from Qwen3VLProcessor (transformers 4.57.1) }
- const std::vector<std::tuple<mtmd_helper_video_src, clip_image_size, std::string>> cases = {
+ const std::vector<std::tuple<mtmd_helper_video_src, mtmd_image_size, std::string>> cases = {
{ { 720, 358, 24.0f, 240 }, { 704, 352 },
"<0.3 seconds>[0][13]<1.3 seconds>[25][38]<2.4 seconds>[50][63]<3.4 seconds>[75][88]<4.5 seconds>[101][113]"
"<5.5 seconds>[126][138]<6.6 seconds>[151][164]<7.6 seconds>[176][189]<8.6 seconds>[201][214]<9.7 seconds>[226][239]" },
@@ -439,7 +440,7 @@ MAKE_TEST(test_video_sequence_qwen3vl) {
// the soundtrack of each group comes after its frames
MAKE_TEST(test_video_sequence_lfm3_omni) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.patch_size = 16;
hparams.n_merge = 2;
const mtmd_image_preprocessor_lfm3vl preproc(hparams);
@@ -462,7 +463,7 @@ MAKE_TEST(test_video_sequence_lfm3_omni) {
//
MAKE_TEST(test_audio_preprocessor_conformer) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.n_mel_bins = 128;
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 512;
@@ -504,7 +505,7 @@ MAKE_TEST(test_audio_preprocessor_conformer) {
if (!t.assert_true("preprocess", !out.entries.empty() && preproc.compute(out))) {
return;
}
- const clip_image_f32 & mel = out.entries[0];
+ const mtmd_image_f32 & mel = out.entries[0];
t.assert_equal("n_len", 101, mel.nx());
for (const auto & [bin, frame, expected] : cases) {
@@ -515,7 +516,7 @@ MAKE_TEST(test_audio_preprocessor_conformer) {
}
MAKE_TEST(test_audio_preprocessor_threads) {
- clip_hparams hparams;
+ mtmd_hparams hparams;
hparams.n_mel_bins = 128;
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 512;
@@ -539,7 +540,7 @@ MAKE_TEST(test_audio_preprocessor_threads) {
if (!t.assert_true("preprocess", ref.entries.size() == 1 && ref.needs_compute())) {
return;
}
- const clip_image_size shape = ref.entries[0].get_size();
+ const mtmd_image_size shape = ref.entries[0].get_size();
if (!t.assert_true("compute", preproc.compute(ref))) {
return;
}
@@ -586,7 +587,7 @@ MAKE_TEST(test_audio_log_mel_precision) {
}
std::vector<float> out((size_t) n_frames * n_mel);
- if (!t.assert_true("log mel", clip_audio_log_mel(config, samples.data(), cache.hann_window.data(), n_frames, out.data()))) {
+ if (!t.assert_true("log mel", mtmd_audio_log_mel(config, samples.data(), cache.hann_window.data(), n_frames, out.data()))) {
return;
}
@@ -650,8 +651,8 @@ MAKE_TEST(test_audio_log_mel_blocks) {
std::vector<float> all((size_t) n_frames * n_mel);
std::vector<float> sub((size_t) n_sub * n_mel);
const float * window = cache.hann_window.data();
- if (!t.assert_true("all", clip_audio_log_mel(config, samples.data(), window, n_frames, all.data())) ||
- !t.assert_true("sub", clip_audio_log_mel(config, samples.data() + (size_t) off * hop, window, n_sub, sub.data()))) {
+ if (!t.assert_true("all", mtmd_audio_log_mel(config, samples.data(), window, n_frames, all.data())) ||
+ !t.assert_true("sub", mtmd_audio_log_mel(config, samples.data() + (size_t) off * hop, window, n_sub, sub.data()))) {
return;
}
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index fd8b24efd..c19815d67 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -22,11 +22,13 @@ set(MTMD_CORE_SOURCES
mtmd-helper-gen.cpp
mtmd-helper-common.h
mtmd-helper.h
- clip.cpp
- clip.h
- clip-impl.h
- clip-model.h
- clip-graph.h
+ mtmd-model.cpp
+ mtmd-subcontext.cpp
+ mtmd-graph.cpp
+ mtmd-subcontext.h
+ mtmd-impl.h
+ mtmd-model.h
+ mtmd-graph.h
)
add_library(mtmd
diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md
index 5dad86100..4ca410c67 100644
--- a/tools/mtmd/README-dev.md
+++ b/tools/mtmd/README-dev.md
@@ -7,6 +7,7 @@ Please refer to [multimodal.md](../../docs/multimodal.md) for a broader context.
In short:
- `libmtmd` started as a wrapper around `libllava` / `clip.cpp`
- Various components that used to be in `clip.cpp` are moved progressively to mtmd. For example, preprocessor is now part of mtmd
+- `clip.cpp` has since been split into `mtmd-model` (hparams and weights, one `mtmd_model` per modality), `mtmd-subcontext` (backends and compute, one `mtmd_subcontext` per modality) and `mtmd-graph` (graph builders, see `models/`). Only the GGUF format keeps the `clip.` name (metadata keys and the `clip` architecture), for compatibility with existing mmproj files
## Terminologies
@@ -15,6 +16,26 @@ In short:
- tiles / slices: for llava-uhd-style models, the preprocessor breaks a large input into smaller square images called tiles or slices
- chunk: a mtmd_input_chunk represents a preprocessed input that can then be passed through `mtmd_encode()`
+## Structure
+
+Overview of the `libmtmd` structure:
+
+- `mtmd_context` (`mtmd.cpp`): the public handle
+ - `mtmd_models`: one mmproj file, loaded by `mtmd_model_load()`
+ - `mtmd_model` `v` / `a` / `gen_a` (`mtmd-model.h`): one per modality, holds hparams, weights (`ctx_data` + `buf`) and requirements
+ - `mtmd_subcontext` `ctx_v` / `ctx_a` / `ctx_gen_a` (`mtmd-subcontext.h`): one per model, holds a const ref to it
+ - backends + `sched`: GPU/CPU backends, compute buffers, resolved flash attn
+ - `mtmd_graph` (`mtmd-graph.h`, `models/`): built for each `encode()` call, temporary
+ - `mtmd_image_preprocessor` / `mtmd_audio_preprocessor` (`mtmd-image.h`, `mtmd-audio.h`): read the hparams of their model
+- `mtmd_gen_state` / `mtmd_audio_stream`: public, one per sequence / stream
+ - `mtmd_subctx_gen_state` / `mtmd_subctx_enc_state`: allocated on the backend of a subcontext, see `gen_state_init()` / `enc_state_init()`
+
+Rules of thumb:
+- `mtmd_model` is read-only once loaded. Anything that depends on the backend (scheduler, compute buffers, flash attn resolution, warmup) belongs to `mtmd_subcontext`
+- A model declares its runtime needs (e.g. `max_nodes`, `flash_attn_auto`) in `load_hparams`, the subcontext applies them
+- Models must outlive their subcontexts, `mtmd_context` declares `models` first for that reason
+- Backends are owned by each subcontext for now; they will move to `mtmd_context` so that subcontexts can share them
+
## Pipeline
A typical pipeline of the core libmtmd is as follows:
@@ -67,21 +88,21 @@ Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system i
- Backbone model should be converted to a normal text model (loadable via `libllama`)
- If model used hard-coded embedding row ID, append them to token embeddings and assign token name for them (see `qwen3tts.py`)
- If model have a specific output logits head for audio codes (usually semantic code), keep the head as-is and pad the logits at inference time (see `src/models/qwen3vl.cpp`)
- - Sidecar models (code2wav, bigvgan, etc) must live inside the mmproj GGUF (but can be in different `clip_context` if necessary)
+ - Sidecar models (code2wav, bigvgan, etc) must live inside the mmproj GGUF (but can be in different `mtmd_subcontext` if necessary)
- Note: it should use `ggml_build_forward_select` to select graphs if multiple graphs living in the same context
- - Reuse existing GGUF metadata key name and tensor name whenever possible; think twice before adding extensive changes to GGUF writer. For example, Qwen3-TTS hard-code part of the hparams to `clip.cpp` as they won't likely to change.
+ - Reuse existing GGUF metadata key name and tensor name whenever possible; think twice before adding extensive changes to GGUF writer. For example, Qwen3-TTS hard-code part of the hparams in `mtmd-model.cpp` as they won't likely to change.
- For tensor naming:
- Prefixed with `a.*` for tensors used by speaker encoder pipeline
- Prefixed with `a.gen.*` for generation stages (code / mel-spectrogram / PCM generation)
- For GGUF metadata:
- Reuse as many existing keys as possible
- - In most cases, you can hard-code model configs in the model graph class, or in `clip_hparams`
+ - In most cases, you can hard-code model configs in the model graph class, or in `mtmd_hparams`
- If some values need to be exposed to the `mtmd_helper` layer, hard-code them in `mtmd_helper` and distinguish by pipeline and `mtmd_gen_audio_info::model_variant` if necessary
- Do NOT add new GGUF metadata or new fields to `mtmd_gen_audio_info` unless you can prove that you absolutely need them
4. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this:
- 10-20% changes is to add new backbone (text) model and conversion
- 60% changes inside `mtmd-helper-gen.cpp`
- - 10% changes inside `libmtmd` and `clip.cpp` systems
+ - 10% changes inside `libmtmd` core (`mtmd-model`, `mtmd-subcontext`, `models/`)
- The rest downstream code (CLI, server) should have no changes at all
5. Update usage documentation in `tools/tts/README.md`
diff --git a/tools/mtmd/README.md b/tools/mtmd/README.md
index 701941947..f0290aae5 100644
--- a/tools/mtmd/README.md
+++ b/tools/mtmd/README.md
@@ -32,7 +32,7 @@ Consequently, running a multimodal model typically requires two GGUF files:
As outlined in the history, `libmtmd` is the modern library designed to replace the original `llava.cpp` implementation for handling multimodal inputs.
-Built upon `clip.cpp` (similar to `llava.cpp`), `libmtmd` offers several advantages:
+Originally built upon `clip.cpp` (similar to `llava.cpp`), `libmtmd` offers several advantages:
- **Unified Interface:** Aims to consolidate interaction for various multimodal models.
- **Improved UX/DX:** Features a more intuitive API, inspired by the `Processor` class in the Hugging Face `transformers` library.
- **Flexibility:** Designed to support multiple input types (text, audio, images) while respecting the wide variety of chat templates used by different models.
diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h
deleted file mode 100644
index f8fd41ef5..000000000
--- a/tools/mtmd/clip.h
+++ /dev/null
@@ -1,200 +0,0 @@
-#pragma once
-
-#include "ggml.h"
-#include "mtmd.h"
-
-#include <stddef.h>
-#include <stdint.h>
-
-#include <map>
-
-// !!! Internal header, to be used by mtmd only !!!
-
-#define MTMD_INTERNAL_HEADER
-
-struct clip_ctx;
-struct clip_enc_state;
-struct clip_gen_state;
-
-struct clip_image_size {
- int width;
- int height;
- bool operator==(const clip_image_size & other) const {
- return width == other.width && height == other.height;
- }
- bool operator!=(const clip_image_size & other) const {
- return !(*this == other);
- }
- int area() const {
- // avoid overflow when computing area
- GGML_ASSERT(width >= 0 && width <= 46000);
- GGML_ASSERT(height >= 0 && height <= 46000);
- return width * height;
- }
-};
-
-struct clip_image_f32;
-struct clip_image_f32_batch;
-
-enum clip_modality {
- CLIP_MODALITY_VISION,
- CLIP_MODALITY_AUDIO,
- CLIP_MODALITY_GEN_AUDIO,
-};
-
-enum clip_flash_attn_type {
- CLIP_FLASH_ATTN_TYPE_AUTO = -1,
- CLIP_FLASH_ATTN_TYPE_DISABLED = 0,
- CLIP_FLASH_ATTN_TYPE_ENABLED = 1,
-};
-
-struct clip_context_params {
- bool use_gpu;
- ggml_backend_dev_t device;
- enum clip_flash_attn_type flash_attn_type;
- int image_min_tokens;
- int image_max_tokens;
- bool warmup;
- ggml_backend_sched_eval_callback cb_eval;
- void * cb_eval_user_data;
- bool no_alloc;
- mtmd_progress_callback progress_callback;
- void * progress_callback_user_data;
- int n_threads;
-};
-
-struct clip_init_result {
- struct clip_ctx * ctx_v; // vision context
- struct clip_ctx * ctx_a; // audio context
- struct clip_ctx * ctx_gen_a; // audio generation context
-};
-
-struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params);
-
-// max number of output tokens per image, -1 if not dynamic size
-int clip_get_image_max_tokens(const struct clip_ctx * ctx);
-
-void clip_free(struct clip_ctx * ctx);
-
-// TODO: should be enum, not string
-const char * clip_patch_merge_type(const struct clip_ctx * ctx);
-
-int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img);
-
-// for M-RoPE, this will be the number of token positions in X and Y directions
-// for other models, X will be the total number of tokens and Y will be 1
-int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img);
-int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img);
-
-// this should be equal to the embedding dimension of the text model
-int clip_n_mmproj_embd(const struct clip_ctx * ctx);
-
-ggml_backend_sched_t clip_get_sched(const struct clip_ctx * ctx);
-
-// compute the log-mel of n_frames frames, the waveform must already be padded and window must hold one whole frame
-// out has the layout [n_frames, n_mel]; like the encoders, must not run at the same time as another pass on config.sched
-struct mtmd_audio_mel_config;
-bool clip_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out);
-
-// TODO: remove clip_image_encode() and always use batched version
-bool clip_image_encode (struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector<float> & out_vec);
-bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, std::vector<float> & out_batch_embd);
-
-enum clip_gen_process_type {
- CLIP_GEN_PROCESS_GEN_UNKNOWN,
- CLIP_GEN_PROCESS_GEN_CODE, // h_state to codes
- CLIP_GEN_PROCESS_GEN_WAV, // codes to raw PCM audio
- CLIP_GEN_PROCESS_TTS, // full utterance of codes to raw PCM audio
- CLIP_GEN_PROCESS_TTS_VOCODE, // internal: mel + source stft to istft input
- CLIP_GEN_PROCESS_TOKENIZE, // raw PCM audio to semantic speech tokens
-};
-
-// gen-side voice reference (chatterbox): speech tokens, mel-rate features and the speaker vector
-struct clip_gen_ref {
- std::vector<int32_t> tokens;
- std::vector<float> feat;
- std::vector<float> spk;
-};
-
-struct clip_encode_params {
- int n_threads = 1;
- const clip_image_f32_batch * imgs = nullptr;
- std::vector<float> * out_embd = nullptr;
-
- // for audio gen, imgs has exactly one entry: hidden state from backbone (GEN_CODE) or unused (GEN_WAV)
- clip_gen_process_type gen_process = CLIP_GEN_PROCESS_GEN_UNKNOWN;
-
- // GEN_CODE: out_embd receives the embd to feed back to the backbone
- int32_t code0 = 0; // semantic code sampled by the backbone
- int32_t top_k = 50;
- float top_p = 1.0f;
- std::vector<int32_t> * out_codes = nullptr; // this frame's 16 sampled codes
- std::vector<float> * out_feats = nullptr; // continuous counterpart of out_codes
- float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders
- bool * out_is_eos = nullptr;
-
- // TOKENIZE: speech embedding rows of the produced codes
- std::vector<float> * out_code_embd = nullptr;
-
- // GEN_WAV
- const std::vector<int32_t> * codes = nullptr; // this frame's 16 RVQ codes
- const std::vector<float> * feats = nullptr; // continuous counterpart of codes
- std::vector<float> * out_audio = nullptr; // decoded PCM samples, F32
- const clip_gen_ref * ref = nullptr; // TTS reference conditioning of the voice
- // TTS_VOCODE internal stage inputs
- const std::vector<float> * mel_in = nullptr;
- const std::vector<float> * sstft_in = nullptr;
- int vocode_n_mel = 0;
- int vocode_n_stft = 0;
- std::vector<float> * out_spec = nullptr;
-
- // audio gen: state of one sequence, clip_encode() updates it in place
- struct clip_gen_state * gen_state = nullptr;
-
- // streaming audio input, updated in place by clip_encode()
- struct clip_enc_state * enc_state = nullptr;
-};
-bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params);
-
-// persistent state of a causal audio encoder, for streaming input (one per stream)
-// it holds the sliding window of past keys/values, so each clip_encode() call only takes the new frames
-// n_frames_max is the maximum number of encoder frames per call
-struct clip_enc_state * clip_enc_state_init(struct clip_ctx * ctx, int32_t n_frames_max);
-void clip_enc_state_free(struct clip_enc_state * state);
-
-// per-sequence state of audio generation: RNG and the slots kept between GEN_WAV calls
-// seed UINT32_MAX means random; reset() clears the slots and keeps the RNG stream
-struct clip_gen_state * clip_gen_state_init(struct clip_ctx * ctx, uint32_t seed);
-void clip_gen_state_free(struct clip_gen_state * state);
-void clip_gen_state_reset(struct clip_gen_state * state);
-
-// read a chatterbox tensor by name as F32, returns the element count (0 if not found), out may be null to query the size
-size_t clip_cbx_read_tensor(struct clip_ctx * ctx, const char * name, float * out, size_t n_max);
-
-// a predefined voice: backbone rows, plus the gen-side reference when the gen stack needs one
-struct clip_gen_voice {
- std::vector<float> rows;
- clip_gen_ref ref;
-};
-
-const std::map<std::string, clip_gen_voice> & clip_gen_get_voices(const struct clip_ctx * ctx);
-
-bool clip_is_llava(const struct clip_ctx * ctx);
-// note for contributor: this clip_is_(model) pattern is deprecated
-// do NOT add new functions like this
-
-bool clip_has_vision_encoder(const struct clip_ctx * ctx);
-bool clip_has_audio_encoder(const struct clip_ctx * ctx);
-
-bool clip_support_batch(const struct clip_ctx * ctx);
-
-int clip_model_n_temporal_merge(const struct clip_ctx * ctx); // TODO @ngxson : remove, refactor this
-
-std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx * ctx);
-
-struct clip_cap {
- bool has_vision;
- bool has_audio;
- bool has_gen_audio;
-};
-struct clip_cap clip_get_cap(const char * fname);
diff --git a/tools/mtmd/debug/mtmd-debug.cpp b/tools/mtmd/debug/mtmd-debug.cpp
index 2719dae9b..894ad7d6f 100644
--- a/tools/mtmd/debug/mtmd-debug.cpp
+++ b/tools/mtmd/debug/mtmd-debug.cpp
@@ -81,7 +81,7 @@ int main(int argc, char ** argv) {
llama_init = common_init_from_params(params);
{
auto * model = llama_init->model();
- const char * clip_path = params.mmproj.path.c_str();
+ const char * mmproj_path = params.mmproj.path.c_str();
mtmd_context_params mparams = mtmd_context_params_default();
mparams.use_gpu = params.mmproj_use_gpu;
mparams.device = params.mmproj_device;
@@ -96,9 +96,9 @@ int main(int argc, char ** argv) {
mparams.cb_eval_user_data = &cb_data;
mparams.cb_eval = common_debug_cb_eval;
}
- ctx_mtmd.reset(mtmd_init_from_file(clip_path, model, mparams));
+ ctx_mtmd.reset(mtmd_init_from_file(mmproj_path, model, mparams));
if (!ctx_mtmd.get()) {
- LOG_ERR("Failed to load vision model from %s\n", clip_path);
+ LOG_ERR("Failed to load vision model from %s\n", mmproj_path);
exit(1);
}
}
diff --git a/tools/mtmd/models/chatterbox-gen.cpp b/tools/mtmd/models/chatterbox-gen.cpp
index 00a9a8d81..017329bf4 100644
--- a/tools/mtmd/models/chatterbox-gen.cpp
+++ b/tools/mtmd/models/chatterbox-gen.cpp
@@ -3,7 +3,7 @@
// Chatterbox generation graphs: flow encoder, cfm estimator, s3 tokenizer and hift vocoder
// x [C, T]: y = W x + b with torch Linear weights stored as [in, out]
-ggml_tensor * clip_graph_chatterbox_base::cbx_linear(ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) const {
+ggml_tensor * mtmd_graph_chatterbox_base::cbx_linear(ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) const {
ggml_tensor * y = ggml_mul_mat(ctx0, w, x);
if (b) {
y = ggml_add(ctx0, y, b);
@@ -12,7 +12,7 @@ ggml_tensor * clip_graph_chatterbox_base::cbx_linear(ggml_tensor * w, ggml_tenso
}
// conv1d over time with asymmetric zero padding: x [C, T] -> [OC, T_out], kernel [K, IC, OC]
-ggml_tensor * clip_graph_chatterbox_base::cbx_conv1d(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x,
+ggml_tensor * mtmd_graph_chatterbox_base::cbx_conv1d(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x,
int stride, int pad_l, int pad_r) const {
ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, C]
if (pad_l > 0) {
@@ -34,7 +34,7 @@ ggml_tensor * clip_graph_chatterbox_base::cbx_conv1d(ggml_tensor * k, ggml_tenso
}
// x [C, T] -> symmetric-padded dilated conv -> [OC, T]
-ggml_tensor * clip_graph_chatterbox_base::cbx_conv1d_dil(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x,
+ggml_tensor * mtmd_graph_chatterbox_base::cbx_conv1d_dil(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x,
int pad, int dil) const {
ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x));
ggml_tensor * y = ggml_conv_1d(ctx0, k, xt, 1, pad, dil);
@@ -61,7 +61,7 @@ static ggml_tensor * cbx_rel_shift(ggml_context * ctx0, ggml_tensor * bd, int T)
}
// espnet rel-pos self attention block, pre-norm, x [512, T], pos [512, 2T-1]
-ggml_tensor * clip_graph_chatterbox::enc_layer(const clip_chatterbox::enc_layer & l, ggml_tensor * x,
+ggml_tensor * mtmd_graph_chatterbox::enc_layer(const mtmd_chatterbox::enc_layer & l, ggml_tensor * x,
ggml_tensor * pos, int T) {
const int n_head = 8;
const int d_head = 64;
@@ -121,14 +121,14 @@ static ggml_tensor * cbx_mish(ggml_context * ctx0, ggml_tensor * x) {
}
// causal block: conv k3 left-padded, layer norm over channels, mish; x [C, T]
-ggml_tensor * clip_graph_chatterbox::causal_block(const clip_chatterbox::causal_block & b, ggml_tensor * x) {
+ggml_tensor * mtmd_graph_chatterbox::causal_block(const mtmd_chatterbox::causal_block & b, ggml_tensor * x) {
x = cbx_conv1d(b.conv_w, b.conv_b, x, 1, (int) b.conv_w->ne[0] - 1, 0);
x = build_norm(x, b.norm_w, b.norm_b, NORM_TYPE_NORMAL, 1e-5f, -1);
return cbx_mish(ctx0, x);
}
// resnet block with time conditioning; x [C, T], temb [1024]
-ggml_tensor * clip_graph_chatterbox::resnet(const clip_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb) {
+ggml_tensor * mtmd_graph_chatterbox::resnet(const mtmd_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb) {
ggml_tensor * h = causal_block(r.block1, x);
ggml_tensor * tproj = cbx_linear(r.mlp_w, r.mlp_b, cbx_mish(ctx0, temb));
h = ggml_add(ctx0, h, tproj); // broadcast [256, 1] over T
@@ -138,7 +138,7 @@ ggml_tensor * clip_graph_chatterbox::resnet(const clip_chatterbox::resnet & r, g
}
// diffusers-style transformer block, full attention; x [256, T]
-ggml_tensor * clip_graph_chatterbox::tfm_block(const clip_chatterbox::tfm_block & b, ggml_tensor * x) {
+ggml_tensor * mtmd_graph_chatterbox::tfm_block(const mtmd_chatterbox::tfm_block & b, ggml_tensor * x) {
const int n_head = 8;
const int d_head = 64;
const int T = (int) x->ne[1];
@@ -157,7 +157,7 @@ ggml_tensor * clip_graph_chatterbox::tfm_block(const clip_chatterbox::tfm_block
}
// one estimator evaluation; x_noise, mu, cond [80, T], spks [80], temb [1024]
-ggml_tensor * clip_graph_chatterbox::estimator(ggml_tensor * x_noise, ggml_tensor * mu, ggml_tensor * spks,
+ggml_tensor * mtmd_graph_chatterbox::estimator(ggml_tensor * x_noise, ggml_tensor * mu, ggml_tensor * spks,
ggml_tensor * cond, ggml_tensor * temb, int T) {
const auto & c = model.cbx;
@@ -199,7 +199,7 @@ ggml_tensor * clip_graph_chatterbox::estimator(ggml_tensor * x_noise, ggml_tenso
}
// s3 tokenizer encoder: log-mel [T, 128] -> fsq code [8, T / 4], rounded to base 3 tokens on the host
-ggml_cgraph * clip_graph_chatterbox::build_s3tok(int T) {
+ggml_cgraph * mtmd_graph_chatterbox::build_s3tok(int T) {
const auto & c = model.cbx;
const int n_head = 20;
const int d_head = 64;
@@ -261,14 +261,14 @@ ggml_cgraph * clip_graph_chatterbox::build_s3tok(int T) {
return gf;
}
-ggml_cgraph * clip_graph_chatterbox::build() {
- if (gen_process == CLIP_GEN_PROCESS_TTS_VOCODE) {
+ggml_cgraph * mtmd_graph_chatterbox::build() {
+ if (gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE) {
return build_vocoder(vocode_n_mel, vocode_n_stft);
}
- if (gen_process == CLIP_GEN_PROCESS_TOKENIZE) {
+ if (gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
return build_s3tok(img.nx());
}
- if (gen_process != CLIP_GEN_PROCESS_TTS) {
+ if (gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS) {
// load-time buffer sizing path
ggml_tensor * inp = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);
ggml_set_name(inp, "inp_stub");
@@ -355,7 +355,7 @@ ggml_cgraph * clip_graph_chatterbox::build() {
const bool meanflow = c.time_embed_mixer_w != nullptr;
const int n_steps = meanflow ? 2 : 10;
- // span points, same schedule as the host side sinusoid fill in clip.cpp
+ // span points, same schedule as the host side sinusoid fill in mtmd-subcontext.cpp
float span[11];
for (int i = 0; i <= n_steps; i++) {
const float u = (float) i / n_steps;
@@ -454,7 +454,7 @@ static ggml_tensor * cbx_snake(ggml_context * ctx0, ggml_tensor * x, ggml_tensor
}
// hifigan-snake resblock; kernels with dilations 1/3/5 on convs1, 1 on convs2
-ggml_tensor * clip_graph_chatterbox::hift_resblock(const clip_chatterbox::hift_res & r, ggml_tensor * x) {
+ggml_tensor * mtmd_graph_chatterbox::hift_resblock(const mtmd_chatterbox::hift_res & r, ggml_tensor * x) {
static const int dil[3] = {1, 3, 5};
for (size_t j = 0; j < r.units.size(); j++) {
const auto & u = r.units[j];
@@ -471,7 +471,7 @@ ggml_tensor * clip_graph_chatterbox::hift_resblock(const clip_chatterbox::hift_r
}
// mel [80, T] + source stft [18, T_stft] -> conv_post output [18, T_stft2]
-ggml_cgraph * clip_graph_chatterbox::build_vocoder(int n_mel, int n_stft) {
+ggml_cgraph * mtmd_graph_chatterbox::build_vocoder(int n_mel, int n_stft) {
const auto & c = model.cbx;
ggml_tensor * mel = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, n_mel);
diff --git a/tools/mtmd/models/chatterbox-spkenc.cpp b/tools/mtmd/models/chatterbox-spkenc.cpp
index afd9f0c29..fc73aa805 100644
--- a/tools/mtmd/models/chatterbox-spkenc.cpp
+++ b/tools/mtmd/models/chatterbox-spkenc.cpp
@@ -5,7 +5,7 @@
static constexpr float bn_eps = 1e-5f;
// per-channel batchnorm on x [C, T], w/b are null on the affine=False variant
-ggml_tensor * clip_graph_chatterbox_spkenc::bn1d(const clip_chatterbox::bn & n, ggml_tensor * x) {
+ggml_tensor * mtmd_graph_chatterbox_spkenc::bn1d(const mtmd_chatterbox::bn & n, ggml_tensor * x) {
ggml_tensor * sd = ggml_sqrt(ctx0, ggml_scale_bias(ctx0, n.var, 1.0f, bn_eps));
if (!n.w) {
return ggml_div(ctx0, ggml_sub(ctx0, x, n.mean), sd);
@@ -16,7 +16,7 @@ ggml_tensor * clip_graph_chatterbox_spkenc::bn1d(const clip_chatterbox::bn & n,
}
// batchnorm on a conv2d activation [W=T, H=F, C, 1], stats on ne2
-static ggml_tensor * cbx_bn2d(ggml_context * ctx0, const clip_chatterbox::bn & n, ggml_tensor * x) {
+static ggml_tensor * cbx_bn2d(ggml_context * ctx0, const mtmd_chatterbox::bn & n, ggml_tensor * x) {
const int C = (int) x->ne[2];
ggml_tensor * mean = ggml_reshape_4d(ctx0, n.mean, 1, 1, C, 1);
ggml_tensor * var = ggml_reshape_4d(ctx0, n.var, 1, 1, C, 1);
@@ -27,12 +27,12 @@ static ggml_tensor * cbx_bn2d(ggml_context * ctx0, const clip_chatterbox::bn & n
return ggml_add(ctx0, ggml_mul(ctx0, x, a), shift);
}
-ggml_tensor * clip_graph_chatterbox_spkenc::bn2d_relu(const clip_chatterbox::bn & n, ggml_tensor * x) {
+ggml_tensor * mtmd_graph_chatterbox_spkenc::bn2d_relu(const mtmd_chatterbox::bn & n, ggml_tensor * x) {
return ggml_relu(ctx0, cbx_bn2d(ctx0, n, x));
}
// fcm residual 2d block, stride on the frequency axis only
-ggml_tensor * clip_graph_chatterbox_spkenc::res2d(const clip_chatterbox::spk_res2d & r, ggml_tensor * x,
+ggml_tensor * mtmd_graph_chatterbox_spkenc::res2d(const mtmd_chatterbox::spk_res2d & r, ggml_tensor * x,
int stride) {
ggml_tensor * cur = ggml_conv_2d(ctx0, r.conv1_w, x, 1, stride, 1, 1, 1, 1);
cur = bn2d_relu(r.bn1, cur);
@@ -48,7 +48,7 @@ ggml_tensor * clip_graph_chatterbox_spkenc::res2d(const clip_chatterbox::spk_res
}
// cam dense tdnn layer: bottleneck then context-gated conv; x [C_in, T] -> [growth, T]
-ggml_tensor * clip_graph_chatterbox_spkenc::cam_layer(const clip_chatterbox::spk_cam_layer & l, ggml_tensor * x,
+ggml_tensor * mtmd_graph_chatterbox_spkenc::cam_layer(const mtmd_chatterbox::spk_cam_layer & l, ggml_tensor * x,
int dil, ggml_tensor * segfix) {
ggml_tensor * h = ggml_relu(ctx0, bn1d(l.nl1_bn, x));
h = cbx_conv1d(l.linear1_w, nullptr, h, 1, 0, 0);
@@ -86,7 +86,7 @@ ggml_tensor * clip_graph_chatterbox_spkenc::cam_layer(const clip_chatterbox::spk
return ggml_mul(ctx0, y, m);
}
-ggml_cgraph * clip_graph_chatterbox_spkenc::build() {
+ggml_cgraph * mtmd_graph_chatterbox_spkenc::build() {
const auto & c = model.cbx;
const int T = img.nx();
@@ -119,7 +119,7 @@ ggml_cgraph * clip_graph_chatterbox_spkenc::build() {
cb(x, "spk_tdnn", -1);
static const int block_dil[3] = {1, 2, 2};
- const clip_chatterbox::spk_cam_block * blocks[3] = { &c.spk_block1, &c.spk_block2, &c.spk_block3 };
+ const mtmd_chatterbox::spk_cam_block * blocks[3] = { &c.spk_block1, &c.spk_block2, &c.spk_block3 };
for (int bi = 1; bi <= 3; bi++) {
const auto & blk = *blocks[bi - 1];
for (const auto & l : blk.layers) {
diff --git a/tools/mtmd/models/cogvlm.cpp b/tools/mtmd/models/cogvlm.cpp
index 44bc88442..aeaa33e4f 100644
--- a/tools/mtmd/models/cogvlm.cpp
+++ b/tools/mtmd/models/cogvlm.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_cogvlm::build() {
+ggml_cgraph * mtmd_graph_cogvlm::build() {
GGML_ASSERT(model.class_embedding != nullptr);
GGML_ASSERT(model.position_embeddings != nullptr);
diff --git a/tools/mtmd/models/conformer.cpp b/tools/mtmd/models/conformer.cpp
index 9463f9f1c..4cd2bee53 100644
--- a/tools/mtmd/models/conformer.cpp
+++ b/tools/mtmd/models/conformer.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_conformer::build() {
+ggml_cgraph * mtmd_graph_conformer::build() {
const int n_frames = img.nx();
const int n_pos = n_frames / 2;
const int n_pos_embd = (((((n_frames + 1) / 2) + 1) / 2 + 1) / 2) * 2 - 1;
diff --git a/tools/mtmd/models/deepseek4v.cpp b/tools/mtmd/models/deepseek4v.cpp
index 06005d52c..e3c0d95ed 100644
--- a/tools/mtmd/models/deepseek4v.cpp
+++ b/tools/mtmd/models/deepseek4v.cpp
@@ -11,14 +11,14 @@
//
// each aligner row ends with a NEWLINE, an odd row count is padded with a full row of PADs
// pairs of adjacent rows are interleaved column-wise ("N-layout")
-// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in clip.cpp)
+// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in mtmd-subcontext.cpp)
//
// ref: inference/vision.py and inference/image_processor.py in the HF repo
//
// DeepSeek-V4.1-Flash (deepseek41v) shares the encoder and aligner; its block is plain reading order
// [START] ([IMAGE]*n_llm_w [NEWLINE])*n_llm_h [END], no lead pads, no pad row, no IMAGE_PAD tensor
-ggml_cgraph * clip_graph_deepseek4v::build() {
+ggml_cgraph * mtmd_graph_deepseek4v::build() {
const int n_merge = hparams.n_merge;
// 2D input positions
@@ -27,7 +27,7 @@ ggml_cgraph * clip_graph_deepseek4v::build() {
ggml_set_input(positions);
int sections[4] = {d_head/4, d_head/4, 0, 0};
- auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) {
return ggml_rope_multi(ctx0, cur, positions, nullptr,
d_head/2, sections, GGML_ROPE_TYPE_VISION,
0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
@@ -92,8 +92,8 @@ ggml_cgraph * clip_graph_deepseek4v::build() {
}
}
- const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge;
- const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge;
+ const int n_llm_w = MTMD_ALIGN(n_patches_x, n_merge) / n_merge;
+ const int n_llm_h = MTMD_ALIGN(n_patches_y, n_merge) / n_merge;
const int n_out = is_v41 ? dsv41_n_block_tokens(n_llm_w, n_llm_h)
: dsv4_get_block_layout(n_llm_w, n_llm_h, img.lead_pad).n_out;
GGML_ASSERT(n_grid == n_llm_w * n_llm_h);
diff --git a/tools/mtmd/models/deepseekocr.cpp b/tools/mtmd/models/deepseekocr.cpp
index b784cdad6..af69c0332 100644
--- a/tools/mtmd/models/deepseekocr.cpp
+++ b/tools/mtmd/models/deepseekocr.cpp
@@ -105,7 +105,7 @@ static ggml_tensor * conv_2d_f32(ggml_context * ctx0, ggml_tensor * a, ggml_tens
}
-ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
+ggml_tensor * mtmd_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
// Building SAM
const int n_embd = hparams.sam_n_embd;
const int n_layer = hparams.sam_n_layer;
@@ -211,7 +211,7 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
mask = ggml_add(ctx0, rw, rh); // [B*n_heads, H*W, H, W]
mask = ggml_reshape_4d(ctx0, mask, W * H, W * H, n_heads, B);
// casting mask to F16 only required when flash-attn is enabled
- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
+ if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) {
mask = ggml_cast(ctx0, mask, GGML_TYPE_F16);
}
@@ -264,7 +264,7 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
return cur;
}
-ggml_cgraph * clip_graph_deepseekocr::build() {
+ggml_cgraph * mtmd_graph_deepseekocr::build() {
// patch embedding
ggml_tensor * inp_raw = build_inp_raw();
diff --git a/tools/mtmd/models/deepseekocr2.cpp b/tools/mtmd/models/deepseekocr2.cpp
index 7088bb97b..fb20e9d1d 100644
--- a/tools/mtmd/models/deepseekocr2.cpp
+++ b/tools/mtmd/models/deepseekocr2.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_deepseekocr2::build() {
+ggml_cgraph * mtmd_graph_deepseekocr2::build() {
GGML_ASSERT(hparams.n_head_kv > 0);
GGML_ASSERT(n_head % hparams.n_head_kv == 0);
@@ -45,7 +45,7 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {
ggml_tensor * inp_pos = ggml_cast(ctx0, ggml_arange(ctx0, 0, seq_len, 1), GGML_TYPE_I32);
- auto add_rope = [&](ggml_tensor * x, const clip_layer &) {
+ auto add_rope = [&](ggml_tensor * x, const mtmd_layer &) {
return ggml_rope_ext(ctx0, x, inp_pos, nullptr, d_head,
GGML_ROPE_TYPE_NEOX, 131072, 1000000, 1, 0, 1, 0, 0);
};
diff --git a/tools/mtmd/models/dots3note.cpp b/tools/mtmd/models/dots3note.cpp
index 93c14fc79..c942ea7da 100644
--- a/tools/mtmd/models/dots3note.cpp
+++ b/tools/mtmd/models/dots3note.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_dots3note_a::build() {
+ggml_cgraph * mtmd_graph_dots3note_a::build() {
// inp_raw: [n_frames, n_mel, 1], one 60s chunk, mel frames not padded
// the reference impl zero-masks conv inputs beyond the valid length at each stage;
// running on exactly the valid frames with the convs' zero padding is equivalent
@@ -37,7 +37,7 @@ ggml_cgraph * clip_graph_dots3note_a::build() {
ggml_set_input(positions);
// partial rotary: first half of each head, NEOX style
- auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) {
return ggml_rope_ext(ctx0, cur, positions, nullptr, d_head/2,
GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
};
diff --git a/tools/mtmd/models/dotsocr.cpp b/tools/mtmd/models/dotsocr.cpp
index 92974bb67..dcede11c3 100644
--- a/tools/mtmd/models/dotsocr.cpp
+++ b/tools/mtmd/models/dotsocr.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_dotsocr::build() {
+ggml_cgraph * mtmd_graph_dotsocr::build() {
const int n_pos = n_patches;
const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position
@@ -11,7 +11,7 @@ ggml_cgraph * clip_graph_dotsocr::build() {
ggml_set_name(positions, "positions");
ggml_set_input(positions);
- auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) {
return ggml_rope_multi(
ctx0, cur, positions, nullptr,
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
diff --git a/tools/mtmd/models/drama3-gen.cpp b/tools/mtmd/models/drama3-gen.cpp
index de7a4f9e8..e77fcf540 100644
--- a/tools/mtmd/models/drama3-gen.cpp
+++ b/tools/mtmd/models/drama3-gen.cpp
@@ -7,13 +7,13 @@
static constexpr int DRAMA3_N_ROT = 64;
static constexpr float DRAMA3_ROPE_THETA = 1e7f;
-std::vector<c2w_state_slot> list_drama3_state_slots(const clip_hparams & hparams, const clip_model & model) {
+std::vector<c2w_state_slot> list_drama3_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
std::vector<c2w_state_slot> slots;
slots.push_back({"tower_pos", 1, 1, true});
for (size_t il = 0; il < model.layers.size(); il++) {
const int64_t n_kv = model.layers[il].k_w->ne[1];
- slots.push_back({"tower_k_" + std::to_string(il), n_kv, clip_graph_drama3_gen::n_pos_max, true});
- slots.push_back({"tower_v_" + std::to_string(il), n_kv, clip_graph_drama3_gen::n_pos_max, true});
+ slots.push_back({"tower_k_" + std::to_string(il), n_kv, mtmd_graph_drama3_gen::n_pos_max, true});
+ slots.push_back({"tower_v_" + std::to_string(il), n_kv, mtmd_graph_drama3_gen::n_pos_max, true});
}
for (auto & slot : list_c2w_state_slots(hparams, model)) {
slots.push_back(slot);
@@ -22,7 +22,7 @@ std::vector<c2w_state_slot> list_drama3_state_slots(const clip_hparams & hparams
}
// h_out = out_norm(Block(eh_proj([enorm(mask * E[code] + extra); hnorm(h_in)])))
-ggml_tensor * clip_graph_drama3_gen::tower_layer(ggml_tensor * x, int il, ggml_tensor * code, ggml_tensor * mask, ggml_tensor * extra,
+ggml_tensor * mtmd_graph_drama3_gen::tower_layer(ggml_tensor * x, int il, ggml_tensor * code, ggml_tensor * mask, ggml_tensor * extra,
ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos) {
const auto & layer = model.layers[il];
const auto & tl = model.drama3_layers[il];
@@ -80,7 +80,7 @@ ggml_tensor * clip_graph_drama3_gen::tower_layer(ggml_tensor * x, int il, ggml_t
}
// one row: backbone hidden + next main code -> other codes of that frame + next backbone input
-ggml_cgraph * clip_graph_drama3_gen::build_tower() {
+ggml_cgraph * mtmd_graph_drama3_gen::build_tower() {
const int n_layer = (int) model.layers.size();
const int n_heads = (int) model.gen_code_head_w->ne[2]; // one per residual codebook
const int n_shared = n_layer - n_heads;
@@ -141,7 +141,7 @@ ggml_cgraph * clip_graph_drama3_gen::build_tower() {
ggml_tensor * extra = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_get_rows(ctx0, cb_all, inp_extra)));
extra = ggml_mul(ctx0, ggml_reshape_2d(ctx0, ggml_sum_rows(ctx0, extra), n_embd_cb, 1), extra_on);
- clip_graph_qwen3tts_gen::code_gen cg(*this, top_k, top_p);
+ mtmd_graph_qwen3tts_gen::code_gen cg(*this, top_k, top_p);
ggml_tensor * out_codes = cg.cache_set(ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, 1, n_cb), 0, inp_code0);
ggml_tensor * code = inp_code0;
@@ -183,7 +183,7 @@ ggml_cgraph * clip_graph_drama3_gen::build_tower() {
}
// frames -> PCM, qwen3tts code2wav blocks without pre_conv and tfm projections
-ggml_cgraph * clip_graph_drama3_gen::build_codec() {
+ggml_cgraph * mtmd_graph_drama3_gen::build_codec() {
const auto & c2w = model.c2w;
const int64_t n_cb = model.gen_code_embd_w->ne[2];
@@ -191,7 +191,7 @@ ggml_cgraph * clip_graph_drama3_gen::build_codec() {
ggml_set_name(inp_codes, "inp_codes");
ggml_set_input(inp_codes);
- clip_graph_qwen3tts_gen::code2wav dec(*this);
+ mtmd_graph_qwen3tts_gen::code2wav dec(*this);
dec.gelu_erf = true;
for (const auto & slot : list_c2w_state_slots(hparams, model)) {
ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
@@ -251,6 +251,6 @@ ggml_cgraph * clip_graph_drama3_gen::build_codec() {
return gf;
}
-ggml_cgraph * clip_graph_drama3_gen::build() {
- return gen_process == CLIP_GEN_PROCESS_GEN_WAV ? build_codec() : build_tower();
+ggml_cgraph * mtmd_graph_drama3_gen::build() {
+ return gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ? build_codec() : build_tower();
}
diff --git a/tools/mtmd/models/exaone4_5.cpp b/tools/mtmd/models/exaone4_5.cpp
index d3372a1cd..768620e59 100644
--- a/tools/mtmd/models/exaone4_5.cpp
+++ b/tools/mtmd/models/exaone4_5.cpp
@@ -1,7 +1,7 @@
// similar to qwen2vl, except for GQA attention
#include "models.h"
-ggml_cgraph * clip_graph_exaone4_5::build() {
+ggml_cgraph * mtmd_graph_exaone4_5::build() {
GGML_ASSERT(model.patch_bias == nullptr);
GGML_ASSERT(model.class_embedding == nullptr);
diff --git a/tools/mtmd/models/gemma4a.cpp b/tools/mtmd/models/gemma4a.cpp
index f98a8b6fc..f1ac867ca 100644
--- a/tools/mtmd/models/gemma4a.cpp
+++ b/tools/mtmd/models/gemma4a.cpp
@@ -1,5 +1,5 @@
/**
- * Gemma 4 Audio Conformer Encoder (clip_graph_gemma4a)
+ * Gemma 4 Audio Conformer Encoder (mtmd_graph_gemma4a)
*
* Architecture: Conformer with dual half-step FFN, full self-attention
* with sinusoidal RPE, depthwise light conv, and output projection.
@@ -8,7 +8,7 @@
#include "models.h"
#include <cmath>
-ggml_cgraph * clip_graph_gemma4a::build() {
+ggml_cgraph * mtmd_graph_gemma4a::build() {
const float res_weight = 0.5f;
const float norm_eps = 1e-6f;
@@ -273,7 +273,7 @@ ggml_cgraph * clip_graph_gemma4a::build() {
return gf;
}
-ggml_tensor * clip_graph_gemma4a::build_mm(ggml_tensor * w, ggml_tensor * x) const {
+ggml_tensor * mtmd_graph_gemma4a::build_mm(ggml_tensor * w, ggml_tensor * x) const {
auto it = model.clamp_info_map.find(w->name);
if (it == model.clamp_info_map.end()) {
return ggml_mul_mat(ctx0, w, x);
diff --git a/tools/mtmd/models/gemma4ua.cpp b/tools/mtmd/models/gemma4ua.cpp
index e24bef2ed..1d77b8889 100644
--- a/tools/mtmd/models/gemma4ua.cpp
+++ b/tools/mtmd/models/gemma4ua.cpp
@@ -1,7 +1,7 @@
#include "models.h"
#include <cmath>
-ggml_cgraph * clip_graph_gemma4ua::build() {
+ggml_cgraph * mtmd_graph_gemma4ua::build() {
ggml_tensor * inp = build_inp_raw(1);
auto cur = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
diff --git a/tools/mtmd/models/gemma4uv.cpp b/tools/mtmd/models/gemma4uv.cpp
index 96031141b..04e139a1b 100644
--- a/tools/mtmd/models/gemma4uv.cpp
+++ b/tools/mtmd/models/gemma4uv.cpp
@@ -1,7 +1,7 @@
#include "models.h"
#include <cmath>
-ggml_cgraph * clip_graph_gemma4uv::build() {
+ggml_cgraph * mtmd_graph_gemma4uv::build() {
ggml_tensor * inp_raw = build_inp_raw();
// Gemma4UnifiedVisionEmbedder uses default pytorch LayerNorm, not RMSNorm
diff --git a/tools/mtmd/models/gemma4v.cpp b/tools/mtmd/models/gemma4v.cpp
index 448438947..46733e0f3 100644
--- a/tools/mtmd/models/gemma4v.cpp
+++ b/tools/mtmd/models/gemma4v.cpp
@@ -1,7 +1,7 @@
#include "models.h"
#include <cmath>
-ggml_cgraph * clip_graph_gemma4v::build() {
+ggml_cgraph * mtmd_graph_gemma4v::build() {
ggml_tensor * inp_raw = build_inp_raw();
// patches = 2 * (patches - 0.5)
@@ -43,7 +43,7 @@ ggml_cgraph * clip_graph_gemma4v::build() {
}
// similar to build_rope_2d, but use neox ordering
- auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) {
const int64_t n_dim = cur->ne[0];
// first half, dims [0, n_dim/2)
@@ -117,7 +117,7 @@ ggml_cgraph * clip_graph_gemma4v::build() {
return gf;
}
-ggml_tensor * clip_graph_gemma4v::build_mm(ggml_tensor * w, ggml_tensor * x) const {
+ggml_tensor * mtmd_graph_gemma4v::build_mm(ggml_tensor * w, ggml_tensor * x) const {
// Gemma4ClippableLinear
auto it = model.clamp_info_map.find(w->name);
diff --git a/tools/mtmd/models/glm4v.cpp b/tools/mtmd/models/glm4v.cpp
index 1cbcea942..2b9a30192 100644
--- a/tools/mtmd/models/glm4v.cpp
+++ b/tools/mtmd/models/glm4v.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_glm4v::build() {
+ggml_cgraph * mtmd_graph_glm4v::build() {
GGML_ASSERT(model.patch_bias != nullptr);
GGML_ASSERT(model.class_embedding == nullptr);
@@ -63,7 +63,7 @@ ggml_cgraph * clip_graph_glm4v::build() {
cb(learned_pos_embd, "learned_pos_embd", -1);
}
- auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) {
return ggml_rope_multi(
ctx0, cur, positions, nullptr,
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
diff --git a/tools/mtmd/models/granite-speech.cpp b/tools/mtmd/models/granite-speech.cpp
index 9725def82..64fb76c41 100644
--- a/tools/mtmd/models/granite-speech.cpp
+++ b/tools/mtmd/models/granite-speech.cpp
@@ -2,7 +2,7 @@
#include <algorithm>
-ggml_cgraph * clip_graph_granite_speech::build() {
+ggml_cgraph * mtmd_graph_granite_speech::build() {
const int n_frames = img.nx();
const int context_size = hparams.audio_chunk_size;
const int ctc_layer = n_layer / 2;
diff --git a/tools/mtmd/models/granite4-vision.cpp b/tools/mtmd/models/granite4-vision.cpp
index a75f1cee9..390d6edf6 100644
--- a/tools/mtmd/models/granite4-vision.cpp
+++ b/tools/mtmd/models/granite4-vision.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-#include "../clip-impl.h"
-#include "../clip-model.h"
+#include "../mtmd-impl.h"
+#include "../mtmd-model.h"
#include <algorithm>
#include <cmath>
@@ -9,7 +9,7 @@
#include <vector>
/*
- * Granite Vision 4.1 clip graph
+ * Granite Vision 4.1 vision graph
*
* Stage 1a: SigLIP vision tower (N layers, post-norm)
* Stage 1b: WindowQFormer blocks (deepstack + spatial)
@@ -22,7 +22,7 @@
// ---------------------------------------------------------------------------
// split the stacked tiles into the batch axis, then run the usual patch embedding
-ggml_tensor * clip_graph_granite4_vision::build_tile_inp() {
+ggml_tensor * mtmd_graph_granite4_vision::build_tile_inp() {
ggml_tensor * inp_raw = build_inp_raw();
if (n_tiles > 1) {
@@ -41,7 +41,7 @@ ggml_tensor * clip_graph_granite4_vision::build_tile_inp() {
return inp;
}
-ggml_tensor * clip_graph_granite4_vision::gather(
+ggml_tensor * mtmd_graph_granite4_vision::gather(
ggml_tensor * src,
const std::string & name,
int idx_len) {
@@ -52,7 +52,7 @@ ggml_tensor * clip_graph_granite4_vision::gather(
return ggml_get_rows(ctx0, src, idx);
}
-ggml_tensor * clip_graph_granite4_vision::interp_down(
+ggml_tensor * mtmd_graph_granite4_vision::interp_down(
ggml_tensor * src,
int side,
int new_side) {
@@ -72,7 +72,7 @@ ggml_tensor * clip_graph_granite4_vision::interp_down(
// build_block - WindowQFormer block implementation
// ---------------------------------------------------------------------------
-ggml_tensor * clip_graph_granite4_vision::build_block(
+ggml_tensor * mtmd_graph_granite4_vision::build_block(
const qf_block & blk,
ggml_tensor * h,
int bid,
@@ -244,7 +244,7 @@ ggml_tensor * clip_graph_granite4_vision::build_block(
// Build the K-tiled, base-scaled newline row tensor.
// Shape: (n_mmproj_embd, 1)
-ggml_tensor * clip_graph_granite4_vision::build_newline_row(ggml_context * ctx0) {
+ggml_tensor * mtmd_graph_granite4_vision::build_newline_row(ggml_context * ctx0) {
const int K = (int) model.qf_proj_blocks.size();
GGML_ASSERT(K > 0);
GGML_ASSERT(n_mmproj_embd % K == 0);
@@ -272,7 +272,7 @@ ggml_tensor * clip_graph_granite4_vision::build_newline_row(ggml_context * ctx0)
// Assemble [overview, tile(0,0), tile(0,1), ...] into one token sequence:
// the overview tokens first, then the tile grid read in raster order with one newline per row.
// ref: https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/llava_next/modeling_llava_next.py#L266
-ggml_tensor * clip_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cur, int out_side) {
+ggml_tensor * mtmd_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cur, int out_side) {
const int n_dim = cur->ne[0];
const int grid_x = anyres.grid_x;
const int grid_y = anyres.grid_y;
@@ -293,7 +293,7 @@ ggml_tensor * clip_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cu
// drop the tokens that only cover the padding added when resizing to the grid
int off_x, off_y, w, h;
- clip_anyres_unpad(cur_w, cur_h, anyres.orig_nx, anyres.orig_ny, off_x, off_y, w, h);
+ mtmd_anyres_unpad(cur_w, cur_h, anyres.orig_nx, anyres.orig_ny, off_x, off_y, w, h);
if (w != cur_w || h != cur_h) {
tiles = ggml_cont(ctx0, ggml_view_3d(ctx0, tiles, n_dim, w, h,
tiles->nb[1], tiles->nb[2],
@@ -307,7 +307,7 @@ ggml_tensor * clip_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cu
return ggml_concat(ctx0, base, tiles, 1);
}
-ggml_cgraph * clip_graph_granite4_vision::build() {
+ggml_cgraph * mtmd_graph_granite4_vision::build() {
GGML_ASSERT(model.patch_embeddings_0 != nullptr);
GGML_ASSERT(model.position_embeddings != nullptr);
GGML_ASSERT(model.class_embedding == nullptr);
diff --git a/tools/mtmd/models/hunyuanvl.cpp b/tools/mtmd/models/hunyuanvl.cpp
index 2c670979d..b6b3fef58 100644
--- a/tools/mtmd/models/hunyuanvl.cpp
+++ b/tools/mtmd/models/hunyuanvl.cpp
@@ -1,12 +1,12 @@
#include "models.h"
-ggml_cgraph * clip_graph_hunyuanvl::build() {
+ggml_cgraph * mtmd_graph_hunyuanvl::build() {
const int merge = hparams.n_merge;
const int pw = n_patches_x;
const int ph = n_patches_y;
// position embedding: declared as a graph input, filled on CPU
- // by clip_image_batch_encode (see PROJECTOR_TYPE_HUNYUANVL branch there).
+ // by mtmd_subcontext::encode() (see PROJECTOR_TYPE_HUNYUANVL branch there).
ggml_tensor * pos_embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, ph * pw);
ggml_set_name(pos_embd, "hunyuanvl_pos_embd");
ggml_set_input(pos_embd);
diff --git a/tools/mtmd/models/internvl.cpp b/tools/mtmd/models/internvl.cpp
index af8f12cdc..5e0b679c4 100644
--- a/tools/mtmd/models/internvl.cpp
+++ b/tools/mtmd/models/internvl.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_internvl::build() {
+ggml_cgraph * mtmd_graph_internvl::build() {
GGML_ASSERT(model.class_embedding != nullptr);
GGML_ASSERT(model.position_embeddings != nullptr);
diff --git a/tools/mtmd/models/kimik25.cpp b/tools/mtmd/models/kimik25.cpp
index cb345f0fc..13baf86d0 100644
--- a/tools/mtmd/models/kimik25.cpp
+++ b/tools/mtmd/models/kimik25.cpp
@@ -2,10 +2,10 @@
#include <cstring>
#include <cmath>
-// note: this is similar to clip_graph::resize_position_embeddings, major difference is having
+// note: this is similar to mtmd_graph::resize_position_embeddings, major difference is having
// the w/h in ne[1] and ne[2] instead of assuming with sqrt. Could try storing the tensor in 2D instead
// with a w*h? Also the permute is a bit different at (2, 1, 0, 3) instead of (2, 0, 1, 3).
-ggml_tensor * clip_graph_kimik25::resize_position_embeddings_3d(uint32_t interpolation_mode) {
+ggml_tensor * mtmd_graph_kimik25::resize_position_embeddings_3d(uint32_t interpolation_mode) {
ggml_tensor * pos_embd = model.position_embeddings;
const int height = img.ny() / patch_size;
const int width = img.nx() / patch_size;
@@ -31,7 +31,7 @@ ggml_tensor * clip_graph_kimik25::resize_position_embeddings_3d(uint32_t interpo
return pos_embd;
}
-ggml_cgraph * clip_graph_kimik25::build() {
+ggml_cgraph * mtmd_graph_kimik25::build() {
ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
ggml_set_name(pos_h, "pos_h");
ggml_set_input(pos_h);
@@ -44,7 +44,7 @@ ggml_cgraph * clip_graph_kimik25::build() {
// Kimi-K2.5 uses interleaved 2D RoPE pattern natively, but
// Q / K are permuted during conversion to use split format.
- auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) {
cur = build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false);
return cur;
};
diff --git a/tools/mtmd/models/kimivl.cpp b/tools/mtmd/models/kimivl.cpp
index 0a06f5090..dd8146708 100644
--- a/tools/mtmd/models/kimivl.cpp
+++ b/tools/mtmd/models/kimivl.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_kimivl::build() {
+ggml_cgraph * mtmd_graph_kimivl::build() {
// 2D input positions
ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
ggml_set_name(pos_h, "pos_h");
@@ -13,7 +13,7 @@ ggml_cgraph * clip_graph_kimivl::build() {
ggml_tensor * learned_pos_embd = resize_position_embeddings();
// build ViT with 2D position embeddings
- auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) {
// first half is X axis and second half is Y axis
return build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false);
};
diff --git a/tools/mtmd/models/lfm3a-gen.cpp b/tools/mtmd/models/lfm3a-gen.cpp
index 998605711..f5aedac6f 100644
--- a/tools/mtmd/models/lfm3a-gen.cpp
+++ b/tools/mtmd/models/lfm3a-gen.cpp
@@ -9,7 +9,7 @@
// GEN_WAV : a window of frames -> polar STFT frames of the liquid-codec decoder, the ISTFT runs on the host
// state carried between GEN_WAV calls: rope offset, short-conv taps and attention window of the codec decoder
-std::vector<c2w_state_slot> list_lfm3a_gen_state_slots(const clip_hparams & hparams, const clip_model & model) {
+std::vector<c2w_state_slot> list_lfm3a_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
std::vector<c2w_state_slot> slots;
if (model.c2w.tfm_layers.empty()) {
return slots;
@@ -29,7 +29,7 @@ std::vector<c2w_state_slot> list_lfm3a_gen_state_slots(const clip_hparams & hpar
}
// pre-norm LFM block, with the LFM3 sandwich norms when the layer has them
-ggml_tensor * clip_graph_lfm3a_gen::build_lfm_layer(ggml_tensor * inpL, const clip_layer & layer, float eps, int il,
+ggml_tensor * mtmd_graph_lfm3a_gen::build_lfm_layer(ggml_tensor * inpL, const mtmd_layer & layer, float eps, int il,
const std::function<ggml_tensor *(ggml_tensor *)> & op) {
ggml_tensor * cur = build_norm(inpL, layer.ln_1_w, nullptr, NORM_TYPE_RMS, eps, il);
cur = op(cur);
@@ -47,7 +47,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_lfm_layer(ggml_tensor * inpL, const cl
}
// GQA with q/k RMSNorm, NEOX rope, an optional cached K/V prefix and the LFM3 per-head sigmoid gate
-ggml_tensor * clip_graph_lfm3a_gen::build_lfm_attn(ggml_tensor * cur, const clip_layer & layer, int n_head, int n_head_kv,
+ggml_tensor * mtmd_graph_lfm3a_gen::build_lfm_attn(ggml_tensor * cur, const mtmd_layer & layer, int n_head, int n_head_kv,
float eps, float rope_theta, int il, ggml_tensor * inp_pos,
ggml_tensor * kq_mask, ggml_tensor * k_cache, ggml_tensor * v_cache,
ggml_tensor ** k_all, ggml_tensor ** v_all) {
@@ -89,7 +89,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_lfm_attn(ggml_tensor * cur, const clip
}
// short conv: in_proj -> b * x -> causal depthwise conv over [conv_state | x] -> c * conv -> out_proj
-ggml_tensor * clip_graph_lfm3a_gen::build_lfm_conv(ggml_tensor * cur, const clip_layer & layer, int il,
+ggml_tensor * mtmd_graph_lfm3a_gen::build_lfm_conv(ggml_tensor * cur, const mtmd_layer & layer, int il,
ggml_tensor * conv_state, ggml_tensor ** conv_state_out) {
const int64_t n_embd_c = cur->ne[0];
const int64_t n_pos = cur->ne[1];
@@ -110,7 +110,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_lfm_conv(ggml_tensor * cur, const clip
return build_mm(layer.conv_pw2_w, cur);
}
-ggml_tensor * clip_graph_lfm3a_gen::build_gen_code() {
+ggml_tensor * mtmd_graph_lfm3a_gen::build_gen_code() {
ggml_tensor * embd_w = model.gen_code_embd_w; // [n_embd, n_vocab, n_codebooks], tied to the output head
const int64_t n_cb = embd_w->ne[2];
const int64_t n_vocab = embd_w->ne[1];
@@ -169,7 +169,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_gen_code() {
return fb;
}
-ggml_tensor * clip_graph_lfm3a_gen::build_gen_wav() {
+ggml_tensor * mtmd_graph_lfm3a_gen::build_gen_wav() {
const auto & c2w = model.c2w;
const int64_t n_cb = model.gen_code_embd_w->ne[2];
const int64_t n_fs = hparams.wav_frame_stack;
@@ -201,7 +201,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_gen_wav() {
cur = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, cur, n_embd_dec, 1, n_frames), n_embd_dec, n_fs, n_frames, 1);
cur = ggml_reshape_2d(ctx0, cur, n_embd_dec, n_pos);
- // positions continue across calls from the "pos" slot, see clip_encode()
+ // positions continue across calls from the "pos" slot, see mtmd_subcontext::encode()
ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
ggml_set_name(inp_pos, "positions");
ggml_set_input(inp_pos);
@@ -250,8 +250,8 @@ ggml_tensor * clip_graph_lfm3a_gen::build_gen_wav() {
return cur;
}
-ggml_cgraph * clip_graph_lfm3a_gen::build() {
- ggml_tensor * out = gen_process == CLIP_GEN_PROCESS_GEN_WAV ? build_gen_wav() : build_gen_code();
+ggml_cgraph * mtmd_graph_lfm3a_gen::build() {
+ ggml_tensor * out = gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ? build_gen_wav() : build_gen_code();
ggml_build_forward_expand(gf, out);
return gf;
}
diff --git a/tools/mtmd/models/lfm3a.cpp b/tools/mtmd/models/lfm3a.cpp
index 4882fc42f..03128a298 100644
--- a/tools/mtmd/models/lfm3a.cpp
+++ b/tools/mtmd/models/lfm3a.cpp
@@ -2,7 +2,7 @@
// streaming encoder state: base position, conv left context, short-conv taps and attention window
// a zero state gives the same result as encoding the whole clip
-std::vector<c2w_state_slot> list_lfm3a_state_slots(const clip_hparams & hparams, const clip_model & model) {
+std::vector<c2w_state_slot> list_lfm3a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
std::vector<c2w_state_slot> slots;
const int64_t n_mel = hparams.n_mel_bins;
@@ -27,7 +27,7 @@ std::vector<c2w_state_slot> list_lfm3a_state_slots(const clip_hparams & hparams,
}
// LFM3-Audio encoder: conv downsampler (8x in time) -> causal LFM2 stack -> MLP projector
-ggml_cgraph * clip_graph_lfm3a::build() {
+ggml_cgraph * mtmd_graph_lfm3a::build() {
const norm_type norm_t = NORM_TYPE_RMS;
const float rope_theta = hparams.rope_theta;
const int64_t prefix = hparams.attn_window_size - 1;
diff --git a/tools/mtmd/models/lfm3vl.cpp b/tools/mtmd/models/lfm3vl.cpp
index e37390976..6ad64ecb1 100644
--- a/tools/mtmd/models/lfm3vl.cpp
+++ b/tools/mtmd/models/lfm3vl.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_lfm3vl::build() {
+ggml_cgraph * mtmd_graph_lfm3vl::build() {
GGML_ASSERT(model.patch_embeddings_0 != nullptr);
GGML_ASSERT(model.lfm3vl_embedding_norm_w != nullptr);
GGML_ASSERT(model.class_embedding == nullptr);
diff --git a/tools/mtmd/models/ling3vl.cpp b/tools/mtmd/models/ling3vl.cpp
index fb9eb7c74..c3b8ed968 100644
--- a/tools/mtmd/models/ling3vl.cpp
+++ b/tools/mtmd/models/ling3vl.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_ling3vl::build() {
+ggml_cgraph * mtmd_graph_ling3vl::build() {
// same vision tower as qwen3vl, but the merger is norm-only (no fc1/fc2) and
// the projector MLP lives at the top level (mm.0 / mm.2)
GGML_ASSERT(model.class_embedding == nullptr);
@@ -57,7 +57,7 @@ ggml_cgraph * clip_graph_ling3vl::build() {
ggml_tensor * inpL = build_vit(
inp, n_pos, norm_t, hparams.ffn_op, learned_pos_embd,
- [&](ggml_tensor * c, const clip_layer &) {
+ [&](ggml_tensor * c, const mtmd_layer &) {
return ggml_rope_multi(
ctx0, c, positions, nullptr,
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);
diff --git a/tools/mtmd/models/llama4.cpp b/tools/mtmd/models/llama4.cpp
index 01af54bba..60bfd5e2e 100644
--- a/tools/mtmd/models/llama4.cpp
+++ b/tools/mtmd/models/llama4.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_llama4::build() {
+ggml_cgraph * mtmd_graph_llama4::build() {
GGML_ASSERT(model.class_embedding != nullptr);
GGML_ASSERT(model.position_embeddings != nullptr);
@@ -31,7 +31,7 @@ ggml_cgraph * clip_graph_llama4::build() {
inp = ggml_concat(ctx0, inp, model.class_embedding, 1);
// build ViT with 2D position embeddings
- auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) {
// first half is X axis and second half is Y axis
// ref: https://github.com/huggingface/transformers/blob/40a493c7ed4f19f08eadb0639cf26d49bfa5e180/src/transformers/models/llama4/modeling_llama4.py#L1312
// ref: https://github.com/Blaizzy/mlx-vlm/blob/a57156aa87b33cca6e5ee6cfc14dd4ef8f611be6/mlx_vlm/models/llama4/vision.py#L441
diff --git a/tools/mtmd/models/llava.cpp b/tools/mtmd/models/llava.cpp
index 47efe68bd..f4f31ffdb 100644
--- a/tools/mtmd/models/llava.cpp
+++ b/tools/mtmd/models/llava.cpp
@@ -2,7 +2,7 @@
// this graph is used by llava, granite and glm
// due to having embedding_stack (used by granite), we cannot reuse build_vit
-ggml_cgraph * clip_graph_llava::build() {
+ggml_cgraph * mtmd_graph_llava::build() {
const int batch_size = 1;
const int n_pos = n_patches + (model.class_embedding ? 1 : 0);
diff --git a/tools/mtmd/models/magevl.cpp b/tools/mtmd/models/magevl.cpp
index f5370ea9f..393f19aaf 100644
--- a/tools/mtmd/models/magevl.cpp
+++ b/tools/mtmd/models/magevl.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_magevl::build() {
+ggml_cgraph * mtmd_graph_magevl::build() {
GGML_ASSERT(hparams.n_merge == 2);
const int n_pair = d_head / 2;
@@ -14,7 +14,7 @@ ggml_cgraph * clip_graph_magevl::build() {
ggml_set_input(positions);
// a pair uses 2 angles: even outputs come from one rope, odd outputs from the other
- auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) {
ggml_tensor * freqs_a = ggml_view_1d(ctx0, model.rope_freqs, n_pair, 0);
ggml_tensor * freqs_b = ggml_view_1d(ctx0, model.rope_freqs, n_pair, ggml_row_size(model.rope_freqs->type, n_pair));
diff --git a/tools/mtmd/models/mimo-audio.cpp b/tools/mtmd/models/mimo-audio.cpp
index 0415bb60d..92be15fc9 100644
--- a/tools/mtmd/models/mimo-audio.cpp
+++ b/tools/mtmd/models/mimo-audio.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_mimo_audio::build() {
+ggml_cgraph * mtmd_graph_mimo_audio::build() {
ggml_tensor * inp = build_inp_raw(1); // [n_frames, n_mel, 1]
ggml_tensor * cur = ggml_conv_1d_ph(ctx0, model.conv1d_1_w, inp, 1, 1);
@@ -43,7 +43,7 @@ ggml_cgraph * clip_graph_mimo_audio::build() {
}
};
- auto add_pos = [&](ggml_tensor * x, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * x, const mtmd_layer &) {
return ggml_rope_ext(ctx0, x, inp_pos, nullptr, d_head,
GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
};
diff --git a/tools/mtmd/models/mimovl.cpp b/tools/mtmd/models/mimovl.cpp
index 4843698e6..6f2b8934f 100644
--- a/tools/mtmd/models/mimovl.cpp
+++ b/tools/mtmd/models/mimovl.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_tensor * clip_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) const {
+ggml_tensor * mtmd_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) const {
ggml_tensor * cur = ggml_mul_mat(ctx0, w, x);
ggml_prec_set_acc(cur, GGML_PREC_BF16);
return cur;
@@ -19,7 +19,7 @@ ggml_tensor * clip_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) cons
// 4. 1D banded sliding window (|q-k| > window_size -> -inf) as a
// single 2D mask broadcast across heads.
// 5. Per-block MLP biases.
-ggml_cgraph * clip_graph_mimovl::build() {
+ggml_cgraph * mtmd_graph_mimovl::build() {
GGML_ASSERT(model.patch_embeddings_0 != nullptr);
GGML_ASSERT(model.patch_embeddings_1 != nullptr);
GGML_ASSERT(model.class_embedding == nullptr);
diff --git a/tools/mtmd/models/minicpmv.cpp b/tools/mtmd/models/minicpmv.cpp
index 0e0fc0381..08ede0dd1 100644
--- a/tools/mtmd/models/minicpmv.cpp
+++ b/tools/mtmd/models/minicpmv.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_minicpmv::build() {
+ggml_cgraph * mtmd_graph_minicpmv::build() {
GGML_ASSERT(model.class_embedding == nullptr);
const int n_pos = n_patches;
const int n_embd_proj = n_mmproj_embd;
@@ -113,7 +113,7 @@ ggml_cgraph * clip_graph_minicpmv::build() {
return gf;
}
-ggml_cgraph * clip_graph_minicpmv4_6::build() {
+ggml_cgraph * mtmd_graph_minicpmv4_6::build() {
const bool is_4x = hparams.n_merge == 2;
const int n_pos = n_patches;
const int half_h = n_patches_y / 2;
diff --git a/tools/mtmd/models/minimax-m3.cpp b/tools/mtmd/models/minimax-m3.cpp
index 256e53105..2ac437bf1 100644
--- a/tools/mtmd/models/minimax-m3.cpp
+++ b/tools/mtmd/models/minimax-m3.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_tensor * clip_graph_minimax_m3::apply_rope(
+ggml_tensor * mtmd_graph_minimax_m3::apply_rope(
ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) {
const int dh = (int) x->ne[0];
const int axd = 2 * ((2 * (dh / 2) / 3) / 2);
@@ -20,7 +20,7 @@ ggml_tensor * clip_graph_minimax_m3::apply_rope(
return x;
}
-ggml_cgraph * clip_graph_minimax_m3::build() {
+ggml_cgraph * mtmd_graph_minimax_m3::build() {
GGML_ASSERT(model.patch_bias == nullptr);
GGML_ASSERT(model.class_embedding == nullptr);
GGML_ASSERT(model.patch_embeddings_0 && model.patch_embeddings_1);
@@ -54,7 +54,7 @@ ggml_cgraph * clip_graph_minimax_m3::build() {
ggml_tensor * inpL = build_vit(
inp, n_pos, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr,
- [&](ggml_tensor * c, const clip_layer &) {
+ [&](ggml_tensor * c, const mtmd_layer &) {
return apply_rope(c, pos_h, pos_w);
});
diff --git a/tools/mtmd/models/mobilenetv5.cpp b/tools/mtmd/models/mobilenetv5.cpp
index 1c42218d2..e127cb31e 100644
--- a/tools/mtmd/models/mobilenetv5.cpp
+++ b/tools/mtmd/models/mobilenetv5.cpp
@@ -2,7 +2,7 @@
// Helpers for MobileNetV5 Blocks
// RMS Norm 2D - normalizes over channels for each spatial position
-ggml_tensor * clip_graph_mobilenetv5::rms_norm_2d(ggml_tensor * inp, ggml_tensor * weight, float eps) {
+ggml_tensor * mtmd_graph_mobilenetv5::rms_norm_2d(ggml_tensor * inp, ggml_tensor * weight, float eps) {
// inp: [W, H, C, B]
ggml_tensor * cur = ggml_permute(ctx0, inp, 2, 1, 0, 3);
@@ -20,7 +20,7 @@ ggml_tensor * clip_graph_mobilenetv5::rms_norm_2d(ggml_tensor * inp, ggml_tensor
}
// Conv2dSame padding - asymmetric SAME padding like PyTorch/TF
-ggml_tensor* clip_graph_mobilenetv5::pad_same_2d(ggml_tensor* inp, int kernel_h, int kernel_w, int stride_h, int stride_w, int dilation_h, int dilation_w) {
+ggml_tensor* mtmd_graph_mobilenetv5::pad_same_2d(ggml_tensor* inp, int kernel_h, int kernel_w, int stride_h, int stride_w, int dilation_h, int dilation_w) {
const int64_t ih = inp->ne[1]; // height
const int64_t iw = inp->ne[0]; // width
@@ -54,7 +54,7 @@ ggml_tensor* clip_graph_mobilenetv5::pad_same_2d(ggml_tensor* inp, int kernel_h,
// Edge Residual Block (Stage 0)
-ggml_tensor * clip_graph_mobilenetv5::build_edge_residual(ggml_tensor * inp, const mobilenetv5_block & block, int stride) {
+ggml_tensor * mtmd_graph_mobilenetv5::build_edge_residual(ggml_tensor * inp, const mobilenetv5_block & block, int stride) {
ggml_tensor * cur = inp;
// 1. Expansion Conv (3x3)
@@ -88,7 +88,7 @@ ggml_tensor * clip_graph_mobilenetv5::build_edge_residual(ggml_tensor * inp, con
}
// Universal Inverted Residual Block (Stage 1+)
-ggml_tensor * clip_graph_mobilenetv5::build_inverted_residual(ggml_tensor * inp, const mobilenetv5_block & block, int stride) {
+ggml_tensor * mtmd_graph_mobilenetv5::build_inverted_residual(ggml_tensor * inp, const mobilenetv5_block & block, int stride) {
ggml_tensor * cur = inp;
// 1. Depthwise Start (Optional)
@@ -149,7 +149,7 @@ ggml_tensor * clip_graph_mobilenetv5::build_inverted_residual(ggml_tensor * inp,
}
// Attention Block (MQA)
-ggml_tensor * clip_graph_mobilenetv5::build_mobilenet_attn(ggml_tensor * inp, const mobilenetv5_block & block) {
+ggml_tensor * mtmd_graph_mobilenetv5::build_mobilenet_attn(ggml_tensor * inp, const mobilenetv5_block & block) {
ggml_tensor * cur = inp;
// Norm
@@ -245,7 +245,7 @@ ggml_tensor * clip_graph_mobilenetv5::build_mobilenet_attn(ggml_tensor * inp, co
return cur;
}
-ggml_cgraph * clip_graph_mobilenetv5::build() {
+ggml_cgraph * mtmd_graph_mobilenetv5::build() {
ggml_tensor * inp = build_inp_raw();
// 1. Stem - Conv2dSame(3, 64, kernel_size=(3, 3), stride=(2, 2))
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 8487a0042..17bd57d95 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -1,6 +1,6 @@
#pragma once
-#include "../clip-graph.h"
+#include "../mtmd-graph.h"
#include <functional>
#include <map>
@@ -13,32 +13,32 @@
* We encourage human contributors to ensure the quality and reliability of the codebase.
*/
-struct clip_graph_siglip : clip_graph {
- clip_graph_siglip(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_siglip : mtmd_graph {
+ mtmd_graph_siglip(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_gemma4v : clip_graph {
- clip_graph_gemma4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_gemma4v : mtmd_graph {
+ mtmd_graph_gemma4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override;
bool support_batch() const override { return true; }
};
-struct clip_graph_gemma4uv : clip_graph {
- clip_graph_gemma4uv(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_gemma4uv : mtmd_graph {
+ mtmd_graph_gemma4uv(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_nemotron_transcribe : clip_graph {
- const clip_enc_state * state; // nullptr during warmup
- clip_graph_nemotron_transcribe(clip_ctx * ctx, const clip_image_f32 & img, const clip_enc_state * state)
- : clip_graph(ctx, img), state(state) {}
+struct mtmd_graph_nemotron_transcribe : mtmd_graph {
+ const mtmd_subctx_enc_state * state; // nullptr during warmup
+ mtmd_graph_nemotron_transcribe(mtmd_subcontext * ctx, const mtmd_image_f32 & img, const mtmd_subctx_enc_state * state)
+ : mtmd_graph(ctx, img), state(state) {}
ggml_cgraph * build() override;
ggml_tensor * build_spk_encoder(ggml_tensor * cur, ggml_tensor * pos);
// one recording: encodes its windows in turn, carrying the speaker state between them
- static bool encode_windows(clip_ctx * ctx, const clip_model & model, clip_encode_params * params);
+ static bool encode_windows(mtmd_subcontext * ctx, const mtmd_model & model, mtmd_subctx_encode_params * params);
// NeMo ParallelExpertEncoder online defaults, in output frames: 40 s windows with 4 s of context on each side
static constexpr int window_core = 500;
@@ -46,54 +46,54 @@ struct clip_graph_nemotron_transcribe : clip_graph {
static constexpr int window_right = 50;
};
-struct clip_graph_pixtral : clip_graph {
- clip_graph_pixtral(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_pixtral : mtmd_graph {
+ mtmd_graph_pixtral(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_deepseek4v : clip_graph {
- clip_graph_deepseek4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_deepseek4v : mtmd_graph {
+ mtmd_graph_deepseek4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_qwen2vl : clip_graph {
- clip_graph_qwen2vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_qwen2vl : mtmd_graph {
+ mtmd_graph_qwen2vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_qwen3vl : clip_graph_qwen2vl {
- clip_graph_qwen3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen2vl(ctx, img) {}
+struct mtmd_graph_qwen3vl : mtmd_graph_qwen2vl {
+ mtmd_graph_qwen3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_qwen2vl(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_magevl : clip_graph {
- clip_graph_magevl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_magevl : mtmd_graph {
+ mtmd_graph_magevl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_sensenova_u1 : clip_graph {
- clip_graph_sensenova_u1(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_sensenova_u1 : mtmd_graph {
+ mtmd_graph_sensenova_u1(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_lfm3vl : clip_graph {
- clip_graph_lfm3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_lfm3vl : mtmd_graph {
+ mtmd_graph_lfm3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_ling3vl : clip_graph_qwen3vl {
- clip_graph_ling3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen3vl(ctx, img) {}
+struct mtmd_graph_ling3vl : mtmd_graph_qwen3vl {
+ mtmd_graph_ling3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_qwen3vl(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_minimax_m3 : clip_graph {
- clip_graph_minimax_m3(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_minimax_m3 : mtmd_graph {
+ mtmd_graph_minimax_m3(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * apply_rope(ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w);
};
-struct clip_graph_mimovl : clip_graph {
- clip_graph_mimovl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_mimovl : mtmd_graph {
+ mtmd_graph_mimovl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
// Force F32 mat-mul accumulation to avoid F16 overflow in the FFN down-proj
// when the mmproj is stored in F16 (the source weights are BF16; downcasting
@@ -101,119 +101,119 @@ struct clip_graph_mimovl : clip_graph {
ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override;
};
-struct clip_graph_step3vl : clip_graph {
- clip_graph_step3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_step3vl : mtmd_graph {
+ mtmd_graph_step3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_youtuvl : clip_graph {
- clip_graph_youtuvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_youtuvl : mtmd_graph {
+ mtmd_graph_youtuvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_yasa2 : clip_graph {
- clip_graph_yasa2(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_yasa2 : mtmd_graph {
+ mtmd_graph_yasa2(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * layer_norm_channels(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b, float eps = 1e-6f);
ggml_tensor * convnext_grn(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b);
};
-struct clip_graph_minicpmv : clip_graph {
- clip_graph_minicpmv(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_minicpmv : mtmd_graph {
+ mtmd_graph_minicpmv(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_minicpmv4_6 : clip_graph {
- clip_graph_minicpmv4_6(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_minicpmv4_6 : mtmd_graph {
+ mtmd_graph_minicpmv4_6(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_internvl : clip_graph {
- clip_graph_internvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_internvl : mtmd_graph {
+ mtmd_graph_internvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
bool support_batch() const override { return true; }
};
-struct clip_graph_nemotron_v2_vl : clip_graph {
- clip_graph_nemotron_v2_vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_nemotron_v2_vl : mtmd_graph {
+ mtmd_graph_nemotron_v2_vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_nemotron_omni : clip_graph {
- clip_graph_nemotron_omni(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_nemotron_omni : mtmd_graph {
+ mtmd_graph_nemotron_omni(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * build_inp_with_temporal_merge();
};
-struct clip_graph_llama4 : clip_graph {
- clip_graph_llama4(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_llama4 : mtmd_graph {
+ mtmd_graph_llama4(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_kimivl : clip_graph {
- clip_graph_kimivl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_kimivl : mtmd_graph {
+ mtmd_graph_kimivl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_paddleocr : clip_graph {
- clip_graph_paddleocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_paddleocr : mtmd_graph {
+ mtmd_graph_paddleocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_dotsocr : clip_graph {
- clip_graph_dotsocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_dotsocr : mtmd_graph {
+ mtmd_graph_dotsocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_dots3note_a : clip_graph {
- clip_graph_dots3note_a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_dots3note_a : mtmd_graph {
+ mtmd_graph_dots3note_a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_cogvlm : clip_graph {
- clip_graph_cogvlm(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_cogvlm : mtmd_graph {
+ mtmd_graph_cogvlm(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_llava : clip_graph {
- clip_graph_llava(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_llava : mtmd_graph {
+ mtmd_graph_llava(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_whisper_enc : clip_graph {
- clip_graph_whisper_enc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_whisper_enc : mtmd_graph {
+ mtmd_graph_whisper_enc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
// causal whisper-like encoder with a sliding window KV cache (voxtral realtime)
-struct clip_graph_voxtral_rt : clip_graph {
- clip_enc_state * state; // nullptr during warmup, the cache is then a graph input
- clip_graph_voxtral_rt(clip_ctx * ctx, const clip_image_f32 & img, clip_enc_state * state) : clip_graph(ctx, img), state(state) {}
+struct mtmd_graph_voxtral_rt : mtmd_graph {
+ mtmd_subctx_enc_state * state; // nullptr during warmup, the cache is then a graph input
+ mtmd_graph_voxtral_rt(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_enc_state * state) : mtmd_graph(ctx, img), state(state) {}
ggml_cgraph * build() override;
};
-struct clip_graph_deepseekocr : clip_graph {
- clip_graph_deepseekocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_deepseekocr : mtmd_graph {
+ mtmd_graph_deepseekocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * build_sam(ggml_tensor * inp); // build the SAM model
bool support_batch() const override { return true; }
};
-struct clip_graph_deepseekocr2 : clip_graph_deepseekocr {
- clip_graph_deepseekocr2(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_deepseekocr(ctx, img) {}
+struct mtmd_graph_deepseekocr2 : mtmd_graph_deepseekocr {
+ mtmd_graph_deepseekocr2(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_deepseekocr(ctx, img) {}
ggml_cgraph * build() override; // reuses build_sam() from base
bool support_batch() const override { return true; }
};
-struct clip_graph_conformer : clip_graph {
- clip_graph_conformer(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_conformer : mtmd_graph {
+ mtmd_graph_conformer(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
// builders shared by the chatterbox gen and spkenc graphs, defined in chatterbox-gen.cpp
-struct clip_graph_chatterbox_base : clip_graph {
- using clip_graph::clip_graph;
+struct mtmd_graph_chatterbox_base : mtmd_graph {
+ using mtmd_graph::mtmd_graph;
protected:
ggml_tensor * cbx_linear(ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) const;
@@ -223,79 +223,79 @@ struct clip_graph_chatterbox_base : clip_graph {
int pad, int dil) const;
};
-struct clip_graph_chatterbox_spkenc : clip_graph_chatterbox_base {
- clip_graph_chatterbox_spkenc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_chatterbox_base(ctx, img) {}
+struct mtmd_graph_chatterbox_spkenc : mtmd_graph_chatterbox_base {
+ mtmd_graph_chatterbox_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_chatterbox_base(ctx, img) {}
ggml_cgraph * build() override;
private:
- ggml_tensor * bn1d(const clip_chatterbox::bn & n, ggml_tensor * x);
- ggml_tensor * bn2d_relu(const clip_chatterbox::bn & n, ggml_tensor * x);
- ggml_tensor * res2d(const clip_chatterbox::spk_res2d & r, ggml_tensor * x, int stride);
- ggml_tensor * cam_layer(const clip_chatterbox::spk_cam_layer & l, ggml_tensor * x, int dil, ggml_tensor * segfix);
+ ggml_tensor * bn1d(const mtmd_chatterbox::bn & n, ggml_tensor * x);
+ ggml_tensor * bn2d_relu(const mtmd_chatterbox::bn & n, ggml_tensor * x);
+ ggml_tensor * res2d(const mtmd_chatterbox::spk_res2d & r, ggml_tensor * x, int stride);
+ ggml_tensor * cam_layer(const mtmd_chatterbox::spk_cam_layer & l, ggml_tensor * x, int dil, ggml_tensor * segfix);
};
-struct clip_graph_chatterbox : clip_graph_chatterbox_base {
- clip_gen_process_type gen_process = CLIP_GEN_PROCESS_GEN_CODE;
+struct mtmd_graph_chatterbox : mtmd_graph_chatterbox_base {
+ mtmd_subctx_gen_process gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
int n_tokens = 0;
int n_prompt_mel = 0;
int vocode_n_mel = 0;
int vocode_n_stft = 0;
- clip_graph_chatterbox(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, int n_tokens,
+ mtmd_graph_chatterbox(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_tokens,
int n_prompt_mel, int vocode_n_mel, int vocode_n_stft)
- : clip_graph_chatterbox_base(ctx, img), gen_process(gen_process), n_tokens(n_tokens), n_prompt_mel(n_prompt_mel),
+ : mtmd_graph_chatterbox_base(ctx, img), gen_process(gen_process), n_tokens(n_tokens), n_prompt_mel(n_prompt_mel),
vocode_n_mel(vocode_n_mel), vocode_n_stft(vocode_n_stft) {}
ggml_cgraph * build() override;
private:
- ggml_tensor * enc_layer(const clip_chatterbox::enc_layer & l, ggml_tensor * x, ggml_tensor * pos, int T);
- ggml_tensor * causal_block(const clip_chatterbox::causal_block & b, ggml_tensor * x);
- ggml_tensor * resnet(const clip_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb);
- ggml_tensor * tfm_block(const clip_chatterbox::tfm_block & b, ggml_tensor * x);
+ ggml_tensor * enc_layer(const mtmd_chatterbox::enc_layer & l, ggml_tensor * x, ggml_tensor * pos, int T);
+ ggml_tensor * causal_block(const mtmd_chatterbox::causal_block & b, ggml_tensor * x);
+ ggml_tensor * resnet(const mtmd_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb);
+ ggml_tensor * tfm_block(const mtmd_chatterbox::tfm_block & b, ggml_tensor * x);
ggml_tensor * estimator(ggml_tensor * x_noise, ggml_tensor * mu, ggml_tensor * spks,
ggml_tensor * cond, ggml_tensor * temb, int T);
- ggml_tensor * hift_resblock(const clip_chatterbox::hift_res & r, ggml_tensor * x);
+ ggml_tensor * hift_resblock(const mtmd_chatterbox::hift_res & r, ggml_tensor * x);
ggml_cgraph * build_s3tok(int T);
ggml_cgraph * build_vocoder(int n_mel, int n_stft);
};
-struct clip_graph_lfm3a : clip_graph {
- clip_graph_lfm3a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_lfm3a : mtmd_graph {
+ mtmd_graph_lfm3a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_granite_speech : clip_graph {
- clip_graph_granite_speech(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_granite_speech : mtmd_graph {
+ mtmd_graph_granite_speech(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_muse_spark_audio : clip_graph {
- clip_graph_muse_spark_audio(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_muse_spark_audio : mtmd_graph {
+ mtmd_graph_muse_spark_audio(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_gemma4a : clip_graph {
- clip_graph_gemma4a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_gemma4a : mtmd_graph {
+ mtmd_graph_gemma4a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override;
};
-struct clip_graph_gemma4ua : clip_graph {
- clip_graph_gemma4ua(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_gemma4ua : mtmd_graph {
+ mtmd_graph_gemma4ua(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_glm4v : clip_graph {
- clip_graph_glm4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_glm4v : mtmd_graph {
+ mtmd_graph_glm4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_hunyuanvl : clip_graph {
- clip_graph_hunyuanvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_hunyuanvl : mtmd_graph {
+ mtmd_graph_hunyuanvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_mobilenetv5 : clip_graph {
- clip_graph_mobilenetv5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_mobilenetv5 : mtmd_graph {
+ mtmd_graph_mobilenetv5(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * rms_norm_2d(
@@ -327,34 +327,34 @@ struct clip_graph_mobilenetv5 : clip_graph {
const mobilenetv5_block & block);
};
-struct clip_graph_qwen3a : clip_graph {
- clip_graph_qwen3a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_qwen3a : mtmd_graph {
+ mtmd_graph_qwen3a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_mimo_audio : clip_graph {
- clip_graph_mimo_audio(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_mimo_audio : mtmd_graph {
+ mtmd_graph_mimo_audio(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_qwen3tts_spkenc : clip_graph {
- clip_graph_qwen3tts_spkenc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_qwen3tts_spkenc : mtmd_graph {
+ mtmd_graph_qwen3tts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const;
- ggml_tensor * res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const;
- ggml_tensor * se_block(ggml_tensor * x, const clip_layer & layer) const;
- ggml_tensor * se_res2net_block(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const;
+ ggml_tensor * res2net(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const;
+ ggml_tensor * se_block(ggml_tensor * x, const mtmd_layer & layer) const;
+ ggml_tensor * se_res2net_block(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const;
ggml_tensor * attentive_stats_pool(ggml_tensor * x) const;
};
-struct clip_graph_qwen3tts_gen : clip_graph {
- clip_graph_qwen3tts_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, int top_k, float top_p)
- : clip_graph(ctx, img), gen_process(gen_process), top_k(top_k), top_p(top_p) {}
+struct mtmd_graph_qwen3tts_gen : mtmd_graph {
+ mtmd_graph_qwen3tts_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int top_k, float top_p)
+ : mtmd_graph(ctx, img), gen_process(gen_process), top_k(top_k), top_p(top_p) {}
ggml_cgraph * build() override;
// which sub-graph build() constructs, fixed at graph-build time
- clip_gen_process_type gen_process;
+ mtmd_subctx_gen_process gen_process;
// sampling params, fixed at graph-build time (GEN_CODE only)
int top_k;
@@ -364,9 +364,9 @@ struct clip_graph_qwen3tts_gen : clip_graph {
// code_gen: backbone hidden state + sampled code0 -> 16 RVQ codes
// MTP-style code predictor, one token per codebook
//
- struct code_gen : clip_graph {
- code_gen(const clip_graph & parent, int top_k, float top_p)
- : clip_graph(parent), top_k(top_k), top_p(top_p) {}
+ struct code_gen : mtmd_graph {
+ code_gen(const mtmd_graph & parent, int top_k, float top_p)
+ : mtmd_graph(parent), top_k(top_k), top_p(top_p) {}
ggml_cgraph * build() override { GGML_ABORT("call prefill()/step() instead"); }
int top_k;
@@ -381,7 +381,7 @@ struct clip_graph_qwen3tts_gen : clip_graph {
ggml_tensor * layer_forward(
ggml_tensor * cur,
- const clip_layer & layer,
+ const mtmd_layer & layer,
ggml_tensor * inp_pos,
ggml_tensor * kq_mask,
ggml_tensor *& k_cache_layer,
@@ -409,8 +409,8 @@ struct clip_graph_qwen3tts_gen : clip_graph {
//
// code2wav: RVQ codes -> raw PCM (quantizer + pre_conv + pre_transformer + upsample + DAC).
//
- struct code2wav : clip_graph {
- code2wav(const clip_graph & parent) : clip_graph(parent) {}
+ struct code2wav : mtmd_graph {
+ code2wav(const mtmd_graph & parent) : mtmd_graph(parent) {}
ggml_cgraph * build() override { GGML_ABORT("call decode() instead"); }
// state_in: previous call's persisted state, by slot name (see list_c2w_state_slots())
@@ -425,9 +425,9 @@ struct clip_graph_qwen3tts_gen : clip_graph {
ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const;
ggml_tensor * quant_decode(ggml_tensor * inp_codes) const;
- ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const;
- ggml_tensor * convnext_block(ggml_tensor * x, const clip_code2wav::upsample_block & blk, const std::string & state_prefix) const;
- ggml_tensor * dac_res_unit(ggml_tensor * x, const clip_code2wav::dac_res & res, int dilation, const std::string & state_name) const;
+ ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il) const;
+ ggml_tensor * convnext_block(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk, const std::string & state_prefix) const;
+ ggml_tensor * dac_res_unit(ggml_tensor * x, const mtmd_code2wav::dac_res & res, int dilation, const std::string & state_name) const;
// drama3: erf GELU in ConvNeXt, not the tanh approximation
bool gelu_erf = false;
@@ -441,8 +441,8 @@ struct clip_graph_qwen3tts_gen : clip_graph {
// pocket-tts: SEANet convolution stack, shared by the voice encoder and the mimi decoder.
// stateless unless state_in is populated: convs then pad instead of carrying left-context.
//
-struct clip_graph_pockettts_seanet : clip_graph {
- clip_graph_pockettts_seanet(const clip_graph & parent) : clip_graph(parent) {}
+struct mtmd_graph_pockettts_seanet : mtmd_graph {
+ mtmd_graph_pockettts_seanet(const mtmd_graph & parent) : mtmd_graph(parent) {}
ggml_cgraph * build() override { GGML_ABORT("call encode()/decode() instead"); }
// per-call streaming state, keyed by slot name (see list_pockettts_state_slots)
@@ -453,7 +453,7 @@ struct clip_graph_pockettts_seanet : clip_graph {
bool pad_replicate = false, const std::string & state_name = "") const;
ggml_tensor * conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride,
const std::string & state_name = "") const;
- ggml_tensor * res_unit(ggml_tensor * x, const clip_seanet::stage & stage, int dilation,
+ ggml_tensor * res_unit(ggml_tensor * x, const mtmd_seanet::stage & stage, int dilation,
const std::string & state_prefix = "") const;
// x: [T, C] -> [T / hop, dim]
@@ -463,11 +463,11 @@ struct clip_graph_pockettts_seanet : clip_graph {
};
// mimi encoder + speaker_proj: reference waveform -> voice conditioning rows
-struct clip_graph_pockettts_spkenc : clip_graph {
- clip_graph_pockettts_spkenc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_pockettts_spkenc : mtmd_graph {
+ mtmd_graph_pockettts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
- ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const;
+ ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const;
};
//
@@ -475,18 +475,18 @@ struct clip_graph_pockettts_spkenc : clip_graph {
// GEN_CODE = flow-matching decoder + end-of-speech head, one latent per call
// GEN_WAV = mimi decoder, a window of latents -> PCM
//
-struct clip_graph_pockettts_gen : clip_graph {
- clip_graph_pockettts_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, int n_step, int n_frames)
- : clip_graph(ctx, img), gen_process(gen_process), n_step(n_step), n_frames(n_frames) {}
+struct mtmd_graph_pockettts_gen : mtmd_graph {
+ mtmd_graph_pockettts_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_step, int n_frames)
+ : mtmd_graph(ctx, img), gen_process(gen_process), n_step(n_step), n_frames(n_frames) {}
ggml_cgraph * build() override;
- clip_gen_process_type gen_process;
+ mtmd_subctx_gen_process gen_process;
int n_step; // lsd_decode steps, fixed at graph-build time
int n_frames; // GEN_WAV only: number of latents to decode
// AdaLN modulation: x * (1 + scale) + shift
ggml_tensor * modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const;
- ggml_tensor * time_embed(const clip_flow_net::time_embd & te, float t) const;
+ ggml_tensor * time_embed(const mtmd_flow_net::time_embd & te, float t) const;
ggml_tensor * flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const;
};
@@ -497,30 +497,30 @@ struct c2w_state_slot {
int64_t ne1;
bool in_place = false; // graph writes the slot directly, no state_in/state_out copy
};
-std::vector<c2w_state_slot> list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model);
+std::vector<c2w_state_slot> list_c2w_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
// drama3: tower KV slots, then list_c2w_state_slots()
-std::vector<c2w_state_slot> list_drama3_state_slots(const clip_hparams & hparams, const clip_model & model);
+std::vector<c2w_state_slot> list_drama3_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
// same, for the streaming mimi decoder (pocket-tts GEN_WAV)
-std::vector<c2w_state_slot> list_pockettts_state_slots(const clip_hparams & hparams, const clip_model & model);
+std::vector<c2w_state_slot> list_pockettts_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
//
// drama3 generation:
// GEN_CODE = one codebook tower row, each layer attends over the whole sequence (KV in gen state)
// GEN_WAV = Fish DAC decoder, frames -> PCM
//
-struct clip_graph_drama3_gen : clip_graph {
- clip_graph_drama3_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, clip_gen_state * gen_state,
+struct mtmd_graph_drama3_gen : mtmd_graph {
+ mtmd_graph_drama3_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, mtmd_subctx_gen_state * gen_state,
int n_frames, int top_k, float top_p, float temp)
- : clip_graph(ctx, img), gen_process(gen_process), gen_state(gen_state), n_frames(n_frames), top_k(top_k), top_p(top_p), temp(temp) {}
+ : mtmd_graph(ctx, img), gen_process(gen_process), gen_state(gen_state), n_frames(n_frames), top_k(top_k), top_p(top_p), temp(temp) {}
ggml_cgraph * build() override;
// tower KV rows, shared by the prompt and the generated frames
static constexpr int n_pos_max = 4096;
- clip_gen_process_type gen_process;
- clip_gen_state * gen_state; // nullptr on buffer reserve
+ mtmd_subctx_gen_process gen_process;
+ mtmd_subctx_gen_state * gen_state; // nullptr on buffer reserve
int n_frames; // GEN_WAV only
int top_k;
float top_p;
@@ -532,10 +532,10 @@ struct clip_graph_drama3_gen : clip_graph {
ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos);
};
-struct clip_graph_soprano : clip_graph {
- clip_graph_soprano(clip_ctx * ctx, const clip_image_f32 & img, int n_frames) : clip_graph(ctx, img), n_frames(n_frames) {}
+struct mtmd_graph_soprano : mtmd_graph {
+ mtmd_graph_soprano(mtmd_subcontext * ctx, const mtmd_image_f32 & img, int n_frames) : mtmd_graph(ctx, img), n_frames(n_frames) {}
ggml_cgraph * build() override;
- static void decode_spectrum(const clip_model & model, const std::vector<float> & spectrum, std::vector<float> & pcm);
+ static void decode_spectrum(const mtmd_model & model, const std::vector<float> & spectrum, std::vector<float> & pcm);
// SopranoDecoder default
static constexpr int upscale = 4;
int n_frames;
@@ -546,54 +546,54 @@ struct clip_graph_soprano : clip_graph {
// GEN_CODE = one MaskGIT pass of the depthformer, or only the feedback embedding once all codes are known
// GEN_WAV = liquid-codec decoder, a window of frames -> polar STFT frames
//
-struct clip_graph_lfm3a_gen : clip_graph {
- clip_graph_lfm3a_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, bool all_known, int n_frames)
- : clip_graph(ctx, img), gen_process(gen_process), all_known(all_known), n_frames(n_frames) {}
+struct mtmd_graph_lfm3a_gen : mtmd_graph {
+ mtmd_graph_lfm3a_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, bool all_known, int n_frames)
+ : mtmd_graph(ctx, img), gen_process(gen_process), all_known(all_known), n_frames(n_frames) {}
ggml_cgraph * build() override;
- clip_gen_process_type gen_process;
+ mtmd_subctx_gen_process gen_process;
bool all_known; // GEN_CODE only
int n_frames; // GEN_WAV only
ggml_tensor * build_gen_code();
ggml_tensor * build_gen_wav();
- ggml_tensor * build_lfm_layer(ggml_tensor * inpL, const clip_layer & layer, float eps, int il,
+ ggml_tensor * build_lfm_layer(ggml_tensor * inpL, const mtmd_layer & layer, float eps, int il,
const std::function<ggml_tensor *(ggml_tensor *)> & op);
- ggml_tensor * build_lfm_attn(ggml_tensor * cur, const clip_layer & layer, int n_head, int n_head_kv,
+ ggml_tensor * build_lfm_attn(ggml_tensor * cur, const mtmd_layer & layer, int n_head, int n_head_kv,
float eps, float rope_theta, int il, ggml_tensor * inp_pos,
ggml_tensor * kq_mask, ggml_tensor * k_cache, ggml_tensor * v_cache,
ggml_tensor ** k_all, ggml_tensor ** v_all);
- ggml_tensor * build_lfm_conv(ggml_tensor * cur, const clip_layer & layer, int il,
+ ggml_tensor * build_lfm_conv(ggml_tensor * cur, const mtmd_layer & layer, int il,
ggml_tensor * conv_state, ggml_tensor ** conv_state_out);
};
// state slots of the streaming codec decoder (lfm3a_gen GEN_WAV)
-std::vector<c2w_state_slot> list_lfm3a_gen_state_slots(const clip_hparams & hparams, const clip_model & model);
+std::vector<c2w_state_slot> list_lfm3a_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
// state slots of the streaming LFM3-Audio encoder (lfm3a)
-std::vector<c2w_state_slot> list_lfm3a_state_slots(const clip_hparams & hparams, const clip_model & model);
+std::vector<c2w_state_slot> list_lfm3a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
-struct clip_graph_kimik25 : clip_graph {
- clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_kimik25 : mtmd_graph {
+ mtmd_graph_kimik25(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode);
};
-struct clip_graph_parakeet : clip_graph {
- clip_graph_parakeet(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_parakeet : mtmd_graph {
+ mtmd_graph_parakeet(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_exaone4_5 : clip_graph {
- clip_graph_exaone4_5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_exaone4_5 : mtmd_graph {
+ mtmd_graph_exaone4_5(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
-struct clip_graph_granite4_vision : clip_graph {
- clip_graph_granite4_vision(clip_ctx * ctx, const clip_image_f32 & img)
- : clip_graph(ctx, img),
+struct mtmd_graph_granite4_vision : mtmd_graph {
+ mtmd_graph_granite4_vision(mtmd_subcontext * ctx, const mtmd_image_f32 & img)
+ : mtmd_graph(ctx, img),
anyres(img.anyres),
n_tiles(img.ny() / img.nx()),
tile_side(img.nx() / patch_size) {}
@@ -602,7 +602,7 @@ struct clip_graph_granite4_vision : clip_graph {
private:
// the input image is a stack of tiles on the Y axis: [overview, tile(0,0), tile(0,1), ...]
- const clip_image_f32::anyres_info anyres;
+ const mtmd_image_f32::anyres_info anyres;
const int n_tiles;
const int tile_side; // patches per tile side
@@ -617,7 +617,7 @@ private:
ggml_tensor * build_anyres_assembly(ggml_tensor * cur, int out_side);
};
-struct clip_graph_muse_glimmer : clip_graph {
- clip_graph_muse_glimmer(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct mtmd_graph_muse_glimmer : mtmd_graph {
+ mtmd_graph_muse_glimmer(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
diff --git a/tools/mtmd/models/muse-glimmer.cpp b/tools/mtmd/models/muse-glimmer.cpp
index a397c1b22..869954f99 100644
--- a/tools/mtmd/models/muse-glimmer.cpp
+++ b/tools/mtmd/models/muse-glimmer.cpp
@@ -5,13 +5,13 @@
// adapter MLP + LLM's vision_projection.
//
// Several quantities are precomputed on host and fed as named graph inputs (filled in
-// clip.cpp set_input, PROJECTOR_TYPE_MUSE_GLIMMER branch):
+// mtmd-subcontext.cpp set_input, PROJECTOR_TYPE_MUSE_GLIMMER branch):
// muse_glimmer_pos_w/_h [n_tok] i32 : 1-indexed RoPE positions (sparse-permuted order)
// muse_glimmer_sp_perm [n_tok] i32 : window grouping permutation (applied after ln_pre)
// muse_glimmer_inv_perm [n_tok] i32 : inverse of sp_perm (applied after blocks)
// muse_glimmer_ds_perm [n_tok] i32 : pixel-shuffle gather (original order)
// attn_mask [n_tok, n_tok] f32 (f16 with flash attn) : block-diagonal window mask (sparse layers)
-ggml_cgraph * clip_graph_muse_glimmer::build() {
+ggml_cgraph * mtmd_graph_muse_glimmer::build() {
const int ds = hparams.n_merge; // downsample factor (2)
const int sf = hparams.muse_glimmer_sparse_factor; // 4
const int n_tok = n_patches;
@@ -50,7 +50,7 @@ ggml_cgraph * clip_graph_muse_glimmer::build() {
}
// 2D RoPE: first half of head_dim uses width pos, second half uses height pos
- auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) {
return build_rope_2d(ctx0, cur, pos_w, pos_h, rope_base, false);
};
diff --git a/tools/mtmd/models/muse-spark-audio.cpp b/tools/mtmd/models/muse-spark-audio.cpp
index d5340d5f9..20e770ef8 100644
--- a/tools/mtmd/models/muse-spark-audio.cpp
+++ b/tools/mtmd/models/muse-spark-audio.cpp
@@ -2,7 +2,7 @@
// Muse Spark audio tower: a causal conv pyramid over the raw waveform, no attention and no spectrogram
// each stage folds `patch` samples into the channel dim, projects, then runs `depth` residual blocks (depthwise causal conv, RMS norm, GELU MLP)
-ggml_cgraph * clip_graph_muse_spark_audio::build() {
+ggml_cgraph * mtmd_graph_muse_spark_audio::build() {
const int kernel = hparams.audio_conv_kernel_size;
const int n_stage = (int) hparams.audio_patch_sizes.size();
diff --git a/tools/mtmd/models/nemotron-omni.cpp b/tools/mtmd/models/nemotron-omni.cpp
index c4b1bff40..fc0f2b2ce 100644
--- a/tools/mtmd/models/nemotron-omni.cpp
+++ b/tools/mtmd/models/nemotron-omni.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_tensor * clip_graph_nemotron_omni::build_inp_with_temporal_merge() {
+ggml_tensor * mtmd_graph_nemotron_omni::build_inp_with_temporal_merge() {
if (n_batch == 1 && !img_batch->is_video) {
return build_inp();
}
@@ -22,7 +22,7 @@ ggml_tensor * clip_graph_nemotron_omni::build_inp_with_temporal_merge() {
return ggml_cont(ctx0, ggml_transpose(ctx0, inp));
}
-ggml_cgraph * clip_graph_nemotron_omni::build() {
+ggml_cgraph * mtmd_graph_nemotron_omni::build() {
GGML_ASSERT(model.class_embedding != nullptr);
GGML_ASSERT(model.position_embeddings != nullptr);
diff --git a/tools/mtmd/models/nemotron-transcribe.cpp b/tools/mtmd/models/nemotron-transcribe.cpp
index 2ce20c81f..b8fe37749 100644
--- a/tools/mtmd/models/nemotron-transcribe.cpp
+++ b/tools/mtmd/models/nemotron-transcribe.cpp
@@ -27,7 +27,7 @@ static constexpr float SPK_WEAK_BOOST = 2.0f; // _boost_topk_scores, scale_fac
static constexpr float SPK_SPEECH_THR = 0.5f; // _disable_low_scores: a speaker's frames at or below it are not cached
// the speaker expert has its own layers, width and head count, which build_vit does not take; full rotary
-ggml_tensor * clip_graph_nemotron_transcribe::build_spk_encoder(ggml_tensor * cur, ggml_tensor * pos) {
+ggml_tensor * mtmd_graph_nemotron_transcribe::build_spk_encoder(ggml_tensor * cur, ggml_tensor * pos) {
const int n_head = hparams.spk_n_head;
const int64_t n_embd = cur->ne[0];
const int64_t n_pos = cur->ne[1];
@@ -57,7 +57,7 @@ ggml_tensor * clip_graph_nemotron_transcribe::build_spk_encoder(ggml_tensor * cu
return build_norm(cur, model.spk_post_ln_w, model.spk_post_ln_b, NORM_TYPE_NORMAL, eps, -1);
}
-ggml_cgraph * clip_graph_nemotron_transcribe::build() {
+ggml_cgraph * mtmd_graph_nemotron_transcribe::build() {
const int sub = hparams.subsampling_factor;
const int64_t n_mel = hparams.n_mel_bins;
const int64_t n_mel_frames = img.nx();
@@ -90,7 +90,7 @@ ggml_cgraph * clip_graph_nemotron_transcribe::build() {
}
ggml_tensor * asr_pos = ggml_view_1d(ctx0, pos, n_pos, 0);
- auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) {
return ggml_rope_ext(ctx0, cur, asr_pos, nullptr, hparams.audio_rope_dims, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f,
0.0f, 1.0f, 0.0f, 0.0f);
};
@@ -231,10 +231,10 @@ struct spk_stream {
};
// every update has a full window, which overflows the FIFO and the cache (NeMo's update period of 300 never applies)
-static_assert(clip_graph_nemotron_transcribe::window_core - SPK_FIFO_LEN > SPK_CACHE_LEN);
+static_assert(mtmd_graph_nemotron_transcribe::window_core - SPK_FIFO_LEN > SPK_CACHE_LEN);
// NeMo SortformerModules.streaming_update: core rows join the FIFO, its overflow moves to the cache, which is compressed
-static void update_spk_state(spk_stream & ss, const clip_enc_state & st, const clip_model & model, int64_t core_begin,
+static void update_spk_state(spk_stream & ss, const mtmd_subctx_enc_state & st, const mtmd_model & model, int64_t core_begin,
int64_t n_core) {
const int64_t d = model.hparams.spk_n_embd;
const int64_t n_spk = model.spk_head_fc2_w->ne[1];
@@ -257,13 +257,13 @@ static void update_spk_state(spk_stream & ss, const clip_enc_state & st, const c
}
// NeMo ParallelExpertEncoder._forward_online: windows with left/right context, one graph each, the speaker state carried
-bool clip_graph_nemotron_transcribe::encode_windows(clip_ctx * ctx, const clip_model & model, clip_encode_params * params) {
+bool mtmd_graph_nemotron_transcribe::encode_windows(mtmd_subcontext * ctx, const mtmd_model & model, mtmd_subctx_encode_params * params) {
GGML_ASSERT(params->imgs->entries.size() == 1 && params->out_embd != nullptr);
- const clip_image_f32 & rec = params->imgs->entries[0];
+ const mtmd_image_f32 & rec = params->imgs->entries[0];
const auto & buf = rec.get_ro_buf();
const int64_t n_frames = rec.nx();
const int64_t n_rows = rec.ny();
- const int64_t n_embd = clip_n_mmproj_embd(ctx);
+ const int64_t n_embd = ctx->model.n_mmproj_embd();
const int sub = model.hparams.subsampling_factor;
const int64_t core = (int64_t) window_core * sub;
const int64_t left = (int64_t) window_left * sub;
@@ -272,7 +272,7 @@ bool clip_graph_nemotron_transcribe::encode_windows(clip_ctx * ctx, const clip_m
GGML_ASSERT((int64_t) buf.size() == n_frames * n_rows);
auto & out = *params->out_embd;
- GGML_ASSERT(out.size() == (size_t) n_embd * clip_n_output_tokens(ctx, &rec));
+ GGML_ASSERT(out.size() == (size_t) n_embd * ctx->model.n_output_tokens(&rec));
// the ASR expert's input: per-feature normalization over the whole recording (NeMo normalize_batch, per_feature)
std::vector<double> mean(n_rows), denom(n_rows);
@@ -292,7 +292,7 @@ bool clip_graph_nemotron_transcribe::encode_windows(clip_ctx * ctx, const clip_m
}
spk_stream ss;
- clip_enc_state st;
+ mtmd_subctx_enc_state st;
size_t n_out = 0;
for (int64_t w = 0; w < n_win; w++) {
const int64_t core_begin = w * core;
@@ -310,18 +310,18 @@ bool clip_graph_nemotron_transcribe::encode_windows(clip_ctx * ctx, const clip_m
st.mel_norm[r * win_len + t] = (float) ((src[t] - mean[r]) / denom[r]);
}
}
- clip_image_f32_batch batch;
+ mtmd_image_f32_batch batch;
batch.is_audio = true;
batch.entries.emplace_back();
batch.entries[0].set_size({(int) win_len, (int) n_rows}, false, true);
batch.entries[0].cpy_buf(data);
- std::vector<float> embd((size_t) n_embd * clip_n_output_tokens(ctx, &batch.entries[0]));
- clip_encode_params wp = *params;
+ std::vector<float> embd((size_t) n_embd * ctx->model.n_output_tokens(&batch.entries[0]));
+ mtmd_subctx_encode_params wp = *params;
wp.imgs = &batch;
wp.out_embd = &embd;
wp.enc_state = &st;
- if (!clip_encode(ctx, &wp)) {
+ if (!ctx->encode(&wp)) {
return false;
}
diff --git a/tools/mtmd/models/nemotron-v2-vl.cpp b/tools/mtmd/models/nemotron-v2-vl.cpp
index 03094be1b..6e644ea97 100644
--- a/tools/mtmd/models/nemotron-v2-vl.cpp
+++ b/tools/mtmd/models/nemotron-v2-vl.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_nemotron_v2_vl::build() {
+ggml_cgraph * mtmd_graph_nemotron_v2_vl::build() {
GGML_ASSERT(model.class_embedding != nullptr);
GGML_ASSERT(model.position_embeddings != nullptr);
diff --git a/tools/mtmd/models/paddleocr.cpp b/tools/mtmd/models/paddleocr.cpp
index 0e6b3f4b6..1ad186935 100644
--- a/tools/mtmd/models/paddleocr.cpp
+++ b/tools/mtmd/models/paddleocr.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_paddleocr::build() {
+ggml_cgraph * mtmd_graph_paddleocr::build() {
GGML_ASSERT(img.nx() % (patch_size * model.hparams.n_merge) == 0);
GGML_ASSERT(img.ny() % (patch_size * model.hparams.n_merge) == 0);
@@ -13,7 +13,7 @@ ggml_cgraph * clip_graph_paddleocr::build() {
ggml_set_name(positions, "positions");
ggml_set_input(positions);
- auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) {
return ggml_rope_multi(
ctx0, cur, positions, nullptr,
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
diff --git a/tools/mtmd/models/parakeet.cpp b/tools/mtmd/models/parakeet.cpp
index a6d7f0739..4a2377b4c 100644
--- a/tools/mtmd/models/parakeet.cpp
+++ b/tools/mtmd/models/parakeet.cpp
@@ -4,7 +4,7 @@ static constexpr int PARAKEET_LOCAL_ATTN_THRESHOLD = 8192;
static constexpr int PARAKEET_LOCAL_ATTN_WINDOW = 128;
// conv subsampling + conformer encoder
-ggml_cgraph * clip_graph_parakeet::build() {
+ggml_cgraph * mtmd_graph_parakeet::build() {
// Conv subsampling
ggml_tensor * inp = build_inp_raw(1);
diff --git a/tools/mtmd/models/pixtral.cpp b/tools/mtmd/models/pixtral.cpp
index edfae0825..6d3f01de0 100644
--- a/tools/mtmd/models/pixtral.cpp
+++ b/tools/mtmd/models/pixtral.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_pixtral::build() {
+ggml_cgraph * mtmd_graph_pixtral::build() {
const int n_merge = hparams.n_merge;
// 2D input positions
@@ -12,7 +12,7 @@ ggml_cgraph * clip_graph_pixtral::build() {
ggml_set_name(pos_w, "pos_w");
ggml_set_input(pos_w);
- auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) {
return build_rope_2d(ctx0, cur, pos_h, pos_w, hparams.rope_theta, true);
};
diff --git a/tools/mtmd/models/pockettts-gen.cpp b/tools/mtmd/models/pockettts-gen.cpp
index 662feff7f..c5f340656 100644
--- a/tools/mtmd/models/pockettts-gen.cpp
+++ b/tools/mtmd/models/pockettts-gen.cpp
@@ -9,13 +9,13 @@
//
// there is no codebook anywhere, "codes" in the mtmd API are continuous features here
-ggml_tensor * clip_graph_pockettts_gen::modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const {
+ggml_tensor * mtmd_graph_pockettts_gen::modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const {
ggml_tensor * cur = ggml_mul(ctx0, x, ggml_scale_bias(ctx0, scale, 1.0f, 1.0f));
return ggml_add(ctx0, cur, shift);
}
// see TimestepEmbedder in the reference
-ggml_tensor * clip_graph_pockettts_gen::time_embed(const clip_flow_net::time_embd & te, float t) const {
+ggml_tensor * mtmd_graph_pockettts_gen::time_embed(const mtmd_flow_net::time_embd & te, float t) const {
// t is a graph-build constant, so the cos/sin table can be folded into a scaled copy
ggml_tensor * args = ggml_scale(ctx0, te.freqs, t);
ggml_tensor * emb = ggml_concat(ctx0, ggml_cos(ctx0, args), ggml_sin(ctx0, args), 0);
@@ -42,7 +42,7 @@ ggml_tensor * clip_graph_pockettts_gen::time_embed(const clip_flow_net::time_emb
}
// one velocity evaluation: v(cond, s, t, x)
-ggml_tensor * clip_graph_pockettts_gen::flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const {
+ggml_tensor * mtmd_graph_pockettts_gen::flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const {
const auto & flow = model.flow;
ggml_tensor * cur = build_mm(flow.input_proj_w, x);
@@ -99,7 +99,7 @@ ggml_tensor * clip_graph_pockettts_gen::flow_forward(ggml_tensor * cond, ggml_te
// state carried between GEN_WAV calls: rope offset, per-layer KV window, conv left context
// and the transposed-conv overlap tails
-std::vector<c2w_state_slot> list_pockettts_state_slots(const clip_hparams & hparams, const clip_model & model) {
+std::vector<c2w_state_slot> list_pockettts_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
std::vector<c2w_state_slot> slots;
if (model.gen_upsample_w == nullptr) {
return slots; // not a pocket-tts decoder
@@ -136,8 +136,8 @@ std::vector<c2w_state_slot> list_pockettts_state_slots(const clip_hparams & hpar
return slots;
}
-ggml_cgraph * clip_graph_pockettts_gen::build() {
- if (gen_process == CLIP_GEN_PROCESS_GEN_CODE) {
+ggml_cgraph * mtmd_graph_pockettts_gen::build() {
+ if (gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE) {
// the backbone hidden state arrives as the single batch entry
ggml_tensor * h_state = build_inp_raw(1);
h_state = ggml_reshape_2d(ctx0, h_state, n_mmproj_embd, 1);
@@ -188,7 +188,7 @@ ggml_cgraph * clip_graph_pockettts_gen::build() {
cur = build_mm(model.gen_quant_out_w, cur);
cb(cur, "quant_out", -1);
- clip_graph_pockettts_seanet seanet(*this);
+ mtmd_graph_pockettts_seanet seanet(*this);
for (const auto & slot : list_pockettts_state_slots(hparams, model)) {
ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
ggml_set_name(t, ("state_in_" + slot.name).c_str());
@@ -225,7 +225,7 @@ ggml_cgraph * clip_graph_pockettts_gen::build() {
keep = ggml_mul(ctx0, keep,
ggml_step(ctx0, ggml_scale_bias(ctx0, ggml_add(ctx0, pos_k, base), 1.0f, 0.5f - (float) prefix)));
ggml_tensor * kq_mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), n_kv, n_pos, 1, 1);
- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
+ if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) {
kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F16);
}
diff --git a/tools/mtmd/models/pockettts-seanet.cpp b/tools/mtmd/models/pockettts-seanet.cpp
index c47207f56..4480aac9d 100644
--- a/tools/mtmd/models/pockettts-seanet.cpp
+++ b/tools/mtmd/models/pockettts-seanet.cpp
@@ -11,7 +11,7 @@ static int64_t div_ceil(int64_t a, int64_t b) {
// x: [T, IC], w: [K, IC, OC] -> [T / stride, OC]
// the convs are causal, so the whole K - stride padding goes on the left
-ggml_tensor * clip_graph_pockettts_seanet::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation,
+ggml_tensor * mtmd_graph_pockettts_seanet::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation,
bool pad_replicate, const std::string & state_name) const {
const int64_t k_size = (w->ne[0] - 1) * dilation + 1;
const int64_t p_total = k_size - stride;
@@ -48,7 +48,7 @@ ggml_tensor * clip_graph_pockettts_seanet::conv1d(ggml_tensor * x, ggml_tensor *
// x: [T, IC], w: [K, OC/groups, IC] -> [T * stride, OC]
// the K - stride overlap tail belongs to the next call: added to its head when streaming, else dropped
-ggml_tensor * clip_graph_pockettts_seanet::conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride,
+ggml_tensor * mtmd_graph_pockettts_seanet::conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride,
const std::string & state_name) const {
const int64_t K = w->ne[0];
const int64_t T = x->ne[0];
@@ -99,7 +99,7 @@ ggml_tensor * clip_graph_pockettts_seanet::conv_transpose1d(ggml_tensor * x, ggm
return out;
}
-ggml_tensor * clip_graph_pockettts_seanet::res_unit(ggml_tensor * x, const clip_seanet::stage & stage, int dilation,
+ggml_tensor * mtmd_graph_pockettts_seanet::res_unit(ggml_tensor * x, const mtmd_seanet::stage & stage, int dilation,
const std::string & state_prefix) const {
ggml_tensor * h = ggml_elu(ctx0, x);
h = conv1d(h, stage.res_conv1_w, stage.res_conv1_b, 1, dilation, false, state_prefix);
@@ -109,7 +109,7 @@ ggml_tensor * clip_graph_pockettts_seanet::res_unit(ggml_tensor * x, const clip_
return ggml_add(ctx0, x, h);
}
-ggml_tensor * clip_graph_pockettts_seanet::encode(ggml_tensor * x) const {
+ggml_tensor * mtmd_graph_pockettts_seanet::encode(ggml_tensor * x) const {
const auto & seanet = model.seanet;
ggml_tensor * cur = conv1d(x, seanet.conv_in_w, seanet.conv_in_b, 1, 1);
@@ -132,7 +132,7 @@ ggml_tensor * clip_graph_pockettts_seanet::encode(ggml_tensor * x) const {
return cur;
}
-ggml_tensor * clip_graph_pockettts_seanet::decode(ggml_tensor * x) const {
+ggml_tensor * mtmd_graph_pockettts_seanet::decode(ggml_tensor * x) const {
const auto & seanet = model.seanet;
const bool stream = !state_in.empty();
diff --git a/tools/mtmd/models/pockettts-spkenc.cpp b/tools/mtmd/models/pockettts-spkenc.cpp
index 280f2a97c..f01727ad5 100644
--- a/tools/mtmd/models/pockettts-spkenc.cpp
+++ b/tools/mtmd/models/pockettts-spkenc.cpp
@@ -4,7 +4,7 @@
// mimi encoder (SEANet + transformer + downsample), then flow_lm.speaker_proj_weight
// pre-norm block with layer scale on both residual paths, see mimi_transformer.py
-ggml_tensor * clip_graph_pockettts_spkenc::tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const {
+ggml_tensor * mtmd_graph_pockettts_spkenc::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const {
ggml_tensor * inp = cur;
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
@@ -36,12 +36,12 @@ ggml_tensor * clip_graph_pockettts_spkenc::tfm_layer_forward(ggml_tensor * cur,
return cur;
}
-ggml_cgraph * clip_graph_pockettts_spkenc::build() {
+ggml_cgraph * mtmd_graph_pockettts_spkenc::build() {
// the preprocessor hands over the waveform as a single-row "mel", already [n_samples, 1]
ggml_tensor * inp_raw = build_inp_raw(1);
ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]);
- clip_graph_pockettts_seanet seanet(*this);
+ mtmd_graph_pockettts_seanet seanet(*this);
cur = seanet.encode(cur);
cb(cur, "mimi_enc", -1);
diff --git a/tools/mtmd/models/qwen2vl.cpp b/tools/mtmd/models/qwen2vl.cpp
index 7516aa23a..8ecd59036 100644
--- a/tools/mtmd/models/qwen2vl.cpp
+++ b/tools/mtmd/models/qwen2vl.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_qwen2vl::build() {
+ggml_cgraph * mtmd_graph_qwen2vl::build() {
GGML_ASSERT(model.patch_bias == nullptr);
GGML_ASSERT(model.class_embedding == nullptr);
diff --git a/tools/mtmd/models/qwen3a.cpp b/tools/mtmd/models/qwen3a.cpp
index 4de96955d..0ed6b9a8f 100644
--- a/tools/mtmd/models/qwen3a.cpp
+++ b/tools/mtmd/models/qwen3a.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_qwen3a::build() {
+ggml_cgraph * mtmd_graph_qwen3a::build() {
// Ref implementation: https://github.com/QwenLM/Qwen3-ASR/blob/main/qwen_asr/core/transformers_backend/modeling_qwen3_asr.py
// inp_raw: [n_frames, n_mel, 1] (nx=n_frames, ny=n_mel)
diff --git a/tools/mtmd/models/qwen3tts-gen.cpp b/tools/mtmd/models/qwen3tts-gen.cpp
index ab619b237..59958fc4c 100644
--- a/tools/mtmd/models/qwen3tts-gen.cpp
+++ b/tools/mtmd/models/qwen3tts-gen.cpp
@@ -3,7 +3,7 @@
#include <string>
// on-device sampling: top-k, top-p, then a random draw
-ggml_tensor * clip_graph_qwen3tts_gen::code_gen::do_sampling(ggml_tensor * logits, ggml_tensor * inp_rand) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::do_sampling(ggml_tensor * logits, ggml_tensor * inp_rand) const {
logits = ggml_reshape_1d(ctx0, logits, ggml_nelements(logits));
const int64_t n_vocab = logits->ne[0];
@@ -68,7 +68,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::do_sampling(ggml_tensor * logit
}
// returns a new cache with row row_idx set to value
-ggml_tensor * clip_graph_qwen3tts_gen::code_gen::cache_set(ggml_tensor * cache, int row_idx, ggml_tensor * value) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::cache_set(ggml_tensor * cache, int row_idx, ggml_tensor * value) const {
const int64_t n_embd = cache->ne[0];
const int64_t n_cache = cache->ne[1];
GGML_ASSERT(row_idx >= 0 && row_idx < n_cache);
@@ -95,7 +95,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::cache_set(ggml_tensor * cache,
}
// builds a const i32 with no host upload: view a tensor, zero it via scale, add value, cast to i32
-ggml_tensor * clip_graph_qwen3tts_gen::code_gen::const_i32(ggml_tensor * anchor, float value) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::const_i32(ggml_tensor * anchor, float value) const {
ggml_tensor * v = ggml_view_1d(ctx0, anchor, 1, 0);
if (v->type != GGML_TYPE_F32) {
v = ggml_cast(ctx0, v, GGML_TYPE_F32);
@@ -104,20 +104,20 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::const_i32(ggml_tensor * anchor,
}
// causal keep-mask row for a query at position pos, window size n_kv_pad
-ggml_tensor * clip_graph_qwen3tts_gen::code_gen::causal_mask_row(int64_t n_kv_pad, int pos) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::causal_mask_row(int64_t n_kv_pad, int pos) const {
ggml_tensor * ones = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv_pad, n_kv_pad), 1.0f);
ggml_tensor * keep = ggml_tri(ctx0, ones, GGML_TRI_TYPE_LOWER_DIAG);
ggml_tensor * row = ggml_view_1d(ctx0, keep, n_kv_pad, (size_t) pos * keep->nb[1]);
ggml_tensor * mask = ggml_log(ctx0, row); // 0 = keep, -inf = masked
mask = ggml_reshape_4d(ctx0, mask, n_kv_pad, 1, 1, 1);
- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
+ if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) {
mask = ggml_cast(ctx0, mask, GGML_TYPE_F16);
}
return mask;
}
// talker hidden size -> predictor hidden size (small_to_mtp_projection)
-ggml_tensor * clip_graph_qwen3tts_gen::code_gen::project_in(ggml_tensor * cur) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::project_in(ggml_tensor * cur) const {
if (!model.gen_code_proj_in_w) {
return cur;
}
@@ -129,9 +129,9 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::project_in(ggml_tensor * cur) c
}
// one transformer layer at position pos; writes k/v into k_cache_layer/v_cache_layer at row pos
-ggml_tensor * clip_graph_qwen3tts_gen::code_gen::layer_forward(
+ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::layer_forward(
ggml_tensor * cur,
- const clip_layer & layer,
+ const mtmd_layer & layer,
ggml_tensor * inp_pos,
ggml_tensor * kq_mask,
ggml_tensor *& k_cache_layer,
@@ -192,7 +192,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::layer_forward(
// position 0: hidden bridge, seeds the k/v cache, no sampling
// position 1: embed(code0), sample with lm_head[0], write out_code_cache[1]
-void clip_graph_qwen3tts_gen::code_gen::prefill(
+void mtmd_graph_qwen3tts_gen::code_gen::prefill(
std::vector<ggml_tensor *> & k_cache,
std::vector<ggml_tensor *> & v_cache,
ggml_tensor *& out_code_cache,
@@ -237,7 +237,7 @@ void clip_graph_qwen3tts_gen::code_gen::prefill(
// - write new kv at cache row g+1, sample with lm_head[g]
// - write result to out_code_cache[g+1]
// step_idx must be in [1, n_acoustic - 1]
-ggml_tensor * clip_graph_qwen3tts_gen::code_gen::step(
+ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::step(
std::vector<ggml_tensor *> & k_cache,
std::vector<ggml_tensor *> & v_cache,
ggml_tensor * out_code_cache,
@@ -290,7 +290,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::step(
// causal conv1d, stride 1: prepend persisted left-context instead of zero-padding, then a plain conv
// x: [T, IC] (T-first). w: [K, IC, OC]. state_name empty means K == 1 (no left-context). returns [T, OC]
-ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation, const std::string & state_name) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::causal_conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation, const std::string & state_name) const {
const int K = (int) w->ne[0];
const int pad = (K - 1) * dilation;
@@ -314,7 +314,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d(ggml_tensor * x,
// causal depthwise conv1d, stride 1, dilation 1, kernel from w's shape.
// x: [T, C]. w: [K, 1, C]. returns [T, C]. see causal_conv1d for the state contract.
-ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const {
const int K = (int) w->ne[0];
const int pad = K - 1;
@@ -338,7 +338,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor *
// causal ConvTranspose1d, the (kernel - stride) overlap tail is kept as state for the next call
// x: [T, IC], w: [K, OC, IC]. state_name empty means K == stride (no overlap). returns [T * stride, OC]
-ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const {
const int K = (int) w->ne[0];
const int OC = (int) w->ne[1];
const int trim = K - stride;
@@ -372,7 +372,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_te
// SnakeBeta activation: y = x + sin(alpha*x)^2 * inv_beta (alpha/inv_beta folded via exp/reciprocal at conversion time)
// x: [T, C]. alpha/beta: [C], broadcasts over T
-ggml_tensor * clip_graph_qwen3tts_gen::code2wav::snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const {
ggml_tensor * a = ggml_reshape_2d(ctx0, alpha, 1, alpha->ne[0]);
ggml_tensor * b = ggml_reshape_2d(ctx0, beta, 1, beta->ne[0]);
@@ -388,7 +388,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::snake(ggml_tensor * x, ggml_ten
// RVQ codebook decode: T frames of 16 codes -> 512-dim hidden (C-first, [512, T])
// codebook 0 (semantic) and 1..15 (acoustic) sum within their group, project separately, then add
-ggml_tensor * clip_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_codes) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_codes) const {
const auto & c2w = model.c2w;
const int64_t T = inp_codes->ne[0];
@@ -418,7 +418,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_
// one pre_transformer layer over a batch of N = sliding_window new frames
// attention runs over [(W-1)-frame prefix from the last batch] + [N new frames]
// RoPE positions come from a persisted counter, so phases line up across batches
-ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il) const {
const int n_head = hparams.wav_tfm_n_head;
const int n_head_kv = hparams.wav_tfm_n_head_kv;
const int64_t d_head = layer.q_w->ne[1] / n_head;
@@ -485,7 +485,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor *
keep = ggml_mul(ctx0, keep, warm);
ggml_tensor * mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), total_kv, N, 1, 1); // 0 = keep, -inf = masked
- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
+ if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) {
mask = ggml_cast(ctx0, mask, GGML_TYPE_F16);
}
@@ -515,7 +515,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor *
// dwconv -> LayerNorm -> pwconv1 -> GELU -> pwconv2 -> layer scale -> residual
// x: [T, C] T-first; LayerNorm/pwconv need C on ne0, so this transposes in and back out
-ggml_tensor * clip_graph_qwen3tts_gen::code2wav::convnext_block(ggml_tensor * x, const clip_code2wav::upsample_block & blk, const std::string & state_prefix) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::convnext_block(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk, const std::string & state_prefix) const {
ggml_tensor * residual = x;
ggml_tensor * h = causal_conv1d_dw(x, blk.dwconv_w, blk.dwconv_b, state_prefix + "_dwconv"); // [T, C]
@@ -538,7 +538,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::convnext_block(ggml_tensor * x,
// SnakeBeta -> dilated causal conv (k=7) -> SnakeBeta -> pointwise causal conv (k=1) -> residual.
// x: [T, C]. returns [T, C].
-ggml_tensor * clip_graph_qwen3tts_gen::code2wav::dac_res_unit(ggml_tensor * x, const clip_code2wav::dac_res & res, int dilation, const std::string & state_name) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::dac_res_unit(ggml_tensor * x, const mtmd_code2wav::dac_res & res, int dilation, const std::string & state_name) const {
ggml_tensor * residual = x;
ggml_tensor * h = snake(x, res.act1_alpha, res.act1_beta);
h = causal_conv1d(h, res.conv1_w, res.conv1_b, dilation, state_name);
@@ -548,7 +548,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::dac_res_unit(ggml_tensor * x, c
}
// RVQ codes -> raw PCM for a batch of N = sliding_window frames
-ggml_tensor * clip_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes) const {
const auto & c2w = model.c2w;
// 1. quantizer decode: N frames of 16 codes -> [512, N] (C-first)
@@ -613,7 +613,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes)
// code2wav's persisted state buffers: RoPE position counter, K/V per pre_transformer layer,
// left-context/tail per stateful conv. shape lookup only, no graph needed
-std::vector<c2w_state_slot> list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model) {
+std::vector<c2w_state_slot> list_c2w_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
const auto & c2w = model.c2w;
std::vector<c2w_state_slot> slots;
@@ -663,20 +663,20 @@ std::vector<c2w_state_slot> list_c2w_state_slots(const clip_hparams & hparams, c
// both sub-graphs are always built, so the topology stays constant
// ggml_build_forward_select() then picks the one that actually runs
-ggml_cgraph * clip_graph_qwen3tts_gen::build() {
+ggml_cgraph * mtmd_graph_qwen3tts_gen::build() {
GGML_ASSERT(n_batch == 1); // this module only ever processes one frame at a time
int idx;
switch (gen_process) {
- case CLIP_GEN_PROCESS_GEN_CODE: idx = 0; break;
- case CLIP_GEN_PROCESS_GEN_WAV: idx = 1; break;
+ case MTMD_SUBCTX_GEN_PROCESS_GEN_CODE: idx = 0; break;
+ case MTMD_SUBCTX_GEN_PROCESS_GEN_WAV: idx = 1; break;
default: GGML_ABORT("unknown gen_process");
}
- // ---- CLIP_GEN_PROCESS_GEN_CODE: backbone hidden state -> 16 RVQ codes + next-step embd ----
+ // ---- MTMD_SUBCTX_GEN_PROCESS_GEN_CODE: backbone hidden state -> 16 RVQ codes + next-step embd ----
// not build_inp_raw(), a GEN_WAV call's `img` has no hidden-state data
ggml_tensor * h_state = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd);
- ggml_set_name(h_state, "inp_raw"); // must keep this exact name, clip_encode() sets it by name
+ ggml_set_name(h_state, "inp_raw"); // must keep this exact name, mtmd_subcontext::encode() sets it by name
ggml_set_input(h_state);
ggml_tensor * code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1);
@@ -740,7 +740,7 @@ ggml_cgraph * clip_graph_qwen3tts_gen::build() {
out_embd = ggml_reshape_2d(ctx0, out_embd, out_embd->ne[0], 1);
cb(out_embd, "gen_audio_out", -1);
- // ---- CLIP_GEN_PROCESS_GEN_WAV: 16 RVQ codes -> raw PCM ----
+ // ---- MTMD_SUBCTX_GEN_PROCESS_GEN_WAV: 16 RVQ codes -> raw PCM ----
const int n_frames = hparams.wav_tfm_swa; // frames per batch, == the attention window
ggml_tensor * inp_codes = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_frames, n_codes);
@@ -764,7 +764,7 @@ ggml_cgraph * clip_graph_qwen3tts_gen::build() {
ggml_set_output(slot.second);
}
- // out_embd goes last, clip_encode() reads it back via ggml_graph_node(gf, -1)
+ // out_embd goes last, mtmd_subcontext::encode() reads it back via ggml_graph_node(gf, -1)
ggml_tensor * outs[2];
outs[0] = out_codes; outs[1] = out_audio;
ggml_build_forward_select(gf, outs, 2, idx);
diff --git a/tools/mtmd/models/qwen3tts-spkenc.cpp b/tools/mtmd/models/qwen3tts-spkenc.cpp
index ac6cafd9a..1717eac6e 100644
--- a/tools/mtmd/models/qwen3tts-spkenc.cpp
+++ b/tools/mtmd/models/qwen3tts-spkenc.cpp
@@ -5,7 +5,7 @@ static constexpr int SPK_DILATIONS[3] = { 2, 3, 4 }; // enc_dilations[1..3]
// conv1d, kernel K, padding "same" (reflect), dilation d
// x: [C, T] (ne[0]=C, ne[1]=T) -> [out_c, T]
-ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const {
+ggml_tensor * mtmd_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const {
const int K = (int) w->ne[0];
const int IC = (int) w->ne[1];
const int OC = (int) w->ne[2];
@@ -36,7 +36,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tens
// Res2Net: split channel axis into `scale` chunks, chain dilated conv1d branches
// x: [C, T] -> [C, T]
-ggml_tensor * clip_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const {
+ggml_tensor * mtmd_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const {
const int64_t C = x->ne[0];
const int64_t T = x->ne[1];
const int64_t Cs = C / scale;
@@ -70,7 +70,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const clip_la
}
// squeeze-and-excitation gate. x: [C, T] -> [C, T]
-ggml_tensor * clip_graph_qwen3tts_spkenc::se_block(ggml_tensor * x, const clip_layer & layer) const {
+ggml_tensor * mtmd_graph_qwen3tts_spkenc::se_block(ggml_tensor * x, const mtmd_layer & layer) const {
// temporal mean, keepdim: transpose so T is on ne[0], reduce, transpose back
ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, C]
ggml_tensor * mean = ggml_mean(ctx0, x_t); // [1, C]
@@ -85,7 +85,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::se_block(ggml_tensor * x, const clip_l
}
// tdnn1 -> res2net -> tdnn2 -> se, plus residual. x: [C, T] -> [C, T]
-ggml_tensor * clip_graph_qwen3tts_spkenc::se_res2net_block(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const {
+ggml_tensor * mtmd_graph_qwen3tts_spkenc::se_res2net_block(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const {
ggml_tensor * residual = x;
ggml_tensor * h = conv1d_same(x, layer.conv_pw1_w, layer.conv_pw1_b, 1); // tdnn1
h = ggml_relu(ctx0, h);
@@ -97,7 +97,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::se_res2net_block(ggml_tensor * x, cons
}
// attentive statistics pooling. x: [C, T] -> [2*C, 1]
-ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) const {
+ggml_tensor * mtmd_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) const {
const int64_t T = x->ne[1];
// mean over T: [C, 1]
@@ -151,7 +151,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x)
return ggml_concat(ctx0, w_mean, w_std, 0); // [2C, 1]
}
-ggml_cgraph * clip_graph_qwen3tts_spkenc::build() {
+ggml_cgraph * mtmd_graph_qwen3tts_spkenc::build() {
// inp_raw: [T, n_mel, 1, 1], from mtmd_audio_preprocessor_qwen3tts_spk
ggml_tensor * inp = build_inp_raw(1);
inp = ggml_reshape_2d(ctx0, inp, inp->ne[0], inp->ne[1]);
diff --git a/tools/mtmd/models/qwen3vl.cpp b/tools/mtmd/models/qwen3vl.cpp
index 2e49dcd01..ada7a129d 100644
--- a/tools/mtmd/models/qwen3vl.cpp
+++ b/tools/mtmd/models/qwen3vl.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_qwen3vl::build() {
+ggml_cgraph * mtmd_graph_qwen3vl::build() {
GGML_ASSERT(model.patch_bias != nullptr);
GGML_ASSERT(model.position_embeddings != nullptr);
GGML_ASSERT(model.class_embedding == nullptr);
diff --git a/tools/mtmd/models/sensenova-u1.cpp b/tools/mtmd/models/sensenova-u1.cpp
index 57875c6ed..8e685d74c 100644
--- a/tools/mtmd/models/sensenova-u1.cpp
+++ b/tools/mtmd/models/sensenova-u1.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_sensenova_u1::build() {
+ggml_cgraph * mtmd_graph_sensenova_u1::build() {
GGML_ASSERT(model.patch_embeddings_0 != nullptr);
GGML_ASSERT(model.patch_bias != nullptr);
GGML_ASSERT(model.mm_0_w != nullptr);
diff --git a/tools/mtmd/models/siglip.cpp b/tools/mtmd/models/siglip.cpp
index 8cc901a28..84387f86d 100644
--- a/tools/mtmd/models/siglip.cpp
+++ b/tools/mtmd/models/siglip.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_siglip::build() {
+ggml_cgraph * mtmd_graph_siglip::build() {
ggml_tensor * inp = build_inp();
ggml_tensor * learned_pos_embd = model.position_embeddings;
diff --git a/tools/mtmd/models/soprano.cpp b/tools/mtmd/models/soprano.cpp
index bd16c6912..43f34f86f 100644
--- a/tools/mtmd/models/soprano.cpp
+++ b/tools/mtmd/models/soprano.cpp
@@ -2,7 +2,7 @@
#include "../mtmd-audio.h"
// Soprano Vocos: normalized Qwen3 hidden states -> STFT coefficients.
-ggml_cgraph * clip_graph_soprano::build() {
+ggml_cgraph * mtmd_graph_soprano::build() {
const auto & v = model.vocos;
const int n_inp = hparams.projection_dim;
const int n_up = upscale * (n_frames - 1) + 1;
@@ -59,7 +59,7 @@ ggml_cgraph * clip_graph_soprano::build() {
return gf;
}
-void clip_graph_soprano::decode_spectrum(const clip_model & model, const std::vector<float> & spectrum, std::vector<float> & pcm) {
+void mtmd_graph_soprano::decode_spectrum(const mtmd_model & model, const std::vector<float> & spectrum, std::vector<float> & pcm) {
const int n_frame_vals = model.vocos.output_w->ne[1];
const int n_fft = n_frame_vals - 2;
const int hop_length = model.hparams.audio_hop_len;
diff --git a/tools/mtmd/models/step3vl.cpp b/tools/mtmd/models/step3vl.cpp
index 5142b0bba..db0245a89 100644
--- a/tools/mtmd/models/step3vl.cpp
+++ b/tools/mtmd/models/step3vl.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_step3vl::build() {
+ggml_cgraph * mtmd_graph_step3vl::build() {
GGML_ASSERT(model.class_embedding == nullptr);
GGML_ASSERT(model.patch_embeddings_0 != nullptr);
GGML_ASSERT(model.position_embeddings != nullptr);
@@ -18,7 +18,7 @@ ggml_cgraph * clip_graph_step3vl::build() {
ggml_tensor * inp = build_inp();
ggml_tensor * learned_pos_embd = resize_position_embeddings();
- auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+ auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) {
return build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false);
};
diff --git a/tools/mtmd/models/voxtral-rt.cpp b/tools/mtmd/models/voxtral-rt.cpp
index 4a01512d2..3f768d6a2 100644
--- a/tools/mtmd/models/voxtral-rt.cpp
+++ b/tools/mtmd/models/voxtral-rt.cpp
@@ -2,7 +2,7 @@
// the input holds 2 context mel frames followed by the new frames, so the causal conv stem
// sees the same left context as on the full spectrogram
-ggml_cgraph * clip_graph_voxtral_rt::build() {
+ggml_cgraph * mtmd_graph_voxtral_rt::build() {
const int n_frames_in = img.nx();
const int n_new = (n_frames_in - 2) / 2; // encoder frames, the conv stem has stride 2
const int n_window = hparams.attn_window_size;
diff --git a/tools/mtmd/models/whisper-enc.cpp b/tools/mtmd/models/whisper-enc.cpp
index 49d5dd5ad..6225b707c 100644
--- a/tools/mtmd/models/whisper-enc.cpp
+++ b/tools/mtmd/models/whisper-enc.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_whisper_enc::build() {
+ggml_cgraph * mtmd_graph_whisper_enc::build() {
const int n_frames = img.nx();
const int n_pos = n_frames / 2;
GGML_ASSERT(model.position_embeddings->ne[1] >= n_pos);
diff --git a/tools/mtmd/models/yasa2.cpp b/tools/mtmd/models/yasa2.cpp
index e8cd3dacb..07fa84e06 100644
--- a/tools/mtmd/models/yasa2.cpp
+++ b/tools/mtmd/models/yasa2.cpp
@@ -25,7 +25,7 @@ static ggml_tensor * mul_channel_weight(
return ggml_mul(ctx0, x_whcb, w4);
}
-ggml_tensor * clip_graph_yasa2::layer_norm_channels(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b, float eps) {
+ggml_tensor * mtmd_graph_yasa2::layer_norm_channels(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b, float eps) {
// Match HF ConvNextLayerNorm(channels_first):
// u = mean_c(x), s = mean_c((x-u)^2), x = (x-u)/sqrt(s+eps)
// cast back to input dtype before affine.
@@ -48,7 +48,7 @@ ggml_tensor * clip_graph_yasa2::layer_norm_channels(ggml_tensor * inp, ggml_tens
return xhat;
}
-ggml_tensor * clip_graph_yasa2::convnext_grn(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b) {
+ggml_tensor * mtmd_graph_yasa2::convnext_grn(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b) {
// Exact ConvNeXtV2 GRN:
// Gx = ||x||_2 over spatial dims (W,H), Nx = Gx / (mean_c(Gx) + eps)
// y = w * (x * Nx) + b + x
@@ -78,7 +78,7 @@ ggml_tensor * clip_graph_yasa2::convnext_grn(ggml_tensor * inp, ggml_tensor * w,
return ggml_add(ctx0, inp, xnx);
}
-ggml_cgraph * clip_graph_yasa2::build() {
+ggml_cgraph * mtmd_graph_yasa2::build() {
ggml_tensor * cur = build_inp_raw();
// Patch embedding Conv2d(kernel=4, stride=4)
diff --git a/tools/mtmd/models/youtuvl.cpp b/tools/mtmd/models/youtuvl.cpp
index cda456c66..8cc714358 100644
--- a/tools/mtmd/models/youtuvl.cpp
+++ b/tools/mtmd/models/youtuvl.cpp
@@ -1,6 +1,6 @@
#include "models.h"
-ggml_cgraph * clip_graph_youtuvl::build() {
+ggml_cgraph * mtmd_graph_youtuvl::build() {
GGML_ASSERT(model.class_embedding == nullptr);
const int batch_size = 1;
const bool use_window_attn = !hparams.wa_layer_indexes.empty();
diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp
index 60087b16d..fcf94445a 100644
--- a/tools/mtmd/mtmd-audio.cpp
+++ b/tools/mtmd/mtmd-audio.cpp
@@ -1,4 +1,5 @@
#include "mtmd-audio.h"
+#include "mtmd-subcontext.h"
#include <cfloat>
#include <cmath>
@@ -269,12 +270,12 @@ static void ifft(const mtmd_audio_cache & cache, float * in, int N, float * out)
fft_impl<true, false>(cache, in, N, out);
}
-static void mtmd_audio_fill_entries(std::vector<mtmd_audio_mel> & mels, bool shape_only, std::vector<clip_image_f32> & entries) {
+static void mtmd_audio_fill_entries(std::vector<mtmd_audio_mel> & mels, bool shape_only, std::vector<mtmd_image_f32> & entries) {
entries.clear();
for (auto & mel : mels) {
GGML_ASSERT(mel.n_len <= INT32_MAX && mel.n_len >= 0);
GGML_ASSERT(mel.n_mel <= INT32_MAX && mel.n_mel >= 0);
- clip_image_f32 entry;
+ mtmd_image_f32 entry;
entry.set_size({(int) mel.n_len, (int) mel.n_mel}, shape_only || mel.data.empty(), /* is_audio */ true);
if (!entry.is_placeholder()) {
entry.cpy_buf(mel.data);
@@ -430,7 +431,7 @@ static bool log_mel_spectrogram(
}
out.data.resize(total_size);
- if (!shape_only && !clip_audio_log_mel(params, samples_padded.data(), hann, out.n_len, out.data.data())) {
+ if (!shape_only && !mtmd_audio_log_mel(params, samples_padded.data(), hann, out.n_len, out.data.data())) {
return false;
}
@@ -547,7 +548,7 @@ bool mtmd_audio_preprocessor_whisper::process(const float * samp
return false;
}
- // because the cgraph in clip.cpp only accepts 3000 frames each, we need to split the mel
+ // because the cgraph only accepts 3000 frames each, we need to split the mel
// we always expect the mel to have 3000 silent frames at the end
if (DEBUG) {
printf("output: n_mel = %d, n_len = %d\n", (int) out_full.n_mel, (int) out_full.n_len);
@@ -2002,10 +2003,10 @@ bool mtmd_audio_preprocessor_pockettts::process(const float * sa
}
// the mimi transformer mask is dense, so cost is quadratic in the reference length
- const int64_t max_samples = (int64_t) clip_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate;
+ const int64_t max_samples = (int64_t) mtmd_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate;
if ((int64_t) n_samples > max_samples) {
LOG_WRN("%s: speaker reference is %.1f s, truncating to the first %d s\n", __func__,
- (double) n_samples / hparams.audio_sample_rate, clip_hparams::pockettts_max_spk_seconds);
+ (double) n_samples / hparams.audio_sample_rate, mtmd_hparams::pockettts_max_spk_seconds);
n_samples = (size_t) max_samples;
}
diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h
index c0751b5e6..ba71913e0 100644
--- a/tools/mtmd/mtmd-audio.h
+++ b/tools/mtmd/mtmd-audio.h
@@ -2,8 +2,7 @@
#include "ggml.h"
#include "ggml-backend.h"
-#include "clip-model.h"
-#include "clip-impl.h"
+#include "mtmd-model.h"
#include <cstdint>
#include <vector>
@@ -77,7 +76,7 @@ struct mtmd_audio_mel_config {
};
struct mtmd_audio_preproc_out {
- std::vector<clip_image_f32> entries; // one per chunk, empty on failure
+ std::vector<mtmd_image_f32> entries; // one per chunk, empty on failure
std::vector<float> samples; // input of compute(), empty if entries already hold the data
bool needs_compute() const {
@@ -86,13 +85,12 @@ struct mtmd_audio_preproc_out {
};
struct mtmd_audio_preprocessor {
- const clip_hparams & hparams;
+ const mtmd_hparams & hparams;
mtmd_audio_mel_config mel_config; // filled by initialize() of the preprocessors that use the log-mel graph
- mtmd_audio_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {
- mel_config.sched = clip_get_sched(ctx);
- }
- mtmd_audio_preprocessor(const clip_hparams & hparams, ggml_backend_sched_t sched): hparams(hparams) {
+ // mel_config.sched must be set before the first compute(), see mtmd_context::init_audio()
+ mtmd_audio_preprocessor(const mtmd_model * model): hparams(model->hparams) {}
+ mtmd_audio_preprocessor(const mtmd_hparams & hparams, ggml_backend_sched_t sched): hparams(hparams) {
mel_config.sched = sched;
}
@@ -115,7 +113,7 @@ struct mtmd_audio_preprocessor {
};
struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
- mtmd_audio_preprocessor_whisper(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
+ mtmd_audio_preprocessor_whisper(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
@@ -124,14 +122,14 @@ struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
};
struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor {
- mtmd_audio_preprocessor_muse_spark_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
+ mtmd_audio_preprocessor_muse_spark_audio(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override {}
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
- mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
- mtmd_audio_preprocessor_conformer(const clip_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {}
+ mtmd_audio_preprocessor_conformer(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
+ mtmd_audio_preprocessor_conformer(const mtmd_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
@@ -146,7 +144,7 @@ struct mtmd_audio_preprocessor_d1omni : mtmd_audio_preprocessor_conformer {
};
struct mtmd_audio_preprocessor_lfm3a : mtmd_audio_preprocessor {
- mtmd_audio_preprocessor_lfm3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
+ mtmd_audio_preprocessor_lfm3a(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
@@ -155,7 +153,7 @@ struct mtmd_audio_preprocessor_lfm3a : mtmd_audio_preprocessor {
};
struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor {
- mtmd_audio_preprocessor_granite_speech(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
+ mtmd_audio_preprocessor_granite_speech(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
@@ -164,7 +162,7 @@ struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor {
};
struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor {
- mtmd_audio_preprocessor_gemma4a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
+ mtmd_audio_preprocessor_gemma4a(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
@@ -173,13 +171,13 @@ struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor {
};
struct mtmd_audio_preprocessor_gemma4ua : mtmd_audio_preprocessor {
- mtmd_audio_preprocessor_gemma4ua(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
+ mtmd_audio_preprocessor_gemma4ua(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor {
- mtmd_audio_preprocessor_qwen3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
+ mtmd_audio_preprocessor_qwen3a(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
@@ -188,7 +186,7 @@ struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor {
};
struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
- mtmd_audio_preprocessor_dots3note(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
+ mtmd_audio_preprocessor_dots3note(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
@@ -197,7 +195,7 @@ struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
};
struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
- mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
+ mtmd_audio_preprocessor_mimo_audio(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
@@ -206,7 +204,7 @@ struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
};
struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor {
- mtmd_audio_preprocessor_qwen3tts_spk(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
+ mtmd_audio_preprocessor_qwen3tts_spk(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
@@ -216,7 +214,7 @@ struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor {
// mimi convolves the waveform directly, so this only pads it to a whole number of frames
struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor {
- mtmd_audio_preprocessor_pockettts(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
+ mtmd_audio_preprocessor_pockettts(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override {}
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
@@ -229,7 +227,7 @@ struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor {
// 4: voice encoder power mel [n_frames x 40] (frame major, trimmed, partial grid)
// turbo loudness-normalizes the clip to -27 LUFS and caps the t3 clip at 15 s instead of 6 s
struct mtmd_audio_preprocessor_chatterbox_ref : mtmd_audio_preprocessor {
- mtmd_audio_preprocessor_chatterbox_ref(const clip_ctx * ctx, bool is_mtl) : mtmd_audio_preprocessor(ctx), is_mtl(is_mtl) {}
+ mtmd_audio_preprocessor_chatterbox_ref(const mtmd_model * model, bool is_mtl) : mtmd_audio_preprocessor(model), is_mtl(is_mtl) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
@@ -242,7 +240,7 @@ struct mtmd_audio_preprocessor_chatterbox_ref : mtmd_audio_preprocessor {
};
struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor {
- mtmd_audio_preprocessor_parakeet(clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) { }
+ mtmd_audio_preprocessor_parakeet(const mtmd_model * model) : mtmd_audio_preprocessor(model) { }
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
@@ -255,7 +253,7 @@ struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor {
// one chunk per recording: the raw log-mel of its valid frames
struct mtmd_audio_preprocessor_nemotron_transcribe : mtmd_audio_preprocessor_parakeet {
- mtmd_audio_preprocessor_nemotron_transcribe(clip_ctx * ctx) : mtmd_audio_preprocessor_parakeet(ctx) { }
+ mtmd_audio_preprocessor_nemotron_transcribe(const mtmd_model * model) : mtmd_audio_preprocessor_parakeet(model) { }
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp
index ba156f1c5..4534527fc 100644
--- a/tools/mtmd/mtmd-cli.cpp
+++ b/tools/mtmd/mtmd-cli.cpp
@@ -153,7 +153,7 @@ struct mtmd_cli_context {
}
void init_vision_context(common_params & params) {
- const char * clip_path = params.mmproj.path.c_str();
+ const char * mmproj_path = params.mmproj.path.c_str();
mtmd_context_params mparams = mtmd_context_params_default();
mparams.use_gpu = params.mmproj_use_gpu;
mparams.device = params.mmproj_device;
@@ -166,7 +166,7 @@ struct mtmd_cli_context {
{
// non-causal models need the whole image in one ubatch
const int n_ubatch = llama_n_ubatch(lctx);
- auto mem = mtmd_get_memory_usage(clip_path, mparams);
+ auto mem = mtmd_get_memory_usage(mmproj_path, mparams);
if (mem.use_non_causal && mem.image_max_tokens > n_ubatch) {
LOG_WRN("%s: cap image_max_tokens (original=%d) to n_ubatch (%d) because model needs non-causal attention on image\n", __func__, mem.image_max_tokens, n_ubatch);
LOG_WRN("%s: increase n_ubatch (-ub) to increase vision token budget\n", __func__);
@@ -178,9 +178,9 @@ struct mtmd_cli_context {
mparams.cb_eval_user_data = &cb_data;
mparams.cb_eval = common_debug_cb_eval;
}
- ctx_vision.reset(mtmd_init_from_file(clip_path, model, mparams));
+ ctx_vision.reset(mtmd_init_from_file(mmproj_path, model, mparams));
if (!ctx_vision.get()) {
- LOG_ERR("Failed to load vision model from %s\n", clip_path);
+ LOG_ERR("Failed to load vision model from %s\n", mmproj_path);
exit(1);
}
diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp
new file mode 100644
index 000000000..3242bddbb
--- /dev/null
+++ b/tools/mtmd/mtmd-graph.cpp
@@ -0,0 +1,1064 @@
+#include "mtmd-graph.h"
+#include "models/models.h"
+
+#include "ggml.h"
+#include "ggml-cpp.h"
+#include "ggml-backend.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <stdexcept>
+#include <vector>
+
+//
+// mtmd_graph
+//
+
+mtmd_graph::mtmd_graph(mtmd_subcontext * ctx, const mtmd_image_f32 & img) :
+ model(ctx->model),
+ hparams(model.hparams),
+ proj_type(ctx->model.proj_type),
+ img(img),
+ patch_size(hparams.patch_size),
+ n_patches_x(img.nx() / patch_size),
+ n_patches_y(img.ny() / patch_size),
+ n_patches(n_patches_x * n_patches_y),
+ n_embd(hparams.n_embd),
+ n_head(hparams.n_head),
+ n_head_kv(hparams.n_head_kv),
+ d_head(hparams.n_embd_head > 0 ? hparams.n_embd_head : (n_head > 0 ? n_embd / n_head : 0)),
+ n_layer(hparams.n_layer),
+ n_mmproj_embd(ctx->model.n_mmproj_embd()),
+ eps(hparams.eps),
+ kq_scale(d_head > 0 ? 1.0f / sqrtf((float)d_head) : 0.0f),
+ flash_attn_type(ctx->flash_attn_type) {
+ struct ggml_init_params params = {
+ /*.mem_size =*/ ctx->buf_compute_meta.size(),
+ /*.mem_buffer =*/ ctx->buf_compute_meta.data(),
+ /*.no_alloc =*/ true,
+ };
+ ctx0_ptr.reset(ggml_init(params));
+ ctx0 = ctx0_ptr.get();
+ gf = ggml_new_graph_custom(ctx0, ctx->max_nodes, false);
+}
+
+mtmd_graph::mtmd_graph(const mtmd_graph & parent) :
+ model(parent.model),
+ hparams(parent.hparams),
+ proj_type(parent.proj_type),
+ img(parent.img),
+ patch_size(parent.patch_size),
+ n_patches_x(parent.n_patches_x),
+ n_patches_y(parent.n_patches_y),
+ n_patches(parent.n_patches),
+ n_embd(parent.n_embd),
+ n_head(parent.n_head),
+ n_head_kv(parent.n_head_kv),
+ d_head(parent.d_head),
+ n_layer(parent.n_layer),
+ n_mmproj_embd(parent.n_mmproj_embd),
+ eps(parent.eps),
+ kq_scale(parent.kq_scale),
+ flash_attn_type(parent.flash_attn_type) {
+ // reuse from parent
+ ctx0 = parent.ctx0;
+ gf = parent.gf;
+}
+
+ggml_tensor * mtmd_graph::build_mm(ggml_tensor * w, ggml_tensor * x) const {
+ return ggml_mul_mat(ctx0, w, x);
+}
+
+void mtmd_graph::cb(ggml_tensor * cur, const char * name, int il) const {
+ if (il >= 0) {
+ ggml_format_name(cur, "%s-%d", name, il);
+ } else {
+ ggml_set_name(cur, name);
+ }
+}
+
+// siglip2 naflex
+ggml_tensor * mtmd_graph::resize_position_embeddings(uint32_t interpolation_mode) {
+ ggml_tensor * pos_embd = model.position_embeddings;
+ const int height = img.ny() / patch_size;
+ const int width = img.nx() / patch_size;
+ const uint32_t mode = interpolation_mode;
+ const int n_per_side = (int)std::sqrt(pos_embd->ne[1]);
+
+ GGML_ASSERT(pos_embd);
+
+ if (height == n_per_side && width == n_per_side) {
+ return pos_embd;
+ }
+
+ pos_embd = ggml_reshape_3d(ctx0, pos_embd, n_embd, n_per_side, n_per_side); // -> (n_embd, n_per_side, n_per_side)
+ pos_embd = ggml_permute(ctx0, pos_embd, 2, 0, 1, 3); // -> (n_per_side, n_per_side, n_embd)
+ pos_embd = ggml_interpolate(ctx0, pos_embd, width, height, n_embd, 1, mode); // -> (width, height, n_embd)
+ pos_embd = ggml_permute(ctx0, pos_embd, 1, 2, 0, 3); // -> (n_embd, width, height)
+ pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, width * height); // -> (n_embd, width * height)
+
+ return pos_embd;
+}
+
+// build vision transformer (ViT) cgraph
+// this function should cover most of the models
+// if your model has specific features, you should probably duplicate this function
+ggml_tensor * mtmd_graph::build_vit(
+ ggml_tensor * inp,
+ int64_t n_pos,
+ norm_type norm_t,
+ ffn_op_type ffn_t,
+ ggml_tensor * learned_pos_embd,
+ std::function<ggml_tensor *(ggml_tensor *, const mtmd_layer &)> add_pos,
+ const build_vit_opts & opts
+ ) {
+ // batch dim: inp is [n_embd, n_pos, B]
+ const int64_t B = inp->ne[2];
+
+ if (learned_pos_embd) {
+ inp = ggml_add(ctx0, inp, learned_pos_embd);
+ cb(inp, "pos_embed", -1);
+ }
+
+ // flatten batch; unflatten again in attention
+ inp = ggml_reshape_2d(ctx0, inp, n_embd, n_pos * B);
+
+ ggml_tensor * inpL = inp;
+
+ // pre-layernorm
+ if (model.pre_ln_w) {
+ inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, norm_t, eps, -1);
+ cb(inpL, "pre_ln", -1);
+ }
+
+ // loop over layers
+ for (int il = 0; il < n_layer; il++) {
+ auto & layer = model.layers[il];
+ ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states
+
+ ggml_tensor * attn_mask = opts.attn_mask;
+ if (opts.attn_mask_layers.size() > (size_t) il) {
+ attn_mask = opts.attn_mask_layers[il];
+ }
+
+ // layernorm1
+ cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
+ cb(cur, "layer_inp_normed", il);
+
+ // self-attention
+ {
+ ggml_tensor * Qcur = nullptr;
+ ggml_tensor * Kcur = nullptr;
+ ggml_tensor * Vcur = nullptr;
+ if (layer.qkv_w != nullptr) {
+ // fused qkv
+ cur = build_mm(layer.qkv_w, cur);
+ if (layer.qkv_b != nullptr) {
+ cur = ggml_add(ctx0, cur, layer.qkv_b);
+ }
+
+ // Q/K/V as [d_head, n_head, n_pos, B], the batch stride is cur->nb[1]*n_pos.
+ Qcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B,
+ /* nb1 */ ggml_row_size(cur->type, d_head),
+ /* nb2 */ cur->nb[1],
+ /* nb3 */ cur->nb[1] * n_pos,
+ /* offset */ 0);
+
+ Kcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B,
+ /* nb1 */ ggml_row_size(cur->type, d_head),
+ /* nb2 */ cur->nb[1],
+ /* nb3 */ cur->nb[1] * n_pos,
+ /* offset */ ggml_row_size(cur->type, n_head * d_head));
+
+ Vcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B,
+ /* nb1 */ ggml_row_size(cur->type, d_head),
+ /* nb2 */ cur->nb[1],
+ /* nb3 */ cur->nb[1] * n_pos,
+ /* offset */ ggml_row_size(cur->type, 2 * n_head * d_head));
+
+ if (layer.q_norm) {
+ GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]);
+ Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il);
+ cb(Qcur, "Qcur_norm", il);
+ }
+
+ if (layer.k_norm) {
+ GGML_ASSERT(layer.k_norm->ne[0] == Kcur->ne[0]);
+ Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il);
+ cb(Kcur, "Kcur_norm", il);
+ }
+
+ } else {
+ // separate q, k, v
+ Qcur = build_mm(layer.q_w, cur);
+ if (layer.q_b) {
+ Qcur = ggml_add(ctx0, Qcur, layer.q_b);
+ }
+
+ Kcur = build_mm(layer.k_w, cur);
+ if (layer.k_b) {
+ Kcur = ggml_add(ctx0, Kcur, layer.k_b);
+ }
+
+ Vcur = build_mm(layer.v_w, cur);
+ if (layer.v_b) {
+ Vcur = ggml_add(ctx0, Vcur, layer.v_b);
+ }
+
+ // if true, norm must be applied after reshaping to (d_head, n_head, n_pos)
+ bool norm_per_head = layer.q_norm && layer.q_norm->ne[0] == d_head;
+
+ if (!norm_per_head) {
+ if (layer.q_norm) {
+ Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il);
+ cb(Qcur, "Qcur_norm", il);
+ }
+ if (layer.k_norm) {
+ Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il);
+ cb(Kcur, "Kcur_norm", il);
+ }
+ }
+
+ Qcur = ggml_reshape_4d(ctx0, Qcur, d_head, n_head, n_pos, B);
+ Kcur = ggml_reshape_4d(ctx0, Kcur, d_head, n_head_kv, n_pos, B);
+ Vcur = ggml_reshape_4d(ctx0, Vcur, d_head, n_head_kv, n_pos, B);
+
+ if (norm_per_head) {
+ if (layer.q_norm) {
+ Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il);
+ cb(Qcur, "Qcur_norm_per_head", il);
+ }
+ if (layer.k_norm) {
+ Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il);
+ cb(Kcur, "Kcur_norm_per_head", il);
+ }
+ }
+ }
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ if (add_pos) {
+ Qcur = add_pos(Qcur, layer);
+ Kcur = add_pos(Kcur, layer);
+ cb(Qcur, "Qcur_pos", il);
+ cb(Kcur, "Kcur_pos", il);
+ }
+
+ if (proj_type == PROJECTOR_TYPE_GEMMA4V) {
+ Vcur = ggml_rms_norm(ctx0, Vcur, eps);
+ cb(Vcur, "Vcur_normed", il);
+ }
+
+ // build_attn returns a flat 2D [n_embd, n_pos*B]
+ cur = build_attn(layer.o_w, layer.o_b,
+ Qcur, Kcur, Vcur, attn_mask, kq_scale, il);
+ cb(cur, "attn_out", il);
+ }
+
+ if (layer.ls_1_w) {
+ cur = ggml_mul(ctx0, cur, layer.ls_1_w);
+ cb(cur, "attn_out_scaled", il);
+ }
+
+ if (layer.attn_post_norm_w) {
+ cur = build_norm(cur, layer.attn_post_norm_w, nullptr, norm_t, eps, il);
+ cb(cur, "attn_post_normed", il);
+ }
+
+ // re-add the layer input, e.g., residual
+ cur = ggml_add(ctx0, cur, inpL);
+
+ inpL = cur; // inpL = residual, cur = hidden_states
+
+ if (opts.callback_layer_out) {
+ opts.callback_layer_out(cur, il);
+ }
+
+ cb(cur, "ffn_inp", il);
+
+ // layernorm2 (pre-ffn norm)
+ cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);
+ cb(cur, "ffn_inp_normed", il);
+
+ // ffn
+ cur = layer.ff_gate_exps_w
+ ? build_moe_ffn(cur, layer, ffn_t, il)
+ : build_ffn(cur,
+ layer.ff_up_w, layer.ff_up_b,
+ layer.ff_gate_w, layer.ff_gate_b,
+ layer.ff_down_w, layer.ff_down_b,
+ ffn_t, il);
+
+ cb(cur, "ffn_out", il);
+
+ if (layer.ff_post_norm_w) {
+ cur = build_norm(cur, layer.ff_post_norm_w, nullptr, norm_t, eps, il);
+ cb(cur, "ffn_post_normed", il);
+ }
+
+ if (layer.ls_2_w) {
+ cur = ggml_mul(ctx0, cur, layer.ls_2_w);
+ cb(cur, "ffn_out_scaled", il);
+ }
+
+ // residual 2
+ cur = ggml_add(ctx0, inpL, cur);
+ cb(cur, "layer_out", il);
+
+ if (layer.ls_out_w) {
+ cur = ggml_mul(ctx0, cur, layer.ls_out_w);
+ cb(cur, "layer_out_scaled", il);
+ }
+
+ inpL = cur;
+ }
+
+ if (model.audio_has_avgpool()) {
+ ggml_tensor * cur = inpL;
+ cur = ggml_transpose(ctx0, cur);
+ cur = ggml_cont(ctx0, cur);
+ cur = ggml_pool_1d(ctx0, cur, GGML_OP_POOL_AVG, 2, 2, 0);
+ cur = ggml_transpose(ctx0, cur);
+ cur = ggml_cont(ctx0, cur);
+ inpL = cur;
+ }
+
+ // post-layernorm
+ if (model.post_ln_w && !opts.skip_post_ln) {
+ inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, -1);
+ }
+
+ // restore the batch dim
+ GGML_ASSERT(inpL->ne[1] % B == 0);
+ inpL = ggml_reshape_3d(ctx0, inpL, n_embd, inpL->ne[1] / B, B);
+ return inpL;
+}
+
+// build the input after conv2d (inp_raw --> patches)
+// returns tensor with shape [n_embd, n_patches]
+ggml_tensor * mtmd_graph::build_inp() {
+ ggml_tensor * inp_raw = build_inp_raw();
+ ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
+ inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, n_batch);
+ inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
+ if (model.patch_bias) {
+ inp = ggml_add(ctx0, inp, model.patch_bias);
+ cb(inp, "patch_bias", -1);
+ }
+ return inp;
+}
+
+ggml_tensor * mtmd_graph::build_inp_raw(int channels) {
+ ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, img.nx(), img.ny(), channels, n_batch);
+ ggml_set_name(inp_raw, "inp_raw");
+ ggml_set_input(inp_raw);
+ return inp_raw;
+}
+
+// qwen-vl style temporal merge: Conv3D(kt=2) split into 2 Conv2D
+ggml_tensor * mtmd_graph::build_inp_temporal_merge() {
+ ggml_tensor * inp_raw = build_inp_raw();
+
+ GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
+ GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
+
+ const size_t nb1 = ggml_row_size(inp_raw->type, img.nx());
+ const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny());
+
+ if (n_batch == 1) {
+ // still image input
+ return ggml_add(ctx0,
+ ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1),
+ ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
+ } else if (n_batch == 2) {
+ // 2 frames input (video input)
+ ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw,
+ img.nx(), img.ny(), 3, nb1, nb2, 0);
+ ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw,
+ img.nx(), img.ny(), 3, nb1, nb2,
+ nb2 * 3); // move to the second frame
+ return ggml_add(ctx0,
+ ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1),
+ ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1));
+ } else {
+ GGML_ASSERT(false && "n_batch > 2 is not supported");
+ }
+}
+
+ggml_tensor * mtmd_graph::build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx) {
+ const ggml_type type = flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED ? GGML_TYPE_F16 : GGML_TYPE_F32;
+ ggml_tensor * mask = ggml_new_tensor_2d(ctx0, type, n_kv, n_q);
+ ggml_set_name(mask, get_attn_mask_name(idx).c_str());
+ ggml_set_input(mask);
+ return mask;
+}
+
+ggml_tensor * mtmd_graph::build_norm(
+ ggml_tensor * cur,
+ ggml_tensor * mw,
+ ggml_tensor * mb,
+ norm_type type,
+ float norm_eps,
+ int il) const {
+
+ cur = type == NORM_TYPE_RMS
+ ? ggml_rms_norm(ctx0, cur, norm_eps)
+ : ggml_norm(ctx0, cur, norm_eps);
+
+ if (mw) {
+ cur = ggml_mul(ctx0, cur, mw);
+ cb(cur, "norm_w", il);
+ }
+
+ if (mb) {
+ cur = ggml_add(ctx0, cur, mb);
+ cb(cur, "norm_b", il);
+ }
+
+ return cur;
+}
+
+ggml_tensor * mtmd_graph::build_ffn(
+ ggml_tensor * cur,
+ ggml_tensor * up,
+ ggml_tensor * up_b,
+ ggml_tensor * gate,
+ ggml_tensor * gate_b,
+ ggml_tensor * down,
+ ggml_tensor * down_b,
+ ffn_op_type type_op,
+ int il) const {
+
+ ggml_tensor * tmp = up ? build_mm(up, cur) : cur;
+ cb(tmp, "ffn_up", il);
+
+ if (up_b) {
+ tmp = ggml_add(ctx0, tmp, up_b);
+ cb(tmp, "ffn_up_b", il);
+ }
+
+ if (gate) {
+ cur = build_mm(gate, cur);
+ cb(cur, "ffn_gate", il);
+
+ if (gate_b) {
+ cur = ggml_add(ctx0, cur, gate_b);
+ cb(cur, "ffn_gate_b", il);
+ }
+ } else {
+ cur = tmp;
+ }
+
+ // we only support parallel ffn for now
+ switch (type_op) {
+ case FFN_SILU:
+ if (gate) {
+ cur = hparams.swiglu_clamp > 0.0f
+ ? ggml_swiglu_clamp(ctx0, cur, tmp, hparams.swiglu_clamp)
+ : ggml_swiglu_split(ctx0, cur, tmp);
+ cb(cur, "ffn_swiglu", il);
+ } else {
+ cur = ggml_silu(ctx0, cur);
+ cb(cur, "ffn_silu", il);
+ } break;
+ case FFN_GELU:
+ if (gate) {
+ cur = ggml_geglu_split(ctx0, cur, tmp);
+ cb(cur, "ffn_geglu", il);
+ } else {
+ cur = ggml_gelu(ctx0, cur);
+ cb(cur, "ffn_gelu", il);
+ } break;
+ case FFN_GELU_ERF:
+ if (gate) {
+ cur = ggml_geglu_erf_split(ctx0, cur, tmp);
+ cb(cur, "ffn_geglu_erf", il);
+ } else {
+ cur = ggml_gelu_erf(ctx0, cur);
+ cb(cur, "ffn_gelu_erf", il);
+ } break;
+ case FFN_GELU_QUICK:
+ if (gate) {
+ cur = ggml_geglu_quick_split(ctx0, cur, tmp);
+ cb(cur, "ffn_geglu_quick", il);
+ } else {
+ cur = ggml_gelu_quick(ctx0, cur);
+ cb(cur, "ffn_gelu_quick", il);
+ } break;
+ case FFN_RELU_SQR:
+ {
+ cur = ggml_relu(ctx0, cur);
+ cur = ggml_sqr(ctx0, cur);
+ cb(cur, "ffn_relu_sqr", il);
+ } break;
+ }
+
+ if (down) {
+ cur = build_mm(down, cur);
+ }
+
+ if (down_b) {
+ cb(cur, "ffn_down", il);
+ }
+
+ if (down_b) {
+ cur = ggml_add(ctx0, cur, down_b);
+ }
+
+ return cur;
+}
+
+// MoE FFN with sigmoid router and normalized top-k weights (dots3note vision)
+// the router runs in fp32; exp_probs_b only affects expert selection, not the weights
+ggml_tensor * mtmd_graph::build_moe_ffn(ggml_tensor * cur, const mtmd_layer & layer, ffn_op_type type_op, int il) const {
+ const int64_t n_tokens = cur->ne[1];
+ const int64_t n_expert = layer.ff_gate_exps_w->ne[2];
+ const int64_t n_expert_used = std::min((int64_t) hparams.n_expert_used, n_expert);
+ GGML_ASSERT(n_expert_used > 0);
+ GGML_ASSERT(type_op == FFN_SILU);
+
+ ggml_tensor * probs = ggml_sigmoid(ctx0, build_mm(layer.ff_gate_inp_w, cur)); // [n_expert, n_tokens]
+ cb(probs, "ffn_moe_probs", il);
+
+ ggml_tensor * sel = layer.ff_exp_probs_b
+ ? ggml_add(ctx0, probs, layer.ff_exp_probs_b)
+ : probs;
+ ggml_tensor * selected = ggml_top_k(ctx0, sel, n_expert_used); // [n_expert_used, n_tokens]
+
+ ggml_tensor * weights = ggml_get_rows(ctx0,
+ ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected);
+ weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens);
+ weights = ggml_div(ctx0, weights, ggml_sum_rows(ctx0, weights));
+ weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens);
+ cb(weights, "ffn_moe_weights", il);
+
+ cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], 1, n_tokens);
+ ggml_tensor * gate = ggml_mul_mat_id(ctx0, layer.ff_gate_exps_w, cur, selected); // [n_ff, n_expert_used, n_tokens]
+ ggml_tensor * up = ggml_mul_mat_id(ctx0, layer.ff_up_exps_w, cur, selected);
+ cur = ggml_mul(ctx0, ggml_silu(ctx0, gate), up);
+ cur = ggml_mul_mat_id(ctx0, layer.ff_down_exps_w, cur, selected); // [n_embd, n_expert_used, n_tokens]
+ cur = ggml_mul(ctx0, cur, weights);
+
+ // sum over the selected experts
+ ggml_tensor * out = nullptr;
+ for (int64_t i = 0; i < n_expert_used; i++) {
+ ggml_tensor * v = ggml_view_2d(ctx0, cur, cur->ne[0], n_tokens, cur->nb[2], i * cur->nb[1]);
+ out = out ? ggml_add(ctx0, out, v) : v;
+ }
+ if (n_expert_used == 1) {
+ out = ggml_cont(ctx0, out);
+ }
+ cb(out, "ffn_moe_out", il);
+ return out;
+}
+
+ggml_tensor * mtmd_graph::build_attn(
+ ggml_tensor * wo,
+ ggml_tensor * wo_b,
+ ggml_tensor * q_cur,
+ ggml_tensor * k_cur,
+ ggml_tensor * v_cur,
+ ggml_tensor * kq_mask,
+ float kq_scale,
+ int il,
+ ggml_tensor * sinks) const {
+ // these nodes are added to the graph together so that they are not reordered
+ // by doing so, the number of splits in the graph is reduced
+ // the order is fixed without the compute flag, so an unselected branch stays out of the compute set
+ ggml_build_forward_order(gf, q_cur);
+ ggml_build_forward_order(gf, k_cur);
+ ggml_build_forward_order(gf, v_cur);
+
+ ggml_tensor * q = ggml_permute(ctx0, q_cur, 0, 2, 1, 3);
+ //cb(q, "q", il);
+
+ ggml_tensor * k = ggml_permute(ctx0, k_cur, 0, 2, 1, 3);
+ //cb(k, "k", il);
+
+ ggml_tensor * cur;
+
+ if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) {
+ ggml_tensor * v = ggml_permute(ctx0, v_cur, 0, 2, 1, 3);
+
+ k = ggml_cast(ctx0, k, GGML_TYPE_F16);
+ v = ggml_cast(ctx0, v, GGML_TYPE_F16);
+ // mask must be f16 here, use build_inp_attn_mask()
+ GGML_ASSERT(!kq_mask || kq_mask->type == GGML_TYPE_F16);
+
+ cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f);
+ ggml_prec_set_acc(cur, GGML_PREC_F32);
+ if (sinks != nullptr) {
+ ggml_flash_attn_ext_add_sinks(cur, sinks);
+ }
+
+ cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]);
+
+ } else {
+ ggml_tensor * v = ggml_permute(ctx0, v_cur, 1, 2, 0, 3);
+ v = ggml_cont(ctx0, v);
+
+ ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
+ // TODO: this most likely requires GGML_PREC_BF16, but will enable it later to avoid scope-creep
+ // ggml_prec_set_acc(kq, GGML_PREC_BF16);
+
+ kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, 0.0f);
+ if (sinks != nullptr) {
+ ggml_soft_max_add_sinks(kq, sinks);
+ }
+
+ ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq);
+ cur = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
+ cur = ggml_cont_2d(ctx0, cur, cur->ne[0] * cur->ne[1], cur->ne[2] * cur->ne[3]);
+ }
+
+ cb(cur, "kqv_out", il);
+
+ if (wo) {
+ cur = build_mm(wo, cur);
+ }
+
+ if (wo_b) {
+ cur = ggml_add(ctx0, cur, wo_b);
+ }
+
+ return cur;
+}
+
+// implementation of the 2D RoPE without adding a new op in ggml
+ggml_tensor * mtmd_graph::build_rope_2d(
+ ggml_context * ctx0,
+ ggml_tensor * cur,
+ ggml_tensor * pos_a, // first half
+ ggml_tensor * pos_b, // second half
+ const float freq_base,
+ const bool interleave_freq
+) {
+ const int64_t n_dim = cur->ne[0];
+
+ // for example, if we have cur tensor of shape (n_dim=8, n_head, n_pos)
+ // we will have a list of 4 inv_freq: 1e-0, 1e-1, 1e-2, 1e-3
+ // first half of cur will use 1e-0, 1e-2 (even)
+ // second half of cur will use 1e-1, 1e-3 (odd)
+ // the trick here is to rotate just half of n_dim, so inv_freq will automatically be even
+ // ^ don't ask me why, it's math! -2(2i) / n_dim == -2i / (n_dim/2)
+ // then for the second half, we use freq_scale to shift the inv_freq
+ // ^ why? replace (2i) with (2i+1) in the above equation
+ const float freq_scale_odd = interleave_freq
+ ? std::pow(freq_base, (float)-2/n_dim)
+ : 1.0;
+
+ // first half, dims [0, n_dim/2)
+ cur = ggml_rope_ext(
+ ctx0,
+ cur,
+ pos_a, // positions
+ nullptr, // freq factors
+ n_dim/2, // n_dims
+ 0, 0, freq_base,
+ 1.0f, 0.0f, 1.0f, 0.0f, 0.0f
+ );
+
+ // second half, dims [n_dim/2, n_dim)
+ cur = ggml_rope_ext(
+ ctx0,
+ cur,
+ pos_b, // positions
+ nullptr, // freq factors
+ n_dim/2, // n_dims
+ 0, 0, freq_base,
+ freq_scale_odd,
+ 0.0f, 1.0f, 0.0f, 0.0f
+ );
+ cur = ggml_rope_set_offset(cur, n_dim/2);
+
+ return cur;
+}
+
+// Generic function to stack frames for audio processing
+// Abstracts out the StackAudioFrames logic used by ultravox
+ggml_tensor * mtmd_graph::build_stack(ggml_tensor * cur, int32_t stack_factor, int32_t n_embed) {
+ if (stack_factor <= 1) {
+ return cur;
+ }
+
+ int64_t total_elements = ggml_nelements(cur);
+ int64_t stride = n_embed * stack_factor;
+
+ // Calculate padded length
+ int64_t padded_len = GGML_PAD(total_elements, stride);
+ int64_t pad = padded_len - total_elements;
+
+ if (pad > 0) {
+ // Pad the tensor to make it divisible by stride
+ cur = ggml_view_1d(ctx0, cur, total_elements, 0);
+ cur = ggml_pad(ctx0, cur, pad, 0, 0, 0);
+ }
+
+ // Reshape to [stride, padded_len / stride]
+ cur = ggml_view_2d(ctx0, cur, stride, padded_len / stride,
+ ggml_row_size(cur->type, stride), 0);
+ return cur;
+}
+
+// aka pixel_shuffle / pixel_unshuffle / patch_merger (Kimi-VL)
+// support dynamic resolution
+ggml_tensor * mtmd_graph::build_suffix(ggml_tensor * cur) {
+ for (int idx : mtmd_suffix_rows(img.suffix_type)) {
+ GGML_ASSERT(model.tok_embd_sep && idx < model.tok_embd_sep->ne[1]);
+ ggml_tensor * row = ggml_view_2d(ctx0, model.tok_embd_sep, model.tok_embd_sep->ne[0], 1,
+ model.tok_embd_sep->nb[1], idx * model.tok_embd_sep->nb[1]);
+ cur = ggml_concat(ctx0, cur, ggml_cast(ctx0, row, cur->type), 1);
+ }
+ return cur;
+}
+
+ggml_tensor * mtmd_graph::build_patch_merge_permute(ggml_tensor * cur, int scale_factor) {
+ GGML_ASSERT(scale_factor > 1);
+
+ const int n_embd = cur->ne[0];
+ int width = img.nx() / patch_size;
+ int height = img.ny() / patch_size;
+
+ // pad width and height to factor
+ const int64_t pad_width = MTMD_ALIGN(width, scale_factor) - width;
+ const int64_t pad_height = MTMD_ALIGN(height, scale_factor) - height;
+ cur = ggml_reshape_3d(ctx0, cur, n_embd, width, height);
+ if (pad_width || pad_height) {
+ cur = ggml_pad(ctx0, cur, 0, pad_width, pad_height, 0);
+ width += pad_width;
+ height += pad_height;
+ }
+
+ // unshuffle h
+ cur = ggml_reshape_3d(ctx0, cur, n_embd * scale_factor, width / scale_factor, height);
+ cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
+
+ // unshuffle w
+ cur = ggml_cont_3d(ctx0, cur, n_embd * scale_factor * scale_factor, height / scale_factor, width / scale_factor);
+ cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
+
+ cur = ggml_cont_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);
+ cb(cur, "pixel_shuffle", -1);
+
+ return cur;
+}
+
+std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const mtmd_image_f32_batch & imgs,
+ const mtmd_subctx_encode_params * params) {
+ const mtmd_image_f32 & img = imgs.entries[0];
+ std::unique_ptr<mtmd_graph> builder;
+
+ switch (ctx->model.proj_type) {
+ case PROJECTOR_TYPE_GEMMA3:
+ case PROJECTOR_TYPE_IDEFICS3:
+ case PROJECTOR_TYPE_COHERE2V:
+ case PROJECTOR_TYPE_LFM2:
+ case PROJECTOR_TYPE_D1OMNI_V:
+ case PROJECTOR_TYPE_JANUS_PRO:
+ case PROJECTOR_TYPE_PHI4:
+ {
+ builder = std::make_unique<mtmd_graph_siglip>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_GEMMA3NV:
+ {
+ builder = std::make_unique<mtmd_graph_mobilenetv5>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_GEMMA4V:
+ {
+ builder = std::make_unique<mtmd_graph_gemma4v>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_GEMMA4UV:
+ {
+ builder = std::make_unique<mtmd_graph_gemma4uv>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_PIXTRAL:
+ case PROJECTOR_TYPE_LIGHTONOCR:
+ {
+ builder = std::make_unique<mtmd_graph_pixtral>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_DOTS_OCR:
+ case PROJECTOR_TYPE_DOTS3NOTE_V: // same ViT + merger; pyramid MoE is handled by build_vit
+ {
+ builder = std::make_unique<mtmd_graph_dotsocr>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_DOTS3NOTE_A:
+ {
+ builder = std::make_unique<mtmd_graph_dots3note_a>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_QWEN2VL:
+ case PROJECTOR_TYPE_QWEN25VL:
+ {
+ builder = std::make_unique<mtmd_graph_qwen2vl>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_QWEN3VL:
+ {
+ builder = std::make_unique<mtmd_graph_qwen3vl>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_MAGEVL:
+ {
+ builder = std::make_unique<mtmd_graph_magevl>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ {
+ builder = std::make_unique<mtmd_graph_sensenova_u1>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_LFM3VL:
+ {
+ builder = std::make_unique<mtmd_graph_lfm3vl>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_LING3VL:
+ {
+ builder = std::make_unique<mtmd_graph_ling3vl>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_EXAONE4_5:
+ {
+ builder = std::make_unique<mtmd_graph_exaone4_5>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_MIMOVL:
+ {
+ builder = std::make_unique<mtmd_graph_mimovl>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_MINIMAX_M3:
+ {
+ builder = std::make_unique<mtmd_graph_minimax_m3>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_MUSE_GLIMMER:
+ {
+ builder = std::make_unique<mtmd_graph_muse_glimmer>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_STEP3VL:
+ {
+ builder = std::make_unique<mtmd_graph_step3vl>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_MINICPMV:
+ {
+ builder = std::make_unique<mtmd_graph_minicpmv>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_MINICPMV4_6:
+ case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
+ {
+ builder = std::make_unique<mtmd_graph_minicpmv4_6>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_INTERNVL:
+ {
+ builder = std::make_unique<mtmd_graph_internvl>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_NEMOTRON_V2_VL:
+ {
+ builder = std::make_unique<mtmd_graph_nemotron_v2_vl>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_NEMOTRON_OMNI:
+ {
+ builder = std::make_unique<mtmd_graph_nemotron_omni>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_LLAMA4:
+ {
+ builder = std::make_unique<mtmd_graph_llama4>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_VOXTRAL_RT:
+ {
+ builder = std::make_unique<mtmd_graph_voxtral_rt>(ctx, img, params ? params->enc_state : nullptr);
+ } break;
+ case PROJECTOR_TYPE_ULTRAVOX:
+ case PROJECTOR_TYPE_VOXTRAL:
+ case PROJECTOR_TYPE_QWEN2A:
+ case PROJECTOR_TYPE_GLMA:
+ case PROJECTOR_TYPE_MERALION:
+ case PROJECTOR_TYPE_MUSIC_FLAMINGO:
+ {
+ builder = std::make_unique<mtmd_graph_whisper_enc>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
+ {
+ builder = std::make_unique<mtmd_graph_muse_spark_audio>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_KIMIVL:
+ {
+ builder = std::make_unique<mtmd_graph_kimivl>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_PADDLEOCR:
+ {
+ builder = std::make_unique<mtmd_graph_paddleocr>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_KIMIK25:
+ {
+ builder = std::make_unique<mtmd_graph_kimik25>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_DEEPSEEK4V:
+ case PROJECTOR_TYPE_DEEPSEEK41V:
+ {
+ builder = std::make_unique<mtmd_graph_deepseek4v>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_COGVLM:
+ {
+ builder = std::make_unique<mtmd_graph_cogvlm>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_HUNYUANVL:
+ {
+ builder = std::make_unique<mtmd_graph_hunyuanvl>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_MLP:
+ case PROJECTOR_TYPE_MLP_NORM:
+ case PROJECTOR_TYPE_LDP:
+ case PROJECTOR_TYPE_LDPV2:
+ case PROJECTOR_TYPE_GLM_EDGE:
+ {
+ builder = std::make_unique<mtmd_graph_llava>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_DEEPSEEKOCR:
+ {
+ builder = std::make_unique<mtmd_graph_deepseekocr>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_DEEPSEEKOCR2:
+ {
+ builder = std::make_unique<mtmd_graph_deepseekocr2>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_LFM2A:
+ case PROJECTOR_TYPE_D1OMNI_A:
+ {
+ builder = std::make_unique<mtmd_graph_conformer>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_LFM3A:
+ {
+ builder = std::make_unique<mtmd_graph_lfm3a>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_GEMMA4A:
+ {
+ builder = std::make_unique<mtmd_graph_gemma4a>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_GEMMA4UA:
+ {
+ builder = std::make_unique<mtmd_graph_gemma4ua>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_GRANITE_SPEECH:
+ {
+ builder = std::make_unique<mtmd_graph_granite_speech>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_GLM4V:
+ case PROJECTOR_TYPE_GLM5V:
+ {
+ builder = std::make_unique<mtmd_graph_glm4v>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_QWEN3A:
+ {
+ builder = std::make_unique<mtmd_graph_qwen3a>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_MIMO_AUDIO:
+ {
+ builder = std::make_unique<mtmd_graph_mimo_audio>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
+ {
+ builder = std::make_unique<mtmd_graph_qwen3tts_spkenc>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_POCKETTTS_SPKENC:
+ {
+ builder = std::make_unique<mtmd_graph_pockettts_spkenc>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_POCKETTTS_GEN:
+ {
+ const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
+ const int n_step = ctx->model.hparams.flow_n_step;
+ const int64_t n_latent = ctx->model.gen_input_lin_w->ne[0];
+ GGML_ASSERT(n_step > 0);
+ GGML_ASSERT(n_latent > 0);
+ // "inp_feats" takes the caller's buffer as-is, the graph must consume all of it
+ if (params && params->feats) {
+ GGML_ASSERT(params->feats->size() % (size_t) n_latent == 0);
+ GGML_ASSERT(params->feats->size() >= (size_t) n_latent);
+ }
+ const int n_frames = params && params->feats ? (int) (params->feats->size() / n_latent) : 1;
+ builder = std::make_unique<mtmd_graph_pockettts_gen>(ctx, img, gen_process, n_step, n_frames);
+ } break;
+ case PROJECTOR_TYPE_CHATTERBOX_SPKENC:
+ {
+ builder = std::make_unique<mtmd_graph_chatterbox_spkenc>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_CHATTERBOX:
+ {
+ const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
+ int n_tokens = params && params->codes ? (int) params->codes->size() : 0;
+ int n_prompt_mel = 0;
+ if (n_tokens > 0) {
+ GGML_ASSERT(params->ref);
+ n_tokens += (int) params->ref->tokens.size();
+ n_prompt_mel = (int) (params->ref->feat.size() / 80);
+ }
+ const int vnm = params ? params->vocode_n_mel : 0;
+ const int vns = params ? params->vocode_n_stft : 0;
+ builder = std::make_unique<mtmd_graph_chatterbox>(ctx, img, gen_process, n_tokens, n_prompt_mel, vnm, vns);
+ } break;
+ case PROJECTOR_TYPE_LFM3A_GEN:
+ {
+ const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
+ const int64_t n_cb = ctx->model.gen_code_embd_w->ne[2];
+ bool all_known = false;
+ int n_frames = 1;
+ if (params && params->codes) {
+ GGML_ASSERT(params->codes->size() % (size_t) n_cb == 0);
+ if (gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
+ n_frames = (int) (params->codes->size() / n_cb);
+ GGML_ASSERT(n_frames > 0);
+ } else {
+ // -1 marks a masked codebook, none masked means only the feedback embedding is needed
+ all_known = std::all_of(params->codes->begin(), params->codes->end(), [](int32_t c) { return c >= 0; });
+ }
+ }
+ builder = std::make_unique<mtmd_graph_lfm3a_gen>(ctx, img, gen_process, all_known, n_frames);
+ } break;
+ case PROJECTOR_TYPE_DRAMA3_GEN:
+ {
+ const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
+ const int64_t n_cb = ctx->model.gen_code_embd_w->ne[2];
+ const int n_frames = params && params->codes ? (int) (params->codes->size() / n_cb) : 1;
+ builder = std::make_unique<mtmd_graph_drama3_gen>(ctx, img, gen_process, params ? params->gen_state : nullptr, n_frames,
+ params ? params->top_k : 30, params ? params->top_p : 0.9f,
+ params ? params->temp : 1.0f);
+ } break;
+ case PROJECTOR_TYPE_QWEN3TTS_GEN:
+ {
+ const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
+ const int top_k = params ? params->top_k : 50;
+ const float top_p = params ? params->top_p : 1.0f;
+ builder = std::make_unique<mtmd_graph_qwen3tts_gen>(ctx, img, gen_process, top_k, top_p);
+ } break;
+ case PROJECTOR_TYPE_SOPRANO:
+ {
+ const int n_frames = params && params->feats ? (int) (params->feats->size() / ctx->model.hparams.projection_dim) : 2;
+ builder = std::make_unique<mtmd_graph_soprano>(ctx, img, n_frames);
+ } break;
+ case PROJECTOR_TYPE_YOUTUVL:
+ {
+ builder = std::make_unique<mtmd_graph_youtuvl>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_YASA2:
+ {
+ builder = std::make_unique<mtmd_graph_yasa2>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_PARAKEET:
+ {
+ builder = std::make_unique<mtmd_graph_parakeet>(ctx, img);
+ } break;
+ case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
+ {
+ builder = std::make_unique<mtmd_graph_nemotron_transcribe>(ctx, img, params ? params->enc_state : nullptr);
+ } break;
+ case PROJECTOR_TYPE_GRANITE4_VISION:
+ {
+ builder = std::make_unique<mtmd_graph_granite4_vision>(ctx, img);
+ } break;
+ default:
+ GGML_ABORT("missing cgraph builder");
+ }
+
+ builder->img_batch = &imgs;
+
+ // TODO [QWEN_VIDEO]: improve this in the future
+ builder->n_batch = imgs.entries.size();
+
+ return builder;
+}
diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/mtmd-graph.h
similarity index 84%
rename from tools/mtmd/clip-graph.h
rename to tools/mtmd/mtmd-graph.h
index a0669d1a0..41765d03f 100644
--- a/tools/mtmd/clip-graph.h
+++ b/tools/mtmd/mtmd-graph.h
@@ -2,9 +2,9 @@
#include "ggml.h"
#include "ggml-cpp.h"
-#include "clip.h"
-#include "clip-impl.h"
-#include "clip-model.h"
+#include "mtmd-subcontext.h"
+#include "mtmd-impl.h"
+#include "mtmd-model.h"
#include <vector>
#include <functional>
@@ -23,13 +23,13 @@ struct build_vit_opts {
bool skip_post_ln = false;
};
-struct clip_graph {
- const clip_model & model;
- const clip_hparams & hparams;
+struct mtmd_graph {
+ const mtmd_model & model;
+ const mtmd_hparams & hparams;
projector_type proj_type;
- const clip_image_f32 & img; // for backward compat
- const clip_image_f32_batch * img_batch = nullptr;
+ const mtmd_image_f32 & img; // for backward compat
+ const mtmd_image_f32_batch * img_batch = nullptr;
const int patch_size;
const int n_patches_x;
@@ -43,7 +43,7 @@ struct clip_graph {
const int n_mmproj_embd;
const float eps;
float kq_scale; // TODO: maybe move this to hparams
- const clip_flash_attn_type flash_attn_type;
+ const mtmd_flash_attn_type flash_attn_type;
// TODO [QWEN_VIDEO]: improve this in the future
int n_batch = 1;
@@ -52,12 +52,12 @@ struct clip_graph {
ggml_context * ctx0;
ggml_cgraph * gf;
- clip_graph(clip_ctx * ctx, const clip_image_f32 & img);
+ mtmd_graph(mtmd_subcontext * ctx, const mtmd_image_f32 & img);
// build sub-graph, reuse buf from parent
- clip_graph(const clip_graph & parent);
+ mtmd_graph(const mtmd_graph & parent);
- virtual ~clip_graph() = default;
+ virtual ~mtmd_graph() = default;
virtual ggml_cgraph * build() = 0;
// wrapper around ggml_mul_mat, allow hooking (e.g. LoRA, clamping) depending on the model
@@ -74,7 +74,7 @@ struct clip_graph {
//
void cb(ggml_tensor * cur0, const char * name, int il) const;
- const clip_image_f32 & get_img(size_t idx) const {
+ const mtmd_image_f32 & get_img(size_t idx) const {
GGML_ASSERT(img_batch);
GGML_ASSERT(idx < img_batch->entries.size());
return img_batch->entries[idx];
@@ -92,7 +92,7 @@ struct clip_graph {
norm_type norm_t,
ffn_op_type ffn_t,
ggml_tensor * learned_pos_embd,
- std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos,
+ std::function<ggml_tensor *(ggml_tensor *, const mtmd_layer &)> add_pos,
const build_vit_opts & opts = {});
// build the input after conv2d (inp_raw --> patches)
@@ -130,7 +130,7 @@ struct clip_graph {
ggml_tensor * build_moe_ffn(
ggml_tensor * cur,
- const clip_layer & layer,
+ const mtmd_layer & layer,
ffn_op_type type_op,
int il) const;
@@ -174,3 +174,11 @@ struct clip_graph {
// append the separators of img.suffix_type after the image tokens
ggml_tensor * build_suffix(ggml_tensor * cur);
};
+
+static inline std::string get_attn_mask_name(int idx) {
+ return idx == 0 ? "attn_mask" : "attn_mask_" + std::to_string(idx);
+}
+
+// create the graph builder of the subcontext's projector, params is only needed by encode()
+std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const mtmd_image_f32_batch & imgs,
+ const mtmd_subctx_encode_params * params = nullptr);
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index d776ed1e8..acdee5d12 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -494,7 +494,7 @@ private:
llama_token tts_eos = LLAMA_TOKEN_NULL;
std::vector<float> tok_embd; // whole token embedding matrix, n_vocab * n_embd
- // must match hparams.wav_tfm_swa hardcoded in clip.cpp
+ // must match hparams.wav_tfm_swa hardcoded in mtmd-model.cpp
size_t window_frames = 72;
// per-generation state, cleared by reset()
@@ -2291,7 +2291,7 @@ private:
// repetition-aware sampling: if a semantic draw repeats one of the last ras_window main tokens, draw again with fixed settings
static constexpr size_t ras_window = 16;
static constexpr size_t n_cb = 10;
- // codebook tower KV rows (clip_graph_drama3_gen::n_pos_max): one per prompt row but the last, one per frame
+ // codebook tower KV rows (mtmd_graph_drama3_gen::n_pos_max): one per prompt row but the last, one per frame
static constexpr size_t n_tower_rows = 4096;
// frames per codec call when streaming (0.33 s)
static constexpr size_t stream_frames = 7;
diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp
index 8c0f433e3..b0543aa43 100644
--- a/tools/mtmd/mtmd-image.cpp
+++ b/tools/mtmd/mtmd-image.cpp
@@ -4,8 +4,8 @@
#include <cmath>
#include <vector>
-void mtmd_image_preproc_out::append(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized) {
- clip_image_f32 dst;
+void mtmd_image_preproc_out::append(const mtmd_hparams & hparams, const mtmd_image_u8 & img, bool normalized) {
+ mtmd_image_f32 dst;
dst.from_u8(img);
if (normalized) {
dst.normalize(hparams.image_mean, hparams.image_std);
@@ -13,20 +13,20 @@ void mtmd_image_preproc_out::append(const clip_hparams & hparams, const clip_ima
entries.push_back(std::move(dst));
}
-void mtmd_image_preproc_out::append(const clip_hparams & hparams, const std::vector<clip_image_u8> & imgs, bool normalized) {
+void mtmd_image_preproc_out::append(const mtmd_hparams & hparams, const std::vector<mtmd_image_u8> & imgs, bool normalized) {
for (const auto & img : imgs) {
append(hparams, img, normalized);
}
}
-void mtmd_image_preproc_out::append(const clip_hparams & hparams, clip_image_f32 & img, bool normalized) {
+void mtmd_image_preproc_out::append(const mtmd_hparams & hparams, mtmd_image_f32 & img, bool normalized) {
if (normalized) {
img.normalize(hparams.image_mean, hparams.image_std);
}
entries.push_back(std::move(img));
}
-void mtmd_image_preproc_out::append_overview(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized) {
+void mtmd_image_preproc_out::append_overview(const mtmd_hparams & hparams, const mtmd_image_u8 & img, bool normalized) {
overview.from_u8(img);
if (normalized) {
overview.normalize(hparams.image_mean, hparams.image_std);
@@ -37,9 +37,9 @@ void mtmd_image_preproc_out::append_overview(const clip_hparams & hparams, const
// in the future, we can have HW acceleration by allowing this struct to access 3rd party lib like imagick or opencv
struct img_tool {
static void resize(
- const clip_image_u8 & src,
- clip_image_u8 & dst,
- const clip_image_size & target_resolution,
+ const mtmd_image_u8 & src,
+ mtmd_image_u8 & dst,
+ const mtmd_image_size & target_resolution,
resize_algo algo,
pad_style padding = PAD_CEIL,
std::array<uint8_t, 3> pad_color = {0, 0, 0}) {
@@ -61,7 +61,7 @@ struct img_tool {
resize_pillow(src, dst, target_resolution.width, target_resolution.height, algo);
} else {
// resize with padding
- clip_image_u8 resized_image;
+ mtmd_image_u8 resized_image;
float scale_w = static_cast<float>(target_resolution.width) / src.get_size().width;
float scale_h = static_cast<float>(target_resolution.height) / src.get_size().height;
float scale = std::min(scale_w, scale_h);
@@ -92,7 +92,7 @@ struct img_tool {
}
}
- static void crop(const clip_image_u8 & image, clip_image_u8 & dst, int x, int y, int w, int h) {
+ static void crop(const mtmd_image_u8 & image, mtmd_image_u8 & dst, int x, int y, int w, int h) {
GGML_ASSERT(x >= 0 && y >= 0 && w > 0 && h > 0);
GGML_ASSERT(x + w <= image.get_size().width && y + h <= image.get_size().height);
dst.set_size({w, h}, image.is_placeholder());
@@ -120,7 +120,7 @@ struct img_tool {
// calculate the size of the **resized** image, while preserving the aspect ratio and
// aligning to the nearest multiple of align_size ("smart_resize" in transformers code)
- static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const calc_size_opt & opts) {
+ static mtmd_image_size calc_size_preserved_ratio(const mtmd_image_size & inp_size, const calc_size_opt & opts) {
GGML_ASSERT(opts.align_size > 0);
const int width = inp_size.width;
const int height = inp_size.height;
@@ -161,7 +161,7 @@ struct img_tool {
}
// draw src image into dst image at offset (offset_x, offset_y)
- static void composite(clip_image_u8 & dst, const clip_image_u8 & src, int offset_x, int offset_y) {
+ static void composite(mtmd_image_u8 & dst, const mtmd_image_u8 & src, int offset_x, int offset_y) {
if (src.is_placeholder()) {
// no-op for placeholder image
return;
@@ -183,7 +183,7 @@ struct img_tool {
}
// fill the image with a solid color
- static void fill(clip_image_u8 & img, const std::array<uint8_t, 3> & color) {
+ static void fill(mtmd_image_u8 & img, const std::array<uint8_t, 3> & color) {
if (img.is_placeholder()) {
// no-op for placeholder image
return;
@@ -208,8 +208,8 @@ private:
//
// RESIZE_ALGO_FLAG_TORCH is the same, except for the int16 weights of torchvision (aten/src/ATen/native/cpu/UpSampleKernel.cpp)
static bool resize_pillow(
- const clip_image_u8 & img,
- clip_image_u8 & dst,
+ const mtmd_image_u8 & img,
+ mtmd_image_u8 & dst,
int target_width,
int target_height,
resize_algo algo) {
@@ -516,8 +516,8 @@ private:
// mtmd_image_preprocessor_llava_uhd
//
-mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const clip_image_u8 & img) const {
- const clip_image_size original_size = img.get_size();
+mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const mtmd_image_u8 & img) const {
+ const mtmd_image_size original_size = img.get_size();
auto const inst = get_slice_instructions(original_size);
auto sliced = slice_image(img, inst);
@@ -530,7 +530,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const clip_
return output;
}
-mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_llava_uhd::get_slice_instructions(const clip_image_size & original_size) const {
+mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_llava_uhd::get_slice_instructions(const mtmd_image_size & original_size) const {
mtmd_image_preprocessor_llava_uhd::slice_instructions res;
const int patch_size = get_slice_align();
const int slice_size = hparams.image_size;
@@ -542,9 +542,9 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_ll
if (!has_slices) {
// skip slicing logic
- res.overview_size = clip_image_size{slice_size, slice_size};
- res.refined_size = clip_image_size{0, 0};
- res.grid_size = clip_image_size{0, 0};
+ res.overview_size = mtmd_image_size{slice_size, slice_size};
+ res.refined_size = mtmd_image_size{0, 0};
+ res.grid_size = mtmd_image_size{0, 0};
return res;
}
@@ -554,9 +554,9 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_ll
auto refine_size = select_best_resolution(
original_size,
hparams.image_res_candidates);
- res.overview_size = clip_image_size{slice_size, slice_size};
+ res.overview_size = mtmd_image_size{slice_size, slice_size};
res.refined_size = refine_size;
- res.grid_size = clip_image_size{0, 0};
+ res.grid_size = mtmd_image_size{0, 0};
LOG_DBG("%s: using pinpoints for slicing\n", __func__);
LOG_DBG("%s: original size: %d x %d, overview size: %d x %d, refined size: %d x %d\n",
@@ -633,7 +633,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_ll
return res;
}
-mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uhd::slice_image(const clip_image_u8 & img, const mtmd_image_preprocessor_llava_uhd::slice_instructions & inst) const {
+mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uhd::slice_image(const mtmd_image_u8 & img, const mtmd_image_preprocessor_llava_uhd::slice_instructions & inst) const {
slice_output output;
// resize to overview size
@@ -646,7 +646,7 @@ mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uh
}
// resize to refined size
- clip_image_u8 refined_img;
+ mtmd_image_u8 refined_img;
img_tool::resize(img, refined_img, inst.refined_size, hparams.image_resize_algo_rf,
hparams.image_pad_rf, hparams.image_pad_color_rf);
@@ -657,7 +657,7 @@ mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uh
int w = slice.size.width;
int h = slice.size.height;
- clip_image_u8 img_slice;
+ mtmd_image_u8 img_slice;
img_tool::crop(refined_img, img_slice, x, y, w, h);
output.slices.push_back(std::move(img_slice));
}
@@ -665,7 +665,7 @@ mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uh
return output;
}
-clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale) const {
+mtmd_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const mtmd_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale) const {
int width = original_size.width;
int height = original_size.height;
if ((width * height > scale_resolution * scale_resolution) || allow_upscale) {
@@ -673,28 +673,28 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const clip_im
height = static_cast<int>(scale_resolution / std::sqrt(r));
width = static_cast<int>(height * r);
}
- clip_image_size res;
+ mtmd_image_size res;
res.width = ensure_divide(width, patch_size);
res.height = ensure_divide(height, patch_size);
return res;
}
-clip_image_size mtmd_image_preprocessor_llava_uhd::resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) const {
+mtmd_image_size mtmd_image_preprocessor_llava_uhd::resize_maintain_aspect_ratio(const mtmd_image_size & orig, const mtmd_image_size & target_max) const {
float scale_width = static_cast<float>(target_max.width) / orig.width;
float scale_height = static_cast<float>(target_max.height) / orig.height;
float scale = std::min(scale_width, scale_height);
- return clip_image_size{
+ return mtmd_image_size{
static_cast<int>(orig.width * scale),
static_cast<int>(orig.height * scale),
};
}
-clip_image_size mtmd_image_preprocessor_llava_uhd::select_best_resolution(const clip_image_size & original_size, const std::vector<clip_image_size> & possible_resolutions) const {
- clip_image_size best_fit;
+mtmd_image_size mtmd_image_preprocessor_llava_uhd::select_best_resolution(const mtmd_image_size & original_size, const std::vector<mtmd_image_size> & possible_resolutions) const {
+ mtmd_image_size best_fit;
int min_wasted_area = std::numeric_limits<int>::max();
int max_effective_resolution = 0;
- for (const clip_image_size & candidate : possible_resolutions) {
+ for (const mtmd_image_size & candidate : possible_resolutions) {
auto target_size = resize_maintain_aspect_ratio(original_size, candidate);
int effective_resolution = std::min(
target_size.width * target_size.height,
@@ -717,7 +717,7 @@ int mtmd_image_preprocessor_llava_uhd::ensure_divide(int length, int patch_size)
return std::max(align_round(static_cast<double>(length) / patch_size) * patch_size, patch_size);
}
-clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale) const {
+mtmd_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const mtmd_image_size & original_size, const mtmd_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale) const {
int width = original_size.width;
int height = original_size.height;
int grid_x = grid.width;
@@ -726,7 +726,7 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_im
int refine_width = ensure_divide(width, grid_x);
int refine_height = ensure_divide(height, grid_y);
- clip_image_size grid_size;
+ mtmd_image_size grid_size;
grid_size.width = refine_width / grid_x;
grid_size.height = refine_height / grid_y;
@@ -734,13 +734,13 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_im
int best_grid_width = best_grid_size.width;
int best_grid_height = best_grid_size.height;
- clip_image_size refine_size;
+ mtmd_image_size refine_size;
refine_size.width = best_grid_width * grid_x;
refine_size.height = best_grid_height * grid_y;
return refine_size;
}
-clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const {
+mtmd_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const {
std::vector<int> candidate_split_grids_nums;
for (int i : {multiple - 1, multiple, multiple + 1}) {
if (i == 1 || i > max_slice_nums) {
@@ -749,18 +749,18 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_s
candidate_split_grids_nums.push_back(i);
}
- std::vector<clip_image_size> candidate_grids;
+ std::vector<mtmd_image_size> candidate_grids;
for (int split_grids_nums : candidate_split_grids_nums) {
int m = 1;
while (m <= split_grids_nums) {
if (split_grids_nums % m == 0) {
- candidate_grids.push_back(clip_image_size{m, split_grids_nums / m});
+ candidate_grids.push_back(mtmd_image_size{m, split_grids_nums / m});
}
++m;
}
}
- clip_image_size best_grid{1, 1};
+ mtmd_image_size best_grid{1, 1};
float min_error = std::numeric_limits<float>::infinity();
for (const auto& grid : candidate_grids) {
float error = std::abs(log_ratio - std::log(1.0 * grid.width / grid.height));
@@ -776,8 +776,8 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_s
// mtmd_image_preprocessor_fixed_size
//
-mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip_image_u8 & img) const {
- clip_image_u8 resized_image;
+mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const mtmd_image_u8 & img) const {
+ mtmd_image_u8 resized_image;
int sz = hparams.image_size;
img_tool::resize(img, resized_image, {sz, sz},
hparams.image_resize_algo,
@@ -792,7 +792,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip
// mtmd_image_preprocessor_dyn_size
//
-clip_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) {
+mtmd_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const mtmd_hparams & hparams, const mtmd_image_size & original_size) {
GGML_ASSERT(hparams.image_min_pixels > 0 && hparams.image_max_pixels > 0);
// the original pixtral model doesn't have n_merge
const int cur_merge = hparams.n_merge;
@@ -807,8 +807,8 @@ clip_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const clip_hpa
});
}
-mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const {
- clip_image_u8 resized_image;
+mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const mtmd_image_u8 & img) const {
+ mtmd_image_u8 resized_image;
img_tool::resize(img, resized_image, target_size(img.get_size()),
hparams.image_resize_algo,
hparams.image_resize_pad,
@@ -823,7 +823,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i
//
// rounds before the pixel checks (an edge may round to 0) and computes in double, as transformers does
-clip_image_size mtmd_image_preprocessor_qwen3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels,
+mtmd_image_size mtmd_image_preprocessor_qwen3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels,
int n_frames, int temporal_factor) {
double h = height;
double w = width;
@@ -850,13 +850,13 @@ clip_image_size mtmd_image_preprocessor_qwen3vl::smart_resize(int width, int hei
return {std::max(factor, w_bar), std::max(factor, h_bar)};
}
-bool mtmd_image_preprocessor_qwen3vl::get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const {
+bool mtmd_image_preprocessor_qwen3vl::get_video_frame_size(int n_frames, int width, int height, mtmd_image_size & out) const {
// Qwen3VLVideoProcessor (and Lfm3VlVideoProcessor) size: shortest_edge 4096, longest_edge 25165824, temporal_patch_size 2
out = smart_resize(width, height, hparams.patch_size * hparams.n_merge, 4096, 25165824, n_frames, 2);
return true;
}
-mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::preprocess_video_frame(const clip_image_u8 & img) const {
+mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::preprocess_video_frame(const mtmd_image_u8 & img) const {
const int f = hparams.patch_size * hparams.n_merge;
const auto size = img.get_size();
return resize_to(img, {
@@ -865,8 +865,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::preprocess_video_frame(c
});
}
-mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::resize_to(const clip_image_u8 & img, const clip_image_size & size) const {
- clip_image_u8 resized_image;
+mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::resize_to(const mtmd_image_u8 & img, const mtmd_image_size & size) const {
+ mtmd_image_u8 resized_image;
img_tool::resize(img, resized_image, size,
hparams.image_resize_algo,
hparams.image_resize_pad,
@@ -880,7 +880,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::resize_to(const clip_ima
// mtmd_image_preprocessor_lfm3vl
//
-clip_image_size mtmd_image_preprocessor_lfm3vl::target_size(const clip_image_size & original_size) const {
+mtmd_image_size mtmd_image_preprocessor_lfm3vl::target_size(const mtmd_image_size & original_size) const {
return smart_resize(original_size.width, original_size.height, hparams.patch_size * hparams.n_merge,
hparams.image_min_pixels, hparams.image_max_pixels);
}
@@ -891,7 +891,7 @@ clip_image_size mtmd_image_preprocessor_lfm3vl::target_size(const clip_image_siz
// The canvas is ceil-aligned to patch_size*n_merge and fitted to the token budget.
// Only rescaled to meet the budget and sits top-left, with black padding on the right and bottom
-mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const clip_image_u8 & img) const {
+mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const mtmd_image_u8 & img) const {
GGML_ASSERT(hparams.image_min_pixels > 0 && hparams.image_max_pixels > 0);
const int64_t factor = hparams.patch_size * hparams.n_merge;
@@ -941,11 +941,11 @@ mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const clip_imag
const int content_h = (int) std::max<int64_t>(1, std::min<int64_t>(canvas_h, (int64_t) std::floor(height * scale)));
const int content_w = (int) std::max<int64_t>(1, std::min<int64_t>(canvas_w, (int64_t) std::floor(width * scale)));
- clip_image_u8 content;
- img_tool::resize(img, content, clip_image_size{content_w, content_h}, hparams.image_resize_algo, PAD_NONE);
+ mtmd_image_u8 content;
+ img_tool::resize(img, content, mtmd_image_size{content_w, content_h}, hparams.image_resize_algo, PAD_NONE);
- clip_image_u8 canvas;
- canvas.set_size(clip_image_size{(int) canvas_w, (int) canvas_h}, img.is_placeholder());
+ mtmd_image_u8 canvas;
+ canvas.set_size(mtmd_image_size{(int) canvas_w, (int) canvas_h}, img.is_placeholder());
img_tool::fill(canvas, {0, 0, 0});
img_tool::composite(canvas, content, 0, 0);
@@ -958,9 +958,9 @@ mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const clip_imag
// mtmd_image_preprocessor_longest_edge
//
-clip_image_size mtmd_image_preprocessor_longest_edge::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) {
+mtmd_image_size mtmd_image_preprocessor_longest_edge::get_target_size(const mtmd_hparams & hparams, const mtmd_image_size & original_size) {
GGML_ASSERT(hparams.image_longest_edge > 0);
- clip_image_size size = original_size;
+ mtmd_image_size size = original_size;
if (hparams.image_floor_before_align) {
const double ratio = double(std::max(size.width, size.height)) / hparams.image_longest_edge;
if (ratio > 1.0) {
@@ -985,9 +985,9 @@ clip_image_size mtmd_image_preprocessor_longest_edge::get_target_size(const clip
});
}
-mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img) const {
- clip_image_u8 resized_image;
- const clip_image_size target_size = get_target_size(hparams, img.get_size());
+mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const mtmd_image_u8 & img) const {
+ mtmd_image_u8 resized_image;
+ const mtmd_image_size target_size = get_target_size(hparams, img.get_size());
img_tool::resize(img, resized_image, target_size,
hparams.image_resize_algo,
hparams.image_resize_pad,
@@ -1001,15 +1001,15 @@ mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const cl
// mtmd_image_preprocessor_minicpmv
//
-mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_minicpmv::get_slice_instructions(const clip_image_size & original_size) const {
+mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_minicpmv::get_slice_instructions(const mtmd_image_size & original_size) const {
// overview only for small images, unlike generic llava-uhd which slices once one side exceeds scale resolution
const int slice_size = hparams.image_size;
const float ratio = (float) original_size.width * original_size.height / (slice_size * slice_size);
if (ratio <= 1.0f) {
mtmd_image_preprocessor_llava_uhd::slice_instructions inst;
inst.overview_size = get_best_resize(original_size, slice_size, get_slice_align(), true);
- inst.refined_size = clip_image_size{0, 0};
- inst.grid_size = clip_image_size{0, 0};
+ inst.refined_size = mtmd_image_size{0, 0};
+ inst.grid_size = mtmd_image_size{0, 0};
return inst;
}
return mtmd_image_preprocessor_llava_uhd::get_slice_instructions(original_size);
@@ -1019,7 +1019,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_mi
// mtmd_image_preprocessor_lfm2
//
-mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image_u8 & img) const {
+mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const mtmd_image_u8 & img) const {
auto const inst = get_slice_instructions(img.get_size());
if (!inst.slices.empty()) {
return mtmd_image_preprocessor_llava_uhd::preprocess(img);
@@ -1034,8 +1034,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image
}
bool mtmd_image_preprocessor_lfm2::should_tile(
- const clip_hparams & hparams,
- const clip_image_size & original_size) {
+ const mtmd_hparams & hparams,
+ const mtmd_image_size & original_size) {
const int align_size = hparams.patch_size * hparams.n_merge;
const auto round_by_factor = [align_size](float x) {
@@ -1052,9 +1052,9 @@ bool mtmd_image_preprocessor_lfm2::should_tile(
}
// same as Lfm2VlImageProcessor.smart_resize: Python's round half to even, double precision
-clip_image_size mtmd_image_preprocessor_lfm2::smart_resize(
- const clip_hparams & hparams,
- const clip_image_size & original_size) {
+mtmd_image_size mtmd_image_preprocessor_lfm2::smart_resize(
+ const mtmd_hparams & hparams,
+ const mtmd_image_size & original_size) {
const int align_size = hparams.patch_size * hparams.n_merge;
const double height = original_size.height;
const double width = original_size.width;
@@ -1075,7 +1075,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::smart_resize(
return {w_bar, h_bar};
}
-mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) const {
+mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const mtmd_image_size & original_size) const {
mtmd_image_preprocessor_llava_uhd::slice_instructions inst;
inst.overview_size = smart_resize(hparams, original_size);
@@ -1083,15 +1083,15 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf
const int tile_size = hparams.preproc_tile_size;
if (!needs_tiling) {
- inst.refined_size = clip_image_size{0, 0};
- inst.grid_size = clip_image_size{0, 0};
+ inst.refined_size = mtmd_image_size{0, 0};
+ inst.grid_size = mtmd_image_size{0, 0};
return inst;
}
- const clip_image_size grid = get_grid_layout(hparams, original_size.height, original_size.width);
+ const mtmd_image_size grid = get_grid_layout(hparams, original_size.height, original_size.width);
inst.grid_size = grid;
- inst.refined_size = clip_image_size{tile_size * grid.width, tile_size * grid.height};
+ inst.refined_size = mtmd_image_size{tile_size * grid.width, tile_size * grid.height};
LOG_DBG("%s: original size: %d x %d, overview size: %d x %d, refined size: %d x %d, grid size: %d x %d\n",
__func__,
@@ -1105,7 +1105,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf
mtmd_image_preprocessor_llava_uhd::slice_coordinates slice;
slice.x = col * tile_size;
slice.y = row * tile_size;
- slice.size = clip_image_size{tile_size, tile_size};
+ slice.size = mtmd_image_size{tile_size, tile_size};
inst.slices.push_back(slice);
LOG_DBG("%s: slice %d: x=%d, y=%d, size=%d x %d\n",
__func__, (int)inst.slices.size() - 1,
@@ -1117,13 +1117,13 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf
}
// double precision as in Python, float breaks exact ties such as 720/1600 between 2/4 and 2/5
-clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(
- const clip_hparams & hparams,
+mtmd_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(
+ const mtmd_hparams & hparams,
double aspect_ratio,
- const std::vector<clip_image_size> & target_ratios,
+ const std::vector<mtmd_image_size> & target_ratios,
int width, int height) {
double best_ratio_diff = std::numeric_limits<double>::max();
- clip_image_size best_ratio = {1, 1};
+ mtmd_image_size best_ratio = {1, 1};
const double area = static_cast<double>(width) * height;
for (const auto & ratio : target_ratios) {
@@ -1143,10 +1143,10 @@ clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(
return best_ratio;
}
-std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios(const clip_hparams & hparams) {
+std::vector<mtmd_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios(const mtmd_hparams & hparams) {
const int min_tiles = hparams.preproc_min_tiles;
const int max_tiles = hparams.preproc_max_tiles;
- std::vector<clip_image_size> ratios;
+ std::vector<mtmd_image_size> ratios;
for (int n = min_tiles; n <= max_tiles; n++) {
for (int w = 1; w <= n; w++) {
for (int h = 1; h <= n; h++) {
@@ -1165,13 +1165,13 @@ std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios(con
}
}
}
- std::sort(ratios.begin(), ratios.end(), [](const clip_image_size & a, const clip_image_size & b) {
+ std::sort(ratios.begin(), ratios.end(), [](const mtmd_image_size & a, const mtmd_image_size & b) {
return a.width * a.height < b.width * b.height;
});
return ratios;
}
-clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(const clip_hparams & hparams, int height, int width) {
+mtmd_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(const mtmd_hparams & hparams, int height, int width) {
const double aspect_ratio = static_cast<double>(width) / height;
const auto ratios = get_target_ratios(hparams);
return find_closest_aspect_ratio(hparams, aspect_ratio, ratios, width, height);
@@ -1181,7 +1181,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(const clip_hparams
// mtmd_image_preprocessor_idefics3
//
-mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_image_u8 & img) const {
+mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const mtmd_image_u8 & img) const {
// The refined size has two steps:
// 1. Resize w/ aspect-ratio preserving such that the longer side is
// the preprocessor longest size
@@ -1189,7 +1189,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i
// multiples of image_size (always rounding up)
//
// CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737
- const clip_image_size original_size = img.get_size();
+ const mtmd_image_size original_size = img.get_size();
// old gguf files have no preprocessor longest size, custom token limits also need the generic size below
if (hparams.image_longest_edge > 0 && hparams.image_min_pixels <= 0 && hparams.image_max_pixels <= 0) {
@@ -1197,7 +1197,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i
const int longest_edge = hparams.image_longest_edge;
const double aspect_ratio = (double) original_size.width / original_size.height;
- clip_image_size resized_size;
+ mtmd_image_size resized_size;
if (original_size.width >= original_size.height) {
resized_size.width = longest_edge;
resized_size.height = (int) (longest_edge / aspect_ratio);
@@ -1210,21 +1210,21 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i
const int grid_x = (resized_size.width + tile_size - 1) / tile_size;
const int grid_y = (resized_size.height + tile_size - 1) / tile_size;
- const clip_image_size refined_size = clip_image_size{grid_x * tile_size, grid_y * tile_size};
+ const mtmd_image_size refined_size = mtmd_image_size{grid_x * tile_size, grid_y * tile_size};
- clip_image_u8 resized_img;
+ mtmd_image_u8 resized_img;
img_tool::resize(img, resized_img, resized_size, hparams.image_resize_algo, PAD_NONE);
- clip_image_u8 refined_img;
+ mtmd_image_u8 refined_img;
img_tool::resize(resized_img, refined_img, refined_size, hparams.image_resize_algo, PAD_NONE);
- clip_image_u8 overview;
+ mtmd_image_u8 overview;
img_tool::resize(refined_img, overview, {tile_size, tile_size}, hparams.image_resize_algo, PAD_NONE);
- std::vector<clip_image_u8> slices;
+ std::vector<mtmd_image_u8> slices;
for (int y = 0; y < grid_y; y++) {
for (int x = 0; x < grid_x; x++) {
- clip_image_u8 slice;
+ mtmd_image_u8 slice;
img_tool::crop(refined_img, slice, x * tile_size, y * tile_size, tile_size, tile_size);
slices.push_back(std::move(slice));
}
@@ -1240,7 +1240,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i
return output;
}
- const clip_image_size refined_size = img_tool::calc_size_preserved_ratio(
+ const mtmd_image_size refined_size = img_tool::calc_size_preserved_ratio(
original_size,
{ hparams.image_size, std::max(0, hparams.image_min_pixels), std::max(0, hparams.image_max_pixels), hparams.image_longest_edge });
// LOG_INF("%s: original size: %d x %d, refined size: %d x %d\n",
@@ -1248,9 +1248,9 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i
// refined_size.width, refined_size.height);
mtmd_image_preprocessor_llava_uhd::slice_instructions instructions;
- instructions.overview_size = clip_image_size{hparams.image_size, hparams.image_size};
+ instructions.overview_size = mtmd_image_size{hparams.image_size, hparams.image_size};
instructions.refined_size = refined_size;
- instructions.grid_size = clip_image_size{
+ instructions.grid_size = mtmd_image_size{
static_cast<int>(std::ceil(static_cast<float>(refined_size.width) / hparams.image_size)),
static_cast<int>(std::ceil(static_cast<float>(refined_size.height) / hparams.image_size)),
};
@@ -1260,7 +1260,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i
instructions.slices.push_back(mtmd_image_preprocessor_llava_uhd::slice_coordinates{
/* x */x,
/* y */y,
- /* size */clip_image_size{
+ /* size */mtmd_image_size{
std::min(hparams.image_size, refined_size.width - x),
std::min(hparams.image_size, refined_size.height - y)
}
@@ -1281,7 +1281,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i
// mtmd_image_preprocessor_cohere2v
//
-mtmd_image_preproc_out mtmd_image_preprocessor_cohere2v::preprocess(const clip_image_u8 & img) const {
+mtmd_image_preproc_out mtmd_image_preprocessor_cohere2v::preprocess(const mtmd_image_u8 & img) const {
const auto inst = get_slice_instructions(img.get_size());
auto sliced = slice_image(img, inst);
@@ -1298,21 +1298,21 @@ mtmd_image_preproc_out mtmd_image_preprocessor_cohere2v::preprocess(const clip_i
return output;
}
-mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_cohere2v::get_slice_instructions(const clip_image_size & original_size) const {
+mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_cohere2v::get_slice_instructions(const mtmd_image_size & original_size) const {
const int tile = hparams.image_size;
// pick the grid with the least upscale; if all grids need downscale, pick the one with the least downscale
// grids are visited by tile count, then by width, same order as HF for ties
double best_down = -1.0;
double best_up = std::numeric_limits<double>::max();
- clip_image_size grid_down = { 1, 1 };
- clip_image_size grid_up = { 0, 0 };
+ mtmd_image_size grid_down = { 1, 1 };
+ mtmd_image_size grid_up = { 0, 0 };
for (int n = 1; n <= hparams.preproc_max_tiles; n++) {
for (int w = 1; w <= n; w++) {
if (n % w != 0) {
continue;
}
- const clip_image_size g = { w, n / w };
+ const mtmd_image_size g = { w, n / w };
const double scale = std::min(
(double) (g.width * tile) / original_size.width,
(double) (g.height * tile) / original_size.height);
@@ -1327,7 +1327,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_co
}
}
}
- const clip_image_size grid = grid_up.width > 0 ? grid_up : grid_down;
+ const mtmd_image_size grid = grid_up.width > 0 ? grid_up : grid_down;
slice_instructions inst;
inst.overview_size = { tile, tile };
@@ -1347,9 +1347,9 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_co
// mtmd_image_preprocessor_internvl
//
-mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const clip_image_u8 & img) const {
+mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const mtmd_image_u8 & img) const {
GGML_ASSERT(!hparams.image_res_candidates.empty());
- const clip_image_size original_size = img.get_size();
+ const mtmd_image_size original_size = img.get_size();
auto const inst = get_slice_instructions(original_size);
auto sliced = slice_image(img, inst);
@@ -1366,8 +1366,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const clip_i
// mtmd_image_preprocessor_deepseekocr
//
-std::vector<clip_image_size> mtmd_image_preprocessor_deepseekocr::get_target_ratios() const {
- std::vector<clip_image_size> ratios;
+std::vector<mtmd_image_size> mtmd_image_preprocessor_deepseekocr::get_target_ratios() const {
+ std::vector<mtmd_image_size> ratios;
for (int n = min_tiles; n <= max_tiles; n++) {
for (int w = 1; w <= n; w++) {
for (int h = 1; h <= n; h++) {
@@ -1387,19 +1387,19 @@ std::vector<clip_image_size> mtmd_image_preprocessor_deepseekocr::get_target_rat
}
}
}
- std::sort(ratios.begin(), ratios.end(), [](const clip_image_size & a, const clip_image_size & b) {
+ std::sort(ratios.begin(), ratios.end(), [](const mtmd_image_size & a, const mtmd_image_size & b) {
return a.width * a.height < b.width * b.height;
});
return ratios;
}
-clip_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio(
+mtmd_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio(
float aspect_ratio,
- const std::vector<clip_image_size> & target_ratios,
+ const std::vector<mtmd_image_size> & target_ratios,
int width,
int height) const {
float best_ratio_diff = std::numeric_limits<float>::max();
- clip_image_size best_ratio = { 1, 1 };
+ mtmd_image_size best_ratio = { 1, 1 };
const float area = static_cast<float>(width * height);
for (const auto & ratio : target_ratios) {
@@ -1482,7 +1482,7 @@ void mtmd_image_preprocessor_deepseek4v::safe_resize(int height, int width, int
}
// ref: load_image()
-mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip_image_u8 & img) const {
+mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const mtmd_image_u8 & img) const {
mtmd_image_preproc_out out;
const int p = hparams.patch_size;
@@ -1490,7 +1490,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip
const int max_n_token = hparams.dsv4_max_n_token;
const int max_wh = hparams.dsv4_max_wh_ratio;
- const clip_image_size orig = img.get_size();
+ const mtmd_image_size orig = img.get_size();
int width = orig.width;
int height = orig.height;
if (max_wh > 0 && width > height * max_wh) {
@@ -1502,11 +1502,11 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip
width = (int) (width * up);
height = (int) (height * up);
}
- int best_width = CLIP_ALIGN(width, p);
- int best_height = CLIP_ALIGN(height, p);
+ int best_width = MTMD_ALIGN(width, p);
+ int best_height = MTMD_ALIGN(height, p);
safe_resize(height, width, best_height, best_width, p, r, max_n_token);
- clip_image_u8 resized;
+ mtmd_image_u8 resized;
if (max_wh > 0 && orig.width >= max_wh * orig.height) {
// extreme aspect ratio: plain stretch resize, no padding
img_tool::resize(img, resized, {best_width, best_height}, hparams.image_resize_algo, PAD_NONE);
@@ -1540,8 +1540,8 @@ void mtmd_image_preprocessor_deepseek41v::plan_grid(int width, int height, int p
width = (int) (width * ratio);
height = (int) (height * ratio);
}
- best_width = CLIP_ALIGN(width, p);
- best_height = CLIP_ALIGN(height, p);
+ best_width = MTMD_ALIGN(width, p);
+ best_height = MTMD_ALIGN(height, p);
if (n_tokens(best_height, best_width) > max_n_token) {
// solve_resize_ratio: largest aspect-preserving size whose token block fits
const double ratio = (double) height / width;
@@ -1563,11 +1563,11 @@ void mtmd_image_preprocessor_deepseek41v::plan_grid(int width, int height, int p
}
}
-mtmd_image_preproc_out mtmd_image_preprocessor_deepseek41v::preprocess(const clip_image_u8 & img) const {
+mtmd_image_preproc_out mtmd_image_preprocessor_deepseek41v::preprocess(const mtmd_image_u8 & img) const {
mtmd_image_preproc_out out;
const int p = hparams.patch_size;
- const clip_image_size orig = img.get_size();
+ const mtmd_image_size orig = img.get_size();
int best_width = 0, best_height = 0;
plan_grid(orig.width, orig.height, p, hparams.n_merge, hparams.dsv4_max_n_token, hparams.image_min_pixels,
best_width, best_height);
@@ -1585,10 +1585,10 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek41v::preprocess(const cli
new_w = round_half_even((double) orig.width / orig.height * best_height);
}
}
- clip_image_u8 resized;
+ mtmd_image_u8 resized;
img_tool::resize(img, resized, {new_w, new_h}, hparams.image_resize_algo, PAD_NONE);
- clip_image_u8 padded;
+ mtmd_image_u8 padded;
if (new_w == best_width && new_h == best_height) {
padded = std::move(resized);
} else {
@@ -1614,14 +1614,14 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek41v::preprocess(const cli
return out;
}
-mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) const {
+mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const mtmd_image_u8 & img) const {
mtmd_image_preproc_out output;
int grid_w = 0;
int grid_h = 0;
const auto img_size = img.get_size();
// global view: aspect-preserving fit-and-pad to base_size
- clip_image_u8 padded;
+ mtmd_image_u8 padded;
img_tool::resize(img, padded,
{ base_size, base_size },
RESIZE_ALGO_BICUBIC,
@@ -1634,12 +1634,12 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const cli
if (img_size.width > tile_size || img_size.height > tile_size) {
const float aspect_ratio = static_cast<float>(img_size.width) / img_size.height;
const auto target_ratios = get_target_ratios();
- const clip_image_size grid =
+ const mtmd_image_size grid =
find_closest_aspect_ratio(aspect_ratio, target_ratios, img_size.width, img_size.height);
grid_w = grid.width;
grid_h = grid.height;
- clip_image_u8 refined;
+ mtmd_image_u8 refined;
img_tool::resize(img, refined, { tile_size * grid_w, tile_size * grid_h }, RESIZE_ALGO_BICUBIC,
PAD_NONE);
@@ -1648,7 +1648,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const cli
// concat all tiles in this row into a single image, along the H axis
// output image size: w = tile_size, h = tile_size * grid_w
// this is to ensure the whole row is always processed together
- clip_image_u8 row_img;
+ mtmd_image_u8 row_img;
row_img.set_size({tile_size, tile_size * grid_w}, false);
for (int col = 0; col < grid_w; col++) {
for (int py = 0; py < tile_size; py++) {
@@ -1661,7 +1661,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const cli
output.append(hparams, row_img, true);
} else {
for (int col = 0; col < grid_w; col++) {
- clip_image_u8 tile;
+ mtmd_image_u8 tile;
img_tool::crop(refined, tile, col * tile_size, row * tile_size, tile_size, tile_size);
output.append(hparams, tile, true);
}
@@ -1685,8 +1685,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const cli
//
void mtmd_image_preprocessor_step3vl::img_u8_resize_bilinear_to_f32(
- const clip_image_u8 & src,
- clip_image_f32 & dst,
+ const mtmd_image_u8 & src,
+ mtmd_image_f32 & dst,
int target_width,
int target_height,
const float mean[3],
@@ -1745,11 +1745,11 @@ void mtmd_image_preprocessor_step3vl::img_u8_resize_bilinear_to_f32(
dst.cpy_buf(local_buf);
}
-int mtmd_image_preprocessor_step3vl::get_image_longest_edge(const clip_hparams & params) {
+int mtmd_image_preprocessor_step3vl::get_image_longest_edge(const mtmd_hparams & params) {
return params.image_longest_edge > 0 ? params.image_longest_edge : default_image_longest_edge;
}
-int mtmd_image_preprocessor_step3vl::determine_window_size(const clip_hparams & params, int longer, int shorter) {
+int mtmd_image_preprocessor_step3vl::determine_window_size(const mtmd_hparams & params, int longer, int shorter) {
const int image_size = params.image_size;
const int crop_size = default_image_crop_size;
const float aspect_ratio = static_cast<float>(longer) / shorter;
@@ -1791,15 +1791,15 @@ std::vector<int> mtmd_image_preprocessor_step3vl::calc_grid(int length, int wind
return starts;
}
-clip_image_u8 mtmd_image_preprocessor_step3vl::prepare_image(const clip_image_u8 & img, const clip_hparams & params) {
- clip_image_u8 resized = img;
+mtmd_image_u8 mtmd_image_preprocessor_step3vl::prepare_image(const mtmd_image_u8 & img, const mtmd_hparams & params) {
+ mtmd_image_u8 resized = img;
const auto img_size = img.get_size();
const float aspect_ratio = img_size.height > 0 ? static_cast<float>(img_size.width) / img_size.height : 1.0f;
if (std::min(img_size.width, img_size.height) < 32 &&
(aspect_ratio > wide_aspect_ratio_limit ||
aspect_ratio < 1.0f / wide_aspect_ratio_limit)) {
const int square_size = std::max(img_size.width, img_size.height);
- clip_image_u8 padded;
+ mtmd_image_u8 padded;
padded.set_size({square_size, square_size}, false);
img_tool::fill(padded, {0, 0, 0});
img_tool::composite(padded, img, 0, 0);
@@ -1810,11 +1810,11 @@ clip_image_u8 mtmd_image_preprocessor_step3vl::prepare_image(const clip_image_u8
const auto resized_size = resized.get_size();
if (std::max(resized_size.width, resized_size.height) > max_image_size) {
const float scale = static_cast<float>(max_image_size) / std::max(resized_size.width, resized_size.height);
- const clip_image_size new_size = {
+ const mtmd_image_size new_size = {
std::max(1, static_cast<int>(std::floor(resized_size.width * scale))),
std::max(1, static_cast<int>(std::floor(resized_size.height * scale))),
};
- clip_image_u8 scaled;
+ mtmd_image_u8 scaled;
img_tool::resize(resized, scaled, new_size, RESIZE_ALGO_BILINEAR, PAD_NONE);
resized = std::move(scaled);
}
@@ -1822,8 +1822,8 @@ clip_image_u8 mtmd_image_preprocessor_step3vl::prepare_image(const clip_image_u8
return resized;
}
-clip_image_u8 mtmd_image_preprocessor_step3vl::crop_with_black_padding(const clip_image_u8 & image, int x, int y, int w, int h) {
- clip_image_u8 dst;
+mtmd_image_u8 mtmd_image_preprocessor_step3vl::crop_with_black_padding(const mtmd_image_u8 & image, int x, int y, int w, int h) {
+ mtmd_image_u8 dst;
dst.set_size({w, h}, false);
img_tool::fill(dst, {0, 0, 0});
@@ -1850,8 +1850,8 @@ clip_image_u8 mtmd_image_preprocessor_step3vl::crop_with_black_padding(const cli
}
mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step3vl::build_slice_instructions(
- const clip_hparams & params,
- const clip_image_size & prepared_size) {
+ const mtmd_hparams & params,
+ const mtmd_image_size & prepared_size) {
slice_instructions instructions;
instructions.overview_size = prepared_size;
@@ -1860,18 +1860,18 @@ mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step
std::max(prepared_size.width, prepared_size.height),
std::min(prepared_size.width, prepared_size.height));
if (window_size <= 0) {
- instructions.refined_size = clip_image_size{0, 0};
- instructions.grid_size = clip_image_size{0, 0};
+ instructions.refined_size = mtmd_image_size{0, 0};
+ instructions.grid_size = mtmd_image_size{0, 0};
return instructions;
}
const int crop_width = calc_crop_extent(prepared_size.width, window_size);
const int crop_height = calc_crop_extent(prepared_size.height, window_size);
- instructions.refined_size = clip_image_size{crop_width, crop_height};
+ instructions.refined_size = mtmd_image_size{crop_width, crop_height};
const auto xs = calc_grid(crop_width, window_size);
const auto ys = calc_grid(crop_height, window_size);
- instructions.grid_size = clip_image_size{
+ instructions.grid_size = mtmd_image_size{
static_cast<int>(xs.size()),
static_cast<int>(ys.size()),
};
@@ -1881,7 +1881,7 @@ mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step
instructions.slices.push_back(slice_coordinates{
/* x */ x,
/* y */ y,
- /* size */ clip_image_size{window_size, window_size},
+ /* size */ mtmd_image_size{window_size, window_size},
});
}
}
@@ -1889,8 +1889,8 @@ mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step
return instructions;
}
-mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_image_u8 & img) const {
- clip_image_u8 prepared = prepare_image(img, hparams);
+mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const mtmd_image_u8 & img) const {
+ mtmd_image_u8 prepared = prepare_image(img, hparams);
const auto instructions = build_slice_instructions(hparams, prepared.get_size());
mtmd_image_preproc_out output;
@@ -1909,10 +1909,10 @@ mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_im
return output;
}
- clip_image_u8 img_for_crop = prepared;
+ mtmd_image_u8 img_for_crop = prepared;
const auto prepared_size = prepared.get_size();
if (instructions.refined_size.width != prepared_size.width || instructions.refined_size.height != prepared_size.height) {
- clip_image_u8 refined;
+ mtmd_image_u8 refined;
img_tool::resize(prepared, refined, instructions.refined_size, RESIZE_ALGO_BILINEAR, PAD_NONE);
img_for_crop = std::move(refined);
}
@@ -1920,9 +1920,9 @@ mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_im
const int crop_size = default_image_crop_size;
for (const auto & slice : instructions.slices) {
// If the requested patch extends past the source image, pad the out-of-bounds area with black.
- clip_image_u8 patch = crop_with_black_padding(img_for_crop, slice.x, slice.y, slice.size.width, slice.size.height);
+ mtmd_image_u8 patch = crop_with_black_padding(img_for_crop, slice.x, slice.y, slice.size.width, slice.size.height);
- clip_image_f32 patch_f32;
+ mtmd_image_f32 patch_f32;
img_u8_resize_bilinear_to_f32(
patch,
patch_f32,
@@ -1943,7 +1943,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_im
// mtmd_image_preprocessor_youtuvl
//
-mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_image_u8 & img) const {
+mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const mtmd_image_u8 & img) const {
const int patch_size = hparams.patch_size; // typically 16
const int merge_size = hparams.n_merge; // typically 2
const int align_size = patch_size * merge_size; // 32
@@ -1981,10 +1981,10 @@ mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_im
}
}
- clip_image_size new_size = {target_width, target_height};
+ mtmd_image_size new_size = {target_width, target_height};
// Resize the image
- clip_image_u8 resized;
+ mtmd_image_u8 resized;
img_tool::resize(img, resized, new_size, hparams.image_resize_algo, hparams.image_resize_pad);
mtmd_image_preproc_out output;
@@ -1992,33 +1992,33 @@ mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_im
return output;
}
-mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_image_u8 & img) const {
+mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const mtmd_image_u8 & img) const {
GGML_ASSERT(!hparams.image_res_candidates.empty());
- const clip_image_size orig_size = img.get_size();
+ const mtmd_image_size orig_size = img.get_size();
const int tile_size = hparams.image_size;
GGML_ASSERT(tile_size > 0);
// llava-next always encodes an overview plus a grid of tiles, even for small images
- const clip_image_size refined_size = select_best_resolution(orig_size, hparams.image_res_candidates);
+ const mtmd_image_size refined_size = select_best_resolution(orig_size, hparams.image_res_candidates);
const int grid_x = refined_size.width / tile_size;
const int grid_y = refined_size.height / tile_size;
// the tiles are stacked on the Y axis, a big grid overflows the stacked image height
GGML_ASSERT(grid_x >= 0 && grid_x <= 1024 && grid_y >= 0 && grid_y <= 1024);
- clip_image_u8 overview;
+ mtmd_image_u8 overview;
img_tool::resize(img, overview, {tile_size, tile_size}, hparams.image_resize_algo_ov,
hparams.image_pad_ov, hparams.image_pad_color_ov);
- clip_image_u8 refined;
+ mtmd_image_u8 refined;
img_tool::resize(img, refined, refined_size, hparams.image_resize_algo_rf,
hparams.image_pad_rf, hparams.image_pad_color_rf);
// stack the overview and the tiles on the Y axis, so the whole grid goes through one graph
- clip_image_u8 stacked;
+ mtmd_image_u8 stacked;
stacked.set_size({tile_size, tile_size * (1 + grid_x * grid_y)}, false);
- auto copy_tile = [&](const clip_image_u8 & src, int src_x, int src_y, int dst_idx) {
+ auto copy_tile = [&](const mtmd_image_u8 & src, int src_x, int src_y, int dst_idx) {
for (int py = 0; py < tile_size; py++) {
for (int px = 0; px < tile_size; px++) {
stacked.set_pixel(px, dst_idx * tile_size + py, src.get_pixel(src_x + px, src_y + py));
@@ -2049,7 +2049,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_im
//
// Replicates transformers' get_aspect_ratio_preserving_size
-static clip_image_size muse_glimmer_grid_size(int img_w, int img_h, int patch_hw, int max_tokens) {
+static mtmd_image_size muse_glimmer_grid_size(int img_w, int img_h, int patch_hw, int max_tokens) {
double i_nph = (double) img_h / patch_hw;
double i_npw = (double) img_w / patch_hw;
const double ratio = i_nph > 0.0 ? i_npw / i_nph : 1.0;
@@ -2084,24 +2084,140 @@ static clip_image_size muse_glimmer_grid_size(int img_w, int img_h, int patch_hw
best_nph = std::max(1, (int) std::lround(i_nph));
best_npw = std::max(1, (int) std::lround(i_npw));
}
- return clip_image_size{ best_npw * patch_hw, best_nph * patch_hw };
+ return mtmd_image_size{ best_npw * patch_hw, best_nph * patch_hw };
}
-mtmd_image_preproc_out mtmd_image_preprocessor_muse_glimmer::preprocess(const clip_image_u8 & img) const {
+mtmd_image_preproc_out mtmd_image_preprocessor_muse_glimmer::preprocess(const mtmd_image_u8 & img) const {
const int patch_hw = hparams.patch_size * hparams.n_merge;
const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
GGML_ASSERT(patch_area > 0 && hparams.image_max_pixels > 0);
const int max_tokens = hparams.image_max_pixels / patch_area;
- const clip_image_size original_size = img.get_size();
- const clip_image_size target_size = muse_glimmer_grid_size(
+ const mtmd_image_size original_size = img.get_size();
+ const mtmd_image_size target_size = muse_glimmer_grid_size(
original_size.width, original_size.height, patch_hw, max_tokens);
// PIL resizes directly to (target_w, target_h) -- a stretch, no padding.
- clip_image_u8 resized_image;
+ mtmd_image_u8 resized_image;
img_tool::resize(img, resized_image, target_size, hparams.image_resize_algo, PAD_NONE);
mtmd_image_preproc_out output;
output.append(hparams, resized_image, true);
return output;
}
+
+//#define MTMD_DEBUG_FUNCTIONS
+
+#ifdef MTMD_DEBUG_FUNCTIONS
+static void mtmd_image_write_image_to_ppm(const mtmd_image_u8& img, const std::string& filename) {
+ std::ofstream file(filename, std::ios::binary);
+ if (!file.is_open()) {
+ LOG_ERR("Failed to open file for writing: %s\n", filename.c_str());
+ return;
+ }
+
+ // PPM header: P6 format, width, height, and max color value
+ const auto ppm_size = img.get_size();
+ file << "P6\n" << ppm_size.width << " " << ppm_size.height << "\n255\n";
+
+ // Write pixel data
+ const auto & ppm_buf = img.get_ro_buf();
+ for (size_t i = 0; i < ppm_buf.size(); i += 3) {
+ // PPM expects binary data in RGB format, which matches our image buffer
+ file.write(reinterpret_cast<const char*>(&ppm_buf[i]), 3);
+ }
+
+ file.close();
+}
+
+static void mtmd_image_save_to_bmp(const mtmd_image_u8& img, const std::string& filename) {
+ std::ofstream file(filename, std::ios::binary);
+ if (!file.is_open()) {
+ LOG_ERR("Failed to open file for writing: %s\n", filename.c_str());
+ return;
+ }
+
+ const auto bmp_size = img.get_size();
+ int fileSize = 54 + 3 * bmp_size.width * bmp_size.height; // File header + info header + pixel data
+ int bytesPerPixel = 3;
+ int widthInBytes = bmp_size.width * bytesPerPixel;
+ int paddingAmount = (4 - (widthInBytes % 4)) % 4;
+ int stride = widthInBytes + paddingAmount;
+
+ // Bitmap file header
+ unsigned char fileHeader[14] = {
+ 'B','M', // Signature
+ 0,0,0,0, // Image file size in bytes
+ 0,0,0,0, // Reserved
+ 54,0,0,0 // Start of pixel array
+ };
+
+ // Total file size
+ fileSize = 54 + (stride * bmp_size.height);
+ fileHeader[2] = (unsigned char)(fileSize);
+ fileHeader[3] = (unsigned char)(fileSize >> 8);
+ fileHeader[4] = (unsigned char)(fileSize >> 16);
+ fileHeader[5] = (unsigned char)(fileSize >> 24);
+
+ // Bitmap information header (BITMAPINFOHEADER)
+ unsigned char infoHeader[40] = {
+ 40,0,0,0, // Size of this header (40 bytes)
+ 0,0,0,0, // Image width
+ 0,0,0,0, // Image height
+ 1,0, // Number of color planes
+ 24,0, // Bits per pixel
+ 0,0,0,0, // No compression
+ 0,0,0,0, // Image size (can be 0 for no compression)
+ 0,0,0,0, // X pixels per meter (not specified)
+ 0,0,0,0, // Y pixels per meter (not specified)
+ 0,0,0,0, // Total colors (color table not used)
+ 0,0,0,0 // Important colors (all are important)
+ };
+
+ // Width and height in the information header
+ infoHeader[4] = (unsigned char)(bmp_size.width);
+ infoHeader[5] = (unsigned char)(bmp_size.width >> 8);
+ infoHeader[6] = (unsigned char)(bmp_size.width >> 16);
+ infoHeader[7] = (unsigned char)(bmp_size.width >> 24);
+ infoHeader[8] = (unsigned char)(bmp_size.height);
+ infoHeader[9] = (unsigned char)(bmp_size.height >> 8);
+ infoHeader[10] = (unsigned char)(bmp_size.height >> 16);
+ infoHeader[11] = (unsigned char)(bmp_size.height >> 24);
+
+ // Write file headers
+ file.write(reinterpret_cast<char*>(fileHeader), sizeof(fileHeader));
+ file.write(reinterpret_cast<char*>(infoHeader), sizeof(infoHeader));
+
+ // Pixel data
+ std::vector<unsigned char> padding(3, 0); // Max padding size to be added to each row
+ for (int y = bmp_size.height - 1; y >= 0; --y) { // BMP files are stored bottom-to-top
+ for (int x = 0; x < bmp_size.width; ++x) {
+ // Each pixel
+ const auto px = img.get_pixel(x, y);
+ unsigned char pixel[3] = {
+ px[2], // BMP stores pixels in BGR format
+ px[1],
+ px[0]
+ };
+ file.write(reinterpret_cast<char*>(pixel), 3);
+ }
+ // Write padding for the row
+ file.write(reinterpret_cast<char*>(padding.data()), paddingAmount);
+ }
+
+ file.close();
+}
+
+// debug function to convert f32 to u8
+static void mtmd_image_convert_f32_to_u8(const mtmd_image_f32& src, mtmd_image_u8& dst) {
+ dst.set_size(src.get_size(), false);
+ const auto & src_buf = src.get_ro_buf();
+ std::vector<uint8_t> dst_buf(src.n_elements());
+ for (size_t i = 0; i < src.n_elements(); ++i) {
+ dst_buf[i] = static_cast<uint8_t>(std::min(std::max(int(src_buf[i] * 255.0f), 0), 255));
+ }
+ dst.cpy_buf(dst_buf);
+}
+#endif
+
+
diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h
index 6eb49ca34..2cb9454f0 100644
--- a/tools/mtmd/mtmd-image.h
+++ b/tools/mtmd/mtmd-image.h
@@ -1,7 +1,7 @@
#pragma once
#include "ggml.h"
-#include "clip-model.h"
+#include "mtmd-model.h"
#include <vector>
#include <string>
@@ -9,18 +9,18 @@
#define MTMD_INTERNAL_HEADER
struct mtmd_image_preproc_out {
- std::vector<clip_image_f32> entries;
+ std::vector<mtmd_image_f32> entries;
// grid size is required for llava-uhd style models
- clip_image_f32 overview; // overview image (downscaled image)
+ mtmd_image_f32 overview; // overview image (downscaled image)
int grid_x = 0;
int grid_y = 0;
- void append(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized = true);
- void append(const clip_hparams & hparams, const std::vector<clip_image_u8> & imgs, bool normalized = true);
- void append(const clip_hparams & hparams, clip_image_f32 & img, bool normalized = true);
+ void append(const mtmd_hparams & hparams, const mtmd_image_u8 & img, bool normalized = true);
+ void append(const mtmd_hparams & hparams, const std::vector<mtmd_image_u8> & imgs, bool normalized = true);
+ void append(const mtmd_hparams & hparams, mtmd_image_f32 & img, bool normalized = true);
- void append_overview(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized = true);
+ void append_overview(const mtmd_hparams & hparams, const mtmd_image_u8 & img, bool normalized = true);
bool has_overview() const {
return overview.nx() > 0 || overview.ny() > 0;
}
@@ -28,17 +28,17 @@ struct mtmd_image_preproc_out {
// base class, models must inherit from this class
struct mtmd_image_preprocessor {
- const clip_hparams & hparams;
+ const mtmd_hparams & hparams;
- mtmd_image_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {}
- mtmd_image_preprocessor(const clip_hparams & hparams): hparams(hparams) {}
+ mtmd_image_preprocessor(const mtmd_model * model): hparams(model->hparams) {}
+ mtmd_image_preprocessor(const mtmd_hparams & hparams): hparams(hparams) {}
virtual ~mtmd_image_preprocessor() = default;
- virtual mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const = 0;
+ virtual mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const = 0;
// frames of a video, which the video helper may have already sized
- virtual mtmd_image_preproc_out preprocess_video_frame(const clip_image_u8 & img) const { return preprocess(img); }
+ virtual mtmd_image_preproc_out preprocess_video_frame(const mtmd_image_u8 & img) const { return preprocess(img); }
// frame size of a video with n_frames sampled frames, as the reference video processor; false if none
- virtual bool get_video_frame_size(int /*n_frames*/, int /*width*/, int /*height*/, clip_image_size & /*out*/) const { return false; }
+ virtual bool get_video_frame_size(int /*n_frames*/, int /*width*/, int /*height*/, mtmd_image_size & /*out*/) const { return false; }
};
/**
@@ -63,30 +63,30 @@ struct mtmd_image_preprocessor {
* NOTE: for the ordering of overview, set "ov_img_first" on the mtmd_context
*/
struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor {
- mtmd_image_preprocessor_llava_uhd(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
- mtmd_image_preprocessor_llava_uhd(const clip_hparams & hparams) : mtmd_image_preprocessor(hparams) {}
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+ mtmd_image_preprocessor_llava_uhd(const mtmd_model * model) : mtmd_image_preprocessor(model) {}
+ mtmd_image_preprocessor_llava_uhd(const mtmd_hparams & hparams) : mtmd_image_preprocessor(hparams) {}
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
struct slice_coordinates {
int x;
int y;
- clip_image_size size;
+ mtmd_image_size size;
};
struct slice_instructions {
- clip_image_size overview_size; // size of downscaled image
- clip_image_size refined_size; // size of image right before slicing (must be multiple of slice size)
- clip_image_size grid_size; // grid_size.width * grid_size.height = number of slices
+ mtmd_image_size overview_size; // size of downscaled image
+ mtmd_image_size refined_size; // size of image right before slicing (must be multiple of slice size)
+ mtmd_image_size grid_size; // grid_size.width * grid_size.height = number of slices
std::vector<slice_coordinates> slices;
};
- virtual slice_instructions get_slice_instructions(const clip_image_size & original_size) const;
+ virtual slice_instructions get_slice_instructions(const mtmd_image_size & original_size) const;
struct slice_output {
- clip_image_u8 overview;
- std::vector<clip_image_u8> slices;
+ mtmd_image_u8 overview;
+ std::vector<mtmd_image_u8> slices;
};
- slice_output slice_image(const clip_image_u8 & img, const slice_instructions & inst) const;
+ slice_output slice_image(const mtmd_image_u8 & img, const slice_instructions & inst) const;
protected:
// align slices to a multiple of the merger factor (integer merger tokens per slice)
@@ -100,7 +100,7 @@ protected:
return static_cast<int>(std::round(v));
}
- clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false) const;
+ mtmd_image_size get_best_resize(const mtmd_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false) const;
/**
* Selects the best resolution from a list of possible resolutions based on the original size.
@@ -117,68 +117,68 @@ protected:
* @param possible_resolutions A list of possible resolutions
* @return The best fit resolution
*/
- clip_image_size select_best_resolution(const clip_image_size & original_size, const std::vector<clip_image_size> & possible_resolutions) const;
+ mtmd_image_size select_best_resolution(const mtmd_image_size & original_size, const std::vector<mtmd_image_size> & possible_resolutions) const;
private:
- clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) const;
+ mtmd_image_size resize_maintain_aspect_ratio(const mtmd_image_size & orig, const mtmd_image_size & target_max) const;
int ensure_divide(int length, int patch_size) const;
- clip_image_size get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false) const;
- clip_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const;
+ mtmd_image_size get_refine_size(const mtmd_image_size & original_size, const mtmd_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false) const;
+ mtmd_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const;
};
// downscale or upscale the input image to fixed size
struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor {
- mtmd_image_preprocessor_fixed_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+ mtmd_image_preprocessor_fixed_size(const mtmd_model * model) : mtmd_image_preprocessor(model) {}
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
};
// resize image to multiple of patch_size*n_merge, while preserving aspect ratio
// if image_resize_pad is true, the resized image will be padded, otherwise it will be either stretched or center-cropped depending on image_resize_pad
// this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etc
struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor {
- mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
- mtmd_image_preprocessor_dyn_size(const clip_hparams & hparams) : mtmd_image_preprocessor(hparams) {}
- static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size);
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
- virtual clip_image_size target_size(const clip_image_size & original_size) const { return get_target_size(hparams, original_size); }
+ mtmd_image_preprocessor_dyn_size(const mtmd_model * model) : mtmd_image_preprocessor(model) {}
+ mtmd_image_preprocessor_dyn_size(const mtmd_hparams & hparams) : mtmd_image_preprocessor(hparams) {}
+ static mtmd_image_size get_target_size(const mtmd_hparams & hparams, const mtmd_image_size & original_size);
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
+ virtual mtmd_image_size target_size(const mtmd_image_size & original_size) const { return get_target_size(hparams, original_size); }
};
// Qwen3-VL, same as dyn_size for images, video frames are sized as Qwen3VLVideoProcessor
struct mtmd_image_preprocessor_qwen3vl : mtmd_image_preprocessor_dyn_size {
using mtmd_image_preprocessor_dyn_size::mtmd_image_preprocessor_dyn_size;
// the video helper sizes the frames (get_video_frame_size), they are only snapped to the grid
- mtmd_image_preproc_out preprocess_video_frame(const clip_image_u8 & img) const override;
- bool get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const override;
+ mtmd_image_preproc_out preprocess_video_frame(const mtmd_image_u8 & img) const override;
+ bool get_video_frame_size(int n_frames, int width, int height, mtmd_image_size & out) const override;
- mtmd_image_preproc_out resize_to(const clip_image_u8 & img, const clip_image_size & size) const;
+ mtmd_image_preproc_out resize_to(const mtmd_image_u8 & img, const mtmd_image_size & size) const;
// with temporal_factor > 1 it is the Qwen3-VL video variant over n_frames
- static clip_image_size smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels,
+ static mtmd_image_size smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels,
int n_frames = 1, int temporal_factor = 1);
};
// LFM3-VL, same as qwen3vl but images are sized exactly as the transformers Qwen2-VL smart_resize
struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_qwen3vl {
using mtmd_image_preprocessor_qwen3vl::mtmd_image_preprocessor_qwen3vl;
- clip_image_size target_size(const clip_image_size & original_size) const override;
+ mtmd_image_size target_size(const mtmd_image_size & original_size) const override;
};
// GLM 5.3 flash, similar to dyn_size, but each edge is aligned up to patch_size*n_merge, and max budget is met by a search over the height with the width scaled proportionally
struct mtmd_image_preprocessor_glm5v : mtmd_image_preprocessor {
- mtmd_image_preprocessor_glm5v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+ mtmd_image_preprocessor_glm5v(const mtmd_model * model) : mtmd_image_preprocessor(model) {}
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
};
// similar to mtmd_image_preprocessor_dyn_size, but cap the longest edge at hparams.image_longest_edge, while preserving aspect ratio
struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor {
- mtmd_image_preprocessor_longest_edge(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
- static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size);
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+ mtmd_image_preprocessor_longest_edge(const mtmd_model * model) : mtmd_image_preprocessor(model) {}
+ static mtmd_image_size get_target_size(const mtmd_hparams & hparams, const mtmd_image_size & original_size);
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
};
// ref: inference/image_processor.py in the HF repo (DeepSeek-V4-Flash-Vision)
struct mtmd_image_preprocessor_deepseek4v : mtmd_image_preprocessor {
- mtmd_image_preprocessor_deepseek4v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+ mtmd_image_preprocessor_deepseek4v(const mtmd_model * model) : mtmd_image_preprocessor(model) {}
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
private:
struct grid_info {
@@ -193,8 +193,8 @@ private:
// ref: inference/image_processor.py in the HF repo (DeepSeek-V4.1-Flash): plan_image_grid / load_image
struct mtmd_image_preprocessor_deepseek41v : mtmd_image_preprocessor {
- mtmd_image_preprocessor_deepseek41v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+ mtmd_image_preprocessor_deepseek41v(const mtmd_model * model) : mtmd_image_preprocessor(model) {}
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
// target pixel size (multiples of patch_size) for an image of the given size, exposed for tests
static void plan_grid(int width, int height, int p, int r, int max_n_token, int min_pixels,
int & best_width, int & best_height);
@@ -203,7 +203,7 @@ struct mtmd_image_preprocessor_deepseek41v : mtmd_image_preprocessor {
// custom llava-uhd slicing logic for MiniCPM-V
struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd {
using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
- slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;
+ slice_instructions get_slice_instructions(const mtmd_image_size & original_size) const override;
protected:
// always patch_size * 4, even in 4x mode (the 2x2 vit_merger slot stays)
@@ -242,50 +242,50 @@ protected:
// ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py
struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {
using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
- slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
+ slice_instructions get_slice_instructions(const mtmd_image_size & original_size) const override;
- static bool should_tile(const clip_hparams & hparams, const clip_image_size & original_size);
- static clip_image_size smart_resize(const clip_hparams & hparams, const clip_image_size & original_size);
- static clip_image_size get_grid_layout(const clip_hparams & hparams, int height, int width);
+ static bool should_tile(const mtmd_hparams & hparams, const mtmd_image_size & original_size);
+ static mtmd_image_size smart_resize(const mtmd_hparams & hparams, const mtmd_image_size & original_size);
+ static mtmd_image_size get_grid_layout(const mtmd_hparams & hparams, int height, int width);
private:
- static clip_image_size find_closest_aspect_ratio(
- const clip_hparams & hparams,
+ static mtmd_image_size find_closest_aspect_ratio(
+ const mtmd_hparams & hparams,
double aspect_ratio,
- const std::vector<clip_image_size> & target_ratios,
+ const std::vector<mtmd_image_size> & target_ratios,
int width, int height);
- static std::vector<clip_image_size> get_target_ratios(const clip_hparams & hparams);
+ static std::vector<mtmd_image_size> get_target_ratios(const mtmd_hparams & hparams);
};
struct mtmd_image_preprocessor_idefics3 : mtmd_image_preprocessor_llava_uhd {
- mtmd_image_preprocessor_idefics3(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+ mtmd_image_preprocessor_idefics3(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {}
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
};
struct mtmd_image_preprocessor_internvl : mtmd_image_preprocessor_llava_uhd {
- mtmd_image_preprocessor_internvl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+ mtmd_image_preprocessor_internvl(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {}
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
};
// stretch the image to a grid of tiles, add a thumbnail if there is more than 1 tile
// ref: https://github.com/huggingface/transformers/blob/main/src/transformers/models/cohere2_vision/image_processing_cohere2_vision.py
struct mtmd_image_preprocessor_cohere2v : mtmd_image_preprocessor_llava_uhd {
- mtmd_image_preprocessor_cohere2v(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
- slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;
+ mtmd_image_preprocessor_cohere2v(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {}
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
+ slice_instructions get_slice_instructions(const mtmd_image_size & original_size) const override;
};
// DeepSeek-OCR (v1/v2) global view + optional local tile grid
struct mtmd_image_preprocessor_deepseekocr : mtmd_image_preprocessor {
- mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx)
- : mtmd_image_preprocessor(ctx),
- fuse_row(clip_get_projector_type(ctx) == PROJECTOR_TYPE_DEEPSEEKOCR),
+ mtmd_image_preprocessor_deepseekocr(const mtmd_model * model)
+ : mtmd_image_preprocessor(model),
+ fuse_row(model->proj_type == PROJECTOR_TYPE_DEEPSEEKOCR),
base_size(hparams.image_size),
tile_size(hparams.preproc_tile_size),
min_tiles(hparams.preproc_min_tiles),
max_tiles(hparams.preproc_max_tiles) {}
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
private:
bool fuse_row; // v1 fuses a tile-row into one image; v2 keeps tiles separate
@@ -294,19 +294,19 @@ private:
int min_tiles;
int max_tiles;
- std::vector<clip_image_size> get_target_ratios() const;
- clip_image_size find_closest_aspect_ratio(
+ std::vector<mtmd_image_size> get_target_ratios() const;
+ mtmd_image_size find_closest_aspect_ratio(
float aspect_ratio,
- const std::vector<clip_image_size> & target_ratios,
+ const std::vector<mtmd_image_size> & target_ratios,
int width, int height) const;
};
// custom image preprocessing for Step3VL
// ref: https://huggingface.co/stepfun-ai/Step3-VL-10B/blob/main/processing_step3.py
struct mtmd_image_preprocessor_step3vl : mtmd_image_preprocessor_llava_uhd {
- mtmd_image_preprocessor_step3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
- static slice_instructions build_slice_instructions(const clip_hparams & params, const clip_image_size & prepared_size);
+ mtmd_image_preprocessor_step3vl(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {}
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
+ static slice_instructions build_slice_instructions(const mtmd_hparams & params, const mtmd_image_size & prepared_size);
private:
static constexpr int default_image_longest_edge = 3024;
@@ -316,33 +316,33 @@ private:
static constexpr float crop_rounding_threshold = 0.2f;
void img_u8_resize_bilinear_to_f32(
- const clip_image_u8 & src,
- clip_image_f32 & dst,
+ const mtmd_image_u8 & src,
+ mtmd_image_f32 & dst,
int target_width,
int target_height,
const float mean[3],
const float std[3]) const;
- static int get_image_longest_edge(const clip_hparams & params);
- static int determine_window_size(const clip_hparams & params, int longer, int shorter);
+ static int get_image_longest_edge(const mtmd_hparams & params);
+ static int determine_window_size(const mtmd_hparams & params, int longer, int shorter);
static int calc_crop_extent(int length, int window_size);
static std::vector<int> calc_grid(int length, int window_size);
- static clip_image_u8 prepare_image(const clip_image_u8 & img, const clip_hparams & params);
- static clip_image_u8 crop_with_black_padding(const clip_image_u8 & image, int x, int y, int w, int h);
+ static mtmd_image_u8 prepare_image(const mtmd_image_u8 & img, const mtmd_hparams & params);
+ static mtmd_image_u8 crop_with_black_padding(const mtmd_image_u8 & image, int x, int y, int w, int h);
};
struct mtmd_image_preprocessor_youtuvl : mtmd_image_preprocessor {
- mtmd_image_preprocessor_youtuvl(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+ mtmd_image_preprocessor_youtuvl(const mtmd_model * model) : mtmd_image_preprocessor(model) {}
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
};
-// llava-next "anyres": stacks the overview and all tiles into one image, assembled by clip in a single graph
+// llava-next "anyres": stacks the overview and all tiles into one image, assembled in a single graph
struct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd {
- mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+ mtmd_image_preprocessor_granite(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {}
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
};
// pick the patch grid closest to the input aspect ratio under the per-image token cap, stretch-resize.
struct mtmd_image_preprocessor_muse_glimmer : mtmd_image_preprocessor {
- mtmd_image_preprocessor_muse_glimmer(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+ mtmd_image_preprocessor_muse_glimmer(const mtmd_model * model) : mtmd_image_preprocessor(model) {}
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
};
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/mtmd-impl.h
similarity index 92%
rename from tools/mtmd/clip-impl.h
rename to tools/mtmd/mtmd-impl.h
index 8a4098da8..e245f4914 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/mtmd-impl.h
@@ -3,7 +3,7 @@
#include "ggml.h"
#include "ggml-cpp.h"
#include "gguf.h"
-#include "clip.h"
+#include "mtmd.h"
#include <array>
#include <climits>
@@ -25,7 +25,7 @@
#include <windows.h>
#endif
-// Internal header for clip.cpp
+// !!! Internal header, to be used by mtmd only !!!
#define MTMD_INTERNAL_HEADER
@@ -490,42 +490,7 @@
#define TN_A_FUSION_KERNEL "a.fusion.spk_kernel"
// align x to upper multiple of n
-#define CLIP_ALIGN(x, n) ((((x) + (n) - 1) / (n)) * (n))
-
-// sliding window KV cache of a causal audio encoder, see clip_enc_state_init()
-struct clip_enc_state {
- int32_t n_ctx = 0; // ring slots: attention window + frames per call
- int32_t n_past = 0; // absolute position of the next frame
-
- std::vector<int32_t> slot_pos; // absolute position held by each slot, -1 if empty
- std::vector<float> stem_tail; // last conv stem output frame, empty on a cold start
-
- ggml_context_ptr ctx_data;
- ggml_backend_buffer_ptr buf;
- std::vector<ggml_tensor *> k; // per layer [n_embd_kv, n_ctx]
- std::vector<ggml_tensor *> v;
-
- // lfm3a: same order as list_lfm3a_state_slots(), F32 [ne0, ne1], updated in place
- std::vector<ggml_tensor *> slots;
-
- // nemotron transcribe: the window's normalized log-mel, cache and FIFO rows in; its speaker rows and predictions out
- std::vector<float> mel_norm;
- std::vector<float> spk_prev;
- std::vector<float> spk_chunk, spk_preds;
-};
-
-// per-sequence state of audio generation, see clip_gen_state_init()
-struct clip_gen_state {
- std::mt19937 rng;
-
- ggml_context_ptr ctx_data;
- ggml_backend_buffer_ptr buf;
- std::vector<ggml_tensor *> slots; // same order as list_gen_state_slots(), F32 [ne0, ne1]
-};
-
-// forward declaration
-// TODO: improve this later
-struct clip_ctx;
+#define MTMD_ALIGN(x, n) ((((x) + (n) - 1) / (n)) * (n))
enum projector_type {
PROJECTOR_TYPE_MLP,
@@ -555,7 +520,7 @@ enum projector_type {
PROJECTOR_TYPE_QWEN2A,
PROJECTOR_TYPE_QWEN3A,
PROJECTOR_TYPE_GLMA,
- PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on clip_ctx
+ PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on mtmd_subcontext
PROJECTOR_TYPE_VOXTRAL,
PROJECTOR_TYPE_VOXTRAL_RT,
PROJECTOR_TYPE_MERALION,
@@ -699,7 +664,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_CHATTERBOX_SPKENC, "chatterbox_spkenc"},
};
-static projector_type clip_projector_type_from_string(const std::string & str) {
+static projector_type projector_type_from_string(const std::string & str) {
for (const auto & pair : PROJECTOR_TYPE_NAMES) {
if (pair.second == str) {
return pair.first;
@@ -708,13 +673,30 @@ static projector_type clip_projector_type_from_string(const std::string & str) {
return PROJECTOR_TYPE_UNKNOWN;
}
+struct mtmd_image_size {
+ int width;
+ int height;
+ bool operator==(const mtmd_image_size & other) const {
+ return width == other.width && height == other.height;
+ }
+ bool operator!=(const mtmd_image_size & other) const {
+ return !(*this == other);
+ }
+ int area() const {
+ // avoid overflow when computing area
+ GGML_ASSERT(width >= 0 && width <= 46000);
+ GGML_ASSERT(height >= 0 && height <= 46000);
+ return width * height;
+ }
+};
+
// RGB uint8 image
-struct clip_image_u8 {
- clip_image_size get_size() const {
+struct mtmd_image_u8 {
+ mtmd_image_size get_size() const {
return { nx, ny };
}
- void set_size(clip_image_size size, bool is_placeholder) {
+ void set_size(mtmd_image_size size, bool is_placeholder) {
nx = size.width;
ny = size.height;
if (is_placeholder) {
@@ -730,7 +712,7 @@ struct clip_image_u8 {
const std::vector<uint8_t> & get_ro_buf() const {
if (is_placeholder()) {
- throw std::runtime_error("this clip_image_u8 is a placeholder");
+ throw std::runtime_error("this mtmd_image_u8 is a placeholder");
}
return buf;
}
@@ -777,20 +759,20 @@ struct clip_image_u8 {
struct mtmd_serialization; // forward declaration
// separators appended after the image tokens of one entry, as rows of v.tok_embd_sep
-enum clip_suffix_type : int32_t {
- CLIP_SUFFIX_NONE = 0,
+enum mtmd_suffix_type : int32_t {
+ MTMD_SUFFIX_NONE = 0,
// MiniCPM-V 4.7 tiles
- CLIP_SUFFIX_MINICPMV_OV, // </image>
- CLIP_SUFFIX_MINICPMV_OV_SLICE, // </image><slice>
- CLIP_SUFFIX_MINICPMV_SLICE, // </slice><slice>
- CLIP_SUFFIX_MINICPMV_ROW_END, // </slice>\n<slice>
- CLIP_SUFFIX_MINICPMV_LAST, // </slice>
- CLIP_SUFFIX_COUNT,
+ MTMD_SUFFIX_MINICPMV_OV, // </image>
+ MTMD_SUFFIX_MINICPMV_OV_SLICE, // </image><slice>
+ MTMD_SUFFIX_MINICPMV_SLICE, // </slice><slice>
+ MTMD_SUFFIX_MINICPMV_ROW_END, // </slice>\n<slice>
+ MTMD_SUFFIX_MINICPMV_LAST, // </slice>
+ MTMD_SUFFIX_COUNT,
};
// rows of v.tok_embd_sep for each suffix type
// MiniCPM-V 4.7 rows (set by the converter): 0 = </image>, 1 = <slice>, 2 = </slice>, 3 = \n
-static inline const std::vector<int> & clip_suffix_rows(clip_suffix_type type) {
+static inline const std::vector<int> & mtmd_suffix_rows(mtmd_suffix_type type) {
static const std::vector<int> none;
static const std::vector<int> minicpmv_ov = { 0 };
static const std::vector<int> minicpmv_ov_slice = { 0, 1 };
@@ -798,12 +780,12 @@ static inline const std::vector<int> & clip_suffix_rows(clip_suffix_type type) {
static const std::vector<int> minicpmv_row_end = { 2, 3, 1 };
static const std::vector<int> minicpmv_last = { 2 };
switch (type) {
- case CLIP_SUFFIX_NONE: return none;
- case CLIP_SUFFIX_MINICPMV_OV: return minicpmv_ov;
- case CLIP_SUFFIX_MINICPMV_OV_SLICE: return minicpmv_ov_slice;
- case CLIP_SUFFIX_MINICPMV_SLICE: return minicpmv_slice;
- case CLIP_SUFFIX_MINICPMV_ROW_END: return minicpmv_row_end;
- case CLIP_SUFFIX_MINICPMV_LAST: return minicpmv_last;
+ case MTMD_SUFFIX_NONE: return none;
+ case MTMD_SUFFIX_MINICPMV_OV: return minicpmv_ov;
+ case MTMD_SUFFIX_MINICPMV_OV_SLICE: return minicpmv_ov_slice;
+ case MTMD_SUFFIX_MINICPMV_SLICE: return minicpmv_slice;
+ case MTMD_SUFFIX_MINICPMV_ROW_END: return minicpmv_row_end;
+ case MTMD_SUFFIX_MINICPMV_LAST: return minicpmv_last;
default: GGML_ABORT("invalid suffix type");
}
}
@@ -814,7 +796,7 @@ static inline const std::vector<int> & clip_suffix_rows(clip_suffix_type type) {
// Memory layout: RGBRGB...RGBRGB... (nt times)
// For audio, only one channel is used, buf.size() == nx*ny
// nx will be n_frames and ny will be n_mel
-struct clip_image_f32 {
+struct mtmd_image_f32 {
// marks the global view in e.g., DeepSeek-OCR Models
bool add_viewsep = false;
// appends a learned newline (or EOI) token after the image
@@ -824,7 +806,7 @@ struct clip_image_f32 {
// depends on the chunk position, set at tokenize time (see mtmd_tokenizer::add_media)
int32_t lead_pad = 0;
// separators appended after the image tokens
- clip_suffix_type suffix_type = CLIP_SUFFIX_NONE;
+ mtmd_suffix_type suffix_type = MTMD_SUFFIX_NONE;
// tile grid of the image group this entry belongs to
// llava-next "anyres" (Granite4 Vision): the whole grid is encoded and assembled in a single graph
@@ -841,14 +823,14 @@ struct clip_image_f32 {
};
anyres_info anyres;
- clip_image_size get_size() const {
+ mtmd_image_size get_size() const {
return { nx_, ny_ };
}
int nx() const { return nx_; }
int ny() const { return ny_; }
- void set_size(clip_image_size size, bool is_placeholder, bool is_audio) {
+ void set_size(mtmd_image_size size, bool is_placeholder, bool is_audio) {
nx_ = size.width;
ny_ = size.height;
if (is_placeholder) {
@@ -866,7 +848,7 @@ struct clip_image_f32 {
buf = new_buf;
}
- void from_u8(const clip_image_u8 & img) {
+ void from_u8(const mtmd_image_u8 & img) {
auto size = img.get_size();
nx_ = size.width;
ny_ = size.height;
@@ -898,7 +880,7 @@ struct clip_image_f32 {
const std::vector<float> & get_ro_buf() const {
if (is_placeholder()) {
- throw std::runtime_error("this clip_image_f32 is a placeholder");
+ throw std::runtime_error("this mtmd_image_f32 is a placeholder");
}
return buf;
}
@@ -924,7 +906,7 @@ struct clip_image_f32 {
// token area kept after removing the padding added by the anyres resize
// ref: https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/llava_next/modeling_llava_next.py#L109
-static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_h,
+static inline void mtmd_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_h,
int & off_x, int & off_y, int & out_w, int & out_h) {
off_x = 0;
off_y = 0;
@@ -966,21 +948,21 @@ static inline dsv4_block_layout dsv4_get_block_layout(int n_llm_w, int n_llm_h,
// logging
//
-static void clip_log_callback_default(enum ggml_log_level level, const char * text, void * user_data) {
+static void mtmd_log_callback_default(enum ggml_log_level level, const char * text, void * user_data) {
(void) level;
(void) user_data;
fputs(text, stderr);
fflush(stderr);
}
-struct clip_logger_state {
+struct mtmd_logger_state {
ggml_log_callback log_callback;
void * log_callback_user_data;
};
-extern struct clip_logger_state g_logger_state;
+extern struct mtmd_logger_state g_logger_state;
-static void clip_log_internal_v(enum ggml_log_level level, const char * format, va_list args) {
+static void mtmd_log_internal_v(enum ggml_log_level level, const char * format, va_list args) {
if (format == NULL) {
return;
}
@@ -1000,31 +982,31 @@ static void clip_log_internal_v(enum ggml_log_level level, const char * format,
va_end(args_copy);
}
-static void clip_log_internal(enum ggml_log_level level, const char * format, ...) {
+static void mtmd_log_internal(enum ggml_log_level level, const char * format, ...) {
va_list args;
va_start(args, format);
- clip_log_internal_v(level, format, args);
+ mtmd_log_internal_v(level, format, args);
va_end(args);
}
-#define LOG_TRC(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
-#define LOG_DBG(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
-#define LOG_INF(...) clip_log_internal(GGML_LOG_LEVEL_INFO, __VA_ARGS__)
-#define LOG_WRN(...) clip_log_internal(GGML_LOG_LEVEL_WARN, __VA_ARGS__)
-#define LOG_ERR(...) clip_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
-#define LOG_CNT(...) clip_log_internal(GGML_LOG_LEVEL_CONT, __VA_ARGS__)
+#define LOG_TRC(...) mtmd_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
+#define LOG_DBG(...) mtmd_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
+#define LOG_INF(...) mtmd_log_internal(GGML_LOG_LEVEL_INFO, __VA_ARGS__)
+#define LOG_WRN(...) mtmd_log_internal(GGML_LOG_LEVEL_WARN, __VA_ARGS__)
+#define LOG_ERR(...) mtmd_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
+#define LOG_CNT(...) mtmd_log_internal(GGML_LOG_LEVEL_CONT, __VA_ARGS__)
//
// cpp wrappers
//
-struct clip_image_f32_batch {
- std::vector<clip_image_f32> entries;
+struct mtmd_image_f32_batch {
+ std::vector<mtmd_image_f32> entries;
bool is_audio = false;
bool is_video = false;
- clip_image_f32_batch clone() const {
- clip_image_f32_batch new_batch{
+ mtmd_image_f32_batch clone() const {
+ mtmd_image_f32_batch new_batch{
/* entries */ {},
/* is_audio */ is_audio,
/* is_video */ is_video,
@@ -1244,9 +1226,3 @@ static void print_tensor_data(ggml_tensor * t, uint8_t * data, int64_t n) {
}
}
-//
-// API used internally with mtmd
-//
-
-projector_type clip_get_projector_type(const struct clip_ctx * ctx);
-void clip_set_debug_output_embeddings(struct clip_ctx * ctx, bool debug);
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/mtmd-model.cpp
similarity index 57%
rename from tools/mtmd/clip.cpp
rename to tools/mtmd/mtmd-model.cpp
index 0a3b3c98d..ae7a942ac 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -1,15 +1,11 @@
-#include "clip.h"
-#include "clip-impl.h"
-#include "clip-model.h"
-#include "clip-graph.h"
+#include "mtmd-impl.h"
+#include "mtmd-model.h"
#include "models/models.h"
-#include "mtmd-audio.h"
#include "ggml.h"
#include "ggml-cpp.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
-#include "ggml-cpu.h"
#include "gguf.h"
#include <algorithm>
@@ -19,7 +15,6 @@
#include <cstring>
#include <fstream>
#include <map>
-#include <memory>
#include <random>
#include <stdexcept>
#include <unordered_set>
@@ -30,1553 +25,245 @@
#include <functional>
#include <float.h>
-struct clip_logger_state g_logger_state = {clip_log_callback_default, NULL};
-
-//#define CLIP_DEBUG_FUNCTIONS
+//
+// mtmd_model_loader
+//
-#ifdef CLIP_DEBUG_FUNCTIONS
-static void clip_image_write_image_to_ppm(const clip_image_u8& img, const std::string& filename) {
- std::ofstream file(filename, std::ios::binary);
- if (!file.is_open()) {
- LOG_ERR("Failed to open file for writing: %s\n", filename.c_str());
- return;
- }
+struct mtmd_model_loader {
+ ggml_context_ptr ctx_meta;
+ gguf_context_ptr ctx_gguf;
- // PPM header: P6 format, width, height, and max color value
- const auto ppm_size = img.get_size();
- file << "P6\n" << ppm_size.width << " " << ppm_size.height << "\n255\n";
+ std::string fname;
- // Write pixel data
- const auto & ppm_buf = img.get_ro_buf();
- for (size_t i = 0; i < ppm_buf.size(); i += 3) {
- // PPM expects binary data in RGB format, which matches our image buffer
- file.write(reinterpret_cast<const char*>(&ppm_buf[i]), 3);
- }
+ size_t model_size = 0; // in bytes
- file.close();
-}
+ bool has_vision = false;
+ bool has_audio = false;
+ bool has_gen_audio = false;
-static void clip_image_save_to_bmp(const clip_image_u8& img, const std::string& filename) {
- std::ofstream file(filename, std::ios::binary);
- if (!file.is_open()) {
- LOG_ERR("Failed to open file for writing: %s\n", filename.c_str());
- return;
- }
+ mtmd_progress_callback progress_callback = nullptr;
+ void * progress_callback_user_data = nullptr;
- const auto bmp_size = img.get_size();
- int fileSize = 54 + 3 * bmp_size.width * bmp_size.height; // File header + info header + pixel data
- int bytesPerPixel = 3;
- int widthInBytes = bmp_size.width * bytesPerPixel;
- int paddingAmount = (4 - (widthInBytes % 4)) % 4;
- int stride = widthInBytes + paddingAmount;
-
- // Bitmap file header
- unsigned char fileHeader[14] = {
- 'B','M', // Signature
- 0,0,0,0, // Image file size in bytes
- 0,0,0,0, // Reserved
- 54,0,0,0 // Start of pixel array
- };
+ mtmd_model_loader(const char * fname,
+ bool skip_tensors = false,
+ mtmd_progress_callback progress_cb = nullptr,
+ void * progress_user_data = nullptr)
+ : fname(fname),
+ progress_callback(progress_cb),
+ progress_callback_user_data(progress_user_data) {
+ struct ggml_context * meta = nullptr;
- // Total file size
- fileSize = 54 + (stride * bmp_size.height);
- fileHeader[2] = (unsigned char)(fileSize);
- fileHeader[3] = (unsigned char)(fileSize >> 8);
- fileHeader[4] = (unsigned char)(fileSize >> 16);
- fileHeader[5] = (unsigned char)(fileSize >> 24);
-
- // Bitmap information header (BITMAPINFOHEADER)
- unsigned char infoHeader[40] = {
- 40,0,0,0, // Size of this header (40 bytes)
- 0,0,0,0, // Image width
- 0,0,0,0, // Image height
- 1,0, // Number of color planes
- 24,0, // Bits per pixel
- 0,0,0,0, // No compression
- 0,0,0,0, // Image size (can be 0 for no compression)
- 0,0,0,0, // X pixels per meter (not specified)
- 0,0,0,0, // Y pixels per meter (not specified)
- 0,0,0,0, // Total colors (color table not used)
- 0,0,0,0 // Important colors (all are important)
- };
+ struct gguf_init_params params = {
+ /*.no_alloc = */ true,
+ /*.ctx = */ &meta,
+ };
- // Width and height in the information header
- infoHeader[4] = (unsigned char)(bmp_size.width);
- infoHeader[5] = (unsigned char)(bmp_size.width >> 8);
- infoHeader[6] = (unsigned char)(bmp_size.width >> 16);
- infoHeader[7] = (unsigned char)(bmp_size.width >> 24);
- infoHeader[8] = (unsigned char)(bmp_size.height);
- infoHeader[9] = (unsigned char)(bmp_size.height >> 8);
- infoHeader[10] = (unsigned char)(bmp_size.height >> 16);
- infoHeader[11] = (unsigned char)(bmp_size.height >> 24);
-
- // Write file headers
- file.write(reinterpret_cast<char*>(fileHeader), sizeof(fileHeader));
- file.write(reinterpret_cast<char*>(infoHeader), sizeof(infoHeader));
-
- // Pixel data
- std::vector<unsigned char> padding(3, 0); // Max padding size to be added to each row
- for (int y = bmp_size.height - 1; y >= 0; --y) { // BMP files are stored bottom-to-top
- for (int x = 0; x < bmp_size.width; ++x) {
- // Each pixel
- const auto px = img.get_pixel(x, y);
- unsigned char pixel[3] = {
- px[2], // BMP stores pixels in BGR format
- px[1],
- px[0]
- };
- file.write(reinterpret_cast<char*>(pixel), 3);
+ ctx_gguf = gguf_context_ptr(gguf_init_from_file(fname, params));
+ if (!ctx_gguf.get()) {
+ throw std::runtime_error(string_format("%s: failed to load mmproj model from %s. Does this file exist?\n", __func__, fname));
}
- // Write padding for the row
- file.write(reinterpret_cast<char*>(padding.data()), paddingAmount);
- }
- file.close();
-}
-
-// debug function to convert f32 to u8
-static void clip_image_convert_f32_to_u8(const clip_image_f32& src, clip_image_u8& dst) {
- dst.set_size(src.get_size(), false);
- const auto & src_buf = src.get_ro_buf();
- std::vector<uint8_t> dst_buf(src.n_elements());
- for (size_t i = 0; i < src.n_elements(); ++i) {
- dst_buf[i] = static_cast<uint8_t>(std::min(std::max(int(src_buf[i] * 255.0f), 0), 255));
- }
- dst.cpy_buf(dst_buf);
-}
-#endif
+ ctx_meta.reset(meta);
+ const int n_tensors = gguf_get_n_tensors(ctx_gguf.get());
-// ggml_backend_cpu_set_n_threads(backend_cpu, n_threads);
-static void clip_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) {
- ggml_backend_dev_t dev = ggml_backend_get_device(backend_cpu);
- ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
- if (reg) {
- auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
- if (ggml_backend_set_n_threads_fn) {
- ggml_backend_set_n_threads_fn(backend_cpu, n_threads);
+ // print gguf info
+ {
+ std::string name;
+ get_string(KEY_NAME, name, false);
+ std::string description;
+ get_string(KEY_DESCRIPTION, description, false);
+ LOG_INF("%s: model name: %s\n", __func__, name.c_str());
+ LOG_INF("%s: description: %s\n", __func__, description.c_str());
+ LOG_INF("%s: GGUF version: %d\n", __func__, gguf_get_version(ctx_gguf.get()));
+ LOG_INF("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx_gguf.get()));
+ LOG_INF("%s: n_tensors: %d\n", __func__, n_tensors);
+ LOG_INF("%s: n_kv: %d\n", __func__, (int)gguf_get_n_kv(ctx_gguf.get()));
+ LOG_INF("\n");
}
- }
-}
-
-static std::shared_ptr<ggml_threadpool> clip_threadpool_init(int n_threads) {
- ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
- ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
- if (!reg || n_threads < 2) {
- return nullptr;
- }
- auto * new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new");
- auto * free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");
- if (!new_fn || !free_fn) {
- return nullptr;
- }
- ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads);
- tpp.poll = 0; // the pool is idle most of the time, don't take CPU away from the text model
- ggml_threadpool_t threadpool = new_fn(&tpp);
- if (!threadpool) {
- throw std::runtime_error("failed to create threadpool");
- }
- return std::shared_ptr<ggml_threadpool>(threadpool, free_fn);
-}
-
-struct clip_ctx {
- clip_model model;
-
- gguf_context_ptr ctx_gguf;
- ggml_context_ptr ctx_data;
-
- std::vector<uint8_t> buf_compute_meta;
-
- std::vector<ggml_backend_t> backend_ptrs;
- std::vector<ggml_backend_buffer_type_t> backend_buft;
-
- std::shared_ptr<ggml_threadpool> threadpool; // shared by the contexts of a model
- ggml_backend_t backend = nullptr;
- ggml_backend_t backend_cpu = nullptr;
- ggml_backend_buffer_ptr buf;
-
-
- int max_nodes = 8192;
- ggml_backend_sched_ptr sched;
- clip_flash_attn_type flash_attn_type = CLIP_FLASH_ATTN_TYPE_AUTO;
- bool is_allocated = false;
-
- bool debug_output_embeddings = false;
-
- // for measuring memory usage
- bool no_alloc = false;
- std::map<ggml_backend_dev_t, size_t> mem_usage;
- std::map<ggml_backend_dev_t, size_t> mem_compute;
-
- bool support_batch = false;
+ // modalities
+ {
+ get_bool(KEY_HAS_VISION_ENC, has_vision, false);
+ get_bool(KEY_HAS_AUDIO_ENC, has_audio, false);
+ get_bool(KEY_HAS_GEN_AUDIO_ENC, has_gen_audio, false);
- clip_ctx(clip_context_params & ctx_params, std::shared_ptr<ggml_threadpool> threadpool) : threadpool(std::move(threadpool)) {
- flash_attn_type = ctx_params.flash_attn_type;
- no_alloc = ctx_params.no_alloc;
- backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
- if (!backend_cpu) {
- throw std::runtime_error("failed to initialize CPU backend");
- }
- if (this->threadpool) {
- ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu));
- auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool");
- if (set_threadpool_fn) {
- set_threadpool_fn(backend_cpu, this->threadpool.get());
+ if (has_vision) {
+ LOG_INF("%s: has vision encoder\n", __func__);
}
- }
- if (ctx_params.use_gpu) {
- if (ctx_params.device != nullptr) {
- backend = ggml_backend_dev_init(ctx_params.device, nullptr);
- if (!backend) {
- throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n",
- __func__, ggml_backend_dev_name(ctx_params.device)));
- }
- } else {
- backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr);
- backend = backend ? backend : ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU, nullptr);
+ if (has_audio) {
+ LOG_INF("%s: has audio encoder\n", __func__);
+ }
+ if (has_gen_audio) {
+ LOG_INF("%s: has audio generation (gen) encoder\n", __func__);
}
}
- if (backend) {
- LOG_INF("%s: CLIP using %s backend\n", __func__, ggml_backend_name(backend));
- backend_ptrs.push_back(backend);
- backend_buft.push_back(ggml_backend_get_default_buffer_type(backend));
- } else {
- backend = backend_cpu;
- LOG_INF("%s: CLIP using CPU backend\n", __func__);
- }
-
- if (ctx_params.image_min_tokens > 0) {
- model.hparams.custom_image_min_tokens = ctx_params.image_min_tokens;
- }
- if (ctx_params.image_max_tokens > 0) {
- model.hparams.custom_image_max_tokens = ctx_params.image_max_tokens;
- }
-
- backend_ptrs.push_back(backend_cpu);
- backend_buft.push_back(ggml_backend_get_default_buffer_type(backend_cpu));
-
- sched.reset(
- ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), 8192, false, true)
- );
-
- if (ctx_params.cb_eval != nullptr) {
- ggml_backend_sched_set_eval_callback(sched.get(), ctx_params.cb_eval, ctx_params.cb_eval_user_data);
- }
-
- debug_output_embeddings = std::getenv("MTMD_DEBUG_EMBEDDINGS") != nullptr;
- }
-
- ~clip_ctx() {
- ggml_backend_free(backend);
- if (backend != backend_cpu) {
- ggml_backend_free(backend_cpu);
+ // tensors
+ if (!skip_tensors) {
+ for (int i = 0; i < n_tensors; ++i) {
+ const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
+ const size_t offset = gguf_get_tensor_offset(ctx_gguf.get(), i);
+ enum ggml_type type = gguf_get_tensor_type(ctx_gguf.get(), i);
+ ggml_tensor * cur = ggml_get_tensor(meta, name);
+ size_t tensor_size = ggml_nbytes(cur);
+ model_size += tensor_size;
+ LOG_DBG("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, offset=%zu, shape:[%" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "], type = %s\n",
+ __func__, i, ggml_n_dims(cur), cur->name, tensor_size, offset, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3], ggml_type_name(type));
+ }
}
}
- // this function is added so that we don't change too much of the existing code
- projector_type proj_type() const {
- return model.proj_type;
- }
-};
-
-//
-// clip_graph
-//
-
-clip_graph::clip_graph(clip_ctx * ctx, const clip_image_f32 & img) :
- model(ctx->model),
- hparams(model.hparams),
- proj_type(ctx->proj_type()),
- img(img),
- patch_size(hparams.patch_size),
- n_patches_x(img.nx() / patch_size),
- n_patches_y(img.ny() / patch_size),
- n_patches(n_patches_x * n_patches_y),
- n_embd(hparams.n_embd),
- n_head(hparams.n_head),
- n_head_kv(hparams.n_head_kv),
- d_head(hparams.n_embd_head > 0 ? hparams.n_embd_head : (n_head > 0 ? n_embd / n_head : 0)),
- n_layer(hparams.n_layer),
- n_mmproj_embd(clip_n_mmproj_embd(ctx)),
- eps(hparams.eps),
- kq_scale(d_head > 0 ? 1.0f / sqrtf((float)d_head) : 0.0f),
- flash_attn_type(ctx->flash_attn_type) {
- struct ggml_init_params params = {
- /*.mem_size =*/ ctx->buf_compute_meta.size(),
- /*.mem_buffer =*/ ctx->buf_compute_meta.data(),
- /*.no_alloc =*/ true,
- };
- ctx0_ptr.reset(ggml_init(params));
- ctx0 = ctx0_ptr.get();
- gf = ggml_new_graph_custom(ctx0, ctx->max_nodes, false);
-}
-
-clip_graph::clip_graph(const clip_graph & parent) :
- model(parent.model),
- hparams(parent.hparams),
- proj_type(parent.proj_type),
- img(parent.img),
- patch_size(parent.patch_size),
- n_patches_x(parent.n_patches_x),
- n_patches_y(parent.n_patches_y),
- n_patches(parent.n_patches),
- n_embd(parent.n_embd),
- n_head(parent.n_head),
- n_head_kv(parent.n_head_kv),
- d_head(parent.d_head),
- n_layer(parent.n_layer),
- n_mmproj_embd(parent.n_mmproj_embd),
- eps(parent.eps),
- kq_scale(parent.kq_scale),
- flash_attn_type(parent.flash_attn_type) {
- // reuse from parent
- ctx0 = parent.ctx0;
- gf = parent.gf;
-}
-
-ggml_tensor * clip_graph::build_mm(ggml_tensor * w, ggml_tensor * x) const {
- return ggml_mul_mat(ctx0, w, x);
-}
-
-void clip_graph::cb(ggml_tensor * cur, const char * name, int il) const {
- if (il >= 0) {
- ggml_format_name(cur, "%s-%d", name, il);
- } else {
- ggml_set_name(cur, name);
- }
-}
-
-// siglip2 naflex
-ggml_tensor * clip_graph::resize_position_embeddings(uint32_t interpolation_mode) {
- ggml_tensor * pos_embd = model.position_embeddings;
- const int height = img.ny() / patch_size;
- const int width = img.nx() / patch_size;
- const uint32_t mode = interpolation_mode;
- const int n_per_side = (int)std::sqrt(pos_embd->ne[1]);
-
- GGML_ASSERT(pos_embd);
-
- if (height == n_per_side && width == n_per_side) {
- return pos_embd;
- }
-
- pos_embd = ggml_reshape_3d(ctx0, pos_embd, n_embd, n_per_side, n_per_side); // -> (n_embd, n_per_side, n_per_side)
- pos_embd = ggml_permute(ctx0, pos_embd, 2, 0, 1, 3); // -> (n_per_side, n_per_side, n_embd)
- pos_embd = ggml_interpolate(ctx0, pos_embd, width, height, n_embd, 1, mode); // -> (width, height, n_embd)
- pos_embd = ggml_permute(ctx0, pos_embd, 1, 2, 0, 3); // -> (n_embd, width, height)
- pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, width * height); // -> (n_embd, width * height)
+ void load_hparams(mtmd_model & model, mtmd_modality modality) {
+ auto & hparams = model.hparams;
+ std::string log_ffn_op; // for logging
- return pos_embd;
-}
+ // sanity check
+ if (modality == MTMD_MODALITY_VISION) {
+ GGML_ASSERT(has_vision);
+ } else if (modality == MTMD_MODALITY_AUDIO) {
+ GGML_ASSERT(has_audio);
+ } else if (modality == MTMD_MODALITY_GEN_AUDIO) {
+ GGML_ASSERT(has_gen_audio);
+ }
+ model.modality = modality;
-// build vision transformer (ViT) cgraph
-// this function should cover most of the models
-// if your model has specific features, you should probably duplicate this function
-ggml_tensor * clip_graph::build_vit(
- ggml_tensor * inp,
- int64_t n_pos,
- norm_type norm_t,
- ffn_op_type ffn_t,
- ggml_tensor * learned_pos_embd,
- std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos,
- const build_vit_opts & opts
- ) {
- // batch dim: inp is [n_embd, n_pos, B]
- const int64_t B = inp->ne[2];
-
- if (learned_pos_embd) {
- inp = ggml_add(ctx0, inp, learned_pos_embd);
- cb(inp, "pos_embed", -1);
- }
- // flatten batch; unflatten again in attention
- inp = ggml_reshape_2d(ctx0, inp, n_embd, n_pos * B);
+ // projector type
+ std::string proj_type;
+ {
+ // default key
+ get_string(KEY_PROJ_TYPE, proj_type, false);
- ggml_tensor * inpL = inp;
+ // for models with mixed modalities
+ if (proj_type.empty()) {
+ if (modality == MTMD_MODALITY_VISION) {
+ get_string(KEY_VISION_PROJ_TYPE, proj_type, false);
+ } else if (modality == MTMD_MODALITY_AUDIO) {
+ get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false);
+ } else if (modality == MTMD_MODALITY_GEN_AUDIO) {
+ get_string(KEY_GEN_AUDIO_PROJ_TYPE, proj_type, false);
+ } else {
+ GGML_ABORT("unknown modality");
+ }
+ }
- // pre-layernorm
- if (model.pre_ln_w) {
- inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, norm_t, eps, -1);
- cb(inpL, "pre_ln", -1);
- }
+ model.proj_type = projector_type_from_string(proj_type);
- // loop over layers
- for (int il = 0; il < n_layer; il++) {
- auto & layer = model.layers[il];
- ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states
+ if (model.proj_type == PROJECTOR_TYPE_UNKNOWN) {
+ throw std::runtime_error(string_format("%s: unknown projector type: %s\n", __func__, proj_type.c_str()));
+ }
- ggml_tensor * attn_mask = opts.attn_mask;
- if (opts.attn_mask_layers.size() > (size_t) il) {
- attn_mask = opts.attn_mask_layers[il];
+ // correct arch for multimodal models (legacy method)
+ if (model.proj_type == PROJECTOR_TYPE_QWEN25O) {
+ model.proj_type = modality == MTMD_MODALITY_VISION
+ ? PROJECTOR_TYPE_QWEN25VL
+ : PROJECTOR_TYPE_QWEN2A;
+ }
}
- // layernorm1
- cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
- cb(cur, "layer_inp_normed", il);
+ const bool is_vision = model.modality == MTMD_MODALITY_VISION;
+ const bool is_audio = model.modality == MTMD_MODALITY_AUDIO;
+ const bool is_gen_audio = model.modality == MTMD_MODALITY_GEN_AUDIO;
- // self-attention
+ // other hparams
{
- ggml_tensor * Qcur = nullptr;
- ggml_tensor * Kcur = nullptr;
- ggml_tensor * Vcur = nullptr;
- if (layer.qkv_w != nullptr) {
- // fused qkv
- cur = build_mm(layer.qkv_w, cur);
- if (layer.qkv_b != nullptr) {
- cur = ggml_add(ctx0, cur, layer.qkv_b);
- }
+ const char * prefix = is_vision ? "vision" : (is_audio ? "audio" : "gen.audio");
+ get_u32(string_format(KEY_N_EMBD, prefix), hparams.n_embd);
+ get_u32(string_format(KEY_N_HEAD, prefix), hparams.n_head);
+ get_u32(string_format(KEY_N_EMBD_HEAD, prefix), hparams.n_embd_head, false);
+ get_u32(string_format(KEY_N_FF, prefix), hparams.n_ff);
+ get_u32(string_format(KEY_N_BLOCK, prefix), hparams.n_layer);
+ get_u32(string_format(KEY_PROJ_DIM, prefix), hparams.projection_dim);
+ get_f32(string_format(KEY_LAYER_NORM_EPS, prefix), hparams.eps);
- // Q/K/V as [d_head, n_head, n_pos, B], the batch stride is cur->nb[1]*n_pos.
- Qcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B,
- /* nb1 */ ggml_row_size(cur->type, d_head),
- /* nb2 */ cur->nb[1],
- /* nb3 */ cur->nb[1] * n_pos,
- /* offset */ 0);
-
- Kcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B,
- /* nb1 */ ggml_row_size(cur->type, d_head),
- /* nb2 */ cur->nb[1],
- /* nb3 */ cur->nb[1] * n_pos,
- /* offset */ ggml_row_size(cur->type, n_head * d_head));
-
- Vcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B,
- /* nb1 */ ggml_row_size(cur->type, d_head),
- /* nb2 */ cur->nb[1],
- /* nb3 */ cur->nb[1] * n_pos,
- /* offset */ ggml_row_size(cur->type, 2 * n_head * d_head));
-
- if (layer.q_norm) {
- GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]);
- Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il);
- cb(Qcur, "Qcur_norm", il);
- }
+ // n_head_kv is optional (for GQA), default to n_head
+ hparams.n_head_kv = hparams.n_head;
+ get_u32(string_format(KEY_N_HEAD_KV, prefix), hparams.n_head_kv, false);
- if (layer.k_norm) {
- GGML_ASSERT(layer.k_norm->ne[0] == Kcur->ne[0]);
- Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il);
- cb(Kcur, "Kcur_norm", il);
+ if (is_vision) {
+ get_u32(KEY_IMAGE_SIZE, hparams.image_size);
+ get_u32(KEY_PATCH_SIZE, hparams.patch_size);
+ get_u32(KEY_MAX_SLICE_NUMS, hparams.max_slice_nums, false);
+ get_i32(KEY_MINICPMV_VERSION, hparams.minicpmv_version, false); // legacy
+ get_u32(KEY_MINICPMV_QUERY_NUM, hparams.minicpmv_query_num, false);
+ if (hparams.minicpmv_query_num == 0) {
+ // Fallback to hardcoded values for legacy models
+ if (hparams.minicpmv_version == 3) {
+ hparams.minicpmv_query_num = 64;
+ } else if (hparams.minicpmv_version == 4) {
+ hparams.minicpmv_query_num = 64;
+ } else if (hparams.minicpmv_version == 5) {
+ hparams.minicpmv_query_num = 64;
+ } else if (hparams.minicpmv_version == 6) {
+ hparams.minicpmv_query_num = 64;
+ } else if (hparams.minicpmv_version == 100045) {
+ hparams.minicpmv_query_num = 64;
+ } else {
+ hparams.minicpmv_query_num = 96;
+ }
}
+ } else if (is_audio) {
+ get_u32(KEY_A_NUM_MEL_BINS, hparams.n_mel_bins);
+ // some hparams are unused, but still need to set to avoid issues
+ hparams.image_size = 0;
+ hparams.patch_size = 1;
+
+ } else if (is_gen_audio) {
+ // these are unused, but still need to be set to avoid issues
+ hparams.image_size = 0;
+ hparams.patch_size = 1;
+ get_string(KEY_GEN_AUDIO_VARIANT, hparams.gen_model_variant, false);
} else {
- // separate q, k, v
- Qcur = build_mm(layer.q_w, cur);
- if (layer.q_b) {
- Qcur = ggml_add(ctx0, Qcur, layer.q_b);
- }
+ GGML_ASSERT(false && "unknown modality");
+ }
- Kcur = build_mm(layer.k_w, cur);
- if (layer.k_b) {
- Kcur = ggml_add(ctx0, Kcur, layer.k_b);
+ // for pinpoints, we need to convert it into a list of resolution candidates
+ {
+ std::vector<int> pinpoints;
+ get_arr_int(KEY_IMAGE_GRID_PINPOINTS, pinpoints, false);
+ if (pinpoints.size() % 2 != 0) {
+ throw std::runtime_error(string_format("%s: image_grid_pinpoints must have an even number of elements, got %zu\n", __func__, pinpoints.size()));
}
-
- Vcur = build_mm(layer.v_w, cur);
- if (layer.v_b) {
- Vcur = ggml_add(ctx0, Vcur, layer.v_b);
+ if (!pinpoints.empty()) {
+ for (size_t i = 0; i < pinpoints.size(); i += 2) {
+ hparams.image_res_candidates.push_back({
+ pinpoints[i],
+ pinpoints[i+1],
+ });
+ }
}
+ }
- // if true, norm must be applied after reshaping to (d_head, n_head, n_pos)
- bool norm_per_head = layer.q_norm && layer.q_norm->ne[0] == d_head;
+ // default warmup value
+ hparams.warmup_image_size = hparams.image_size;
- if (!norm_per_head) {
- if (layer.q_norm) {
- Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il);
- cb(Qcur, "Qcur_norm", il);
- }
- if (layer.k_norm) {
- Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il);
- cb(Kcur, "Kcur_norm", il);
- }
+ {
+ bool use_gelu = false;
+ bool use_silu = false;
+ get_bool(KEY_USE_GELU, use_gelu, false);
+ get_bool(KEY_USE_SILU, use_silu, false);
+ if (use_gelu && use_silu) {
+ throw std::runtime_error(string_format("%s: both use_gelu and use_silu are set to true\n", __func__));
}
+ if (use_gelu) {
+ hparams.ffn_op = FFN_GELU;
+ log_ffn_op = "gelu";
+ } else if (use_silu) {
+ hparams.ffn_op = FFN_SILU;
+ log_ffn_op = "silu";
+ } else {
+ hparams.ffn_op = FFN_GELU_QUICK;
+ log_ffn_op = "gelu_quick";
+ }
+ }
- Qcur = ggml_reshape_4d(ctx0, Qcur, d_head, n_head, n_pos, B);
- Kcur = ggml_reshape_4d(ctx0, Kcur, d_head, n_head_kv, n_pos, B);
- Vcur = ggml_reshape_4d(ctx0, Vcur, d_head, n_head_kv, n_pos, B);
-
- if (norm_per_head) {
- if (layer.q_norm) {
- Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il);
- cb(Qcur, "Qcur_norm_per_head", il);
- }
- if (layer.k_norm) {
- Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il);
- cb(Kcur, "Kcur_norm_per_head", il);
- }
- }
- }
-
- cb(Qcur, "Qcur", il);
- cb(Kcur, "Kcur", il);
- cb(Vcur, "Vcur", il);
-
- if (add_pos) {
- Qcur = add_pos(Qcur, layer);
- Kcur = add_pos(Kcur, layer);
- cb(Qcur, "Qcur_pos", il);
- cb(Kcur, "Kcur_pos", il);
- }
-
- if (proj_type == PROJECTOR_TYPE_GEMMA4V) {
- Vcur = ggml_rms_norm(ctx0, Vcur, eps);
- cb(Vcur, "Vcur_normed", il);
- }
-
- // build_attn returns a flat 2D [n_embd, n_pos*B]
- cur = build_attn(layer.o_w, layer.o_b,
- Qcur, Kcur, Vcur, attn_mask, kq_scale, il);
- cb(cur, "attn_out", il);
- }
-
- if (layer.ls_1_w) {
- cur = ggml_mul(ctx0, cur, layer.ls_1_w);
- cb(cur, "attn_out_scaled", il);
- }
-
- if (layer.attn_post_norm_w) {
- cur = build_norm(cur, layer.attn_post_norm_w, nullptr, norm_t, eps, il);
- cb(cur, "attn_post_normed", il);
- }
-
- // re-add the layer input, e.g., residual
- cur = ggml_add(ctx0, cur, inpL);
-
- inpL = cur; // inpL = residual, cur = hidden_states
-
- if (opts.callback_layer_out) {
- opts.callback_layer_out(cur, il);
- }
-
- cb(cur, "ffn_inp", il);
-
- // layernorm2 (pre-ffn norm)
- cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);
- cb(cur, "ffn_inp_normed", il);
-
- // ffn
- cur = layer.ff_gate_exps_w
- ? build_moe_ffn(cur, layer, ffn_t, il)
- : build_ffn(cur,
- layer.ff_up_w, layer.ff_up_b,
- layer.ff_gate_w, layer.ff_gate_b,
- layer.ff_down_w, layer.ff_down_b,
- ffn_t, il);
-
- cb(cur, "ffn_out", il);
-
- if (layer.ff_post_norm_w) {
- cur = build_norm(cur, layer.ff_post_norm_w, nullptr, norm_t, eps, il);
- cb(cur, "ffn_post_normed", il);
- }
-
- if (layer.ls_2_w) {
- cur = ggml_mul(ctx0, cur, layer.ls_2_w);
- cb(cur, "ffn_out_scaled", il);
- }
-
- // residual 2
- cur = ggml_add(ctx0, inpL, cur);
- cb(cur, "layer_out", il);
-
- if (layer.ls_out_w) {
- cur = ggml_mul(ctx0, cur, layer.ls_out_w);
- cb(cur, "layer_out_scaled", il);
- }
-
- inpL = cur;
- }
-
- if (model.audio_has_avgpool()) {
- ggml_tensor * cur = inpL;
- cur = ggml_transpose(ctx0, cur);
- cur = ggml_cont(ctx0, cur);
- cur = ggml_pool_1d(ctx0, cur, GGML_OP_POOL_AVG, 2, 2, 0);
- cur = ggml_transpose(ctx0, cur);
- cur = ggml_cont(ctx0, cur);
- inpL = cur;
- }
-
- // post-layernorm
- if (model.post_ln_w && !opts.skip_post_ln) {
- inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, -1);
- }
-
- // restore the batch dim
- GGML_ASSERT(inpL->ne[1] % B == 0);
- inpL = ggml_reshape_3d(ctx0, inpL, n_embd, inpL->ne[1] / B, B);
- return inpL;
-}
-
-// build the input after conv2d (inp_raw --> patches)
-// returns tensor with shape [n_embd, n_patches]
-ggml_tensor * clip_graph::build_inp() {
- ggml_tensor * inp_raw = build_inp_raw();
- ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
- inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, n_batch);
- inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
- if (model.patch_bias) {
- inp = ggml_add(ctx0, inp, model.patch_bias);
- cb(inp, "patch_bias", -1);
- }
- return inp;
-}
-
-ggml_tensor * clip_graph::build_inp_raw(int channels) {
- ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, img.nx(), img.ny(), channels, n_batch);
- ggml_set_name(inp_raw, "inp_raw");
- ggml_set_input(inp_raw);
- return inp_raw;
-}
-
-// qwen-vl style temporal merge: Conv3D(kt=2) split into 2 Conv2D
-ggml_tensor * clip_graph::build_inp_temporal_merge() {
- ggml_tensor * inp_raw = build_inp_raw();
-
- GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
- GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
-
- const size_t nb1 = ggml_row_size(inp_raw->type, img.nx());
- const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny());
-
- if (n_batch == 1) {
- // still image input
- return ggml_add(ctx0,
- ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1),
- ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
- } else if (n_batch == 2) {
- // 2 frames input (video input)
- ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw,
- img.nx(), img.ny(), 3, nb1, nb2, 0);
- ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw,
- img.nx(), img.ny(), 3, nb1, nb2,
- nb2 * 3); // move to the second frame
- return ggml_add(ctx0,
- ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1),
- ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1));
- } else {
- GGML_ASSERT(false && "n_batch > 2 is not supported");
- }
-}
-
-static std::string get_attn_mask_name(int idx) {
- return idx == 0 ? "attn_mask" : "attn_mask_" + std::to_string(idx);
-}
-
-ggml_tensor * clip_graph::build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx) {
- const ggml_type type = flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED ? GGML_TYPE_F16 : GGML_TYPE_F32;
- ggml_tensor * mask = ggml_new_tensor_2d(ctx0, type, n_kv, n_q);
- ggml_set_name(mask, get_attn_mask_name(idx).c_str());
- ggml_set_input(mask);
- return mask;
-}
-
-ggml_tensor * clip_graph::build_norm(
- ggml_tensor * cur,
- ggml_tensor * mw,
- ggml_tensor * mb,
- norm_type type,
- float norm_eps,
- int il) const {
-
- cur = type == NORM_TYPE_RMS
- ? ggml_rms_norm(ctx0, cur, norm_eps)
- : ggml_norm(ctx0, cur, norm_eps);
-
- if (mw) {
- cur = ggml_mul(ctx0, cur, mw);
- cb(cur, "norm_w", il);
- }
-
- if (mb) {
- cur = ggml_add(ctx0, cur, mb);
- cb(cur, "norm_b", il);
- }
-
- return cur;
-}
-
-ggml_tensor * clip_graph::build_ffn(
- ggml_tensor * cur,
- ggml_tensor * up,
- ggml_tensor * up_b,
- ggml_tensor * gate,
- ggml_tensor * gate_b,
- ggml_tensor * down,
- ggml_tensor * down_b,
- ffn_op_type type_op,
- int il) const {
-
- ggml_tensor * tmp = up ? build_mm(up, cur) : cur;
- cb(tmp, "ffn_up", il);
-
- if (up_b) {
- tmp = ggml_add(ctx0, tmp, up_b);
- cb(tmp, "ffn_up_b", il);
- }
-
- if (gate) {
- cur = build_mm(gate, cur);
- cb(cur, "ffn_gate", il);
-
- if (gate_b) {
- cur = ggml_add(ctx0, cur, gate_b);
- cb(cur, "ffn_gate_b", il);
- }
- } else {
- cur = tmp;
- }
-
- // we only support parallel ffn for now
- switch (type_op) {
- case FFN_SILU:
- if (gate) {
- cur = hparams.swiglu_clamp > 0.0f
- ? ggml_swiglu_clamp(ctx0, cur, tmp, hparams.swiglu_clamp)
- : ggml_swiglu_split(ctx0, cur, tmp);
- cb(cur, "ffn_swiglu", il);
- } else {
- cur = ggml_silu(ctx0, cur);
- cb(cur, "ffn_silu", il);
- } break;
- case FFN_GELU:
- if (gate) {
- cur = ggml_geglu_split(ctx0, cur, tmp);
- cb(cur, "ffn_geglu", il);
- } else {
- cur = ggml_gelu(ctx0, cur);
- cb(cur, "ffn_gelu", il);
- } break;
- case FFN_GELU_ERF:
- if (gate) {
- cur = ggml_geglu_erf_split(ctx0, cur, tmp);
- cb(cur, "ffn_geglu_erf", il);
- } else {
- cur = ggml_gelu_erf(ctx0, cur);
- cb(cur, "ffn_gelu_erf", il);
- } break;
- case FFN_GELU_QUICK:
- if (gate) {
- cur = ggml_geglu_quick_split(ctx0, cur, tmp);
- cb(cur, "ffn_geglu_quick", il);
- } else {
- cur = ggml_gelu_quick(ctx0, cur);
- cb(cur, "ffn_gelu_quick", il);
- } break;
- case FFN_RELU_SQR:
- {
- cur = ggml_relu(ctx0, cur);
- cur = ggml_sqr(ctx0, cur);
- cb(cur, "ffn_relu_sqr", il);
- } break;
- }
-
- if (down) {
- cur = build_mm(down, cur);
- }
-
- if (down_b) {
- cb(cur, "ffn_down", il);
- }
-
- if (down_b) {
- cur = ggml_add(ctx0, cur, down_b);
- }
-
- return cur;
-}
-
-// MoE FFN with sigmoid router and normalized top-k weights (dots3note vision)
-// the router runs in fp32; exp_probs_b only affects expert selection, not the weights
-ggml_tensor * clip_graph::build_moe_ffn(ggml_tensor * cur, const clip_layer & layer, ffn_op_type type_op, int il) const {
- const int64_t n_tokens = cur->ne[1];
- const int64_t n_expert = layer.ff_gate_exps_w->ne[2];
- const int64_t n_expert_used = std::min((int64_t) hparams.n_expert_used, n_expert);
- GGML_ASSERT(n_expert_used > 0);
- GGML_ASSERT(type_op == FFN_SILU);
-
- ggml_tensor * probs = ggml_sigmoid(ctx0, build_mm(layer.ff_gate_inp_w, cur)); // [n_expert, n_tokens]
- cb(probs, "ffn_moe_probs", il);
-
- ggml_tensor * sel = layer.ff_exp_probs_b
- ? ggml_add(ctx0, probs, layer.ff_exp_probs_b)
- : probs;
- ggml_tensor * selected = ggml_top_k(ctx0, sel, n_expert_used); // [n_expert_used, n_tokens]
-
- ggml_tensor * weights = ggml_get_rows(ctx0,
- ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected);
- weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens);
- weights = ggml_div(ctx0, weights, ggml_sum_rows(ctx0, weights));
- weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens);
- cb(weights, "ffn_moe_weights", il);
-
- cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], 1, n_tokens);
- ggml_tensor * gate = ggml_mul_mat_id(ctx0, layer.ff_gate_exps_w, cur, selected); // [n_ff, n_expert_used, n_tokens]
- ggml_tensor * up = ggml_mul_mat_id(ctx0, layer.ff_up_exps_w, cur, selected);
- cur = ggml_mul(ctx0, ggml_silu(ctx0, gate), up);
- cur = ggml_mul_mat_id(ctx0, layer.ff_down_exps_w, cur, selected); // [n_embd, n_expert_used, n_tokens]
- cur = ggml_mul(ctx0, cur, weights);
-
- // sum over the selected experts
- ggml_tensor * out = nullptr;
- for (int64_t i = 0; i < n_expert_used; i++) {
- ggml_tensor * v = ggml_view_2d(ctx0, cur, cur->ne[0], n_tokens, cur->nb[2], i * cur->nb[1]);
- out = out ? ggml_add(ctx0, out, v) : v;
- }
- if (n_expert_used == 1) {
- out = ggml_cont(ctx0, out);
- }
- cb(out, "ffn_moe_out", il);
- return out;
-}
-
-ggml_tensor * clip_graph::build_attn(
- ggml_tensor * wo,
- ggml_tensor * wo_b,
- ggml_tensor * q_cur,
- ggml_tensor * k_cur,
- ggml_tensor * v_cur,
- ggml_tensor * kq_mask,
- float kq_scale,
- int il,
- ggml_tensor * sinks) const {
- // these nodes are added to the graph together so that they are not reordered
- // by doing so, the number of splits in the graph is reduced
- // the order is fixed without the compute flag, so an unselected branch stays out of the compute set
- ggml_build_forward_order(gf, q_cur);
- ggml_build_forward_order(gf, k_cur);
- ggml_build_forward_order(gf, v_cur);
-
- ggml_tensor * q = ggml_permute(ctx0, q_cur, 0, 2, 1, 3);
- //cb(q, "q", il);
-
- ggml_tensor * k = ggml_permute(ctx0, k_cur, 0, 2, 1, 3);
- //cb(k, "k", il);
-
- ggml_tensor * cur;
-
- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
- ggml_tensor * v = ggml_permute(ctx0, v_cur, 0, 2, 1, 3);
-
- k = ggml_cast(ctx0, k, GGML_TYPE_F16);
- v = ggml_cast(ctx0, v, GGML_TYPE_F16);
- // mask must be f16 here, use build_inp_attn_mask()
- GGML_ASSERT(!kq_mask || kq_mask->type == GGML_TYPE_F16);
-
- cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f);
- ggml_prec_set_acc(cur, GGML_PREC_F32);
- if (sinks != nullptr) {
- ggml_flash_attn_ext_add_sinks(cur, sinks);
- }
-
- cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]);
-
- } else {
- ggml_tensor * v = ggml_permute(ctx0, v_cur, 1, 2, 0, 3);
- v = ggml_cont(ctx0, v);
-
- ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
- // TODO: this most likely requires GGML_PREC_BF16, but will enable it later to avoid scope-creep
- // ggml_prec_set_acc(kq, GGML_PREC_BF16);
-
- kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, 0.0f);
- if (sinks != nullptr) {
- ggml_soft_max_add_sinks(kq, sinks);
- }
-
- ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq);
- cur = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
- cur = ggml_cont_2d(ctx0, cur, cur->ne[0] * cur->ne[1], cur->ne[2] * cur->ne[3]);
- }
-
- cb(cur, "kqv_out", il);
-
- if (wo) {
- cur = build_mm(wo, cur);
- }
-
- if (wo_b) {
- cur = ggml_add(ctx0, cur, wo_b);
- }
-
- return cur;
-}
-
-// implementation of the 2D RoPE without adding a new op in ggml
-ggml_tensor * clip_graph::build_rope_2d(
- ggml_context * ctx0,
- ggml_tensor * cur,
- ggml_tensor * pos_a, // first half
- ggml_tensor * pos_b, // second half
- const float freq_base,
- const bool interleave_freq
-) {
- const int64_t n_dim = cur->ne[0];
-
- // for example, if we have cur tensor of shape (n_dim=8, n_head, n_pos)
- // we will have a list of 4 inv_freq: 1e-0, 1e-1, 1e-2, 1e-3
- // first half of cur will use 1e-0, 1e-2 (even)
- // second half of cur will use 1e-1, 1e-3 (odd)
- // the trick here is to rotate just half of n_dim, so inv_freq will automatically be even
- // ^ don't ask me why, it's math! -2(2i) / n_dim == -2i / (n_dim/2)
- // then for the second half, we use freq_scale to shift the inv_freq
- // ^ why? replace (2i) with (2i+1) in the above equation
- const float freq_scale_odd = interleave_freq
- ? std::pow(freq_base, (float)-2/n_dim)
- : 1.0;
-
- // first half, dims [0, n_dim/2)
- cur = ggml_rope_ext(
- ctx0,
- cur,
- pos_a, // positions
- nullptr, // freq factors
- n_dim/2, // n_dims
- 0, 0, freq_base,
- 1.0f, 0.0f, 1.0f, 0.0f, 0.0f
- );
-
- // second half, dims [n_dim/2, n_dim)
- cur = ggml_rope_ext(
- ctx0,
- cur,
- pos_b, // positions
- nullptr, // freq factors
- n_dim/2, // n_dims
- 0, 0, freq_base,
- freq_scale_odd,
- 0.0f, 1.0f, 0.0f, 0.0f
- );
- cur = ggml_rope_set_offset(cur, n_dim/2);
-
- return cur;
-}
-
-// Generic function to stack frames for audio processing
-// Abstracts out the StackAudioFrames logic used by ultravox
-ggml_tensor * clip_graph::build_stack(ggml_tensor * cur, int32_t stack_factor, int32_t n_embed) {
- if (stack_factor <= 1) {
- return cur;
- }
-
- int64_t total_elements = ggml_nelements(cur);
- int64_t stride = n_embed * stack_factor;
-
- // Calculate padded length
- int64_t padded_len = GGML_PAD(total_elements, stride);
- int64_t pad = padded_len - total_elements;
-
- if (pad > 0) {
- // Pad the tensor to make it divisible by stride
- cur = ggml_view_1d(ctx0, cur, total_elements, 0);
- cur = ggml_pad(ctx0, cur, pad, 0, 0, 0);
- }
-
- // Reshape to [stride, padded_len / stride]
- cur = ggml_view_2d(ctx0, cur, stride, padded_len / stride,
- ggml_row_size(cur->type, stride), 0);
- return cur;
-}
-
-// aka pixel_shuffle / pixel_unshuffle / patch_merger (Kimi-VL)
-// support dynamic resolution
-ggml_tensor * clip_graph::build_suffix(ggml_tensor * cur) {
- for (int idx : clip_suffix_rows(img.suffix_type)) {
- GGML_ASSERT(model.tok_embd_sep && idx < model.tok_embd_sep->ne[1]);
- ggml_tensor * row = ggml_view_2d(ctx0, model.tok_embd_sep, model.tok_embd_sep->ne[0], 1,
- model.tok_embd_sep->nb[1], idx * model.tok_embd_sep->nb[1]);
- cur = ggml_concat(ctx0, cur, ggml_cast(ctx0, row, cur->type), 1);
- }
- return cur;
-}
-
-ggml_tensor * clip_graph::build_patch_merge_permute(ggml_tensor * cur, int scale_factor) {
- GGML_ASSERT(scale_factor > 1);
-
- const int n_embd = cur->ne[0];
- int width = img.nx() / patch_size;
- int height = img.ny() / patch_size;
-
- // pad width and height to factor
- const int64_t pad_width = CLIP_ALIGN(width, scale_factor) - width;
- const int64_t pad_height = CLIP_ALIGN(height, scale_factor) - height;
- cur = ggml_reshape_3d(ctx0, cur, n_embd, width, height);
- if (pad_width || pad_height) {
- cur = ggml_pad(ctx0, cur, 0, pad_width, pad_height, 0);
- width += pad_width;
- height += pad_height;
- }
-
- // unshuffle h
- cur = ggml_reshape_3d(ctx0, cur, n_embd * scale_factor, width / scale_factor, height);
- cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
-
- // unshuffle w
- cur = ggml_cont_3d(ctx0, cur, n_embd * scale_factor * scale_factor, height / scale_factor, width / scale_factor);
- cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
-
- cur = ggml_cont_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);
- cb(cur, "pixel_shuffle", -1);
-
- return cur;
-}
-
-static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const clip_image_f32_batch & imgs,
- const clip_encode_params * params = nullptr) {
- const clip_image_f32 & img = imgs.entries[0];
- std::unique_ptr<clip_graph> builder;
-
- switch (ctx->proj_type()) {
- case PROJECTOR_TYPE_GEMMA3:
- case PROJECTOR_TYPE_IDEFICS3:
- case PROJECTOR_TYPE_COHERE2V:
- case PROJECTOR_TYPE_LFM2:
- case PROJECTOR_TYPE_D1OMNI_V:
- case PROJECTOR_TYPE_JANUS_PRO:
- case PROJECTOR_TYPE_PHI4:
- {
- builder = std::make_unique<clip_graph_siglip>(ctx, img);
- } break;
- case PROJECTOR_TYPE_GEMMA3NV:
- {
- builder = std::make_unique<clip_graph_mobilenetv5>(ctx, img);
- } break;
- case PROJECTOR_TYPE_GEMMA4V:
- {
- builder = std::make_unique<clip_graph_gemma4v>(ctx, img);
- } break;
- case PROJECTOR_TYPE_GEMMA4UV:
- {
- builder = std::make_unique<clip_graph_gemma4uv>(ctx, img);
- } break;
- case PROJECTOR_TYPE_PIXTRAL:
- case PROJECTOR_TYPE_LIGHTONOCR:
- {
- builder = std::make_unique<clip_graph_pixtral>(ctx, img);
- } break;
- case PROJECTOR_TYPE_DOTS_OCR:
- case PROJECTOR_TYPE_DOTS3NOTE_V: // same ViT + merger; pyramid MoE is handled by build_vit
- {
- builder = std::make_unique<clip_graph_dotsocr>(ctx, img);
- } break;
- case PROJECTOR_TYPE_DOTS3NOTE_A:
- {
- builder = std::make_unique<clip_graph_dots3note_a>(ctx, img);
- } break;
- case PROJECTOR_TYPE_QWEN2VL:
- case PROJECTOR_TYPE_QWEN25VL:
- {
- builder = std::make_unique<clip_graph_qwen2vl>(ctx, img);
- } break;
- case PROJECTOR_TYPE_QWEN3VL:
- {
- builder = std::make_unique<clip_graph_qwen3vl>(ctx, img);
- } break;
- case PROJECTOR_TYPE_MAGEVL:
- {
- builder = std::make_unique<clip_graph_magevl>(ctx, img);
- } break;
- case PROJECTOR_TYPE_SENSENOVA_U1:
- {
- builder = std::make_unique<clip_graph_sensenova_u1>(ctx, img);
- } break;
- case PROJECTOR_TYPE_LFM3VL:
- {
- builder = std::make_unique<clip_graph_lfm3vl>(ctx, img);
- } break;
- case PROJECTOR_TYPE_LING3VL:
- {
- builder = std::make_unique<clip_graph_ling3vl>(ctx, img);
- } break;
- case PROJECTOR_TYPE_EXAONE4_5:
- {
- builder = std::make_unique<clip_graph_exaone4_5>(ctx, img);
- } break;
- case PROJECTOR_TYPE_MIMOVL:
- {
- builder = std::make_unique<clip_graph_mimovl>(ctx, img);
- } break;
- case PROJECTOR_TYPE_MINIMAX_M3:
- {
- builder = std::make_unique<clip_graph_minimax_m3>(ctx, img);
- } break;
- case PROJECTOR_TYPE_MUSE_GLIMMER:
- {
- builder = std::make_unique<clip_graph_muse_glimmer>(ctx, img);
- } break;
- case PROJECTOR_TYPE_STEP3VL:
- {
- builder = std::make_unique<clip_graph_step3vl>(ctx, img);
- } break;
- case PROJECTOR_TYPE_MINICPMV:
- {
- builder = std::make_unique<clip_graph_minicpmv>(ctx, img);
- } break;
- case PROJECTOR_TYPE_MINICPMV4_6:
- case PROJECTOR_TYPE_MINICPMV4_7:
- case PROJECTOR_TYPE_MINICPM_OCR_1_0:
- {
- builder = std::make_unique<clip_graph_minicpmv4_6>(ctx, img);
- } break;
- case PROJECTOR_TYPE_INTERNVL:
- {
- builder = std::make_unique<clip_graph_internvl>(ctx, img);
- } break;
- case PROJECTOR_TYPE_NEMOTRON_V2_VL:
- {
- builder = std::make_unique<clip_graph_nemotron_v2_vl>(ctx, img);
- } break;
- case PROJECTOR_TYPE_NEMOTRON_OMNI:
- {
- builder = std::make_unique<clip_graph_nemotron_omni>(ctx, img);
- } break;
- case PROJECTOR_TYPE_LLAMA4:
- {
- builder = std::make_unique<clip_graph_llama4>(ctx, img);
- } break;
- case PROJECTOR_TYPE_VOXTRAL_RT:
- {
- builder = std::make_unique<clip_graph_voxtral_rt>(ctx, img, params ? params->enc_state : nullptr);
- } break;
- case PROJECTOR_TYPE_ULTRAVOX:
- case PROJECTOR_TYPE_VOXTRAL:
- case PROJECTOR_TYPE_QWEN2A:
- case PROJECTOR_TYPE_GLMA:
- case PROJECTOR_TYPE_MERALION:
- case PROJECTOR_TYPE_MUSIC_FLAMINGO:
- {
- builder = std::make_unique<clip_graph_whisper_enc>(ctx, img);
- } break;
- case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
- {
- builder = std::make_unique<clip_graph_muse_spark_audio>(ctx, img);
- } break;
- case PROJECTOR_TYPE_KIMIVL:
- {
- builder = std::make_unique<clip_graph_kimivl>(ctx, img);
- } break;
- case PROJECTOR_TYPE_PADDLEOCR:
- {
- builder = std::make_unique<clip_graph_paddleocr>(ctx, img);
- } break;
- case PROJECTOR_TYPE_KIMIK25:
- {
- builder = std::make_unique<clip_graph_kimik25>(ctx, img);
- } break;
- case PROJECTOR_TYPE_DEEPSEEK4V:
- case PROJECTOR_TYPE_DEEPSEEK41V:
- {
- builder = std::make_unique<clip_graph_deepseek4v>(ctx, img);
- } break;
- case PROJECTOR_TYPE_COGVLM:
- {
- builder = std::make_unique<clip_graph_cogvlm>(ctx, img);
- } break;
- case PROJECTOR_TYPE_HUNYUANVL:
- {
- builder = std::make_unique<clip_graph_hunyuanvl>(ctx, img);
- } break;
- case PROJECTOR_TYPE_MLP:
- case PROJECTOR_TYPE_MLP_NORM:
- case PROJECTOR_TYPE_LDP:
- case PROJECTOR_TYPE_LDPV2:
- case PROJECTOR_TYPE_GLM_EDGE:
- {
- builder = std::make_unique<clip_graph_llava>(ctx, img);
- } break;
- case PROJECTOR_TYPE_DEEPSEEKOCR:
- {
- builder = std::make_unique<clip_graph_deepseekocr>(ctx, img);
- } break;
- case PROJECTOR_TYPE_DEEPSEEKOCR2:
- {
- builder = std::make_unique<clip_graph_deepseekocr2>(ctx, img);
- } break;
- case PROJECTOR_TYPE_LFM2A:
- case PROJECTOR_TYPE_D1OMNI_A:
- {
- builder = std::make_unique<clip_graph_conformer>(ctx, img);
- } break;
- case PROJECTOR_TYPE_LFM3A:
- {
- builder = std::make_unique<clip_graph_lfm3a>(ctx, img);
- } break;
- case PROJECTOR_TYPE_GEMMA4A:
- {
- builder = std::make_unique<clip_graph_gemma4a>(ctx, img);
- } break;
- case PROJECTOR_TYPE_GEMMA4UA:
- {
- builder = std::make_unique<clip_graph_gemma4ua>(ctx, img);
- } break;
- case PROJECTOR_TYPE_GRANITE_SPEECH:
- {
- builder = std::make_unique<clip_graph_granite_speech>(ctx, img);
- } break;
- case PROJECTOR_TYPE_GLM4V:
- case PROJECTOR_TYPE_GLM5V:
- {
- builder = std::make_unique<clip_graph_glm4v>(ctx, img);
- } break;
- case PROJECTOR_TYPE_QWEN3A:
- {
- builder = std::make_unique<clip_graph_qwen3a>(ctx, img);
- } break;
- case PROJECTOR_TYPE_MIMO_AUDIO:
- {
- builder = std::make_unique<clip_graph_mimo_audio>(ctx, img);
- } break;
- case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
- {
- builder = std::make_unique<clip_graph_qwen3tts_spkenc>(ctx, img);
- } break;
- case PROJECTOR_TYPE_POCKETTTS_SPKENC:
- {
- builder = std::make_unique<clip_graph_pockettts_spkenc>(ctx, img);
- } break;
- case PROJECTOR_TYPE_POCKETTTS_GEN:
- {
- const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE;
- const int n_step = ctx->model.hparams.flow_n_step;
- const int64_t n_latent = ctx->model.gen_input_lin_w->ne[0];
- GGML_ASSERT(n_step > 0);
- GGML_ASSERT(n_latent > 0);
- // "inp_feats" takes the caller's buffer as-is, the graph must consume all of it
- if (params && params->feats) {
- GGML_ASSERT(params->feats->size() % (size_t) n_latent == 0);
- GGML_ASSERT(params->feats->size() >= (size_t) n_latent);
- }
- const int n_frames = params && params->feats ? (int) (params->feats->size() / n_latent) : 1;
- builder = std::make_unique<clip_graph_pockettts_gen>(ctx, img, gen_process, n_step, n_frames);
- } break;
- case PROJECTOR_TYPE_CHATTERBOX_SPKENC:
- {
- builder = std::make_unique<clip_graph_chatterbox_spkenc>(ctx, img);
- } break;
- case PROJECTOR_TYPE_CHATTERBOX:
- {
- const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE;
- int n_tokens = params && params->codes ? (int) params->codes->size() : 0;
- int n_prompt_mel = 0;
- if (n_tokens > 0) {
- GGML_ASSERT(params->ref);
- n_tokens += (int) params->ref->tokens.size();
- n_prompt_mel = (int) (params->ref->feat.size() / 80);
- }
- const int vnm = params ? params->vocode_n_mel : 0;
- const int vns = params ? params->vocode_n_stft : 0;
- builder = std::make_unique<clip_graph_chatterbox>(ctx, img, gen_process, n_tokens, n_prompt_mel, vnm, vns);
- } break;
- case PROJECTOR_TYPE_LFM3A_GEN:
- {
- const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE;
- const int64_t n_cb = ctx->model.gen_code_embd_w->ne[2];
- bool all_known = false;
- int n_frames = 1;
- if (params && params->codes) {
- GGML_ASSERT(params->codes->size() % (size_t) n_cb == 0);
- if (gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
- n_frames = (int) (params->codes->size() / n_cb);
- GGML_ASSERT(n_frames > 0);
- } else {
- // -1 marks a masked codebook, none masked means only the feedback embedding is needed
- all_known = std::all_of(params->codes->begin(), params->codes->end(), [](int32_t c) { return c >= 0; });
- }
- }
- builder = std::make_unique<clip_graph_lfm3a_gen>(ctx, img, gen_process, all_known, n_frames);
- } break;
- case PROJECTOR_TYPE_DRAMA3_GEN:
- {
- const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE;
- const int64_t n_cb = ctx->model.gen_code_embd_w->ne[2];
- const int n_frames = params && params->codes ? (int) (params->codes->size() / n_cb) : 1;
- builder = std::make_unique<clip_graph_drama3_gen>(ctx, img, gen_process, params ? params->gen_state : nullptr, n_frames,
- params ? params->top_k : 30, params ? params->top_p : 0.9f,
- params ? params->temp : 1.0f);
- } break;
- case PROJECTOR_TYPE_QWEN3TTS_GEN:
- {
- const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE;
- const int top_k = params ? params->top_k : 50;
- const float top_p = params ? params->top_p : 1.0f;
- builder = std::make_unique<clip_graph_qwen3tts_gen>(ctx, img, gen_process, top_k, top_p);
- } break;
- case PROJECTOR_TYPE_SOPRANO:
- {
- const int n_frames = params && params->feats ? (int) (params->feats->size() / ctx->model.hparams.projection_dim) : 2;
- builder = std::make_unique<clip_graph_soprano>(ctx, img, n_frames);
- } break;
- case PROJECTOR_TYPE_YOUTUVL:
- {
- builder = std::make_unique<clip_graph_youtuvl>(ctx, img);
- } break;
- case PROJECTOR_TYPE_YASA2:
- {
- builder = std::make_unique<clip_graph_yasa2>(ctx, img);
- } break;
- case PROJECTOR_TYPE_PARAKEET:
- {
- builder = std::make_unique<clip_graph_parakeet>(ctx, img);
- } break;
- case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
- {
- builder = std::make_unique<clip_graph_nemotron_transcribe>(ctx, img, params ? params->enc_state : nullptr);
- } break;
- case PROJECTOR_TYPE_GRANITE4_VISION:
- {
- builder = std::make_unique<clip_graph_granite4_vision>(ctx, img);
- } break;
- default:
- GGML_ABORT("missing cgraph builder");
- }
-
- builder->img_batch = &imgs;
-
- // TODO [QWEN_VIDEO]: improve this in the future
- builder->n_batch = imgs.entries.size();
-
- return builder;
-}
-
-//
-// clip_model_loader
-//
-
-struct clip_model_loader {
- ggml_context_ptr ctx_meta;
- gguf_context_ptr ctx_gguf;
-
- std::string fname;
-
- size_t model_size = 0; // in bytes
-
- bool has_vision = false;
- bool has_audio = false;
- bool has_gen_audio = false;
-
- mtmd_progress_callback progress_callback = nullptr;
- void * progress_callback_user_data = nullptr;
-
- // TODO @ngxson : we should not pass clip_ctx here, it should be clip_model
- clip_model_loader(const char * fname,
- bool skip_tensors = false,
- mtmd_progress_callback progress_cb = nullptr,
- void * progress_user_data = nullptr)
- : fname(fname),
- progress_callback(progress_cb),
- progress_callback_user_data(progress_user_data) {
- struct ggml_context * meta = nullptr;
-
- struct gguf_init_params params = {
- /*.no_alloc = */ true,
- /*.ctx = */ &meta,
- };
-
- ctx_gguf = gguf_context_ptr(gguf_init_from_file(fname, params));
- if (!ctx_gguf.get()) {
- throw std::runtime_error(string_format("%s: failed to load CLIP model from %s. Does this file exist?\n", __func__, fname));
- }
-
- ctx_meta.reset(meta);
-
- const int n_tensors = gguf_get_n_tensors(ctx_gguf.get());
-
- // print gguf info
- {
- std::string name;
- get_string(KEY_NAME, name, false);
- std::string description;
- get_string(KEY_DESCRIPTION, description, false);
- LOG_INF("%s: model name: %s\n", __func__, name.c_str());
- LOG_INF("%s: description: %s\n", __func__, description.c_str());
- LOG_INF("%s: GGUF version: %d\n", __func__, gguf_get_version(ctx_gguf.get()));
- LOG_INF("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx_gguf.get()));
- LOG_INF("%s: n_tensors: %d\n", __func__, n_tensors);
- LOG_INF("%s: n_kv: %d\n", __func__, (int)gguf_get_n_kv(ctx_gguf.get()));
- LOG_INF("\n");
- }
-
- // modalities
- {
- get_bool(KEY_HAS_VISION_ENC, has_vision, false);
- get_bool(KEY_HAS_AUDIO_ENC, has_audio, false);
- get_bool(KEY_HAS_GEN_AUDIO_ENC, has_gen_audio, false);
-
- if (has_vision) {
- LOG_INF("%s: has vision encoder\n", __func__);
- }
- if (has_audio) {
- LOG_INF("%s: has audio encoder\n", __func__);
- }
- if (has_gen_audio) {
- LOG_INF("%s: has audio generation (gen) encoder\n", __func__);
- }
- }
-
- // tensors
- if (!skip_tensors) {
- for (int i = 0; i < n_tensors; ++i) {
- const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
- const size_t offset = gguf_get_tensor_offset(ctx_gguf.get(), i);
- enum ggml_type type = gguf_get_tensor_type(ctx_gguf.get(), i);
- ggml_tensor * cur = ggml_get_tensor(meta, name);
- size_t tensor_size = ggml_nbytes(cur);
- model_size += tensor_size;
- LOG_DBG("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, offset=%zu, shape:[%" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "], type = %s\n",
- __func__, i, ggml_n_dims(cur), cur->name, tensor_size, offset, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3], ggml_type_name(type));
- }
- }
- }
-
- void load_hparams(clip_model & model, clip_modality modality) {
- auto & hparams = model.hparams;
- std::string log_ffn_op; // for logging
-
- // sanity check
- if (modality == CLIP_MODALITY_VISION) {
- GGML_ASSERT(has_vision);
- } else if (modality == CLIP_MODALITY_AUDIO) {
- GGML_ASSERT(has_audio);
- } else if (modality == CLIP_MODALITY_GEN_AUDIO) {
- GGML_ASSERT(has_gen_audio);
- }
- model.modality = modality;
-
-
- // projector type
- std::string proj_type;
- {
- // default key
- get_string(KEY_PROJ_TYPE, proj_type, false);
-
- // for models with mixed modalities
- if (proj_type.empty()) {
- if (modality == CLIP_MODALITY_VISION) {
- get_string(KEY_VISION_PROJ_TYPE, proj_type, false);
- } else if (modality == CLIP_MODALITY_AUDIO) {
- get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false);
- } else if (modality == CLIP_MODALITY_GEN_AUDIO) {
- get_string(KEY_GEN_AUDIO_PROJ_TYPE, proj_type, false);
- } else {
- GGML_ABORT("unknown modality");
- }
- }
-
- model.proj_type = clip_projector_type_from_string(proj_type);
-
- if (model.proj_type == PROJECTOR_TYPE_UNKNOWN) {
- throw std::runtime_error(string_format("%s: unknown projector type: %s\n", __func__, proj_type.c_str()));
- }
-
- // correct arch for multimodal models (legacy method)
- if (model.proj_type == PROJECTOR_TYPE_QWEN25O) {
- model.proj_type = modality == CLIP_MODALITY_VISION
- ? PROJECTOR_TYPE_QWEN25VL
- : PROJECTOR_TYPE_QWEN2A;
- }
- }
-
- const bool is_vision = model.modality == CLIP_MODALITY_VISION;
- const bool is_audio = model.modality == CLIP_MODALITY_AUDIO;
- const bool is_gen_audio = model.modality == CLIP_MODALITY_GEN_AUDIO;
-
- // other hparams
- {
- const char * prefix = is_vision ? "vision" : (is_audio ? "audio" : "gen.audio");
- get_u32(string_format(KEY_N_EMBD, prefix), hparams.n_embd);
- get_u32(string_format(KEY_N_HEAD, prefix), hparams.n_head);
- get_u32(string_format(KEY_N_EMBD_HEAD, prefix), hparams.n_embd_head, false);
- get_u32(string_format(KEY_N_FF, prefix), hparams.n_ff);
- get_u32(string_format(KEY_N_BLOCK, prefix), hparams.n_layer);
- get_u32(string_format(KEY_PROJ_DIM, prefix), hparams.projection_dim);
- get_f32(string_format(KEY_LAYER_NORM_EPS, prefix), hparams.eps);
-
- // n_head_kv is optional (for GQA), default to n_head
- hparams.n_head_kv = hparams.n_head;
- get_u32(string_format(KEY_N_HEAD_KV, prefix), hparams.n_head_kv, false);
-
- if (is_vision) {
- get_u32(KEY_IMAGE_SIZE, hparams.image_size);
- get_u32(KEY_PATCH_SIZE, hparams.patch_size);
- get_u32(KEY_MAX_SLICE_NUMS, hparams.max_slice_nums, false);
- get_i32(KEY_MINICPMV_VERSION, hparams.minicpmv_version, false); // legacy
- get_u32(KEY_MINICPMV_QUERY_NUM, hparams.minicpmv_query_num, false);
- if (hparams.minicpmv_query_num == 0) {
- // Fallback to hardcoded values for legacy models
- if (hparams.minicpmv_version == 3) {
- hparams.minicpmv_query_num = 64;
- } else if (hparams.minicpmv_version == 4) {
- hparams.minicpmv_query_num = 64;
- } else if (hparams.minicpmv_version == 5) {
- hparams.minicpmv_query_num = 64;
- } else if (hparams.minicpmv_version == 6) {
- hparams.minicpmv_query_num = 64;
- } else if (hparams.minicpmv_version == 100045) {
- hparams.minicpmv_query_num = 64;
- } else {
- hparams.minicpmv_query_num = 96;
- }
- }
- } else if (is_audio) {
- get_u32(KEY_A_NUM_MEL_BINS, hparams.n_mel_bins);
- // some hparams are unused, but still need to set to avoid issues
- hparams.image_size = 0;
- hparams.patch_size = 1;
-
- } else if (is_gen_audio) {
- // these are unused, but still need to be set to avoid issues
- hparams.image_size = 0;
- hparams.patch_size = 1;
- get_string(KEY_GEN_AUDIO_VARIANT, hparams.gen_model_variant, false);
-
- } else {
- GGML_ASSERT(false && "unknown modality");
- }
-
- // for pinpoints, we need to convert it into a list of resolution candidates
- {
- std::vector<int> pinpoints;
- get_arr_int(KEY_IMAGE_GRID_PINPOINTS, pinpoints, false);
- if (pinpoints.size() % 2 != 0) {
- throw std::runtime_error(string_format("%s: image_grid_pinpoints must have an even number of elements, got %zu\n", __func__, pinpoints.size()));
- }
- if (!pinpoints.empty()) {
- for (size_t i = 0; i < pinpoints.size(); i += 2) {
- hparams.image_res_candidates.push_back({
- pinpoints[i],
- pinpoints[i+1],
- });
- }
- }
- }
-
- // default warmup value
- hparams.warmup_image_size = hparams.image_size;
-
- {
- bool use_gelu = false;
- bool use_silu = false;
- get_bool(KEY_USE_GELU, use_gelu, false);
- get_bool(KEY_USE_SILU, use_silu, false);
- if (use_gelu && use_silu) {
- throw std::runtime_error(string_format("%s: both use_gelu and use_silu are set to true\n", __func__));
- }
- if (use_gelu) {
- hparams.ffn_op = FFN_GELU;
- log_ffn_op = "gelu";
- } else if (use_silu) {
- hparams.ffn_op = FFN_SILU;
- log_ffn_op = "silu";
- } else {
- hparams.ffn_op = FFN_GELU_QUICK;
- log_ffn_op = "gelu_quick";
- }
- }
-
- {
- std::string mm_patch_merge_type;
- get_string(KEY_MM_PATCH_MERGE_TYPE, mm_patch_merge_type, false);
- if (mm_patch_merge_type == "spatial_unpad") {
- hparams.mm_patch_merge_type = PATCH_MERGE_SPATIAL_UNPAD;
+ {
+ std::string mm_patch_merge_type;
+ get_string(KEY_MM_PATCH_MERGE_TYPE, mm_patch_merge_type, false);
+ if (mm_patch_merge_type == "spatial_unpad") {
+ hparams.mm_patch_merge_type = PATCH_MERGE_SPATIAL_UNPAD;
}
}
@@ -1692,7 +379,7 @@ struct clip_model_loader {
{
get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor);
GGML_ASSERT(hparams.subsampling_factor == 8 &&
- "subsampling_factor must match the conv strides in clip_graph_parakeet::build()");
+ "subsampling_factor must match the conv strides in mtmd_graph_parakeet::build()");
get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size);
GGML_ASSERT(hparams.audio_conv_kernel_size > 0 && hparams.audio_conv_kernel_size % 2 == 1 &&
"audio_conv_kernel_size must be a positive odd integer");
@@ -1720,7 +407,7 @@ struct clip_model_loader {
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
// one full window
- using nt = clip_graph_nemotron_transcribe;
+ using nt = mtmd_graph_nemotron_transcribe;
hparams.warmup_audio_size = (nt::window_left + nt::window_core + nt::window_right) * hparams.subsampling_factor;
} break;
case PROJECTOR_TYPE_IDEFICS3:
@@ -2054,7 +741,7 @@ struct clip_model_loader {
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.image_resize_pad = PAD_NONE;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
- // n_merge is used as a divisor in clip_image_batch_encode
+ // n_merge is used as a divisor in mtmd_subcontext::encode()
// (gh / n_merge); reject 0 to avoid int div-by-zero (DoS).
GGML_ASSERT(hparams.n_merge > 0);
hparams.rope_theta = 10000.0f; // vision_config.rope_theta
@@ -2221,7 +908,7 @@ struct clip_model_loader {
hparams.audio_n_fft = 400;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
- // 2 context frames + 8 s of new frames, see clip_enc_state_init()
+ // 2 context frames + 8 s of new frames, see mtmd_subcontext::enc_state_init()
hparams.warmup_audio_size = 2 + 2*400;
} break;
case PROJECTOR_TYPE_MIMO_AUDIO:
@@ -2281,6 +968,9 @@ struct clip_model_loader {
} break;
case PROJECTOR_TYPE_LFM3A_GEN:
{
+ // TODO: CUDA has no flash attention kernel for the depthformer head dim 32, enable it once it does
+ model.flash_attn_auto = false;
+
// fixed by LiquidCodecConfig, head counts come from the tensors
hparams.rope_theta = 1000000.0f;
@@ -2359,6 +1049,9 @@ struct clip_model_loader {
} break;
case PROJECTOR_TYPE_CHATTERBOX:
{
+ // the classic cfm solver unrolls up to 10 steps x 2 cfg estimator evaluations in one graph
+ model.max_nodes = 65536;
+
// s3gen output rate
hparams.audio_sample_rate = 24000;
} break;
@@ -2617,8 +1310,7 @@ struct clip_model_loader {
}
}
- void load_tensors(clip_ctx & ctx_clip) {
- auto & model = ctx_clip.model;
+ void load_tensors(mtmd_model & model, ggml_backend_buffer_type_t buft, bool no_alloc) {
auto & hparams = model.hparams;
std::map<std::string, size_t> tensor_offset;
std::vector<ggml_tensor *> tensors_to_load;
@@ -2629,8 +1321,8 @@ struct clip_model_loader {
}
// TODO @ngxson : support both audio and video in the future
- const char * prefix = model.modality == CLIP_MODALITY_AUDIO ? "a"
- : model.modality == CLIP_MODALITY_GEN_AUDIO ? "a.gen.code"
+ const char * prefix = model.modality == MTMD_MODALITY_AUDIO ? "a"
+ : model.modality == MTMD_MODALITY_GEN_AUDIO ? "a.gen.code"
: "v";
// get offsets
@@ -2645,8 +1337,8 @@ struct clip_model_loader {
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ true,
};
- ctx_clip.ctx_data.reset(ggml_init(params));
- if (!ctx_clip.ctx_data) {
+ model.ctx_data.reset(ggml_init(params));
+ if (!model.ctx_data) {
throw std::runtime_error(string_format("%s: failed to init ggml context\n", __func__));
}
@@ -2663,18 +1355,18 @@ struct clip_model_loader {
}
if (cur) {
tensors_to_load.push_back(cur);
- ggml_tensor * data_tensor = ggml_dup_tensor(ctx_clip.ctx_data.get(), cur);
+ ggml_tensor * data_tensor = ggml_dup_tensor(model.ctx_data.get(), cur);
ggml_set_name(data_tensor, cur->name);
loaded_tensor_names.insert(name);
cur = data_tensor;
// add to weight memory counter
- ctx_clip.mem_usage[ggml_backend_get_device(ctx_clip.backend)] += ggml_nbytes(cur);
+ model.mem_weights[ggml_backend_buft_get_device(buft)] += ggml_nbytes(cur);
}
return cur;
};
// pocket-tts: the encoder and the decoder share the same layout, only the prefix differs
- auto load_seanet = [&](clip_seanet & seanet, bool is_decoder) {
+ auto load_seanet = [&](mtmd_seanet & seanet, bool is_decoder) {
const char * conv_in = is_decoder ? TN_A_GEN_WAV_SEANET_CONV_IN : TN_A_SEANET_CONV_IN;
const char * conv_out = is_decoder ? TN_A_GEN_WAV_SEANET_CONV_OUT : TN_A_SEANET_CONV_OUT;
const char * res1 = is_decoder ? TN_A_GEN_WAV_SEANET_RES_CONV1 : TN_A_SEANET_RES_CONV1;
@@ -3706,7 +2398,7 @@ struct clip_model_loader {
if (probe == nullptr) {
break;
}
- clip_flow_net::block blk;
+ mtmd_flow_net::block blk;
blk.norm_w = probe;
blk.norm_b = get_tensor(string_format(TN_A_GEN_FLOW_BLK_NORM, il, "bias"));
blk.up_w = get_tensor(string_format(TN_A_GEN_FLOW_BLK_UP, il, "weight"));
@@ -3866,7 +2558,7 @@ struct clip_model_loader {
auto has = [&](const std::string & name) {
return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0;
};
- auto load_enc_layer = [&](const std::string & p, clip_chatterbox::enc_layer & l) {
+ auto load_enc_layer = [&](const std::string & p, mtmd_chatterbox::enc_layer & l) {
l.norm_mha_w = get_tensor(p + ".norm_mha.weight");
l.norm_mha_b = get_tensor(p + ".norm_mha.bias");
l.attn_q_w = get_tensor(p + ".self_attn.linear_q.weight");
@@ -3887,13 +2579,13 @@ struct clip_model_loader {
l.ffn_2_w = get_tensor(p + ".feed_forward.w_2.weight");
l.ffn_2_b = get_tensor(p + ".feed_forward.w_2.bias");
};
- auto load_causal = [&](const std::string & p, clip_chatterbox::causal_block & b) {
+ auto load_causal = [&](const std::string & p, mtmd_chatterbox::causal_block & b) {
b.conv_w = get_tensor(p + ".block.0.weight");
b.conv_b = get_tensor(p + ".block.0.bias");
b.norm_w = get_tensor(p + ".block.2.weight");
b.norm_b = get_tensor(p + ".block.2.bias");
};
- auto load_resnet = [&](const std::string & p, clip_chatterbox::resnet & r) {
+ auto load_resnet = [&](const std::string & p, mtmd_chatterbox::resnet & r) {
load_causal(p + ".block1", r.block1);
load_causal(p + ".block2", r.block2);
r.mlp_w = get_tensor(p + ".mlp.1.weight");
@@ -3901,7 +2593,7 @@ struct clip_model_loader {
r.res_conv_w = get_tensor(p + ".res_conv.weight");
r.res_conv_b = get_tensor(p + ".res_conv.bias");
};
- auto load_tfm = [&](const std::string & p, clip_chatterbox::tfm_block & b) {
+ auto load_tfm = [&](const std::string & p, mtmd_chatterbox::tfm_block & b) {
b.norm1_w = get_tensor(p + ".norm1.weight");
b.norm1_b = get_tensor(p + ".norm1.bias");
b.attn_q_w = get_tensor(p + ".attn1.to_q.weight");
@@ -3916,10 +2608,10 @@ struct clip_model_loader {
b.ff_out_w = get_tensor(p + ".ff.net.2.weight");
b.ff_out_b = get_tensor(p + ".ff.net.2.bias");
};
- auto load_stage = [&](const std::string & p, clip_chatterbox::est_stage & s, bool boundary) {
+ auto load_stage = [&](const std::string & p, mtmd_chatterbox::est_stage & s, bool boundary) {
load_resnet(p + ".0", s.res);
for (int j = 0; has(p + ".1." + std::to_string(j) + ".norm1.weight"); j++) {
- clip_chatterbox::tfm_block b;
+ mtmd_chatterbox::tfm_block b;
load_tfm(p + ".1." + std::to_string(j), b);
s.tfm.push_back(b);
}
@@ -3928,10 +2620,10 @@ struct clip_model_loader {
s.conv_b = get_tensor(p + ".2.bias");
}
};
- auto load_hres = [&](const std::string & p, clip_chatterbox::hift_res & r) {
+ auto load_hres = [&](const std::string & p, mtmd_chatterbox::hift_res & r) {
for (int j = 0; has(p + ".convs1." + std::to_string(j) + ".weight"); j++) {
const std::string js = std::to_string(j);
- clip_chatterbox::hift_res_unit u;
+ mtmd_chatterbox::hift_res_unit u;
u.act1_alpha = get_tensor(p + ".activations1." + js + ".alpha");
u.act2_alpha = get_tensor(p + ".activations2." + js + ".alpha");
u.conv1_w = get_tensor(p + ".convs1." + js + ".weight");
@@ -3955,7 +2647,7 @@ struct clip_model_loader {
c.pre_conv2_w = get_tensor("a.gen.fenc.pre_lookahead_layer.conv2.weight");
c.pre_conv2_b = get_tensor("a.gen.fenc.pre_lookahead_layer.conv2.bias");
for (int i = 0; has("a.gen.fenc.encoders." + std::to_string(i) + ".norm_mha.weight"); i++) {
- clip_chatterbox::enc_layer l;
+ mtmd_chatterbox::enc_layer l;
load_enc_layer("a.gen.fenc.encoders." + std::to_string(i), l);
c.enc.push_back(l);
}
@@ -3966,7 +2658,7 @@ struct clip_model_loader {
c.up_embed_norm_w = get_tensor("a.gen.fenc.up_embed.out.1.weight");
c.up_embed_norm_b = get_tensor("a.gen.fenc.up_embed.out.1.bias");
for (int i = 0; has("a.gen.fenc.up_encoders." + std::to_string(i) + ".norm_mha.weight"); i++) {
- clip_chatterbox::enc_layer l;
+ mtmd_chatterbox::enc_layer l;
load_enc_layer("a.gen.fenc.up_encoders." + std::to_string(i), l);
c.up_enc.push_back(l);
}
@@ -3983,7 +2675,7 @@ struct clip_model_loader {
c.time_embed_mixer_w = get_tensor("a.gen.est.time_embed_mixer.weight", false);
load_stage("a.gen.est.down_blocks.0", c.est_down, true);
for (int i = 0; has("a.gen.est.mid_blocks." + std::to_string(i) + ".0.block1.block.0.weight"); i++) {
- clip_chatterbox::est_stage s;
+ mtmd_chatterbox::est_stage s;
load_stage("a.gen.est.mid_blocks." + std::to_string(i), s, false);
c.est_mid.push_back(std::move(s));
}
@@ -3999,7 +2691,7 @@ struct clip_model_loader {
c.hift_post_b = get_tensor("a.gen.hift.conv_post.bias");
for (int i = 0; has("a.gen.hift.ups." + std::to_string(i) + ".weight"); i++) {
const std::string is = std::to_string(i);
- clip_chatterbox::hift_up up;
+ mtmd_chatterbox::hift_up up;
up.up_w = get_tensor("a.gen.hift.ups." + is + ".weight");
up.up_b = get_tensor("a.gen.hift.ups." + is + ".bias");
up.source_down_w = get_tensor("a.gen.hift.source_downs." + is + ".weight");
@@ -4012,7 +2704,7 @@ struct clip_model_loader {
}
for (int i = 0; has("a.gen.hift.f0_predictor.condnet." + std::to_string(i) + ".weight"); i += 2) {
const std::string is = std::to_string(i);
- clip_chatterbox::f0_conv fc;
+ mtmd_chatterbox::f0_conv fc;
fc.w = get_tensor("a.gen.hift.f0_predictor.condnet." + is + ".weight");
fc.b = get_tensor("a.gen.hift.f0_predictor.condnet." + is + ".bias");
c.f0_condnet.push_back(fc);
@@ -4027,7 +2719,7 @@ struct clip_model_loader {
c.s3tok_conv2_b = get_tensor("a.s3tok.encoder.conv2.bias");
for (int i = 0; has("a.s3tok.encoder.blocks." + std::to_string(i) + ".attn_ln.weight"); i++) {
const std::string p = "a.s3tok.encoder.blocks." + std::to_string(i);
- clip_chatterbox::s3tok_block b;
+ mtmd_chatterbox::s3tok_block b;
b.attn_ln_w = get_tensor(p + ".attn_ln.weight");
b.attn_ln_b = get_tensor(p + ".attn_ln.bias");
b.attn_q_w = get_tensor(p + ".attn.query.weight");
@@ -4121,13 +2813,13 @@ struct clip_model_loader {
auto has = [&](const std::string & name) {
return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0;
};
- auto load_bn = [&](const std::string & p, clip_chatterbox::bn & n) {
+ auto load_bn = [&](const std::string & p, mtmd_chatterbox::bn & n) {
n.mean = get_tensor(p + ".running_mean");
n.var = get_tensor(p + ".running_var");
n.w = get_tensor(p + ".weight", false);
n.b = get_tensor(p + ".bias", false);
};
- auto load_res2d = [&](const std::string & p, clip_chatterbox::spk_res2d & r) {
+ auto load_res2d = [&](const std::string & p, mtmd_chatterbox::spk_res2d & r) {
r.conv1_w = get_tensor(p + ".conv1.weight");
load_bn(p + ".bn1", r.bn1);
r.conv2_w = get_tensor(p + ".conv2.weight");
@@ -4137,10 +2829,10 @@ struct clip_model_loader {
load_bn(p + ".shortcut.1", r.shortcut_bn);
}
};
- auto load_cam_block = [&](const std::string & bp, const std::string & tp, clip_chatterbox::spk_cam_block & blk) {
+ auto load_cam_block = [&](const std::string & bp, const std::string & tp, mtmd_chatterbox::spk_cam_block & blk) {
for (int li = 1; has(bp + ".tdnnd" + std::to_string(li) + ".linear1.weight"); li++) {
const std::string p = bp + ".tdnnd" + std::to_string(li);
- clip_chatterbox::spk_cam_layer l;
+ mtmd_chatterbox::spk_cam_layer l;
load_bn(p + ".nonlinear1.batchnorm", l.nl1_bn);
l.linear1_w = get_tensor(p + ".linear1.weight");
load_bn(p + ".nonlinear2.batchnorm", l.nl2_bn);
@@ -4155,4017 +2847,1361 @@ struct clip_model_loader {
blk.transit_w = get_tensor(tp + ".linear.weight");
};
- c.spk_conv1_w = get_tensor("a.spk.head.conv1.weight");
- load_bn("a.spk.head.bn1", c.spk_bn1);
- load_res2d("a.spk.head.layer1.0", c.spk_layer1_0);
- load_res2d("a.spk.head.layer1.1", c.spk_layer1_1);
- load_res2d("a.spk.head.layer2.0", c.spk_layer2_0);
- load_res2d("a.spk.head.layer2.1", c.spk_layer2_1);
- c.spk_conv2_w = get_tensor("a.spk.head.conv2.weight");
- load_bn("a.spk.head.bn2", c.spk_bn2);
- c.spk_tdnn_w = get_tensor("a.spk.xvector.tdnn.linear.weight");
- load_bn("a.spk.xvector.tdnn.nonlinear.batchnorm", c.spk_tdnn_bn);
- load_cam_block("a.spk.xvector.block1", "a.spk.xvector.transit1", c.spk_block1);
- load_cam_block("a.spk.xvector.block2", "a.spk.xvector.transit2", c.spk_block2);
- load_cam_block("a.spk.xvector.block3", "a.spk.xvector.transit3", c.spk_block3);
- load_bn("a.spk.xvector.out_nonlinear.batchnorm", c.spk_out_bn);
- c.spk_dense_w = get_tensor("a.spk.xvector.dense.linear.weight");
- load_bn("a.spk.xvector.dense.nonlinear.batchnorm", c.spk_dense_bn);
-
- // read on the host: voice encoder lstm, conditioning encoder
- for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) {
- const std::string name = t->name;
- if (name.rfind("a.ve.", 0) == 0 || name.rfind("a.cenc.", 0) == 0) {
- model.cbx_tensors[name] = get_tensor(name);
- }
- }
- } break;
- case PROJECTOR_TYPE_SOPRANO:
- {
- auto & v = model.vocos;
- v.input_w = get_tensor(string_format(TN_A_GEN_WAV_INPUT, "weight"));
- v.input_b = get_tensor(string_format(TN_A_GEN_WAV_INPUT, "bias"));
- v.norm_w = get_tensor(string_format(TN_A_GEN_WAV_NORM, "weight"));
- v.norm_b = get_tensor(string_format(TN_A_GEN_WAV_NORM, "bias"));
- v.output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT_NORM, "weight"));
- v.output_norm_b = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT_NORM, "bias"));
- v.output_w = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT, "weight"));
- v.output_b = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT, "bias"));
- v.blocks.resize(hparams.n_layer);
- for (int il = 0; il < hparams.n_layer; ++il) {
- auto & b = v.blocks[il];
- b.dwconv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight"));
- b.dwconv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias"));
- b.norm_w = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "weight"));
- b.norm_b = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "bias"));
- b.pw1_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "weight"));
- b.pw1_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "bias"));
- b.pw2_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "weight"));
- b.pw2_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "bias"));
- b.gamma = get_tensor(string_format(TN_A_GEN_WAV_UP_GAMMA, il));
- }
- } break;
- case PROJECTOR_TYPE_VOXTRAL:
- case PROJECTOR_TYPE_VOXTRAL_RT:
- {
- model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
- model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
- model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
- model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
- model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
- model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
- } break;
- case PROJECTOR_TYPE_MUSIC_FLAMINGO:
- {
- model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
- model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
- model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
- model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
- model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
- model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
- model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
- model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias"));
- } break;
- case PROJECTOR_TYPE_INTERNVL:
- {
- model.mm_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
- model.mm_0_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "bias"));
- model.mm_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
- model.mm_1_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "bias"));
- model.mm_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
- model.mm_3_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "bias"));
- } break;
- case PROJECTOR_TYPE_NEMOTRON_V2_VL:
- case PROJECTOR_TYPE_NEMOTRON_OMNI:
- {
- model.mm_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
- model.mm_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
- model.mm_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
- } break;
- case PROJECTOR_TYPE_GLMA:
- {
- model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
- model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
- model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
- model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
- model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
- model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
- model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
- model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias"));
- model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"));
- model.mm_norm_pre_b = get_tensor(string_format(TN_MM_NORM_PRE, "bias"));
- model.mm_boi = get_tensor(string_format(TN_TOK_BOI));
- model.mm_eoi = get_tensor(string_format(TN_TOK_EOI));
- } break;
- case PROJECTOR_TYPE_LLAMA4:
- {
- model.mm_model_proj = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
- model.mm_model_mlp_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
- model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 2, "weight"));
- } break;
- case PROJECTOR_TYPE_COGVLM:
- {
- model.mm_model_proj = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
- model.mm_post_fc_norm_w = get_tensor(string_format(TN_MM_POST_FC_NORM, "weight"));
- model.mm_post_fc_norm_b = get_tensor(string_format(TN_MM_POST_FC_NORM, "bias"));
- model.mm_h_to_4h_w = get_tensor(string_format(TN_MM_H_TO_4H, "weight"));
- model.mm_gate_w = get_tensor(string_format(TN_MM_GATE, "weight"));
- model.mm_4h_to_h_w = get_tensor(string_format(TN_MM_4H_TO_H, "weight"));
- model.mm_boi = get_tensor(TN_TOK_BOI);
- model.mm_eoi = get_tensor(TN_TOK_EOI);
- } break;
- case PROJECTOR_TYPE_HUNYUANVL:
- {
- // proj.0 -> mm.0 (conv1), proj.2 -> mm.2 (conv2), mlp -> mm.model.fc (linear)
- model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
- model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
- model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
- model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
- model.mm_model_proj = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
- model.mm_model_proj_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias"));
- model.mm_pre_norm_w = get_tensor(string_format(TN_MM_PRE_NORM, "weight"));
- model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
- model.mm_img_begin = get_tensor(TN_MM_IMG_BEGIN);
- model.mm_img_end = get_tensor(TN_MM_IMG_END);
- model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
- model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false);
- } break;
- case PROJECTOR_TYPE_JANUS_PRO:
- {
- model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
- model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
- model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
- model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
- } break;
- case PROJECTOR_TYPE_PHI4:
- {
- model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
- model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
- model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
- model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
- } break;
- case PROJECTOR_TYPE_DEEPSEEKOCR:
- case PROJECTOR_TYPE_DEEPSEEKOCR2:
- {
- model.pos_embed = get_tensor(string_format(TN_SAM_POS_EMBD, "weight"));
- model.patch_embed_proj_w = get_tensor(string_format(TN_SAM_PATCH_EMBD, "weight"));
- model.patch_embed_proj_b = get_tensor(string_format(TN_SAM_PATCH_EMBD, "bias"));
- model.n_sam_layers = hparams.sam_n_layer;
- model.sam_layers.resize(model.n_sam_layers);
- for (int il = 0; il < model.n_sam_layers; ++il) {
- auto & layer = model.sam_layers[il];
- layer.qkv_w = get_tensor(string_format(TN_SAM_ATTN_QKV, il, "weight"));
- layer.qkv_b = get_tensor(string_format(TN_SAM_ATTN_QKV, il, "bias"));
- layer.o_w = get_tensor(string_format(TN_SAM_ATTN_OUT, il, "weight"));
- layer.o_b = get_tensor(string_format(TN_SAM_ATTN_OUT, il, "bias"));
- layer.ln_1_w = get_tensor(string_format(TN_SAM_PRE_NORM, il, "weight"));
- layer.ln_1_b = get_tensor(string_format(TN_SAM_PRE_NORM, il, "bias"));
- layer.ln_2_w = get_tensor(string_format(TN_SAM_POST_NORM, il, "weight"));
- layer.ln_2_b = get_tensor(string_format(TN_SAM_POST_NORM, il, "bias"));
- layer.rel_pos_h = get_tensor(string_format(TN_SAM_ATTN_POS_H, il, "weight"));
- layer.rel_pos_w = get_tensor(string_format(TN_SAM_ATTN_POS_W, il, "weight"));
- layer.ff_up_w = get_tensor(string_format(TN_SAM_FFN_UP, il, "weight"));
- layer.ff_up_b = get_tensor(string_format(TN_SAM_FFN_UP, il, "bias"));
- layer.ff_down_w = get_tensor(string_format(TN_SAM_FFN_DOWN, il, "weight"));
- layer.ff_down_b = get_tensor(string_format(TN_SAM_FFN_DOWN, il, "bias"));
- }
- model.neck_0_w = get_tensor(string_format(TN_SAM_NECK, 0, "weight"));
- model.neck_1_b = get_tensor(string_format(TN_SAM_NECK, 1, "bias"));
- model.neck_1_w = get_tensor(string_format(TN_SAM_NECK, 1, "weight"));
- model.neck_2_w = get_tensor(string_format(TN_SAM_NECK, 2, "weight"));
- model.neck_3_b = get_tensor(string_format(TN_SAM_NECK, 3, "bias"));
- model.neck_3_w = get_tensor(string_format(TN_SAM_NECK, 3, "weight"));
- model.net_2 = get_tensor(string_format(TN_SAM_NET, 2, "weight"));
- model.net_3 = get_tensor(string_format(TN_SAM_NET, 3, "weight"));
- model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false);
- model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR);
- model.mm_fc_w = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
- model.mm_fc_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias"));
- model.resample_query_768 = get_tensor(string_format(TN_RESMPL_QUERY, 768, "weight"), false);
- model.resample_query_1024 = get_tensor(string_format(TN_RESMPL_QUERY, 1024, "weight"), false);
- } break;
- case PROJECTOR_TYPE_GEMMA4A:
- {
- for (int i = 0; i < 2; i++) {
- model.sscp_conv_w[i] = get_tensor(string_format(TN_A_CONV1D, i, "weight"));
- model.sscp_conv_b[i] = get_tensor(string_format(TN_A_CONV1D, i, "bias"), false);
- model.sscp_norm_w[i] = get_tensor(string_format(TN_A_CONV1D_NORM, i, "weight"), false);
- }
- model.sscp_inp_proj_w = get_tensor(string_format(TN_A_INP_PROJ, "weight"));
- model.sscp_inp_proj_b = get_tensor(string_format(TN_A_INP_PROJ, "bias"), false);
- model.audio_out_proj_w = get_tensor(string_format(TN_A_OUT_PROJ, "weight"), false);
- model.audio_out_proj_b = get_tensor(string_format(TN_A_OUT_PROJ, "bias"), false);
- // audio multimodal embedder (mm.a.* namespace, not mm.*)
- model.mm_soft_emb_norm_w = get_tensor(string_format(TN_A_MM_SOFT_EMB_N, "weight"), false);
- model.mm_input_proj_w = get_tensor(string_format(TN_A_MM_INP_PROJ, "weight"), false);
-
- // Per-layer tensors NOT loaded by the generic loop above
- for (int il = 0; il < hparams.n_layer; ++il) {
- auto & layer = model.layers[il];
-
- // Gemma4 audio conformer-specific tensors
- layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight"));
- layer.attn_pre_norm_w = get_tensor(string_format(TN_A_ATTN_PRE_NORM, prefix, il, "weight"), false);
- layer.per_dim_scale_w = get_tensor(string_format(TN_A_PER_DIM_SCALE, prefix, il, "weight"), false);
- layer.per_dim_k_scale_w = get_tensor(string_format(TN_A_PER_DIM_K_SCALE, prefix, il, "weight"), false);
- layer.attn_k_rel_w = get_tensor(string_format(TN_A_ATTN_K_REL, prefix, il, "weight"), false);
-
- // Convolution module
- // Note: conv_norm / norm_conv are swapped in GGUF due to
- // upstream tensor_mapping.py, so we load them in reverse order
- layer.norm_conv_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight"), false);
- layer.norm_conv_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias"), false);
- layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight"));
- layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias"), false);
- layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight"));
- layer.conv_dw_b = get_tensor(string_format(TN_CONV_DW, prefix, il, "bias"), false);
- layer.conv_norm_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"), false);
- layer.conv_norm_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"), false);
- layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight"));
- layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"), false);
-
- // FFN2 (second half-step)
- layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight"));
- layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight"));
- layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias"), false);
- layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight"));
- layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"), false);
- layer.ff_post_norm_1_w = get_tensor(string_format(TN_A_FFN_POST_NORM_1, prefix, il, "weight"), false);
- }
+ c.spk_conv1_w = get_tensor("a.spk.head.conv1.weight");
+ load_bn("a.spk.head.bn1", c.spk_bn1);
+ load_res2d("a.spk.head.layer1.0", c.spk_layer1_0);
+ load_res2d("a.spk.head.layer1.1", c.spk_layer1_1);
+ load_res2d("a.spk.head.layer2.0", c.spk_layer2_0);
+ load_res2d("a.spk.head.layer2.1", c.spk_layer2_1);
+ c.spk_conv2_w = get_tensor("a.spk.head.conv2.weight");
+ load_bn("a.spk.head.bn2", c.spk_bn2);
+ c.spk_tdnn_w = get_tensor("a.spk.xvector.tdnn.linear.weight");
+ load_bn("a.spk.xvector.tdnn.nonlinear.batchnorm", c.spk_tdnn_bn);
+ load_cam_block("a.spk.xvector.block1", "a.spk.xvector.transit1", c.spk_block1);
+ load_cam_block("a.spk.xvector.block2", "a.spk.xvector.transit2", c.spk_block2);
+ load_cam_block("a.spk.xvector.block3", "a.spk.xvector.transit3", c.spk_block3);
+ load_bn("a.spk.xvector.out_nonlinear.batchnorm", c.spk_out_bn);
+ c.spk_dense_w = get_tensor("a.spk.xvector.dense.linear.weight");
+ load_bn("a.spk.xvector.dense.nonlinear.batchnorm", c.spk_dense_bn);
- // Load clamp info for ClippableLinear AFTER all tensors are loaded
- for (auto * tensor : tensors_to_load) {
- std::string name = tensor->name;
- if (string_ends_with(name, ".weight")) {
- std::string name_inp_max = name;
- std::string name_inp_min = name;
- std::string name_out_max = name;
- std::string name_out_min = name;
- string_replace_all(name_inp_max, ".weight", ".input_max");
- string_replace_all(name_inp_min, ".weight", ".input_min");
- string_replace_all(name_out_max, ".weight", ".output_max");
- string_replace_all(name_out_min, ".weight", ".output_min");
- model.clamp_info_map[name] = {
- get_scalar(name_inp_max, FLT_MAX),
- get_scalar(name_inp_min, -FLT_MAX),
- get_scalar(name_out_max, FLT_MAX),
- get_scalar(name_out_min, -FLT_MAX)
- };
+ // read on the host: voice encoder lstm, conditioning encoder
+ for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) {
+ const std::string name = t->name;
+ if (name.rfind("a.ve.", 0) == 0 || name.rfind("a.cenc.", 0) == 0) {
+ model.cbx_tensors[name] = get_tensor(name);
}
}
} break;
- case PROJECTOR_TYPE_GEMMA4UA:
+ case PROJECTOR_TYPE_SOPRANO:
{
- model.mm_input_proj_w = get_tensor(string_format(TN_A_MM_INP_PROJ, "weight"));
+ auto & v = model.vocos;
+ v.input_w = get_tensor(string_format(TN_A_GEN_WAV_INPUT, "weight"));
+ v.input_b = get_tensor(string_format(TN_A_GEN_WAV_INPUT, "bias"));
+ v.norm_w = get_tensor(string_format(TN_A_GEN_WAV_NORM, "weight"));
+ v.norm_b = get_tensor(string_format(TN_A_GEN_WAV_NORM, "bias"));
+ v.output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT_NORM, "weight"));
+ v.output_norm_b = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT_NORM, "bias"));
+ v.output_w = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT, "weight"));
+ v.output_b = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT, "bias"));
+ v.blocks.resize(hparams.n_layer);
+ for (int il = 0; il < hparams.n_layer; ++il) {
+ auto & b = v.blocks[il];
+ b.dwconv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight"));
+ b.dwconv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias"));
+ b.norm_w = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "weight"));
+ b.norm_b = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "bias"));
+ b.pw1_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "weight"));
+ b.pw1_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "bias"));
+ b.pw2_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "weight"));
+ b.pw2_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "bias"));
+ b.gamma = get_tensor(string_format(TN_A_GEN_WAV_UP_GAMMA, il));
+ }
} break;
- case PROJECTOR_TYPE_LFM2A:
- case PROJECTOR_TYPE_D1OMNI_A:
+ case PROJECTOR_TYPE_VOXTRAL:
+ case PROJECTOR_TYPE_VOXTRAL_RT:
{
- for (int i : {0, 2, 3, 5, 6}) {
- model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight"));
- model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias"));
- }
- model.pre_encode_out_w = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight"));
- model.pre_encode_out_b = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias"));
-
- model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "weight"));
- model.mm_0_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "bias"));
+ model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
+ model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
+ model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
+ model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
+ model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
+ model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
+ } break;
+ case PROJECTOR_TYPE_MUSIC_FLAMINGO:
+ {
+ model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
+ model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
+ model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
+ model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
- model.mm_3_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight"));
- model.mm_3_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias"));
-
- // residual block after the projector: norm, down, up
- if (model.proj_type == PROJECTOR_TYPE_D1OMNI_A) {
- model.mm_4_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 4, "weight"));
- model.mm_4_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 4, "bias"));
- model.mm_5_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 5, "weight"));
- model.mm_5_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 5, "bias"));
- model.mm_6_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 6, "weight"));
- model.mm_6_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 6, "bias"));
- }
-
- for (int il = 0; il < hparams.n_layer; ++il) {
- auto & layer = model.layers[il];
-
- layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight"));
- layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, il, "bias"));
- layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight"));
- layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias"));
- layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight"));
- layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias"));
- layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight"));
- layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"));
-
- layer.pos_bias_u = get_tensor(string_format(TN_POS_BIAS_U, prefix, il));
- layer.pos_bias_v = get_tensor(string_format(TN_POS_BIAS_V, prefix, il));
-
- layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"));
- layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"));
-
- layer.linear_pos_w = get_tensor(string_format(TN_LINEAR_POS, prefix, il, "weight"));
-
- layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight"));
- layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias"));
- layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight"));
- layer.conv_dw_b = get_tensor(string_format(TN_CONV_DW, prefix, il, "bias"));
- layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight"));
- layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias"));
- layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight"));
- layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"));
- }
+ model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
+ model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias"));
} break;
- case PROJECTOR_TYPE_LFM3A:
+ case PROJECTOR_TYPE_INTERNVL:
{
- for (int i : {0, 2, 3, 5, 6}) {
- model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight"));
- model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias"));
- }
- model.pre_encode_out_w = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight"));
- model.pre_encode_out_b = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias"));
-
- model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "weight"));
- model.mm_0_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "bias"));
+ model.mm_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
+ model.mm_0_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "bias"));
+ model.mm_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
+ model.mm_1_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "bias"));
+ model.mm_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
+ model.mm_3_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "bias"));
+ } break;
+ case PROJECTOR_TYPE_NEMOTRON_V2_VL:
+ case PROJECTOR_TYPE_NEMOTRON_OMNI:
+ {
+ model.mm_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
+ model.mm_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
+ model.mm_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
+ } break;
+ case PROJECTOR_TYPE_GLMA:
+ {
+ model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
+ model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
+ model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
+ model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
- model.mm_3_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight"));
- model.mm_3_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias"));
-
- if (!model.post_ln_w) {
- throw std::runtime_error(string_format("%s: missing tensor %s\n", __func__, string_format(TN_LN_POST, prefix, "weight").c_str()));
- }
-
- // each layer is either a short conv block or a complete attention block
- for (int il = 0; il < hparams.n_layer; ++il) {
- auto & layer = model.layers[il];
- layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight"), false);
- layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight"), false);
- layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight"), false);
- const bool is_conv = layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w;
- const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w && layer.q_norm && layer.k_norm;
- if (is_conv == is_attn) {
- throw std::runtime_error(string_format("%s: lfm3a layer %d is neither a complete conv nor a complete attention block\n", __func__, il));
- }
- }
+ model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
+ model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias"));
+ model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"));
+ model.mm_norm_pre_b = get_tensor(string_format(TN_MM_NORM_PRE, "bias"));
+ model.mm_boi = get_tensor(string_format(TN_TOK_BOI));
+ model.mm_eoi = get_tensor(string_format(TN_TOK_EOI));
} break;
- case PROJECTOR_TYPE_PARAKEET:
+ case PROJECTOR_TYPE_LLAMA4:
{
-
- hparams.mel_filters = get_vector(TN_MEL_FILTERS);
- hparams.window = get_vector(TN_WINDOW);
-
- // Subsampling layers (conv1d)
- for (int i : {0, 2, 3, 5, 6}) {
- model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight"));
- model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias"));
- }
- model.pre_encode_out_w = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight"));
- model.pre_encode_out_b = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias"));
-
- // Projection layers
- model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"), false);
- model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"), false);
- model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"), false);
-
- // Encoder layers
- for (int il = 0; il < hparams.n_layer; ++il) {
- auto & layer = model.layers[il];
-
- // Attention (from shared above)
-
- // Relative position encoding
- layer.linear_pos_w = get_tensor(string_format(TN_LINEAR_POS, prefix, il, "weight"));
- layer.pos_bias_u = get_tensor(string_format(TN_POS_BIAS_U, prefix, il));
- layer.pos_bias_v = get_tensor(string_format(TN_POS_BIAS_V, prefix, il));
-
- // Convolution module
- layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight"));
- layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias"), false);
- layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight"));
- layer.conv_dw_b = get_tensor(string_format(TN_CONV_DW, prefix, il, "bias"), false);
- layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight"));
- layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias"));
- layer.conv_norm_mean = get_tensor(string_format(TN_CONV_NORM_MEAN, prefix, il));
- layer.conv_norm_var = get_tensor(string_format(TN_CONV_NORM_VAR, prefix, il));
- layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight"));
- layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"), false);
-
- // Feed-forward networks
- layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight"));
- layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, il, "bias"));
-
- layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight"));
- layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias"));
- layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight"));
- layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias"), false);
- layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight"));
- layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"), false);
-
- // Layer norms
- layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"));
- layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"));
- }
-
- model.mm_model_mlp_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
- model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
- model.mm_model_mlp_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
+ model.mm_model_proj = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
+ model.mm_model_mlp_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
+ model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 2, "weight"));
+ } break;
+ case PROJECTOR_TYPE_COGVLM:
+ {
+ model.mm_model_proj = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
+ model.mm_post_fc_norm_w = get_tensor(string_format(TN_MM_POST_FC_NORM, "weight"));
+ model.mm_post_fc_norm_b = get_tensor(string_format(TN_MM_POST_FC_NORM, "bias"));
+ model.mm_h_to_4h_w = get_tensor(string_format(TN_MM_H_TO_4H, "weight"));
+ model.mm_gate_w = get_tensor(string_format(TN_MM_GATE, "weight"));
+ model.mm_4h_to_h_w = get_tensor(string_format(TN_MM_4H_TO_H, "weight"));
+ model.mm_boi = get_tensor(TN_TOK_BOI);
+ model.mm_eoi = get_tensor(TN_TOK_EOI);
} break;
- case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
+ case PROJECTOR_TYPE_HUNYUANVL:
{
- hparams.mel_filters = get_vector(TN_MEL_FILTERS);
- hparams.window = get_vector(TN_WINDOW);
-
- // ASR expert: blocks, pre_ln and post_ln come from the generic loader
- model.inp_proj_w = get_tensor(string_format(TN_INP_PROJ, "weight"));
- for (int il = 0; il < hparams.n_layer; ++il) {
- auto & layer = model.layers[il];
- layer.q_norm_b = get_tensor(string_format(TN_ATTN_Q_NORM, prefix, il, "bias"));
- layer.k_norm_b = get_tensor(string_format(TN_ATTN_K_NORM, prefix, il, "bias"));
- }
-
- // speaker expert
- const char * sp = "a.spk";
- model.spk_inp_proj_w = get_tensor(string_format(TN_A_SPK_INP_PROJ, "weight"));
- model.spk_pre_ln_w = get_tensor(string_format(TN_LN_PRE, sp, "weight"));
- model.spk_pre_ln_b = get_tensor(string_format(TN_LN_PRE, sp, "bias"));
- model.spk_post_ln_w = get_tensor(string_format(TN_LN_POST, sp, "weight"));
- model.spk_post_ln_b = get_tensor(string_format(TN_LN_POST, sp, "bias"));
- model.spk_layers.resize(hparams.spk_n_layer);
- for (int il = 0; il < hparams.spk_n_layer; ++il) {
- auto & layer = model.spk_layers[il];
- layer.ln_1_w = get_tensor(string_format(TN_LN_1, sp, il, "weight"));
- layer.ln_1_b = get_tensor(string_format(TN_LN_1, sp, il, "bias"));
- layer.qkv_w = get_tensor(string_format(TN_ATTN_QKV, sp, il, "weight"));
- layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, sp, il, "weight"));
- layer.o_b = get_tensor(string_format(TN_ATTN_OUTPUT, sp, il, "bias"));
- layer.ln_2_w = get_tensor(string_format(TN_LN_2, sp, il, "weight"));
- layer.ln_2_b = get_tensor(string_format(TN_LN_2, sp, il, "bias"));
- layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, sp, il, "weight"));
- layer.ff_up_b = get_tensor(string_format(TN_FFN_UP, sp, il, "bias"));
- layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, sp, il, "weight"));
- layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, sp, il, "bias"));
- }
-
- // Sortformer head, fusion
- model.spk_head_proj_w = get_tensor(string_format(TN_A_SPK_HEAD, "proj", "weight"));
- model.spk_head_proj_b = get_tensor(string_format(TN_A_SPK_HEAD, "proj", "bias"));
- model.spk_head_up_w = get_tensor(string_format(TN_A_SPK_HEAD, "upsample", "weight"));
- model.spk_head_up_b = get_tensor(string_format(TN_A_SPK_HEAD, "upsample", "bias"));
- model.spk_head_fc1_w = get_tensor(string_format(TN_A_SPK_HEAD, "fc1", "weight"));
- model.spk_head_fc1_b = get_tensor(string_format(TN_A_SPK_HEAD, "fc1", "bias"));
- model.spk_head_fc2_w = get_tensor(string_format(TN_A_SPK_HEAD, "fc2", "weight"));
- model.spk_head_fc2_b = get_tensor(string_format(TN_A_SPK_HEAD, "fc2", "bias"));
- model.fusion_asr_norm_w = get_tensor(string_format(TN_A_FUSION_NORM, "asr_norm", "weight"));
- model.fusion_asr_norm_b = get_tensor(string_format(TN_A_FUSION_NORM, "asr_norm", "bias"));
- model.fusion_spk_norm_w = get_tensor(string_format(TN_A_FUSION_NORM, "spk_norm", "weight"));
- model.fusion_spk_norm_b = get_tensor(string_format(TN_A_FUSION_NORM, "spk_norm", "bias"));
- model.fusion_spk_kernel = get_tensor(TN_A_FUSION_KERNEL);
-
- model.spk_sil_emb = get_vector(TN_A_SPK_SIL_EMB);
-
- // the head layouts and the widths the graph and the speaker state use must match the tensors
- hparams.spk_n_embd = model.spk_inp_proj_w->ne[1];
- const int64_t d_head = hparams.n_head > 0 ? hparams.n_embd / hparams.n_head : 0;
- const int64_t d_head_spk = hparams.spk_n_head > 0 ? hparams.spk_n_embd / hparams.spk_n_head : 0;
- const int64_t n_spk = model.spk_head_fc2_w->ne[1];
- // the graph splits each fused QKV into three views of the layer width
- bool qkv_ok = true;
- for (const auto & layer : model.layers) {
- qkv_ok = qkv_ok && layer.qkv_w && layer.qkv_w->ne[0] == hparams.n_embd && layer.qkv_w->ne[1] == 3 * hparams.n_embd;
- }
- for (const auto & layer : model.spk_layers) {
- qkv_ok = qkv_ok && layer.qkv_w->ne[0] == hparams.spk_n_embd && layer.qkv_w->ne[1] == 3 * hparams.spk_n_embd;
- }
- if (!qkv_ok || model.inp_proj_w->ne[1] != hparams.n_embd || d_head <= 0 || d_head * hparams.n_head != hparams.n_embd ||
- (hparams.n_embd_head > 0 && hparams.n_embd_head != d_head) ||
- hparams.audio_rope_dims <= 0 || hparams.audio_rope_dims % 2 != 0 || hparams.audio_rope_dims > d_head ||
- d_head_spk <= 0 || d_head_spk % 2 != 0 || d_head_spk * hparams.spk_n_head != hparams.spk_n_embd ||
- (int64_t) model.spk_sil_emb.size() != hparams.spk_n_embd || n_spk <= 0 ||
- model.fusion_spk_kernel->ne[0] != n_spk || model.fusion_spk_kernel->ne[1] != hparams.n_embd ||
- hparams.mel_filters.size() != (size_t) hparams.n_mel_bins * (hparams.audio_n_fft / 2 + 1) ||
- hparams.window.size() != (size_t) hparams.audio_window_len) {
- throw std::runtime_error(string_format("%s: hparams do not match the tensor shapes\n", __func__));
- }
-
- // projection to the text model
- model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight"));
- model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias"));
+ // proj.0 -> mm.0 (conv1), proj.2 -> mm.2 (conv2), mlp -> mm.model.fc (linear)
+ model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
+ model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
+ model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
+ model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
+ model.mm_model_proj = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
+ model.mm_model_proj_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias"));
+ model.mm_pre_norm_w = get_tensor(string_format(TN_MM_PRE_NORM, "weight"));
+ model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
+ model.mm_img_begin = get_tensor(TN_MM_IMG_BEGIN);
+ model.mm_img_end = get_tensor(TN_MM_IMG_END);
+ model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
+ model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false);
} break;
- case PROJECTOR_TYPE_GRANITE_SPEECH:
+ case PROJECTOR_TYPE_JANUS_PRO:
{
- model.inp_proj_w = get_tensor(string_format(TN_INP_PROJ, "weight"));
- model.inp_proj_b = get_tensor(string_format(TN_INP_PROJ, "bias"));
- model.ctc_out_w = get_tensor(string_format(TN_CTC_OUT, "weight"));
- model.ctc_out_b = get_tensor(string_format(TN_CTC_OUT, "bias"));
- model.ctc_out_mid_w = get_tensor(string_format(TN_CTC_OUT_MID, "weight"));
- model.ctc_out_mid_b = get_tensor(string_format(TN_CTC_OUT_MID, "bias"));
-
- // per-layer tensors not loaded by the generic loop above
- for (int il = 0; il < hparams.n_layer; ++il) {
- auto & layer = model.layers[il];
-
- layer.attn_rel_pos_emb = get_tensor(string_format(TN_ATTN_REL_POS_EMB, prefix, il));
-
- layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight"));
- layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, il, "bias"));
-
- layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight"));
- layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias"));
- layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight"));
- layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias"));
- layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight"));
- layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"));
-
- layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"));
- layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"));
- layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight"));
- layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias"));
- layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight"));
- layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight"));
- layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias"));
- layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight"));
- layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"));
- }
-
- model.qf_proj_blocks.resize(1);
- auto & qf = model.qf_proj_blocks[0];
- qf.qf_proj_query = get_tensor(string_format(TN_QF_PROJ_QUERY, prefix));
- qf.qf_proj_norm_w = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "weight"));
- qf.qf_proj_norm_b = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "bias"));
- qf.qf_proj_linear_w = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "weight"));
- qf.qf_proj_linear_b = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "bias"));
-
- const int n_proj_layers = 2;
- qf.qf_proj_layers.resize(n_proj_layers);
- for (int il = 0; il < n_proj_layers; ++il) {
- auto & pl = qf.qf_proj_layers[il];
-
- pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "weight"));
- pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "bias"));
- pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, il, "weight"));
- pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, il, "bias"));
- pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, il, "weight"));
- pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, il, "bias"));
- pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, il, "weight"));
- pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, il, "bias"));
- pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, il, "weight"));
- pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, il, "bias"));
-
- pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, il, "weight"));
- pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, il, "bias"));
- pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, il, "weight"));
- pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, il, "bias"));
- pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, il, "weight"));
- pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, il, "bias"));
- pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, il, "weight"));
- pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, il, "bias"));
- pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, il, "weight"));
- pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, il, "bias"));
-
- pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, prefix, il, "weight"));
- pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, prefix, il, "bias"));
- pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, il, "weight"));
- pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, il, "bias"));
- pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "weight"));
- pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "bias"));
- }
+ model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
+ model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
+ model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
+ model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
} break;
- case PROJECTOR_TYPE_GRANITE4_VISION:
+ case PROJECTOR_TYPE_PHI4:
{
- // image_newline lives at the top-level.
- model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
-
- // Load separate layerwise and spatial projector tensors
- const auto projector_count = hparams.feature_layers.size();
- model.qf_proj_blocks.resize(projector_count);
- for (size_t bid = 0; bid < projector_count; ++bid) {
- auto & b = model.qf_proj_blocks[bid];
-
- // non-layerwise tensors
- b.qf_proj_img_pos = get_tensor(string_format(TN_MULTI_PROJ_IMG_POS, bid));
- b.qf_proj_query = get_tensor(string_format(TN_MULTI_PROJ_QUERY, prefix, bid));
- b.qf_proj_linear_w = get_tensor(string_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "weight"));
- b.qf_proj_linear_b = get_tensor(string_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "bias"));
- b.qf_proj_norm_w = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "weight"));
- b.qf_proj_norm_b = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "bias"));
- b.qf_proj_post_norm_w = get_tensor(string_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "weight"));
- b.qf_proj_post_norm_b = get_tensor(string_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "bias"));
+ model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
+ model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
+ model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
+ model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
+ } break;
+ case PROJECTOR_TYPE_DEEPSEEKOCR:
+ case PROJECTOR_TYPE_DEEPSEEKOCR2:
+ {
+ model.pos_embed = get_tensor(string_format(TN_SAM_POS_EMBD, "weight"));
+ model.patch_embed_proj_w = get_tensor(string_format(TN_SAM_PATCH_EMBD, "weight"));
+ model.patch_embed_proj_b = get_tensor(string_format(TN_SAM_PATCH_EMBD, "bias"));
+ model.n_sam_layers = hparams.sam_n_layer;
+ model.sam_layers.resize(model.n_sam_layers);
+ for (int il = 0; il < model.n_sam_layers; ++il) {
+ auto & layer = model.sam_layers[il];
+ layer.qkv_w = get_tensor(string_format(TN_SAM_ATTN_QKV, il, "weight"));
+ layer.qkv_b = get_tensor(string_format(TN_SAM_ATTN_QKV, il, "bias"));
+ layer.o_w = get_tensor(string_format(TN_SAM_ATTN_OUT, il, "weight"));
+ layer.o_b = get_tensor(string_format(TN_SAM_ATTN_OUT, il, "bias"));
+ layer.ln_1_w = get_tensor(string_format(TN_SAM_PRE_NORM, il, "weight"));
+ layer.ln_1_b = get_tensor(string_format(TN_SAM_PRE_NORM, il, "bias"));
+ layer.ln_2_w = get_tensor(string_format(TN_SAM_POST_NORM, il, "weight"));
+ layer.ln_2_b = get_tensor(string_format(TN_SAM_POST_NORM, il, "bias"));
+ layer.rel_pos_h = get_tensor(string_format(TN_SAM_ATTN_POS_H, il, "weight"));
+ layer.rel_pos_w = get_tensor(string_format(TN_SAM_ATTN_POS_W, il, "weight"));
+ layer.ff_up_w = get_tensor(string_format(TN_SAM_FFN_UP, il, "weight"));
+ layer.ff_up_b = get_tensor(string_format(TN_SAM_FFN_UP, il, "bias"));
+ layer.ff_down_w = get_tensor(string_format(TN_SAM_FFN_DOWN, il, "weight"));
+ layer.ff_down_b = get_tensor(string_format(TN_SAM_FFN_DOWN, il, "bias"));
+ }
+ model.neck_0_w = get_tensor(string_format(TN_SAM_NECK, 0, "weight"));
+ model.neck_1_b = get_tensor(string_format(TN_SAM_NECK, 1, "bias"));
+ model.neck_1_w = get_tensor(string_format(TN_SAM_NECK, 1, "weight"));
+ model.neck_2_w = get_tensor(string_format(TN_SAM_NECK, 2, "weight"));
+ model.neck_3_b = get_tensor(string_format(TN_SAM_NECK, 3, "bias"));
+ model.neck_3_w = get_tensor(string_format(TN_SAM_NECK, 3, "weight"));
+ model.net_2 = get_tensor(string_format(TN_SAM_NET, 2, "weight"));
+ model.net_3 = get_tensor(string_format(TN_SAM_NET, 3, "weight"));
+ model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false);
+ model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR);
+ model.mm_fc_w = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
+ model.mm_fc_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias"));
+ model.resample_query_768 = get_tensor(string_format(TN_RESMPL_QUERY, 768, "weight"), false);
+ model.resample_query_1024 = get_tensor(string_format(TN_RESMPL_QUERY, 1024, "weight"), false);
+ } break;
+ case PROJECTOR_TYPE_GEMMA4A:
+ {
+ for (int i = 0; i < 2; i++) {
+ model.sscp_conv_w[i] = get_tensor(string_format(TN_A_CONV1D, i, "weight"));
+ model.sscp_conv_b[i] = get_tensor(string_format(TN_A_CONV1D, i, "bias"), false);
+ model.sscp_norm_w[i] = get_tensor(string_format(TN_A_CONV1D_NORM, i, "weight"), false);
+ }
+ model.sscp_inp_proj_w = get_tensor(string_format(TN_A_INP_PROJ, "weight"));
+ model.sscp_inp_proj_b = get_tensor(string_format(TN_A_INP_PROJ, "bias"), false);
+ model.audio_out_proj_w = get_tensor(string_format(TN_A_OUT_PROJ, "weight"), false);
+ model.audio_out_proj_b = get_tensor(string_format(TN_A_OUT_PROJ, "bias"), false);
+ // audio multimodal embedder (mm.a.* namespace, not mm.*)
+ model.mm_soft_emb_norm_w = get_tensor(string_format(TN_A_MM_SOFT_EMB_N, "weight"), false);
+ model.mm_input_proj_w = get_tensor(string_format(TN_A_MM_INP_PROJ, "weight"), false);
- // laywerwise tensors
- // NOTE: If any model uses multi-layer qformers, this will need to change
- b.qf_proj_layers.resize(1);
- auto & pl = b.qf_proj_layers[0];
+ // Per-layer tensors NOT loaded by the generic loop above
+ for (int il = 0; il < hparams.n_layer; ++il) {
+ auto & layer = model.layers[il];
- pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, bid, "weight"));
- pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, bid, "bias"));
- pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, bid, "weight"));
- pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, bid, "bias"));
- pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, bid, "weight"));
- pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, bid, "bias"));
- pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, bid, "weight"));
- pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, bid, "bias"));
- pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, bid, "weight"));
- pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, bid, "bias"));
+ // Gemma4 audio conformer-specific tensors
+ layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight"));
+ layer.attn_pre_norm_w = get_tensor(string_format(TN_A_ATTN_PRE_NORM, prefix, il, "weight"), false);
+ layer.per_dim_scale_w = get_tensor(string_format(TN_A_PER_DIM_SCALE, prefix, il, "weight"), false);
+ layer.per_dim_k_scale_w = get_tensor(string_format(TN_A_PER_DIM_K_SCALE, prefix, il, "weight"), false);
+ layer.attn_k_rel_w = get_tensor(string_format(TN_A_ATTN_K_REL, prefix, il, "weight"), false);
- pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "weight"));
- pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "bias"));
- pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, bid, "weight"));
- pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, bid, "bias"));
- pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, bid, "weight"));
- pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, bid, "bias"));
- pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, bid, "weight"));
- pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, bid, "bias"));
- pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, bid, "weight"));
- pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, bid, "bias"));
+ // Convolution module
+ // Note: conv_norm / norm_conv are swapped in GGUF due to
+ // upstream tensor_mapping.py, so we load them in reverse order
+ layer.norm_conv_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight"), false);
+ layer.norm_conv_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias"), false);
+ layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight"));
+ layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias"), false);
+ layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight"));
+ layer.conv_dw_b = get_tensor(string_format(TN_CONV_DW, prefix, il, "bias"), false);
+ layer.conv_norm_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"), false);
+ layer.conv_norm_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"), false);
+ layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight"));
+ layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"), false);
- pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, prefix, bid, "weight"));
- pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, prefix, bid, "bias"));
- pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, bid, "weight"));
- pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, bid, "bias"));
- pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, prefix, bid, "weight"));
- pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, prefix, bid, "bias"));
+ // FFN2 (second half-step)
+ layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight"));
+ layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight"));
+ layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias"), false);
+ layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight"));
+ layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"), false);
+ layer.ff_post_norm_1_w = get_tensor(string_format(TN_A_FFN_POST_NORM_1, prefix, il, "weight"), false);
}
+ // Load clamp info for ClippableLinear AFTER all tensors are loaded
+ for (auto * tensor : tensors_to_load) {
+ std::string name = tensor->name;
+ if (string_ends_with(name, ".weight")) {
+ std::string name_inp_max = name;
+ std::string name_inp_min = name;
+ std::string name_out_max = name;
+ std::string name_out_min = name;
+ string_replace_all(name_inp_max, ".weight", ".input_max");
+ string_replace_all(name_inp_min, ".weight", ".input_min");
+ string_replace_all(name_out_max, ".weight", ".output_max");
+ string_replace_all(name_out_min, ".weight", ".output_min");
+ model.clamp_info_map[name] = {
+ get_scalar(name_inp_max, FLT_MAX),
+ get_scalar(name_inp_min, -FLT_MAX),
+ get_scalar(name_out_max, FLT_MAX),
+ get_scalar(name_out_min, -FLT_MAX)
+ };
+ }
+ }
} break;
- case PROJECTOR_TYPE_SENSENOVA_U1:
+ case PROJECTOR_TYPE_GEMMA4UA:
{
- model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
- model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
+ model.mm_input_proj_w = get_tensor(string_format(TN_A_MM_INP_PROJ, "weight"));
} break;
- default:
- GGML_ASSERT(false && "unknown projector type");
- }
-
- // load data
- {
- std::vector<uint8_t> read_buf;
-
- // start loading event
- if (progress_callback){
- progress_callback(0.0, progress_callback_user_data);
- }
-
- // compute total tensor data size for progress reporting
- size_t total_data_size = 0;
- for (auto & t : tensors_to_load) {
- total_data_size += ggml_nbytes(t);
- }
-
- // alloc memory and offload data
- ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(ctx_clip.backend);
- ctx_clip.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_clip.ctx_data.get(), buft));
- ggml_backend_buffer_set_usage(ctx_clip.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
- // read the weight from file
- if (!ctx_clip.no_alloc) {
- size_t data_loaded = 0;
- for (auto & t : tensors_to_load) {
- ggml_tensor * cur = ggml_get_tensor(ctx_clip.ctx_data.get(), t->name);
- GGML_ASSERT(cur && "tensor not found in ctx_data");
- auto it_off = tensor_offset.find(t->name);
- GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor");
- const size_t offset = it_off->second;
- fin.seekg(offset, std::ios::beg);
- if (!fin) {
- throw std::runtime_error(string_format("%s: failed to seek for tensor %s\n", __func__, t->name));
- }
- size_t num_bytes = ggml_nbytes(cur);
- if (ggml_backend_buft_is_host(buft)) {
- // for the CPU and Metal backend, we can read directly into the tensor
- fin.read(reinterpret_cast<char *>(cur->data), num_bytes);
- } else {
- // read into a temporary buffer first, then copy to device memory
- read_buf.resize(num_bytes);
- fin.read(reinterpret_cast<char *>(read_buf.data()), num_bytes);
- ggml_backend_tensor_set(cur, read_buf.data(), 0, num_bytes);
- }
- data_loaded += num_bytes;
- if (progress_callback && total_data_size > 0) {
- const float progress = (float)data_loaded / (float)total_data_size;
- if (!progress_callback(progress, progress_callback_user_data)) {
- throw std::runtime_error(string_format("%s: model loading cancelled by progress_callback\n", __func__));
- }
+ case PROJECTOR_TYPE_LFM2A:
+ case PROJECTOR_TYPE_D1OMNI_A:
+ {
+ for (int i : {0, 2, 3, 5, 6}) {
+ model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight"));
+ model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias"));
}
- }
- LOG_DBG("%s: loaded %zu tensors from %s\n", __func__, tensors_to_load.size(), fname.c_str());
- } else {
- LOG_DBG("%s: no_alloc is set, skipping tensor data loading (%zu tensors)\n", __func__, tensors_to_load.size());
- }
- fin.close();
- }
-
- }
-
- struct support_info_op {
- ggml_tensor * op;
-
- // true if the op runs on the accelerated ctx_clip.backend
- bool is_accel = true;
- };
-
- struct support_info_graph {
- // whether the clip_ctx.backend supports flash attention
- bool fattn = true;
- ggml_tensor * fattn_op = nullptr; // for debugging
-
- std::vector<support_info_op> ops;
- };
-
- static clip_image_f32_batch get_dummy_batch(clip_ctx & ctx_clip) {
- // create a fake batch
- const auto & hparams = ctx_clip.model.hparams;
- clip_image_f32_batch batch;
- clip_image_f32 img;
- if (ctx_clip.model.modality == CLIP_MODALITY_VISION) {
- const int sz = hparams.warmup_image_size;
- img.set_size({sz, sz}, false, false);
- LOG_INF("%s: warmup with image size = %d x %d\n", __func__, sz, sz);
- } else {
- // GEMMA4UA uses n_mel_bins as a raw-waveform frame size (640), not a mel-bin count,
- // so the [1, 256] bound only applies to FFT-based models.
- const bool fft_based = ctx_clip.model.proj_type != PROJECTOR_TYPE_GEMMA4UA;
- if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) {
- throw std::runtime_error(string_format("%s: invalid n_mel_bins (%d), must be in [1, 256]\n", __func__, hparams.n_mel_bins));
- }
- img.set_size({hparams.warmup_audio_size, hparams.n_mel_bins}, false, false);
- LOG_INF("%s: warmup with audio size = %d\n", __func__, hparams.warmup_audio_size);
- }
- batch.entries.push_back(img);
- return batch;
- }
-
- static void init_ctx(clip_ctx & ctx_clip) {
- ctx_clip.buf_compute_meta.resize(ctx_clip.max_nodes * ggml_tensor_overhead() + ggml_graph_overhead());
-
- // check batching support
- auto batch = get_dummy_batch(ctx_clip);
- auto builder = clip_get_graph_builder(&ctx_clip, batch);
- ctx_clip.support_batch = builder->support_batch();
- }
-
- static void warmup(clip_ctx & ctx_clip) {
- auto batch = get_dummy_batch(ctx_clip);
- warmup(ctx_clip, batch);
- }
-
- static void warmup(clip_ctx & ctx_clip, const clip_image_f32_batch & batch) {
- support_info_graph info;
-
- if (ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_AUTO) {
- // try to enable flash attention to see if it's supported
- ctx_clip.flash_attn_type = CLIP_FLASH_ATTN_TYPE_ENABLED;
- info = reserve_compute_meta(ctx_clip, batch);
- if (!info.fattn && info.fattn_op) {
- auto op = info.fattn_op;
- LOG_WRN("%s: *****************************************************************\n", __func__);
- LOG_WRN("%s: WARNING: flash attention not supported by %s, memory usage will increase\n", __func__, ggml_backend_name(ctx_clip.backend));
- LOG_WRN("%s: op params: \n", __func__);
- static auto print_shape = [](const char * fn, const char * name, ggml_tensor * t) {
- LOG_WRN("%s: %s: type = %s, ne = [%d %d %d %d], nb = [%d %d %d %d]\n", fn,
- name, ggml_type_name(t->type),
- t->ne[0], t->ne[1], t->ne[2], t->ne[3],
- t->nb[0], t->nb[1], t->nb[2], t->nb[3]);
- };
- print_shape(__func__, " dst", op);
- print_shape(__func__, "src0", op->src[0]);
- print_shape(__func__, "src1", op->src[1]);
- print_shape(__func__, "src2", op->src[2]);
- LOG_WRN("%s: please report this on github as an issue\n", __func__);
- LOG_WRN("%s: *****************************************************************\n", __func__);
- ctx_clip.flash_attn_type = CLIP_FLASH_ATTN_TYPE_DISABLED;
- reserve_compute_meta(ctx_clip, batch);
- }
- } else {
- info = reserve_compute_meta(ctx_clip, batch);
- if (!info.fattn && ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
- LOG_WRN("%s: flash attention is not supported by the current backend; falling back to CPU (performance will be degraded)\n", __func__);
- }
- }
-
- ctx_clip.is_allocated = true; // mark buffers as allocated
-
- LOG_INF("%s: flash attention is %s\n", __func__,
- (ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled");
-
- // print ops that are not supported by the GPU backend (if there is one)
- if (ctx_clip.backend && ctx_clip.backend != ctx_clip.backend_cpu) {
- std::vector<support_info_op> unsupported_ops;
- for (const auto & op : info.ops) {
- if (!op.is_accel) {
- unsupported_ops.push_back(op);
- }
- }
- if (!unsupported_ops.empty()) {
- LOG_WRN("%s: *****************************************************************\n", __func__);
- LOG_WRN("%s: WARNING: the CLIP graph uses unsupported operators by the backend\n", __func__);
- LOG_WRN("%s: the performance will be suboptimal \n", __func__);
- LOG_WRN("%s: list of unsupported ops (backend=%s):\n", __func__, ggml_backend_name(ctx_clip.backend));
- for (const auto & op : unsupported_ops) {
- LOG_WRN("%s: %16s: type = %s, ne = [%d %d %d %d]\n", __func__,
- ggml_op_name(op.op->op),
- ggml_type_name(op.op->type),
- op.op->ne[0], op.op->ne[1], op.op->ne[2], op.op->ne[3]);
- }
- LOG_WRN("%s: flash attention is %s\n", __func__,
- (ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled");
- LOG_WRN("%s: please report this on github as an issue\n", __func__);
- LOG_WRN("%s: ref: https://github.com/ggml-org/llama.cpp/pull/16837#issuecomment-3461676118\n", __func__);
- LOG_WRN("%s: *****************************************************************\n", __func__);
- }
- }
- }
-
- // only initialize backend buffers, but do not allocate them yet
- static support_info_graph reserve_compute_meta(clip_ctx & ctx_clip, const clip_image_f32_batch & batch) {
- ggml_cgraph * gf = clip_get_graph_builder(&ctx_clip, batch)->build();
- ggml_backend_sched_reserve(ctx_clip.sched.get(), gf);
-
- ctx_clip.mem_compute.clear();
- for (size_t i = 0; i < ctx_clip.backend_ptrs.size(); ++i) {
- ggml_backend_t backend = ctx_clip.backend_ptrs[i];
- ggml_backend_buffer_type_t buft = ctx_clip.backend_buft[i];
- size_t size = ggml_backend_sched_get_buffer_size(ctx_clip.sched.get(), backend);
- if (size > 1) {
- LOG_INF("%s: %10s compute buffer size = %8.2f MiB\n", __func__,
- ggml_backend_buft_name(buft),
- size / 1024.0 / 1024.0);
- }
- ctx_clip.mem_compute[ggml_backend_get_device(backend)] += size;
- }
-
- const int n_splits = ggml_backend_sched_get_n_splits(ctx_clip.sched.get());
- const int n_nodes = ggml_graph_n_nodes(gf);
-
- LOG_INF("%s: graph splits = %d, nodes = %d\n", __func__, n_splits, n_nodes);
-
- support_info_graph res {
- /*.fattn = */ true,
- /*.fattn_op = */ nullptr,
- /*.ops = */ {},
- };
-
- // check op support
- for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
- ggml_tensor * node = ggml_graph_node(gf, i);
- res.ops.push_back({node, true});
- if (!ggml_backend_supports_op(ctx_clip.backend, node)) {
- res.ops.back().is_accel = false;
- if (node->op == GGML_OP_FLASH_ATTN_EXT) {
- res.fattn = false;
- res.fattn_op = node;
- }
- }
- }
-
- return res;
- }
-
- void get_bool(const std::string & key, bool & output, bool required = true) const {
- const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
- if (i < 0) {
- if (required) {
- throw std::runtime_error("Key not found: " + key);
- }
- return;
- }
- output = gguf_get_val_bool(ctx_gguf.get(), i);
- }
-
- void get_i32(const std::string & key, int & output, bool required = true) const {
- const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
- if (i < 0) {
- if (required) {
- throw std::runtime_error("Key not found: " + key);
- }
- return;
- }
- output = gguf_get_val_i32(ctx_gguf.get(), i);
- }
+ model.pre_encode_out_w = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight"));
+ model.pre_encode_out_b = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias"));
- void get_u32(const std::string & key, int & output, bool required = true) const {
- const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
- if (i < 0) {
- if (required) {
- throw std::runtime_error("Key not found: " + key);
- }
- return;
- }
- const uint32_t val = gguf_get_val_u32(ctx_gguf.get(), i);
- // sanity check
- if (val > (uint32_t) INT32_MAX) {
- throw std::runtime_error(string_format("%s: value %u for key '%s' exceeds INT32_MAX\n",
- __func__, val, key.c_str()));
- }
- output = (int) val;
- }
+ model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "weight"));
+ model.mm_0_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "bias"));
+ model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
+ model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
+ model.mm_3_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight"));
+ model.mm_3_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias"));
- void get_f32(const std::string & key, float & output, bool required = true) const {
- const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
- if (i < 0) {
- if (required) {
- throw std::runtime_error("Key not found: " + key);
- }
- return;
- }
- output = gguf_get_val_f32(ctx_gguf.get(), i);
- }
+ // residual block after the projector: norm, down, up
+ if (model.proj_type == PROJECTOR_TYPE_D1OMNI_A) {
+ model.mm_4_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 4, "weight"));
+ model.mm_4_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 4, "bias"));
+ model.mm_5_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 5, "weight"));
+ model.mm_5_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 5, "bias"));
+ model.mm_6_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 6, "weight"));
+ model.mm_6_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 6, "bias"));
+ }
- void get_arr_f32(const std::string & key, std::vector<float> & output, bool required = true) const {
- const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
- if (i < 0) {
- if (required) {
- throw std::runtime_error("Key not found: " + key);
- }
- return;
- }
- if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) {
- throw std::runtime_error(string_format("%s: key '%s' is not an array\n", __func__, key.c_str()));
- }
- const auto type = gguf_get_arr_type(ctx_gguf.get(), i);
- if (type != GGUF_TYPE_FLOAT32) {
- throw std::runtime_error(string_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_FLOAT32)\n", __func__, key.c_str(), type, GGUF_TYPE_FLOAT32));
- }
- const size_t n = gguf_get_arr_n(ctx_gguf.get(), i);
- if (n > (size_t) std::numeric_limits<int>::max()) {
- throw std::runtime_error(string_format("%s: array '%s' is too large (%zu elements)\n", __func__, key.c_str(), n));
- }
- output.resize(n);
- const float * values = (const float *)gguf_get_arr_data(ctx_gguf.get(), i);
- for (size_t j = 0; j < n; ++j) {
- output[j] = values[j];
- }
- }
+ for (int il = 0; il < hparams.n_layer; ++il) {
+ auto & layer = model.layers[il];
- void get_string(const std::string & key, std::string & output, bool required = true) const {
- const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
- if (i < 0) {
- if (required) {
- throw std::runtime_error("Key not found: " + key);
- }
- return;
- }
- output = std::string(gguf_get_val_str(ctx_gguf.get(), i));
- }
+ layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight"));
+ layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, il, "bias"));
+ layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight"));
+ layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias"));
+ layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight"));
+ layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias"));
+ layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight"));
+ layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"));
- void get_arr_int(const std::string & key, std::vector<int> & output, bool required = true) const {
- const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
- if (i < 0) {
- if (required) {
- throw std::runtime_error("Key not found: " + key);
- }
- return;
- }
- if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) {
- throw std::runtime_error(string_format("%s: key '%s' is not an array\n", __func__, key.c_str()));
- }
- const auto type = gguf_get_arr_type(ctx_gguf.get(), i);
- if (type != GGUF_TYPE_INT32) {
- throw std::runtime_error(string_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_INT32)\n", __func__, key.c_str(), type, GGUF_TYPE_INT32));
- }
- const size_t n = gguf_get_arr_n(ctx_gguf.get(), i);
- if (n > (size_t) std::numeric_limits<int>::max()) {
- throw std::runtime_error(string_format("%s: array '%s' is too large (%zu elements)\n", __func__, key.c_str(), n));
- }
- output.resize(n);
- const int32_t * values = (const int32_t *)gguf_get_arr_data(ctx_gguf.get(), i);
- for (size_t j = 0; j < n; ++j) {
- output[j] = values[j];
- }
- }
+ layer.pos_bias_u = get_tensor(string_format(TN_POS_BIAS_U, prefix, il));
+ layer.pos_bias_v = get_tensor(string_format(TN_POS_BIAS_V, prefix, il));
- static void set_llava_uhd_res_candidates(clip_model & model, const int max_patches_per_side) {
- auto & hparams = model.hparams;
- for (int x = 1; x <= max_patches_per_side; x++) {
- for (int y = 1; y <= max_patches_per_side; y++) {
- if (x == 1 && y == 1) {
- continue; // skip the first point
- }
- hparams.image_res_candidates.push_back(clip_image_size{
- x*hparams.image_size,
- y*hparams.image_size,
- });
- }
- }
- }
+ layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"));
+ layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"));
- static void set_internvl_dhr_res_candidates(clip_model & model) {
- auto & hparams = model.hparams;
- int min_num = hparams.preproc_min_tiles;
- int max_num = hparams.preproc_max_tiles;
- if (min_num < 1) {
- return; // avoid divide by 0
- }
- for (int a = min_num; a <= max_num; ++a) {
- int b_lo = (min_num + a - 1) / a;
- int b_hi = max_num / a;
- b_lo = std::max(b_lo, min_num);
- b_hi = std::min(b_hi, max_num);
- for (int b = b_lo; b <= b_hi; ++b) {
- hparams.image_res_candidates.push_back(clip_image_size {
- a*hparams.image_size,
- b*hparams.image_size,
- });
- }
- }
- }
-};
+ layer.linear_pos_w = get_tensor(string_format(TN_LINEAR_POS, prefix, il, "weight"));
-struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params) {
- clip_ctx * ctx_vision = nullptr;
- clip_ctx * ctx_audio = nullptr;
- clip_ctx * ctx_gen_audio = nullptr;
-
- std::shared_ptr<ggml_threadpool> threadpool;
-
- try {
- // no_alloc only measures memory, nothing is computed
- threadpool = clip_threadpool_init(ctx_params.no_alloc ? 1 : ctx_params.n_threads);
-
- clip_model_loader loader(fname,
- /* skip_tensors */ false,
- ctx_params.progress_callback,
- ctx_params.progress_callback_user_data);
- bool skip_audio = false;
-
- if (loader.has_vision) {
- ctx_vision = new clip_ctx(ctx_params, threadpool);
- loader.load_hparams(ctx_vision->model, CLIP_MODALITY_VISION);
- loader.load_tensors(*ctx_vision);
- loader.init_ctx(*ctx_vision);
- if (ctx_params.warmup) {
- loader.warmup(*ctx_vision);
- }
+ layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight"));
+ layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias"));
+ layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight"));
+ layer.conv_dw_b = get_tensor(string_format(TN_CONV_DW, prefix, il, "bias"));
+ layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight"));
+ layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias"));
+ layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight"));
+ layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"));
+ }
+ } break;
+ case PROJECTOR_TYPE_LFM3A:
+ {
+ for (int i : {0, 2, 3, 5, 6}) {
+ model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight"));
+ model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias"));
+ }
+ model.pre_encode_out_w = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight"));
+ model.pre_encode_out_b = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias"));
- // TODO: we don't support audio for Gemma 3N, but GGUF contains audio tensors
- // we can remove this check when we implement audio support for Gemma 3N
- skip_audio = ctx_vision->model.proj_type == PROJECTOR_TYPE_GEMMA3NV;
- }
+ model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "weight"));
+ model.mm_0_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "bias"));
+ model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
+ model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
+ model.mm_3_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight"));
+ model.mm_3_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias"));
- if (loader.has_audio && !skip_audio) {
- ctx_audio = new clip_ctx(ctx_params, threadpool);
- loader.load_hparams(ctx_audio->model, CLIP_MODALITY_AUDIO);
- loader.load_tensors(*ctx_audio);
- loader.init_ctx(*ctx_audio);
- if (ctx_params.warmup) {
- loader.warmup(*ctx_audio);
- }
- }
+ if (!model.post_ln_w) {
+ throw std::runtime_error(string_format("%s: missing tensor %s\n", __func__, string_format(TN_LN_POST, prefix, "weight").c_str()));
+ }
- if (loader.has_gen_audio) {
- ctx_gen_audio = new clip_ctx(ctx_params, threadpool);
- loader.load_hparams(ctx_gen_audio->model, CLIP_MODALITY_GEN_AUDIO);
- // TODO: CUDA has no flash attention kernel for the depthformer head dim 32, enable it once it does
- if (ctx_gen_audio->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN && ctx_gen_audio->flash_attn_type == CLIP_FLASH_ATTN_TYPE_AUTO) {
- ctx_gen_audio->flash_attn_type = CLIP_FLASH_ATTN_TYPE_DISABLED;
- }
- loader.load_tensors(*ctx_gen_audio);
- if (ctx_gen_audio->model.proj_type == PROJECTOR_TYPE_CHATTERBOX) {
- // the classic cfm solver unrolls up to 10 steps x 2 cfg estimator evaluations in one graph
- ctx_gen_audio->max_nodes = 65536;
- ctx_gen_audio->sched.reset(
- ggml_backend_sched_new(ctx_gen_audio->backend_ptrs.data(), ctx_gen_audio->backend_buft.data(),
- ctx_gen_audio->backend_ptrs.size(), ctx_gen_audio->max_nodes, false, true));
- if (ctx_params.cb_eval != nullptr) {
- ggml_backend_sched_set_eval_callback(ctx_gen_audio->sched.get(), ctx_params.cb_eval, ctx_params.cb_eval_user_data);
- }
- }
- // TODO: fix warmup
- ctx_gen_audio->buf_compute_meta.resize(ctx_gen_audio->max_nodes * ggml_tensor_overhead() +
- ggml_graph_overhead_custom(ctx_gen_audio->max_nodes, false));
- }
+ // each layer is either a short conv block or a complete attention block
+ for (int il = 0; il < hparams.n_layer; ++il) {
+ auto & layer = model.layers[il];
+ layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight"), false);
+ layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight"), false);
+ layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight"), false);
+ const bool is_conv = layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w;
+ const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w && layer.q_norm && layer.k_norm;
+ if (is_conv == is_attn) {
+ throw std::runtime_error(string_format("%s: lfm3a layer %d is neither a complete conv nor a complete attention block\n", __func__, il));
+ }
+ }
+ } break;
+ case PROJECTOR_TYPE_PARAKEET:
+ {
- } catch (const std::exception & e) {
- LOG_ERR("%s: failed to load model '%s': %s\n", __func__, fname, e.what());
+ hparams.mel_filters = get_vector(TN_MEL_FILTERS);
+ hparams.window = get_vector(TN_WINDOW);
- delete ctx_vision;
- delete ctx_audio;
- delete ctx_gen_audio;
+ // Subsampling layers (conv1d)
+ for (int i : {0, 2, 3, 5, 6}) {
+ model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight"));
+ model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias"));
+ }
+ model.pre_encode_out_w = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight"));
+ model.pre_encode_out_b = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias"));
- return {nullptr, nullptr, nullptr};
- }
+ // Projection layers
+ model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"), false);
+ model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"), false);
+ model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"), false);
- return {ctx_vision, ctx_audio, ctx_gen_audio};
-}
+ // Encoder layers
+ for (int il = 0; il < hparams.n_layer; ++il) {
+ auto & layer = model.layers[il];
-struct clip_cap clip_get_cap(const char * fname) {
- clip_cap res;
- clip_model_loader loader(fname, /* skip_tensors= */ true);
- res.has_vision = loader.has_vision;
- res.has_audio = loader.has_audio;
- res.has_gen_audio = loader.has_gen_audio;
- return res;
-}
+ // Attention (from shared above)
-int clip_get_image_max_tokens(const clip_ctx * ctx) {
- const auto & hparams = ctx->model.hparams;
- if (ctx->proj_type() == PROJECTOR_TYPE_DEEPSEEK4V || ctx->proj_type() == PROJECTOR_TYPE_DEEPSEEK41V) {
- return hparams.dsv4_max_n_token;
- }
- if (hparams.image_max_pixels <= 0) {
- return -1;
- }
- const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
- return hparams.image_max_pixels / patch_area;
-}
+ // Relative position encoding
+ layer.linear_pos_w = get_tensor(string_format(TN_LINEAR_POS, prefix, il, "weight"));
+ layer.pos_bias_u = get_tensor(string_format(TN_POS_BIAS_U, prefix, il));
+ layer.pos_bias_v = get_tensor(string_format(TN_POS_BIAS_V, prefix, il));
-void clip_free(clip_ctx * ctx) {
- if (ctx == nullptr) {
- return;
- }
- delete ctx;
-}
+ // Convolution module
+ layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight"));
+ layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias"), false);
+ layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight"));
+ layer.conv_dw_b = get_tensor(string_format(TN_CONV_DW, prefix, il, "bias"), false);
+ layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight"));
+ layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias"));
+ layer.conv_norm_mean = get_tensor(string_format(TN_CONV_NORM_MEAN, prefix, il));
+ layer.conv_norm_var = get_tensor(string_format(TN_CONV_NORM_VAR, prefix, il));
+ layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight"));
+ layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"), false);
-const char * clip_patch_merge_type(const struct clip_ctx * ctx) {
- return ctx->model.hparams.mm_patch_merge_type == PATCH_MERGE_SPATIAL_UNPAD ? "spatial_unpad" : "flat";
-}
+ // Feed-forward networks
+ layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight"));
+ layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, il, "bias"));
-int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
- const auto & params = ctx->model.hparams;
- const int n_total = clip_n_output_tokens(ctx, img);
- const auto & proj = ctx->proj_type();
- switch (proj) {
- case PROJECTOR_TYPE_QWEN2VL:
- case PROJECTOR_TYPE_QWEN25VL:
- case PROJECTOR_TYPE_QWEN3VL:
- case PROJECTOR_TYPE_MAGEVL:
- case PROJECTOR_TYPE_SENSENOVA_U1:
- case PROJECTOR_TYPE_LING3VL:
- case PROJECTOR_TYPE_EXAONE4_5:
- case PROJECTOR_TYPE_MIMOVL:
- case PROJECTOR_TYPE_GLM4V:
- case PROJECTOR_TYPE_GLM5V:
- case PROJECTOR_TYPE_PADDLEOCR:
- case PROJECTOR_TYPE_HUNYUANVL:
- case PROJECTOR_TYPE_YOUTUVL:
- case PROJECTOR_TYPE_MUSE_GLIMMER:
- case PROJECTOR_TYPE_LFM3VL:
- return (img->nx() / params.patch_size) / 2;
- case PROJECTOR_TYPE_STEP3VL:
- case PROJECTOR_TYPE_MINICPMV4_6:
- case PROJECTOR_TYPE_MINICPMV4_7:
- case PROJECTOR_TYPE_MINICPM_OCR_1_0:
- return img->nx() / (params.patch_size * params.n_merge);
- case PROJECTOR_TYPE_DEEPSEEKOCR:
- case PROJECTOR_TYPE_DEEPSEEKOCR2:
- return (img->nx() / params.patch_size) / 4;
- default:
- break;
- }
- return n_total;
-}
+ layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight"));
+ layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias"));
+ layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight"));
+ layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias"), false);
+ layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight"));
+ layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"), false);
+
+ // Layer norms
+ layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"));
+ layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"));
+ }
+
+ model.mm_model_mlp_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
+ model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
+ model.mm_model_mlp_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
+ } break;
+ case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
+ {
+ hparams.mel_filters = get_vector(TN_MEL_FILTERS);
+ hparams.window = get_vector(TN_WINDOW);
-int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {
- const auto & params = ctx->model.hparams;
- const auto & proj = ctx->proj_type();
- switch (proj) {
- case PROJECTOR_TYPE_QWEN2VL:
- case PROJECTOR_TYPE_QWEN25VL:
- case PROJECTOR_TYPE_QWEN3VL:
- case PROJECTOR_TYPE_MAGEVL:
- case PROJECTOR_TYPE_SENSENOVA_U1:
- case PROJECTOR_TYPE_LING3VL:
- case PROJECTOR_TYPE_EXAONE4_5:
- case PROJECTOR_TYPE_MIMOVL:
- case PROJECTOR_TYPE_GLM4V:
- case PROJECTOR_TYPE_GLM5V:
- case PROJECTOR_TYPE_PADDLEOCR:
- case PROJECTOR_TYPE_HUNYUANVL:
- case PROJECTOR_TYPE_YOUTUVL:
- case PROJECTOR_TYPE_MUSE_GLIMMER:
- case PROJECTOR_TYPE_LFM3VL:
- return (img->ny() / params.patch_size) / 2;
- case PROJECTOR_TYPE_STEP3VL:
- case PROJECTOR_TYPE_MINICPMV4_6:
- case PROJECTOR_TYPE_MINICPMV4_7:
- case PROJECTOR_TYPE_MINICPM_OCR_1_0:
- return img->ny() / (params.patch_size * params.n_merge);
- default:
- break;
- }
- return 1;
-}
+ // ASR expert: blocks, pre_ln and post_ln come from the generic loader
+ model.inp_proj_w = get_tensor(string_format(TN_INP_PROJ, "weight"));
+ for (int il = 0; il < hparams.n_layer; ++il) {
+ auto & layer = model.layers[il];
+ layer.q_norm_b = get_tensor(string_format(TN_ATTN_Q_NORM, prefix, il, "bias"));
+ layer.k_norm_b = get_tensor(string_format(TN_ATTN_K_NORM, prefix, il, "bias"));
+ }
-int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
- const auto & params = ctx->model.hparams;
+ // speaker expert
+ const char * sp = "a.spk";
+ model.spk_inp_proj_w = get_tensor(string_format(TN_A_SPK_INP_PROJ, "weight"));
+ model.spk_pre_ln_w = get_tensor(string_format(TN_LN_PRE, sp, "weight"));
+ model.spk_pre_ln_b = get_tensor(string_format(TN_LN_PRE, sp, "bias"));
+ model.spk_post_ln_w = get_tensor(string_format(TN_LN_POST, sp, "weight"));
+ model.spk_post_ln_b = get_tensor(string_format(TN_LN_POST, sp, "bias"));
+ model.spk_layers.resize(hparams.spk_n_layer);
+ for (int il = 0; il < hparams.spk_n_layer; ++il) {
+ auto & layer = model.spk_layers[il];
+ layer.ln_1_w = get_tensor(string_format(TN_LN_1, sp, il, "weight"));
+ layer.ln_1_b = get_tensor(string_format(TN_LN_1, sp, il, "bias"));
+ layer.qkv_w = get_tensor(string_format(TN_ATTN_QKV, sp, il, "weight"));
+ layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, sp, il, "weight"));
+ layer.o_b = get_tensor(string_format(TN_ATTN_OUTPUT, sp, il, "bias"));
+ layer.ln_2_w = get_tensor(string_format(TN_LN_2, sp, il, "weight"));
+ layer.ln_2_b = get_tensor(string_format(TN_LN_2, sp, il, "bias"));
+ layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, sp, il, "weight"));
+ layer.ff_up_b = get_tensor(string_format(TN_FFN_UP, sp, il, "bias"));
+ layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, sp, il, "weight"));
+ layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, sp, il, "bias"));
+ }
- // for models with fixed size image, the input image is already pre-processed and resized to square
- int patch_size = params.patch_size;
- int n_patches = (img->nx() / patch_size) * (img->ny() / patch_size);
+ // Sortformer head, fusion
+ model.spk_head_proj_w = get_tensor(string_format(TN_A_SPK_HEAD, "proj", "weight"));
+ model.spk_head_proj_b = get_tensor(string_format(TN_A_SPK_HEAD, "proj", "bias"));
+ model.spk_head_up_w = get_tensor(string_format(TN_A_SPK_HEAD, "upsample", "weight"));
+ model.spk_head_up_b = get_tensor(string_format(TN_A_SPK_HEAD, "upsample", "bias"));
+ model.spk_head_fc1_w = get_tensor(string_format(TN_A_SPK_HEAD, "fc1", "weight"));
+ model.spk_head_fc1_b = get_tensor(string_format(TN_A_SPK_HEAD, "fc1", "bias"));
+ model.spk_head_fc2_w = get_tensor(string_format(TN_A_SPK_HEAD, "fc2", "weight"));
+ model.spk_head_fc2_b = get_tensor(string_format(TN_A_SPK_HEAD, "fc2", "bias"));
+ model.fusion_asr_norm_w = get_tensor(string_format(TN_A_FUSION_NORM, "asr_norm", "weight"));
+ model.fusion_asr_norm_b = get_tensor(string_format(TN_A_FUSION_NORM, "asr_norm", "bias"));
+ model.fusion_spk_norm_w = get_tensor(string_format(TN_A_FUSION_NORM, "spk_norm", "weight"));
+ model.fusion_spk_norm_b = get_tensor(string_format(TN_A_FUSION_NORM, "spk_norm", "bias"));
+ model.fusion_spk_kernel = get_tensor(TN_A_FUSION_KERNEL);
- projector_type proj = ctx->proj_type();
+ model.spk_sil_emb = get_vector(TN_A_SPK_SIL_EMB);
- switch (proj) {
- case PROJECTOR_TYPE_MLP:
- case PROJECTOR_TYPE_MLP_NORM:
- case PROJECTOR_TYPE_JANUS_PRO:
- case PROJECTOR_TYPE_PHI4:
- {
- // do nothing
- } break;
- case PROJECTOR_TYPE_YASA2:
- {
- n_patches = 64; // adaptive average pooling to 8x8 tokens
- } break;
- case PROJECTOR_TYPE_LDP:
- case PROJECTOR_TYPE_LDPV2:
- case PROJECTOR_TYPE_GLM_EDGE:
- {
- n_patches /= 4;
- if (ctx->model.mm_boi) {
- n_patches += 2; // for BOI and EOI token embeddings
- }
- } break;
- case PROJECTOR_TYPE_MINICPMV:
- {
- // Use actual config value if available, otherwise fall back to hardcoded values
- if (params.minicpmv_query_num > 0) {
- n_patches = params.minicpmv_query_num;
- } else {
- // Fallback to hardcoded values for legacy models
- if (params.minicpmv_version == 2) {
- n_patches = 96;
- } else if (params.minicpmv_version == 3) {
- n_patches = 64;
- } else if (params.minicpmv_version == 4) {
- n_patches = 64;
- } else if (params.minicpmv_version == 5) {
- // MiniCPM-V 4.0
- n_patches = 64;
- } else if (params.minicpmv_version == 6) {
- // MiniCPM-V 4.5
- n_patches = 64;
- } else if (params.minicpmv_version == 100045) {
- // MiniCPM-o 4.5
- n_patches = 64;
- } else {
- GGML_ABORT("Unknown minicpmv version");
+ // the head layouts and the widths the graph and the speaker state use must match the tensors
+ hparams.spk_n_embd = model.spk_inp_proj_w->ne[1];
+ const int64_t d_head = hparams.n_head > 0 ? hparams.n_embd / hparams.n_head : 0;
+ const int64_t d_head_spk = hparams.spk_n_head > 0 ? hparams.spk_n_embd / hparams.spk_n_head : 0;
+ const int64_t n_spk = model.spk_head_fc2_w->ne[1];
+ // the graph splits each fused QKV into three views of the layer width
+ bool qkv_ok = true;
+ for (const auto & layer : model.layers) {
+ qkv_ok = qkv_ok && layer.qkv_w && layer.qkv_w->ne[0] == hparams.n_embd && layer.qkv_w->ne[1] == 3 * hparams.n_embd;
}
- }
- } break;
- case PROJECTOR_TYPE_MINICPMV4_6:
- case PROJECTOR_TYPE_MINICPMV4_7:
- case PROJECTOR_TYPE_MINICPM_OCR_1_0:
- {
- n_patches /= params.n_merge * params.n_merge;
- } break;
- case PROJECTOR_TYPE_QWEN2VL:
- case PROJECTOR_TYPE_QWEN25VL:
- case PROJECTOR_TYPE_QWEN3VL:
- case PROJECTOR_TYPE_MAGEVL:
- case PROJECTOR_TYPE_SENSENOVA_U1:
- case PROJECTOR_TYPE_LING3VL:
- case PROJECTOR_TYPE_EXAONE4_5:
- case PROJECTOR_TYPE_MIMOVL:
- case PROJECTOR_TYPE_MINIMAX_M3:
- case PROJECTOR_TYPE_GLM4V:
- case PROJECTOR_TYPE_GLM5V:
- case PROJECTOR_TYPE_YOUTUVL:
- case PROJECTOR_TYPE_MUSE_GLIMMER:
- case PROJECTOR_TYPE_LFM3VL:
- {
- // dynamic size (2 conv, so double patch size)
- int x_patch = img->nx() / (params.patch_size * 2);
- int y_patch = img->ny() / (params.patch_size * 2);
- n_patches = x_patch * y_patch;
- } break;
- case PROJECTOR_TYPE_STEP3VL:
- {
- int x_patch = img->nx() / (params.patch_size * params.n_merge);
- int y_patch = img->ny() / (params.patch_size * params.n_merge);
- n_patches = x_patch * y_patch;
- } break;
- case PROJECTOR_TYPE_GEMMA3:
- case PROJECTOR_TYPE_GEMMA4V:
- case PROJECTOR_TYPE_GEMMA4UV:
- case PROJECTOR_TYPE_IDEFICS3:
- case PROJECTOR_TYPE_COHERE2V:
- case PROJECTOR_TYPE_INTERNVL:
- case PROJECTOR_TYPE_NEMOTRON_V2_VL:
- case PROJECTOR_TYPE_NEMOTRON_OMNI:
- case PROJECTOR_TYPE_LLAMA4:
- {
- // both X and Y are downscaled by the scale factor
- int scale_factor = ctx->model.hparams.n_merge;
- n_patches /= (scale_factor * scale_factor);
- } break;
- case PROJECTOR_TYPE_GEMMA3NV:
- {
- // MobileNetV5 MSFA adapter always outputs fixed 16x16 resolution
- // regardless of input size (see architecture description)
- n_patches = ctx->model.hparams.image_size / ctx->model.hparams.patch_size;
- } break;
- case PROJECTOR_TYPE_LFM2:
- case PROJECTOR_TYPE_D1OMNI_V:
- case PROJECTOR_TYPE_KIMIVL:
- case PROJECTOR_TYPE_KIMIK25:
- {
- // dynamic size
- int out_patch_size = params.patch_size * ctx->model.hparams.n_merge;
- int x_patch = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size;
- int y_patch = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
- n_patches = x_patch * y_patch;
- } break;
- case PROJECTOR_TYPE_DEEPSEEK4V:
- {
- const int out_patch_size = params.patch_size * params.n_merge;
- const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size;
- const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
- n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out;
- } break;
- case PROJECTOR_TYPE_DEEPSEEK41V:
- {
- const int out_patch_size = params.patch_size * params.n_merge;
- const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size;
- const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
- n_patches = dsv41_n_block_tokens(n_llm_w, n_llm_h);
- } break;
- case PROJECTOR_TYPE_PADDLEOCR:
- case PROJECTOR_TYPE_DOTS_OCR:
- case PROJECTOR_TYPE_DOTS3NOTE_V:
- {
- // dynamic size
- int n_merge = ctx->model.hparams.n_merge;
- int stride = n_merge * n_merge;
- n_patches = CLIP_ALIGN(n_patches, stride) / stride;
- } break;
- case PROJECTOR_TYPE_DOTS3NOTE_A:
- {
- // 3x stride-2 conv2d over mel frames
- n_patches = (img->nx() + 7) / 8;
- } break;
- case PROJECTOR_TYPE_PIXTRAL:
- case PROJECTOR_TYPE_LIGHTONOCR:
- {
- // dynamic size
- int n_merge = ctx->model.hparams.n_merge;
- int n_patches_x = img->nx() / patch_size / n_merge;
- int n_patches_y = img->ny() / patch_size / n_merge;
- if (ctx->model.token_embd_img_break) {
- n_patches = n_patches_y * n_patches_x + n_patches_y - 1; // + one [IMG_BREAK] per row, except the last row
- } else {
- n_patches = n_patches_y * n_patches_x;
- }
- } break;
- case PROJECTOR_TYPE_VOXTRAL_RT:
- {
- // 2 context frames, conv stride 2, then stacked frames
- n_patches = (img->nx() - 2) / 2 / ctx->model.hparams.proj_stack_factor;
- } break;
- case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
- {
- const int64_t fold = ctx->model.hparams.audio_samples_per_token();
- n_patches = (int) ((img->nx() + fold - 1) / fold);
- } break;
- case PROJECTOR_TYPE_VOXTRAL:
- case PROJECTOR_TYPE_ULTRAVOX:
- case PROJECTOR_TYPE_QWEN2A:
- case PROJECTOR_TYPE_MERALION:
- case PROJECTOR_TYPE_MUSIC_FLAMINGO:
- {
- n_patches = img->nx();
+ for (const auto & layer : model.spk_layers) {
+ qkv_ok = qkv_ok && layer.qkv_w->ne[0] == hparams.spk_n_embd && layer.qkv_w->ne[1] == 3 * hparams.spk_n_embd;
+ }
+ if (!qkv_ok || model.inp_proj_w->ne[1] != hparams.n_embd || d_head <= 0 || d_head * hparams.n_head != hparams.n_embd ||
+ (hparams.n_embd_head > 0 && hparams.n_embd_head != d_head) ||
+ hparams.audio_rope_dims <= 0 || hparams.audio_rope_dims % 2 != 0 || hparams.audio_rope_dims > d_head ||
+ d_head_spk <= 0 || d_head_spk % 2 != 0 || d_head_spk * hparams.spk_n_head != hparams.spk_n_embd ||
+ (int64_t) model.spk_sil_emb.size() != hparams.spk_n_embd || n_spk <= 0 ||
+ model.fusion_spk_kernel->ne[0] != n_spk || model.fusion_spk_kernel->ne[1] != hparams.n_embd ||
+ hparams.mel_filters.size() != (size_t) hparams.n_mel_bins * (hparams.audio_n_fft / 2 + 1) ||
+ hparams.window.size() != (size_t) hparams.audio_window_len) {
+ throw std::runtime_error(string_format("%s: hparams do not match the tensor shapes\n", __func__));
+ }
+
+ // projection to the text model
+ model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight"));
+ model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias"));
+ } break;
+ case PROJECTOR_TYPE_GRANITE_SPEECH:
+ {
+ model.inp_proj_w = get_tensor(string_format(TN_INP_PROJ, "weight"));
+ model.inp_proj_b = get_tensor(string_format(TN_INP_PROJ, "bias"));
+ model.ctc_out_w = get_tensor(string_format(TN_CTC_OUT, "weight"));
+ model.ctc_out_b = get_tensor(string_format(TN_CTC_OUT, "bias"));
+ model.ctc_out_mid_w = get_tensor(string_format(TN_CTC_OUT_MID, "weight"));
+ model.ctc_out_mid_b = get_tensor(string_format(TN_CTC_OUT_MID, "bias"));
- const int proj_stack_factor = ctx->model.hparams.proj_stack_factor;
- if (ctx->model.audio_has_stack_frames()) {
- GGML_ASSERT(proj_stack_factor > 0);
- const int n_len = CLIP_ALIGN(n_patches, proj_stack_factor);
- n_patches = n_len / proj_stack_factor;
- }
+ // per-layer tensors not loaded by the generic loop above
+ for (int il = 0; il < hparams.n_layer; ++il) {
+ auto & layer = model.layers[il];
- // whisper downscales input token by half after conv1d
- n_patches /= 2;
+ layer.attn_rel_pos_emb = get_tensor(string_format(TN_ATTN_REL_POS_EMB, prefix, il));
- if (ctx->model.audio_has_avgpool()) {
- // divide by 2 because of nn.AvgPool1d(2, stride=2)
- n_patches /= 2;
- }
- } break;
- case PROJECTOR_TYPE_QWEN3A:
- {
- // chunk_size=100 frames --> 3x stride-2 conv2d --> 13 tokens per chunk
- const int chunk_size = 100;
- const int tokens_per_chunk = 13;
- n_patches = (img->nx() / chunk_size) * tokens_per_chunk;
- } break;
- case PROJECTOR_TYPE_GLMA:
- {
- n_patches = img->nx();
- // whisper downscales input token by half after conv1d
- n_patches /= 2;
- // reshape by merge_factor
- n_patches /= ctx->model.hparams.proj_stack_factor;
- // for BOI and EOI token embeddings
- n_patches += 2;
- } break;
- case PROJECTOR_TYPE_COGVLM:
- {
- n_patches += 2; // for BOI and EOI token embeddings
- } break;
- case PROJECTOR_TYPE_DEEPSEEKOCR:
- {
- // SAM encoder applies two stride-2 convolutions (net_2 and net_3)
- // that reduce spatial dimensions by 4x in each direction (16x total)
- // E.g., 64x64 -> 16x16 patches
- n_patches /= 16;
+ layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight"));
+ layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, il, "bias"));
- if (img->add_viewsep) {
- // global view: one image-newline per token-row + trailing view separator
- const int h = static_cast<int>(std::sqrt(static_cast<float>(n_patches)));
- n_patches = h * (h + 1) + 1;
- } else if (img->ny() >= img->nx() && img->ny() % img->nx() == 0) {
- // tile row: one image-newline per token-row
- const int grid_w = img->ny() / img->nx();
- const int tile_patches = img->nx() / (patch_size * 4); // patches per tile side (SAM divides by 4)
- const int h = tile_patches;
- n_patches = (tile_patches * grid_w + 1) * h;
- }
- } break;
- case PROJECTOR_TYPE_HUNYUANVL:
- {
- int merge = ctx->model.hparams.n_merge;
- int ow = (img->nx() / patch_size) / merge;
- int oh = (img->ny() / patch_size) / merge;
- n_patches = (ow + 1) * oh + 2;
- } break;
- case PROJECTOR_TYPE_DEEPSEEKOCR2:
- {
- // 1024 global view -> 256 query tokens + 1 view separator = 257;
- // 768 local tile -> 144 query tokens, no separator.
- n_patches /= 16;
- if (img->add_viewsep) {
- n_patches += 1; // view separator, appended only after the global view
- }
- } break;
- case PROJECTOR_TYPE_LFM2A:
- case PROJECTOR_TYPE_D1OMNI_A:
- case PROJECTOR_TYPE_LFM3A:
- {
- n_patches = ((((img->nx() + 1) / 2) + 1) / 2 + 1) / 2;
- } break;
- case PROJECTOR_TYPE_GEMMA4A:
- {
- // Two Conv2D stride-2: O = floor((I + 2p - k) / s) + 1, p=1, k=3, s=2
- // O = floor((I - 1) / 2) + 1
- int n = img->nx();
- for (int i = 0; i < 2; i++) {
- n = (n - 1) / 2 + 1;
- }
- n_patches = n;
- } break;
- case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
- case PROJECTOR_TYPE_PARAKEET:
- {
- n_patches = (img->nx() + (params.subsampling_factor - 1)) / params.subsampling_factor;
- } break;
- case PROJECTOR_TYPE_GEMMA4UA:
- {
- n_patches = img->nx(); // no downsampling: one token per raw waveform frame
- } break;
- case PROJECTOR_TYPE_MIMO_AUDIO:
- {
- // conv1(s=1) + conv2(s=2) -> RVQ-encoder downsample conv(k=2,s=2)
- int n = img->nx();
- n = (n - 1) / 2 + 1; // conv1 + conv2
- n = (n - 2) / 2 + 1; // downsample conv
- const int group_size = params.audio_local_group_size;
- n_patches = (n + group_size - 1) / group_size;
- } break;
- case PROJECTOR_TYPE_GRANITE_SPEECH:
- {
- const int ws = ctx->model.hparams.audio_proj_window_size;
- const int ds = ctx->model.hparams.audio_proj_downsample_rate;
- n_patches = ((img->nx() + ws - 1) / ws) * (ws / ds);
- } break;
- case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
- {
- // pooling gives one speaker embedding, whatever the clip length is
- n_patches = 1;
- } break;
- case PROJECTOR_TYPE_CHATTERBOX_SPKENC:
- {
- // statistics pooling gives one speaker embedding, whatever the clip length is
- n_patches = 1;
- } break;
- case PROJECTOR_TYPE_QWEN3TTS_GEN:
- case PROJECTOR_TYPE_LFM3A_GEN:
- case PROJECTOR_TYPE_DRAMA3_GEN:
- {
- // one hidden-state vector fed back to the talker per call
- n_patches = 1;
- } break;
- case PROJECTOR_TYPE_POCKETTTS_SPKENC:
- {
- // one conditioning row per 12.5Hz frame
- const int hop = ctx->model.hparams.mimi_downsample * 120;
- n_patches = img->nx() / hop;
- } break;
- case PROJECTOR_TYPE_POCKETTTS_GEN:
- {
- // one latent per call for GEN_CODE, GEN_WAV sizes its input from the caller
- n_patches = 1;
- } break;
- case PROJECTOR_TYPE_GRANITE4_VISION:
- {
- // Per-tile output token count: each projector block outputs
- // query_side^2 tokens per window x n^2 windows.
- // For 384x384 input: n = 24/8 = 3, query_side = 4 -> 144.
- const int window_side = ctx->model.hparams.downsample_window_side;
- const int query_side = ctx->model.hparams.downsample_query_side;
- const int side = img->nx() / params.patch_size;
- const int n = side / window_side;
- const int out_side = query_side * n;
- n_patches = out_side * out_side;
- if (img->anyres.is_tiled()) {
- // overview tile, then the unpadded tile grid with one newline per row
- int off_x, off_y, w, h;
- clip_anyres_unpad(img->anyres.grid_x * out_side, img->anyres.grid_y * out_side,
- img->anyres.orig_nx, img->anyres.orig_ny, off_x, off_y, w, h);
- n_patches += h * (w + 1);
- }
- } break;
- default:
- GGML_ABORT("unsupported projector type");
- }
+ layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight"));
+ layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias"));
+ layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight"));
+ layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias"));
+ layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight"));
+ layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"));
- n_patches += (int) clip_suffix_rows(img->suffix_type).size();
+ layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"));
+ layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"));
+ layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight"));
+ layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias"));
+ layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight"));
+ layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight"));
+ layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias"));
+ layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight"));
+ layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"));
+ }
- return n_patches;
-}
+ model.qf_proj_blocks.resize(1);
+ auto & qf = model.qf_proj_blocks[0];
+ qf.qf_proj_query = get_tensor(string_format(TN_QF_PROJ_QUERY, prefix));
+ qf.qf_proj_norm_w = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "weight"));
+ qf.qf_proj_norm_b = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "bias"));
+ qf.qf_proj_linear_w = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "weight"));
+ qf.qf_proj_linear_b = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "bias"));
-ggml_backend_sched_t clip_get_sched(const struct clip_ctx * ctx) {
- return ctx->sched.get();
-}
+ const int n_proj_layers = 2;
+ qf.qf_proj_layers.resize(n_proj_layers);
+ for (int il = 0; il < n_proj_layers; ++il) {
+ auto & pl = qf.qf_proj_layers[il];
-// TODO: remove this and use ggml_mul_mat with GGML_PREC_F32 once all backends honor it for F32 x F32
-// ggml_mul_mat for 2D F32 tensors, always in F32 on CPU: some GPU backends round the inputs to F16 or TF32, which loses the quiet bins of a spectrum
-static void mtmd_mul_mat_prec_op(ggml_tensor * dst, int ith, int nth, void * /*userdata*/) {
- const ggml_tensor * a = dst->src[0];
- const ggml_tensor * b = dst->src[1];
- const int64_t n_k = a->ne[0];
-
- for (int64_t n = ith; n < b->ne[1]; n += nth) {
- const float * b_row = (const float *) ((const char *) b->data + n * b->nb[1]);
- float * dst_row = (float *) ((char *) dst->data + n * dst->nb[1]);
- for (int64_t m = 0; m < a->ne[1]; m++) {
- const float * a_row = (const float *) ((const char *) a->data + m * a->nb[1]);
- // independent partial sums, so that the compiler can vectorize the loop
- float acc[16] = {};
- int64_t k = 0;
- for (; k + 16 <= n_k; k += 16) {
- for (int l = 0; l < 16; l++) {
- acc[l] += a_row[k + l] * b_row[k + l];
- }
- }
- float sum = 0.0f;
- for (; k < n_k; k++) {
- sum += a_row[k] * b_row[k];
- }
- for (int l = 0; l < 16; l++) {
- sum += acc[l];
- }
- dst_row[m] = sum;
- }
- }
-}
+ pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "weight"));
+ pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "bias"));
+ pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, il, "weight"));
+ pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, il, "bias"));
+ pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, il, "weight"));
+ pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, il, "bias"));
+ pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, il, "weight"));
+ pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, il, "bias"));
+ pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, il, "weight"));
+ pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, il, "bias"));
-static ggml_tensor * mtmd_mul_mat_prec(ggml_context * ctx, ggml_tensor * a, ggml_tensor * b) {
- GGML_ASSERT(a->type == GGML_TYPE_F32 && b->type == GGML_TYPE_F32);
- GGML_ASSERT(ggml_n_dims(a) <= 2 && ggml_n_dims(b) <= 2 && a->ne[0] == b->ne[0]);
- GGML_ASSERT(a->nb[0] == sizeof(float) && b->nb[0] == sizeof(float));
- ggml_tensor * args[] = { a, b };
- return ggml_custom_4d(ctx, GGML_TYPE_F32, a->ne[1], b->ne[1], 1, 1, args, 2, mtmd_mul_mat_prec_op, GGML_N_TASKS_MAX, nullptr);
-}
+ pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, il, "weight"));
+ pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, il, "bias"));
+ pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, il, "weight"));
+ pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, il, "bias"));
+ pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, il, "weight"));
+ pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, il, "bias"));
+ pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, il, "weight"));
+ pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, il, "bias"));
+ pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, il, "weight"));
+ pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, il, "bias"));
-bool clip_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out) {
- ggml_backend_sched_t sched = config.sched;
- ggml_backend_t backend_cpu = ggml_backend_sched_get_backend(sched, ggml_backend_sched_get_n_backends(sched) - 1);
- clip_backend_cpu_set_n_threads(backend_cpu, std::max(1, config.n_threads));
-
- const auto & cache = *config.cache;
- const auto & filters = cache.filters.data;
- const int64_t n_fft_bins = config.n_fft_bins;
- const int64_t frame_size = (n_fft_bins - 1) * 2;
- const int64_t n_mel = config.n_mel;
- GGML_ASSERT(filters.size() == (size_t) (n_mel * n_fft_bins));
- GGML_ASSERT(cache.cos_vals.size() == (size_t) frame_size && cache.sin_vals.size() == (size_t) frame_size);
-
- // DFT matrix with the window folded in, the first n_fft_bins rows give the real part
- std::vector<float> dft((size_t) 2 * n_fft_bins * frame_size);
- for (int64_t k = 0; k < n_fft_bins; k++) {
- for (int64_t j = 0; j < frame_size; j++) {
- const int64_t q = (j * k) % frame_size;
- dft[k * frame_size + j] = window[j] * cache.cos_vals[q];
- dft[(k + n_fft_bins) * frame_size + j] = window[j] * cache.sin_vals[q];
- }
- }
+ pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, prefix, il, "weight"));
+ pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, prefix, il, "bias"));
+ pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, il, "weight"));
+ pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, il, "bias"));
+ pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "weight"));
+ pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "bias"));
+ }
+ } break;
+ case PROJECTOR_TYPE_GRANITE4_VISION:
+ {
+ // image_newline lives at the top-level.
+ model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
+
+ // Load separate layerwise and spatial projector tensors
+ const auto projector_count = hparams.feature_layers.size();
+ model.qf_proj_blocks.resize(projector_count);
+ for (size_t bid = 0; bid < projector_count; ++bid) {
+ auto & b = model.qf_proj_blocks[bid];
+
+ // non-layerwise tensors
+ b.qf_proj_img_pos = get_tensor(string_format(TN_MULTI_PROJ_IMG_POS, bid));
+ b.qf_proj_query = get_tensor(string_format(TN_MULTI_PROJ_QUERY, prefix, bid));
+ b.qf_proj_linear_w = get_tensor(string_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "weight"));
+ b.qf_proj_linear_b = get_tensor(string_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "bias"));
+ b.qf_proj_norm_w = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "weight"));
+ b.qf_proj_norm_b = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "bias"));
+ b.qf_proj_post_norm_w = get_tensor(string_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "weight"));
+ b.qf_proj_post_norm_b = get_tensor(string_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "bias"));
+
+ // laywerwise tensors
+ // NOTE: If any model uses multi-layer qformers, this will need to change
+ b.qf_proj_layers.resize(1);
+ auto & pl = b.qf_proj_layers[0];
- // the spectrum is several times larger than the result, so long audio is processed in blocks
- const int64_t n_frames_max = 8192;
+ pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, bid, "weight"));
+ pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, bid, "bias"));
+ pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, bid, "weight"));
+ pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, bid, "bias"));
+ pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, bid, "weight"));
+ pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, bid, "bias"));
+ pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, bid, "weight"));
+ pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, bid, "bias"));
+ pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, bid, "weight"));
+ pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, bid, "bias"));
- for (int64_t off = 0; off < n_frames; off += n_frames_max) {
- const int64_t n_cur = std::min(n_frames_max, n_frames - off);
+ pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "weight"));
+ pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "bias"));
+ pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, bid, "weight"));
+ pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, bid, "bias"));
+ pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, bid, "weight"));
+ pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, bid, "bias"));
+ pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, bid, "weight"));
+ pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, bid, "bias"));
+ pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, bid, "weight"));
+ pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, bid, "bias"));
- ggml_init_params params = {
- /*.mem_size =*/ ggml_tensor_overhead() * 32 + ggml_graph_overhead(),
- /*.mem_buffer =*/ nullptr,
- /*.no_alloc =*/ true,
- };
- ggml_context_ptr ctx_ptr(ggml_init(params));
- ggml_context * ctx0 = ctx_ptr.get();
-
- ggml_tensor * inp_samples = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, (n_cur - 1) * config.hop_length + frame_size);
- ggml_tensor * inp_dft = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, frame_size, 2 * n_fft_bins);
- ggml_tensor * inp_filters = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_fft_bins, n_mel);
- ggml_set_input(inp_samples);
- ggml_set_input(inp_dft);
- ggml_set_input(inp_filters);
-
- ggml_tensor * cur = ggml_cont(ctx0, ggml_view_2d(ctx0, inp_samples, frame_size, n_cur, config.hop_length * ggml_element_size(inp_samples), 0));
- cur = mtmd_mul_mat_prec(ctx0, inp_dft, cur); // [2 * n_fft_bins, n_cur]
-
- cur = ggml_sqr(ctx0, cur);
- cur = ggml_add(ctx0,
- ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], 0),
- ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], n_fft_bins * ggml_element_size(cur)));
- if (config.use_magnitude) {
- cur = ggml_sqrt(ctx0, cur);
- }
+ pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, prefix, bid, "weight"));
+ pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, prefix, bid, "bias"));
+ pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, bid, "weight"));
+ pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, bid, "bias"));
+ pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, prefix, bid, "weight"));
+ pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, prefix, bid, "bias"));
+ }
- cur = mtmd_mul_mat_prec(ctx0, cur, inp_filters); // [n_cur, n_mel]
- cur = config.mel_floor_add
- ? ggml_scale_bias(ctx0, cur, 1.0f, config.mel_floor)
- : ggml_clamp(ctx0, cur, config.mel_floor, FLT_MAX);
- cur = ggml_log(ctx0, cur);
- if (!config.use_natural_log) {
- cur = ggml_scale(ctx0, cur, 1.0f / logf(10.0f));
+ } break;
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ {
+ model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
+ model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
+ } break;
+ default:
+ GGML_ASSERT(false && "unknown projector type");
}
- ggml_set_output(cur);
- ggml_cgraph * gf = ggml_new_graph(ctx0);
- ggml_build_forward_expand(gf, cur);
+ // load data
+ {
+ std::vector<uint8_t> read_buf;
- ggml_backend_sched_reset(sched);
- if (!ggml_backend_sched_alloc_graph(sched, gf)) {
- LOG_ERR("%s: failed to allocate compute graph\n", __func__);
- return false;
- }
+ // start loading event
+ if (progress_callback){
+ progress_callback(0.0, progress_callback_user_data);
+ }
- ggml_backend_tensor_set(inp_samples, samples + off * config.hop_length, 0, ggml_nbytes(inp_samples));
- ggml_backend_tensor_set(inp_dft, dft.data(), 0, ggml_nbytes(inp_dft));
- ggml_backend_tensor_set(inp_filters, filters.data(), 0, ggml_nbytes(inp_filters));
+ // compute total tensor data size for progress reporting
+ size_t total_data_size = 0;
+ for (auto & t : tensors_to_load) {
+ total_data_size += ggml_nbytes(t);
+ }
- auto status = ggml_backend_sched_graph_compute(sched, gf);
- if (status != GGML_STATUS_SUCCESS) {
- LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status);
- return false;
+ // alloc memory and offload data
+ model.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(model.ctx_data.get(), buft));
+ ggml_backend_buffer_set_usage(model.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
+ // read the weight from file
+ if (!no_alloc) {
+ size_t data_loaded = 0;
+ for (auto & t : tensors_to_load) {
+ ggml_tensor * cur = ggml_get_tensor(model.ctx_data.get(), t->name);
+ GGML_ASSERT(cur && "tensor not found in ctx_data");
+ auto it_off = tensor_offset.find(t->name);
+ GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor");
+ const size_t offset = it_off->second;
+ fin.seekg(offset, std::ios::beg);
+ if (!fin) {
+ throw std::runtime_error(string_format("%s: failed to seek for tensor %s\n", __func__, t->name));
+ }
+ size_t num_bytes = ggml_nbytes(cur);
+ if (ggml_backend_buft_is_host(buft)) {
+ // for the CPU and Metal backend, we can read directly into the tensor
+ fin.read(reinterpret_cast<char *>(cur->data), num_bytes);
+ } else {
+ // read into a temporary buffer first, then copy to device memory
+ read_buf.resize(num_bytes);
+ fin.read(reinterpret_cast<char *>(read_buf.data()), num_bytes);
+ ggml_backend_tensor_set(cur, read_buf.data(), 0, num_bytes);
+ }
+ data_loaded += num_bytes;
+ if (progress_callback && total_data_size > 0) {
+ const float progress = (float)data_loaded / (float)total_data_size;
+ if (!progress_callback(progress, progress_callback_user_data)) {
+ throw std::runtime_error(string_format("%s: model loading cancelled by progress_callback\n", __func__));
+ }
+ }
+ }
+ LOG_DBG("%s: loaded %zu tensors from %s\n", __func__, tensors_to_load.size(), fname.c_str());
+ } else {
+ LOG_DBG("%s: no_alloc is set, skipping tensor data loading (%zu tensors)\n", __func__, tensors_to_load.size());
+ }
+ fin.close();
}
- for (int64_t j = 0; j < n_mel; j++) {
- ggml_backend_tensor_get(cur, out + j * n_frames + off, j * cur->nb[1], n_cur * ggml_element_size(cur));
- }
}
- return true;
-}
-
-bool clip_image_encode(struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector<float> & out_vec) {
- clip_image_f32_batch imgs;
- clip_image_f32 img_copy = *img;
- imgs.entries.push_back(std::move(img_copy));
-
- return clip_image_batch_encode(ctx, n_threads, &imgs, out_vec);
-}
-
-bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32_batch * imgs_c_ptr, std::vector<float> & out_batch_embd) {
- clip_encode_params params;
- params.imgs = imgs_c_ptr;
- params.n_threads = n_threads;
- params.out_embd = &out_batch_embd;
-
- if (ctx->proj_type() == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) {
- return clip_graph_nemotron_transcribe::encode_windows(ctx, ctx->model, ¶ms);
+ void get_bool(const std::string & key, bool & output, bool required = true) const {
+ const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
+ if (i < 0) {
+ if (required) {
+ throw std::runtime_error("Key not found: " + key);
+ }
+ return;
+ }
+ output = gguf_get_val_bool(ctx_gguf.get(), i);
}
- return clip_encode(ctx, ¶ms);
-}
-// persisted state slots of the gen-audio decoders and the streaming audio encoders
-static std::vector<c2w_state_slot> list_gen_state_slots(const clip_hparams & hparams, const clip_model & model) {
- switch (model.proj_type) {
- case PROJECTOR_TYPE_QWEN3TTS_GEN: return list_c2w_state_slots(hparams, model);
- case PROJECTOR_TYPE_DRAMA3_GEN: return list_drama3_state_slots(hparams, model);
- case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model);
- case PROJECTOR_TYPE_LFM3A_GEN: return list_lfm3a_gen_state_slots(hparams, model);
- default: return {};
+ void get_i32(const std::string & key, int & output, bool required = true) const {
+ const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
+ if (i < 0) {
+ if (required) {
+ throw std::runtime_error("Key not found: " + key);
+ }
+ return;
+ }
+ output = gguf_get_val_i32(ctx_gguf.get(), i);
}
-}
-// state_out can have another shape than its slot, only the byte size must match
-static void copy_gen_state_out(ggml_tensor * src, ggml_tensor * dst) {
- GGML_ASSERT(ggml_nbytes(src) == ggml_nbytes(dst));
- if (src->type == dst->type && ggml_are_same_shape(src, dst) && ggml_is_contiguous(src)) {
- ggml_backend_tensor_copy(src, dst);
- return;
+ void get_u32(const std::string & key, int & output, bool required = true) const {
+ const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
+ if (i < 0) {
+ if (required) {
+ throw std::runtime_error("Key not found: " + key);
+ }
+ return;
+ }
+ const uint32_t val = gguf_get_val_u32(ctx_gguf.get(), i);
+ // sanity check
+ if (val > (uint32_t) INT32_MAX) {
+ throw std::runtime_error(string_format("%s: value %u for key '%s' exceeds INT32_MAX\n",
+ __func__, val, key.c_str()));
+ }
+ output = (int) val;
}
- std::vector<uint8_t> tmp(ggml_nbytes(src));
- ggml_backend_tensor_get(src, tmp.data(), 0, tmp.size());
- ggml_backend_tensor_set(dst, tmp.data(), 0, tmp.size());
-}
-clip_gen_state * clip_gen_state_init(clip_ctx * ctx, uint32_t seed) {
- auto st = std::make_unique<clip_gen_state>();
- st->rng.seed(seed == UINT32_MAX ? std::random_device{}() : seed);
-
- const auto slots = list_gen_state_slots(ctx->model.hparams, ctx->model);
- if (slots.empty()) {
- return st.release();
+ void get_f32(const std::string & key, float & output, bool required = true) const {
+ const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
+ if (i < 0) {
+ if (required) {
+ throw std::runtime_error("Key not found: " + key);
+ }
+ return;
+ }
+ output = gguf_get_val_f32(ctx_gguf.get(), i);
}
- ggml_init_params params = {
- /*.mem_size =*/ ggml_tensor_overhead() * slots.size(),
- /*.mem_buffer =*/ nullptr,
- /*.no_alloc =*/ true,
- };
- st->ctx_data.reset(ggml_init(params));
- for (const auto & slot : slots) {
- ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1);
- ggml_format_name(t, "gen_state_%s", slot.name.c_str());
- st->slots.push_back(t);
- }
- st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), ctx->backend));
- if (!st->buf) {
- LOG_ERR("%s: failed to allocate the gen state\n", __func__);
- return nullptr;
+ void get_arr_f32(const std::string & key, std::vector<float> & output, bool required = true) const {
+ const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
+ if (i < 0) {
+ if (required) {
+ throw std::runtime_error("Key not found: " + key);
+ }
+ return;
+ }
+ if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) {
+ throw std::runtime_error(string_format("%s: key '%s' is not an array\n", __func__, key.c_str()));
+ }
+ const auto type = gguf_get_arr_type(ctx_gguf.get(), i);
+ if (type != GGUF_TYPE_FLOAT32) {
+ throw std::runtime_error(string_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_FLOAT32)\n", __func__, key.c_str(), type, GGUF_TYPE_FLOAT32));
+ }
+ const size_t n = gguf_get_arr_n(ctx_gguf.get(), i);
+ if (n > (size_t) std::numeric_limits<int>::max()) {
+ throw std::runtime_error(string_format("%s: array '%s' is too large (%zu elements)\n", __func__, key.c_str(), n));
+ }
+ output.resize(n);
+ const float * values = (const float *)gguf_get_arr_data(ctx_gguf.get(), i);
+ for (size_t j = 0; j < n; ++j) {
+ output[j] = values[j];
+ }
}
- // a zero state is the cold start
- ggml_backend_buffer_clear(st->buf.get(), 0);
-
- return st.release();
-}
-
-void clip_gen_state_free(clip_gen_state * state) {
- delete state;
-}
-void clip_gen_state_reset(clip_gen_state * state) {
- if (state->buf) {
- ggml_backend_buffer_clear(state->buf.get(), 0);
+ void get_string(const std::string & key, std::string & output, bool required = true) const {
+ const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
+ if (i < 0) {
+ if (required) {
+ throw std::runtime_error("Key not found: " + key);
+ }
+ return;
+ }
+ output = std::string(gguf_get_val_str(ctx_gguf.get(), i));
}
-}
-
-const std::map<std::string, clip_gen_voice> & clip_gen_get_voices(const clip_ctx * ctx) {
- return ctx->model.gen_voices;
-}
-clip_enc_state * clip_enc_state_init(clip_ctx * ctx, int32_t n_frames_max) {
- const auto & hparams = ctx->model.hparams;
- if (ctx->proj_type() == PROJECTOR_TYPE_LFM3A) {
- // downsampler context, short-conv taps and attention window, see list_lfm3a_state_slots()
- const auto slots = list_lfm3a_state_slots(hparams, ctx->model);
- auto st = std::make_unique<clip_enc_state>();
- ggml_init_params params = {
- /*.mem_size =*/ ggml_tensor_overhead() * slots.size(),
- /*.mem_buffer =*/ nullptr,
- /*.no_alloc =*/ true,
- };
- st->ctx_data.reset(ggml_init(params));
- for (const auto & slot : slots) {
- ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1);
- ggml_format_name(t, "enc_state_%s", slot.name.c_str());
- st->slots.push_back(t);
+ void get_arr_int(const std::string & key, std::vector<int> & output, bool required = true) const {
+ const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
+ if (i < 0) {
+ if (required) {
+ throw std::runtime_error("Key not found: " + key);
+ }
+ return;
}
- st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), ctx->backend));
- if (!st->buf) {
- LOG_ERR("%s: failed to allocate the encoder state\n", __func__);
- return nullptr;
+ if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) {
+ throw std::runtime_error(string_format("%s: key '%s' is not an array\n", __func__, key.c_str()));
+ }
+ const auto type = gguf_get_arr_type(ctx_gguf.get(), i);
+ if (type != GGUF_TYPE_INT32) {
+ throw std::runtime_error(string_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_INT32)\n", __func__, key.c_str(), type, GGUF_TYPE_INT32));
+ }
+ const size_t n = gguf_get_arr_n(ctx_gguf.get(), i);
+ if (n > (size_t) std::numeric_limits<int>::max()) {
+ throw std::runtime_error(string_format("%s: array '%s' is too large (%zu elements)\n", __func__, key.c_str(), n));
+ }
+ output.resize(n);
+ const int32_t * values = (const int32_t *)gguf_get_arr_data(ctx_gguf.get(), i);
+ for (size_t j = 0; j < n; ++j) {
+ output[j] = values[j];
}
- // a zero state is the cold start
- ggml_backend_buffer_clear(st->buf.get(), 0);
- return st.release();
- }
- if (ctx->proj_type() != PROJECTOR_TYPE_VOXTRAL_RT || hparams.attn_window_size <= 0) {
- LOG_ERR("%s: the encoder does not support streaming input\n", __func__);
- return nullptr;
- }
- if (n_frames_max <= 0) {
- LOG_ERR("%s: invalid n_frames_max %d\n", __func__, n_frames_max);
- return nullptr;
}
- const int n_layer = hparams.n_layer;
- const int d_head = hparams.n_embd_head > 0 ? hparams.n_embd_head : hparams.n_embd / hparams.n_head;
- const int n_embd_kv = hparams.n_head_kv * d_head;
-
- auto st = std::make_unique<clip_enc_state>();
- st->n_ctx = hparams.attn_window_size + n_frames_max;
- st->slot_pos.assign(st->n_ctx, -1);
-
- ggml_init_params params = {
- /*.mem_size =*/ ggml_tensor_overhead() * 2 * n_layer,
- /*.mem_buffer =*/ nullptr,
- /*.no_alloc =*/ true,
- };
- st->ctx_data.reset(ggml_init(params));
- for (int il = 0; il < n_layer; il++) {
- ggml_tensor * k = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F16, n_embd_kv, st->n_ctx);
- ggml_tensor * v = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F16, n_embd_kv, st->n_ctx);
- ggml_format_name(k, "enc_state_k_%d", il);
- ggml_format_name(v, "enc_state_v_%d", il);
- st->k.push_back(k);
- st->v.push_back(v);
- }
- st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), ctx->backend));
- if (!st->buf) {
- LOG_ERR("%s: failed to allocate the encoder state\n", __func__);
- return nullptr;
+ static void set_llava_uhd_res_candidates(mtmd_model & model, const int max_patches_per_side) {
+ auto & hparams = model.hparams;
+ for (int x = 1; x <= max_patches_per_side; x++) {
+ for (int y = 1; y <= max_patches_per_side; y++) {
+ if (x == 1 && y == 1) {
+ continue; // skip the first point
+ }
+ hparams.image_res_candidates.push_back(mtmd_image_size{
+ x*hparams.image_size,
+ y*hparams.image_size,
+ });
+ }
+ }
}
- // empty slots are masked out, but they must not hold NaN
- ggml_backend_buffer_clear(st->buf.get(), 0);
-
- LOG_INF("%s: encoder state: %d slots, %.2f MiB\n", __func__, st->n_ctx, ggml_backend_buffer_get_size(st->buf.get()) / 1024.0 / 1024.0);
-
- return st.release();
-}
-
-void clip_enc_state_free(clip_enc_state * state) {
- delete state;
-}
-size_t clip_cbx_read_tensor(struct clip_ctx * ctx, const char * name, float * out, size_t n_max) {
- const auto & tensors = ctx->model.cbx_tensors;
- auto it = tensors.find(name);
- if (it == tensors.end() || !it->second) {
- return 0;
- }
- ggml_tensor * t = it->second;
- const size_t n = (size_t) ggml_nelements(t);
- if (!out) {
- return n;
- }
- if (n_max < n) {
- return 0;
- }
- if (t->type == GGML_TYPE_F32) {
- ggml_backend_tensor_get(t, out, 0, n * sizeof(float));
- } else if (t->type == GGML_TYPE_F16) {
- std::vector<ggml_fp16_t> tmp(n);
- ggml_backend_tensor_get(t, tmp.data(), 0, n * sizeof(ggml_fp16_t));
- for (size_t i = 0; i < n; i++) out[i] = ggml_fp16_to_fp32(tmp[i]);
- } else {
- return 0;
+ static void set_internvl_dhr_res_candidates(mtmd_model & model) {
+ auto & hparams = model.hparams;
+ int min_num = hparams.preproc_min_tiles;
+ int max_num = hparams.preproc_max_tiles;
+ if (min_num < 1) {
+ return; // avoid divide by 0
+ }
+ for (int a = min_num; a <= max_num; ++a) {
+ int b_lo = (min_num + a - 1) / a;
+ int b_hi = max_num / a;
+ b_lo = std::max(b_lo, min_num);
+ b_hi = std::min(b_hi, max_num);
+ for (int b = b_lo; b <= b_hi; ++b) {
+ hparams.image_res_candidates.push_back(mtmd_image_size {
+ a*hparams.image_size,
+ b*hparams.image_size,
+ });
+ }
+ }
}
- return n;
-}
+};
-bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
- const clip_image_f32_batch & imgs = *params->imgs;
- int n_batch_cur = imgs.entries.size();
+//
+// loading
+//
- // [QWEN_VIDEO] for video models, the batch dimension is used as temporal dimension for merged frames
- if (!ctx->support_batch && n_batch_cur > clip_model_n_temporal_merge(ctx)) {
- LOG_ERR("%s: batch size %d exceeds maximum supported batch/temporal-merge size %d\n", __func__, n_batch_cur, clip_model_n_temporal_merge(ctx));
- return false;
- }
+mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params, ggml_backend_buffer_type_t buft) {
+ mtmd_model_loader loader(fname,
+ /* skip_tensors */ false,
+ params.progress_callback,
+ params.progress_callback_user_data);
+
+ auto load = [&](mtmd_modality modality) {
+ auto model = std::make_unique<mtmd_model>();
+ model->hparams.custom_image_min_tokens = params.image_min_tokens;
+ model->hparams.custom_image_max_tokens = params.image_max_tokens;
+ loader.load_hparams(*model, modality);
+ loader.load_tensors(*model, buft, params.no_alloc);
+ return model;
+ };
- // if buffers are not allocated, we need to do a warmup run to allocate them
- if (!ctx->is_allocated) {
- clip_model_loader::warmup(*ctx, *params->imgs);
+ mtmd_models res;
+ if (loader.has_vision) {
+ res.v = load(MTMD_MODALITY_VISION);
}
- const bool needs_gen_state = params->gen_process == CLIP_GEN_PROCESS_GEN_CODE ||
- params->gen_process == CLIP_GEN_PROCESS_GEN_WAV ||
- params->gen_process == CLIP_GEN_PROCESS_TTS;
- if (needs_gen_state && params->gen_state == nullptr) {
- LOG_ERR("%s: audio gen requires a gen state\n", __func__);
- return false;
- }
- if (ctx->proj_type() == PROJECTOR_TYPE_DRAMA3_GEN && params->gen_process == CLIP_GEN_PROCESS_GEN_CODE) {
- float pos = 0.0f;
- ggml_backend_tensor_get(params->gen_state->slots[0], &pos, 0, sizeof(float));
- if ((int) pos >= clip_graph_drama3_gen::n_pos_max) {
- LOG_ERR("%s: the codebook tower holds at most %d rows\n", __func__, clip_graph_drama3_gen::n_pos_max);
- return false;
- }
+ // TODO: we don't support audio for Gemma 3N, but GGUF contains audio tensors
+ // we can remove this check when we implement audio support for Gemma 3N
+ const bool skip_audio = res.v && res.v->proj_type == PROJECTOR_TYPE_GEMMA3NV;
+ if (loader.has_audio && !skip_audio) {
+ res.a = load(MTMD_MODALITY_AUDIO);
}
- // build the inference graph
- ggml_backend_sched_reset(ctx->sched.get());
- ggml_cgraph * gf = clip_get_graph_builder(ctx, imgs, params)->build();
-
- // the last node is the embedding tensor, code2wav has no out_embd
- // read it before scheduling: backends may reorder the nodes of graphs with several outputs (Metal)
- ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr;
-
- if (!ggml_backend_sched_alloc_graph(ctx->sched.get(), gf)) {
- LOG_ERR("%s: failed to allocate compute graph\n", __func__);
- return false;
+ if (loader.has_gen_audio) {
+ res.gen_a = load(MTMD_MODALITY_GEN_AUDIO);
}
- // set inputs
- const auto & model = ctx->model;
- const auto & hparams = model.hparams;
-
- const int image_size_width = imgs.entries[0].nx();
- const int image_size_height = imgs.entries[0].ny();
-
- const int patch_size = hparams.patch_size;
- const int num_patches = ((image_size_width / patch_size) * (image_size_height / patch_size));
- const int n_pos = num_patches + (model.class_embedding ? 1 : 0);
- const int pos_w = image_size_width / patch_size;
- const int pos_h = image_size_height / patch_size;
-
+ return res;
+}
- auto get_inp_tensor = [&gf](const char * name) {
- ggml_tensor * inp = ggml_graph_get_tensor(gf, name);
- if (inp == nullptr) {
- GGML_ABORT("Failed to get tensor %s", name);
- }
- if (!(inp->flags & GGML_TENSOR_FLAG_INPUT)) {
- GGML_ABORT("Tensor %s is not an input tensor", name);
- }
- return inp;
- };
+mtmd_model_caps mtmd_model_get_caps(const char * fname) {
+ mtmd_model_caps res;
+ mtmd_model_loader loader(fname, /* skip_tensors= */ true);
+ res.has_vision = loader.has_vision;
+ res.has_audio = loader.has_audio;
+ res.has_gen_audio = loader.has_gen_audio;
+ return res;
+}
- auto set_input_f32 = [&get_inp_tensor](const char * name, const std::vector<float> & values) {
- ggml_tensor * cur = get_inp_tensor(name);
- GGML_ASSERT(cur->type == GGML_TYPE_F32);
- GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size());
- ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur));
- };
+//
+// mtmd_model
+//
- // mask from build_inp_attn_mask(), f16 if flash attn is enabled
- auto set_input_attn_mask = [&get_inp_tensor](const std::vector<float> & values, int idx = 0) {
- ggml_tensor * cur = get_inp_tensor(get_attn_mask_name(idx).c_str());
- GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size());
- if (cur->type == GGML_TYPE_F16) {
- std::vector<ggml_fp16_t> values_f16(values.size());
- ggml_fp32_to_fp16_row(values.data(), values_f16.data(), values.size());
- ggml_backend_tensor_set(cur, values_f16.data(), 0, ggml_nbytes(cur));
- } else {
- GGML_ASSERT(cur->type == GGML_TYPE_F32);
- ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur));
- }
- };
+int mtmd_model::image_max_tokens() const {
+ if (proj_type == PROJECTOR_TYPE_DEEPSEEK4V || proj_type == PROJECTOR_TYPE_DEEPSEEK41V) {
+ return hparams.dsv4_max_n_token;
+ }
+ if (hparams.image_max_pixels <= 0) {
+ return -1;
+ }
+ const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
+ return hparams.image_max_pixels / patch_area;
+}
- auto set_input_i32 = [&get_inp_tensor](const char * name, std::vector<int32_t> & values) {
- ggml_tensor * cur = get_inp_tensor(name);
- GGML_ASSERT(cur->type == GGML_TYPE_I32);
- GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size());
- ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur));
- };
+int mtmd_model::n_output_tokens_x(const mtmd_image_f32 * img) const {
+ const auto & params = hparams;
+ const int n_total = n_output_tokens(img);
+ const auto & proj = proj_type;
+ switch (proj) {
+ case PROJECTOR_TYPE_QWEN2VL:
+ case PROJECTOR_TYPE_QWEN25VL:
+ case PROJECTOR_TYPE_QWEN3VL:
+ case PROJECTOR_TYPE_MAGEVL:
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ case PROJECTOR_TYPE_LING3VL:
+ case PROJECTOR_TYPE_EXAONE4_5:
+ case PROJECTOR_TYPE_MIMOVL:
+ case PROJECTOR_TYPE_GLM4V:
+ case PROJECTOR_TYPE_GLM5V:
+ case PROJECTOR_TYPE_PADDLEOCR:
+ case PROJECTOR_TYPE_HUNYUANVL:
+ case PROJECTOR_TYPE_YOUTUVL:
+ case PROJECTOR_TYPE_MUSE_GLIMMER:
+ case PROJECTOR_TYPE_LFM3VL:
+ return (img->nx() / params.patch_size) / 2;
+ case PROJECTOR_TYPE_STEP3VL:
+ case PROJECTOR_TYPE_MINICPMV4_6:
+ case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
+ return img->nx() / (params.patch_size * params.n_merge);
+ case PROJECTOR_TYPE_DEEPSEEKOCR:
+ case PROJECTOR_TYPE_DEEPSEEKOCR2:
+ return (img->nx() / params.patch_size) / 4;
+ default:
+ break;
+ }
+ return n_total;
+}
- // state of the previous call, zero on a cold start
- auto set_gen_state_in = [&]() {
- const auto & slots = params->gen_state->slots;
- size_t i = 0;
- for (const auto & slot : list_gen_state_slots(hparams, model)) {
- GGML_ASSERT(i < slots.size());
- if (slot.in_place) {
- i++;
- continue;
- }
- ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str()));
- }
- };
+int mtmd_model::n_output_tokens_y(const mtmd_image_f32 * img) const {
+ const auto & params = hparams;
+ const auto & proj = proj_type;
+ switch (proj) {
+ case PROJECTOR_TYPE_QWEN2VL:
+ case PROJECTOR_TYPE_QWEN25VL:
+ case PROJECTOR_TYPE_QWEN3VL:
+ case PROJECTOR_TYPE_MAGEVL:
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ case PROJECTOR_TYPE_LING3VL:
+ case PROJECTOR_TYPE_EXAONE4_5:
+ case PROJECTOR_TYPE_MIMOVL:
+ case PROJECTOR_TYPE_GLM4V:
+ case PROJECTOR_TYPE_GLM5V:
+ case PROJECTOR_TYPE_PADDLEOCR:
+ case PROJECTOR_TYPE_HUNYUANVL:
+ case PROJECTOR_TYPE_YOUTUVL:
+ case PROJECTOR_TYPE_MUSE_GLIMMER:
+ case PROJECTOR_TYPE_LFM3VL:
+ return (img->ny() / params.patch_size) / 2;
+ case PROJECTOR_TYPE_STEP3VL:
+ case PROJECTOR_TYPE_MINICPMV4_6:
+ case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
+ return img->ny() / (params.patch_size * params.n_merge);
+ default:
+ break;
+ }
+ return 1;
+}
- // lfm3a encoder and codec: causal sliding window over [cached window | this call]
- auto set_lfm3a_window_inputs = [&](int base, int n_pos, int window) {
- std::vector<int32_t> positions(n_pos);
- for (int i = 0; i < n_pos; i++) {
- positions[i] = base + i;
- }
- set_input_i32("positions", positions);
-
- // key j holds position base - prefix + j
- const int prefix = window - 1;
- const int n_kv = prefix + n_pos;
- std::vector<float> mask((size_t) n_kv * n_pos, -INFINITY);
- for (int q = 0; q < n_pos; q++) {
- for (int j = std::max(0, q + prefix - window + 1); j <= q + prefix; j++) {
- if (base - prefix + j >= 0) {
- mask[(size_t) q * n_kv + j] = 0.0f;
- }
- }
- }
- set_input_attn_mask(mask);
- };
+int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
+ const auto & params = hparams;
- // rope positions and attention mask of the mimi transformers (pocket-tts).
- // the mask is causal with a sliding window, see _build_attention_mask() in the reference
- auto set_pockettts_tfm_inputs = [&]() {
- const int64_t n_pos = ggml_nelements(get_inp_tensor("inp_pos"));
- GGML_ASSERT(n_pos > 0);
- std::vector<int32_t> positions((size_t) n_pos);
- for (int64_t i = 0; i < n_pos; i++) {
- positions[(size_t) i] = (int32_t) i;
- }
- set_input_i32("inp_pos", positions);
-
- // the preprocessor truncates the waveform to keep this mask bounded
- const int64_t max_pos = (int64_t) clip_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate / 120;
- GGML_ASSERT(n_pos <= max_pos && "pocket-tts speaker reference too long for a dense mask");
-
- const int64_t context = hparams.mimi_tfm_context;
- std::vector<float> mask((size_t) n_pos * n_pos, -INFINITY);
- for (int64_t q = 0; q < n_pos; q++) {
- for (int64_t k = 0; k < n_pos; k++) {
- const int64_t delta = q - k;
- if (delta >= 0 && delta < context) {
- mask[(size_t) q * n_pos + k] = 0.0f;
- }
- }
- }
- set_input_attn_mask(mask);
- };
+ // for models with fixed size image, the input image is already pre-processed and resized to square
+ int patch_size = params.patch_size;
+ int n_patches = (img->nx() / patch_size) * (img->ny() / patch_size);
- // set input pixel values
- if (!imgs.is_audio) {
- size_t nelem = 0;
- for (const auto & img : imgs.entries) {
- nelem += img.nx() * img.ny() * 3;
- }
- std::vector<float> inp_raw(nelem);
-
- // layout of data (note: the channel dim is unrolled to better visualize the layout):
- //
- // ┌──W──┐
- // │ H │ channel = R
- // ├─────┤ │
- // │ H │ channel = G
- // ├─────┤ │
- // │ H │ channel = B
- // └─────┘ │
- // ──────┘ x B
-
- // IMPORTANT: [QWEN_VIDEO] the batch dim is currently used for temporal dim in Qwen-VL models
- // All entries must have the same spatial size (enforced by can_batch_with() during merging)
- {
- const int nx = imgs.entries[0].nx();
- const int ny = imgs.entries[0].ny();
- const int n = nx * ny;
-
- for (int b = 0; b < n_batch_cur; b++) {
- LOG_DBG("%s: copying image %d/%d to input buffer (nx=%d, ny=%d)\n", __func__, b+1, n_batch_cur, nx, ny);
- const auto & buf = imgs.entries[b].get_ro_buf();
- float * batch_entry = inp_raw.data() + b * (3*n);
- for (int y = 0; y < ny; y++) {
- for (int x = 0; x < nx; x++) {
- size_t base_src = 3*(y * nx + x);
- size_t base_dst = y * nx + x;
- batch_entry[ base_dst] = buf[base_src ];
- batch_entry[1*n + base_dst] = buf[base_src + 1];
- batch_entry[2*n + base_dst] = buf[base_src + 2];
- }
- }
- }
- }
- set_input_f32("inp_raw", inp_raw);
-
- } else if (params->gen_process != CLIP_GEN_PROCESS_GEN_WAV &&
- params->gen_process != CLIP_GEN_PROCESS_TTS &&
- params->gen_process != CLIP_GEN_PROCESS_TTS_VOCODE &&
- ggml_graph_get_tensor(gf, "inp_raw") != nullptr) {
- // audio input. GEN_WAV and the chatterbox TTS stages are not here: they take codes or feats, set in the switch below.
- // a GEN_CODE graph may have no hidden-state input (lfm3a_gen feedback embedding)
- GGML_ASSERT(imgs.entries.size() == 1);
-
- const auto & mel_inp = imgs.entries[0];
- const auto & buf = mel_inp.get_ro_buf();
- const int n_step = mel_inp.nx();
- const int n_mel = mel_inp.ny();
- GGML_ASSERT((size_t)n_step * n_mel == buf.size());
-
- set_input_f32("inp_raw", buf);
- }
+ projector_type proj = proj_type;
- // set input per projector
- switch (ctx->model.proj_type) {
- case PROJECTOR_TYPE_VOXTRAL_RT:
+ switch (proj) {
+ case PROJECTOR_TYPE_MLP:
+ case PROJECTOR_TYPE_MLP_NORM:
+ case PROJECTOR_TYPE_JANUS_PRO:
+ case PROJECTOR_TYPE_PHI4:
{
- clip_enc_state * st = params->enc_state;
- if (!st) {
- LOG_ERR("%s: this encoder needs a clip_enc_state\n", __func__);
- return false;
- }
- const int n_window = hparams.attn_window_size;
- const int n_new = (imgs.entries[0].nx() - 2) / 2;
- GGML_ASSERT(n_new <= st->n_ctx - n_window);
-
- std::vector<int32_t> pos(n_new);
- std::vector<int64_t> idx(n_new);
- std::vector<int32_t> slot_pos = st->slot_pos; // including the frames written by this call
- for (int i = 0; i < n_new; i++) {
- pos[i] = st->n_past + i;
- idx[i] = pos[i] % st->n_ctx;
- slot_pos[idx[i]] = pos[i];
- }
- set_input_i32("inp_pos", pos);
- ggml_backend_tensor_set(get_inp_tensor("inp_kv_idx"), idx.data(), 0, idx.size() * sizeof(int64_t));
-
- // each frame sees itself and the n_window - 1 frames before it
- std::vector<float> mask((size_t) st->n_ctx * n_new, -INFINITY);
- for (int i = 0; i < n_new; i++) {
- for (int s = 0; s < st->n_ctx; s++) {
- const int32_t p = slot_pos[s];
- if (p >= 0 && p <= pos[i] && p > pos[i] - n_window) {
- mask[(size_t) i * st->n_ctx + s] = 0.0f;
- }
- }
- }
- set_input_attn_mask(mask);
-
- std::vector<float> tail = st->stem_tail;
- tail.resize(hparams.n_embd, 0.0f);
- set_input_f32("inp_stem_tail", tail);
+ // do nothing
} break;
- case PROJECTOR_TYPE_MUSE_GLIMMER:
+ case PROJECTOR_TYPE_YASA2:
{
- const int grid_w = pos_w; // image_size_width / patch_size
- const int grid_h = pos_h; // image_size_height / patch_size
- const int n_tok = grid_w * grid_h;
- const int pgrid = (int) std::sqrt((double) ctx->model.position_embeddings->ne[1]); // 32
- const int f = hparams.n_merge; // downsample 2
-
- // pixel patchify runs inside the graph via build_inp() (ggml_conv_2d);
- // pos-emb bilinear interp via resize_position_embeddings().
-
- // --- sparse window grouping (pgrid x pgrid windows) ---
- const int win = pgrid;
- const int nwin_h = (grid_h + win - 1) / win;
- const int nwin_w = (grid_w + win - 1) / win;
- std::vector<int32_t> sp_perm; sp_perm.reserve(n_tok);
- std::vector<int> sp_slens;
- for (int wy = 0; wy < nwin_h; wy++) {
- for (int wx = 0; wx < nwin_w; wx++) {
- int cnt = 0;
- for (int hh = 0; hh < win; hh++) {
- for (int ww = 0; ww < win; ww++) {
- const int gy = wy * win + hh;
- const int gx = wx * win + ww;
- if (gy < grid_h && gx < grid_w) { sp_perm.push_back(gy * grid_w + gx); cnt++; }
- }
- }
- if (cnt > 0) sp_slens.push_back(cnt);
- }
- }
- std::vector<int32_t> rpos_w(n_tok), rpos_h(n_tok), inv_perm(n_tok);
- for (int i = 0; i < n_tok; i++) {
- const int orig = sp_perm[i];
- rpos_w[i] = (orig % grid_w) + 1; // 1-indexed
- rpos_h[i] = (orig / grid_w) + 1;
- inv_perm[orig] = i;
- }
- set_input_i32("muse_glimmer_sp_perm", sp_perm);
- set_input_i32("muse_glimmer_inv_perm", inv_perm);
- set_input_i32("muse_glimmer_pos_w", rpos_w);
- set_input_i32("muse_glimmer_pos_h", rpos_h);
-
- // block-diagonal window mask (permuted order)
- std::vector<float> sp_mask((size_t) n_tok * n_tok, -INFINITY);
- {
- int off = 0;
- for (int s : sp_slens) {
- for (int a = 0; a < s; a++)
- for (int b = 0; b < s; b++)
- sp_mask[(size_t) (off + a) * n_tok + (off + b)] = 0.0f;
- off += s;
- }
+ n_patches = 64; // adaptive average pooling to 8x8 tokens
+ } break;
+ case PROJECTOR_TYPE_LDP:
+ case PROJECTOR_TYPE_LDPV2:
+ case PROJECTOR_TYPE_GLM_EDGE:
+ {
+ n_patches /= 4;
+ if (mm_boi) {
+ n_patches += 2; // for BOI and EOI token embeddings
}
- set_input_attn_mask(sp_mask);
-
- // pixel-shuffle gather (original order): f*f spatial neighbours grouped
- std::vector<int32_t> dsp; dsp.reserve(n_tok);
- for (int oy = 0; oy < grid_h / f; oy++)
- for (int ox = 0; ox < grid_w / f; ox++)
- for (int ry = 0; ry < f; ry++)
- for (int rx = 0; rx < f; rx++)
- dsp.push_back((oy * f + ry) * grid_w + (ox * f + rx));
- set_input_i32("muse_glimmer_ds_perm", dsp);
} break;
case PROJECTOR_TYPE_MINICPMV:
{
- // inspired from siglip:
- // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit
- // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316
- std::vector<int32_t> positions(pos_h * pos_w);
- // note: sized by the actual patch counts; a tall/wide image produces more
- // than 1024 patches per side and a fixed [1024] array would be overrun
- std::vector<int> bucket_coords_h(pos_h);
- std::vector<int> bucket_coords_w(pos_w);
- for (int i = 0; i < pos_h; i++){
- bucket_coords_h[i] = std::floor(70.0*i/pos_h);
- }
- for (int i = 0; i < pos_w; i++){
- bucket_coords_w[i] = std::floor(70.0*i/pos_w);
- }
- for (int i = 0, id = 0; i < pos_h; i++){
- for (int j = 0; j < pos_w; j++){
- positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j];
+ // Use actual config value if available, otherwise fall back to hardcoded values
+ if (params.minicpmv_query_num > 0) {
+ n_patches = params.minicpmv_query_num;
+ } else {
+ // Fallback to hardcoded values for legacy models
+ if (params.minicpmv_version == 2) {
+ n_patches = 96;
+ } else if (params.minicpmv_version == 3) {
+ n_patches = 64;
+ } else if (params.minicpmv_version == 4) {
+ n_patches = 64;
+ } else if (params.minicpmv_version == 5) {
+ // MiniCPM-V 4.0
+ n_patches = 64;
+ } else if (params.minicpmv_version == 6) {
+ // MiniCPM-V 4.5
+ n_patches = 64;
+ } else if (params.minicpmv_version == 100045) {
+ // MiniCPM-o 4.5
+ n_patches = 64;
+ } else {
+ GGML_ABORT("Unknown minicpmv version");
}
}
- set_input_i32("positions", positions);
-
- // inputs for resampler projector
- // set the 2D positions (using float for sinusoidal embedding)
- int n_patches_per_col = image_size_width / patch_size;
- std::vector<float> pos_data(n_pos);
- // dimension H
- for (int i = 0; i < n_pos; i++) {
- pos_data[i] = static_cast<float>(i / n_patches_per_col);
- }
- set_input_f32("pos_h", pos_data);
- // dimension W
- for (int i = 0; i < n_pos; i++) {
- pos_data[i] = static_cast<float>(i % n_patches_per_col);
- }
- set_input_f32("pos_w", pos_data);
- // base frequency omega
- const float base_freq = 10000.0f;
- const int n_embd_proj = clip_n_mmproj_embd(ctx);
- std::vector<float> omega(n_embd_proj / 4);
- for (int i = 0; i < n_embd_proj / 4; ++i) {
- omega[i] = 1.0f / std::pow(base_freq, static_cast<float>(i) / (n_embd_proj / 4));
- }
- set_input_f32("omega", omega);
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
- const bool is_4x = hparams.n_merge == 2;
-
- // SigLIP position buckets (same as resampler path)
- std::vector<int32_t> positions(pos_h * pos_w);
- std::vector<int> bucket_coords_h(pos_h);
- std::vector<int> bucket_coords_w(pos_w);
- for (int i = 0; i < pos_h; i++){
- bucket_coords_h[i] = std::floor(70.0*i/pos_h);
- }
- for (int i = 0; i < pos_w; i++){
- bucket_coords_w[i] = std::floor(70.0*i/pos_w);
- }
- for (int i = 0, id = 0; i < pos_h; i++){
- for (int j = 0; j < pos_w; j++){
- positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j];
- }
- }
- set_input_i32("positions", positions);
-
- const int half_h = pos_h / 2;
- const int half_w = pos_w / 2;
-
- auto make_ds_idx = [](int off_r, int off_c, int ds_h, int ds_w, int stride_w) {
- std::vector<int32_t> idx(ds_h * ds_w);
- for (int i = 0; i < ds_h; i++) {
- for (int j = 0; j < ds_w; j++) {
- idx[i * ds_w + j] = (2*i + off_r) * stride_w + (2*j + off_c);
- }
- }
- return idx;
- };
-
- if (!is_4x) {
- // window reorder indices for 2x2 windows
- std::vector<int32_t> window_idx(n_pos);
- std::vector<int32_t> inv_window_idx(n_pos);
- {
- int k = 0;
- for (int wi = 0; wi < half_h; wi++) {
- for (int wj = 0; wj < half_w; wj++) {
- window_idx[k++] = (2*wi ) * pos_w + (2*wj );
- window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1);
- window_idx[k++] = (2*wi + 1) * pos_w + (2*wj );
- window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1);
- }
- }
- for (int i = 0; i < n_pos; i++) {
- inv_window_idx[window_idx[i]] = i;
- }
- }
- set_input_i32("vit_merger_window_idx", window_idx);
- set_input_i32("vit_merger_inv_window_idx", inv_window_idx);
-
- // block-diagonal attention mask: tokens in the same 4-token
- // window attend to each other (mask = 0), all other positions
- // are masked out (-inf). matches the window-major reorder above.
- std::vector<float> window_mask_data(n_pos * n_pos, std::numeric_limits<float>::lowest());
- for (int wi = 0; wi < n_pos / 4; wi++) {
- for (int i = 0; i < 4; i++) {
- for (int j = 0; j < 4; j++) {
- window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f;
- }
- }
- }
- set_input_attn_mask(window_mask_data);
-
- // ViT merger 2x2 downsample indices
- auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w);
- auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w);
- auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w);
- auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w);
- set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0);
- set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1);
- set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2);
- set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3);
- }
-
- const int merger_h = is_4x ? pos_h : half_h;
- const int merger_w = is_4x ? pos_w : half_w;
- auto m_ds_0 = make_ds_idx(0, 0, merger_h / 2, merger_w / 2, merger_w);
- auto m_ds_1 = make_ds_idx(0, 1, merger_h / 2, merger_w / 2, merger_w);
- auto m_ds_2 = make_ds_idx(1, 0, merger_h / 2, merger_w / 2, merger_w);
- auto m_ds_3 = make_ds_idx(1, 1, merger_h / 2, merger_w / 2, merger_w);
- set_input_i32("merger_ds_idx_0", m_ds_0);
- set_input_i32("merger_ds_idx_1", m_ds_1);
- set_input_i32("merger_ds_idx_2", m_ds_2);
- set_input_i32("merger_ds_idx_3", m_ds_3);
- } break;
- case PROJECTOR_TYPE_LFM3VL:
- {
- // patches in raster-scan order (no interleave before encoder layers)
- const int pw = image_size_width / patch_size;
- const int ph = image_size_height / patch_size;
- std::vector<int> positions(n_pos * 4);
- int ptr = 0;
- for (int y = 0; y < ph; y++) {
- for (int x = 0; x < pw; x++) {
- positions[ ptr] = y;
- positions[ num_patches + ptr] = x;
- positions[2 * num_patches + ptr] = y;
- positions[3 * num_patches + ptr] = x;
- ptr++;
- }
- }
- set_input_i32("positions", positions);
- } break;
- case PROJECTOR_TYPE_MAGEVL:
- {
- // t, h, w in 2x2 block order, t is always 0 for an image
- const int merge_ratio = hparams.n_merge;
- const int pw = image_size_width / patch_size / merge_ratio;
- const int ph = image_size_height / patch_size / merge_ratio;
- std::vector<int> positions(n_pos * 4, 0);
- int ptr = 0;
- for (int ty = 0; ty < ph; ty++) {
- for (int tx = 0; tx < pw; tx++) {
- for (int dy = 0; dy < merge_ratio; dy++) {
- for (int dx = 0; dx < merge_ratio; dx++) {
- positions[ num_patches + ptr] = ty * merge_ratio + dy;
- positions[2 * num_patches + ptr] = tx * merge_ratio + dx;
- ptr++;
- }
- }
- }
- }
- set_input_i32("positions", positions);
+ n_patches /= params.n_merge * params.n_merge;
} break;
case PROJECTOR_TYPE_QWEN2VL:
+ case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
+ case PROJECTOR_TYPE_MAGEVL:
+ case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_LING3VL:
+ case PROJECTOR_TYPE_EXAONE4_5:
+ case PROJECTOR_TYPE_MIMOVL:
+ case PROJECTOR_TYPE_MINIMAX_M3:
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_GLM5V:
+ case PROJECTOR_TYPE_YOUTUVL:
+ case PROJECTOR_TYPE_MUSE_GLIMMER:
+ case PROJECTOR_TYPE_LFM3VL:
{
- const int merge_ratio = hparams.n_merge;
- const int pw = image_size_width / patch_size / merge_ratio;
- const int ph = image_size_height / patch_size / merge_ratio;
- std::vector<int> positions(n_pos * 4);
- int ptr = 0;
- for (int ty = 0; ty < ph; ty++) {
- for (int tx = 0; tx < pw; tx++) {
- for (int dy = 0; dy < merge_ratio; dy++) {
- for (int dx = 0; dx < merge_ratio; dx++) {
- const int y = ty * merge_ratio + dy;
- const int x = tx * merge_ratio + dx;
- positions[ ptr] = y;
- positions[ num_patches + ptr] = x;
- positions[2 * num_patches + ptr] = y;
- positions[3 * num_patches + ptr] = x;
- ptr++;
- }
- }
- }
- }
-
- set_input_i32("positions", positions);
+ // dynamic size (2 conv, so double patch size)
+ int x_patch = img->nx() / (params.patch_size * 2);
+ int y_patch = img->ny() / (params.patch_size * 2);
+ n_patches = x_patch * y_patch;
} break;
case PROJECTOR_TYPE_STEP3VL:
{
- std::vector<int32_t> pos_data(n_pos);
- for (int i = 0; i < n_pos; i++) {
- pos_data[i] = i / pos_w;
- }
- set_input_i32("pos_h", pos_data);
- for (int i = 0; i < n_pos; i++) {
- pos_data[i] = i % pos_w;
- }
- set_input_i32("pos_w", pos_data);
- } break;
- case PROJECTOR_TYPE_PADDLEOCR:
- {
- const int merge_ratio = hparams.n_merge;
- const int pw = image_size_width / patch_size / merge_ratio;
- const int ph = image_size_height / patch_size / merge_ratio;
- std::vector<int> positions(n_pos * 4);
- int ptr = 0;
- // NOTE: same as Qwen-VL, but x and y are swapped
- for (int ty = 0; ty < ph; ty++) {
- for (int dy = 0; dy < merge_ratio; dy++) {
- for (int tx = 0; tx < pw; tx++) {
- for (int dx = 0; dx < merge_ratio; dx++) {
- const int y = ty * merge_ratio + dy;
- const int x = tx * merge_ratio + dx;
- positions[ ptr] = y;
- positions[ num_patches + ptr] = x;
- positions[2 * num_patches + ptr] = y;
- positions[3 * num_patches + ptr] = x;
- ptr++;
- }
- }
- }
- }
-
- set_input_i32("positions", positions);
- } break;
- case PROJECTOR_TYPE_MINIMAX_M3:
- {
- const int n_merge = hparams.n_merge;
- const int gh = image_size_height / patch_size;
- const int gw = image_size_width / patch_size;
- std::vector<int32_t> pos_h, pos_w;
- pos_h.reserve(gh * gw);
- pos_w.reserve(gh * gw);
- for (int bh = 0; bh < gh / n_merge; bh++)
- for (int bw = 0; bw < gw / n_merge; bw++)
- for (int mh = 0; mh < n_merge; mh++)
- for (int mw = 0; mw < n_merge; mw++) {
- pos_h.push_back(bh * n_merge + mh);
- pos_w.push_back(bw * n_merge + mw);
- }
- set_input_i32("minimax_pos_h", pos_h);
- set_input_i32("minimax_pos_w", pos_w);
- } break;
- case PROJECTOR_TYPE_DOTS_OCR:
- case PROJECTOR_TYPE_DOTS3NOTE_V:
- {
- const int pw = image_size_width / patch_size;
- const int ph = image_size_height / patch_size;
- const int n_pos = ph * pw;
- std::vector<int> positions(n_pos * 4);
- int ptr = 0;
-
- // flat layout: [h, w, h, w] for each patch
- // patches are in raster order (matching conv2d output)
- for (int y = 0; y < ph; y++) {
- for (int x = 0; x < pw; x++) {
- positions[ ptr] = y;
- positions[ n_pos + ptr] = x;
- positions[2*n_pos + ptr] = y;
- positions[3*n_pos + ptr] = x;
- ptr++;
- }
- }
-
- set_input_i32("positions", positions);
+ int x_patch = img->nx() / (params.patch_size * params.n_merge);
+ int y_patch = img->ny() / (params.patch_size * params.n_merge);
+ n_patches = x_patch * y_patch;
} break;
- case PROJECTOR_TYPE_QWEN25VL:
- case PROJECTOR_TYPE_EXAONE4_5:
- case PROJECTOR_TYPE_YOUTUVL:
+ case PROJECTOR_TYPE_GEMMA3:
+ case PROJECTOR_TYPE_GEMMA4V:
+ case PROJECTOR_TYPE_GEMMA4UV:
+ case PROJECTOR_TYPE_IDEFICS3:
+ case PROJECTOR_TYPE_COHERE2V:
+ case PROJECTOR_TYPE_INTERNVL:
+ case PROJECTOR_TYPE_NEMOTRON_V2_VL:
+ case PROJECTOR_TYPE_NEMOTRON_OMNI:
+ case PROJECTOR_TYPE_LLAMA4:
{
- // pw * ph = number of tokens output by ViT after apply patch merger
- // ipw * ipw = number of vision token been processed inside ViT
- const bool use_window_attn =
- (ctx->model.proj_type == PROJECTOR_TYPE_QWEN25VL || ctx->model.proj_type == PROJECTOR_TYPE_EXAONE4_5)
- ? hparams.n_wa_pattern > 0
- : !hparams.wa_layer_indexes.empty();
- const int merge_ratio = 2;
- const int pw = image_size_width / patch_size / merge_ratio;
- const int ph = image_size_height / patch_size / merge_ratio;
- const int ipw = image_size_width / patch_size;
- const int iph = image_size_height / patch_size;
-
- std::vector<int> idx (ph * pw);
- std::vector<int> inv_idx(ph * pw);
-
- if (use_window_attn) {
- const int attn_window_size = hparams.attn_window_size > 0 ? hparams.attn_window_size : 112;
- const int grid_window = attn_window_size / patch_size / merge_ratio;
- int dst = 0;
- // [num_vision_tokens, num_vision_tokens] attention mask tensor
- std::vector<float> mask(pow(ipw * iph, 2), std::numeric_limits<float>::lowest());
- int mask_row = 0;
-
- for (int y = 0; y < ph; y += grid_window) {
- for (int x = 0; x < pw; x += grid_window) {
- const int win_h = std::min(grid_window, ph - y);
- const int win_w = std::min(grid_window, pw - x);
- const int dst_0 = dst;
- // group all tokens belong to the same window togather (to a continue range)
- for (int dy = 0; dy < win_h; dy++) {
- for (int dx = 0; dx < win_w; dx++) {
- const int src = (y + dy) * pw + (x + dx);
- GGML_ASSERT(src < (int)idx.size());
- GGML_ASSERT(dst < (int)inv_idx.size());
- idx [src] = dst;
- inv_idx[dst] = src;
- dst++;
- }
- }
-
- for (int r=0; r < win_h * win_w * merge_ratio * merge_ratio; r++) {
- int row_offset = mask_row * (ipw * iph);
- std::fill(
- mask.begin() + row_offset + (dst_0 * merge_ratio * merge_ratio),
- mask.begin() + row_offset + (dst * merge_ratio * merge_ratio),
- 0.0);
- mask_row++;
- }
- }
- }
-
- set_input_i32("window_idx", idx);
- set_input_i32("inv_window_idx", inv_idx);
- set_input_attn_mask(mask);
- } else {
- for (int i = 0; i < ph * pw; i++) {
- idx[i] = i;
- }
- }
-
- const int mpow = merge_ratio * merge_ratio;
- std::vector<int> positions(n_pos * 4);
-
- int ptr = 0;
- for (int ty = 0; ty < ph; ty++) {
- for (int tx = 0; tx < pw; tx++) {
- for (int dy = 0; dy < merge_ratio; dy++) {
- for (int dx = 0; dx < merge_ratio; dx++) {
- auto remap = idx[ptr / mpow];
- remap = (remap * mpow) + (ptr % mpow);
-
- const int y = ty * merge_ratio + dy;
- const int x = tx * merge_ratio + dx;
- positions[ remap] = y;
- positions[ num_patches + remap] = x;
- positions[2 * num_patches + remap] = y;
- positions[3 * num_patches + remap] = x;
- ptr++;
- }
- }
- }
- }
-
- set_input_i32("positions", positions);
+ // both X and Y are downscaled by the scale factor
+ int scale_factor = hparams.n_merge;
+ n_patches /= (scale_factor * scale_factor);
} break;
- case PROJECTOR_TYPE_MIMOVL:
+ case PROJECTOR_TYPE_GEMMA3NV:
{
- const int merge = hparams.n_merge; // 2
- const int merge_unit = merge * merge; // 4
- const int patch = hparams.patch_size; // 16
- const int H = image_size_height / patch;
- const int W = image_size_width / patch;
- const int n_pos_full = H * W;
- const int llm_h = H / merge;
- const int llm_w = W / merge;
- const int n_units = llm_h * llm_w; // n_pos / merge_unit
-
- // Row-major merge-tile-ordered (h, w) positions
- std::vector<int32_t> pos_h_row(n_pos_full);
- std::vector<int32_t> pos_w_row(n_pos_full);
- {
- int idx = 0;
- for (int ty = 0; ty < llm_h; ty++) {
- for (int tx = 0; tx < llm_w; tx++) {
- for (int dy = 0; dy < merge; dy++) {
- for (int dx = 0; dx < merge; dx++) {
- pos_h_row[idx] = ty * merge + dy;
- pos_w_row[idx] = tx * merge + dx;
- idx++;
- }
- }
- }
- }
- }
-
- // Col-major merge-unit permutation
- std::vector<float> idx_col(n_units);
- for (int r = 0; r < llm_h; r++) {
- for (int c = 0; c < llm_w; c++) {
- int u_row = r * llm_w + c;
- int u_col = c * llm_h + r;
- idx_col[u_col] = (float) u_row;
- }
- }
-
- // Col-mode positions: permute pos_*_row by idx_col
- std::vector<int32_t> pos_h_col(n_pos_full);
- std::vector<int32_t> pos_w_col(n_pos_full);
- for (int u = 0; u < n_units; u++) {
- int src = (int) idx_col[u];
- for (int k = 0; k < merge_unit; k++) {
- pos_h_col[u * merge_unit + k] = pos_h_row[src * merge_unit + k];
- pos_w_col[u * merge_unit + k] = pos_w_row[src * merge_unit + k];
- }
- }
-
- // Pack into ggml_rope_multi VISION-mode layout. The non-CPU kernels
- // only read slots 0 and 1, so pack h in slot 0, w in slot 1:
- // positions[0..n_pos) = h
- // positions[n_pos..2*n_pos) = w
- // positions[2*n_pos..3*n_pos) = 0
- // positions[3*n_pos..4*n_pos) = 0
- std::vector<int32_t> positions_row(static_cast<size_t>(n_pos_full) * 4, 0);
- std::vector<int32_t> positions_col(static_cast<size_t>(n_pos_full) * 4, 0);
- for (int i = 0; i < n_pos_full; i++) {
- positions_row[0 * n_pos_full + i] = pos_h_row[i];
- positions_row[1 * n_pos_full + i] = pos_w_row[i];
- positions_col[0 * n_pos_full + i] = pos_h_col[i];
- positions_col[1 * n_pos_full + i] = pos_w_col[i];
- }
-
- // Banded 1D sliding-window mask
- const int window = hparams.attn_window_size;
- GGML_ASSERT(window > 0);
- std::vector<float> mask(static_cast<size_t>(n_pos_full) * n_pos_full, std::numeric_limits<float>::lowest());
- for (int q = 0; q < n_pos_full; q++) {
- int lo = std::max(0, q - window);
- int hi = std::min(n_pos_full - 1, q + window);
- for (int k = lo; k <= hi; k++) {
- mask[static_cast<size_t>(q) * n_pos_full + k] = 0.0f;
- }
- }
-
- set_input_i32("mimovl_positions_row", positions_row);
- set_input_i32("mimovl_positions_col", positions_col);
- set_input_f32("mimovl_idx_col", idx_col);
- set_input_attn_mask(mask);
+ // MobileNetV5 MSFA adapter always outputs fixed 16x16 resolution
+ // regardless of input size (see architecture description)
+ n_patches = hparams.image_size / hparams.patch_size;
} break;
- case PROJECTOR_TYPE_PIXTRAL:
+ case PROJECTOR_TYPE_LFM2:
+ case PROJECTOR_TYPE_D1OMNI_V:
case PROJECTOR_TYPE_KIMIVL:
case PROJECTOR_TYPE_KIMIK25:
- case PROJECTOR_TYPE_LIGHTONOCR:
{
- // set the 2D positions
- int n_patches_per_col = image_size_width / patch_size;
- std::vector<int> pos_data(n_pos);
- // dimension H
- for (int i = 0; i < n_pos; i++) {
- pos_data[i] = i / n_patches_per_col;
- }
- set_input_i32("pos_h", pos_data);
- // dimension W
- for (int i = 0; i < n_pos; i++) {
- pos_data[i] = i % n_patches_per_col;
- }
- set_input_i32("pos_w", pos_data);
- } break;
- case PROJECTOR_TYPE_SENSENOVA_U1:
- {
- const int n_patches_per_row = image_size_width / patch_size;
- std::vector<int> pos_data(n_pos);
- for (int i = 0; i < n_pos; i++) {
- pos_data[i] = i / n_patches_per_row;
- }
- set_input_i32("pos_y", pos_data);
- for (int i = 0; i < n_pos; i++) {
- pos_data[i] = i % n_patches_per_row;
- }
- set_input_i32("pos_x", pos_data);
+ // dynamic size
+ int out_patch_size = params.patch_size * hparams.n_merge;
+ int x_patch = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size;
+ int y_patch = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size;
+ n_patches = x_patch * y_patch;
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
- // set the 2D positions (mrope layout, only the first 2 channels are used)
- int n_patches_per_row = image_size_width / patch_size;
- std::vector<int32_t> positions(n_pos * 4, 0);
- for (int i = 0; i < n_pos; i++) {
- positions[i] = i / n_patches_per_row; // row
- positions[n_pos + i] = i % n_patches_per_row; // col
- }
- set_input_i32("positions", positions);
-
- // token block layout index (see clip_graph_deepseek4v::build)
- // rows [0, n_grid) are the aligner output, the sentinels follow
- const int n_merge = hparams.n_merge;
- const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge;
- const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge;
- const int n_grid = n_llm_w * n_llm_h;
- const int idx_start = n_grid;
- const int idx_end = n_grid + 1;
- const int idx_newline = n_grid + 2;
- const int idx_pad = n_grid + 3;
-
- const int lead_pad = imgs.entries[0].lead_pad;
- const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad);
-
- std::vector<int32_t> idx;
- idx.reserve(bl.n_out);
- for (int i = 0; i < lead_pad; i++) {
- idx.push_back(idx_pad);
- }
- idx.push_back(idx_start);
- // pairs of adjacent rows are interleaved column-wise ("N-layout")
- // ref: build_image_block in inference/image_processor.py
- for (int t = 0; t < bl.rows * bl.row_len; t++) {
- const int g = t / (2 * bl.row_len);
- const int rem = t % (2 * bl.row_len);
- const int c = rem / 2; // column
- const int r = 2 * g + rem % 2; // row
- if (r >= n_llm_h) {
- idx.push_back(idx_pad);
- } else if (c == n_llm_w) {
- idx.push_back(idx_newline);
- } else {
- idx.push_back(r * n_llm_w + c);
- }
- }
- for (int i = 0; i < bl.pad_last; i++) {
- idx.push_back(idx_pad);
- }
- idx.push_back(idx_end);
- set_input_i32("layout_idx", idx);
+ const int out_patch_size = params.patch_size * params.n_merge;
+ const int n_llm_w = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size;
+ const int n_llm_h = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size;
+ n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out;
} break;
case PROJECTOR_TYPE_DEEPSEEK41V:
{
- // mrope layout, only the first 2 channels are used
- const int n_patches_per_row = image_size_width / patch_size;
- std::vector<int32_t> positions(n_pos * 4, 0);
- for (int i = 0; i < n_pos; i++) {
- positions[i] = i / n_patches_per_row;
- positions[n_pos + i] = i % n_patches_per_row;
- }
- set_input_i32("positions", positions);
-
- // reading-order block: rows [0, n_grid) are the aligner output, then start / end / newline
- const int n_merge = hparams.n_merge;
- const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge;
- const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge;
- const int n_grid = n_llm_w * n_llm_h;
- std::vector<int32_t> idx;
- idx.reserve(dsv41_n_block_tokens(n_llm_w, n_llm_h));
- idx.push_back(n_grid); // IMAGE_START
- for (int r = 0; r < n_llm_h; r++) {
- for (int c = 0; c < n_llm_w; c++) {
- idx.push_back(r * n_llm_w + c);
- }
- idx.push_back(n_grid + 2); // IMAGE_NEW_LINE
- }
- idx.push_back(n_grid + 1); // IMAGE_END
- set_input_i32("layout_idx", idx);
+ const int out_patch_size = params.patch_size * params.n_merge;
+ const int n_llm_w = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size;
+ const int n_llm_h = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size;
+ n_patches = dsv41_n_block_tokens(n_llm_w, n_llm_h);
} break;
- case PROJECTOR_TYPE_GLM_EDGE:
- {
- // llava and other models
- std::vector<int32_t> positions(n_pos);
- for (int i = 0; i < n_pos; i++) {
- positions[i] = i;
- }
- set_input_i32("positions", positions);
- } break;
- case PROJECTOR_TYPE_MLP:
- case PROJECTOR_TYPE_MLP_NORM:
- case PROJECTOR_TYPE_LDP:
- case PROJECTOR_TYPE_LDPV2:
+ case PROJECTOR_TYPE_PADDLEOCR:
+ case PROJECTOR_TYPE_DOTS_OCR:
+ case PROJECTOR_TYPE_DOTS3NOTE_V:
{
- // llava and other models
- std::vector<int32_t> positions(n_pos);
- for (int i = 0; i < n_pos; i++) {
- positions[i] = i;
- }
- set_input_i32("positions", positions);
-
- // The patches vector is used to get rows to index into the embeds with;
- // we should skip dim 0 only if we have CLS to avoid going out of bounds
- // when retrieving the rows.
- int patch_offset = model.class_embedding ? 1 : 0;
- std::vector<int32_t> patches(num_patches);
- for (int i = 0; i < num_patches; i++) {
- patches[i] = i + patch_offset;
- }
- set_input_i32("patches", patches);
+ // dynamic size
+ int n_merge = hparams.n_merge;
+ int stride = n_merge * n_merge;
+ n_patches = MTMD_ALIGN(n_patches, stride) / stride;
} break;
- case PROJECTOR_TYPE_POCKETTTS_SPKENC:
+ case PROJECTOR_TYPE_DOTS3NOTE_A:
{
- set_pockettts_tfm_inputs();
+ // 3x stride-2 conv2d over mel frames
+ n_patches = (img->nx() + 7) / 8;
} break;
- case PROJECTOR_TYPE_POCKETTTS_GEN:
+ case PROJECTOR_TYPE_PIXTRAL:
+ case PROJECTOR_TYPE_LIGHTONOCR:
{
- if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
- GGML_ASSERT(params->feats != nullptr);
- set_input_f32("inp_feats", *params->feats);
- // positions and mask are derived in-graph from the persisted counter
- set_gen_state_in();
+ // dynamic size
+ int n_merge = hparams.n_merge;
+ int n_patches_x = img->nx() / patch_size / n_merge;
+ int n_patches_y = img->ny() / patch_size / n_merge;
+ if (token_embd_img_break) {
+ n_patches = n_patches_y * n_patches_x + n_patches_y - 1; // + one [IMG_BREAK] per row, except the last row
} else {
- // flow matching starts from gaussian noise, std = sqrt(temp)
- ggml_tensor * t = get_inp_tensor("inp_noise");
- // Config.default_temperature, for a caller that does not set one
- const float temp = params->temp > 0.0f ? params->temp : 0.7f;
- std::normal_distribution<float> dist(0.0f, std::sqrt(temp));
- std::vector<float> noise(ggml_nelements(t));
- for (auto & v : noise) {
- v = dist(params->gen_state->rng);
- }
- set_input_f32("inp_noise", noise);
+ n_patches = n_patches_y * n_patches_x;
}
} break;
- case PROJECTOR_TYPE_SOPRANO:
+ case PROJECTOR_TYPE_VOXTRAL_RT:
{
- set_input_f32("inp_feats", *params->feats);
+ // 2 context frames, conv stride 2, then stacked frames
+ n_patches = (img->nx() - 2) / 2 / hparams.proj_stack_factor;
} break;
- case PROJECTOR_TYPE_GEMMA4V:
- case PROJECTOR_TYPE_GEMMA4UV:
+ case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
{
- // set (col, row) patch positions for learned positional embedding
- const int n_cols = image_size_width / patch_size;
- std::vector<int> pos_x(num_patches), pos_y(num_patches);
- for (int i = 0; i < num_patches; i++) {
- pos_x[i] = i % n_cols;
- pos_y[i] = i / n_cols;
- }
- set_input_i32("pos_x", pos_x);
- set_input_i32("pos_y", pos_y);
+ const int64_t fold = hparams.audio_samples_per_token();
+ n_patches = (int) ((img->nx() + fold - 1) / fold);
} break;
- case PROJECTOR_TYPE_DEEPSEEKOCR:
- case PROJECTOR_TYPE_DEEPSEEKOCR2:
+ case PROJECTOR_TYPE_VOXTRAL:
+ case PROJECTOR_TYPE_ULTRAVOX:
+ case PROJECTOR_TYPE_QWEN2A:
+ case PROJECTOR_TYPE_MERALION:
+ case PROJECTOR_TYPE_MUSIC_FLAMINGO:
{
- GGML_ASSERT(
- (pos_w == pos_h) // overview image
- || (pos_h >= pos_w && pos_h % pos_w == 0) // tile images
- );
-
- const int window = hparams.attn_window_size;
- const int pos = pos_w;
- std::vector<int32_t> rel_pos_indices_local(window * window);
- std::vector<int32_t> rel_pos_indices_global(pos * pos);
-
- for (int q = 0; q < window; q++) {
- for (int k = 0; k < window; k++) {
- rel_pos_indices_local[q * window + k] = q - k + window - 1;
- }
- }
+ n_patches = img->nx();
- for (int q = 0; q < pos; q++) {
- for (int k = 0; k < pos; k++) {
- rel_pos_indices_global[q * pos + k] = q - k + pos - 1;
- }
+ const int proj_stack_factor = hparams.proj_stack_factor;
+ if (audio_has_stack_frames()) {
+ GGML_ASSERT(proj_stack_factor > 0);
+ const int n_len = MTMD_ALIGN(n_patches, proj_stack_factor);
+ n_patches = n_len / proj_stack_factor;
}
- set_input_i32("rel_pos_indices_local", rel_pos_indices_local);
- set_input_i32("rel_pos_indices_global", rel_pos_indices_global);
-
- if (ctx->proj_type() == PROJECTOR_TYPE_DEEPSEEKOCR2) {
-
- // qwen2 encoder attention mask
-
- // num_image_tokens = num_patches / 16
- // 256 for 1024 global view
- // 144 for 768 tile views
- const int num_image_tokens = num_patches / 16;
- const int seq_len = num_image_tokens * 2;
- std::vector qwen2_mask(static_cast<size_t>(seq_len) * seq_len, 0.0f);
-
- // attention mask layout
- // +--------------+---------------+
- // | all 0 | all -inf |
- // +--------------+---------------+
- // | all 0 | lower tri 0 |
- // +--------------+---------------+
- for (int i = 0; i < seq_len; i++) {
- for (int j = 0; j < seq_len; j++) {
- const bool zero = i < num_image_tokens ?
- j < num_image_tokens :
- j < num_image_tokens || j <= i;
- qwen2_mask[static_cast<size_t>(i) * seq_len + j] = zero ? 0.0f : -1e9f;
- }
- }
- set_input_attn_mask(qwen2_mask);
- }
- } break;
- case PROJECTOR_TYPE_CHATTERBOX:
- {
- if (params->gen_process == CLIP_GEN_PROCESS_TTS_VOCODE) {
- set_input_f32("inp_mel", *params->mel_in);
- set_input_f32("inp_sstft", *params->sstft_in);
- break;
- }
- if (params->gen_process == CLIP_GEN_PROCESS_TOKENIZE) {
- const int T1 = (imgs.entries[0].nx() - 1) / 2 + 1;
- const int T2 = (T1 - 1) / 2 + 1;
- std::vector<int32_t> pos(T2);
- for (int i = 0; i < T2; i++) {
- pos[(size_t) i] = i;
- }
- set_input_i32("inp_pos", pos);
- break;
- }
- if (params->gen_process != CLIP_GEN_PROCESS_TTS) {
- break;
- }
- const int n_gen = (int) params->codes->size();
- GGML_ASSERT(params->ref);
- const int n_prompt = (int) params->ref->tokens.size();
- std::vector<int32_t> tokens = params->ref->tokens;
- tokens.resize((size_t) n_prompt + n_gen);
- memcpy(tokens.data() + n_prompt, params->codes->data(), (size_t) n_gen * sizeof(int32_t));
- const int64_t n_codes = model.cbx.input_embedding_w->ne[1];
- for (int32_t t : tokens) {
- if (t < 0 || t >= n_codes) {
- LOG_ERR("%s: speech token out of range (%d, codebook size %lld)\n", __func__, t, (long long) n_codes);
- return false;
- }
- }
- const int T1 = n_prompt + n_gen;
- const int T2 = 2 * T1;
- set_input_i32("inp_tokens", tokens);
-
- // mel-rate reference conditioning of the voice
- set_input_f32("inp_prompt_feat", params->ref->feat);
- set_input_f32("inp_xvec", params->ref->spk);
-
- // espnet relative positional encoding, entry k holds the sinusoid of relative position (T-1) - k
- auto fill_pos = [&](const char * name, int T) {
- const int d = 512;
- std::vector<float> pos((size_t) (2 * T - 1) * d);
- for (int k = 0; k < 2 * T - 1; k++) {
- const double rel = (double) (T - 1 - k);
- for (int i = 0; i < d / 2; i++) {
- const double div = exp(-(double) (2 * i) * log(10000.0) / d);
- pos[(size_t) k * d + 2 * i ] = (float) sin(rel * div);
- pos[(size_t) k * d + 2 * i + 1] = (float) cos(rel * div);
- }
- }
- set_input_f32(name, pos);
- };
- fill_pos("inp_pos1", T1);
- fill_pos("inp_pos2", T2);
-
- // noise and the time embeddings of the solver span points, matcha layout (sines then cosines, scale 1000)
- std::vector<float> noise((size_t) 80 * T2);
- std::normal_distribution<float> nd(0.0f, 1.0f);
- for (auto & f : noise) f = nd(params->gen_state->rng);
- set_input_f32("inp_noise", noise);
-
- const bool meanflow = model.cbx.time_embed_mixer_w != nullptr;
- const int n_steps = meanflow ? 2 : 10;
- std::vector<float> temb((size_t) 320 * (n_steps + 1));
- for (int s = 0; s <= n_steps; s++) {
- const double u = (double) s / n_steps;
- const double t = meanflow ? u : 1.0 - cos(u * M_PI / 2.0);
- for (int i = 0; i < 160; i++) {
- const double div = exp(-(double) i * log(10000.0) / 159.0);
- temb[(size_t) s * 320 + i ] = (float) sin(1000.0 * t * div);
- temb[(size_t) s * 320 + i + 160] = (float) cos(1000.0 * t * div);
- }
+ // whisper downscales input token by half after conv1d
+ n_patches /= 2;
+
+ if (audio_has_avgpool()) {
+ // divide by 2 because of nn.AvgPool1d(2, stride=2)
+ n_patches /= 2;
}
- set_input_f32("inp_temb", temb);
} break;
- case PROJECTOR_TYPE_GEMMA3:
- case PROJECTOR_TYPE_GEMMA3NV:
- case PROJECTOR_TYPE_IDEFICS3:
- case PROJECTOR_TYPE_COHERE2V:
- case PROJECTOR_TYPE_INTERNVL:
- case PROJECTOR_TYPE_NEMOTRON_V2_VL:
- case PROJECTOR_TYPE_NEMOTRON_OMNI:
- case PROJECTOR_TYPE_QWEN2A:
case PROJECTOR_TYPE_QWEN3A:
- case PROJECTOR_TYPE_GLMA:
- case PROJECTOR_TYPE_ULTRAVOX:
- case PROJECTOR_TYPE_LFM2:
- case PROJECTOR_TYPE_D1OMNI_V:
- case PROJECTOR_TYPE_VOXTRAL:
- case PROJECTOR_TYPE_MERALION:
- case PROJECTOR_TYPE_MUSIC_FLAMINGO:
- case PROJECTOR_TYPE_JANUS_PRO:
- case PROJECTOR_TYPE_PHI4:
- case PROJECTOR_TYPE_COGVLM:
- case PROJECTOR_TYPE_YASA2:
- case PROJECTOR_TYPE_GEMMA4UA:
- case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
- case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
- {
- // do nothing
- } break;
- case PROJECTOR_TYPE_CHATTERBOX_SPKENC:
{
- // the ceil-mode seg pooling divides by the full segment length, rescale the last partial segment
- const int T1 = (imgs.entries[0].nx() - 1) / 2 + 1;
- const int S = (T1 + 99) / 100;
- std::vector<float> segfix((size_t) S, 1.0f);
- const int last = T1 - (S - 1) * 100;
- segfix[(size_t) S - 1] = 100.0f / (float) last;
- set_input_f32("inp_segfix", segfix);
+ // chunk_size=100 frames --> 3x stride-2 conv2d --> 13 tokens per chunk
+ const int chunk_size = 100;
+ const int tokens_per_chunk = 13;
+ n_patches = (img->nx() / chunk_size) * tokens_per_chunk;
} break;
- case PROJECTOR_TYPE_LFM3A_GEN:
+ case PROJECTOR_TYPE_GLMA:
{
- const int64_t n_cb = model.gen_code_embd_w->ne[2];
- const int64_t n_vocab = model.gen_code_embd_w->ne[1];
- if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
- GGML_ASSERT(params->codes != nullptr);
- // codes index the FSQ table via ggml_get_rows, the end-of-audio code has no waveform
- const int64_t n_codes = model.c2w.quant_first_cb_w->ne[1];
- for (int32_t c : *params->codes) {
- if (c < 0 || c >= n_codes) {
- LOG_ERR("%s: code out of range (%d, codebook size %lld)\n", __func__, c, (long long) n_codes);
- return false;
- }
- }
- std::vector<int32_t> codes = *params->codes;
- set_input_i32("inp_codes", codes);
- set_gen_state_in();
- // the "pos" slot leads the state
- float base = 0.0f;
- ggml_backend_tensor_get(params->gen_state->slots[0], &base, 0, sizeof(float));
- set_lfm3a_window_inputs((int) base, (int) ggml_nelements(get_inp_tensor("positions")), hparams.wav_tfm_swa);
- } else {
- // rows of the merged per-codebook tables; a masked codebook (-1) takes row 0, the mask drops it
- std::vector<int32_t> codes(n_cb, -1);
- if (params->codes) {
- GGML_ASSERT(params->codes->size() == (size_t) n_cb);
- codes = *params->codes;
- }
- std::vector<float> mask(n_cb, 0.0f);
- for (int64_t i = 0; i < n_cb; i++) {
- if (codes[i] >= n_vocab) {
- LOG_ERR("%s: code out of range (%d, vocab %lld)\n", __func__, codes[i], (long long) n_vocab);
- return false;
- }
- if (codes[i] < 0) {
- codes[i] = 0;
- } else {
- mask[i] = 1.0f;
- }
- codes[i] += (int32_t) (i * n_vocab);
- }
- set_input_i32("inp_codes", codes);
- if (ggml_graph_get_tensor(gf, "inp_mask") != nullptr) {
- set_input_f32("inp_mask", mask);
- }
- }
+ n_patches = img->nx();
+ // whisper downscales input token by half after conv1d
+ n_patches /= 2;
+ // reshape by merge_factor
+ n_patches /= hparams.proj_stack_factor;
+ // for BOI and EOI token embeddings
+ n_patches += 2;
} break;
- case PROJECTOR_TYPE_DRAMA3_GEN:
+ case PROJECTOR_TYPE_COGVLM:
{
- const int64_t n_cb = model.gen_code_embd_w->ne[2];
- const int64_t n_vocab = model.gen_code_embd_w->ne[1];
- if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
- // frame-major to group-major, clamp: codec residual codebooks are smaller than the tower vocab
- GGML_ASSERT(params->codes != nullptr && params->codes->size() % n_cb == 0);
- const int64_t n_frames = (int64_t) params->codes->size() / n_cb;
- const int64_t n_first = model.c2w.quant_first_cb_w->ne[1];
- const int64_t n_rest = model.c2w.quant_rest_cb_w->ne[1];
- std::vector<int32_t> codes(params->codes->size());
- for (int64_t f = 0; f < n_frames; f++) {
- for (int64_t g = 0; g < n_cb; g++) {
- const int64_t n = g == 0 ? n_first : n_rest;
- codes[g * n_frames + f] = std::clamp<int32_t>((*params->codes)[f * n_cb + g], 0, (int32_t) n - 1);
- }
- }
- set_input_i32("inp_codes", codes);
- set_gen_state_in();
- } else {
- // code0 < 0: next row is text (prompt), the tower only fills its KV
- // codes: input frame codes for a speech row, empty for a text row
- const bool next_speech = params->code0 >= 0;
- const bool cur_speech = params->codes && (int64_t) params->codes->size() == n_cb;
- if (params->code0 >= n_vocab) {
- LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) n_vocab);
- return false;
- }
- std::vector<int32_t> code0 = { std::max(params->code0, 0) };
- set_input_i32("inp_code0", code0);
-
- std::vector<int32_t> extra(n_cb - 2);
- for (int64_t g = 2; g < n_cb; g++) {
- const int32_t c = cur_speech ? std::clamp<int32_t>((*params->codes)[g], 0, (int32_t) n_vocab - 1) : 0;
- extra[g - 2] = c + (int32_t) (g * n_vocab);
- }
- set_input_i32("inp_extra", extra);
- set_input_f32("inp_scale", { next_speech ? 1.0f : 0.0f, cur_speech ? 1.0f : 0.0f });
-
- float pos = 0.0f;
- ggml_backend_tensor_get(params->gen_state->slots[0], &pos, 0, sizeof(float));
- std::vector<int32_t> inp_pos = { (int32_t) pos };
- set_input_i32("inp_pos", inp_pos);
-
- std::uniform_real_distribution<float> dist(0.0f, 1.0f);
- std::vector<float> r(model.gen_code_head_w->ne[2]);
- for (auto & v : r) {
- v = dist(params->gen_state->rng);
- }
- set_input_f32("inp_rand", r);
- }
+ n_patches += 2; // for BOI and EOI token embeddings
} break;
- case PROJECTOR_TYPE_QWEN3TTS_GEN:
- {
- if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
- GGML_ASSERT(params->codes != nullptr);
-
- // frame-major input to group-major, rear-padded with code 0 up to one window
- const int64_t n_codes = model.gen_code_head_w->ne[2] + 1;
- const int64_t n_frames_w = hparams.wav_tfm_swa;
- const int64_t n_frames = (int64_t) params->codes->size() / n_codes;
- GGML_ASSERT(n_frames > 0 && n_frames <= n_frames_w);
-
- // codes are used as ggml_get_rows indices, so check them against the codebook vocab
- const int64_t vocab_first = model.c2w.quant_first_cb_w->ne[1];
- const int64_t vocab_rest = model.c2w.quant_rest_cb_w->ne[1];
- for (int64_t f = 0; f < n_frames; f++) {
- for (int64_t g = 0; g < n_codes; g++) {
- const int32_t c = (*params->codes)[f * n_codes + g];
- const int64_t vocab = (g == 0) ? vocab_first : vocab_rest;
- if (c < 0 || (int64_t) c >= vocab) {
- LOG_ERR("%s: code out of range (frame %lld, group %lld, code %d, vocab %lld)\n",
- __func__, (long long) f, (long long) g, c, (long long) vocab);
- return false;
- }
- }
- }
+ case PROJECTOR_TYPE_DEEPSEEKOCR:
+ {
+ // SAM encoder applies two stride-2 convolutions (net_2 and net_3)
+ // that reduce spatial dimensions by 4x in each direction (16x total)
+ // E.g., 64x64 -> 16x16 patches
+ n_patches /= 16;
- std::vector<int32_t> codes(n_frames_w * n_codes, 0);
- for (int64_t f = 0; f < n_frames; f++) {
- for (int64_t g = 0; g < n_codes; g++) {
- codes[g * n_frames_w + f] = (*params->codes)[f * n_codes + g];
- }
- }
- set_input_i32("inp_codes", codes);
- set_gen_state_in();
- } else {
- // code0 indexes gen_code_out_embd_w via ggml_get_rows; bound it
- const int64_t vocab0 = model.gen_code_out_embd_w->ne[1];
- if (params->code0 < 0 || (int64_t) params->code0 >= vocab0) {
- LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) vocab0);
- return false;
- }
- std::vector<int32_t> code0 = { params->code0 };
- set_input_i32("inp_code0", code0);
-
- // one uniform(0,1) draw per codebook, used by do_sampling()
- std::uniform_real_distribution<float> dist(0.0f, 1.0f);
- const int64_t n_acoustic = model.gen_code_head_w->ne[2];
- for (int64_t g = 0; g < n_acoustic; g++) {
- std::vector<float> r = { dist(params->gen_state->rng) };
- set_input_f32(("inp_rand_" + std::to_string(g)).c_str(), r);
- }
- }
- } break;
+ if (img->add_viewsep) {
+ // global view: one image-newline per token-row + trailing view separator
+ const int h = static_cast<int>(std::sqrt(static_cast<float>(n_patches)));
+ n_patches = h * (h + 1) + 1;
+ } else if (img->ny() >= img->nx() && img->ny() % img->nx() == 0) {
+ // tile row: one image-newline per token-row
+ const int grid_w = img->ny() / img->nx();
+ const int tile_patches = img->nx() / (patch_size * 4); // patches per tile side (SAM divides by 4)
+ const int h = tile_patches;
+ n_patches = (tile_patches * grid_w + 1) * h;
+ }
+ } break;
case PROJECTOR_TYPE_HUNYUANVL:
{
- // Compute the HunyuanVL 2D position embedding on CPU (with the
- // custom sf=(target+0.1)/n_grid bilinear sampling that the
- // reference implementation uses) and upload it to the graph
- // input declared in clip_graph_hunyuanvl::build().
- GGML_ASSERT(model.position_embeddings != nullptr);
- ggml_tensor * src_t = model.position_embeddings;
- const int64_t n_embd = src_t->ne[0];
- const int64_t n_pos = src_t->ne[1]; // = n_grid * n_grid
- const int n_grid = (int)std::lround(std::sqrt((double)n_pos));
- GGML_ASSERT((int64_t)n_grid * n_grid == n_pos);
- const int out_w = pos_w; // pw
- const int out_h = pos_h; // ph
-
- // Pull weight to host.
- std::vector<float> src(n_embd * n_pos);
- ggml_backend_tensor_get(src_t, src.data(), 0, ggml_nbytes(src_t));
-
- // Output layout matches ggml_new_tensor_2d(F32, n_embd, out_h*out_w):
- // ne[0] = n_embd (fastest), ne[1] = out_h*out_w
- // dst[(y*out_w + x) * n_embd + c]
- std::vector<float> dst((size_t)n_embd * out_h * out_w);
-
- const float sx = (float)(out_w + 0.1f) / (float)n_grid;
- const float sy = (float)(out_h + 0.1f) / (float)n_grid;
-
- for (int y = 0; y < out_h; ++y) {
- // Match ggml_compute_forward_upscale_f32 pixel-center
- // convention (align_corners=False): src_y = (y+0.5)/sy - 0.5.
- const float fy = ((float)y + 0.5f) / sy - 0.5f;
- int y0 = (int)std::floor(fy);
- int y1 = y0 + 1;
- y0 = std::clamp(y0, 0, n_grid - 1);
- y1 = std::clamp(y1, 0, n_grid - 1);
- float wy1 = std::clamp(fy - (float)y0, 0.0f, 1.0f);
- const float wy0 = 1.0f - wy1;
- for (int x = 0; x < out_w; ++x) {
- const float fx = ((float)x + 0.5f) / sx - 0.5f;
- int x0 = (int)std::floor(fx);
- int x1 = x0 + 1;
- x0 = std::clamp(x0, 0, n_grid - 1);
- x1 = std::clamp(x1, 0, n_grid - 1);
- float wx1 = std::clamp(fx - (float)x0, 0.0f, 1.0f);
- const float wx0 = 1.0f - wx1;
-
- const float w00 = wy0 * wx0;
- const float w01 = wy0 * wx1;
- const float w10 = wy1 * wx0;
- const float w11 = wy1 * wx1;
-
- const float * s00 = &src[((size_t)y0 * n_grid + x0) * n_embd];
- const float * s01 = &src[((size_t)y0 * n_grid + x1) * n_embd];
- const float * s10 = &src[((size_t)y1 * n_grid + x0) * n_embd];
- const float * s11 = &src[((size_t)y1 * n_grid + x1) * n_embd];
- float * d = &dst[((size_t)y * out_w + x) * n_embd];
- for (int c = 0; c < n_embd; ++c) {
- d[c] = w00 * s00[c] + w01 * s01[c] + w10 * s10[c] + w11 * s11[c];
- }
- }
- }
-
- set_input_f32("hunyuanvl_pos_embd", dst);
+ int merge = hparams.n_merge;
+ int ow = (img->nx() / patch_size) / merge;
+ int oh = (img->ny() / patch_size) / merge;
+ n_patches = (ow + 1) * oh + 2;
+ } break;
+ case PROJECTOR_TYPE_DEEPSEEKOCR2:
+ {
+ // 1024 global view -> 256 query tokens + 1 view separator = 257;
+ // 768 local tile -> 144 query tokens, no separator.
+ n_patches /= 16;
+ if (img->add_viewsep) {
+ n_patches += 1; // view separator, appended only after the global view
+ }
+ } break;
+ case PROJECTOR_TYPE_LFM2A:
+ case PROJECTOR_TYPE_D1OMNI_A:
+ case PROJECTOR_TYPE_LFM3A:
+ {
+ n_patches = ((((img->nx() + 1) / 2) + 1) / 2 + 1) / 2;
} break;
- case PROJECTOR_TYPE_LLAMA4:
+ case PROJECTOR_TYPE_GEMMA4A:
{
- // set the 2D positions
- int n_patches_per_col = image_size_width / patch_size;
- std::vector<int> pos_data(num_patches + 1, 0); // +1 for the [CLS] token
- // last pos is always kept 0, it's for CLS
- // dimension H
- for (int i = 0; i < num_patches; i++) {
- pos_data[i] = (i / n_patches_per_col) + 1;
- }
- set_input_i32("pos_h", pos_data);
- // dimension W
- for (int i = 0; i < num_patches; i++) {
- pos_data[i] = (i % n_patches_per_col) + 1;
+ // Two Conv2D stride-2: O = floor((I + 2p - k) / s) + 1, p=1, k=3, s=2
+ // O = floor((I - 1) / 2) + 1
+ int n = img->nx();
+ for (int i = 0; i < 2; i++) {
+ n = (n - 1) / 2 + 1;
}
- set_input_i32("pos_w", pos_data);
+ n_patches = n;
} break;
- case PROJECTOR_TYPE_DOTS3NOTE_A:
+ case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
+ case PROJECTOR_TYPE_PARAKEET:
{
- GGML_ASSERT(imgs.entries.size() == 1);
- const int n_pos = (imgs.entries.front().nx() + 7) / 8; // 3x stride-2 conv2d
- std::vector<int32_t> positions(n_pos);
- for (int i = 0; i < n_pos; i++) {
- positions[i] = i;
- }
- set_input_i32("positions", positions);
+ n_patches = (img->nx() + (params.subsampling_factor - 1)) / params.subsampling_factor;
} break;
- case PROJECTOR_TYPE_GEMMA4A:
+ case PROJECTOR_TYPE_GEMMA4UA:
{
- GGML_ASSERT(imgs.entries.size() == 1);
- const auto & img0 = imgs.entries.front();
- // Compute n_pos matching SSCP output: two stride-2 convs
- int n_pos = img0.nx();
- for (int i = 0; i < 2; i++) { n_pos = (n_pos - 1) / 2 + 1; }
-
- // Chunked local attention: blocked causal mask and RPE
- const int chunk_size = 12;
- const int max_past = 12;
- const int context_size = chunk_size + max_past;
- const int num_blocks = (n_pos + chunk_size - 1) / chunk_size;
-
- // Blocked causal attention mask: [context_size, chunk_size, num_blocks]
- {
- std::vector<float> mask(context_size * chunk_size * num_blocks, -1e9f);
- for (int b = 0; b < num_blocks; b++) {
- for (int q = 0; q < chunk_size; q++) {
- int gq = b * chunk_size + q;
- for (int k = 0; k < context_size; k++) {
- int gk = b * chunk_size - max_past + k;
- if (gq < n_pos && gk >= 0 && gk < n_pos && gk <= gq && (gq - gk) < max_past) {
- mask[k + q * context_size + b * context_size * chunk_size] = 0.0f;
- }
- }
- }
- }
- set_input_f32("kq_mask", mask);
- }
-
- // Sinusoidal RPE: 13 positions [12, 11, ..., 0]
- {
- const int n_embd = ctx->model.hparams.n_embd;
- const int num_timescales = n_embd / 2;
- const float log_timescale_increment = logf(10000.0f) / std::max(num_timescales - 1, 1);
- const int rpe_len = max_past + 1;
- std::vector<float> pos_emb(n_embd * rpe_len, 0.0f);
- for (int p = 0; p < rpe_len; p++) {
- float position = (float)(max_past - p);
- for (int i = 0; i < num_timescales; i++) {
- float inv_ts = expf(-(float)i * log_timescale_increment);
- float scaled = position * inv_ts;
- pos_emb[p * n_embd + i] = sinf(scaled);
- pos_emb[p * n_embd + i + num_timescales] = cosf(scaled);
- }
- }
- set_input_f32("pos_emb", pos_emb);
- }
+ n_patches = img->nx(); // no downsampling: one token per raw waveform frame
} break;
case PROJECTOR_TYPE_MIMO_AUDIO:
{
- GGML_ASSERT(imgs.entries.size() == 1);
- const int n_frames = imgs.entries.front().nx();
- const int n_pos = (n_frames - 1) / 2 + 1; // matches conv1(s=1)+conv2(s=2) output length
-
- std::vector<int32_t> positions(n_pos);
- for (int i = 0; i < n_pos; i++) {
- positions[i] = i;
- }
- set_input_i32("mimo_audio_positions", positions);
-
- const int window = hparams.attn_window_size;
- GGML_ASSERT(window > 0);
-
- const float neg_inf = std::numeric_limits<float>::lowest();
- std::vector<float> full_mask((size_t) n_pos * n_pos);
- std::vector<float> window_mask((size_t) n_pos * n_pos);
- for (int q = 0; q < n_pos; q++) {
- for (int k = 0; k < n_pos; k++) {
- const bool causal_ok = k <= q;
- full_mask[(size_t) q * n_pos + k] = causal_ok ? 0.0f : neg_inf;
- window_mask[(size_t) q * n_pos + k] = (causal_ok && (q - k) <= window) ? 0.0f : neg_inf;
- }
- }
- set_input_attn_mask(full_mask, 0);
- set_input_attn_mask(window_mask, 1);
-
- // input_local_transformer: block-diagonal mask + in-group positions
- {
- const int n_pos_ds = (n_pos - 2) / 2 + 1; // matches downsample conv (k=2,s=2,p=0)
- const int group_size = hparams.audio_local_group_size;
- GGML_ASSERT(group_size > 0);
- const int n_groups = (n_pos_ds + group_size - 1) / group_size;
- const int n_padded = n_groups * group_size;
-
- std::vector<int32_t> local_positions(n_padded);
- for (int i = 0; i < n_padded; i++) {
- local_positions[i] = i % group_size;
- }
- set_input_i32("mimo_audio_local_positions", local_positions);
-
- std::vector<float> local_mask((size_t) n_padded * n_padded);
- for (int q = 0; q < n_padded; q++) {
- for (int k = 0; k < n_padded; k++) {
- const bool same_group = (q / group_size) == (k / group_size);
- local_mask[(size_t) q * n_padded + k] = same_group ? 0.0f : neg_inf;
- }
- }
- set_input_attn_mask(local_mask, 2);
- }
+ // conv1(s=1) + conv2(s=2) -> RVQ-encoder downsample conv(k=2,s=2)
+ int n = img->nx();
+ n = (n - 1) / 2 + 1; // conv1 + conv2
+ n = (n - 2) / 2 + 1; // downsample conv
+ const int group_size = params.audio_local_group_size;
+ n_patches = (n + group_size - 1) / group_size;
} break;
- case PROJECTOR_TYPE_LFM2A:
- case PROJECTOR_TYPE_D1OMNI_A:
+ case PROJECTOR_TYPE_GRANITE_SPEECH:
{
- GGML_ASSERT(imgs.entries.size() == 1);
- const auto n_frames = clip_n_output_tokens(ctx, &imgs.entries.front());
-
- auto d_model = 512;
- auto seq_len = n_frames * 2 - 1;
- std::vector<float> pos_emb(d_model*seq_len);
- std::vector<double> inv_freq(d_model / 2);
- for (size_t i = 0; i < inv_freq.size(); ++i) {
- inv_freq[i] = std::exp(-(std::log(10000.0) / (float)d_model) * (2.0f * (float)(i)));
- }
- for (int64_t pos = 0; pos < seq_len; ++pos) {
- for (size_t i = 0; i < inv_freq.size(); ++i) {
- const float ang = (n_frames - pos - 1) * inv_freq[i];
- pos_emb[pos*d_model + 2*i + 0] = sinf(ang); // even
- pos_emb[pos*d_model + 2*i + 1] = cosf(ang); // odd
- }
- }
- set_input_f32("pos_emb", pos_emb);
+ const int ws = hparams.audio_proj_window_size;
+ const int ds = hparams.audio_proj_downsample_rate;
+ n_patches = ((img->nx() + ws - 1) / ws) * (ws / ds);
} break;
- case PROJECTOR_TYPE_LFM3A:
+ case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
- GGML_ASSERT(imgs.entries.size() == 1);
- const int n_pos = clip_n_output_tokens(ctx, &imgs.entries.front());
-
- // the "pos" slot leads the state; without a stream the state is zero, the clip is encoded from its start
- int base = 0;
- if (params->enc_state != nullptr) {
- const auto & slots = params->enc_state->slots;
- float base_f = 0.0f;
- ggml_backend_tensor_get(slots[0], &base_f, 0, sizeof(float));
- base = (int) base_f;
- size_t i = 0;
- for (const auto & slot : list_lfm3a_state_slots(hparams, model)) {
- ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str()));
- }
- } else {
- for (const auto & slot : list_lfm3a_state_slots(hparams, model)) {
- ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str());
- ggml_backend_tensor_memset(t, 0, 0, ggml_nbytes(t));
- }
- }
-
- set_lfm3a_window_inputs(base, n_pos, hparams.attn_window_size);
+ // pooling gives one speaker embedding, whatever the clip length is
+ n_patches = 1;
} break;
- case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
+ case PROJECTOR_TYPE_CHATTERBOX_SPKENC:
{
- GGML_ASSERT(imgs.entries.size() == 1);
- std::vector<int32_t> positions(ggml_nelements(get_inp_tensor("positions")));
- for (size_t i = 0; i < positions.size(); i++) {
- positions[i] = (int32_t) i;
- }
- set_input_i32("positions", positions);
- GGML_ASSERT(params->enc_state != nullptr);
- set_input_f32("inp_norm", params->enc_state->mel_norm);
- if (ggml_graph_get_tensor(gf, "spk_prev") != nullptr) {
- set_input_f32("spk_prev", params->enc_state->spk_prev);
- }
+ // statistics pooling gives one speaker embedding, whatever the clip length is
+ n_patches = 1;
} break;
- case PROJECTOR_TYPE_PARAKEET:
+ case PROJECTOR_TYPE_QWEN3TTS_GEN:
+ case PROJECTOR_TYPE_LFM3A_GEN:
+ case PROJECTOR_TYPE_DRAMA3_GEN:
{
- GGML_ASSERT(imgs.entries.size() == 1);
- struct ggml_tensor * attn_mask = ggml_graph_get_tensor(gf, "attn_mask");
- const int n_q = attn_mask->ne[1];
- const int n_k = attn_mask->ne[0];
- const int n_frames = imgs.entries.front().nx();
- const int n_tokens_real = (n_frames + hparams.subsampling_factor-1) / hparams.subsampling_factor;
- const float mask_value = -1e30f;
-
- std::vector<float> mask_data(n_q * n_k);
- if (n_k == n_q) {
- // full attention: mask keys that are padding
- for (int q = 0; q < n_q; ++q) {
- for (int k = 0; k < n_k; ++k) {
- mask_data[q * n_k + k] = (k >= n_tokens_real) ? mask_value : 0.0f;
- }
- }
- } else {
- // local attention: mask keys outside the valid window
- const int att_left = n_k / 2;
- for (int q = 0; q < n_q; ++q) {
- for (int k = 0; k < n_k; ++k) {
- const int key = q - att_left + k;
- mask_data[q * n_k + k] = (key >= 0 && key < n_tokens_real) ? 0.0f : mask_value;
- }
- }
- }
- set_input_f32(attn_mask->name, mask_data);
-
- // local attention skew mask: zeroes out the probs that were
- // computed for keys outside the valid sliding window.
- if (struct ggml_tensor * local_mask = ggml_graph_get_tensor(gf, "local_mask")) {
- const int lm_k = local_mask->ne[0];
- const int lm_q = local_mask->ne[1];
- const int window_size = lm_k - lm_q + 1;
- std::vector<float> lm_data(lm_q * lm_k);
- for (int q = 0; q < lm_q; ++q) {
- for (int k = 0; k < lm_k; ++k) {
- const int rel = k - q;
- lm_data[q * lm_k + k] = (rel >= 0 && rel < window_size) ? 1.0f : 0.0f;
- }
- }
- set_input_f32(local_mask->name, lm_data);
- }
-
- // Generate rotation frequencies for relative positional encoding.
- {
- const int n_state = hparams.n_embd;
- const int d_half = n_state / 2;
- const float log_10000 = logf(10000.0f);
- std::vector<float> freqs(d_half);
- for (int k = 0; k < d_half; ++k) {
- freqs[k] = expf(-(float(k * 2) * log_10000 / float(n_state)));
- }
- set_input_f32("pos_freqs", freqs);
- }
-
- // Generate relative positional distance values which scaled by
- // the frequency to produce the angles for sin/cos.
- {
- // window_size is only known after graph construction since it depends on
- // n_time from the conv output, so we read it back from the graph tensor.
- struct ggml_tensor * rel_pos = ggml_graph_get_tensor(gf, "rel_positions");
- const int window_size = rel_pos->ne[1];
- std::vector<float> pos(window_size);
- // local attention: window is fixed at [att_left, att_right]
- // full attention: window covers the full sequence, centered
- if (ggml_graph_get_tensor(gf, "local_mask")) {
- const int att_left = window_size / 2;
- for (int t = 0; t < window_size; ++t) {
- pos[t] = float(att_left - t);
- }
- } else {
- const int n_time = (window_size + 1) / 2;
- for (int t = 0; t < window_size; ++t) {
- pos[t] = float(n_time - 1 - t);
- }
- }
- set_input_f32(rel_pos->name, pos);
- }
+ // one hidden-state vector fed back to the talker per call
+ n_patches = 1;
} break;
- case PROJECTOR_TYPE_GRANITE_SPEECH:
+ case PROJECTOR_TYPE_POCKETTTS_SPKENC:
{
- const int context_size = ctx->model.hparams.audio_chunk_size;
- const int max_pos_emb = ctx->model.hparams.audio_max_pos_emb;
-
- std::vector<int32_t> dists((size_t) context_size * (size_t) context_size);
- for (int i = 0; i < context_size; i++) {
- for (int j = 0; j < context_size; j++) {
- int d = i - j;
- if (d < -context_size) d = -context_size;
- if (d > context_size) d = context_size;
- dists[(size_t) i * (size_t) context_size + (size_t) j] = d + max_pos_emb;
- }
- }
- set_input_i32("attn_dists", dists);
-
- const int n_frames = image_size_width;
- const int remainder = n_frames % context_size;
- if (remainder > 0) {
- const int num_blocks = (n_frames + context_size - 1) / context_size;
- std::vector<float> mask((size_t) context_size * (size_t) context_size * (size_t) num_blocks, 0.0f);
- const float neg_inf = -INFINITY;
- const size_t last_block_offset = (size_t) (num_blocks - 1) * (size_t) context_size * (size_t) context_size;
- for (int q = 0; q < context_size; q++) {
- for (int k = 0; k < context_size; k++) {
- if (q >= remainder || k >= remainder) {
- mask[last_block_offset + (size_t) q * (size_t) context_size + (size_t) k] = neg_inf;
- }
- }
- }
- set_input_f32("attn_mask", mask);
- }
+ // one conditioning row per 12.5Hz frame
+ const int hop = hparams.mimi_downsample * 120;
+ n_patches = img->nx() / hop;
+ } break;
+ case PROJECTOR_TYPE_POCKETTTS_GEN:
+ {
+ // one latent per call for GEN_CODE, GEN_WAV sizes its input from the caller
+ n_patches = 1;
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
- // Granite Vision 4.1 uses precomputed permutation index
- // tensors to express the _win / _unwin / spatial sampling
- // reshapes as ggml_get_rows gathers. The names are set
- // by g4v_gather() in models/granite4-vision.cpp.
- const int patch_size = model.hparams.patch_size;
- const int image_side = imgs.entries.front().nx() / patch_size;
+ // Per-tile output token count: each projector block outputs
+ // query_side^2 tokens per window x n^2 windows.
+ // For 384x384 input: n = 24/8 = 3, query_side = 4 -> 144.
const int window_side = hparams.downsample_window_side;
const int query_side = hparams.downsample_query_side;
- const int n = image_side / window_side;
- const int new_side = n * query_side;
-
- // Builds the raster→window permutation indices for a
- // (side, side) grid split into (n × n) windows of (win × win)
- // tokens each. dst[w * win*win + p] = source raster index.
- auto make_win_idx = [](int side, int win) {
- const int nn = side / win;
- std::vector<int32_t> idx(static_cast<size_t>(side) * side);
- for (int wy = 0; wy < nn; ++wy) {
- for (int wx = 0; wx < nn; ++wx) {
- for (int iy = 0; iy < win; ++iy) {
- for (int ix = 0; ix < win; ++ix) {
- const int w = wy * nn + wx;
- const int p = iy * win + ix;
- const int y = wy * win + iy;
- const int x = wx * win + ix;
- idx[static_cast<size_t>(w) * (win*win) + p] = y * side + x;
- }
- }
- }
- }
- return idx;
- };
-
- auto make_unwin_idx = [&](int side, int win) {
- const std::vector<int32_t> fwd = make_win_idx(side, win);
- std::vector<int32_t> inv(fwd.size());
- for (size_t i = 0; i < fwd.size(); ++i) {
- inv[fwd[i]] = static_cast<int32_t>(i);
- }
- return inv;
- };
-
- auto make_spatial_idx = [](int side, int offset) {
- const int off_y = (offset >> 1) & 1;
- const int off_x = offset & 1;
- const int new_s = side / 2;
- std::vector<int32_t> idx(static_cast<size_t>(new_s) * new_s);
- for (int y = 0; y < new_s; ++y) {
- for (int x = 0; x < new_s; ++x) {
- idx[y * new_s + x] = (y * 2 + off_y) * side + (x * 2 + off_x);
- }
- }
- return idx;
- };
-
- // the same permutation is applied to every tile of the stacked image
- auto upload = [&](const std::string & name, const std::vector<int32_t> & idx) {
- ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str());
- GGML_ASSERT(t);
- GGML_ASSERT(ggml_nelements(t) % (int64_t) idx.size() == 0);
- const int n_rep = ggml_nelements(t) / idx.size();
- std::vector<int32_t> buf;
- buf.reserve(idx.size() * n_rep);
- for (int i = 0; i < n_rep; ++i) {
- buf.insert(buf.end(), idx.begin(), idx.end());
- }
- ggml_backend_tensor_set(t, buf.data(), 0, ggml_nbytes(t));
- };
-
- // Stage 1b only uses block 0's permutations; future stages
- // will upload all blocks.
- for (size_t bid = 0; bid < hparams.feature_layers.size(); ++bid) {
- const std::string prefix = "g4v_blk" + std::to_string(bid) + "_";
- upload(prefix + "win_idx", make_win_idx(image_side, window_side));
- upload(prefix + "qwin_idx", make_win_idx(new_side, query_side));
- upload(prefix + "unwin_idx", make_unwin_idx(new_side, query_side));
- const auto spatial_offset = hparams.proj_spatial_offsets[bid];
- if (spatial_offset >= 0) {
- upload(prefix + "spatial_idx", make_spatial_idx(image_side,spatial_offset));
- }
+ const int side = img->nx() / params.patch_size;
+ const int n = side / window_side;
+ const int out_side = query_side * n;
+ n_patches = out_side * out_side;
+ if (img->anyres.is_tiled()) {
+ // overview tile, then the unpadded tile grid with one newline per row
+ int off_x, off_y, w, h;
+ mtmd_anyres_unpad(img->anyres.grid_x * out_side, img->anyres.grid_y * out_side,
+ img->anyres.orig_nx, img->anyres.orig_ny, off_x, off_y, w, h);
+ n_patches += h * (w + 1);
}
} break;
default:
- GGML_ABORT("Unknown projector type");
- }
-
- clip_backend_cpu_set_n_threads(ctx->backend_cpu, params->n_threads);
-
- auto status = ggml_backend_sched_graph_compute(ctx->sched.get(), gf);
- if (status != GGML_STATUS_SUCCESS) {
- LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status);
- return false;
- }
-
- if (embeddings != nullptr) {
- // sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one)
- const int n_tokens_out = embeddings->ne[1];
- const int expected_n_tokens_out = clip_n_output_tokens(ctx, &imgs.entries[0]);
- if (n_tokens_out != expected_n_tokens_out) {
- LOG_ERR("%s: expected output %d tokens, got %d\n", __func__, expected_n_tokens_out, n_tokens_out);
- GGML_ABORT("Invalid number of output tokens");
- }
-
- LOG_DBG("%s: output embedding shape [%d, %d, %d]\n", __func__,
- (int)embeddings->ne[0], (int)embeddings->ne[1], (int)embeddings->ne[2]);
-
- // copy output to user buffer if provided
- // if output is empty, skip the copy
- auto & out_batch_embd = *params->out_embd;
- if (!out_batch_embd.empty()) {
- if (out_batch_embd.size() != (size_t)ggml_nelements(embeddings)) {
- LOG_ERR("%s: output buffer has %zu elements but expected %zu\n", __func__, out_batch_embd.size(), (size_t)ggml_nelements(embeddings));
- GGML_ABORT("Output buffer size mismatch");
- }
- ggml_backend_tensor_get(embeddings, out_batch_embd.data(), 0, ggml_nbytes(embeddings));
- } else {
- LOG_WRN("%s: output buffer is empty, skipping copy\n", __func__);
- }
- }
-
- //
- // for audio gen models
- //
-
- if (ctx->proj_type() == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == CLIP_GEN_PROCESS_TOKENIZE) {
- ggml_tensor * fsq = ggml_graph_get_tensor(gf, "out_fsq");
- GGML_ASSERT(fsq != nullptr && params->out_codes);
- const int n_tok = (int) fsq->ne[1];
- std::vector<float> h((size_t) 8 * n_tok);
- ggml_backend_tensor_get(fsq, h.data(), 0, ggml_nbytes(fsq));
-
- // fsq round to base 3: h in (-1, 1) maps to digits {0, 1, 2}
- params->out_codes->resize(n_tok);
- for (int t = 0; t < n_tok; t++) {
- int32_t code = 0;
- for (int i = 7; i >= 0; i--) {
- const int32_t d = (int32_t) roundf(h[(size_t) t * 8 + i] * 0.9990000128746033f) + 1;
- code = code * 3 + d;
- }
- (*params->out_codes)[(size_t) t] = code;
- }
-
- // speech table rows of the codes, the multilingual variant adds its learned positions as its perceiver expects
- if (params->out_code_embd) {
- const auto & tensors = ctx->model.cbx_tensors;
- auto tab_it = tensors.find("a.gen.code.out_embd.weight");
- GGML_ASSERT(tab_it != tensors.end());
- ggml_tensor * tab = tab_it->second;
- const int n_e = (int) tab->ne[0];
- auto pos_it = tensors.find("a.gen.t3.speech_pos_emb");
- ggml_tensor * pos = pos_it == tensors.end() ? nullptr : pos_it->second;
- if ((tab->type != GGML_TYPE_F16 && tab->type != GGML_TYPE_F32) ||
- (pos && (pos->type != GGML_TYPE_F32 || pos->ne[0] != n_e || pos->ne[1] < n_tok))) {
- LOG_ERR("%s: unexpected speech table or positions\n", __func__);
- return false;
- }
-
- params->out_code_embd->resize((size_t) n_tok * n_e);
- std::vector<ggml_fp16_t> h16(n_e);
- std::vector<float> pr(n_e);
- for (int t = 0; t < n_tok; t++) {
- float * dst = params->out_code_embd->data() + (size_t) t * n_e;
- if ((*params->out_codes)[(size_t) t] >= tab->ne[1]) {
- LOG_ERR("%s: speech code beyond the speech table\n", __func__);
- return false;
- }
- const size_t r = (size_t) (*params->out_codes)[(size_t) t] * n_e;
- if (tab->type == GGML_TYPE_F16) {
- ggml_backend_tensor_get(tab, h16.data(), r * sizeof(ggml_fp16_t), (size_t) n_e * sizeof(ggml_fp16_t));
- for (int j = 0; j < n_e; j++) {
- dst[j] = ggml_fp16_to_fp32(h16[j]);
- }
- } else {
- ggml_backend_tensor_get(tab, dst, r * sizeof(float), (size_t) n_e * sizeof(float));
- }
- if (pos) {
- ggml_backend_tensor_get(pos, pr.data(), (size_t) t * n_e * sizeof(float), (size_t) n_e * sizeof(float));
- for (int j = 0; j < n_e; j++) {
- dst[j] += pr[j];
- }
- }
- }
- }
- return true;
- }
-
- if (ctx->proj_type() == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == CLIP_GEN_PROCESS_TTS_VOCODE) {
- ggml_tensor * sp = ggml_graph_get_tensor(gf, "out_spec");
- GGML_ASSERT(sp != nullptr && params->out_spec);
- params->out_spec->resize(ggml_nelements(sp));
- ggml_backend_tensor_get(sp, params->out_spec->data(), 0, ggml_nbytes(sp));
- return true;
- }
-
- if (ctx->proj_type() == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == CLIP_GEN_PROCESS_TTS) {
- ggml_tensor * mel = ggml_graph_get_tensor(gf, "out_mel");
- GGML_ASSERT(mel != nullptr);
- std::vector<float> mel_data(ggml_nelements(mel));
- ggml_backend_tensor_get(mel, mel_data.data(), 0, ggml_nbytes(mel));
-
- // hift: f0 -> harmonic source -> source stft on the host, then the vocoder graph and the istft
- ggml_tensor * f0_t = ggml_graph_get_tensor(gf, "out_f0");
- GGML_ASSERT(f0_t != nullptr);
- const int n_mel_out = (int) ggml_nelements(f0_t);
- std::vector<float> f0(n_mel_out);
- ggml_backend_tensor_get(f0_t, f0.data(), 0, ggml_nbytes(f0_t));
-
- // SineGen: f0 upsampled x480, 9 harmonics with cumulative phase, uv gating and noise, merged by l_linear + tanh
- const int ups_total = 480;
- const double sr = 24000.0;
- const int64_t n_wav = (int64_t) n_mel_out * ups_total;
- std::vector<float> lw(9);
- float lb = 0.0f;
- if (clip_cbx_read_tensor(ctx, "a.gen.hift.m_source.l_linear.weight", lw.data(), lw.size()) != 9 ||
- clip_cbx_read_tensor(ctx, "a.gen.hift.m_source.l_linear.bias", &lb, 1) != 1) {
- LOG_ERR("%s: unexpected hift source module\n", __func__);
- return false;
- }
-
- std::mt19937 & srng = params->gen_state->rng;
- std::uniform_real_distribution<double> ud(-M_PI, M_PI);
- std::normal_distribution<float> snd(0.0f, 1.0f);
- double phase[9];
- for (int h = 0; h < 9; h++) {
- phase[h] = h == 0 ? 0.0 : ud(srng);
- }
- std::vector<float> src((size_t) n_wav);
- double cum[9] = {0.0};
- for (int64_t t = 0; t < n_wav; t++) {
- const float f = f0[(size_t) (t / ups_total)];
- const float uv = f > 10.0f ? 1.0f : 0.0f; // nsf_voiced_threshold
- const float namp = uv * 0.003f + (1.0f - uv) * 0.1f / 3.0f;
- float merged = lb;
- for (int h = 0; h < 9; h++) {
- cum[h] += (double) f * (h + 1) / sr;
- cum[h] -= floor(cum[h]);
- float sine = 0.1f * (float) sin(2.0 * M_PI * cum[h] + phase[h]);
- sine = sine * uv + namp * snd(srng);
- merged += lw[(size_t) h] * sine;
- }
- src[(size_t) t] = tanhf(merged);
- }
-
- // stft of the source: n_fft 16, hop 4, hann window, centered
- const int n_fft = 16, hop = 4, n_bins = 9;
- std::vector<float> win(n_fft);
- for (int i = 0; i < n_fft; i++) win[(size_t) i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / n_fft);
- const int n_stft = (int) (n_wav / hop) + 1;
- std::vector<float> sstft((size_t) n_stft * 18);
- for (int fr = 0; fr < n_stft; fr++) {
- const int64_t c0 = (int64_t) fr * hop - n_fft / 2; // centered, reflect padded
- for (int k = 0; k < n_bins; k++) {
- double re = 0.0, im = 0.0;
- for (int i = 0; i < n_fft; i++) {
- int64_t idx = c0 + i;
- if (idx < 0) idx = -idx;
- if (idx >= n_wav) idx = 2 * (n_wav - 1) - idx;
- const double v = (double) src[(size_t) idx] * win[(size_t) i];
- const double a = 2.0 * M_PI * k * i / n_fft;
- re += v * cos(a);
- im -= v * sin(a);
- }
- sstft[(size_t) fr * 18 + k ] = (float) re;
- sstft[(size_t) fr * 18 + 9 + k] = (float) im;
- }
- }
-
- // vocoder graph on mel + source stft
- std::vector<float> spec;
- {
- clip_encode_params vp = *params;
- vp.gen_process = CLIP_GEN_PROCESS_TTS_VOCODE;
- vp.mel_in = &mel_data;
- vp.sstft_in = &sstft;
- vp.vocode_n_mel = n_mel_out;
- vp.vocode_n_stft = n_stft;
- vp.out_audio = nullptr;
- vp.out_spec = &spec;
- if (!clip_encode(ctx, &vp)) {
- LOG_ERR("%s: vocoder stage failed\n", __func__);
- return false;
- }
- }
-
- // istft: mag = clipped exp, phase = sin, hann overlap-add
- const int n_frames_out = (int) (spec.size() / 18);
- const int64_t n_out = (int64_t) (n_frames_out - 1) * hop;
- std::vector<double> acc((size_t) n_out + n_fft, 0.0);
- std::vector<double> wsum((size_t) n_out + n_fft, 0.0);
- for (int fr = 0; fr < n_frames_out; fr++) {
- double frame[16];
- for (int i = 0; i < n_fft; i++) {
- double v = 0.0;
- for (int k = 0; k < n_bins; k++) {
- const double mag = fmin(exp((double) spec[(size_t) fr * 18 + k]), 1e2);
- const double ph = sin((double) spec[(size_t) fr * 18 + 9 + k]);
- const double re = mag * cos(ph), im = mag * sin(ph);
- const double a = 2.0 * M_PI * k * i / n_fft;
- const double w = (k == 0 || k == n_fft / 2) ? 1.0 : 2.0;
- v += w * (re * cos(a) - im * sin(a));
- }
- frame[i] = v / n_fft;
- }
- const int64_t o = (int64_t) fr * hop;
- for (int i = 0; i < n_fft; i++) {
- acc [(size_t) (o + i)] += frame[i] * win[(size_t) i];
- wsum[(size_t) (o + i)] += (double) win[(size_t) i] * win[(size_t) i];
- }
- }
- GGML_ASSERT(params->out_audio);
- auto & out_audio = *params->out_audio;
- out_audio.resize((size_t) std::min<int64_t>(n_out - n_fft / 2, n_wav));
- for (size_t i = 0; i < out_audio.size(); i++) {
- const size_t j = i + n_fft / 2; // drop the centering pad
- const double v = wsum[j] > 1e-11 ? acc[j] / wsum[j] : 0.0;
- out_audio[(size_t) i] = (float) fmax(-0.99, fmin(0.99, v));
- }
-
- // trim_fade of the reference: silence 20 ms then fade in 20 ms, hides the artifact of the prompt boundary
- const size_t n_trim = 24000 / 50;
- for (size_t i = 0; i < 2 * n_trim && i < out_audio.size(); i++) {
- const double g = i < n_trim ? 0.0
- : (cos(M_PI * (1.0 - (double) (i - n_trim) / n_trim)) + 1.0) / 2.0;
- out_audio[(size_t) i] *= (float) g;
- }
- return true;
+ GGML_ABORT("unsupported projector type");
}
- // optional outputs: a pipeline yields codes or feats, and not all have an eos head
- if (params->out_codes != nullptr) {
- ggml_tensor * codes = ggml_graph_get_tensor(gf, "out_codes");
- if (codes != nullptr) {
- auto & out_codes = *params->out_codes;
- out_codes.resize(ggml_nelements(codes));
- ggml_backend_tensor_get(codes, out_codes.data(), 0, ggml_nbytes(codes));
- }
- }
- if (params->out_feats != nullptr) {
- ggml_tensor * feats = ggml_graph_get_tensor(gf, "out_feats");
- if (feats != nullptr) {
- auto & out_feats = *params->out_feats;
- out_feats.resize(ggml_nelements(feats));
- ggml_backend_tensor_get(feats, out_feats.data(), 0, ggml_nbytes(feats));
- }
- }
- if (params->out_is_eos != nullptr) {
- ggml_tensor * eos = ggml_graph_get_tensor(gf, "out_eos_score");
- if (eos != nullptr) {
- GGML_ASSERT(ggml_nelements(eos) == 1);
- float score = 0.0f;
- ggml_backend_tensor_get(eos, &score, 0, sizeof(float));
- *params->out_is_eos = score > hparams.gen_eos_threshold;
- }
- }
- if (params->out_audio != nullptr) {
- ggml_tensor * audio = ggml_graph_get_tensor(gf, "out_audio");
- if (audio == nullptr) {
- GGML_ABORT("out_audio requested but graph has no \"out_audio\" tensor");
- }
- auto & out_audio = *params->out_audio;
- out_audio.resize(ggml_nelements(audio));
- ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio));
+ n_patches += (int) mtmd_suffix_rows(img->suffix_type).size();
- if (model.proj_type == PROJECTOR_TYPE_SOPRANO) {
- std::vector<float> spectrum = std::move(out_audio);
- clip_graph_soprano::decode_spectrum(model, spectrum, out_audio);
- }
+ return n_patches;
+}
- // drop the tail audio that comes from the code-0 rear padding, drama3 has no padding
- if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN) {
- const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0;
- const int64_t n_frames_w = hparams.wav_tfm_swa;
- const int64_t n_frames = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w;
- if (n_frames < n_frames_w) {
- const size_t hop = out_audio.size() / n_frames_w;
- out_audio.resize((size_t) n_frames * hop);
- }
- }
+size_t mtmd_model::cbx_read_tensor(const char * name, float * out, size_t n_max) const {
+ const auto & tensors = cbx_tensors;
+ auto it = tensors.find(name);
+ if (it == tensors.end() || !it->second) {
+ return 0;
}
- if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_LFM3A) {
- const auto & slots = params->enc_state->slots;
- size_t i = 0;
- for (const auto & slot : list_lfm3a_state_slots(hparams, model)) {
- ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str());
- GGML_ASSERT(t != nullptr);
- copy_gen_state_out(t, slots[i++]);
- }
- } else if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) {
- clip_enc_state * st = params->enc_state;
- ggml_tensor * chunk = ggml_graph_get_tensor(gf, "spk_chunk");
- ggml_tensor * preds = ggml_graph_get_tensor(gf, "spk_preds");
- GGML_ASSERT(chunk != nullptr && preds != nullptr);
- st->spk_chunk.resize(ggml_nelements(chunk));
- st->spk_preds.resize(ggml_nelements(preds));
- ggml_backend_tensor_get(chunk, st->spk_chunk.data(), 0, ggml_nbytes(chunk));
- ggml_backend_tensor_get(preds, st->spk_preds.data(), 0, ggml_nbytes(preds));
- } else if (params->enc_state != nullptr) {
- clip_enc_state * st = params->enc_state;
- ggml_tensor * tail = ggml_graph_get_tensor(gf, "out_stem_tail");
- GGML_ASSERT(tail != nullptr);
- st->stem_tail.resize(ggml_nelements(tail));
- ggml_backend_tensor_get(tail, st->stem_tail.data(), 0, ggml_nbytes(tail));
-
- const int n_new = (imgs.entries[0].nx() - 2) / 2;
- for (int i = 0; i < n_new; i++) {
- st->slot_pos[(st->n_past + i) % st->n_ctx] = st->n_past + i;
- }
- st->n_past += n_new;
+ ggml_tensor * t = it->second;
+ const size_t n = (size_t) ggml_nelements(t);
+ if (!out) {
+ return n;
}
-
- if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
- const auto & slots = params->gen_state->slots;
- size_t i = 0;
- for (const auto & slot : list_gen_state_slots(hparams, model)) {
- if (slot.in_place) {
- i++;
- continue;
- }
- ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str());
- if (t == nullptr) {
- GGML_ABORT("graph has no \"state_out_%s\" tensor", slot.name.c_str());
- }
- GGML_ASSERT(i < slots.size());
- copy_gen_state_out(t, slots[i++]);
- }
+ if (n_max < n) {
+ return 0;
}
-
- //
- // Debug: dump final embeddings if MTMD_DEBUG_EMBEDDINGS is set
- //
-
- if (ctx->debug_output_embeddings && embeddings != nullptr) {
- const int64_t n_embd = embeddings->ne[0];
- const int64_t n_tokens = embeddings->ne[1];
- std::vector<float> emb_data(ggml_nelements(embeddings));
- ggml_backend_tensor_get(embeddings, emb_data.data(), 0, ggml_nbytes(embeddings));
-
- LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n");
- LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens);
-
- // TEMP debugging (parity validation), will be removed before merge
- // when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data]
- const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS");
- if (dump_path && strcmp(dump_path, "1") != 0) {
- FILE * f = fopen(dump_path, "wb");
- if (f) {
- const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd };
- fwrite(hdr, sizeof(hdr), 1, f);
- fwrite(emb_data.data(), sizeof(float), emb_data.size(), f);
- fclose(f);
- }
- }
-
- // Print first few values of first token
- LOG_INF("Token 0 (first 16 values): ");
- for (int i = 0; i < std::min((int64_t)16, n_embd); i++) {
- LOG_INF("%.6f ", emb_data[i]);
- }
- LOG_INF("\n");
-
- // Print last few values of first token
- if (n_embd > 16) {
- LOG_INF("Token 0 (last 16 values): ");
- for (int64_t i = n_embd - 16; i < n_embd; i++) {
- LOG_INF("%.6f ", emb_data[i]);
- }
- LOG_INF("\n");
- }
-
- // Compute and print statistics
- float sum = 0.0f, sum_sq = 0.0f, min_val = emb_data[0], max_val = emb_data[0];
- for (size_t i = 0; i < emb_data.size(); i++) {
- sum += emb_data[i];
- sum_sq += emb_data[i] * emb_data[i];
- min_val = std::min(min_val, emb_data[i]);
- max_val = std::max(max_val, emb_data[i]);
- }
- float mean = sum / emb_data.size();
- float variance = (sum_sq / emb_data.size()) - (mean * mean);
- LOG_INF("Stats: mean=%.6f, std=%.6f, min=%.6f, max=%.6f, sum=%.6f\n",
- mean, sqrtf(variance), min_val, max_val, sum);
- LOG_INF("=== END MTMD_DEBUG_EMBEDDINGS ===\n\n");
+ if (t->type == GGML_TYPE_F32) {
+ ggml_backend_tensor_get(t, out, 0, n * sizeof(float));
+ } else if (t->type == GGML_TYPE_F16) {
+ std::vector<ggml_fp16_t> tmp(n);
+ ggml_backend_tensor_get(t, tmp.data(), 0, n * sizeof(ggml_fp16_t));
+ for (size_t i = 0; i < n; i++) out[i] = ggml_fp16_to_fp32(tmp[i]);
+ } else {
+ return 0;
}
-
- return true;
+ return n;
}
-int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
- switch (ctx->model.proj_type) {
+int mtmd_model::n_mmproj_embd() const {
+ switch (proj_type) {
case PROJECTOR_TYPE_LDP:
- return ctx->model.mm_model_block_1_block_2_1_b->ne[0];
+ return mm_model_block_1_block_2_1_b->ne[0];
case PROJECTOR_TYPE_LDPV2:
- return ctx->model.mm_model_peg_0_b->ne[0];
+ return mm_model_peg_0_b->ne[0];
case PROJECTOR_TYPE_MLP:
case PROJECTOR_TYPE_PHI4:
case PROJECTOR_TYPE_PIXTRAL:
@@ -8173,21 +4209,21 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
case PROJECTOR_TYPE_DOTS3NOTE_A:
- return ctx->model.mm_2_w->ne[1];
+ return mm_2_w->ne[1];
case PROJECTOR_TYPE_MLP_NORM:
- return ctx->model.mm_3_b->ne[0];
+ return mm_3_b->ne[0];
case PROJECTOR_TYPE_MINICPMV:
- return ctx->model.mm_model_proj->ne[0];
+ return mm_model_proj->ne[0];
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
- return ctx->model.mm_ffn_down_w->ne[1];
+ return mm_ffn_down_w->ne[1];
case PROJECTOR_TYPE_GLM_EDGE:
- return ctx->model.mm_model_mlp_3_w->ne[1];
+ return mm_model_mlp_3_w->ne[1];
case PROJECTOR_TYPE_MINIMAX_M3:
- return ctx->model.mm_merger_fc2_b->ne[0];
+ return mm_merger_fc2_b->ne[0];
case PROJECTOR_TYPE_MUSE_GLIMMER:
- return ctx->model.mm_2_w->ne[1];
+ return mm_2_w->ne[1];
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_MAGEVL:
@@ -8195,47 +4231,47 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_JANUS_PRO:
case PROJECTOR_TYPE_YOUTUVL:
- return ctx->model.mm_1_b->ne[0];
+ return mm_1_b->ne[0];
case PROJECTOR_TYPE_SENSENOVA_U1:
- return ctx->model.hparams.projection_dim;
+ return hparams.projection_dim;
case PROJECTOR_TYPE_QWEN3VL:
// main path + deepstack paths
- return ctx->model.mm_1_b->ne[0] * (1 + ctx->model.n_deepstack_layers);
+ return mm_1_b->ne[0] * (1 + n_deepstack_layers);
case PROJECTOR_TYPE_LING3VL:
- return ctx->model.mm_1_b->ne[0];
+ return mm_1_b->ne[0];
case PROJECTOR_TYPE_MIMOVL:
- return ctx->model.mm_1_w->ne[1];
+ return mm_1_w->ne[1];
case PROJECTOR_TYPE_STEP3VL:
- return ctx->model.mm_model_proj->ne[1];
+ return mm_model_proj->ne[1];
case PROJECTOR_TYPE_GEMMA3:
case PROJECTOR_TYPE_GEMMA3NV:
- return ctx->model.mm_input_proj_w->ne[0];
+ return mm_input_proj_w->ne[0];
case PROJECTOR_TYPE_GEMMA4V:
case PROJECTOR_TYPE_GEMMA4UV:
case PROJECTOR_TYPE_GEMMA4A:
case PROJECTOR_TYPE_GEMMA4UA:
- return ctx->model.mm_input_proj_w->ne[1];
+ return mm_input_proj_w->ne[1];
case PROJECTOR_TYPE_IDEFICS3:
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
- return ctx->model.mm_fc_w->ne[1];
+ return mm_fc_w->ne[1];
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_VOXTRAL_RT:
case PROJECTOR_TYPE_MUSIC_FLAMINGO:
- return ctx->model.mm_2_w->ne[1];
+ return mm_2_w->ne[1];
case PROJECTOR_TYPE_MERALION:
- return ctx->model.mm_3_w->ne[1]; // out_proj output dim
+ return mm_3_w->ne[1]; // out_proj output dim
case PROJECTOR_TYPE_INTERNVL:
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
case PROJECTOR_TYPE_NEMOTRON_OMNI:
- return ctx->model.mm_3_w->ne[1];
+ return mm_3_w->ne[1];
case PROJECTOR_TYPE_LLAMA4:
- return ctx->model.mm_model_proj->ne[1];
+ return mm_model_proj->ne[1];
case PROJECTOR_TYPE_QWEN2A:
case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
- return ctx->model.mm_fc_w->ne[1];
+ return mm_fc_w->ne[1];
case PROJECTOR_TYPE_QWEN3A:
- return ctx->model.mm_2_w->ne[1];
+ return mm_2_w->ne[1];
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_LFM2:
case PROJECTOR_TYPE_D1OMNI_V:
@@ -8246,74 +4282,58 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_DEEPSEEK4V:
case PROJECTOR_TYPE_DEEPSEEK41V:
case PROJECTOR_TYPE_COHERE2V:
- return ctx->model.mm_2_w->ne[1];
+ return mm_2_w->ne[1];
case PROJECTOR_TYPE_HUNYUANVL:
- return ctx->model.mm_model_proj->ne[1];
+ return mm_model_proj->ne[1];
case PROJECTOR_TYPE_COGVLM:
- return ctx->model.mm_4h_to_h_w->ne[1];
+ return mm_4h_to_h_w->ne[1];
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
- return ctx->model.mm_fc_w->ne[1];
+ return mm_fc_w->ne[1];
case PROJECTOR_TYPE_LFM2A:
- return ctx->model.position_embeddings->ne[0];
+ return position_embeddings->ne[0];
case PROJECTOR_TYPE_D1OMNI_A:
case PROJECTOR_TYPE_LFM3A:
- return ctx->model.mm_3_w->ne[1];
+ return mm_3_w->ne[1];
case PROJECTOR_TYPE_GRANITE_SPEECH:
- return ctx->model.qf_proj_blocks[0].qf_proj_linear_w->ne[1];
+ return qf_proj_blocks[0].qf_proj_linear_w->ne[1];
case PROJECTOR_TYPE_GRANITE4_VISION:
- return ctx->model.qf_proj_blocks.size() * ctx->model.hparams.projection_dim;
+ return qf_proj_blocks.size() * hparams.projection_dim;
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_GLM5V:
- return ctx->model.mm_ffn_down_w->ne[1];
+ return mm_ffn_down_w->ne[1];
case PROJECTOR_TYPE_MIMO_AUDIO:
- return ctx->model.mm_2_w->ne[1];
+ return mm_2_w->ne[1];
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
- return ctx->model.mm_fc_w->ne[2];
+ return mm_fc_w->ne[2];
case PROJECTOR_TYPE_QWEN3TTS_GEN:
case PROJECTOR_TYPE_LFM3A_GEN:
case PROJECTOR_TYPE_DRAMA3_GEN:
- return ctx->model.gen_code_out_embd_w->ne[0];
+ return gen_code_out_embd_w->ne[0];
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
- return ctx->model.spk_proj_w->ne[1];
+ return spk_proj_w->ne[1];
case PROJECTOR_TYPE_POCKETTTS_GEN:
- return ctx->model.gen_input_lin_w->ne[1];
+ return gen_input_lin_w->ne[1];
case PROJECTOR_TYPE_CHATTERBOX:
// talker width, carried by the speech table
- return ctx->model.cbx_tensors.at("a.gen.code.out_embd.weight")->ne[0];
+ return cbx_tensors.at("a.gen.code.out_embd.weight")->ne[0];
case PROJECTOR_TYPE_CHATTERBOX_SPKENC:
// talker conditioning rows, the speaker projection carries their width
- return ctx->model.cbx_tensors.at("a.cenc.spkr_enc.weight")->ne[1];
+ return cbx_tensors.at("a.cenc.spkr_enc.weight")->ne[1];
case PROJECTOR_TYPE_SOPRANO:
- return ctx->model.hparams.projection_dim;
+ return hparams.projection_dim;
case PROJECTOR_TYPE_PARAKEET:
- return ctx->model.mm_1_w->ne[1];
+ return mm_1_w->ne[1];
default:
GGML_ABORT("Unknown projector type");
}
}
-bool clip_is_llava(const struct clip_ctx * ctx) {
- return ctx->model.hparams.has_llava_projector;
-}
-
-bool clip_has_vision_encoder(const struct clip_ctx * ctx) {
- return ctx->model.modality == CLIP_MODALITY_VISION;
-}
-
-bool clip_has_audio_encoder(const struct clip_ctx * ctx) {
- return ctx->model.modality == CLIP_MODALITY_AUDIO;
-}
-
-bool clip_support_batch(const struct clip_ctx * ctx) {
- return ctx->support_batch;
-}
-
// TODO @ngxson : this is no longer correct with mtmd_batch API
// this was only meant to be used by qwen-vl-based models, to fuse 2 input images into one (qwen-vl video support)
// this logic should be refactored in near future to distinctly handle "merge frames" and "batching"
-int clip_model_n_temporal_merge(const struct clip_ctx * ctx) {
- switch (ctx->proj_type()) {
+int mtmd_model::n_temporal_merge() const {
+ switch (proj_type) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
@@ -8325,31 +4345,3 @@ int clip_model_n_temporal_merge(const struct clip_ctx * ctx) {
return 1;
}
}
-
-//
-// API used internally with mtmd
-//
-
-projector_type clip_get_projector_type(const struct clip_ctx * ctx) {
- return ctx->proj_type();
-}
-
-const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx) {
- return &ctx->model.hparams;
-}
-
-std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx * ctx) {
- std::map<ggml_backend_dev_t, size_t> result = ctx->mem_usage;
- for (auto & [dev, size] : ctx->mem_compute) {
- result[dev] += size;
- }
- return result;
-}
-
-//
-// API for debugging
-//
-
-void clip_set_debug_output_embeddings(clip_ctx * ctx, bool enable) {
- ctx->debug_output_embeddings = enable;
-}
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/mtmd-model.h
similarity index 92%
rename from tools/mtmd/clip-model.h
rename to tools/mtmd/mtmd-model.h
index f50fc40b9..0d975ba36 100644
--- a/tools/mtmd/clip-model.h
+++ b/tools/mtmd/mtmd-model.h
@@ -1,8 +1,8 @@
#pragma once
#include "ggml.h"
-#include "clip.h"
-#include "clip-impl.h"
+#include "ggml-backend.h"
+#include "mtmd-impl.h"
#include <algorithm>
#include <array>
@@ -12,6 +12,25 @@
#include <cstdint>
#include <cmath>
+enum mtmd_modality {
+ MTMD_MODALITY_VISION,
+ MTMD_MODALITY_AUDIO,
+ MTMD_MODALITY_GEN_AUDIO,
+};
+
+// gen-side voice reference (chatterbox): speech tokens, mel-rate features and the speaker vector
+struct mtmd_gen_ref {
+ std::vector<int32_t> tokens;
+ std::vector<float> feat;
+ std::vector<float> spk;
+};
+
+// a predefined voice: backbone rows, plus the gen-side reference when the gen stack needs one
+struct mtmd_gen_voice {
+ std::vector<float> rows;
+ mtmd_gen_ref ref;
+};
+
enum ffn_op_type {
FFN_GELU,
FFN_GELU_ERF,
@@ -50,7 +69,7 @@ enum pad_style {
PAD_NEAREST,
};
-struct clip_hparams {
+struct mtmd_hparams {
int32_t image_size = 0;
int32_t patch_size = 0;
int32_t n_embd = 0;
@@ -74,7 +93,7 @@ struct clip_hparams {
std::array<uint8_t, 3> image_pad_color = {0, 0, 0};
// (preprocessor) for llava-uhd style models
- std::vector<clip_image_size> image_res_candidates;
+ std::vector<mtmd_image_size> image_res_candidates;
int32_t preproc_min_tiles = 0;
int32_t preproc_max_tiles = 0;
int32_t max_slice_nums = 9; // llava-uhd slice cap; per-model, carried in the GGUF
@@ -277,7 +296,7 @@ struct clip_hparams {
}
};
-struct clip_layer {
+struct mtmd_layer {
// layernorm 1 (or layer input norm, or pre-attention norm)
ggml_tensor * ln_1_w = nullptr;
ggml_tensor * ln_1_b = nullptr;
@@ -484,13 +503,13 @@ struct qf_block {
ggml_tensor * qf_proj_post_norm_w = nullptr;
ggml_tensor * qf_proj_post_norm_b = nullptr;
ggml_tensor * qf_proj_img_pos = nullptr; // Vision only
- std::vector<clip_layer> qf_proj_layers;
+ std::vector<mtmd_layer> qf_proj_layers;
};
// pocket-tts SEANet stack, used in both directions:
// encoder = conv_in -> per stage (residual unit, strided conv) -> conv_out
// decoder = conv_in -> per stage (strided convtr, residual unit) -> conv_out
-struct clip_seanet {
+struct mtmd_seanet {
// one residual unit: ELU -> dilated conv -> ELU -> pointwise conv, added to the input
struct stage {
ggml_tensor * res_conv1_w = nullptr;
@@ -509,7 +528,7 @@ struct clip_seanet {
};
// pocket-tts flow-matching decoder (SimpleMLPAdaLN)
-struct clip_flow_net {
+struct mtmd_flow_net {
// AdaLN res block: in_ln -> modulate -> Linear -> SiLU -> Linear, gated residual
struct block {
ggml_tensor * norm_w = nullptr;
@@ -545,7 +564,7 @@ struct clip_flow_net {
};
// qwen3tts code2wav: RVQ codes -> raw PCM
-struct clip_code2wav {
+struct mtmd_code2wav {
// "upsample" stage: one ConvNeXt block plus the causal ConvTranspose1d before it
struct upsample_block {
ggml_tensor * conv_w = nullptr; // causal ConvTranspose1d, 2x
@@ -599,7 +618,7 @@ struct clip_code2wav {
ggml_tensor * tfm_out_proj_w = nullptr;
ggml_tensor * tfm_out_proj_b = nullptr;
ggml_tensor * tfm_output_norm_w = nullptr;
- std::vector<clip_layer> tfm_layers; // reuses the generic block fields (ln_1/attn/ln_2/ffn/ls_1/ls_2)
+ std::vector<mtmd_layer> tfm_layers; // reuses the generic block fields (ln_1/attn/ln_2/ffn/ls_1/ls_2)
std::vector<upsample_block> upsample;
@@ -612,7 +631,7 @@ struct clip_code2wav {
ggml_tensor * dac_post_conv_b = nullptr;
};
-struct clip_chatterbox {
+struct mtmd_chatterbox {
// espnet rel-pos conformer layer of the flow encoder
struct enc_layer {
ggml_tensor * norm_mha_w = nullptr;
@@ -842,7 +861,7 @@ struct clip_chatterbox {
};
// drama3 tower: extra tensors around the generic block
-struct clip_drama3_layer {
+struct mtmd_drama3_layer {
ggml_tensor * enorm_w = nullptr;
ggml_tensor * hnorm_w = nullptr;
ggml_tensor * eh_proj_w = nullptr; // [code embd; hidden] -> hidden
@@ -850,7 +869,7 @@ struct clip_drama3_layer {
ggml_tensor * out_norm_w = nullptr;
};
-struct clip_vocos {
+struct mtmd_vocos {
ggml_tensor * input_w = nullptr;
ggml_tensor * input_b = nullptr;
ggml_tensor * norm_w = nullptr;
@@ -859,13 +878,23 @@ struct clip_vocos {
ggml_tensor * output_norm_b = nullptr;
ggml_tensor * output_w = nullptr;
ggml_tensor * output_b = nullptr;
- std::vector<clip_code2wav::upsample_block> blocks;
+ std::vector<mtmd_code2wav::upsample_block> blocks;
};
-struct clip_model {
- clip_modality modality = CLIP_MODALITY_VISION;
+// the weights and hparams of one modality of an mmproj file, read-only once loaded
+struct mtmd_model {
+ mtmd_modality modality = MTMD_MODALITY_VISION;
projector_type proj_type = PROJECTOR_TYPE_MLP;
- clip_hparams hparams;
+ mtmd_hparams hparams;
+
+ // runtime requirements declared by the model, applied by mtmd_subcontext
+ int max_nodes = 8192;
+ bool flash_attn_auto = true; // false: MTMD_FLASH_ATTN_TYPE_AUTO resolves to disabled
+
+ // weights
+ ggml_context_ptr ctx_data;
+ ggml_backend_buffer_ptr buf;
+ std::map<ggml_backend_dev_t, size_t> mem_weights;
// embeddings
ggml_tensor * class_embedding = nullptr;
@@ -888,7 +917,7 @@ struct clip_model {
ggml_tensor * pre_ln_w = nullptr;
ggml_tensor * pre_ln_b = nullptr;
- std::vector<clip_layer> layers;
+ std::vector<mtmd_layer> layers;
// one patch embedder per audio pyramid stage (muse-spark)
struct audio_patch {
@@ -898,7 +927,7 @@ struct clip_model {
};
std::vector<audio_patch> a_patches;
- int32_t n_deepstack_layers = 0; // used by Qwen3-VL, calculated from clip_layer
+ int32_t n_deepstack_layers = 0; // used by Qwen3-VL, calculated from mtmd_layer
ggml_tensor * post_ln_w;
ggml_tensor * post_ln_b;
@@ -1085,7 +1114,7 @@ struct clip_model {
// mimo-v2.5: LLM-side connector (input_local_transformer, separate from the
// audio_tokenizer's own encoder `layers`)
- std::vector<clip_layer> mm_a_local_layers;
+ std::vector<mtmd_layer> mm_a_local_layers;
ggml_tensor * mm_a_local_norm_w = nullptr;
// qwen3a
@@ -1116,18 +1145,18 @@ struct clip_model {
ggml_tensor * gen_code_mask_embd = nullptr;
// qwen3tts code2wav: RVQ codes -> raw PCM
- clip_code2wav c2w;
- std::vector<clip_drama3_layer> drama3_layers;
+ mtmd_code2wav c2w;
+ std::vector<mtmd_drama3_layer> drama3_layers;
// pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path)
- clip_seanet seanet;
+ mtmd_seanet seanet;
// pocket-tts: voice latent -> backbone embd (speaker path)
ggml_tensor * spk_proj_w = nullptr;
ggml_tensor * downsample_w = nullptr;
// pocket-tts: flow-matching decoder, backbone hidden state -> next latent
- clip_flow_net flow;
+ mtmd_flow_net flow;
ggml_tensor * gen_out_eos_w = nullptr;
ggml_tensor * gen_out_eos_b = nullptr;
ggml_tensor * gen_input_lin_w = nullptr; // latent -> backbone embd
@@ -1135,18 +1164,18 @@ struct clip_model {
ggml_tensor * gen_emb_std = nullptr;
ggml_tensor * gen_quant_out_w = nullptr; // latent -> decoder dim
ggml_tensor * gen_upsample_w = nullptr; // depthwise convtr, frame rate -> encoder frame rate
- std::vector<clip_layer> gen_tfm_layers; // mimi decoder_transformer
+ std::vector<mtmd_layer> gen_tfm_layers; // mimi decoder_transformer
// chatterbox flow encoder, cfm estimator, hift vocoder, s3 tokenizer and CAMPPlus speaker encoder
- clip_chatterbox cbx;
+ mtmd_chatterbox cbx;
- // chatterbox tensors read on the host, see clip_cbx_read_tensor()
+ // chatterbox tensors read on the host, see cbx_read_tensor()
std::map<std::string, ggml_tensor *> cbx_tensors;
// predefined voices, no rows if the voice is named in the prompt (lfm3a)
- std::map<std::string, clip_gen_voice> gen_voices;
+ std::map<std::string, mtmd_gen_voice> gen_voices;
// soprano: Vocos decoder, hidden states -> STFT coefficients
- clip_vocos vocos;
+ mtmd_vocos vocos;
// cogvlm
ggml_tensor * mm_post_fc_norm_w = nullptr;
@@ -1178,10 +1207,10 @@ struct clip_model {
int32_t n_sam_layers = 12; // used by deepseek-ocr sam encoder
- std::vector<clip_layer> sam_layers;
+ std::vector<mtmd_layer> sam_layers;
// nemotron transcribe: speaker expert, Sortformer head, fusion
- std::vector<clip_layer> spk_layers;
+ std::vector<mtmd_layer> spk_layers;
ggml_tensor * spk_inp_proj_w = nullptr;
ggml_tensor * spk_pre_ln_w = nullptr;
ggml_tensor * spk_pre_ln_b = nullptr;
@@ -1254,6 +1283,51 @@ struct clip_model {
|| proj_type == PROJECTOR_TYPE_VOXTRAL
|| proj_type == PROJECTOR_TYPE_MERALION;
}
+
+ // this should be equal to the embedding dimension of the text model
+ int n_mmproj_embd() const;
+
+ int n_output_tokens(const mtmd_image_f32 * img) const;
+
+ // for M-RoPE, this will be the number of token positions in X and Y directions
+ // for other models, X will be the total number of tokens and Y will be 1
+ int n_output_tokens_x(const mtmd_image_f32 * img) const;
+ int n_output_tokens_y(const mtmd_image_f32 * img) const;
+
+ // max number of output tokens per image, -1 if not dynamic size
+ int image_max_tokens() const;
+
+ // TODO @ngxson : this is no longer correct with mtmd_batch API
+ int n_temporal_merge() const;
+
+ // read a chatterbox tensor by name as F32, returns the element count (0 if not found), out may be null to query the size
+ size_t cbx_read_tensor(const char * name, float * out, size_t n_max) const;
+};
+
+struct mtmd_model_params {
+ // custom value provided by user, -1 if not set
+ int32_t image_min_tokens = -1;
+ int32_t image_max_tokens = -1;
+ bool no_alloc = false; // only create the tensors, do not read their data
+ mtmd_progress_callback progress_callback = nullptr;
+ void * progress_callback_user_data = nullptr;
+};
+
+// the modalities of an mmproj file, a missing one is nullptr
+struct mtmd_models {
+ std::unique_ptr<mtmd_model> v;
+ std::unique_ptr<mtmd_model> a;
+ std::unique_ptr<mtmd_model> gen_a;
+};
+
+// the weights are allocated in buft, throws on error
+mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params, ggml_backend_buffer_type_t buft);
+
+struct mtmd_model_caps {
+ bool has_vision;
+ bool has_audio;
+ bool has_gen_audio;
};
-const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx);
+// read the modalities of an mmproj file without loading its tensors
+mtmd_model_caps mtmd_model_get_caps(const char * fname);
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
new file mode 100644
index 000000000..fa33d5887
--- /dev/null
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -0,0 +1,2701 @@
+#include "mtmd-subcontext.h"
+#include "mtmd-graph.h"
+#include "models/models.h"
+#include "mtmd-audio.h"
+
+#include "ggml.h"
+#include "ggml-cpp.h"
+#include "ggml-alloc.h"
+#include "ggml-backend.h"
+#include "ggml-cpu.h"
+
+#include <algorithm>
+#include <cfloat>
+#include <cmath>
+#include <cstdlib>
+#include <cstring>
+#include <map>
+#include <memory>
+#include <random>
+#include <stdexcept>
+#include <vector>
+
+// ggml_backend_cpu_set_n_threads(backend_cpu, n_threads);
+static void mtmd_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) {
+ ggml_backend_dev_t dev = ggml_backend_get_device(backend_cpu);
+ ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
+ if (reg) {
+ auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
+ if (ggml_backend_set_n_threads_fn) {
+ ggml_backend_set_n_threads_fn(backend_cpu, n_threads);
+ }
+ }
+}
+
+std::shared_ptr<ggml_threadpool> mtmd_threadpool_init(int n_threads) {
+ ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
+ ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
+ if (!reg || n_threads < 2) {
+ return nullptr;
+ }
+ auto * new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new");
+ auto * free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");
+ if (!new_fn || !free_fn) {
+ return nullptr;
+ }
+ ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads);
+ tpp.poll = 0; // the pool is idle most of the time, don't take CPU away from the text model
+ ggml_threadpool_t threadpool = new_fn(&tpp);
+ if (!threadpool) {
+ throw std::runtime_error("failed to create threadpool");
+ }
+ return std::shared_ptr<ggml_threadpool>(threadpool, free_fn);
+}
+
+ggml_backend_dev_t mtmd_select_device(bool use_gpu, ggml_backend_dev_t device) {
+ if (!use_gpu) {
+ return nullptr;
+ }
+ if (device != nullptr) {
+ return device;
+ }
+ ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);
+ return dev ? dev : ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU);
+}
+
+mtmd_subcontext::mtmd_subcontext(const mtmd_model & model, const mtmd_subctx_params & params) :
+ model(model),
+ threadpool(params.threadpool),
+ max_nodes(model.max_nodes),
+ flash_attn_type(params.flash_attn_type) {
+ if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_AUTO && !model.flash_attn_auto) {
+ flash_attn_type = MTMD_FLASH_ATTN_TYPE_DISABLED;
+ }
+
+ backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
+ if (!backend_cpu) {
+ throw std::runtime_error("failed to initialize CPU backend");
+ }
+ if (threadpool) {
+ ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu));
+ auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool");
+ if (set_threadpool_fn) {
+ set_threadpool_fn(backend_cpu, threadpool.get());
+ }
+ }
+ if (params.device != nullptr) {
+ backend = ggml_backend_dev_init(params.device, nullptr);
+ if (!backend) {
+ ggml_backend_free(backend_cpu);
+ throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n",
+ __func__, ggml_backend_dev_name(params.device)));
+ }
+ }
+
+ if (backend) {
+ LOG_INF("%s: using %s backend\n", __func__, ggml_backend_name(backend));
+ backend_ptrs.push_back(backend);
+ backend_buft.push_back(ggml_backend_get_default_buffer_type(backend));
+ } else {
+ backend = backend_cpu;
+ LOG_INF("%s: using CPU backend\n", __func__);
+ }
+
+ backend_ptrs.push_back(backend_cpu);
+ backend_buft.push_back(ggml_backend_get_default_buffer_type(backend_cpu));
+
+ sched.reset(
+ ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, true)
+ );
+
+ if (params.cb_eval != nullptr) {
+ ggml_backend_sched_set_eval_callback(sched.get(), params.cb_eval, params.cb_eval_user_data);
+ }
+
+ debug_output_embeddings = std::getenv("MTMD_DEBUG_EMBEDDINGS") != nullptr;
+
+ buf_compute_meta.resize(max_nodes * ggml_tensor_overhead() + ggml_graph_overhead_custom(max_nodes, false));
+
+ // TODO: fix warmup for audio gen
+ if (model.modality != MTMD_MODALITY_GEN_AUDIO) {
+ // check batching support
+ auto batch = get_dummy_batch();
+ support_batch = mtmd_get_graph_builder(this, batch)->support_batch();
+
+ if (params.warmup) {
+ warmup(batch);
+ }
+ }
+}
+
+mtmd_subcontext::~mtmd_subcontext() {
+ // the scheduler references the backends
+ sched.reset();
+ ggml_backend_free(backend);
+ if (backend != backend_cpu) {
+ ggml_backend_free(backend_cpu);
+ }
+}
+
+mtmd_image_f32_batch mtmd_subcontext::get_dummy_batch() const {
+ // create a fake batch
+ const auto & hparams = model.hparams;
+ mtmd_image_f32_batch batch;
+ mtmd_image_f32 img;
+ if (model.modality == MTMD_MODALITY_VISION) {
+ const int sz = hparams.warmup_image_size;
+ img.set_size({sz, sz}, false, false);
+ LOG_INF("%s: warmup with image size = %d x %d\n", __func__, sz, sz);
+ } else {
+ // GEMMA4UA uses n_mel_bins as a raw-waveform frame size (640), not a mel-bin count,
+ // so the [1, 256] bound only applies to FFT-based models.
+ const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA;
+ if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) {
+ throw std::runtime_error(string_format("%s: invalid n_mel_bins (%d), must be in [1, 256]\n", __func__, hparams.n_mel_bins));
+ }
+ img.set_size({hparams.warmup_audio_size, hparams.n_mel_bins}, false, false);
+ LOG_INF("%s: warmup with audio size = %d\n", __func__, hparams.warmup_audio_size);
+ }
+ batch.entries.push_back(img);
+ return batch;
+}
+
+void mtmd_subcontext::warmup(const mtmd_image_f32_batch & batch) {
+ support_info_graph info;
+
+ if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_AUTO) {
+ // try to enable flash attention to see if it's supported
+ flash_attn_type = MTMD_FLASH_ATTN_TYPE_ENABLED;
+ info = reserve_compute_meta(batch);
+ if (!info.fattn && info.fattn_op) {
+ auto op = info.fattn_op;
+ LOG_WRN("%s: *****************************************************************\n", __func__);
+ LOG_WRN("%s: WARNING: flash attention not supported by %s, memory usage will increase\n", __func__, ggml_backend_name(backend));
+ LOG_WRN("%s: op params: \n", __func__);
+ static auto print_shape = [](const char * fn, const char * name, ggml_tensor * t) {
+ LOG_WRN("%s: %s: type = %s, ne = [%d %d %d %d], nb = [%d %d %d %d]\n", fn,
+ name, ggml_type_name(t->type),
+ t->ne[0], t->ne[1], t->ne[2], t->ne[3],
+ t->nb[0], t->nb[1], t->nb[2], t->nb[3]);
+ };
+ print_shape(__func__, " dst", op);
+ print_shape(__func__, "src0", op->src[0]);
+ print_shape(__func__, "src1", op->src[1]);
+ print_shape(__func__, "src2", op->src[2]);
+ LOG_WRN("%s: please report this on github as an issue\n", __func__);
+ LOG_WRN("%s: *****************************************************************\n", __func__);
+ flash_attn_type = MTMD_FLASH_ATTN_TYPE_DISABLED;
+ reserve_compute_meta(batch);
+ }
+ } else {
+ info = reserve_compute_meta(batch);
+ if (!info.fattn && flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) {
+ LOG_WRN("%s: flash attention is not supported by the current backend; falling back to CPU (performance will be degraded)\n", __func__);
+ }
+ }
+
+ is_allocated = true; // mark buffers as allocated
+
+ LOG_INF("%s: flash attention is %s\n", __func__,
+ (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled");
+
+ // print ops that are not supported by the GPU backend (if there is one)
+ if (backend && backend != backend_cpu) {
+ std::vector<support_info_op> unsupported_ops;
+ for (const auto & op : info.ops) {
+ if (!op.is_accel) {
+ unsupported_ops.push_back(op);
+ }
+ }
+ if (!unsupported_ops.empty()) {
+ LOG_WRN("%s: *****************************************************************\n", __func__);
+ LOG_WRN("%s: WARNING: the mmproj graph uses unsupported operators by the backend\n", __func__);
+ LOG_WRN("%s: the performance will be suboptimal \n", __func__);
+ LOG_WRN("%s: list of unsupported ops (backend=%s):\n", __func__, ggml_backend_name(backend));
+ for (const auto & op : unsupported_ops) {
+ LOG_WRN("%s: %16s: type = %s, ne = [%d %d %d %d]\n", __func__,
+ ggml_op_name(op.op->op),
+ ggml_type_name(op.op->type),
+ op.op->ne[0], op.op->ne[1], op.op->ne[2], op.op->ne[3]);
+ }
+ LOG_WRN("%s: flash attention is %s\n", __func__,
+ (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled");
+ LOG_WRN("%s: please report this on github as an issue\n", __func__);
+ LOG_WRN("%s: ref: https://github.com/ggml-org/llama.cpp/pull/16837#issuecomment-3461676118\n", __func__);
+ LOG_WRN("%s: *****************************************************************\n", __func__);
+ }
+ }
+}
+
+mtmd_subcontext::support_info_graph mtmd_subcontext::reserve_compute_meta(const mtmd_image_f32_batch & batch) {
+ ggml_cgraph * gf = mtmd_get_graph_builder(this, batch)->build();
+ ggml_backend_sched_reserve(sched.get(), gf);
+
+ mem_compute.clear();
+ for (size_t i = 0; i < backend_ptrs.size(); ++i) {
+ ggml_backend_t backend = backend_ptrs[i];
+ ggml_backend_buffer_type_t buft = backend_buft[i];
+ size_t size = ggml_backend_sched_get_buffer_size(sched.get(), backend);
+ if (size > 1) {
+ LOG_INF("%s: %10s compute buffer size = %8.2f MiB\n", __func__,
+ ggml_backend_buft_name(buft),
+ size / 1024.0 / 1024.0);
+ }
+ mem_compute[ggml_backend_get_device(backend)] += size;
+ }
+
+ const int n_splits = ggml_backend_sched_get_n_splits(sched.get());
+ const int n_nodes = ggml_graph_n_nodes(gf);
+
+ LOG_INF("%s: graph splits = %d, nodes = %d\n", __func__, n_splits, n_nodes);
+
+ support_info_graph res {
+ /*.fattn = */ true,
+ /*.fattn_op = */ nullptr,
+ /*.ops = */ {},
+ };
+
+ // check op support
+ for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
+ ggml_tensor * node = ggml_graph_node(gf, i);
+ res.ops.push_back({node, true});
+ if (!ggml_backend_supports_op(backend, node)) {
+ res.ops.back().is_accel = false;
+ if (node->op == GGML_OP_FLASH_ATTN_EXT) {
+ res.fattn = false;
+ res.fattn_op = node;
+ }
+ }
+ }
+
+ return res;
+}
+
+// TODO: remove this and use ggml_mul_mat with GGML_PREC_F32 once all backends honor it for F32 x F32
+// ggml_mul_mat for 2D F32 tensors, always in F32 on CPU: some GPU backends round the inputs to F16 or TF32, which loses the quiet bins of a spectrum
+static void mtmd_mul_mat_prec_op(ggml_tensor * dst, int ith, int nth, void * /*userdata*/) {
+ const ggml_tensor * a = dst->src[0];
+ const ggml_tensor * b = dst->src[1];
+ const int64_t n_k = a->ne[0];
+
+ for (int64_t n = ith; n < b->ne[1]; n += nth) {
+ const float * b_row = (const float *) ((const char *) b->data + n * b->nb[1]);
+ float * dst_row = (float *) ((char *) dst->data + n * dst->nb[1]);
+ for (int64_t m = 0; m < a->ne[1]; m++) {
+ const float * a_row = (const float *) ((const char *) a->data + m * a->nb[1]);
+ // independent partial sums, so that the compiler can vectorize the loop
+ float acc[16] = {};
+ int64_t k = 0;
+ for (; k + 16 <= n_k; k += 16) {
+ for (int l = 0; l < 16; l++) {
+ acc[l] += a_row[k + l] * b_row[k + l];
+ }
+ }
+ float sum = 0.0f;
+ for (; k < n_k; k++) {
+ sum += a_row[k] * b_row[k];
+ }
+ for (int l = 0; l < 16; l++) {
+ sum += acc[l];
+ }
+ dst_row[m] = sum;
+ }
+ }
+}
+
+static ggml_tensor * mtmd_mul_mat_prec(ggml_context * ctx, ggml_tensor * a, ggml_tensor * b) {
+ GGML_ASSERT(a->type == GGML_TYPE_F32 && b->type == GGML_TYPE_F32);
+ GGML_ASSERT(ggml_n_dims(a) <= 2 && ggml_n_dims(b) <= 2 && a->ne[0] == b->ne[0]);
+ GGML_ASSERT(a->nb[0] == sizeof(float) && b->nb[0] == sizeof(float));
+ ggml_tensor * args[] = { a, b };
+ return ggml_custom_4d(ctx, GGML_TYPE_F32, a->ne[1], b->ne[1], 1, 1, args, 2, mtmd_mul_mat_prec_op, GGML_N_TASKS_MAX, nullptr);
+}
+
+bool mtmd_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out) {
+ ggml_backend_sched_t sched = config.sched;
+ ggml_backend_t backend_cpu = ggml_backend_sched_get_backend(sched, ggml_backend_sched_get_n_backends(sched) - 1);
+ mtmd_backend_cpu_set_n_threads(backend_cpu, std::max(1, config.n_threads));
+
+ const auto & cache = *config.cache;
+ const auto & filters = cache.filters.data;
+ const int64_t n_fft_bins = config.n_fft_bins;
+ const int64_t frame_size = (n_fft_bins - 1) * 2;
+ const int64_t n_mel = config.n_mel;
+ GGML_ASSERT(filters.size() == (size_t) (n_mel * n_fft_bins));
+ GGML_ASSERT(cache.cos_vals.size() == (size_t) frame_size && cache.sin_vals.size() == (size_t) frame_size);
+
+ // DFT matrix with the window folded in, the first n_fft_bins rows give the real part
+ std::vector<float> dft((size_t) 2 * n_fft_bins * frame_size);
+ for (int64_t k = 0; k < n_fft_bins; k++) {
+ for (int64_t j = 0; j < frame_size; j++) {
+ const int64_t q = (j * k) % frame_size;
+ dft[k * frame_size + j] = window[j] * cache.cos_vals[q];
+ dft[(k + n_fft_bins) * frame_size + j] = window[j] * cache.sin_vals[q];
+ }
+ }
+
+ // the spectrum is several times larger than the result, so long audio is processed in blocks
+ const int64_t n_frames_max = 8192;
+
+ for (int64_t off = 0; off < n_frames; off += n_frames_max) {
+ const int64_t n_cur = std::min(n_frames_max, n_frames - off);
+
+ ggml_init_params params = {
+ /*.mem_size =*/ ggml_tensor_overhead() * 32 + ggml_graph_overhead(),
+ /*.mem_buffer =*/ nullptr,
+ /*.no_alloc =*/ true,
+ };
+ ggml_context_ptr ctx_ptr(ggml_init(params));
+ ggml_context * ctx0 = ctx_ptr.get();
+
+ ggml_tensor * inp_samples = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, (n_cur - 1) * config.hop_length + frame_size);
+ ggml_tensor * inp_dft = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, frame_size, 2 * n_fft_bins);
+ ggml_tensor * inp_filters = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_fft_bins, n_mel);
+ ggml_set_input(inp_samples);
+ ggml_set_input(inp_dft);
+ ggml_set_input(inp_filters);
+
+ ggml_tensor * cur = ggml_cont(ctx0, ggml_view_2d(ctx0, inp_samples, frame_size, n_cur, config.hop_length * ggml_element_size(inp_samples), 0));
+ cur = mtmd_mul_mat_prec(ctx0, inp_dft, cur); // [2 * n_fft_bins, n_cur]
+
+ cur = ggml_sqr(ctx0, cur);
+ cur = ggml_add(ctx0,
+ ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], 0),
+ ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], n_fft_bins * ggml_element_size(cur)));
+ if (config.use_magnitude) {
+ cur = ggml_sqrt(ctx0, cur);
+ }
+
+ cur = mtmd_mul_mat_prec(ctx0, cur, inp_filters); // [n_cur, n_mel]
+ cur = config.mel_floor_add
+ ? ggml_scale_bias(ctx0, cur, 1.0f, config.mel_floor)
+ : ggml_clamp(ctx0, cur, config.mel_floor, FLT_MAX);
+ cur = ggml_log(ctx0, cur);
+ if (!config.use_natural_log) {
+ cur = ggml_scale(ctx0, cur, 1.0f / logf(10.0f));
+ }
+ ggml_set_output(cur);
+
+ ggml_cgraph * gf = ggml_new_graph(ctx0);
+ ggml_build_forward_expand(gf, cur);
+
+ ggml_backend_sched_reset(sched);
+ if (!ggml_backend_sched_alloc_graph(sched, gf)) {
+ LOG_ERR("%s: failed to allocate compute graph\n", __func__);
+ return false;
+ }
+
+ ggml_backend_tensor_set(inp_samples, samples + off * config.hop_length, 0, ggml_nbytes(inp_samples));
+ ggml_backend_tensor_set(inp_dft, dft.data(), 0, ggml_nbytes(inp_dft));
+ ggml_backend_tensor_set(inp_filters, filters.data(), 0, ggml_nbytes(inp_filters));
+
+ auto status = ggml_backend_sched_graph_compute(sched, gf);
+ if (status != GGML_STATUS_SUCCESS) {
+ LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status);
+ return false;
+ }
+
+ for (int64_t j = 0; j < n_mel; j++) {
+ ggml_backend_tensor_get(cur, out + j * n_frames + off, j * cur->nb[1], n_cur * ggml_element_size(cur));
+ }
+ }
+
+ return true;
+}
+
+bool mtmd_subcontext::encode_image(int n_threads, const mtmd_image_f32 * img, std::vector<float> & out_vec) {
+ mtmd_image_f32_batch imgs;
+ mtmd_image_f32 img_copy = *img;
+ imgs.entries.push_back(std::move(img_copy));
+
+ return encode_batch(n_threads, &imgs, out_vec);
+}
+
+bool mtmd_subcontext::encode_batch(int n_threads, const mtmd_image_f32_batch * imgs_c_ptr, std::vector<float> & out_batch_embd) {
+ mtmd_subctx_encode_params params;
+ params.imgs = imgs_c_ptr;
+ params.n_threads = n_threads;
+ params.out_embd = &out_batch_embd;
+
+ if (model.proj_type == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) {
+ return mtmd_graph_nemotron_transcribe::encode_windows(this, model, ¶ms);
+ }
+ return encode(¶ms);
+}
+
+// persisted state slots of the gen-audio decoders and the streaming audio encoders
+static std::vector<c2w_state_slot> list_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
+ switch (model.proj_type) {
+ case PROJECTOR_TYPE_QWEN3TTS_GEN: return list_c2w_state_slots(hparams, model);
+ case PROJECTOR_TYPE_DRAMA3_GEN: return list_drama3_state_slots(hparams, model);
+ case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model);
+ case PROJECTOR_TYPE_LFM3A_GEN: return list_lfm3a_gen_state_slots(hparams, model);
+ default: return {};
+ }
+}
+
+// state_out can have another shape than its slot, only the byte size must match
+static void copy_gen_state_out(ggml_tensor * src, ggml_tensor * dst) {
+ GGML_ASSERT(ggml_nbytes(src) == ggml_nbytes(dst));
+ if (src->type == dst->type && ggml_are_same_shape(src, dst) && ggml_is_contiguous(src)) {
+ ggml_backend_tensor_copy(src, dst);
+ return;
+ }
+ std::vector<uint8_t> tmp(ggml_nbytes(src));
+ ggml_backend_tensor_get(src, tmp.data(), 0, tmp.size());
+ ggml_backend_tensor_set(dst, tmp.data(), 0, tmp.size());
+}
+
+mtmd_subctx_gen_state * mtmd_subcontext::gen_state_init(uint32_t seed) {
+ auto st = std::make_unique<mtmd_subctx_gen_state>();
+ st->rng.seed(seed == UINT32_MAX ? std::random_device{}() : seed);
+
+ const auto slots = list_gen_state_slots(model.hparams, model);
+ if (slots.empty()) {
+ return st.release();
+ }
+
+ ggml_init_params params = {
+ /*.mem_size =*/ ggml_tensor_overhead() * slots.size(),
+ /*.mem_buffer =*/ nullptr,
+ /*.no_alloc =*/ true,
+ };
+ st->ctx_data.reset(ggml_init(params));
+ for (const auto & slot : slots) {
+ ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1);
+ ggml_format_name(t, "gen_state_%s", slot.name.c_str());
+ st->slots.push_back(t);
+ }
+ st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backend));
+ if (!st->buf) {
+ LOG_ERR("%s: failed to allocate the gen state\n", __func__);
+ return nullptr;
+ }
+ // a zero state is the cold start
+ ggml_backend_buffer_clear(st->buf.get(), 0);
+
+ return st.release();
+}
+
+mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) {
+ const auto & hparams = model.hparams;
+ if (model.proj_type == PROJECTOR_TYPE_LFM3A) {
+ // downsampler context, short-conv taps and attention window, see list_lfm3a_state_slots()
+ const auto slots = list_lfm3a_state_slots(hparams, model);
+ auto st = std::make_unique<mtmd_subctx_enc_state>();
+ ggml_init_params params = {
+ /*.mem_size =*/ ggml_tensor_overhead() * slots.size(),
+ /*.mem_buffer =*/ nullptr,
+ /*.no_alloc =*/ true,
+ };
+ st->ctx_data.reset(ggml_init(params));
+ for (const auto & slot : slots) {
+ ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1);
+ ggml_format_name(t, "enc_state_%s", slot.name.c_str());
+ st->slots.push_back(t);
+ }
+ st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backend));
+ if (!st->buf) {
+ LOG_ERR("%s: failed to allocate the encoder state\n", __func__);
+ return nullptr;
+ }
+ // a zero state is the cold start
+ ggml_backend_buffer_clear(st->buf.get(), 0);
+ return st.release();
+ }
+ if (model.proj_type != PROJECTOR_TYPE_VOXTRAL_RT || hparams.attn_window_size <= 0) {
+ LOG_ERR("%s: the encoder does not support streaming input\n", __func__);
+ return nullptr;
+ }
+ if (n_frames_max <= 0) {
+ LOG_ERR("%s: invalid n_frames_max %d\n", __func__, n_frames_max);
+ return nullptr;
+ }
+
+ const int n_layer = hparams.n_layer;
+ const int d_head = hparams.n_embd_head > 0 ? hparams.n_embd_head : hparams.n_embd / hparams.n_head;
+ const int n_embd_kv = hparams.n_head_kv * d_head;
+
+ auto st = std::make_unique<mtmd_subctx_enc_state>();
+ st->n_ctx = hparams.attn_window_size + n_frames_max;
+ st->slot_pos.assign(st->n_ctx, -1);
+
+ ggml_init_params params = {
+ /*.mem_size =*/ ggml_tensor_overhead() * 2 * n_layer,
+ /*.mem_buffer =*/ nullptr,
+ /*.no_alloc =*/ true,
+ };
+ st->ctx_data.reset(ggml_init(params));
+ for (int il = 0; il < n_layer; il++) {
+ ggml_tensor * k = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F16, n_embd_kv, st->n_ctx);
+ ggml_tensor * v = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F16, n_embd_kv, st->n_ctx);
+ ggml_format_name(k, "enc_state_k_%d", il);
+ ggml_format_name(v, "enc_state_v_%d", il);
+ st->k.push_back(k);
+ st->v.push_back(v);
+ }
+ st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backend));
+ if (!st->buf) {
+ LOG_ERR("%s: failed to allocate the encoder state\n", __func__);
+ return nullptr;
+ }
+ // empty slots are masked out, but they must not hold NaN
+ ggml_backend_buffer_clear(st->buf.get(), 0);
+
+ LOG_INF("%s: encoder state: %d slots, %.2f MiB\n", __func__, st->n_ctx, ggml_backend_buffer_get_size(st->buf.get()) / 1024.0 / 1024.0);
+
+ return st.release();
+}
+
+bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
+ const mtmd_image_f32_batch & imgs = *params->imgs;
+ int n_batch_cur = imgs.entries.size();
+
+ // [QWEN_VIDEO] for video models, the batch dimension is used as temporal dimension for merged frames
+ if (!support_batch && n_batch_cur > model.n_temporal_merge()) {
+ LOG_ERR("%s: batch size %d exceeds maximum supported batch/temporal-merge size %d\n", __func__, n_batch_cur, model.n_temporal_merge());
+ return false;
+ }
+
+ // if buffers are not allocated, we need to do a warmup run to allocate them
+ if (!is_allocated) {
+ warmup(*params->imgs);
+ }
+
+ const bool needs_gen_state = params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE ||
+ params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ||
+ params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS;
+ if (needs_gen_state && params->gen_state == nullptr) {
+ LOG_ERR("%s: audio gen requires a gen state\n", __func__);
+ return false;
+ }
+ if (model.proj_type == PROJECTOR_TYPE_DRAMA3_GEN && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE) {
+ float pos = 0.0f;
+ ggml_backend_tensor_get(params->gen_state->slots[0], &pos, 0, sizeof(float));
+ if ((int) pos >= mtmd_graph_drama3_gen::n_pos_max) {
+ LOG_ERR("%s: the codebook tower holds at most %d rows\n", __func__, mtmd_graph_drama3_gen::n_pos_max);
+ return false;
+ }
+ }
+
+ // build the inference graph
+ ggml_backend_sched_reset(sched.get());
+ ggml_cgraph * gf = mtmd_get_graph_builder(this, imgs, params)->build();
+
+ // the last node is the embedding tensor, code2wav has no out_embd
+ // read it before scheduling: backends may reorder the nodes of graphs with several outputs (Metal)
+ ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr;
+
+ if (!ggml_backend_sched_alloc_graph(sched.get(), gf)) {
+ LOG_ERR("%s: failed to allocate compute graph\n", __func__);
+ return false;
+ }
+
+ // set inputs
+ const auto & hparams = model.hparams;
+
+ const int image_size_width = imgs.entries[0].nx();
+ const int image_size_height = imgs.entries[0].ny();
+
+ const int patch_size = hparams.patch_size;
+ const int num_patches = ((image_size_width / patch_size) * (image_size_height / patch_size));
+ const int n_pos = num_patches + (model.class_embedding ? 1 : 0);
+ const int pos_w = image_size_width / patch_size;
+ const int pos_h = image_size_height / patch_size;
+
+
+ auto get_inp_tensor = [&gf](const char * name) {
+ ggml_tensor * inp = ggml_graph_get_tensor(gf, name);
+ if (inp == nullptr) {
+ GGML_ABORT("Failed to get tensor %s", name);
+ }
+ if (!(inp->flags & GGML_TENSOR_FLAG_INPUT)) {
+ GGML_ABORT("Tensor %s is not an input tensor", name);
+ }
+ return inp;
+ };
+
+ auto set_input_f32 = [&get_inp_tensor](const char * name, const std::vector<float> & values) {
+ ggml_tensor * cur = get_inp_tensor(name);
+ GGML_ASSERT(cur->type == GGML_TYPE_F32);
+ GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size());
+ ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur));
+ };
+
+ // mask from build_inp_attn_mask(), f16 if flash attn is enabled
+ auto set_input_attn_mask = [&get_inp_tensor](const std::vector<float> & values, int idx = 0) {
+ ggml_tensor * cur = get_inp_tensor(get_attn_mask_name(idx).c_str());
+ GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size());
+ if (cur->type == GGML_TYPE_F16) {
+ std::vector<ggml_fp16_t> values_f16(values.size());
+ ggml_fp32_to_fp16_row(values.data(), values_f16.data(), values.size());
+ ggml_backend_tensor_set(cur, values_f16.data(), 0, ggml_nbytes(cur));
+ } else {
+ GGML_ASSERT(cur->type == GGML_TYPE_F32);
+ ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur));
+ }
+ };
+
+ auto set_input_i32 = [&get_inp_tensor](const char * name, std::vector<int32_t> & values) {
+ ggml_tensor * cur = get_inp_tensor(name);
+ GGML_ASSERT(cur->type == GGML_TYPE_I32);
+ GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size());
+ ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur));
+ };
+
+ // state of the previous call, zero on a cold start
+ auto set_gen_state_in = [&]() {
+ const auto & slots = params->gen_state->slots;
+ size_t i = 0;
+ for (const auto & slot : list_gen_state_slots(hparams, model)) {
+ GGML_ASSERT(i < slots.size());
+ if (slot.in_place) {
+ i++;
+ continue;
+ }
+ ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str()));
+ }
+ };
+
+ // lfm3a encoder and codec: causal sliding window over [cached window | this call]
+ auto set_lfm3a_window_inputs = [&](int base, int n_pos, int window) {
+ std::vector<int32_t> positions(n_pos);
+ for (int i = 0; i < n_pos; i++) {
+ positions[i] = base + i;
+ }
+ set_input_i32("positions", positions);
+
+ // key j holds position base - prefix + j
+ const int prefix = window - 1;
+ const int n_kv = prefix + n_pos;
+ std::vector<float> mask((size_t) n_kv * n_pos, -INFINITY);
+ for (int q = 0; q < n_pos; q++) {
+ for (int j = std::max(0, q + prefix - window + 1); j <= q + prefix; j++) {
+ if (base - prefix + j >= 0) {
+ mask[(size_t) q * n_kv + j] = 0.0f;
+ }
+ }
+ }
+ set_input_attn_mask(mask);
+ };
+
+ // rope positions and attention mask of the mimi transformers (pocket-tts).
+ // the mask is causal with a sliding window, see _build_attention_mask() in the reference
+ auto set_pockettts_tfm_inputs = [&]() {
+ const int64_t n_pos = ggml_nelements(get_inp_tensor("inp_pos"));
+ GGML_ASSERT(n_pos > 0);
+ std::vector<int32_t> positions((size_t) n_pos);
+ for (int64_t i = 0; i < n_pos; i++) {
+ positions[(size_t) i] = (int32_t) i;
+ }
+ set_input_i32("inp_pos", positions);
+
+ // the preprocessor truncates the waveform to keep this mask bounded
+ const int64_t max_pos = (int64_t) mtmd_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate / 120;
+ GGML_ASSERT(n_pos <= max_pos && "pocket-tts speaker reference too long for a dense mask");
+
+ const int64_t context = hparams.mimi_tfm_context;
+ std::vector<float> mask((size_t) n_pos * n_pos, -INFINITY);
+ for (int64_t q = 0; q < n_pos; q++) {
+ for (int64_t k = 0; k < n_pos; k++) {
+ const int64_t delta = q - k;
+ if (delta >= 0 && delta < context) {
+ mask[(size_t) q * n_pos + k] = 0.0f;
+ }
+ }
+ }
+ set_input_attn_mask(mask);
+ };
+
+ // set input pixel values
+ if (!imgs.is_audio) {
+ size_t nelem = 0;
+ for (const auto & img : imgs.entries) {
+ nelem += img.nx() * img.ny() * 3;
+ }
+ std::vector<float> inp_raw(nelem);
+
+ // layout of data (note: the channel dim is unrolled to better visualize the layout):
+ //
+ // ┌──W──┐
+ // │ H │ channel = R
+ // ├─────┤ │
+ // │ H │ channel = G
+ // ├─────┤ │
+ // │ H │ channel = B
+ // └─────┘ │
+ // ──────┘ x B
+
+ // IMPORTANT: [QWEN_VIDEO] the batch dim is currently used for temporal dim in Qwen-VL models
+ // All entries must have the same spatial size (enforced by can_batch_with() during merging)
+ {
+ const int nx = imgs.entries[0].nx();
+ const int ny = imgs.entries[0].ny();
+ const int n = nx * ny;
+
+ for (int b = 0; b < n_batch_cur; b++) {
+ LOG_DBG("%s: copying image %d/%d to input buffer (nx=%d, ny=%d)\n", __func__, b+1, n_batch_cur, nx, ny);
+ const auto & buf = imgs.entries[b].get_ro_buf();
+ float * batch_entry = inp_raw.data() + b * (3*n);
+ for (int y = 0; y < ny; y++) {
+ for (int x = 0; x < nx; x++) {
+ size_t base_src = 3*(y * nx + x);
+ size_t base_dst = y * nx + x;
+ batch_entry[ base_dst] = buf[base_src ];
+ batch_entry[1*n + base_dst] = buf[base_src + 1];
+ batch_entry[2*n + base_dst] = buf[base_src + 2];
+ }
+ }
+ }
+ }
+ set_input_f32("inp_raw", inp_raw);
+
+ } else if (params->gen_process != MTMD_SUBCTX_GEN_PROCESS_GEN_WAV &&
+ params->gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS &&
+ params->gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE &&
+ ggml_graph_get_tensor(gf, "inp_raw") != nullptr) {
+ // audio input. GEN_WAV and the chatterbox TTS stages are not here: they take codes or feats, set in the switch below.
+ // a GEN_CODE graph may have no hidden-state input (lfm3a_gen feedback embedding)
+ GGML_ASSERT(imgs.entries.size() == 1);
+
+ const auto & mel_inp = imgs.entries[0];
+ const auto & buf = mel_inp.get_ro_buf();
+ const int n_step = mel_inp.nx();
+ const int n_mel = mel_inp.ny();
+ GGML_ASSERT((size_t)n_step * n_mel == buf.size());
+
+ set_input_f32("inp_raw", buf);
+ }
+
+ // set input per projector
+ switch (model.proj_type) {
+ case PROJECTOR_TYPE_VOXTRAL_RT:
+ {
+ mtmd_subctx_enc_state * st = params->enc_state;
+ if (!st) {
+ LOG_ERR("%s: this encoder needs a mtmd_subctx_enc_state\n", __func__);
+ return false;
+ }
+ const int n_window = hparams.attn_window_size;
+ const int n_new = (imgs.entries[0].nx() - 2) / 2;
+ GGML_ASSERT(n_new <= st->n_ctx - n_window);
+
+ std::vector<int32_t> pos(n_new);
+ std::vector<int64_t> idx(n_new);
+ std::vector<int32_t> slot_pos = st->slot_pos; // including the frames written by this call
+ for (int i = 0; i < n_new; i++) {
+ pos[i] = st->n_past + i;
+ idx[i] = pos[i] % st->n_ctx;
+ slot_pos[idx[i]] = pos[i];
+ }
+ set_input_i32("inp_pos", pos);
+ ggml_backend_tensor_set(get_inp_tensor("inp_kv_idx"), idx.data(), 0, idx.size() * sizeof(int64_t));
+
+ // each frame sees itself and the n_window - 1 frames before it
+ std::vector<float> mask((size_t) st->n_ctx * n_new, -INFINITY);
+ for (int i = 0; i < n_new; i++) {
+ for (int s = 0; s < st->n_ctx; s++) {
+ const int32_t p = slot_pos[s];
+ if (p >= 0 && p <= pos[i] && p > pos[i] - n_window) {
+ mask[(size_t) i * st->n_ctx + s] = 0.0f;
+ }
+ }
+ }
+ set_input_attn_mask(mask);
+
+ std::vector<float> tail = st->stem_tail;
+ tail.resize(hparams.n_embd, 0.0f);
+ set_input_f32("inp_stem_tail", tail);
+ } break;
+ case PROJECTOR_TYPE_MUSE_GLIMMER:
+ {
+ const int grid_w = pos_w; // image_size_width / patch_size
+ const int grid_h = pos_h; // image_size_height / patch_size
+ const int n_tok = grid_w * grid_h;
+ const int pgrid = (int) std::sqrt((double) model.position_embeddings->ne[1]); // 32
+ const int f = hparams.n_merge; // downsample 2
+
+ // pixel patchify runs inside the graph via build_inp() (ggml_conv_2d);
+ // pos-emb bilinear interp via resize_position_embeddings().
+
+ // --- sparse window grouping (pgrid x pgrid windows) ---
+ const int win = pgrid;
+ const int nwin_h = (grid_h + win - 1) / win;
+ const int nwin_w = (grid_w + win - 1) / win;
+ std::vector<int32_t> sp_perm; sp_perm.reserve(n_tok);
+ std::vector<int> sp_slens;
+ for (int wy = 0; wy < nwin_h; wy++) {
+ for (int wx = 0; wx < nwin_w; wx++) {
+ int cnt = 0;
+ for (int hh = 0; hh < win; hh++) {
+ for (int ww = 0; ww < win; ww++) {
+ const int gy = wy * win + hh;
+ const int gx = wx * win + ww;
+ if (gy < grid_h && gx < grid_w) { sp_perm.push_back(gy * grid_w + gx); cnt++; }
+ }
+ }
+ if (cnt > 0) sp_slens.push_back(cnt);
+ }
+ }
+ std::vector<int32_t> rpos_w(n_tok), rpos_h(n_tok), inv_perm(n_tok);
+ for (int i = 0; i < n_tok; i++) {
+ const int orig = sp_perm[i];
+ rpos_w[i] = (orig % grid_w) + 1; // 1-indexed
+ rpos_h[i] = (orig / grid_w) + 1;
+ inv_perm[orig] = i;
+ }
+ set_input_i32("muse_glimmer_sp_perm", sp_perm);
+ set_input_i32("muse_glimmer_inv_perm", inv_perm);
+ set_input_i32("muse_glimmer_pos_w", rpos_w);
+ set_input_i32("muse_glimmer_pos_h", rpos_h);
+
+ // block-diagonal window mask (permuted order)
+ std::vector<float> sp_mask((size_t) n_tok * n_tok, -INFINITY);
+ {
+ int off = 0;
+ for (int s : sp_slens) {
+ for (int a = 0; a < s; a++)
+ for (int b = 0; b < s; b++)
+ sp_mask[(size_t) (off + a) * n_tok + (off + b)] = 0.0f;
+ off += s;
+ }
+ }
+ set_input_attn_mask(sp_mask);
+
+ // pixel-shuffle gather (original order): f*f spatial neighbours grouped
+ std::vector<int32_t> dsp; dsp.reserve(n_tok);
+ for (int oy = 0; oy < grid_h / f; oy++)
+ for (int ox = 0; ox < grid_w / f; ox++)
+ for (int ry = 0; ry < f; ry++)
+ for (int rx = 0; rx < f; rx++)
+ dsp.push_back((oy * f + ry) * grid_w + (ox * f + rx));
+ set_input_i32("muse_glimmer_ds_perm", dsp);
+ } break;
+ case PROJECTOR_TYPE_MINICPMV:
+ {
+ // inspired from siglip:
+ // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit
+ // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316
+ std::vector<int32_t> positions(pos_h * pos_w);
+ // note: sized by the actual patch counts; a tall/wide image produces more
+ // than 1024 patches per side and a fixed [1024] array would be overrun
+ std::vector<int> bucket_coords_h(pos_h);
+ std::vector<int> bucket_coords_w(pos_w);
+ for (int i = 0; i < pos_h; i++){
+ bucket_coords_h[i] = std::floor(70.0*i/pos_h);
+ }
+ for (int i = 0; i < pos_w; i++){
+ bucket_coords_w[i] = std::floor(70.0*i/pos_w);
+ }
+ for (int i = 0, id = 0; i < pos_h; i++){
+ for (int j = 0; j < pos_w; j++){
+ positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j];
+ }
+ }
+ set_input_i32("positions", positions);
+
+ // inputs for resampler projector
+ // set the 2D positions (using float for sinusoidal embedding)
+ int n_patches_per_col = image_size_width / patch_size;
+ std::vector<float> pos_data(n_pos);
+ // dimension H
+ for (int i = 0; i < n_pos; i++) {
+ pos_data[i] = static_cast<float>(i / n_patches_per_col);
+ }
+ set_input_f32("pos_h", pos_data);
+ // dimension W
+ for (int i = 0; i < n_pos; i++) {
+ pos_data[i] = static_cast<float>(i % n_patches_per_col);
+ }
+ set_input_f32("pos_w", pos_data);
+ // base frequency omega
+ const float base_freq = 10000.0f;
+ const int n_embd_proj = model.n_mmproj_embd();
+ std::vector<float> omega(n_embd_proj / 4);
+ for (int i = 0; i < n_embd_proj / 4; ++i) {
+ omega[i] = 1.0f / std::pow(base_freq, static_cast<float>(i) / (n_embd_proj / 4));
+ }
+ set_input_f32("omega", omega);
+ } break;
+ case PROJECTOR_TYPE_MINICPMV4_6:
+ case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
+ {
+ const bool is_4x = hparams.n_merge == 2;
+
+ // SigLIP position buckets (same as resampler path)
+ std::vector<int32_t> positions(pos_h * pos_w);
+ std::vector<int> bucket_coords_h(pos_h);
+ std::vector<int> bucket_coords_w(pos_w);
+ for (int i = 0; i < pos_h; i++){
+ bucket_coords_h[i] = std::floor(70.0*i/pos_h);
+ }
+ for (int i = 0; i < pos_w; i++){
+ bucket_coords_w[i] = std::floor(70.0*i/pos_w);
+ }
+ for (int i = 0, id = 0; i < pos_h; i++){
+ for (int j = 0; j < pos_w; j++){
+ positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j];
+ }
+ }
+ set_input_i32("positions", positions);
+
+ const int half_h = pos_h / 2;
+ const int half_w = pos_w / 2;
+
+ auto make_ds_idx = [](int off_r, int off_c, int ds_h, int ds_w, int stride_w) {
+ std::vector<int32_t> idx(ds_h * ds_w);
+ for (int i = 0; i < ds_h; i++) {
+ for (int j = 0; j < ds_w; j++) {
+ idx[i * ds_w + j] = (2*i + off_r) * stride_w + (2*j + off_c);
+ }
+ }
+ return idx;
+ };
+
+ if (!is_4x) {
+ // window reorder indices for 2x2 windows
+ std::vector<int32_t> window_idx(n_pos);
+ std::vector<int32_t> inv_window_idx(n_pos);
+ {
+ int k = 0;
+ for (int wi = 0; wi < half_h; wi++) {
+ for (int wj = 0; wj < half_w; wj++) {
+ window_idx[k++] = (2*wi ) * pos_w + (2*wj );
+ window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1);
+ window_idx[k++] = (2*wi + 1) * pos_w + (2*wj );
+ window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1);
+ }
+ }
+ for (int i = 0; i < n_pos; i++) {
+ inv_window_idx[window_idx[i]] = i;
+ }
+ }
+ set_input_i32("vit_merger_window_idx", window_idx);
+ set_input_i32("vit_merger_inv_window_idx", inv_window_idx);
+
+ // block-diagonal attention mask: tokens in the same 4-token
+ // window attend to each other (mask = 0), all other positions
+ // are masked out (-inf). matches the window-major reorder above.
+ std::vector<float> window_mask_data(n_pos * n_pos, std::numeric_limits<float>::lowest());
+ for (int wi = 0; wi < n_pos / 4; wi++) {
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < 4; j++) {
+ window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f;
+ }
+ }
+ }
+ set_input_attn_mask(window_mask_data);
+
+ // ViT merger 2x2 downsample indices
+ auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w);
+ auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w);
+ auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w);
+ auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w);
+ set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0);
+ set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1);
+ set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2);
+ set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3);
+ }
+
+ const int merger_h = is_4x ? pos_h : half_h;
+ const int merger_w = is_4x ? pos_w : half_w;
+ auto m_ds_0 = make_ds_idx(0, 0, merger_h / 2, merger_w / 2, merger_w);
+ auto m_ds_1 = make_ds_idx(0, 1, merger_h / 2, merger_w / 2, merger_w);
+ auto m_ds_2 = make_ds_idx(1, 0, merger_h / 2, merger_w / 2, merger_w);
+ auto m_ds_3 = make_ds_idx(1, 1, merger_h / 2, merger_w / 2, merger_w);
+ set_input_i32("merger_ds_idx_0", m_ds_0);
+ set_input_i32("merger_ds_idx_1", m_ds_1);
+ set_input_i32("merger_ds_idx_2", m_ds_2);
+ set_input_i32("merger_ds_idx_3", m_ds_3);
+ } break;
+ case PROJECTOR_TYPE_LFM3VL:
+ {
+ // patches in raster-scan order (no interleave before encoder layers)
+ const int pw = image_size_width / patch_size;
+ const int ph = image_size_height / patch_size;
+ std::vector<int> positions(n_pos * 4);
+ int ptr = 0;
+ for (int y = 0; y < ph; y++) {
+ for (int x = 0; x < pw; x++) {
+ positions[ ptr] = y;
+ positions[ num_patches + ptr] = x;
+ positions[2 * num_patches + ptr] = y;
+ positions[3 * num_patches + ptr] = x;
+ ptr++;
+ }
+ }
+ set_input_i32("positions", positions);
+ } break;
+ case PROJECTOR_TYPE_MAGEVL:
+ {
+ // t, h, w in 2x2 block order, t is always 0 for an image
+ const int merge_ratio = hparams.n_merge;
+ const int pw = image_size_width / patch_size / merge_ratio;
+ const int ph = image_size_height / patch_size / merge_ratio;
+ std::vector<int> positions(n_pos * 4, 0);
+ int ptr = 0;
+ for (int ty = 0; ty < ph; ty++) {
+ for (int tx = 0; tx < pw; tx++) {
+ for (int dy = 0; dy < merge_ratio; dy++) {
+ for (int dx = 0; dx < merge_ratio; dx++) {
+ positions[ num_patches + ptr] = ty * merge_ratio + dy;
+ positions[2 * num_patches + ptr] = tx * merge_ratio + dx;
+ ptr++;
+ }
+ }
+ }
+ }
+ set_input_i32("positions", positions);
+ } break;
+ case PROJECTOR_TYPE_QWEN2VL:
+ case PROJECTOR_TYPE_QWEN3VL:
+ case PROJECTOR_TYPE_LING3VL:
+ case PROJECTOR_TYPE_GLM4V:
+ case PROJECTOR_TYPE_GLM5V:
+ {
+ const int merge_ratio = hparams.n_merge;
+ const int pw = image_size_width / patch_size / merge_ratio;
+ const int ph = image_size_height / patch_size / merge_ratio;
+ std::vector<int> positions(n_pos * 4);
+ int ptr = 0;
+ for (int ty = 0; ty < ph; ty++) {
+ for (int tx = 0; tx < pw; tx++) {
+ for (int dy = 0; dy < merge_ratio; dy++) {
+ for (int dx = 0; dx < merge_ratio; dx++) {
+ const int y = ty * merge_ratio + dy;
+ const int x = tx * merge_ratio + dx;
+ positions[ ptr] = y;
+ positions[ num_patches + ptr] = x;
+ positions[2 * num_patches + ptr] = y;
+ positions[3 * num_patches + ptr] = x;
+ ptr++;
+ }
+ }
+ }
+ }
+
+ set_input_i32("positions", positions);
+ } break;
+ case PROJECTOR_TYPE_STEP3VL:
+ {
+ std::vector<int32_t> pos_data(n_pos);
+ for (int i = 0; i < n_pos; i++) {
+ pos_data[i] = i / pos_w;
+ }
+ set_input_i32("pos_h", pos_data);
+ for (int i = 0; i < n_pos; i++) {
+ pos_data[i] = i % pos_w;
+ }
+ set_input_i32("pos_w", pos_data);
+ } break;
+ case PROJECTOR_TYPE_PADDLEOCR:
+ {
+ const int merge_ratio = hparams.n_merge;
+ const int pw = image_size_width / patch_size / merge_ratio;
+ const int ph = image_size_height / patch_size / merge_ratio;
+ std::vector<int> positions(n_pos * 4);
+ int ptr = 0;
+ // NOTE: same as Qwen-VL, but x and y are swapped
+ for (int ty = 0; ty < ph; ty++) {
+ for (int dy = 0; dy < merge_ratio; dy++) {
+ for (int tx = 0; tx < pw; tx++) {
+ for (int dx = 0; dx < merge_ratio; dx++) {
+ const int y = ty * merge_ratio + dy;
+ const int x = tx * merge_ratio + dx;
+ positions[ ptr] = y;
+ positions[ num_patches + ptr] = x;
+ positions[2 * num_patches + ptr] = y;
+ positions[3 * num_patches + ptr] = x;
+ ptr++;
+ }
+ }
+ }
+ }
+
+ set_input_i32("positions", positions);
+ } break;
+ case PROJECTOR_TYPE_MINIMAX_M3:
+ {
+ const int n_merge = hparams.n_merge;
+ const int gh = image_size_height / patch_size;
+ const int gw = image_size_width / patch_size;
+ std::vector<int32_t> pos_h, pos_w;
+ pos_h.reserve(gh * gw);
+ pos_w.reserve(gh * gw);
+ for (int bh = 0; bh < gh / n_merge; bh++)
+ for (int bw = 0; bw < gw / n_merge; bw++)
+ for (int mh = 0; mh < n_merge; mh++)
+ for (int mw = 0; mw < n_merge; mw++) {
+ pos_h.push_back(bh * n_merge + mh);
+ pos_w.push_back(bw * n_merge + mw);
+ }
+ set_input_i32("minimax_pos_h", pos_h);
+ set_input_i32("minimax_pos_w", pos_w);
+ } break;
+ case PROJECTOR_TYPE_DOTS_OCR:
+ case PROJECTOR_TYPE_DOTS3NOTE_V:
+ {
+ const int pw = image_size_width / patch_size;
+ const int ph = image_size_height / patch_size;
+ const int n_pos = ph * pw;
+ std::vector<int> positions(n_pos * 4);
+ int ptr = 0;
+
+ // flat layout: [h, w, h, w] for each patch
+ // patches are in raster order (matching conv2d output)
+ for (int y = 0; y < ph; y++) {
+ for (int x = 0; x < pw; x++) {
+ positions[ ptr] = y;
+ positions[ n_pos + ptr] = x;
+ positions[2*n_pos + ptr] = y;
+ positions[3*n_pos + ptr] = x;
+ ptr++;
+ }
+ }
+
+ set_input_i32("positions", positions);
+ } break;
+ case PROJECTOR_TYPE_QWEN25VL:
+ case PROJECTOR_TYPE_EXAONE4_5:
+ case PROJECTOR_TYPE_YOUTUVL:
+ {
+ // pw * ph = number of tokens output by ViT after apply patch merger
+ // ipw * ipw = number of vision token been processed inside ViT
+ const bool use_window_attn =
+ (model.proj_type == PROJECTOR_TYPE_QWEN25VL || model.proj_type == PROJECTOR_TYPE_EXAONE4_5)
+ ? hparams.n_wa_pattern > 0
+ : !hparams.wa_layer_indexes.empty();
+ const int merge_ratio = 2;
+ const int pw = image_size_width / patch_size / merge_ratio;
+ const int ph = image_size_height / patch_size / merge_ratio;
+ const int ipw = image_size_width / patch_size;
+ const int iph = image_size_height / patch_size;
+
+ std::vector<int> idx (ph * pw);
+ std::vector<int> inv_idx(ph * pw);
+
+ if (use_window_attn) {
+ const int attn_window_size = hparams.attn_window_size > 0 ? hparams.attn_window_size : 112;
+ const int grid_window = attn_window_size / patch_size / merge_ratio;
+ int dst = 0;
+ // [num_vision_tokens, num_vision_tokens] attention mask tensor
+ std::vector<float> mask(pow(ipw * iph, 2), std::numeric_limits<float>::lowest());
+ int mask_row = 0;
+
+ for (int y = 0; y < ph; y += grid_window) {
+ for (int x = 0; x < pw; x += grid_window) {
+ const int win_h = std::min(grid_window, ph - y);
+ const int win_w = std::min(grid_window, pw - x);
+ const int dst_0 = dst;
+ // group all tokens belong to the same window togather (to a continue range)
+ for (int dy = 0; dy < win_h; dy++) {
+ for (int dx = 0; dx < win_w; dx++) {
+ const int src = (y + dy) * pw + (x + dx);
+ GGML_ASSERT(src < (int)idx.size());
+ GGML_ASSERT(dst < (int)inv_idx.size());
+ idx [src] = dst;
+ inv_idx[dst] = src;
+ dst++;
+ }
+ }
+
+ for (int r=0; r < win_h * win_w * merge_ratio * merge_ratio; r++) {
+ int row_offset = mask_row * (ipw * iph);
+ std::fill(
+ mask.begin() + row_offset + (dst_0 * merge_ratio * merge_ratio),
+ mask.begin() + row_offset + (dst * merge_ratio * merge_ratio),
+ 0.0);
+ mask_row++;
+ }
+ }
+ }
+
+ set_input_i32("window_idx", idx);
+ set_input_i32("inv_window_idx", inv_idx);
+ set_input_attn_mask(mask);
+ } else {
+ for (int i = 0; i < ph * pw; i++) {
+ idx[i] = i;
+ }
+ }
+
+ const int mpow = merge_ratio * merge_ratio;
+ std::vector<int> positions(n_pos * 4);
+
+ int ptr = 0;
+ for (int ty = 0; ty < ph; ty++) {
+ for (int tx = 0; tx < pw; tx++) {
+ for (int dy = 0; dy < merge_ratio; dy++) {
+ for (int dx = 0; dx < merge_ratio; dx++) {
+ auto remap = idx[ptr / mpow];
+ remap = (remap * mpow) + (ptr % mpow);
+
+ const int y = ty * merge_ratio + dy;
+ const int x = tx * merge_ratio + dx;
+ positions[ remap] = y;
+ positions[ num_patches + remap] = x;
+ positions[2 * num_patches + remap] = y;
+ positions[3 * num_patches + remap] = x;
+ ptr++;
+ }
+ }
+ }
+ }
+
+ set_input_i32("positions", positions);
+ } break;
+ case PROJECTOR_TYPE_MIMOVL:
+ {
+ const int merge = hparams.n_merge; // 2
+ const int merge_unit = merge * merge; // 4
+ const int patch = hparams.patch_size; // 16
+ const int H = image_size_height / patch;
+ const int W = image_size_width / patch;
+ const int n_pos_full = H * W;
+ const int llm_h = H / merge;
+ const int llm_w = W / merge;
+ const int n_units = llm_h * llm_w; // n_pos / merge_unit
+
+ // Row-major merge-tile-ordered (h, w) positions
+ std::vector<int32_t> pos_h_row(n_pos_full);
+ std::vector<int32_t> pos_w_row(n_pos_full);
+ {
+ int idx = 0;
+ for (int ty = 0; ty < llm_h; ty++) {
+ for (int tx = 0; tx < llm_w; tx++) {
+ for (int dy = 0; dy < merge; dy++) {
+ for (int dx = 0; dx < merge; dx++) {
+ pos_h_row[idx] = ty * merge + dy;
+ pos_w_row[idx] = tx * merge + dx;
+ idx++;
+ }
+ }
+ }
+ }
+ }
+
+ // Col-major merge-unit permutation
+ std::vector<float> idx_col(n_units);
+ for (int r = 0; r < llm_h; r++) {
+ for (int c = 0; c < llm_w; c++) {
+ int u_row = r * llm_w + c;
+ int u_col = c * llm_h + r;
+ idx_col[u_col] = (float) u_row;
+ }
+ }
+
+ // Col-mode positions: permute pos_*_row by idx_col
+ std::vector<int32_t> pos_h_col(n_pos_full);
+ std::vector<int32_t> pos_w_col(n_pos_full);
+ for (int u = 0; u < n_units; u++) {
+ int src = (int) idx_col[u];
+ for (int k = 0; k < merge_unit; k++) {
+ pos_h_col[u * merge_unit + k] = pos_h_row[src * merge_unit + k];
+ pos_w_col[u * merge_unit + k] = pos_w_row[src * merge_unit + k];
+ }
+ }
+
+ // Pack into ggml_rope_multi VISION-mode layout. The non-CPU kernels
+ // only read slots 0 and 1, so pack h in slot 0, w in slot 1:
+ // positions[0..n_pos) = h
+ // positions[n_pos..2*n_pos) = w
+ // positions[2*n_pos..3*n_pos) = 0
+ // positions[3*n_pos..4*n_pos) = 0
+ std::vector<int32_t> positions_row(static_cast<size_t>(n_pos_full) * 4, 0);
+ std::vector<int32_t> positions_col(static_cast<size_t>(n_pos_full) * 4, 0);
+ for (int i = 0; i < n_pos_full; i++) {
+ positions_row[0 * n_pos_full + i] = pos_h_row[i];
+ positions_row[1 * n_pos_full + i] = pos_w_row[i];
+ positions_col[0 * n_pos_full + i] = pos_h_col[i];
+ positions_col[1 * n_pos_full + i] = pos_w_col[i];
+ }
+
+ // Banded 1D sliding-window mask
+ const int window = hparams.attn_window_size;
+ GGML_ASSERT(window > 0);
+ std::vector<float> mask(static_cast<size_t>(n_pos_full) * n_pos_full, std::numeric_limits<float>::lowest());
+ for (int q = 0; q < n_pos_full; q++) {
+ int lo = std::max(0, q - window);
+ int hi = std::min(n_pos_full - 1, q + window);
+ for (int k = lo; k <= hi; k++) {
+ mask[static_cast<size_t>(q) * n_pos_full + k] = 0.0f;
+ }
+ }
+
+ set_input_i32("mimovl_positions_row", positions_row);
+ set_input_i32("mimovl_positions_col", positions_col);
+ set_input_f32("mimovl_idx_col", idx_col);
+ set_input_attn_mask(mask);
+ } break;
+ case PROJECTOR_TYPE_PIXTRAL:
+ case PROJECTOR_TYPE_KIMIVL:
+ case PROJECTOR_TYPE_KIMIK25:
+ case PROJECTOR_TYPE_LIGHTONOCR:
+ {
+ // set the 2D positions
+ int n_patches_per_col = image_size_width / patch_size;
+ std::vector<int> pos_data(n_pos);
+ // dimension H
+ for (int i = 0; i < n_pos; i++) {
+ pos_data[i] = i / n_patches_per_col;
+ }
+ set_input_i32("pos_h", pos_data);
+ // dimension W
+ for (int i = 0; i < n_pos; i++) {
+ pos_data[i] = i % n_patches_per_col;
+ }
+ set_input_i32("pos_w", pos_data);
+ } break;
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ {
+ const int n_patches_per_row = image_size_width / patch_size;
+ std::vector<int> pos_data(n_pos);
+ for (int i = 0; i < n_pos; i++) {
+ pos_data[i] = i / n_patches_per_row;
+ }
+ set_input_i32("pos_y", pos_data);
+ for (int i = 0; i < n_pos; i++) {
+ pos_data[i] = i % n_patches_per_row;
+ }
+ set_input_i32("pos_x", pos_data);
+ } break;
+ case PROJECTOR_TYPE_DEEPSEEK4V:
+ {
+ // set the 2D positions (mrope layout, only the first 2 channels are used)
+ int n_patches_per_row = image_size_width / patch_size;
+ std::vector<int32_t> positions(n_pos * 4, 0);
+ for (int i = 0; i < n_pos; i++) {
+ positions[i] = i / n_patches_per_row; // row
+ positions[n_pos + i] = i % n_patches_per_row; // col
+ }
+ set_input_i32("positions", positions);
+
+ // token block layout index (see mtmd_graph_deepseek4v::build)
+ // rows [0, n_grid) are the aligner output, the sentinels follow
+ const int n_merge = hparams.n_merge;
+ const int n_llm_w = MTMD_ALIGN(pos_w, n_merge) / n_merge;
+ const int n_llm_h = MTMD_ALIGN(pos_h, n_merge) / n_merge;
+ const int n_grid = n_llm_w * n_llm_h;
+ const int idx_start = n_grid;
+ const int idx_end = n_grid + 1;
+ const int idx_newline = n_grid + 2;
+ const int idx_pad = n_grid + 3;
+
+ const int lead_pad = imgs.entries[0].lead_pad;
+ const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad);
+
+ std::vector<int32_t> idx;
+ idx.reserve(bl.n_out);
+ for (int i = 0; i < lead_pad; i++) {
+ idx.push_back(idx_pad);
+ }
+ idx.push_back(idx_start);
+ // pairs of adjacent rows are interleaved column-wise ("N-layout")
+ // ref: build_image_block in inference/image_processor.py
+ for (int t = 0; t < bl.rows * bl.row_len; t++) {
+ const int g = t / (2 * bl.row_len);
+ const int rem = t % (2 * bl.row_len);
+ const int c = rem / 2; // column
+ const int r = 2 * g + rem % 2; // row
+ if (r >= n_llm_h) {
+ idx.push_back(idx_pad);
+ } else if (c == n_llm_w) {
+ idx.push_back(idx_newline);
+ } else {
+ idx.push_back(r * n_llm_w + c);
+ }
+ }
+ for (int i = 0; i < bl.pad_last; i++) {
+ idx.push_back(idx_pad);
+ }
+ idx.push_back(idx_end);
+ set_input_i32("layout_idx", idx);
+ } break;
+ case PROJECTOR_TYPE_DEEPSEEK41V:
+ {
+ // mrope layout, only the first 2 channels are used
+ const int n_patches_per_row = image_size_width / patch_size;
+ std::vector<int32_t> positions(n_pos * 4, 0);
+ for (int i = 0; i < n_pos; i++) {
+ positions[i] = i / n_patches_per_row;
+ positions[n_pos + i] = i % n_patches_per_row;
+ }
+ set_input_i32("positions", positions);
+
+ // reading-order block: rows [0, n_grid) are the aligner output, then start / end / newline
+ const int n_merge = hparams.n_merge;
+ const int n_llm_w = MTMD_ALIGN(pos_w, n_merge) / n_merge;
+ const int n_llm_h = MTMD_ALIGN(pos_h, n_merge) / n_merge;
+ const int n_grid = n_llm_w * n_llm_h;
+ std::vector<int32_t> idx;
+ idx.reserve(dsv41_n_block_tokens(n_llm_w, n_llm_h));
+ idx.push_back(n_grid); // IMAGE_START
+ for (int r = 0; r < n_llm_h; r++) {
+ for (int c = 0; c < n_llm_w; c++) {
+ idx.push_back(r * n_llm_w + c);
+ }
+ idx.push_back(n_grid + 2); // IMAGE_NEW_LINE
+ }
+ idx.push_back(n_grid + 1); // IMAGE_END
+ set_input_i32("layout_idx", idx);
+ } break;
+ case PROJECTOR_TYPE_GLM_EDGE:
+ {
+ // llava and other models
+ std::vector<int32_t> positions(n_pos);
+ for (int i = 0; i < n_pos; i++) {
+ positions[i] = i;
+ }
+ set_input_i32("positions", positions);
+ } break;
+ case PROJECTOR_TYPE_MLP:
+ case PROJECTOR_TYPE_MLP_NORM:
+ case PROJECTOR_TYPE_LDP:
+ case PROJECTOR_TYPE_LDPV2:
+ {
+ // llava and other models
+ std::vector<int32_t> positions(n_pos);
+ for (int i = 0; i < n_pos; i++) {
+ positions[i] = i;
+ }
+ set_input_i32("positions", positions);
+
+ // The patches vector is used to get rows to index into the embeds with;
+ // we should skip dim 0 only if we have CLS to avoid going out of bounds
+ // when retrieving the rows.
+ int patch_offset = model.class_embedding ? 1 : 0;
+ std::vector<int32_t> patches(num_patches);
+ for (int i = 0; i < num_patches; i++) {
+ patches[i] = i + patch_offset;
+ }
+ set_input_i32("patches", patches);
+ } break;
+ case PROJECTOR_TYPE_POCKETTTS_SPKENC:
+ {
+ set_pockettts_tfm_inputs();
+ } break;
+ case PROJECTOR_TYPE_POCKETTTS_GEN:
+ {
+ if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
+ GGML_ASSERT(params->feats != nullptr);
+ set_input_f32("inp_feats", *params->feats);
+ // positions and mask are derived in-graph from the persisted counter
+ set_gen_state_in();
+ } else {
+ // flow matching starts from gaussian noise, std = sqrt(temp)
+ ggml_tensor * t = get_inp_tensor("inp_noise");
+ // Config.default_temperature, for a caller that does not set one
+ const float temp = params->temp > 0.0f ? params->temp : 0.7f;
+ std::normal_distribution<float> dist(0.0f, std::sqrt(temp));
+ std::vector<float> noise(ggml_nelements(t));
+ for (auto & v : noise) {
+ v = dist(params->gen_state->rng);
+ }
+ set_input_f32("inp_noise", noise);
+ }
+ } break;
+ case PROJECTOR_TYPE_SOPRANO:
+ {
+ set_input_f32("inp_feats", *params->feats);
+ } break;
+ case PROJECTOR_TYPE_GEMMA4V:
+ case PROJECTOR_TYPE_GEMMA4UV:
+ {
+ // set (col, row) patch positions for learned positional embedding
+ const int n_cols = image_size_width / patch_size;
+ std::vector<int> pos_x(num_patches), pos_y(num_patches);
+ for (int i = 0; i < num_patches; i++) {
+ pos_x[i] = i % n_cols;
+ pos_y[i] = i / n_cols;
+ }
+ set_input_i32("pos_x", pos_x);
+ set_input_i32("pos_y", pos_y);
+ } break;
+ case PROJECTOR_TYPE_DEEPSEEKOCR:
+ case PROJECTOR_TYPE_DEEPSEEKOCR2:
+ {
+ GGML_ASSERT(
+ (pos_w == pos_h) // overview image
+ || (pos_h >= pos_w && pos_h % pos_w == 0) // tile images
+ );
+
+ const int window = hparams.attn_window_size;
+ const int pos = pos_w;
+ std::vector<int32_t> rel_pos_indices_local(window * window);
+ std::vector<int32_t> rel_pos_indices_global(pos * pos);
+
+ for (int q = 0; q < window; q++) {
+ for (int k = 0; k < window; k++) {
+ rel_pos_indices_local[q * window + k] = q - k + window - 1;
+ }
+ }
+
+ for (int q = 0; q < pos; q++) {
+ for (int k = 0; k < pos; k++) {
+ rel_pos_indices_global[q * pos + k] = q - k + pos - 1;
+ }
+ }
+
+ set_input_i32("rel_pos_indices_local", rel_pos_indices_local);
+ set_input_i32("rel_pos_indices_global", rel_pos_indices_global);
+
+ if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) {
+
+ // qwen2 encoder attention mask
+
+ // num_image_tokens = num_patches / 16
+ // 256 for 1024 global view
+ // 144 for 768 tile views
+ const int num_image_tokens = num_patches / 16;
+ const int seq_len = num_image_tokens * 2;
+ std::vector qwen2_mask(static_cast<size_t>(seq_len) * seq_len, 0.0f);
+
+ // attention mask layout
+ // +--------------+---------------+
+ // | all 0 | all -inf |
+ // +--------------+---------------+
+ // | all 0 | lower tri 0 |
+ // +--------------+---------------+
+ for (int i = 0; i < seq_len; i++) {
+ for (int j = 0; j < seq_len; j++) {
+ const bool zero = i < num_image_tokens ?
+ j < num_image_tokens :
+ j < num_image_tokens || j <= i;
+ qwen2_mask[static_cast<size_t>(i) * seq_len + j] = zero ? 0.0f : -1e9f;
+ }
+ }
+ set_input_attn_mask(qwen2_mask);
+ }
+ } break;
+ case PROJECTOR_TYPE_CHATTERBOX:
+ {
+ if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE) {
+ set_input_f32("inp_mel", *params->mel_in);
+ set_input_f32("inp_sstft", *params->sstft_in);
+ break;
+ }
+ if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
+ const int T1 = (imgs.entries[0].nx() - 1) / 2 + 1;
+ const int T2 = (T1 - 1) / 2 + 1;
+ std::vector<int32_t> pos(T2);
+ for (int i = 0; i < T2; i++) {
+ pos[(size_t) i] = i;
+ }
+ set_input_i32("inp_pos", pos);
+ break;
+ }
+ if (params->gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS) {
+ break;
+ }
+ const int n_gen = (int) params->codes->size();
+ GGML_ASSERT(params->ref);
+ const int n_prompt = (int) params->ref->tokens.size();
+ std::vector<int32_t> tokens = params->ref->tokens;
+ tokens.resize((size_t) n_prompt + n_gen);
+ memcpy(tokens.data() + n_prompt, params->codes->data(), (size_t) n_gen * sizeof(int32_t));
+ const int64_t n_codes = model.cbx.input_embedding_w->ne[1];
+ for (int32_t t : tokens) {
+ if (t < 0 || t >= n_codes) {
+ LOG_ERR("%s: speech token out of range (%d, codebook size %lld)\n", __func__, t, (long long) n_codes);
+ return false;
+ }
+ }
+ const int T1 = n_prompt + n_gen;
+ const int T2 = 2 * T1;
+ set_input_i32("inp_tokens", tokens);
+
+ // mel-rate reference conditioning of the voice
+ set_input_f32("inp_prompt_feat", params->ref->feat);
+ set_input_f32("inp_xvec", params->ref->spk);
+
+ // espnet relative positional encoding, entry k holds the sinusoid of relative position (T-1) - k
+ auto fill_pos = [&](const char * name, int T) {
+ const int d = 512;
+ std::vector<float> pos((size_t) (2 * T - 1) * d);
+ for (int k = 0; k < 2 * T - 1; k++) {
+ const double rel = (double) (T - 1 - k);
+ for (int i = 0; i < d / 2; i++) {
+ const double div = exp(-(double) (2 * i) * log(10000.0) / d);
+ pos[(size_t) k * d + 2 * i ] = (float) sin(rel * div);
+ pos[(size_t) k * d + 2 * i + 1] = (float) cos(rel * div);
+ }
+ }
+ set_input_f32(name, pos);
+ };
+ fill_pos("inp_pos1", T1);
+ fill_pos("inp_pos2", T2);
+
+ // noise and the time embeddings of the solver span points, matcha layout (sines then cosines, scale 1000)
+ std::vector<float> noise((size_t) 80 * T2);
+ std::normal_distribution<float> nd(0.0f, 1.0f);
+ for (auto & f : noise) f = nd(params->gen_state->rng);
+ set_input_f32("inp_noise", noise);
+
+ const bool meanflow = model.cbx.time_embed_mixer_w != nullptr;
+ const int n_steps = meanflow ? 2 : 10;
+ std::vector<float> temb((size_t) 320 * (n_steps + 1));
+ for (int s = 0; s <= n_steps; s++) {
+ const double u = (double) s / n_steps;
+ const double t = meanflow ? u : 1.0 - cos(u * M_PI / 2.0);
+ for (int i = 0; i < 160; i++) {
+ const double div = exp(-(double) i * log(10000.0) / 159.0);
+ temb[(size_t) s * 320 + i ] = (float) sin(1000.0 * t * div);
+ temb[(size_t) s * 320 + i + 160] = (float) cos(1000.0 * t * div);
+ }
+ }
+ set_input_f32("inp_temb", temb);
+ } break;
+ case PROJECTOR_TYPE_GEMMA3:
+ case PROJECTOR_TYPE_GEMMA3NV:
+ case PROJECTOR_TYPE_IDEFICS3:
+ case PROJECTOR_TYPE_COHERE2V:
+ case PROJECTOR_TYPE_INTERNVL:
+ case PROJECTOR_TYPE_NEMOTRON_V2_VL:
+ case PROJECTOR_TYPE_NEMOTRON_OMNI:
+ case PROJECTOR_TYPE_QWEN2A:
+ case PROJECTOR_TYPE_QWEN3A:
+ case PROJECTOR_TYPE_GLMA:
+ case PROJECTOR_TYPE_ULTRAVOX:
+ case PROJECTOR_TYPE_LFM2:
+ case PROJECTOR_TYPE_D1OMNI_V:
+ case PROJECTOR_TYPE_VOXTRAL:
+ case PROJECTOR_TYPE_MERALION:
+ case PROJECTOR_TYPE_MUSIC_FLAMINGO:
+ case PROJECTOR_TYPE_JANUS_PRO:
+ case PROJECTOR_TYPE_PHI4:
+ case PROJECTOR_TYPE_COGVLM:
+ case PROJECTOR_TYPE_YASA2:
+ case PROJECTOR_TYPE_GEMMA4UA:
+ case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
+ case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
+ {
+ // do nothing
+ } break;
+ case PROJECTOR_TYPE_CHATTERBOX_SPKENC:
+ {
+ // the ceil-mode seg pooling divides by the full segment length, rescale the last partial segment
+ const int T1 = (imgs.entries[0].nx() - 1) / 2 + 1;
+ const int S = (T1 + 99) / 100;
+ std::vector<float> segfix((size_t) S, 1.0f);
+ const int last = T1 - (S - 1) * 100;
+ segfix[(size_t) S - 1] = 100.0f / (float) last;
+ set_input_f32("inp_segfix", segfix);
+ } break;
+ case PROJECTOR_TYPE_LFM3A_GEN:
+ {
+ const int64_t n_cb = model.gen_code_embd_w->ne[2];
+ const int64_t n_vocab = model.gen_code_embd_w->ne[1];
+ if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
+ GGML_ASSERT(params->codes != nullptr);
+ // codes index the FSQ table via ggml_get_rows, the end-of-audio code has no waveform
+ const int64_t n_codes = model.c2w.quant_first_cb_w->ne[1];
+ for (int32_t c : *params->codes) {
+ if (c < 0 || c >= n_codes) {
+ LOG_ERR("%s: code out of range (%d, codebook size %lld)\n", __func__, c, (long long) n_codes);
+ return false;
+ }
+ }
+ std::vector<int32_t> codes = *params->codes;
+ set_input_i32("inp_codes", codes);
+ set_gen_state_in();
+ // the "pos" slot leads the state
+ float base = 0.0f;
+ ggml_backend_tensor_get(params->gen_state->slots[0], &base, 0, sizeof(float));
+ set_lfm3a_window_inputs((int) base, (int) ggml_nelements(get_inp_tensor("positions")), hparams.wav_tfm_swa);
+ } else {
+ // rows of the merged per-codebook tables; a masked codebook (-1) takes row 0, the mask drops it
+ std::vector<int32_t> codes(n_cb, -1);
+ if (params->codes) {
+ GGML_ASSERT(params->codes->size() == (size_t) n_cb);
+ codes = *params->codes;
+ }
+ std::vector<float> mask(n_cb, 0.0f);
+ for (int64_t i = 0; i < n_cb; i++) {
+ if (codes[i] >= n_vocab) {
+ LOG_ERR("%s: code out of range (%d, vocab %lld)\n", __func__, codes[i], (long long) n_vocab);
+ return false;
+ }
+ if (codes[i] < 0) {
+ codes[i] = 0;
+ } else {
+ mask[i] = 1.0f;
+ }
+ codes[i] += (int32_t) (i * n_vocab);
+ }
+ set_input_i32("inp_codes", codes);
+ if (ggml_graph_get_tensor(gf, "inp_mask") != nullptr) {
+ set_input_f32("inp_mask", mask);
+ }
+ }
+ } break;
+ case PROJECTOR_TYPE_DRAMA3_GEN:
+ {
+ const int64_t n_cb = model.gen_code_embd_w->ne[2];
+ const int64_t n_vocab = model.gen_code_embd_w->ne[1];
+ if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
+ // frame-major to group-major, clamp: codec residual codebooks are smaller than the tower vocab
+ GGML_ASSERT(params->codes != nullptr && params->codes->size() % n_cb == 0);
+ const int64_t n_frames = (int64_t) params->codes->size() / n_cb;
+ const int64_t n_first = model.c2w.quant_first_cb_w->ne[1];
+ const int64_t n_rest = model.c2w.quant_rest_cb_w->ne[1];
+ std::vector<int32_t> codes(params->codes->size());
+ for (int64_t f = 0; f < n_frames; f++) {
+ for (int64_t g = 0; g < n_cb; g++) {
+ const int64_t n = g == 0 ? n_first : n_rest;
+ codes[g * n_frames + f] = std::clamp<int32_t>((*params->codes)[f * n_cb + g], 0, (int32_t) n - 1);
+ }
+ }
+ set_input_i32("inp_codes", codes);
+ set_gen_state_in();
+ } else {
+ // code0 < 0: next row is text (prompt), the tower only fills its KV
+ // codes: input frame codes for a speech row, empty for a text row
+ const bool next_speech = params->code0 >= 0;
+ const bool cur_speech = params->codes && (int64_t) params->codes->size() == n_cb;
+ if (params->code0 >= n_vocab) {
+ LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) n_vocab);
+ return false;
+ }
+ std::vector<int32_t> code0 = { std::max(params->code0, 0) };
+ set_input_i32("inp_code0", code0);
+
+ std::vector<int32_t> extra(n_cb - 2);
+ for (int64_t g = 2; g < n_cb; g++) {
+ const int32_t c = cur_speech ? std::clamp<int32_t>((*params->codes)[g], 0, (int32_t) n_vocab - 1) : 0;
+ extra[g - 2] = c + (int32_t) (g * n_vocab);
+ }
+ set_input_i32("inp_extra", extra);
+ set_input_f32("inp_scale", { next_speech ? 1.0f : 0.0f, cur_speech ? 1.0f : 0.0f });
+
+ float pos = 0.0f;
+ ggml_backend_tensor_get(params->gen_state->slots[0], &pos, 0, sizeof(float));
+ std::vector<int32_t> inp_pos = { (int32_t) pos };
+ set_input_i32("inp_pos", inp_pos);
+
+ std::uniform_real_distribution<float> dist(0.0f, 1.0f);
+ std::vector<float> r(model.gen_code_head_w->ne[2]);
+ for (auto & v : r) {
+ v = dist(params->gen_state->rng);
+ }
+ set_input_f32("inp_rand", r);
+ }
+ } break;
+ case PROJECTOR_TYPE_QWEN3TTS_GEN:
+ {
+ if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
+ GGML_ASSERT(params->codes != nullptr);
+
+ // frame-major input to group-major, rear-padded with code 0 up to one window
+ const int64_t n_codes = model.gen_code_head_w->ne[2] + 1;
+ const int64_t n_frames_w = hparams.wav_tfm_swa;
+ const int64_t n_frames = (int64_t) params->codes->size() / n_codes;
+ GGML_ASSERT(n_frames > 0 && n_frames <= n_frames_w);
+
+ // codes are used as ggml_get_rows indices, so check them against the codebook vocab
+ const int64_t vocab_first = model.c2w.quant_first_cb_w->ne[1];
+ const int64_t vocab_rest = model.c2w.quant_rest_cb_w->ne[1];
+ for (int64_t f = 0; f < n_frames; f++) {
+ for (int64_t g = 0; g < n_codes; g++) {
+ const int32_t c = (*params->codes)[f * n_codes + g];
+ const int64_t vocab = (g == 0) ? vocab_first : vocab_rest;
+ if (c < 0 || (int64_t) c >= vocab) {
+ LOG_ERR("%s: code out of range (frame %lld, group %lld, code %d, vocab %lld)\n",
+ __func__, (long long) f, (long long) g, c, (long long) vocab);
+ return false;
+ }
+ }
+ }
+
+ std::vector<int32_t> codes(n_frames_w * n_codes, 0);
+ for (int64_t f = 0; f < n_frames; f++) {
+ for (int64_t g = 0; g < n_codes; g++) {
+ codes[g * n_frames_w + f] = (*params->codes)[f * n_codes + g];
+ }
+ }
+ set_input_i32("inp_codes", codes);
+ set_gen_state_in();
+ } else {
+ // code0 indexes gen_code_out_embd_w via ggml_get_rows; bound it
+ const int64_t vocab0 = model.gen_code_out_embd_w->ne[1];
+ if (params->code0 < 0 || (int64_t) params->code0 >= vocab0) {
+ LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) vocab0);
+ return false;
+ }
+ std::vector<int32_t> code0 = { params->code0 };
+ set_input_i32("inp_code0", code0);
+
+ // one uniform(0,1) draw per codebook, used by do_sampling()
+ std::uniform_real_distribution<float> dist(0.0f, 1.0f);
+ const int64_t n_acoustic = model.gen_code_head_w->ne[2];
+ for (int64_t g = 0; g < n_acoustic; g++) {
+ std::vector<float> r = { dist(params->gen_state->rng) };
+ set_input_f32(("inp_rand_" + std::to_string(g)).c_str(), r);
+ }
+ }
+ } break;
+ case PROJECTOR_TYPE_HUNYUANVL:
+ {
+ // Compute the HunyuanVL 2D position embedding on CPU (with the
+ // custom sf=(target+0.1)/n_grid bilinear sampling that the
+ // reference implementation uses) and upload it to the graph
+ // input declared in mtmd_graph_hunyuanvl::build().
+ GGML_ASSERT(model.position_embeddings != nullptr);
+ ggml_tensor * src_t = model.position_embeddings;
+ const int64_t n_embd = src_t->ne[0];
+ const int64_t n_pos = src_t->ne[1]; // = n_grid * n_grid
+ const int n_grid = (int)std::lround(std::sqrt((double)n_pos));
+ GGML_ASSERT((int64_t)n_grid * n_grid == n_pos);
+ const int out_w = pos_w; // pw
+ const int out_h = pos_h; // ph
+
+ // Pull weight to host.
+ std::vector<float> src(n_embd * n_pos);
+ ggml_backend_tensor_get(src_t, src.data(), 0, ggml_nbytes(src_t));
+
+ // Output layout matches ggml_new_tensor_2d(F32, n_embd, out_h*out_w):
+ // ne[0] = n_embd (fastest), ne[1] = out_h*out_w
+ // dst[(y*out_w + x) * n_embd + c]
+ std::vector<float> dst((size_t)n_embd * out_h * out_w);
+
+ const float sx = (float)(out_w + 0.1f) / (float)n_grid;
+ const float sy = (float)(out_h + 0.1f) / (float)n_grid;
+
+ for (int y = 0; y < out_h; ++y) {
+ // Match ggml_compute_forward_upscale_f32 pixel-center
+ // convention (align_corners=False): src_y = (y+0.5)/sy - 0.5.
+ const float fy = ((float)y + 0.5f) / sy - 0.5f;
+ int y0 = (int)std::floor(fy);
+ int y1 = y0 + 1;
+ y0 = std::clamp(y0, 0, n_grid - 1);
+ y1 = std::clamp(y1, 0, n_grid - 1);
+ float wy1 = std::clamp(fy - (float)y0, 0.0f, 1.0f);
+ const float wy0 = 1.0f - wy1;
+ for (int x = 0; x < out_w; ++x) {
+ const float fx = ((float)x + 0.5f) / sx - 0.5f;
+ int x0 = (int)std::floor(fx);
+ int x1 = x0 + 1;
+ x0 = std::clamp(x0, 0, n_grid - 1);
+ x1 = std::clamp(x1, 0, n_grid - 1);
+ float wx1 = std::clamp(fx - (float)x0, 0.0f, 1.0f);
+ const float wx0 = 1.0f - wx1;
+
+ const float w00 = wy0 * wx0;
+ const float w01 = wy0 * wx1;
+ const float w10 = wy1 * wx0;
+ const float w11 = wy1 * wx1;
+
+ const float * s00 = &src[((size_t)y0 * n_grid + x0) * n_embd];
+ const float * s01 = &src[((size_t)y0 * n_grid + x1) * n_embd];
+ const float * s10 = &src[((size_t)y1 * n_grid + x0) * n_embd];
+ const float * s11 = &src[((size_t)y1 * n_grid + x1) * n_embd];
+ float * d = &dst[((size_t)y * out_w + x) * n_embd];
+ for (int c = 0; c < n_embd; ++c) {
+ d[c] = w00 * s00[c] + w01 * s01[c] + w10 * s10[c] + w11 * s11[c];
+ }
+ }
+ }
+
+ set_input_f32("hunyuanvl_pos_embd", dst);
+ } break;
+ case PROJECTOR_TYPE_LLAMA4:
+ {
+ // set the 2D positions
+ int n_patches_per_col = image_size_width / patch_size;
+ std::vector<int> pos_data(num_patches + 1, 0); // +1 for the [CLS] token
+ // last pos is always kept 0, it's for CLS
+ // dimension H
+ for (int i = 0; i < num_patches; i++) {
+ pos_data[i] = (i / n_patches_per_col) + 1;
+ }
+ set_input_i32("pos_h", pos_data);
+ // dimension W
+ for (int i = 0; i < num_patches; i++) {
+ pos_data[i] = (i % n_patches_per_col) + 1;
+ }
+ set_input_i32("pos_w", pos_data);
+ } break;
+ case PROJECTOR_TYPE_DOTS3NOTE_A:
+ {
+ GGML_ASSERT(imgs.entries.size() == 1);
+ const int n_pos = (imgs.entries.front().nx() + 7) / 8; // 3x stride-2 conv2d
+ std::vector<int32_t> positions(n_pos);
+ for (int i = 0; i < n_pos; i++) {
+ positions[i] = i;
+ }
+ set_input_i32("positions", positions);
+ } break;
+ case PROJECTOR_TYPE_GEMMA4A:
+ {
+ GGML_ASSERT(imgs.entries.size() == 1);
+ const auto & img0 = imgs.entries.front();
+ // Compute n_pos matching SSCP output: two stride-2 convs
+ int n_pos = img0.nx();
+ for (int i = 0; i < 2; i++) { n_pos = (n_pos - 1) / 2 + 1; }
+
+ // Chunked local attention: blocked causal mask and RPE
+ const int chunk_size = 12;
+ const int max_past = 12;
+ const int context_size = chunk_size + max_past;
+ const int num_blocks = (n_pos + chunk_size - 1) / chunk_size;
+
+ // Blocked causal attention mask: [context_size, chunk_size, num_blocks]
+ {
+ std::vector<float> mask(context_size * chunk_size * num_blocks, -1e9f);
+ for (int b = 0; b < num_blocks; b++) {
+ for (int q = 0; q < chunk_size; q++) {
+ int gq = b * chunk_size + q;
+ for (int k = 0; k < context_size; k++) {
+ int gk = b * chunk_size - max_past + k;
+ if (gq < n_pos && gk >= 0 && gk < n_pos && gk <= gq && (gq - gk) < max_past) {
+ mask[k + q * context_size + b * context_size * chunk_size] = 0.0f;
+ }
+ }
+ }
+ }
+ set_input_f32("kq_mask", mask);
+ }
+
+ // Sinusoidal RPE: 13 positions [12, 11, ..., 0]
+ {
+ const int n_embd = model.hparams.n_embd;
+ const int num_timescales = n_embd / 2;
+ const float log_timescale_increment = logf(10000.0f) / std::max(num_timescales - 1, 1);
+ const int rpe_len = max_past + 1;
+ std::vector<float> pos_emb(n_embd * rpe_len, 0.0f);
+ for (int p = 0; p < rpe_len; p++) {
+ float position = (float)(max_past - p);
+ for (int i = 0; i < num_timescales; i++) {
+ float inv_ts = expf(-(float)i * log_timescale_increment);
+ float scaled = position * inv_ts;
+ pos_emb[p * n_embd + i] = sinf(scaled);
+ pos_emb[p * n_embd + i + num_timescales] = cosf(scaled);
+ }
+ }
+ set_input_f32("pos_emb", pos_emb);
+ }
+ } break;
+ case PROJECTOR_TYPE_MIMO_AUDIO:
+ {
+ GGML_ASSERT(imgs.entries.size() == 1);
+ const int n_frames = imgs.entries.front().nx();
+ const int n_pos = (n_frames - 1) / 2 + 1; // matches conv1(s=1)+conv2(s=2) output length
+
+ std::vector<int32_t> positions(n_pos);
+ for (int i = 0; i < n_pos; i++) {
+ positions[i] = i;
+ }
+ set_input_i32("mimo_audio_positions", positions);
+
+ const int window = hparams.attn_window_size;
+ GGML_ASSERT(window > 0);
+
+ const float neg_inf = std::numeric_limits<float>::lowest();
+ std::vector<float> full_mask((size_t) n_pos * n_pos);
+ std::vector<float> window_mask((size_t) n_pos * n_pos);
+ for (int q = 0; q < n_pos; q++) {
+ for (int k = 0; k < n_pos; k++) {
+ const bool causal_ok = k <= q;
+ full_mask[(size_t) q * n_pos + k] = causal_ok ? 0.0f : neg_inf;
+ window_mask[(size_t) q * n_pos + k] = (causal_ok && (q - k) <= window) ? 0.0f : neg_inf;
+ }
+ }
+ set_input_attn_mask(full_mask, 0);
+ set_input_attn_mask(window_mask, 1);
+
+ // input_local_transformer: block-diagonal mask + in-group positions
+ {
+ const int n_pos_ds = (n_pos - 2) / 2 + 1; // matches downsample conv (k=2,s=2,p=0)
+ const int group_size = hparams.audio_local_group_size;
+ GGML_ASSERT(group_size > 0);
+ const int n_groups = (n_pos_ds + group_size - 1) / group_size;
+ const int n_padded = n_groups * group_size;
+
+ std::vector<int32_t> local_positions(n_padded);
+ for (int i = 0; i < n_padded; i++) {
+ local_positions[i] = i % group_size;
+ }
+ set_input_i32("mimo_audio_local_positions", local_positions);
+
+ std::vector<float> local_mask((size_t) n_padded * n_padded);
+ for (int q = 0; q < n_padded; q++) {
+ for (int k = 0; k < n_padded; k++) {
+ const bool same_group = (q / group_size) == (k / group_size);
+ local_mask[(size_t) q * n_padded + k] = same_group ? 0.0f : neg_inf;
+ }
+ }
+ set_input_attn_mask(local_mask, 2);
+ }
+ } break;
+ case PROJECTOR_TYPE_LFM2A:
+ case PROJECTOR_TYPE_D1OMNI_A:
+ {
+ GGML_ASSERT(imgs.entries.size() == 1);
+ const auto n_frames = model.n_output_tokens(&imgs.entries.front());
+
+ auto d_model = 512;
+ auto seq_len = n_frames * 2 - 1;
+ std::vector<float> pos_emb(d_model*seq_len);
+ std::vector<double> inv_freq(d_model / 2);
+ for (size_t i = 0; i < inv_freq.size(); ++i) {
+ inv_freq[i] = std::exp(-(std::log(10000.0) / (float)d_model) * (2.0f * (float)(i)));
+ }
+ for (int64_t pos = 0; pos < seq_len; ++pos) {
+ for (size_t i = 0; i < inv_freq.size(); ++i) {
+ const float ang = (n_frames - pos - 1) * inv_freq[i];
+ pos_emb[pos*d_model + 2*i + 0] = sinf(ang); // even
+ pos_emb[pos*d_model + 2*i + 1] = cosf(ang); // odd
+ }
+ }
+ set_input_f32("pos_emb", pos_emb);
+ } break;
+ case PROJECTOR_TYPE_LFM3A:
+ {
+ GGML_ASSERT(imgs.entries.size() == 1);
+ const int n_pos = model.n_output_tokens(&imgs.entries.front());
+
+ // the "pos" slot leads the state; without a stream the state is zero, the clip is encoded from its start
+ int base = 0;
+ if (params->enc_state != nullptr) {
+ const auto & slots = params->enc_state->slots;
+ float base_f = 0.0f;
+ ggml_backend_tensor_get(slots[0], &base_f, 0, sizeof(float));
+ base = (int) base_f;
+ size_t i = 0;
+ for (const auto & slot : list_lfm3a_state_slots(hparams, model)) {
+ ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str()));
+ }
+ } else {
+ for (const auto & slot : list_lfm3a_state_slots(hparams, model)) {
+ ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str());
+ ggml_backend_tensor_memset(t, 0, 0, ggml_nbytes(t));
+ }
+ }
+
+ set_lfm3a_window_inputs(base, n_pos, hparams.attn_window_size);
+ } break;
+ case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
+ {
+ GGML_ASSERT(imgs.entries.size() == 1);
+ std::vector<int32_t> positions(ggml_nelements(get_inp_tensor("positions")));
+ for (size_t i = 0; i < positions.size(); i++) {
+ positions[i] = (int32_t) i;
+ }
+ set_input_i32("positions", positions);
+ GGML_ASSERT(params->enc_state != nullptr);
+ set_input_f32("inp_norm", params->enc_state->mel_norm);
+ if (ggml_graph_get_tensor(gf, "spk_prev") != nullptr) {
+ set_input_f32("spk_prev", params->enc_state->spk_prev);
+ }
+ } break;
+ case PROJECTOR_TYPE_PARAKEET:
+ {
+ GGML_ASSERT(imgs.entries.size() == 1);
+ struct ggml_tensor * attn_mask = ggml_graph_get_tensor(gf, "attn_mask");
+ const int n_q = attn_mask->ne[1];
+ const int n_k = attn_mask->ne[0];
+ const int n_frames = imgs.entries.front().nx();
+ const int n_tokens_real = (n_frames + hparams.subsampling_factor-1) / hparams.subsampling_factor;
+ const float mask_value = -1e30f;
+
+ std::vector<float> mask_data(n_q * n_k);
+ if (n_k == n_q) {
+ // full attention: mask keys that are padding
+ for (int q = 0; q < n_q; ++q) {
+ for (int k = 0; k < n_k; ++k) {
+ mask_data[q * n_k + k] = (k >= n_tokens_real) ? mask_value : 0.0f;
+ }
+ }
+ } else {
+ // local attention: mask keys outside the valid window
+ const int att_left = n_k / 2;
+ for (int q = 0; q < n_q; ++q) {
+ for (int k = 0; k < n_k; ++k) {
+ const int key = q - att_left + k;
+ mask_data[q * n_k + k] = (key >= 0 && key < n_tokens_real) ? 0.0f : mask_value;
+ }
+ }
+ }
+ set_input_f32(attn_mask->name, mask_data);
+
+ // local attention skew mask: zeroes out the probs that were
+ // computed for keys outside the valid sliding window.
+ if (struct ggml_tensor * local_mask = ggml_graph_get_tensor(gf, "local_mask")) {
+ const int lm_k = local_mask->ne[0];
+ const int lm_q = local_mask->ne[1];
+ const int window_size = lm_k - lm_q + 1;
+ std::vector<float> lm_data(lm_q * lm_k);
+ for (int q = 0; q < lm_q; ++q) {
+ for (int k = 0; k < lm_k; ++k) {
+ const int rel = k - q;
+ lm_data[q * lm_k + k] = (rel >= 0 && rel < window_size) ? 1.0f : 0.0f;
+ }
+ }
+ set_input_f32(local_mask->name, lm_data);
+ }
+
+ // Generate rotation frequencies for relative positional encoding.
+ {
+ const int n_state = hparams.n_embd;
+ const int d_half = n_state / 2;
+ const float log_10000 = logf(10000.0f);
+ std::vector<float> freqs(d_half);
+ for (int k = 0; k < d_half; ++k) {
+ freqs[k] = expf(-(float(k * 2) * log_10000 / float(n_state)));
+ }
+ set_input_f32("pos_freqs", freqs);
+ }
+
+ // Generate relative positional distance values which scaled by
+ // the frequency to produce the angles for sin/cos.
+ {
+ // window_size is only known after graph construction since it depends on
+ // n_time from the conv output, so we read it back from the graph tensor.
+ struct ggml_tensor * rel_pos = ggml_graph_get_tensor(gf, "rel_positions");
+ const int window_size = rel_pos->ne[1];
+ std::vector<float> pos(window_size);
+ // local attention: window is fixed at [att_left, att_right]
+ // full attention: window covers the full sequence, centered
+ if (ggml_graph_get_tensor(gf, "local_mask")) {
+ const int att_left = window_size / 2;
+ for (int t = 0; t < window_size; ++t) {
+ pos[t] = float(att_left - t);
+ }
+ } else {
+ const int n_time = (window_size + 1) / 2;
+ for (int t = 0; t < window_size; ++t) {
+ pos[t] = float(n_time - 1 - t);
+ }
+ }
+ set_input_f32(rel_pos->name, pos);
+ }
+ } break;
+ case PROJECTOR_TYPE_GRANITE_SPEECH:
+ {
+ const int context_size = model.hparams.audio_chunk_size;
+ const int max_pos_emb = model.hparams.audio_max_pos_emb;
+
+ std::vector<int32_t> dists((size_t) context_size * (size_t) context_size);
+ for (int i = 0; i < context_size; i++) {
+ for (int j = 0; j < context_size; j++) {
+ int d = i - j;
+ if (d < -context_size) d = -context_size;
+ if (d > context_size) d = context_size;
+ dists[(size_t) i * (size_t) context_size + (size_t) j] = d + max_pos_emb;
+ }
+ }
+ set_input_i32("attn_dists", dists);
+
+ const int n_frames = image_size_width;
+ const int remainder = n_frames % context_size;
+ if (remainder > 0) {
+ const int num_blocks = (n_frames + context_size - 1) / context_size;
+ std::vector<float> mask((size_t) context_size * (size_t) context_size * (size_t) num_blocks, 0.0f);
+ const float neg_inf = -INFINITY;
+ const size_t last_block_offset = (size_t) (num_blocks - 1) * (size_t) context_size * (size_t) context_size;
+ for (int q = 0; q < context_size; q++) {
+ for (int k = 0; k < context_size; k++) {
+ if (q >= remainder || k >= remainder) {
+ mask[last_block_offset + (size_t) q * (size_t) context_size + (size_t) k] = neg_inf;
+ }
+ }
+ }
+ set_input_f32("attn_mask", mask);
+ }
+ } break;
+ case PROJECTOR_TYPE_GRANITE4_VISION:
+ {
+ // Granite Vision 4.1 uses precomputed permutation index
+ // tensors to express the _win / _unwin / spatial sampling
+ // reshapes as ggml_get_rows gathers. The names are set
+ // by g4v_gather() in models/granite4-vision.cpp.
+ const int patch_size = model.hparams.patch_size;
+ const int image_side = imgs.entries.front().nx() / patch_size;
+ const int window_side = hparams.downsample_window_side;
+ const int query_side = hparams.downsample_query_side;
+ const int n = image_side / window_side;
+ const int new_side = n * query_side;
+
+ // Builds the raster→window permutation indices for a
+ // (side, side) grid split into (n × n) windows of (win × win)
+ // tokens each. dst[w * win*win + p] = source raster index.
+ auto make_win_idx = [](int side, int win) {
+ const int nn = side / win;
+ std::vector<int32_t> idx(static_cast<size_t>(side) * side);
+ for (int wy = 0; wy < nn; ++wy) {
+ for (int wx = 0; wx < nn; ++wx) {
+ for (int iy = 0; iy < win; ++iy) {
+ for (int ix = 0; ix < win; ++ix) {
+ const int w = wy * nn + wx;
+ const int p = iy * win + ix;
+ const int y = wy * win + iy;
+ const int x = wx * win + ix;
+ idx[static_cast<size_t>(w) * (win*win) + p] = y * side + x;
+ }
+ }
+ }
+ }
+ return idx;
+ };
+
+ auto make_unwin_idx = [&](int side, int win) {
+ const std::vector<int32_t> fwd = make_win_idx(side, win);
+ std::vector<int32_t> inv(fwd.size());
+ for (size_t i = 0; i < fwd.size(); ++i) {
+ inv[fwd[i]] = static_cast<int32_t>(i);
+ }
+ return inv;
+ };
+
+ auto make_spatial_idx = [](int side, int offset) {
+ const int off_y = (offset >> 1) & 1;
+ const int off_x = offset & 1;
+ const int new_s = side / 2;
+ std::vector<int32_t> idx(static_cast<size_t>(new_s) * new_s);
+ for (int y = 0; y < new_s; ++y) {
+ for (int x = 0; x < new_s; ++x) {
+ idx[y * new_s + x] = (y * 2 + off_y) * side + (x * 2 + off_x);
+ }
+ }
+ return idx;
+ };
+
+ // the same permutation is applied to every tile of the stacked image
+ auto upload = [&](const std::string & name, const std::vector<int32_t> & idx) {
+ ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str());
+ GGML_ASSERT(t);
+ GGML_ASSERT(ggml_nelements(t) % (int64_t) idx.size() == 0);
+ const int n_rep = ggml_nelements(t) / idx.size();
+ std::vector<int32_t> buf;
+ buf.reserve(idx.size() * n_rep);
+ for (int i = 0; i < n_rep; ++i) {
+ buf.insert(buf.end(), idx.begin(), idx.end());
+ }
+ ggml_backend_tensor_set(t, buf.data(), 0, ggml_nbytes(t));
+ };
+
+ // Stage 1b only uses block 0's permutations; future stages
+ // will upload all blocks.
+ for (size_t bid = 0; bid < hparams.feature_layers.size(); ++bid) {
+ const std::string prefix = "g4v_blk" + std::to_string(bid) + "_";
+ upload(prefix + "win_idx", make_win_idx(image_side, window_side));
+ upload(prefix + "qwin_idx", make_win_idx(new_side, query_side));
+ upload(prefix + "unwin_idx", make_unwin_idx(new_side, query_side));
+ const auto spatial_offset = hparams.proj_spatial_offsets[bid];
+ if (spatial_offset >= 0) {
+ upload(prefix + "spatial_idx", make_spatial_idx(image_side,spatial_offset));
+ }
+ }
+ } break;
+ default:
+ GGML_ABORT("Unknown projector type");
+ }
+
+ mtmd_backend_cpu_set_n_threads(backend_cpu, params->n_threads);
+
+ auto status = ggml_backend_sched_graph_compute(sched.get(), gf);
+ if (status != GGML_STATUS_SUCCESS) {
+ LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status);
+ return false;
+ }
+
+ if (embeddings != nullptr) {
+ // sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one)
+ const int n_tokens_out = embeddings->ne[1];
+ const int expected_n_tokens_out = model.n_output_tokens(&imgs.entries[0]);
+ if (n_tokens_out != expected_n_tokens_out) {
+ LOG_ERR("%s: expected output %d tokens, got %d\n", __func__, expected_n_tokens_out, n_tokens_out);
+ GGML_ABORT("Invalid number of output tokens");
+ }
+
+ LOG_DBG("%s: output embedding shape [%d, %d, %d]\n", __func__,
+ (int)embeddings->ne[0], (int)embeddings->ne[1], (int)embeddings->ne[2]);
+
+ // copy output to user buffer if provided
+ // if output is empty, skip the copy
+ auto & out_batch_embd = *params->out_embd;
+ if (!out_batch_embd.empty()) {
+ if (out_batch_embd.size() != (size_t)ggml_nelements(embeddings)) {
+ LOG_ERR("%s: output buffer has %zu elements but expected %zu\n", __func__, out_batch_embd.size(), (size_t)ggml_nelements(embeddings));
+ GGML_ABORT("Output buffer size mismatch");
+ }
+ ggml_backend_tensor_get(embeddings, out_batch_embd.data(), 0, ggml_nbytes(embeddings));
+ } else {
+ LOG_WRN("%s: output buffer is empty, skipping copy\n", __func__);
+ }
+ }
+
+ //
+ // for audio gen models
+ //
+
+ if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
+ ggml_tensor * fsq = ggml_graph_get_tensor(gf, "out_fsq");
+ GGML_ASSERT(fsq != nullptr && params->out_codes);
+ const int n_tok = (int) fsq->ne[1];
+ std::vector<float> h((size_t) 8 * n_tok);
+ ggml_backend_tensor_get(fsq, h.data(), 0, ggml_nbytes(fsq));
+
+ // fsq round to base 3: h in (-1, 1) maps to digits {0, 1, 2}
+ params->out_codes->resize(n_tok);
+ for (int t = 0; t < n_tok; t++) {
+ int32_t code = 0;
+ for (int i = 7; i >= 0; i--) {
+ const int32_t d = (int32_t) roundf(h[(size_t) t * 8 + i] * 0.9990000128746033f) + 1;
+ code = code * 3 + d;
+ }
+ (*params->out_codes)[(size_t) t] = code;
+ }
+
+ // speech table rows of the codes, the multilingual variant adds its learned positions as its perceiver expects
+ if (params->out_code_embd) {
+ const auto & tensors = model.cbx_tensors;
+ auto tab_it = tensors.find("a.gen.code.out_embd.weight");
+ GGML_ASSERT(tab_it != tensors.end());
+ ggml_tensor * tab = tab_it->second;
+ const int n_e = (int) tab->ne[0];
+ auto pos_it = tensors.find("a.gen.t3.speech_pos_emb");
+ ggml_tensor * pos = pos_it == tensors.end() ? nullptr : pos_it->second;
+ if ((tab->type != GGML_TYPE_F16 && tab->type != GGML_TYPE_F32) ||
+ (pos && (pos->type != GGML_TYPE_F32 || pos->ne[0] != n_e || pos->ne[1] < n_tok))) {
+ LOG_ERR("%s: unexpected speech table or positions\n", __func__);
+ return false;
+ }
+
+ params->out_code_embd->resize((size_t) n_tok * n_e);
+ std::vector<ggml_fp16_t> h16(n_e);
+ std::vector<float> pr(n_e);
+ for (int t = 0; t < n_tok; t++) {
+ float * dst = params->out_code_embd->data() + (size_t) t * n_e;
+ if ((*params->out_codes)[(size_t) t] >= tab->ne[1]) {
+ LOG_ERR("%s: speech code beyond the speech table\n", __func__);
+ return false;
+ }
+ const size_t r = (size_t) (*params->out_codes)[(size_t) t] * n_e;
+ if (tab->type == GGML_TYPE_F16) {
+ ggml_backend_tensor_get(tab, h16.data(), r * sizeof(ggml_fp16_t), (size_t) n_e * sizeof(ggml_fp16_t));
+ for (int j = 0; j < n_e; j++) {
+ dst[j] = ggml_fp16_to_fp32(h16[j]);
+ }
+ } else {
+ ggml_backend_tensor_get(tab, dst, r * sizeof(float), (size_t) n_e * sizeof(float));
+ }
+ if (pos) {
+ ggml_backend_tensor_get(pos, pr.data(), (size_t) t * n_e * sizeof(float), (size_t) n_e * sizeof(float));
+ for (int j = 0; j < n_e; j++) {
+ dst[j] += pr[j];
+ }
+ }
+ }
+ }
+ return true;
+ }
+
+ if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE) {
+ ggml_tensor * sp = ggml_graph_get_tensor(gf, "out_spec");
+ GGML_ASSERT(sp != nullptr && params->out_spec);
+ params->out_spec->resize(ggml_nelements(sp));
+ ggml_backend_tensor_get(sp, params->out_spec->data(), 0, ggml_nbytes(sp));
+ return true;
+ }
+
+ if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS) {
+ ggml_tensor * mel = ggml_graph_get_tensor(gf, "out_mel");
+ GGML_ASSERT(mel != nullptr);
+ std::vector<float> mel_data(ggml_nelements(mel));
+ ggml_backend_tensor_get(mel, mel_data.data(), 0, ggml_nbytes(mel));
+
+ // hift: f0 -> harmonic source -> source stft on the host, then the vocoder graph and the istft
+ ggml_tensor * f0_t = ggml_graph_get_tensor(gf, "out_f0");
+ GGML_ASSERT(f0_t != nullptr);
+ const int n_mel_out = (int) ggml_nelements(f0_t);
+ std::vector<float> f0(n_mel_out);
+ ggml_backend_tensor_get(f0_t, f0.data(), 0, ggml_nbytes(f0_t));
+
+ // SineGen: f0 upsampled x480, 9 harmonics with cumulative phase, uv gating and noise, merged by l_linear + tanh
+ const int ups_total = 480;
+ const double sr = 24000.0;
+ const int64_t n_wav = (int64_t) n_mel_out * ups_total;
+ std::vector<float> lw(9);
+ float lb = 0.0f;
+ if (model.cbx_read_tensor("a.gen.hift.m_source.l_linear.weight", lw.data(), lw.size()) != 9 ||
+ model.cbx_read_tensor("a.gen.hift.m_source.l_linear.bias", &lb, 1) != 1) {
+ LOG_ERR("%s: unexpected hift source module\n", __func__);
+ return false;
+ }
+
+ std::mt19937 & srng = params->gen_state->rng;
+ std::uniform_real_distribution<double> ud(-M_PI, M_PI);
+ std::normal_distribution<float> snd(0.0f, 1.0f);
+ double phase[9];
+ for (int h = 0; h < 9; h++) {
+ phase[h] = h == 0 ? 0.0 : ud(srng);
+ }
+ std::vector<float> src((size_t) n_wav);
+ double cum[9] = {0.0};
+ for (int64_t t = 0; t < n_wav; t++) {
+ const float f = f0[(size_t) (t / ups_total)];
+ const float uv = f > 10.0f ? 1.0f : 0.0f; // nsf_voiced_threshold
+ const float namp = uv * 0.003f + (1.0f - uv) * 0.1f / 3.0f;
+ float merged = lb;
+ for (int h = 0; h < 9; h++) {
+ cum[h] += (double) f * (h + 1) / sr;
+ cum[h] -= floor(cum[h]);
+ float sine = 0.1f * (float) sin(2.0 * M_PI * cum[h] + phase[h]);
+ sine = sine * uv + namp * snd(srng);
+ merged += lw[(size_t) h] * sine;
+ }
+ src[(size_t) t] = tanhf(merged);
+ }
+
+ // stft of the source: n_fft 16, hop 4, hann window, centered
+ const int n_fft = 16, hop = 4, n_bins = 9;
+ std::vector<float> win(n_fft);
+ for (int i = 0; i < n_fft; i++) win[(size_t) i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / n_fft);
+ const int n_stft = (int) (n_wav / hop) + 1;
+ std::vector<float> sstft((size_t) n_stft * 18);
+ for (int fr = 0; fr < n_stft; fr++) {
+ const int64_t c0 = (int64_t) fr * hop - n_fft / 2; // centered, reflect padded
+ for (int k = 0; k < n_bins; k++) {
+ double re = 0.0, im = 0.0;
+ for (int i = 0; i < n_fft; i++) {
+ int64_t idx = c0 + i;
+ if (idx < 0) idx = -idx;
+ if (idx >= n_wav) idx = 2 * (n_wav - 1) - idx;
+ const double v = (double) src[(size_t) idx] * win[(size_t) i];
+ const double a = 2.0 * M_PI * k * i / n_fft;
+ re += v * cos(a);
+ im -= v * sin(a);
+ }
+ sstft[(size_t) fr * 18 + k ] = (float) re;
+ sstft[(size_t) fr * 18 + 9 + k] = (float) im;
+ }
+ }
+
+ // vocoder graph on mel + source stft
+ std::vector<float> spec;
+ {
+ mtmd_subctx_encode_params vp = *params;
+ vp.gen_process = MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE;
+ vp.mel_in = &mel_data;
+ vp.sstft_in = &sstft;
+ vp.vocode_n_mel = n_mel_out;
+ vp.vocode_n_stft = n_stft;
+ vp.out_audio = nullptr;
+ vp.out_spec = &spec;
+ if (!encode(&vp)) {
+ LOG_ERR("%s: vocoder stage failed\n", __func__);
+ return false;
+ }
+ }
+
+ // istft: mag = clipped exp, phase = sin, hann overlap-add
+ const int n_frames_out = (int) (spec.size() / 18);
+ const int64_t n_out = (int64_t) (n_frames_out - 1) * hop;
+ std::vector<double> acc((size_t) n_out + n_fft, 0.0);
+ std::vector<double> wsum((size_t) n_out + n_fft, 0.0);
+ for (int fr = 0; fr < n_frames_out; fr++) {
+ double frame[16];
+ for (int i = 0; i < n_fft; i++) {
+ double v = 0.0;
+ for (int k = 0; k < n_bins; k++) {
+ const double mag = fmin(exp((double) spec[(size_t) fr * 18 + k]), 1e2);
+ const double ph = sin((double) spec[(size_t) fr * 18 + 9 + k]);
+ const double re = mag * cos(ph), im = mag * sin(ph);
+ const double a = 2.0 * M_PI * k * i / n_fft;
+ const double w = (k == 0 || k == n_fft / 2) ? 1.0 : 2.0;
+ v += w * (re * cos(a) - im * sin(a));
+ }
+ frame[i] = v / n_fft;
+ }
+ const int64_t o = (int64_t) fr * hop;
+ for (int i = 0; i < n_fft; i++) {
+ acc [(size_t) (o + i)] += frame[i] * win[(size_t) i];
+ wsum[(size_t) (o + i)] += (double) win[(size_t) i] * win[(size_t) i];
+ }
+ }
+ GGML_ASSERT(params->out_audio);
+ auto & out_audio = *params->out_audio;
+ out_audio.resize((size_t) std::min<int64_t>(n_out - n_fft / 2, n_wav));
+ for (size_t i = 0; i < out_audio.size(); i++) {
+ const size_t j = i + n_fft / 2; // drop the centering pad
+ const double v = wsum[j] > 1e-11 ? acc[j] / wsum[j] : 0.0;
+ out_audio[(size_t) i] = (float) fmax(-0.99, fmin(0.99, v));
+ }
+
+ // trim_fade of the reference: silence 20 ms then fade in 20 ms, hides the artifact of the prompt boundary
+ const size_t n_trim = 24000 / 50;
+ for (size_t i = 0; i < 2 * n_trim && i < out_audio.size(); i++) {
+ const double g = i < n_trim ? 0.0
+ : (cos(M_PI * (1.0 - (double) (i - n_trim) / n_trim)) + 1.0) / 2.0;
+ out_audio[(size_t) i] *= (float) g;
+ }
+ return true;
+ }
+
+ // optional outputs: a pipeline yields codes or feats, and not all have an eos head
+ if (params->out_codes != nullptr) {
+ ggml_tensor * codes = ggml_graph_get_tensor(gf, "out_codes");
+ if (codes != nullptr) {
+ auto & out_codes = *params->out_codes;
+ out_codes.resize(ggml_nelements(codes));
+ ggml_backend_tensor_get(codes, out_codes.data(), 0, ggml_nbytes(codes));
+ }
+ }
+ if (params->out_feats != nullptr) {
+ ggml_tensor * feats = ggml_graph_get_tensor(gf, "out_feats");
+ if (feats != nullptr) {
+ auto & out_feats = *params->out_feats;
+ out_feats.resize(ggml_nelements(feats));
+ ggml_backend_tensor_get(feats, out_feats.data(), 0, ggml_nbytes(feats));
+ }
+ }
+ if (params->out_is_eos != nullptr) {
+ ggml_tensor * eos = ggml_graph_get_tensor(gf, "out_eos_score");
+ if (eos != nullptr) {
+ GGML_ASSERT(ggml_nelements(eos) == 1);
+ float score = 0.0f;
+ ggml_backend_tensor_get(eos, &score, 0, sizeof(float));
+ *params->out_is_eos = score > hparams.gen_eos_threshold;
+ }
+ }
+ if (params->out_audio != nullptr) {
+ ggml_tensor * audio = ggml_graph_get_tensor(gf, "out_audio");
+ if (audio == nullptr) {
+ GGML_ABORT("out_audio requested but graph has no \"out_audio\" tensor");
+ }
+ auto & out_audio = *params->out_audio;
+ out_audio.resize(ggml_nelements(audio));
+ ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio));
+
+ if (model.proj_type == PROJECTOR_TYPE_SOPRANO) {
+ std::vector<float> spectrum = std::move(out_audio);
+ mtmd_graph_soprano::decode_spectrum(model, spectrum, out_audio);
+ }
+
+ // drop the tail audio that comes from the code-0 rear padding, drama3 has no padding
+ if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN) {
+ const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0;
+ const int64_t n_frames_w = hparams.wav_tfm_swa;
+ const int64_t n_frames = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w;
+ if (n_frames < n_frames_w) {
+ const size_t hop = out_audio.size() / n_frames_w;
+ out_audio.resize((size_t) n_frames * hop);
+ }
+ }
+ }
+ if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_LFM3A) {
+ const auto & slots = params->enc_state->slots;
+ size_t i = 0;
+ for (const auto & slot : list_lfm3a_state_slots(hparams, model)) {
+ ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str());
+ GGML_ASSERT(t != nullptr);
+ copy_gen_state_out(t, slots[i++]);
+ }
+ } else if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) {
+ mtmd_subctx_enc_state * st = params->enc_state;
+ ggml_tensor * chunk = ggml_graph_get_tensor(gf, "spk_chunk");
+ ggml_tensor * preds = ggml_graph_get_tensor(gf, "spk_preds");
+ GGML_ASSERT(chunk != nullptr && preds != nullptr);
+ st->spk_chunk.resize(ggml_nelements(chunk));
+ st->spk_preds.resize(ggml_nelements(preds));
+ ggml_backend_tensor_get(chunk, st->spk_chunk.data(), 0, ggml_nbytes(chunk));
+ ggml_backend_tensor_get(preds, st->spk_preds.data(), 0, ggml_nbytes(preds));
+ } else if (params->enc_state != nullptr) {
+ mtmd_subctx_enc_state * st = params->enc_state;
+ ggml_tensor * tail = ggml_graph_get_tensor(gf, "out_stem_tail");
+ GGML_ASSERT(tail != nullptr);
+ st->stem_tail.resize(ggml_nelements(tail));
+ ggml_backend_tensor_get(tail, st->stem_tail.data(), 0, ggml_nbytes(tail));
+
+ const int n_new = (imgs.entries[0].nx() - 2) / 2;
+ for (int i = 0; i < n_new; i++) {
+ st->slot_pos[(st->n_past + i) % st->n_ctx] = st->n_past + i;
+ }
+ st->n_past += n_new;
+ }
+
+ if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
+ const auto & slots = params->gen_state->slots;
+ size_t i = 0;
+ for (const auto & slot : list_gen_state_slots(hparams, model)) {
+ if (slot.in_place) {
+ i++;
+ continue;
+ }
+ ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str());
+ if (t == nullptr) {
+ GGML_ABORT("graph has no \"state_out_%s\" tensor", slot.name.c_str());
+ }
+ GGML_ASSERT(i < slots.size());
+ copy_gen_state_out(t, slots[i++]);
+ }
+ }
+
+ //
+ // Debug: dump final embeddings if MTMD_DEBUG_EMBEDDINGS is set
+ //
+
+ if (debug_output_embeddings && embeddings != nullptr) {
+ const int64_t n_embd = embeddings->ne[0];
+ const int64_t n_tokens = embeddings->ne[1];
+ std::vector<float> emb_data(ggml_nelements(embeddings));
+ ggml_backend_tensor_get(embeddings, emb_data.data(), 0, ggml_nbytes(embeddings));
+
+ LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n");
+ LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens);
+
+ // TEMP debugging (parity validation), will be removed before merge
+ // when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data]
+ const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS");
+ if (dump_path && strcmp(dump_path, "1") != 0) {
+ FILE * f = fopen(dump_path, "wb");
+ if (f) {
+ const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd };
+ fwrite(hdr, sizeof(hdr), 1, f);
+ fwrite(emb_data.data(), sizeof(float), emb_data.size(), f);
+ fclose(f);
+ }
+ }
+
+ // Print first few values of first token
+ LOG_INF("Token 0 (first 16 values): ");
+ for (int i = 0; i < std::min((int64_t)16, n_embd); i++) {
+ LOG_INF("%.6f ", emb_data[i]);
+ }
+ LOG_INF("\n");
+
+ // Print last few values of first token
+ if (n_embd > 16) {
+ LOG_INF("Token 0 (last 16 values): ");
+ for (int64_t i = n_embd - 16; i < n_embd; i++) {
+ LOG_INF("%.6f ", emb_data[i]);
+ }
+ LOG_INF("\n");
+ }
+
+ // Compute and print statistics
+ float sum = 0.0f, sum_sq = 0.0f, min_val = emb_data[0], max_val = emb_data[0];
+ for (size_t i = 0; i < emb_data.size(); i++) {
+ sum += emb_data[i];
+ sum_sq += emb_data[i] * emb_data[i];
+ min_val = std::min(min_val, emb_data[i]);
+ max_val = std::max(max_val, emb_data[i]);
+ }
+ float mean = sum / emb_data.size();
+ float variance = (sum_sq / emb_data.size()) - (mean * mean);
+ LOG_INF("Stats: mean=%.6f, std=%.6f, min=%.6f, max=%.6f, sum=%.6f\n",
+ mean, sqrtf(variance), min_val, max_val, sum);
+ LOG_INF("=== END MTMD_DEBUG_EMBEDDINGS ===\n\n");
+ }
+
+ return true;
+}
+
+std::map<ggml_backend_dev_t, size_t> mtmd_subcontext::get_mem_usage() const {
+ std::map<ggml_backend_dev_t, size_t> result = model.mem_weights;
+ for (auto & [dev, size] : mem_compute) {
+ result[dev] += size;
+ }
+ return result;
+}
diff --git a/tools/mtmd/mtmd-subcontext.h b/tools/mtmd/mtmd-subcontext.h
new file mode 100644
index 000000000..d757c9b6e
--- /dev/null
+++ b/tools/mtmd/mtmd-subcontext.h
@@ -0,0 +1,202 @@
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+#include "ggml-cpp.h"
+#include "mtmd-impl.h"
+#include "mtmd-model.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <map>
+#include <memory>
+#include <random>
+#include <vector>
+
+// !!! Internal header, to be used by mtmd only !!!
+
+// a subcontext runs the graphs of one mtmd_model (one modality)
+// it owns the backends and the compute buffers, the weights are owned by the model
+
+enum mtmd_flash_attn_type {
+ MTMD_FLASH_ATTN_TYPE_AUTO = -1,
+ MTMD_FLASH_ATTN_TYPE_DISABLED = 0,
+ MTMD_FLASH_ATTN_TYPE_ENABLED = 1,
+};
+
+struct mtmd_subctx_params {
+ ggml_backend_dev_t device; // nullptr means CPU, see mtmd_select_device()
+ std::shared_ptr<ggml_threadpool> threadpool; // CPU threadpool shared by the subcontexts, can be nullptr
+ enum mtmd_flash_attn_type flash_attn_type;
+ bool warmup;
+ ggml_backend_sched_eval_callback cb_eval;
+ void * cb_eval_user_data;
+};
+
+// the device the weights and the graphs are placed on, nullptr means CPU
+ggml_backend_dev_t mtmd_select_device(bool use_gpu, ggml_backend_dev_t device);
+
+// nullptr if n_threads < 2 or the CPU backend has no threadpool support
+std::shared_ptr<ggml_threadpool> mtmd_threadpool_init(int n_threads);
+
+// compute the log-mel of n_frames frames, the waveform must already be padded and window must hold one whole frame
+// out has the layout [n_frames, n_mel]; like the encoders, must not run at the same time as another pass on config.sched
+struct mtmd_audio_mel_config;
+bool mtmd_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out);
+
+enum mtmd_subctx_gen_process {
+ MTMD_SUBCTX_GEN_PROCESS_GEN_UNKNOWN,
+ MTMD_SUBCTX_GEN_PROCESS_GEN_CODE, // h_state to codes
+ MTMD_SUBCTX_GEN_PROCESS_GEN_WAV, // codes to raw PCM audio
+ MTMD_SUBCTX_GEN_PROCESS_TTS, // full utterance of codes to raw PCM audio
+ MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE, // internal: mel + source stft to istft input
+ MTMD_SUBCTX_GEN_PROCESS_TOKENIZE, // raw PCM audio to semantic speech tokens
+};
+
+// persistent state of a causal audio encoder, for streaming input (one per stream)
+// it holds the sliding window of past keys/values, so each encode() call only takes the new frames
+struct mtmd_subctx_enc_state {
+ int32_t n_ctx = 0; // ring slots: attention window + frames per call
+ int32_t n_past = 0; // absolute position of the next frame
+
+ std::vector<int32_t> slot_pos; // absolute position held by each slot, -1 if empty
+ std::vector<float> stem_tail; // last conv stem output frame, empty on a cold start
+
+ ggml_context_ptr ctx_data;
+ ggml_backend_buffer_ptr buf;
+ std::vector<ggml_tensor *> k; // per layer [n_embd_kv, n_ctx]
+ std::vector<ggml_tensor *> v;
+
+ // lfm3a: same order as list_lfm3a_state_slots(), F32 [ne0, ne1], updated in place
+ std::vector<ggml_tensor *> slots;
+
+ // nemotron transcribe: the window's normalized log-mel, cache and FIFO rows in; its speaker rows and predictions out
+ std::vector<float> mel_norm;
+ std::vector<float> spk_prev;
+ std::vector<float> spk_chunk, spk_preds;
+};
+
+// per-sequence state of audio generation: RNG and the slots kept between GEN_WAV calls
+struct mtmd_subctx_gen_state {
+ std::mt19937 rng;
+
+ ggml_context_ptr ctx_data;
+ ggml_backend_buffer_ptr buf;
+ std::vector<ggml_tensor *> slots; // same order as list_gen_state_slots(), F32 [ne0, ne1]
+
+ // clears the slots and keeps the RNG stream
+ void reset() {
+ if (buf) {
+ ggml_backend_buffer_clear(buf.get(), 0);
+ }
+ }
+};
+
+struct mtmd_subctx_encode_params {
+ int n_threads = 1;
+ const mtmd_image_f32_batch * imgs = nullptr;
+ std::vector<float> * out_embd = nullptr;
+
+ // for audio gen, imgs has exactly one entry: hidden state from backbone (GEN_CODE) or unused (GEN_WAV)
+ mtmd_subctx_gen_process gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_UNKNOWN;
+
+ // GEN_CODE: out_embd receives the embd to feed back to the backbone
+ int32_t code0 = 0; // semantic code sampled by the backbone
+ int32_t top_k = 50;
+ float top_p = 1.0f;
+ std::vector<int32_t> * out_codes = nullptr; // this frame's 16 sampled codes
+ std::vector<float> * out_feats = nullptr; // continuous counterpart of out_codes
+ float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders
+ bool * out_is_eos = nullptr;
+
+ // TOKENIZE: speech embedding rows of the produced codes
+ std::vector<float> * out_code_embd = nullptr;
+
+ // GEN_WAV
+ const std::vector<int32_t> * codes = nullptr; // this frame's 16 RVQ codes
+ const std::vector<float> * feats = nullptr; // continuous counterpart of codes
+ std::vector<float> * out_audio = nullptr; // decoded PCM samples, F32
+ const mtmd_gen_ref * ref = nullptr; // TTS reference conditioning of the voice
+ // TTS_VOCODE internal stage inputs
+ const std::vector<float> * mel_in = nullptr;
+ const std::vector<float> * sstft_in = nullptr;
+ int vocode_n_mel = 0;
+ int vocode_n_stft = 0;
+ std::vector<float> * out_spec = nullptr;
+
+ // audio gen: state of one sequence, encode() updates it in place
+ struct mtmd_subctx_gen_state * gen_state = nullptr;
+
+ // streaming audio input, updated in place by encode()
+ struct mtmd_subctx_enc_state * enc_state = nullptr;
+};
+
+struct mtmd_subcontext {
+ const mtmd_model & model;
+
+ std::vector<uint8_t> buf_compute_meta;
+
+ std::vector<ggml_backend_t> backend_ptrs;
+ std::vector<ggml_backend_buffer_type_t> backend_buft;
+
+ std::shared_ptr<ggml_threadpool> threadpool; // shared by the subcontexts of a model
+
+ ggml_backend_t backend = nullptr;
+ ggml_backend_t backend_cpu = nullptr;
+
+ int max_nodes = 8192;
+ ggml_backend_sched_ptr sched;
+ mtmd_flash_attn_type flash_attn_type = MTMD_FLASH_ATTN_TYPE_AUTO;
+ bool is_allocated = false;
+
+ bool debug_output_embeddings = false;
+
+ // for measuring memory usage
+ std::map<ggml_backend_dev_t, size_t> mem_compute;
+
+ bool support_batch = false;
+
+ // the model must outlive the subcontext
+ mtmd_subcontext(const mtmd_model & model, const mtmd_subctx_params & params);
+ ~mtmd_subcontext();
+
+ // reserve the compute buffers for the given batch
+ void warmup(const mtmd_image_f32_batch & batch);
+
+ bool encode(mtmd_subctx_encode_params * params);
+
+ // TODO: remove encode_image() and always use batched version
+ bool encode_image(int n_threads, const mtmd_image_f32 * img, std::vector<float> & out_vec);
+ bool encode_batch(int n_threads, const mtmd_image_f32_batch * imgs, std::vector<float> & out_batch_embd);
+
+ // n_frames_max is the maximum number of encoder frames per encode() call, free the result with delete
+ mtmd_subctx_enc_state * enc_state_init(int32_t n_frames_max);
+
+ // seed UINT32_MAX means random, free the result with delete
+ mtmd_subctx_gen_state * gen_state_init(uint32_t seed);
+
+ // weights and compute buffers
+ std::map<ggml_backend_dev_t, size_t> get_mem_usage() const;
+
+private:
+ struct support_info_op {
+ ggml_tensor * op;
+
+ // true if the op runs on the accelerated backend
+ bool is_accel = true;
+ };
+
+ struct support_info_graph {
+ // whether the backend supports flash attention
+ bool fattn = true;
+ ggml_tensor * fattn_op = nullptr; // for debugging
+
+ std::vector<support_info_op> ops;
+ };
+
+ mtmd_image_f32_batch get_dummy_batch() const;
+
+ // only initialize backend buffers, but do not allocate them yet
+ support_info_graph reserve_compute_meta(const mtmd_image_f32_batch & batch);
+};
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 538f90013..6b4fc4d0a 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -1,5 +1,5 @@
-#include "clip.h"
-#include "clip-impl.h"
+#include "mtmd-subcontext.h"
+#include "mtmd-impl.h"
#include "mtmd.h"
#include "mtmd-internal.h"
#include "mtmd-audio.h"
@@ -106,10 +106,10 @@ std::string mtmd_serialization::read<std::string>() {
return str;
}
-// only mtmd.cpp needs these, so they're implemented here rather than in clip-impl.h
-void clip_image_f32::serialize(mtmd_serialization & ser) const {
+// only mtmd.cpp needs these, so they're implemented here rather than in mtmd-impl.h
+void mtmd_image_f32::serialize(mtmd_serialization & ser) const {
// remember to bump MTMD_SERIALIZATION_VERSION if this is changed
- // note: buf is intentionally NOT serialized; the loaded clip_image_f32 will always be a placeholder
+ // note: buf is intentionally NOT serialized; the loaded mtmd_image_f32 will always be a placeholder
ser.write(add_viewsep);
ser.write(add_newline);
ser.write(lead_pad);
@@ -121,16 +121,16 @@ void clip_image_f32::serialize(mtmd_serialization & ser) const {
ser.write((int32_t)nx_);
ser.write((int32_t)ny_);
}
-void clip_image_f32::deserialize(mtmd_serialization & ser) {
+void mtmd_image_f32::deserialize(mtmd_serialization & ser) {
add_viewsep = ser.read<bool>();
add_newline = ser.read<bool>();
lead_pad = ser.read<int32_t>();
if (ser.version >= 3) {
const int32_t suffix_raw = ser.read<int32_t>();
- if (suffix_raw < 0 || suffix_raw >= CLIP_SUFFIX_COUNT) {
+ if (suffix_raw < 0 || suffix_raw >= MTMD_SUFFIX_COUNT) {
throw std::runtime_error("invalid suffix type");
}
- suffix_type = (clip_suffix_type)suffix_raw;
+ suffix_type = (mtmd_suffix_type)suffix_raw;
anyres.grid_x = ser.read<int32_t>();
anyres.grid_y = ser.read<int32_t>();
anyres.orig_nx = ser.read<int32_t>();
@@ -141,7 +141,7 @@ void clip_image_f32::deserialize(mtmd_serialization & ser) {
buf.clear(); // always a placeholder after loading
}
-void clip_image_f32_batch::serialize(mtmd_serialization & ser) const {
+void mtmd_image_f32_batch::serialize(mtmd_serialization & ser) const {
// remember to bump MTMD_SERIALIZATION_VERSION if this is changed
ser.write(is_audio);
ser.write<uint64_t>(entries.size());
@@ -149,7 +149,7 @@ void clip_image_f32_batch::serialize(mtmd_serialization & ser) const {
entry.serialize(ser);
}
}
-void clip_image_f32_batch::deserialize(mtmd_serialization & ser) {
+void mtmd_image_f32_batch::deserialize(mtmd_serialization & ser) {
is_audio = ser.read<bool>();
uint64_t n = ser.read<uint64_t>();
constexpr size_t min_entry_bytes = sizeof(uint8_t) * 2 + sizeof(int32_t) * 2;
@@ -159,7 +159,7 @@ void clip_image_f32_batch::deserialize(mtmd_serialization & ser) {
entries.clear();
entries.reserve(n);
for (uint64_t i = 0; i < n; i++) {
- clip_image_f32 entry;
+ mtmd_image_f32 entry;
entry.deserialize(ser);
entries.push_back(std::move(entry));
}
@@ -176,7 +176,7 @@ struct mtmd_bitmap {
std::string id; // optional user-defined id, for ex: can be set to image hash, useful for KV cache tracking
bool is_audio = false; // true if the bitmap is audio
bool is_voice = false;
- std::shared_ptr<const clip_gen_ref> gen_ref; // gen-side reference of a predefined voice, if any
+ std::shared_ptr<const mtmd_gen_ref> gen_ref; // gen-side reference of a predefined voice, if any
bool mergeable = false; // [QWEN_VIDEO] set only on frames of the same video
bool is_video = false; // see mtmd_bitmap_set_is_video()
@@ -202,13 +202,13 @@ struct mtmd_bitmap {
}
}
- static mtmd_bitmap * init_voice(const std::string & name, const clip_gen_voice & voice, uint32_t n_embd) {
+ static mtmd_bitmap * init_voice(const std::string & name, const mtmd_gen_voice & voice, uint32_t n_embd) {
const float * rows = voice.rows.data();
const uint32_t n_rows = voice.rows.size() / n_embd;
auto * bitmap = new mtmd_bitmap(nullptr, n_embd, n_rows);
bitmap->is_voice = true;
if (!voice.ref.tokens.empty()) {
- bitmap->gen_ref = std::make_shared<const clip_gen_ref>(voice.ref);
+ bitmap->gen_ref = std::make_shared<const mtmd_gen_ref>(voice.ref);
}
bitmap->id = name;
bitmap->data.assign((const unsigned char *) rows, (const unsigned char *) (rows + (size_t) n_embd * n_rows));
@@ -268,7 +268,7 @@ struct mtmd_image_tokens {
uint32_t n = 0;
for (size_t k = 0; k < batch_f32.entries.size(); ++k) {
const auto [gw, gh] = canvas_tile_grid(k);
- n += gw * gh + (uint32_t) clip_suffix_rows(batch_f32.entries[k].suffix_type).size();
+ n += gw * gh + (uint32_t) mtmd_suffix_rows(batch_f32.entries[k].suffix_type).size();
}
return n;
}
@@ -284,7 +284,7 @@ struct mtmd_image_tokens {
}
return nx * ny * nz;
}
- clip_image_f32_batch batch_f32; // preprocessed image patches
+ mtmd_image_f32_batch batch_f32; // preprocessed image patches
std::string id; // optional user-defined ID, useful for KV cache tracking
// true if one of entries in batch_f32 is a placeholder
@@ -354,16 +354,16 @@ using mtmd_audio_source_ptr = std::shared_ptr<mtmd_audio_preproc_out>;
// an audio track split across chunks (e.g. the soundtrack of a video): it is encoded once, as one clip, and every
// chunk takes its rows of it
struct mtmd_audio_track {
- clip_image_f32_batch batch_f32; // the mel of the whole track
+ mtmd_image_f32_batch batch_f32; // the mel of the whole track
mtmd_audio_source_ptr source;
uint32_t n_tokens = 0;
std::vector<float> embd; // filled by the first encode
- std::shared_ptr<const clip_gen_ref> gen_ref; // gen-side reference of a predefined voice, if any
+ std::shared_ptr<const mtmd_gen_ref> gen_ref; // gen-side reference of a predefined voice, if any
};
struct mtmd_audio_tokens {
uint32_t n_tokens = 0; // number of tokens
- clip_image_f32_batch batch_f32; // preprocessed image patches
+ mtmd_image_f32_batch batch_f32; // preprocessed image patches
std::string id; // optional user-defined ID, useful for KV cache tracking
mtmd_audio_source_ptr source; // if set, batch_f32 has the shape of the mel of chunk source_idx
@@ -554,13 +554,13 @@ const char * mtmd_default_marker() {
return "<__media__>";
}
-static clip_flash_attn_type mtmd_get_clip_flash_attn_type(enum llama_flash_attn_type flash_attn_type) {
+static mtmd_flash_attn_type mtmd_to_flash_attn_type(enum llama_flash_attn_type flash_attn_type) {
switch (flash_attn_type) {
- case LLAMA_FLASH_ATTN_TYPE_AUTO: return CLIP_FLASH_ATTN_TYPE_AUTO;
- case LLAMA_FLASH_ATTN_TYPE_DISABLED: return CLIP_FLASH_ATTN_TYPE_DISABLED;
- case LLAMA_FLASH_ATTN_TYPE_ENABLED: return CLIP_FLASH_ATTN_TYPE_ENABLED;
+ case LLAMA_FLASH_ATTN_TYPE_AUTO: return MTMD_FLASH_ATTN_TYPE_AUTO;
+ case LLAMA_FLASH_ATTN_TYPE_DISABLED: return MTMD_FLASH_ATTN_TYPE_DISABLED;
+ case LLAMA_FLASH_ATTN_TYPE_ENABLED: return MTMD_FLASH_ATTN_TYPE_ENABLED;
}
- return CLIP_FLASH_ATTN_TYPE_AUTO;
+ return MTMD_FLASH_ATTN_TYPE_AUTO;
}
mtmd_context_params mtmd_context_params_default() {
@@ -585,20 +585,23 @@ mtmd_context_params mtmd_context_params_default() {
}
// chatterbox multilingual has a perceiver in its conditioning encoder (speaker encoding context)
-static bool cbx_is_mtl(clip_ctx * ctx_a) {
- return clip_cbx_read_tensor(ctx_a, "a.cenc.perceiver.pre_attention_query", nullptr, 0) > 0;
+static bool cbx_is_mtl(const mtmd_model & model) {
+ return model.cbx_read_tensor("a.cenc.perceiver.pre_attention_query", nullptr, 0) > 0;
}
struct mtmd_context {
- struct clip_ctx * ctx_v; // vision
- struct clip_ctx * ctx_a; // audio
+ // must outlive the subcontexts below
+ mtmd_models models;
+
+ std::unique_ptr<mtmd_subcontext> ctx_v; // vision
+ std::unique_ptr<mtmd_subcontext> ctx_a; // audio
std::vector<float> out_embd; // image embedding vector
// generation context
- struct clip_ctx * ctx_gen_a; // audio
+ std::unique_ptr<mtmd_subcontext> ctx_gen_a; // audio
// gen-side reference data of the last encoded chunk (chatterbox), see mtmd_gen_state_set_ref()
- clip_gen_ref gen_ref;
+ mtmd_gen_ref gen_ref;
bool print_timings;
int n_threads;
@@ -683,34 +686,48 @@ struct mtmd_context {
}
}
- clip_context_params ctx_clip_params {
- /* use_gpu */ ctx_params.use_gpu,
- /* device */ ctx_params.device,
- /* flash_attn_type */ mtmd_get_clip_flash_attn_type(ctx_params.flash_attn_type),
- /* image_min_tokens */ ctx_params.image_min_tokens,
- /* image_max_tokens */ ctx_params.image_max_tokens,
+ ggml_backend_dev_t device = mtmd_select_device(ctx_params.use_gpu, ctx_params.device);
+
+ mtmd_model_params model_params;
+ model_params.image_min_tokens = ctx_params.image_min_tokens;
+ model_params.image_max_tokens = ctx_params.image_max_tokens;
+ model_params.no_alloc = no_alloc;
+ model_params.progress_callback = ctx_params.progress_callback;
+ model_params.progress_callback_user_data = ctx_params.progress_callback_user_data;
+
+ ggml_backend_buffer_type_t buft = device
+ ? ggml_backend_dev_buffer_type(device)
+ : ggml_backend_dev_buffer_type(ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU));
+ models = mtmd_model_load(mmproj_fname, model_params, buft);
+
+ // soprano and pocket-tts without voice cloning have no encoder
+ if (!models.v && !models.a && !models.gen_a) {
+ throw std::runtime_error(string_format("Failed to load mmproj model from %s\n", mmproj_fname));
+ }
+
+ mtmd_subctx_params sctx_params {
+ /* device */ device,
+ // no_alloc only measures memory, nothing is computed
+ /* threadpool */ mtmd_threadpool_init(no_alloc ? 1 : ctx_params.n_threads),
+ /* flash_attn_type */ mtmd_to_flash_attn_type(ctx_params.flash_attn_type),
/* warmup */ ctx_params.warmup,
/* cb_eval */ ctx_params.cb_eval,
/* cb_eval_user_data */ ctx_params.cb_eval_user_data,
- /* no_alloc */ no_alloc,
- /* progress_callback */ ctx_params.progress_callback,
- /* progress_callback_user_data */ ctx_params.progress_callback_user_data,
- /* n_threads */ ctx_params.n_threads,
};
-
- auto res = clip_init(mmproj_fname, ctx_clip_params);
- ctx_v = res.ctx_v;
- ctx_a = res.ctx_a;
- ctx_gen_a = res.ctx_gen_a;
- // soprano and pocket-tts without voice cloning have no encoder
- if (!ctx_v && !ctx_a && !ctx_gen_a) {
- throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname));
+ if (models.v) {
+ ctx_v = std::make_unique<mtmd_subcontext>(*models.v, sctx_params);
+ }
+ if (models.a) {
+ ctx_a = std::make_unique<mtmd_subcontext>(*models.a, sctx_params);
+ }
+ if (models.gen_a) {
+ ctx_gen_a = std::make_unique<mtmd_subcontext>(*models.gen_a, sctx_params);
}
// if both vision and audio mmproj are present, we need to validate their n_embd
if (ctx_v && ctx_a) {
- int n_embd_v = clip_n_mmproj_embd(ctx_v);
- int n_embd_a = clip_n_mmproj_embd(ctx_a);
+ int n_embd_v = ctx_v->model.n_mmproj_embd();
+ int n_embd_a = ctx_a->model.n_mmproj_embd();
if (n_embd_v != n_embd_a) {
throw std::runtime_error(string_format(
"mismatch between vision and audio mmproj (n_embd_v = %d, n_embd_a = %d)\n",
@@ -721,15 +738,15 @@ struct mtmd_context {
// since we already validate n_embd of vision and audio mmproj,
// we can safely assume that they are the same
// a gen-only mmproj is validated below
- int n_embd_clip = ctx_v || ctx_a ? clip_n_mmproj_embd(ctx_v ? ctx_v : ctx_a) : n_embd_text;
- if (n_embd_text > 0 && n_embd_text != n_embd_clip) {
+ int n_embd_mmproj = ctx_v || ctx_a ? (ctx_v ? ctx_v : ctx_a)->model.n_mmproj_embd() : n_embd_text;
+ if (n_embd_text > 0 && n_embd_text != n_embd_mmproj) {
throw std::runtime_error(string_format(
"mismatch between text model (n_embd = %d) and mmproj (n_embd = %d)\n"
"hint: you may be using wrong mmproj\n",
- n_embd_text, n_embd_clip));
+ n_embd_text, n_embd_mmproj));
}
if (ctx_gen_a) {
- int n_embd_gen = clip_n_mmproj_embd(ctx_gen_a);
+ int n_embd_gen = ctx_gen_a->model.n_mmproj_embd();
if (n_embd_text > 0 && n_embd_text != n_embd_gen) {
throw std::runtime_error(string_format(
"mismatch between text model (n_embd = %d) and gen-audio mmproj (n_embd = %d)\n"
@@ -749,7 +766,7 @@ struct mtmd_context {
GGML_ASSERT(ctx_v != nullptr);
image_preproc.reset();
- projector_type proj = clip_get_projector_type(ctx_v);
+ projector_type proj = ctx_v->model.proj_type;
switch (proj) {
case PROJECTOR_TYPE_MLP:
@@ -760,16 +777,16 @@ struct mtmd_context {
case PROJECTOR_TYPE_JANUS_PRO:
case PROJECTOR_TYPE_GLM_EDGE:
{
- bool has_pinpoints = !clip_get_hparams(ctx_v)->image_res_candidates.empty();
+ bool has_pinpoints = !ctx_v->model.hparams.image_res_candidates.empty();
if (has_pinpoints) {
- image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(models.v.get());
} else {
- image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(models.v.get());
}
} break;
case PROJECTOR_TYPE_MINICPMV:
{
- int minicpmv_version = clip_get_hparams(ctx_v)->minicpmv_version;
+ int minicpmv_version = ctx_v->model.hparams.minicpmv_version;
if (minicpmv_version == 2) {
// minicpmv 2.5 format:
// <image> (overview) </image><slice><image> (slice) </image><image> (slice) </image>\n ... </slice>
@@ -798,7 +815,7 @@ struct mtmd_context {
} else if (minicpmv_version != 0) {
throw std::runtime_error(string_format("unsupported minicpmv version: %d\n", minicpmv_version));
}
- image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(models.v.get());
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
@@ -813,10 +830,10 @@ struct mtmd_context {
tok_row_end_trail = false; // no trailing end-of-row token
ov_img_first = true;
use_image_id = true;
- if (clip_get_projector_type(ctx_v) == PROJECTOR_TYPE_MINICPM_OCR_1_0) {
- image_preproc = std::make_unique<mtmd_image_preprocessor_minicpm_ocr_1_0>(ctx_v);
+ if (ctx_v->model.proj_type == PROJECTOR_TYPE_MINICPM_OCR_1_0) {
+ image_preproc = std::make_unique<mtmd_image_preprocessor_minicpm_ocr_1_0>(models.v.get());
} else {
- image_preproc = std::make_unique<mtmd_image_preprocessor_minicpmv>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_minicpmv>(models.v.get());
}
} break;
case PROJECTOR_TYPE_QWEN3VL:
@@ -824,7 +841,7 @@ struct mtmd_context {
// <|vision_start|> ... (image embeddings) ... <|vision_end|>
img_beg = "<|vision_start|>";
img_end = "<|vision_end|>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_qwen3vl>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_qwen3vl>(models.v.get());
} break;
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
@@ -835,26 +852,26 @@ struct mtmd_context {
// <|vision_start|> ... (image embeddings) ... <|vision_end|>
img_beg = "<|vision_start|>";
img_end = "<|vision_end|>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} break;
case PROJECTOR_TYPE_MINIMAX_M3:
{
// ]<]start of image[>[ ... (image embeddings) ... ]<]end of image[>[
img_beg = "]<]start of image[>[";
img_end = "]<]end of image[>[";
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} break;
case PROJECTOR_TYPE_SENSENOVA_U1:
{
img_beg = "<img>";
img_end = "</img>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} break;
case PROJECTOR_TYPE_MUSE_GLIMMER:
{
img_beg = "<|image_start|>";
img_end = "<|image_end|>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_muse_glimmer>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_muse_glimmer>(models.v.get());
} break;
case PROJECTOR_TYPE_LFM3VL:
{
@@ -863,14 +880,14 @@ struct mtmd_context {
img_end = "<|vision_end|>";
vid_beg = img_beg;
vid_end = img_end;
- image_preproc = std::make_unique<mtmd_image_preprocessor_lfm3vl>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_lfm3vl>(models.v.get());
} break;
case PROJECTOR_TYPE_YOUTUVL:
{
// <|vision_start|> ... (image embeddings) ... <|vision_end|>
img_beg = "<|vision_start|>";
img_end = "<|vision_end|>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_youtuvl>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_youtuvl>(models.v.get());
} break;
case PROJECTOR_TYPE_YASA2:
{
@@ -883,7 +900,7 @@ struct mtmd_context {
// images. This will be implemented in a future PR (dispatch on has_pinpoints
// - see LDP/COGVLM branch above) and emit image_grid_pinpoints in the conversion
// script.
- image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(models.v.get());
} break;
case PROJECTOR_TYPE_GEMMA3:
case PROJECTOR_TYPE_GEMMA3NV:
@@ -891,7 +908,7 @@ struct mtmd_context {
// <start_of_image> ... (image embeddings) ... <end_of_image>
img_beg = "<start_of_image>";
img_end = "<end_of_image>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(models.v.get());
} break;
case PROJECTOR_TYPE_IDEFICS3:
{
@@ -901,22 +918,22 @@ struct mtmd_context {
tok_ov_img_end = {lookup_token("<fake_token_around_image>")};
tok_row_end = {lookup_token("\n")};
sli_img_start_tmpl = "<fake_token_around_image><row_%d_col_%d>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_idefics3>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_idefics3>(models.v.get());
} break;
case PROJECTOR_TYPE_PIXTRAL:
{
// https://github.com/huggingface/transformers/blob/1cd110c6cb6a6237614130c470e9a902dbc1a4bd/docs/source/en/model_doc/pixtral.md
img_end = "[IMG_END]";
- if (clip_get_hparams(ctx_v)->image_longest_edge > 0) {
- image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(ctx_v);
+ if (ctx_v->model.hparams.image_longest_edge > 0) {
+ image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(models.v.get());
} else {
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
}
} break;
case PROJECTOR_TYPE_PHI4:
{
// Phi-4 uses media marker insertion only. Keep image boundary text empty.
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} break;
case PROJECTOR_TYPE_LLAMA4:
{
@@ -925,7 +942,7 @@ struct mtmd_context {
img_end = "<|image_end|>";
LOG_WRN("%s: llama 4 vision is known to have degraded quality:\n"
" https://github.com/ggml-org/llama.cpp/pull/13282\n", __func__);
- image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(models.v.get());
ov_img_first = false;
} break;
case PROJECTOR_TYPE_STEP3VL:
@@ -942,14 +959,14 @@ struct mtmd_context {
tok_row_end = {lookup_token("<patch_newline>")};
tok_row_end_trail = false;
ov_img_first = false; // patches first, overview last
- image_preproc = std::make_unique<mtmd_image_preprocessor_step3vl>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_step3vl>(models.v.get());
} break;
case PROJECTOR_TYPE_INTERNVL:
{
// <img> ... (image embeddings) ... </img>
img_beg = "<img>";
img_end = "</img>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_internvl>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_internvl>(models.v.get());
ov_img_first = false;
} break;
case PROJECTOR_TYPE_COHERE2V:
@@ -960,14 +977,14 @@ struct mtmd_context {
tok_sli_img_end = {lookup_token("<|IMG_LINE_BREAK|>")};
tok_ov_img_end = tok_sli_img_end;
ov_img_first = false;
- image_preproc = std::make_unique<mtmd_image_preprocessor_cohere2v>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_cohere2v>(models.v.get());
} break;
case PROJECTOR_TYPE_KIMIVL:
{
// <|media_start|> ... (image embeddings) ... <|media_end|>
img_beg = "<|media_start|>";
img_end = "<|media_end|>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} break;
case PROJECTOR_TYPE_KIMIK25:
{
@@ -982,24 +999,24 @@ struct mtmd_context {
img_beg = "<|media_begin|>";
img_end = "<|media_end|>";
}
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} break;
case PROJECTOR_TYPE_LIGHTONOCR:
{
// <|im_start|> ... (image embeddings) ... <|im_end|>
img_beg = "<|im_start|>";
img_end = "<|im_end|>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(models.v.get());
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
// no vocab tokens are added; the start/end/newline markers are learned embeddings emitted by the encoder
- image_preproc = std::make_unique<mtmd_image_preprocessor_deepseek4v>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_deepseek4v>(models.v.get());
} break;
case PROJECTOR_TYPE_DEEPSEEK41V:
{
// as deepseek4v: markers come from the encoder, the block is decoded causally
- image_preproc = std::make_unique<mtmd_image_preprocessor_deepseek41v>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_deepseek41v>(models.v.get());
} break;
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
@@ -1007,17 +1024,17 @@ struct mtmd_context {
// <|img|> ... (image embeddings) ... <|endofimg|>
img_beg = "<|img|>";
img_end = "<|endofimg|>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} break;
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
{
- image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(models.v.get());
} break;
case PROJECTOR_TYPE_NEMOTRON_OMNI:
{
img_beg = "<img>";
img_end = "</img>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} break;
case PROJECTOR_TYPE_LFM2:
{
@@ -1034,33 +1051,33 @@ struct mtmd_context {
sli_img_start_tmpl = "<|img_row_%d_col_%d|>";
tok_ov_img_start = {lookup_token("<|img_thumbnail|>")};
ov_img_first = false;
- image_preproc = std::make_unique<mtmd_image_preprocessor_lfm2>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_lfm2>(models.v.get());
} break;
case PROJECTOR_TYPE_D1OMNI_V:
{
// same tiles and thumbnail as lfm2, without separator tokens
- image_preproc = std::make_unique<mtmd_image_preprocessor_lfm2>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_lfm2>(models.v.get());
} break;
case PROJECTOR_TYPE_GLM4V:
{
// <|begin_of_image|> ... (image embeddings) ... <|end_of_image|>
img_beg = "<|begin_of_image|>";
img_end = "<|end_of_image|>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} break;
case PROJECTOR_TYPE_GLM5V:
{
// <|begin_of_image|> ... (image embeddings) ... <|end_of_image|>
img_beg = "<|begin_of_image|>";
img_end = "<|end_of_image|>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_glm5v>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_glm5v>(models.v.get());
} break;
case PROJECTOR_TYPE_PADDLEOCR:
{
// <|IMAGE_START|> ... (image embeddings) ... <|IMAGE_END|>
img_beg = "<|IMAGE_START|>";
img_end = "<|IMAGE_END|>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} break;
case PROJECTOR_TYPE_GEMMA4V:
case PROJECTOR_TYPE_GEMMA4UV:
@@ -1068,13 +1085,13 @@ struct mtmd_context {
// <|image> ... (image embeddings) ... <image|>
img_beg = "<|image>";
img_end = "<image|>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} break;
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
{
img_end = "\n"; // prevent empty batch on llama-server
- image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(models.v.get());
ov_img_first = false;
} break;
case PROJECTOR_TYPE_HUNYUANVL:
@@ -1082,21 +1099,21 @@ struct mtmd_context {
// note: these use fullwidth ｜ (U+FF5C) and ▁ (U+2581) to match the tokenizer vocabulary
img_beg = "<｜hy_place▁holder▁no▁100｜>";
img_end = "<｜hy_place▁holder▁no▁101｜>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} break;
case PROJECTOR_TYPE_EXAONE4_5:
{
// <vision> ... (image embeddings) ... </vision>
img_beg = "<vision>";
img_end = "</vision>";
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
// ... (image embeddings) \n ...
img_beg = "";
img_end = "\n";
- image_preproc = std::make_unique<mtmd_image_preprocessor_granite>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_granite>(models.v.get());
} break;
default:
throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj));
@@ -1109,7 +1126,7 @@ struct mtmd_context {
GGML_ASSERT(ctx_a != nullptr);
audio_preproc.reset();
- projector_type proj = clip_get_projector_type(ctx_a);
+ projector_type proj = ctx_a->model.proj_type;
LOG_WRN("%s: audio input is in experimental stage and may have reduced quality:\n"
" https://github.com/ggml-org/llama.cpp/discussions/13759\n", __func__);
@@ -1122,31 +1139,31 @@ struct mtmd_context {
// <|audio_bos|> ... (embeddings) ... <|audio_eos|>
aud_beg = "<|audio_bos|>";
aud_end = "<|audio_eos|>";
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(models.a.get());
} break;
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
{
aud_beg = "<|soft_audio_start|>";
aud_end = "<|soft_audio_end|>";
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_muse_spark_audio>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_muse_spark_audio>(models.a.get());
} break;
case PROJECTOR_TYPE_QWEN3A:
{
aud_beg = "<|audio_start|>";
aud_end = "<|audio_end|>";
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_qwen3a>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_qwen3a>(models.a.get());
} break;
case PROJECTOR_TYPE_VOXTRAL:
{
// [BEGIN_AUDIO] ... (embeddings) ...
aud_beg = "[BEGIN_AUDIO]";
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(models.a.get());
} break;
case PROJECTOR_TYPE_MUSIC_FLAMINGO:
{
// <sound> ... (embeddings) ...
aud_beg = "<sound>";
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(models.a.get());
} break;
case PROJECTOR_TYPE_VOXTRAL_RT:
{
@@ -1156,15 +1173,15 @@ struct mtmd_context {
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_MERALION:
{
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(models.a.get());
} break;
case PROJECTOR_TYPE_LFM2A:
{
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_conformer>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_conformer>(models.a.get());
} break;
case PROJECTOR_TYPE_D1OMNI_A:
{
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_d1omni>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_d1omni>(models.a.get());
} break;
case PROJECTOR_TYPE_LFM3A:
{
@@ -1173,56 +1190,56 @@ struct mtmd_context {
aud_beg = img_beg;
aud_end = img_end;
}
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_lfm3a>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_lfm3a>(models.a.get());
} break;
case PROJECTOR_TYPE_GRANITE_SPEECH:
{
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_granite_speech>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_granite_speech>(models.a.get());
} break;
case PROJECTOR_TYPE_GEMMA4A:
{
aud_beg = "<|audio>";
aud_end = "<audio|>";
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_gemma4a>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_gemma4a>(models.a.get());
} break;
case PROJECTOR_TYPE_PARAKEET:
{
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_parakeet>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_parakeet>(models.a.get());
} break;
case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
{
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_nemotron_transcribe>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_nemotron_transcribe>(models.a.get());
} break;
case PROJECTOR_TYPE_GEMMA4UA:
{
aud_beg = "<|audio>";
aud_end = "<audio|>";
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_gemma4ua>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_gemma4ua>(models.a.get());
} break;
case PROJECTOR_TYPE_DOTS3NOTE_A:
{
// <|audio_comp_start|> ... (embeddings) ... <|audio_comp_end|>
aud_beg = "<|audio_comp_start|>";
aud_end = "<|audio_comp_end|>";
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_dots3note>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_dots3note>(models.a.get());
} break;
case PROJECTOR_TYPE_MIMO_AUDIO:
{
aud_beg = "<|mimo_audio_start|>";
aud_end = "<|mimo_audio_end|>";
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_mimo_audio>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_mimo_audio>(models.a.get());
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_qwen3tts_spk>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_qwen3tts_spk>(models.a.get());
} break;
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
{
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_pockettts>(ctx_a);
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_pockettts>(models.a.get());
} break;
case PROJECTOR_TYPE_CHATTERBOX_SPKENC:
{
- audio_preproc = std::make_unique<mtmd_audio_preprocessor_chatterbox_ref>(ctx_a, cbx_is_mtl(ctx_a));
+ audio_preproc = std::make_unique<mtmd_audio_preprocessor_chatterbox_ref>(models.a.get(), cbx_is_mtl(*models.a));
} break;
default:
throw std::runtime_error(string_format("%s: unexpected audio projector type %d\n", __func__, proj));
@@ -1231,26 +1248,27 @@ struct mtmd_context {
// initialize audio preprocessor
if (audio_preproc) {
audio_preproc->mel_config.n_threads = n_threads;
+ audio_preproc->mel_config.sched = ctx_a->sched.get();
audio_preproc->initialize();
}
}
- // get clip ctx based on chunk type
- clip_ctx * get_clip_ctx(const mtmd_input_chunk * chunk) const {
+ // get the subcontext based on chunk type
+ mtmd_subcontext * get_subctx(const mtmd_input_chunk * chunk) const {
if (chunk->type == MTMD_INPUT_CHUNK_TYPE_IMAGE) {
- return ctx_v;
+ return ctx_v.get();
} else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {
- return ctx_a;
+ return ctx_a.get();
}
GGML_ABORT("unknown chunk type");
}
projector_type proj_type_v() const {
- return ctx_v ? clip_get_projector_type(ctx_v) : PROJECTOR_TYPE_UNKNOWN;
+ return ctx_v ? ctx_v->model.proj_type : PROJECTOR_TYPE_UNKNOWN;
}
projector_type proj_type_a() const {
- return ctx_a ? clip_get_projector_type(ctx_a) : PROJECTOR_TYPE_UNKNOWN;
+ return ctx_a ? ctx_a->model.proj_type : PROJECTOR_TYPE_UNKNOWN;
}
// [LFM3_OMNI] the audio of a video is its soundtrack, see MTMD_VIDEO_TEMPLATE_LFM3_OMNI
@@ -1260,20 +1278,14 @@ struct mtmd_context {
int64_t n_embd_out() const {
if (ctx_v) {
- return clip_n_mmproj_embd(ctx_v);
+ return ctx_v->model.n_mmproj_embd();
} else if (ctx_a) {
- return clip_n_mmproj_embd(ctx_a);
+ return ctx_a->model.n_mmproj_embd();
} else {
- throw std::runtime_error("no CLIP model loaded");
+ throw std::runtime_error("no mmproj model loaded");
}
}
- ~mtmd_context() {
- clip_free(ctx_a);
- clip_free(ctx_v);
- clip_free(ctx_gen_a);
- }
-
private:
llama_token lookup_token(const std::string & token_text) {
if (vocab == nullptr) {
@@ -1498,7 +1510,7 @@ struct mtmd_tokenizer {
// [QWEN_VIDEO] handle frame merging for models that support it (i.e. qwen-vl)
int n_merge_frames = 1;
if (ctx->ctx_v) {
- n_merge_frames = clip_model_n_temporal_merge(ctx->ctx_v);
+ n_merge_frames = ctx->ctx_v->model.n_temporal_merge();
GGML_ASSERT(n_merge_frames <= 2 && "we only support merging maximum 2 images for now; open an issue if this model supports merging more");
}
@@ -1624,8 +1636,8 @@ struct mtmd_tokenizer {
return 2;
}
- // convert mtmd_bitmap to clip_image_u8
- clip_image_u8 img_u8;
+ // convert mtmd_bitmap to mtmd_image_u8
+ mtmd_image_u8 img_u8;
img_u8.set_size(
{(int)bmp->nx, (int)bmp->ny},
bmp->is_placeholder());
@@ -1667,7 +1679,7 @@ struct mtmd_tokenizer {
add_text("<image_id>" + std::to_string(n_images_added) + "</image_id>", true);
}
add_text(ctx->tok_ov_img_start);
- // the separators after <image> are appended by clip, see add_canvas_chunk()
+ // the separators after <image> are appended by the graph, see add_canvas_chunk()
add_canvas_chunk(std::move(preproc_out), bitmaps[0]->id);
} else if (has_tiling_grid) {
// [QWEN_VIDEO] we do not support "frame merging" for llama-uhd style, so no batching for now
@@ -1762,8 +1774,8 @@ struct mtmd_tokenizer {
size_t n_tokens = 0;
for (auto & e : preproc_out.entries) {
- n_tokens += clip_n_output_tokens(ctx->ctx_v, &e);
- if (clip_model_n_temporal_merge(ctx->ctx_v) == 2) {
+ n_tokens += ctx->ctx_v->model.n_output_tokens(&e);
+ if (ctx->ctx_v->model.n_temporal_merge() == 2) {
// [QWEN_VIDEO] pair input is merged to the same embd, so only count as one image
break;
}
@@ -1772,12 +1784,12 @@ struct mtmd_tokenizer {
mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens);
// [QWEN_VIDEO] improve this in the future
- image_tokens->n_temporal_merge = clip_model_n_temporal_merge(ctx->ctx_v);
+ image_tokens->n_temporal_merge = ctx->ctx_v->model.n_temporal_merge();
if (mtmd_decode_use_mrope(ctx)) {
// for Qwen2VL, we need this information for M-RoPE decoding positions
- image_tokens->nx = clip_n_output_tokens_x(ctx->ctx_v, &preproc_out.entries[0]);
- image_tokens->ny = clip_n_output_tokens_y(ctx->ctx_v, &preproc_out.entries[0]);
+ image_tokens->nx = ctx->ctx_v->model.n_output_tokens_x(&preproc_out.entries[0]);
+ image_tokens->ny = ctx->ctx_v->model.n_output_tokens_y(&preproc_out.entries[0]);
} else {
// other models, we only need the total number of tokens
image_tokens->nx = n_tokens;
@@ -1798,7 +1810,7 @@ struct mtmd_tokenizer {
GGML_ASSERT(n_tokens == (size_t)image_tokens->n_tokens());
}
- clip_image_f32_batch batch_f32;
+ mtmd_image_f32_batch batch_f32;
batch_f32.is_audio = false;
batch_f32.is_video = bitmaps[0]->is_video_frame();
batch_f32.entries = std::move(preproc_out.entries);
@@ -1887,13 +1899,13 @@ struct mtmd_tokenizer {
}
// one chunk for the five entries of a chatterbox clip, one token per talker conditioning row
- if (clip_get_projector_type(ctx->ctx_a) == PROJECTOR_TYPE_CHATTERBOX_SPKENC) {
+ if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_CHATTERBOX_SPKENC) {
GGML_ASSERT(source->entries.size() == 5);
// mtl: spkr, perceiver query rows, emotion; turbo: spkr, t3 speech tokens
- const size_t n_query = clip_cbx_read_tensor(ctx->ctx_a, "a.cenc.perceiver.pre_attention_query", nullptr, 0) / ctx->n_embd_out();
- const size_t n_tokens = cbx_is_mtl(ctx->ctx_a) ? 2 + n_query : 1 + (size_t) source->entries[2].nx() / 4;
+ const size_t n_query = ctx->ctx_a->model.cbx_read_tensor("a.cenc.perceiver.pre_attention_query", nullptr, 0) / ctx->n_embd_out();
+ const size_t n_tokens = cbx_is_mtl(ctx->ctx_a->model) ? 2 + n_query : 1 + (size_t) source->entries[2].nx() / 4;
- clip_image_f32_batch batch_f32;
+ mtmd_image_f32_batch batch_f32;
batch_f32.is_audio = true;
batch_f32.entries = std::move(source->entries);
@@ -1913,11 +1925,11 @@ struct mtmd_tokenizer {
// consider each mel_spec as a separate audio chunk
// TODO: maybe support batching, but this may come with memory cost
for (size_t i = 0; i < source->entries.size(); i++) {
- clip_image_f32 mel_f32 = source->entries[i];
+ mtmd_image_f32 mel_f32 = source->entries[i];
- size_t n_tokens = clip_n_output_tokens(ctx->ctx_a, &mel_f32);
+ size_t n_tokens = ctx->ctx_a->model.n_output_tokens(&mel_f32);
- clip_image_f32_batch batch_f32;
+ mtmd_image_f32_batch batch_f32;
batch_f32.is_audio = true;
batch_f32.entries.push_back(std::move(mel_f32));
@@ -1995,9 +2007,9 @@ struct mtmd_tokenizer {
LOG_ERR("%s: unable to preprocess the soundtrack\n", __func__);
return 2;
}
- clip_image_f32 mel_f32 = source->entries[0];
+ mtmd_image_f32 mel_f32 = source->entries[0];
st.track = std::make_shared<mtmd_audio_track>();
- st.track->n_tokens = clip_n_output_tokens(ctx->ctx_a, &mel_f32);
+ st.track->n_tokens = ctx->ctx_a->model.n_output_tokens(&mel_f32);
if (source->needs_compute()) {
st.track->source = source;
}
@@ -2006,7 +2018,7 @@ struct mtmd_tokenizer {
}
// rows of [begin, end): round(sample / samples per row), the last piece takes the rest of the track
- const double samples_per_row = 8.0 * clip_get_hparams(ctx->ctx_a)->audio_hop_len; // one row per 8 mel frames
+ const double samples_per_row = 8.0 * ctx->ctx_a->model.hparams.audio_hop_len; // one row per 8 mel frames
const uint32_t n_tokens = st.track->n_tokens;
st.n_samples += bitmap.nx;
const uint32_t begin = st.n_assigned;
@@ -2031,7 +2043,7 @@ struct mtmd_tokenizer {
return 0;
}
- // MiniCPM-V 4.7: the overview and all slices go in one chunk, clip appends the separators after each tile:
+ // MiniCPM-V 4.7: the overview and all slices go in one chunk, the graph appends the separators after each tile:
// [ov] </image><slice> [S00] </slice><slice> [S01] </slice>\n<slice> [S10] </slice><slice> [S11] </slice>
void add_canvas_chunk(mtmd_image_preproc_out && preproc_out, const std::string & id) {
const int n_col = preproc_out.grid_x;
@@ -2041,9 +2053,9 @@ struct mtmd_tokenizer {
GGML_ASSERT((int) slices.size() == n_col * n_row);
auto & ov = preproc_out.overview;
- ov.suffix_type = CLIP_SUFFIX_MINICPMV_OV;
+ ov.suffix_type = MTMD_SUFFIX_MINICPMV_OV;
if (!slices.empty()) {
- ov.suffix_type = CLIP_SUFFIX_MINICPMV_OV_SLICE;
+ ov.suffix_type = MTMD_SUFFIX_MINICPMV_OV_SLICE;
ov.anyres.grid_x = n_col;
ov.anyres.grid_y = n_row;
}
@@ -2051,11 +2063,11 @@ struct mtmd_tokenizer {
for (int x = 0; x < n_col; x++) {
auto & suffix = slices[y * n_col + x].suffix_type;
if (y == n_row - 1 && x == n_col - 1) {
- suffix = CLIP_SUFFIX_MINICPMV_LAST;
+ suffix = MTMD_SUFFIX_MINICPMV_LAST;
} else if (x == n_col - 1) {
- suffix = CLIP_SUFFIX_MINICPMV_ROW_END;
+ suffix = MTMD_SUFFIX_MINICPMV_ROW_END;
} else {
- suffix = CLIP_SUFFIX_MINICPMV_SLICE;
+ suffix = MTMD_SUFFIX_MINICPMV_SLICE;
}
}
}
@@ -2069,12 +2081,12 @@ struct mtmd_tokenizer {
entries.push_back(std::move(slice));
}
// token grid of the last entry, the grids of the other entries are scaled from it
- image_tokens->nx = clip_n_output_tokens_x(ctx->ctx_v, &entries.back());
- image_tokens->ny = clip_n_output_tokens_y(ctx->ctx_v, &entries.back());
+ image_tokens->nx = ctx->ctx_v->model.n_output_tokens_x(&entries.back());
+ image_tokens->ny = ctx->ctx_v->model.n_output_tokens_y(&entries.back());
size_t n_tokens = 0;
for (const auto & entry : entries) {
- n_tokens += clip_n_output_tokens(ctx->ctx_v, &entry);
+ n_tokens += ctx->ctx_v->model.n_output_tokens(&entry);
}
GGML_ASSERT(n_tokens == image_tokens->n_tokens());
@@ -2090,9 +2102,9 @@ struct mtmd_tokenizer {
std::vector<mtmd_input_chunk> split_batch_to_chunk(mtmd_image_preproc_out && preproc_out, const std::string & id) {
std::vector<mtmd_input_chunk> chunks;
- auto process_chunk = [&](clip_image_f32 && img) {
+ auto process_chunk = [&](mtmd_image_f32 && img) {
mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens);
- image_tokens->nx = clip_n_output_tokens(ctx->ctx_v, &img);
+ image_tokens->nx = ctx->ctx_v->model.n_output_tokens(&img);
image_tokens->ny = 1;
image_tokens->batch_f32.entries.push_back(std::move(img));
image_tokens->id = id;
@@ -2213,8 +2225,8 @@ int32_t mtmd_tokenize_from_parts(const mtmd_context * ctx,
}
static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * image_tokens, std::vector<float> & out_embd) {
- clip_ctx * ctx_clip = ctx->ctx_v;
- if (!ctx_clip) {
+ mtmd_subcontext * sctx = ctx->ctx_v.get();
+ if (!sctx) {
LOG_ERR("%s: this API does not support non-vision input, please use mtmd_encode_chunk instead\n", __func__);
return 1;
}
@@ -2232,10 +2244,10 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im
// the tiles differ in size, encode them one by one
size_t offset = 0;
for (const auto & entry : image_tokens->batch_f32.entries) {
- clip_image_f32_batch one;
+ mtmd_image_f32_batch one;
one.entries.push_back(entry);
- std::vector<float> embd((size_t) n_embd_out * clip_n_output_tokens(ctx_clip, &entry));
- if (!clip_image_batch_encode(ctx_clip, ctx->n_threads, &one, embd)) {
+ std::vector<float> embd((size_t) n_embd_out * sctx->model.n_output_tokens(&entry));
+ if (!sctx->encode_batch(ctx->n_threads, &one, embd)) {
return 1;
}
GGML_ASSERT(offset + embd.size() <= out_embd.size());
@@ -2246,9 +2258,7 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im
return 0;
}
- bool ok = clip_image_batch_encode(
- ctx_clip,
- ctx->n_threads,
+ bool ok = sctx->encode_batch(ctx->n_threads,
&image_tokens->batch_f32,
out_embd);
@@ -2257,8 +2267,8 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im
static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd);
-static bool mtmd_audio_batch_encode(mtmd_context * ctx, const clip_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector<float> & out_embd) {
- clip_image_f32_batch batch = batch_f32.clone();
+static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector<float> & out_embd) {
+ mtmd_image_f32_batch batch = batch_f32.clone();
if (source) {
if (!ctx->audio_preproc->compute(*source)) {
LOG_ERR("%s: unable to compute the mel\n", __func__);
@@ -2267,7 +2277,7 @@ static bool mtmd_audio_batch_encode(mtmd_context * ctx, const clip_image_f32_bat
GGML_ASSERT(idx < source->entries.size());
batch.entries[0] = source->entries[idx];
}
- return clip_image_batch_encode(ctx->ctx_a, ctx->n_threads, &batch, out_embd);
+ return ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd);
}
static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk * chunk, std::vector<float> & out_embd) {
@@ -2321,7 +2331,7 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk
}
return 0;
}
- if (clip_get_projector_type(ctx->ctx_a) == PROJECTOR_TYPE_CHATTERBOX_SPKENC) {
+ if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_CHATTERBOX_SPKENC) {
return cbx_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
}
out_embd.resize((size_t)chunk->tokens_audio->n_tokens * ctx->n_embd_out());
@@ -2367,9 +2377,9 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
info.type = MTMD_GEN_AUDIO_TYPE_NONE;
return info;
}
- info.model_variant = clip_get_hparams(ctx->ctx_gen_a)->gen_model_variant.c_str();
- info.n_voices = (int32_t) clip_gen_get_voices(ctx->ctx_gen_a).size();
- switch (clip_get_projector_type(ctx->ctx_gen_a)) {
+ info.model_variant = ctx->ctx_gen_a->model.hparams.gen_model_variant.c_str();
+ info.n_voices = (int32_t) ctx->ctx_gen_a->model.gen_voices.size();
+ switch (ctx->ctx_gen_a->model.proj_type) {
case PROJECTOR_TYPE_QWEN3TTS_GEN:
info.type = MTMD_GEN_AUDIO_TYPE_QWEN3TTS;
info.sample_rate = 24000;
@@ -2405,19 +2415,19 @@ size_t mtmd_gen_audio_read_tensor(mtmd_context * ctx, const char * name, float *
if (!ctx->ctx_gen_a) {
return 0;
}
- return clip_cbx_read_tensor(ctx->ctx_gen_a, name, out, n_max);
+ return ctx->ctx_gen_a->model.cbx_read_tensor(name, out, n_max);
}
// raw 192-dim CAMPPlus x-vector of the fbank entry
-static bool cbx_ref_xvec(mtmd_context * ctx, const clip_image_f32 & fbank, std::vector<float> & xvec) {
- clip_image_f32 mel_img;
+static bool cbx_ref_xvec(mtmd_context * ctx, const mtmd_image_f32 & fbank, std::vector<float> & xvec) {
+ mtmd_image_f32 mel_img;
mel_img.set_size(fbank.get_size(), false, true);
mel_img.cpy_buf(fbank.get_ro_buf());
- clip_image_f32_batch batch;
+ mtmd_image_f32_batch batch;
batch.is_audio = true;
batch.entries.push_back(std::move(mel_img));
xvec.resize(192);
- if (!clip_image_batch_encode(ctx->ctx_a, ctx->n_threads, &batch, xvec)) {
+ if (!ctx->ctx_a->encode_batch(ctx->n_threads, &batch, xvec)) {
LOG_ERR("%s: speaker encoder failed\n", __func__);
return false;
}
@@ -2425,28 +2435,28 @@ static bool cbx_ref_xvec(mtmd_context * ctx, const clip_image_f32 & fbank, std::
}
// s3 speech tokens of a log-mel entry, rows optionally receives their speech embedding rows
-static int32_t cbx_ref_tokenize(mtmd_context * ctx, const clip_image_f32 & mel,
+static int32_t cbx_ref_tokenize(mtmd_context * ctx, const mtmd_image_f32 & mel,
std::vector<int32_t> & codes, std::vector<float> * rows) {
- clip_image_f32 mel_img;
+ mtmd_image_f32 mel_img;
mel_img.set_size(mel.get_size(), false, true);
mel_img.cpy_buf(mel.get_ro_buf());
- clip_image_f32_batch batch;
+ mtmd_image_f32_batch batch;
batch.is_audio = true;
batch.entries.push_back(std::move(mel_img));
std::vector<int32_t> out_codes;
std::vector<float> out_embd;
- clip_encode_params params;
+ mtmd_subctx_encode_params params;
params.imgs = &batch;
params.n_threads = ctx->n_threads;
- params.gen_process = CLIP_GEN_PROCESS_TOKENIZE;
+ params.gen_process = MTMD_SUBCTX_GEN_PROCESS_TOKENIZE;
params.out_codes = &out_codes;
params.out_code_embd = rows ? &out_embd : nullptr;
- if (!clip_encode(ctx->ctx_gen_a, ¶ms)) {
- LOG_ERR("%s: clip_encode failed (tokenize)\n", __func__);
+ if (!ctx->ctx_gen_a->encode(¶ms)) {
+ LOG_ERR("%s: encode failed (tokenize)\n", __func__);
return 1;
}
@@ -2458,17 +2468,17 @@ static int32_t cbx_ref_tokenize(mtmd_context * ctx, const clip_image_f32 & mel,
}
// talker conditioning rows: the speaker row, then on mtl the perceiver block and the emotion row
-static int32_t cbx_ref_cond(mtmd_context * ctx, const clip_image_f32 & ve_mel,
+static int32_t cbx_ref_cond(mtmd_context * ctx, const mtmd_image_f32 & ve_mel,
const std::vector<float> & pse, std::vector<float> & out_rows) {
- clip_ctx * ctx_clip = ctx->ctx_a;
+ mtmd_subcontext * sctx = ctx->ctx_a.get();
// n_expect 0 accepts any non-empty size
auto read_t = [&](const char * name, std::vector<float> & v, size_t n_expect) -> bool {
- const size_t n = clip_cbx_read_tensor(ctx_clip, name, nullptr, 0);
+ const size_t n = sctx->model.cbx_read_tensor(name, nullptr, 0);
if (n == 0 || (n_expect > 0 && n != n_expect)) {
return false;
}
v.resize(n);
- return clip_cbx_read_tensor(ctx_clip, name, v.data(), n) == n;
+ return sctx->model.cbx_read_tensor(name, v.data(), n) == n;
};
// voice encoder (embeds_from_wavs): 3-layer lstm per 160-frame partial, the normalized partial embeddings are averaged
@@ -2725,7 +2735,7 @@ static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audi
}
// [spkr, perceiver block] on mtl, [spkr, reference speech rows] on turbo
- const bool mtl = cbx_is_mtl(ctx->ctx_a);
+ const bool mtl = cbx_is_mtl(ctx->ctx_a->model);
std::vector<float> pse;
if (mtl) {
pse = std::move(t3_rows);
@@ -2754,7 +2764,7 @@ const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx) {
if (!ctx->ctx_gen_a || idx < 0) {
return nullptr;
}
- const auto & voices = clip_gen_get_voices(ctx->ctx_gen_a);
+ const auto & voices = ctx->ctx_gen_a->model.gen_voices;
return idx < (int32_t) voices.size() ? std::next(voices.begin(), idx)->first.c_str() : nullptr;
}
@@ -2763,7 +2773,7 @@ mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * na
return nullptr;
}
// the name is matched case-insensitively, as the lfm3a "tts" chat template lowers it
- const auto & voices = clip_gen_get_voices(ctx->ctx_gen_a);
+ const auto & voices = ctx->ctx_gen_a->model.gen_voices;
const auto it = std::find_if(voices.begin(), voices.end(), [name](const auto & voice) {
return std::equal(voice.first.begin(), voice.first.end(), name, name + strlen(name),
[](unsigned char a, unsigned char b) { return std::tolower(a) == std::tolower(b); });
@@ -2771,7 +2781,7 @@ mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * na
if (it == voices.end()) {
return nullptr;
}
- return mtmd_bitmap::init_voice(it->first, it->second, clip_n_mmproj_embd(ctx->ctx_gen_a));
+ return mtmd_bitmap::init_voice(it->first, it->second, ctx->ctx_gen_a->model.n_mmproj_embd());
}
mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
@@ -2781,7 +2791,7 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
return inp;
}
- switch (clip_get_projector_type(ctx->ctx_gen_a)) {
+ switch (ctx->ctx_gen_a->model.proj_type) {
case PROJECTOR_TYPE_QWEN3TTS_GEN:
// https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base/blob/main/generation_config.json
inp.top_k = 50;
@@ -2806,7 +2816,7 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
case PROJECTOR_TYPE_LFM3A_GEN:
{
// MaskGIT sampling runs in the helper, with the settings of the checkpoint
- const auto * hparams = clip_get_hparams(ctx->ctx_gen_a);
+ const auto * hparams = &ctx->ctx_gen_a->model.hparams;
inp.top_k = hparams->gen_top_k;
inp.top_p = 1.0f;
inp.temp = hparams->gen_temp;
@@ -2820,8 +2830,8 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
}
struct mtmd_gen_state {
- clip_ctx * ctx_clip = nullptr;
- clip_gen_state * clip_state = nullptr;
+ mtmd_subcontext * sctx = nullptr;
+ mtmd_subctx_gen_state * gen_state = nullptr;
std::vector<int32_t> out_codes; // this frame's 16 sampled codes (GEN_CODE)
std::vector<float> out_feats; // this frame's continuous features, if any (GEN_CODE)
@@ -2829,13 +2839,13 @@ struct mtmd_gen_state {
std::vector<float> out_audio; // decoded PCM samples for the current frame (GEN_WAV)
// voice reference for GEN_WAV, from a speaker clip or a predefined voice (chatterbox)
- clip_gen_ref ref;
+ mtmd_gen_ref ref;
// lfm3a: the codec graph yields STFT frames, the ISTFT runs on the host
std::unique_ptr<mtmd_audio_streaming_istft> istft;
~mtmd_gen_state() {
- clip_gen_state_free(clip_state);
+ delete gen_state;
}
};
@@ -2851,13 +2861,13 @@ mtmd_gen_state * mtmd_gen_state_init(mtmd_context * ctx, mtmd_gen_state_params p
return nullptr;
}
auto st = std::make_unique<mtmd_gen_state>();
- st->ctx_clip = ctx->ctx_gen_a;
- st->clip_state = clip_gen_state_init(ctx->ctx_gen_a, params.seed);
- if (!st->clip_state) {
+ st->sctx = ctx->ctx_gen_a.get();
+ st->gen_state = ctx->ctx_gen_a->gen_state_init(params.seed);
+ if (!st->gen_state) {
return nullptr;
}
- if (clip_get_projector_type(ctx->ctx_gen_a) == PROJECTOR_TYPE_LFM3A_GEN) {
- const auto * hparams = clip_get_hparams(ctx->ctx_gen_a);
+ if (ctx->ctx_gen_a->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN) {
+ const auto * hparams = &ctx->ctx_gen_a->model.hparams;
st->istft = std::make_unique<mtmd_audio_streaming_istft>(hparams->audio_n_fft, hparams->audio_hop_len);
}
return st.release();
@@ -2868,17 +2878,17 @@ void mtmd_gen_state_free(mtmd_gen_state * state) {
}
void mtmd_gen_state_reset(mtmd_gen_state * state) {
- clip_gen_state_reset(state->clip_state);
+ state->gen_state->reset();
if (state->istft) {
- const auto * hparams = clip_get_hparams(state->ctx_clip);
+ const auto * hparams = &state->sctx->model.hparams;
state->istft = std::make_unique<mtmd_audio_streaming_istft>(hparams->audio_n_fft, hparams->audio_hop_len);
}
}
// lfm3a GEN_WAV: ISTFT of the codec output, n_codes == 0 flushes and resets the state
static int32_t mtmd_gen_wav_lfm3a(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
- clip_ctx * ctx_clip = ctx->ctx_gen_a;
- const auto & hparams = *clip_get_hparams(ctx_clip);
+ mtmd_subcontext * sctx = ctx->ctx_gen_a.get();
+ const auto & hparams = sctx->model.hparams;
const int n_bins = hparams.audio_n_fft / 2 + 1;
const int n_trim = (hparams.audio_n_fft - hparams.audio_hop_len) / 2;
@@ -2896,23 +2906,23 @@ static int32_t mtmd_gen_wav_lfm3a(mtmd_context * ctx, mtmd_gen_state * state, co
std::vector<float> feats;
// no hidden-state input, the batch entry is an unused placeholder
- clip_image_f32 dummy;
+ mtmd_image_f32 dummy;
dummy.set_size({1, 1}, false, true);
dummy.cpy_buf(std::vector<float>(1, 0.0f));
- clip_image_f32_batch batch;
+ mtmd_image_f32_batch batch;
batch.is_audio = true;
batch.entries.push_back(std::move(dummy));
- clip_encode_params params;
+ mtmd_subctx_encode_params params;
params.imgs = &batch;
params.n_threads = ctx->n_threads;
- params.gen_process = CLIP_GEN_PROCESS_GEN_WAV;
+ params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV;
params.codes = &in_codes;
params.out_feats = &feats;
- params.gen_state = state->clip_state;
- if (!clip_encode(ctx_clip, ¶ms)) {
- LOG_ERR("%s: clip_encode failed (codec)\n", __func__);
+ params.gen_state = state->gen_state;
+ if (!sctx->encode(¶ms)) {
+ LOG_ERR("%s: encode failed (codec)\n", __func__);
return 1;
}
@@ -2938,7 +2948,7 @@ static int32_t mtmd_gen_wav_lfm3a(mtmd_context * ctx, mtmd_gen_state * state, co
}
int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) {
- if (!state || state->ctx_clip != ctx->ctx_gen_a) {
+ if (!state || state->sctx != ctx->ctx_gen_a.get()) {
LOG_ERR("%s: invalid gen state\n", __func__);
return 1;
}
@@ -2951,35 +2961,35 @@ int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) {
}
static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
- clip_ctx * ctx_clip = ctx->ctx_gen_a;
- if (!ctx_clip) {
+ mtmd_subcontext * sctx = ctx->ctx_gen_a.get();
+ if (!sctx) {
LOG_ERR("%s: model does not support audio generation\n", __func__);
return 1;
}
- if (!state || state->ctx_clip != ctx_clip) {
+ if (!state || state->sctx != sctx) {
LOG_ERR("%s: invalid gen state\n", __func__);
return 1;
}
*out = {};
- if (clip_get_projector_type(ctx_clip) == PROJECTOR_TYPE_SOPRANO) {
- const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip);
+ if (sctx->model.proj_type == PROJECTOR_TYPE_SOPRANO) {
+ const size_t n_embd = (size_t) sctx->model.n_mmproj_embd();
if (inp->type != MTMD_GEN_PROCESS_TYPE_GEN_WAV || !inp->feats || inp->n_feats < 2 * n_embd || inp->n_feats % n_embd != 0 || inp->codes) {
LOG_ERR("%s: soprano requires at least 2 frames of features, without codes\n", __func__);
return 1;
}
}
- if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_WAV && clip_get_projector_type(ctx_clip) == PROJECTOR_TYPE_LFM3A_GEN) {
+ if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_WAV && sctx->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN) {
return mtmd_gen_wav_lfm3a(ctx, state, inp, out);
}
if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE) {
- const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip);
+ const size_t n_embd = (size_t) sctx->model.n_mmproj_embd();
// the hidden state may be omitted when the graph does not use it
- clip_image_f32 hidden_state;
+ mtmd_image_f32 hidden_state;
hidden_state.set_size({(int) n_embd, 1}, false, true);
hidden_state.cpy_buf(inp->embd ? std::vector<float>(inp->embd, inp->embd + n_embd) : std::vector<float>(n_embd, 0.0f));
@@ -2988,7 +2998,7 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
in_codes.assign(inp->codes, inp->codes + inp->n_codes);
}
- clip_image_f32_batch batch;
+ mtmd_image_f32_batch batch;
batch.is_audio = true;
batch.entries.push_back(std::move(hidden_state));
@@ -2997,10 +3007,10 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
std::vector<float> out_feats;
bool is_eos = false;
- clip_encode_params params;
+ mtmd_subctx_encode_params params;
params.imgs = &batch;
params.n_threads = ctx->n_threads;
- params.gen_process = CLIP_GEN_PROCESS_GEN_CODE;
+ params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
params.out_embd = &out_embd;
params.out_codes = &out_codes;
params.out_feats = &out_feats;
@@ -3010,10 +3020,10 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
params.temp = inp->temp;
params.codes = inp->codes ? &in_codes : nullptr;
params.out_is_eos = &is_eos;
- params.gen_state = state->clip_state;
+ params.gen_state = state->gen_state;
- if (!clip_encode(ctx_clip, ¶ms)) {
- LOG_ERR("%s: clip_encode failed (gen_code)\n", __func__);
+ if (!sctx->encode(¶ms)) {
+ LOG_ERR("%s: encode failed (gen_code)\n", __func__);
return 1;
}
@@ -3031,7 +3041,7 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
}
// MTMD_GEN_PROCESS_TYPE_GEN_WAV
- if (clip_get_projector_type(ctx_clip) == PROJECTOR_TYPE_CHATTERBOX) {
+ if (sctx->model.proj_type == PROJECTOR_TYPE_CHATTERBOX) {
if (!inp->codes || inp->n_codes == 0) {
LOG_ERR("%s: codes required for gen_wav\n", __func__);
return 1;
@@ -3043,24 +3053,24 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
std::vector<int32_t> in_codes(inp->codes, inp->codes + inp->n_codes);
// the batch entry is unused, present to satisfy the encode interface
- clip_image_f32 dummy;
+ mtmd_image_f32 dummy;
dummy.set_size({1, 1}, false, true);
dummy.cpy_buf(std::vector<float>(1, 0.0f));
- clip_image_f32_batch batch;
+ mtmd_image_f32_batch batch;
batch.is_audio = true;
batch.entries.push_back(std::move(dummy));
- clip_encode_params params;
+ mtmd_subctx_encode_params params;
params.imgs = &batch;
params.n_threads = ctx->n_threads;
- params.gen_process = CLIP_GEN_PROCESS_TTS;
+ params.gen_process = MTMD_SUBCTX_GEN_PROCESS_TTS;
params.codes = &in_codes;
params.out_audio = &state->out_audio;
params.ref = &state->ref;
- params.gen_state = state->clip_state;
+ params.gen_state = state->gen_state;
- if (!clip_encode(ctx_clip, ¶ms)) {
- LOG_ERR("%s: clip_encode failed (gen_wav)\n", __func__);
+ if (!sctx->encode(¶ms)) {
+ LOG_ERR("%s: encode failed (gen_wav)\n", __func__);
return 1;
}
@@ -3085,25 +3095,25 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
// gen_wav has no hidden-state input, the batch entry is an unused placeholder
// TODO @ngxson : some models in the future may require hidden-state input, need to update this code later
- clip_image_f32 dummy;
+ mtmd_image_f32 dummy;
dummy.set_size({1, 1}, false, true);
dummy.cpy_buf(std::vector<float>(1, 0.0f));
- clip_image_f32_batch batch;
+ mtmd_image_f32_batch batch;
batch.is_audio = true;
batch.entries.push_back(std::move(dummy));
- clip_encode_params params;
+ mtmd_subctx_encode_params params;
params.imgs = &batch;
params.n_threads = ctx->n_threads;
- params.gen_process = CLIP_GEN_PROCESS_GEN_WAV;
+ params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV;
params.codes = has_codes ? &in_codes : nullptr;
params.feats = has_feats ? &in_feats : nullptr;
params.out_audio = &state->out_audio;
- params.gen_state = state->clip_state;
+ params.gen_state = state->gen_state;
- if (!clip_encode(ctx_clip, ¶ms)) {
- LOG_ERR("%s: clip_encode failed (code2wav)\n", __func__);
+ if (!sctx->encode(¶ms)) {
+ LOG_ERR("%s: encode failed (code2wav)\n", __func__);
return 1;
}
@@ -3138,7 +3148,7 @@ int32_t mtmd_batch_add_chunk(mtmd_batch * batch, const mtmd_input_chunk * chunk)
return 1;
}
- auto * ctx = batch->ctx->get_clip_ctx(chunk);
+ auto * ctx = batch->ctx->get_subctx(chunk);
if (!ctx) {
LOG_ERR("%s: model does not support input chunk type %d\n", __func__, (int)chunk->type);
return 1;
@@ -3150,7 +3160,7 @@ int32_t mtmd_batch_add_chunk(mtmd_batch * batch, const mtmd_input_chunk * chunk)
return 0;
}
- if (!clip_support_batch(ctx)) {
+ if (!ctx->support_batch) {
// if no batching support, batch can only have one single chunk
return 2; // "batch too large" error code
}
@@ -3260,7 +3270,7 @@ bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk
{
// E2B (n_embd = 1536) and E4B (n_embd = 2560) always use causal
// note: use mmproj n_embd, because text model may not be provided (e.g. mtmd_get_memory_usage)
- const int n_embd = clip_n_mmproj_embd(ctx->ctx_v);
+ const int n_embd = ctx->ctx_v->model.n_mmproj_embd();
return n_embd != 1536 && n_embd != 2560;
}
case PROJECTOR_TYPE_GEMMA4UV:
@@ -3289,7 +3299,7 @@ int mtmd_get_audio_sample_rate(const mtmd_context * ctx) {
if (!ctx->ctx_a) {
return -1;
}
- return clip_get_hparams(ctx->ctx_a)->audio_sample_rate;
+ return ctx->ctx_a->model.hparams.audio_sample_rate;
}
mtmd_video_config mtmd_video_config_init(mtmd_video_template tmpl, const mtmd_image_preprocessor * preproc,
@@ -3312,7 +3322,7 @@ mtmd_video_config mtmd_video_config_init(mtmd_video_template tmpl, const mtmd_im
n = std::min(n, src_n_frames);
cfg.n_frames = n;
- clip_image_size size;
+ mtmd_image_size size;
if (preproc && preproc->get_video_frame_size(n, src_width, src_height, size)) {
cfg.frame_width = size.width;
cfg.frame_height = size.height;
@@ -3324,7 +3334,7 @@ mtmd_video_config mtmd_get_video_config(const mtmd_context * ctx,
int32_t src_width, int32_t src_height,
int32_t src_n_frames, float src_fps) {
mtmd_video_template tmpl = MTMD_VIDEO_TEMPLATE_DEFAULT;
- switch (ctx->ctx_v ? clip_get_projector_type(ctx->ctx_v) : PROJECTOR_TYPE_UNKNOWN) {
+ switch (ctx->ctx_v ? ctx->ctx_v->model.proj_type : PROJECTOR_TYPE_UNKNOWN) {
case PROJECTOR_TYPE_LFM3VL:
tmpl = ctx->video_has_soundtrack() ? MTMD_VIDEO_TEMPLATE_LFM3_OMNI : MTMD_VIDEO_TEMPLATE_LFM3;
break;
@@ -3650,7 +3660,7 @@ static mtmd_decoder_pos mtmd_canvas_decoder_pos(const mtmd_image_tokens * image_
}
i -= gw * gh;
- const size_t n_sep = clip_suffix_rows(entries[k].suffix_type).size();
+ const size_t n_sep = mtmd_suffix_rows(entries[k].suffix_type).size();
if (i < n_sep) {
const uint32_t p = t + 1 + i;
pos.t = p;
@@ -3752,7 +3762,7 @@ llama_pos mtmd_image_tokens_get_n_pos(const mtmd_image_tokens * image_tokens) {
// one position per tile, plus one per separator
llama_pos n_pos = 0;
for (const auto & entry : image_tokens->batch_f32.entries) {
- n_pos += 1 + (llama_pos) clip_suffix_rows(entry.suffix_type).size();
+ n_pos += 1 + (llama_pos) mtmd_suffix_rows(entry.suffix_type).size();
}
return n_pos;
}
@@ -3796,14 +3806,16 @@ mtmd_input_chunks * mtmd_test_create_input_chunks() {
return chunks;
}
+struct mtmd_logger_state g_logger_state = {mtmd_log_callback_default, NULL};
+
void mtmd_log_set(ggml_log_callback log_callback, void * user_data) {
- g_logger_state.log_callback = log_callback ? log_callback : clip_log_callback_default;
+ g_logger_state.log_callback = log_callback ? log_callback : mtmd_log_callback_default;
g_logger_state.log_callback_user_data = user_data;
}
struct mtmd_caps mtmd_get_cap_from_file(const char * fname) {
try {
- auto tmp = clip_get_cap(fname);
+ auto tmp = mtmd_model_get_caps(fname);
mtmd_caps cap;
cap.inp_audio = tmp.has_audio;
cap.inp_vision = tmp.has_vision;
@@ -3819,14 +3831,12 @@ struct mtmd_caps mtmd_get_cap_from_file(const char * fname) {
// Debugging API (NOT intended for public use)
//
-static void mtmd_debug_encode_impl(mtmd_context * ctx, clip_ctx * ctx_clip, clip_image_f32 & image) {
- clip_set_debug_output_embeddings(ctx_clip, true);
- int n_mmproj_embd = clip_n_mmproj_embd(ctx_clip);
- int n_tokens = clip_n_output_tokens(ctx_clip, &image);
+static void mtmd_debug_encode_impl(mtmd_context * ctx, mtmd_subcontext * sctx, mtmd_image_f32 & image) {
+ sctx->debug_output_embeddings = true;
+ int n_mmproj_embd = sctx->model.n_mmproj_embd();
+ int n_tokens = sctx->model.n_output_tokens(&image);
std::vector<float> embd_output(n_tokens * n_mmproj_embd, 0.0f);
- bool ok = clip_image_encode(
- ctx_clip,
- ctx->n_threads,
+ bool ok = sctx->encode_image(ctx->n_threads,
&image,
embd_output);
if (!ok) {
@@ -3845,11 +3855,11 @@ void mtmd_debug_encode_image(mtmd_context * ctx, const std::vector<std::vector<f
for (const auto & row : image) {
img_buf.insert(img_buf.end(), row.begin(), row.end());
}
- clip_image_f32 inp_image;
+ mtmd_image_f32 inp_image;
inp_image.set_size({img_sz, img_sz}, false, false);
inp_image.cpy_buf(img_buf);
LOG_INF("%s: created input image with nx=%d, ny=%d\n", __func__, img_sz, img_sz);
- mtmd_debug_encode_impl(ctx, ctx->ctx_v, inp_image);
+ mtmd_debug_encode_impl(ctx, ctx->ctx_v.get(), inp_image);
}
void mtmd_debug_encode_audio(mtmd_context * ctx, const std::vector<float> & input) {
@@ -3857,7 +3867,7 @@ void mtmd_debug_encode_audio(mtmd_context * ctx, const std::vector<float> & inpu
LOG_ERR("%s: model does not support audio input\n", __func__);
return;
}
- int n_mel = clip_get_hparams(ctx->ctx_a)->n_mel_bins;
+ int n_mel = ctx->ctx_a->model.hparams.n_mel_bins;
const int audio_nx = (int)input.size();
std::vector<float> audio_buf(audio_nx * n_mel);
for (int i = 0; i < audio_nx; i++) {
@@ -3865,11 +3875,11 @@ void mtmd_debug_encode_audio(mtmd_context * ctx, const std::vector<float> & inpu
audio_buf[j * audio_nx + i] = input[i];
}
}
- clip_image_f32 inp_audio;
+ mtmd_image_f32 inp_audio;
inp_audio.set_size({audio_nx, n_mel}, false, true);
inp_audio.cpy_buf(audio_buf);
LOG_INF("%s: created input audio with nx=%d, ny=%d\n", __func__, audio_nx, n_mel);
- mtmd_debug_encode_impl(ctx, ctx->ctx_a, inp_audio);
+ mtmd_debug_encode_impl(ctx, ctx->ctx_a.get(), inp_audio);
}
void mtmd_debug_preprocess_image(mtmd_context * ctx, const std::vector<uint8_t> & rgb_values, int nx, int ny) {
@@ -3877,13 +3887,13 @@ void mtmd_debug_preprocess_image(mtmd_context * ctx, const std::vector<uint8_t>
LOG_ERR("%s: model does not support vision input\n", __func__);
return;
}
- clip_image_u8 img_u8;
+ mtmd_image_u8 img_u8;
img_u8.set_size({nx, ny}, false);
img_u8.cpy_buf(rgb_values);
GGML_ASSERT(ctx->image_preproc != nullptr);
mtmd_image_preproc_out preproc_out = ctx->image_preproc->preprocess(img_u8);
- clip_image_f32_batch batch_f32;
+ mtmd_image_f32_batch batch_f32;
batch_f32.is_audio = false;
for (auto & entry : preproc_out.entries) {
batch_f32.entries.push_back(std::move(entry));
@@ -3938,23 +3948,23 @@ mtmd_memory_usage mtmd_get_memory_usage(const char * mmproj_fname,
ctx.reset(new mtmd_context(mmproj_fname, nullptr, ctx_params, true));
mtmd_log_set(saved_log_callback, saved_log_user_data); // restore log callback
std::map<ggml_backend_dev_t, size_t> total_mem;
- auto merge = [&](const struct clip_ctx * c) {
- for (auto & [dev, size] : clip_get_mem_usage(c)) {
+ auto merge = [&](const struct mtmd_subcontext * c) {
+ for (auto & [dev, size] : c->get_mem_usage()) {
total_mem[dev] += size;
}
};
if (ctx->ctx_v) {
- merge(ctx->ctx_v);
+ merge(ctx->ctx_v.get());
}
if (ctx->ctx_a) {
- merge(ctx->ctx_a);
+ merge(ctx->ctx_a.get());
}
if (ctx->ctx_gen_a) {
- merge(ctx->ctx_gen_a);
+ merge(ctx->ctx_gen_a.get());
}
mtmd_memory_usage res;
res.backend_mem_usage = std::move(total_mem);
- res.image_max_tokens = ctx->ctx_v ? clip_get_image_max_tokens(ctx->ctx_v) : -1;
+ res.image_max_tokens = ctx->ctx_v ? ctx->ctx_v->model.image_max_tokens() : -1;
res.use_non_causal = ctx->ctx_v ? mtmd_decode_use_non_causal(ctx.get(), nullptr) : false;
return res;
} catch (const std::exception & e) {
@@ -3971,7 +3981,7 @@ mtmd_memory_usage mtmd_get_memory_usage(const char * mmproj_fname,
// the model-independent part of a stream, the model-specific part is in the subclasses below
struct mtmd_audio_stream {
mtmd_context * ctx;
- clip_ctx * ctx_clip;
+ mtmd_subcontext * sctx;
// model constants
int n_mel;
@@ -4015,10 +4025,10 @@ struct mtmd_audio_stream {
// Voxtral Realtime: a row is an audio embedding plus the embedding of the previous token
struct mtmd_audio_stream_voxtral_rt : mtmd_audio_stream {
- clip_enc_state * enc_state = nullptr;
+ mtmd_subctx_enc_state * enc_state = nullptr;
int n_fft;
- int n_frames_max; // encoder frames per clip_encode() call
+ int n_frames_max; // encoder frames per encode() call
int n_right_pad; // silence tokens added by flush()
float log_mel_max; // global_log_mel_max in params.json
@@ -4030,7 +4040,7 @@ struct mtmd_audio_stream_voxtral_rt : mtmd_audio_stream {
std::vector<float> tok_embd;
~mtmd_audio_stream_voxtral_rt() override {
- clip_enc_state_free(enc_state);
+ delete enc_state;
}
bool init(const mtmd_audio_stream_params & params) override;
@@ -4043,7 +4053,7 @@ struct mtmd_audio_stream_voxtral_rt : mtmd_audio_stream {
// training samples are max 30 s: restart the sequence at a silent chunk after 15 s, always after 30 s; the encoder keeps its state
struct mtmd_audio_stream_lfm3a : mtmd_audio_stream {
int win; // samples per mel frame
- clip_enc_state * enc_state = nullptr;
+ mtmd_subctx_enc_state * enc_state = nullptr;
// live ASR
static constexpr int64_t chunk_rows = 12;
@@ -4073,7 +4083,7 @@ struct mtmd_audio_stream_lfm3a : mtmd_audio_stream {
bool end_done = false;
~mtmd_audio_stream_lfm3a() override {
- clip_enc_state_free(enc_state);
+ delete enc_state;
}
bool init(const mtmd_audio_stream_params & params) override;
@@ -4088,7 +4098,7 @@ bool mtmd_support_audio_stream(const mtmd_context * ctx) {
if (!ctx->ctx_a || !ctx->text_model) {
return false;
}
- const projector_type proj = clip_get_projector_type(ctx->ctx_a);
+ const projector_type proj = ctx->ctx_a->model.proj_type;
return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A;
}
@@ -4105,19 +4115,19 @@ mtmd_audio_stream * mtmd_audio_stream_init(mtmd_context * ctx, mtmd_audio_stream
LOG_ERR("%s: the model does not support streaming audio input\n", __func__);
return nullptr;
}
- const clip_hparams * hparams = clip_get_hparams(ctx->ctx_a);
+ const mtmd_hparams * hparams = &ctx->ctx_a->model.hparams;
std::unique_ptr<mtmd_audio_stream> s;
- switch (clip_get_projector_type(ctx->ctx_a)) {
+ switch (ctx->ctx_a->model.proj_type) {
case PROJECTOR_TYPE_VOXTRAL_RT: s = std::make_unique<mtmd_audio_stream_voxtral_rt>(); break;
case PROJECTOR_TYPE_LFM3A: s = std::make_unique<mtmd_audio_stream_lfm3a>(); break;
default: GGML_ABORT("unsupported projector");
}
s->ctx = ctx;
- s->ctx_clip = ctx->ctx_a;
+ s->sctx = ctx->ctx_a.get();
s->n_mel = hparams->n_mel_bins;
s->hop = hparams->audio_hop_len;
- s->n_embd = clip_n_mmproj_embd(ctx->ctx_a);
+ s->n_embd = ctx->ctx_a->model.n_mmproj_embd();
if (s->n_embd != ctx->n_embd_text) {
LOG_ERR("%s: mmproj embedding size %d does not match the text model (%d)\n", __func__, s->n_embd, ctx->n_embd_text);
return nullptr;
@@ -4166,7 +4176,7 @@ bool mtmd_audio_stream_is_seq_start(const mtmd_audio_stream * s) {
//
bool mtmd_audio_stream_voxtral_rt::init(const mtmd_audio_stream_params & params) {
- const clip_hparams * hparams = clip_get_hparams(ctx_clip);
+ const mtmd_hparams * hparams = &sctx->model.hparams;
const llama_vocab * vocab = ctx->vocab;
n_fft = hparams->audio_n_fft;
@@ -4180,7 +4190,7 @@ bool mtmd_audio_stream_voxtral_rt::init(const mtmd_audio_stream_params & params)
// the encoder works in whole decoder rows
int n_step = (int) ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 / hop);
n_step = std::max(n_step, frames_per_tok);
- n_step = CLIP_ALIGN(n_step, frames_per_tok);
+ n_step = MTMD_ALIGN(n_step, frames_per_tok);
n_step_mel = std::min(n_step, 2 * n_frames_max);
if (params.n_prompt > 0) {
@@ -4209,7 +4219,7 @@ bool mtmd_audio_stream_voxtral_rt::init(const mtmd_audio_stream_params & params)
cache.fill_hann_window(hparams->audio_window_len, true);
cache.fill_mel_filterbank_matrix(n_mel, n_fft, hparams->audio_sample_rate);
- enc_state = clip_enc_state_init(ctx_clip, n_frames_max);
+ enc_state = sctx->enc_state_init(n_frames_max);
if (!enc_state) {
return false;
}
@@ -4293,20 +4303,20 @@ static bool mtmd_audio_stream_encode(mtmd_audio_stream_voxtral_rt * s, bool forc
}
}
- clip_image_f32 img;
+ mtmd_image_f32 img;
img.set_size({n_in, s->n_mel}, false, /* is_audio */ true);
img.cpy_buf(data);
- clip_image_f32_batch batch;
+ mtmd_image_f32_batch batch;
batch.is_audio = true;
batch.entries.push_back(std::move(img));
std::vector<float> res((size_t) s->n_embd * (n / s->frames_per_tok));
- clip_encode_params params;
+ mtmd_subctx_encode_params params;
params.n_threads = s->ctx->n_threads;
params.imgs = &batch;
params.out_embd = &res;
params.enc_state = s->enc_state;
- if (!clip_encode(s->ctx_clip, ¶ms)) {
+ if (!s->sctx->encode(¶ms)) {
LOG_ERR("%s: failed to encode audio\n", __func__);
return false;
}
@@ -4385,7 +4395,7 @@ int32_t mtmd_audio_stream_voxtral_rt::step(llama_token last_token) {
//
bool mtmd_audio_stream_lfm3a::init(const mtmd_audio_stream_params & params) {
- const clip_hparams * hparams = clip_get_hparams(ctx_clip);
+ const mtmd_hparams * hparams = &sctx->model.hparams;
// one row per 8 mel frames (three stride-2 convs), mel frames are not centered
win = hparams->audio_window_len;
@@ -4393,7 +4403,7 @@ bool mtmd_audio_stream_lfm3a::init(const mtmd_audio_stream_params & params) {
samples_per_tok = frames_per_tok * hop;
const int n_step = (int) ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 / hop);
- n_step_mel = CLIP_ALIGN(std::max(n_step, frames_per_tok), frames_per_tok);
+ n_step_mel = MTMD_ALIGN(std::max(n_step, frames_per_tok), frames_per_tok);
if (params.n_prompt > 0) {
// live ASR, a chunk per encoder call
@@ -4416,7 +4426,7 @@ bool mtmd_audio_stream_lfm3a::init(const mtmd_audio_stream_params & params) {
max_seq_rows = std::lround(30.0 * hparams->audio_sample_rate / samples_per_tok);
}
- enc_state = clip_enc_state_init(ctx_clip, n_step_mel);
+ enc_state = sctx->enc_state_init(n_step_mel);
if (!enc_state) {
return false;
}
@@ -4508,22 +4518,22 @@ static bool mtmd_audio_stream_encode(mtmd_audio_stream_lfm3a * s, bool flush) {
data[(size_t) m * n + t] = s->mel[(size_t) t * s->n_mel + m];
}
}
- clip_image_f32 img;
+ mtmd_image_f32 img;
img.set_size({n, s->n_mel}, false, /* is_audio */ true);
img.cpy_buf(data);
- const int n_rows = clip_n_output_tokens(s->ctx_clip, &img);
+ const int n_rows = s->sctx->model.n_output_tokens(&img);
- clip_image_f32_batch batch;
+ mtmd_image_f32_batch batch;
batch.is_audio = true;
batch.entries.push_back(std::move(img));
std::vector<float> res((size_t) s->n_embd * n_rows);
- clip_encode_params params;
+ mtmd_subctx_encode_params params;
params.n_threads = s->ctx->n_threads;
params.imgs = &batch;
params.out_embd = &res;
params.enc_state = s->enc_state;
- if (!clip_encode(s->ctx_clip, ¶ms)) {
+ if (!s->sctx->encode(¶ms)) {
LOG_ERR("%s: failed to encode audio\n", __func__);
return false;
}