Commit 8c388364e for llama.cpp
commit 8c388364e1808eba617b3e51b34dc267f7652886
Author: Pascal <admin@serveurperso.com>
Date: Thu Nov 12 12:14:36 2026 +0100
mtmd: require the MLP projector and patch embedding tensors (#32129)
* mtmd: require the MLP projector and patch embedding tensors
The loader marks the patch embedding and the LLaVA projector tensors
as optional, while the graphs and n_mmproj_embd() dereference them
unconditionally. A GGUF without them crashes on a null tensor during
the warmup.
Load the patch embedding as required for the vision encoders that
start with it, which is all of them except GEMMA3NV, DEEPSEEKOCR and
DEEPSEEKOCR2. Load mm.0 and mm.2 as required for MLP, and mm.0, mm.1,
mm.3 and mm.4 for the Yi-type MLP_NORM variant, selected by mm.3.
The graph no longer tests mm.2 and the yasa2 assert on the patch
embedding is covered by the loader.
* mtmd: keep the Yi-type llava TODO
The Yi-type variant is still detected by the presence of mm.3, the
TODO on that detection stays.
diff --git a/tools/mtmd/models/llava.cpp b/tools/mtmd/models/llava.cpp
index f4f31ffdb..8e8c7638b 100644
--- a/tools/mtmd/models/llava.cpp
+++ b/tools/mtmd/models/llava.cpp
@@ -167,10 +167,8 @@ ggml_cgraph * mtmd_graph_llava::build() {
embeddings = ggml_add(ctx0, embeddings, model.mm_0_b);
embeddings = ggml_gelu(ctx0, embeddings);
- if (model.mm_2_w) {
- embeddings = build_mm(model.mm_2_w, embeddings);
- embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);
- }
+ embeddings = build_mm(model.mm_2_w, embeddings);
+ embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);
}
else if (proj_type == PROJECTOR_TYPE_MLP_NORM) {
embeddings = build_mm(model.mm_0_w, embeddings);
diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp
index e00043818..6cd9509f9 100644
--- a/tools/mtmd/mtmd-model.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -1448,8 +1448,15 @@ struct mtmd_model_loader {
model.post_ln_w = get_tensor(mtmd_format(TN_LN_POST, prefix, "weight"), false);
model.post_ln_b = get_tensor(mtmd_format(TN_LN_POST, prefix, "bias"), false);
+ // every vision encoder starts with a patch embedding, except the ones with their own stem
+ const bool has_patch_embd = (
+ model.modality == MTMD_MODALITY_VISION &&
+ model.proj_type != PROJECTOR_TYPE_GEMMA3NV &&
+ model.proj_type != PROJECTOR_TYPE_DEEPSEEKOCR &&
+ model.proj_type != PROJECTOR_TYPE_DEEPSEEKOCR2);
+
model.patch_bias = get_tensor(TN_PATCH_BIAS, false);
- model.patch_embeddings_0 = get_tensor(TN_PATCH_EMBD, false);
+ model.patch_embeddings_0 = get_tensor(TN_PATCH_EMBD, has_patch_embd);
model.patch_embeddings_1 = get_tensor(TN_PATCH_EMBD_1, false);
model.norm_embd_w = get_tensor(mtmd_format(TN_NORM_EMBD, "weight"), false);
@@ -1660,23 +1667,22 @@ struct mtmd_model_loader {
case PROJECTOR_TYPE_MLP:
case PROJECTOR_TYPE_MLP_NORM:
{
- // LLaVA projection
- model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"), false);
- model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"), false);
- // Yi-type llava
- model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"), false);
- model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"), false);
- // missing in Yi-type llava
- model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"), false);
- model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false);
- // Yi-type llava
- model.mm_3_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 3, "weight"), false);
- model.mm_3_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 3, "bias"), false);
- model.mm_4_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 4, "weight"), false);
- model.mm_4_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 4, "bias"), false);
+ // LLaVA projection: mm.0 -> gelu -> mm.2
+ // Yi-type llava (MLP_NORM): mm.0 -> norm mm.1 -> gelu -> mm.3 -> norm mm.4
+ model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight"));
+ model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"));
+ model.mm_3_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 3, "weight"), model.proj_type == PROJECTOR_TYPE_MLP_NORM);
if (model.mm_3_w) {
// TODO: this is a hack to support Yi-type llava
model.proj_type = PROJECTOR_TYPE_MLP_NORM;
+ model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
+ model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"));
+ model.mm_3_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 3, "bias"));
+ model.mm_4_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 4, "weight"));
+ model.mm_4_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 4, "bias"));
+ } else {
+ model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
+ model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"));
}
model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false);
} break;
@@ -1925,7 +1931,6 @@ struct mtmd_model_loader {
{
// reuse tensors already loaded by the common section
// (TN_PATCH_EMBD and TN_PATCH_BIAS have the same tensor names)
- GGML_ASSERT(model.patch_embeddings_0 && "yasa2 requires v.patch_embd.weight");
model.yasa_patch_w = model.patch_embeddings_0;
model.yasa_patch_b = model.patch_bias;
model.yasa_patch_ln_w = get_tensor(TN_YASA_PATCH_LN_W, false);