Commit 9581154a2 for llama.cpp
commit 9581154a2250cc9e6948e4c2a2fc4b4e19d1a57e
Author: Sidhartha Reddy Potu <73529763+sid-rp@users.noreply.github.com>
Date: Mon Nov 16 15:29:02 2026 -0800
mtmd : fix Muse Glimmer vision embedding order (#32433)
Assisted-by: Claude
diff --git a/tools/mtmd/models/muse-glimmer.cpp b/tools/mtmd/models/muse-glimmer.cpp
index 869954f99..baff2c823 100644
--- a/tools/mtmd/models/muse-glimmer.cpp
+++ b/tools/mtmd/models/muse-glimmer.cpp
@@ -34,8 +34,16 @@ ggml_cgraph * mtmd_graph_muse_glimmer::build() {
ggml_tensor * sp_mask = build_inp_attn_mask(n_tok, n_tok);
// patchify via build_inp (conv2d over raw pixels) + bilinear-resized learned pos-emb
+ // pos-emb grid is resized with height and width swapped, to match the trainer reference
+ const int n_per_side = (int) std::sqrt(model.position_embeddings->ne[1]);
+ ggml_tensor * pos_embd = ggml_reshape_3d(ctx0, model.position_embeddings, n_embd, n_per_side, n_per_side);
+ pos_embd = ggml_permute(ctx0, pos_embd, 2, 0, 1, 3);
+ pos_embd = ggml_interpolate(ctx0, pos_embd, n_patches_y, n_patches_x, n_embd, 1, GGML_SCALE_MODE_BILINEAR);
+ pos_embd = ggml_permute(ctx0, pos_embd, 1, 2, 0, 3);
+ pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, n_tok);
+
ggml_tensor * x = build_inp(); // [n_embd, n_tok, 1]
- x = ggml_add(ctx0, x, resize_position_embeddings(GGML_SCALE_MODE_BILINEAR));
+ x = ggml_add(ctx0, x, pos_embd);
cb(x, "after_posemb", -1);
// group patches into pgrid x pgrid windows (sparse attention order)