Commit 7894e8b92 for llama.cpp
commit 7894e8b92481a8bb8ffe94f73b656ffe0dd01e10
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Sat Oct 24 03:38:25 2026 +0200
mtmd: move build_inp_temporal_merge to base class (#31065)
diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/clip-graph.h
index 33e546b7d..a0669d1a0 100644
--- a/tools/mtmd/clip-graph.h
+++ b/tools/mtmd/clip-graph.h
@@ -101,6 +101,10 @@ struct clip_graph {
ggml_tensor * build_inp_raw(int channels = 3);
+ // qwen-vl style temporal merge, handles n_batch == 1 (image) or n_batch == 2 (2 video frames)
+ // requires patch_embeddings_0 and patch_embeddings_1; returns tensor with shape [w, h, n_embd, 1]
+ ggml_tensor * build_inp_temporal_merge();
+
// f16 if flash attn is enabled, set it with set_input_attn_mask()
// idx is only needed when the graph has more than one mask
ggml_tensor * build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx = 0);
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 2ab55c52a..d204708ee 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -584,6 +584,36 @@ ggml_tensor * clip_graph::build_inp_raw(int channels) {
return inp_raw;
}
+// qwen-vl style temporal merge: Conv3D(kt=2) split into 2 Conv2D
+ggml_tensor * clip_graph::build_inp_temporal_merge() {
+ ggml_tensor * inp_raw = build_inp_raw();
+
+ GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
+ GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
+
+ const size_t nb1 = ggml_row_size(inp_raw->type, img.nx());
+ const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny());
+
+ if (n_batch == 1) {
+ // still image input
+ return ggml_add(ctx0,
+ ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1),
+ ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
+ } else if (n_batch == 2) {
+ // 2 frames input (video input)
+ ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw,
+ img.nx(), img.ny(), 3, nb1, nb2, 0);
+ ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw,
+ img.nx(), img.ny(), 3, nb1, nb2,
+ nb2 * 3); // move to the second frame
+ return ggml_add(ctx0,
+ ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1),
+ ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1));
+ } else {
+ GGML_ASSERT(false && "n_batch > 2 is not supported");
+ }
+}
+
static std::string get_attn_mask_name(int idx) {
return idx == 0 ? "attn_mask" : "attn_mask_" + std::to_string(idx);
}
diff --git a/tools/mtmd/models/ling3vl.cpp b/tools/mtmd/models/ling3vl.cpp
index ef8eb3a83..fb9eb7c74 100644
--- a/tools/mtmd/models/ling3vl.cpp
+++ b/tools/mtmd/models/ling3vl.cpp
@@ -14,7 +14,7 @@ ggml_cgraph * clip_graph_ling3vl::build() {
// vision M-RoPE, same layout as qwen3vl: [row, col, row, col] quarters
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
- ggml_tensor * inp = build_inp_with_temporal_merge();
+ ggml_tensor * inp = build_inp_temporal_merge();
// spatial merge
{
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 8ae2a5b25..3b34548c4 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -42,7 +42,6 @@ struct clip_graph_deepseek4v : clip_graph {
struct clip_graph_qwen2vl : clip_graph {
clip_graph_qwen2vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
- ggml_tensor * build_inp_with_temporal_merge();
};
struct clip_graph_qwen3vl : clip_graph_qwen2vl {
diff --git a/tools/mtmd/models/qwen2vl.cpp b/tools/mtmd/models/qwen2vl.cpp
index 4998d95bc..7516aa23a 100644
--- a/tools/mtmd/models/qwen2vl.cpp
+++ b/tools/mtmd/models/qwen2vl.cpp
@@ -1,34 +1,5 @@
#include "models.h"
-ggml_tensor * clip_graph_qwen2vl::build_inp_with_temporal_merge() {
- ggml_tensor * inp_raw = build_inp_raw();
-
- GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
- GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
-
- const size_t nb1 = ggml_row_size(inp_raw->type, img.nx());
- const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny());
-
- if (n_batch == 1) {
- // still image input
- return ggml_add(ctx0,
- ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1),
- ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
- } else if (n_batch == 2) {
- // 2 frames input (video input)
- ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw,
- img.nx(), img.ny(), 3, nb1, nb2, 0);
- ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw,
- img.nx(), img.ny(), 3, nb1, nb2,
- nb2 * 3); // move to the second frame
- return ggml_add(ctx0,
- ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1),
- ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1));
- } else {
- GGML_ASSERT(false && "n_batch > 2 is not supported");
- }
-}
-
ggml_cgraph * clip_graph_qwen2vl::build() {
GGML_ASSERT(model.patch_bias == nullptr);
GGML_ASSERT(model.class_embedding == nullptr);
@@ -45,7 +16,7 @@ ggml_cgraph * clip_graph_qwen2vl::build() {
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
- ggml_tensor * inp = build_inp_with_temporal_merge();
+ ggml_tensor * inp = build_inp_temporal_merge();
// second conv dimension
{
diff --git a/tools/mtmd/models/qwen3vl.cpp b/tools/mtmd/models/qwen3vl.cpp
index 48626b221..2e49dcd01 100644
--- a/tools/mtmd/models/qwen3vl.cpp
+++ b/tools/mtmd/models/qwen3vl.cpp
@@ -13,7 +13,7 @@ ggml_cgraph * clip_graph_qwen3vl::build() {
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
- ggml_tensor * inp = build_inp_with_temporal_merge();
+ ggml_tensor * inp = build_inp_temporal_merge();
// spatial merge
{