Commit 0fd868cbc for llama.cpp
commit 0fd868cbcadcbcd80373d1cb542ecbcfa115ca75
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Sat Oct 10 11:22:12 2026 +0200
mtmd: add build_inp_attn_mask (#30259)
diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/clip-graph.h
index bbee35bea..06eee4697 100644
--- a/tools/mtmd/clip-graph.h
+++ b/tools/mtmd/clip-graph.h
@@ -101,6 +101,10 @@ struct clip_graph {
ggml_tensor * build_inp_raw(int channels = 3);
+ // f16 if flash attn is enabled, set it with set_input_attn_mask()
+ // idx is only needed when the graph has more than one mask
+ ggml_tensor * build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx = 0);
+
ggml_tensor * build_norm(
ggml_tensor * cur,
ggml_tensor * mw,
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index ca38d76df..db6abf633 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -588,6 +588,18 @@ ggml_tensor * clip_graph::build_inp_raw(int channels) {
return inp_raw;
}
+static std::string get_attn_mask_name(int idx) {
+ return idx == 0 ? "attn_mask" : "attn_mask_" + std::to_string(idx);
+}
+
+ggml_tensor * clip_graph::build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx) {
+ const ggml_type type = flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED ? GGML_TYPE_F16 : GGML_TYPE_F32;
+ ggml_tensor * mask = ggml_new_tensor_2d(ctx0, type, n_kv, n_q);
+ ggml_set_name(mask, get_attn_mask_name(idx).c_str());
+ ggml_set_input(mask);
+ return mask;
+}
+
ggml_tensor * clip_graph::build_norm(
ggml_tensor * cur,
ggml_tensor * mw,
@@ -777,9 +789,8 @@ ggml_tensor * clip_graph::build_attn(
k = ggml_cast(ctx0, k, GGML_TYPE_F16);
v = ggml_cast(ctx0, v, GGML_TYPE_F16);
- if (kq_mask) {
- kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F16);
- }
+ // mask must be f16 here, use build_inp_attn_mask()
+ GGML_ASSERT(!kq_mask || kq_mask->type == GGML_TYPE_F16);
cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f);
ggml_prec_set_acc(cur, GGML_PREC_F32);
@@ -4591,6 +4602,20 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur));
};
+ // mask from build_inp_attn_mask(), f16 if flash attn is enabled
+ auto set_input_attn_mask = [&get_inp_tensor](const std::vector<float> & values, int idx = 0) {
+ ggml_tensor * cur = get_inp_tensor(get_attn_mask_name(idx).c_str());
+ GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size());
+ if (cur->type == GGML_TYPE_F16) {
+ std::vector<ggml_fp16_t> values_f16(values.size());
+ ggml_fp32_to_fp16_row(values.data(), values_f16.data(), values.size());
+ ggml_backend_tensor_set(cur, values_f16.data(), 0, ggml_nbytes(cur));
+ } else {
+ GGML_ASSERT(cur->type == GGML_TYPE_F32);
+ ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur));
+ }
+ };
+
auto set_input_i32 = [&get_inp_tensor](const char * name, std::vector<int32_t> & values) {
ggml_tensor * cur = get_inp_tensor(name);
GGML_ASSERT(cur->type == GGML_TYPE_I32);
@@ -4639,7 +4664,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
}
}
- set_input_f32("kq_mask", mask);
+ set_input_attn_mask(mask);
};
// set input pixel values
@@ -4753,7 +4778,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
off += s;
}
}
- set_input_f32("muse_glimmer_sp_mask", sp_mask);
+ set_input_attn_mask(sp_mask);
// pixel-shuffle gather (original order): f*f spatial neighbours grouped
std::vector<int32_t> dsp; dsp.reserve(n_tok);
@@ -4876,7 +4901,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
}
}
- set_input_f32("vit_merger_window_mask", window_mask_data);
+ set_input_attn_mask(window_mask_data);
// ViT merger 2x2 downsample indices
auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w);
@@ -5061,7 +5086,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
set_input_i32("window_idx", idx);
set_input_i32("inv_window_idx", inv_idx);
- set_input_f32("window_mask", mask);
+ set_input_attn_mask(mask);
} else {
for (int i = 0; i < ph * pw; i++) {
idx[i] = i;
@@ -5172,7 +5197,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
set_input_i32("mimovl_positions_row", positions_row);
set_input_i32("mimovl_positions_col", positions_col);
set_input_f32("mimovl_idx_col", idx_col);
- set_input_f32("mimovl_window_mask", mask);
+ set_input_attn_mask(mask);
} break;
case PROJECTOR_TYPE_PIXTRAL:
case PROJECTOR_TYPE_KIMIVL:
@@ -5366,7 +5391,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
qwen2_mask[static_cast<size_t>(i) * seq_len + j] = zero ? 0.0f : -1e9f;
}
}
- set_input_f32("qwen2_attn_mask", qwen2_mask);
+ set_input_attn_mask(qwen2_mask);
}
} break;
case PROJECTOR_TYPE_GEMMA3:
@@ -5612,8 +5637,8 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
window_mask[(size_t) q * n_pos + k] = (causal_ok && (q - k) <= window) ? 0.0f : neg_inf;
}
}
- set_input_f32("mimo_audio_full_mask", full_mask);
- set_input_f32("mimo_audio_window_mask", window_mask);
+ set_input_attn_mask(full_mask, 0);
+ set_input_attn_mask(window_mask, 1);
// input_local_transformer: block-diagonal mask + in-group positions
{
@@ -5636,7 +5661,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
local_mask[(size_t) q * n_padded + k] = same_group ? 0.0f : neg_inf;
}
}
- set_input_f32("mimo_audio_local_mask", local_mask);
+ set_input_attn_mask(local_mask, 2);
}
} break;
case PROJECTOR_TYPE_LFM2A:
diff --git a/tools/mtmd/models/deepseekocr2.cpp b/tools/mtmd/models/deepseekocr2.cpp
index 3e8b40941..7088bb97b 100644
--- a/tools/mtmd/models/deepseekocr2.cpp
+++ b/tools/mtmd/models/deepseekocr2.cpp
@@ -41,9 +41,7 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {
auto seq_len = inp->ne[1];
// qwen2 encoder attention mask
- ggml_tensor * attn_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, seq_len, seq_len);
- ggml_set_name(attn_mask, "qwen2_attn_mask");
- ggml_set_input(attn_mask);
+ ggml_tensor * attn_mask = build_inp_attn_mask(seq_len, seq_len);
ggml_tensor * inp_pos = ggml_cast(ctx0, ggml_arange(ctx0, 0, seq_len, 1), GGML_TYPE_I32);
diff --git a/tools/mtmd/models/exaone4_5.cpp b/tools/mtmd/models/exaone4_5.cpp
index bd9e8c748..d3372a1cd 100644
--- a/tools/mtmd/models/exaone4_5.cpp
+++ b/tools/mtmd/models/exaone4_5.cpp
@@ -58,13 +58,7 @@ ggml_cgraph * clip_graph_exaone4_5::build() {
ggml_set_name(inv_window_idx, "inv_window_idx");
ggml_set_input(inv_window_idx);
- window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
- ggml_set_name(window_mask, "window_mask");
- ggml_set_input(window_mask);
-
- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
- window_mask = ggml_cast(ctx0, window_mask, GGML_TYPE_F16);
- }
+ window_mask = build_inp_attn_mask(n_pos, n_pos);
}
ggml_tensor * inpL = inp;
diff --git a/tools/mtmd/models/mimo-audio.cpp b/tools/mtmd/models/mimo-audio.cpp
index 481b36cc8..0415bb60d 100644
--- a/tools/mtmd/models/mimo-audio.cpp
+++ b/tools/mtmd/models/mimo-audio.cpp
@@ -21,13 +21,8 @@ ggml_cgraph * clip_graph_mimo_audio::build() {
ggml_set_name(inp_pos, "mimo_audio_positions");
ggml_set_input(inp_pos);
- ggml_tensor * full_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
- ggml_set_name(full_mask, "mimo_audio_full_mask");
- ggml_set_input(full_mask);
-
- ggml_tensor * window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
- ggml_set_name(window_mask, "mimo_audio_window_mask");
- ggml_set_input(window_mask);
+ ggml_tensor * full_mask = build_inp_attn_mask(n_pos, n_pos, 0);
+ ggml_tensor * window_mask = build_inp_attn_mask(n_pos, n_pos, 1);
build_vit_opts opts;
opts.attn_mask_layers.resize(n_layer);
@@ -150,9 +145,7 @@ ggml_cgraph * clip_graph_mimo_audio::build() {
ggml_set_name(local_pos, "mimo_audio_local_positions");
ggml_set_input(local_pos);
- ggml_tensor * local_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_padded, n_padded);
- ggml_set_name(local_mask, "mimo_audio_local_mask");
- ggml_set_input(local_mask);
+ ggml_tensor * local_mask = build_inp_attn_mask(n_padded, n_padded, 2);
const float local_rope_theta = 640000.0f; // audio_config.rope_theta (differs from the encoder's)
auto apply_local_rope = [&](ggml_tensor * x) {
diff --git a/tools/mtmd/models/mimovl.cpp b/tools/mtmd/models/mimovl.cpp
index e1fbe2671..0cde450bf 100644
--- a/tools/mtmd/models/mimovl.cpp
+++ b/tools/mtmd/models/mimovl.cpp
@@ -84,13 +84,7 @@ ggml_cgraph * clip_graph_mimovl::build() {
ggml_tensor * idx_col = ggml_cast(ctx0, idx_col_f, GGML_TYPE_I32);
ggml_tensor * idx_col_inv = ggml_argsort(ctx0, idx_col_f, GGML_SORT_ORDER_ASC);
- ggml_tensor * window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
- ggml_set_name(window_mask, "mimovl_window_mask");
- ggml_set_input(window_mask);
-
- ggml_tensor * window_mask_attn = (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED)
- ? ggml_cast(ctx0, window_mask, GGML_TYPE_F16)
- : window_mask;
+ ggml_tensor * window_mask = build_inp_attn_mask(n_pos, n_pos);
// Reorder helper: permute patches at merge-unit granularity. The patch
// sequence is laid out as n_units groups of merge_unit (=4) consecutive
@@ -151,7 +145,7 @@ ggml_cgraph * clip_graph_mimovl::build() {
cb(Kcur, "Kcur_rope", il);
// Full layers: plain attention. Windowed layers: banded mask and per-head sinks.
- ggml_tensor * mask = is_full ? nullptr : window_mask_attn;
+ ggml_tensor * mask = is_full ? nullptr : window_mask;
ggml_tensor * sinks = is_full ? nullptr : layer.attn_sinks;
if (!is_full) {
GGML_ASSERT(layer.attn_sinks != nullptr);
diff --git a/tools/mtmd/models/minicpmv.cpp b/tools/mtmd/models/minicpmv.cpp
index 3e9c4c2a1..16514a337 100644
--- a/tools/mtmd/models/minicpmv.cpp
+++ b/tools/mtmd/models/minicpmv.cpp
@@ -146,12 +146,7 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() {
// so each window-major group of 4 tokens only attends to itself)
vit_merger_window_idx = add_i32_input("vit_merger_window_idx", n_pos);
vit_merger_inv_window_idx = add_i32_input("vit_merger_inv_window_idx", n_pos);
- vit_merger_window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
- ggml_set_name(vit_merger_window_mask, "vit_merger_window_mask");
- ggml_set_input(vit_merger_window_mask);
- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
- vit_merger_window_mask = ggml_cast(ctx0, vit_merger_window_mask, GGML_TYPE_F16);
- }
+ vit_merger_window_mask = build_inp_attn_mask(n_pos, n_pos);
// ViT merger 2x2 downsample gather indices
vit_merger_ds_idx_0 = add_i32_input("vit_merger_ds_idx_0", n_ds);
diff --git a/tools/mtmd/models/muse-glimmer.cpp b/tools/mtmd/models/muse-glimmer.cpp
index b201536f5..a397c1b22 100644
--- a/tools/mtmd/models/muse-glimmer.cpp
+++ b/tools/mtmd/models/muse-glimmer.cpp
@@ -10,7 +10,7 @@
// muse_glimmer_sp_perm [n_tok] i32 : window grouping permutation (applied after ln_pre)
// muse_glimmer_inv_perm [n_tok] i32 : inverse of sp_perm (applied after blocks)
// muse_glimmer_ds_perm [n_tok] i32 : pixel-shuffle gather (original order)
-// muse_glimmer_sp_mask [n_tok, n_tok] f32 : block-diagonal window mask (sparse layers)
+// attn_mask [n_tok, n_tok] f32 (f16 with flash attn) : block-diagonal window mask (sparse layers)
ggml_cgraph * clip_graph_muse_glimmer::build() {
const int ds = hparams.n_merge; // downsample factor (2)
const int sf = hparams.muse_glimmer_sparse_factor; // 4
@@ -31,9 +31,7 @@ ggml_cgraph * clip_graph_muse_glimmer::build() {
ggml_tensor * inv_perm = inp_i32("muse_glimmer_inv_perm", n_tok);
ggml_tensor * ds_perm = inp_i32("muse_glimmer_ds_perm", n_tok);
- ggml_tensor * sp_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tok, n_tok);
- ggml_set_name(sp_mask, "muse_glimmer_sp_mask");
- ggml_set_input(sp_mask);
+ ggml_tensor * sp_mask = build_inp_attn_mask(n_tok, n_tok);
// patchify via build_inp (conv2d over raw pixels) + bilinear-resized learned pos-emb
ggml_tensor * x = build_inp(); // [n_embd, n_tok, 1]
diff --git a/tools/mtmd/models/pockettts-gen.cpp b/tools/mtmd/models/pockettts-gen.cpp
index 3fd613e5f..662feff7f 100644
--- a/tools/mtmd/models/pockettts-gen.cpp
+++ b/tools/mtmd/models/pockettts-gen.cpp
@@ -225,6 +225,9 @@ ggml_cgraph * clip_graph_pockettts_gen::build() {
keep = ggml_mul(ctx0, keep,
ggml_step(ctx0, ggml_scale_bias(ctx0, ggml_add(ctx0, pos_k, base), 1.0f, 0.5f - (float) prefix)));
ggml_tensor * kq_mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), n_kv, n_pos, 1, 1);
+ if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
+ kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F16);
+ }
for (int il = 0; il < n_layer; il++) {
const auto & layer = model.gen_tfm_layers[il];
diff --git a/tools/mtmd/models/pockettts-spkenc.cpp b/tools/mtmd/models/pockettts-spkenc.cpp
index f802d9068..280f2a97c 100644
--- a/tools/mtmd/models/pockettts-spkenc.cpp
+++ b/tools/mtmd/models/pockettts-spkenc.cpp
@@ -53,9 +53,7 @@ ggml_cgraph * clip_graph_pockettts_spkenc::build() {
ggml_set_input(inp_pos);
// the mimi transformer is causal with a sliding window, see _build_attention_mask()
- ggml_tensor * kq_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, cur->ne[1], cur->ne[1]);
- ggml_set_name(kq_mask, "kq_mask");
- ggml_set_input(kq_mask);
+ ggml_tensor * kq_mask = build_inp_attn_mask(cur->ne[1], cur->ne[1]);
for (int il = 0; il < n_layer; il++) {
cur = tfm_layer_forward(cur, model.layers[il], inp_pos, kq_mask, il);
diff --git a/tools/mtmd/models/qwen2vl.cpp b/tools/mtmd/models/qwen2vl.cpp
index 2220c2692..4998d95bc 100644
--- a/tools/mtmd/models/qwen2vl.cpp
+++ b/tools/mtmd/models/qwen2vl.cpp
@@ -82,14 +82,7 @@ ggml_cgraph * clip_graph_qwen2vl::build() {
ggml_set_name(inv_window_idx, "inv_window_idx");
ggml_set_input(inv_window_idx);
// mask for window attention
- window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
- ggml_set_name(window_mask, "window_mask");
- ggml_set_input(window_mask);
-
- // if flash attn is used, we need to pad the mask and cast to f16
- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
- window_mask = ggml_cast(ctx0, window_mask, GGML_TYPE_F16);
- }
+ window_mask = build_inp_attn_mask(n_pos, n_pos);
// inpL shape: [n_embd, n_patches_x * n_patches_y, batch_size]
GGML_ASSERT(batch_size == 1);
diff --git a/tools/mtmd/models/qwen3tts-gen.cpp b/tools/mtmd/models/qwen3tts-gen.cpp
index 84c77f4fa..9826190e6 100644
--- a/tools/mtmd/models/qwen3tts-gen.cpp
+++ b/tools/mtmd/models/qwen3tts-gen.cpp
@@ -109,7 +109,11 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::causal_mask_row(int64_t n_kv_pa
ggml_tensor * keep = ggml_tri(ctx0, ones, GGML_TRI_TYPE_LOWER_DIAG);
ggml_tensor * row = ggml_view_1d(ctx0, keep, n_kv_pad, (size_t) pos * keep->nb[1]);
ggml_tensor * mask = ggml_log(ctx0, row); // 0 = keep, -inf = masked
- return ggml_reshape_4d(ctx0, mask, n_kv_pad, 1, 1, 1);
+ mask = ggml_reshape_4d(ctx0, mask, n_kv_pad, 1, 1, 1);
+ if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
+ mask = ggml_cast(ctx0, mask, GGML_TYPE_F16);
+ }
+ return mask;
}
// talker hidden size -> predictor hidden size (small_to_mtp_projection)
@@ -481,6 +485,9 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor *
keep = ggml_mul(ctx0, keep, warm);
ggml_tensor * mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), total_kv, N, 1, 1); // 0 = keep, -inf = masked
+ if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
+ mask = ggml_cast(ctx0, mask, GGML_TYPE_F16);
+ }
ggml_tensor * q_cur = ggml_reshape_4d(ctx0, q, d_head, n_head, N, 1);
ggml_tensor * k_cur = ggml_reshape_4d(ctx0, k_full, d_head, n_head_kv, total_kv, 1);
diff --git a/tools/mtmd/models/youtuvl.cpp b/tools/mtmd/models/youtuvl.cpp
index cd8f6d446..cda456c66 100644
--- a/tools/mtmd/models/youtuvl.cpp
+++ b/tools/mtmd/models/youtuvl.cpp
@@ -69,14 +69,7 @@ ggml_cgraph * clip_graph_youtuvl::build() {
ggml_set_name(inv_window_idx, "inv_window_idx");
ggml_set_input(inv_window_idx);
// mask for window attention
- window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
- ggml_set_name(window_mask, "window_mask");
- ggml_set_input(window_mask);
-
- // if flash attn is used, we need to pad the mask and cast to f16
- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
- window_mask = ggml_cast(ctx0, window_mask, GGML_TYPE_F16);
- }
+ window_mask = build_inp_attn_mask(n_pos, n_pos);
// inpL shape: [n_embd, n_patches_x * n_patches_y, batch_size]
GGML_ASSERT(batch_size == 1);