Commit 6f311d7ad for llama.cpp
commit 6f311d7ad63dce97a3ad1deaafb7cc044bb8c26c
Author: Masashi Yoshimura <yoshimura.masashi.frbs@gmail.com>
Date: Thu Nov 19 01:08:02 2026 +0900
webgpu: (fix CI) allow the FA mask to be broadcast over the Q batch (#32535)
* allow the mask batch to be broadcast over the Q batch
* use Q_ne3 for blk_batch_count of get_alloc_size function
diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp
index 898c89399..fc2c2797d 100644
--- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp
+++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp
@@ -2106,6 +2106,7 @@ static ggml_webgpu_flash_attn_op ggml_webgpu_flash_attn_prepare(webgpu_context &
op.has_mask ? (uint32_t) (mask->nb[3] / ggml_type_size(mask->type)) : 0, // stride of mask dim 3
op.has_mask ? (uint32_t) (mask->nb[2] / ggml_type_size(mask->type)) : 0, // stride of mask dim 2
op.has_mask ? (uint32_t) mask->ne[2] : 1u, // mask heads, broadcast over the Q heads
+ op.has_mask ? (uint32_t) mask->ne[3] : 1u, // mask batches, broadcast over the Q batches
(uint32_t) (Q->ne[2] / K->ne[2]), // repeat factor for K/V in dim 2 (MHA/MQA/GQA)
ggml_webgpu_u32_from_f32(scale), // scale (possibly adjusted for logit softcap)
ggml_webgpu_u32_from_f32(max_bias),
@@ -2215,7 +2216,7 @@ static webgpu_encoded_op ggml_webgpu_flash_attn_vec(webgpu_context & ct
const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
const uint32_t stride_mask2 = (uint32_t) (mask->nb[2] / ggml_type_size(mask->type));
const uint32_t mask_ne2 = (uint32_t) mask->ne[2];
- blk_batch_count = (stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u) * mask_ne2;
+ blk_batch_count = (uint32_t) (mask->ne[3] * mask_ne2);
const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
blk_size_bytes = ROUNDUP_POW2(blk_elems * sizeof(uint32_t), WEBGPU_STORAGE_BUF_BINDING_MULT);
const ggml_webgpu_shader_lib_context blk_shader_ctx = op.shader_lib_ctx;
@@ -4448,12 +4449,10 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer
res += WEBGPU_STORAGE_BUF_BINDING_MULT + align;
}
if (mask != nullptr) {
- const uint32_t blk_nblk0 = CEIL_DIV((uint32_t) K->ne[1], kv_tile);
- const uint32_t blk_nblk1 = (uint32_t) n_q;
- const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
- const uint32_t blk_batch_count =
- (stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u) * (uint32_t) mask->ne[2];
- const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
+ const uint32_t blk_nblk0 = CEIL_DIV((uint32_t) K->ne[1], kv_tile);
+ const uint32_t blk_nblk1 = (uint32_t) n_q;
+ const uint32_t blk_batch_count = (uint32_t) Q->ne[3] * (uint32_t) mask->ne[2];
+ const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
const size_t blk_size_bytes =
ROUNDUP_POW2(blk_elems * sizeof(uint32_t), WEBGPU_STORAGE_BUF_BINDING_MULT);
res += blk_size_bytes + align;
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl
index 3e671f116..357145f4d 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl
@@ -116,7 +116,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
#ifdef MASK
// mask offset
- let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
+ let mask_global_offset = params.offset_mask + (batch_idx % params.mask_ne3) * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
#endif
// note that the output is permuted, the layout is [head_dim_v, n_heads, seq_len_q, batch_size]
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl
index 4ab4c088b..eabd06902 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl
@@ -67,6 +67,7 @@ struct Params {
stride_mask3: u32,
stride_mask2: u32,
mask_ne2: u32,
+ mask_ne3: u32,
// repeat factors for K/V, e.g., MHA vs. MQA vs. GQA
q_per_kv: u32,
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl
index 70ef4d03e..273f51dd1 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl
@@ -65,7 +65,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
let row_active = subgroup_id < Q_TILE && global_q_row < params.seq_len_q;
#ifdef MASK
- let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
+ let mask_global_offset = params.offset_mask + (batch_idx % params.mask_ne3) * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
#endif
let dst_global_offset = dst_batch_offset + q_row_start * dst2_stride + head_idx * HEAD_DIM_V;
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl
index 985527d28..19d6d40a1 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl
@@ -45,8 +45,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
let q_start = q_blk;
let k_start = kv_blk * KV_TILE;
- let mask_batch = select(0u, batch_idx, params.stride_mask3 > 0u);
- let mask_batch_base = params.offset_mask + mask_batch * params.stride_mask3 + mask_head * params.stride_mask2;
+ let mask_batch_base = params.offset_mask + batch_idx * params.stride_mask3 + mask_head * params.stride_mask2;
// We keep min/max to classify:
// - fully masked (max <= MASK_MIN)
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl
index 1bff2ab98..79fb65443 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl
@@ -107,7 +107,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
#ifdef MASK
// mask offset
- let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
+ let mask_global_offset = params.offset_mask + (batch_idx % params.mask_ne3) * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
#endif
let head = f32(head_idx);
@@ -128,7 +128,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
#ifdef BLK
let q_blk = q_row_start;
let kv_blk = kv_tile / KV_TILE;
- let blk_batch = select(0u, batch_idx, params.stride_mask3 > 0u);
+ let blk_batch = batch_idx % params.mask_ne3;
let blk_head = head_idx % params.mask_ne2;
let blk_idx = params.blk_base + ((blk_batch * params.mask_ne2 + blk_head) * params.blk_nblk1 + q_blk) * params.blk_nblk0 + kv_blk;
let blk_state_local = blk[blk_idx];