Commit 164ab018f for llama.cpp
commit 164ab018f9b7368ff8122094994c4b74e468aa0e
Author: Pascal <admin@serveurperso.com>
Date: Mon Nov 16 14:20:18 2026 +0100
webgpu: support per-head flash attention masks (#32394)
The flash attention shaders indexed the mask without its head
dimension, so masks with ne[2] > 1 gave wrong results. The mask offset
and the vec block states now include the head, broadcast as
head % mask->ne[2] like the other backends. The params buffer grows to
256 bytes to fit the two new mask fields.
The shared memory estimate dropped kv_shmem when only one of K and V
is read directly, while the shaders keep it unless both are, which made
the K f16 / V q8_0 cases exceed the workgroup storage limit. The vec kv
tile now takes both flags: the memory estimate drops kv_shmem only when
both are direct, and the tile still divides the KV padding when either
is. get_alloc_size and supports_op use the same rule as the pipelines,
and supports_op accounts for the register tile fallback.
diff --git a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
index 3cc6a70c8..db612da23 100644
--- a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
+++ b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
@@ -962,13 +962,16 @@ inline uint32_t ggml_webgpu_flash_attn_get_vec_kv_tile(size_t wg_mem_limit_byt
uint32_t head_dim_qk,
uint32_t head_dim_v,
bool has_mask,
- bool kv_direct) {
- const uint32_t max_kv_tile =
- ggml_webgpu_flash_attn_max_kv_tile(wg_mem_limit_bytes, 1u, 1u, head_dim_qk, head_dim_v, has_mask, kv_direct);
+ bool k_direct,
+ bool v_direct) {
+ // kv_shmem is dropped only when both K and V are read directly
+ const uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(wg_mem_limit_bytes, 1u, 1u, head_dim_qk, head_dim_v,
+ has_mask, k_direct && v_direct);
GGML_ASSERT(max_kv_tile > 0);
uint32_t kv_tile = std::min(GGML_WEBGPU_FLASH_ATTN_VEC_MAX_KV_TILE, max_kv_tile);
- if (kv_direct) {
+ // a direct read has no bounds check, so the tile must divide the KV padding
+ if (k_direct || v_direct) {
kv_tile = std::min(kv_tile, GGML_WEBGPU_KV_SEQ_PAD);
while (GGML_WEBGPU_KV_SEQ_PAD % kv_tile != 0) {
kv_tile -= 1u;
@@ -3050,7 +3053,7 @@ class ggml_webgpu_shader_lib {
uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(
context.wg_mem_limit_bytes, decisions.q_tile, decisions.use_sg_matrix ? context.sg_mat_n : 1u,
key.common.head_dim_qk, key.common.head_dim_v, key.common.has_mask,
- key.common.k_direct || key.common.v_direct);
+ key.common.k_direct && key.common.v_direct);
// WorkGroup storage size isn't enough for some params with subgroup matrices path (ref. https://github.com/ggml-org/llama.cpp/pull/26566)
if (max_kv_tile == 0) {
@@ -3063,7 +3066,7 @@ class ggml_webgpu_shader_lib {
key.use_sg_matrix = false;
max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(
context.wg_mem_limit_bytes, decisions.q_tile, 1u, key.common.head_dim_qk, key.common.head_dim_v,
- key.common.has_mask, key.common.k_direct || key.common.v_direct);
+ key.common.has_mask, key.common.k_direct && key.common.v_direct);
GGML_ASSERT(max_kv_tile > 0);
}
@@ -3125,7 +3128,7 @@ class ggml_webgpu_shader_lib {
ggml_webgpu_flash_attn_vec_decisions decisions = {};
decisions.kv_tile = ggml_webgpu_flash_attn_get_vec_kv_tile(context.wg_mem_limit_bytes, key.common.head_dim_qk,
key.common.head_dim_v, key.common.has_mask,
- key.common.k_direct || key.common.v_direct);
+ key.common.k_direct, key.common.v_direct);
decisions.wg_size = context.max_subgroup_size;
std::string variant = "flash_attn_vec";
diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp
index c08aa415f..a09b8a5e1 100644
--- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp
+++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp
@@ -90,7 +90,7 @@ static inline uint32_t ggml_webgpu_u32_from_f32(float value) {
#define WEBGPU_NUM_PARAM_SLOT_SAFETY_MARGIN 10u
#define WEBGPU_RUNTIME_WAIT_TIMEOUT_MS 30000u
#define WEBGPU_RUNTIME_WAIT_TIMEOUT_NS (WEBGPU_RUNTIME_WAIT_TIMEOUT_MS * 1e6)
-#define WEBGPU_PARAMS_BUF_SIZE_BYTES 128 // enough for 32 parameters
+#define WEBGPU_PARAMS_BUF_SIZE_BYTES 256 // enough for 64 parameters
#define WEBGPU_SET_ROWS_ERROR_BUF_SIZE_BYTES 4
#define WEBGPU_STORAGE_BUF_BINDING_MULT 4 // a storage buffer binding size must be a multiple of 4
@@ -2104,8 +2104,10 @@ static ggml_webgpu_flash_attn_op ggml_webgpu_flash_attn_prepare(webgpu_context &
(uint32_t) (V->nb[2] / ggml_type_size(V->type)), // stride (elements/blocks) of V in dimension 2
(uint32_t) (V->nb[3] / ggml_type_size(V->type)), // stride (elements/blocks) of V in dimension 3
op.has_mask ? (uint32_t) (mask->nb[3] / ggml_type_size(mask->type)) : 0, // stride of mask dim 3
- (uint32_t) (Q->ne[2] / K->ne[2]), // repeat factor for K/V in dim 2 (MHA/MQA/GQA)
- ggml_webgpu_u32_from_f32(scale), // scale (possibly adjusted for logit softcap)
+ op.has_mask ? (uint32_t) (mask->nb[2] / ggml_type_size(mask->type)) : 0, // stride of mask dim 2
+ op.has_mask ? (uint32_t) mask->ne[2] : 1u, // mask heads, broadcast over the Q heads
+ (uint32_t) (Q->ne[2] / K->ne[2]), // repeat factor for K/V in dim 2 (MHA/MQA/GQA)
+ ggml_webgpu_u32_from_f32(scale), // scale (possibly adjusted for logit softcap)
ggml_webgpu_u32_from_f32(max_bias),
ggml_webgpu_u32_from_f32(logit_softcap),
ggml_webgpu_u32_from_f32(n_head_log2),
@@ -2211,7 +2213,9 @@ static webgpu_encoded_op ggml_webgpu_flash_attn_vec(webgpu_context & ct
blk_nblk1 = (uint32_t) Q->ne[1];
blk_buf = ggml_webgpu_tensor_buf(dst);
const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
- blk_batch_count = stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u;
+ const uint32_t stride_mask2 = (uint32_t) (mask->nb[2] / ggml_type_size(mask->type));
+ const uint32_t mask_ne2 = (uint32_t) mask->ne[2];
+ blk_batch_count = (stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u) * mask_ne2;
const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
blk_size_bytes = ROUNDUP_POW2(blk_elems * sizeof(uint32_t), WEBGPU_STORAGE_BUF_BINDING_MULT);
const ggml_webgpu_shader_lib_context blk_shader_ctx = op.shader_lib_ctx;
@@ -2222,6 +2226,8 @@ static webgpu_encoded_op ggml_webgpu_flash_attn_vec(webgpu_context & ct
(uint32_t) Q->ne[1], // seq_len_q
(uint32_t) K->ne[1], // seq_len_kv
stride_mask3, // stride_mask3
+ stride_mask2, // stride_mask2
+ mask_ne2, // mask_ne2
blk_nblk0, // nblk0
blk_nblk1, // nblk1
};
@@ -4381,12 +4387,11 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer
// longest vec batch and the size stays monotonic in the batch length
const int64_t n_q = std::min<int64_t>(Q->ne[1], GGML_WEBGPU_FLASH_ATTN_VEC_MAX_SEQ_LEN - 1);
if (ggml_webgpu_flash_attn_use_vec_path(ctx->webgpu_global_ctx, Q, K, V, n_q)) {
- const bool kv_direct =
- ggml_webgpu_flash_attn_k_direct(Q, K, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH) ||
- ggml_webgpu_flash_attn_v_direct(Q, V, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH);
const uint32_t kv_tile = ggml_webgpu_flash_attn_get_vec_kv_tile(
capabilities.limits.maxComputeWorkgroupStorageSize, (uint32_t) Q->ne[0], (uint32_t) V->ne[0],
- mask != nullptr, kv_direct);
+ mask != nullptr,
+ ggml_webgpu_flash_attn_k_direct(Q, K, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH),
+ ggml_webgpu_flash_attn_v_direct(Q, V, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH));
const uint32_t vec_nwg_cap = capabilities.min_subgroup_size;
uint32_t nwg = ggml_webgpu_flash_attn_vec_nwg(vec_nwg_cap, kv_tile, (uint32_t) K->ne[1]);
@@ -4403,11 +4408,12 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer
res += WEBGPU_STORAGE_BUF_BINDING_MULT + align;
}
if (mask != nullptr) {
- const uint32_t blk_nblk0 = CEIL_DIV((uint32_t) K->ne[1], kv_tile);
- const uint32_t blk_nblk1 = (uint32_t) n_q;
- const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
- const uint32_t blk_batch_count = stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u;
- const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
+ const uint32_t blk_nblk0 = CEIL_DIV((uint32_t) K->ne[1], kv_tile);
+ const uint32_t blk_nblk1 = (uint32_t) n_q;
+ const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
+ const uint32_t blk_batch_count =
+ (stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u) * (uint32_t) mask->ne[2];
+ const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
const size_t blk_size_bytes =
ROUNDUP_POW2(blk_elems * sizeof(uint32_t), WEBGPU_STORAGE_BUF_BINDING_MULT);
res += blk_size_bytes + align;
@@ -5014,14 +5020,22 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
const uint32_t q_tile =
use_subgroup_matrix ? capabilities.sg_mat_m : GGML_WEBGPU_FLASH_ATTN_TILE_Q_TILE;
const uint32_t kv_granularity = use_subgroup_matrix ? capabilities.sg_mat_n : 1u;
- const bool kv_direct = use_subgroup_matrix ?
- ggml_webgpu_flash_attn_k_direct(src0, src1, capabilities.sg_mat_k) ||
- ggml_webgpu_flash_attn_v_direct(src0, src2, capabilities.sg_mat_k) :
- false;
- const uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(
+ // same direct read rule as get_flash_attn_pipeline: F16 only, on the subgroup matrix path
+ const bool kv_direct = use_subgroup_matrix && src1->type == GGML_TYPE_F16 &&
+ src2->type == GGML_TYPE_F16 &&
+ ggml_webgpu_flash_attn_k_direct(src0, src1, capabilities.sg_mat_k) &&
+ ggml_webgpu_flash_attn_v_direct(src0, src2, capabilities.sg_mat_k);
+ const uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(
capabilities.limits.maxComputeWorkgroupStorageSize, q_tile, kv_granularity, (uint32_t) src0->ne[0],
(uint32_t) src2->ne[0], op->src[3] != nullptr, kv_direct);
- supports_op = max_kv_tile > 0;
+ // get_flash_attn_pipeline falls back to the register tile path when the subgroup matrix tile does not fit
+ const uint32_t fallback_kv_tile =
+ use_subgroup_matrix ?
+ ggml_webgpu_flash_attn_max_kv_tile(
+ capabilities.limits.maxComputeWorkgroupStorageSize, GGML_WEBGPU_FLASH_ATTN_TILE_Q_TILE, 1u,
+ (uint32_t) src0->ne[0], (uint32_t) src2->ne[0], op->src[3] != nullptr, false) :
+ 0u;
+ supports_op = max_kv_tile > 0 || fallback_kv_tile > 0;
break;
}
case GGML_OP_GROUP_NORM:
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl
index a7dee6512..3e671f116 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl
@@ -116,7 +116,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
#ifdef MASK
// mask offset
- let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + q_row_start * params.seq_len_kv;
+ let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
#endif
// note that the output is permuted, the layout is [head_dim_v, n_heads, seq_len_q, batch_size]
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl
index 48a79b6ce..4ab4c088b 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl
@@ -65,6 +65,8 @@ struct Params {
stride_v2: u32,
stride_v3: u32,
stride_mask3: u32,
+ stride_mask2: u32,
+ mask_ne2: u32,
// repeat factors for K/V, e.g., MHA vs. MQA vs. GQA
q_per_kv: u32,
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl
index 7edca84fc..70ef4d03e 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl
@@ -65,7 +65,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
let row_active = subgroup_id < Q_TILE && global_q_row < params.seq_len_q;
#ifdef MASK
- let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + q_row_start * params.seq_len_kv;
+ let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
#endif
let dst_global_offset = dst_batch_offset + q_row_start * dst2_stride + head_idx * HEAD_DIM_V;
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl
index b4f7c16c3..985527d28 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl
@@ -9,6 +9,8 @@ struct Params {
seq_len_q: u32,
seq_len_kv: u32,
stride_mask3: u32,
+ stride_mask2: u32,
+ mask_ne2: u32,
// Number of KV blocks and Q blocks per batch.
// nblk0 = ceil(seq_len_kv / KV_TILE), nblk1 = seq_len_q.
nblk0: u32,
@@ -30,11 +32,12 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
@builtin(local_invocation_id) local_id: vec3<u32>) {
// Dispatch mapping:
// - x indexes KV blocks
- // - y flattens (batch_idx, q_blk) as y = batch_idx * nblk1 + q_blk
+ // - y flattens (batch_idx, mask_head, q_blk) as y = (batch_idx * mask_ne2 + mask_head) * nblk1 + q_blk
let kv_blk = wg_id.x;
let y = wg_id.y;
let q_blk = y % params.nblk1;
- let batch_idx = y / params.nblk1;
+ let mask_head = (y / params.nblk1) % params.mask_ne2;
+ let batch_idx = y / (params.nblk1 * params.mask_ne2);
if (kv_blk >= params.nblk0) {
return;
}
@@ -43,7 +46,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
let k_start = kv_blk * KV_TILE;
let mask_batch = select(0u, batch_idx, params.stride_mask3 > 0u);
- let mask_batch_base = params.offset_mask + mask_batch * params.stride_mask3;
+ let mask_batch_base = params.offset_mask + mask_batch * params.stride_mask3 + mask_head * params.stride_mask2;
// We keep min/max to classify:
// - fully masked (max <= MASK_MIN)
@@ -95,7 +98,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
}
}
- let blk_idx = (batch_idx * params.nblk1 + q_blk) * params.nblk0 + kv_blk;
+ let blk_idx = ((batch_idx * params.mask_ne2 + mask_head) * params.nblk1 + q_blk) * params.nblk0 + kv_blk;
blk[blk_idx] = state;
}
}
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl
index ae941245c..1bff2ab98 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl
@@ -107,7 +107,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
#ifdef MASK
// mask offset
- let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + q_row_start * params.seq_len_kv;
+ let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
#endif
let head = f32(head_idx);
@@ -129,7 +129,8 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
let q_blk = q_row_start;
let kv_blk = kv_tile / KV_TILE;
let blk_batch = select(0u, batch_idx, params.stride_mask3 > 0u);
- let blk_idx = params.blk_base + (blk_batch * params.blk_nblk1 + q_blk) * params.blk_nblk0 + kv_blk;
+ let blk_head = head_idx % params.mask_ne2;
+ let blk_idx = params.blk_base + ((blk_batch * params.mask_ne2 + blk_head) * params.blk_nblk1 + q_blk) * params.blk_nblk0 + kv_blk;
let blk_state_local = blk[blk_idx];
#else
let blk_state_local = 1u;