Commit 164ab018f for llama.cpp

commit 164ab018f9b7368ff8122094994c4b74e468aa0e
Author: Pascal <admin@serveurperso.com>
Date:   Mon Nov 16 14:20:18 2026 +0100

    webgpu: support per-head flash attention masks (#32394)

    The flash attention shaders indexed the mask without its head
    dimension, so masks with ne[2] > 1 gave wrong results. The mask offset
    and the vec block states now include the head, broadcast as
    head % mask->ne[2] like the other backends. The params buffer grows to
    256 bytes to fit the two new mask fields.

    The shared memory estimate dropped kv_shmem when only one of K and V
    is read directly, while the shaders keep it unless both are, which made
    the K f16 / V q8_0 cases exceed the workgroup storage limit. The vec kv
    tile now takes both flags: the memory estimate drops kv_shmem only when
    both are direct, and the tile still divides the KV padding when either
    is. get_alloc_size and supports_op use the same rule as the pipelines,
    and supports_op accounts for the register tile fallback.

diff --git a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
index 3cc6a70c8..db612da23 100644
--- a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
+++ b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
@@ -962,13 +962,16 @@ inline uint32_t ggml_webgpu_flash_attn_get_vec_kv_tile(size_t   wg_mem_limit_byt
                                                        uint32_t head_dim_qk,
                                                        uint32_t head_dim_v,
                                                        bool     has_mask,
-                                                       bool     kv_direct) {
-    const uint32_t max_kv_tile =
-        ggml_webgpu_flash_attn_max_kv_tile(wg_mem_limit_bytes, 1u, 1u, head_dim_qk, head_dim_v, has_mask, kv_direct);
+                                                       bool     k_direct,
+                                                       bool     v_direct) {
+    // kv_shmem is dropped only when both K and V are read directly
+    const uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(wg_mem_limit_bytes, 1u, 1u, head_dim_qk, head_dim_v,
+                                                                    has_mask, k_direct && v_direct);
     GGML_ASSERT(max_kv_tile > 0);

     uint32_t kv_tile = std::min(GGML_WEBGPU_FLASH_ATTN_VEC_MAX_KV_TILE, max_kv_tile);
-    if (kv_direct) {
+    // a direct read has no bounds check, so the tile must divide the KV padding
+    if (k_direct || v_direct) {
         kv_tile = std::min(kv_tile, GGML_WEBGPU_KV_SEQ_PAD);
         while (GGML_WEBGPU_KV_SEQ_PAD % kv_tile != 0) {
             kv_tile -= 1u;
@@ -3050,7 +3053,7 @@ class ggml_webgpu_shader_lib {
         uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(
             context.wg_mem_limit_bytes, decisions.q_tile, decisions.use_sg_matrix ? context.sg_mat_n : 1u,
             key.common.head_dim_qk, key.common.head_dim_v, key.common.has_mask,
-            key.common.k_direct || key.common.v_direct);
+            key.common.k_direct && key.common.v_direct);

         // WorkGroup storage size isn't enough for some params with subgroup matrices path (ref. https://github.com/ggml-org/llama.cpp/pull/26566)
         if (max_kv_tile == 0) {
@@ -3063,7 +3066,7 @@ class ggml_webgpu_shader_lib {
             key.use_sg_matrix       = false;
             max_kv_tile             = ggml_webgpu_flash_attn_max_kv_tile(
                 context.wg_mem_limit_bytes, decisions.q_tile, 1u, key.common.head_dim_qk, key.common.head_dim_v,
-                key.common.has_mask, key.common.k_direct || key.common.v_direct);
+                key.common.has_mask, key.common.k_direct && key.common.v_direct);
             GGML_ASSERT(max_kv_tile > 0);
         }

@@ -3125,7 +3128,7 @@ class ggml_webgpu_shader_lib {
         ggml_webgpu_flash_attn_vec_decisions decisions = {};
         decisions.kv_tile = ggml_webgpu_flash_attn_get_vec_kv_tile(context.wg_mem_limit_bytes, key.common.head_dim_qk,
                                                                    key.common.head_dim_v, key.common.has_mask,
-                                                                   key.common.k_direct || key.common.v_direct);
+                                                                   key.common.k_direct, key.common.v_direct);
         decisions.wg_size = context.max_subgroup_size;

         std::string              variant = "flash_attn_vec";
diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp
index c08aa415f..a09b8a5e1 100644
--- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp
+++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp
@@ -90,7 +90,7 @@ static inline uint32_t ggml_webgpu_u32_from_f32(float value) {
 #define WEBGPU_NUM_PARAM_SLOT_SAFETY_MARGIN      10u
 #define WEBGPU_RUNTIME_WAIT_TIMEOUT_MS           30000u
 #define WEBGPU_RUNTIME_WAIT_TIMEOUT_NS           (WEBGPU_RUNTIME_WAIT_TIMEOUT_MS * 1e6)
-#define WEBGPU_PARAMS_BUF_SIZE_BYTES             128  // enough for 32 parameters
+#define WEBGPU_PARAMS_BUF_SIZE_BYTES             256  // enough for 64 parameters
 #define WEBGPU_SET_ROWS_ERROR_BUF_SIZE_BYTES     4
 #define WEBGPU_STORAGE_BUF_BINDING_MULT          4    // a storage buffer binding size must be a multiple of 4

@@ -2104,8 +2104,10 @@ static ggml_webgpu_flash_attn_op ggml_webgpu_flash_attn_prepare(webgpu_context &
         (uint32_t) (V->nb[2] / ggml_type_size(V->type)),  // stride (elements/blocks) of V in dimension 2
         (uint32_t) (V->nb[3] / ggml_type_size(V->type)),  // stride (elements/blocks) of V in dimension 3
         op.has_mask ? (uint32_t) (mask->nb[3] / ggml_type_size(mask->type)) : 0,  // stride of mask dim 3
-        (uint32_t) (Q->ne[2] / K->ne[2]),  // repeat factor for K/V in dim 2 (MHA/MQA/GQA)
-        ggml_webgpu_u32_from_f32(scale),   // scale (possibly adjusted for logit softcap)
+        op.has_mask ? (uint32_t) (mask->nb[2] / ggml_type_size(mask->type)) : 0,  // stride of mask dim 2
+        op.has_mask ? (uint32_t) mask->ne[2] : 1u,  // mask heads, broadcast over the Q heads
+        (uint32_t) (Q->ne[2] / K->ne[2]),           // repeat factor for K/V in dim 2 (MHA/MQA/GQA)
+        ggml_webgpu_u32_from_f32(scale),            // scale (possibly adjusted for logit softcap)
         ggml_webgpu_u32_from_f32(max_bias),
         ggml_webgpu_u32_from_f32(logit_softcap),
         ggml_webgpu_u32_from_f32(n_head_log2),
@@ -2211,7 +2213,9 @@ static webgpu_encoded_op ggml_webgpu_flash_attn_vec(webgpu_context &          ct
         blk_nblk1                   = (uint32_t) Q->ne[1];
         blk_buf                     = ggml_webgpu_tensor_buf(dst);
         const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
-        blk_batch_count             = stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u;
+        const uint32_t stride_mask2 = (uint32_t) (mask->nb[2] / ggml_type_size(mask->type));
+        const uint32_t mask_ne2     = (uint32_t) mask->ne[2];
+        blk_batch_count             = (stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u) * mask_ne2;
         const uint64_t blk_elems    = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
         blk_size_bytes              = ROUNDUP_POW2(blk_elems * sizeof(uint32_t), WEBGPU_STORAGE_BUF_BINDING_MULT);
         const ggml_webgpu_shader_lib_context blk_shader_ctx = op.shader_lib_ctx;
@@ -2222,6 +2226,8 @@ static webgpu_encoded_op ggml_webgpu_flash_attn_vec(webgpu_context &          ct
             (uint32_t) Q->ne[1],                                                                   // seq_len_q
             (uint32_t) K->ne[1],                                                                   // seq_len_kv
             stride_mask3,                                                                          // stride_mask3
+            stride_mask2,                                                                          // stride_mask2
+            mask_ne2,                                                                              // mask_ne2
             blk_nblk0,                                                                             // nblk0
             blk_nblk1,                                                                             // nblk1
         };
@@ -4381,12 +4387,11 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer
                 // longest vec batch and the size stays monotonic in the batch length
                 const int64_t       n_q = std::min<int64_t>(Q->ne[1], GGML_WEBGPU_FLASH_ATTN_VEC_MAX_SEQ_LEN - 1);
                 if (ggml_webgpu_flash_attn_use_vec_path(ctx->webgpu_global_ctx, Q, K, V, n_q)) {
-                    const bool kv_direct =
-                        ggml_webgpu_flash_attn_k_direct(Q, K, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH) ||
-                        ggml_webgpu_flash_attn_v_direct(Q, V, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH);
                     const uint32_t kv_tile = ggml_webgpu_flash_attn_get_vec_kv_tile(
                         capabilities.limits.maxComputeWorkgroupStorageSize, (uint32_t) Q->ne[0], (uint32_t) V->ne[0],
-                        mask != nullptr, kv_direct);
+                        mask != nullptr,
+                        ggml_webgpu_flash_attn_k_direct(Q, K, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH),
+                        ggml_webgpu_flash_attn_v_direct(Q, V, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH));

                     const uint32_t vec_nwg_cap = capabilities.min_subgroup_size;
                     uint32_t       nwg = ggml_webgpu_flash_attn_vec_nwg(vec_nwg_cap, kv_tile, (uint32_t) K->ne[1]);
@@ -4403,11 +4408,12 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer
                         res += WEBGPU_STORAGE_BUF_BINDING_MULT + align;
                     }
                     if (mask != nullptr) {
-                        const uint32_t blk_nblk0       = CEIL_DIV((uint32_t) K->ne[1], kv_tile);
-                        const uint32_t blk_nblk1       = (uint32_t) n_q;
-                        const uint32_t stride_mask3    = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
-                        const uint32_t blk_batch_count = stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u;
-                        const uint64_t blk_elems       = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
+                        const uint32_t blk_nblk0    = CEIL_DIV((uint32_t) K->ne[1], kv_tile);
+                        const uint32_t blk_nblk1    = (uint32_t) n_q;
+                        const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
+                        const uint32_t blk_batch_count =
+                            (stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u) * (uint32_t) mask->ne[2];
+                        const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
                         const size_t   blk_size_bytes =
                             ROUNDUP_POW2(blk_elems * sizeof(uint32_t), WEBGPU_STORAGE_BUF_BINDING_MULT);
                         res += blk_size_bytes + align;
@@ -5014,14 +5020,22 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
                 const uint32_t q_tile =
                     use_subgroup_matrix ? capabilities.sg_mat_m : GGML_WEBGPU_FLASH_ATTN_TILE_Q_TILE;
                 const uint32_t kv_granularity = use_subgroup_matrix ? capabilities.sg_mat_n : 1u;
-                const bool     kv_direct = use_subgroup_matrix ?
-                                               ggml_webgpu_flash_attn_k_direct(src0, src1, capabilities.sg_mat_k) ||
-                                                   ggml_webgpu_flash_attn_v_direct(src0, src2, capabilities.sg_mat_k) :
-                                               false;
-                const uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(
+                // same direct read rule as get_flash_attn_pipeline: F16 only, on the subgroup matrix path
+                const bool     kv_direct      = use_subgroup_matrix && src1->type == GGML_TYPE_F16 &&
+                                                src2->type == GGML_TYPE_F16 &&
+                                                ggml_webgpu_flash_attn_k_direct(src0, src1, capabilities.sg_mat_k) &&
+                                                ggml_webgpu_flash_attn_v_direct(src0, src2, capabilities.sg_mat_k);
+                const uint32_t max_kv_tile    = ggml_webgpu_flash_attn_max_kv_tile(
                     capabilities.limits.maxComputeWorkgroupStorageSize, q_tile, kv_granularity, (uint32_t) src0->ne[0],
                     (uint32_t) src2->ne[0], op->src[3] != nullptr, kv_direct);
-                supports_op = max_kv_tile > 0;
+                // get_flash_attn_pipeline falls back to the register tile path when the subgroup matrix tile does not fit
+                const uint32_t fallback_kv_tile =
+                    use_subgroup_matrix ?
+                        ggml_webgpu_flash_attn_max_kv_tile(
+                            capabilities.limits.maxComputeWorkgroupStorageSize, GGML_WEBGPU_FLASH_ATTN_TILE_Q_TILE, 1u,
+                            (uint32_t) src0->ne[0], (uint32_t) src2->ne[0], op->src[3] != nullptr, false) :
+                        0u;
+                supports_op = max_kv_tile > 0 || fallback_kv_tile > 0;
                 break;
             }
         case GGML_OP_GROUP_NORM:
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl
index a7dee6512..3e671f116 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl
@@ -116,7 +116,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,

 #ifdef MASK
     // mask offset
-    let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + q_row_start * params.seq_len_kv;
+    let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
 #endif

     // note that the output is permuted, the layout is [head_dim_v, n_heads, seq_len_q, batch_size]
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl
index 48a79b6ce..4ab4c088b 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl
@@ -65,6 +65,8 @@ struct Params {
     stride_v2: u32,
     stride_v3: u32,
     stride_mask3: u32,
+    stride_mask2: u32,
+    mask_ne2: u32,

     // repeat factors for K/V, e.g., MHA vs. MQA vs. GQA
     q_per_kv: u32,
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl
index 7edca84fc..70ef4d03e 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl
@@ -65,7 +65,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
     let row_active = subgroup_id < Q_TILE && global_q_row < params.seq_len_q;

 #ifdef MASK
-    let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + q_row_start * params.seq_len_kv;
+    let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
 #endif

     let dst_global_offset = dst_batch_offset + q_row_start * dst2_stride + head_idx * HEAD_DIM_V;
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl
index b4f7c16c3..985527d28 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl
@@ -9,6 +9,8 @@ struct Params {
     seq_len_q: u32,
     seq_len_kv: u32,
     stride_mask3: u32,
+    stride_mask2: u32,
+    mask_ne2: u32,
     // Number of KV blocks and Q blocks per batch.
     // nblk0 = ceil(seq_len_kv / KV_TILE), nblk1 = seq_len_q.
     nblk0: u32,
@@ -30,11 +32,12 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
         @builtin(local_invocation_id) local_id: vec3<u32>) {
     // Dispatch mapping:
     //  - x indexes KV blocks
-    //  - y flattens (batch_idx, q_blk) as y = batch_idx * nblk1 + q_blk
+    //  - y flattens (batch_idx, mask_head, q_blk) as y = (batch_idx * mask_ne2 + mask_head) * nblk1 + q_blk
     let kv_blk = wg_id.x;
     let y = wg_id.y;
     let q_blk = y % params.nblk1;
-    let batch_idx = y / params.nblk1;
+    let mask_head = (y / params.nblk1) % params.mask_ne2;
+    let batch_idx = y / (params.nblk1 * params.mask_ne2);
     if (kv_blk >= params.nblk0) {
         return;
     }
@@ -43,7 +46,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
     let k_start = kv_blk * KV_TILE;

     let mask_batch = select(0u, batch_idx, params.stride_mask3 > 0u);
-    let mask_batch_base = params.offset_mask + mask_batch * params.stride_mask3;
+    let mask_batch_base = params.offset_mask + mask_batch * params.stride_mask3 + mask_head * params.stride_mask2;

     // We keep min/max to classify:
     //  - fully masked (max <= MASK_MIN)
@@ -95,7 +98,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
             }
         }

-        let blk_idx = (batch_idx * params.nblk1 + q_blk) * params.nblk0 + kv_blk;
+        let blk_idx = ((batch_idx * params.mask_ne2 + mask_head) * params.nblk1 + q_blk) * params.nblk0 + kv_blk;
         blk[blk_idx] = state;
     }
 }
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl
index ae941245c..1bff2ab98 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl
@@ -107,7 +107,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,

 #ifdef MASK
     // mask offset
-    let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + q_row_start * params.seq_len_kv;
+    let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
 #endif

     let head = f32(head_idx);
@@ -129,7 +129,8 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
         let q_blk = q_row_start;
         let kv_blk = kv_tile / KV_TILE;
         let blk_batch = select(0u, batch_idx, params.stride_mask3 > 0u);
-        let blk_idx = params.blk_base + (blk_batch * params.blk_nblk1 + q_blk) * params.blk_nblk0 + kv_blk;
+        let blk_head = head_idx % params.mask_ne2;
+        let blk_idx = params.blk_base + ((blk_batch * params.mask_ne2 + blk_head) * params.blk_nblk1 + q_blk) * params.blk_nblk0 + kv_blk;
         let blk_state_local = blk[blk_idx];
 #else
         let blk_state_local = 1u;