Commit 362ac9297 for llama.cpp

commit 362ac929749d56e433878585a8506f24968698b4
Author: Ruben Ortlam <rortlam@redhat.com>
Date:   Sat Nov 21 12:46:20 2026 +0100

    vulkan: unroll FA coopmat1 array due to a driver bug on AMD Windows (#32466)

    * vulkan: unroll FA coopmat1 array due to a driver bug on AMD Windows

    * simplify code to make it clearer

diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index d4bbf9913..ae5a05c72 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -1323,7 +1323,9 @@ static vk_fa_tuning_params get_fa_tuning_params_coopmat1(const vk_device& device
     }

     // Br=32 spills registers, only prefill benefits
-    result.q_in_regs = result.block_rows == coopmat_block_rows && device->vendor_id == VK_VENDOR_ID_AMD && n_rows >= 64;
+    // AMD proprietary driver crashes compiling the loop-indexed Qreg array
+    result.q_in_regs = result.block_rows == coopmat_block_rows && device->vendor_id == VK_VENDOR_ID_AMD && n_rows >= 64 &&
+                       device->driver_id != vk::DriverId::eAmdProprietary;

     // Flipped ownership: each subgroup owns 16 Q rows.
     // uma, nvidia: K/V tiles staged in separate LDS buffers; discrete AMD GQA: K/V loaded directly from global memory.
@@ -1355,6 +1357,9 @@ static vk_fa_tuning_params get_fa_tuning_params_coopmat1(const vk_device& device
         result.limit_occupancy_shmem = 14 * 1024 / 16;
     }

+    // non-flip shader handles at most two row tiles
+    GGML_ASSERT(result.flip || result.block_rows <= 2 * coopmat_block_rows);
+
     return result;
 }

diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp
index bb9755ab8..ea945ed07 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp
@@ -25,6 +25,7 @@
 // These need to be supported N,M values for a MatBc x MatBr x 16 coopmatmuladd
 const uint32_t MatBr = 16;
 const uint32_t MatBc = 16;
+const uint32_t q_tiles = Br / MatBr;

 const uint32_t rows_per_thread = Br / row_split;
 const uint32_t cols_per_iter = gl_WorkGroupSize.x / row_split;
@@ -64,15 +65,14 @@ shared FLOAT_TYPEV4 kvsh[(kvsh_stride >= vsh_stride) ? (Bc * kvsh_stride) : (Bc
 shared ACC_TYPE slope[Br];

 // row-tiled builds stage the PV result through sfsh
-const bool PV_IN_SFSH = Br > MatBr;
 const uint32_t osh_stride = row_split * MatBr / 4;
-shared O_TYPEV4 pvsh[PV_IN_SFSH ? 2 : Br * osh_stride];
+shared O_TYPEV4 pvsh[q_tiles > 1 ? 2 : Br * osh_stride];

 shared vec4 occupancy_limiter[LIMIT_OCCUPANCY_SHMEM > 0 ? LIMIT_OCCUPANCY_SHMEM : 1];
 #if defined(FLOAT16) && !defined(ACC_TYPE_MAX)
 const uint32_t pv_per_acc = 2;
 O_TYPEV4 pvsh_load(const uint32_t i) {
-    if (!PV_IN_SFSH) {
+    if (q_tiles == 1) {
         return pvsh[i];
     }
     const uvec2 u = floatBitsToUint(vec2(sfsh[i / 2][(i % 2) * 2], sfsh[i / 2][(i % 2) * 2 + 1]));
@@ -81,7 +81,7 @@ O_TYPEV4 pvsh_load(const uint32_t i) {
 #else
 const uint32_t pv_per_acc = 1;
 O_TYPEV4 pvsh_load(const uint32_t i) {
-    if (!PV_IN_SFSH) {
+    if (q_tiles == 1) {
         return pvsh[i];
     }
     return O_TYPEV4(sfsh[i]);
@@ -117,7 +117,6 @@ void main() {

 #define tile_row(r) (row_tid * rows_per_thread + (r))

-    const uint32_t q_tiles = Br / MatBr;
     const uint32_t d_tiles = HSK_pad / 16;

     uint32_t q_offset = gqa_iq1*p.nb01 + (iq2*p.nb02+iq3*p.nb03) / 4;
@@ -329,10 +328,9 @@ void main() {
         // K * Q^T -> S^T: Bc x HSK_pad * HSK_pad x Br -> Bc x Br
         // Bc split across workgroup (four subgroups), loop over HSK in chunks of 16: 16 x 16 * 16 x 16 -> 16 x 16
         // This is written transposed in order to allow for N being 8 if implementations need it
-        coopmat<ACC_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator> SfMat[q_tiles];
-        [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) {
-            SfMat[q] = coopmat<ACC_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator>(0);
-        }
+        // No coopmat arrays: AMD's Windows compiler crashes on loop-indexed ones. q_tiles <= 2.
+        coopmat<ACC_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator> SfMat0 = coopmat<ACC_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator>(0);
+        coopmat<ACC_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator> SfMat1 = coopmat<ACC_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator>(0);
         coopmat<FLOAT_TYPE, gl_ScopeSubgroup, MatBc, 16, gl_MatrixUseA> KMat;
         coopmat<FLOAT_TYPE, gl_ScopeSubgroup, 16, MatBr, gl_MatrixUseB> QMat;

@@ -386,19 +384,24 @@ void main() {
                 coopMatLoad(KMat, kvsh, coord, kvsh_stride, gl_CooperativeMatrixLayoutRowMajor);
             }

-            [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) {
-                if (Q_IN_REGS) {
-                    SfMat[q] = coopMatMulAdd(KMat, Qreg[d * q_tiles + q], SfMat[q]);
-                } else {
-                    coopMatLoad(QMat, Qf, d * 16 / 4 + q * MatBr * qstride, qstride, gl_CooperativeMatrixLayoutColumnMajor);
-                    SfMat[q] = coopMatMulAdd(KMat, QMat, SfMat[q]);
+            if (Q_IN_REGS) {
+                SfMat0 = coopMatMulAdd(KMat, Qreg[d * q_tiles], SfMat0);
+                if (q_tiles > 1) {
+                    SfMat1 = coopMatMulAdd(KMat, Qreg[d * q_tiles + 1], SfMat1);
+                }
+            } else {
+                coopMatLoad(QMat, Qf, d * 16 / 4, qstride, gl_CooperativeMatrixLayoutColumnMajor);
+                SfMat0 = coopMatMulAdd(KMat, QMat, SfMat0);
+                if (q_tiles > 1) {
+                    coopMatLoad(QMat, Qf, d * 16 / 4 + MatBr * qstride, qstride, gl_CooperativeMatrixLayoutColumnMajor);
+                    SfMat1 = coopMatMulAdd(KMat, QMat, SfMat1);
                 }
             }
         }

-        [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) {
-            uint coord = gl_SubgroupID * MatBc * sfshstride + q * (MatBr / 4);
-            coopMatStore(SfMat[q], sfsh, coord, sfshstride, gl_CooperativeMatrixLayoutRowMajor);
+        coopMatStore(SfMat0, sfsh, gl_SubgroupID * MatBc * sfshstride, sfshstride, gl_CooperativeMatrixLayoutRowMajor);
+        if (q_tiles > 1) {
+            coopMatStore(SfMat1, sfsh, gl_SubgroupID * MatBc * sfshstride + MatBr / 4, sfshstride, gl_CooperativeMatrixLayoutRowMajor);
         }
         barrier();

@@ -516,10 +519,8 @@ void main() {
         [[unroll]] for (uint32_t hsv_tile = 0; hsv_tile < num_hsv_tiles; ++hsv_tile) {
             const uint hsv_offset = (hsv_tile * row_split + gl_SubgroupID) * 16;

-            coopmat<O_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator> PVMat[q_tiles];
-            [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) {
-                PVMat[q] = coopmat<O_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator>(0);
-            }
+            coopmat<O_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator> PVMat0 = coopmat<O_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator>(0);
+            coopmat<O_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator> PVMat1 = coopmat<O_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator>(0);

             // Preload V tiles for [Bc, 16 * num subgroups]
             const uint v_rows = Bc;
@@ -584,18 +585,19 @@ void main() {
                         coopMatLoad(QMat, kvsh, v_tile_offset, kvsh_stride, gl_CooperativeMatrixLayoutRowMajor);
                     }

-                    [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) {
-                        coopMatLoad(KMat, Psh, bc_chunk * MatBc * psh_stride + q * (MatBc / 4), psh_stride, gl_CooperativeMatrixLayoutColumnMajor);
-                        PVMat[q] = coopMatMulAdd(KMat, QMat, PVMat[q]);
+                    coopMatLoad(KMat, Psh, bc_chunk * MatBc * psh_stride, psh_stride, gl_CooperativeMatrixLayoutColumnMajor);
+                    PVMat0 = coopMatMulAdd(KMat, QMat, PVMat0);
+                    if (q_tiles > 1) {
+                        coopMatLoad(KMat, Psh, bc_chunk * MatBc * psh_stride + MatBc / 4, psh_stride, gl_CooperativeMatrixLayoutColumnMajor);
+                        PVMat1 = coopMatMulAdd(KMat, QMat, PVMat1);
                     }
                 }

-                [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) {
-                    if (PV_IN_SFSH) {
-                        coopMatStore(PVMat[q], sfsh, (o_offset + q * MatBc * osh_stride) / pv_per_acc, osh_stride / pv_per_acc, gl_CooperativeMatrixLayoutRowMajor);
-                    } else {
-                        coopMatStore(PVMat[q], pvsh, o_offset + q * MatBc * osh_stride, osh_stride, gl_CooperativeMatrixLayoutRowMajor);
-                    }
+                if (q_tiles > 1) {
+                    coopMatStore(PVMat0, sfsh, o_offset / pv_per_acc, osh_stride / pv_per_acc, gl_CooperativeMatrixLayoutRowMajor);
+                    coopMatStore(PVMat1, sfsh, (o_offset + MatBc * osh_stride) / pv_per_acc, osh_stride / pv_per_acc, gl_CooperativeMatrixLayoutRowMajor);
+                } else {
+                    coopMatStore(PVMat0, pvsh, o_offset, osh_stride, gl_CooperativeMatrixLayoutRowMajor);
                 }
             }