Commit 362ac9297 for llama.cpp
commit 362ac929749d56e433878585a8506f24968698b4
Author: Ruben Ortlam <rortlam@redhat.com>
Date: Sat Nov 21 12:46:20 2026 +0100
vulkan: unroll FA coopmat1 array due to a driver bug on AMD Windows (#32466)
* vulkan: unroll FA coopmat1 array due to a driver bug on AMD Windows
* simplify code to make it clearer
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index d4bbf9913..ae5a05c72 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -1323,7 +1323,9 @@ static vk_fa_tuning_params get_fa_tuning_params_coopmat1(const vk_device& device
}
// Br=32 spills registers, only prefill benefits
- result.q_in_regs = result.block_rows == coopmat_block_rows && device->vendor_id == VK_VENDOR_ID_AMD && n_rows >= 64;
+ // AMD proprietary driver crashes compiling the loop-indexed Qreg array
+ result.q_in_regs = result.block_rows == coopmat_block_rows && device->vendor_id == VK_VENDOR_ID_AMD && n_rows >= 64 &&
+ device->driver_id != vk::DriverId::eAmdProprietary;
// Flipped ownership: each subgroup owns 16 Q rows.
// uma, nvidia: K/V tiles staged in separate LDS buffers; discrete AMD GQA: K/V loaded directly from global memory.
@@ -1355,6 +1357,9 @@ static vk_fa_tuning_params get_fa_tuning_params_coopmat1(const vk_device& device
result.limit_occupancy_shmem = 14 * 1024 / 16;
}
+ // non-flip shader handles at most two row tiles
+ GGML_ASSERT(result.flip || result.block_rows <= 2 * coopmat_block_rows);
+
return result;
}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp
index bb9755ab8..ea945ed07 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp
@@ -25,6 +25,7 @@
// These need to be supported N,M values for a MatBc x MatBr x 16 coopmatmuladd
const uint32_t MatBr = 16;
const uint32_t MatBc = 16;
+const uint32_t q_tiles = Br / MatBr;
const uint32_t rows_per_thread = Br / row_split;
const uint32_t cols_per_iter = gl_WorkGroupSize.x / row_split;
@@ -64,15 +65,14 @@ shared FLOAT_TYPEV4 kvsh[(kvsh_stride >= vsh_stride) ? (Bc * kvsh_stride) : (Bc
shared ACC_TYPE slope[Br];
// row-tiled builds stage the PV result through sfsh
-const bool PV_IN_SFSH = Br > MatBr;
const uint32_t osh_stride = row_split * MatBr / 4;
-shared O_TYPEV4 pvsh[PV_IN_SFSH ? 2 : Br * osh_stride];
+shared O_TYPEV4 pvsh[q_tiles > 1 ? 2 : Br * osh_stride];
shared vec4 occupancy_limiter[LIMIT_OCCUPANCY_SHMEM > 0 ? LIMIT_OCCUPANCY_SHMEM : 1];
#if defined(FLOAT16) && !defined(ACC_TYPE_MAX)
const uint32_t pv_per_acc = 2;
O_TYPEV4 pvsh_load(const uint32_t i) {
- if (!PV_IN_SFSH) {
+ if (q_tiles == 1) {
return pvsh[i];
}
const uvec2 u = floatBitsToUint(vec2(sfsh[i / 2][(i % 2) * 2], sfsh[i / 2][(i % 2) * 2 + 1]));
@@ -81,7 +81,7 @@ O_TYPEV4 pvsh_load(const uint32_t i) {
#else
const uint32_t pv_per_acc = 1;
O_TYPEV4 pvsh_load(const uint32_t i) {
- if (!PV_IN_SFSH) {
+ if (q_tiles == 1) {
return pvsh[i];
}
return O_TYPEV4(sfsh[i]);
@@ -117,7 +117,6 @@ void main() {
#define tile_row(r) (row_tid * rows_per_thread + (r))
- const uint32_t q_tiles = Br / MatBr;
const uint32_t d_tiles = HSK_pad / 16;
uint32_t q_offset = gqa_iq1*p.nb01 + (iq2*p.nb02+iq3*p.nb03) / 4;
@@ -329,10 +328,9 @@ void main() {
// K * Q^T -> S^T: Bc x HSK_pad * HSK_pad x Br -> Bc x Br
// Bc split across workgroup (four subgroups), loop over HSK in chunks of 16: 16 x 16 * 16 x 16 -> 16 x 16
// This is written transposed in order to allow for N being 8 if implementations need it
- coopmat<ACC_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator> SfMat[q_tiles];
- [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) {
- SfMat[q] = coopmat<ACC_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator>(0);
- }
+ // No coopmat arrays: AMD's Windows compiler crashes on loop-indexed ones. q_tiles <= 2.
+ coopmat<ACC_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator> SfMat0 = coopmat<ACC_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator>(0);
+ coopmat<ACC_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator> SfMat1 = coopmat<ACC_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator>(0);
coopmat<FLOAT_TYPE, gl_ScopeSubgroup, MatBc, 16, gl_MatrixUseA> KMat;
coopmat<FLOAT_TYPE, gl_ScopeSubgroup, 16, MatBr, gl_MatrixUseB> QMat;
@@ -386,19 +384,24 @@ void main() {
coopMatLoad(KMat, kvsh, coord, kvsh_stride, gl_CooperativeMatrixLayoutRowMajor);
}
- [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) {
- if (Q_IN_REGS) {
- SfMat[q] = coopMatMulAdd(KMat, Qreg[d * q_tiles + q], SfMat[q]);
- } else {
- coopMatLoad(QMat, Qf, d * 16 / 4 + q * MatBr * qstride, qstride, gl_CooperativeMatrixLayoutColumnMajor);
- SfMat[q] = coopMatMulAdd(KMat, QMat, SfMat[q]);
+ if (Q_IN_REGS) {
+ SfMat0 = coopMatMulAdd(KMat, Qreg[d * q_tiles], SfMat0);
+ if (q_tiles > 1) {
+ SfMat1 = coopMatMulAdd(KMat, Qreg[d * q_tiles + 1], SfMat1);
+ }
+ } else {
+ coopMatLoad(QMat, Qf, d * 16 / 4, qstride, gl_CooperativeMatrixLayoutColumnMajor);
+ SfMat0 = coopMatMulAdd(KMat, QMat, SfMat0);
+ if (q_tiles > 1) {
+ coopMatLoad(QMat, Qf, d * 16 / 4 + MatBr * qstride, qstride, gl_CooperativeMatrixLayoutColumnMajor);
+ SfMat1 = coopMatMulAdd(KMat, QMat, SfMat1);
}
}
}
- [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) {
- uint coord = gl_SubgroupID * MatBc * sfshstride + q * (MatBr / 4);
- coopMatStore(SfMat[q], sfsh, coord, sfshstride, gl_CooperativeMatrixLayoutRowMajor);
+ coopMatStore(SfMat0, sfsh, gl_SubgroupID * MatBc * sfshstride, sfshstride, gl_CooperativeMatrixLayoutRowMajor);
+ if (q_tiles > 1) {
+ coopMatStore(SfMat1, sfsh, gl_SubgroupID * MatBc * sfshstride + MatBr / 4, sfshstride, gl_CooperativeMatrixLayoutRowMajor);
}
barrier();
@@ -516,10 +519,8 @@ void main() {
[[unroll]] for (uint32_t hsv_tile = 0; hsv_tile < num_hsv_tiles; ++hsv_tile) {
const uint hsv_offset = (hsv_tile * row_split + gl_SubgroupID) * 16;
- coopmat<O_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator> PVMat[q_tiles];
- [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) {
- PVMat[q] = coopmat<O_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator>(0);
- }
+ coopmat<O_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator> PVMat0 = coopmat<O_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator>(0);
+ coopmat<O_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator> PVMat1 = coopmat<O_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator>(0);
// Preload V tiles for [Bc, 16 * num subgroups]
const uint v_rows = Bc;
@@ -584,18 +585,19 @@ void main() {
coopMatLoad(QMat, kvsh, v_tile_offset, kvsh_stride, gl_CooperativeMatrixLayoutRowMajor);
}
- [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) {
- coopMatLoad(KMat, Psh, bc_chunk * MatBc * psh_stride + q * (MatBc / 4), psh_stride, gl_CooperativeMatrixLayoutColumnMajor);
- PVMat[q] = coopMatMulAdd(KMat, QMat, PVMat[q]);
+ coopMatLoad(KMat, Psh, bc_chunk * MatBc * psh_stride, psh_stride, gl_CooperativeMatrixLayoutColumnMajor);
+ PVMat0 = coopMatMulAdd(KMat, QMat, PVMat0);
+ if (q_tiles > 1) {
+ coopMatLoad(KMat, Psh, bc_chunk * MatBc * psh_stride + MatBc / 4, psh_stride, gl_CooperativeMatrixLayoutColumnMajor);
+ PVMat1 = coopMatMulAdd(KMat, QMat, PVMat1);
}
}
- [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) {
- if (PV_IN_SFSH) {
- coopMatStore(PVMat[q], sfsh, (o_offset + q * MatBc * osh_stride) / pv_per_acc, osh_stride / pv_per_acc, gl_CooperativeMatrixLayoutRowMajor);
- } else {
- coopMatStore(PVMat[q], pvsh, o_offset + q * MatBc * osh_stride, osh_stride, gl_CooperativeMatrixLayoutRowMajor);
- }
+ if (q_tiles > 1) {
+ coopMatStore(PVMat0, sfsh, o_offset / pv_per_acc, osh_stride / pv_per_acc, gl_CooperativeMatrixLayoutRowMajor);
+ coopMatStore(PVMat1, sfsh, (o_offset + MatBc * osh_stride) / pv_per_acc, osh_stride / pv_per_acc, gl_CooperativeMatrixLayoutRowMajor);
+ } else {
+ coopMatStore(PVMat0, pvsh, o_offset, osh_stride, gl_CooperativeMatrixLayoutRowMajor);
}
}