Commit 054a1babf for llama.cpp
commit 054a1babf3f57074b0f63f415fc34bdf13435a64
Author: Gaurav Garg <gaugarg@nvidia.com>
Date: Tue Nov 17 16:49:28 2026 +0530
Rotate K/V read order for better L2 cache hit rates (#32407)
diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh
index 2ae59969d..dea8d2ad3 100644
--- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh
+++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh
@@ -639,6 +639,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
float * const __restrict__ KQ_rowsum,
const int jt,
const int kb0,
+ const int kb0_next,
const int k_VKQ_sup) {
#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
@@ -1030,10 +1031,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
if (!last_iter) {
if (ncols2 > 1 || mask_h) {
flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
- (mask_h, tile_mask, stride_mask, k_VKQ_0 + nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);
+ (mask_h, tile_mask, stride_mask, kb0_next*nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);
}
flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
- (K_h2, tile_K, min(nbatch_K2, DKQ/2), stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, indices);
+ (K_h2, tile_K, min(nbatch_K2, DKQ/2), stride_K, kb0_next*nbatch_fa, k_VKQ_sup, indices);
}
}
@@ -1108,7 +1109,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
scale, slope, logit_softcap, ne01, ne02,
stride_K, stride_V, stride_mask,
tile_Q, tile_K, tile_V, tile_mask,
- Q_B, VKQ_C, KQ_max, KQ_rowsum, kb0);
+ Q_B, VKQ_C, KQ_max, KQ_rowsum, kb0, kb0_next);
NO_DEVICE_CODE;
#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
}
@@ -1285,7 +1286,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
const int jt,
const int zt_gqa,
const int kb0_start,
- const int kb0_stop) {
+ const int kb0_stop,
+ const int kb0_max) {
#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
//In this kernel Q, K, V are matrices while i, j, k are matrix indices.
@@ -1438,7 +1440,13 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
__syncthreads();
}
- int kb0 = kb0_start;
+ // With multi-stage loading the K/V block order is rotated, see flash_attn_ext_f16.
+ // Then K/V blocks >= iter_k wrap around to 0 and the fully masked K/V blocks >= kb0_max are skipped.
+ const int iter_k = (ne11 + nbatch_fa - 1) / nbatch_fa;
+ const int nkb_pre = max(0, min(kb0_stop, kb0_max) - kb0_start);
+ const int nkb_post = max(0, min(kb0_stop - iter_k, kb0_max));
+ const int nkb = max(1, nkb_pre + nkb_post);
+ int kb0 = nstages > 1 && nkb_pre == 0 && nkb_post > 0 ? 0 : kb0_start;
// Preload mask and K data for first iteration when using cp_async with multiple stages:
if constexpr (nstages > 1) {
@@ -1456,7 +1464,29 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
}
// kb0_start is always < kb0_stop so the last iter can be executed unconditionally.
- if constexpr (ncols2 == 1 || use_sparse) {
+ if constexpr (nstages > 1) {
+ constexpr bool oob_check = false;
+ for (int ikb = 1; ikb < nkb; ++ikb) {
+ const int kb0_next = kb0 + 1 == kb0_max ? 0 : kb0 + 1;
+ constexpr bool last_iter = false;
+ constexpr int k_VKQ_sup = nbatch_fa;
+ flash_attn_ext_f16_iter
+ <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,
+ T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
+ (Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
+ ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
+ KQ_max, KQ_rowsum, jt, kb0, kb0_next, k_VKQ_sup);
+ kb0 = kb0_next;
+ }
+ constexpr bool last_iter = true;
+ constexpr int k_VKQ_sup = nbatch_fa;
+ flash_attn_ext_f16_iter
+ <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,
+ T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
+ (Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
+ ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
+ KQ_max, KQ_rowsum, jt, kb0, kb0, k_VKQ_sup);
+ } else if constexpr (ncols2 == 1 || use_sparse) {
constexpr bool oob_check = true;
for (; kb0 < kb0_stop-1; ++kb0) {
constexpr bool last_iter = false;
@@ -1466,7 +1496,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
- KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
+ KQ_max, KQ_rowsum, jt, kb0, kb0 + 1, k_VKQ_sup);
}
constexpr bool last_iter = true;
const int k_VKQ_sup = ne11 - kb0*nbatch_fa;
@@ -1475,7 +1505,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
- KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
+ KQ_max, KQ_rowsum, jt, kb0, kb0, k_VKQ_sup);
} else {
constexpr bool oob_check = false;
for (; kb0 < kb0_stop-1; ++kb0) {
@@ -1486,7 +1516,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
- KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
+ KQ_max, KQ_rowsum, jt, kb0, kb0 + 1, k_VKQ_sup);
}
constexpr bool last_iter = true;
constexpr int k_VKQ_sup = nbatch_fa;
@@ -1495,7 +1525,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
- KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
+ KQ_max, KQ_rowsum, jt, kb0, kb0, k_VKQ_sup);
}
// With multi-stage loading there is no __syncthreads at the end of the iter,
@@ -1893,7 +1923,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dstk_fixup,
scale, slope, logit_softcap, ne01, ne02, gqa_ratio,
stride_Q1, stride_Q2, stride_K, stride_V, stride_mask,
- jt, kb0_start, kb0_stop);
+ jt, kb0_start, kb0_stop, kb0_max);
NO_DEVICE_CODE;
#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
}
@@ -2021,6 +2051,18 @@ static __global__ void flash_attn_ext_f16(
int kb0_start = kbc % iter_k;
int kb0_stop = min(iter_k, kb0_start + kbc_stop - kbc);
+ // Rotate the K/V block order: kbc uses K/V block (kbc - kbc_tile) % iter_k instead of kbc % iter_k, kbc_tile is the start of the CUDA block that started the tile.
+ // Then concurrent CUDA blocks use the same K/V blocks and can share them in L2. Only with multi-stage loading, else they all wait for the same K/V data.
+ constexpr bool rotate = ggml_cuda_fattn_mma_get_nstages(DKQ, DV, ncols1, ncols2, use_sparse) > 1;
+ const int kbc0 = kbc;
+ int kbc_tile = kbc;
+ if (rotate && kb0_start != 0) {
+ // Get the CUDA block that started the tile, inverse of the kbc formula above:
+ const int64_t nkbc = int64_t(iter_k)*iter_j*iter_z_gqa*ne12*ne03;
+ const int64_t b_first = ((kbc - kb0_start + 1)*int64_t(gridDim.x) - 1) / nkbc;
+ kbc_tile = b_first*nkbc / gridDim.x;
+ }
+
while (kbc < kbc_stop && kb0_stop == iter_k) {
// z_KV == K/V head index, zt_gqa = Q head start index per K/V head, jt = token position start index
const int sequence = kbc /(iter_k*iter_j*iter_z_gqa*ne12);
@@ -2043,26 +2085,30 @@ static __global__ void flash_attn_ext_f16(
const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;
+ // With rotation kb0_stop is not cut, process_tile skips the fully masked K/V blocks >= kb0_max.
if (use_sparse) {
kb0_stop = min(kb0_stop, (KV_max[(sequence % ne33)*iter_j + jt] + nbatch_fa - 1) / nbatch_fa);
- } else if (KV_max) {
+ } else if (KV_max && !rotate) {
kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa);
}
+ const int kb0_max = rotate && KV_max ? KV_max[sequence*iter_j + jt] / nbatch_fa : iter_k;
+ const int kb0_rot = rotate ? (kbc - kbc_tile) % iter_k : kb0_start;
constexpr bool is_fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.
if (kb0_start == 0) {
constexpr bool needs_fixup = false; // CUDA block is working on an entire tile.
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
- ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
+ ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_rot, kb0_rot + kb0_stop - kb0_start, kb0_max);
} else {
constexpr bool needs_fixup = true; // CUDA block is missing the beginning of a tile.
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
- ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
+ ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_rot, kb0_rot + kb0_stop - kb0_start, kb0_max);
}
kbc += iter_k;
kbc -= kbc % iter_k;
+ kbc_tile = kbc0;
kb0_start = 0;
kb0_stop = min(iter_k, kbc_stop - kbc);
@@ -2095,15 +2141,17 @@ static __global__ void flash_attn_ext_f16(
if (use_sparse) {
kb0_stop = min(kb0_stop, (KV_max[(sequence % ne33)*iter_j + jt] + nbatch_fa - 1) / nbatch_fa);
- } else if (KV_max) {
+ } else if (KV_max && !rotate) {
kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa);
}
+ const int kb0_max = rotate && KV_max ? KV_max[sequence*iter_j + jt] / nbatch_fa : iter_k;
+ const int kb0_rot = rotate ? (kbc - kbc_tile) % iter_k : kb0_start;
constexpr bool is_fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.
constexpr bool needs_fixup = false;
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
- ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
+ ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_rot, kb0_rot + kb0_stop - kb0_start, kb0_max);
#else
GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, KV_rows_ptr, dst_ptr, dst_meta_ptr, scale,
max_bias, m0, m1, n_head_log2, logit_softcap,