Commit 08c252e22 for llama.cpp
commit 08c252e221759e0f69e80f0450b295c92d2344e1
Author: lhez <lih@qti.qualcomm.com>
Date: Wed Nov 18 20:11:46 2026 -0800
opencl: allow adjusting KV padding in decomposed FA (#32522)
Assisted-by: Codex
diff --git a/docs/backend/OPENCL.md b/docs/backend/OPENCL.md
index acaaedfbf..0a9a324fd 100644
--- a/docs/backend/OPENCL.md
+++ b/docs/backend/OPENCL.md
@@ -100,6 +100,12 @@ The OpenCL backend has the following CMake options that control the behavior of
| `GGML_OPENCL_USE_ADRENO_KERNELS` | `ON` | Use kernels optimized for Adreno. |
| `GGML_OPENCL_USE_ADRENO_BIN_KERNELS` | `OFF` | Allow using binary kernel lib for Adreno. |
+## Decomposed Flash Attention
+
+The P8 path rounds scratch row pitches for P and transposed V up to a multiple of 32 KiB before adding the existing 64-byte padding. This reduces cache conflicts when the KV length exceeds 16384 and the V head size is at most 256. Larger V heads keep the previous layout. The extra scratch space is less than 32 KiB per row at the default granule.
+
+`GGML_OPENCL_FA_KV_PITCH_ROUND` sets the rounding granule in bytes to a positive power of two; `0` restores the previous layout. `GGML_OPENCL_FA_KV_PITCH_ROUND_MIN` sets the KV-length threshold (default `16384`), and `GGML_OPENCL_FA_KV_PITCH_ROUND_MAX_DV` sets the largest V head size to round (default `256`). These settings do not change the existing attention dispatch conditions.
+
## Program Binary Cache
Compiled `cl_program` binaries are cached on disk, so subsequent runs skip the expensive
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 3a15efabf..0d9212232 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -23277,8 +23277,31 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
const int64_t kqv_m = pad_up(dv, 64);
const int64_t tile_n = pad_up(n_q_chunk, 32);
- // Pad the int8 row pitch by one cache line, as in the P8 KQV kernel.
- const int kv_pitch = (int) n_kv + 64;
+ static const int64_t kv_pitch_round = [] {
+ const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND");
+ const int64_t v = (e && e[0]) ? strtoll(e, nullptr, 10) : 32768;
+ return v > 0 && v <= INT_MAX && (v & (v - 1)) == 0 ? v : (int64_t) 0;
+ }();
+ static const int64_t kv_pitch_round_min = [] {
+ const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND_MIN");
+ return (e && e[0]) ? strtoll(e, nullptr, 10) : (int64_t) 16384;
+ }();
+ static const int64_t kv_pitch_round_max_dv = [] {
+ const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND_MAX_DV");
+ return (e && e[0]) ? strtoll(e, nullptr, 10) : (int64_t) 256;
+ }();
+
+ // Round long P and V^T rows to reduce cache conflicts, then add one cache line.
+ int64_t kv_pitch_value = n_kv + 64;
+ if (kv_pitch_round > 0 && n_kv > kv_pitch_round_min && dv <= kv_pitch_round_max_dv) {
+ const int64_t rounded_pitch = pad_up(n_kv, kv_pitch_round) + 64;
+ if (rounded_pitch <= INT_MAX &&
+ (size_t) rounded_pitch <= SIZE_MAX / n_q_chunk / n_head &&
+ (size_t) rounded_pitch <= SIZE_MAX / dv / n_head_kv) {
+ kv_pitch_value = rounded_pitch;
+ }
+ }
+ const int kv_pitch = (int) kv_pitch_value;
const size_t nblk = (size_t) kv_pitch / 32;
backend_ctx->prealloc_fa_pq.allocate(backend_ctx->context, (size_t) kv_pitch * n_q_chunk * n_head);