Commit 7e960f40f for llama.cpp
commit 7e960f40fb0bd2eede76406f807d6bc3f7aa339c
Author: shaofeiqi <shaoqi@qti.qualcomm.com>
Date: Thu Nov 19 22:26:28 2026 -0800
opencl: fold GQA head for f16 FA for better efficiency (#32631)
Assisted-by: Codex
Co-authored-by: Li He <lih@qti.qualcomm.com>
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 776a8e757..894f8235e 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -23128,8 +23128,17 @@ static void ggml_cl_fa_kq_p8_dispatch(
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
- const size_t n_tiles = (N + 31) / 32;
- size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
+ // Pack query heads of each KV group into a short batch's tile.
+ static const bool fold_enabled = [] {
+ const char * e = getenv("GGML_OPENCL_FA_KQ_P8_FOLD");
+ return !e || e[0] != '0';
+ }();
+ const int gqa = D_B / D_A;
+ const int fold = fold_enabled && gqa > 1 && N < 32 && gqa <= INT_MAX / N ? gqa : 1;
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &fold));
+
+ const size_t n_tiles = ((size_t) N * fold + 31) / 32;
+ size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * (fold > 1 ? D_A : D_B)};
const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl b/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl
index 2c0a56c85..9c4f71a6d 100644
--- a/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl
+++ b/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl
@@ -119,7 +119,8 @@ __kernel void mul_mm_f16_f32_kq_p8(
int N,
int D_A,
int D_B,
- int p_pitch // bytes per P row, >= M, multiple of 32
+ int p_pitch, // bytes per P row, >= M, multiple of 32
+ int gqa_fold
) {
// Match the n-tile-first grid in ggml_cl_fa_kq_p8_dispatch.
@@ -135,7 +136,8 @@ __kernel void mul_mm_f16_f32_kq_p8(
const uint col = block_id_m * TILESIZE_M;
const uint row = block_id_n * TILESIZE_N;
- const uint depth_A = block_id_d / (D_B/D_A);
+ const uint fold = (uint)gqa_fold;
+ const uint depth_A = fold > 1 ? block_id_d : block_id_d / (D_B/D_A);
const uint depth_B = block_id_d;
int line_stride_matrix_A_in_bytes = K * D_A * 2;
@@ -158,11 +160,18 @@ __kernel void mul_mm_f16_f32_kq_p8(
__local float matrix_B_local[1024];
- // Clamp Q loads in the last tile; columns past N are not stored.
- const uint c0 = min(row + b_globalOffsetInWords_xy.y, (uint)N - 1);
- const uint c1 = min(row + b_globalOffsetInWords_xy.y + 16, (uint)N - 1);
- const uint b_globalOffsetInWords00 = c0*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
- const uint b_globalOffsetInWords16 = c1*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
+ // Folded columns enumerate (query, head within the KV group).
+ const uint c0 = min(row + b_globalOffsetInWords_xy.y, (uint)N * fold - 1);
+ const uint c1 = min(row + b_globalOffsetInWords_xy.y + 16, (uint)N * fold - 1);
+ uint b_globalOffsetInWords00;
+ uint b_globalOffsetInWords16;
+ if (fold > 1) {
+ b_globalOffsetInWords00 = (c0/fold)*strideBinElements + (depth_B*fold + c0%fold)*K + b_globalOffsetInWords_xy.x;
+ b_globalOffsetInWords16 = (c1/fold)*strideBinElements + (depth_B*fold + c1%fold)*K + b_globalOffsetInWords_xy.x;
+ } else {
+ b_globalOffsetInWords00 = c0*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
+ b_globalOffsetInWords16 = c1*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
+ }
for (uint step=0; step < K; step+=TILESIZE_K) {
regA = mm_load_a(matrix_A, subMatrixAStartInElements, line_stride_matrix_A_in_bytes);
@@ -199,9 +208,10 @@ __kernel void mul_mm_f16_f32_kq_p8(
if (lane < 32) {
const int j = lane >> 1;
const int b = lane & 1;
- const uint n = row + half_*16 + j;
- const uint h = depth_B;
- if (n < (uint)N) {
+ const uint c = row + half_*16 + j;
+ const uint n = c / fold;
+ const uint h = depth_B*fold + c % fold;
+ if (c < (uint)N * fold) {
float slope = 1.0f;
if (max_bias > 0.0f) {
const float base = (int)h < n_head_log2 ? m0 : m1;