Commit 7e960f40f for llama.cpp

commit 7e960f40fb0bd2eede76406f807d6bc3f7aa339c
Author: shaofeiqi <shaoqi@qti.qualcomm.com>
Date:   Thu Nov 19 22:26:28 2026 -0800

    opencl: fold GQA head for f16 FA for better efficiency (#32631)

    Assisted-by: Codex

    Co-authored-by: Li He <lih@qti.qualcomm.com>
    Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>

diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 776a8e757..894f8235e 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -23128,8 +23128,17 @@ static void ggml_cl_fa_kq_p8_dispatch(
         CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
         CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));

-        const size_t n_tiles = (N + 31) / 32;
-        size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
+        // Pack query heads of each KV group into a short batch's tile.
+        static const bool fold_enabled = [] {
+            const char * e = getenv("GGML_OPENCL_FA_KQ_P8_FOLD");
+            return !e || e[0] != '0';
+        }();
+        const int gqa = D_B / D_A;
+        const int fold = fold_enabled && gqa > 1 && N < 32 && gqa <= INT_MAX / N ? gqa : 1;
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &fold));
+
+        const size_t n_tiles = ((size_t) N * fold + 31) / 32;
+        size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * (fold > 1 ? D_A : D_B)};

         const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
         size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl b/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl
index 2c0a56c85..9c4f71a6d 100644
--- a/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl
+++ b/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl
@@ -119,7 +119,8 @@ __kernel void mul_mm_f16_f32_kq_p8(
         int N,
         int D_A,
         int D_B,
-        int p_pitch              // bytes per P row, >= M, multiple of 32
+        int p_pitch,             // bytes per P row, >= M, multiple of 32
+        int gqa_fold
 ) {

     // Match the n-tile-first grid in ggml_cl_fa_kq_p8_dispatch.
@@ -135,7 +136,8 @@ __kernel void mul_mm_f16_f32_kq_p8(

     const uint col   = block_id_m * TILESIZE_M;
     const uint row   = block_id_n * TILESIZE_N;
-    const uint depth_A = block_id_d / (D_B/D_A);
+    const uint fold = (uint)gqa_fold;
+    const uint depth_A = fold > 1 ? block_id_d : block_id_d / (D_B/D_A);
     const uint depth_B = block_id_d;

     int line_stride_matrix_A_in_bytes = K * D_A * 2;
@@ -158,11 +160,18 @@ __kernel void mul_mm_f16_f32_kq_p8(

     __local float matrix_B_local[1024];

-    // Clamp Q loads in the last tile; columns past N are not stored.
-    const uint c0 = min(row + b_globalOffsetInWords_xy.y, (uint)N - 1);
-    const uint c1 = min(row + b_globalOffsetInWords_xy.y + 16, (uint)N - 1);
-    const uint b_globalOffsetInWords00 = c0*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
-    const uint b_globalOffsetInWords16 = c1*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
+    // Folded columns enumerate (query, head within the KV group).
+    const uint c0 = min(row + b_globalOffsetInWords_xy.y, (uint)N * fold - 1);
+    const uint c1 = min(row + b_globalOffsetInWords_xy.y + 16, (uint)N * fold - 1);
+    uint b_globalOffsetInWords00;
+    uint b_globalOffsetInWords16;
+    if (fold > 1) {
+        b_globalOffsetInWords00 = (c0/fold)*strideBinElements + (depth_B*fold + c0%fold)*K + b_globalOffsetInWords_xy.x;
+        b_globalOffsetInWords16 = (c1/fold)*strideBinElements + (depth_B*fold + c1%fold)*K + b_globalOffsetInWords_xy.x;
+    } else {
+        b_globalOffsetInWords00 = c0*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
+        b_globalOffsetInWords16 = c1*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
+    }

     for (uint step=0; step < K; step+=TILESIZE_K) {
         regA = mm_load_a(matrix_A, subMatrixAStartInElements, line_stride_matrix_A_in_bytes);
@@ -199,9 +208,10 @@ __kernel void mul_mm_f16_f32_kq_p8(
         if (lane < 32) {
             const int  j = lane >> 1;
             const int  b = lane & 1;
-            const uint n = row + half_*16 + j;
-            const uint h = depth_B;
-            if (n < (uint)N) {
+            const uint c = row + half_*16 + j;
+            const uint n = c / fold;
+            const uint h = depth_B*fold + c % fold;
+            if (c < (uint)N * fold) {
                 float slope = 1.0f;
                 if (max_bias > 0.0f) {
                     const float base = (int)h < n_head_log2 ? m0 : m1;