Commit c406f4367 for llama.cpp

commit c406f4367e19bea7930e7f74e1984c9acc6d44bf
Author: shaofeiqi <shaoqi@qti.qualcomm.com>
Date:   Thu Nov 19 11:30:19 2026 -0800

    opencl: support dp4a for KQ mm in decomposed FA (#32577)

    * opencl: support int8 KQ for DK = 512

    Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
    Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
    Assisted-by: Codex

    * opencl: allow query chunk up to 96 for int8 KQ

    Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
    Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
    Assisted-by: Codex

    * opencl: extend `build_program_from_source` to allow non-fatal failure

    Assisted-by: Codex

    * opencl: add int8 KQ for decomposed FA with DK <= 256

    Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
    Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
    Assisted-by: Codex

    ---------

    Co-authored-by: Li He <lih@qti.qualcomm.com>
    Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>

diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt
index cdf4a5d54..325f5c4bf 100644
--- a/ggml/src/ggml-opencl/CMakeLists.txt
+++ b/ggml/src/ggml-opencl/CMakeLists.txt
@@ -323,6 +323,9 @@ set(GGML_OPENCL_KERNELS
     mul_mm_f16_f32_kq_kqv
     mul_mm_f16_f32_kq_p8
     mul_mm_q8_f32_kqv
+    mul_mm_q8_kq_p8
+    mul_mm_q8_kq_p8_dk512
+    flash_attn_q8_rows
     conv2d
     conv2d_f16_f32
     conv2_dw
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 8162ace2e..776a8e757 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -781,6 +781,10 @@ struct ggml_backend_opencl_context {
     ggml_cl_buffer prealloc_fa_pd;
     ggml_cl_buffer prealloc_fa_bmax;
     ggml_cl_buffer prealloc_fa_bsum;
+    ggml_cl_buffer prealloc_fa_k8;
+    ggml_cl_buffer prealloc_fa_k8_scale;
+    ggml_cl_buffer prealloc_fa_q8;
+    ggml_cl_buffer prealloc_fa_q8_scale;

 #ifdef GGML_OPENCL_USE_ADRENO_KERNELS
     ggml_cl_buffer prealloc_adreno_xmem_const;
@@ -899,6 +903,10 @@ struct ggml_backend_opencl_context {
     cl_kernel kernel_fa_kqv_direct_f16 = nullptr;
     cl_kernel kernel_fa_kqv_direct_reduce = nullptr;
     cl_kernel kernel_fa_v_transpose_q8 = nullptr;
+    cl_kernel kernel_fa_q8_rows_f16 = nullptr;
+    cl_kernel kernel_fa_q8_rows_f32 = nullptr;
+    cl_kernel kernel_mul_mm_q8_kq_p8 = nullptr;
+    cl_kernel kernel_mul_mm_q8_kq_p8_dk512 = nullptr;
     ggml_opencl_fa_kernels fa;
 #ifdef GGML_OPENCL_USE_ADRENO_KERNELS
     ggml_cl_adreno_xmem_attn_state adreno_xmem_attn;
@@ -1506,7 +1514,18 @@ struct ggml_backend_opencl_context {
 #endif // GGML_OPENCL_USE_ADRENO_KERNELS

     ~ggml_backend_opencl_context() {
-        for (cl_kernel kernel : {kernel_mul_mm_f16_f32_kq_p8, kernel_mul_mm_q8_f32_kqv, kernel_fa_p8_fixup, kernel_fa_v_transpose_q8, kernel_fa_kqv_direct_f16, kernel_fa_kqv_direct_reduce}) {
+        for (cl_kernel kernel : {
+                kernel_mul_mm_f16_f32_kq_p8,
+                kernel_mul_mm_q8_f32_kqv,
+                kernel_fa_p8_fixup,
+                kernel_fa_v_transpose_q8,
+                kernel_fa_kqv_direct_f16,
+                kernel_fa_kqv_direct_reduce,
+                kernel_fa_q8_rows_f16,
+                kernel_fa_q8_rows_f32,
+                kernel_mul_mm_q8_kq_p8,
+                kernel_mul_mm_q8_kq_p8_dk512
+        }) {
             if (kernel != nullptr) {
                 CL_CHECK(clReleaseKernel(kernel));
             }
@@ -1648,7 +1667,7 @@ static cl_program build_program_from_source_ex(cl_context ctx, cl_device_id dev,
     return NULL;
 }

-static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts) {
+static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts, bool fatal = true) {
     cl_context   ctx = backend_ctx->context;
     cl_device_id dev = backend_ctx->device;

@@ -1660,7 +1679,7 @@ static cl_program build_program_from_source(ggml_backend_opencl_context * backen
         return p_cached;
     }

-    cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, /*fatal=*/true);
+    cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, fatal);

     // Best-effort save of the freshly-built binary (no-op if cache disabled).
     if (p != nullptr) {
@@ -3995,6 +4014,52 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
     }

 #ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+    if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+        const std::string rows_src {
+            #include "flash_attn_q8_rows.cl.h"
+        };
+        const std::string kq_src {
+            #include "mul_mm_q8_kq_p8.cl.h"
+        };
+#else
+        const std::string rows_src = read_file("flash_attn_q8_rows.cl");
+        const std::string kq_src = read_file("mul_mm_q8_kq_p8.cl");
+#endif
+        cl_program prog = build_program_from_source(backend_ctx, rows_src.c_str(), compile_opts, /*fatal=*/false);
+        if (prog != nullptr) {
+            CL_CHECK((backend_ctx->kernel_fa_q8_rows_f16 = clCreateKernel(prog, "kernel_fa_q8_rows_f16", &err), err));
+            CL_CHECK((backend_ctx->kernel_fa_q8_rows_f32 = clCreateKernel(prog, "kernel_fa_q8_rows_f32", &err), err));
+            CL_CHECK(clReleaseProgram(prog));
+        }
+        prog = build_program_from_source(backend_ctx, kq_src.c_str(), compile_opts, /*fatal=*/false);
+        if (prog != nullptr) {
+            CL_CHECK((backend_ctx->kernel_mul_mm_q8_kq_p8 = clCreateKernel(prog, "kernel_mul_mm_q8_kq_p8", &err), err));
+            CL_CHECK(clReleaseProgram(prog));
+        }
+        GGML_LOG_CONT(".");
+    }
+
+    // mul_mm_q8_kq_p8_dk512
+    if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+        const std::string kernel_src {
+            #include "mul_mm_q8_kq_p8_dk512.cl.h"
+        };
+#else
+        const std::string kernel_src = read_file("mul_mm_q8_kq_p8_dk512.cl");
+#endif
+        cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts, /*fatal=*/false);
+        if (prog != nullptr) {
+            cl_kernel kernel = clCreateKernel(prog, "kernel_mul_mm_q8_kq_p8_dk512", &err);
+            CL_CHECK(clReleaseProgram(prog));
+            if (err == CL_SUCCESS) {
+                backend_ctx->kernel_mul_mm_q8_kq_p8_dk512 = kernel;
+            }
+        }
+        GGML_LOG_CONT(".");
+    }
+
     // mul_mm_f16_f32_kq_p8
     if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -22922,12 +22987,58 @@ static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_te
 }
 #endif // GGML_OPENCL_USE_ADRENO_KERNELS

+static void ggml_cl_fa_q8_rows(
+    ggml_backend_t backend, const ggml_tensor * src, const ggml_tensor * dst,
+    cl_mem values, cl_mem scales, int64_t row0, int nrow
+) {
+    auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
+    auto * extra = (ggml_tensor_extra_cl *) (src->view_src ? src->view_src->extra : src->extra);
+    const cl_ulong offset = extra->offset + src->view_offs + row0 * src->nb[1];
+    const cl_ulong nb1 = src->nb[1];
+    const cl_ulong nb2 = src->nb[2];
+    const int dk = src->ne[0];
+    const int nhead = src->ne[2];
+    cl_kernel kernel;
+    switch (src->type) {
+        case GGML_TYPE_F16:  kernel = backend_ctx->kernel_fa_q8_rows_f16;  break;
+        case GGML_TYPE_F32:  kernel = backend_ctx->kernel_fa_q8_rows_f32;  break;
+        default: GGML_ABORT("unsupported int8 KQ input type");
+    }
+    cl_uint a = 0;
+    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem),   &extra->data_device));
+    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &offset));
+    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb1));
+    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb2));
+    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem),   &values));
+    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem),   &scales));
+    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int),      &dk));
+    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int),      &nrow));
+    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int),      &nhead));
+    const size_t blocks = (size_t) (dk / 32) * nrow * nhead;
+    size_t gws[3] = {(blocks + 63) / 64 * 64, 1, 1};
+    size_t lws[3] = {64, 1, 1};
+    backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
+}
+
+static void ggml_cl_fa_kq_int8_prepare(ggml_backend_t backend, const ggml_tensor * dst, int64_t n_q_chunk) {
+    auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
+    const ggml_tensor * q = dst->src[0];
+    const ggml_tensor * k = dst->src[1];
+    const size_t k_bytes = (size_t) k->ne[0] * k->ne[1] * k->ne[2];
+    const size_t q_bytes = (size_t) q->ne[0] * n_q_chunk * q->ne[2];
+    backend_ctx->prealloc_fa_k8.allocate(backend_ctx->context, k_bytes);
+    backend_ctx->prealloc_fa_k8_scale.allocate(backend_ctx->context, k_bytes / 32 * sizeof(cl_half));
+    backend_ctx->prealloc_fa_q8.allocate(backend_ctx->context, q_bytes);
+    backend_ctx->prealloc_fa_q8_scale.allocate(backend_ctx->context, q_bytes / 32 * sizeof(cl_half));
+    ggml_cl_fa_q8_rows(backend, k, dst, backend_ctx->prealloc_fa_k8.buffer, backend_ctx->prealloc_fa_k8_scale.buffer, 0, k->ne[1]);
+}
+
 static void ggml_cl_fa_kq_p8_dispatch(
     ggml_backend_t backend,
     const ggml_tensor * q, const ggml_tensor * k,
     const ggml_tensor * mask, const ggml_tensor * sinks,
     const ggml_tensor * dst,
-    int64_t q0, int nqc, float scale, float max_bias, int p_pitch
+    int64_t q0, int nqc, float scale, float max_bias, int p_pitch, bool kq_int8
 ) {
     auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
     auto * extra_k = (ggml_tensor_extra_cl *) (k->view_src ? k->view_src->extra : k->extra);
@@ -22940,64 +23051,94 @@ static void ggml_cl_fa_kq_p8_dispatch(
     const int D_A = k->ne[2];
     const int D_B = q->ne[2];

-    cl_int err;
-
-    cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]};
-    cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
-    CL_CHECK(err);
-
-    cl_image_format fmt = {CL_RGBA, CL_FLOAT};
-    cl_image_desc desc = {};
-    desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
-    desc.image_width = region.size / 16;
-    desc.buffer = k_sub;
-    cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err);
-    CL_CHECK(err);
-
-    region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1];
-    region.size = (size_t) K * N * D_B * sizeof(float);
-    cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
-    CL_CHECK(err);
-
     const cl_ulong off_mask = extra_m->offset + mask->view_offs + q0 * mask->nb[1];
     const cl_ulong mask_nb1 = mask->nb[1];
     const int n_head_log2 = 1u << (unsigned) floorf(log2f((float) D_B));
     const float m0 = powf(2.0f, -max_bias / n_head_log2);
     const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);

+    cl_kernel kernel;
     cl_uint a = 0;

-    cl_kernel kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8;
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
+    if (kq_int8) {
+        ggml_cl_fa_q8_rows(backend, q, dst, backend_ctx->prealloc_fa_q8.buffer, backend_ctx->prealloc_fa_q8_scale.buffer, q0, nqc);
+        kernel = K <= 256 ? backend_ctx->kernel_mul_mm_q8_kq_p8 : backend_ctx->kernel_mul_mm_q8_kq_p8_dk512;
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_k8.buffer));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_k8_scale.buffer));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_q8.buffer));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_q8_scale.buffer));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
+        size_t gws[3] = {64 * (((size_t) N + 31) / 32) * (D_B / D_A), ((size_t) M + 511) / 512, (size_t) D_A};
+        size_t lws[3] = {64, 1, 1};
+        backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
+    } else {
+        cl_int err;

-    const size_t n_tiles = (N + 31) / 32;
-    size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
+        cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]};
+        cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
+        CL_CHECK(err);

-    const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
-    size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
+        cl_image_format fmt = {CL_RGBA, CL_FLOAT};
+        cl_image_desc desc = {};
+        desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+        desc.image_width = region.size / 16;
+        desc.buffer = k_sub;
+        cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err);
+        CL_CHECK(err);

-    backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
-    CL_CHECK(clReleaseMemObject(k_img));
-    CL_CHECK(clReleaseMemObject(k_sub));
-    CL_CHECK(clReleaseMemObject(q_sub));
+        region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1];
+        region.size = (size_t) K * N * D_B * sizeof(float);
+        cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
+        CL_CHECK(err);
+
+        kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8;
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
+        CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
+
+        const size_t n_tiles = (N + 31) / 32;
+        size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
+
+        const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
+        size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
+
+        backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
+        CL_CHECK(clReleaseMemObject(k_img));
+        CL_CHECK(clReleaseMemObject(k_sub));
+        CL_CHECK(clReleaseMemObject(q_sub));
+    }

     cl_mem sinks_buf = extra_q->data_device;
     cl_ulong sinks_off = 0;
@@ -23261,7 +23402,7 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
     ggml_backend_t backend,
     const ggml_tensor * q, const ggml_tensor * k,
     ggml_tensor * dst,
-    int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct
+    int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct, bool kq_int8
 ) {
     auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
     const ggml_tensor * v = dst->src[2];
@@ -23345,13 +23486,17 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
         backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
     }

+    if (kq_int8) {
+        ggml_cl_fa_kq_int8_prepare(backend, dst, n_q_chunk);
+    }
+
     ggml_tensor_extra_cl extra_kqv;
     ggml_tensor kqv;

     for (int64_t q0 = 0; q0 < n_q; q0 += n_q_chunk) {
         const int64_t nqc = MIN(n_q_chunk, n_q - q0);

-        ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch);
+        ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch, kq_int8);

         ggml_cl_fa_decompose_scratch_tensor(kqv, extra_kqv, backend_ctx->prealloc_fa_kqv.buffer, GGML_TYPE_F32, dv, nqc, n_head);

@@ -23408,6 +23553,7 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
 // head size. GGML_OPENCL_FA_DECOMPOSE_MB sets the KQ scratch budget (default 256).
 // GGML_OPENCL_FA_KQ_P8=0 disables U8 probabilities and int8 KQV.
 // GGML_OPENCL_FA_KQV_DIRECT=0 disables direct PV for small query batches.
+// GGML_OPENCL_FA_KQ_INT8=0 keeps F16 KQ; =1 enables int8 KQ for supported shapes.
 static bool ggml_cl_flash_attn_decompose_dispatch(
     ggml_backend_t backend,
     const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst
@@ -23537,12 +23683,46 @@ static bool ggml_cl_flash_attn_decompose_dispatch(
         return false;
     }

+    static const int kq_int8_env = [] {
+        const char * e = getenv("GGML_OPENCL_FA_KQ_INT8");
+        return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1;
+    }();
+    static const int dk512_env = [] {
+        const char * e = getenv("GGML_OPENCL_FA_KQ_DK512_F16");
+        return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1;
+    }();
+    const bool dk512 = dk > 256 && dk <= 512 && n_q >= 32 &&
+        (dk512_env == 1 || (dk512_env < 0 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E));
+    const bool kq_int8_possible = kq_p8 && (dk <= 256 || dk512) && dk % 32 == 0 &&
+        backend_ctx->kernel_fa_q8_rows_f16 != nullptr && backend_ctx->kernel_fa_q8_rows_f32 != nullptr &&
+        n_head <= INT_MAX && n_head_kv <= INT_MAX &&
+        n_q <= INT_MAX / n_head / (dk / 32) && n_kv <= INT_MAX / n_head_kv / (dk / 32) &&
+        (dst->src[4] == nullptr || dk > 64) &&
+        (kq_int8_env == 1 || (kq_int8_env < 0 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && n_kv >= 2048)) &&
+        backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fa_q8_rows_f16) >= 64 &&
+        backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fa_q8_rows_f32) >= 64;
+    cl_kernel kq_int8_kernel = nullptr;
+    if (kq_int8_possible) {
+        kq_int8_kernel = dk <= 256 ? backend_ctx->kernel_mul_mm_q8_kq_p8 : backend_ctx->kernel_mul_mm_q8_kq_p8_dk512;
+    }
+    const bool kq_int8 = kq_int8_kernel != nullptr && backend_ctx->get_kernel_workgroup_size(kq_int8_kernel) >= 64;
+
+    // Fewer query tiles per KV block reduce cache pressure in the int8 KQ kernel.
+    static const int64_t kq_int8_max_chunk = [] {
+        const char * e = getenv("GGML_OPENCL_FA_KQ_INT8_MAX_CHUNK");
+        const int v = (e && e[0]) ? atoi(e) : 96;
+        return (int64_t) (v >= 32 ? (v / 32) * 32 : 96);
+    }();
+    if (kq_int8) {
+        n_q_chunk = MIN(n_q_chunk, kq_int8_max_chunk);
+    }
+
     // even out the chunks, then align to the 32-wide n-tile
     const int64_t n_chunks = (n_q + n_q_chunk - 1) / n_q_chunk;
     n_q_chunk = MIN(n_q_chunk, ((n_q + n_chunks - 1) / n_chunks + 31) & ~(int64_t) 31);

     if (kq_p8) {
-        ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct);
+        ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct, kq_int8);
     } else {
         ggml_cl_flash_attn_decompose(backend, q, k, dst, n_q_chunk, scale, max_bias);
     }
diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_q8_rows.cl b/ggml/src/ggml-opencl/kernels/flash_attn_q8_rows.cl
new file mode 100644
index 000000000..f0eda2539
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/flash_attn_q8_rows.cl
@@ -0,0 +1,77 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+// Store 32-element blocks as separate int8 values and half scales.
+
+kernel void kernel_fa_q8_rows_f16(
+        global const char * src, ulong off_src, ulong nb1, ulong nb2,
+        global char * dq, global half * dd,
+        int ne0, int nrow, int nhead
+) {
+    const int nblk = ne0 / 32;
+    const int gid  = get_global_id(0);
+    if (gid >= nblk*nrow*nhead) {
+        return;
+    }
+    const int b = gid % nblk;
+    const int r = (gid / nblk) % nrow;
+    const int h = gid / (nblk*nrow);
+
+    global const half * p = (global const half *)(src + off_src + (ulong)h*nb2 + (ulong)r*nb1) + b*32;
+    float v[32];
+    float amax = 0.0f;
+    #pragma unroll
+    for (int i = 0; i < 8; ++i) {
+        const float4 x = convert_float4(vload4(i, p));
+        v[i*4+0] = x.s0;
+        v[i*4+1] = x.s1;
+        v[i*4+2] = x.s2;
+        v[i*4+3] = x.s3;
+        amax = fmax(amax, fmax(fmax(fabs(x.s0), fabs(x.s1)), fmax(fabs(x.s2), fabs(x.s3))));
+    }
+    const float d  = amax / 127.0f;
+    const float id = amax > 0.0f ? 127.0f / amax : 0.0f;
+    const size_t row = (size_t)h*nrow + r;
+    dd[row*nblk + b] = (half)d;
+    global char * q = dq + row*ne0 + b*32;
+    #pragma unroll
+    for (int i = 0; i < 32; ++i) {
+        q[i] = (char)clamp((int)rint(v[i]*id), -127, 127);
+    }
+}
+
+kernel void kernel_fa_q8_rows_f32(
+        global const char * src, ulong off_src, ulong nb1, ulong nb2,
+        global char * dq, global half * dd,
+        int ne0, int nrow, int nhead
+) {
+    const int nblk = ne0 / 32;
+    const int gid  = get_global_id(0);
+    if (gid >= nblk*nrow*nhead) {
+        return;
+    }
+    const int b = gid % nblk;
+    const int r = (gid / nblk) % nrow;
+    const int h = gid / (nblk*nrow);
+
+    global const float * p = (global const float *)(src + off_src + (ulong)h*nb2 + (ulong)r*nb1) + b*32;
+    float v[32];
+    float amax = 0.0f;
+    #pragma unroll
+    for (int i = 0; i < 8; ++i) {
+        const float4 x = vload4(i, p);
+        v[i*4+0] = x.s0;
+        v[i*4+1] = x.s1;
+        v[i*4+2] = x.s2;
+        v[i*4+3] = x.s3;
+        amax = fmax(amax, fmax(fmax(fabs(x.s0), fabs(x.s1)), fmax(fabs(x.s2), fabs(x.s3))));
+    }
+    const float d  = amax / 127.0f;
+    const float id = amax > 0.0f ? 127.0f / amax : 0.0f;
+    const size_t row = (size_t)h*nrow + r;
+    dd[row*nblk + b] = (half)d;
+    global char * q = dq + row*ne0 + b*32;
+    #pragma unroll
+    for (int i = 0; i < 32; ++i) {
+        q[i] = (char)clamp((int)rint(v[i]*id), -127, 127);
+    }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q8_kq_p8.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q8_kq_p8.cl
new file mode 100644
index 000000000..8e04ad393
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mm_q8_kq_p8.cl
@@ -0,0 +1,174 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define KQ_TN 32
+#define KQ_MB 8
+#define KQ_TM 64
+#define KQ_WG 64
+#define KQ_DK_MAX 256
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_mul_mm_q8_kq_p8(
+        global const uint  * kq,        // K int8, [head_kv][n_kv][dk]   as packed uints
+        global const half  * kd,        // K scales, [head_kv][n_kv][dk/32]
+        global const uint  * qq,        // Q int8, [head][n_q][dk]       as packed uints
+        global const half  * qd,        // Q scales, [head][n_q][dk/32]
+        global       uchar * qp,        // u8 P, [head][n_q][p_pitch]
+        global       float * bmax,      // per 32-row block max,          [head][n_q][p_pitch/32]
+        global       float * bsum,      // per block sum of exp(s - bmax), [head][n_q][p_pitch/32]
+        global const char  * mask,      // f16 [>=n_kv][n_q] view, row stride mask_nb1 bytes
+        ulong                offset_mask,
+        ulong                mask_nb1,
+        float                scale,
+        float                max_bias,
+        float                m0,
+        float                m1,
+        int                  n_head_log2,
+        int                  dk,
+        int                  n_kv,
+        int                  n_q,
+        int                  n_head,
+        int                  n_head_kv,
+        int                  p_pitch     // bytes per P row, >= n_kv, multiple of 32
+) {
+    const int lid  = get_local_id(0);
+    const int gsz  = n_head / n_head_kv;
+    const int g0   = get_group_id(0);
+    const int head_kv = get_group_id(2);
+    const int head = head_kv*gsz + (g0 % gsz);
+    const int qn0  = (g0 / gsz) * KQ_TN;
+    const int kv0  = get_group_id(1)*KQ_TM*KQ_MB + lid;
+
+    const int nblk = dk / 32;
+    const int nu   = dk / 4;
+
+    __local uint  sh_q [KQ_TN][KQ_DK_MAX/4];
+    __local half  sh_qd[KQ_TN][KQ_DK_MAX/32];
+    __local float p8_lds[1024];
+
+    const size_t qbase = (size_t)head*n_q;
+    for (int i = lid; i < KQ_TN*(KQ_DK_MAX/16); i += KQ_WG) {
+        const int t = i / (KQ_DK_MAX/16);
+        const int u = (i % (KQ_DK_MAX/16)) * 4;
+        const int qi = qn0 + t;
+        uint4 v = (uint4)(0u);
+        if (qi < n_q && u < nu) {
+            v = vload4(0, &qq[(qbase + qi)*nu + u]);
+        }
+        vstore4(v, 0, &sh_q[t][u]);
+    }
+    for (int i = lid; i < KQ_TN*(KQ_DK_MAX/32); i += KQ_WG) {
+        const int t = i / (KQ_DK_MAX/32);
+        const int b = i % (KQ_DK_MAX/32);
+        const int qi = qn0 + t;
+        sh_qd[t][b] = (qi < n_q && b < nblk) ? qd[(qbase + qi)*nblk + b] : (half)0.0f;
+    }
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    float slope = 1.0f;
+    if (max_bias > 0.0f) {
+        const float base = head < n_head_log2 ? m0 : m1;
+        const int   ex   = head < n_head_log2 ? head + 1 : 2*(head - n_head_log2) + 1;
+        slope = pow(base, ex);
+    }
+    global const half * mrow = (global const half *)(mask + offset_mask);
+    const uint mstride = (uint)(mask_nb1 / 2);
+    const int  pblk    = p_pitch / 32;
+
+    for (int m = 0; m < KQ_MB; ++m) {
+        const int kv = kv0 + m*KQ_TM;
+        if (kv - lid >= n_kv) {
+            break; // uniform across the workgroup
+        }
+
+        float acc[KQ_TN];
+        #pragma unroll
+        for (int t = 0; t < KQ_TN; ++t) {
+            acc[t] = 0.0f;
+        }
+
+        const int    kl    = min(kv, n_kv - 1);
+        const size_t kbase = ((size_t)head_kv*n_kv + kl)*nu;
+        const size_t kdbas = ((size_t)head_kv*n_kv + kl)*nblk;
+
+        for (int b = 0; b < nblk; ++b) {
+            const uint4 w0 = vload4(0, &kq[kbase + b*8]);
+            const uint4 w1 = vload4(0, &kq[kbase + b*8 + 4]);
+            const float dks = (float)kd[kdbas + b];
+
+            #pragma unroll
+            for (int t = 0; t < KQ_TN; ++t) {
+                const uint4 a0 = vload4(0, &sh_q[t][b*8]);
+                const uint4 a1 = vload4(0, &sh_q[t][b*8 + 4]);
+                int raw = 0;
+                raw = dot_acc_sat_4x8packed_ss_int(w0.s0, a0.s0, raw);
+                raw = dot_acc_sat_4x8packed_ss_int(w0.s1, a0.s1, raw);
+                raw = dot_acc_sat_4x8packed_ss_int(w0.s2, a0.s2, raw);
+                raw = dot_acc_sat_4x8packed_ss_int(w0.s3, a0.s3, raw);
+                raw = dot_acc_sat_4x8packed_ss_int(w1.s0, a1.s0, raw);
+                raw = dot_acc_sat_4x8packed_ss_int(w1.s1, a1.s1, raw);
+                raw = dot_acc_sat_4x8packed_ss_int(w1.s2, a1.s2, raw);
+                raw = dot_acc_sat_4x8packed_ss_int(w1.s3, a1.s3, raw);
+                acc[t] += dks * (float)sh_qd[t][b] * (float)raw;
+            }
+        }
+
+        // Each lane owns one complete 32-row softmax block after the transpose.
+        const int kvblk0 = kv - lid;
+        #pragma unroll
+        for (int half_ = 0; half_ < 2; ++half_) {
+            barrier(CLK_LOCAL_MEM_FENCE);
+            #pragma unroll
+            for (int j = 0; j < 16; ++j) {
+                p8_lds[j*64 + lid] = acc[half_*16 + j];
+            }
+            barrier(CLK_LOCAL_MEM_FENCE);
+            if (lid < 32) {
+                const int j = lid >> 1;
+                const int b = lid & 1;
+                const int n = qn0 + half_*16 + j;
+                if (n < n_q) {
+                    const int kvb = kvblk0 + b*32;
+                    const __local float * src = p8_lds + j*64 + b*32;
+                    float16 s0 = vload16(0, src);
+                    float16 s1 = vload16(1, src);
+                    const global half * mp = mrow + (size_t)n*mstride + kvb;
+                    const float16 m0v = convert_float16(vload16(0, mp));
+                    const float16 m1v = convert_float16(vload16(1, mp));
+                    s0 = s0*scale + slope*m0v;
+                    s1 = s1*scale + slope*m1v;
+                    const float16 mx16 = fmax(s0, s1);
+                    const float8  mx8  = fmax(mx16.lo, mx16.hi);
+                    const float4  mx4  = fmax(mx8.lo, mx8.hi);
+                    const float2  mx2  = fmax(mx4.lo, mx4.hi);
+                    const float   amax = fmax(mx2.x, mx2.y);
+                    const size_t prow = qbase + n;
+                    const int    blk  = kvb >> 5;
+                    float sum = 0.0f;
+                    uchar16 q0 = (uchar16)(0);
+                    uchar16 q1 = (uchar16)(0);
+                    // finite sentinel: the program builds with -cl-finite-math-only
+                    if (amax > -1.0e30f) {
+                        const float16 e0 = exp(s0 - amax);
+                        const float16 e1 = exp(s1 - amax);
+                        const float16 a16 = e0 + e1;
+                        const float8  a8  = a16.lo + a16.hi;
+                        const float4  a4  = a8.lo + a8.hi;
+                        const float2  a2  = a4.lo + a4.hi;
+                        sum = a2.x + a2.y;
+                        q0 = convert_uchar16_sat_rte(e0*255.0f);
+                        q1 = convert_uchar16_sat_rte(e1*255.0f);
+                    }
+                    global uchar * qdst = qp + prow*(size_t)p_pitch + (size_t)kvb;
+                    vstore16(q0, 0, qdst);
+                    vstore16(q1, 1, qdst);
+                    bmax[prow*pblk + blk] = amax;
+                    bsum[prow*pblk + blk] = sum;
+                }
+            }
+        }
+    }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q8_kq_p8_dk512.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q8_kq_p8_dk512.cl
new file mode 100644
index 000000000..f6ba2b6c2
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mm_q8_kq_p8_dk512.cl
@@ -0,0 +1,170 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define KQ_TN 32
+#define KQ_MB 8
+#define KQ_TM 64
+#define KQ_WG 64
+#define KQ_DKC 256
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_mul_mm_q8_kq_p8_dk512(
+        global const uint  * kq,        // K int8, [head_kv][n_kv][dk]   as packed uints
+        global const half  * kd,        // K scales, [head_kv][n_kv][dk/32]
+        global const uint  * qq,        // Q int8, [head][n_q][dk]       as packed uints
+        global const half  * qd,        // Q scales, [head][n_q][dk/32]
+        global       uchar * qp,        // u8 P, [head][n_q][p_pitch]
+        global       float * bmax,      // per 32-row block max,          [head][n_q][p_pitch/32]
+        global       float * bsum,      // per block sum of exp(s - bmax), [head][n_q][p_pitch/32]
+        global const char  * mask,      // f16 [>=n_kv][n_q] view, row stride mask_nb1 bytes
+        ulong                offset_mask,
+        ulong                mask_nb1,
+        float                scale,
+        float                max_bias,
+        float                m0,
+        float                m1,
+        int                  n_head_log2,
+        int                  dk,
+        int                  n_kv,
+        int                  n_q,
+        int                  n_head,
+        int                  n_head_kv,
+        int                  p_pitch     // bytes per P row, >= n_kv, multiple of 32
+) {
+    const int lid  = get_local_id(0);
+    const int gsz  = n_head / n_head_kv;
+    const int g0   = get_group_id(0);
+    const int head_kv = get_group_id(2);
+    const int head = head_kv*gsz + (g0 % gsz);
+    const int qn0  = (g0 / gsz) * KQ_TN;
+    const int kv0  = get_group_id(1)*KQ_TM*KQ_MB + lid;
+
+    const int nblk = dk / 32;
+    const int nu   = dk / 4;
+
+    // Reuse the 256-wide Q slice for the F32 softmax scores after the contraction.
+    __local uint  sh_q [KQ_TN*KQ_DKC/4];
+    __local half  sh_qd[KQ_TN][KQ_DKC/32];
+    __local float * p8_lds = (__local float *)sh_q;
+
+    const size_t qbase = (size_t)head*n_q;
+
+    float slope = 1.0f;
+    if (max_bias > 0.0f) {
+        const float base = head < n_head_log2 ? m0 : m1;
+        const int   ex   = head < n_head_log2 ? head + 1 : 2*(head - n_head_log2) + 1;
+        slope = pow(base, ex);
+    }
+    global const half * mrow = (global const half *)(mask + offset_mask);
+    const uint mstride = (uint)(mask_nb1 / 2);
+    const int  pblk    = p_pitch / 32;
+
+    for (int m = 0; m < KQ_MB; ++m) {
+        const int kv = kv0 + m*KQ_TM;
+        if (kv - lid >= n_kv) {
+            break; // uniform across the workgroup
+        }
+
+        float acc[KQ_TN];
+        #pragma unroll
+        for (int t = 0; t < KQ_TN; ++t) {
+            acc[t] = 0.0f;
+        }
+
+        const int    kl    = min(kv, n_kv - 1);
+        const size_t kbase = ((size_t)head_kv*n_kv + kl)*nu;
+        const size_t kdbas = ((size_t)head_kv*n_kv + kl)*nblk;
+
+        for (int c0 = 0; c0 < nblk; c0 += KQ_DKC/32) {
+            const int cb = min(KQ_DKC/32, nblk - c0);
+            barrier(CLK_LOCAL_MEM_FENCE);
+            for (int i = lid; i < KQ_TN*(KQ_DKC/16); i += KQ_WG) {
+                const int t  = i / (KQ_DKC/16);
+                const int u  = (i % (KQ_DKC/16)) * 4;
+                const int qi = qn0 + t;
+                uint4 v = (uint4)(0u);
+                if (qi < n_q && u < cb*8) {
+                    v = vload4(0, &qq[(qbase + qi)*nu + c0*8 + u]);
+                }
+                vstore4(v, 0, &sh_q[t*(KQ_DKC/4) + u]);
+            }
+            for (int i = lid; i < KQ_TN*(KQ_DKC/32); i += KQ_WG) {
+                const int t  = i / (KQ_DKC/32);
+                const int b  = i % (KQ_DKC/32);
+                const int qi = qn0 + t;
+                sh_qd[t][b] = (qi < n_q && b < cb) ? qd[(qbase + qi)*nblk + c0 + b] : (half)0.0f;
+            }
+            barrier(CLK_LOCAL_MEM_FENCE);
+
+            for (int b = 0; b < cb; ++b) {
+                const uint4 w0 = vload4(0, &kq[kbase + (c0 + b)*8]);
+                const uint4 w1 = vload4(0, &kq[kbase + (c0 + b)*8 + 4]);
+                const float dks = (float)kd[kdbas + c0 + b];
+
+                #pragma unroll
+                for (int t = 0; t < KQ_TN; ++t) {
+                    const uint4 a0 = vload4(0, &sh_q[t*(KQ_DKC/4) + b*8]);
+                    const uint4 a1 = vload4(0, &sh_q[t*(KQ_DKC/4) + b*8 + 4]);
+                    int raw = 0;
+                    raw = dot_acc_sat_4x8packed_ss_int(w0.s0, a0.s0, raw);
+                    raw = dot_acc_sat_4x8packed_ss_int(w0.s1, a0.s1, raw);
+                    raw = dot_acc_sat_4x8packed_ss_int(w0.s2, a0.s2, raw);
+                    raw = dot_acc_sat_4x8packed_ss_int(w0.s3, a0.s3, raw);
+                    raw = dot_acc_sat_4x8packed_ss_int(w1.s0, a1.s0, raw);
+                    raw = dot_acc_sat_4x8packed_ss_int(w1.s1, a1.s1, raw);
+                    raw = dot_acc_sat_4x8packed_ss_int(w1.s2, a1.s2, raw);
+                    raw = dot_acc_sat_4x8packed_ss_int(w1.s3, a1.s3, raw);
+                    acc[t] += dks * (float)sh_qd[t][b] * (float)raw;
+                }
+            }
+        }
+
+        // Finish reading the Q slice before writing scores to the same local memory.
+        const int kvblk0 = kv - lid;
+        barrier(CLK_LOCAL_MEM_FENCE);
+        #pragma unroll
+        for (int t = 0; t < KQ_TN; ++t) {
+            p8_lds[t*64 + lid] = acc[t];
+        }
+        barrier(CLK_LOCAL_MEM_FENCE);
+        {
+            const int j = lid >> 1;
+            const int b = lid & 1;
+            const int n = qn0 + j;
+            if (n < n_q) {
+                const int kvb = kvblk0 + b*32;
+                __local float * src = p8_lds + j*64 + b*32;
+                const global half * mp = mrow + (size_t)n*mstride + kvb;
+                float4 s = vload4(0, src)*scale + slope*convert_float4(vload4(0, mp));
+                vstore4(s, 0, src);
+                float amax = fmax(fmax(s.x, s.y), fmax(s.z, s.w));
+                for (int c = 1; c < 8; ++c) {
+                    s = vload4(c, src)*scale + slope*convert_float4(vload4(c, mp));
+                    vstore4(s, c, src);
+                    amax = fmax(amax, fmax(fmax(s.x, s.y), fmax(s.z, s.w)));
+                }
+                const size_t prow = qbase + n;
+                const int    blk  = kvb >> 5;
+                global uchar * qdst = qp + prow*(size_t)p_pitch + (size_t)kvb;
+                float sum = 0.0f;
+                // finite sentinel: the program builds with -cl-finite-math-only
+                if (amax > -1.0e30f) {
+                    for (int c = 0; c < 8; ++c) {
+                        const float4 e = exp(vload4(c, src) - amax);
+                        sum += (e.x + e.y) + (e.z + e.w);
+                        vstore4(convert_uchar4_sat_rte(e*255.0f), c, qdst);
+                    }
+                } else {
+                    for (int c = 0; c < 8; ++c) {
+                        vstore4((uchar4)(0), c, qdst);
+                    }
+                }
+                bmax[prow*pblk + blk] = amax;
+                bsum[prow*pblk + blk] = sum;
+            }
+        }
+    }
+}