Commit c406f4367 for llama.cpp
commit c406f4367e19bea7930e7f74e1984c9acc6d44bf
Author: shaofeiqi <shaoqi@qti.qualcomm.com>
Date: Thu Nov 19 11:30:19 2026 -0800
opencl: support dp4a for KQ mm in decomposed FA (#32577)
* opencl: support int8 KQ for DK = 512
Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
Assisted-by: Codex
* opencl: allow query chunk up to 96 for int8 KQ
Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
Assisted-by: Codex
* opencl: extend `build_program_from_source` to allow non-fatal failure
Assisted-by: Codex
* opencl: add int8 KQ for decomposed FA with DK <= 256
Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
Assisted-by: Codex
---------
Co-authored-by: Li He <lih@qti.qualcomm.com>
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt
index cdf4a5d54..325f5c4bf 100644
--- a/ggml/src/ggml-opencl/CMakeLists.txt
+++ b/ggml/src/ggml-opencl/CMakeLists.txt
@@ -323,6 +323,9 @@ set(GGML_OPENCL_KERNELS
mul_mm_f16_f32_kq_kqv
mul_mm_f16_f32_kq_p8
mul_mm_q8_f32_kqv
+ mul_mm_q8_kq_p8
+ mul_mm_q8_kq_p8_dk512
+ flash_attn_q8_rows
conv2d
conv2d_f16_f32
conv2_dw
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 8162ace2e..776a8e757 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -781,6 +781,10 @@ struct ggml_backend_opencl_context {
ggml_cl_buffer prealloc_fa_pd;
ggml_cl_buffer prealloc_fa_bmax;
ggml_cl_buffer prealloc_fa_bsum;
+ ggml_cl_buffer prealloc_fa_k8;
+ ggml_cl_buffer prealloc_fa_k8_scale;
+ ggml_cl_buffer prealloc_fa_q8;
+ ggml_cl_buffer prealloc_fa_q8_scale;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
ggml_cl_buffer prealloc_adreno_xmem_const;
@@ -899,6 +903,10 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_fa_kqv_direct_f16 = nullptr;
cl_kernel kernel_fa_kqv_direct_reduce = nullptr;
cl_kernel kernel_fa_v_transpose_q8 = nullptr;
+ cl_kernel kernel_fa_q8_rows_f16 = nullptr;
+ cl_kernel kernel_fa_q8_rows_f32 = nullptr;
+ cl_kernel kernel_mul_mm_q8_kq_p8 = nullptr;
+ cl_kernel kernel_mul_mm_q8_kq_p8_dk512 = nullptr;
ggml_opencl_fa_kernels fa;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
ggml_cl_adreno_xmem_attn_state adreno_xmem_attn;
@@ -1506,7 +1514,18 @@ struct ggml_backend_opencl_context {
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
~ggml_backend_opencl_context() {
- for (cl_kernel kernel : {kernel_mul_mm_f16_f32_kq_p8, kernel_mul_mm_q8_f32_kqv, kernel_fa_p8_fixup, kernel_fa_v_transpose_q8, kernel_fa_kqv_direct_f16, kernel_fa_kqv_direct_reduce}) {
+ for (cl_kernel kernel : {
+ kernel_mul_mm_f16_f32_kq_p8,
+ kernel_mul_mm_q8_f32_kqv,
+ kernel_fa_p8_fixup,
+ kernel_fa_v_transpose_q8,
+ kernel_fa_kqv_direct_f16,
+ kernel_fa_kqv_direct_reduce,
+ kernel_fa_q8_rows_f16,
+ kernel_fa_q8_rows_f32,
+ kernel_mul_mm_q8_kq_p8,
+ kernel_mul_mm_q8_kq_p8_dk512
+ }) {
if (kernel != nullptr) {
CL_CHECK(clReleaseKernel(kernel));
}
@@ -1648,7 +1667,7 @@ static cl_program build_program_from_source_ex(cl_context ctx, cl_device_id dev,
return NULL;
}
-static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts) {
+static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts, bool fatal = true) {
cl_context ctx = backend_ctx->context;
cl_device_id dev = backend_ctx->device;
@@ -1660,7 +1679,7 @@ static cl_program build_program_from_source(ggml_backend_opencl_context * backen
return p_cached;
}
- cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, /*fatal=*/true);
+ cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, fatal);
// Best-effort save of the freshly-built binary (no-op if cache disabled).
if (p != nullptr) {
@@ -3995,6 +4014,52 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
}
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string rows_src {
+ #include "flash_attn_q8_rows.cl.h"
+ };
+ const std::string kq_src {
+ #include "mul_mm_q8_kq_p8.cl.h"
+ };
+#else
+ const std::string rows_src = read_file("flash_attn_q8_rows.cl");
+ const std::string kq_src = read_file("mul_mm_q8_kq_p8.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, rows_src.c_str(), compile_opts, /*fatal=*/false);
+ if (prog != nullptr) {
+ CL_CHECK((backend_ctx->kernel_fa_q8_rows_f16 = clCreateKernel(prog, "kernel_fa_q8_rows_f16", &err), err));
+ CL_CHECK((backend_ctx->kernel_fa_q8_rows_f32 = clCreateKernel(prog, "kernel_fa_q8_rows_f32", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ }
+ prog = build_program_from_source(backend_ctx, kq_src.c_str(), compile_opts, /*fatal=*/false);
+ if (prog != nullptr) {
+ CL_CHECK((backend_ctx->kernel_mul_mm_q8_kq_p8 = clCreateKernel(prog, "kernel_mul_mm_q8_kq_p8", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ }
+ GGML_LOG_CONT(".");
+ }
+
+ // mul_mm_q8_kq_p8_dk512
+ if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mm_q8_kq_p8_dk512.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mm_q8_kq_p8_dk512.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts, /*fatal=*/false);
+ if (prog != nullptr) {
+ cl_kernel kernel = clCreateKernel(prog, "kernel_mul_mm_q8_kq_p8_dk512", &err);
+ CL_CHECK(clReleaseProgram(prog));
+ if (err == CL_SUCCESS) {
+ backend_ctx->kernel_mul_mm_q8_kq_p8_dk512 = kernel;
+ }
+ }
+ GGML_LOG_CONT(".");
+ }
+
// mul_mm_f16_f32_kq_p8
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -22922,12 +22987,58 @@ static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_te
}
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
+static void ggml_cl_fa_q8_rows(
+ ggml_backend_t backend, const ggml_tensor * src, const ggml_tensor * dst,
+ cl_mem values, cl_mem scales, int64_t row0, int nrow
+) {
+ auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
+ auto * extra = (ggml_tensor_extra_cl *) (src->view_src ? src->view_src->extra : src->extra);
+ const cl_ulong offset = extra->offset + src->view_offs + row0 * src->nb[1];
+ const cl_ulong nb1 = src->nb[1];
+ const cl_ulong nb2 = src->nb[2];
+ const int dk = src->ne[0];
+ const int nhead = src->ne[2];
+ cl_kernel kernel;
+ switch (src->type) {
+ case GGML_TYPE_F16: kernel = backend_ctx->kernel_fa_q8_rows_f16; break;
+ case GGML_TYPE_F32: kernel = backend_ctx->kernel_fa_q8_rows_f32; break;
+ default: GGML_ABORT("unsupported int8 KQ input type");
+ }
+ cl_uint a = 0;
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra->data_device));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &offset));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb1));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb2));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &values));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &scales));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &dk));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nrow));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nhead));
+ const size_t blocks = (size_t) (dk / 32) * nrow * nhead;
+ size_t gws[3] = {(blocks + 63) / 64 * 64, 1, 1};
+ size_t lws[3] = {64, 1, 1};
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
+}
+
+static void ggml_cl_fa_kq_int8_prepare(ggml_backend_t backend, const ggml_tensor * dst, int64_t n_q_chunk) {
+ auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
+ const ggml_tensor * q = dst->src[0];
+ const ggml_tensor * k = dst->src[1];
+ const size_t k_bytes = (size_t) k->ne[0] * k->ne[1] * k->ne[2];
+ const size_t q_bytes = (size_t) q->ne[0] * n_q_chunk * q->ne[2];
+ backend_ctx->prealloc_fa_k8.allocate(backend_ctx->context, k_bytes);
+ backend_ctx->prealloc_fa_k8_scale.allocate(backend_ctx->context, k_bytes / 32 * sizeof(cl_half));
+ backend_ctx->prealloc_fa_q8.allocate(backend_ctx->context, q_bytes);
+ backend_ctx->prealloc_fa_q8_scale.allocate(backend_ctx->context, q_bytes / 32 * sizeof(cl_half));
+ ggml_cl_fa_q8_rows(backend, k, dst, backend_ctx->prealloc_fa_k8.buffer, backend_ctx->prealloc_fa_k8_scale.buffer, 0, k->ne[1]);
+}
+
static void ggml_cl_fa_kq_p8_dispatch(
ggml_backend_t backend,
const ggml_tensor * q, const ggml_tensor * k,
const ggml_tensor * mask, const ggml_tensor * sinks,
const ggml_tensor * dst,
- int64_t q0, int nqc, float scale, float max_bias, int p_pitch
+ int64_t q0, int nqc, float scale, float max_bias, int p_pitch, bool kq_int8
) {
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
auto * extra_k = (ggml_tensor_extra_cl *) (k->view_src ? k->view_src->extra : k->extra);
@@ -22940,64 +23051,94 @@ static void ggml_cl_fa_kq_p8_dispatch(
const int D_A = k->ne[2];
const int D_B = q->ne[2];
- cl_int err;
-
- cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]};
- cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
- CL_CHECK(err);
-
- cl_image_format fmt = {CL_RGBA, CL_FLOAT};
- cl_image_desc desc = {};
- desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
- desc.image_width = region.size / 16;
- desc.buffer = k_sub;
- cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err);
- CL_CHECK(err);
-
- region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1];
- region.size = (size_t) K * N * D_B * sizeof(float);
- cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
- CL_CHECK(err);
-
const cl_ulong off_mask = extra_m->offset + mask->view_offs + q0 * mask->nb[1];
const cl_ulong mask_nb1 = mask->nb[1];
const int n_head_log2 = 1u << (unsigned) floorf(log2f((float) D_B));
const float m0 = powf(2.0f, -max_bias / n_head_log2);
const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
+ cl_kernel kernel;
cl_uint a = 0;
- cl_kernel kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8;
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
- CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
+ if (kq_int8) {
+ ggml_cl_fa_q8_rows(backend, q, dst, backend_ctx->prealloc_fa_q8.buffer, backend_ctx->prealloc_fa_q8_scale.buffer, q0, nqc);
+ kernel = K <= 256 ? backend_ctx->kernel_mul_mm_q8_kq_p8 : backend_ctx->kernel_mul_mm_q8_kq_p8_dk512;
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_k8.buffer));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_k8_scale.buffer));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_q8.buffer));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_q8_scale.buffer));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
+ size_t gws[3] = {64 * (((size_t) N + 31) / 32) * (D_B / D_A), ((size_t) M + 511) / 512, (size_t) D_A};
+ size_t lws[3] = {64, 1, 1};
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
+ } else {
+ cl_int err;
- const size_t n_tiles = (N + 31) / 32;
- size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
+ cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]};
+ cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
- const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
- size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
+ cl_image_format fmt = {CL_RGBA, CL_FLOAT};
+ cl_image_desc desc = {};
+ desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ desc.image_width = region.size / 16;
+ desc.buffer = k_sub;
+ cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err);
+ CL_CHECK(err);
- backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
- CL_CHECK(clReleaseMemObject(k_img));
- CL_CHECK(clReleaseMemObject(k_sub));
- CL_CHECK(clReleaseMemObject(q_sub));
+ region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1];
+ region.size = (size_t) K * N * D_B * sizeof(float);
+ cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+
+ kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8;
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
+
+ const size_t n_tiles = (N + 31) / 32;
+ size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
+
+ const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
+ size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
+
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
+ CL_CHECK(clReleaseMemObject(k_img));
+ CL_CHECK(clReleaseMemObject(k_sub));
+ CL_CHECK(clReleaseMemObject(q_sub));
+ }
cl_mem sinks_buf = extra_q->data_device;
cl_ulong sinks_off = 0;
@@ -23261,7 +23402,7 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
ggml_backend_t backend,
const ggml_tensor * q, const ggml_tensor * k,
ggml_tensor * dst,
- int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct
+ int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct, bool kq_int8
) {
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
const ggml_tensor * v = dst->src[2];
@@ -23345,13 +23486,17 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
}
+ if (kq_int8) {
+ ggml_cl_fa_kq_int8_prepare(backend, dst, n_q_chunk);
+ }
+
ggml_tensor_extra_cl extra_kqv;
ggml_tensor kqv;
for (int64_t q0 = 0; q0 < n_q; q0 += n_q_chunk) {
const int64_t nqc = MIN(n_q_chunk, n_q - q0);
- ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch);
+ ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch, kq_int8);
ggml_cl_fa_decompose_scratch_tensor(kqv, extra_kqv, backend_ctx->prealloc_fa_kqv.buffer, GGML_TYPE_F32, dv, nqc, n_head);
@@ -23408,6 +23553,7 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
// head size. GGML_OPENCL_FA_DECOMPOSE_MB sets the KQ scratch budget (default 256).
// GGML_OPENCL_FA_KQ_P8=0 disables U8 probabilities and int8 KQV.
// GGML_OPENCL_FA_KQV_DIRECT=0 disables direct PV for small query batches.
+// GGML_OPENCL_FA_KQ_INT8=0 keeps F16 KQ; =1 enables int8 KQ for supported shapes.
static bool ggml_cl_flash_attn_decompose_dispatch(
ggml_backend_t backend,
const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst
@@ -23537,12 +23683,46 @@ static bool ggml_cl_flash_attn_decompose_dispatch(
return false;
}
+ static const int kq_int8_env = [] {
+ const char * e = getenv("GGML_OPENCL_FA_KQ_INT8");
+ return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1;
+ }();
+ static const int dk512_env = [] {
+ const char * e = getenv("GGML_OPENCL_FA_KQ_DK512_F16");
+ return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1;
+ }();
+ const bool dk512 = dk > 256 && dk <= 512 && n_q >= 32 &&
+ (dk512_env == 1 || (dk512_env < 0 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E));
+ const bool kq_int8_possible = kq_p8 && (dk <= 256 || dk512) && dk % 32 == 0 &&
+ backend_ctx->kernel_fa_q8_rows_f16 != nullptr && backend_ctx->kernel_fa_q8_rows_f32 != nullptr &&
+ n_head <= INT_MAX && n_head_kv <= INT_MAX &&
+ n_q <= INT_MAX / n_head / (dk / 32) && n_kv <= INT_MAX / n_head_kv / (dk / 32) &&
+ (dst->src[4] == nullptr || dk > 64) &&
+ (kq_int8_env == 1 || (kq_int8_env < 0 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && n_kv >= 2048)) &&
+ backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fa_q8_rows_f16) >= 64 &&
+ backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fa_q8_rows_f32) >= 64;
+ cl_kernel kq_int8_kernel = nullptr;
+ if (kq_int8_possible) {
+ kq_int8_kernel = dk <= 256 ? backend_ctx->kernel_mul_mm_q8_kq_p8 : backend_ctx->kernel_mul_mm_q8_kq_p8_dk512;
+ }
+ const bool kq_int8 = kq_int8_kernel != nullptr && backend_ctx->get_kernel_workgroup_size(kq_int8_kernel) >= 64;
+
+ // Fewer query tiles per KV block reduce cache pressure in the int8 KQ kernel.
+ static const int64_t kq_int8_max_chunk = [] {
+ const char * e = getenv("GGML_OPENCL_FA_KQ_INT8_MAX_CHUNK");
+ const int v = (e && e[0]) ? atoi(e) : 96;
+ return (int64_t) (v >= 32 ? (v / 32) * 32 : 96);
+ }();
+ if (kq_int8) {
+ n_q_chunk = MIN(n_q_chunk, kq_int8_max_chunk);
+ }
+
// even out the chunks, then align to the 32-wide n-tile
const int64_t n_chunks = (n_q + n_q_chunk - 1) / n_q_chunk;
n_q_chunk = MIN(n_q_chunk, ((n_q + n_chunks - 1) / n_chunks + 31) & ~(int64_t) 31);
if (kq_p8) {
- ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct);
+ ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct, kq_int8);
} else {
ggml_cl_flash_attn_decompose(backend, q, k, dst, n_q_chunk, scale, max_bias);
}
diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_q8_rows.cl b/ggml/src/ggml-opencl/kernels/flash_attn_q8_rows.cl
new file mode 100644
index 000000000..f0eda2539
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/flash_attn_q8_rows.cl
@@ -0,0 +1,77 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+// Store 32-element blocks as separate int8 values and half scales.
+
+kernel void kernel_fa_q8_rows_f16(
+ global const char * src, ulong off_src, ulong nb1, ulong nb2,
+ global char * dq, global half * dd,
+ int ne0, int nrow, int nhead
+) {
+ const int nblk = ne0 / 32;
+ const int gid = get_global_id(0);
+ if (gid >= nblk*nrow*nhead) {
+ return;
+ }
+ const int b = gid % nblk;
+ const int r = (gid / nblk) % nrow;
+ const int h = gid / (nblk*nrow);
+
+ global const half * p = (global const half *)(src + off_src + (ulong)h*nb2 + (ulong)r*nb1) + b*32;
+ float v[32];
+ float amax = 0.0f;
+ #pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ const float4 x = convert_float4(vload4(i, p));
+ v[i*4+0] = x.s0;
+ v[i*4+1] = x.s1;
+ v[i*4+2] = x.s2;
+ v[i*4+3] = x.s3;
+ amax = fmax(amax, fmax(fmax(fabs(x.s0), fabs(x.s1)), fmax(fabs(x.s2), fabs(x.s3))));
+ }
+ const float d = amax / 127.0f;
+ const float id = amax > 0.0f ? 127.0f / amax : 0.0f;
+ const size_t row = (size_t)h*nrow + r;
+ dd[row*nblk + b] = (half)d;
+ global char * q = dq + row*ne0 + b*32;
+ #pragma unroll
+ for (int i = 0; i < 32; ++i) {
+ q[i] = (char)clamp((int)rint(v[i]*id), -127, 127);
+ }
+}
+
+kernel void kernel_fa_q8_rows_f32(
+ global const char * src, ulong off_src, ulong nb1, ulong nb2,
+ global char * dq, global half * dd,
+ int ne0, int nrow, int nhead
+) {
+ const int nblk = ne0 / 32;
+ const int gid = get_global_id(0);
+ if (gid >= nblk*nrow*nhead) {
+ return;
+ }
+ const int b = gid % nblk;
+ const int r = (gid / nblk) % nrow;
+ const int h = gid / (nblk*nrow);
+
+ global const float * p = (global const float *)(src + off_src + (ulong)h*nb2 + (ulong)r*nb1) + b*32;
+ float v[32];
+ float amax = 0.0f;
+ #pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ const float4 x = vload4(i, p);
+ v[i*4+0] = x.s0;
+ v[i*4+1] = x.s1;
+ v[i*4+2] = x.s2;
+ v[i*4+3] = x.s3;
+ amax = fmax(amax, fmax(fmax(fabs(x.s0), fabs(x.s1)), fmax(fabs(x.s2), fabs(x.s3))));
+ }
+ const float d = amax / 127.0f;
+ const float id = amax > 0.0f ? 127.0f / amax : 0.0f;
+ const size_t row = (size_t)h*nrow + r;
+ dd[row*nblk + b] = (half)d;
+ global char * q = dq + row*ne0 + b*32;
+ #pragma unroll
+ for (int i = 0; i < 32; ++i) {
+ q[i] = (char)clamp((int)rint(v[i]*id), -127, 127);
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q8_kq_p8.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q8_kq_p8.cl
new file mode 100644
index 000000000..8e04ad393
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mm_q8_kq_p8.cl
@@ -0,0 +1,174 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define KQ_TN 32
+#define KQ_MB 8
+#define KQ_TM 64
+#define KQ_WG 64
+#define KQ_DK_MAX 256
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_mul_mm_q8_kq_p8(
+ global const uint * kq, // K int8, [head_kv][n_kv][dk] as packed uints
+ global const half * kd, // K scales, [head_kv][n_kv][dk/32]
+ global const uint * qq, // Q int8, [head][n_q][dk] as packed uints
+ global const half * qd, // Q scales, [head][n_q][dk/32]
+ global uchar * qp, // u8 P, [head][n_q][p_pitch]
+ global float * bmax, // per 32-row block max, [head][n_q][p_pitch/32]
+ global float * bsum, // per block sum of exp(s - bmax), [head][n_q][p_pitch/32]
+ global const char * mask, // f16 [>=n_kv][n_q] view, row stride mask_nb1 bytes
+ ulong offset_mask,
+ ulong mask_nb1,
+ float scale,
+ float max_bias,
+ float m0,
+ float m1,
+ int n_head_log2,
+ int dk,
+ int n_kv,
+ int n_q,
+ int n_head,
+ int n_head_kv,
+ int p_pitch // bytes per P row, >= n_kv, multiple of 32
+) {
+ const int lid = get_local_id(0);
+ const int gsz = n_head / n_head_kv;
+ const int g0 = get_group_id(0);
+ const int head_kv = get_group_id(2);
+ const int head = head_kv*gsz + (g0 % gsz);
+ const int qn0 = (g0 / gsz) * KQ_TN;
+ const int kv0 = get_group_id(1)*KQ_TM*KQ_MB + lid;
+
+ const int nblk = dk / 32;
+ const int nu = dk / 4;
+
+ __local uint sh_q [KQ_TN][KQ_DK_MAX/4];
+ __local half sh_qd[KQ_TN][KQ_DK_MAX/32];
+ __local float p8_lds[1024];
+
+ const size_t qbase = (size_t)head*n_q;
+ for (int i = lid; i < KQ_TN*(KQ_DK_MAX/16); i += KQ_WG) {
+ const int t = i / (KQ_DK_MAX/16);
+ const int u = (i % (KQ_DK_MAX/16)) * 4;
+ const int qi = qn0 + t;
+ uint4 v = (uint4)(0u);
+ if (qi < n_q && u < nu) {
+ v = vload4(0, &qq[(qbase + qi)*nu + u]);
+ }
+ vstore4(v, 0, &sh_q[t][u]);
+ }
+ for (int i = lid; i < KQ_TN*(KQ_DK_MAX/32); i += KQ_WG) {
+ const int t = i / (KQ_DK_MAX/32);
+ const int b = i % (KQ_DK_MAX/32);
+ const int qi = qn0 + t;
+ sh_qd[t][b] = (qi < n_q && b < nblk) ? qd[(qbase + qi)*nblk + b] : (half)0.0f;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ float slope = 1.0f;
+ if (max_bias > 0.0f) {
+ const float base = head < n_head_log2 ? m0 : m1;
+ const int ex = head < n_head_log2 ? head + 1 : 2*(head - n_head_log2) + 1;
+ slope = pow(base, ex);
+ }
+ global const half * mrow = (global const half *)(mask + offset_mask);
+ const uint mstride = (uint)(mask_nb1 / 2);
+ const int pblk = p_pitch / 32;
+
+ for (int m = 0; m < KQ_MB; ++m) {
+ const int kv = kv0 + m*KQ_TM;
+ if (kv - lid >= n_kv) {
+ break; // uniform across the workgroup
+ }
+
+ float acc[KQ_TN];
+ #pragma unroll
+ for (int t = 0; t < KQ_TN; ++t) {
+ acc[t] = 0.0f;
+ }
+
+ const int kl = min(kv, n_kv - 1);
+ const size_t kbase = ((size_t)head_kv*n_kv + kl)*nu;
+ const size_t kdbas = ((size_t)head_kv*n_kv + kl)*nblk;
+
+ for (int b = 0; b < nblk; ++b) {
+ const uint4 w0 = vload4(0, &kq[kbase + b*8]);
+ const uint4 w1 = vload4(0, &kq[kbase + b*8 + 4]);
+ const float dks = (float)kd[kdbas + b];
+
+ #pragma unroll
+ for (int t = 0; t < KQ_TN; ++t) {
+ const uint4 a0 = vload4(0, &sh_q[t][b*8]);
+ const uint4 a1 = vload4(0, &sh_q[t][b*8 + 4]);
+ int raw = 0;
+ raw = dot_acc_sat_4x8packed_ss_int(w0.s0, a0.s0, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(w0.s1, a0.s1, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(w0.s2, a0.s2, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(w0.s3, a0.s3, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(w1.s0, a1.s0, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(w1.s1, a1.s1, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(w1.s2, a1.s2, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(w1.s3, a1.s3, raw);
+ acc[t] += dks * (float)sh_qd[t][b] * (float)raw;
+ }
+ }
+
+ // Each lane owns one complete 32-row softmax block after the transpose.
+ const int kvblk0 = kv - lid;
+ #pragma unroll
+ for (int half_ = 0; half_ < 2; ++half_) {
+ barrier(CLK_LOCAL_MEM_FENCE);
+ #pragma unroll
+ for (int j = 0; j < 16; ++j) {
+ p8_lds[j*64 + lid] = acc[half_*16 + j];
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (lid < 32) {
+ const int j = lid >> 1;
+ const int b = lid & 1;
+ const int n = qn0 + half_*16 + j;
+ if (n < n_q) {
+ const int kvb = kvblk0 + b*32;
+ const __local float * src = p8_lds + j*64 + b*32;
+ float16 s0 = vload16(0, src);
+ float16 s1 = vload16(1, src);
+ const global half * mp = mrow + (size_t)n*mstride + kvb;
+ const float16 m0v = convert_float16(vload16(0, mp));
+ const float16 m1v = convert_float16(vload16(1, mp));
+ s0 = s0*scale + slope*m0v;
+ s1 = s1*scale + slope*m1v;
+ const float16 mx16 = fmax(s0, s1);
+ const float8 mx8 = fmax(mx16.lo, mx16.hi);
+ const float4 mx4 = fmax(mx8.lo, mx8.hi);
+ const float2 mx2 = fmax(mx4.lo, mx4.hi);
+ const float amax = fmax(mx2.x, mx2.y);
+ const size_t prow = qbase + n;
+ const int blk = kvb >> 5;
+ float sum = 0.0f;
+ uchar16 q0 = (uchar16)(0);
+ uchar16 q1 = (uchar16)(0);
+ // finite sentinel: the program builds with -cl-finite-math-only
+ if (amax > -1.0e30f) {
+ const float16 e0 = exp(s0 - amax);
+ const float16 e1 = exp(s1 - amax);
+ const float16 a16 = e0 + e1;
+ const float8 a8 = a16.lo + a16.hi;
+ const float4 a4 = a8.lo + a8.hi;
+ const float2 a2 = a4.lo + a4.hi;
+ sum = a2.x + a2.y;
+ q0 = convert_uchar16_sat_rte(e0*255.0f);
+ q1 = convert_uchar16_sat_rte(e1*255.0f);
+ }
+ global uchar * qdst = qp + prow*(size_t)p_pitch + (size_t)kvb;
+ vstore16(q0, 0, qdst);
+ vstore16(q1, 1, qdst);
+ bmax[prow*pblk + blk] = amax;
+ bsum[prow*pblk + blk] = sum;
+ }
+ }
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q8_kq_p8_dk512.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q8_kq_p8_dk512.cl
new file mode 100644
index 000000000..f6ba2b6c2
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mm_q8_kq_p8_dk512.cl
@@ -0,0 +1,170 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define KQ_TN 32
+#define KQ_MB 8
+#define KQ_TM 64
+#define KQ_WG 64
+#define KQ_DKC 256
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_mul_mm_q8_kq_p8_dk512(
+ global const uint * kq, // K int8, [head_kv][n_kv][dk] as packed uints
+ global const half * kd, // K scales, [head_kv][n_kv][dk/32]
+ global const uint * qq, // Q int8, [head][n_q][dk] as packed uints
+ global const half * qd, // Q scales, [head][n_q][dk/32]
+ global uchar * qp, // u8 P, [head][n_q][p_pitch]
+ global float * bmax, // per 32-row block max, [head][n_q][p_pitch/32]
+ global float * bsum, // per block sum of exp(s - bmax), [head][n_q][p_pitch/32]
+ global const char * mask, // f16 [>=n_kv][n_q] view, row stride mask_nb1 bytes
+ ulong offset_mask,
+ ulong mask_nb1,
+ float scale,
+ float max_bias,
+ float m0,
+ float m1,
+ int n_head_log2,
+ int dk,
+ int n_kv,
+ int n_q,
+ int n_head,
+ int n_head_kv,
+ int p_pitch // bytes per P row, >= n_kv, multiple of 32
+) {
+ const int lid = get_local_id(0);
+ const int gsz = n_head / n_head_kv;
+ const int g0 = get_group_id(0);
+ const int head_kv = get_group_id(2);
+ const int head = head_kv*gsz + (g0 % gsz);
+ const int qn0 = (g0 / gsz) * KQ_TN;
+ const int kv0 = get_group_id(1)*KQ_TM*KQ_MB + lid;
+
+ const int nblk = dk / 32;
+ const int nu = dk / 4;
+
+ // Reuse the 256-wide Q slice for the F32 softmax scores after the contraction.
+ __local uint sh_q [KQ_TN*KQ_DKC/4];
+ __local half sh_qd[KQ_TN][KQ_DKC/32];
+ __local float * p8_lds = (__local float *)sh_q;
+
+ const size_t qbase = (size_t)head*n_q;
+
+ float slope = 1.0f;
+ if (max_bias > 0.0f) {
+ const float base = head < n_head_log2 ? m0 : m1;
+ const int ex = head < n_head_log2 ? head + 1 : 2*(head - n_head_log2) + 1;
+ slope = pow(base, ex);
+ }
+ global const half * mrow = (global const half *)(mask + offset_mask);
+ const uint mstride = (uint)(mask_nb1 / 2);
+ const int pblk = p_pitch / 32;
+
+ for (int m = 0; m < KQ_MB; ++m) {
+ const int kv = kv0 + m*KQ_TM;
+ if (kv - lid >= n_kv) {
+ break; // uniform across the workgroup
+ }
+
+ float acc[KQ_TN];
+ #pragma unroll
+ for (int t = 0; t < KQ_TN; ++t) {
+ acc[t] = 0.0f;
+ }
+
+ const int kl = min(kv, n_kv - 1);
+ const size_t kbase = ((size_t)head_kv*n_kv + kl)*nu;
+ const size_t kdbas = ((size_t)head_kv*n_kv + kl)*nblk;
+
+ for (int c0 = 0; c0 < nblk; c0 += KQ_DKC/32) {
+ const int cb = min(KQ_DKC/32, nblk - c0);
+ barrier(CLK_LOCAL_MEM_FENCE);
+ for (int i = lid; i < KQ_TN*(KQ_DKC/16); i += KQ_WG) {
+ const int t = i / (KQ_DKC/16);
+ const int u = (i % (KQ_DKC/16)) * 4;
+ const int qi = qn0 + t;
+ uint4 v = (uint4)(0u);
+ if (qi < n_q && u < cb*8) {
+ v = vload4(0, &qq[(qbase + qi)*nu + c0*8 + u]);
+ }
+ vstore4(v, 0, &sh_q[t*(KQ_DKC/4) + u]);
+ }
+ for (int i = lid; i < KQ_TN*(KQ_DKC/32); i += KQ_WG) {
+ const int t = i / (KQ_DKC/32);
+ const int b = i % (KQ_DKC/32);
+ const int qi = qn0 + t;
+ sh_qd[t][b] = (qi < n_q && b < cb) ? qd[(qbase + qi)*nblk + c0 + b] : (half)0.0f;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ for (int b = 0; b < cb; ++b) {
+ const uint4 w0 = vload4(0, &kq[kbase + (c0 + b)*8]);
+ const uint4 w1 = vload4(0, &kq[kbase + (c0 + b)*8 + 4]);
+ const float dks = (float)kd[kdbas + c0 + b];
+
+ #pragma unroll
+ for (int t = 0; t < KQ_TN; ++t) {
+ const uint4 a0 = vload4(0, &sh_q[t*(KQ_DKC/4) + b*8]);
+ const uint4 a1 = vload4(0, &sh_q[t*(KQ_DKC/4) + b*8 + 4]);
+ int raw = 0;
+ raw = dot_acc_sat_4x8packed_ss_int(w0.s0, a0.s0, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(w0.s1, a0.s1, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(w0.s2, a0.s2, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(w0.s3, a0.s3, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(w1.s0, a1.s0, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(w1.s1, a1.s1, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(w1.s2, a1.s2, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(w1.s3, a1.s3, raw);
+ acc[t] += dks * (float)sh_qd[t][b] * (float)raw;
+ }
+ }
+ }
+
+ // Finish reading the Q slice before writing scores to the same local memory.
+ const int kvblk0 = kv - lid;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ #pragma unroll
+ for (int t = 0; t < KQ_TN; ++t) {
+ p8_lds[t*64 + lid] = acc[t];
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ {
+ const int j = lid >> 1;
+ const int b = lid & 1;
+ const int n = qn0 + j;
+ if (n < n_q) {
+ const int kvb = kvblk0 + b*32;
+ __local float * src = p8_lds + j*64 + b*32;
+ const global half * mp = mrow + (size_t)n*mstride + kvb;
+ float4 s = vload4(0, src)*scale + slope*convert_float4(vload4(0, mp));
+ vstore4(s, 0, src);
+ float amax = fmax(fmax(s.x, s.y), fmax(s.z, s.w));
+ for (int c = 1; c < 8; ++c) {
+ s = vload4(c, src)*scale + slope*convert_float4(vload4(c, mp));
+ vstore4(s, c, src);
+ amax = fmax(amax, fmax(fmax(s.x, s.y), fmax(s.z, s.w)));
+ }
+ const size_t prow = qbase + n;
+ const int blk = kvb >> 5;
+ global uchar * qdst = qp + prow*(size_t)p_pitch + (size_t)kvb;
+ float sum = 0.0f;
+ // finite sentinel: the program builds with -cl-finite-math-only
+ if (amax > -1.0e30f) {
+ for (int c = 0; c < 8; ++c) {
+ const float4 e = exp(vload4(c, src) - amax);
+ sum += (e.x + e.y) + (e.z + e.w);
+ vstore4(convert_uchar4_sat_rte(e*255.0f), c, qdst);
+ }
+ } else {
+ for (int c = 0; c < 8; ++c) {
+ vstore4((uchar4)(0), c, qdst);
+ }
+ }
+ bmax[prow*pblk + blk] = amax;
+ bsum[prow*pblk + blk] = sum;
+ }
+ }
+ }
+}