Commit 74a4fb003 for llama.cpp
commit 74a4fb0037243f1b6196d0f990ff0aff031356cf
Author: Hongqiang Wang <wangh@qti.qualcomm.com>
Date: Wed Oct 28 14:55:28 2026 -0700
opencl: add tiled q4_k GEMM for adreno (#30412)
* opencl: run the tiled q4_K lm_head through a tiled GEMM at batch > 1
* opencl: split tiled dp4a q4_k gemm into its own file and test for dp4a
Assisted-by: Codex
---------
Co-authored-by: Li He <lih@qti.qualcomm.com>
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt
index 1963c6943..4d651ddb6 100644
--- a/ggml/src/ggml-opencl/CMakeLists.txt
+++ b/ggml/src/ggml-opencl/CMakeLists.txt
@@ -206,6 +206,7 @@ set(GGML_OPENCL_KERNELS
gemv_noshuffle_q4_k_f32
gemv_noshuffle_q4_k_f32_o4
gemv_noshuffle_q4_k_f32_tiled
+ gemm_noshuffle_q4_K_f32_tiled_dp4a
gemm_noshuffle_q4_k_f32
gemv_noshuffle_q4_k_f32_32b_trans
gemv_noshuffle_q6_k_f32
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 67248264e..8167e5e22 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -1286,6 +1286,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_gemv_noshuffle_q4_k_f32;
cl_kernel kernel_gemv_noshuffle_q4_k_f32_o4; // 4-output-per-WI, long-vocab lm_head
cl_kernel kernel_gemv_noshuffle_q4_k_f32_tiled; // tiled-wide layout (opt-in)
+ cl_kernel kernel_gemm_noshuffle_q4_K_f32_tiled_dp4a = nullptr;
cl_kernel kernel_gemv_noshuffle_q4_k_f32_splitk; // split-K across WGs (small-M decode)
cl_kernel kernel_gemv_splitk_reduce_f32; // sums split-K per-slice partials
cl_kernel kernel_gemv_noshuffle_q4_k_f32_glu; // fused gate+up GEMV + GLU (FFN)
@@ -5092,6 +5093,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
+ // gemm_noshuffle_q4_K_f32_tiled_dp4a
+ if (backend_ctx->has_integer_dot) {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_q4_K_f32_tiled_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_q4_K_f32_tiled_dp4a.cl");
+#endif
+ std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable ";
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_K_f32_tiled_dp4a =
+ clCreateKernel(prog, "kernel_gemm_noshuffle_q4_K_f32_tiled_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = nullptr;
backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = nullptr;
backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin = nullptr;
@@ -9622,9 +9640,11 @@ inline bool use_q6k_tiled(const ggml_backend_opencl_context *backend_ctx, const
use_adreno_kernels(backend_ctx, tensor);
}
-// q4_K analog of the tiled-wide lm_head/embed GEMV (default OFF;
-// GGML_OPENCL_Q4K_GEMV_TILED forces either way: =0 off, else on). Same gate.
+// Select untiled Q4_K before upload if the batched DP4A reader is unavailable.
inline bool q4k_gemv_tiled_enabled(const ggml_backend_opencl_context *backend_ctx) {
+ if (!backend_ctx->has_integer_dot) {
+ return false;
+ }
static const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_TILED");
if (e && e[0] != '\0') {
return e[0] != '0';
@@ -23506,6 +23526,48 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t
region.size = K * N * sizeof(float);
CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+ // Only the tiled kernels can read the tiled layout (use_q4k_tiled).
+ if (use_q4k_tiled(backend_ctx, src0)) {
+ const size_t n_blocks = (size_t)N * (K / 32);
+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));
+
+ cl_int tbq = (cl_int)n_blocks;
+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf));
+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tbq));
+ size_t q_local[1] = { 64 };
+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
+
+ cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q4_K_f32_tiled_dp4a;
+ CL_CHECK(clSetKernelArg(dk, 0, sizeof(cl_mem), &extra0_q4_k->q));
+ CL_CHECK(clSetKernelArg(dk, 1, sizeof(cl_mem), &extra0_q4_k->d));
+ CL_CHECK(clSetKernelArg(dk, 2, sizeof(cl_mem), &extra0_q4_k->dm));
+ CL_CHECK(clSetKernelArg(dk, 3, sizeof(cl_mem), &extra0_q4_k->s));
+ CL_CHECK(clSetKernelArg(dk, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, 6, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(dk, 7, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(dk, 8, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(dk, 9, sizeof(cl_int), &ne00));
+ CL_CHECK(clSetKernelArg(dk, 10, sizeof(cl_int), &ne01));
+ CL_CHECK(clSetKernelArg(dk, 11, sizeof(cl_int), &ne1));
+
+ const int WROWS_D = 4 * 64;
+ const int BN_DP = 8;
+ size_t dlocal[3] = {64, 4, 1};
+ size_t dglobal[3] = {(size_t)CEIL_DIV(ne01, WROWS_D) * 64, 4, (size_t)CEIL_DIV(ne1, BN_DP)};
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dglobal, dlocal, dst);
+
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+ return;
+ }
+
// CoopK handles without _bin use the same untiled weight layout as gemm_noshuffle_q4_k_f32.
// Activations use K-major Q8_1 SoA (K contiguous).
if (ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q4_k)
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_K_f32_tiled_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_K_f32_tiled_dp4a.cl
new file mode 100644
index 000000000..af642b4df
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_K_f32_tiled_dp4a.cl
@@ -0,0 +1,131 @@
+// DP4A GEMM for the canonical 64-row Q4_K layout.
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#ifdef cl_qcom_reqd_sub_group_size
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#endif
+
+#define QK_K 256
+#define TILE_ROWS 64
+#define NTILES 4
+#define BN_DP 8
+#define WG_THREADS (NTILES * TILE_ROWS)
+
+inline void q4k_scale_min(int j, __global const uchar * q, uchar * d, uchar * m) {
+ if (j < 4) {
+ *d = q[j] & 0x3F;
+ *m = q[j+4] & 0x3F;
+ } else {
+ *d = (q[j+4] & 0x0F) | ((q[j-4] & 0xC0) >> 2);
+ *m = ((q[j+4] >> 4) & 0x0F) | ((q[j] & 0xC0) >> 2);
+ }
+}
+
+// dp4a GEMM over the tiled q4_K layout (ne1 > 1) with a q8_1 activation; per 32-element sub-block:
+// acc += d*sd*qad*dp4a(q, qa) - dm*sm*qas
+#if defined(ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemm_noshuffle_q4_K_f32_tiled_dp4a(
+ __global uint4 * src0_q,
+ __global half * src0_d,
+ __global half * src0_dm,
+ __global uchar * src0_s,
+ __global uint * qa, // q8_1 int8 activation, 4/uint [ne11][ne00/4]
+ __global half * qad, // q8_1 per-32-block scale [ne11][ne00/32]
+ __global half * qas, // q8_1 per-32-block sum (qad*Sum)[ne11][ne00/32]
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int ne11
+) {
+ int rit = get_local_id(0);
+ int sg = get_local_id(1);
+ int lid = sg * TILE_ROWS + rit;
+ int row = get_group_id(0) * WG_THREADS + lid;
+ int rt = row / TILE_ROWS;
+ int col0 = get_global_id(2) * BN_DP;
+ int nb = ne00 / QK_K;
+ int qcs = ne00 / 4;
+ int dcs = ne00 / 32;
+
+ const bool row_ok = row < ne01;
+ __local uint sh_qa[BN_DP * 64];
+ __local half sh_qad[BN_DP * 8];
+ __local half sh_qas[BN_DP * 8];
+
+ float acc[BN_DP];
+ #pragma unroll
+ for (int j = 0; j < BN_DP; ++j) acc[j] = 0.0f;
+
+ for (int sb = 0; sb < nb; ++sb) {
+ for (int p = lid; p < BN_DP * 64; p += WG_THREADS) {
+ int j = p >> 6, u = p & 63, c = col0 + j;
+ sh_qa[p] = (c < ne11) ? qa[c * qcs + sb * 64 + u] : 0u;
+ }
+ for (int p = lid; p < BN_DP * 8; p += WG_THREADS) {
+ int j = p >> 3, b = p & 7, c = col0 + j;
+ sh_qad[p] = (c < ne11) ? qad[c * dcs + sb * 8 + b] : (half)0;
+ sh_qas[p] = (c < ne11) ? qas[c * dcs + sb * 8 + b] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (row_ok) {
+ int tile_blk = rt * nb + sb;
+ float dval = (float)src0_d [tile_blk * TILE_ROWS + rit];
+ float dmval = (float)src0_dm[tile_blk * TILE_ROWS + rit];
+ __global uchar * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 12;
+
+ uint q[32];
+ #pragma unroll
+ for (int g = 0; g < 8; ++g) {
+ uint4 v = src0_q[(tile_blk * 8 + g) * TILE_ROWS + rit];
+ q[g*4+0] = v.x; q[g*4+1] = v.y; q[g*4+2] = v.z; q[g*4+3] = v.w;
+ }
+
+ for (int is = 0; is < 8; ++is) {
+ uchar sd, sm;
+ q4k_scale_min(is, sc, &sd, &sm);
+ float scl = dval * (float)sd, mnv = dmval * (float)sm;
+
+ uint cw[8];
+ #pragma unroll
+ for (int t = 0; t < 8; ++t) {
+ uint pk = 0;
+ #pragma unroll
+ for (int u = 0; u < 4; ++u) {
+ int e = is * 32 + t * 4 + u;
+ uint code = (q[e >> 3] >> ((e & 7) * 4)) & 0xF;
+ pk |= (code & 0xFF) << (u * 8);
+ }
+ cw[t] = pk;
+ }
+ #pragma unroll
+ for (int j = 0; j < BN_DP; ++j) {
+ __local uint * a = sh_qa + j * 64 + is * 8;
+ int raw = 0;
+ #pragma unroll
+ for (int t = 0; t < 8; ++t) raw = dot_acc_sat_4x8packed_ss_int(cw[t], a[t], raw);
+ acc[j] += scl * (float)sh_qad[j*8 + is] * (float)raw
+ - mnv * (float)sh_qas[j*8 + is];
+ }
+ }
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (row_ok) {
+ dst = (global float*)((global char*)dst + offsetd);
+ #pragma unroll
+ for (int j = 0; j < BN_DP; ++j) {
+ int c = col0 + j;
+ if (c < ne11) dst[(ulong)c * ne01 + row] = acc[j];
+ }
+ }
+}