Commit 9f70b2cec for llama.cpp
commit 9f70b2cecd1a9a3f73ac525c47ca22a6ee9a7b69
Author: shaofeiqi <shaoqi@qti.qualcomm.com>
Date: Fri Sep 25 20:41:47 2026 -0700
opencl: add A8 Q8_0 non-MoE dp4a binary kernel (#29439)
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 0b685e697..19cd02583 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -1224,6 +1224,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_gemm_noshuffle_q8_0_f32, kernel_gemm_noshuffle_q8_0_f32_bin;
cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q8_0 prefill GEMM (opt-in)
cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = nullptr; // q8_0 dense dp4a, weights via texture (opt-in)
+ cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = nullptr;
cl_kernel kernel_gemv_noshuffle_q8_0_f32;
cl_kernel kernel_gemv_noshuffle_q8_0_f32_splitk; // split-K across WGs (small-M decode)
cl_kernel kernel_gemm_noshuffle_q1_0_f32;
@@ -4154,6 +4155,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
}
}
+ if (backend_ctx->has_integer_dot) {
+ size_t bin_size = 0;
+ backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = nullptr;
+
+ if (use_adreno_bin_kernels(backend_ctx)) {
+ const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8", &bin_size);
+ if (kernel_bin && bin_size > 0) {
+ cl_program prog =
+ build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, compile_opts, bin_size);
+
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+ }
+ }
+
// gemv_noshuffle_general_q8_0_f32
{
std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +
@@ -20949,6 +20967,80 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clReleaseMemObject(b_img));
CL_CHECK(clReleaseMemObject(b_sub_buf));
} else {
+ static const char * q8_bin_dp4a_env = getenv("GGML_OPENCL_Q8_0_BIN_DP4A");
+ const bool q8_bin_dp4a_on = backend_ctx->has_integer_dot &&
+ (q8_bin_dp4a_env ? (atoi(q8_bin_dp4a_env) != 0) : true);
+
+ if (q8_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin
+ && M % 64 == 0) {
+ cl_mem a_sub = nullptr;
+ region.origin = offset1;
+ region.size = (size_t)K * N * sizeof(float);
+ CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+
+ const size_t n_blocks = (size_t)N * (K / 32);
+
+ const size_t N_pad = ((size_t)N + 31) / 32 * 32;
+ const size_t n_blocks_pad = N_pad * (K / 32);
+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N_pad * K * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks_pad * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_pad * sizeof(cl_half));
+
+ cl_int tb = (cl_int)n_blocks;
+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub));
+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));
+ size_t q_local[1] = { 64 };
+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
+
+ cl_mem q_img = nullptr;
+ img_fmt = { CL_R, CL_UNSIGNED_INT32 };
+ memset(&img_desc, 0, sizeof(img_desc));
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = (size_t)M * (size_t)K / 4;
+ img_desc.buffer = extra0_q8_0->q;
+ CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+
+ cl_mem d_sub_buf = nullptr;
+ cl_mem d_img = nullptr;
+ region.origin = extrad->offset;
+ region.size = (size_t)M * N * sizeof(float);
+ CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+
+ img_fmt = { CL_R, CL_FLOAT };
+ memset(&img_desc, 0, sizeof(img_desc));
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = (size_t)M * N;
+ img_desc.buffer = d_sub_buf;
+ CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+
+ cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin;
+ cl_uint ne00_u = (cl_uint)K;
+ cl_uint ne01_u = (cl_uint)M;
+ CL_CHECK(clSetKernelArg(dk, 0, sizeof(cl_mem), &q_img));
+ CL_CHECK(clSetKernelArg(dk, 1, sizeof(cl_mem), &extra0_q8_0->d));
+ CL_CHECK(clSetKernelArg(dk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, 4, sizeof(cl_mem), &d_img));
+ CL_CHECK(clSetKernelArg(dk, 5, sizeof(cl_uint), &ne00_u));
+ CL_CHECK(clSetKernelArg(dk, 6, sizeof(cl_uint), &ne01_u));
+ CL_CHECK(clSetKernelArg(dk, 7, sizeof(int), &N));
+
+ size_t d_local[3] = { 64, 1, 1 };
+ size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);
+
+ CL_CHECK(clReleaseMemObject(q_img));
+ CL_CHECK(clReleaseMemObject(d_img));
+ CL_CHECK(clReleaseMemObject(d_sub_buf));
+ CL_CHECK(clReleaseMemObject(a_sub));
+ return;
+ }
+
// dp4a dense q8_0 prefill GEMM. Quantizes the [N,K] activations to
// q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch
// (ne1>8) only; q8_0 weights are already int8 (no requant) and symmetric