Commit e0949d09b for llama.cpp
commit e0949d09b130707b3fa93d2cff616024231cf939
Author: shaofeiqi <shaoqi@qti.qualcomm.com>
Date: Wed Nov 11 21:53:51 2026 -0800
opencl: add bin kernel `kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin` (#30947)
* opencl: add A7 Q4_0 non-MoE binary kernel
* opencl: refactor
---------
Co-authored-by: Li He <lih@qti.qualcomm.com>
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 53ad1cc40..91675b6d4 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -1400,6 +1400,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_gemv_noshuffle_q4_0_f32;
cl_kernel kernel_gemv_noshuffle_q4_0_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP)
cl_kernel kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin;
+ cl_kernel kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin;
cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin;
cl_kernel kernel_gemv_noshuffle_q4_0_f32_32b_trans;
cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_11008;
@@ -6166,6 +6167,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = nullptr;
backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = nullptr;
+ backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin = nullptr;
backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin = nullptr;
{
{
@@ -6210,6 +6212,17 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
CL_CHECK(clReleaseProgram(bin_prog));
GGML_LOG_CONT(".");
}
+
+ kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_f32_32b_trans_ila_a7", &bin_size);
+ if (kernel_bin && bin_size > 0) {
+ cl_program bin_prog =
+ build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size);
+
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin =
+ clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7", &err), err));
+ CL_CHECK(clReleaseProgram(bin_prog));
+ GGML_LOG_CONT(".");
+ }
}
}
@@ -11615,7 +11628,8 @@ inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *b
inline bool use_q4_0_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
if (!backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans ||
- !backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin) {
+ !(backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin ||
+ backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin)) {
return false;
}
@@ -25357,7 +25371,8 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const gg
CL_CHECK(clReleaseMemObject(b_sub_buf));
CL_CHECK(clReleaseMemObject(b_img));
} else {
- const int gemm_tile_n = 64;
+ const bool have_a8 = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin != nullptr;
+ const int gemm_tile_n = have_a8 ? 64 : 32;
int N_pad = (N + gemm_tile_n - 1) & ~(gemm_tile_n - 1);
cl_mem a_img = extra0_q4_0->q_img;
@@ -25457,6 +25472,53 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const gg
cl_mem b_img;
CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+ int line_stride_matrix_A_in_bytes = M * 4;
+ int line_stride_matrix_S_in_bytes = M * 2;
+ int line_stride_matrix_B_in_bytes = K * 4;
+ int line_stride_matrix_C_in_bytes = M * 4;
+
+ const int b_offset_for_kernel = 0;
+
+ if (!have_a8) {
+ const int c_offset_for_kernel = (int)offsetd;
+
+ kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin;
+
+ cl_uint k_arg = 0;
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_0->d));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &b_offset_for_kernel));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_A_in_bytes));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_S_in_bytes));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_B_in_bytes));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_C_in_bytes));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N));
+
+ size_t m_tiles = (size_t)CEIL_DIV(M, 64);
+ if ((m_tiles % 2) != 0) {
+ // keep the work-group count in y even so every work-group is full
+ m_tiles += 1;
+ }
+
+ size_t local_work_size[3] = { 64, 2, 4 };
+ size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) };
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+
+ CL_CHECK(clReleaseMemObject(b_img));
+ if (b_sub_buf) {
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+ }
+ if (b_padded) {
+ CL_CHECK(clReleaseMemObject(b_padded));
+ }
+ return;
+ }
+
region.origin = offsetd;
region.size = (size_t)M * N * sizeof(float);
cl_mem d_sub_buf;
@@ -25470,13 +25532,7 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const gg
cl_mem d_img;
CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err));
- int line_stride_matrix_A_in_bytes = M * 4;
- int line_stride_matrix_S_in_bytes = M * 2;
- int line_stride_matrix_B_in_bytes = K * 4;
- int line_stride_matrix_C_in_bytes = M * 4;
-
- int c_offset_for_kernel = 0;
- int b_offset_for_kernel = 0;
+ const int c_offset_for_kernel = 0;
kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin;
@@ -25496,19 +25552,19 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const gg
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N));
size_t local_work_size[3] = { 64, 2, 2 };
- size_t m_tiles = (size_t)CEIL_DIV(M, 64);
- size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) };
+ size_t global_work_size[3] = { 64, (size_t)CEIL_DIV(M, 64), (size_t)CEIL_DIV(N_pad, gemm_tile_n) };
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+ CL_CHECK(clReleaseMemObject(d_img));
+ CL_CHECK(clReleaseMemObject(d_sub_buf));
CL_CHECK(clReleaseMemObject(b_img));
+
if (b_sub_buf) {
CL_CHECK(clReleaseMemObject(b_sub_buf));
}
if (b_padded) {
CL_CHECK(clReleaseMemObject(b_padded));
}
- CL_CHECK(clReleaseMemObject(d_img));
- CL_CHECK(clReleaseMemObject(d_sub_buf));
}
}
#endif // GGML_OPENCL_USE_ADRENO_KERNELS