Commit 8415bd7bb for llama.cpp
commit 8415bd7bb39aa29823a72ef832d671ee6898fc0b
Author: shaofeiqi <shaoqi@qti.qualcomm.com>
Date: Wed Nov 4 19:14:11 2026 -0800
opencl: add DSV4_HC_PRE and DSV4_HC_POST (#31623)
* opencl: add DSV4_HC_PRE and DSV4_HC_POST
Co-authored-by: Shafei Qi <shaoqi@qti.qualcomm.com>
* opencl: improve dsv4_hc_post
Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
---------
Co-authored-by: Li He <lih@qti.qualcomm.com>
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt
index 20dd595c7..58a6d438e 100644
--- a/ggml/src/ggml-opencl/CMakeLists.txt
+++ b/ggml/src/ggml-opencl/CMakeLists.txt
@@ -78,6 +78,7 @@ set(GGML_OPENCL_KERNELS
diag_mask_inf
diag
div
+ dsv4_hc
gelu
get_rows
glu
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index a2620135d..8aa013edb 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -1104,6 +1104,10 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_argmax_f32_i32;
cl_kernel kernel_argsort_f32_i32;
cl_kernel kernel_sum_rows_f32, kernel_sum_rows_f32_4;
+ cl_kernel kernel_dsv4_hc_pre = nullptr;
+ cl_kernel kernel_dsv4_hc_post[2] = {};
+ size_t local_dsv4_hc_pre = 0;
+ size_t local_dsv4_hc_post[2] = {};
cl_kernel kernel_cumsum_blk, kernel_cumsum_add;
cl_kernel kernel_repeat_f32;
cl_kernel kernel_pad;
@@ -1463,6 +1467,14 @@ struct ggml_backend_opencl_context {
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
~ggml_backend_opencl_context() {
+ if (kernel_dsv4_hc_pre) {
+ CL_CHECK(clReleaseKernel(kernel_dsv4_hc_pre));
+ }
+ for (int i = 0; i < 2; ++i) {
+ if (kernel_dsv4_hc_post[i]) {
+ CL_CHECK(clReleaseKernel(kernel_dsv4_hc_post[i]));
+ }
+ }
if (iq1_s_grid_image) {
CL_CHECK(clReleaseMemObject(iq1_s_grid_image));
}
@@ -4227,6 +4239,28 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
+ // dsv4_hc
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "dsv4_hc.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("dsv4_hc.cl");
+#endif
+ for (int i = 0; i < 2; ++i) {
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + (i == 0 ? " -DHC_VEC=1" : " -DHC_VEC=4"));
+ if (i == 0) {
+ CL_CHECK((backend_ctx->kernel_dsv4_hc_pre = clCreateKernel(prog, "kernel_dsv4_hc_pre", &err), err));
+ backend_ctx->local_dsv4_hc_pre = MIN((size_t) 64, backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_dsv4_hc_pre));
+ }
+ CL_CHECK((backend_ctx->kernel_dsv4_hc_post[i] = clCreateKernel(prog, "kernel_dsv4_hc_post", &err), err));
+ backend_ctx->local_dsv4_hc_post[i] = MIN((size_t) 64, backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_dsv4_hc_post[i]));
+ CL_CHECK(clReleaseProgram(prog));
+ }
+ GGML_LOG_CONT(".");
+ }
+
// sum_rows
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -12048,6 +12082,14 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
case GGML_OP_SUM_ROWS:
case GGML_OP_CUMSUM:
return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]);
+ case GGML_OP_DSV4_HC_PRE:
+ case GGML_OP_DSV4_HC_POST:
+ for (int i = 0; i < (op->op == GGML_OP_DSV4_HC_PRE ? 2 : 4); ++i) {
+ if (op->src[i] && op->src[i]->type != GGML_TYPE_F32) {
+ return false;
+ }
+ }
+ return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op);
case GGML_OP_MEAN:
return op->src[0]->type == GGML_TYPE_F32;
case GGML_OP_FLASH_ATTN_EXT: {
@@ -18274,7 +18316,7 @@ static void ggml_cl_add(ggml_backend_t backend, const ggml_tensor * src0, const
cl_kernel kernel;
- const bool bcast_row = ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0;
+ const bool bcast_row = ggml_is_contiguous(src0) && ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0;
if (bcast_row) {
GGML_ASSERT(ggml_is_contiguous(src0));
@@ -37839,6 +37881,47 @@ static void ggml_cl_argsort(ggml_backend_t backend, const ggml_tensor * src0, co
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
}
+static void ggml_cl_dsv4_hc(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
+ const bool pre = dst->op == GGML_OP_DSV4_HC_PRE;
+ const bool vec = !pre && src0->ne[0] % 4 == 0 && src0->nb[0] == sizeof(float) && src1->nb[0] == sizeof(float);
+ cl_kernel kernel = pre ? backend_ctx->kernel_dsv4_hc_pre : backend_ctx->kernel_dsv4_hc_post[vec];
+
+ cl_uint arg = 0;
+ for (int i = 0; i < (pre ? 2 : 4); ++i) {
+ const ggml_tensor * src = dst->src[i];
+ auto * extra = src ? (ggml_tensor_extra_cl *) src->extra : nullptr;
+ const cl_mem buffer = extra ? extra->data_device : nullptr;
+ const cl_ulong offset = src ? extra->offset + src->view_offs : 0;
+ CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &buffer));
+ CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &offset));
+ for (int j = 0; j < 3; ++j) {
+ const cl_ulong stride = src ? src->nb[j] : 0;
+ CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &stride));
+ }
+ }
+
+ auto * extra = (ggml_tensor_extra_cl *) dst->extra;
+ const cl_ulong offset = extra->offset + dst->view_offs;
+ const cl_long n_embd = src0->ne[0];
+ const cl_long n_hc = pre ? src0->ne[1] : src1->ne[1];
+ CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &extra->data_device));
+ CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &offset));
+ CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_long), &n_embd));
+ CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_long), &n_hc));
+ if (pre) {
+ const float scale = ggml_get_op_params_f32(dst, 0);
+ const cl_int gated = ggml_get_op_params_i32(dst, 1);
+ CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &scale));
+ CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_int), &gated));
+ }
+
+ const size_t n_work = (size_t) n_embd / (vec ? 4 : 1);
+ size_t local_work_size[] = { pre ? backend_ctx->local_dsv4_hc_pre : backend_ctx->local_dsv4_hc_post[vec], 1, 1 };
+ size_t global_work_size[] = { CEIL_DIV(n_work, local_work_size[0]) * local_work_size[0], (size_t) dst->ne[1], (size_t) dst->ne[2] };
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+}
+
static void ggml_cl_sum_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
GGML_ASSERT(src0);
GGML_ASSERT(src0->extra);
@@ -38726,6 +38809,10 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor
}
func = ggml_cl_sum_rows;
break;
+ case GGML_OP_DSV4_HC_PRE:
+ case GGML_OP_DSV4_HC_POST:
+ func = ggml_cl_dsv4_hc;
+ break;
case GGML_OP_CUMSUM:
if (!any_on_device) {
return false;
diff --git a/ggml/src/ggml-opencl/kernels/dsv4_hc.cl b/ggml/src/ggml-opencl/kernels/dsv4_hc.cl
new file mode 100644
index 000000000..164ab1dae
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/dsv4_hc.cl
@@ -0,0 +1,101 @@
+#if HC_VEC == 4
+typedef float4 hc_float;
+#define HC_LOAD(p) vload4(0, (global const float *) (p))
+#define HC_STORE(v, p) vstore4(v, 0, (global float *) (p))
+#else
+typedef float hc_float;
+#define HC_LOAD(p) (*(global const float *) (p))
+#define HC_STORE(v, p) (*(global float *) (p) = (v))
+#endif
+
+#if HC_VEC == 1
+kernel void kernel_dsv4_hc_pre(
+ global const char * x,
+ ulong offset_x,
+ ulong nbx0,
+ ulong nbx1,
+ ulong nbx2,
+ global const char * weights,
+ ulong offset_w,
+ ulong nbw0,
+ ulong nbw1,
+ ulong nbw2,
+ global char * dst,
+ ulong offset_dst,
+ long n_embd,
+ long n_hc,
+ float scale,
+ int gated
+) {
+ const size_t i = get_global_id(0);
+ const size_t t = get_global_id(1);
+ if (i >= n_embd) {
+ return;
+ }
+
+ x += offset_x + i*nbx0 + t*nbx2;
+ weights += offset_w + (gated ? i*nbw0 + t*nbw2 : t*nbw1);
+
+ float sum = 0.0f;
+ for (long h = 0; h < n_hc; ++h) {
+ const float xv = *(global const float *) (x + h*nbx1);
+ float wv = *(global const float *) (weights + h*(gated ? nbw1 : nbw0));
+ if (gated) {
+ wv = 1.0f / (1.0f + exp(-wv));
+ }
+ sum += xv * wv;
+ }
+ ((global float *) (dst + offset_dst))[i + t*n_embd] = scale * sum;
+}
+#endif
+
+kernel void kernel_dsv4_hc_post(
+ global const char * x,
+ ulong offset_x,
+ ulong nbx0,
+ ulong nbx1,
+ ulong nbx2,
+ global const char * residual,
+ ulong offset_r,
+ ulong nbr0,
+ ulong nbr1,
+ ulong nbr2,
+ global const char * post,
+ ulong offset_p,
+ ulong nbp0,
+ ulong nbp1,
+ ulong nbp2,
+ global const char * comb,
+ ulong offset_c,
+ ulong nbc0,
+ ulong nbc1,
+ ulong nbc2,
+ global char * dst,
+ ulong offset_dst,
+ long n_embd,
+ long n_hc
+) {
+ const size_t i = get_global_id(0) * HC_VEC;
+ const size_t h = get_global_id(1);
+ const size_t t = get_global_id(2);
+ if (i >= n_embd) {
+ return;
+ }
+
+ const hc_float xv = HC_LOAD(x + offset_x + i*nbx0 + t*nbx1);
+ const float pv = *(global const float *) (post + offset_p + h*nbp0 + t*nbp1);
+ residual += offset_r + i*nbr0 + t*nbr2;
+
+ hc_float sum = xv * pv;
+ if (comb) {
+ comb += offset_c + h*nbc0 + t*nbc2;
+ for (long s = 0; s < n_hc; ++s) {
+ const hc_float rv = HC_LOAD(residual + s*nbr1);
+ const float cv = *(global const float *) (comb + s*nbc1);
+ sum += rv * cv;
+ }
+ } else {
+ sum += HC_LOAD(residual + h*nbr1);
+ }
+ HC_STORE(sum, dst + offset_dst + (i + (h + t*n_hc)*n_embd)*sizeof(float));
+}