Commit 6fa4f6cd3 for llama.cpp
commit 6fa4f6cd3ef4b65188acb251d5ef37f8300719a3
Author: lhez <lih@qti.qualcomm.com>
Date: Mon Oct 19 12:22:34 2026 -0700
opencl: add get_rows for i32, cpy for i32 and f32 (#30756)
* opencl: support i32 GET_ROWS
* opencl: support i32 and f32 copy conversions
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index de93b0134..96461bcd9 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -868,7 +868,7 @@ struct ggml_backend_opencl_context {
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
ggml_cl_adreno_xmem_attn_state adreno_xmem_attn;
#endif
- cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_q4_0;
+ cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_i32, kernel_get_rows_q4_0;
cl_kernel kernel_set_rows_f32_i64, kernel_set_rows_f32_i32, kernel_set_rows_f16_i64, kernel_set_rows_f16_i32;
cl_kernel kernel_set_rows_q8_0_i64, kernel_set_rows_q8_0_i32;
cl_kernel kernel_set_rows_q8_0_soa_i64, kernel_set_rows_q8_0_soa_i32;
@@ -876,7 +876,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_set_rows_q4_0_soa_i64, kernel_set_rows_q4_0_soa_i32;
cl_kernel kernel_rope_norm_f32, kernel_rope_norm_f16, kernel_rope_neox_f32, kernel_rope_neox_f16;
cl_kernel kernel_rope_multi_f32, kernel_rope_multi_f16, kernel_rope_vision_f32, kernel_rope_vision_f16;
- cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_i32_i32;
+ cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_i32_i32, kernel_cpy_i32_f32, kernel_cpy_f32_i32;
cl_kernel kernel_cpy_f32_f32_pack = nullptr;
cl_kernel kernel_cpy_f32_f32_flat = nullptr;
cl_kernel kernel_mul_mat_f32_f32;
@@ -1652,6 +1652,8 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
}
}
CL_CHECK((backend_ctx->kernel_cpy_i32_i32 = clCreateKernel(prog, "kernel_cpy_i32_i32", &err), err));
+ CL_CHECK((backend_ctx->kernel_cpy_i32_f32 = clCreateKernel(prog, "kernel_cpy_i32_f32", &err), err));
+ CL_CHECK((backend_ctx->kernel_cpy_f32_i32 = clCreateKernel(prog, "kernel_cpy_f32_i32", &err), err));
GGML_LOG_CONT(".");
}
@@ -1840,6 +1842,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_get_rows_f32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f32", &err), err));
+ CL_CHECK((backend_ctx->kernel_get_rows_i32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_i32", &err), err));
CL_CHECK((backend_ctx->kernel_get_rows_f16 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f16", &err), err));
CL_CHECK((backend_ctx->kernel_get_rows_q4_0 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_q4_0", &err), err));
GGML_LOG_CONT(".");
@@ -9102,6 +9105,8 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
return true;
case GGML_OP_GET_ROWS:
switch (op->src[0]->type) {
+ case GGML_TYPE_I32:
+ return op->type == GGML_TYPE_I32;
case GGML_TYPE_F32:
case GGML_TYPE_F16:
return true;
@@ -9139,6 +9144,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
switch (op->src[0]->type) {
case GGML_TYPE_F32:
switch (op->type) {
+ case GGML_TYPE_I32:
case GGML_TYPE_F16:
case GGML_TYPE_F32:
return true;
@@ -9155,6 +9161,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
}
case GGML_TYPE_I32:
switch (op->type) {
+ case GGML_TYPE_F32:
case GGML_TYPE_I32:
return true;
default:
@@ -13619,6 +13626,9 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, c
case GGML_TYPE_F32:
kernel = backend_ctx->kernel_get_rows_f32;
break;
+ case GGML_TYPE_I32:
+ kernel = backend_ctx->kernel_get_rows_i32;
+ break;
case GGML_TYPE_F16:
kernel = backend_ctx->kernel_get_rows_f16;
break;
@@ -13654,7 +13664,7 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, c
}
int nchunks = 1;
- if (src0->type == GGML_TYPE_F32) {
+ if (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32) {
const int chunk_target = nth * 4;
nchunks = (ne00 + chunk_target - 1) / chunk_target;
nchunks = MAX(1, MIN(nchunks, 64));
@@ -28726,6 +28736,9 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const
switch (src0t) {
case GGML_TYPE_F32:
switch (src1t) {
+ case GGML_TYPE_I32:
+ kernel = backend_ctx->kernel_cpy_f32_i32;
+ break;
case GGML_TYPE_F16:
kernel = backend_ctx->kernel_cpy_f32_f16;
break;
@@ -28752,6 +28765,9 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const
break;
case GGML_TYPE_I32:
switch (src1t) {
+ case GGML_TYPE_F32:
+ kernel = backend_ctx->kernel_cpy_i32_f32;
+ break;
case GGML_TYPE_I32:
kernel = backend_ctx->kernel_cpy_i32_i32;
break;
diff --git a/ggml/src/ggml-opencl/kernels/cpy.cl b/ggml/src/ggml-opencl/kernels/cpy.cl
index 11d7a57c1..18400d37f 100644
--- a/ggml/src/ggml-opencl/kernels/cpy.cl
+++ b/ggml/src/ggml-opencl/kernels/cpy.cl
@@ -313,3 +313,88 @@ kernel void kernel_cpy_f32_f32_flat(
}
}
}
+
+kernel void kernel_cpy_i32_f32(
+ global int * src0,
+ ulong offset0,
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int ne02,
+ int ne03,
+ ulong nb00,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne0,
+ int ne1,
+ int ne2,
+ int ne3,
+ ulong nb0,
+ ulong nb1,
+ ulong nb2,
+ ulong nb3
+) {
+ src0 = (global int *) ((global char *) src0 + offset0);
+ dst = (global float *) ((global char *) dst + offsetd);
+
+ const ulong i01 = get_group_id(0);
+ const ulong i02 = get_group_id(1);
+ const ulong i03 = get_group_id(2);
+
+ for (ulong i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
+ const ulong n = ((i03 * ne02 + i02) * ne01 + i01) * ne00 + i00;
+ const ulong i0 = n % ne0;
+ const ulong i1 = n / ne0 % ne1;
+ const ulong i2 = n / ne0 / ne1 % ne2;
+ const ulong i3 = n / ne0 / ne1 / ne2;
+
+ global const int * src = (global const int *) ((global char *) src0 + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+ global float * out = (global float *) ((global char *) dst + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
+ out[0] = convert_float(src[0]);
+ }
+}
+
+
+kernel void kernel_cpy_f32_i32(
+ global float * src0,
+ ulong offset0,
+ global int * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int ne02,
+ int ne03,
+ ulong nb00,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne0,
+ int ne1,
+ int ne2,
+ int ne3,
+ ulong nb0,
+ ulong nb1,
+ ulong nb2,
+ ulong nb3
+) {
+ src0 = (global float *) ((global char *) src0 + offset0);
+ dst = (global int *) ((global char *) dst + offsetd);
+
+ const ulong i01 = get_group_id(0);
+ const ulong i02 = get_group_id(1);
+ const ulong i03 = get_group_id(2);
+
+ for (ulong i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
+ const ulong n = ((i03 * ne02 + i02) * ne01 + i01) * ne00 + i00;
+ const ulong i0 = n % ne0;
+ const ulong i1 = n / ne0 % ne1;
+ const ulong i2 = n / ne0 / ne1 % ne2;
+ const ulong i3 = n / ne0 / ne1 / ne2;
+
+ global const float * src = (global const float *) ((global char *) src0 + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+ global int * out = (global int *) ((global char *) dst + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
+ out[0] = convert_int_rtz(src[0]);
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/get_rows.cl b/ggml/src/ggml-opencl/kernels/get_rows.cl
index 9ae4fff09..f060e46ae 100644
--- a/ggml/src/ggml-opencl/kernels/get_rows.cl
+++ b/ggml/src/ggml-opencl/kernels/get_rows.cl
@@ -106,6 +106,53 @@ kernel void kernel_get_rows_f32(
}
}
+kernel void kernel_get_rows_i32(
+ global void * src0,
+ ulong offset0,
+ global int * src1,
+ ulong offset1,
+ global int * dst,
+ ulong offsetd,
+ int ne00,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne10,
+ ulong nb10,
+ ulong nb11,
+ ulong nb12,
+ ulong nb1,
+ ulong nb2,
+ ulong nb3
+) {
+ src0 = (global void*)((global char*)src0 + offset0);
+ src1 = (global int*)((global char*)src1 + offset1);
+ dst = (global int*)((global char*)dst + offsetd);
+
+ int nchunks = get_num_groups(0) / ne10;
+ int g = get_group_id(0);
+ int i10 = g / nchunks;
+ int chunk = g - i10 * nchunks;
+ int i11 = get_group_id(1);
+ int i12 = get_group_id(2);
+
+ int r = ((global int *) ((global char *) src1 + i12*nb12 + i11*nb11 + i10*nb10))[0];
+
+ int i02 = i11;
+ int i03 = i12;
+
+ global int * dst_row = (global int *) ((global char *) dst + i12*nb3 + i11*nb2 + i10*nb1);
+ global int * src_row = (global int *) ((global char *) src0 + r*nb01 + i02*nb02 + i03*nb03);
+
+ int span = (ne00 + nchunks - 1) / nchunks;
+ int start = chunk * span;
+ int end = min(start + span, ne00);
+
+ for (int ind = start + get_local_id(0); ind < end; ind += get_local_size(0)) {
+ dst_row[ind] = src_row[ind];
+ }
+}
+
kernel void kernel_get_rows_f16(
global void * src0,
ulong offset0,