Commit 53eb66943 for llama.cpp
commit 53eb66943217eb32fb2b3b6240d0320d320c1768
Author: lhez <lih@qti.qualcomm.com>
Date: Mon Nov 9 00:10:59 2026 -0800
opencl: add f32 MUL_MAT_ID (#31783)
* opencl: add f32 mul_mat_id
* opencl: handle -1 index
* opencl: make loading moe_sort_by_expert general
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt
index 0ef7f8fe6..e2a82d5bc 100644
--- a/ggml/src/ggml-opencl/CMakeLists.txt
+++ b/ggml/src/ggml-opencl/CMakeLists.txt
@@ -195,6 +195,8 @@ set(GGML_OPENCL_KERNELS
mul_mv_mxfp4_f32
mul_mv_mxfp4_f32_flat
mul_mv_id_q4_0_f32_8x_flat
+ mul_mv_id_f32_f32
+ mul_mm_id_f32_f32
mul_mv_id_q8_0_f32
mul_mv_id_q8_0_f32_flat
mul_mv_id_mxfp4_f32
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index a985e0882..bf7987691 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -1185,6 +1185,8 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_moe_zero_dst;
cl_kernel kernel_moe_combine_f32 = nullptr; // fused router-weight mul + cross-expert sum
cl_kernel kernel_moe_combine_bias_f32 = nullptr; // same, with the down-projection bias add folded in
+ cl_kernel kernel_mul_mv_id_f32_f32;
+ cl_kernel kernel_mul_mm_id_f32_f32;
cl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat;
cl_kernel kernel_mul_mv_id_q8_0_f32, kernel_mul_mv_id_q8_0_f32_flat;
cl_kernel kernel_mul_mv_id_mxfp4_f32;
@@ -1997,6 +1999,26 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
+ // moe_sort_by_expert
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "moe_sort_by_expert.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("moe_sort_by_expert.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+
+ CL_CHECK((backend_ctx->kernel_moe_histogram = clCreateKernel(prog, "kernel_moe_histogram", &err), err));
+ CL_CHECK((backend_ctx->kernel_moe_scan = clCreateKernel(prog, "kernel_moe_scan", &err), err));
+ CL_CHECK((backend_ctx->kernel_moe_fill = clCreateKernel(prog, "kernel_moe_fill", &err), err));
+ CL_CHECK((backend_ctx->kernel_moe_scatter = clCreateKernel(prog, "kernel_moe_scatter", &err), err));
+ CL_CHECK((backend_ctx->kernel_moe_scatter_stable = clCreateKernel(prog, "kernel_moe_scatter_stable", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
// moe_add_id_glu
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -4869,6 +4891,36 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
+ // mul_mv_id_f32_f32
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mv_id_f32_f32.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mv_id_f32_f32.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_mul_mv_id_f32_f32 = clCreateKernel(prog, "kernel_mul_mv_id_f32_f32", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // mul_mm_id_f32_f32
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mm_id_f32_f32.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mm_id_f32_f32.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_mul_mm_id_f32_f32 = clCreateKernel(prog, "kernel_mul_mm_id_f32_f32", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
// mul_mv_id_q4_0_f32_8x_flat
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -7357,27 +7409,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
- // moe_sort_by_expert
- {
-#ifdef GGML_OPENCL_EMBED_KERNELS
- const std::string kernel_src {
- #include "moe_sort_by_expert.cl.h"
- };
-#else
- const std::string kernel_src = read_file("moe_sort_by_expert.cl");
-#endif
- cl_program prog =
- build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts);
-
- CL_CHECK((backend_ctx->kernel_moe_histogram = clCreateKernel(prog, "kernel_moe_histogram", &err), err));
- CL_CHECK((backend_ctx->kernel_moe_scan = clCreateKernel(prog, "kernel_moe_scan", &err), err));
- CL_CHECK((backend_ctx->kernel_moe_fill = clCreateKernel(prog, "kernel_moe_fill", &err), err));
- CL_CHECK((backend_ctx->kernel_moe_scatter = clCreateKernel(prog, "kernel_moe_scatter", &err), err));
- CL_CHECK((backend_ctx->kernel_moe_scatter_stable = clCreateKernel(prog, "kernel_moe_scatter_stable", &err), err));
- CL_CHECK(clReleaseProgram(prog));
- GGML_LOG_CONT(".");
- }
-
// gemv_noshuffle_q6_k_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -12017,6 +12048,12 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
}
return false;
case GGML_OP_MUL_MAT_ID:
+ if (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) {
+ return op->src[1]->type == GGML_TYPE_F32 && op->src[2]->type == GGML_TYPE_I32 && op->type == GGML_TYPE_F32 &&
+ op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && op->src[1]->nb[0] == sizeof(float) &&
+ op->nb[0] == sizeof(float) && op->src[0]->ne[0] <= INT_MAX - 256 && op->ne[0] <= INT_MAX - 64 &&
+ op->src[1]->ne[1] <= INT_MAX && op->ne[1] <= INT_MAX && op->ne[2] <= INT_MAX - 64;
+ }
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
if ((ggml_get_op_params_i32(op, 3) == GGML_PREC_BF16 || ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) &&
use_adreno_moe_kernels(backend_ctx, op->src[0])) {
@@ -34092,7 +34129,11 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src,
cl_mem emap_buf = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
CL_CHECK(err);
- backend_ctx->prealloc_hist.allocate(backend_ctx->context, sizeof(int) * ne02);
+ if (sizeof(int) * ne02 > backend_ctx->prealloc_hist.size) {
+ backend_ctx->prealloc_hist.allocate(backend_ctx->context, sizeof(int) * ne02);
+ const int zero = 0;
+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_hist.buffer, &zero, sizeof(zero), 0, sizeof(int) * ne02, 0, nullptr, nullptr));
+ }
region.origin = 0;
region.size = sizeof(int) * ne02;
cl_mem hist_buf = clCreateSubBuffer(backend_ctx->prealloc_hist.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
@@ -34233,7 +34274,108 @@ static bool use_adreno_moe_gemv_small_batch(const ggml_backend_opencl_context *
2 * (int64_t) n_tokens * n_used < (int64_t) n_expert;
}
+static void ggml_cl_mul_mat_id_float(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context;
+ cl_kernel kernel = backend_ctx->kernel_mul_mv_id_f32_f32;
+ const ggml_tensor * src2 = dst->src[2];
+ ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) src0->extra;
+ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *) src1->extra;
+ ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *) src2->extra;
+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra;
+
+ const cl_ulong offset0 = extra0->offset + src0->view_offs;
+ const cl_ulong offset1 = extra1->offset + src1->view_offs;
+ const cl_ulong offset2 = extra2->offset + src2->view_offs;
+ const cl_ulong offsetd = extrad->offset + dst->view_offs;
+
+ const int ncols = src0->ne[0];
+ const int ne11 = src1->ne[1];
+ const int nrows = dst->ne[0];
+ // BF16 tensors are stored as F16 on the device. Activations remain F32.
+ const int src0_f16 = src0->type == GGML_TYPE_F32 ? 0 : 1;
+
+ const cl_ulong nb01 = src0->nb[1];
+ const cl_ulong nb02 = src0->nb[2];
+ const cl_ulong nb11 = src1->nb[1];
+ const cl_ulong nb12 = src1->nb[2];
+ const cl_ulong nb20 = src2->nb[0];
+ const cl_ulong nb21 = src2->nb[1];
+ const cl_ulong nb1 = dst->nb[1];
+ const cl_ulong nb2 = dst->nb[2];
+
+ // The router uses the IDs row stride as its expert count.
+ const size_t router_experts = src2->nb[1]/sizeof(int32_t);
+ if (src1->ne[2] >= 16 && ncols >= 64 && nrows >= 32 && ggml_is_contiguous(dst) &&
+ src2->nb[0] == sizeof(int32_t) && src2->nb[1] % sizeof(int32_t) == 0 &&
+ router_experts >= (size_t) src0->ne[2] && router_experts <= 1024 &&
+ dst->ne[1]*dst->ne[2] >= (int64_t) router_experts*4 &&
+ dst->ne[1]*dst->ne[2] >= 16384/nrows &&
+ dst->ne[1]*dst->ne[2] <= INT_MAX - 32*(int64_t) router_experts) {
+ const int nused = dst->ne[1];
+ const size_t max_tiles = (size_t) nused*dst->ne[2]/32 + router_experts;
+ moe_router_reoerder(backend, src2, nused);
+ backend_ctx->toggle_reorder = true;
+ const float zero = 0.0f;
+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, extrad->data_device, &zero, sizeof(zero), offsetd, ggml_nbytes(dst), 0, nullptr, nullptr));
+ kernel = backend_ctx->kernel_mul_mm_id_f32_f32;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &backend_ctx->prealloc_post_router.buffer));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &backend_ctx->prealloc_emap.buffer));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &backend_ctx->prealloc_total_tiles.buffer));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ncols));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &nrows));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &nused));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &src0_f16));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb01));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb02));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb11));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb12));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb1));
+ CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb2));
+ size_t global_work_size[] = {(size_t) ((nrows + 63)/64)*128, max_tiles};
+ size_t local_work_size[] = {128, 1};
+ backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst);
+ return;
+ }
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ncols));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &src0_f16));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb11));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb12));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb20));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb21));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb1));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb2));
+ CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &nrows));
+
+ size_t global_work_size[] = {(size_t) ((dst->ne[0] + 3)/4)*64, (size_t) dst->ne[1], (size_t) dst->ne[2]};
+ size_t local_work_size[] = {64, 1, 1};
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+}
+
static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ if (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16) {
+ ggml_cl_mul_mat_id_float(backend, src0, src1, dst);
+ return;
+ }
+
GGML_ASSERT(src0);
GGML_ASSERT(src0->extra);
GGML_ASSERT(src1);
diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_id_f32_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mm_id_f32_f32.cl
new file mode 100644
index 000000000..fdcbd6113
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mm_id_f32_f32.cl
@@ -0,0 +1,98 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#define BM 64
+#define BN 32
+#define BK 16
+#define TM 4
+#define TN 4
+
+kernel void kernel_mul_mm_id_f32_f32(
+ global const char * src0, ulong offset0,
+ global const char * src1, ulong offset1,
+ global const int * routes,
+ global const ushort * experts,
+ global const int * total_tiles,
+ global char * dst, ulong offsetd,
+ int ncols, int nrows, int ne11, int nused, int src0_f16,
+ ulong nb01, ulong nb02, ulong nb11, ulong nb12, ulong nb1, ulong nb2) {
+ const int tile = get_group_id(1);
+ if (tile >= total_tiles[0]) {
+ return;
+ }
+ const int tid = get_local_id(0);
+ const int row = get_group_id(0)*BM;
+ const int mr = tid % (BM/TM)*TM;
+ const int nr = tid / (BM/TM)*TN;
+ src0 += offset0 + experts[tile]*nb02;
+ src1 += offset1;
+ dst += offsetd;
+ local float a[BM*BK];
+ local float b[BN*BK];
+ float sums[TM*TN] = {0.0f};
+
+ for (int base = 0; base < ncols; base += BK) {
+ for (int i = tid; i < BM*BK/4; i += 128) {
+ const int r = i/(BK/4);
+ const int k = base + i%(BK/4)*4;
+ float4 value = 0.0f;
+ if (row + r < nrows && k + 3 < ncols) {
+ global const char * arow = src0 + (row + r)*nb01;
+ value = src0_f16 ? convert_float4(vload4(0, (global const half *) arow + k)) : vload4(0, (global const float *) arow + k);
+ } else if (row + r < nrows) {
+ float tail[4] = {0.0f};
+ for (int j = 0; j < 4 && k + j < ncols; ++j) {
+ global const char * arow = src0 + (row + r)*nb01;
+ tail[j] = src0_f16 ? (float) ((global const half *) arow)[k + j] : ((global const float *) arow)[k + j];
+ }
+ value = (float4) (tail[0], tail[1], tail[2], tail[3]);
+ }
+ a[(i%(BK/4)*4 + 0)*BM + r] = value.s0;
+ a[(i%(BK/4)*4 + 1)*BM + r] = value.s1;
+ a[(i%(BK/4)*4 + 2)*BM + r] = value.s2;
+ a[(i%(BK/4)*4 + 3)*BM + r] = value.s3;
+ }
+ for (int i = tid; i < BN*BK/4; i += 128) {
+ const int route = routes[tile*BN + i/(BK/4)];
+ const int k = base + i%(BK/4)*4;
+ float4 value = 0.0f;
+ if (route >= 0) {
+ global const float * brow = (global const float *) (src1 + (route%nused%ne11)*nb11 + (route/nused)*nb12);
+ if (k + 3 < ncols) {
+ value = vload4(0, brow + k);
+ } else {
+ float tail[4] = {0.0f};
+ for (int j = 0; j < 4 && k + j < ncols; ++j) {
+ tail[j] = brow[k + j];
+ }
+ value = (float4) (tail[0], tail[1], tail[2], tail[3]);
+ }
+ }
+ b[(i%(BK/4)*4 + 0)*BN + i/(BK/4)] = value.s0;
+ b[(i%(BK/4)*4 + 1)*BN + i/(BK/4)] = value.s1;
+ b[(i%(BK/4)*4 + 2)*BN + i/(BK/4)] = value.s2;
+ b[(i%(BK/4)*4 + 3)*BN + i/(BK/4)] = value.s3;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ #pragma unroll
+ for (int k = 0; k < BK; ++k) {
+ #pragma unroll
+ for (int n = 0; n < TN; ++n) {
+ const float bv = b[k*BN + nr + n];
+ #pragma unroll
+ for (int m = 0; m < TM; ++m) {
+ sums[n*TM + m] = mad(a[k*BM + mr + m], bv, sums[n*TM + m]);
+ }
+ }
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+ for (int n = 0; n < TN; ++n) {
+ const int route = routes[tile*BN + nr + n];
+ if (route >= 0) {
+ global float * out = (global float *) (dst + (route%nused)*nb1 + (route/nused)*nb2);
+ for (int m = 0; m < TM && row + mr + m < nrows; ++m) {
+ out[row + mr + m] = sums[n*TM + m];
+ }
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_id_f32_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_id_f32_f32.cl
new file mode 100644
index 000000000..b82ca90c8
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_id_f32_f32.cl
@@ -0,0 +1,89 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_qcom_reqd_sub_group_size
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+__attribute__((qcom_reqd_sub_group_size("half")))
+#endif
+kernel void kernel_mul_mv_id_f32_f32(
+ global const char * src0,
+ ulong offset0,
+ global const char * src1,
+ ulong offset1,
+ global const char * ids,
+ ulong offset2,
+ global char * dst,
+ ulong offsetd,
+ int ncols,
+ int ne11,
+ int src0_f16,
+ ulong nb01,
+ ulong nb02,
+ ulong nb11,
+ ulong nb12,
+ ulong nb20,
+ ulong nb21,
+ ulong nb1,
+ ulong nb2,
+ int nrows) {
+ const int row = get_group_id(0)*4;
+ const int slot = get_group_id(1);
+ const int token = get_group_id(2);
+ const int tid = get_local_id(0);
+ const int expert = *(global const int *) (ids + offset2 + slot*nb20 + token*nb21);
+ dst += offsetd + slot*nb1 + token*nb2;
+ if (expert == -1) {
+ if (tid == 0) {
+ for (int r = 0; r < 4 && row + r < nrows; ++r) {
+ ((global float *) dst)[row + r] = 0.0f;
+ }
+ }
+ return;
+ }
+ src0 += offset0 + row*nb01 + expert*nb02;
+ src1 += offset1 + (slot % ne11)*nb11 + token*nb12;
+ float sum[4] = {0.0f, 0.0f, 0.0f, 0.0f};
+ for (int i = tid*4; i + 3 < ncols; i += 64*4) {
+ const float4 b = vload4(0, (global const float *) src1 + i);
+ for (int r = 0; r < 4 && row + r < nrows; ++r) {
+ global const char * arow = src0 + r*nb01;
+ const float4 a = src0_f16 ? convert_float4(vload4(0, (global const half *) arow + i)) : vload4(0, (global const float *) arow + i);
+ sum[r] += dot(a, b);
+ }
+ }
+ for (int i = ncols/4*4 + tid; i < ncols; i += 64) {
+ const float b = ((global const float *) src1)[i];
+ for (int r = 0; r < 4 && row + r < nrows; ++r) {
+ global const char * arow = src0 + r*nb01;
+ const float a = src0_f16 ? (float) ((global const half *) arow)[i] : ((global const float *) arow)[i];
+ sum[r] = mad(a, b, sum[r]);
+ }
+ }
+#ifdef cl_qcom_reqd_sub_group_size
+ for (int r = 0; r < 4; ++r) {
+ sum[r] = sub_group_reduce_add(sum[r]);
+ }
+#else
+ local float sums[4][64];
+ for (int r = 0; r < 4; ++r) {
+ sums[r][tid] = sum[r];
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ for (int stride = 32; stride > 0; stride /= 2) {
+ for (int r = 0; r < 4; ++r) {
+ if (tid < stride) {
+ sums[r][tid] += sums[r][tid + stride];
+ }
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+ for (int r = 0; r < 4; ++r) {
+ sum[r] = sums[r][0];
+ }
+#endif
+ if (tid == 0) {
+ for (int r = 0; r < 4 && row + r < nrows; ++r) {
+ ((global float *) dst)[row + r] = sum[r];
+ }
+ }
+}