Commit 547208dca for llama.cpp
commit 547208dca250ceb3e4c8615598d043f2b602a527
Author: Todd Malsbary <todd.malsbary@intel.com>
Date: Sat Oct 17 21:02:38 2026 -0700
Restore ESIMD kernels on FFN path (#30239)
* Reenable ESIMD on the FFN path
Signed-off-by: Todd Malsbary <todd.malsbary@intel.com>
* Extend ESIMD fusion on FFN path to include Q8_0
Signed-off-by: Todd Malsbary <todd.malsbary@intel.com>
* Remove comment explicitly referencing ESIMD in test-backend-ops
Signed-off-by: Todd Malsbary <todd.malsbary@intel.com>
---------
Signed-off-by: Todd Malsbary <todd.malsbary@intel.com>
diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp
index 475509372..0bf6facec 100644
--- a/ggml/src/ggml-sycl/common.hpp
+++ b/ggml/src/ggml-sycl/common.hpp
@@ -37,6 +37,10 @@ namespace syclexp = sycl::ext::oneapi::experimental;
#endif
#endif
+#if defined(__INTEL_LLVM_COMPILER)
+ #define GGML_SYCL_DMMV_HAS_ESIMD
+#endif
+
#if GGML_SYCL_DNNL
#include "dnnl.hpp"
#include "dnnl_sycl.hpp"
diff --git a/ggml/src/ggml-sycl/dmmv.cpp b/ggml/src/ggml-sycl/dmmv.cpp
index 90f926829..d720bc1f8 100644
--- a/ggml/src/ggml-sycl/dmmv.cpp
+++ b/ggml/src/ggml-sycl/dmmv.cpp
@@ -10,7 +10,6 @@
#endif
#include <sycl/ext/intel/esimd.hpp>
#include "esimd.hpp"
- #define GGML_SYCL_DMMV_HAS_ESIMD
#endif
static void convert_f16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
@@ -1873,11 +1872,67 @@ static void dequantize_mul_mat_vec_q6_K_sycl(const void *vx, const float *y,
#ifdef GGML_SYCL_DMMV_HAS_ESIMD
using ggml_sycl_esimd::GGML_SYCL_DMMV_ESIMD_WG_SIZE;
-// generic reordered dequantize-matvec: each work-group owns a pair of
-// consecutive output rows and updates one 32-wide accumulator per row
-template <ggml_type T>
+// policies for dequantize_mul_mat_vec_reorder_esimd: row mapping and what to do with the two sums
+// plain mat-vec: a work-group owns two consecutive rows of one matrix
+struct esimd_dmmv_row_pair_policy {
+ const void * vx;
+ float * dst;
+
+ ESIMD_INLINE const void * w_a() const { return vx; }
+ ESIMD_INLINE const void * w_b() const { return vx; }
+
+ ESIMD_INLINE int row_a(int g) const { return g * 2; }
+ ESIMD_INLINE int row_b(int g) const { return g * 2 + 1; }
+ ESIMD_INLINE bool has_b(int g, int nrows) const { return row_b(g) < nrows; }
+
+ ESIMD_INLINE void epilogue(int g, int nrows, float sum_a, float sum_b) const {
+ dst[row_a(g)] = sum_a;
+ if (has_b(g, nrows)) {
+ dst[row_b(g)] = sum_b;
+ }
+ }
+};
+
+// fused gate+up+GLU: a work-group owns one row of both matrices; a is the gate, b the up
+struct esimd_dmmv_glu_policy {
+ const void * vgate;
+ const void * vup;
+ float * dst;
+ ggml_glu_op glu_op;
+
+ ESIMD_INLINE const void * w_a() const { return vgate; }
+ ESIMD_INLINE const void * w_b() const { return vup; }
+
+ ESIMD_INLINE int row_a(int g) const { return g; }
+ ESIMD_INLINE int row_b(int g) const { return g; }
+ ESIMD_INLINE bool has_b(int, int) const { return true; } // up always contributes
+
+ ESIMD_INLINE void epilogue(int g, int, float gate, float up) const {
+ using namespace sycl::ext::intel::esimd;
+
+ // SwiGLU: silu(gate)*up = gate*sigmoid(gate)*up. GEGLU's tanh form reduces
+ // to the same shape via tanh(y) = 2*sigmoid(2y) - 1, so only the sigmoid
+ // argument differs. sigmoid via exp2 (width-1 simd is a degenerate case for
+ // the EM pipe, so broadcast to 16 lanes); lane 0 holds the result.
+ const float GELU_COEF_A = 0.044715f;
+ const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
+ const float NEG_LOG2E = -1.44269504088896341f;
+
+ const float arg = (glu_op == GGML_GLU_OP_GEGLU) ?
+ 2.0f * SQRT_2_OVER_PI * gate * (1.0f + GELU_COEF_A * gate * gate) :
+ gate;
+
+ simd<float, 16> t = arg * NEG_LOG2E;
+ simd<float, 16> e = exp2(t);
+ dst[g] = gate / (1.0f + e[0]) * up;
+ }
+};
+
+// generic reordered dequantize-matvec: each work-group owns a pair of rows chosen by the policy
+// and updates one 32-wide accumulator per row
+template <ggml_type T, typename Policy>
ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd(
- const void * vx, const float * y, float * dst,
+ const Policy & policy, const float * y,
const int ncols, const int nrows,
sycl::local_accessor<float, 1> lmem,
const sycl::nd_item<1> & it) {
@@ -1886,41 +1941,41 @@ ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd(
const int num_blocks_per_row = ncols / QK_K;
const size_t nb = (size_t) nrows * num_blocks_per_row;
- const auto ps = traits::make_ptrs(vx, nb);
+ const auto pa = traits::make_ptrs(policy.w_a(), nb);
+ const auto pb = traits::make_ptrs(policy.w_b(), nb);
- const int tid = it.get_local_id(0);
- const int row_pair = it.get_group(0);
- const int row0 = row_pair * 2; // two consecutive output rows
- const bool has_row1 = row0 + 1 < nrows;
+ const int tid = it.get_local_id(0);
+ const int g = it.get_group(0);
+ const int row_a = policy.row_a(g);
+ const int row_b = policy.row_b(g);
+ const bool has_b = policy.has_b(g, nrows);
- // one 32-wide accumulator per output row (small footprint, no spill)
- simd<float, 32> acc0 = 0.0f;
- simd<float, 32> acc1 = 0.0f;
+ // one 32-wide accumulator per row (small footprint, no spill)
+ simd<float, 32> acc_a = 0.0f;
+ simd<float, 32> acc_b = 0.0f;
for (int ib = tid; ib < num_blocks_per_row; ib += GGML_SYCL_DMMV_ESIMD_WG_SIZE) {
simd<float, 256> y_vec = block_load<float, 256>(y + (size_t) ib * QK_K);
- const size_t bi0 = (size_t) (row0 + 0) * num_blocks_per_row + ib;
- const size_t bi1 = (size_t) (row0 + 1) * num_blocks_per_row + ib;
+ const size_t bi_a = (size_t) row_a * num_blocks_per_row + ib;
+ const size_t bi_b = (size_t) row_b * num_blocks_per_row + ib;
- traits::mac_pair(ps, bi0, ps, bi1, has_row1, y_vec, acc0, acc1);
+ traits::mac_pair(pa, bi_a, pb, bi_b, has_b, y_vec, acc_a, acc_b);
}
- lmem[tid * 2 + 0] = reduce<float>(acc0, std::plus<>{});
- lmem[tid * 2 + 1] = reduce<float>(acc1, std::plus<>{});
+ lmem[tid * 2 + 0] = reduce<float>(acc_a, std::plus<>{});
+ lmem[tid * 2 + 1] = reduce<float>(acc_b, std::plus<>{});
it.barrier(sycl::access::fence_space::local_space);
if (tid == 0) {
- float sum0 = 0.0f;
- float sum1 = 0.0f;
+ float sum_a = 0.0f;
+ float sum_b = 0.0f;
for (int p = 0; p < GGML_SYCL_DMMV_ESIMD_WG_SIZE; ++p) {
- sum0 += lmem[p * 2 + 0];
- sum1 += lmem[p * 2 + 1];
- }
- dst[row0 + 0] = sum0;
- if (has_row1) {
- dst[row0 + 1] = sum1;
+ sum_a += lmem[p * 2 + 0];
+ sum_b += lmem[p * 2 + 1];
}
+
+ policy.epilogue(g, nrows, sum_a, sum_b);
}
}
@@ -1936,7 +1991,7 @@ static void dequantize_mul_mat_vec_q2_K_sycl_reorder_esimd(const void *vx, const
sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q2_K>(
- vx, y, dst, ncols, nrows, lmem, it);
+ esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
});
});
}
@@ -1953,7 +2008,7 @@ static void dequantize_mul_mat_vec_q3_K_sycl_reorder_esimd(const void *vx, const
sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q3_K>(
- vx, y, dst, ncols, nrows, lmem, it);
+ esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
});
});
}
@@ -1970,7 +2025,7 @@ static void dequantize_mul_mat_vec_q4_K_sycl_reorder_esimd(const void *vx, const
sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q4_K>(
- vx, y, dst, ncols, nrows, lmem, it);
+ esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
});
});
}
@@ -1987,7 +2042,7 @@ static void dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(const void *vx, const
sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q5_K>(
- vx, y, dst, ncols, nrows, lmem, it);
+ esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
});
});
}
@@ -2004,7 +2059,7 @@ static void dequantize_mul_mat_vec_q6_K_sycl_reorder_esimd(const void *vx, const
sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q6_K>(
- vx, y, dst, ncols, nrows, lmem, it);
+ esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
});
});
}
@@ -2045,9 +2100,9 @@ ESIMD_INLINE void q8_0_mac_stripe(
}
}
-template <int WG>
+template <int WG, typename Policy>
ESIMD_INLINE void dequantize_mul_mat_vec_q8_0_reorder_esimd(
- const void * vx, const float * y, float * dst,
+ const Policy & policy, const float * y,
const int ncols, const int nrows,
sycl::local_accessor<float, 1> lmem,
const sycl::nd_item<1> & it) {
@@ -2057,84 +2112,137 @@ ESIMD_INLINE void dequantize_mul_mat_vec_q8_0_reorder_esimd(
const int nblk_row = ncols / QK8_0;
const size_t nb = (size_t) nrows * nblk_row;
- const int8_t * qs = (const int8_t *) vx;
- const sycl::half * d = (const sycl::half *) (qs + nb * QK8_0);
+ const int8_t * qs_a = (const int8_t *) policy.w_a();
+ const int8_t * qs_b = (const int8_t *) policy.w_b();
+ const sycl::half * d_a = (const sycl::half *) (qs_a + nb * QK8_0);
+ const sycl::half * d_b = (const sycl::half *) (qs_b + nb * QK8_0);
- const int tid = it.get_local_id(0);
- const int row_pair = it.get_group(0);
- const int row0 = row_pair * 2;
- const bool has_row1 = row0 + 1 < nrows;
+ const int tid = it.get_local_id(0);
+ const int g = it.get_group(0);
+ const bool has_b = policy.has_b(g, nrows);
- const size_t base0 = (size_t) row0 * nblk_row;
- const size_t base1 = has_row1 ? (size_t) (row0 + 1) * nblk_row : base0;
+ const size_t base_a = (size_t) policy.row_a(g) * nblk_row;
+ const size_t base_b = (size_t) policy.row_b(g) * nblk_row;
- simd<float, 32> acc0 = 0.0f;
- simd<float, 32> acc1 = 0.0f;
+ simd<float, 32> acc_a = 0.0f;
+ simd<float, 32> acc_b = 0.0f;
// Each thread processes one contiguous stripe.
int ib = 0;
for (; ib + WG * STRIPE <= nblk_row; ib += WG * STRIPE) {
const int b = ib + tid * STRIPE;
simd<float, 256> y_vec = block_load<float, 256>(y + (size_t) b * QK8_0);
- q8_0_mac_stripe<STRIPE>(qs + (base0 + b) * QK8_0, qs + (base1 + b) * QK8_0,
- d + base0 + b, d + base1 + b, has_row1, y_vec, acc0, acc1);
+ q8_0_mac_stripe<STRIPE>(qs_a + (base_a + b) * QK8_0, qs_b + (base_b + b) * QK8_0,
+ d_a + base_a + b, d_b + base_b + b, has_b, y_vec, acc_a, acc_b);
}
// Distribute remaining blocks across the work-group.
for (int b = ib + tid; b < nblk_row; b += WG) {
simd<float, 32> y_vec = block_load<float, 32>(y + (size_t) b * QK8_0);
- q8_0_mac_stripe<1>(qs + (base0 + b) * QK8_0, qs + (base1 + b) * QK8_0,
- d + base0 + b, d + base1 + b, has_row1, y_vec, acc0, acc1);
+ q8_0_mac_stripe<1>(qs_a + (base_a + b) * QK8_0, qs_b + (base_b + b) * QK8_0,
+ d_a + base_a + b, d_b + base_b + b, has_b, y_vec, acc_a, acc_b);
}
- lmem[tid * 2 + 0] = reduce<float>(acc0, std::plus<>{});
- lmem[tid * 2 + 1] = reduce<float>(acc1, std::plus<>{});
+ lmem[tid * 2 + 0] = reduce<float>(acc_a, std::plus<>{});
+ lmem[tid * 2 + 1] = reduce<float>(acc_b, std::plus<>{});
it.barrier(sycl::access::fence_space::local_space);
if (tid == 0) {
- float sum0 = 0.0f;
- float sum1 = 0.0f;
+ float sum_a = 0.0f;
+ float sum_b = 0.0f;
for (int p = 0; p < WG; ++p) {
- sum0 += lmem[p * 2 + 0];
- sum1 += lmem[p * 2 + 1];
- }
- dst[row0 + 0] = sum0;
- if (has_row1) {
- dst[row0 + 1] = sum1;
+ sum_a += lmem[p * 2 + 0];
+ sum_b += lmem[p * 2 + 1];
}
+
+ policy.epilogue(g, nrows, sum_a, sum_b);
}
}
-template <int WG>
-static void q8_0_esimd_launch(const void * vx, const float * y, float * dst, const int ncols,
- const int nrows, dpct::queue_ptr stream) {
- const int workgroups = (nrows + 1) / 2;
+template <int WG, typename Policy>
+static void q8_0_esimd_launch(const Policy & policy, const float * y, const int ncols, const int nrows,
+ const int workgroups, dpct::queue_ptr stream) {
stream->submit([&](sycl::handler & h) {
sycl::local_accessor<float, 1> lmem(sycl::range<1>(WG * 2), h);
h.parallel_for(
sycl::nd_range<1>(sycl::range<1>((size_t) workgroups * WG), sycl::range<1>(WG)),
[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
- dequantize_mul_mat_vec_q8_0_reorder_esimd<WG>(vx, y, dst, ncols, nrows, lmem, it);
+ dequantize_mul_mat_vec_q8_0_reorder_esimd<WG>(policy, y, ncols, nrows, lmem, it);
});
});
}
-static void dequantize_mul_mat_vec_q8_0_sycl_reorder_esimd(const void *vx, const float *y,
- float *dst, const int ncols,
- const int nrows,
- dpct::queue_ptr stream) {
+// Scale the work-group with the number of blocks per row.
+template <typename Policy>
+static void q8_0_esimd_launch_scaled(const Policy & policy, const float * y, const int ncols, const int nrows,
+ const int workgroups, dpct::queue_ptr stream) {
GGML_ASSERT(ncols % QK8_0 == 0);
- // Scale the work-group with the number of blocks per row.
const int nblk_row = ncols / QK8_0;
if (nblk_row >= 64) {
- q8_0_esimd_launch<8>(vx, y, dst, ncols, nrows, stream);
+ q8_0_esimd_launch<8>(policy, y, ncols, nrows, workgroups, stream);
} else if (nblk_row >= 32) {
- q8_0_esimd_launch<4>(vx, y, dst, ncols, nrows, stream);
+ q8_0_esimd_launch<4>(policy, y, ncols, nrows, workgroups, stream);
} else if (nblk_row >= 16) {
- q8_0_esimd_launch<2>(vx, y, dst, ncols, nrows, stream);
+ q8_0_esimd_launch<2>(policy, y, ncols, nrows, workgroups, stream);
} else {
- q8_0_esimd_launch<1>(vx, y, dst, ncols, nrows, stream);
+ q8_0_esimd_launch<1>(policy, y, ncols, nrows, workgroups, stream);
+ }
+}
+
+static void dequantize_mul_mat_vec_q8_0_sycl_reorder_esimd(const void *vx, const float *y,
+ float *dst, const int ncols,
+ const int nrows,
+ dpct::queue_ptr stream) {
+ q8_0_esimd_launch_scaled(esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, (nrows + 1) / 2, stream);
+}
+
+// ESIMD fused gate+up+GLU on reordered K-quant SOA weights: one output row (gate
+// and up) per work-group, two independent 32-wide accumulators per super-block.
+// Reuses the DMMV kernels' dequant+MAC primitive (esimd_reorder_q_traits<T>::mac_pair).
+template <ggml_type T>
+static void dequantize_mul_mat_vec_glu_reorder_esimd(const void * vx, const void * vgate,
+ const float * y, float * dst,
+ const int ncols, const int nrows,
+ const ggml_glu_op glu_op,
+ dpct::queue_ptr stream) {
+ GGML_ASSERT(ncols % QK_K == 0);
+ const int workgroups = nrows; // one output row (gate+up) per work-group
+ stream->submit([&](sycl::handler & h) {
+ sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);
+ h.parallel_for(
+ sycl::nd_range<1>(sycl::range<1>((size_t) workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
+ [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
+ dequantize_mul_mat_vec_reorder_esimd<T>(
+ esimd_dmmv_glu_policy{ vgate, vx, dst, glu_op }, y, ncols, nrows, lmem, it);
+ });
+ });
+}
+
+bool ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(enum ggml_type src0_type, enum ggml_glu_op glu_op,
+ const void * vx, const void * vgate, const float * y,
+ float * dst, int ncols, int nrows, dpct::queue_ptr stream) {
+ switch (src0_type) {
+ case GGML_TYPE_Q2_K:
+ dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q2_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
+ return true;
+ case GGML_TYPE_Q3_K:
+ dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q3_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
+ return true;
+ case GGML_TYPE_Q4_K:
+ dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q4_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
+ return true;
+ case GGML_TYPE_Q5_K:
+ dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q5_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
+ return true;
+ case GGML_TYPE_Q6_K:
+ dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q6_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
+ return true;
+ case GGML_TYPE_Q8_0:
+ q8_0_esimd_launch_scaled(esimd_dmmv_glu_policy{ vgate, vx, dst, glu_op }, y, ncols, nrows, nrows, stream);
+ return true;
+ default:
+ return false;
}
}
diff --git a/ggml/src/ggml-sycl/dmmv.hpp b/ggml/src/ggml-sycl/dmmv.hpp
index bd8373564..318e3489d 100644
--- a/ggml/src/ggml-sycl/dmmv.hpp
+++ b/ggml/src/ggml-sycl/dmmv.hpp
@@ -24,4 +24,13 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
const int64_t src1_ncols, const int64_t src1_padded_row_size,
const dpct::queue_ptr &stream);
+// Fused dense-FFN mat-vec: writes glu(gate . y, up . y) for same-type reordered
+// weights (Q2_K..Q6_K, Q8_0) and a single F32 activation column, using the ESIMD DMMV kernels.
+// vx is the up weight, vgate the gate weight. Only defined when GGML_SYCL_DMMV_HAS_ESIMD;
+// returns false if the type is unhandled.
+bool ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(
+ enum ggml_type src0_type, enum ggml_glu_op glu_op,
+ const void * vx, const void * vgate, const float * y,
+ float * dst, int ncols, int nrows, dpct::queue_ptr stream);
+
#endif // GGML_SYCL_DMMV_HPP
diff --git a/ggml/src/ggml-sycl/fusion.cpp b/ggml/src/ggml-sycl/fusion.cpp
index 79fe13a1d..cb4274e79 100644
--- a/ggml/src/ggml-sycl/fusion.cpp
+++ b/ggml/src/ggml-sycl/fusion.cpp
@@ -4,7 +4,7 @@
#include <algorithm>
// mul_mat(gate) + mul_mat(up) + GLU: graph shape and tensor properties only. Backend state
-// (weight layout, split buffers, DMMV) is checked by ggml_sycl_mul_mat_glu_mmvq_fused().
+// (weight layout, split buffers, DMMV) is checked by ggml_sycl_op_mul_mat_glu_fused().
static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const ggml_tensor * up,
const ggml_tensor * glu) {
// the fused epilogue implements these two; the rest fall back to the standalone GLU kernels
@@ -32,12 +32,22 @@ static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const gg
return false;
}
- // fused GEMVs walk whole QK_K super-blocks: the reorder kernel covers same-type
+ // fused K-quant GEMVs walk whole QK_K super-blocks: the reorder kernel covers same-type
// q4_K, the plain-layout kernel covers q5_K / iq4_xs pairs incl. mixed gate/up types
const bool reorder_pair = wu->type == GGML_TYPE_Q4_K && wg->type == GGML_TYPE_Q4_K;
const bool plain_pair = (wu->type == GGML_TYPE_Q5_K || wu->type == GGML_TYPE_IQ4_XS) &&
(wg->type == GGML_TYPE_Q5_K || wg->type == GGML_TYPE_IQ4_XS);
- if ((!reorder_pair && !plain_pair) || wu->ne[0] % QK_K != 0) {
+ // the ESIMD kernel covers same-type q2_K..q6_K and q8_0, single column only (as unfused DMMV)
+#ifdef GGML_SYCL_DMMV_HAS_ESIMD
+ const bool esimd_pair = g_ggml_sycl_enable_esimd && wu->type == wg->type && act->ne[1] == 1 &&
+ (wu->type == GGML_TYPE_Q2_K || wu->type == GGML_TYPE_Q3_K || wu->type == GGML_TYPE_Q4_K ||
+ wu->type == GGML_TYPE_Q5_K || wu->type == GGML_TYPE_Q6_K || wu->type == GGML_TYPE_Q8_0);
+#else
+ const bool esimd_pair = false;
+#endif
+ // q8_0 rows are walked in 32-wide blocks, so they only need to be a multiple of QK8_0
+ const int64_t row_align = esimd_pair && wu->type == GGML_TYPE_Q8_0 ? QK8_0 : QK_K;
+ if ((!reorder_pair && !plain_pair && !esimd_pair) || wu->ne[0] % row_align != 0) {
return false;
}
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index b380cbedf..3cc565d6b 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -46,9 +46,6 @@
# include <sycl/ext/oneapi/virtual_mem/virtual_mem.hpp>
# define GGML_SYCL_SUPPORT_VMM
#endif
-#if defined(__INTEL_LLVM_COMPILER)
- #define GGML_SYCL_DMMV_HAS_ESIMD
-#endif
#include <sycl/half_type.hpp>
#include "ggml.h"
@@ -5287,6 +5284,12 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, gg
return false;
}
+ // the plain kernel only covers q5_K / iq4_xs pairs: bail before quantizing the activation
+ const auto plain_type = [](ggml_type t) { return t == GGML_TYPE_Q5_K || t == GGML_TYPE_IQ4_XS; };
+ if (!plain_type(wu->type) || !plain_type(wg->type)) {
+ return false;
+ }
+
// log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises
scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (plain layout)");
@@ -5311,9 +5314,42 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, gg
GGML_UNUSED(gate);
}
+#ifdef GGML_SYCL_DMMV_HAS_ESIMD
+// {mul_mat(gate), mul_mat(up), GLU} on the reorder layout via the ESIMD DMMV kernel, for
+// same-type K-quant or Q8_0 pairs and a single activation column. It reads the F32 activation
+// directly, so one launch replaces three and no q8_1 quantization is needed.
+static bool ggml_sycl_mul_mat_glu_dmmv_esimd(ggml_backend_sycl_context & ctx, ggml_tensor * glu,
+ ggml_tensor * gate, ggml_tensor * up, const ggml_tensor * wu,
+ const ggml_tensor * wg, const ggml_tensor * act) {
+ // the kernel reads the activation while writing the output; glu may reuse the dead activation's memory
+ const char * act_begin = (const char *) act->data;
+ const char * glu_begin = (const char *) glu->data;
+ if (act_begin < glu_begin + ggml_nbytes(glu) && glu_begin < act_begin + ggml_nbytes(act)) {
+ return false;
+ }
+
+ // install the reorder (SoA) layout the kernel needs, as the unfused DMMV path would
+ opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::DMMV);
+ opt_for_reorder(&ctx, wg, act, gate, mul_mat_algo::DMMV);
+
+ const auto * extra_u = static_cast<const ggml_tensor_extra_gpu *>(wu->extra);
+ const auto * extra_g = static_cast<const ggml_tensor_extra_gpu *>(wg->extra);
+ if (!extra_u || !extra_g || !extra_u->optimized_feature.reorder || !extra_g->optimized_feature.reorder) {
+ return false;
+ }
+
+ // log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises
+ scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (ESIMD)");
+
+ return ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(wu->type, ggml_get_glu_op(glu), wu->data, wg->data,
+ (const float *) act->data, (float *) glu->data,
+ (int) wu->ne[0], (int) wu->ne[1], ctx.stream());
+}
+#endif
+
// Fused dense-FFN mat-vec for the {mul_mat(gate), mul_mat(up), GLU} subgraph at node_idx.
// Returns false if it declined, in which case the caller runs the three nodes normally.
-static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) {
+static bool ggml_sycl_mul_mat_glu_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) {
if (!ggml_sycl_can_fuse(cgraph, node_idx, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }, {})) {
return false;
}
@@ -5331,6 +5367,16 @@ static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, gg
return false;
}
+#ifdef GGML_SYCL_DMMV_HAS_ESIMD
+ // single-column same-type K-quant or Q8_0 pairs take the ESIMD kernel, as the unfused mul-mats
+ // would via DMMV; if the reorder layout could not be installed, fall through to mmvq
+ if (g_ggml_sycl_enable_esimd && wu->type == wg->type && act->ne[1] == 1 &&
+ ggml_sycl_supports_reorder_esimd(wu->type) &&
+ ggml_sycl_mul_mat_glu_dmmv_esimd(ctx, glu, gate, up, wu, wg, act)) {
+ return true;
+ }
+#endif
+
// with DMMV prioritised the unfused path would not have gone through mmvq at all
if (g_ggml_sycl_prioritize_dmmv) {
return false;
@@ -6594,7 +6640,7 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc
continue;
}
- if (node->op == GGML_OP_MUL_MAT && ggml_sycl_mul_mat_glu_mmvq_fused(*sycl_ctx, cgraph, i)) {
+ if (node->op == GGML_OP_MUL_MAT && ggml_sycl_mul_mat_glu_fused(*sycl_ctx, cgraph, i)) {
i += 2;
continue;
}
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 8826b274b..011f93865 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -11927,6 +11927,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
true, 16, 8, b, false, true, false));
}
+ for (int64_t k : {96, 896, 1024, 2880}) {
+ for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) {
+ test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q8_0, glu_op, 1, 32, k,
+ false, 16, 8, false, false, true, false, {1, 1}));
+ }
+ }
+
// Fused row-pair coverage: minimum rows, an even pair, and an odd tail.
// TODO: the max_nmse_err() for these cases is not estimated correctly causing sporadic false failures.
//for (ggml_glu_op glu_op : { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU }) {