Commit c66103334 for llama.cpp
commit c6610333490e1d28d2700982064b76baf2bcb08c
Author: Titaniumtown <titaniumtown@proton.me>
Date: Sat Nov 21 05:11:14 2026 -0800
sycl: add IQ4_NL and IQ4_XS weight reorder (#31998)
diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp
index 1e7edf3b8..e991e29e7 100644
--- a/ggml/src/ggml-sycl/convert.cpp
+++ b/ggml/src/ggml-sycl/convert.cpp
@@ -516,6 +516,16 @@ static void dequantize_row_iq4_nl_sycl(const void *vx, dst_t *y, const int64_t k
}
}
+template <ggml_type type, typename dst_t>
+static void dequantize_row_iq4_sycl_reorder(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) {
+ // flat range<1>, four qs bytes (8 outputs) per item
+ GGML_ASSERT(k % ggml_sycl_reordered::block_q_t<type>::traits::qk == 0);
+ stream->parallel_for(sycl::range<1>((size_t) (k / 8)),
+ [=](sycl::id<1> idx) {
+ dequantize_block_iq4_reorder_wide<type>(vx, y, k, idx);
+ });
+}
+
template <typename dst_t>
static void dequantize_row_mxfp4_sycl(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) {
const int nb = (k + QK_K - 1) / QK_K;
@@ -743,9 +753,17 @@ static to_t_sycl_t<dst_t> ggml_get_dequant_sycl(ggml_type type, ggml_tensor * ds
return dequantize_row_iq3_s_sycl;
}
case GGML_TYPE_IQ4_XS:
- return dequantize_row_iq4_xs_sycl;
+ if (is_reordered_src0(dst)) {
+ return dequantize_row_iq4_sycl_reorder<GGML_TYPE_IQ4_XS>;
+ } else {
+ return dequantize_row_iq4_xs_sycl;
+ }
case GGML_TYPE_IQ4_NL:
- return dequantize_row_iq4_nl_sycl;
+ if (is_reordered_src0(dst)) {
+ return dequantize_row_iq4_sycl_reorder<GGML_TYPE_IQ4_NL>;
+ } else {
+ return dequantize_row_iq4_nl_sycl;
+ }
case GGML_TYPE_MXFP4:
if (is_reordered_src0(dst)) {
return dequantize_row_mxfp4_sycl_reorder;
diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hpp
index eb94771d6..98f1aff6f 100644
--- a/ggml/src/ggml-sycl/dequantize.hpp
+++ b/ggml/src/ggml-sycl/dequantize.hpp
@@ -15,6 +15,7 @@
#include "common.hpp"
#include "convert.hpp"
+#include "quants.hpp"
typedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, dfloat2 & v);
typedef void (*dequantize_kernel_t_reorder)(const void *d, const int64_t ib, const void *qs,
@@ -674,6 +675,15 @@ static __dpct_inline__ void dequantize_iq4_nl(const void *vx, const int64_t ib,
v.y() = dequantize_one(iqs + 1);
}
+static __dpct_inline__ void dequantize_iq4_nl_reorder(const void * d_ptr, const int64_t ib, const void * qs,
+ const int iqs, dfloat2 & v) {
+ const float d = (float) ((const sycl::half *) d_ptr)[ib];
+ const uint8_t q = ((const uint8_t *) qs)[iqs];
+
+ v.x() = d * kvalues_iq4nl[q & 0xF];
+ v.y() = d * kvalues_iq4nl[q >> 4];
+}
+
static __dpct_inline__ void dequantize_iq4_xs(const void *vx, const int64_t ib,
const int iqs, dfloat2 &v) {
#if QK_K == 256
@@ -1758,6 +1768,60 @@ dequantize_block_iq4_xs(const void *__restrict__ vx, dst_t *__restrict__ yy,
}
}
+// reordered IQ4_NL / IQ4_XS: qs bytes iq .. iq + 3 of 32-value group g to y[iq + j] and y[iq + 16 + j], with y the
+// group's first output, in two vec<dst_t, 4> stores as for MXFP4; tail is the first byte after all qs
+template <ggml_type type, typename dst_t>
+static __dpct_inline__ void dequantize_iq4_reorder_quad(const uint8_t * __restrict__ x,
+ const uint8_t * __restrict__ tail, const int64_t g,
+ const int iq, dst_t * __restrict__ y) {
+ const uint8_t * qs = x + 16 * g + iq;
+ const float d = ggml_sycl_reordered::iq4_group_scale<type>(tail, g);
+
+ sycl::vec<dst_t, 4> lo, hi;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ lo[j] = d * kvalues_iq4nl[qs[j] & 0xf];
+ hi[j] = d * kvalues_iq4nl[qs[j] >> 4];
+ }
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + iq) = lo;
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + iq + 16) = hi;
+}
+
+// flat range over k / 8 items: item i takes group i / 4 from qs byte 4 * (i % 4)
+template <ggml_type type, typename dst_t>
+static void dequantize_block_iq4_reorder_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k,
+ const sycl::id<1> idx) {
+ const int64_t i = (int64_t) idx[0];
+ const int64_t g = i / 4;
+ const uint8_t * x = (const uint8_t *) vx;
+ dequantize_iq4_reorder_quad<type>(x, x + k / 2, g, (int) (4 * (i % 4)), yy + 32 * g);
+}
+
+// item t (0..31) of block ib dequantizes four qs bytes of sub-block t/4 into y, the output of block ib
+template <typename dst_t>
+static __dpct_inline__ void dequantize_block_iq4_xs_reorder(const void * __restrict__ vx, dst_t * __restrict__ y,
+ const int64_t ib, const int t, const int64_t nb) {
+ const int ib32 = t / 4;
+ const int iq = 4 * (t % 4);
+
+ const uint8_t * qs = (const uint8_t *) vx + ib * (QK_K / 2) + 16 * ib32 + iq;
+ const uint8_t * meta = (const uint8_t *) vx + nb * (QK_K / 2) + ib * offsetof(block_iq4_xs, qs);
+ const uint16_t scales_h = *(const uint16_t *) (meta + offsetof(block_iq4_xs, scales_h));
+ const uint8_t scales_l = meta[offsetof(block_iq4_xs, scales_l) + ib32 / 2];
+ const float d = (float) *(const ggml_half *) (meta + offsetof(block_iq4_xs, d)) *
+ ((((scales_l >> (4 * (ib32 % 2))) & 0xf) | (((scales_h >> (2 * ib32)) & 3) << 4)) - 32);
+
+ sycl::vec<dst_t, 4> lo, hi;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ lo[j] = d * kvalues_iq4nl[qs[j] & 0xf];
+ hi[j] = d * kvalues_iq4nl[qs[j] >> 4];
+ }
+ dst_t * yb = y + 32 * ib32 + iq;
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb) = lo;
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb + 16) = hi;
+}
+
template<typename dst_t>
static void dequantize_block_mxfp4(const void * __restrict__ vx, dst_t * __restrict__ yy,
const sycl::nd_item<3> &item_ct1) {
diff --git a/ggml/src/ggml-sycl/fused-gemm.cpp b/ggml/src/ggml-sycl/fused-gemm.cpp
index d0525e3a0..89096d561 100644
--- a/ggml/src/ggml-sycl/fused-gemm.cpp
+++ b/ggml/src/ggml-sycl/fused-gemm.cpp
@@ -1,5 +1,6 @@
#include "fused-gemm.hpp"
#include "fused-gemm-esimd.hpp"
+#include "quants.hpp"
#include <sycl/ext/oneapi/matrix/matrix.hpp>
@@ -421,14 +422,10 @@ static __dpct_inline__ void fg_pack_quarter(const float * __restrict__ t, typena
}
}
+// 32 values from 16 qs bytes, as in IQ4_NL blocks and IQ4_XS sub-blocks: the low nibbles fill the
+// first half of the step and the high nibbles the second, so the halves land at a[0..7] and a[8..15]
template <typename E>
-static __dpct_inline__ void fg_stage_a(const block_iq4_xs * __restrict__ xrow, const int kb, typename E::pair * a) {
- FG_SUPERBLOCK(block_iq4_xs);
- // low nibbles fill the first half of the step, high nibbles the second, so the two halves
- // land at a[0..7] and a[8..15] and no quarter loop is needed
- const float d = (float) blk->d *
- ((((blk->scales_l[ib / 2] >> (4 * (ib % 2))) & 0xf) | (((blk->scales_h >> (2 * ib)) & 3) << 4)) - 32);
- const uint8_t * q4 = blk->qs + 16 * ib;
+static __dpct_inline__ void fg_decode_iq4(const uint8_t * __restrict__ q4, const float d, typename E::pair * a) {
#pragma unroll
for (int j = 0; j < 8; ++j) {
a[j] = E::make(d * kvalues_iq4nl[q4[2 * j] & 0xf], d * kvalues_iq4nl[q4[2 * j + 1] & 0xf]);
@@ -436,6 +433,14 @@ static __dpct_inline__ void fg_stage_a(const block_iq4_xs * __restrict__ xrow, c
}
}
+template <typename E>
+static __dpct_inline__ void fg_stage_a(const block_iq4_xs * __restrict__ xrow, const int kb, typename E::pair * a) {
+ FG_SUPERBLOCK(block_iq4_xs);
+ const float d = (float) blk->d *
+ ((((blk->scales_l[ib / 2] >> (4 * (ib % 2))) & 0xf) | (((blk->scales_h >> (2 * ib)) & 3) << 4)) - 32);
+ fg_decode_iq4<E>(blk->qs + 16 * ib, d, a);
+}
+
template <typename E>
static __dpct_inline__ void fg_stage_a(const block_iq3_xxs * __restrict__ xrow, const int kb, typename E::pair * a) {
FG_SUPERBLOCK(block_iq3_xxs);
@@ -821,6 +826,22 @@ template <> struct fg_soa<block_q6_K> {
}
};
+template <ggml_type type> struct fg_soa_iq4 {
+ static constexpr bool supported = true;
+ // [qs][per-block records]; k step kb is 32-value group kb % groups of block ib
+ template <typename E>
+ static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int kb,
+ typename E::pair * a) {
+ constexpr int groups = ggml_sycl_reordered::block_q_t<type>::traits::qk / QK4_NL;
+ const int64_t g = (int64_t) ib * groups + kb % groups;
+ fg_decode_iq4<E>(x + 16 * g, ggml_sycl_reordered::iq4_group_scale<type>(x + nblocks * groups * 16, g), a);
+ }
+};
+
+template <> struct fg_soa<block_iq4_nl> : fg_soa_iq4<GGML_TYPE_IQ4_NL> {};
+
+template <> struct fg_soa<block_iq4_xs> : fg_soa_iq4<GGML_TYPE_IQ4_XS> {};
+
// one SG_ROWS x BN output tile: B columns [b0, b0 + BN) of packed_b go to dst columns [n0, n1),
// n1 - n0 <= BN
diff --git a/ggml/src/ggml-sycl/getrows.cpp b/ggml/src/ggml-sycl/getrows.cpp
index 7b53e3dcc..4830186d8 100644
--- a/ggml/src/ggml-sycl/getrows.cpp
+++ b/ggml/src/ggml-sycl/getrows.cpp
@@ -588,10 +588,24 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
src1_i32, (float *)dst->data, ctx.stream());
break;
case GGML_TYPE_IQ4_NL:
+ if (src0_reordered) {
+ get_rows_sycl_reorder<QK4_NL, QR4_NL, dequantize_iq4_nl_reorder>(dst->src[0], dst->src[1], dst, dst->src[0]->data,
+ src1_i32, (float *)dst->data, ctx.stream());
+ break;
+ }
get_rows_sycl<QK4_NL, 1, dequantize_iq4_nl>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,
src1_i32, (float *)dst->data, ctx.stream());
break;
case GGML_TYPE_IQ4_XS:
+ if (src0_reordered) {
+ const void * vx = dst->src[0]->data;
+ get_rows_sycl_reorder_k<1, QK_K / 8, 1>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(),
+ [=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<1> & item_ct1, int64_t nb) {
+ dequantize_block_iq4_xs_reorder(vx, y, ib, (int) item_ct1.get_local_id(0), nb);
+ GGML_UNUSED(scales_local);
+ });
+ break;
+ }
get_rows_sycl<QK_K, 1, dequantize_iq4_xs>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,
src1_i32, (float *)dst->data, ctx.stream());
break;
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index 3de5f1bd5..7dda19dfd 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -832,6 +832,8 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer,
case GGML_TYPE_Q6_K:
case GGML_TYPE_IQ3_XXS:
case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_IQ4_XS:
case GGML_TYPE_MXFP4:{
ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};
tensor->extra = extra;
@@ -4386,6 +4388,8 @@ inline bool ggml_sycl_supports_reorder_mmvq(enum ggml_type type) {
case GGML_TYPE_Q6_K:
case GGML_TYPE_IQ3_XXS:
case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_IQ4_XS:
return true;
default:
return false;
@@ -4975,6 +4979,20 @@ static bool reorder_qw_iq3_xxs(uint8_t * data_device, size_t size, dpct::queue_p
}, stream);
}
+// IQ4_NL / IQ4_XS: the qs go to the front, everything before them (d, plus the IQ4_XS scales) is the block's
+// record in the tail
+template <typename block_t>
+static bool reorder_qw_iq4(uint8_t * data_device, size_t size, dpct::queue_ptr stream) {
+ constexpr size_t record_size = offsetof(block_t, qs);
+ return reorder_qw_planar<block_t>(__func__, data_device, size, record_size, sizeof(block_t) - record_size,
+ [](const block_t & x, int64_t ib, uint8_t * tail, int64_t) {
+ const uint8_t * record = reinterpret_cast<const uint8_t *>(&x);
+ for (size_t j = 0; j < record_size; ++j) {
+ tail[ib * record_size + j] = record[j];
+ }
+ }, stream);
+}
+
static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) {
uint8_t * data_device = (uint8_t *) src0->data;
size_t ncols = src0->ne[0];
@@ -5022,6 +5040,10 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) {
return reorder_qw_iq3_xxs(data_device, size, stream);
case GGML_TYPE_IQ3_S:
return reorder_qw_iq3_s(data_device, size, stream);
+ case GGML_TYPE_IQ4_NL:
+ return reorder_qw_iq4<block_iq4_nl>(data_device, size, stream);
+ case GGML_TYPE_IQ4_XS:
+ return reorder_qw_iq4<block_iq4_xs>(data_device, size, stream);
default:
return false;
}
@@ -5384,8 +5406,8 @@ static bool ggml_sycl_mul_mat_glu_fused(ggml_backend_sycl_context & ctx, ggml_cg
}
// gate and up of one type the XMX kernel handles, it needs the reorder layout like the unfused mmvq path
- if (wg->type == wu->type && gate->src[1] == act && act->ne[1] >= ggml_sycl_xmx_min_cols(wu->type) &&
- act->ne[1] <= g_ggml_sycl_xmx_glu_max_cols) {
+ if (wg->type == wu->type && ggml_sycl_xmx_supports_type(wu->type) && gate->src[1] == act &&
+ act->ne[1] >= ggml_sycl_xmx_min_cols(wu->type) && act->ne[1] <= g_ggml_sycl_xmx_glu_max_cols) {
opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::MMVQ);
opt_for_reorder(&ctx, wg, act, gate, mul_mat_algo::MMVQ);
if (can_use_xmx_batch(ctx.device, wu, act) && can_use_xmx_batch(ctx.device, wg, act) &&
diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp
index 03464bfae..614b38b22 100644
--- a/ggml/src/ggml-sycl/mmvq.cpp
+++ b/ggml/src/ggml-sycl/mmvq.cpp
@@ -1776,10 +1776,11 @@ static void mul_mat_vec_iq3_xxs_q8_1_sycl(const void *vx, const void *vy,
}
template <ggml_type type>
-static void reorder_mul_mat_vec_iq3_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
- const int nrows, dpct::queue_ptr stream) {
- static_assert(type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S);
- GGML_ASSERT(ncols % QK_K == 0);
+static void reorder_mul_mat_vec_iq_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
+ const int nrows, dpct::queue_ptr stream) {
+ static_assert(type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ4_NL ||
+ type == GGML_TYPE_IQ4_XS);
+ GGML_ASSERT(ncols % ggml_sycl_reordered::block_q_t<type>::traits::qk == 0);
constexpr size_t num_subgroups = WARP_SIZE;
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
const sycl::range<3> block_nums(1, 1, block_num_y);
@@ -1795,13 +1796,14 @@ static void reorder_mul_mat_vec_iq3_q8_1_sycl(const void * vx, const void * vy,
}
template <ggml_type type, int ncols_dst>
-static void reorder_mul_mat_vec_iq3_q8_1_sycl_ncols(
+static void reorder_mul_mat_vec_iq_q8_1_sycl_ncols(
const void * vx, const void * vy, float * dst,
const int ncols, const int nrows,
const int stride_col_y_bytes, const int stride_col_dst,
dpct::queue_ptr stream) {
- static_assert(type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S);
- GGML_ASSERT(ncols % QK_K == 0);
+ static_assert(type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ4_NL ||
+ type == GGML_TYPE_IQ4_XS);
+ GGML_ASSERT(ncols % ggml_sycl_reordered::block_q_t<type>::traits::qk == 0);
constexpr size_t num_subgroups = WARP_SIZE;
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
const sycl::range<3> block_nums(1, 1, block_num_y);
@@ -1818,21 +1820,21 @@ static void reorder_mul_mat_vec_iq3_q8_1_sycl_ncols(
}
template <ggml_type type>
-static void reorder_mul_mat_vec_iq3_q8_1_sycl_switch_ncols(
+static void reorder_mul_mat_vec_iq_q8_1_sycl_switch_ncols(
const void * vx, const void * vy, float * dst,
const int ncols, const int nrows, const int ncols_dst,
const int stride_col_y_bytes, const int stride_col_dst,
dpct::queue_ptr stream) {
switch (ncols_dst) {
- case 1: reorder_mul_mat_vec_iq3_q8_1_sycl<type>(vx, vy, dst, ncols, nrows, stream); break;
- case 2: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
- case 3: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
- case 4: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
- case 5: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
- case 6: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
- case 7: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
- case 8: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
- default: GGML_ABORT("unsupported ncols_dst=%d for reordered IQ3 multi-col MMVQ", ncols_dst);
+ case 1: reorder_mul_mat_vec_iq_q8_1_sycl<type>(vx, vy, dst, ncols, nrows, stream); break;
+ case 2: reorder_mul_mat_vec_iq_q8_1_sycl_ncols<type, 2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+ case 3: reorder_mul_mat_vec_iq_q8_1_sycl_ncols<type, 3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+ case 4: reorder_mul_mat_vec_iq_q8_1_sycl_ncols<type, 4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+ case 5: reorder_mul_mat_vec_iq_q8_1_sycl_ncols<type, 5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+ case 6: reorder_mul_mat_vec_iq_q8_1_sycl_ncols<type, 6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+ case 7: reorder_mul_mat_vec_iq_q8_1_sycl_ncols<type, 7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+ case 8: reorder_mul_mat_vec_iq_q8_1_sycl_ncols<type, 8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+ default: GGML_ABORT("unsupported ncols_dst=%d for reordered IQ multi-col MMVQ", ncols_dst);
}
}
@@ -2518,13 +2520,13 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens
const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
const int stride_col_dst = dst->ne[0];
GGML_SYCL_DEBUG("Calling reordered IQ3_XXS multi-col MMVQ ncols=%d\n", (int) src1_ncols);
- reorder_mul_mat_vec_iq3_q8_1_sycl_switch_ncols<GGML_TYPE_IQ3_XXS>(
+ reorder_mul_mat_vec_iq_q8_1_sycl_switch_ncols<GGML_TYPE_IQ3_XXS>(
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
return;
} else {
GGML_SYCL_DEBUG("Calling reordered IQ3_XXS MMVQ\n");
- reorder_mul_mat_vec_iq3_q8_1_sycl<GGML_TYPE_IQ3_XXS>(
+ reorder_mul_mat_vec_iq_q8_1_sycl<GGML_TYPE_IQ3_XXS>(
src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
}
} else {
@@ -2539,13 +2541,13 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens
const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
const int stride_col_dst = dst->ne[0];
GGML_SYCL_DEBUG("Calling reordered IQ3_S multi-col MMVQ ncols=%d\n", (int) src1_ncols);
- reorder_mul_mat_vec_iq3_q8_1_sycl_switch_ncols<GGML_TYPE_IQ3_S>(
+ reorder_mul_mat_vec_iq_q8_1_sycl_switch_ncols<GGML_TYPE_IQ3_S>(
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
return;
} else {
GGML_SYCL_DEBUG("Calling reordered IQ3_S MMVQ\n");
- reorder_mul_mat_vec_iq3_q8_1_sycl<GGML_TYPE_IQ3_S>(
+ reorder_mul_mat_vec_iq_q8_1_sycl<GGML_TYPE_IQ3_S>(
src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
}
} else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
@@ -2561,10 +2563,42 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens
}
break;
case GGML_TYPE_IQ4_NL:
- mul_mat_vec_iq4_nl_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
+ const int stride_col_dst = dst->ne[0];
+ GGML_SYCL_DEBUG("Calling reordered IQ4_NL multi-col MMVQ ncols=%d\n", (int) src1_ncols);
+ reorder_mul_mat_vec_iq_q8_1_sycl_switch_ncols<GGML_TYPE_IQ4_NL>(
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
+ return;
+ } else {
+ GGML_SYCL_DEBUG("Calling reordered IQ4_NL MMVQ\n");
+ reorder_mul_mat_vec_iq_q8_1_sycl<GGML_TYPE_IQ4_NL>(
+ src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
+ }
+ } else {
+ mul_mat_vec_iq4_nl_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
+ }
break;
case GGML_TYPE_IQ4_XS:
- if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
+ const int stride_col_dst = dst->ne[0];
+ GGML_SYCL_DEBUG("Calling reordered IQ4_XS multi-col MMVQ ncols=%d\n", (int) src1_ncols);
+ reorder_mul_mat_vec_iq_q8_1_sycl_switch_ncols<GGML_TYPE_IQ4_XS>(
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
+ return;
+ } else {
+ GGML_SYCL_DEBUG("Calling reordered IQ4_XS MMVQ\n");
+ reorder_mul_mat_vec_iq_q8_1_sycl<GGML_TYPE_IQ4_XS>(
+ src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
+ }
+ } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
const int stride_col_y = src1_padded_col_size / QK8_1;
const int stride_col_dst = dst->ne[0];
GGML_SYCL_DEBUG("Calling mul_mat_vec_iq4_xs_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
diff --git a/ggml/src/ggml-sycl/quants.hpp b/ggml/src/ggml-sycl/quants.hpp
index 9fdcc22f2..79b5c8de5 100644
--- a/ggml/src/ggml-sycl/quants.hpp
+++ b/ggml/src/ggml-sycl/quants.hpp
@@ -14,6 +14,7 @@
#ifndef GGML_SYCL_QUANTS_HPP
#define GGML_SYCL_QUANTS_HPP
+#include <cstddef>
#include <utility>
#include "ggml-common.h"
@@ -225,6 +226,70 @@ template <> struct block_q_t<GGML_TYPE_IQ3_S> {
static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
};
+template <> struct block_q_t<GGML_TYPE_IQ4_NL> {
+ struct traits {
+ static constexpr uint32_t qk = QK4_NL;
+ static constexpr uint32_t qi = QI4_NL;
+ static constexpr uint32_t qr = QR4_NL;
+ static constexpr uint32_t vdr_mmvq = 4;
+ };
+
+ // Reordered layout: [qs (QK4_NL/2 per block)][d]
+ static constexpr int record_size = offsetof(block_iq4_nl, qs);
+
+ static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
+ return { block_index * (QK4_NL / 2), 0 };
+ }
+
+ static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
+ return { (ncols / QR4_NL * nrows) + block_index * sizeof(ggml_half), 0 };
+ }
+
+ static float group_scale(const uint8_t * record, const int /* group */) {
+ return *reinterpret_cast<const ggml_half *>(record);
+ }
+
+ static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
+};
+
+template <> struct block_q_t<GGML_TYPE_IQ4_XS> {
+ struct traits {
+ static constexpr uint32_t qk = QK_K;
+ static constexpr uint32_t qi = QI4_XS;
+ static constexpr uint32_t qr = QR4_XS;
+ static constexpr uint32_t vdr_mmvq = 4;
+ };
+
+ // Reordered layout: [qs (QK_K/2 per block)][{d, scales_h, scales_l} per block, as in block_iq4_xs]
+ static constexpr int record_size = offsetof(block_iq4_xs, qs);
+
+ static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
+ return { block_index * (QK_K / 2), 0 };
+ }
+
+ static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
+ const int nblocks = nrows * (ncols / QK_K);
+ return { nblocks * (QK_K / 2) + block_index * (int) offsetof(block_iq4_xs, qs), 0 };
+ }
+
+ static float group_scale(const uint8_t * record, const int group) {
+ const uint16_t scales_h = *reinterpret_cast<const uint16_t *>(record + offsetof(block_iq4_xs, scales_h));
+ const uint8_t scales_l = record[offsetof(block_iq4_xs, scales_l) + group / 2];
+ const int ls = ((scales_l >> (4 * (group % 2))) & 0xf) | (((scales_h >> (2 * group)) & 3) << 4);
+ return (float) *reinterpret_cast<const ggml_half *>(record + offsetof(block_iq4_xs, d)) * (ls - 32);
+ }
+
+ static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
+};
+
+// IQ4_NL blocks and IQ4_XS sub-blocks are 32-value groups of 16 qs bytes, so in both layouts the qs of
+// group g start at 16 * g; tail is the first byte after all qs, where the per-block records start
+template <ggml_type type> inline float iq4_group_scale(const uint8_t * tail, const int64_t g) {
+ using block_t = block_q_t<type>;
+ constexpr int64_t groups = block_t::traits::qk / QK4_NL;
+ return block_t::group_scale(tail + g / groups * block_t::record_size, (int) (g % groups));
+}
+
template <> struct block_q_t<GGML_TYPE_Q8_0> {
struct traits {
static constexpr uint32_t qk = QK8_0; // 32
diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp
index f547b20e6..f2f34667a 100644
--- a/ggml/src/ggml-sycl/vecdotq.hpp
+++ b/ggml/src/ggml-sycl/vecdotq.hpp
@@ -388,6 +388,14 @@ template <> struct reorder_vec_dot_shared_weights<GGML_TYPE_Q5_K> {
static constexpr bool value = true;
};
+template <> struct reorder_vec_dot_shared_weights<GGML_TYPE_IQ4_NL> {
+ static constexpr bool value = true;
+};
+
+template <> struct reorder_vec_dot_shared_weights<GGML_TYPE_IQ4_XS> {
+ static constexpr bool value = true;
+};
+
template <ggml_type T> struct reorder_vec_dot_shared_activations {
static constexpr bool value = false;
};
@@ -1956,4 +1964,88 @@ vec_dot_iq4_xs_q8_1(const void *__restrict__ vbq,
#endif
}
+// IQ4_NL blocks and IQ4_XS sub-blocks are both 32 values in 16 qs bytes: the low nibbles pair with
+// q8_1 ints 0..3 and the high nibbles with ints 4..7. A lane decodes them once for all columns.
+struct reorder_vec_dot_iq4_32 {
+ struct weights {
+ int v[8];
+ float d;
+ };
+
+ static __dpct_inline__ weights decode(const uint8_t * qs, const float d) {
+ const sycl::int4 q4 = *reinterpret_cast<const sycl::int4 *>(qs);
+ const uint8_t * values = (const uint8_t *) kvalues_iq4nl;
+
+ weights w;
+ get_int_from_table_16((uint32_t) q4.x(), values, w.v[0], w.v[4]);
+ get_int_from_table_16((uint32_t) q4.y(), values, w.v[1], w.v[5]);
+ get_int_from_table_16((uint32_t) q4.z(), values, w.v[2], w.v[6]);
+ get_int_from_table_16((uint32_t) q4.w(), values, w.v[3], w.v[7]);
+ w.d = d;
+ return w;
+ }
+
+ static __dpct_inline__ float dot32(const weights & w, const int8_t * q8, const sycl::half2 ds8) {
+ const int * u = (const int *) q8;
+ int sumi1 = 0;
+ int sumi2 = 0;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ sumi1 = dpct::dp4a(w.v[j + 0], u[j + 0], sumi1);
+ sumi2 = dpct::dp4a(w.v[j + 4], u[j + 4], sumi2);
+ }
+ return w.d * (float) ds8[0] * (sumi1 + sumi2);
+ }
+};
+
+template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_IQ4_NL> : reorder_vec_dot_iq4_32 {
+ static constexpr ggml_type gtype = GGML_TYPE_IQ4_NL;
+
+ // a lane holds a whole block, so iqs is always 0
+ __dpct_inline__ static weights load(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
+ const std::pair<int, int> d_offset, const int & /* iqs */) {
+ const uint8_t * base = static_cast<const uint8_t *>(vbq);
+ return decode(base + ibx_offset.first, *reinterpret_cast<const ggml_half *>(base + d_offset.first));
+ }
+
+ __dpct_inline__ static float dot(const weights & w, const int8_t * q8_1_quant_ptr, const sycl::half2 * q8_1_ds,
+ const int & /* iqs */) {
+ return dot32(w, q8_1_quant_ptr, *q8_1_ds);
+ }
+
+ __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
+ const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
+ const sycl::half2 * q8_1_ds, const int & iqs) {
+ return dot(load(vbq, ibx_offset, d_offset, iqs), q8_1_quant_ptr, q8_1_ds, iqs);
+ }
+};
+
+template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_IQ4_XS> : reorder_vec_dot_iq4_32 {
+ static constexpr ggml_type gtype = GGML_TYPE_IQ4_XS;
+
+ // iqs is the first qs int of the lane's sub-block
+ __dpct_inline__ static weights load(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
+ const std::pair<int, int> d_offset, const int & iqs) {
+ const uint8_t * base = static_cast<const uint8_t *>(vbq);
+ const uint8_t * meta = base + d_offset.first;
+ const int ib32 = iqs / 4;
+ const uint16_t scales_h = *reinterpret_cast<const uint16_t *>(meta + offsetof(block_iq4_xs, scales_h));
+ const uint8_t scales_l = meta[offsetof(block_iq4_xs, scales_l) + ib32 / 2];
+ const int ls = ((scales_l >> (4 * (ib32 % 2))) & 0xf) | (((scales_h >> (2 * ib32)) & 3) << 4);
+ const float d = (float) *reinterpret_cast<const ggml_half *>(meta + offsetof(block_iq4_xs, d));
+ return decode(base + ibx_offset.first + sizeof(int) * iqs, d * (ls - 32));
+ }
+
+ __dpct_inline__ static float dot(const weights & w, const int8_t * q8_1_quant_ptr, const sycl::half2 * q8_1_ds,
+ const int & iqs) {
+ return dot32(w, q8_1_quant_ptr + (iqs / 4) * QK8_1, q8_1_ds[iqs / 4]);
+ }
+
+ __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
+ const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
+ const sycl::half2 * q8_1_ds, const int & iqs) {
+ return dot(load(vbq, ibx_offset, d_offset, iqs), q8_1_quant_ptr, q8_1_ds, iqs);
+ }
+};
+
#endif // GGML_SYCL_VECDOTQ_HPP
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 711674958..a0d3349e6 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -10948,10 +10948,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_get_rows(GGML_TYPE_F32, 1, 8, 2, 1, 1, false));
for (ggml_type type : {GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K,
- GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S}) {
+ GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ4_XS}) {
test_cases.emplace_back(new test_get_rows_after_mul_mat(type));
}
- for (ggml_type type : {GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S}) {
+ for (ggml_type type : {GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ4_XS}) {
test_cases.emplace_back(new test_mul_mat_reorder_then_big_batch(type));
}
for (int n : {1, 3, 257}) {
@@ -10959,6 +10959,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
// many rows of a wide table, as a per-layer embedding lookup does
test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, 160, 100000, 32768, 1, 1, false));
+ // past the fused dequant GEMM's column limit the reordered weight goes through the conversion kernels
+ for (ggml_type type : {GGML_TYPE_IQ4_NL, GGML_TYPE_IQ4_XS}) {
+ test_cases.emplace_back(new test_mul_mat_reorder_then_big_batch(type, 128));
+ }
for (ggml_type type : all_types) {
for (int b : {1, 7}) {
for (bool v : {false, true}) {
@@ -12127,8 +12131,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, 8, 16*256, { 1, 1}, {1, 1}));
}
- // Multi-column MMVQ coverage for the Q4_K weight-reuse path and a Q5_K control.
- for (ggml_type type_a : { GGML_TYPE_Q4_K, GGML_TYPE_Q5_K }) {
+ // Multi-column MMVQ coverage for the Q4_K and IQ4 weight-reuse paths and a Q5_K control.
+ for (ggml_type type_a : { GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ4_XS }) {
for (int n = 1; n <= 8; ++n) {
test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 4096, n, 1024, { 1, 1 }, { 1, 1 }));
test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1023, n, 4096, { 1, 1 }, { 1, 1 }));