Commit c66103334 for llama.cpp

commit c6610333490e1d28d2700982064b76baf2bcb08c
Author: Titaniumtown <titaniumtown@proton.me>
Date:   Sat Nov 21 05:11:14 2026 -0800

    sycl: add IQ4_NL and IQ4_XS weight reorder (#31998)

diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp
index 1e7edf3b8..e991e29e7 100644
--- a/ggml/src/ggml-sycl/convert.cpp
+++ b/ggml/src/ggml-sycl/convert.cpp
@@ -516,6 +516,16 @@ static void dequantize_row_iq4_nl_sycl(const void *vx, dst_t *y, const int64_t k
       }
 }

+template <ggml_type type, typename dst_t>
+static void dequantize_row_iq4_sycl_reorder(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) {
+    // flat range<1>, four qs bytes (8 outputs) per item
+    GGML_ASSERT(k % ggml_sycl_reordered::block_q_t<type>::traits::qk == 0);
+    stream->parallel_for(sycl::range<1>((size_t) (k / 8)),
+        [=](sycl::id<1> idx) {
+            dequantize_block_iq4_reorder_wide<type>(vx, y, k, idx);
+        });
+}
+
 template <typename dst_t>
 static void dequantize_row_mxfp4_sycl(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) {
     const int nb = (k + QK_K - 1) / QK_K;
@@ -743,9 +753,17 @@ static to_t_sycl_t<dst_t> ggml_get_dequant_sycl(ggml_type type, ggml_tensor * ds
                 return dequantize_row_iq3_s_sycl;
             }
         case GGML_TYPE_IQ4_XS:
-            return dequantize_row_iq4_xs_sycl;
+            if (is_reordered_src0(dst)) {
+                return dequantize_row_iq4_sycl_reorder<GGML_TYPE_IQ4_XS>;
+            } else {
+                return dequantize_row_iq4_xs_sycl;
+            }
         case GGML_TYPE_IQ4_NL:
-            return dequantize_row_iq4_nl_sycl;
+            if (is_reordered_src0(dst)) {
+                return dequantize_row_iq4_sycl_reorder<GGML_TYPE_IQ4_NL>;
+            } else {
+                return dequantize_row_iq4_nl_sycl;
+            }
         case GGML_TYPE_MXFP4:
             if (is_reordered_src0(dst)) {
                 return dequantize_row_mxfp4_sycl_reorder;
diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hpp
index eb94771d6..98f1aff6f 100644
--- a/ggml/src/ggml-sycl/dequantize.hpp
+++ b/ggml/src/ggml-sycl/dequantize.hpp
@@ -15,6 +15,7 @@

 #include "common.hpp"
 #include "convert.hpp"
+#include "quants.hpp"

 typedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, dfloat2 & v);
 typedef void (*dequantize_kernel_t_reorder)(const void *d, const int64_t ib, const void *qs,
@@ -674,6 +675,15 @@ static __dpct_inline__ void dequantize_iq4_nl(const void *vx, const int64_t ib,
     v.y() = dequantize_one(iqs + 1);
 }

+static __dpct_inline__ void dequantize_iq4_nl_reorder(const void * d_ptr, const int64_t ib, const void * qs,
+                                                      const int iqs, dfloat2 & v) {
+    const float   d = (float) ((const sycl::half *) d_ptr)[ib];
+    const uint8_t q = ((const uint8_t *) qs)[iqs];
+
+    v.x() = d * kvalues_iq4nl[q & 0xF];
+    v.y() = d * kvalues_iq4nl[q >> 4];
+}
+
 static __dpct_inline__ void dequantize_iq4_xs(const void *vx, const int64_t ib,
                                               const int iqs, dfloat2 &v) {
 #if QK_K == 256
@@ -1758,6 +1768,60 @@ dequantize_block_iq4_xs(const void *__restrict__ vx, dst_t *__restrict__ yy,
     }
 }

+// reordered IQ4_NL / IQ4_XS: qs bytes iq .. iq + 3 of 32-value group g to y[iq + j] and y[iq + 16 + j], with y the
+// group's first output, in two vec<dst_t, 4> stores as for MXFP4; tail is the first byte after all qs
+template <ggml_type type, typename dst_t>
+static __dpct_inline__ void dequantize_iq4_reorder_quad(const uint8_t * __restrict__ x,
+                                                        const uint8_t * __restrict__ tail, const int64_t g,
+                                                        const int iq, dst_t * __restrict__ y) {
+    const uint8_t * qs = x + 16 * g + iq;
+    const float     d  = ggml_sycl_reordered::iq4_group_scale<type>(tail, g);
+
+    sycl::vec<dst_t, 4> lo, hi;
+#pragma unroll
+    for (int j = 0; j < 4; ++j) {
+        lo[j] = d * kvalues_iq4nl[qs[j] & 0xf];
+        hi[j] = d * kvalues_iq4nl[qs[j] >> 4];
+    }
+    *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + iq)      = lo;
+    *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + iq + 16) = hi;
+}
+
+// flat range over k / 8 items: item i takes group i / 4 from qs byte 4 * (i % 4)
+template <ggml_type type, typename dst_t>
+static void dequantize_block_iq4_reorder_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k,
+                                              const sycl::id<1> idx) {
+    const int64_t   i = (int64_t) idx[0];
+    const int64_t   g = i / 4;
+    const uint8_t * x = (const uint8_t *) vx;
+    dequantize_iq4_reorder_quad<type>(x, x + k / 2, g, (int) (4 * (i % 4)), yy + 32 * g);
+}
+
+// item t (0..31) of block ib dequantizes four qs bytes of sub-block t/4 into y, the output of block ib
+template <typename dst_t>
+static __dpct_inline__ void dequantize_block_iq4_xs_reorder(const void * __restrict__ vx, dst_t * __restrict__ y,
+                                                           const int64_t ib, const int t, const int64_t nb) {
+    const int ib32 = t / 4;
+    const int iq   = 4 * (t % 4);
+
+    const uint8_t * qs       = (const uint8_t *) vx + ib * (QK_K / 2) + 16 * ib32 + iq;
+    const uint8_t * meta     = (const uint8_t *) vx + nb * (QK_K / 2) + ib * offsetof(block_iq4_xs, qs);
+    const uint16_t  scales_h = *(const uint16_t *) (meta + offsetof(block_iq4_xs, scales_h));
+    const uint8_t   scales_l = meta[offsetof(block_iq4_xs, scales_l) + ib32 / 2];
+    const float     d        = (float) *(const ggml_half *) (meta + offsetof(block_iq4_xs, d)) *
+                               ((((scales_l >> (4 * (ib32 % 2))) & 0xf) | (((scales_h >> (2 * ib32)) & 3) << 4)) - 32);
+
+    sycl::vec<dst_t, 4> lo, hi;
+#pragma unroll
+    for (int j = 0; j < 4; ++j) {
+        lo[j] = d * kvalues_iq4nl[qs[j] & 0xf];
+        hi[j] = d * kvalues_iq4nl[qs[j] >> 4];
+    }
+    dst_t * yb = y + 32 * ib32 + iq;
+    *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb)      = lo;
+    *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb + 16) = hi;
+}
+
 template<typename dst_t>
 static void dequantize_block_mxfp4(const void * __restrict__ vx, dst_t * __restrict__ yy,
                                    const sycl::nd_item<3> &item_ct1) {
diff --git a/ggml/src/ggml-sycl/fused-gemm.cpp b/ggml/src/ggml-sycl/fused-gemm.cpp
index d0525e3a0..89096d561 100644
--- a/ggml/src/ggml-sycl/fused-gemm.cpp
+++ b/ggml/src/ggml-sycl/fused-gemm.cpp
@@ -1,5 +1,6 @@
 #include "fused-gemm.hpp"
 #include "fused-gemm-esimd.hpp"
+#include "quants.hpp"

 #include <sycl/ext/oneapi/matrix/matrix.hpp>

@@ -421,14 +422,10 @@ static __dpct_inline__ void fg_pack_quarter(const float * __restrict__ t, typena
     }
 }

+// 32 values from 16 qs bytes, as in IQ4_NL blocks and IQ4_XS sub-blocks: the low nibbles fill the
+// first half of the step and the high nibbles the second, so the halves land at a[0..7] and a[8..15]
 template <typename E>
-static __dpct_inline__ void fg_stage_a(const block_iq4_xs * __restrict__ xrow, const int kb, typename E::pair * a) {
-    FG_SUPERBLOCK(block_iq4_xs);
-    // low nibbles fill the first half of the step, high nibbles the second, so the two halves
-    // land at a[0..7] and a[8..15] and no quarter loop is needed
-    const float d = (float) blk->d *
-        ((((blk->scales_l[ib / 2] >> (4 * (ib % 2))) & 0xf) | (((blk->scales_h >> (2 * ib)) & 3) << 4)) - 32);
-    const uint8_t * q4 = blk->qs + 16 * ib;
+static __dpct_inline__ void fg_decode_iq4(const uint8_t * __restrict__ q4, const float d, typename E::pair * a) {
 #pragma unroll
     for (int j = 0; j < 8; ++j) {
         a[j]     = E::make(d * kvalues_iq4nl[q4[2 * j] & 0xf], d * kvalues_iq4nl[q4[2 * j + 1] & 0xf]);
@@ -436,6 +433,14 @@ static __dpct_inline__ void fg_stage_a(const block_iq4_xs * __restrict__ xrow, c
     }
 }

+template <typename E>
+static __dpct_inline__ void fg_stage_a(const block_iq4_xs * __restrict__ xrow, const int kb, typename E::pair * a) {
+    FG_SUPERBLOCK(block_iq4_xs);
+    const float d = (float) blk->d *
+        ((((blk->scales_l[ib / 2] >> (4 * (ib % 2))) & 0xf) | (((blk->scales_h >> (2 * ib)) & 3) << 4)) - 32);
+    fg_decode_iq4<E>(blk->qs + 16 * ib, d, a);
+}
+
 template <typename E>
 static __dpct_inline__ void fg_stage_a(const block_iq3_xxs * __restrict__ xrow, const int kb, typename E::pair * a) {
     FG_SUPERBLOCK(block_iq3_xxs);
@@ -821,6 +826,22 @@ template <> struct fg_soa<block_q6_K> {
     }
 };

+template <ggml_type type> struct fg_soa_iq4 {
+    static constexpr bool supported = true;
+    // [qs][per-block records]; k step kb is 32-value group kb % groups of block ib
+    template <typename E>
+    static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int kb,
+                                      typename E::pair * a) {
+        constexpr int groups = ggml_sycl_reordered::block_q_t<type>::traits::qk / QK4_NL;
+        const int64_t g      = (int64_t) ib * groups + kb % groups;
+        fg_decode_iq4<E>(x + 16 * g, ggml_sycl_reordered::iq4_group_scale<type>(x + nblocks * groups * 16, g), a);
+    }
+};
+
+template <> struct fg_soa<block_iq4_nl> : fg_soa_iq4<GGML_TYPE_IQ4_NL> {};
+
+template <> struct fg_soa<block_iq4_xs> : fg_soa_iq4<GGML_TYPE_IQ4_XS> {};
+

 // one SG_ROWS x BN output tile: B columns [b0, b0 + BN) of packed_b go to dst columns [n0, n1),
 // n1 - n0 <= BN
diff --git a/ggml/src/ggml-sycl/getrows.cpp b/ggml/src/ggml-sycl/getrows.cpp
index 7b53e3dcc..4830186d8 100644
--- a/ggml/src/ggml-sycl/getrows.cpp
+++ b/ggml/src/ggml-sycl/getrows.cpp
@@ -588,10 +588,24 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
             src1_i32, (float *)dst->data, ctx.stream());
             break;
         case GGML_TYPE_IQ4_NL:
+            if (src0_reordered) {
+                get_rows_sycl_reorder<QK4_NL, QR4_NL, dequantize_iq4_nl_reorder>(dst->src[0], dst->src[1], dst, dst->src[0]->data,
+                src1_i32, (float *)dst->data, ctx.stream());
+                break;
+            }
             get_rows_sycl<QK4_NL, 1, dequantize_iq4_nl>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,
             src1_i32, (float *)dst->data, ctx.stream());
             break;
         case GGML_TYPE_IQ4_XS:
+            if (src0_reordered) {
+                const void * vx = dst->src[0]->data;
+                get_rows_sycl_reorder_k<1, QK_K / 8, 1>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(),
+                    [=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<1> & item_ct1, int64_t nb) {
+                        dequantize_block_iq4_xs_reorder(vx, y, ib, (int) item_ct1.get_local_id(0), nb);
+                        GGML_UNUSED(scales_local);
+                    });
+                break;
+            }
             get_rows_sycl<QK_K, 1, dequantize_iq4_xs>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,
             src1_i32, (float *)dst->data, ctx.stream());
             break;
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index 3de5f1bd5..7dda19dfd 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -832,6 +832,8 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer,
             case GGML_TYPE_Q6_K:
             case GGML_TYPE_IQ3_XXS:
             case GGML_TYPE_IQ3_S:
+            case GGML_TYPE_IQ4_NL:
+            case GGML_TYPE_IQ4_XS:
             case GGML_TYPE_MXFP4:{
                 ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};
                 tensor->extra                 = extra;
@@ -4386,6 +4388,8 @@ inline bool ggml_sycl_supports_reorder_mmvq(enum ggml_type type) {
         case GGML_TYPE_Q6_K:
         case GGML_TYPE_IQ3_XXS:
         case GGML_TYPE_IQ3_S:
+        case GGML_TYPE_IQ4_NL:
+        case GGML_TYPE_IQ4_XS:
             return true;
         default:
             return false;
@@ -4975,6 +4979,20 @@ static bool reorder_qw_iq3_xxs(uint8_t * data_device, size_t size, dpct::queue_p
         }, stream);
 }

+// IQ4_NL / IQ4_XS: the qs go to the front, everything before them (d, plus the IQ4_XS scales) is the block's
+// record in the tail
+template <typename block_t>
+static bool reorder_qw_iq4(uint8_t * data_device, size_t size, dpct::queue_ptr stream) {
+    constexpr size_t record_size = offsetof(block_t, qs);
+    return reorder_qw_planar<block_t>(__func__, data_device, size, record_size, sizeof(block_t) - record_size,
+        [](const block_t & x, int64_t ib, uint8_t * tail, int64_t) {
+            const uint8_t * record = reinterpret_cast<const uint8_t *>(&x);
+            for (size_t j = 0; j < record_size; ++j) {
+                tail[ib * record_size + j] = record[j];
+            }
+        }, stream);
+}
+
 static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) {
     uint8_t * data_device = (uint8_t *) src0->data;
     size_t ncols = src0->ne[0];
@@ -5022,6 +5040,10 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) {
             return reorder_qw_iq3_xxs(data_device, size, stream);
         case GGML_TYPE_IQ3_S:
             return reorder_qw_iq3_s(data_device, size, stream);
+        case GGML_TYPE_IQ4_NL:
+            return reorder_qw_iq4<block_iq4_nl>(data_device, size, stream);
+        case GGML_TYPE_IQ4_XS:
+            return reorder_qw_iq4<block_iq4_xs>(data_device, size, stream);
         default:
             return false;
     }
@@ -5384,8 +5406,8 @@ static bool ggml_sycl_mul_mat_glu_fused(ggml_backend_sycl_context & ctx, ggml_cg
     }

     // gate and up of one type the XMX kernel handles, it needs the reorder layout like the unfused mmvq path
-    if (wg->type == wu->type && gate->src[1] == act && act->ne[1] >= ggml_sycl_xmx_min_cols(wu->type) &&
-        act->ne[1] <= g_ggml_sycl_xmx_glu_max_cols) {
+    if (wg->type == wu->type && ggml_sycl_xmx_supports_type(wu->type) && gate->src[1] == act &&
+        act->ne[1] >= ggml_sycl_xmx_min_cols(wu->type) && act->ne[1] <= g_ggml_sycl_xmx_glu_max_cols) {
         opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::MMVQ);
         opt_for_reorder(&ctx, wg, act, gate, mul_mat_algo::MMVQ);
         if (can_use_xmx_batch(ctx.device, wu, act) && can_use_xmx_batch(ctx.device, wg, act) &&
diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp
index 03464bfae..614b38b22 100644
--- a/ggml/src/ggml-sycl/mmvq.cpp
+++ b/ggml/src/ggml-sycl/mmvq.cpp
@@ -1776,10 +1776,11 @@ static void mul_mat_vec_iq3_xxs_q8_1_sycl(const void *vx, const void *vy,
 }

 template <ggml_type type>
-static void reorder_mul_mat_vec_iq3_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
-                                              const int nrows, dpct::queue_ptr stream) {
-    static_assert(type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S);
-    GGML_ASSERT(ncols % QK_K == 0);
+static void reorder_mul_mat_vec_iq_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
+                                             const int nrows, dpct::queue_ptr stream) {
+    static_assert(type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ4_NL ||
+                  type == GGML_TYPE_IQ4_XS);
+    GGML_ASSERT(ncols % ggml_sycl_reordered::block_q_t<type>::traits::qk == 0);
     constexpr size_t num_subgroups = WARP_SIZE;
     const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
     const sycl::range<3> block_nums(1, 1, block_num_y);
@@ -1795,13 +1796,14 @@ static void reorder_mul_mat_vec_iq3_q8_1_sycl(const void * vx, const void * vy,
 }

 template <ggml_type type, int ncols_dst>
-static void reorder_mul_mat_vec_iq3_q8_1_sycl_ncols(
+static void reorder_mul_mat_vec_iq_q8_1_sycl_ncols(
         const void * vx, const void * vy, float * dst,
         const int ncols, const int nrows,
         const int stride_col_y_bytes, const int stride_col_dst,
         dpct::queue_ptr stream) {
-    static_assert(type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S);
-    GGML_ASSERT(ncols % QK_K == 0);
+    static_assert(type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ4_NL ||
+                  type == GGML_TYPE_IQ4_XS);
+    GGML_ASSERT(ncols % ggml_sycl_reordered::block_q_t<type>::traits::qk == 0);
     constexpr size_t num_subgroups = WARP_SIZE;
     const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
     const sycl::range<3> block_nums(1, 1, block_num_y);
@@ -1818,21 +1820,21 @@ static void reorder_mul_mat_vec_iq3_q8_1_sycl_ncols(
 }

 template <ggml_type type>
-static void reorder_mul_mat_vec_iq3_q8_1_sycl_switch_ncols(
+static void reorder_mul_mat_vec_iq_q8_1_sycl_switch_ncols(
         const void * vx, const void * vy, float * dst,
         const int ncols, const int nrows, const int ncols_dst,
         const int stride_col_y_bytes, const int stride_col_dst,
         dpct::queue_ptr stream) {
     switch (ncols_dst) {
-        case 1: reorder_mul_mat_vec_iq3_q8_1_sycl<type>(vx, vy, dst, ncols, nrows, stream); break;
-        case 2: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 3: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 4: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 5: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 6: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 7: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 8: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        default: GGML_ABORT("unsupported ncols_dst=%d for reordered IQ3 multi-col MMVQ", ncols_dst);
+        case 1: reorder_mul_mat_vec_iq_q8_1_sycl<type>(vx, vy, dst, ncols, nrows, stream); break;
+        case 2: reorder_mul_mat_vec_iq_q8_1_sycl_ncols<type, 2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+        case 3: reorder_mul_mat_vec_iq_q8_1_sycl_ncols<type, 3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+        case 4: reorder_mul_mat_vec_iq_q8_1_sycl_ncols<type, 4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+        case 5: reorder_mul_mat_vec_iq_q8_1_sycl_ncols<type, 5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+        case 6: reorder_mul_mat_vec_iq_q8_1_sycl_ncols<type, 6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+        case 7: reorder_mul_mat_vec_iq_q8_1_sycl_ncols<type, 7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+        case 8: reorder_mul_mat_vec_iq_q8_1_sycl_ncols<type, 8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+        default: GGML_ABORT("unsupported ncols_dst=%d for reordered IQ multi-col MMVQ", ncols_dst);
     }
 }

@@ -2518,13 +2520,13 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens
                         const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
                         const int stride_col_dst     = dst->ne[0];
                         GGML_SYCL_DEBUG("Calling reordered IQ3_XXS multi-col MMVQ ncols=%d\n", (int) src1_ncols);
-                        reorder_mul_mat_vec_iq3_q8_1_sycl_switch_ncols<GGML_TYPE_IQ3_XXS>(
+                        reorder_mul_mat_vec_iq_q8_1_sycl_switch_ncols<GGML_TYPE_IQ3_XXS>(
                             src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
                             src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
                         return;
                     } else {
                         GGML_SYCL_DEBUG("Calling reordered IQ3_XXS MMVQ\n");
-                        reorder_mul_mat_vec_iq3_q8_1_sycl<GGML_TYPE_IQ3_XXS>(
+                        reorder_mul_mat_vec_iq_q8_1_sycl<GGML_TYPE_IQ3_XXS>(
                             src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
                     }
                 } else {
@@ -2539,13 +2541,13 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens
                         const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
                         const int stride_col_dst     = dst->ne[0];
                         GGML_SYCL_DEBUG("Calling reordered IQ3_S multi-col MMVQ ncols=%d\n", (int) src1_ncols);
-                        reorder_mul_mat_vec_iq3_q8_1_sycl_switch_ncols<GGML_TYPE_IQ3_S>(
+                        reorder_mul_mat_vec_iq_q8_1_sycl_switch_ncols<GGML_TYPE_IQ3_S>(
                             src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
                             src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
                         return;
                     } else {
                         GGML_SYCL_DEBUG("Calling reordered IQ3_S MMVQ\n");
-                        reorder_mul_mat_vec_iq3_q8_1_sycl<GGML_TYPE_IQ3_S>(
+                        reorder_mul_mat_vec_iq_q8_1_sycl<GGML_TYPE_IQ3_S>(
                             src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
                     }
                 } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
@@ -2561,10 +2563,42 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens
                 }
                 break;
             case GGML_TYPE_IQ4_NL:
-                mul_mat_vec_iq4_nl_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
+                if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
+                    ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+                    if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
+                        const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
+                        const int stride_col_dst     = dst->ne[0];
+                        GGML_SYCL_DEBUG("Calling reordered IQ4_NL multi-col MMVQ ncols=%d\n", (int) src1_ncols);
+                        reorder_mul_mat_vec_iq_q8_1_sycl_switch_ncols<GGML_TYPE_IQ4_NL>(
+                            src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
+                            src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
+                        return;
+                    } else {
+                        GGML_SYCL_DEBUG("Calling reordered IQ4_NL MMVQ\n");
+                        reorder_mul_mat_vec_iq_q8_1_sycl<GGML_TYPE_IQ4_NL>(
+                            src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
+                    }
+                } else {
+                    mul_mat_vec_iq4_nl_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
+                }
                 break;
             case GGML_TYPE_IQ4_XS:
-                if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
+                if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
+                    ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+                    if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
+                        const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
+                        const int stride_col_dst     = dst->ne[0];
+                        GGML_SYCL_DEBUG("Calling reordered IQ4_XS multi-col MMVQ ncols=%d\n", (int) src1_ncols);
+                        reorder_mul_mat_vec_iq_q8_1_sycl_switch_ncols<GGML_TYPE_IQ4_XS>(
+                            src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
+                            src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
+                        return;
+                    } else {
+                        GGML_SYCL_DEBUG("Calling reordered IQ4_XS MMVQ\n");
+                        reorder_mul_mat_vec_iq_q8_1_sycl<GGML_TYPE_IQ4_XS>(
+                            src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
+                    }
+                } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
                     const int stride_col_y   = src1_padded_col_size / QK8_1;
                     const int stride_col_dst = dst->ne[0];
                     GGML_SYCL_DEBUG("Calling mul_mat_vec_iq4_xs_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
diff --git a/ggml/src/ggml-sycl/quants.hpp b/ggml/src/ggml-sycl/quants.hpp
index 9fdcc22f2..79b5c8de5 100644
--- a/ggml/src/ggml-sycl/quants.hpp
+++ b/ggml/src/ggml-sycl/quants.hpp
@@ -14,6 +14,7 @@
 #ifndef GGML_SYCL_QUANTS_HPP
 #define GGML_SYCL_QUANTS_HPP

+#include <cstddef>
 #include <utility>

 #include "ggml-common.h"
@@ -225,6 +226,70 @@ template <> struct block_q_t<GGML_TYPE_IQ3_S> {
     static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
 };

+template <> struct block_q_t<GGML_TYPE_IQ4_NL> {
+    struct traits {
+        static constexpr uint32_t qk       = QK4_NL;
+        static constexpr uint32_t qi       = QI4_NL;
+        static constexpr uint32_t qr       = QR4_NL;
+        static constexpr uint32_t vdr_mmvq = 4;
+    };
+
+    // Reordered layout: [qs (QK4_NL/2 per block)][d]
+    static constexpr int record_size = offsetof(block_iq4_nl, qs);
+
+    static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
+        return { block_index * (QK4_NL / 2), 0 };
+    }
+
+    static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
+        return { (ncols / QR4_NL * nrows) + block_index * sizeof(ggml_half), 0 };
+    }
+
+    static float group_scale(const uint8_t * record, const int /* group */) {
+        return *reinterpret_cast<const ggml_half *>(record);
+    }
+
+    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
+};
+
+template <> struct block_q_t<GGML_TYPE_IQ4_XS> {
+    struct traits {
+        static constexpr uint32_t qk       = QK_K;
+        static constexpr uint32_t qi       = QI4_XS;
+        static constexpr uint32_t qr       = QR4_XS;
+        static constexpr uint32_t vdr_mmvq = 4;
+    };
+
+    // Reordered layout: [qs (QK_K/2 per block)][{d, scales_h, scales_l} per block, as in block_iq4_xs]
+    static constexpr int record_size = offsetof(block_iq4_xs, qs);
+
+    static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
+        return { block_index * (QK_K / 2), 0 };
+    }
+
+    static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
+        const int nblocks = nrows * (ncols / QK_K);
+        return { nblocks * (QK_K / 2) + block_index * (int) offsetof(block_iq4_xs, qs), 0 };
+    }
+
+    static float group_scale(const uint8_t * record, const int group) {
+        const uint16_t scales_h = *reinterpret_cast<const uint16_t *>(record + offsetof(block_iq4_xs, scales_h));
+        const uint8_t  scales_l = record[offsetof(block_iq4_xs, scales_l) + group / 2];
+        const int      ls       = ((scales_l >> (4 * (group % 2))) & 0xf) | (((scales_h >> (2 * group)) & 3) << 4);
+        return (float) *reinterpret_cast<const ggml_half *>(record + offsetof(block_iq4_xs, d)) * (ls - 32);
+    }
+
+    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
+};
+
+// IQ4_NL blocks and IQ4_XS sub-blocks are 32-value groups of 16 qs bytes, so in both layouts the qs of
+// group g start at 16 * g; tail is the first byte after all qs, where the per-block records start
+template <ggml_type type> inline float iq4_group_scale(const uint8_t * tail, const int64_t g) {
+    using block_t = block_q_t<type>;
+    constexpr int64_t groups = block_t::traits::qk / QK4_NL;
+    return block_t::group_scale(tail + g / groups * block_t::record_size, (int) (g % groups));
+}
+
 template <> struct block_q_t<GGML_TYPE_Q8_0> {
     struct traits {
         static constexpr uint32_t qk       = QK8_0;      // 32
diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp
index f547b20e6..f2f34667a 100644
--- a/ggml/src/ggml-sycl/vecdotq.hpp
+++ b/ggml/src/ggml-sycl/vecdotq.hpp
@@ -388,6 +388,14 @@ template <> struct reorder_vec_dot_shared_weights<GGML_TYPE_Q5_K> {
     static constexpr bool value = true;
 };

+template <> struct reorder_vec_dot_shared_weights<GGML_TYPE_IQ4_NL> {
+    static constexpr bool value = true;
+};
+
+template <> struct reorder_vec_dot_shared_weights<GGML_TYPE_IQ4_XS> {
+    static constexpr bool value = true;
+};
+
 template <ggml_type T> struct reorder_vec_dot_shared_activations {
     static constexpr bool value = false;
 };
@@ -1956,4 +1964,88 @@ vec_dot_iq4_xs_q8_1(const void *__restrict__ vbq,
 #endif
 }

+// IQ4_NL blocks and IQ4_XS sub-blocks are both 32 values in 16 qs bytes: the low nibbles pair with
+// q8_1 ints 0..3 and the high nibbles with ints 4..7. A lane decodes them once for all columns.
+struct reorder_vec_dot_iq4_32 {
+    struct weights {
+        int   v[8];
+        float d;
+    };
+
+    static __dpct_inline__ weights decode(const uint8_t * qs, const float d) {
+        const sycl::int4 q4     = *reinterpret_cast<const sycl::int4 *>(qs);
+        const uint8_t *  values = (const uint8_t *) kvalues_iq4nl;
+
+        weights w;
+        get_int_from_table_16((uint32_t) q4.x(), values, w.v[0], w.v[4]);
+        get_int_from_table_16((uint32_t) q4.y(), values, w.v[1], w.v[5]);
+        get_int_from_table_16((uint32_t) q4.z(), values, w.v[2], w.v[6]);
+        get_int_from_table_16((uint32_t) q4.w(), values, w.v[3], w.v[7]);
+        w.d = d;
+        return w;
+    }
+
+    static __dpct_inline__ float dot32(const weights & w, const int8_t * q8, const sycl::half2 ds8) {
+        const int * u     = (const int *) q8;
+        int         sumi1 = 0;
+        int         sumi2 = 0;
+#pragma unroll
+        for (int j = 0; j < 4; ++j) {
+            sumi1 = dpct::dp4a(w.v[j + 0], u[j + 0], sumi1);
+            sumi2 = dpct::dp4a(w.v[j + 4], u[j + 4], sumi2);
+        }
+        return w.d * (float) ds8[0] * (sumi1 + sumi2);
+    }
+};
+
+template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_IQ4_NL> : reorder_vec_dot_iq4_32 {
+    static constexpr ggml_type gtype = GGML_TYPE_IQ4_NL;
+
+    // a lane holds a whole block, so iqs is always 0
+    __dpct_inline__ static weights load(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
+                                        const std::pair<int, int> d_offset, const int & /* iqs */) {
+        const uint8_t * base = static_cast<const uint8_t *>(vbq);
+        return decode(base + ibx_offset.first, *reinterpret_cast<const ggml_half *>(base + d_offset.first));
+    }
+
+    __dpct_inline__ static float dot(const weights & w, const int8_t * q8_1_quant_ptr, const sycl::half2 * q8_1_ds,
+                                     const int & /* iqs */) {
+        return dot32(w, q8_1_quant_ptr, *q8_1_ds);
+    }
+
+    __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
+                                     const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
+                                     const sycl::half2 * q8_1_ds, const int & iqs) {
+        return dot(load(vbq, ibx_offset, d_offset, iqs), q8_1_quant_ptr, q8_1_ds, iqs);
+    }
+};
+
+template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_IQ4_XS> : reorder_vec_dot_iq4_32 {
+    static constexpr ggml_type gtype = GGML_TYPE_IQ4_XS;
+
+    // iqs is the first qs int of the lane's sub-block
+    __dpct_inline__ static weights load(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
+                                        const std::pair<int, int> d_offset, const int & iqs) {
+        const uint8_t * base     = static_cast<const uint8_t *>(vbq);
+        const uint8_t * meta     = base + d_offset.first;
+        const int       ib32     = iqs / 4;
+        const uint16_t  scales_h = *reinterpret_cast<const uint16_t *>(meta + offsetof(block_iq4_xs, scales_h));
+        const uint8_t   scales_l = meta[offsetof(block_iq4_xs, scales_l) + ib32 / 2];
+        const int       ls       = ((scales_l >> (4 * (ib32 % 2))) & 0xf) | (((scales_h >> (2 * ib32)) & 3) << 4);
+        const float     d        = (float) *reinterpret_cast<const ggml_half *>(meta + offsetof(block_iq4_xs, d));
+        return decode(base + ibx_offset.first + sizeof(int) * iqs, d * (ls - 32));
+    }
+
+    __dpct_inline__ static float dot(const weights & w, const int8_t * q8_1_quant_ptr, const sycl::half2 * q8_1_ds,
+                                     const int & iqs) {
+        return dot32(w, q8_1_quant_ptr + (iqs / 4) * QK8_1, q8_1_ds[iqs / 4]);
+    }
+
+    __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
+                                     const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
+                                     const sycl::half2 * q8_1_ds, const int & iqs) {
+        return dot(load(vbq, ibx_offset, d_offset, iqs), q8_1_quant_ptr, q8_1_ds, iqs);
+    }
+};
+
 #endif // GGML_SYCL_VECDOTQ_HPP
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 711674958..a0d3349e6 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -10948,10 +10948,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {

     test_cases.emplace_back(new test_get_rows(GGML_TYPE_F32, 1, 8, 2, 1, 1, false));
     for (ggml_type type : {GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K,
-                          GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S}) {
+                          GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ4_XS}) {
         test_cases.emplace_back(new test_get_rows_after_mul_mat(type));
     }
-    for (ggml_type type : {GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S}) {
+    for (ggml_type type : {GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ4_XS}) {
         test_cases.emplace_back(new test_mul_mat_reorder_then_big_batch(type));
     }
     for (int n : {1, 3, 257}) {
@@ -10959,6 +10959,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
     }
     // many rows of a wide table, as a per-layer embedding lookup does
     test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, 160, 100000, 32768, 1, 1, false));
+    // past the fused dequant GEMM's column limit the reordered weight goes through the conversion kernels
+    for (ggml_type type : {GGML_TYPE_IQ4_NL, GGML_TYPE_IQ4_XS}) {
+        test_cases.emplace_back(new test_mul_mat_reorder_then_big_batch(type, 128));
+    }
     for (ggml_type type : all_types) {
         for (int b : {1, 7}) {
             for (bool v : {false, true}) {
@@ -12127,8 +12131,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
         test_cases.emplace_back(new test_mul_mat(type_a,    GGML_TYPE_F32, 16,  8, 16*256, { 1,  1}, {1, 1}));
     }

-    // Multi-column MMVQ coverage for the Q4_K weight-reuse path and a Q5_K control.
-    for (ggml_type type_a : { GGML_TYPE_Q4_K, GGML_TYPE_Q5_K }) {
+    // Multi-column MMVQ coverage for the Q4_K and IQ4 weight-reuse paths and a Q5_K control.
+    for (ggml_type type_a : { GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ4_XS }) {
         for (int n = 1; n <= 8; ++n) {
             test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 4096, n, 1024, { 1, 1 }, { 1, 1 }));
             test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1023, n, 4096, { 1, 1 }, { 1, 1 }));