Commit 8777c0c83 for llama.cpp

commit 8777c0c833effb3dec51f8db4f49f5c8b234c644
Author: anujj <ajalota@nvidia.com>
Date:   Tue Nov 17 21:01:40 2026 +0530

    Title: CUDA: few-row MMQ for MUL_MAT_ID (#32065)

    * CUDA: few-row MMQ for MUL_MAT_ID

    * CUDA: few-row MMQ for MUL_MAT_ID from 2 tokens on DGX Spark

    * CUDA: few-row MUL_MAT_ID: one launcher, tighter pair count, more tests

    * CUDA: few-row MUL_MAT_ID: dispatch order MMVQ, few-row, MMQ and review fixes

diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index cdd889caa..b0fd03654 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -2275,6 +2275,11 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) {
         return false;
     }

+    // do not fuse the matmuls that few-row MMQ takes in ggml_cuda_mul_mat_id
+    if (tensor->op == GGML_OP_MUL_MAT_ID && ggml_cuda_should_use_mmq_few(src0, src1, tensor->src[2], cc)) {
+        return false;
+    }
+
     return use_mul_mat_vec_q;
 }

@@ -2426,13 +2431,13 @@ static bool ggml_cuda_should_fuse_mul_mat_q(const ggml_tensor * up, const ggml_t
     const ggml_tensor * y    = up->src[1];
     const int           cc   = ggml_cuda_info().devices[device].cc;

-    // For MUL_MAT_ID: same kernel choice as ggml_cuda_mul_mat_id, with the nominal tiles of all experts (the per-expert token counts are only known on the GPU).
+    // For MUL_MAT_ID: the MMVQ/MMQ choice of ggml_cuda_mul_mat_id, with the nominal tiles of all experts (the per-expert token counts are only known on the GPU).
     const bool    is_id     = up->op == GGML_OP_MUL_MAT_ID;
     const int64_t ncols     = is_id ? y->ne[2]    : y->ne[1];
     const int64_t nchannels = is_id ? x_up->ne[2] : y->ne[2]*y->ne[3];
     const bool    use_mmvq  = is_id ? ncols <= get_mmvq_mmid_max_batch(x_up->type, cc) : ggml_cuda_should_use_mmvq(x_up->type, cc, ncols);
-    // do not fuse the matmuls that few-row MMQ takes in ggml_cuda_mul_mat
-    if (!is_id && ggml_cuda_should_use_mmq_few(x_up, y, cc)) {
+    // do not fuse the matmuls that few-row MMQ takes in ggml_cuda_mul_mat; for MUL_MAT_ID the fused MMQ keeps the expert gate and up
+    if (!is_id && ggml_cuda_should_use_mmq_few(x_up, y, nullptr, cc)) {
         return false;
     }
     if (use_mmvq || !ggml_cuda_should_use_mmq(x_up->type, cc, ncols, /*n_experts =*/ is_id ? x_up->ne[2] : 0)) {
@@ -2488,13 +2493,13 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
         return;
     }
     // from a per-type number of src1 columns up to 24, few-row MMQ is faster than MMVQ and MMQ
-    const bool use_mmq_few = ggml_cuda_should_use_mmq_few(src0, src1, cc);
+    const bool use_mmq_few = ggml_cuda_should_use_mmq_few(src0, src1, nullptr, cc);
     if (!use_mmq_few && ggml_cuda_should_use_mmvq(src0->type, cc, ne11)) {
         ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst);
         return;
     }
     if (use_mmq_few) {
-        ggml_cuda_mul_mat_q_few(ctx, src0, src1, dst);
+        ggml_cuda_mul_mat_q_few(ctx, src0, src1, nullptr, dst);
         return;
     }
     if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) {
@@ -2531,6 +2536,10 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int c
         }
     }

+    if (ggml_cuda_should_use_mmq_few(src0, src1, dst->src[2], cc)) {
+        return false;
+    }
+
     if (ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[2], /*n_experts=*/src0->ne[2])) {
         return false;
     }
@@ -2557,8 +2566,11 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *

     // [TAG_MUL_MAT_ID_CUDA_GRAPHS]
     if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
+        // up to 24 tokens few-row MMQ is faster than MMQ, on DGX Spark also faster than MMVQ
+        const bool use_mmq_few = ggml_cuda_should_use_mmq_few(src0, src1, ids, cc);
+
         static_assert(MMVQ_MAX_BATCH_SIZE == MMVF_MAX_BATCH_SIZE);
-        if (ne2 <= MMVQ_MAX_BATCH_SIZE) {
+        if (!use_mmq_few && ne2 <= MMVQ_MAX_BATCH_SIZE) {
             if (ggml_is_quantized(src0->type)) {
                 const int mmvq_mmid_max = get_mmvq_mmid_max_batch(src0->type, cc);
                 if (ne2 <= mmvq_mmid_max) {
@@ -2578,6 +2590,11 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
             }
         }

+        if (use_mmq_few) {
+            ggml_cuda_mul_mat_q_few(ctx, src0, src1, ids, dst);
+            return;
+        }
+
         if (ggml_cuda_should_use_mmq(src0->type, cc, ne12, /*n_experts=*/ne02)) {
             ggml_cuda_mul_mat_q(ctx, src0, src1, ids, dst);
             return;
diff --git a/ggml/src/ggml-cuda/mmq-few.cu b/ggml/src/ggml-cuda/mmq-few.cu
index 2c2a2745e..a141cc718 100644
--- a/ggml/src/ggml-cuda/mmq-few.cu
+++ b/ggml/src/ggml-cuda/mmq-few.cu
@@ -1,6 +1,7 @@
 #include "mmq-few.cuh"
 #include "cp-async.cuh"
 #include "mma.cuh"
+#include "mmvq.cuh"
 #include "quantize.cuh"
 #include "vecdotq.cuh"

@@ -13,6 +14,7 @@ using namespace ggml_cuda_mma;

 // Few-row MMQ: int8 MMA mat-mul for 2-24 src1 columns. A CUDA block has 16 src0 rows (32 for 2 column groups on large matrices).
 // Its warps split K in units of 256 values and copy raw src0 rows into their own cp.async ring, src1 is quantized to the MMA fragment layout.
+// MUL_MAT_ID: one column group per expert and CUDA block, for short rows the warps split the rows instead of K (64 rows per block).

 static constexpr int MMQ_FEW_MAX_NCG = 3;  // max. number of column groups
 static constexpr int MMQ_FEW_NROWS   = 16; // src0 rows of one MMA tile
@@ -80,6 +82,11 @@ static constexpr __host__ __device__ bool mmq_few_tall_supported(const ggml_type
            type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ4_NL || type == GGML_TYPE_IQ4_XS;
 }

+// MUL_MAT_ID with fewer than 2 units of 256 values per warp: the warps split the rows, each warp has its own 16 rows.
+static constexpr __host__ __device__ bool mmq_few_ksplit(const bool has_ids, const int nunits) {
+    return !has_ids || nunits >= 2*MMQ_FEW_NWARPS;
+}
+
 #if defined(CP_ASYNC_AVAILABLE)

 // The src1 fragments of a lane for one unit of 256 values, see quantize.cuh.
@@ -484,17 +491,19 @@ static __device__ __forceinline__ void mmq_few_vec_dot(const char * const (&x)[2

 #endif // defined(CP_ASYNC_AVAILABLE)

-template <ggml_type type, int ncg, int nmt>
+// has_ids: MUL_MAT_ID, blockIdx.y is a pair of quantize_mmq_few_id_q8_1, its expert is the src0 channel and dst dims 1 and 2 are slot and token.
+template <ggml_type type, int ncg, int nmt, bool has_ids>
 __launch_bounds__(MMQ_FEW_NWARPS*ggml_cuda_get_physical_warp_size(), 1)
 static __global__ void mul_mat_q_few(
         const char * x, const int4 * y_ptr, float * dst,
         const int nunits, const int nrows_x, const int ncols_dst, const int64_t stride_row_x, const int stride_col_dst,
         const int channel_ratio, const int64_t stride_channel_x, const int stride_channel_dst,
-        const int sample_ratio, const int64_t stride_sample_x, const int stride_sample_dst, const bool x_is_constant) {
+        const int sample_ratio, const int64_t stride_sample_x, const int stride_sample_dst, const bool x_is_constant, const int * pairs) {
 #if defined(CP_ASYNC_AVAILABLE)
     constexpr int nwarps    = MMQ_FEW_NWARPS;
     constexpr int warp_size = ggml_cuda_get_physical_warp_size();
     static_assert(warp_size == WARP_SIZE, "unsupported warp size");
+    static_assert(!has_ids || (ncg == 1 && nmt == 1), "unsupported MUL_MAT_ID config");

     constexpr int  unit_size    = mmq_few_unit_size(type);
     constexpr int  nrows        = nmt*MMQ_FEW_NROWS;
@@ -510,13 +519,30 @@ static __global__ void mul_mat_q_few(
     extern __shared__ char data_mmq_few[];
     char * ring = data_mmq_few + threadIdx.y*nstages*nbytes_stage;

-    const int row0        = blockIdx.x*nrows;
+    const bool ksplit   = mmq_few_ksplit(has_ids, nunits);
+    const int  row0_blk = blockIdx.x*(ksplit ? nrows : nwarps*nrows);
+    const int  row0     = ksplit ? row0_blk : row0_blk + threadIdx.y*nrows;
+    const int  iu_begin = ksplit ? nunits_stage*threadIdx.y : 0;
+    const int  iu_step  = ksplit ? nunits_stage*nwarps : nunits_stage;
+
     const int channel_dst = blockIdx.y;
     const int sample_dst  = blockIdx.z;
     const int nchannels   = gridDim.y;

-    x   += int64_t(sample_dst/sample_ratio)*stride_sample_x + int64_t(channel_dst/channel_ratio)*stride_channel_x;
-    dst += int64_t(sample_dst)*stride_sample_dst + int64_t(channel_dst)*stride_channel_dst;
+    int ncols = ncols_dst;
+    [[maybe_unused]] const int * pair = nullptr;
+    if constexpr (has_ids) {
+        ggml_cuda_pdl_sync(); // the pairs come from the preceding kernel
+        pair  = pairs + channel_dst*MMQ_FEW_PAIR_INTS;
+        ncols = pair[1];
+        if (ncols == 0) {
+            return;
+        }
+        x += int64_t(pair[0])*stride_channel_x;
+    } else {
+        x   += int64_t(sample_dst/sample_ratio)*stride_sample_x + int64_t(channel_dst/channel_ratio)*stride_channel_x;
+        dst += int64_t(sample_dst)*stride_sample_dst + int64_t(channel_dst)*stride_channel_dst;
+    }
     const int4 * GGML_CUDA_RESTRICT y = y_ptr + (int64_t(sample_dst)*nchannels + channel_dst)*nunits*ncg*MMQ_FEW_Q8_1_INT4;

     // No bounds check for the units, the gate makes nunits a multiple of nunits_stage.
@@ -562,11 +588,11 @@ static __global__ void mul_mat_q_few(
     // DGX Spark DRAM serves the short row pieces of the stage copies slowly, so the rows of the block are first prefetched into L2 in address order.
     {
         const int     tid       = threadIdx.y*warp_size + threadIdx.x;
-        const int     nrows_blk = min(nrows, nrows_x - row0);
+        const int     nrows_blk = min(ksplit ? nrows : nwarps*nrows, nrows_x - row0_blk);
         const int64_t row_bytes = int64_t(nunits)*unit_size;
         const bool    contig    = stride_row_x == row_bytes;
         for (int ir = 0; ir < (contig ? 1 : nrows_blk); ++ir) {
-            const uintptr_t p0 = (uintptr_t) (x + int64_t(row0 + ir)*stride_row_x);
+            const uintptr_t p0 = (uintptr_t) (x + int64_t(row0_blk + ir)*stride_row_x);
             const uintptr_t p1 = p0 + (contig ? nrows_blk*row_bytes : row_bytes);
             for (uintptr_t p = (p0 & ~uintptr_t(127)) + 128*tid; p < p1; p += 128*nwarps*warp_size) {
                 ggml_cuda_prefetch_l2((const void *) p);
@@ -575,11 +601,9 @@ static __global__ void mul_mat_q_few(
     }
 #endif // __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK

-    const int iu_step = nunits_stage*nwarps;
-
 #pragma unroll
     for (int stage = 0; stage < nstages - 1; ++stage) {
-        const int iu0 = nunits_stage*threadIdx.y + stage*iu_step;
+        const int iu0 = iu_begin + stage*iu_step;
         if (iu0 < nunits) {
             load_stage(stage, iu0);
         }
@@ -598,16 +622,16 @@ static __global__ void mul_mat_q_few(

     // The next src1 fragments are loaded right after the MMAs that use the current ones, an earlier load is slower.
     mmq_few_y_frag yf[nunits_stage][ncg];
-    if (nunits_stage*threadIdx.y < nunits) {
+    if (iu_begin < nunits) {
 #pragma unroll
         for (int k = 0; k < nunits_stage; ++k) {
-            load_y(yf[k], nunits_stage*threadIdx.y + k);
+            load_y(yf[k], iu_begin + k);
         }
     }

     float sum[nmt][ncg][4] = {{{0.0f}}};

-    for (int iter = 0, iu0 = nunits_stage*threadIdx.y; iu0 < nunits; ++iter, iu0 += iu_step) {
+    for (int iter = 0, iu0 = iu_begin; iu0 < nunits; ++iter, iu0 += iu_step) {
         const int iu_load = iu0 + (nstages - 1)*iu_step;
         if (iu_load < nunits) {
             load_stage((iter + nstages - 1) % nstages, iu_load);
@@ -643,26 +667,28 @@ static __global__ void mul_mat_q_few(
     ggml_cuda_pdl_lc();

     // reduce the sums over the warps as one flat array
-    constexpr int nsum = nmt*ncg*4;
-    float * sum_flat = &sum[0][0][0];
+    if (ksplit) {
+        constexpr int nsum = nmt*ncg*4;
+        float * sum_flat = &sum[0][0][0];

-    float * buf_sum = (float *) data_mmq_few;
-    __syncthreads();
-    if (threadIdx.y > 0) {
+        float * buf_sum = (float *) data_mmq_few;
+        __syncthreads();
+        if (threadIdx.y > 0) {
 #pragma unroll
-        for (int i = 0; i < nsum; ++i) {
-            buf_sum[((threadIdx.y - 1)*nsum + i)*warp_size + threadIdx.x] = sum_flat[i];
+            for (int i = 0; i < nsum; ++i) {
+                buf_sum[((threadIdx.y - 1)*nsum + i)*warp_size + threadIdx.x] = sum_flat[i];
+            }
+        }
+        __syncthreads();
+        if (threadIdx.y > 0) {
+            return;
         }
-    }
-    __syncthreads();
-    if (threadIdx.y > 0) {
-        return;
-    }
 #pragma unroll
-    for (int w = 0; w < nwarps - 1; ++w) {
+        for (int w = 0; w < nwarps - 1; ++w) {
 #pragma unroll
-        for (int i = 0; i < nsum; ++i) {
-            sum_flat[i] += buf_sum[(w*nsum + i)*warp_size + threadIdx.x];
+            for (int i = 0; i < nsum; ++i) {
+                sum_flat[i] += buf_sum[(w*nsum + i)*warp_size + threadIdx.x];
+            }
         }
     }

@@ -674,8 +700,13 @@ static __global__ void mul_mat_q_few(
             for (int l = 0; l < 4; ++l) {
                 const int i = row0 + MMQ_FEW_NROWS*m + tile<16, 8, int>::get_i(l);
                 const int j = c*MMQ_FEW_NCOLS + tile<16, 8, int>::get_j(l);
-                if (i < nrows_x && j < ncols_dst) {
-                    dst[int64_t(j)*stride_col_dst + i] = sum[m][c][l];
+                if (i < nrows_x && j < ncols) {
+                    if constexpr (has_ids) {
+                        const int e = pair[2 + j]; // token*n_expert_used + slot
+                        dst[int64_t(e / ncols_dst)*stride_channel_dst + int64_t(e % ncols_dst)*stride_col_dst + i] = sum[m][c][l];
+                    } else {
+                        dst[int64_t(j)*stride_col_dst + i] = sum[m][c][l];
+                    }
                 }
             }
         }
@@ -683,7 +714,7 @@ static __global__ void mul_mat_q_few(
 #else
     GGML_UNUSED_VARS(x, y_ptr, dst, nunits, nrows_x, ncols_dst, stride_row_x, stride_col_dst,
         channel_ratio, stride_channel_x, stride_channel_dst,
-        sample_ratio, stride_sample_x, stride_sample_dst, x_is_constant);
+        sample_ratio, stride_sample_x, stride_sample_dst, x_is_constant, pairs);
     NO_DEVICE_CODE;
 #endif // defined(CP_ASYNC_AVAILABLE)
 }
@@ -695,9 +726,10 @@ static bool mmq_few_is_constant(const ggml_tensor * src0) {
         !(x_root->flags & GGML_TENSOR_FLAG_PARAM);
 }

-template <ggml_type type, int ncg, int nmt>
+// MUL_MAT_ID: grid y is the pairs, the kernel takes the expert from the pair and does not use the channel and sample ratios.
+template <ggml_type type, int ncg, int nmt, bool has_ids = false>
 static void launch_mul_mat_q_few(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
-        const int4 * src1_q, cudaStream_t stream) {
+        const int4 * src1_q, cudaStream_t stream, const int * pairs = nullptr, const int npairs_max = 0) {
     GGML_TENSOR_BINARY_OP_LOCALS;

     const int nunits = ne00 / QK_K;
@@ -705,18 +737,19 @@ static void launch_mul_mat_q_few(const ggml_tensor * src0, const ggml_tensor * s

     const int warp_size     = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size;
     const int nbytes_shared = mmq_few_nbytes_shared(type, ncg, nmt);
-    CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q_few<type, ncg, nmt>), nbytes_shared);
+    CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q_few<type, ncg, nmt, has_ids>), nbytes_shared);

     const bool x_is_constant = mmq_few_is_constant(src0);

-    const dim3 block_nums((ne01 + nmt*MMQ_FEW_NROWS - 1) / (nmt*MMQ_FEW_NROWS), ne12, ne13);
+    const int  nrows_blk = (mmq_few_ksplit(has_ids, nunits) ? 1 : MMQ_FEW_NWARPS)*nmt*MMQ_FEW_NROWS;
+    const dim3 block_nums((ne01 + nrows_blk - 1) / nrows_blk, has_ids ? npairs_max : ne12, ne13);
     const dim3 block_dims(warp_size, MMQ_FEW_NWARPS, 1);
     const ggml_cuda_kernel_launch_params params(block_nums, block_dims, nbytes_shared, stream);
-    ggml_cuda_kernel_launch(mul_mat_q_few<type, ncg, nmt>, params,
+    ggml_cuda_kernel_launch(mul_mat_q_few<type, ncg, nmt, has_ids>, params,
         (const char *) src0->data, src1_q, (float *) dst->data,
         nunits, (int) ne01, (int) ne1, (int64_t) nb01, (int) (nb1/sizeof(float)),
         (int) (ne12/ne02), (int64_t) nb02, (int) (nb2/sizeof(float)),
-        (int) (ne13/ne03), (int64_t) nb03, (int) (nb3/sizeof(float)), x_is_constant);
+        (int) (ne13/ne03), (int64_t) nb03, (int) (nb3/sizeof(float)), x_is_constant, pairs);
 }

 static int mmq_few_ncg(const int64_t ncols) {
@@ -743,7 +776,11 @@ static void launch_mul_mat_q_few_tall(const ggml_tensor * src0, const ggml_tenso

 template <ggml_type type>
 static void launch_mul_mat_q_few_switch_ncg(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
-        const int4 * src1_q, const int ncg, cudaStream_t stream) {
+        const int4 * src1_q, const int * pairs, const int npairs_max, const int ncg, cudaStream_t stream) {
+    if (pairs) {
+        launch_mul_mat_q_few<type, 1, 1, true>(src0, src1, dst, src1_q, stream, pairs, npairs_max);
+        return;
+    }
     switch (ncg) {
         case 1:
             launch_mul_mat_q_few<type, 1, 1>(src0, src1, dst, src1_q, stream);
@@ -759,7 +796,7 @@ static void launch_mul_mat_q_few_switch_ncg(const ggml_tensor * src0, const ggml
     }
 }

-void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {
     GGML_TENSOR_BINARY_OP_LOCALS;

     GGML_ASSERT(src1->type == GGML_TYPE_F32);
@@ -767,51 +804,73 @@ void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor
     GGML_ASSERT(nb00 == ggml_type_size(src0->type));
     GGML_ASSERT(nb10 == sizeof(float));
     GGML_ASSERT(nb0  == sizeof(float));
-    GGML_ASSERT(ne11 <= MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS);
     GGML_ASSERT(ne00 % QK_K == 0);
-    GGML_ASSERT(ne12 % ne02 == 0 && ne13 % ne03 == 0);

     cudaStream_t stream = ctx.stream();

     const int nunits = ne00 / QK_K;
-    const int ncg    = mmq_few_ncg(ne11);
+    const int ncg    = ids ? 1 : mmq_few_ncg(ne11);
+
+    ggml_cuda_pool_alloc<int>  pairs(ctx.pool());
+    ggml_cuda_pool_alloc<int4> src1_q(ctx.pool());
+    int npairs_max = 0;
+    if (ids) {
+        GGML_ASSERT(ids->type == GGML_TYPE_I32);
+        GGML_ASSERT(ids->nb[0] == ggml_type_size(ids->type));
+        GGML_ASSERT(ne1 == ids->ne[0] && ne13 == 1 && ne3 == 1);
+
+        const int64_t n_expert_used = ids->ne[0];
+        const int64_t n_tokens      = ids->ne[1];
+        const int64_t nentries      = n_tokens*n_expert_used;
+        const int64_t nused         = std::min(ne02, nentries);
+        // an expert with c > 0 entries (at most one per token) has ceil(c/MMQ_FEW_NCOLS) pairs
+        npairs_max = (int) std::min(nused + (nentries - nused)/MMQ_FEW_NCOLS, ne02*((n_tokens + MMQ_FEW_NCOLS - 1) / MMQ_FEW_NCOLS));
+
+        pairs.alloc(int64_t(npairs_max)*MMQ_FEW_PAIR_INTS);
+        src1_q.alloc(int64_t(npairs_max)*nunits*MMQ_FEW_Q8_1_INT4);
+        quantize_mmq_few_id_q8_1_cuda((const float *) src1->data, (const int32_t *) ids->data, src1_q.get(), pairs.get(), mmq_few_ds_layout(src0->type),
+            ne00, nb11/sizeof(float), nb12/sizeof(float), ne11, ne02, n_expert_used, n_tokens, ids->nb[1]/sizeof(int32_t), npairs_max, stream);
+    } else {
+        GGML_ASSERT(ne11 <= MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS);
+        GGML_ASSERT(ne12 % ne02 == 0 && ne13 % ne03 == 0);

-    ggml_cuda_pool_alloc<int4> src1_q(ctx.pool(), int64_t(ne12)*ne13*nunits*ncg*MMQ_FEW_Q8_1_INT4);
-    quantize_mmq_few_q8_1_cuda((const float *) src1->data, src1_q.get(), mmq_few_ds_layout(src0->type), ne10, nb11/sizeof(float), nb12/sizeof(float), nb13/sizeof(float),
-        ne11, ne12, ne13, stream);
+        src1_q.alloc(int64_t(ne12)*ne13*nunits*ncg*MMQ_FEW_Q8_1_INT4);
+        quantize_mmq_few_q8_1_cuda((const float *) src1->data, src1_q.get(), mmq_few_ds_layout(src0->type), ne10, nb11/sizeof(float), nb12/sizeof(float), nb13/sizeof(float),
+            ne11, ne12, ne13, stream);
+    }

     switch (src0->type) {
         case GGML_TYPE_Q8_0:
-            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q8_0>(src0, src1, dst, src1_q.get(), ncg, stream);
+            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q8_0>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
             break;
         case GGML_TYPE_Q3_K:
-            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q3_K>(src0, src1, dst, src1_q.get(), ncg, stream);
+            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q3_K>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
             break;
         case GGML_TYPE_Q4_K:
-            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q4_K>(src0, src1, dst, src1_q.get(), ncg, stream);
+            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q4_K>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
             break;
         case GGML_TYPE_Q5_K:
-            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q5_K>(src0, src1, dst, src1_q.get(), ncg, stream);
+            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q5_K>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
             break;
         case GGML_TYPE_Q6_K:
-            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q6_K>(src0, src1, dst, src1_q.get(), ncg, stream);
+            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q6_K>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
             break;
         case GGML_TYPE_IQ3_S:
-            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ3_S>(src0, src1, dst, src1_q.get(), ncg, stream);
+            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ3_S>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
             break;
         case GGML_TYPE_IQ4_NL:
-            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ4_NL>(src0, src1, dst, src1_q.get(), ncg, stream);
+            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ4_NL>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
             break;
         case GGML_TYPE_IQ4_XS:
-            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ4_XS>(src0, src1, dst, src1_q.get(), ncg, stream);
+            launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ4_XS>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
             break;
         default:
             GGML_ABORT("unsupported type for few-row MMQ");
     }
 }

-// Min. number of src1 columns for the few-row MMQ, 0 if it is not used for this type, GPU and row length.
-static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const int64_t ne00) {
+// Min. number of src1 columns (MUL_MAT_ID: tokens) for the few-row MMQ, 0 if it is not used for this type, GPU and row length.
+static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const int64_t ne00, const bool has_ids) {
 #ifdef GGML_CUDA_FORCE_CUBLAS
     return 0;
 #endif // GGML_CUDA_FORCE_CUBLAS
@@ -824,13 +883,16 @@ static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const
     if (cc == GGML_CUDA_CC_DGX_SPARK && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_DGX_SPARK) {
         return 0;
     }
-    // with fewer than 8 units of 256 values per row the warps idle, MMVQ and MMQ are better there
-    if (mmq_few_unit_size(type) == 0 || ne00 < 8*QK_K) {
+    // with fewer than 8 units of 256 values per row the warps idle, MMVQ and MMQ are better there (MUL_MAT_ID splits the rows instead)
+    if (mmq_few_unit_size(type) == 0 || (!has_ids && ne00 < 8*QK_K)) {
         return 0;
     }
-    if (cc == GGML_CUDA_CC_DGX_SPARK) { // tuned on DGX Spark GB10
+    if (cc == GGML_CUDA_CC_DGX_SPARK) { // tuned on DGX Spark GB10, also for MUL_MAT_ID
         return type == GGML_TYPE_IQ4_XS || type == GGML_TYPE_IQ4_NL ? 8 : 2;
     }
+    if (has_ids) { // after MMVQ, which is faster up to its max. batch size
+        return get_mmvq_mmid_max_batch(type, cc) + 1;
+    }
     switch (type) { // all GPUs except DGX Spark, tuned on RTX 5090
         case GGML_TYPE_Q3_K:   return 3;
         case GGML_TYPE_Q5_K:   return 3;
@@ -844,14 +906,19 @@ static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const
     }
 }

-bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const int cc) {
-    const int min_ncols = ggml_cuda_mmq_few_min_ncols(src0->type, cc, src0->ne[0]);
-    if (min_ncols == 0 || src1->ne[1] < min_ncols || src1->ne[1] > MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS) {
+bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, const int cc) {
+    const int64_t ncols     = ids ? ids->ne[1] : src1->ne[1];
+    const int     min_ncols = ggml_cuda_mmq_few_min_ncols(src0->type, cc, src0->ne[0], ids != nullptr);
+    if (min_ncols == 0 || ncols < min_ncols || ncols > MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS) {
+        return false;
+    }
+    // MUL_MAT_ID: the pairs are built in shared memory
+    if (ids && (2*src0->ne[2] + 1)*sizeof(int) > ggml_cuda_info().devices[ggml_cuda_get_device()].smpb) {
         return false;
     }
     // the row must hold a whole number of ring stages of the launched kernel
-    const int ncg          = mmq_few_ncg(src1->ne[1]);
-    const int nmt          = mmq_few_nmt(src0->type, ncg, src0->ne[1]);
+    const int ncg          = ids ? 1 : mmq_few_ncg(ncols);
+    const int nmt          = ids ? 1 : mmq_few_nmt(src0->type, ncg, src0->ne[1]);
     const int nunits_stage = mmq_few_nunits_stage(src0->type, ncg, nmt);
     if (src0->ne[0] % (nunits_stage*QK_K) != 0) {
         return false;
@@ -863,13 +930,13 @@ bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor *

 #else // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)

-void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
-    GGML_UNUSED_VARS(ctx, src0, src1, dst);
+void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {
+    GGML_UNUSED_VARS(ctx, src0, src1, ids, dst);
     GGML_ABORT("few-row MMQ is not supported on HIP or MUSA");
 }

-bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const int cc) {
-    GGML_UNUSED_VARS(src0, src1, cc);
+bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, const int cc) {
+    GGML_UNUSED_VARS(src0, src1, ids, cc);
     return false;
 }

diff --git a/ggml/src/ggml-cuda/mmq-few.cuh b/ggml/src/ggml-cuda/mmq-few.cuh
index 016834998..133297c00 100644
--- a/ggml/src/ggml-cuda/mmq-few.cuh
+++ b/ggml/src/ggml-cuda/mmq-few.cuh
@@ -2,6 +2,7 @@

 #include "common.cuh"

-bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, int cc);
+// ids != nullptr: MUL_MAT_ID
+bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, int cc);

-void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst);
diff --git a/ggml/src/ggml-cuda/quantize.cu b/ggml/src/ggml-cuda/quantize.cu
index 6483da821..b37ab3d0a 100644
--- a/ggml/src/ggml-cuda/quantize.cu
+++ b/ggml/src/ggml-cuda/quantize.cu
@@ -600,25 +600,9 @@ void quantize_mmq_q8_1_cuda(
     }
 }

-// Each thread quantizes 4 values of one column, see quantize.cuh for the layout.
+// Stores the 4 values xi of a thread, values [iu, iu + 4) of column g in a unit of 256 values, in the column group yg of the unit, see quantize.cuh for the layout.
 template <mmq_q8_1_ds_layout ds_layout>
-static __global__ void quantize_mmq_few_q8_1(
-        const float * x_ptr, void * vy_ptr, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
-        const int ne1, const int ne2) {
-    ggml_cuda_pdl_lc();
-    const float * GGML_CUDA_RESTRICT x  = x_ptr;
-    int4        * GGML_CUDA_RESTRICT vy = (int4 *) vy_ptr;
-    const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4;
-
-    if (i0 >= ne00) {
-        return;
-    }
-    ggml_cuda_pdl_sync();
-
-    const int64_t i2 = blockIdx.z % ne2;
-    const int64_t i3 = blockIdx.z / ne2;
-    const float4  xi = (int) blockIdx.x < ne1 ? *(const float4 *) (x + i3*s03 + i2*s02 + blockIdx.x*s01 + i0) : make_float4(0.0f, 0.0f, 0.0f, 0.0f);
-
+static __device__ __forceinline__ void quantize_mmq_few_store(const float4 xi, int4 * yg, const int iu, const int g) {
     // the 32 values of a sub-block are held by 8 threads
     float amax = fmaxf(fmaxf(fabsf(xi.x), fabsf(xi.y)), fmaxf(fabsf(xi.z), fabsf(xi.w)));
     amax = warp_reduce_max<8>(amax);
@@ -637,10 +621,8 @@ static __global__ void quantize_mmq_few_q8_1(
         sum = d*warp_reduce_sum<8>(float(q.x + q.y + q.z + q.w));
     }

-    const int s = i0 % QK_K / QK8_1;          // sub-block
-    const int k = i0 % QK8_1;                 // first value in sub-block
-    const int g = blockIdx.x % MMQ_FEW_NCOLS; // column in group
-    int4 * yg = vy + ((blockIdx.z*(ne00/QK_K) + i0/QK_K)*(gridDim.x/MMQ_FEW_NCOLS) + blockIdx.x/MMQ_FEW_NCOLS)*MMQ_FEW_Q8_1_INT4;
+    const int s = iu / QK8_1; // sub-block
+    const int k = iu % QK8_1; // first value in sub-block
     ((int *) (yg + (s/2)*WARP_SIZE + 4*g + (k % 16)/4))[2*(s % 2) + k/16] = *(const int *) &q;
     if (k == 0) {
         int4 * ds = yg + 4*WARP_SIZE + 4*(g/2) + s/2;
@@ -652,6 +634,120 @@ static __global__ void quantize_mmq_few_q8_1(
     }
 }

+// Each thread quantizes 4 values of one column, see quantize.cuh for the layout.
+template <mmq_q8_1_ds_layout ds_layout>
+static __global__ void quantize_mmq_few_q8_1(
+        const float * x_ptr, void * vy_ptr, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
+        const int ne1, const int ne2) {
+    ggml_cuda_pdl_lc();
+    const float * GGML_CUDA_RESTRICT x  = x_ptr;
+    int4        * GGML_CUDA_RESTRICT vy = (int4 *) vy_ptr;
+    const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4;
+
+    if (i0 >= ne00) {
+        return;
+    }
+    ggml_cuda_pdl_sync();
+
+    const int64_t i2 = blockIdx.z % ne2;
+    const int64_t i3 = blockIdx.z / ne2;
+    const float4  xi = (int) blockIdx.x < ne1 ? *(const float4 *) (x + i3*s03 + i2*s02 + blockIdx.x*s01 + i0) : make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+
+    const int g = blockIdx.x % MMQ_FEW_NCOLS; // column in group
+    int4 * yg = vy + ((blockIdx.z*(ne00/QK_K) + i0/QK_K)*(gridDim.x/MMQ_FEW_NCOLS) + blockIdx.x/MMQ_FEW_NCOLS)*MMQ_FEW_Q8_1_INT4;
+    quantize_mmq_few_store<ds_layout>(xi, yg, i0 % QK_K, g);
+}
+
+// MUL_MAT_ID: block x is the (token, slot) entry, ids are as in mm_ids_helper. Each block finds the pair of its entry in shared memory, block (0, 0) also writes all pairs.
+// Entry rank r = number of earlier tokens with the same expert, pair = first pair of the expert + r/MMQ_FEW_NCOLS, column = r % MMQ_FEW_NCOLS.
+template <mmq_q8_1_ds_layout ds_layout>
+static __global__ void quantize_mmq_few_id_q8_1(
+        const float * x_ptr, const int32_t * ids, void * vy_ptr, int * pairs, const int64_t ne00, const int64_t s11, const int64_t s12,
+        const int ne11, const int n_experts, const int n_expert_used, const int64_t si1, const int npairs_max) {
+    // per expert: bit t set if token t uses it and the index of its first pair; then the total number of pairs
+    extern __shared__ unsigned int data_mmq_few_id[];
+    unsigned int * tokens = data_mmq_few_id;
+    int          * pair0  = (int *) (tokens + n_experts);
+    int          & npairs = pair0[n_experts];
+
+    ggml_cuda_pdl_lc();
+    const float * GGML_CUDA_RESTRICT x  = x_ptr;
+    int4        * GGML_CUDA_RESTRICT vy = (int4 *) vy_ptr;
+    ggml_cuda_pdl_sync();
+
+    const int nentries = gridDim.x;
+    auto expert_of = [&](const int i) {
+        const int e = ids[(i / n_expert_used)*si1 + i % n_expert_used];
+        return e >= 0 && e < n_experts ? e : -1; // other ids skip the slot
+    };
+
+    for (int e = threadIdx.x; e < n_experts; e += blockDim.x) {
+        tokens[e] = 0;
+    }
+    __syncthreads();
+    for (int i = threadIdx.x; i < nentries; i += blockDim.x) {
+        const int e = expert_of(i);
+        if (e >= 0) {
+            atomicOr(&tokens[e], 1u << (i / n_expert_used));
+        }
+    }
+    __syncthreads();
+    if (threadIdx.x < WARP_SIZE) {
+        const int per_lane = (n_experts + WARP_SIZE - 1) / WARP_SIZE;
+        const int e0       = min(int(threadIdx.x)*per_lane, n_experts);
+        const int e1       = min(e0 + per_lane, n_experts);
+        int n = 0;
+        for (int e = e0; e < e1; ++e) {
+            n += (__popc(tokens[e]) + MMQ_FEW_NCOLS - 1) / MMQ_FEW_NCOLS;
+        }
+        int p = warp_prefix_inclusive_sum<int>(n) - n;
+        for (int e = e0; e < e1; ++e) {
+            pair0[e] = p;
+            p += (__popc(tokens[e]) + MMQ_FEW_NCOLS - 1) / MMQ_FEW_NCOLS;
+        }
+        if (threadIdx.x == WARP_SIZE - 1) {
+            npairs = p;
+        }
+    }
+    __syncthreads();
+
+    auto rank = [&](const int i, const int e) {
+        return __popc(tokens[e] & ((1u << (i / n_expert_used)) - 1u));
+    };
+
+    if (blockIdx.x == 0 && blockIdx.y == 0) {
+        for (int i = threadIdx.x; i < nentries; i += blockDim.x) {
+            const int e = expert_of(i);
+            if (e < 0) {
+                continue;
+            }
+            const int r = rank(i, e);
+            int * pair = pairs + (pair0[e] + r/MMQ_FEW_NCOLS)*MMQ_FEW_PAIR_INTS;
+            pair[2 + r % MMQ_FEW_NCOLS] = i;
+            if (r % MMQ_FEW_NCOLS == 0) {
+                pair[0] = e;
+                pair[1] = min(__popc(tokens[e]) - r, MMQ_FEW_NCOLS);
+            }
+        }
+        for (int p = npairs + threadIdx.x; p < npairs_max; p += blockDim.x) {
+            pairs[p*MMQ_FEW_PAIR_INTS + 1] = 0;
+        }
+    }
+
+    const int     e  = expert_of(blockIdx.x);
+    const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4;
+    if (e < 0 || i0 >= ne00) {
+        return;
+    }
+    const int    it = blockIdx.x / n_expert_used;
+    const int    is = blockIdx.x % n_expert_used;
+    const float4 xi = *(const float4 *) (x + it*s12 + (is % ne11)*s11 + i0);
+
+    const int r = rank(blockIdx.x, e);
+    int4 * yg = vy + (int64_t(pair0[e] + r/MMQ_FEW_NCOLS)*(ne00/QK_K) + i0/QK_K)*MMQ_FEW_Q8_1_INT4;
+    quantize_mmq_few_store<ds_layout>(xi, yg, i0 % QK_K, r % MMQ_FEW_NCOLS);
+}
+
 void quantize_mmq_few_q8_1_cuda(
         const float * x, void * vy, const mmq_q8_1_ds_layout ds_layout, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
         const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {
@@ -675,6 +771,32 @@ void quantize_mmq_few_q8_1_cuda(
     }
 }

+void quantize_mmq_few_id_q8_1_cuda(
+        const float * x, const int32_t * ids, void * vy, int * pairs, const mmq_q8_1_ds_layout ds_layout, const int64_t ne00, const int64_t s11, const int64_t s12,
+        const int64_t ne11, const int64_t n_experts, const int64_t n_expert_used, const int64_t n_tokens, const int64_t si1, const int64_t npairs_max, cudaStream_t stream) {
+    GGML_ASSERT(ne00 % QK_K == 0);
+    GGML_ASSERT(n_tokens <= 32); // one bit per token
+
+    const int64_t block_num_y   = (ne00 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ);
+    const size_t  nbytes_shared = (2*n_experts + 1)*sizeof(int);
+    GGML_ASSERT(nbytes_shared <= ggml_cuda_info().devices[ggml_cuda_get_device()].smpb);
+    const dim3 num_blocks(n_tokens*n_expert_used, block_num_y, 1);
+    const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
+    const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, nbytes_shared, stream);
+    switch (ds_layout) {
+        case MMQ_Q8_1_DS_LAYOUT_D4:
+            ggml_cuda_kernel_launch(quantize_mmq_few_id_q8_1<MMQ_Q8_1_DS_LAYOUT_D4>, launch_params,
+                x, ids, vy, pairs, ne00, s11, s12, (int) ne11, (int) n_experts, (int) n_expert_used, si1, (int) npairs_max);
+            break;
+        case MMQ_Q8_1_DS_LAYOUT_DS4:
+            ggml_cuda_kernel_launch(quantize_mmq_few_id_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4>, launch_params,
+                x, ids, vy, pairs, ne00, s11, s12, (int) ne11, (int) n_experts, (int) n_expert_used, si1, (int) npairs_max);
+            break;
+        default:
+            GGML_ABORT("fatal error");
+    }
+}
+
 // scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row)
 void quantize_scatter_mmq_q8_1_cuda(
         const float * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0,
diff --git a/ggml/src/ggml-cuda/quantize.cuh b/ggml/src/ggml-cuda/quantize.cuh
index 11720ef93..0613cf367 100644
--- a/ggml/src/ggml-cuda/quantize.cuh
+++ b/ggml/src/ggml-cuda/quantize.cuh
@@ -32,6 +32,14 @@ void quantize_mmq_few_q8_1_cuda(
         const float * x, void * vy, mmq_q8_1_ds_layout ds_layout, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
         int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);

+// MUL_MAT_ID: a pair is an expert and one column group of up to MMQ_FEW_NCOLS of its (token, slot) entries. The src1 of pair p is channel p above, its columns past the entries are not written.
+// A pair is MMQ_FEW_PAIR_INTS ints: expert, number of entries (0 for an unused pair), then the entries token*n_expert_used + slot.
+static constexpr int MMQ_FEW_PAIR_INTS = 2 + MMQ_FEW_NCOLS;
+
+void quantize_mmq_few_id_q8_1_cuda(
+        const float * x, const int32_t * ids, void * vy, int * pairs, mmq_q8_1_ds_layout ds_layout, int64_t ne00, int64_t s11, int64_t s12,
+        int64_t ne11, int64_t n_experts, int64_t n_expert_used, int64_t n_tokens, int64_t si1, int64_t npairs_max, cudaStream_t stream);
+
 void quantize_mmq_fp4_cuda(const float *   x,
                              const int32_t * ids,
                              void *          vy,
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index da91b51e9..f2ede4290 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -12030,6 +12030,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
             test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 6170, 13, k, {1, 1}, {1, 1}));
             test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1000, 21, k, {1, 1}, {1, 1}));
         }
+        // MUL_MAT_ID: K = 8 units (the warps split K, broadcast src1), K = 2 units (the warps split the rows, more than 8 tokens per expert, unaligned experts)
+        test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 16, 4, true,   72, 13, 2048));
+        test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32,  4, 2, false, 202, 24,  512));
     }
     // permuted and broadcast src0/src1 for the 2 byte aligned types
     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K,  GGML_TYPE_F32, 64, 13, 2304, {2, 3}, {1, 1}, {0, 2, 1, 3}));
@@ -12044,6 +12047,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
     test_cases.emplace_back(new test_mul_mat_pos(GGML_TYPE_Q5_K, GGML_TYPE_F32, 1000, 13, 4608, {1, 1}, {1, 1}));
     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 6170, 13, 4608, {1, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 0, 0, 0, true));
     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 1000, 21, 4608, {1, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 0, 0, 0, true));
+    // CUDA few-row MMQ for MUL_MAT_ID: strided experts, more than 32 experts
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32,  16, 4, true,  72, 13, 2048, 80));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 128, 8, false, 72, 16, 2048));

 #if 0
     {
@@ -13411,6 +13417,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
     test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150,  2560, 256, true, 16, 8, false, false, true, true));
     // K = 2048: few-row MMQ, where enabled, takes the gate and up matmuls instead of the fused MMQ path
     test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K,   GGML_GLU_OP_SWIGLU,         9,   128, 2048, false, 1, 1, false, false, true, false, { 1, 1 }));
+    // on DGX Spark also the expert gate and up matmuls at 8 tokens, instead of the fused MMVQ path
+    test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K,   GGML_GLU_OP_SWIGLU,         8,   128, 2048, true, 16, 4, true));

     for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT, GATING_FUNC_SQRT_SOFTPLUS}) {
         for (bool with_norm : {false, true}) {