Commit 05db09a4d for llama.cpp
commit 05db09a4d3c3c1caa7cda31f98eca6ae28d92eb5
Author: thelittlefireman <5165783+thelittlefireman@users.noreply.github.com>
Date: Mon Nov 16 06:37:42 2026 +0100
ggml-cuda: fix MoE regression on HIP with #29948 (#32344)
diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh
index d645a8aac..efe6527c2 100644
--- a/ggml/src/ggml-cuda/mmq.cuh
+++ b/ggml/src/ggml-cuda/mmq.cuh
@@ -1079,8 +1079,14 @@ static __global__ void mul_mat_q(
const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);
const int wt = tmp2.x;
int zt = tmp2.y;
+#if defined(GGML_USE_HIP)
+ // Row tiles innermost for MoE, see launch_mul_mat_q.
+ const int jt = ids_dst ? blockIdx.y : blockIdx.x;
+ const int it = ids_dst ? blockIdx.x : blockIdx.y;
+#else
const int jt = blockIdx.x;
const int it = blockIdx.y;
+#endif // defined(GGML_USE_HIP)
// Defaults for regular matrix multiplication:
int col_low = jt*J;
@@ -1483,8 +1489,15 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
const int I_dst = has_fusion ? config.I/2 : config.I; // Rows of dst per tile, see mul_mat_q.
const int nti = (args.nrows_x + I_dst - 1) / I_dst;
const int ntzw = args.nchannels_y * args.nsamples_y;
- // Column tiles innermost so that the concurrently running blocks share the same tile of x and read it from L2.
- const dim3 block_nums_xy_tiling(args.ntj, nti, ntzw);
+#if defined(GGML_USE_HIP)
+ // For MoE the column tiles are tied to experts, so the column-innermost order does not improve the reuse of x in L2.
+ // It also groups blocks with very different runtimes which schedules poorly on AMD GPUs, see #31599.
+ // Therefore the row tiles are innermost for MoE. The column-innermost order is kept for dense multiplication.
+ const bool rows_innermost = args.ids_dst != nullptr;
+#else
+ constexpr bool rows_innermost = false;
+#endif // defined(GGML_USE_HIP)
+ const dim3 block_nums_xy_tiling = rows_innermost ? dim3(nti, args.ntj, ntzw) : dim3(args.ntj, nti, ntzw);
GGML_ASSERT(args.ids_dst || args.nchannels_y % args.nchannels_x == 0);
GGML_ASSERT( args.nsamples_y % args.nsamples_x == 0);
@@ -1499,7 +1512,7 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio);
if (has_fusion || !config.stream_k) {
- GGML_ASSERT(nti <= 65535); // gridDim.y limit
+ GGML_ASSERT((rows_innermost ? args.ntj : nti) <= 65535); // gridDim.y limit
mul_mat_q<type, J, fallback, prec_src1, has_fusion><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>
(args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.tile_spec, args.dst, nullptr,
args.y_scale, args.fusion.glu_op, args.fusion.glu_limit, (const float *) args.fusion.x_scale, (const float *) args.fusion.gate_scale,