Commit adc0dbc2 for whisper.cpp
commit adc0dbc2ecf4d6b4dc781f4f30141c956bd1c68f
Author: Konrad Moren <kmoren@nvidia.com>
Date: Thu Nov 12 10:28:36 2026 +0100
CUDA: Optimize prefil via fuse of w_s scale in epilogue MMQ for nvfp4 checkpoints (llama/26048)
* ggml-cuda: Apply NVFP4 weight scale in MMQ write-back
* ggml-cuda: Fuse mul_mat(_id) + scale into MMQ
* ggml-cuda: Tests for the fused scale
* ggml-cuda: reuse existing graph to test fusion in mmq epiloque
* ggml-cuda: rename test_mul_mat_vec_fusion to test_mul_mat_fusion for consistency
* ggml-cuda: refine comments for clarity on MMQ dispatch and fusion checks in mul_mat functions
* ggml-cuda: refactor fusion checks for mul_mat operations and remove unused function
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 92ae9848..28073a17 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -5009,6 +5009,23 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
return 1;
}
+ if (!with_bias && src0->type == GGML_TYPE_NVFP4 && src1->type == GGML_TYPE_F32 && mm_node->type == GGML_TYPE_F32) {
+ const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc;
+ const bool is_id = op == GGML_OP_MUL_MAT_ID;
+ const int64_t ncols = is_id ? src1->ne[2] : src1->ne[1];
+ const bool use_mmvq = is_id ? ncols <= get_mmvq_mmid_max_batch(src0->type, cc) :
+ ggml_cuda_should_use_mmvq(src0->type, cc, ncols);
+ const bool bad_padding_clear = ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE &&
+ ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) &&
+ src0->view_src;
+ if (!use_mmvq && !bad_padding_clear &&
+ ggml_cuda_should_use_mmq(src0->type, cc, ncols, /*n_experts =*/ is_id ? src0->ne[2] : 0)) {
+ fusion_data.prec_src1 = ggml_prec_get_src(mm_node, 1);
+ ggml_cuda_mul_mat_q(*cuda_ctx, src0, src1, ids, out_node, &fusion_data);
+ return n_ops - 1;
+ }
+ }
+
if (ggml_cuda_should_fuse_mul_mat_vec_q(mm_node)) {
ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, out_node, &fusion_data);
fused_mul_mat_vec = true;
diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
index 36765750..2e619976 100644
--- a/ggml/src/ggml-cuda/mmq.cu
+++ b/ggml/src/ggml-cuda/mmq.cu
@@ -169,21 +169,27 @@ void ggml_cuda_mul_mat_q(
GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.
// Fused gate/up/GLU: src0 is the up weight, dst the GLU output.
+ // Fused scale (no gate): dst is the MUL that applies the weight scale to the MUL_MAT.
const ggml_tensor * gate = fusion ? fusion->gate : nullptr;
ggml_cuda_mm_fusion_args_device fusion_device = {};
if (fusion) {
- GGML_ASSERT(gate && !fusion->x_bias && !fusion->gate_bias && !fusion->x_scale == !fusion->gate_scale);
- GGML_ASSERT(gate->type == src0->type && ggml_are_same_shape(gate, src0) && ggml_are_same_stride(gate, src0));
+ GGML_ASSERT(!fusion->x_bias && !fusion->gate_bias);
// NVFP4 weight scales (only applied for NVFP4): one value per tensor, or one per expert with ids.
for (const ggml_tensor * scale : { fusion->x_scale, fusion->gate_scale }) {
GGML_ASSERT(!scale || (src0->type == GGML_TYPE_NVFP4 && scale->type == GGML_TYPE_F32 && ggml_is_contiguous(scale) &&
ggml_nelements(scale) == (ids ? src0->ne[2] : 1)));
}
- fusion_device.gate = gate->data;
- fusion_device.x_scale = fusion->x_scale ? fusion->x_scale->data : nullptr;
- fusion_device.gate_scale = fusion->gate_scale ? fusion->gate_scale->data : nullptr;
- fusion_device.glu_op = fusion->glu_op;
- fusion_device.glu_limit = fusion->glu_limit;
+ if (gate) {
+ GGML_ASSERT(!fusion->x_scale == !fusion->gate_scale);
+ GGML_ASSERT(gate->type == src0->type && ggml_are_same_shape(gate, src0) && ggml_are_same_stride(gate, src0));
+ fusion_device.gate = gate->data;
+ fusion_device.gate_scale = fusion->gate_scale ? fusion->gate_scale->data : nullptr;
+ fusion_device.glu_op = fusion->glu_op;
+ fusion_device.glu_limit = fusion->glu_limit;
+ } else {
+ GGML_ASSERT(!fusion->gate_scale && fusion->x_scale);
+ }
+ fusion_device.x_scale = fusion->x_scale ? fusion->x_scale->data : nullptr;
}
GGML_TENSOR_BINARY_OP_LOCALS;
@@ -231,8 +237,8 @@ void ggml_cuda_mul_mat_q(
const bool fallback = ggml_cuda_mmq_needs_fallback(ne01);
- // With fusion dst is the GLU output, not a MUL_MAT; the src1 precision requested by both fused MUL_MATs is in the fusion args.
- const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0->type, gate ? fusion->prec_src1 : ggml_prec_get_src(dst, 1), cc);
+ // With fusion dst is the GLU output or the scale node, not a MUL_MAT; the src1 precision requested by the fused MUL_MATs is in the fusion args.
+ const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0->type, fusion ? fusion->prec_src1 : ggml_prec_get_src(dst, 1), cc);
const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4;
const size_t y_block_size = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq);
diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh
index 42a3afd2..d645a8aa 100644
--- a/ggml/src/ggml-cuda/mmq.cuh
+++ b/ggml/src/ggml-cuda/mmq.cuh
@@ -465,7 +465,11 @@ template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __de
const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst,
const float * __restrict__ y_scale, const int stride, const int i_max, const int j_sup,
const ggml_glu_op glu_op, const float glu_limit, const float * __restrict__ x_scale, const float * __restrict__ gate_scale) {
- GGML_UNUSED_VARS(glu_op, glu_limit, x_scale, gate_scale);
+ GGML_UNUSED_VARS(glu_op, glu_limit, gate_scale);
+
+ // NVFP4 weight scale of a fused MUL (per tensor, or of the expert of this tile), 1 without.
+ const float scale = type == GGML_TYPE_NVFP4 && x_scale ? *x_scale : 1.0f;
+
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;
constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);
@@ -489,11 +493,15 @@ template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __de
continue;
}
- if (y_scale_used) {
- dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
+ float value = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ if (y_scale_used) {
+ value *= y_scale[j];
+ }
} else {
- dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
+ GGML_UNUSED(y_scale_used);
}
+ dst[ids_dst[j]*stride + i] = value * scale;
}
}
}
@@ -520,9 +528,9 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(
const bool y_scale_used = type == GGML_TYPE_NVFP4 && y_scale != nullptr;
- // Fused NVFP4: load the weight scales once, 1 without scales.
+ // NVFP4: load the weight scales once, 1 without scales. Without fusion x_scale is the scale of a fused MUL.
constexpr bool fused_nvfp4 = has_fusion && type == GGML_TYPE_NVFP4;
- const float scale_up = fused_nvfp4 && x_scale ? *x_scale : 1.0f;
+ const float scale_up = type == GGML_TYPE_NVFP4 && x_scale ? *x_scale : 1.0f;
const float scale_gate = fused_nvfp4 && gate_scale ? *gate_scale : 1.0f;
#pragma unroll
@@ -556,10 +564,16 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(
gate *= scale_gate;
}
dst[ids_dst[j]*stride + i] = ggml_cuda_op_glu_single(glu_op, gate, up, glu_limit);
- } else if (y_scale_used) {
- dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l];
} else {
- dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l];
+ float value = sum[(j0/tile_C::J + n)*tile_C::ne + l];
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ if (y_scale_used) {
+ value *= y_scale[j];
+ }
+ } else {
+ GGML_UNUSED(y_scale_used);
+ }
+ dst[ids_dst[j]*stride + i] = value * scale_up;
}
}
}
@@ -1201,10 +1215,12 @@ static __global__ void mul_mat_q(
const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;
+ const float * x_scale_tile = x_scale && ids_dst ? x_scale + zt : x_scale;
+
constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.
mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1, false>
(x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
- glu_op, glu_limit, x_scale, gate_scale, stride_row_x, ncols_y, stride_col_dst,
+ glu_op, glu_limit, x_scale_tile, gate_scale, stride_row_x, ncols_y, stride_col_dst,
tile_x_i_max, col_high - col_low, kb0_start, kb0_stop);
kbc += blocks_per_ne00.z;
@@ -1275,10 +1291,12 @@ static __global__ void mul_mat_q(
const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;
+ const float * x_scale_tile = x_scale && ids_dst ? x_scale + zt : x_scale;
+
constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.
mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1, false>
(x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
- glu_op, glu_limit, x_scale, gate_scale, stride_row_x, ncols_y, stride_col_dst,
+ glu_op, glu_limit, x_scale_tile, gate_scale, stride_row_x, ncols_y, stride_col_dst,
tile_x_i_max, col_high - col_low, kb0_start, kb0_stop);
}