Commit adc0dbc2 for whisper.cpp

commit adc0dbc2ecf4d6b4dc781f4f30141c956bd1c68f
Author: Konrad Moren <kmoren@nvidia.com>
Date:   Thu Nov 12 10:28:36 2026 +0100

    CUDA: Optimize prefil via fuse of w_s scale in epilogue MMQ for nvfp4 checkpoints (llama/26048)

    * ggml-cuda: Apply NVFP4 weight scale in MMQ write-back

    * ggml-cuda: Fuse mul_mat(_id) + scale into MMQ

    * ggml-cuda: Tests for the fused scale

    * ggml-cuda: reuse existing graph to test fusion in mmq epiloque

    * ggml-cuda: rename test_mul_mat_vec_fusion to test_mul_mat_fusion for consistency

    * ggml-cuda: refine comments for clarity on MMQ dispatch and fusion checks in mul_mat functions

    * ggml-cuda: refactor fusion checks for mul_mat operations and remove unused function

diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 92ae9848..28073a17 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -5009,6 +5009,23 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
                 return 1;
             }

+            if (!with_bias && src0->type == GGML_TYPE_NVFP4 && src1->type == GGML_TYPE_F32 && mm_node->type == GGML_TYPE_F32) {
+                const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc;
+                const bool is_id = op == GGML_OP_MUL_MAT_ID;
+                const int64_t ncols = is_id ? src1->ne[2] : src1->ne[1];
+                const bool use_mmvq = is_id ? ncols <= get_mmvq_mmid_max_batch(src0->type, cc) :
+                                            ggml_cuda_should_use_mmvq(src0->type, cc, ncols);
+                const bool bad_padding_clear = ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE &&
+                                               ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) &&
+                                               src0->view_src;
+                if (!use_mmvq && !bad_padding_clear &&
+                        ggml_cuda_should_use_mmq(src0->type, cc, ncols, /*n_experts =*/ is_id ? src0->ne[2] : 0)) {
+                    fusion_data.prec_src1 = ggml_prec_get_src(mm_node, 1);
+                    ggml_cuda_mul_mat_q(*cuda_ctx, src0, src1, ids, out_node, &fusion_data);
+                    return n_ops - 1;
+                }
+            }
+
             if (ggml_cuda_should_fuse_mul_mat_vec_q(mm_node)) {
                 ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, out_node, &fusion_data);
                 fused_mul_mat_vec = true;
diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
index 36765750..2e619976 100644
--- a/ggml/src/ggml-cuda/mmq.cu
+++ b/ggml/src/ggml-cuda/mmq.cu
@@ -169,21 +169,27 @@ void ggml_cuda_mul_mat_q(
     GGML_ASSERT(!ids || ids->type  == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.

     // Fused gate/up/GLU: src0 is the up weight, dst the GLU output.
+    // Fused scale (no gate): dst is the MUL that applies the weight scale to the MUL_MAT.
     const ggml_tensor * gate = fusion ? fusion->gate : nullptr;
     ggml_cuda_mm_fusion_args_device fusion_device = {};
     if (fusion) {
-        GGML_ASSERT(gate && !fusion->x_bias && !fusion->gate_bias && !fusion->x_scale == !fusion->gate_scale);
-        GGML_ASSERT(gate->type == src0->type && ggml_are_same_shape(gate, src0) && ggml_are_same_stride(gate, src0));
+        GGML_ASSERT(!fusion->x_bias && !fusion->gate_bias);
         // NVFP4 weight scales (only applied for NVFP4): one value per tensor, or one per expert with ids.
         for (const ggml_tensor * scale : { fusion->x_scale, fusion->gate_scale }) {
             GGML_ASSERT(!scale || (src0->type == GGML_TYPE_NVFP4 && scale->type == GGML_TYPE_F32 && ggml_is_contiguous(scale) &&
                 ggml_nelements(scale) == (ids ? src0->ne[2] : 1)));
         }
-        fusion_device.gate       = gate->data;
-        fusion_device.x_scale    = fusion->x_scale    ? fusion->x_scale->data    : nullptr;
-        fusion_device.gate_scale = fusion->gate_scale ? fusion->gate_scale->data : nullptr;
-        fusion_device.glu_op     = fusion->glu_op;
-        fusion_device.glu_limit  = fusion->glu_limit;
+        if (gate) {
+            GGML_ASSERT(!fusion->x_scale == !fusion->gate_scale);
+            GGML_ASSERT(gate->type == src0->type && ggml_are_same_shape(gate, src0) && ggml_are_same_stride(gate, src0));
+            fusion_device.gate       = gate->data;
+            fusion_device.gate_scale = fusion->gate_scale ? fusion->gate_scale->data : nullptr;
+            fusion_device.glu_op     = fusion->glu_op;
+            fusion_device.glu_limit  = fusion->glu_limit;
+        } else {
+            GGML_ASSERT(!fusion->gate_scale && fusion->x_scale);
+        }
+        fusion_device.x_scale = fusion->x_scale ? fusion->x_scale->data : nullptr;
     }

     GGML_TENSOR_BINARY_OP_LOCALS;
@@ -231,8 +237,8 @@ void ggml_cuda_mul_mat_q(

     const bool fallback = ggml_cuda_mmq_needs_fallback(ne01);

-    // With fusion dst is the GLU output, not a MUL_MAT; the src1 precision requested by both fused MUL_MATs is in the fusion args.
-    const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0->type, gate ? fusion->prec_src1 : ggml_prec_get_src(dst, 1), cc);
+    // With fusion dst is the GLU output or the scale node, not a MUL_MAT; the src1 precision requested by the fused MUL_MATs is in the fusion args.
+    const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0->type, fusion ? fusion->prec_src1 : ggml_prec_get_src(dst, 1), cc);

     const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4;
     const size_t y_block_size       = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq);
diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh
index 42a3afd2..d645a8aa 100644
--- a/ggml/src/ggml-cuda/mmq.cuh
+++ b/ggml/src/ggml-cuda/mmq.cuh
@@ -465,7 +465,11 @@ template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __de
         const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst,
         const float * __restrict__ y_scale, const int stride, const int i_max, const int j_sup,
         const ggml_glu_op glu_op, const float glu_limit, const float * __restrict__ x_scale, const float * __restrict__ gate_scale) {
-    GGML_UNUSED_VARS(glu_op, glu_limit, x_scale, gate_scale);
+    GGML_UNUSED_VARS(glu_op, glu_limit, gate_scale);
+
+    // NVFP4 weight scale of a fused MUL (per tensor, or of the expert of this tile), 1 without.
+    const float scale = type == GGML_TYPE_NVFP4 && x_scale ? *x_scale : 1.0f;
+
     constexpr int warp_size = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps    = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;
     constexpr int I         = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);
@@ -489,11 +493,15 @@ template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __de
                 continue;
             }

-            if (y_scale_used) {
-                dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
+            float value = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
+            if constexpr (type == GGML_TYPE_NVFP4) {
+                if (y_scale_used) {
+                    value *= y_scale[j];
+                }
             } else {
-                dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
+                GGML_UNUSED(y_scale_used);
             }
+            dst[ids_dst[j]*stride + i] = value * scale;
         }
     }
 }
@@ -520,9 +528,9 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(

     const bool y_scale_used = type == GGML_TYPE_NVFP4 && y_scale != nullptr;

-    // Fused NVFP4: load the weight scales once, 1 without scales.
+    // NVFP4: load the weight scales once, 1 without scales. Without fusion x_scale is the scale of a fused MUL.
     constexpr bool fused_nvfp4 = has_fusion && type == GGML_TYPE_NVFP4;
-    const float scale_up   = fused_nvfp4 && x_scale ? *x_scale    : 1.0f;
+    const float scale_up   = type == GGML_TYPE_NVFP4 && x_scale ? *x_scale : 1.0f;
     const float scale_gate = fused_nvfp4 && gate_scale ? *gate_scale : 1.0f;

 #pragma unroll
@@ -556,10 +564,16 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(
                         gate *= scale_gate;
                     }
                     dst[ids_dst[j]*stride + i] = ggml_cuda_op_glu_single(glu_op, gate, up, glu_limit);
-                } else if (y_scale_used) {
-                    dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l];
                 } else {
-                    dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l];
+                    float value = sum[(j0/tile_C::J + n)*tile_C::ne + l];
+                    if constexpr (type == GGML_TYPE_NVFP4) {
+                        if (y_scale_used) {
+                            value *= y_scale[j];
+                        }
+                    } else {
+                        GGML_UNUSED(y_scale_used);
+                    }
+                    dst[ids_dst[j]*stride + i] = value * scale_up;
                 }
             }
         }
@@ -1201,10 +1215,12 @@ static __global__ void mul_mat_q(

         const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;

+        const float * x_scale_tile = x_scale && ids_dst ? x_scale + zt : x_scale;
+
         constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.
         mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1, false>
             (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
-             glu_op, glu_limit, x_scale, gate_scale, stride_row_x, ncols_y, stride_col_dst,
+             glu_op, glu_limit, x_scale_tile, gate_scale, stride_row_x, ncols_y, stride_col_dst,
              tile_x_i_max, col_high - col_low, kb0_start, kb0_stop);

         kbc += blocks_per_ne00.z;
@@ -1275,10 +1291,12 @@ static __global__ void mul_mat_q(

     const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;

+    const float * x_scale_tile = x_scale && ids_dst ? x_scale + zt : x_scale;
+
     constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.
     mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1, false>
         (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
-         glu_op, glu_limit, x_scale, gate_scale, stride_row_x, ncols_y, stride_col_dst,
+         glu_op, glu_limit, x_scale_tile, gate_scale, stride_row_x, ncols_y, stride_col_dst,
          tile_x_i_max, col_high - col_low, kb0_start, kb0_stop);
 }