Commit 6afbdfb7 for whisper.cpp
commit 6afbdfb7a00b4c5963a105643b000f2201a6ca03
Author: Aman Gupta <amangupta052@gmail.com>
Date: Wed Nov 11 16:46:13 2026 +0800
cuda : use MMVF for BF16 matrices of few rows at small batch sizes (llama/31959)
MMF tiles 32 rows per block, so a BF16 matrix with few rows (e.g. the
qwen4exp hyper-connection projections, 320 rows with K = 10240) runs only
a handful of blocks and leaves most SMs idle at batch sizes 2..8. MMVF
runs a block per row and wins there when K is long enough.
MMVF's cost grows with rows x batch while MMF's stays roughly flat in that
range, so use MMVF only while rows * ne11 * 256 <= nsm * min(K, max_k) and
K >= 1024. The budget scales with the SM count, which keeps GPUs with few
SMs (e.g. the A16) on MMF. Blackwell allows up to 8 columns and
max_k = 6144; older GPUs stop at 5 columns and max_k = 4096.
Add eval cases at 128x2560, 512x2560 and 320x10240 for n = 2, 3, 4, 5 and 8.
diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu
index d96dbd04..154a621f 100644
--- a/ggml/src/ggml-cuda/mmvf.cu
+++ b/ggml/src/ggml-cuda/mmvf.cu
@@ -941,6 +941,18 @@ bool ggml_cuda_should_use_mmvf(const ggml_tensor * src0, int cc, int warp_size,
if (src0_small && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) {
return ne11 <= MMVF_MAX_BATCH_SIZE;
}
+ // MMF tiles 32 rows per block, so a matrix of few rows leaves most SMs idle while MMVF runs a block
+ // per row: with a long enough K, MMVF wins for the small batches of speculative decoding as long as
+ // its work (rows x batch) stays small relative to the SM count, with a budget that grows with K
+ if (src0_small && src0_ne[0] >= 1024) {
+ const bool blackwell = cc >= GGML_CUDA_CC_BLACKWELL;
+ const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm;
+ const int64_t max_ncols = blackwell ? MMVF_MAX_BATCH_SIZE : 5;
+ const int64_t max_k = blackwell ? 6144 : 4096;
+ if (ne11 <= max_ncols && src0_ne[1]*ne11*256 <= nsm*std::min(src0_ne[0], max_k)) {
+ return true;
+ }
+ }
if (ampere_mma_available(cc)) {
return src0_small && ne11 == 1;
}