Commit 459ab1cdd for llama.cpp

commit 459ab1cdd3c566168c814b6f50cb1a7e6eb79914
Author: Aman Gupta <amangupta052@gmail.com>
Date:   Wed Nov 11 16:46:13 2026 +0800

    cuda : use MMVF for BF16 matrices of few rows at small batch sizes (#31959)

    MMF tiles 32 rows per block, so a BF16 matrix with few rows (e.g. the
    qwen4exp hyper-connection projections, 320 rows with K = 10240) runs only
    a handful of blocks and leaves most SMs idle at batch sizes 2..8. MMVF
    runs a block per row and wins there when K is long enough.

    MMVF's cost grows with rows x batch while MMF's stays roughly flat in that
    range, so use MMVF only while rows * ne11 * 256 <= nsm * min(K, max_k) and
    K >= 1024. The budget scales with the SM count, which keeps GPUs with few
    SMs (e.g. the A16) on MMF. Blackwell allows up to 8 columns and
    max_k = 6144; older GPUs stop at 5 columns and max_k = 4096.

    Add eval cases at 128x2560, 512x2560 and 320x10240 for n = 2, 3, 4, 5 and 8.

diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu
index d96dbd04d..154a621fa 100644
--- a/ggml/src/ggml-cuda/mmvf.cu
+++ b/ggml/src/ggml-cuda/mmvf.cu
@@ -941,6 +941,18 @@ bool ggml_cuda_should_use_mmvf(const ggml_tensor * src0, int cc, int warp_size,
                 if (src0_small && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) {
                     return ne11 <= MMVF_MAX_BATCH_SIZE;
                 }
+                // MMF tiles 32 rows per block, so a matrix of few rows leaves most SMs idle while MMVF runs a block
+                // per row: with a long enough K, MMVF wins for the small batches of speculative decoding as long as
+                // its work (rows x batch) stays small relative to the SM count, with a budget that grows with K
+                if (src0_small && src0_ne[0] >= 1024) {
+                    const bool    blackwell = cc >= GGML_CUDA_CC_BLACKWELL;
+                    const int     nsm       = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm;
+                    const int64_t max_ncols = blackwell ? MMVF_MAX_BATCH_SIZE : 5;
+                    const int64_t max_k     = blackwell ? 6144 : 4096;
+                    if (ne11 <= max_ncols && src0_ne[1]*ne11*256 <= nsm*std::min(src0_ne[0], max_k)) {
+                        return true;
+                    }
+                }
                 if (ampere_mma_available(cc)) {
                     return src0_small && ne11 == 1;
                 }
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 7be4977d7..c8b1d02e4 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -11984,6 +11984,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
         }
     }

+    // BF16 matrices of few rows and a long K at small batch sizes (MMVF instead of MMF)
+    for (int n : {2, 3, 4, 5, 8}) {
+        test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32,  320, n, 10240, {1, 1}, {1, 1}));
+        test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32,  512, n,  2560, {1, 1}, {1, 1}));
+        test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32,  128, n,  2560, {1, 1}, {1, 1}));
+    }
+
     // BF16 is absent from base_types: add the 3 standard non-contig permutations explicitly
     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16,  1, 256, {2, 3}, {1, 1}, {0, 2, 1, 3}));
     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16,  1, 256, {2, 3}, {1, 1}, {0, 1, 3, 2}));