Commit 9d3aba6b5 for llama.cpp
commit 9d3aba6b5e10a6dd086bd0555bd12c00366a8916
Author: ynankani <ynankani@nvidia.com>
Date: Mon Oct 5 04:56:50 2026 +0000
CUDA: use MMVF for thin f16/bf16 mul_mat at small batch size (#29633)
* CUDA: use MMVF for thin f16/bf16 mul_mat at small batch size
Signed-off-by: ynankani <ynankani@nvidia.com>
* adjust kernel selection logic
---------
Signed-off-by: ynankani <ynankani@nvidia.com>
Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 303c9f463..303d67e4c 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -1778,8 +1778,9 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor) {
(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16) &&
src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;
- const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
- use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, is_mul_mat_id ? src1->ne[2] : src1->ne[1]);
+ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
+ const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size;
+ use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, is_mul_mat_id ? src1->ne[2] : src1->ne[1]);
//we only support fusion for ncols_dst = 1
if (tensor->op == GGML_OP_MUL_MAT && dst->ne[1] != 1) {
@@ -1892,7 +1893,7 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
const int cc = ggml_cuda_info().devices[ctx.device].cc;
const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size;
- if (ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, ne11)) {
+ if (ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11)) {
// The custom F16 vector kernel can be used over batched cuBLAS GEMM.
// But this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention)
ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst);
@@ -1902,7 +1903,7 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
if (ne01 == 1 && ne11 > MMVF_MAX_BATCH_SIZE && ne2 == 1 && ne3 == 1
&& src0->type == GGML_TYPE_F32
&& ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst)
- && ggml_cuda_should_use_mmvf(src1->type, cc, src1->ne, src1->nb, /*ne11 =*/ 1)) {
+ && ggml_cuda_should_use_mmvf(src1->type, cc, warp_size, src1->ne, src1->nb, /*ne11 =*/ 1)) {
ggml_tensor dst_vec = *dst;
dst_vec.ne[0] = ne11;
dst_vec.ne[1] = 1;
diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu
index bd5c5d421..d0a572e15 100644
--- a/ggml/src/ggml-cuda/mmvf.cu
+++ b/ggml/src/ggml-cuda/mmvf.cu
@@ -2,6 +2,7 @@
#include "common.cuh"
#include "unary.cuh"
#include "mmvf.cuh"
+#include "mmf.cuh"
#include "convert.cuh"
template <typename T, typename type_acc, int ncols_dst, int block_size, bool has_fusion = false, bool is_multi_token_id = false>
@@ -789,7 +790,7 @@ void ggml_cuda_op_mul_mat_vec_f(
GGML_UNUSED_VARS(ctx, src1, dst, src1_ddq_i, src1_ncols, src1_padded_row_size);
}
-bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11) {
+bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11) {
if (src0_ne[0] % 2 != 0) {
return false;
}
@@ -820,12 +821,16 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0
if (fp32_mma_hardware_available(cc)) {
return ne11 <= 3;
}
- return ne11 <= 8;
+ return ne11 <= MMVF_MAX_BATCH_SIZE;
}
- return ne11 <= 8;
+ return ne11 <= MMVF_MAX_BATCH_SIZE;
case GGML_TYPE_F16:
if (GGML_CUDA_CC_IS_NVIDIA(cc)) {
const bool src0_small = (src0_ne[1] <= 512 || src0_ne[2]*src0_ne[3] == 1);
+ // MMF needs full row tiles, for other row counts MMVF still beats cuBLAS at small batch size
+ if (src0_small && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) {
+ return ne11 <= MMVF_MAX_BATCH_SIZE;
+ }
if (ampere_mma_available(cc)) {
return src0_small && ne11 == 1;
}
@@ -835,8 +840,11 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0
if (fp16_mma_hardware_available(cc)) {
return src0_small && ne11 <= 3;
}
- return ne11 <= 8;
+ return ne11 <= MMVF_MAX_BATCH_SIZE;
} else if (GGML_CUDA_CC_IS_AMD(cc)) {
+ if (GGML_CUDA_CC_IS_RDNA(cc) && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) {
+ return ne11 <= MMVF_MAX_BATCH_SIZE;
+ }
if (fp16_mma_hardware_available(cc)) {
if (GGML_CUDA_CC_IS_RDNA3(cc)) {
return ne11 <= 3;
@@ -846,12 +854,16 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0
}
return ne11 <= 2;
}
- return ne11 <= 8;
+ return ne11 <= MMVF_MAX_BATCH_SIZE;
}
- return ne11 <= 8;
+ return ne11 <= MMVF_MAX_BATCH_SIZE;
case GGML_TYPE_BF16:
if (GGML_CUDA_CC_IS_NVIDIA(cc)) {
const bool src0_small = (src0_ne[1] <= 512 || src0_ne[2]*src0_ne[3] == 1);
+ // MMF needs full row tiles, for other row counts MMVF still beats cuBLAS at small batch size
+ if (src0_small && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) {
+ return ne11 <= MMVF_MAX_BATCH_SIZE;
+ }
if (ampere_mma_available(cc)) {
return src0_small && ne11 == 1;
}
@@ -861,14 +873,17 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0
if (bf16_mma_hardware_available(cc)) {
return src0_small && ne11 <= 3;
}
- return ne11 <= 8;
+ return ne11 <= MMVF_MAX_BATCH_SIZE;
} else if (GGML_CUDA_CC_IS_AMD(cc)) {
+ if (GGML_CUDA_CC_IS_RDNA(cc) && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) {
+ return ne11 <= MMVF_MAX_BATCH_SIZE;
+ }
if (bf16_mma_hardware_available(cc)) {
return ne11 <= 3;
}
- return ne11 <= 8;
+ return ne11 <= MMVF_MAX_BATCH_SIZE;
}
- return ne11 <= 8;
+ return ne11 <= MMVF_MAX_BATCH_SIZE;
default:
return false;
}
diff --git a/ggml/src/ggml-cuda/mmvf.cuh b/ggml/src/ggml-cuda/mmvf.cuh
index a50f7c021..350b5cc06 100644
--- a/ggml/src/ggml-cuda/mmvf.cuh
+++ b/ggml/src/ggml-cuda/mmvf.cuh
@@ -11,4 +11,4 @@ void ggml_cuda_op_mul_mat_vec_f(
const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
const int64_t src1_padded_row_size, cudaStream_t stream);
-bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11);
+bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11);