Commit 6788edb4f for llama.cpp
commit 6788edb4f325c1cb4210997eb79edcab2e27aeaa
Author: Jeff Bolz <jbolz@nvidia.com>
Date: Thu Sep 10 12:20:18 2026 -0500
vulkan: small M matrix optimizations for qwen (#28457)
* vulkan: optimize m=1 mul_mat by swapping A/B
* vulkan: Improve small M perf
Allow split_k with small M.
Make small vs med tile selection (for coopmat2) depend on M, not just N.
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index 870fa1155..34400a1b7 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -5424,8 +5424,9 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
bool prefer_large = tiles_m > shader_core_count || tiles_l > shader_core_count ||
(tiles_l <= shader_core_count / 3 && tiles_m > shader_core_count / 2);
if (n > crossover_large && prefer_large) return last;
- uint32_t crossover_medium = configs[0].unaligned->wg_denoms[1];
- if (n > crossover_medium) return 1;
+ uint32_t crossover_medium_m = configs[0].unaligned->wg_denoms[0];
+ uint32_t crossover_medium_n = configs[0].unaligned->wg_denoms[1];
+ if (m > crossover_medium_m && n > crossover_medium_n) return 1;
return 0;
};
device->matmul_id_tile_selector = [](uint32_t /*m*/, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/,
@@ -9027,7 +9028,7 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m,
}
uint32_t split_k = 1;
- if (ctx->device->shader_core_count != 0 && m >= pipeline->wg_denoms[0] && n >= pipeline->wg_denoms[1]) {
+ if (ctx->device->shader_core_count != 0 && n >= pipeline->wg_denoms[1]) {
// If k is 'large' and the SMs will fill less than halfway, use split_k.
uint32_t m_tiles = CEIL_DIV(m, pipeline->wg_denoms[0]);
uint32_t n_tiles = CEIL_DIV(n, pipeline->wg_denoms[1]);
@@ -9780,10 +9781,10 @@ static bool ggml_vk_should_use_mmvq(const vk_device& device, uint32_t m, uint32_
GGML_UNUSED(m);
}
-static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {
+static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, bool swap_inputs = false) {
ggml_tensor * dst = cgraph->nodes[node_idx];
- const ggml_tensor * src0 = dst->src[0];
- const ggml_tensor * src1 = dst->src[1];
+ const ggml_tensor * src0 = dst->src[swap_inputs ? 1 : 0];
+ const ggml_tensor * src1 = dst->src[swap_inputs ? 0 : 1];
VK_LOG_DEBUG("ggml_vk_mul_mat_vec_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3];
std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3];
@@ -9802,8 +9803,8 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context&
const uint64_t ne12 = src1->ne[2];
const uint64_t ne13 = src1->ne[3];
- const uint64_t ne20 = dst->ne[0];
- const uint64_t ne21 = dst->ne[1];
+ const uint64_t ne20 = dst->ne[swap_inputs ? 1 : 0];
+ const uint64_t ne21 = dst->ne[swap_inputs ? 0 : 1];
// const uint64_t ne22 = dst->ne[2];
// const uint64_t ne23 = dst->ne[3];
@@ -10417,6 +10418,16 @@ static void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, c
src0->ne[1] <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] &&
src1->ne[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]) {
ggml_vk_mul_mat_vec_nc_f16_f32(ctx, subctx, cgraph, node_idx);
+ // With one output row, B^T*A has the same flat output as A^T*B.
+ } else if (ctx->num_additional_fused_ops == 0 &&
+ (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) &&
+ (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_BF16 || ggml_is_quantized(src1->type)) &&
+ dst->ne[0] == 1 && dst->ne[1] > mul_mat_vec_max_cols &&
+ src0->ne[2] == 1 && src0->ne[3] == 1 &&
+ src1->ne[2] == 1 && src1->ne[3] == 1 &&
+ ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst) &&
+ get_misalign_bytes(ctx, src0) == 0 && get_misalign_bytes(ctx, src1) == 0 && get_misalign_bytes(ctx, dst) == 0) {
+ ggml_vk_mul_mat_vec_q_f16(ctx, subctx, cgraph, node_idx, true);
// mul_mat_vec supports batching ne12*ne13 when ne11==1, or treating ne11 as the batch size (up to four)
// when ne12 and ne13 are one.
} else if ((dst->ne[1] == 1 || (dst->ne[1] <= mul_mat_vec_max_cols && src1->ne[2] * src1->ne[3] == 1)) &&
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 0a6516e61..3428629fd 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -9683,9 +9683,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));
// m == 1, with n on both sides of MMVF_MAX_BATCH_SIZE (8): mmvf below, operand swap above
- for (int64_t n : {1, 7, 8, 9, 16, 128, 512}) {
+ for (int64_t n : {1, 7, 8, 9, 16, 127, 128, 511, 512}) {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 1, n, 2048, {1, 1}, {1, 1}));
}
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1, 512, 2048, {1, 1}, {1, 1}));
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1, 512, 2048, {1, 1}, {1, 1}));
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1, 509, 2051, {1, 1}, {1, 1}));
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1, 509, 2051, {1, 1}, {1, 1}));
+
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 31, 509, 2051, {1, 1}, {1, 1}));
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 32, 509, 2112, {1, 1}, {1, 1}));
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 32, 509, 2112, {1, 1}, {1, 1}));
#if 0
{