Commit ef3848f9d for llama.cpp
commit ef3848f9d7d4cf7b6b2abfc3b4c960b55d6d9e91
Author: Jeff Bolz <jbolz@nvidia.com>
Date: Mon Nov 16 07:25:12 2026 -0600
vulkan: Support split_k for conv3d. (#29929)
This tries to use the 256x128 shape (coopmat2-only) with split_k when
it appears profitable. It reuses the mul_mat split_k reduce shader.
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
index 64a39bf25..e19e2d029 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
@@ -466,15 +466,15 @@ struct vk_conv2d_pipeline_state {
struct vk_conv3d_pipeline_state {
vk_conv3d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t s2, uint32_t p0, uint32_t p1, uint32_t p2,
- uint32_t d0, uint32_t d1, uint32_t d2, uint32_t KW, uint32_t KH, uint32_t KD, uint32_t aligned)
- : s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), KW(KW), KH(KH), KD(KD), aligned(aligned) {}
+ uint32_t d0, uint32_t d1, uint32_t d2, uint32_t KW, uint32_t KH, uint32_t KD, uint32_t aligned, uint32_t split_k)
+ : s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), KW(KW), KH(KH), KD(KD), aligned(aligned), split_k(split_k) {}
uint32_t s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD;
- uint32_t aligned;
+ uint32_t aligned, split_k;
bool operator<(const vk_conv3d_pipeline_state &b) const {
- return std::tie(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned) <
- std::tie(b.s0, b.s1, b.s2, b.p0, b.p1, b.p2, b.d0, b.d1, b.d2, b.KW, b.KH, b.KD, b.aligned);
+ return std::tie(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned, split_k) <
+ std::tie(b.s0, b.s1, b.s2, b.p0, b.p1, b.p2, b.d0, b.d1, b.d2, b.KW, b.KH, b.KD, b.aligned, b.split_k);
}
};
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index f07135351..b5b4264fb 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -4294,6 +4294,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
spec_constants_cpy.push_back(conv2d_csh_store); \
spec_constants_cpy.push_back(conv2d_WM); \
spec_constants_cpy.push_back(conv2d_WN); \
+ spec_constants_cpy.push_back(state.split_k); \
ggml_vk_create_pipeline( \
device, c.second, "conv3d" #type_suffix, \
conv3d##type_suffix##spv_suffix##_len, conv3d##type_suffix##spv_suffix##_data, "main", 3, \
@@ -9285,24 +9286,58 @@ static vk_conv_shapes ggml_vk_conv_select_shape(ggml_backend_vk_context * ctx, u
}
}
-static vk_conv_shapes ggml_vk_conv3d_select_shape(ggml_backend_vk_context * ctx, uint32_t K, uint32_t NPQ) {
- const auto & block = vk_conv_block_sizes[CONV_SHAPE_256x128];
+static vk_conv_shapes ggml_vk_conv3d_select_shape(ggml_backend_vk_context * ctx, const ggml_tensor * kernel, const ggml_tensor * dst, uint32_t & split_k) {
+ split_k = 1;
+ const uint32_t K = ggml_get_op_params_i32(dst, 11);
+ const uint32_t NPQ = ggml_get_op_params_i32(dst, 10) * dst->ne[0] * dst->ne[1] * dst->ne[2];
+ const vk_conv_shapes fallback = ggml_vk_conv_select_shape(ctx, K, NPQ);
const uint32_t shader_core_count = ctx->device->shader_core_count;
// Slowdowns were observed on DGX Spark with drivers older than R615.
const bool driver_supported = ctx->device->driver_id != vk::DriverId::eNvidiaProprietary ||
(ctx->device->properties.driverVersion >> 22) >= 615;
- if (ctx->device->coopmat2 && shader_core_count > 0 && K % block.K == 0 && driver_supported) {
- const uint64_t n_tiles = (uint64_t)(K / block.K) * CEIL_DIV((uint64_t)NPQ, block.NPQ);
- // Use the largest cm2 padding and include the dummy Csh[8].
- const uint32_t pad = 8, csh_elems = 8;
- const uint32_t shmem = (block.K * (block.CRS + pad) + block.CRS * (block.NPQ + pad) + csh_elems) * sizeof(uint16_t);
- // Reuse each loaded input across 256 output channels instead of 128.
- // Require full channel tiles and at least as many workgroups as GPU cores.
- if (n_tiles >= shader_core_count && shmem <= ctx->device->properties.limits.maxComputeSharedMemorySize) {
- return CONV_SHAPE_256x128;
- }
+ if (!ctx->device->coopmat2 || shader_core_count == 0 || K == 0 || NPQ == 0 || !driver_supported) {
+ return fallback;
}
- return ggml_vk_conv_select_shape(ctx, K, NPQ);
+ const auto & block = vk_conv_block_sizes[CONV_SHAPE_256x128];
+ const auto & limits = ctx->device->properties.limits;
+ // Use the largest cm2 padding and include the dummy Csh[8].
+ const uint32_t pad = 8, csh_elems = 8;
+ const uint32_t shmem = (block.K * (block.CRS + pad) + block.CRS * (block.NPQ + pad) + csh_elems) * sizeof(uint16_t);
+ if (shmem > limits.maxComputeSharedMemorySize) {
+ return fallback;
+ }
+ const uint32_t channel_tiles = CEIL_DIV(K, block.K);
+ const uint64_t n_tiles = (uint64_t)channel_tiles * CEIL_DIV(NPQ, block.NPQ);
+ // Keep full channel tiles when the grid already fills the GPU.
+ if (n_tiles >= shader_core_count) {
+ return K % block.K == 0 ? CONV_SHAPE_256x128 : fallback;
+ }
+ static const bool disable_split_k = getenv("GGML_VK_DISABLE_CONV3D_SPLIT_K") != nullptr;
+ // Do not use a tile with more than half its output channels padded.
+ if (disable_split_k || K < block.K / 2 || !ggml_is_contiguous(dst) || get_misalign_bytes(ctx, dst) != 0) {
+ return fallback;
+ }
+ // Use power-of-two splits to give each core at least one workgroup.
+ uint64_t splits = 1;
+ while (n_tiles * splits < shader_core_count) {
+ splits *= 2;
+ }
+ const uint64_t crs = (uint64_t)ggml_get_op_params_i32(dst, 9) * kernel->ne[0] * kernel->ne[1] * kernel->ne[2];
+ const uint64_t operand_bytes = block.K * ggml_type_size(kernel->type) + block.NPQ * sizeof(float);
+ const uint64_t scratch_bytes = 2 * sizeof(float) * block.K * block.NPQ;
+ // Do not add more scratch bytes than the estimated operand load bytes.
+ if (crs * operand_bytes < splits * scratch_bytes) {
+ return fallback;
+ }
+ const uint64_t ne = ggml_nelements(dst);
+ const uint64_t bytes = ggml_nbytes(dst);
+ if (ne == 0 || CEIL_DIV(ne, ctx->device->pipeline_matmul_split_k_reduce->wg_denoms[0]) > limits.maxComputeWorkGroupCount[0] ||
+ splits > limits.maxStorageBufferRange / bytes ||
+ splits > limits.maxComputeWorkGroupCount[0] / channel_tiles) {
+ return fallback;
+ }
+ split_k = (uint32_t)splits;
+ return CONV_SHAPE_256x128;
}
static vk_pipeline ggml_vk_rms_norm_pipeline(const std::array<vk_pipeline, 3> & pipelines, int64_t ncols) {
@@ -10011,7 +10046,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
const uint32_t IC = (uint32_t)ggml_get_op_params_i32(dst, 9);
const uint32_t N = (uint32_t)ggml_get_op_params_i32(dst, 10);
const uint32_t NPQ = N * dst->ne[2] * dst->ne[1] * dst->ne[0];
- const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, OC, NPQ);
+ uint32_t split_k;
+ const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, src0, dst, split_k);
const uint32_t KW = (uint32_t)src0->ne[0];
const uint32_t KH = (uint32_t)src0->ne[1];
@@ -10034,7 +10070,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
(CRS % BS_CRS == 0) &&
(NPQ % BS_NPQ == 0)) ? 1u : 0u;
- vk_conv3d_pipeline_state conv3d_pipeline_state(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned);
+ vk_conv3d_pipeline_state conv3d_pipeline_state(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned, split_k);
std::map<vk_conv3d_pipeline_state, vk_pipeline> *pipelines = nullptr;
if (src0->type == GGML_TYPE_F32) {
@@ -10140,6 +10176,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
init_pushconst_tensor_offsets(ctx, pc, src0, src1, src2, src3, dst);
std::array<uint32_t, 3> elements;
+ uint32_t split_k = 1;
switch (op) {
case GGML_OP_NORM:
@@ -10315,10 +10352,12 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
case GGML_OP_CONV_3D:
if constexpr (std::is_same_v<PC, vk_op_conv3d_push_constants>) {
const uint32_t NPQ = pc.N * pc.OD * pc.OH * pc.OW;
- const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, pc.OC, NPQ);
- const uint32_t NPQ_blocks = CEIL_DIV(NPQ, vk_conv_block_sizes[shape].NPQ);
+ const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, src0, dst, split_k);
+ const auto & block = vk_conv_block_sizes[shape];
+ const uint32_t NPQ_blocks = CEIL_DIV(NPQ, block.NPQ);
+ const uint32_t channel_groups = split_k > 1 ? CEIL_DIV(pc.OC, block.K) * block.K * split_k : pc.OC;
- elements = { pc.OC, NPQ_blocks, 1 };
+ elements = { channel_groups, NPQ_blocks, 1 };
if (elements[1] > 512) {
elements[2] = CEIL_DIV(elements[1], 512);
elements[1] = 512;
@@ -10449,6 +10488,29 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
break;
}
+ if constexpr (std::is_same_v<PC, vk_op_conv3d_push_constants>) {
+ if (split_k > 1) {
+ const uint32_t ne = (uint32_t)ggml_nelements(dst);
+ const size_t scratch_size = ggml_nbytes(dst) * split_k;
+ if (ctx->prealloc_size_split_k < scratch_size) {
+ ctx->prealloc_size_split_k = scratch_size;
+ ggml_vk_preallocate_buffers(ctx, subctx);
+ }
+ if (ctx->prealloc_split_k_need_sync) {
+ ggml_vk_sync_buffers(ctx, subctx);
+ }
+ vk_subbuffer scratch = { ctx->prealloc_split_k, 0, scratch_size };
+ pc.dst_offset = 0;
+ ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, 1);
+ ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, scratch }, pc, elements);
+ ggml_vk_sync_buffers(ctx, subctx);
+ const std::array<uint32_t, 2> reduce_pc = { ne, split_k };
+ ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_matmul_split_k_reduce, { scratch, dst_buf }, reduce_pc, { ne, 1, 1 });
+ ctx->prealloc_split_k_need_sync = true;
+ return;
+ }
+ }
+
if (op == GGML_OP_ADD || op == GGML_OP_RMS_NORM) {
vk_subbuffer a_buf = src0_buf;
if (ctx->do_add_rms_partials) {
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp
index 6919861b3..42ab48fce 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp
@@ -112,6 +112,8 @@ const uint warps_M = BS_K / WM;
const uint warps_N = BS_NPQ / WN;
#endif
+layout(constant_id = 22) const uint split_k = 1;
+
// without padding, ID_idx/IH_idx/IW_idx are in bounds by construction
const bool dhw_in_bounds = (p0 == 0) && (p1 == 0) && (p2 == 0);
@@ -170,8 +172,11 @@ D,R,S=KD,KH,KW
Z,P,Q=OD,OH,OW
*/
-uint32_t B_idx_K = gl_WorkGroupID.x;
+// Packing splits into the low bits would avoid runtime division, but adjacent channel tiles give better input cache reuse.
+uint32_t B_idx_K = split_k > 1 ? gl_WorkGroupID.x % splitWork(K, BS_K) : gl_WorkGroupID.x;
+uint32_t B_idx_split = split_k > 1 ? gl_WorkGroupID.x / splitWork(K, BS_K) : 0;
uint32_t B_idx_NPQ = gl_WorkGroupID.y + gl_WorkGroupID.z * 512;
+uint32_t split_offset = B_idx_split * K * NPQ;
uint32_t T_y = tid / NT_NPQ;
uint32_t T_x = tid % NT_NPQ;
@@ -219,7 +224,7 @@ ACC_TYPE perElemOpStore(const in uint32_t r, const in uint32_t c, const in ACC_T
split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx);
uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3;
if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
- dst_data[dst_idx + p.dst_offset] = D_TYPE(elem);
+ dst_data[dst_idx + p.dst_offset + split_offset] = D_TYPE(elem);
}
return elem;
}
@@ -249,7 +254,11 @@ void main() {
}
#endif
/* Advance block in CRS dim */
- [[dont_unroll]] for (uint32_t B_idx_CRS = 0; B_idx_CRS < NB_CRS; B_idx_CRS++) {
+ const uint32_t crs_count = NB_CRS / split_k;
+ const uint32_t crs_extra = NB_CRS % split_k;
+ const uint32_t crs_begin = B_idx_split * crs_count + min(B_idx_split, crs_extra);
+ const uint32_t crs_end = crs_begin + crs_count + uint32_t(B_idx_split < crs_extra);
+ [[dont_unroll]] for (uint32_t B_idx_CRS = crs_begin; B_idx_CRS < crs_end; B_idx_CRS++) {
uint32_t CRS_idx_a = B_idx_CRS * BS_CRS + Ac;
uint32_t IC_idx_a;
uint32_t KD_idx_a;
@@ -398,7 +407,7 @@ void main() {
split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx);
uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3;
if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
- dst_data[dst_idx + p.dst_offset] = D_TYPE(Csh[k_local * Csh_stride + npq_thread]);
+ dst_data[dst_idx + p.dst_offset + split_offset] = D_TYPE(Csh[k_local * Csh_stride + npq_thread]);
}
}
}
@@ -420,7 +429,7 @@ void main() {
split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx);
uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3;
if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
- dst_data[dst_idx + p.dst_offset] = D_TYPE(regC[T_ly][T_lx]);
+ dst_data[dst_idx + p.dst_offset + split_offset] = D_TYPE(regC[T_ly][T_lx]);
}
}
}
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 0a1b592d5..74aac916e 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -11179,6 +11179,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
}
+ test_cases.emplace_back(new test_conv_3d(2, 4097, 2, 32, 16, 128, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type));
// Case with kernel size 1
test_cases.emplace_back(new test_conv_3d(1, 4, 8, 8, 8, 8, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type));
test_cases.emplace_back(new test_conv_3d(2, 8, 5, 11, 9, 65, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, kernel_type));