Commit ef3848f9d for llama.cpp

commit ef3848f9d7d4cf7b6b2abfc3b4c960b55d6d9e91
Author: Jeff Bolz <jbolz@nvidia.com>
Date:   Mon Nov 16 07:25:12 2026 -0600

    vulkan: Support split_k for conv3d. (#29929)

    This tries to use the 256x128 shape (coopmat2-only) with split_k when
    it appears profitable. It reuses the mul_mat split_k reduce shader.

diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
index 64a39bf25..e19e2d029 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
@@ -466,15 +466,15 @@ struct vk_conv2d_pipeline_state {

 struct vk_conv3d_pipeline_state {
     vk_conv3d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t s2, uint32_t p0, uint32_t p1, uint32_t p2,
-                             uint32_t d0, uint32_t d1, uint32_t d2, uint32_t KW, uint32_t KH, uint32_t KD, uint32_t aligned)
-        : s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), KW(KW), KH(KH), KD(KD), aligned(aligned) {}
+                             uint32_t d0, uint32_t d1, uint32_t d2, uint32_t KW, uint32_t KH, uint32_t KD, uint32_t aligned, uint32_t split_k)
+        : s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), KW(KW), KH(KH), KD(KD), aligned(aligned), split_k(split_k) {}

     uint32_t s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD;
-    uint32_t aligned;
+    uint32_t aligned, split_k;

     bool operator<(const vk_conv3d_pipeline_state &b) const {
-        return std::tie(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned) <
-               std::tie(b.s0, b.s1, b.s2, b.p0, b.p1, b.p2, b.d0, b.d1, b.d2, b.KW, b.KH, b.KD, b.aligned);
+        return std::tie(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned, split_k) <
+               std::tie(b.s0, b.s1, b.s2, b.p0, b.p1, b.p2, b.d0, b.d1, b.d2, b.KW, b.KH, b.KD, b.aligned, b.split_k);
     }
 };

diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index f07135351..b5b4264fb 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -4294,6 +4294,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
             spec_constants_cpy.push_back(conv2d_csh_store); \
             spec_constants_cpy.push_back(conv2d_WM); \
             spec_constants_cpy.push_back(conv2d_WN); \
+            spec_constants_cpy.push_back(state.split_k); \
             ggml_vk_create_pipeline( \
                 device, c.second, "conv3d" #type_suffix, \
                 conv3d##type_suffix##spv_suffix##_len, conv3d##type_suffix##spv_suffix##_data, "main", 3, \
@@ -9285,24 +9286,58 @@ static vk_conv_shapes ggml_vk_conv_select_shape(ggml_backend_vk_context * ctx, u
     }
 }

-static vk_conv_shapes ggml_vk_conv3d_select_shape(ggml_backend_vk_context * ctx, uint32_t K, uint32_t NPQ) {
-    const auto & block = vk_conv_block_sizes[CONV_SHAPE_256x128];
+static vk_conv_shapes ggml_vk_conv3d_select_shape(ggml_backend_vk_context * ctx, const ggml_tensor * kernel, const ggml_tensor * dst, uint32_t & split_k) {
+    split_k = 1;
+    const uint32_t K = ggml_get_op_params_i32(dst, 11);
+    const uint32_t NPQ = ggml_get_op_params_i32(dst, 10) * dst->ne[0] * dst->ne[1] * dst->ne[2];
+    const vk_conv_shapes fallback = ggml_vk_conv_select_shape(ctx, K, NPQ);
     const uint32_t shader_core_count = ctx->device->shader_core_count;
     // Slowdowns were observed on DGX Spark with drivers older than R615.
     const bool driver_supported = ctx->device->driver_id != vk::DriverId::eNvidiaProprietary ||
                                   (ctx->device->properties.driverVersion >> 22) >= 615;
-    if (ctx->device->coopmat2 && shader_core_count > 0 && K % block.K == 0 && driver_supported) {
-        const uint64_t n_tiles = (uint64_t)(K / block.K) * CEIL_DIV((uint64_t)NPQ, block.NPQ);
-        // Use the largest cm2 padding and include the dummy Csh[8].
-        const uint32_t pad = 8, csh_elems = 8;
-        const uint32_t shmem = (block.K * (block.CRS + pad) + block.CRS * (block.NPQ + pad) + csh_elems) * sizeof(uint16_t);
-        // Reuse each loaded input across 256 output channels instead of 128.
-        // Require full channel tiles and at least as many workgroups as GPU cores.
-        if (n_tiles >= shader_core_count && shmem <= ctx->device->properties.limits.maxComputeSharedMemorySize) {
-            return CONV_SHAPE_256x128;
-        }
+    if (!ctx->device->coopmat2 || shader_core_count == 0 || K == 0 || NPQ == 0 || !driver_supported) {
+        return fallback;
     }
-    return ggml_vk_conv_select_shape(ctx, K, NPQ);
+    const auto & block = vk_conv_block_sizes[CONV_SHAPE_256x128];
+    const auto & limits = ctx->device->properties.limits;
+    // Use the largest cm2 padding and include the dummy Csh[8].
+    const uint32_t pad = 8, csh_elems = 8;
+    const uint32_t shmem = (block.K * (block.CRS + pad) + block.CRS * (block.NPQ + pad) + csh_elems) * sizeof(uint16_t);
+    if (shmem > limits.maxComputeSharedMemorySize) {
+        return fallback;
+    }
+    const uint32_t channel_tiles = CEIL_DIV(K, block.K);
+    const uint64_t n_tiles = (uint64_t)channel_tiles * CEIL_DIV(NPQ, block.NPQ);
+    // Keep full channel tiles when the grid already fills the GPU.
+    if (n_tiles >= shader_core_count) {
+        return K % block.K == 0 ? CONV_SHAPE_256x128 : fallback;
+    }
+    static const bool disable_split_k = getenv("GGML_VK_DISABLE_CONV3D_SPLIT_K") != nullptr;
+    // Do not use a tile with more than half its output channels padded.
+    if (disable_split_k || K < block.K / 2 || !ggml_is_contiguous(dst) || get_misalign_bytes(ctx, dst) != 0) {
+        return fallback;
+    }
+    // Use power-of-two splits to give each core at least one workgroup.
+    uint64_t splits = 1;
+    while (n_tiles * splits < shader_core_count) {
+        splits *= 2;
+    }
+    const uint64_t crs = (uint64_t)ggml_get_op_params_i32(dst, 9) * kernel->ne[0] * kernel->ne[1] * kernel->ne[2];
+    const uint64_t operand_bytes = block.K * ggml_type_size(kernel->type) + block.NPQ * sizeof(float);
+    const uint64_t scratch_bytes = 2 * sizeof(float) * block.K * block.NPQ;
+    // Do not add more scratch bytes than the estimated operand load bytes.
+    if (crs * operand_bytes < splits * scratch_bytes) {
+        return fallback;
+    }
+    const uint64_t ne = ggml_nelements(dst);
+    const uint64_t bytes = ggml_nbytes(dst);
+    if (ne == 0 || CEIL_DIV(ne, ctx->device->pipeline_matmul_split_k_reduce->wg_denoms[0]) > limits.maxComputeWorkGroupCount[0] ||
+        splits > limits.maxStorageBufferRange / bytes ||
+        splits > limits.maxComputeWorkGroupCount[0] / channel_tiles) {
+        return fallback;
+    }
+    split_k = (uint32_t)splits;
+    return CONV_SHAPE_256x128;
 }

 static vk_pipeline ggml_vk_rms_norm_pipeline(const std::array<vk_pipeline, 3> & pipelines, int64_t ncols) {
@@ -10011,7 +10046,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
             const uint32_t IC = (uint32_t)ggml_get_op_params_i32(dst, 9);
             const uint32_t N  = (uint32_t)ggml_get_op_params_i32(dst, 10);
             const uint32_t NPQ = N * dst->ne[2] * dst->ne[1] * dst->ne[0];
-            const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, OC, NPQ);
+            uint32_t split_k;
+            const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, src0, dst, split_k);

             const uint32_t KW = (uint32_t)src0->ne[0];
             const uint32_t KH = (uint32_t)src0->ne[1];
@@ -10034,7 +10070,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
                                       (CRS % BS_CRS == 0) &&
                                       (NPQ % BS_NPQ == 0)) ? 1u : 0u;

-            vk_conv3d_pipeline_state conv3d_pipeline_state(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned);
+            vk_conv3d_pipeline_state conv3d_pipeline_state(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned, split_k);

             std::map<vk_conv3d_pipeline_state, vk_pipeline> *pipelines = nullptr;
             if (src0->type == GGML_TYPE_F32) {
@@ -10140,6 +10176,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
     init_pushconst_tensor_offsets(ctx, pc, src0, src1, src2, src3, dst);

     std::array<uint32_t, 3> elements;
+    uint32_t split_k = 1;

     switch (op) {
     case GGML_OP_NORM:
@@ -10315,10 +10352,12 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
     case GGML_OP_CONV_3D:
         if constexpr (std::is_same_v<PC, vk_op_conv3d_push_constants>) {
             const uint32_t NPQ = pc.N * pc.OD * pc.OH * pc.OW;
-            const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, pc.OC, NPQ);
-            const uint32_t NPQ_blocks = CEIL_DIV(NPQ, vk_conv_block_sizes[shape].NPQ);
+            const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, src0, dst, split_k);
+            const auto & block = vk_conv_block_sizes[shape];
+            const uint32_t NPQ_blocks = CEIL_DIV(NPQ, block.NPQ);
+            const uint32_t channel_groups = split_k > 1 ? CEIL_DIV(pc.OC, block.K) * block.K * split_k : pc.OC;

-            elements = { pc.OC, NPQ_blocks, 1 };
+            elements = { channel_groups, NPQ_blocks, 1 };
             if (elements[1] > 512) {
                 elements[2] = CEIL_DIV(elements[1], 512);
                 elements[1] = 512;
@@ -10449,6 +10488,29 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
         break;
     }

+    if constexpr (std::is_same_v<PC, vk_op_conv3d_push_constants>) {
+        if (split_k > 1) {
+            const uint32_t ne = (uint32_t)ggml_nelements(dst);
+            const size_t scratch_size = ggml_nbytes(dst) * split_k;
+            if (ctx->prealloc_size_split_k < scratch_size) {
+                ctx->prealloc_size_split_k = scratch_size;
+                ggml_vk_preallocate_buffers(ctx, subctx);
+            }
+            if (ctx->prealloc_split_k_need_sync) {
+                ggml_vk_sync_buffers(ctx, subctx);
+            }
+            vk_subbuffer scratch = { ctx->prealloc_split_k, 0, scratch_size };
+            pc.dst_offset = 0;
+            ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, 1);
+            ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, scratch }, pc, elements);
+            ggml_vk_sync_buffers(ctx, subctx);
+            const std::array<uint32_t, 2> reduce_pc = { ne, split_k };
+            ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_matmul_split_k_reduce, { scratch, dst_buf }, reduce_pc, { ne, 1, 1 });
+            ctx->prealloc_split_k_need_sync = true;
+            return;
+        }
+    }
+
     if (op == GGML_OP_ADD || op == GGML_OP_RMS_NORM) {
         vk_subbuffer a_buf = src0_buf;
         if (ctx->do_add_rms_partials) {
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp
index 6919861b3..42ab48fce 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp
@@ -112,6 +112,8 @@ const uint warps_M     = BS_K / WM;
 const uint warps_N     = BS_NPQ / WN;
 #endif

+layout(constant_id = 22) const uint split_k = 1;
+
 // without padding, ID_idx/IH_idx/IW_idx are in bounds by construction
 const bool dhw_in_bounds = (p0 == 0) && (p1 == 0) && (p2 == 0);

@@ -170,8 +172,11 @@ D,R,S=KD,KH,KW
 Z,P,Q=OD,OH,OW
 */

-uint32_t B_idx_K   = gl_WorkGroupID.x;
+// Packing splits into the low bits would avoid runtime division, but adjacent channel tiles give better input cache reuse.
+uint32_t B_idx_K   = split_k > 1 ? gl_WorkGroupID.x % splitWork(K, BS_K) : gl_WorkGroupID.x;
+uint32_t B_idx_split = split_k > 1 ? gl_WorkGroupID.x / splitWork(K, BS_K) : 0;
 uint32_t B_idx_NPQ = gl_WorkGroupID.y + gl_WorkGroupID.z * 512;
+uint32_t split_offset = B_idx_split * K * NPQ;

 uint32_t T_y = tid / NT_NPQ;
 uint32_t T_x = tid % NT_NPQ;
@@ -219,7 +224,7 @@ ACC_TYPE perElemOpStore(const in uint32_t r, const in uint32_t c, const in ACC_T
     split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx);
     uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3;
     if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
-        dst_data[dst_idx + p.dst_offset] = D_TYPE(elem);
+        dst_data[dst_idx + p.dst_offset + split_offset] = D_TYPE(elem);
     }
     return elem;
 }
@@ -249,7 +254,11 @@ void main() {
     }
 #endif
     /* Advance block in CRS dim */
-    [[dont_unroll]] for (uint32_t B_idx_CRS = 0; B_idx_CRS < NB_CRS; B_idx_CRS++) {
+    const uint32_t crs_count = NB_CRS / split_k;
+    const uint32_t crs_extra = NB_CRS % split_k;
+    const uint32_t crs_begin = B_idx_split * crs_count + min(B_idx_split, crs_extra);
+    const uint32_t crs_end = crs_begin + crs_count + uint32_t(B_idx_split < crs_extra);
+    [[dont_unroll]] for (uint32_t B_idx_CRS = crs_begin; B_idx_CRS < crs_end; B_idx_CRS++) {
         uint32_t CRS_idx_a = B_idx_CRS * BS_CRS + Ac;
         uint32_t IC_idx_a;
         uint32_t KD_idx_a;
@@ -398,7 +407,7 @@ void main() {
             split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx);
             uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3;
             if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
-                dst_data[dst_idx + p.dst_offset] = D_TYPE(Csh[k_local * Csh_stride + npq_thread]);
+                dst_data[dst_idx + p.dst_offset + split_offset] = D_TYPE(Csh[k_local * Csh_stride + npq_thread]);
             }
         }
     }
@@ -420,7 +429,7 @@ void main() {
                 split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx);
                 uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3;
                 if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
-                    dst_data[dst_idx + p.dst_offset] = D_TYPE(regC[T_ly][T_lx]);
+                    dst_data[dst_idx + p.dst_offset + split_offset] = D_TYPE(regC[T_ly][T_lx]);
                 }
             }
         }
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 0a1b592d5..74aac916e 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -11179,6 +11179,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
                 }
             }
         }
+        test_cases.emplace_back(new test_conv_3d(2, 4097, 2, 32, 16, 128, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type));
         // Case with kernel size 1
         test_cases.emplace_back(new test_conv_3d(1, 4, 8, 8, 8, 8, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type));
         test_cases.emplace_back(new test_conv_3d(2, 8, 5, 11, 9, 65, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, kernel_type));