Commit 573aa89de for llama.cpp

commit 573aa89de6b6912b7433c4da20a013f743dd1009
Author: Jeff Bolz <jbolz@nvidia.com>
Date:   Thu Nov 19 00:14:14 2026 -0600

    vulkan: Implement stream-k MUL_MAT (#28528)

    * vulkan: Implement stream-k MUL_MAT

    Stream-k tries to split chunks of 256 K elements evenly across all SMs.
    Workgroups that finish a tile write to dst, others write to scratch, and
    there's a resolve shader similar to split-k.

    The shader code is implemented for scalar+cm1+cm2, but only currently enabled
    for cm2. The heuristic for enabling on coopmat2 is pretty simple due to the
    large tile size (one workgroup per SM), but would be more complex for the tile
    sizes being used for cm1, and it doesn't make sense for me to tune that on
    NVIDIA devices.

    Assisted-by: Codex

    * address review feedback

diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-common.h b/ggml/src/ggml-vulkan/ggml-vulkan-common.h
index 69028c042..ea3b3b4b8 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-common.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-common.h
@@ -101,7 +101,7 @@ void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<vk_stagi

 // matmul
 vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type);
-void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t padded_n, uint64_t b_col_bytes, uint64_t d_col_bytes);
+void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, uint32_t split_k, uint32_t stream_k_groups, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t padded_n, uint64_t b_col_bytes, uint64_t d_col_bytes);
 bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor);
 vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src, const ggml_tensor * dst, ggml_type to);
 vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx, ggml_type type);
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp
index 2c9c09c06..56fa6ada3 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp
@@ -225,7 +225,7 @@ void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size
             ctx, subctx, p, ggml_vk_subbuffer(ctx, d_X), ggml_vk_subbuffer(ctx, d_Y), ggml_vk_subbuffer(ctx, d_D), ggml_vk_subbuffer(ctx, ctx->prealloc_split_k),
             m, n, k,
             k, k, m, k*m, k*n, m*n,
-            split_k, batch, batch, batch, 1, 1, n,
+            split_k, 0, batch, batch, batch, 1, 1, n,
             k * sizeof(Y_TYPE), m * sizeof(float)
         );
     }
@@ -693,7 +693,7 @@ void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t
                 ctx, subctx, p, { qx_buf, 0, qx_sz }, { qy_buf, 0, qy_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k },
                 m, n, k,
                 k, k, m, k*m, k*n, m*n,
-                split_k, batch, batch, batch, 1, 1, n,
+                split_k, 0, batch, batch, batch, 1, 1, n,
                 (uint64_t)(k / ggml_blck_size(GGML_TYPE_Q8_1)) * ggml_type_size(GGML_TYPE_Q8_1), m * sizeof(float)
             );
         }
@@ -703,7 +703,7 @@ void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t
                 ctx, subctx, p, { qx_buf, 0, qx_sz }, { y_buf, 0, y_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k },
                 m, n, k,
                 k, k, m, k*m, k*n, m*n,
-                split_k, batch, batch, batch, 1, 1, n,
+                split_k, 0, batch, batch, batch, 1, 1, n,
                 k * sizeof(float), m * sizeof(float)
             );
         }
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h
index caa852e28..176dbb72a 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h
@@ -13,8 +13,38 @@ struct vk_mat_mat_push_constants {
     uint32_t k_split;
     uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3;
     uint32_t padded_N;
+    uint32_t k_units_per_tile;
+    uint32_t work_units_per_group;
+    uint32_t groups_with_extra_unit;
+    uint32_t k_units_mp;
+    uint32_t blocks_m_mp;
+    uint32_t blocks_n_mp;
+    uint32_t ne12_mp;
+    uint32_t broadcast2_mp;
+    uint32_t broadcast3_mp;
+    uint32_t div_Ls0;
+    uint32_t div_Ls1;
 };

+struct vk_mat_mat_stream_k_fixup_push_constants {
+    uint32_t M; uint32_t N;
+    uint32_t BM; uint32_t BN;
+    uint32_t k_units_per_tile;
+    uint32_t blocks_m;
+    uint32_t blocks_n;
+    uint32_t work_units_per_group;
+    uint32_t groups_with_extra_unit;
+    uint32_t k_units_mp;
+    uint32_t blocks_m_mp;
+    uint32_t blocks_n_mp;
+    uint32_t work_units_mp;
+    uint32_t large_group_work_mp;
+    uint32_t div_Ls0;
+    uint32_t div_Ls1;
+};
+
+static_assert(sizeof(vk_mat_mat_push_constants) <= 128);
+
 struct vk_mat_vec_push_constants {
     uint32_t ncols;
     uint32_t stride_a;
@@ -403,6 +433,34 @@ template <typename T> void init_pushconst_fastdiv(T &p) {
     static_assert(!std::is_const<T>::value, "unexpected type");
 }

+inline void init_pushconst_fastdiv(vk_mat_mat_push_constants &p) {
+    uint32_t ne12_L, broadcast2_L, broadcast3_L;
+    init_fastdiv_values(p.ne12,       p.ne12_mp,       ne12_L);
+    init_fastdiv_values(p.broadcast2, p.broadcast2_mp, broadcast2_L);
+    init_fastdiv_values(p.broadcast3, p.broadcast3_mp, broadcast3_L);
+    p.div_Ls1 = pack_fastdiv_L(ne12_L, broadcast2_L, broadcast3_L);
+}
+
+inline void init_pushconst_fastdiv(vk_mat_mat_push_constants &p, uint32_t blocks_m, uint32_t blocks_n) {
+    uint32_t k_units_L, blocks_m_L, blocks_n_L;
+    init_fastdiv_values(p.k_units_per_tile, p.k_units_mp, k_units_L);
+    init_fastdiv_values(blocks_m, p.blocks_m_mp, blocks_m_L);
+    init_fastdiv_values(blocks_n, p.blocks_n_mp, blocks_n_L);
+    p.div_Ls0 = pack_fastdiv_L(k_units_L, blocks_m_L, blocks_n_L);
+    init_pushconst_fastdiv(p);
+}
+
+template <> inline void init_pushconst_fastdiv(vk_mat_mat_stream_k_fixup_push_constants &p) {
+    uint32_t k_units_L, blocks_m_L, blocks_n_L, work_units_L, large_group_work_L;
+    init_fastdiv_values(p.k_units_per_tile, p.k_units_mp, k_units_L);
+    init_fastdiv_values(p.blocks_m, p.blocks_m_mp, blocks_m_L);
+    init_fastdiv_values(p.blocks_n, p.blocks_n_mp, blocks_n_L);
+    init_fastdiv_values(p.work_units_per_group, p.work_units_mp, work_units_L);
+    init_fastdiv_values(p.work_units_per_group + 1, p.large_group_work_mp, large_group_work_L);
+    p.div_Ls0 = pack_fastdiv_L(k_units_L, blocks_m_L, blocks_n_L);
+    p.div_Ls1 = pack_fastdiv_L(work_units_L, large_group_work_L, 0);
+}
+
 template <> inline void init_pushconst_fastdiv(vk_op_unary_push_constants &p) {
     // Compute magic values to divide by these six numbers.
     uint32_t ne0_012L;
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
index 54c7a3690..f976c5795 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
@@ -259,10 +259,11 @@ struct vk_matmul_pipeline_key {
     ggml_type type_b;
     bool mul_mat_id;
     bool f16acc;
+    bool stream_k = false;

     bool operator<(const vk_matmul_pipeline_key & o) const {
-        return std::tie(type_a, type_b, mul_mat_id, f16acc)
-             < std::tie(o.type_a, o.type_b, o.mul_mat_id, o.f16acc);
+        return std::tie(type_a, type_b, mul_mat_id, f16acc, stream_k)
+             < std::tie(o.type_a, o.type_b, o.mul_mat_id, o.f16acc, o.stream_k);
     }
 };

@@ -819,6 +820,7 @@ struct vk_device_struct {
     matmul_tile_selector_t matmul_id_tile_selector;

     vk_pipeline pipeline_matmul_split_k_reduce;
+    vk_pipeline pipeline_matmul_stream_k_fixup;
     vk_pipeline pipeline_quantize_q8_1_x4;

     vk_pipeline pipeline_dequant[GGML_TYPE_COUNT];
@@ -1023,6 +1025,8 @@ struct vk_device_struct {
     std::map<std::string, uint64_t> fusion_counts;
     bool disable_descriptor_reuse;
     std::atomic<uint64_t> buffer_destroy_count {};
+    bool disable_stream_k;
+    bool force_stream_k;
     bool disable_host_visible_vidmem;
     bool allow_sysmem_fallback;
     bool disable_graph_optimize;
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index 616a17c4c..e279693cb 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -2467,6 +2467,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
     FOR_EACH_LUT_FP4_TYPE(X)

     const int mul_mat_id_param_count = 5;
+    constexpr uint32_t stream_k_spec_id = 15;

     using spec_fn_t = std::function<std::vector<uint32_t>(const std::vector<uint32_t>&, bool)>;
     auto const &create_mm_pipelines = [&](
@@ -2478,6 +2479,11 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
         bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0,
         bool create_aligned = true, bool pin_subgroup_to_warp = false
     ) {
+        GGML_ASSERT(!key.stream_k || (!key.mul_mat_id && param_count == 3 && push_constant_size == sizeof(vk_mat_mat_push_constants) && create_aligned));
+        // Non-ID matmul shaders declare the Stream-K scratch binding in all specializations.
+        if (!key.mul_mat_id) {
+            ++param_count;
+        }
         auto& vec = device->pipeline_matmul[key];
         const bool first_call = vec.empty();
         for (size_t i = 0; i < tile_configs.size(); i++) {
@@ -2495,25 +2501,40 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
                 vk_matmul_pipeline_pair pair{};
                 pair.align = tc.align;
                 pair.narrow_n_max = tc.narrow_n_max;
-                std::string suffix = "_" + std::to_string(i);
-                pair.unaligned = std::make_shared<vk_pipeline_struct>();
+                if (!key.stream_k) {
+                    pair.unaligned = std::make_shared<vk_pipeline_struct>();
+                }
                 if (create_aligned) {
                     pair.aligned = std::make_shared<vk_pipeline_struct>();
                 }
+                if (key.stream_k) {
+                    // Tile selection reads unaligned even when selecting an aligned pipeline.
+                    pair.unaligned = pair.aligned;
+                }
                 vec.push_back(pair);
             }

-            ggml_vk_create_pipeline(device, vec[i].unaligned,
-                vec[i].unaligned->name.empty() ? (shader_name + "_" + std::to_string(i)).c_str() : vec[i].unaligned->name.c_str(),
-                spv_len, spv_data, "main", param_count, push_constant_size,
-                tc.wg_denoms, spec_fn(tc.warptile, false), 1,
-                disable_robustness, rfs, rsgs);
+            if (!key.stream_k) {
+                ggml_vk_create_pipeline(device, vec[i].unaligned,
+                    vec[i].unaligned->name.empty() ? (shader_name + "_" + std::to_string(i)).c_str() : vec[i].unaligned->name.c_str(),
+                    spv_len, spv_data, "main", param_count, push_constant_size,
+                    tc.wg_denoms, spec_fn(tc.warptile, false), 1,
+                    disable_robustness, rfs, rsgs);
+            }

             if (vec[i].aligned) {
+                auto spec = spec_fn(tc.warptile, true);
+                if (key.stream_k) {
+                    if (spec.size() <= stream_k_spec_id) {
+                        spec.resize(stream_k_spec_id + 1);
+                    }
+                    spec[stream_k_spec_id] = 1;
+                }
+                const std::string name = shader_name + (key.stream_k ? "_stream_k_" : "_aligned_") + std::to_string(i);
                 ggml_vk_create_pipeline(device, vec[i].aligned,
-                    vec[i].aligned->name.empty() ? (shader_name + "_aligned_" + std::to_string(i)).c_str() : vec[i].aligned->name.c_str(),
+                    vec[i].aligned->name.empty() ? name.c_str() : vec[i].aligned->name.c_str(),
                     spv_len, spv_data, "main", param_count, push_constant_size,
-                    tc.wg_denoms, spec_fn(tc.warptile, true), tc.align,
+                    tc.wg_denoms, spec, tc.align,
                     disable_robustness, rfs, rsgs);
             }
         }
@@ -2611,13 +2632,19 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
         }

         spec_fn_t cm2_spec = [&](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a); };
+        auto create_cm2_pipelines = [&](vk_matmul_pipeline_key key, const std::vector<vk_tile_config>& tc,
+                                        const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc, const spec_fn_t& spec) {
+            create_mm_pipelines(key, tc, name, len, data, pc_size, pc, spec, true);
+            key.stream_k = true;
+            create_mm_pipelines(key, tc, name, len, data, pc_size, pc, spec, true);
+        };

         // F16 x F16
-        create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, true},  tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm2_len, matmul_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
-        create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16",        matmul_f16_cm2_len,        matmul_f16_cm2_data,        sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
+        create_cm2_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, true},  tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm2_len, matmul_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
+        create_cm2_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16",        matmul_f16_cm2_len,        matmul_f16_cm2_data,        sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
 #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)
         if (device->coopmat_bf16_support) {
-            create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm2_len, matmul_bf16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
+            create_cm2_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm2_len, matmul_bf16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
         }
 #endif
         for (const auto type : non_lut_quant_types) {
@@ -2627,18 +2654,18 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
             }
             auto& tc = ((type >= GGML_TYPE_Q2_K && type <= GGML_TYPE_Q6_K) || type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0) ? tc_mmq_k : tc_mmq;
             spec_fn_t qs = [&, type](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); };
-            create_mm_pipelines({type, GGML_TYPE_F16, false, true},  tc, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm2_len, matmul_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true);
-            create_mm_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16",        matmul_quant_f16_cm2_len,        matmul_quant_f16_cm2_data,        sizeof(vk_mat_mat_push_constants), 3, qs, true);
+            create_cm2_pipelines({type, GGML_TYPE_F16, false, true},  tc, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm2_len, matmul_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs);
+            create_cm2_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16",        matmul_quant_f16_cm2_len,        matmul_quant_f16_cm2_data,        sizeof(vk_mat_mat_push_constants), 3, qs);
         }
-        create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, true},  tc_mmq_k, "matmul_q4_k_f16_f16acc", matmul_q4_k_f16_f16acc_cm2_len, matmul_q4_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
-        create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q4_k_f16",        matmul_q4_k_f16_cm2_len,        matmul_q4_k_f16_cm2_data,        sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
-        create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, true},  tc_mmq_k, "matmul_q5_k_f16_f16acc", matmul_q5_k_f16_f16acc_cm2_len, matmul_q5_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
-        create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q5_k_f16",        matmul_q5_k_f16_cm2_len,        matmul_q5_k_f16_cm2_data,        sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
+        create_cm2_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, true},  tc_mmq_k, "matmul_q4_k_f16_f16acc", matmul_q4_k_f16_f16acc_cm2_len, matmul_q4_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
+        create_cm2_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q4_k_f16",        matmul_q4_k_f16_cm2_len,        matmul_q4_k_f16_cm2_data,        sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
+        create_cm2_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, true},  tc_mmq_k, "matmul_q5_k_f16_f16acc", matmul_q5_k_f16_f16acc_cm2_len, matmul_q5_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
+        create_cm2_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q5_k_f16",        matmul_q5_k_f16_cm2_len,        matmul_q5_k_f16_cm2_data,        sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
 #define X_CM2(TYPE, tstr) \
         { auto tc = filter_tc(tc_mmq, TYPE, false); \
           if (!tc.empty()) { \
-              create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true},  tc, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm2_len, matmul_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \
-              create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16",        matmul_##tstr##_f16_cm2_len,        matmul_##tstr##_f16_cm2_data,        sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \
+              create_cm2_pipelines({TYPE, GGML_TYPE_F16, false, true},  tc, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm2_len, matmul_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec); \
+              create_cm2_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16",        matmul_##tstr##_f16_cm2_len,        matmul_##tstr##_f16_cm2_data,        sizeof(vk_mat_mat_push_constants), 3, cm2_spec); \
           } }
         FOR_EACH_LUT_TYPE_NONFP4(X_CM2)
 #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT)
@@ -2646,8 +2673,8 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
 #define X_CM2_OCP(TYPE, tstr) \
             { auto tc = filter_tc(tc_mmq, TYPE, false); \
               if (!tc.empty()) { \
-                  create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true},  tc, "matmul_" #tstr "_f16_ocp_f16acc", matmul_##tstr##_f16_ocp_f16acc_cm2_len, matmul_##tstr##_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \
-                  create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16_ocp",        matmul_##tstr##_f16_ocp_cm2_len,        matmul_##tstr##_f16_ocp_cm2_data,        sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \
+                  create_cm2_pipelines({TYPE, GGML_TYPE_F16, false, true},  tc, "matmul_" #tstr "_f16_ocp_f16acc", matmul_##tstr##_f16_ocp_f16acc_cm2_len, matmul_##tstr##_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec); \
+                  create_cm2_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16_ocp",        matmul_##tstr##_f16_ocp_cm2_len,        matmul_##tstr##_f16_ocp_cm2_data,        sizeof(vk_mat_mat_push_constants), 3, cm2_spec); \
               } }
             FOR_EACH_LUT_FP4_TYPE(X_CM2_OCP)
 #undef X_CM2_OCP
@@ -3515,6 +3542,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
     ggml_vk_create_pipeline(device, device->pipeline_get_rows_back_f32, "get_rows_back_f32", get_rows_back_f32_len, get_rows_back_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {256, 1, 1}, {}, 1, true);

     ggml_vk_create_pipeline(device, device->pipeline_matmul_split_k_reduce, "split_k_reduce", split_k_reduce_len, split_k_reduce_data, "main", 2, 2 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1);
+    ggml_vk_create_pipeline(device, device->pipeline_matmul_stream_k_fixup, "stream_k_fixup", stream_k_fixup_len, stream_k_fixup_data, "main", 2, sizeof(vk_mat_mat_stream_k_fixup_push_constants), {1, 32, 1}, {}, 1);
     ggml_vk_create_pipeline(device, device->pipeline_flash_attn_split_k_reduce, "fa_split_k_reduce", fa_split_k_reduce_len, fa_split_k_reduce_data, "main", 3, sizeof(vk_op_flash_attn_split_k_reduce_push_constants), {1, device->subgroup_size, 1}, {device->subgroup_size}, 1, true);

 #if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT)
@@ -5375,6 +5403,8 @@ vk_device ggml_vk_get_device(size_t idx) {
         device->serialize_submissions = getenv("GGML_VK_SERIALIZE_SUBMISSIONS") != nullptr;

         device->disable_fusion = getenv("GGML_VK_DISABLE_FUSION") != nullptr;
+        device->disable_stream_k = getenv("GGML_VK_DISABLE_STREAM_K") != nullptr;
+        device->force_stream_k = getenv("GGML_VK_FORCE_STREAM_K") != nullptr;

         device->disable_descriptor_reuse = getenv("GGML_VK_DISABLE_DESCRIPTOR_REUSE") != nullptr;

@@ -6408,14 +6438,113 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m,
     return split_k;
 }

+static uint32_t ggml_vk_guess_stream_k_groups(
+        ggml_backend_vk_context * ctx, const vk_pipeline & pipeline,
+        uint32_t m, uint32_t n, uint32_t k, uint32_t stride_d, uint32_t batch, uint32_t split_k, bool disable_split_k) {
+    if (disable_split_k) {
+        return 0;
+    }
+    if (ctx->device->disable_stream_k) {
+        return 0;
+    }
+    // TODO: Tune the CM1 and scalar workgroup counts before enabling Stream-K.
+    if (!ctx->device->coopmat2) {
+        return 0;
+    }
+    if (ctx->device->shader_core_count == 0) {
+        return 0;
+    }
+
+    const uint32_t bm = pipeline->wg_denoms[0];
+    const uint32_t bn = pipeline->wg_denoms[1];
+    if (bm % 32 != 0 || m % bm != 0 || n % bn != 0 || k % 256 != 0 || stride_d != m) {
+        return 0;
+    }
+
+    const uint32_t k_units = k / 256;
+    if (k_units == 1) {
+        return 0;
+    }
+
+    const uint64_t tiles = uint64_t(m / bm) * (n / bn) * batch;
+    const uint64_t total_work_units = tiles * k_units;
+    if (total_work_units > UINT32_MAX) {
+        return 0;
+    }
+
+    // Coopmat2 tiles are tuned for one resident workgroup per SM.
+    const uint32_t target_groups = ctx->device->shader_core_count;
+    const uint64_t regular_groups = tiles * split_k;
+    const uint64_t regular_waves = CEIL_DIV(regular_groups, target_groups);
+    const uint64_t regular_capacity = regular_waves * target_groups;
+
+    const uint32_t stream_k_groups = std::min<uint64_t>(target_groups, total_work_units);
+    if (stream_k_groups > ctx->device->properties.limits.maxComputeWorkGroupCount[0] ||
+        bm / 32 > ctx->device->properties.limits.maxComputeWorkGroupCount[1]) {
+        return 0;
+    }
+    if (!ctx->device->force_stream_k) {
+        if (regular_waves == 1 && stream_k_groups == regular_groups) {
+            return 0;
+        }
+        // Stream-K does not offset its overhead when the regular grid fills most of the last wave.
+        // Use a lower cutoff for short K with no K split and narrow N tiles.
+        if ((split_k == 1 && k <= 4096 && bn <= 128 && regular_groups * 4 >= regular_capacity * 3) ||
+            regular_groups * 10 >= regular_capacity * 9) {
+            return 0;
+        }
+    }
+
+    return stream_k_groups;
+}
+
 void ggml_vk_matmul(
         ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline,
         vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer,
         uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d,
         uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d,
-        uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3,
+        uint32_t split_k, uint32_t stream_k_groups, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3,
         uint32_t padded_n, uint64_t b_col_bytes, uint64_t d_col_bytes) {
         VK_LOG_DEBUG("ggml_vk_matmul(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), split_k: (" << (split_k_buffer.buffer != nullptr ? split_k_buffer.buffer->buffer : VK_NULL_HANDLE) << ", " << split_k_buffer.offset << ", " << split_k_buffer.size << "), m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", split_k: " << split_k << ", batch: " << batch << ", ne02: " << ne02 << ", ne12: " << ne12 << ", broadcast2: " << broadcast2 << ", broadcast3: " << broadcast3 << ", padded_n: " << padded_n << ")");
+    if (stream_k_groups != 0) {
+        GGML_ASSERT(batch_stride_d == m * n);
+        if (ctx->prealloc_split_k_need_sync) {
+            ggml_vk_sync_buffers(ctx, subctx);
+        }
+
+        ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
+        ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_stream_k_fixup, 1);
+
+        const uint32_t bm = pipeline->wg_denoms[0];
+        const uint32_t bn = pipeline->wg_denoms[1];
+        const uint32_t k_units_per_tile = k / 256;
+        const uint32_t blocks_m = m / bm;
+        const uint32_t blocks_n = n / bn;
+        const uint32_t total_work_units = blocks_m * blocks_n * batch * k_units_per_tile;
+        const uint32_t work_units_per_group = total_work_units / stream_k_groups;
+        const uint32_t groups_with_extra_unit = total_work_units % stream_k_groups;
+
+        vk_mat_mat_push_constants pc1 = {
+            m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, 0, batch, k, ne02, ne12, broadcast2, broadcast3, padded_n,
+            k_units_per_tile, work_units_per_group, groups_with_extra_unit,
+            0, 0, 0, 0, 0, 0, 0, 0
+        };
+        init_pushconst_fastdiv(pc1, blocks_m, blocks_n);
+
+        ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, split_k_buffer }, pc1, { stream_k_groups * pipeline->wg_denoms[0], 1, 1 });
+        ggml_vk_sync_buffers(ctx, subctx);
+
+        vk_mat_mat_stream_k_fixup_push_constants pc2 = {
+            m, n, bm, bn, k_units_per_tile, blocks_m, blocks_n, work_units_per_group, groups_with_extra_unit,
+            0, 0, 0, 0, 0, 0, 0
+        };
+        init_pushconst_fastdiv(pc2);
+
+        ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_matmul_stream_k_fixup, { split_k_buffer, d }, pc2, { stream_k_groups, bm, 1 });
+        ctx->prealloc_split_k_need_sync = true;
+        return;
+    }
+
     if (split_k == 1) {
         // a small tile can still produce more workgroups than the device allows on the n axis:
         // split the dispatch into n-chunks, shifting the B and D column windows
@@ -6438,8 +6567,12 @@ void ggml_vk_matmul(
             while (base_work_group_z < batch) {
                 uint32_t groups_z = std::min(batch - base_work_group_z, ctx->device->properties.limits.maxComputeWorkGroupCount[2]);

-                const vk_mat_mat_push_constants pc = { m, n_chunk, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k, ne02, ne12, broadcast2, broadcast3, uint32_t(padded_n - n_base) };
-                ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b_chunk, d_chunk }, pc, { m, n_chunk, groups_z });
+                vk_mat_mat_push_constants pc = {
+                    m, n_chunk, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k, ne02, ne12, broadcast2, broadcast3, uint32_t(padded_n - n_base),
+                    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+                };
+                init_pushconst_fastdiv(pc);
+                ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b_chunk, d_chunk, d_chunk }, pc, { m, n_chunk, groups_z });
                 base_work_group_z += groups_z;
             }
         }
@@ -6462,9 +6595,13 @@ void ggml_vk_matmul(
     while (base_work_group_z < batch) {
         uint32_t groups_z = std::min(batch - base_work_group_z, ctx->device->properties.limits.maxComputeWorkGroupCount[2]);

-        const vk_mat_mat_push_constants pc1 = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k_split, ne02, ne12, broadcast2, broadcast3, padded_n };
+        vk_mat_mat_push_constants pc1 = {
+            m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k_split, ne02, ne12, broadcast2, broadcast3, padded_n,
+            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+        };
+        init_pushconst_fastdiv(pc1);
         // Make sure enough workgroups get assigned for split k to work
-        ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, split_k_buffer }, pc1, { (CEIL_DIV(m, pipeline->wg_denoms[0]) * pipeline->wg_denoms[0]) * split_k, n, groups_z });
+        ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, split_k_buffer, split_k_buffer }, pc1, { (CEIL_DIV(m, pipeline->wg_denoms[0]) * pipeline->wg_denoms[0]) * split_k, n, groups_z });
         base_work_group_z += groups_z;
     }
     ggml_vk_sync_buffers(ctx, subctx);
@@ -6518,10 +6655,11 @@ static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_t
     return ctx->device->fp16 && prec_allows_f16;
 }

-static const std::vector<vk_matmul_pipeline_pair>* ggml_vk_get_mul_mat_mat_pipeline_map(
-        ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec, bool mul_mat_id = false) {
+static const std::vector<vk_matmul_pipeline_pair> * ggml_vk_get_mul_mat_mat_pipeline_map(
+        ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec,
+        bool mul_mat_id = false, bool stream_k = false) {
     bool f16acc = ggml_vk_get_mul_mat_mat_f16acc(ctx, src0_type, src1_type, prec);
-    vk_matmul_pipeline_key key{src0_type, src1_type, mul_mat_id, f16acc};
+    vk_matmul_pipeline_key key{src0_type, src1_type, mul_mat_id, f16acc, stream_k};
     auto it = ctx->device->pipeline_matmul.find(key);
     if (it == ctx->device->pipeline_matmul.end() || it->second.empty()) {
         // Try without f16acc
@@ -6894,25 +7032,50 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
     const bool y_f32_kernel     = y_kernel_type == GGML_TYPE_F32;
     const bool y_needs_reformat = !quantize_y && (!ggml_vk_dim01_contiguous(src1) || y_kernel_type != src1->type);

-    const std::vector<vk_matmul_pipeline_pair>* mmp_map =
-        ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, ggml_prec_get_acc(dst));
+    ggml_type mmp_type_a = src0->type;
+    const ggml_type mmp_type_b = y_kernel_type;
+    const std::vector<vk_matmul_pipeline_pair> * mmp_map =
+        ggml_vk_get_mul_mat_mat_pipeline_map(ctx, mmp_type_a, mmp_type_b, ggml_prec_get_acc(dst));

     const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig;
     const bool qy_needs_dequant = y_needs_reformat;

     if (qx_needs_dequant) {
         // dequant src0 to f16
-        mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, ggml_prec_get_acc(dst));
+        mmp_type_a = f16_type;
+        mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, mmp_type_a, mmp_type_b, ggml_prec_get_acc(dst));
     }

     GGML_ASSERT(mmp_map != nullptr);

+    const std::vector<vk_matmul_pipeline_pair> * regular_mmp_map = mmp_map;
+    bool use_stream_k = ctx->device->coopmat2 && !ctx->device->disable_stream_k && !disable_split_k;
+    if (use_stream_k) {
+        mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, mmp_type_a, mmp_type_b, ggml_prec_get_acc(dst), false, true);
+        if (mmp_map == nullptr) {
+            mmp_map = regular_mmp_map;
+            use_stream_k = false;
+        }
+    }
+
     const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, ne11, false));
     const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && ne11 > 8;
+    const bool needs_64b_indexing = ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange;
+    if (use_stream_k && !aligned) {
+        mmp_map = regular_mmp_map;
+        use_stream_k = false;
+    }

     vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, ne11, aligned, false);

-    if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) {
+    const bool narrow_n = mmp_map->back().narrow_n_max != 0 && ne11 <= mmp_map->back().narrow_n_max;
+    const uint32_t split_k = ggml_vk_guess_split_k(ctx, ne01, ne11, ne10, disable_split_k, pipeline, narrow_n);
+    const uint32_t stream_k_groups = use_stream_k ? ggml_vk_guess_stream_k_groups(ctx, pipeline, ne01, ne11, ne10, stride_d, ne12 * ne13, split_k, disable_split_k) : 0;
+    if (use_stream_k && stream_k_groups == 0) {
+        pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *regular_mmp_map, ne01, ne11, aligned, false);
+    }
+
+    if (needs_64b_indexing) {
         pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline);
     }

@@ -6923,9 +7086,6 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
     const uint64_t y_ne = padded_n * ne10 * ne12 * ne13;
     const uint64_t d_ne = ggml_nelements(dst);

-    const bool narrow_n = mmp_map->back().narrow_n_max != 0 && ne11 <= mmp_map->back().narrow_n_max;
-    const uint32_t split_k = ggml_vk_guess_split_k(ctx, ne01, ne11, ne10, disable_split_k, pipeline, narrow_n);
-
     const uint64_t qx_sz = ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type);
     const uint64_t qy_sz = ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type);
     const uint64_t x_sz = !qx_needs_dequant ? qx_sz : sizeof(ggml_fp16_t) * x_ne;
@@ -6953,12 +7113,13 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
         to_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1);
     }

+    const uint64_t stream_k_size = stream_k_groups != 0 ? uint64_t(stream_k_groups) * pipeline->wg_denoms[0] * pipeline->wg_denoms[1] * sizeof(float) : 0;
+    const uint64_t split_k_size = stream_k_groups != 0 ? stream_k_size : (split_k > 1 ? d_sz * split_k : 0);
     {
-        const uint64_t split_k_size = split_k > 1 ? d_sz * split_k : 0;
         if (
                 (qx_needs_dequant && x_sz > ctx->device->properties.limits.maxStorageBufferRange) ||
                 (qy_needs_dequant && y_sz > ctx->device->properties.limits.maxStorageBufferRange) ||
-                (split_k > 1 && split_k_size > ctx->device->properties.limits.maxStorageBufferRange)) {
+                (split_k_size > ctx->device->properties.limits.maxStorageBufferRange)) {
             GGML_ABORT("Requested preallocation size is too large");
         }
         if (qx_needs_dequant && ctx->prealloc_size_x < x_sz) {
@@ -6969,7 +7130,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
             ctx->prealloc_size_y = y_sz;
             ggml_vk_preallocate_buffers(ctx, subctx);
         }
-        if (split_k > 1 && ctx->prealloc_size_split_k < split_k_size) {
+        if (split_k_size != 0 && ctx->prealloc_size_split_k < split_k_size) {
             ctx->prealloc_size_split_k = split_k_size;
             ggml_vk_preallocate_buffers(ctx, subctx);
         }
@@ -6984,7 +7145,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
         if (quantize_y) {
             ggml_pipeline_request_descriptor_sets(ctx, to_q8_1, 1);
         }
-        if (split_k > 1) {
+        if (stream_k_groups == 0 && split_k > 1) {
             ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, 1);
         }
     }
@@ -7092,10 +7253,10 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
     ggml_vk_matmul(
         ctx, subctx, pipeline,
         { d_X, x_buf_offset, x_range }, { d_Y, y_buf_offset, y_range },
-        ggml_vk_subbuffer(ctx, d_D, d_buf_offset), { ctx->prealloc_split_k, 0, d_sz * split_k },
+        ggml_vk_subbuffer(ctx, d_D, d_buf_offset), { ctx->prealloc_split_k, 0, split_k_size },
         ne01, ne11, ne10,
         ne10, ne10, stride_d, stride_batch_x, stride_batch_y, stride_batch_d,
-        split_k, ne12*ne13, ne02, ne12, r2, r3, padded_n,
+        split_k, stream_k_groups, ne12*ne13, ne02, ne12, r2, r3, padded_n,
         b_col_bytes, (uint64_t)dst->nb[1]
     );  // NOLINT

diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_stream_k_fixup.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_stream_k_fixup.comp
new file mode 100644
index 000000000..c8a196156
--- /dev/null
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_stream_k_fixup.comp
@@ -0,0 +1,86 @@
+#version 450
+
+#include "utils.glsl"
+
+layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in;
+
+layout (binding = 0) readonly buffer S4 {vec4 data_s4[];};
+layout (binding = 1) buffer D4 {vec4 data_d4[];};
+
+layout (push_constant) uniform parameter {
+    uint M;
+    uint N;
+    uint BM;
+    uint BN;
+    uint k_units_per_tile;
+    uint blocks_m;
+    uint blocks_n;
+    uint work_units_per_group;
+    uint groups_with_extra_unit;
+    uint k_units_mp;
+    uint blocks_m_mp;
+    uint blocks_n_mp;
+    uint work_units_mp;
+    uint large_group_work_mp;
+    uint div_Ls0;
+    uint div_Ls1;
+} p;
+
+const uint RESOLVE_ROWS = 32;
+
+// A normal matmul workgroup accumulates one BM x BN output tile over the full K range.
+// Stream-K divides that accumulation into work units. Each work unit covers 256 consecutive K values and produces a BM x BN partial sum.
+// Work units are ordered [K unit, M tile, N tile, batch] and divided into one contiguous range per main workgroup.
+// A main workgroup writes its partial sum to D when it reaches the end of an output tile, or to scratch when its range stops within a tile.
+// Scratch has shape [BM, BN, stream_k_groups] and stores float partial sums.
+// Element [r, c, g] is workgroup g's contribution to local output element [r, c] in the last output tile of its range.
+// The output tile for each scratch slot is derived from the workgroup's range and is not stored in scratch.
+// Only the last output tile in a workgroup's range can stop early, so one scratch slot is enough.
+// Fixup workgroup (x, y) handles main workgroup x and rows 32*y through 32*y+31.
+// If main workgroup x starts within a tile and completes it, fixup adds the earlier groups' partial sums from scratch to D.
+
+// Find the main workgroup whose range contains work.
+// With q = work_units_per_group and r = groups_with_extra_unit, the first r workgroups get q + 1 units and all others get q.
+// group(work) = work / (q + 1) when work < r*(q + 1), otherwise r + (work - r*(q + 1))/q.
+uint group_for_work(uint work) {
+    const uint large_group_work = p.work_units_per_group + 1;
+    const uint large_work = large_group_work * p.groups_with_extra_unit;
+    if (work < large_work) {
+        return fastdiv(work, p.large_group_work_mp, fastdiv_L(p.div_Ls1, 1));
+    }
+    return p.groups_with_extra_unit + fastdiv(work - large_work, p.work_units_mp, fastdiv_L(p.div_Ls1, 0));
+}
+
+void main() {
+    const uint group_idx = gl_WorkGroupID.x;
+    const uint work_unit = group_idx * p.work_units_per_group + min(group_idx, p.groups_with_extra_unit);
+    const uint work_unit_end = work_unit + p.work_units_per_group + uint(group_idx < p.groups_with_extra_unit);
+
+    const uint tile_idx = fastdiv(work_unit, p.k_units_mp, fastdiv_L(p.div_Ls0, 0));
+    const uint tile_work = tile_idx * p.k_units_per_tile;
+    // Only a range that starts within a tile and completes it needs fixup.
+    if (work_unit == tile_work || work_unit_end < tile_work + p.k_units_per_tile) {
+        return;
+    }
+
+    const uint first_group = group_for_work(tile_work);
+    const uint outer_tile = fastdiv(tile_idx, p.blocks_m_mp, fastdiv_L(p.div_Ls0, 1));
+    const uint ir = tile_idx - outer_tile * p.blocks_m;
+    const uint batch_idx = fastdiv(outer_tile, p.blocks_n_mp, fastdiv_L(p.div_Ls0, 2));
+    const uint ic = outer_tile - batch_idx * p.blocks_n;
+    const uint row_base = gl_WorkGroupID.y * RESOLVE_ROWS;
+
+    for (uint stripe_elem = gl_LocalInvocationIndex * 4; stripe_elem < RESOLVE_ROWS * p.BN; stripe_elem += gl_WorkGroupSize.x * 4) {
+        const uint r = row_base + (stripe_elem & (RESOLVE_ROWS - 1));
+        const uint c = stripe_elem / RESOLVE_ROWS;
+        const uint elem_idx = c * p.BM + r;
+        const uint dst_idx = batch_idx * p.M * p.N + (ic * p.BN + c) * p.M + ir * p.BM + r;
+
+        // D contains the completing segment. Add the earlier segments from scratch.
+        vec4 result = data_d4[dst_idx / 4];
+        for (uint previous_group = first_group; previous_group < group_idx; ++previous_group) {
+            result += data_s4[(previous_group * p.BM * p.BN + elem_idx) / 4];
+        }
+        data_d4[dst_idx / 4] = result;
+    }
+}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp
index 19d37870b..68de76373 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp
@@ -38,6 +38,7 @@ layout (constant_id = 12) const uint MmTypeA = 0;

 #include "types.glsl"
 #include "dot_product_funcs.glsl"
+#include "utils.glsl"

 #ifndef MULMAT_QUANT
 #ifndef LOAD_VEC_A
@@ -121,6 +122,9 @@ layout (binding = 0) readonly buffer BUF_Q5_K_P32   { block_q5_K_packed32 data[]
 layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
 layout (binding = 1) readonly buffer B_SCALAR {B_TYPE_SCALAR data_b_scalar[];};
 layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
+#ifndef MUL_MAT_ID
+layout (binding = 3) writeonly buffer S {D_TYPE data_s[];};
+#endif

 #ifdef MUL_MAT_ID
 layout (binding = 3) readonly buffer IDS {int data_ids[];};
@@ -155,6 +159,18 @@ layout (push_constant) uniform parameter
     uint ne12;
     uint broadcast2;
     uint broadcast3;
+    uint padded_N;
+    uint k_units_per_tile;
+    uint work_units_per_group;
+    uint groups_with_extra_unit;
+    uint k_units_mp;
+    uint blocks_m_mp;
+    uint blocks_n_mp;
+    uint ne12_mp;
+    uint broadcast2_mp;
+    uint broadcast3_mp;
+    uint div_Ls0;
+    uint div_Ls1;
 #endif
 } p;

@@ -189,6 +205,9 @@ layout(constant_id = 13) const bool APPLY_SLM_A_RESHAPE = false;
 const uint SHMEM_STRIDE_PAD = 1;
 const bool APPLY_SLM_A_RESHAPE = false;
 #endif
+#ifndef MUL_MAT_ID
+layout(constant_id = 15) const bool stream_k = false;
+#endif
 #define SHMEM_STRIDE (BK / 2 + SHMEM_STRIDE_PAD)

 shared FLOAT_TYPEV2 buf_a[BM * SHMEM_STRIDE];
@@ -218,27 +237,24 @@ void main() {
     _ne1 = data_row_map[2 + 2 * tile_idx];
     const uint ic = 0;
 #else
-    const uint ic = gl_WorkGroupID.y;
+    const uint group_idx = gl_WorkGroupID.x;
+    uint stream_k_work_unit;
+    uint stream_k_work_unit_end;
+
+    if (stream_k) {
+        stream_k_work_unit = group_idx * p.work_units_per_group + min(group_idx, p.groups_with_extra_unit);
+        stream_k_work_unit_end = stream_k_work_unit + p.work_units_per_group + uint(group_idx < p.groups_with_extra_unit);
+    }
 #endif
 #if defined(NEEDS_INIT_IQ_SHMEM) || defined(MULMAT_QUANT)
     init_iq_shmem(gl_WorkGroupSize);
 #endif

-#ifndef MUL_MAT_ID
-    const uint batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
-
-    const uint i13 = batch_idx / p.ne12;
-    const uint i12 = batch_idx % p.ne12;
-
-    const uint i03 = i13 / p.broadcast3;
-    const uint i02 = i12 / p.broadcast2;
-
-    const uint batch_idx_a = i03 * p.ne02 + i02;
-#endif
-
+#ifdef MUL_MAT_ID
     const uint blocks_m = (p.M + BM - 1) / BM;
     const uint ir = gl_WorkGroupID.x % blocks_m;
     const uint ik = gl_WorkGroupID.x / blocks_m;
+#endif

     const uint WNITER = (WM * WN) / (WARP * TM * TN * WMITER);
     const uint WSUBM = WM / WMITER;
@@ -288,6 +304,41 @@ void main() {
     const uint loadstride_a = gl_WorkGroupSize.x * LOAD_VEC_A_EFF * LOAD_VEC_BATCH_A / BK;
     const uint loadstride_b = gl_WorkGroupSize.x * LOAD_VEC_B_EFF * LOAD_VEC_BATCH_B / BK;

+#ifndef MUL_MAT_ID
+    do {
+    uint stream_k_tile_k_unit;
+    uint stream_k_tile_k_units;
+    uint ir;
+    uint ic;
+    uint batch_idx;
+    uint ik;
+
+    if (stream_k) {
+        const uint tile_idx = fastdiv(stream_k_work_unit, p.k_units_mp, fastdiv_L(p.div_Ls0, 0));
+        stream_k_tile_k_unit = stream_k_work_unit - tile_idx * p.k_units_per_tile;
+        stream_k_tile_k_units = min(stream_k_work_unit_end - stream_k_work_unit, p.k_units_per_tile - stream_k_tile_k_unit);
+
+        const uint blocks_m = p.M / BM;
+        const uint blocks_n = p.N / BN;
+        const uint outer_tile = fastdiv(tile_idx, p.blocks_m_mp, fastdiv_L(p.div_Ls0, 1));
+        ir = tile_idx - outer_tile * blocks_m;
+        batch_idx = fastdiv(outer_tile, p.blocks_n_mp, fastdiv_L(p.div_Ls0, 2));
+        ic = outer_tile - batch_idx * blocks_n;
+    } else {
+        const uint blocks_m = (p.M + BM - 1) / BM;
+        ic = gl_WorkGroupID.y;
+        batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
+        ir = group_idx % blocks_m;
+        ik = group_idx / blocks_m;
+    }
+
+    const uint i13 = fastdiv(batch_idx, p.ne12_mp, fastdiv_L(p.div_Ls1, 0));
+    const uint i12 = batch_idx - i13 * p.ne12;
+    const uint i03 = fastdiv(i13, p.broadcast3_mp, fastdiv_L(p.div_Ls1, 2));
+    const uint i02 = fastdiv(i12, p.broadcast2_mp, fastdiv_L(p.div_Ls1, 1));
+    const uint batch_idx_a = i03 * p.ne02 + i02;
+#endif
+
 #ifdef MUL_MAT_ID
     load_row_ids(row_begin);
     uint required_work_items = _ne1 * BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B;
@@ -298,8 +349,15 @@ void main() {
     const uint start_k = 0;
     const uint end_k = p.K;
 #else
-    const uint start_k = ik * p.k_split;
-    const uint end_k = min(p.K, (ik + 1) * p.k_split);
+    uint start_k;
+    uint end_k;
+    if (stream_k) {
+        start_k = stream_k_tile_k_unit * 256;
+        end_k = (stream_k_tile_k_unit + stream_k_tile_k_units) * 256;
+    } else {
+        start_k = ik * p.k_split;
+        end_k = min(p.K, (ik + 1) * p.k_split);
+    }
 #endif

     uint pos_a =
@@ -438,7 +496,17 @@ void main() {
     const uint dc = ic * BN + warp_c * WN;

 #ifndef MUL_MAT_ID
-    const uint offsets = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches;
+    uint offsets = batch_idx * p.batch_stride_d;
+    if (!stream_k) {
+        offsets += ik * p.batch_stride_d * p.num_batches;
+    }
+#endif
+
+#ifndef MUL_MAT_ID
+    bool stream_k_store_scratch;
+    if (stream_k) {
+        stream_k_store_scratch = stream_k_tile_k_unit + stream_k_tile_k_units != p.k_units_per_tile;
+    }
 #endif

 #ifdef COOPMAT
@@ -462,36 +530,44 @@ void main() {
         }
     }
 #else
-    const bool is_aligned = p.stride_d % 4 == 0;  // Assumption: D_TYPE == float
-
     [[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) {
         [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) {
-            const bool is_in_bounds = dr + (cm_row + 1) * TM <= p.M && dc + (cm_col + 1) * TN <= p.N;
-
-            if (is_aligned && is_in_bounds) {
-                // Full coopMat is within bounds and stride_d is aligned with 16B
-                coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> cm_dtype = coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(sums[cm_col * cms_per_row + cm_row]);
-                coopMatStore(cm_dtype, data_d, offsets + (dc + cm_col * TN) * p.stride_d + dr + cm_row * TM, p.stride_d, gl_CooperativeMatrixLayoutColumnMajor);
-            } else if (is_in_bounds) {
-                // Full coopMat is within bounds, but stride_d is not aligned
-                coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
-
-                controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
-                [[unroll]] for (uint col = 0; col < TN; col += storestride) {
-                    data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
+            coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> cm_dtype = coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(sums[cm_col * cms_per_row + cm_row]);
+            if (stream_k) {
+                if (stream_k_store_scratch) {
+                    const uint scratch_offset = group_idx * BM * BN + (warp_c * WN + cm_col * TN) * BM + warp_r * WM + cm_row * TM;
+                    coopMatStore(cm_dtype, data_s, scratch_offset, BM, gl_CooperativeMatrixLayoutColumnMajor);
+                } else {
+                    coopMatStore(cm_dtype, data_d, offsets + (dc + cm_col * TN) * p.stride_d + dr + cm_row * TM, p.stride_d, gl_CooperativeMatrixLayoutColumnMajor);
                 }
-                controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
-            } else if (dr + cm_row * TM < p.M && dc + cm_col * TN < p.N) {
-                // Partial coopMat is within bounds
-                coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
-
-                controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
-                [[unroll]] for (uint col = 0; col < TN; col += storestride) {
-                    if (dr + cm_row * TM + store_r < p.M && dc + cm_col * TN + col + store_c < p.N) {
+            } else {
+                const bool is_aligned = p.stride_d % 4 == 0;  // Assumption: D_TYPE == float
+                const bool is_in_bounds = dr + (cm_row + 1) * TM <= p.M && dc + (cm_col + 1) * TN <= p.N;
+
+                if (is_aligned && is_in_bounds) {
+                    // Full coopMat is within bounds and stride_d is aligned with 16B
+                    coopMatStore(cm_dtype, data_d, offsets + (dc + cm_col * TN) * p.stride_d + dr + cm_row * TM, p.stride_d, gl_CooperativeMatrixLayoutColumnMajor);
+                } else if (is_in_bounds) {
+                    // Full coopMat is within bounds, but stride_d is not aligned
+                    coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
+
+                    controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
+                    [[unroll]] for (uint col = 0; col < TN; col += storestride) {
                         data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
                     }
+                    controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
+                } else if (dr + cm_row * TM < p.M && dc + cm_col * TN < p.N) {
+                    // Partial coopMat is within bounds
+                    coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
+
+                    controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
+                    [[unroll]] for (uint col = 0; col < TN; col += storestride) {
+                        if (dr + cm_row * TM + store_r < p.M && dc + cm_col * TN + col + store_c < p.N) {
+                            data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
+                        }
+                    }
+                    controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
                 }
-                controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
             }
         }
     }
@@ -519,11 +595,22 @@ void main() {
                         data_d[row_idx.y * p.batch_stride_d + row_idx.x * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
                     }
 #else
-                    if (dr_warp + 2 * cr < p.M && dc_warp + cc < p.N) {
-                        data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr] = D_TYPE(sums[sums_idx].x);
-                    }
-                    if (dr_warp + 2 * cr + 1 < p.M && dc_warp + cc < p.N) {
-                        data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
+                    if (stream_k) {
+                        const uint scratch_offset = group_idx * BM * BN + (dc_warp + cc - ic * BN) * BM + dr_warp + 2 * cr - ir * BM;
+                        if (stream_k_store_scratch) {
+                            data_s[scratch_offset] = D_TYPE(sums[sums_idx].x);
+                            data_s[scratch_offset + 1] = D_TYPE(sums[sums_idx].y);
+                        } else {
+                            data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr] = D_TYPE(sums[sums_idx].x);
+                            data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
+                        }
+                    } else {
+                        if (dr_warp + 2 * cr < p.M && dc_warp + cc < p.N) {
+                            data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr] = D_TYPE(sums[sums_idx].x);
+                        }
+                        if (dr_warp + 2 * cr + 1 < p.M && dc_warp + cc < p.N) {
+                            data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
+                        }
                     }
 #endif // MUL_MAT_ID
                 }
@@ -531,4 +618,10 @@ void main() {
         }
     }
 #endif // COOPMAT
+#ifndef MUL_MAT_ID
+    if (stream_k) {
+        stream_k_work_unit += stream_k_tile_k_units;
+    }
+    } while (stream_k && stream_k_work_unit < stream_k_work_unit_end);
+#endif
 }
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp
index dad9260b9..1416cf72a 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp
@@ -61,6 +61,10 @@ uint mm_quant_k() {
 }
 #endif

+#ifndef MUL_MAT_ID
+layout (constant_id = 15) const bool stream_k = false;
+#endif
+
 layout (push_constant) uniform parameter
 {
     uint M;
@@ -91,6 +95,17 @@ layout (push_constant) uniform parameter
     uint broadcast3;
     // N dimension for the B matrix can be >= p.N
     uint padded_N;
+    uint k_units_per_tile;
+    uint work_units_per_group;
+    uint groups_with_extra_unit;
+    uint k_units_mp;
+    uint blocks_m_mp;
+    uint blocks_n_mp;
+    uint ne12_mp;
+    uint broadcast2_mp;
+    uint broadcast3_mp;
+    uint div_Ls0;
+    uint div_Ls1;
 #endif
 } p;

@@ -102,6 +117,9 @@ layout (binding = 0) readonly buffer A {uint8_t data_a[];};
 #endif
 layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
 layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
+#ifndef MUL_MAT_ID
+layout (binding = 3) writeonly buffer S {D_TYPE data_s[];};
+#endif
 #if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)
 layout (binding = 1) readonly buffer B4 {B_TYPEV4 data_b_v4[];};
 #endif
@@ -261,6 +279,17 @@ void main() {
     const uint row_begin = tile_idx * BN;
     _ne1 = data_row_map[2 + 2 * tile_idx];
     const uint ic = 0;
+#else
+    const uint group_idx = gl_WorkGroupID.x;
+    uint stream_k_work_unit;
+    uint stream_k_work_unit_end;
+
+    if (stream_k) {
+        // A work unit covers 256 consecutive K elements of one output tile.
+        // Work units are flattened as [K unit, M tile, N tile, batch].
+        stream_k_work_unit = group_idx * p.work_units_per_group + min(group_idx, p.groups_with_extra_unit);
+        stream_k_work_unit_end = stream_k_work_unit + p.work_units_per_group + uint(group_idx < p.groups_with_extra_unit);
+    }
 #endif

 #ifdef MUL_MAT_ID
@@ -278,43 +307,61 @@ void main() {
 #endif

 #ifndef MUL_MAT_ID
-    const uint batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
-
-    const uint i13 = batch_idx / p.ne12;
-    const uint i12 = batch_idx % p.ne12;
-
-    const uint i03 = i13 / p.broadcast3;
-    const uint i02 = i12 / p.broadcast2;
+    do {
+    uint stream_k_tile_k_unit;
+    uint stream_k_tile_k_units;
+    uint ir;
+    uint ic;
+    uint batch_idx;
+    uint ik;
+
+    if (stream_k) {
+        // The 256-element granularity does not split any supported quant block.
+        const uint tile_idx = fastdiv(stream_k_work_unit, p.k_units_mp, fastdiv_L(p.div_Ls0, 0));
+        stream_k_tile_k_unit = stream_k_work_unit - tile_idx * p.k_units_per_tile;
+        stream_k_tile_k_units = min(stream_k_work_unit_end - stream_k_work_unit, p.k_units_per_tile - stream_k_tile_k_unit);
+
+        const uint blocks_m = p.M / BM;
+        const uint blocks_n = p.N / BN;
+        const uint outer_tile = fastdiv(tile_idx, p.blocks_m_mp, fastdiv_L(p.div_Ls0, 1));
+        ir = tile_idx - outer_tile * blocks_m;
+        batch_idx = fastdiv(outer_tile, p.blocks_n_mp, fastdiv_L(p.div_Ls0, 2));
+        ic = outer_tile - batch_idx * blocks_n;
+    } else {
+        const uint blocks_m = (p.M + BM - 1) / BM;
+        // L2-friendly workgroup scheduling
+        const uint blocks_n = (p.N + BN - 1) / BN;
+#if defined(MULMAT_QUANT)
+        const uint a_panel_bytes = (BM * p.K / mm_quant_k()) * MmABlockBytes;
+#elif defined(DATA_A_F32)
+        const uint a_panel_bytes = BM * p.K * 4;
+#elif QUANT_K == 1
+        const uint a_panel_bytes = BM * p.K * 2;
+#else
+        const uint a_panel_bytes = (BM * p.K) / 2 + (BM * p.K) / 16;
+#endif
+        const uint group_m = clamp(GROUP_A_BUDGET / max(a_panel_bytes, 1u), 1u, min(blocks_m, 32u));
+        const uint tiles_per_group = group_m * blocks_n;
+        const uint lin = gl_WorkGroupID.y * blocks_m + (gl_WorkGroupID.x % blocks_m);
+        const uint group_id = lin / tiles_per_group;
+        const uint first_m = group_id * group_m;
+        const uint gsize = min(blocks_m - first_m, group_m);
+        const uint in_group = lin - group_id * tiles_per_group;
+        ir = first_m + in_group % gsize;
+        ic = in_group / gsize;
+        batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
+        ik = group_idx / blocks_m;
+    }

+    const uint i13 = fastdiv(batch_idx, p.ne12_mp, fastdiv_L(p.div_Ls1, 0));
+    const uint i12 = batch_idx - i13 * p.ne12;
+    const uint i03 = fastdiv(i13, p.broadcast3_mp, fastdiv_L(p.div_Ls1, 2));
+    const uint i02 = fastdiv(i12, p.broadcast2_mp, fastdiv_L(p.div_Ls1, 1));
     const uint batch_idx_a = i03 * p.ne02 + i02;
-#endif
-
+#else
     const uint blocks_m = (p.M + BM - 1) / BM;
     const uint ik = gl_WorkGroupID.x / blocks_m;
-
-#ifdef MUL_MAT_ID
     const uint ir = gl_WorkGroupID.x % blocks_m;
-#else
-    // L2-friendly workgroup scheduling
-    const uint blocks_n = (p.N + BN - 1) / BN;
-#if defined(MULMAT_QUANT)
-    const uint a_panel_bytes = (BM * p.K / mm_quant_k()) * MmABlockBytes;
-#elif defined(DATA_A_F32)
-    const uint a_panel_bytes = BM * p.K * 4;
-#elif QUANT_K == 1
-    const uint a_panel_bytes = BM * p.K * 2;
-#else
-    const uint a_panel_bytes = (BM * p.K) / 2 + (BM * p.K) / 16;
-#endif
-    const uint group_m = clamp(GROUP_A_BUDGET / max(a_panel_bytes, 1u), 1u, min(blocks_m, 32u));
-    const uint tiles_per_group = group_m * blocks_n;
-    const uint lin = gl_WorkGroupID.y * blocks_m + (gl_WorkGroupID.x % blocks_m);
-    const uint group_id = lin / tiles_per_group;
-    const uint first_m = group_id * group_m;
-    const uint gsize = min(blocks_m - first_m, group_m);
-    const uint in_group = lin - group_id * tiles_per_group;
-    const uint ir = first_m + in_group % gsize;
-    const uint ic = in_group / gsize;
 #endif

 #ifdef MUL_MAT_ID
@@ -325,8 +372,15 @@ void main() {
     uint start_k = 0;
     const uint end_k = p.K;
 #else
-    uint start_k = ik * p.k_split;
-    const uint end_k = min(p.K, (ik + 1) * p.k_split);
+    uint start_k;
+    uint end_k;
+    if (stream_k) {
+        start_k = stream_k_tile_k_unit * 256;
+        end_k = (stream_k_tile_k_unit + stream_k_tile_k_units) * 256;
+    } else {
+        start_k = ik * p.k_split;
+        end_k = min(p.K, (ik + 1) * p.k_split);
+    }
 #endif

 #ifdef MULMAT_QUANT
@@ -341,7 +395,10 @@ void main() {
 #else
     uint pos_a = batch_idx_a * (p.batch_stride_a / qk);
     uint pos_b = batch_idx * p.batch_stride_b;
-    uint pos_d = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches;
+    uint pos_d = batch_idx * p.batch_stride_d;
+    if (!stream_k) {
+        pos_d += ik * p.batch_stride_d * p.num_batches;
+    }
 #endif

 #ifdef MULMAT_QUANT
@@ -404,12 +461,15 @@ void main() {
     const uint UNROLL_K = (qk == 256) ? 256 : (BK * 8);
     const uint unroll_count = UNROLL_K / BK;

-    // Detect a fast path where all loads are entirely in bounds and no clamping is required
-    if ((ir + 1) * BM <= p.M && (ic + 1) * BN <= p.padded_N && (start_k % START_ALIGN_K) == 0 && (end_k % BK) == 0 &&
+    bool fast_path = stream_k;
+    if (!stream_k) {
+        fast_path = (ir + 1) * BM <= p.M && (ic + 1) * BN <= p.padded_N && (start_k % START_ALIGN_K) == 0 && (end_k % BK) == 0 &&
 #if !defined(MULMAT_QUANT) && QUANT_K == 1
-        (stride_a % 8) == 0 &&
+                    (stride_a % 8) == 0 &&
 #endif
-        (stride_b % 8) == 0) {
+                    (stride_b % 8) == 0;
+    }
+    if (fast_path) {
         // Hint to the compiler that values are aligned (want 16B alignment)
         start_k &= ~(START_ALIGN_K-1);
         stride_b &= ~7;
@@ -427,7 +487,7 @@ void main() {
         // The fetches and stores are pipelined to hide the latency.
         fetch_scales(ir * BM, pos_a, stride_a, start_k, tid, true);

-        if (enable_smaller_matrices && ic * BN + BNover4 >= p.N) {
+        if (!stream_k && enable_smaller_matrices && ic * BN + BNover4 >= p.N) {
             coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator> sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator>(0.0);
             for (uint i = 0; i < k_iters; ++i) {

@@ -470,7 +530,7 @@ void main() {

             coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BNover4, ir * BM, BM), tensorViewTranspose);
             return;
-        } else if (enable_smaller_matrices && ic * BN + BNover2 >= p.N) {
+        } else if (!stream_k && enable_smaller_matrices && ic * BN + BNover2 >= p.N) {
             coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator> sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator>(0.0);
             for (uint i = 0; i < k_iters; ++i) {

@@ -555,8 +615,18 @@ void main() {

             coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator> mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator>(sum);

-            coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
-            return;
+            if (stream_k) {
+                if (stream_k_tile_k_unit + stream_k_tile_k_units != p.k_units_per_tile) {
+                    tensorLayoutNV<2> tensorLayoutS = createTensorLayoutNV(2);
+                    tensorLayoutS = setTensorLayoutDimensionNV(tensorLayoutS, BN, BM);
+                    tensorLayoutS = setTensorLayoutStrideNV(tensorLayoutS, BM, 1);
+                    coopMatStoreTensorNV(mat_d, data_s, group_idx * BM * BN, tensorLayoutS, tensorViewTranspose);
+                } else {
+                    coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
+                }
+            } else {
+                coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
+            }
         }
     } else
 #endif // !defined(MUL_MAT_ID)
@@ -720,4 +790,10 @@ void main() {
         coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
 #endif
     }
+#ifndef MUL_MAT_ID
+    if (stream_k) {
+        stream_k_work_unit += stream_k_tile_k_units;
+    }
+    } while (stream_k && stream_k_work_unit < stream_k_work_unit_end);
+#endif
 }
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
index 86b69d15f..2a0774696 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
@@ -948,6 +948,7 @@ void process_shaders() {
     string_to_spv("acc_f32", "acc.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});

     string_to_spv("split_k_reduce", "mul_mat_split_k_reduce.comp", {});
+    string_to_spv("stream_k_fixup", "mul_mat_stream_k_fixup.comp", {});
     string_to_spv("fa_split_k_reduce", "flash_attn_split_k_reduce.comp", {});

     string_to_spv("fa_mask_opt", "flash_attn_mask_opt.comp", {});
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index b001c79d1..be577b36c 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -12265,6 +12265,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {

     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 576, 512, 576, {1,1}, {1,1}));
     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 1, 2048, 8192, {1,  1}, {1, 1}));
+
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 128, 128,  4352, {1, 1}, {1, 1}));
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32,  64, 192, 17664, {1, 1}, {1, 1}));
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 128, 128, 4096, {1, 1}, {2, 1}));
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 128, 256, 4096, {1, 1}, {1, 1}));
+
     for (ggml_type type_a : all_types) {
         test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1, 64, 256, {1,  1}, {1, 1}));
     }