Commit 16f80bd0c for llama.cpp
commit 16f80bd0c0fa6b27e7d0ec7fcb5a0e0dc5f3fd47
Author: Ruben Ortlam <rortlam@redhat.com>
Date: Fri Nov 20 08:38:14 2026 +0100
vulkan: chunked GATED_DELTA_NET implementation (#30207)
* vulkan: add cm2 chunked GATED_DELTA_NET shader
* fix shmem bank conflict bottleneck
* optimize further
* add coopmat maintenance1 variant
* optimize for RDNA4
* add gmem version RDNA4
* optimize for Ampere
* fix nvidia shmem overuse
* fix cm2 validation errors
* improve cm2 perf
* improve cm1 perf
* clean up
* enable by default
* show cm maintenance1 marker in device info
* properly check for cm_m1 features
* add tests
* add v32 mode, improve chunked condition
* allow other subgroup sizes
* vulkan: specialize the GDN CM2 workgroup size
Assisted-by: Codex
* vulkan: coopmat2 GDN implementation
* work around Nvidia shader compiler crash
* remove shared shader cm2 code
---------
Co-authored-by: Jeff Bolz <jbolz@nvidia.com>
diff --git a/ggml/src/ggml-vulkan/CMakeLists.txt b/ggml/src/ggml-vulkan/CMakeLists.txt
index 791753a8c..d8fa6ef40 100644
--- a/ggml/src/ggml-vulkan/CMakeLists.txt
+++ b/ggml/src/ggml-vulkan/CMakeLists.txt
@@ -101,6 +101,12 @@ if (Vulkan_FOUND)
"GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT"
)
+ test_shader_extension_support(
+ "GL_EXT_cooperative_matrix_maintenance1"
+ "${CMAKE_CURRENT_SOURCE_DIR}/vulkan-shaders/feature-tests/coopmat_maintenance1.comp"
+ "GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT"
+ )
+
test_shader_extension_support(
"GL_EXT_integer_dot_product"
"${CMAKE_CURRENT_SOURCE_DIR}/vulkan-shaders/feature-tests/integer_dot.comp"
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h
index 859cca181..0a04c1d56 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h
@@ -787,6 +787,7 @@ struct vk_op_gated_delta_net_push_constants {
uint32_t n_seqs;
uint32_t s_off;
uint32_t sq1, sq2, sq3;
+ uint32_t sk1, sk2, sk3;
uint32_t sv1, sv2, sv3;
uint32_t sb1, sb2, sb3;
uint32_t neq1, rq3;
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
index b4fec2da4..211948ee7 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
@@ -36,6 +36,21 @@ typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV {
} VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV;
#endif
+#ifndef VK_EXT_cooperative_matrix_maintenance1
+#define VK_EXT_cooperative_matrix_maintenance1 1
+#define VK_EXT_COOPERATIVE_MATRIX_MAINTENANCE_1_EXTENSION_NAME "VK_EXT_cooperative_matrix_maintenance1"
+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_MAINTENANCE_1_FEATURES_EXT ((VkStructureType)1000659000)
+typedef struct VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT {
+ VkStructureType sType;
+ void* pNext;
+ VkBool32 cooperativeMatrixProperties2;
+ VkBool32 cooperativeMatrixReductions;
+ VkBool32 cooperativeMatrixConversions;
+ VkBool32 cooperativeMatrixPerElementOperations;
+ VkBool32 cooperativeMatrixGetCoordinate;
+} VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT;
+#endif
+
#if __has_include(<spirv/unified1/spirv.hpp>)
# include <spirv/unified1/spirv.hpp>
#elif __has_include(<spirv-headers/spirv.hpp>)
@@ -221,6 +236,12 @@ static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; }
#define MAX_FUSED_ADDS (MAX_PARAMETER_COUNT - 3)
+#define GGML_VK_GDN_CM_V 64u
+#define GGML_VK_GDN_CM2_V 32u
+#define GGML_VK_GDN_CM_D 128u
+#define GGML_VK_GDN_CM_PAD 8u
+#define GGML_VK_GDN_CM_LDP (GGML_VK_GDN_CM_D + GGML_VK_GDN_CM_PAD)
+
struct vk_pipeline_struct;
typedef std::shared_ptr<struct vk_pipeline_struct> vk_pipeline;
@@ -767,6 +788,12 @@ struct vk_device_struct {
bool coopmat_acc_f32_support {};
bool coopmat_acc_f16_support {};
bool coopmat_bf16_support {};
+ bool coopmat_bf16_acc_support {};
+ bool coopmat_maintenance1 {};
+ bool coopmat_m1_reductions {};
+ bool coopmat_m1_conversions {};
+ bool coopmat_m1_per_element_ops {};
+ bool coopmat_m1_get_coordinate {};
bool coopmat_support_16x16x16_f16acc {};
bool coopmat_support_16x16x16_f32acc {};
bool coopmat1_fa_support {};
@@ -781,6 +808,7 @@ struct vk_device_struct {
bool coopmat2;
bool coopmat2_bf16_support {};
+ bool coopmat2_gdn_support {};
bool coopmat2_decode_vector;
bool dot2_f16 {};
@@ -987,6 +1015,9 @@ struct vk_device_struct {
vk_pipeline pipeline_lightning_indexer_f32[GGML_TYPE_COUNT];
// [size_idx][kda] where size_idx: 0=d16, 1=d32, 2=d64, 3=d128
vk_pipeline pipeline_gated_delta_net[4][2];
+ vk_pipeline pipeline_gated_delta_net_cm;
+ vk_pipeline pipeline_gated_delta_net_cm_v32;
+ vk_pipeline pipeline_gated_delta_net_cm_v128;
vk_pipeline pipeline_ssm_scan_f32_d128;
vk_pipeline pipeline_ssm_scan_f32_d256;
vk_pipeline pipeline_ssm_conv_f32;
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index cd8ef9ba3..a9d60d6af 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -1667,6 +1667,22 @@ static uint32_t ggml_vk_mul_mat_vec_max_cols(const vk_device & device) {
return device->coopmat2 && !device->disable_narrow_mul_mat ? 4u : mul_mat_vec_max_cols;
}
+static uint32_t ggml_vk_gated_delta_net_shmem_size(uint32_t v, bool coopmat2) {
+ const uint32_t C = 16;
+ const uint32_t LDP = GGML_VK_GDN_CM_LDP;
+ const uint32_t bf16_size = sizeof(ggml_bf16_t);
+ const bool gmem_mirror = !coopmat2 && v == 128;
+
+ const uint32_t qk = 4 * C * LDP * bf16_size;
+ const uint32_t state = gmem_mirror ? 0 : 2 * v * LDP * bf16_size;
+ const uint32_t p_inv = 4 * C * C * bf16_size;
+ const uint32_t delta_solved = coopmat2 ? 2 * C * (v + GGML_VK_GDN_CM_PAD) * bf16_size : 4 * C * v * bf16_size;
+ const uint32_t lower = (coopmat2 ? C * (C - 1) / 2 : C * C) * sizeof(float);
+ const uint32_t scan = 4 * C * sizeof(float);
+
+ return qk + state + p_inv + delta_solved + lower + scan;
+}
+
static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vector<uint32_t>& warptile, bool mul_mat_id, ggml_type src0_type) {
uint32_t lut_size = 0;
@@ -4124,13 +4140,14 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
device->subgroup_min_size <= 16u && device->subgroup_max_size >= 16u) {
gdn_subgroup_size = 16u;
}
+ const bool gdn_packed_rows = S_V == 128u && device->vendor_id == VK_VENDOR_ID_NVIDIA &&
+ gdn_subgroup_size == 32u && device->subgroup_arithmetic && device->subgroup_clustered;
uint32_t lanes_per_column;
if (device->vendor_id == VK_VENDOR_ID_INTEL) {
// Intel Xe: full-width reduction (min rows/lane) ~10x over the COLS_PER_WG=8 rule.
lanes_per_column = std::min(gdn_subgroup_size, S_V);
} else if (S_V >= 128u && device->subgroup_clustered) {
- // COLS_PER_WG=8: measured optimum (Ampere sg32->4, Vega20 sg64->8).
- lanes_per_column = std::max(1u, gdn_subgroup_size / 8u);
+ lanes_per_column = gdn_packed_rows ? 8u : std::max(1u, gdn_subgroup_size / 8u);
} else {
// Use largest power-of-two that divides both S_V and subgroup_size so that
// (1) S_V % lanes_per_column == 0 and (2) S_V % (subgroup_size / lanes_per_column) == 0.
@@ -4177,9 +4194,60 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
for (uint32_t kda = 0; kda < 2; kda++) {
ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net[si][kda],
gdn_names[si][kda], gdn_len, gdn_data, "main", 7, sizeof(vk_op_gated_delta_net_push_constants),
- wg_denoms, {S_V, kda, gdn_subgroup_size, lanes_per_column}, 1, true, use_subgroup_ops, gdn_subgroup_size);
+ wg_denoms, {S_V, kda, gdn_subgroup_size, lanes_per_column, uint32_t(gdn_packed_rows)}, 1, true, use_subgroup_ops, gdn_subgroup_size);
+ }
+ }
+
+ // Chunked GDN prefill uses workgroup matrices on CM2 and subgroup matrices on CM1.
+ const bool gdn_chunked = getenv("GGML_VK_DISABLE_GDN_CHUNKED") == nullptr;
+ const uint32_t gdn_shmem = device->properties.limits.maxComputeSharedMemorySize;
+#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)
+ if (gdn_chunked && device->coopmat2 && device->coopmat2_gdn_support) {
+ ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm,
+ "gated_delta_net_f32_cm2", gated_delta_net_f32_cm2_len, gated_delta_net_f32_cm2_data,
+ "main", 7, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, {}, 1, true, true, 32);
+ }
+#endif
+#if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)
+ // shader needs >= 16 lanes and 16 subgroups per WG; prefer wave64
+ auto gdn_sg_ok = [&](uint32_t sg) {
+ const bool reachable = device->subgroup_size_control ? device->subgroup_min_size <= sg && sg <= device->subgroup_max_size
+ : device->subgroup_size == sg;
+ return reachable && 16u * sg <= device->properties.limits.maxComputeWorkGroupInvocations &&
+ 16u * sg <= device->properties.limits.maxComputeWorkGroupSize[0];
+ };
+ const uint32_t gdn_sg = gdn_sg_ok(64) ? 64u : device->subgroup_size;
+ if (gdn_chunked && !device->coopmat2 && device->coopmat_support && device->coopmat_bf16_support && device->coopmat_m1_per_element_ops &&
+ gdn_sg >= 16 && gdn_sg_ok(gdn_sg)) {
+ const bool bf16acc = device->coopmat_bf16_acc_support;
+ const std::vector<uint32_t> gdn_spec = { 16u * gdn_sg };
+ // V=64: state mirror in shared, 7 bindings
+ if (gdn_shmem >= ggml_vk_gated_delta_net_shmem_size(GGML_VK_GDN_CM_V, false)) {
+ ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm,
+ bf16acc ? "gated_delta_net_f32_cm1_bf16acc" : "gated_delta_net_f32_cm1",
+ bf16acc ? gated_delta_net_f32_cm1_bf16acc_len : gated_delta_net_f32_cm1_len,
+ bf16acc ? (const void *)gated_delta_net_f32_cm1_bf16acc_data : (const void *)gated_delta_net_f32_cm1_data,
+ "main", 7, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, gdn_spec, 1, true, true, gdn_sg);
+ }
+ if (gdn_shmem >= ggml_vk_gated_delta_net_shmem_size(32u, false)) {
+ ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm_v32,
+ bf16acc ? "gated_delta_net_f32_cm1_bf16acc_v32" : "gated_delta_net_f32_cm1_v32",
+ bf16acc ? gated_delta_net_f32_cm1_bf16acc_v32_len : gated_delta_net_f32_cm1_v32_len,
+ bf16acc ? (const void *)gated_delta_net_f32_cm1_bf16acc_v32_data : (const void *)gated_delta_net_f32_cm1_v32_data,
+ "main", 7, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, gdn_spec, 1, true, true, gdn_sg);
+ }
+ // V=128: state mirror in a gmem scratch buffer, 8 bindings
+ if (gdn_shmem >= ggml_vk_gated_delta_net_shmem_size(128u, false)) {
+ ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm_v128,
+ bf16acc ? "gated_delta_net_f32_cm1_bf16acc_v128" : "gated_delta_net_f32_cm1_v128",
+ bf16acc ? gated_delta_net_f32_cm1_bf16acc_v128_len : gated_delta_net_f32_cm1_v128_len,
+ bf16acc ? (const void *)gated_delta_net_f32_cm1_bf16acc_v128_data : (const void *)gated_delta_net_f32_cm1_v128_data,
+ "main", 8, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, gdn_spec, 1, true, true, gdn_sg);
}
}
+#endif
+ GGML_UNUSED(gdn_chunked);
+ GGML_UNUSED(gdn_shmem);
}
if (device->subgroup_arithmetic && device->subgroup_require_full_support) {
@@ -4554,6 +4622,11 @@ vk_device ggml_vk_get_device(size_t idx) {
} else if (strcmp("VK_NV_cooperative_matrix2", properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_COOPMAT2")) {
coopmat2_support = true;
+#endif
+#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
+ } else if (strcmp("VK_EXT_cooperative_matrix_maintenance1", properties.extensionName) == 0 &&
+ !getenv("GGML_VK_DISABLE_COOPMAT_MAINTENANCE1")) {
+ device->coopmat_maintenance1 = true;
#endif
} else if (strcmp(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME, properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_COOPMAT2_DECODE_VECTOR")) {
@@ -4880,6 +4953,15 @@ vk_device ggml_vk_get_device(size_t idx) {
}
#endif
+#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
+ VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT coopmat_m1_features {};
+ coopmat_m1_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_MAINTENANCE_1_FEATURES_EXT;
+ if (device->coopmat_support && device->coopmat_maintenance1) {
+ last_struct->pNext = (VkBaseOutStructure *)&coopmat_m1_features;
+ last_struct = (VkBaseOutStructure *)&coopmat_m1_features;
+ }
+#endif
+
VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV coopmat2_decode_vector_features {};
coopmat2_decode_vector_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV;
if (coopmat2_decode_vector_support) {
@@ -5064,6 +5146,17 @@ vk_device ggml_vk_get_device(size_t idx) {
device->coopmat1_fa_support = device->coopmat_support && device->subgroup_require_full_support;
#endif
+#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
+ device->coopmat_maintenance1 = device->coopmat_support && device->coopmat_maintenance1;
+ if (device->coopmat_maintenance1) {
+ device->coopmat_m1_reductions = coopmat_m1_features.cooperativeMatrixReductions;
+ device->coopmat_m1_conversions = coopmat_m1_features.cooperativeMatrixConversions;
+ device->coopmat_m1_per_element_ops = coopmat_m1_features.cooperativeMatrixPerElementOperations;
+ device->coopmat_m1_get_coordinate = coopmat_m1_features.cooperativeMatrixGetCoordinate;
+ device_extensions.push_back(VK_EXT_COOPERATIVE_MATRIX_MAINTENANCE_1_EXTENSION_NAME);
+ }
+#endif
+
if (coopmat2_support) {
#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)
if (coopmat2_features.cooperativeMatrixWorkgroupScope &&
@@ -5147,6 +5240,13 @@ vk_device ggml_vk_get_device(size_t idx) {
prop.KGranularity <= 16) {
found_bf16_128 = true;
}
+ // GDN uses M=32, with N/K dimensions that are multiples of 16.
+ if (prop.workgroupInvocations == 128 &&
+ prop.MGranularity != 0 && 32 % prop.MGranularity == 0 &&
+ prop.NGranularity != 0 && 16 % prop.NGranularity == 0 &&
+ prop.KGranularity != 0 && 16 % prop.KGranularity == 0) {
+ device->coopmat2_gdn_support = true;
+ }
if (prop.workgroupInvocations == 256 &&
prop.MGranularity <= 32 &&
prop.NGranularity <= 32 &&
@@ -5157,6 +5257,12 @@ vk_device ggml_vk_get_device(size_t idx) {
#endif
}
}
+ device->coopmat2_gdn_support = device->coopmat2_gdn_support &&
+ coopmat2_props.cooperativeMatrixWorkgroupScopeMaxWorkgroupSize >= 128 &&
+ device->properties.limits.maxComputeWorkGroupInvocations >= 128 &&
+ device->properties.limits.maxComputeWorkGroupSize[0] >= 128 &&
+ uint64_t(ggml_vk_gated_delta_net_shmem_size(GGML_VK_GDN_CM2_V, true)) + coopmat2_props.cooperativeMatrixWorkgroupScopeReservedSharedMemory <=
+ device->properties.limits.maxComputeSharedMemorySize;
if (found_fp16_128 && found_fp16_256 &&
found_fp32_128 && found_fp32_256 &&
coopmat2_props.cooperativeMatrixFlexibleDimensionsMaxDimension >= 512) {
@@ -5274,6 +5380,18 @@ vk_device ggml_vk_get_device(size_t idx) {
device->coopmat_bf16_support = true;
}
}
+ // bf16 accumulator (bf16/bf16/bf16/bf16) at the shader's fixed 16x16x16
+ // shape lets the GDN coopmat1 path keep the hi/lo split as accumulators.
+ if (bfloat16_support &&
+ prop.AType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
+ prop.BType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
+ prop.CType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
+ prop.ResultType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
+ (vk::ScopeKHR)prop.scope == vk::ScopeKHR::eSubgroup &&
+ prop.MSize == 16 && prop.NSize == 16 && prop.KSize == 16
+ ) {
+ device->coopmat_bf16_acc_support = true;
+ }
#endif
}
@@ -5489,6 +5607,7 @@ static void ggml_vk_print_gpu_info(size_t idx) {
bool coopmat_support = false;
bool coopmat2_support = false;
bool coopmat2_decode_vector_support = false;
+ bool coopmat_maintenance1_support = false;
bool integer_dot_product = false;
bool bfloat16_support = false;
bool dot2_f16_support = false;
@@ -5509,6 +5628,11 @@ static void ggml_vk_print_gpu_info(size_t idx) {
} else if (strcmp("VK_NV_cooperative_matrix2", properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_COOPMAT2")) {
coopmat2_support = true;
+#endif
+#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
+ } else if (strcmp("VK_EXT_cooperative_matrix_maintenance1", properties.extensionName) == 0 &&
+ !getenv("GGML_VK_DISABLE_COOPMAT_MAINTENANCE1")) {
+ coopmat_maintenance1_support = true;
#endif
} else if (strcmp(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME, properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_COOPMAT2_DECODE_VECTOR")) {
@@ -5592,6 +5716,15 @@ static void ggml_vk_print_gpu_info(size_t idx) {
}
#endif
+#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
+ VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT coopmat_m1_features {};
+ coopmat_m1_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_MAINTENANCE_1_FEATURES_EXT;
+ if (coopmat_support && coopmat_maintenance1_support) {
+ last_struct->pNext = (VkBaseOutStructure *)&coopmat_m1_features;
+ last_struct = (VkBaseOutStructure *)&coopmat_m1_features;
+ }
+#endif
+
VkPhysicalDeviceShaderIntegerDotProductFeaturesKHR shader_integer_dot_product_features {};
shader_integer_dot_product_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_INTEGER_DOT_PRODUCT_FEATURES_KHR;
if (integer_dot_product) {
@@ -5685,13 +5818,17 @@ static void ggml_vk_print_gpu_info(size_t idx) {
coopmat2_support = false;
#endif
+#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
+ coopmat_maintenance1_support = coopmat_maintenance1_support && coopmat_m1_features.cooperativeMatrixPerElementOperations;
+#endif
+
coopmat2_decode_vector_support = coopmat2_decode_vector_support && coopmat2_decode_vector_features.cooperativeMatrixDecodeVector;
#if !defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT)
coopmat2_decode_vector_support = false;
#endif
std::string matrix_cores = coopmat2_support ? (coopmat2_decode_vector_support ? "NV_coopmat2v" : "NV_coopmat2")
- : coopmat_support ? "KHR_coopmat"
+ : coopmat_support ? (coopmat_maintenance1_support ? "KHR_coopmat_m1" : "KHR_coopmat")
: "none";
bool dot2_f16 = dot2_f16_support && dot2_features.shaderMixedFloatDotProductFloat16AccFloat32;
@@ -11172,6 +11309,7 @@ void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context& subctx
void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
const ggml_tensor * src_q = dst->src[0];
+ const ggml_tensor * src_k = dst->src[1];
const ggml_tensor * src_v = dst->src[2];
const ggml_tensor * src_beta = dst->src[4];
@@ -11187,7 +11325,39 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx,
const uint32_t s_off = S_v * H * n_tokens * n_seqs;
- vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op);
+ const ggml_tensor * src_g = dst->src[3];
+ // RDNA3: the scan is faster below ~4k tokens
+ const uint32_t cm_min_tokens = ctx->device->architecture == AMD_RDNA3 ? 4096u : 64u;
+ const bool use_cm = (ctx->device->pipeline_gated_delta_net_cm != nullptr ||
+ ctx->device->pipeline_gated_delta_net_cm_v32 != nullptr ||
+ ctx->device->pipeline_gated_delta_net_cm_v128 != nullptr) &&
+ src_g->ne[0] == 1 && S_v == 128 && dst->src[0]->ne[0] == 128 && K == 1 && n_tokens >= cm_min_tokens;
+
+ const bool use_cm2 = use_cm && ctx->device->coopmat2;
+ const bool have_v64 = ctx->device->pipeline_gated_delta_net_cm != nullptr;
+ bool use_v32 = use_cm && !use_cm2 && !have_v64 && ctx->device->pipeline_gated_delta_net_cm_v32 != nullptr;
+ bool use_v128 = use_cm && !use_cm2 && !have_v64 && !use_v32;
+ const uint32_t nsm = ctx->device->shader_core_count;
+ if (use_cm && !use_cm2 && have_v64 && nsm != 0) {
+ const uint32_t wg64 = 2u * H * n_seqs;
+ // V=32 only pays off while V=64 leaves at least half the cores idle
+ use_v32 = ctx->device->pipeline_gated_delta_net_cm_v32 != nullptr && 2u * wg64 <= nsm;
+ if (!use_v32 && ctx->device->pipeline_gated_delta_net_cm_v128 != nullptr) {
+ const uint32_t wg128 = H * n_seqs;
+ const uint32_t waves64 = (wg64 + nsm - 1) / nsm;
+ const uint32_t waves128 = (wg128 + nsm - 1) / nsm;
+ // prefer v128 when occupancy (wgs / (waves * nsm)) is at least as good
+ use_v128 = (uint64_t)wg128 * waves64 >= (uint64_t)wg64 * waves128;
+ }
+ }
+
+ const uint32_t cm_V = use_cm2 ? GGML_VK_GDN_CM2_V : (use_v128 ? 128u : (use_v32 ? 32u : GGML_VK_GDN_CM_V));
+
+ vk_pipeline pipeline = use_v128
+ ? ctx->device->pipeline_gated_delta_net_cm_v128
+ : use_v32 ? ctx->device->pipeline_gated_delta_net_cm_v32
+ : (use_cm ? ctx->device->pipeline_gated_delta_net_cm
+ : ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op));
GGML_ASSERT(pipeline != nullptr);
ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
@@ -11198,9 +11368,27 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx,
src_buf[i] = ggml_vk_tensor_subbuffer(ctx, dst->src[i]);
}
+ vk_subbuffer mirror_buf{};
+ if (use_v128) {
+ const uint32_t LDP = GGML_VK_GDN_CM_LDP;
+ const uint32_t blocks = (S_v / cm_V) * n_seqs * H;
+ const size_t scratch_size = (size_t)blocks * LDP * cm_V * 2u * sizeof(uint16_t);
+ if (ctx->prealloc_size_x < scratch_size) {
+ ctx->prealloc_size_x = scratch_size;
+ ggml_vk_preallocate_buffers(ctx, subctx);
+ }
+ if (ctx->prealloc_x_need_sync) {
+ ggml_vk_sync_buffers(ctx, subctx);
+ }
+ mirror_buf = vk_subbuffer{ ctx->prealloc_x, 0, ctx->prealloc_x->size };
+ }
+
const uint32_t sq1 = (uint32_t)(src_q->nb[1] / sizeof(float));
const uint32_t sq2 = (uint32_t)(src_q->nb[2] / sizeof(float));
const uint32_t sq3 = (uint32_t)(src_q->nb[3] / sizeof(float));
+ const uint32_t sk1 = (uint32_t)(src_k->nb[1] / sizeof(float));
+ const uint32_t sk2 = (uint32_t)(src_k->nb[2] / sizeof(float));
+ const uint32_t sk3 = (uint32_t)(src_k->nb[3] / sizeof(float));
const uint32_t sv1 = (uint32_t)(src_v->nb[1] / sizeof(float));
const uint32_t sv2 = (uint32_t)(src_v->nb[2] / sizeof(float));
const uint32_t sv3 = (uint32_t)(src_v->nb[3] / sizeof(float));
@@ -11215,6 +11403,7 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx,
const vk_op_gated_delta_net_push_constants pc = {
H, n_tokens, n_seqs, s_off,
sq1, sq2, sq3,
+ sk1, sk2, sk3,
sv1, sv2, sv3,
sb1, sb2, sb3,
neq1, rq3,
@@ -11222,9 +11411,16 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx,
K
};
- ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
- {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf},
- pc, { H, n_seqs, S_v });
+ if (use_v128) {
+ ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
+ {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf, mirror_buf},
+ pc, std::array<uint32_t, 3>{ S_v / cm_V, n_seqs, H });
+ ctx->prealloc_x_need_sync = true;
+ } else {
+ ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
+ {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf},
+ pc, use_cm ? std::array<uint32_t, 3>{ S_v / cm_V, n_seqs, H } : std::array<uint32_t, 3>{ H, n_seqs, S_v });
+ }
}
void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt b/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt
index cbe7a68bf..273a80233 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt
@@ -15,6 +15,10 @@ if (GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT)
add_compile_definitions(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT)
message(STATUS "Enabling coopmat2 decode_vector glslc support")
endif()
+if (GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
+ add_compile_definitions(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
+ message(STATUS "Enabling coopmat maintenance1 glslc support")
+endif()
if (GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)
add_compile_definitions(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)
message(STATUS "Enabling dot glslc support")
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat_maintenance1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat_maintenance1.comp
new file mode 100644
index 000000000..049ca2867
--- /dev/null
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat_maintenance1.comp
@@ -0,0 +1,8 @@
+#version 460
+
+#extension GL_KHR_cooperative_matrix : require
+#extension GL_EXT_cooperative_matrix_maintenance1 : require
+
+void main()
+{
+}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp b/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp
index 0e384330b..ad2ced78a 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp
@@ -15,10 +15,19 @@ layout(constant_id = 0) const uint S_V = 128;
layout(constant_id = 1) const uint KDA = 0;
layout(constant_id = 2) const uint SUBGROUP_SIZE = 32;
layout(constant_id = 3) const uint LANES_PER_COLUMN = 32;
+layout(constant_id = 4) const bool PACKED_ROWS = false;
const uint COLS_PER_WG = SUBGROUP_SIZE / LANES_PER_COLUMN;
const uint ROWS_PER_LANE = S_V / LANES_PER_COLUMN;
+uint row_index(uint r, uint lane) {
+ // Keep four adjacent rows together in each lane.
+ if (PACKED_ROWS) {
+ return (r / 4u * LANES_PER_COLUMN + lane) * 4u + r % 4u;
+ }
+ return r * LANES_PER_COLUMN + lane;
+}
+
layout(local_size_x_id = 2, local_size_y = 1, local_size_z = 1) in;
layout(push_constant) uniform Parameters {
@@ -27,6 +36,7 @@ layout(push_constant) uniform Parameters {
uint n_seqs;
uint s_off;
uint sq1, sq2, sq3;
+ uint sk1, sk2, sk3;
uint sv1, sv2, sv3;
uint sb1, sb2, sb3;
uint neq1, rq3;
@@ -110,7 +120,7 @@ void main() {
FLOAT_TYPE s_shard[ROWS_PER_LANE];
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
- s_shard[r] = FLOAT_TYPE(data_state[state_in_base + col * S_V + r * LANES_PER_COLUMN + lane]);
+ s_shard[r] = FLOAT_TYPE(data_state[state_in_base + col * S_V + row_index(r, lane)]);
}
// snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back.
@@ -120,7 +130,7 @@ void main() {
for (uint t = 0; t < n_tokens; t++) {
const uint q_off = iq3 * sq3 + t * sq2 + iq1 * sq1;
- const uint k_off = q_off;
+ const uint k_off = iq3 * sk3 + t * sk2 + iq1 * sk1;
const uint v_off = seq_id * sv3 + t * sv2 + head_id * sv1;
const uint gb_off = seq_id * sb3 + t * sb2 + head_id * sb1;
const FLOAT_TYPE beta_val = FLOAT_TYPE(data_beta[gb_off]);
@@ -128,7 +138,7 @@ void main() {
FLOAT_TYPE k_reg[ROWS_PER_LANE];
FLOAT_TYPE q_reg[ROWS_PER_LANE];
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
- const uint i = r * LANES_PER_COLUMN + lane;
+ const uint i = row_index(r, lane);
k_reg[r] = FLOAT_TYPE(data_k[k_off + i]);
q_reg[r] = FLOAT_TYPE(data_q[q_off + i]);
}
@@ -142,7 +152,7 @@ void main() {
} else {
const uint g_base = gb_off * S_V;
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
- const uint i = r * LANES_PER_COLUMN + lane;
+ const uint i = row_index(r, lane);
g_exp[r] = exp(FLOAT_TYPE(data_g[g_base + i]));
}
}
@@ -175,7 +185,7 @@ void main() {
if (target_slot >= 0 && target_slot < int(K)) {
const uint slot_base = s_off + uint(target_slot) * state_size_per_snap + state_out_base;
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
- data_dst[slot_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[r];
+ data_dst[slot_base + col * S_V + row_index(r, lane)] = s_shard[r];
}
}
}
@@ -183,7 +193,7 @@ void main() {
if (K == 1u) {
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
- data_dst[s_off + state_out_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[r];
+ data_dst[s_off + state_out_base + col * S_V + row_index(r, lane)] = s_shard[r];
}
}
}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net_cm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net_cm.comp
new file mode 100644
index 000000000..c5b4e4084
--- /dev/null
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net_cm.comp
@@ -0,0 +1,437 @@
+#version 450
+
+#extension GL_EXT_control_flow_attributes : enable
+#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
+#extension GL_EXT_bfloat16 : require
+#extension GL_KHR_memory_scope_semantics : enable
+#extension GL_KHR_shader_subgroup_basic : enable
+#extension GL_KHR_shader_subgroup_arithmetic : enable
+#extension GL_KHR_shader_subgroup_shuffle : enable
+#extension GL_KHR_cooperative_matrix : enable
+#extension GL_EXT_cooperative_matrix_maintenance1 : enable
+
+// Chunked gated-delta-net prefill (non-KDA, S_v == D == 128, K == 1), port of CUDA PR 29353.
+// State in register tiles, mirrored ([D x V]) for the projections.
+// bf16 hi/lo 3x products for ~fp32 accuracy.
+#define OPD bfloat16_t
+
+#define C 16 // chunk size (tokens per step)
+#define D 128 // head dim / value dim
+#define TM 16 // tile rows
+#define TN 16 // tile cols (bf16 cm has only m16n16k16; n8 would pad)
+#ifndef V
+#define V 64 // value columns per workgroup block (V=128 = whole head)
+#endif
+#define NSUB 16 // subgroups per workgroup
+layout(constant_id = 0) const uint WGS = NSUB * 32; // NSUB * subgroup size
+#define KT 16 // K-tile width for deep (K=D) matmuls (bf16 cm: m16n16k16 only)
+#define PAD 8 // shared padding to break COL-load bank conflicts (keep LDP*2 a multiple of 16B)
+#define LDP (D + PAD)
+
+// GMEM_MIRROR: state mirror in a gmem scratch buffer (too big for shared)
+
+#define STATE_TILES ((D / TM) * (V / TN))
+#define PER_WARP (STATE_TILES / NSUB)
+#define VT (V / TN) // value tiles
+#define CT (C / TN) // C-column tiles (gram/pmat halves)
+const uint NQK = C * D / WGS;
+
+#define TILE_R(ti) (((ti) / VT) * TM)
+#define TILE_C(ti) (((ti) % VT) * TN)
+
+layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
+
+layout(push_constant) uniform Parameters {
+ uint H;
+ uint n_tokens;
+ uint n_seqs;
+ uint s_off;
+ uint sq1, sq2, sq3;
+ uint sk1, sk2, sk3;
+ uint sv1, sv2, sv3;
+ uint sb1, sb2, sb3;
+ uint neq1, rq3;
+ float scale;
+ uint K;
+} p;
+
+layout(binding = 0) readonly buffer QBuf { float data_q[]; };
+layout(binding = 1) readonly buffer KBuf { float data_k[]; };
+layout(binding = 2) readonly buffer VBuf { float data_v[]; };
+layout(binding = 3) readonly buffer GBuf { float data_g[]; };
+layout(binding = 4) readonly buffer BetaBuf { float data_beta[]; };
+layout(binding = 5) readonly buffer StateBuf { float data_state[]; };
+layout(binding = 6) buffer DstBuf { float data_dst[]; };
+#ifdef GMEM_MIRROR
+layout(binding = 7) coherent buffer MirrorBuf { OPD data_mirror[]; };
+#endif
+
+shared OPD sh_qh[C * LDP], sh_ql[C * LDP]; // q*scale hi/lo (D padded)
+shared OPD sh_kh[C * LDP], sh_kl[C * LDP]; // k hi/lo (D padded)
+#ifndef GMEM_MIRROR
+shared OPD sh_sh[LDP * V], sh_sl[LDP * V]; // state mirror [D x V] column-major (D padded)
+#endif
+shared OPD sh_dh[C * V], sh_dl[C * V]; // delta (pre-inv) / delta2 [C x V] row-major
+shared OPD sh_soh[C * V], sh_sol[C * V]; // solved [C x V] row-major
+shared OPD sh_ph[C * C], sh_pl[C * C]; // decayed P
+shared OPD sh_ih[C * C], sh_il[C * C]; // (I + L)^-1
+shared float sh_lower[C * C]; // strictly-lower decayed L
+
+shared float sh_prefix[C], sh_beta[C], sh_ep[C], sh_carry[C];
+
+#define SG gl_ScopeSubgroup
+#define MUA gl_MatrixUseA
+#define MUB gl_MatrixUseB
+#define MACC gl_MatrixUseAccumulator
+#define ROW gl_CooperativeMatrixLayoutRowMajor
+#define COL gl_CooperativeMatrixLayoutColumnMajor
+
+
+// 3x bf16 product accumulated into acc (hi/lo split for ~fp32 accuracy).
+#define PRODUCT(acc, ah, al, bh, bl) \
+ acc = coopMatMulAdd(al, bh, acc); \
+ acc = coopMatMulAdd(ah, bl, acc); \
+ acc = coopMatMulAdd(ah, bh, acc);
+
+void split(const float x, out OPD h, out OPD l) {
+ h = OPD(x);
+ l = OPD(x - float(h));
+}
+
+#ifdef BF16ACC
+// split through a bf16 accumulator
+#define SPLIT_STORE(m, arr_h, off_h, arr_l, off_l, ld, layout) { \
+ const coopmat<OPD, SG, TM, TN, MACC> h_ = coopmat<OPD, SG, TM, TN, MACC>(m); \
+ const coopmat<OPD, SG, TM, TN, MACC> l_ = coopmat<OPD, SG, TM, TN, MACC>(m - coopmat<float, SG, TM, TN, MACC>(h_)); \
+ coopMatStore(h_, arr_h, off_h, ld, layout); \
+ coopMatStore(l_, arr_l, off_l, ld, layout); \
+}
+
+#ifdef GMEM_MIRROR
+#define STORE_MIRROR(m, off) SPLIT_STORE(m, data_mirror, mir_h + off, data_mirror, mir_l + off, LDP, COL)
+#else
+#define STORE_MIRROR(m, off) SPLIT_STORE(m, sh_sh, off, sh_sl, off, LDP, COL)
+#endif
+#define STORE_DELTA(m, off) SPLIT_STORE(m, sh_dh, off, sh_dl, off, V, ROW)
+#define STORE_SOLVED(m, off) SPLIT_STORE(m, sh_soh, off, sh_sol, off, V, ROW)
+#else
+// split per element
+#ifdef GMEM_MIRROR
+float SplitColS(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) {
+ const uint idx = off + c * LDP + r;
+ split(elem, data_mirror[idx], data_mirror[idx + LDP * V]);
+ return elem;
+}
+#define STORE_MIRROR(m, off) coopMatPerElementEXT(m, m, SplitColS, mir_h + off)
+#else
+float SplitColS(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) {
+ const uint idx = off + c * LDP + r;
+ split(elem, sh_sh[idx], sh_sl[idx]);
+ return elem;
+}
+#define STORE_MIRROR(m, off) coopMatPerElementEXT(m, m, SplitColS, off)
+#endif
+
+float SplitRowD(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) {
+ const uint idx = off + r * V + c;
+ split(elem, sh_dh[idx], sh_dl[idx]);
+ return elem;
+}
+
+float SplitRowSO(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) {
+ const uint idx = off + r * V + c;
+ split(elem, sh_soh[idx], sh_sol[idx]);
+ return elem;
+}
+
+#define STORE_DELTA(m, off) coopMatPerElementEXT(m, m, SplitRowD, off)
+#define STORE_SOLVED(m, off) coopMatPerElementEXT(m, m, SplitRowSO, off)
+#endif
+
+#ifdef GMEM_MIRROR
+#define LOAD_S(h, l, off) \
+ coopMatLoad(h, data_mirror, mir_h + off, LDP, COL); \
+ coopMatLoad(l, data_mirror, mir_l + off, LDP, COL);
+// mirror is workgroup-private
+#define MIRROR_BARRIER() controlBarrier(gl_ScopeWorkgroup, gl_ScopeWorkgroup, \
+ gl_StorageSemanticsBuffer | gl_StorageSemanticsShared, \
+ gl_SemanticsAcquireRelease)
+#else
+#define LOAD_S(h, l, off) \
+ coopMatLoad(h, sh_sh, off, LDP, COL); \
+ coopMatLoad(l, sh_sl, off, LDP, COL);
+#define MIRROR_BARRIER() barrier()
+#endif
+
+float StoreOut(const in uint32_t r, const in uint32_t c, const in float elem,
+ const in uint32_t out_base, const in uint32_t valid) {
+ if (r < valid) {
+ data_dst[out_base + r * (D * p.H) + c] = elem;
+ }
+ return elem;
+}
+
+float StoreState(const in uint32_t r, const in uint32_t c, const in float elem,
+ const in uint32_t base) {
+ data_dst[base + c * D + r] = elem;
+ return elem;
+}
+
+float LoadV(const in uint32_t r, const in uint32_t c, const in float elem,
+ const in uint32_t v_base, const in uint32_t sv2, const in uint32_t valid) {
+ return r < valid ? data_v[v_base + r * sv2 + c] : 0.0;
+}
+
+// sh_beta[r] == 0 for r >= valid
+float ComputeDeltaV(const in uint32_t r, const in uint32_t c, const in float ks,
+ const in float vv) {
+ return sh_beta[r] * (vv - sh_ep[r] * ks);
+}
+
+float ApplyExpPrefix(const in uint32_t r, const in uint32_t c, const in float qs) {
+ return sh_ep[r] * qs;
+}
+
+float StoreLower(const in uint32_t r, const in uint32_t c, const in float elem,
+ const in uint32_t c_o) {
+ const uint cc = c_o + c;
+ if (cc < r) {
+ sh_lower[r * C + cc] = elem * exp(sh_prefix[r] - sh_prefix[cc]) * sh_beta[r];
+ }
+ return elem;
+}
+
+float StorePmat(const in uint32_t r, const in uint32_t c, const in float elem,
+ const in uint32_t c_o) {
+ const uint cc = c_o + c;
+ float v = 0.0;
+ if (cc <= r) {
+ v = elem * exp(sh_prefix[r] - sh_prefix[cc]);
+ }
+ split(v, sh_ph[r * C + cc], sh_pl[r * C + cc]);
+ return elem;
+}
+
+void prefetch(const uint t0, const uint iq1, const uint iq3, const uint seq_id, const uint head_id,
+ out float q[NQK], out float k[NQK], out float g, out float b) {
+ const uint n = t0 < p.n_tokens ? min(C, p.n_tokens - t0) : 0u;
+ [[unroll]] for (uint j = 0; j < NQK; j++) {
+ const uint i = gl_LocalInvocationID.x + j * WGS;
+ const uint t = i / D;
+ q[j] = 0.0;
+ k[j] = 0.0;
+ if (t < n) {
+ const uint q_off = iq3 * p.sq3 + (t0 + t) * p.sq2 + iq1 * p.sq1 + i % D;
+ const uint k_off = iq3 * p.sk3 + (t0 + t) * p.sk2 + iq1 * p.sk1 + i % D;
+ q[j] = data_q[q_off] * p.scale;
+ k[j] = data_k[k_off];
+ }
+ }
+ g = 0.0;
+ b = 0.0;
+ const uint t = gl_SubgroupInvocationID;
+ if (gl_SubgroupID == 0 && t < n) {
+ const uint gb = seq_id * p.sb3 + (t0 + t) * p.sb2 + head_id * p.sb1;
+ g = data_g[gb];
+ b = data_beta[gb];
+ }
+}
+
+// sg0: gram K@K^T -> L, sg1: pmat Q@K^T -> P
+void gram_pmat(const uint sg) {
+ const bool gram = sg < CT;
+ const uint c_o = (sg % CT) * TN;
+ coopmat<float, SG, C, TN, MACC> x = coopmat<float, SG, C, TN, MACC>(0);
+ [[unroll]] for (uint kk = 0; kk < D; kk += KT) {
+ coopmat<OPD, SG, C, KT, MUA> ah, al;
+ if (gram) {
+ coopMatLoad(ah, sh_kh, kk, LDP, ROW);
+ coopMatLoad(al, sh_kl, kk, LDP, ROW);
+ } else {
+ coopMatLoad(ah, sh_qh, kk, LDP, ROW);
+ coopMatLoad(al, sh_ql, kk, LDP, ROW);
+ }
+ coopmat<OPD, SG, KT, TN, MUB> bh, bl;
+ coopMatLoad(bh, sh_kh, c_o * LDP + kk, LDP, COL);
+ coopMatLoad(bl, sh_kl, c_o * LDP + kk, LDP, COL);
+ PRODUCT(x, ah, al, bh, bl);
+ }
+ if (gram) {
+ coopMatPerElementEXT(x, x, StoreLower, c_o);
+ } else {
+ coopMatPerElementEXT(x, x, StorePmat, c_o);
+ }
+}
+
+// (I + L)^-1, forward substitution: subgroup per row, lane per column
+void invert(const uint sg) {
+ const uint col = gl_SubgroupInvocationID;
+ [[unroll]] for (uint row = sg; row < C; row += NSUB) {
+ float x = col < C ? (col == row ? 1.0 : (col < row ? -sh_lower[row * C + col] : 0.0)) : 0.0;
+ for (int pivot = int(row) - 1; pivot > 0; pivot--) {
+ const float xp = subgroupShuffle(x, uint(pivot));
+ if (col < uint(pivot)) {
+ x -= xp * sh_lower[uint(pivot) * C + col];
+ }
+ }
+ if (col < C) {
+ split(x, sh_ih[row * C + col], sh_il[row * C + col]);
+ }
+ }
+}
+
+float carry_decay(const uint valid) {
+ const float plast = sh_prefix[valid - 1];
+ if (gl_LocalInvocationID.x < C) {
+ sh_carry[gl_LocalInvocationID.x] = exp(plast - sh_prefix[gl_LocalInvocationID.x]);
+ }
+ return exp(plast);
+}
+
+void main() {
+ const uint head_id = gl_WorkGroupID.z;
+ const uint seq_id = gl_WorkGroupID.y;
+ const uint sg = gl_SubgroupID;
+ const uint tid = gl_LocalInvocationID.x;
+ const uint v0 = gl_WorkGroupID.x * V;
+
+ const uint iq1 = head_id % p.neq1;
+ const uint iq3 = seq_id / p.rq3;
+
+ const uint state_base = (seq_id * p.H + head_id) * (D * D);
+
+#ifdef GMEM_MIRROR
+ const uint mir_stride = uint(LDP) * V;
+ const uint mir_block = (head_id * p.n_seqs + seq_id) * gl_NumWorkGroups.x + gl_WorkGroupID.x;
+ const uint mir_h = mir_block * mir_stride * 2u;
+ const uint mir_l = mir_h + mir_stride;
+#endif
+
+ coopmat<float, SG, TM, TN, MACC> state[PER_WARP];
+ [[unroll]] for (uint j = 0; j < PER_WARP; j++) {
+ const uint ti = sg + j * NSUB;
+ coopMatLoad(state[j], data_state, state_base + (v0 + TILE_C(ti)) * D + TILE_R(ti), D, COL);
+ }
+
+ float qreg[NQK], kreg[NQK], greg, breg;
+ prefetch(0u, iq1, iq3, seq_id, head_id, qreg, kreg, greg, breg);
+
+ for (uint t0 = 0; t0 < p.n_tokens; t0 += C) {
+ const uint valid = min(C, p.n_tokens - t0);
+
+ barrier();
+ if (sg == 0) {
+ const uint t = gl_SubgroupInvocationID;
+ const float acc = subgroupInclusiveAdd(greg);
+ if (t < C) {
+ sh_prefix[t] = acc;
+ sh_ep[t] = exp(acc);
+ sh_beta[t] = breg;
+ }
+ }
+ [[unroll]] for (uint j = 0; j < NQK; j++) {
+ const uint i = tid + j * WGS;
+ const uint pi = (i / D) * LDP + (i % D);
+ split(qreg[j], sh_qh[pi], sh_ql[pi]);
+ split(kreg[j], sh_kh[pi], sh_kl[pi]);
+ }
+
+ [[unroll]] for (uint j = 0; j < PER_WARP; j++) {
+ const uint ti = sg + j * NSUB;
+ STORE_MIRROR(state[j], TILE_C(ti) * LDP + TILE_R(ti));
+ }
+ MIRROR_BARRIER();
+
+ prefetch(t0 + C, iq1, iq3, seq_id, head_id, qreg, kreg, greg, breg);
+ const float cdecay = carry_decay(valid);
+
+ // qs -> base_r and ks -> delta on sg2..2+VT-1
+ coopmat<float, SG, C, TN, MACC> base_r = coopmat<float, SG, C, TN, MACC>(0);
+ if (sg < 2 * CT) {
+ gram_pmat(sg);
+ } else if (sg - 2 * CT < VT) {
+ const uint c_o = (sg - 2 * CT) * TN;
+ const uint v_base = seq_id * p.sv3 + head_id * p.sv1 + v0 + c_o + t0 * p.sv2;
+ coopmat<float, SG, C, TN, MACC> vmat = coopmat<float, SG, C, TN, MACC>(0);
+ coopMatPerElementEXT(vmat, vmat, LoadV, v_base, p.sv2, valid);
+ coopmat<float, SG, C, TN, MACC> qs = coopmat<float, SG, C, TN, MACC>(0);
+ coopmat<float, SG, C, TN, MACC> ks = coopmat<float, SG, C, TN, MACC>(0);
+ [[unroll]] for (uint kk = 0; kk < D; kk += KT) {
+ coopmat<OPD, SG, C, KT, MUA> qh, ql, kh, kl;
+ coopMatLoad(qh, sh_qh, kk, LDP, ROW);
+ coopMatLoad(ql, sh_ql, kk, LDP, ROW);
+ coopMatLoad(kh, sh_kh, kk, LDP, ROW);
+ coopMatLoad(kl, sh_kl, kk, LDP, ROW);
+ coopmat<OPD, SG, KT, TN, MUB> sbh, sbl;
+ LOAD_S(sbh, sbl, c_o * LDP + kk);
+ PRODUCT(qs, qh, ql, sbh, sbl);
+ PRODUCT(ks, kh, kl, sbh, sbl);
+ }
+ coopMatPerElementEXT(base_r, qs, ApplyExpPrefix);
+ coopmat<float, SG, C, TN, MACC> delta;
+ coopMatPerElementEXT(delta, ks, ComputeDeltaV, vmat);
+ STORE_DELTA(delta, c_o);
+ }
+ barrier();
+
+ invert(sg);
+ barrier();
+
+ // solved = (I+L)^-1 @ delta (sg0..VT-1)
+ if (sg < VT) {
+ const uint c_o = sg * TN;
+ coopmat<float, SG, C, TN, MACC> solved = coopmat<float, SG, C, TN, MACC>(0);
+ coopmat<OPD, SG, C, C, MUA> ih, il;
+ coopMatLoad(ih, sh_ih, 0, C, ROW);
+ coopMatLoad(il, sh_il, 0, C, ROW);
+ coopmat<OPD, SG, C, TN, MUB> dh, dl;
+ coopMatLoad(dh, sh_dh, c_o, V, ROW);
+ coopMatLoad(dl, sh_dl, c_o, V, ROW);
+ PRODUCT(solved, ih, il, dh, dl);
+ STORE_SOLVED(solved, c_o);
+ }
+ barrier();
+
+ // out = base + P@solved (sg2..2+VT-1)
+ if (sg >= 2 * CT && sg - 2 * CT < VT) {
+ const uint c_o = (sg - 2 * CT) * TN;
+ coopmat<float, SG, C, TN, MACC> O = base_r;
+ coopmat<OPD, SG, C, C, MUA> ph, pl;
+ coopMatLoad(ph, sh_ph, 0, C, ROW);
+ coopMatLoad(pl, sh_pl, 0, C, ROW);
+ coopmat<OPD, SG, C, TN, MUB> svh, svl;
+ coopMatLoad(svh, sh_soh, c_o, V, ROW);
+ coopMatLoad(svl, sh_sol, c_o, V, ROW);
+ PRODUCT(O, ph, pl, svh, svl);
+ const uint out_base = (seq_id * p.n_tokens * p.H + head_id) * D + v0 + c_o + t0 * (D * p.H);
+ coopMatPerElementEXT(O, O, StoreOut, out_base, valid);
+ }
+ // delta2 = solved * carry -> sh_dh/dl (all threads)
+ for (uint i = tid; i < C * V; i += WGS) {
+ const float sv = float(sh_soh[i]) + float(sh_sol[i]);
+ split(sv * sh_carry[i / V], sh_dh[i], sh_dl[i]);
+ }
+ barrier();
+
+ // state = cdecay*state + K^T @ delta2
+ [[unroll]] for (uint j = 0; j < PER_WARP; j++) {
+ const uint ti = sg + j * NSUB;
+ coopmat<OPD, SG, TM, C, MUA> kth, ktl;
+ coopMatLoad(kth, sh_kh, TILE_R(ti), LDP, COL);
+ coopMatLoad(ktl, sh_kl, TILE_R(ti), LDP, COL);
+ coopmat<OPD, SG, C, TN, MUB> dh, dl;
+ coopMatLoad(dh, sh_dh, TILE_C(ti), V, ROW);
+ coopMatLoad(dl, sh_dl, TILE_C(ti), V, ROW);
+ state[j] = state[j] * cdecay;
+ PRODUCT(state[j], kth, ktl, dh, dl);
+ }
+ // no end-of-chunk barrier: covered by the chunk-top barrier
+ }
+
+ [[unroll]] for (uint j = 0; j < PER_WARP; j++) {
+ const uint ti = sg + j * NSUB;
+ coopMatPerElementEXT(state[j], state[j], StoreState,
+ p.s_off + state_base + (v0 + TILE_C(ti)) * D + TILE_R(ti));
+ }
+}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net_cm2.comp
new file mode 100644
index 000000000..270ef35db
--- /dev/null
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net_cm2.comp
@@ -0,0 +1,320 @@
+#version 450
+
+#extension GL_EXT_control_flow_attributes : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
+#extension GL_EXT_bfloat16 : require
+#extension GL_KHR_memory_scope_semantics : require
+#extension GL_KHR_shader_subgroup_basic : require
+#extension GL_KHR_shader_subgroup_arithmetic : require
+#extension GL_KHR_shader_subgroup_shuffle : require
+#extension GL_KHR_shader_subgroup_shuffle_relative : require
+#extension GL_KHR_cooperative_matrix : require
+#extension GL_NV_cooperative_matrix2 : require
+
+// Chunked gated-delta-net prefill (non-KDA, S_v == D == 128, K == 1).
+// BF16 hi/lo products use FP32 accumulators.
+#define OPD bfloat16_t
+
+#define C 16 // chunk size (tokens per step)
+#define D 128 // head dim / value dim
+#define V 32 // value columns per workgroup block
+#define WGS 128
+#define NSUB (WGS / 32)
+#define KT 32 // K-tile width for deep (K=D) matmuls
+#define PAD 8
+#define LDP (D + PAD)
+#define LDV (V + PAD)
+#define KQ_ROWS (2 * C)
+#define NQK (C * D / WGS)
+
+layout(local_size_x = WGS, local_size_y = 1, local_size_z = 1) in;
+
+layout(push_constant) uniform Parameters {
+ uint H;
+ uint n_tokens;
+ uint n_seqs;
+ uint s_off;
+ uint sq1, sq2, sq3;
+ uint sk1, sk2, sk3;
+ uint sv1, sv2, sv3;
+ uint sb1, sb2, sb3;
+ uint neq1, rq3;
+ float scale;
+ uint K;
+} p;
+
+layout(binding = 0) readonly buffer QBuf { float data_q[]; };
+layout(binding = 1) readonly buffer KBuf { float data_k[]; };
+layout(binding = 2) readonly buffer VBuf { float data_v[]; };
+layout(binding = 3) readonly buffer GBuf { float data_g[]; };
+layout(binding = 4) readonly buffer BetaBuf { float data_beta[]; };
+layout(binding = 5) readonly buffer StateBuf { float data_state[]; };
+layout(binding = 6) buffer DstBuf { float data_dst[]; };
+
+// K and scaled Q occupy the two row blocks of one 32-row operand.
+shared OPD sh_kqh[2 * C * LDP], sh_kql[2 * C * LDP];
+shared OPD sh_sh[V * LDP], sh_sl[V * LDP]; // state mirror [V x D] row-major (D padded)
+shared OPD sh_ph[C * C], sh_pl[C * C];
+shared OPD sh_ih[C * C], sh_il[C * C];
+shared OPD sh_dh[C * LDV], sh_dl[C * LDV]; // delta / solved [C x V] row-major (V padded)
+shared float sh_lower[C * (C - 1) / 2]; // packed strictly-lower Gram entries
+shared float sh_prefix[C], sh_ep[C], sh_beta[C], sh_carry[C];
+
+#define WG gl_ScopeWorkgroup
+#define MUA gl_MatrixUseA
+#define MUB gl_MatrixUseB
+#define MACC gl_MatrixUseAccumulator
+#define ROW gl_CooperativeMatrixLayoutRowMajor
+#define COL gl_CooperativeMatrixLayoutColumnMajor
+
+#define PRODUCT(acc, ah, al, bh, bl) \
+ acc = coopMatMulAdd(ah, bh, acc); \
+ acc = coopMatMulAdd(al, bh, acc); \
+ acc = coopMatMulAdd(ah, bl, acc);
+
+void split(const float x, out OPD h, out OPD l) {
+ h = OPD(x);
+ l = OPD(x - float(h));
+}
+
+float StoreGramPmat(const in uint32_t r, const in uint32_t c, const in float elem) {
+ if (r < C) {
+ if (c < r) {
+ sh_lower[r * (r - 1) / 2 + c] = elem * exp(sh_prefix[r] - sh_prefix[c]) * sh_beta[r];
+ }
+ } else {
+ const uint row = r - C;
+ const float y = c <= row ? elem * exp(sh_prefix[row] - sh_prefix[c]) : 0.0;
+ split(y, sh_ph[row * C + c], sh_pl[row * C + c]);
+ }
+ return elem;
+}
+
+float SplitRowS(const in uint32_t r, const in uint32_t c, const in float elem) {
+ const uint i = r * LDP + c;
+ split(elem, sh_sh[i], sh_sl[i]);
+ return elem;
+}
+
+float ApplyExpPrefix(const in uint32_t r, const in uint32_t c, const in float qs) {
+ return sh_ep[c] * qs;
+}
+
+float LoadV(const in uint32_t r, const in uint32_t c, const in float elem,
+ const in uint32_t v_base, const in uint32_t sv2, const in uint32_t valid) {
+ return c < valid ? data_v[v_base + c * sv2 + r] : 0.0;
+}
+
+float StoreDeltaV(const in uint32_t r, const in uint32_t c, const in float ks, const in float vv) {
+ const float x = sh_beta[c] * (vv - sh_ep[c] * ks);
+ const uint i = c * LDV + r;
+ split(x, sh_dh[i], sh_dl[i]);
+ return ks;
+}
+
+float StoreSolved(const in uint32_t r, const in uint32_t c, const in float elem) {
+ const uint i = c * LDV + r;
+ split(elem, sh_dh[i], sh_dl[i]);
+ // The state mirror is no longer needed after KS/QS.
+ const uint j = r * LDP + c;
+ split(sh_carry[c] * elem, sh_sh[j], sh_sl[j]);
+ return elem;
+}
+
+float StoreOut(const in uint32_t r, const in uint32_t c, const in float elem,
+ const in uint32_t out_base, const in uint32_t valid) {
+ if (c < valid) {
+ data_dst[out_base + c * (p.H * D) + r] = elem;
+ }
+ return elem;
+}
+
+void prefetch(const uint t0, const uint q_base, const uint k_base, const uint gb_base,
+ out float q[NQK], out float k[NQK], out float g, out float b) {
+ const uint valid = t0 < p.n_tokens ? min(C, p.n_tokens - t0) : 0u;
+ const uint r = gl_SubgroupInvocationID;
+ g = 0.0;
+ b = 0.0;
+ if (gl_SubgroupID == 0 && r < valid) {
+ const uint off = gb_base + (t0 + r) * p.sb2;
+ g = data_g[off];
+ b = data_beta[off];
+ }
+ [[unroll]] for (uint j = 0; j < NQK; ++j) {
+ const uint i = gl_LocalInvocationID.x + j * WGS;
+ const uint token = i / D;
+ q[j] = 0.0;
+ k[j] = 0.0;
+ if (token < valid) {
+ const uint q_off = q_base + (t0 + token) * p.sq2 + i % D;
+ const uint k_off = k_base + (t0 + token) * p.sk2 + i % D;
+ q[j] = data_q[q_off];
+ k[j] = data_k[k_off];
+ }
+ }
+}
+
+void stage_inputs(const in float q[NQK], const in float k[NQK]) {
+ [[unroll]] for (uint j = 0; j < NQK; ++j) {
+ const uint i = gl_LocalInvocationID.x + j * WGS;
+ const uint token = i / D;
+ const uint key = i % D;
+ const uint ki = token * LDP + key;
+ const uint qi = (C + token) * LDP + key;
+ split(k[j], sh_kqh[ki], sh_kql[ki]);
+ split(p.scale * q[j], sh_kqh[qi], sh_kql[qi]);
+ }
+}
+
+// [K; Q] * K^T computes Gram and P in one 32-row matrix.
+void gram_pmat() {
+ coopmat<float, WG, KQ_ROWS, C, MACC> x = coopmat<float, WG, KQ_ROWS, C, MACC>(0.0);
+ [[unroll]] for (uint kk = 0; kk < D; kk += KT) {
+ coopmat<OPD, WG, KQ_ROWS, KT, MUA> ah, al;
+ coopmat<OPD, WG, KT, C, MUB> bh, bl;
+ coopMatLoad(ah, sh_kqh, kk, LDP, ROW);
+ coopMatLoad(al, sh_kql, kk, LDP, ROW);
+ coopMatLoad(bh, sh_kqh, kk, LDP, COL);
+ coopMatLoad(bl, sh_kql, kk, LDP, COL);
+ PRODUCT(x, ah, al, bh, bl);
+ }
+ coopMatPerElementNV(x, x, StoreGramPmat);
+}
+
+void invert() {
+ const uint col = gl_SubgroupInvocationID % C;
+ // Each half-subgroup covers all columns of its assigned rows.
+ const uint section = gl_SubgroupInvocationID / C;
+ const uint row_base = (gl_SubgroupID * 2u + section) * (C / (2 * NSUB));
+ // Keep the diagonal zero so pivots at or above the row do nothing.
+ float x[C / (2 * NSUB)];
+ [[unroll]] for (uint j = 0; j < C / (2 * NSUB); ++j) {
+ const uint row = row_base + j;
+ x[j] = col < row ? -sh_lower[row * (row - 1) / 2 + col] : 0.0;
+ }
+ [[dont_unroll]] for (int pivot = C - 2; pivot > 0; --pivot) {
+ const float lower = col < uint(pivot) ? sh_lower[uint(pivot) * uint(pivot - 1) / 2 + col] : 0.0;
+ [[unroll]] for (uint j = 0; j < C / (2 * NSUB); ++j) {
+ const float xp = subgroupShuffle(x[j], uint(pivot) + section * C);
+ x[j] -= xp * lower;
+ }
+ }
+ [[unroll]] for (uint j = 0; j < C / (2 * NSUB); ++j) {
+ const uint row = row_base + j;
+ split(col == row ? 1.0 : x[j], sh_ih[row * C + col], sh_il[row * C + col]);
+ }
+}
+
+void main() {
+ const uint head_id = gl_WorkGroupID.z;
+ const uint seq_id = gl_WorkGroupID.y;
+ const uint v0 = gl_WorkGroupID.x * V;
+ const uint iq1 = head_id % p.neq1;
+ const uint iq3 = seq_id / p.rq3;
+ const uint q_base = iq3 * p.sq3 + iq1 * p.sq1;
+ const uint k_base = iq3 * p.sk3 + iq1 * p.sk1;
+ const uint v_base = seq_id * p.sv3 + head_id * p.sv1 + v0;
+ const uint gb_base = seq_id * p.sb3 + head_id * p.sb1;
+ const uint state_base = (seq_id * p.H + head_id) * (D * D) + v0 * D;
+ const uint out_base = (seq_id * p.n_tokens * p.H + head_id) * D + v0;
+
+ // Transposing the state-side products gives them V=32 rows with C=16 columns.
+ coopmat<float, WG, V, D, MACC> state;
+ coopMatLoad(state, data_state, state_base, D, ROW);
+
+ float qreg[NQK], kreg[NQK], greg, breg;
+ prefetch(0u, q_base, k_base, gb_base, qreg, kreg, greg, breg);
+
+ [[dont_unroll]] for (uint t0 = 0; t0 < p.n_tokens; t0 += C) {
+ const uint valid = min(C, p.n_tokens - t0);
+ barrier();
+ if (gl_SubgroupID == 0) {
+ const uint r = gl_SubgroupInvocationID;
+ // shuffle scan instead of subgroupInclusiveAdd: works around an NVIDIA r620 shader compiler crash
+ float sum = greg;
+ [[unroll]] for (uint d = 1; d < C; d <<= 1) {
+ const float t = subgroupShuffleUp(sum, d);
+ if (r >= d) {
+ sum += t;
+ }
+ }
+ const float last = subgroupShuffle(sum, valid - 1);
+ if (r < C) {
+ sh_prefix[r] = sum;
+ sh_ep[r] = exp(sum);
+ sh_beta[r] = breg;
+ sh_carry[r] = exp(last - sum);
+ }
+ }
+ stage_inputs(qreg, kreg);
+ coopmat<float, WG, V, C, MACC> vmat;
+ if (valid == C && (v_base % 4u) == 0 && (p.sv2 % 4u) == 0) {
+ coopMatLoad(vmat, data_v, v_base + t0 * p.sv2, p.sv2, COL);
+ } else {
+ vmat = coopmat<float, WG, V, C, MACC>(0.0);
+ coopMatPerElementNV(vmat, vmat, LoadV, v_base + t0 * p.sv2, p.sv2, valid);
+ }
+ barrier();
+ // Current inputs are in shared memory; keep the next chunk in registers.
+ prefetch(t0 + C, q_base, k_base, gb_base, qreg, kreg, greg, breg);
+
+ gram_pmat();
+ barrier();
+ invert();
+ coopMatPerElementNV(state, state, SplitRowS);
+ barrier();
+
+ coopmat<float, WG, V, C, MACC> ks = coopmat<float, WG, V, C, MACC>(0.0), base_r = coopmat<float, WG, V, C, MACC>(0.0);
+ [[unroll]] for (uint kk = 0; kk < D; kk += KT) {
+ coopmat<OPD, WG, V, KT, MUA> ah, al;
+ coopmat<OPD, WG, KT, C, MUB> kh, kl, qh, ql;
+ coopMatLoad(ah, sh_sh, kk, LDP, ROW);
+ coopMatLoad(al, sh_sl, kk, LDP, ROW);
+ coopMatLoad(kh, sh_kqh, kk, LDP, COL);
+ coopMatLoad(kl, sh_kql, kk, LDP, COL);
+ coopMatLoad(qh, sh_kqh, C * LDP + kk, LDP, COL);
+ coopMatLoad(ql, sh_kql, C * LDP + kk, LDP, COL);
+ PRODUCT(ks, ah, al, kh, kl);
+ PRODUCT(base_r, ah, al, qh, ql);
+ }
+ coopMatPerElementNV(base_r, base_r, ApplyExpPrefix);
+ coopMatPerElementNV(ks, ks, StoreDeltaV, vmat);
+ barrier();
+
+ coopmat<float, WG, V, C, MACC> solved = coopmat<float, WG, V, C, MACC>(0.0);
+ {
+ coopmat<OPD, WG, V, C, MUA> rh, rl;
+ coopmat<OPD, WG, C, C, MUB> ih, il;
+ coopMatLoad(rh, sh_dh, 0, LDV, COL);
+ coopMatLoad(rl, sh_dl, 0, LDV, COL);
+ coopMatLoad(ih, sh_ih, 0, C, COL);
+ coopMatLoad(il, sh_il, 0, C, COL);
+ PRODUCT(solved, rh, rl, ih, il);
+ }
+ barrier();
+ coopMatPerElementNV(solved, solved, StoreSolved);
+ barrier();
+
+ {
+ coopmat<OPD, WG, V, C, MUA> xh, xl;
+ coopmat<OPD, WG, C, C, MUB> ph, pl;
+ coopMatLoad(xh, sh_dh, 0, LDV, COL);
+ coopMatLoad(xl, sh_dl, 0, LDV, COL);
+ coopMatLoad(ph, sh_ph, 0, C, COL);
+ coopMatLoad(pl, sh_pl, 0, C, COL);
+ PRODUCT(base_r, xh, xl, ph, pl);
+ }
+ coopMatPerElementNV(base_r, base_r, StoreOut, out_base + t0 * (p.H * D), valid);
+
+ coopmat<OPD, WG, V, C, MUA> xh, xl;
+ coopmat<OPD, WG, C, D, MUB> kh, kl;
+ coopMatLoad(xh, sh_sh, 0, LDP, ROW);
+ coopMatLoad(xl, sh_sl, 0, LDP, ROW);
+ coopMatLoad(kh, sh_kqh, 0, LDP, ROW);
+ coopMatLoad(kl, sh_kql, 0, LDP, ROW);
+ state *= sh_ep[valid - 1];
+ PRODUCT(state, xh, xl, kh, kl);
+ }
+
+ coopMatStore(state, data_dst, p.s_off + state_base, D, ROW);
+}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
index e988600e3..e20c94758 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
@@ -348,7 +348,10 @@ compile_count_guard acquire_compile_slot() {
}
void string_to_spv_func(std::string name, std::string in_path, std::string out_path, std::map<std::string, std::string> defines, bool coopmat, bool dep_file, compile_count_guard slot) {
- std::string target_env = (name.find("_cm2") != std::string::npos) ? "--target-env=vulkan1.3" : "--target-env=vulkan1.2";
+ // coopmat2 and the maintenance1 GDN coopmat1 variants need SPIR-V 1.6 (vulkan1.3)
+ const bool needs_spv16 = name.find("_cm2") != std::string::npos ||
+ name.find("gated_delta_net_f32_cm1") != std::string::npos;
+ std::string target_env = needs_spv16 ? "--target-env=vulkan1.3" : "--target-env=vulkan1.2";
#ifdef _WIN32
std::vector<std::string> cmd = {GLSLC, "-fshader-stage=compute", target_env, "\"" + in_path + "\"", "-o", "\"" + out_path + "\""};
@@ -1128,6 +1131,20 @@ void process_shaders() {
string_to_spv("gated_delta_net_f32_nocluster", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}, {"USE_SUBGROUP_CLUSTERED", "0"}}));
string_to_spv("gated_delta_net_f32_shmem", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "0"}, {"USE_SUBGROUP_CLUSTERED", "0"}}));
+#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)
+ string_to_spv("gated_delta_net_f32", "gated_delta_net_cm2.comp", base_dict, true, false, true);
+#endif
+#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
+ string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"BF16ACC", "1"}}), true, true, false, false, "_bf16acc");
+ string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", base_dict, true, true);
+ string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"BF16ACC", "1"}, {"V", "32"}}), true, true, false, false, "_bf16acc_v32");
+ string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"V", "32"}}), true, true, false, false, "_v32");
+ // whole-head V=128 variant: state mirror in a gmem scratch buffer (won't fit LDS).
+ // For head counts that tail the CU grid at V=64 (e.g. 48-head 27B on 64 CUs).
+ string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"BF16ACC", "1"}, {"GMEM_MIRROR", "1"}, {"V", "128"}}), true, true, false, false, "_bf16acc_v128");
+ string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"GMEM_MIRROR", "1"}, {"V", "128"}}), true, true, false, false, "_v128");
+#endif
+
string_to_spv("opt_step_adamw_f32", "opt_step_adamw.comp", merge_maps(base_dict, {{"A_TYPE", "float"}}));
string_to_spv("opt_step_sgd_f32", "opt_step_sgd.comp", merge_maps(base_dict, {{"A_TYPE", "float"}}));
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index a870016b0..f75d93bc3 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -5261,16 +5261,20 @@ struct test_gated_delta_net : public test_case {
const bool permuted;
const bool kda;
const int64_t K; // snapshot slot count: 1 = final-only, >1 = last K states
+ const bool correlated;
+ const bool padded_v;
+ const bool padded_q;
+ const bool permuted_k;
std::string vars() override {
- return VARS_TO_STR9(type, head_count, head_size, n_seq_tokens, n_seqs, v_repeat, permuted, kda, K);
+ return VARS_TO_STR13(type, head_count, head_size, n_seq_tokens, n_seqs, v_repeat, permuted, kda, K, correlated, padded_v, padded_q, permuted_k);
}
test_gated_delta_net(ggml_type type = GGML_TYPE_F32,
int64_t head_count = 4, int64_t head_size = 16, int64_t n_seq_tokens = 1, int64_t n_seqs = 1,
- int v_repeat = 1, bool permuted = false, bool kda = false, int64_t K = 1)
+ int v_repeat = 1, bool permuted = false, bool kda = false, int64_t K = 1, bool correlated = false, bool padded_v = false, bool padded_q = false, bool permuted_k = false)
: type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs),
- v_repeat(v_repeat), permuted(permuted), kda(kda), K(K) {}
+ v_repeat(v_repeat), permuted(permuted), kda(kda), K(K), correlated(correlated), padded_v(padded_v), padded_q(padded_q), permuted_k(permuted_k) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
ggml_tensor * q;
@@ -5280,11 +5284,15 @@ struct test_gated_delta_net : public test_case {
// create with dims 1 and 2 swapped, then permute back to get non-contiguous layout
q = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs), 0, 2, 1, 3);
k = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs), 0, 2, 1, 3);
- v = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count * v_repeat, n_seqs), 0, 2, 1, 3);
+ v = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size + padded_v, n_seq_tokens, head_count * v_repeat, n_seqs), 0, 2, 1, 3);
} else {
q = ggml_new_tensor_4d(ctx, type, head_size, head_count, n_seq_tokens, n_seqs);
k = ggml_new_tensor_4d(ctx, type, head_size, head_count, n_seq_tokens, n_seqs);
- v = ggml_new_tensor_4d(ctx, type, head_size, head_count * v_repeat, n_seq_tokens, n_seqs);
+ v = ggml_new_tensor_4d(ctx, type, head_size + padded_v, head_count * v_repeat, n_seq_tokens, n_seqs);
+ }
+ if (padded_v) {
+ ggml_set_name(v, "v");
+ v = ggml_view_4d(ctx, v, head_size, head_count * v_repeat, n_seq_tokens, n_seqs, v->nb[1], v->nb[2], v->nb[3], 0);
}
ggml_set_name(q, "q");
ggml_set_name(k, "k");
@@ -5299,6 +5307,17 @@ struct test_gated_delta_net : public test_case {
// q/k are L2-normalised in qwen35/kimi-linear before delta_net
q = ggml_l2_norm(ctx, q, 1e-6f);
k = ggml_l2_norm(ctx, k, 1e-6f);
+ // Set layouts after normalization, which produces contiguous tensors.
+ if (padded_q) {
+ ggml_tensor * q_padded = ggml_new_tensor_4d(ctx, type, head_size + 1, head_count, n_seq_tokens, n_seqs);
+ q_padded = ggml_view_4d(ctx, q_padded, head_size, head_count, n_seq_tokens, n_seqs, q_padded->nb[1], q_padded->nb[2], q_padded->nb[3], 0);
+ q = ggml_cpy(ctx, q, q_padded);
+ }
+ if (permuted_k) {
+ ggml_tensor * k_permuted = ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs);
+ k_permuted = ggml_permute(ctx, k_permuted, 0, 2, 1, 3);
+ k = ggml_cpy(ctx, k, k_permuted);
+ }
ggml_tensor * out = ggml_gated_delta_net(ctx, q, k, v, g, beta, state, K);
return out;
}
@@ -5307,9 +5326,17 @@ struct test_gated_delta_net : public test_case {
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
if (ggml_is_view_op(t->op)) { continue; }
if (strcmp(t->name, "g") == 0) {
- init_tensor_uniform(t, -20.0f, -1e-4f);
+ init_tensor_uniform(t, correlated ? -1e-4f : -20.0f, -1e-4f);
} else if (strcmp(t->name, "beta") == 0) {
- init_tensor_uniform(t, 0.0f, 1.0f);
+ init_tensor_uniform(t, correlated ? 0.9f : 0.0f, correlated ? 0.9f : 1.0f);
+ } else if (correlated && strcmp(t->name, "k") == 0) {
+ // Identical unit keys and weak decay stress cancellation in the triangular solve.
+ GGML_ASSERT(t->type == GGML_TYPE_F32);
+ std::vector<float> data(ggml_nelements(t), 0.0f);
+ for (size_t i = 0; i < data.size(); i += size_t(t->ne[0])) {
+ data[i] = 1.0f;
+ }
+ ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(float));
} else if (strcmp(t->name, "v") == 0) {
init_tensor_uniform(t, -0.3f, 5.0f);
} else {
@@ -13634,6 +13661,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 33, 1, 1, false, true));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 100, 1, 1, false, true));
+ // KDA at D=128: decode, prefill, tails, strides, and snapshots.
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1, 1, false, true));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1, 1, false, true));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 65, 2, 2, true, true));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 257, 1, 1, false, true));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 65, 1, 1, false, true, 1, true));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 257, 1, 1, true, true, 1, true, true));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4, 2, 2, false, true, 4));
+
// MMA-eligible GDN shapes: whole heads, value slices, tails, and the HIP prefill threshold.
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 65, 1, 1));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 65, 3, 4));
@@ -13645,6 +13681,30 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2048, 1, 2));
// Keep the WebGPU dispatch below 65535 workgroups (32 per token for this shape).
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2047, 2, 3));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4097, 1, 2));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 127, 2));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1, 2));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1, 3));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 513, 1, 2));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 512, 2, 2));
+
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 257, 2, 2, true));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 3, 128, 129, 1, 1, false, false, 1, false, true));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4095, 1, 2));
+
+ // Correlated keys, high beta, and weak decay stress the chunked triangular solve.
+ for (int64_t n_tokens : {64, 65, 257}) {
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, n_tokens, 1, 1, false, false, 1, true));
+ }
+
+ // Odd V strides require element loads instead of aligned matrix loads.
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 3, 128, 65, 1, 1, false, false, 1, false, true));
+
+ // Q and K can have different strides in both scalar and chunked paths.
+ for (int64_t n_tokens : {4, 65}) {
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 2, 128, n_tokens, 2, 2, false, false, 1, false, false, true));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 2, 128, n_tokens, 2, 2, false, false, 1, false, false, false, true));
+ }
// K > 1: output keeps the last min(n_tokens, K) per-token snapshots, ordered most-recent-first
// (slot 0 = final state, slot s = state s tokens back).
@@ -14296,12 +14356,34 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 256, 1)); // PP-256
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 512, 1)); // PP-512
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1024, 1)); // PP-1024
+ // MMA-eligible prefill sweep (16 heads, d=128, K=1): Vulkan cm2 vs CUDA MMA
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 64, 1));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 128, 1));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 512, 1));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 1024, 1));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2048, 1));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 4096, 1));
// Small model configs (fewer heads = less GPU occupancy for autoregressive)
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1)); // 4h PP-64
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 256, 1)); // 4h PP-256
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 512, 1)); // 4h PP-512
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1024, 1)); // 4h PP-1024
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 64, 1, 1, false, true)); // KDA PP-64
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1, 1, false, true));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1, 1, false, true));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1024, 1, 1, false, true));
+ // GDN model shapes: 35B (v_repeat=2), 27B (v_repeat=3), batch sweep
+ for (int T : { 2048, 4096, 8192, 16384, 32768 }) {
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, T, 1, 2));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, T, 1, 3));
+ }
+
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 2048, 1));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4096, 1));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 8192, 1));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 20, 128, 1536, 1));
+ test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 20, 128, 2048, 1));
// lightning_indexer
for (int kv : { 256, 4096, 65536 }) {