Commit b1c11aa7 for whisper.cpp
commit b1c11aa7f8549c2933ab17f95cdcb7503775ea78
Author: Jeff Bolz <jbolz@nvidia.com>
Date: Wed Nov 11 04:08:06 2026 -0600
vulkan: allow dequantized KV optimization with CM2 (llama/31526)
* additional FA test cases
Assisted-by: Codex
* vulkan: allow dequantized KV optimization with CM2
Enable the KV dequantization optimization with coopmat2, guarded by a heuristic
based on query tokens*GQA ratio.
While doing this we saw very large speedups for certain head sizes, and it was
because those head sizes were incorrectly treated as unaligned. So also fix
those to be treated as aligned, which should help for when the dequant
optimization doesn't apply.
* gqa uses nbq2 for the stride, so we need to check that value for alignment
* tests : use VARS_TO_STR18 for flash attention
Assisted-by: Codex
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index 749f57e7..502f7334 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -8290,6 +8290,10 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
const size_t kv_scratch_ts = kv_raw_strided ? ggml_type_size(k->type) : sizeof(ggml_fp16_t);
// K and V share one scratch buffer, so the sum must fit in one allocation
const uint64_t dequant_kv_sz = ((uint64_t)ggml_nelements(k) + (uint64_t)ggml_nelements(v)) * kv_scratch_ts;
+ const uint32_t cm2_min_queries = KV >= 8192 ? 512 : 1024;
+ const uint32_t cm2_min_reuse = KV >= 8192 ? 4096 : 8192;
+ // Query reuse must cover the conversion and synchronization cost.
+ const bool cm2_dequant_kv = KV >= 4096 && neq1 >= cm2_min_queries && (uint64_t) neq1 * qk_ratio >= cm2_min_reuse;
const bool use_dequant_kv = ((k_quant && v_quant) || kv_raw_strided) && neq1 >= 64 &&
is_dense_kv_cache(k) && is_dense_kv_cache(v) &&
(uint64_t)ggml_nelements(k) * kv_scratch_ts <= ctx->device->properties.limits.maxStorageBufferRange &&
@@ -8299,11 +8303,10 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
(kv_raw_strided ||
(ctx->device->pipeline_dequant_transpose[k->type] != nullptr &&
ctx->device->pipeline_dequant_transpose[v->type] != nullptr)) &&
- // coopmat2 path does not benefit from the f16 scratch
- !ctx->device->coopmat2 &&
// Intel Xe1 regresses, see PR 25494
(ctx->device->vendor_id != VK_VENDOR_ID_INTEL ||
- (ctx->device->coopmat_support && ctx->device->architecture != vk_device_architecture::INTEL_XE1));
+ (ctx->device->coopmat_support && ctx->device->architecture != vk_device_architecture::INTEL_XE1)) &&
+ (!ctx->device->coopmat2 || cm2_dequant_kv);
const ggml_type k_type_eff = (use_dequant_kv && !kv_raw_strided) ? GGML_TYPE_F16 : k->type;
const ggml_type v_type_eff = (use_dequant_kv && !kv_raw_strided) ? GGML_TYPE_F16 : v->type;
@@ -8356,6 +8359,7 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
(gqa_ratio > 1 || (tuning_params.path == FA_SCALAR && N == 1));
const uint32_t q_stride = (uint32_t)(nbq1 / ggml_type_size(q->type));
+ const uint32_t q_stride_eff = gqa_ratio > 1 ? (uint32_t)(nbq2 / ggml_type_size(q->type)) : q_stride;
uint32_t k_stride = (uint32_t)(nbk1 / ggml_type_size(k->type));
uint32_t v_stride = (uint32_t)(nbv1 / ggml_type_size(v->type));
@@ -8378,9 +8382,13 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
nbv3_eff = (uint32_t)((uint64_t)HSV * KV * nev2 * sizeof(ggml_fp16_t));
}
const uint32_t alignment = tuning_params.block_cols;
- bool aligned = (KV % alignment) == 0 &&
- // the "aligned" shader variant will forcibly align strides, for performance
- (q_stride & 7) == 0 && (k_stride & 7) == 0 && (v_stride & 7) == 0;
+ // CM2 decoded K/V strides count whole blocks and do not need to be multiples of eight blocks.
+ const bool cm2_decode_k = tuning_params.path == FA_COOPMAT2 &&
+ k_type_eff != GGML_TYPE_F16 && k_type_eff != GGML_TYPE_BF16;
+ const bool cm2_decode_v = tuning_params.path == FA_COOPMAT2 &&
+ v_type_eff != GGML_TYPE_F16 && v_type_eff != GGML_TYPE_BF16;
+ bool aligned = (KV % alignment) == 0 && (q_stride_eff & 7) == 0 &&
+ (cm2_decode_k || (k_stride & 7) == 0) && (cm2_decode_v || (v_stride & 7) == 0);
// Need to use the coopmat2 variant that clamps loads when HSK/HSV aren't sufficiently aligned.
if (((HSK | HSV) % 16) != 0 && tuning_params.path == FA_COOPMAT2) {