Commit ae4893a09 for llama.cpp
commit ae4893a092e060cfbd4979600bf3c1e48a710874
Author: Ruben Ortlam <rortlam@redhat.com>
Date: Tue Nov 10 08:41:45 2026 +0100
vulkan: add tiled-transpose concat shader for Qwen models (#30591)
* vulkan: tiled transpose for dim-0 concat with a transposed source
The delta-net conv-state path does ggml_transpose() straight into a dim-0
ggml_concat(), so the generic concat kernel reads src1 with the transposed
stride, one element per thread, and neighbouring lanes land on different cache
lines. Stage a 32x32 tile through shared memory instead, which keeps both the
load and the store coalesced. 33 columns pads away the bank conflicts.
Gated narrowly: dim 0, 2D, unquantized 4-byte elements, contiguous src0/dst
rows, and src1 actually transposed. Everything else falls through to the
existing kernel.
Qwen3.6-35B-A3B on gfx1151/RADV, 30 gated-delta-net layers: the CONCAT op goes
11877 us -> 957 us, and pp2048 at ub2048 goes 1224 -> 1628 t/s. Generated text
is unchanged.
The approach follows the tiled transposed concat Gaetan Puleo wrote for the HIP
backend in his Strix Halo fork.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
* enable by default, add smaller shader sizes
* clean up comments
* improve shader
* Add non-contiguous concat tests
Co-authored-by: ravel7524 <58877666+ravel7524@users.noreply.github.com>
* handle x or y axis overflow
---------
Co-authored-by: Nathan Wilson <nath.flagman151@passmail.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: ravel7524 <58877666+ravel7524@users.noreply.github.com>
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
index 5b30f0267..786f8f019 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
@@ -837,6 +837,7 @@ struct vk_device_struct {
vk_pipeline pipeline_add_id_f32;
vk_pipeline pipeline_concat_i8, pipeline_concat_i16, pipeline_concat_i32, pipeline_concat_i64;
+ vk_pipeline pipeline_concat_transpose_i8, pipeline_concat_transpose_i16, pipeline_concat_transpose_i32;
vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32;
vk_pipeline pipeline_scale_f32;
vk_pipeline pipeline_log[2];
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index 3efbbc38b..2e8f0d2ec 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -3416,6 +3416,9 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
ggml_vk_create_pipeline(device, device->pipeline_concat_i8, "concat_i8", concat_i8_len, concat_i8_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_concat_i16, "concat_i16", concat_i16_len, concat_i16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_concat_i32, "concat_i32", concat_i32_len, concat_i32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1);
+ ggml_vk_create_pipeline(device, device->pipeline_concat_transpose_i8, "concat_transpose_i8", concat_transpose_i8_len, concat_transpose_i8_data, "main", 3, sizeof(vk_op_binary_push_constants), {32, 32, 1}, {}, 1);
+ ggml_vk_create_pipeline(device, device->pipeline_concat_transpose_i16, "concat_transpose_i16", concat_transpose_i16_len, concat_transpose_i16_data, "main", 3, sizeof(vk_op_binary_push_constants), {32, 32, 1}, {}, 1);
+ ggml_vk_create_pipeline(device, device->pipeline_concat_transpose_i32, "concat_transpose_i32", concat_transpose_i32_len, concat_transpose_i32_data, "main", 3, sizeof(vk_op_binary_push_constants), {32, 32, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_concat_i64, "concat_i64", concat_i64_len, concat_i64_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_upscale_nearest_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_NEAREST}, 1);
@@ -8853,6 +8856,26 @@ static vk_pipeline ggml_vk_rms_norm_pipeline(const std::array<vk_pipeline, 3> &
return pipelines[2];
}
+static bool ggml_vk_concat_is_transposed(const vk_device & device, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) {
+ if (ggml_get_op_params_i32(dst, 0) != 0) { // dim 0 only
+ return false;
+ }
+ if (src0->ne[2] != 1 || src0->ne[3] != 1 || src1->ne[2] != 1 || src1->ne[3] != 1) {
+ return false;
+ }
+ const size_t ts = ggml_type_size(src0->type);
+ if (src0->nb[0] != ts || dst->nb[0] != ts) { // src0 and dst rows must be contiguous
+ return false;
+ }
+ if (src1->nb[0] <= src1->nb[1]) { // src1 must actually be transposed
+ return false;
+ }
+ if (CEIL_DIV(src1->ne[1], 32) > device->properties.limits.maxComputeWorkGroupCount[0]) {
+ return false;
+ }
+ return src0->ne[1] == src1->ne[1] && dst->ne[1] == src1->ne[1];
+}
+
static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * dst, ggml_op op) {
switch (op) {
case GGML_OP_GET_ROWS:
@@ -8945,6 +8968,18 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
if (!ggml_vk_concat_supported(src0, src1, dst)) {
return nullptr;
}
+ if (!ggml_is_quantized(src0->type) && ggml_vk_concat_is_transposed(ctx->device, src0, src1, dst)) {
+ switch (ggml_vk_concat_unit_size(src0->type)) {
+ case 1:
+ return ctx->device->pipeline_concat_transpose_i8;
+ case 2:
+ return ctx->device->pipeline_concat_transpose_i16;
+ case 4:
+ return ctx->device->pipeline_concat_transpose_i32;
+ default:
+ break;
+ }
+ }
switch (ggml_vk_concat_unit_size(src0->type)) {
case 1:
return ctx->device->pipeline_concat_i8;
@@ -9882,6 +9917,16 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
case GGML_OP_GLU:
case GGML_OP_CONV_2D_DW:
{
+ // dispatched per 32x32 tile, not per element
+ if (op == GGML_OP_CONCAT &&
+ (pipeline == ctx->device->pipeline_concat_transpose_i8 ||
+ pipeline == ctx->device->pipeline_concat_transpose_i16 ||
+ pipeline == ctx->device->pipeline_concat_transpose_i32)) {
+ const uint32_t tiles_y = CEIL_DIV(dst->ne[0], 32);
+ const uint32_t groups_y = std::min(tiles_y, ctx->device->properties.limits.maxComputeWorkGroupCount[1]);
+ elements = { (uint32_t)src1->ne[1], groups_y * 32, CEIL_DIV(tiles_y, groups_y) };
+ break;
+ }
uint32_t ne = ggml_nelements(dst);
if (op == GGML_OP_CPY && ggml_is_quantized(src0->type) && ggml_is_quantized(dst->type)) {
// Convert from number of logical elements to 2- or 4-byte units.
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/concat_transpose.comp b/ggml/src/ggml-vulkan/vulkan-shaders/concat_transpose.comp
new file mode 100644
index 000000000..e598ad818
--- /dev/null
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/concat_transpose.comp
@@ -0,0 +1,39 @@
+#version 450
+
+#include "types.glsl"
+#include "generic_binary_head.glsl"
+
+#define TILE_DIM 32
+layout(local_size_x = TILE_DIM, local_size_y = 8, local_size_z = 1) in;
+
+// 33 columns (not 32) pads away shared-memory bank conflicts.
+shared A_TYPE tile[TILE_DIM][TILE_DIM + 1];
+
+void main() {
+ const uint tid_x = gl_LocalInvocationID.x;
+ const uint tid_y = gl_LocalInvocationID.y;
+
+ // workgroup x walks rows (src1's contiguous dim1) so consecutive workgroups stream src1, y walks dst columns
+ const uint row0 = gl_WorkGroupID.x * TILE_DIM;
+ const uint col0 = (gl_WorkGroupID.z * gl_NumWorkGroups.y + gl_WorkGroupID.y) * TILE_DIM;
+
+ const uint i1 = row0 + tid_x;
+ [[unroll]] for (uint j = 0; j < TILE_DIM; j += gl_WorkGroupSize.y) {
+ const uint i0 = col0 + tid_y + j;
+ if (i0 >= p.ne00 && i0 - p.ne00 < p.ne10 && i1 < p.ne11) {
+ tile[tid_y + j][tid_x] = A_TYPE(data_b[get_boffset() + (i0 - p.ne00)*p.nb10 + i1*p.nb11]);
+ }
+ }
+
+ barrier();
+
+ const uint d0 = col0 + tid_x;
+ [[unroll]] for (uint j = 0; j < TILE_DIM; j += gl_WorkGroupSize.y) {
+ const uint d1 = row0 + tid_y + j;
+ if (d0 < p.ne20 && d1 < p.ne21) {
+ data_d[get_doffset() + d1*p.nb21 + d0*p.nb20] = d0 < p.ne00
+ ? D_TYPE(data_a[get_aoffset() + d1*p.nb01 + d0*p.nb00])
+ : D_TYPE(tile[tid_x][tid_y + j]);
+ }
+ }
+}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
index dba9d8ca8..c1696e4cd 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
@@ -969,6 +969,9 @@ void process_shaders() {
string_to_spv("concat_i16", "concat.comp", {{"A_TYPE", "uint16_t"}, {"B_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}});
string_to_spv("concat_i32", "concat.comp", {{"A_TYPE", "uint"}, {"B_TYPE", "uint"}, {"D_TYPE", "uint"}});
string_to_spv("concat_i64", "concat.comp", {{"A_TYPE", "uvec2"}, {"B_TYPE", "uvec2"}, {"D_TYPE", "uvec2"}});
+ string_to_spv("concat_transpose_i8", "concat_transpose.comp", {{"A_TYPE", "uint8_t"}, {"B_TYPE", "uint8_t"}, {"D_TYPE", "uint8_t"}});
+ string_to_spv("concat_transpose_i16", "concat_transpose.comp", {{"A_TYPE", "uint16_t"}, {"B_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}});
+ string_to_spv("concat_transpose_i32", "concat_transpose.comp", {{"A_TYPE", "uint"}, {"B_TYPE", "uint"}, {"D_TYPE", "uint"}});
string_to_spv("upscale_f32", "upscale.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}});
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index ddbbae9dc..840bfecd5 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -12413,6 +12413,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_concat(GGML_TYPE_Q8_0, {8192, 2, 1, 1}, dim == 0 ? 4096 : 1, dim, 0));
}
+ // GDN conv_input
+ for (int64_t ne_b_d : { 31, 32, 33, 64 }) {
+ test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {3, 10240, 1, 1}, ne_b_d, 0, 16));
+ test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {1, 37, 1, 1}, ne_b_d, 0, 16));
+ }
+ test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {64, 37, 1, 1}, 32, 0, 16));
+ // Empty src0
+ test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {0, 64, 1, 1}, 33, 0, 16));
+
for (ggml_sort_order order : {GGML_SORT_ORDER_ASC, GGML_SORT_ORDER_DESC}) {
for (uint32_t i = 4; i <= 1024*1024; i *= 2) {
test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {i-1, 1, 1, 1}));
@@ -13181,9 +13190,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
test_cases.emplace_back(new test_arange(GGML_TYPE_F32, 0.0f, n, 1.0f));
}
- // delta-net conv input: conv state followed by the transposed qkv of the ubatch
- test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {3, 8192, 1, 1}, 512, 0, 16));
-
// LoRA at prefill: B is [rank, n_out], so K = rank
for (int64_t k : {16, 32, 48, 64}) {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 2048, 512, k, {1, 1}, {1, 1}));
@@ -13216,6 +13222,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
}
}
+ // GDN conv_input shapes
+ for (int64_t channels : {8192, 10240}) {
+ for (int64_t n_tokens : {512, 2048, 4096}) {
+ test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {3, channels, 1, 1}, n_tokens, 0, 16));
+ }
+ }
+
// CONT of a 0<->2 permute at DeepSeek-V4 lightning-indexer shapes:
// indexer_kq is [n_kv, n_tokens, n_head=64] and gets ggml_cont(ggml_permute(.., 2,1,0,3)).
for (int64_t n_kv : { 1024, 1280, 2048, 2304 }) {