Commit 47fe13331 for llama.cpp

commit 47fe1333180210d01b000f9da614ea272f2f04c3
Author: uvos <carl@uvos.xyz>
Date:   Mon Nov 16 14:24:19 2026 +0100

    HIP/CUDA: enable GDN mma kernel on CDNA (#31327)

diff --git a/ggml/src/ggml-cuda/gated-delta-net-mma.cu b/ggml/src/ggml-cuda/gated-delta-net-mma.cu
index 5fc9ef330..d3b8e575b 100644
--- a/ggml/src/ggml-cuda/gated-delta-net-mma.cu
+++ b/ggml/src/ggml-cuda/gated-delta-net-mma.cu
@@ -9,11 +9,16 @@ constexpr int D = 128;
 using bf16      = nv_bfloat16;

 template <int R, int C> struct matrix {
-    bf16 hi[R * C], lo[R * C];
+#ifdef GGML_USE_HIP
+    static constexpr int alloc = C + 4;
+#else
+    static constexpr int alloc = C;
+#endif // GGML_USE_HIP
+    bf16 hi[R * alloc], lo[R * alloc];

     static __device__ __forceinline__ int index(int r, int c) {
 #ifdef GGML_USE_HIP
-        return r * C + c;
+        return r * alloc + c;
 #else
         return 2 * ggml_cuda_mma::swizzle<C / 2, nv_bfloat162>(r * (C / 2) + c / 2, r) + c % 2;
 #endif // GGML_USE_HIP
@@ -56,11 +61,11 @@ template <int C, int V> struct alignas(128) shared {
     static __device__ __forceinline__ int lower_index(int r, int c) { return r * C + c; }
 };

-#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
+#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
 using namespace ggml_cuda_mma;
 #ifdef GGML_USE_HIP
 using acc    = tile<16, 16, float, DATA_LAYOUT_J_MAJOR>;
-using a_tile = tile<16, 8, nv_bfloat162, DATA_LAYOUT_I_MAJOR_MIRRORED>;
+using a_tile = tile<16, 8, nv_bfloat162, get_input_data_layout()>;
 using b_tile = a_tile;
 #else
 using acc    = tile<16, 8, float>;
@@ -73,6 +78,23 @@ __device__ __forceinline__ tile_t load(const matrix<R, C> & m, int row, int col,
     tile_t       t;
     const bf16 * data = low ? m.lo : m.hi;
 #ifdef GGML_USE_HIP
+#ifdef AMD_MFMA_AVAILABLE
+    if constexpr (!TRANS) {
+#pragma unroll
+        for (int i = 0; i < tile_t::ne; ++i) {
+            const int r = row + tile_t::get_i(i), c = col + 2 * tile_t::get_j(i);
+            ggml_cuda_memcpy_1<4>(t.x + i, data + m.index(r, c));
+        }
+    } else {
+#pragma unroll
+        for (int i = 0; i < tile_t::ne; ++i) {
+            const int  r = row + tile_t::get_i(i), c = col + 2 * tile_t::get_j(i);
+            const bf16 x = data[m.index(c, r)];
+            const bf16 y = data[m.index(c + 1, r)];
+            t.x[i]       = __float22bfloat162_rn(make_float2(__bfloat162float(x), __bfloat162float(y)));
+        }
+    }
+#else
 #pragma unroll
     for (int i = 0; i < tile_t::ne; ++i) {
         const int  r = row + tile_t::get_i(i), c = col + 2 * tile_t::get_j(i);
@@ -80,6 +102,7 @@ __device__ __forceinline__ tile_t load(const matrix<R, C> & m, int row, int col,
         const bf16 y = data[TRANS ? m.index(c + 1, r) : m.index(r, c + 1)];
         t.x[i]       = __float22bfloat162_rn(make_float2(__bfloat162float(x), __bfloat162float(y)));
     }
+#endif // AMD_MFMA_AVAILABLE
 #else
     const auto * packed = reinterpret_cast<const nv_bfloat162 *>(data);
     if constexpr (TRANS) {
@@ -118,15 +141,17 @@ template <int R, int C> __device__ __forceinline__ void store(matrix<R, C> & m,
     }
 #endif // GGML_USE_HIP
 }
-#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
+#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)

 // Each block owns V independent value columns and advances by C tokens.
 // Keep recurrent state in FP32; pack high/residual BF16 operands inside the block.
 template <int C, int V, int BLOCKS = 1>
 static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_mma_args a) {
-#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
-    static_assert(ggml_cuda_get_physical_warp_size() == 32, "GDN MMA requires 32-lane warps");
-    constexpr int WARPS = 8;
+#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
+    constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+    static_assert(256 % warp_size == 0, "block must span a whole number of waves");
+    constexpr int warps = 256 / warp_size;
+    constexpr int input_rows = warp_size / 32;  // input rows a wave stages per step
 #ifdef GGML_USE_HIP
     constexpr int N = 16;
 #else
@@ -134,34 +159,37 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
 #endif // GGML_USE_HIP
     static_assert(C >= 16 && (C & (C - 1)) == 0 && V % N == 0 && D % V == 0, "invalid GDN tile");
     extern __shared__ __align__(128) unsigned char bytes[];
-    auto &                                         s    = *reinterpret_cast<shared<C, V> *>(bytes);
-    const int                                      lane = threadIdx.x, warp = threadIdx.y, tid = warp * 32 + lane;
-    const int                                      slice = blockIdx.x % (D / V);
-    const int                                      h     = (blockIdx.x / (D / V)) % a.H;
-    const int                                      seq   = blockIdx.x / ((D / V) * a.H);
-    const int                                      v0    = slice * V;
-    const int64_t                                  soff  = ((int64_t) seq * a.H + h) * D * D;
-    const int64_t                                  qoff  = (seq / a.rq3) * a.sq3 + (h % a.H_k) * a.sq1;
-    const int64_t                                  voff  = seq * a.sv3 + h * a.sv1 + v0;
-    const int64_t                                  goff  = seq * a.sb3 + h * a.sb1;
-    float *                                        out   = a.dst + ((int64_t) seq * a.n_tokens * a.H + h) * D + v0;
-    constexpr int                                  STATE_TILES = (D / 16) * (V / N);
-    constexpr int                                  PER_WARP    = STATE_TILES / WARPS;
-    static_assert(STATE_TILES % WARPS == 0, "state must divide across warps");
+    auto & s    = *reinterpret_cast<shared<C, V> *>(bytes);
+    const int lane = threadIdx.x;
+    const int warp = threadIdx.y;
+    const int tid = warp * warp_size + lane;
+    const int sub = warp_size > 32 ? lane / 32 : 0;
+    const int slane = warp_size > 32 ? lane % 32 : lane;
+    const int slice = blockIdx.x % (D / V);
+    const int h = (blockIdx.x / (D / V)) % a.H;
+    const int seq = blockIdx.x / ((D / V) * a.H);
+    const int v0 = slice * V;
+    const int64_t soff = ((int64_t) seq * a.H + h) * D * D;
+    const int64_t qoff = (seq / a.rq3) * a.sq3 + (h % a.H_k) * a.sq1;
+    const int64_t voff = seq * a.sv3 + h * a.sv1 + v0;
+    const int64_t goff = seq * a.sb3 + h * a.sb1;
+    float * out = a.dst + ((int64_t) seq * a.n_tokens * a.H + h) * D + v0;
+    constexpr int state_tiles = (D / 16) * (V / N);
+    constexpr int PER_WARP = state_tiles / warps;
+    static_assert(state_tiles % warps == 0, "state must divide across warps");
     acc state[PER_WARP];
 #pragma unroll
     for (int j = 0; j < PER_WARP; ++j) {
-        const int tile_index = warp + j * WARPS, r = (tile_index / (V / N)) * 16, c = (tile_index % (V / N)) * N;
+        const int tile_index = warp + j * warps, r = (tile_index / (V / N)) * 16, c = (tile_index % (V / N)) * N;
 #pragma unroll
         for (int i = 0; i < acc::ne; ++i) {
             state[j].x[i] = a.state[soff + (v0 + c + acc::get_j(i)) * D + r + acc::get_i(i)];
         }
     }
     auto read_qk = [&](int64_t t, float4 & q, float4 & k) {
-        t += warp;
         q = k = make_float4(0, 0, 0, 0);
         if (t < a.n_tokens) {
-            const int64_t off = qoff + t * a.sq2 + 4 * lane;
+            const int64_t off = qoff + t * a.sq2 + 4 * slane;
             if ((((uintptr_t) (a.q + off) | (uintptr_t) (a.k + off)) & 15) == 0) {
                 q = *reinterpret_cast<const float4 *>(a.q + off);
                 k = *reinterpret_cast<const float4 *>(a.k + off);
@@ -172,22 +200,26 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
         }
     };
     // Carry the first eight input rows across chunks to hide global-load latency.
+    constexpr int  QK_ROWS  = warps * input_rows;
     constexpr bool PREFETCH = C == 16 && V == 32;
-    float4         next_q, next_k;
+    float4 next_q;
+    float4 next_k;
     if constexpr (PREFETCH) {
-        read_qk(0, next_q, next_k);
+        read_qk(warp * input_rows + sub, next_q, next_k);
     }
     for (int64_t t0 = 0; t0 < a.n_tokens; t0 += C) {
         const int valid = min((int64_t) C, a.n_tokens - t0);
 #pragma unroll
-        for (int j = 0; j < C / 8; ++j) {
-            const int t = warp + 8 * j, d = 4 * lane;
-            float4    q, k;
+        for (int j = 0; j < C / QK_ROWS; ++j) {
+            const int t = warp * input_rows + sub + QK_ROWS * j;
+            const int d = 4 * slane;
+            float4 q;
+            float4 k;
             if (PREFETCH && j == 0) {
                 q = next_q;
                 k = next_k;
             } else {
-                read_qk(t0 + 8 * j, q, k);
+                read_qk(t0 + t, q, k);
             }
             s.q.store2(t, d, make_float2(q.x * a.scale, q.y * a.scale));
             s.q.store2(t, d + 2, make_float2(q.z * a.scale, q.w * a.scale));
@@ -197,13 +229,13 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
         // Load gates in parallel before the FP32 prefix scan.
         if (warp == 0) {
             float carry = 0.f;
-            for (int base = 0; base < C; base += 32) {
+            for (int base = 0; base < C; base += warp_size) {
                 const int   t    = base + lane;
                 float       g    = t < valid ? a.g[goff + (t0 + t) * a.sb2] : 0.f;
                 const float beta = t < valid ? a.beta[goff + (t0 + t) * a.sb2] : 0.f;
 #pragma unroll
-                for (int offset = 1; offset < 32; offset *= 2) {
-                    const float previous = ggml_cuda_shfl_up(g, offset, 32);
+                for (int offset = 1; offset < warp_size; offset *= 2) {
+                    const float previous = ggml_cuda_shfl_up(g, offset, warp_size);
                     if (lane >= offset) {
                         g += previous;
                     }
@@ -213,23 +245,23 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
                     s.prefix[t] = g;
                     s.beta[t]   = beta;
                 }
-                carry = ggml_cuda_shfl(g, 31, 32);
+                carry = ggml_cuda_shfl(g, warp_size - 1, warp_size);
             }
         }
 #pragma unroll
         for (int j = 0; j < PER_WARP; ++j) {
-            const int ti = warp + j * WARPS;
+            const int ti = warp + j * warps;
             store(s.scratch.state, state[j], (ti / (V / N)) * 16, (ti % (V / N)) * N);
         }
         __syncthreads();
         if constexpr (PREFETCH) {
             if (t0 + C < a.n_tokens) {
-                read_qk(t0 + C, next_q, next_k);
+                read_qk(t0 + C + warp * input_rows + sub, next_q, next_k);
             }
         }
         // V32 overlaps Gram work with one output tile per projection warp.
         constexpr bool SPLIT_PROJECTION = C == 16 && V == 32;
-        acc            base_output[((C / 16) * (V / N) + WARPS - 1) / WARPS];
+        acc            base_output[((C / 16) * (V / N) + warps - 1) / warps];
         // delta = beta * (v - decay * K S), base output = decay * Q S
         auto           project = [&]() {
             if constexpr (V == 32) {
@@ -237,23 +269,26 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
                     return;
                 }
             }
-            for (int ti = (V == 32 ? warp - 4 : warp); ti < (C / 16) * (V / N); ti += (V == 32 ? 4 : WARPS)) {
-                const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
-                acc       qs, ks;
+            for (int ti = (V == 32 ? warp - 4 : warp); ti < (C / 16) * (V / N); ti += (V == 32 ? 4 : warps)) {
+                const int r = (ti / (V / N)) * 16;
+                const int c = (ti % (V / N)) * N;
+                acc qs;
+                acc ks;
                 product<D, false, true>(qs, s.q, s.scratch.state, r, c);
                 product<D, false, true>(ks, s.k, s.scratch.state, r, c);
 #pragma unroll
                 for (int i = 0; i < acc::ne; ++i) {
-                    const int   t = r + acc::get_i(i), v = c + acc::get_j(i);
+                    const int t = r + acc::get_i(i);
+                    const int v = c + acc::get_j(i);
                     const float decay = expf(s.prefix[t]);
                     const float vv    = t < valid ? a.v[voff + (t0 + t) * a.sv2 + v] : 0.f;
                     s.delta.store(t, v, s.beta[t] * (vv - decay * ks.x[i]));
-                    base_output[ti / WARPS].x[i] = decay * qs.x[i];
+                    base_output[ti / warps].x[i] = decay * qs.x[i];
                 }
             }
         };
         // A = beta * strict_tril(K K^T * decay), P = tril(Q K^T * decay)
-        for (int ti = warp; ti < 2 * (C / 16) * (C / N) && (V != 32 || warp < 4); ti += (V == 32 ? 4 : WARPS)) {
+        for (int ti = warp; ti < 2 * (C / 16) * (C / N) && (V != 32 || warp < 4); ti += (V == 32 ? 4 : warps)) {
             const bool gram = ti < (C / 16) * (C / N);
             const int  t = ti % ((C / 16) * (C / N)), r = (t / (C / N)) * 16, c = (t % (C / N)) * N;
             acc        x;
@@ -280,27 +315,27 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
         }
         __syncthreads();
         // inverse = (I + A)^-1 by forward substitution
-        for (int row = warp; row < C; row += WARPS) {
-            float x[(C + 31) / 32];
+        for (int row = warp; row < C; row += warps) {
+            float x[(C + warp_size - 1) / warp_size];
 #pragma unroll
-            for (int j = 0; j < (C + 31) / 32; ++j) {
-                const int col = lane + 32 * j;
-                x[j]          = col == row ? 1.f : (col < row ? -s.lower[s.lower_index(row, col)] : 0.f);
+            for (int j = 0; j < (C + warp_size - 1) / warp_size; ++j) {
+                const int col = lane + warp_size * j;
+                x[j] = col == row ? 1.f : (col < row ? -s.lower[s.lower_index(row, col)] : 0.f);
             }
             for (int pivot = row - 1; pivot > 0; --pivot) {
-                const float xp = ggml_cuda_shfl(x[pivot / 32], pivot % 32, 32);
+                const float xp = ggml_cuda_shfl(x[pivot / warp_size], pivot % warp_size, warp_size);
 #pragma unroll
-                for (int j = 0; j < (C + 31) / 32; ++j) {
-                    const int col = lane + 32 * j;
+                for (int j = 0; j < (C + warp_size - 1) / warp_size; ++j) {
+                    const int col = lane + warp_size * j;
                     if (col < pivot) {
                         x[j] -= xp * s.lower[s.lower_index(pivot, col)];
                     }
                 }
             }
 #pragma unroll
-            for (int j = 0; j < (C + 31) / 32; ++j) {
-                if (lane + 32 * j < C) {
-                    s.inverse.store(row, lane + 32 * j, x[j]);
+            for (int j = 0; j < (C + warp_size - 1) / warp_size; ++j) {
+                if (lane + warp_size * j < C) {
+                    s.inverse.store(row, lane + warp_size * j, x[j]);
                 }
             }
         }
@@ -309,8 +344,9 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
         }
         __syncthreads();
         // solved = inverse * delta
-        for (int ti = warp; ti < (C / 16) * (V / N); ti += WARPS) {
-            const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
+        for (int ti = warp; ti < (C / 16) * (V / N); ti += warps) {
+            const int r = (ti / (V / N)) * 16;
+            const int c = (ti % (V / N)) * N;
             acc       x;
             product<C, false, true>(x, s.inverse, s.delta, r, c);
             store(s.scratch.solved, x, r, c);
@@ -318,21 +354,25 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
         __syncthreads();
         // out = base output + P * solved
         for (int ti = (SPLIT_PROJECTION ? warp - 4 : warp); ti < (C / 16) * (V / N) && (!SPLIT_PROJECTION || warp >= 4);
-             ti += WARPS) {
-            const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
+             ti += warps) {
+            const int r = (ti / (V / N)) * 16;
+            const int c = (ti % (V / N)) * N;
             acc       x;
             product<C, false, true>(x, s.p, s.scratch.solved, r, c);
 #pragma unroll
             for (int i = 0; i < acc::ne; ++i) {
-                const int t = r + acc::get_i(i), v = c + acc::get_j(i);
+                const int t = r + acc::get_i(i);
+                const int v = c + acc::get_j(i);
                 if (t < valid) {
-                    out[(t0 + t) * a.H * D + v] = base_output[ti / WARPS].x[i] + x.x[i];
+                    out[(t0 + t) * a.H * D + v] = base_output[ti / warps].x[i] + x.x[i];
                 }
             }
         }
         // S = decay(chunk) * S + K^T * (solved * decay to chunk end)
-        for (int i = tid; i < C * V; i += 256) {
-            const int   t = i / V, v = i % V, j = s.scratch.solved.index(t, v);
+        for (int i = tid; i < C * V; i += warps * warp_size) {
+            const int t = i / V;
+            const int v = i % V;
+            const int j = s.scratch.solved.index(t, v);
             const float x = __bfloat162float(s.scratch.solved.hi[j]) + __bfloat162float(s.scratch.solved.lo[j]);
             s.delta.store(t, v, x * expf(s.prefix[valid - 1] - s.prefix[t]));
         }
@@ -340,7 +380,9 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
         const float decay = expf(s.prefix[valid - 1]);
 #pragma unroll
         for (int j = 0; j < PER_WARP; ++j) {
-            const int ti = warp + j * WARPS, r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
+            const int ti = warp + j * warps;
+            const int r = (ti / (V / N)) * 16;
+            const int c = (ti % (V / N)) * N;
 #pragma unroll
             for (int i = 0; i < acc::ne; ++i) {
                 state[j].x[i] *= decay;
@@ -351,7 +393,9 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
     }
 #pragma unroll
     for (int j = 0; j < PER_WARP; ++j) {
-        const int ti = warp + j * WARPS, r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
+        const int ti = warp + j * warps;
+        const int r = (ti / (V / N)) * 16;
+        const int c = (ti % (V / N)) * N;
 #pragma unroll
         for (int i = 0; i < acc::ne; ++i) {
             a.state_out[soff + (v0 + c + acc::get_j(i)) * D + r + acc::get_i(i)] = state[j].x[i];
@@ -360,10 +404,11 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
 #else
     GGML_UNUSED_VARS(a);
     NO_DEVICE_CODE;
-#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
+#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
 }

-template <int C, int V, int BLOCKS = 1> bool init(int device) {
+template <int C, int V, int BLOCKS = 1>
+static bool init(int device) {
     static std::once_flag once[GGML_CUDA_MAX_DEVICES];
     static bool           available[GGML_CUDA_MAX_DEVICES] = {};
     std::call_once(once[device], [device] {
@@ -381,7 +426,13 @@ template <int C, int V, int BLOCKS = 1> bool init(int device) {
     return available[device];
 }

-enum class gdn_path { ar, slice32_one_block, slice32_two_blocks, slice64, whole_head };
+enum class gdn_path {
+    ar,
+    slice32_one_block,
+    slice32_two_blocks,
+    slice64,
+    whole_head
+};

 static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a) {
     GGML_ASSERT(device >= 0 && device < GGML_CUDA_MAX_DEVICES);
@@ -391,10 +442,25 @@ static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a
     }
     const auto & info = ggml_cuda_info().devices[device];
 #ifdef GGML_USE_HIP
-    if (!GGML_CUDA_CC_IS_RDNA3_5(info.cc) || a.n_tokens < 2048) {
-        return gdn_path::ar;
+    if (GGML_CUDA_CC_IS_RDNA3_5(info.cc)) {
+        if (a.n_tokens < 2048) {
+            return gdn_path::ar;
+        }
+        return init<16, 64>(device) ? gdn_path::slice64 : gdn_path::ar;
     }
-    return init<16, 64>(device) ? gdn_path::slice64 : gdn_path::ar;
+    if (GGML_CUDA_CC_IS_CDNA(info.cc)) {
+        if (!amd_mfma_available(info.cc)) {
+            return gdn_path::ar;
+        }
+        const int64_t blocks            = a.H * a.n_seqs * 2;
+        const int64_t blocks_nwaves     = (blocks + info.nsm - 1) / info.nsm;
+        const int64_t blocks_efficiency = 100 * blocks / (info.nsm * blocks_nwaves);
+        if (blocks_nwaves > 1 && blocks_efficiency < 60) {
+            return gdn_path::ar;
+        }
+        return init<16, 64>(device) ? gdn_path::slice64 : gdn_path::ar;
+    }
+    return gdn_path::ar;
 #elif defined(GGML_USE_MUSA)
     GGML_UNUSED_VARS(info);
     return gdn_path::ar;
@@ -428,7 +494,8 @@ bool ggml_cuda_gdn_mma_launch(int device, const ggml_cuda_gdn_mma_args & a, cuda
         return false;
     }
 #ifdef GGML_USE_HIP
-    gdn_single<16, 64><<<a.H * a.n_seqs * 2, dim3(32, 8), sizeof(shared<16, 64>), stream>>>(a);
+    const int warp_size = ggml_cuda_info().devices[device].warp_size;
+    gdn_single<16, 64><<<a.H * a.n_seqs * 2, dim3(warp_size, 256 / warp_size), sizeof(shared<16, 64>), stream>>>(a);
 #else
     if (path == gdn_path::slice32_two_blocks) {
         gdn_single<16, 32, 2><<<a.H * a.n_seqs * 4, dim3(32, 8), sizeof(shared<16, 32>), stream>>>(a);