Commit aaa807c56 for llama.cpp
commit aaa807c568f2ffdb50d51455658ee91f5cbeedc1
Author: Pascal <admin@serveurperso.com>
Date: Wed Nov 18 07:45:14 2026 +0100
cuda: decode the conv2d patch indices with fastdiv (#32492)
The implicit GEMM conv2d kernel recovers the channel and the kernel
position of each reduction index with three integer divisions by
runtime divisors, once per loaded row of every tile. Outside the 3x3
and 1x1 specializations this integer division bounds the kernel:
removing it halves the time on long reductions.
Divide with the precomputed magic numbers of fastdiv instead. The 3x3
and 1x1 specializations keep their compile time divisors.
test-backend-ops -o CONV_2D: 2006/2006 on CUDA0.
CONV_2D, F16 kernel, us/run on an RTX PRO 6000:
[2054,1,768] k [7,1,768,768] 221 -> 138
[16390,1,384] k [7,1,384,384] 433 -> 266
[65542,1,192] k [7,1,192,192] 420 -> 271
[8240,1,256] k [11,1,256,256] 165 -> 108
[19,19,256,16] k [4,4,256,4096] 1672 -> 1076
diff --git a/ggml/src/ggml-cuda/conv2d.cu b/ggml/src/ggml-cuda/conv2d.cu
index 919a33325..ea586eb75 100644
--- a/ggml/src/ggml-cuda/conv2d.cu
+++ b/ggml/src/ggml-cuda/conv2d.cu
@@ -128,7 +128,9 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
const half * __restrict__ weight,
float * __restrict__ output,
const conv_params P,
- const int split_k) {
+ const int split_k,
+ const uint3 kw_fd,
+ const uint3 kwkh_fd) {
using namespace ggml_cuda_mma;
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
constexpr int nthreads = 4 * warp_size;
@@ -202,7 +204,10 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
}
for (int k = load_row; k < BK; k += nthreads / (BN / 2)) {
const int ki = k0 + k;
- const int ci = ki / (kw * kh), ky = ki / kw % kh, kx = ki % kw;
+ const int ci = KW && KH ? ki / (KW * KH) : int(fastdiv(uint32_t(ki), kwkh_fd));
+ const int r = ki - ci * kw * kh;
+ const int ky = KW ? r / KW : int(fastdiv(uint32_t(r), kw_fd));
+ const int kx = r - ky * kw;
const int offset = ki < k_total ? (n * ic + ci) * ih * iw + ky * dy * iw + kx * dx : 0;
half lo = __float2half(0.0f), hi = lo;
if (ki < k_total && spatial < ow * oh) {
@@ -314,12 +319,17 @@ static void conv2d_launch_implicit_gemm(const half * input,
dim3 grid,
dim3 block,
cudaStream_t stream) {
+ const uint3 kw_fd = init_fastdiv_values(uint64_t(params.KW));
+ const uint3 kwkh_fd = init_fastdiv_values(uint64_t(params.KW * params.KH));
if (params.KW == 3 && params.KH == 3) {
- conv2d_implicit_gemm_f16<3, 3, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k);
+ conv2d_implicit_gemm_f16<3, 3, use_mma>
+ <<<grid, block, 0, stream>>>(input, weight, output, params, split_k, kw_fd, kwkh_fd);
} else if (params.KW == 1 && params.KH == 1) {
- conv2d_implicit_gemm_f16<1, 1, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k);
+ conv2d_implicit_gemm_f16<1, 1, use_mma>
+ <<<grid, block, 0, stream>>>(input, weight, output, params, split_k, kw_fd, kwkh_fd);
} else {
- conv2d_implicit_gemm_f16<0, 0, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k);
+ conv2d_implicit_gemm_f16<0, 0, use_mma>
+ <<<grid, block, 0, stream>>>(input, weight, output, params, split_k, kw_fd, kwkh_fd);
}
}