Commit ebc9fb933 for llama.cpp
commit ebc9fb933e774bb4ac0ba497a67948e16b7e5656
Author: Giorgio Sarno <giorgio.srn@gmail.com>
Date: Tue Nov 17 15:51:15 2026 +0100
CUDA: use block reductions for whole-row pool_1d (#31855)
* CUDA: use block reductions for whole-row pool_1d
Assisted-by: Claude Code
* CUDA: template pool1d row kernel by ggml_op_pool
diff --git a/ggml/src/ggml-cuda/pool1d.cu b/ggml/src/ggml-cuda/pool1d.cu
index ac6fb0cbd..495bea0bf 100644
--- a/ggml/src/ggml-cuda/pool1d.cu
+++ b/ggml/src/ggml-cuda/pool1d.cu
@@ -49,6 +49,36 @@ static __global__ void pool1d_nchw_kernel(
o_ptr[cur_ow] = res;
}
+// a window covering the whole row gives one output per row: reduce it with a block instead of one thread
+template <ggml_op_pool op>
+static __global__ void pool1d_row_kernel(const int iw, const float * src, float * dst) {
+ static_assert(op == GGML_OP_POOL_MAX || op == GGML_OP_POOL_AVG, "unsupported pool op");
+
+ const float * row = src + (int64_t) blockIdx.x*iw;
+
+ float res = op == GGML_OP_POOL_MAX ? -FLT_MAX : 0.0f;
+ for (int i = threadIdx.x; i < iw; i += blockDim.x) {
+ res = op == GGML_OP_POOL_MAX ? max(res, row[i]) : res + row[i];
+ }
+
+ __shared__ float buf[WARP_SIZE];
+ res = block_reduce<op == GGML_OP_POOL_MAX ? block_reduce_method::MAX : block_reduce_method::SUM>(res, buf);
+
+ if (threadIdx.x == 0) {
+ dst[blockIdx.x] = op == GGML_OP_POOL_MAX ? res : res/iw;
+ }
+}
+
+static void pool1d_row_f32_cuda(const int iw, const int64_t nr, const float * src, float * dst, const enum ggml_op_pool op, cudaStream_t stream) {
+ const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm;
+ const dim3 block_dims(nr/nsm < 2 ? 512 : (iw < 1024 ? 32 : 128));
+ if (op == GGML_OP_POOL_MAX) {
+ pool1d_row_kernel<GGML_OP_POOL_MAX><<<nr, block_dims, 0, stream>>>(iw, src, dst);
+ } else {
+ pool1d_row_kernel<GGML_OP_POOL_AVG><<<nr, block_dims, 0, stream>>>(iw, src, dst);
+ }
+}
+
static void pool1d_nchw_kernel_f32_f32_cuda(
const int iw, const int ow,
const int kw, const int sw, const int pw,
@@ -79,6 +109,12 @@ void ggml_cuda_op_pool1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
const int64_t OW = dst->ne[0];
const int64_t nr = ggml_nrows(src0);
+ // short rows: one thread per row is faster
+ if (OW == 1 && p0 == 0 && k0 == IW && IW >= CUDA_POOL1D_ROW_MIN_IW && (op == GGML_OP_POOL_MAX || op == GGML_OP_POOL_AVG)) {
+ pool1d_row_f32_cuda(IW, nr, src0_d, dst_d, op, stream);
+ return;
+ }
+
const int parallel_elements = (int)(nr * OW);
pool1d_nchw_kernel_f32_f32_cuda(IW, OW, k0, s0, p0, parallel_elements, src0_d, dst_d, op, stream);
diff --git a/ggml/src/ggml-cuda/pool1d.cuh b/ggml/src/ggml-cuda/pool1d.cuh
index c79461dd8..008b75ef5 100644
--- a/ggml/src/ggml-cuda/pool1d.cuh
+++ b/ggml/src/ggml-cuda/pool1d.cuh
@@ -1,5 +1,6 @@
#include "common.cuh"
#define CUDA_POOL1D_BLOCK_SIZE 256
+#define CUDA_POOL1D_ROW_MIN_IW 256
void ggml_cuda_op_pool1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index a45d9bad5..da91b51e9 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -10920,6 +10920,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
}
+ // one window over the whole row (a row reduction)
+ for (int64_t iw : {10, 128, 256, 1000, 2048, 4099}) {
+ test_cases.emplace_back(new test_pool1d(pool_type, type_input, { iw, 3, 2, 1 }, (int) iw, (int) iw, 0));
+ test_cases.emplace_back(new test_pool1d(pool_type, type_input, { iw, 1024, 1, 1 }, (int) iw, (int) iw, 0));
+ }
}
}
@@ -13652,6 +13657,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
}
}
}
+ // POOL_1D whole-row reduction with short rows and many rows
+ for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) {
+ test_cases.emplace_back(new test_pool1d(pool_type, GGML_TYPE_F32, {16, 65536, 1, 1}, 16, 16, 0));
+ }
// LEAKY_RELU at FFN activation width, for direct comparison with RELU
for (int64_t n_tokens : {512, 2048}) {