Commit 03ec5b432 for llama.cpp
commit 03ec5b4326d0a7aec5573dfe7810811d7da12de4
Author: losewayy <losemywayy@gmail.com>
Date: Wed Nov 18 18:43:52 2026 +0800
cuda : fuse NORM + MUL (+ ADD) (#32454)
Assisted-by: Codex
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index b7842d289..3d8d30d81 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -4169,6 +4169,26 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph,
return false;
}
+ if ((ops.size() == 2 || ops.size() == 3) && ops.begin()[0] == GGML_OP_NORM && ops.begin()[1] == GGML_OP_MUL) {
+ const ggml_tensor * norm = cgraph->nodes[node_idx];
+ const ggml_tensor * mul = cgraph->nodes[node_idx + 1];
+ const ggml_tensor * weight = mul->src[0] == norm ? mul->src[1] : mul->src[0];
+ if (norm->src[0]->type != GGML_TYPE_F32 || norm->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32 ||
+ weight->type != GGML_TYPE_F32 || !ggml_is_vector(weight) || weight->ne[0] != norm->ne[0] ||
+ !ggml_is_contiguous(weight)) {
+ return false;
+ }
+ if (ops.size() == 3) {
+ const ggml_tensor * add = cgraph->nodes[node_idx + 2];
+ const ggml_tensor * bias = add->src[0] == mul ? add->src[1] : add->src[0];
+ if (ops.begin()[2] != GGML_OP_ADD || add->type != GGML_TYPE_F32 || bias->type != GGML_TYPE_F32 ||
+ !ggml_is_vector(bias) || bias->ne[0] != norm->ne[0] || !ggml_is_contiguous(bias)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
if ((ops.size() == 2 || ops.size() == 3) && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) {
const ggml_tensor *rms_norm = cgraph->nodes[node_idx];
const ggml_tensor *mul = cgraph->nodes[node_idx+1];
@@ -5136,6 +5156,16 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
return 2;
}
+ if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) {
+ ggml_cuda_op_norm(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]);
+ return 2;
+ }
+
+ if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_NORM, GGML_OP_MUL }, {})) {
+ ggml_cuda_op_norm(*cuda_ctx, node, cgraph->nodes[i + 1]);
+ return 1;
+ }
+
if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) {
ggml_cuda_op_rms_norm_fused_add(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]);
return 2;
diff --git a/ggml/src/ggml-cuda/norm.cu b/ggml/src/ggml-cuda/norm.cu
index d6471bcc4..95c3b0a0e 100644
--- a/ggml/src/ggml-cuda/norm.cu
+++ b/ggml/src/ggml-cuda/norm.cu
@@ -2,10 +2,18 @@
#include "convert.cuh"
#include <cstdint>
-template <int block_size>
-static __global__ void norm_f32(
- const float * x, float * dst, const int ncols, const int nchannels, const int nsamples,
- const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps) {
+template <int block_size, bool do_multiply = false, bool do_add = false>
+static __global__ void norm_f32(const float * x,
+ float * dst,
+ const int ncols,
+ const int nchannels,
+ const int nsamples,
+ const int64_t stride_row,
+ const int64_t stride_channel,
+ const int64_t stride_sample,
+ const float eps,
+ const float * mul,
+ const float * add) {
const int nrows = gridDim.x;
const int row = blockIdx.x;
const int tid = threadIdx.x;
@@ -36,7 +44,14 @@ static __global__ void norm_f32(
const float inv_std = rsqrtf(var + eps);
for (int col = tid; col < ncols; col += block_size) {
- dstc[col] = (xc[col] - mean) * inv_std;
+ float v = (xc[col] - mean) * inv_std;
+ if constexpr (do_multiply) {
+ v *= mul[col];
+ }
+ if constexpr (do_add) {
+ v += add[col];
+ }
+ dstc[col] = v;
}
if constexpr (block_size > WARP_SIZE) {
@@ -384,16 +399,33 @@ static __global__ void l2_norm_f32(
}
}
-static void norm_f32_cuda(
- const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples,
- const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) {
+template <bool do_multiply = false, bool do_add = false>
+static void norm_f32_cuda(const float * x,
+ float * dst,
+ const int ncols,
+ const int nrows,
+ const int nchannels,
+ const int nsamples,
+ const int64_t stride_row,
+ const int64_t stride_channel,
+ const int64_t stride_sample,
+ const float eps,
+ cudaStream_t stream,
+ const float * mul = nullptr,
+ const float * add = nullptr) {
const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX));
- if (ncols < 1024) {
+ if (do_multiply && ncols <= 2048) {
+ const dim3 block_dims(256, 1, 1);
+ norm_f32<256, do_multiply, do_add><<<blocks_num, block_dims, 32 * sizeof(float2), stream>>>(
+ x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, add);
+ } else if (!do_multiply && ncols < 1024) {
const dim3 block_dims(WARP_SIZE, 1, 1);
- norm_f32<WARP_SIZE><<<blocks_num, block_dims, 0, stream>>>(x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps);
+ norm_f32<WARP_SIZE, do_multiply, do_add><<<blocks_num, block_dims, 0, stream>>>(
+ x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, add);
} else {
const dim3 block_dims(1024, 1, 1);
- norm_f32<1024><<<blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float2): 0, stream>>>(x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps);
+ norm_f32<1024, do_multiply, do_add><<<blocks_num, block_dims, 32 * sizeof(float2), stream>>>(
+ x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, add);
}
}
@@ -591,10 +623,10 @@ static void l2_norm_f32_cuda(
}
}
-void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul, ggml_tensor * add) {
const ggml_tensor * src0 = dst->src[0];
const float * src0_d = (const float *) src0->data;
- float * dst_d = (float *) dst->data;
+ float * dst_d = (float *) (add ? add : mul ? mul : dst)->data;
cudaStream_t stream = ctx.stream();
GGML_ASSERT(src0->type == GGML_TYPE_F32);
@@ -612,7 +644,19 @@ void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
const int64_t s02 = nb02 / ts0;
const int64_t s03 = nb03 / ts0;
- norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream);
+ if (mul) {
+ const ggml_tensor * weight = mul->src[0] == dst ? mul->src[1] : mul->src[0];
+ const float * weight_d = (const float *) weight->data;
+ if (add) {
+ const ggml_tensor * bias = add->src[0] == mul ? add->src[1] : add->src[0];
+ norm_f32_cuda<true, true>(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream, weight_d,
+ (const float *) bias->data);
+ } else {
+ norm_f32_cuda<true, false>(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream, weight_d);
+ }
+ } else {
+ norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream);
+ }
}
void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
diff --git a/ggml/src/ggml-cuda/norm.cuh b/ggml/src/ggml-cuda/norm.cuh
index 95618df5f..741a1d1ae 100644
--- a/ggml/src/ggml-cuda/norm.cuh
+++ b/ggml/src/ggml-cuda/norm.cuh
@@ -1,6 +1,9 @@
#include "common.cuh"
-void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx,
+ ggml_tensor * dst,
+ ggml_tensor * mul = nullptr,
+ ggml_tensor * add = nullptr);
void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index cddc2dbaa..49ae976a4 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -3924,6 +3924,8 @@ struct test_norm_mul_add : public test_case {
const std::array<int64_t, 4> ne;
float eps;
const bool broadcast;
+ const bool affine;
+ const bool bias;
std::string op_desc(ggml_tensor * t) override {
GGML_UNUSED(t);
@@ -3932,22 +3934,28 @@ struct test_norm_mul_add : public test_case {
bool run_whole_graph() override { return true; }
- std::string vars() override {
- return VARS_TO_STR4(type, ne, eps, broadcast);
- }
+ std::string vars() override { return VARS_TO_STR6(type, ne, eps, broadcast, affine, bias); }
- test_norm_mul_add(ggml_type type = GGML_TYPE_F32,
- std::array<int64_t, 4> ne = {128, 2, 1, 1},
- float eps = 1e-5f,
- bool broadcast = false)
- : type(type), ne(ne), eps(eps), broadcast(broadcast) {}
+ test_norm_mul_add(ggml_type type = GGML_TYPE_F32,
+ std::array<int64_t, 4> ne = { 128, 2, 1, 1 },
+ float eps = 1e-5f,
+ bool broadcast = false,
+ bool affine = false,
+ bool bias = true) :
+ type(type),
+ ne(ne),
+ eps(eps),
+ broadcast(broadcast),
+ affine(affine),
+ bias(bias) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
std::array<int64_t, 4> broadcast_dims = {ne[0], ne[1] * 2, ne[2] * 2, ne[3] * 2};
ggml_tensor * a = ggml_new_tensor(ctx, type, 4, broadcast ? broadcast_dims.data() : ne.data());
- ggml_tensor * w = ggml_new_tensor(ctx, type, 4, ne.data());
- ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data());
+ std::array<int64_t, 4> affine_dims = { ne[0], 1, 1, 1 };
+ ggml_tensor * w = ggml_new_tensor(ctx, type, 4, affine ? affine_dims.data() : ne.data());
+ ggml_tensor * b = ggml_new_tensor(ctx, type, 4, affine ? affine_dims.data() : ne.data());
ggml_set_param(a); ggml_set_param(w); ggml_set_param(b);
ggml_set_name(a, "a"); ggml_set_name(w, "w"); ggml_set_name(b, "b");
@@ -3956,7 +3964,7 @@ struct test_norm_mul_add : public test_case {
ggml_tensor * n = ggml_norm(ctx, a, eps);
ggml_tensor * m = ggml_mul(ctx, n, w);
- ggml_tensor * out = ggml_add(ctx, m, b);
+ ggml_tensor * out = bias ? ggml_add(ctx, m, b) : m;
ggml_set_name(out, "out");
return out;
}
@@ -11671,6 +11679,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
test_cases.emplace_back(new test_add_rms_norm(GGML_TYPE_F32, {n, 1, 1, 1}, 1e-6f, false));
}
+ test_cases.emplace_back(new test_norm_mul_add(GGML_TYPE_F32, { 1025, 2, 3, 2 }, 1e-5f, false, true));
+ test_cases.emplace_back(new test_norm_mul_add(GGML_TYPE_F32, { 8192, 1, 1, 1 }, 1e-5f, false, true));
+ for (uint32_t n : { 768, 1024, 1280 }) {
+ for (bool bias : { false, true }) {
+ test_cases.emplace_back(new test_norm_mul_add(GGML_TYPE_F32, { n, 257, 1, 1 }, 1e-5f, false, true, bias));
+ }
+ }
+
for (uint32_t n : {64, 1025}) {
test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, false, false));
test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, true, false));