Commit a818a46f3 for llama.cpp
commit a818a46f3df4b6dee7f59f174283deaeae083c4b
Author: Clemens Wasser <clemens.wasser@gmail.com>
Date: Sat Nov 21 13:20:22 2026 +0100
sycl: add fused ADD, RMS_NORM, MUL and vectorize rms norm (#31753)
* sycl: vectorize contiguous rms norm
* sycl: add fused ADD, RMS_NORM, MUL
* tests: test fused ADD, RMS_NORM, MUL at decode widths
diff --git a/ggml/src/ggml-sycl/fusion.cpp b/ggml/src/ggml-sycl/fusion.cpp
index 9b553829b..9ea6af6b0 100644
--- a/ggml/src/ggml-sycl/fusion.cpp
+++ b/ggml/src/ggml-sycl/fusion.cpp
@@ -121,6 +121,28 @@ bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializ
return ggml_sycl_should_fuse_mul_mat_glu(gate, up, glu);
}
+ // the residual sum stays materialised (a later residual add reads it), so ggml_can_fuse cannot
+ // express this either: the add and the mul are the subgraph's outputs
+ if (ops.size() == 3 && ops.begin()[0] == GGML_OP_ADD && ops.begin()[1] == GGML_OP_RMS_NORM &&
+ ops.begin()[2] == GGML_OP_MUL) {
+ if (!ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx, node_idx + 2 })) {
+ return false;
+ }
+
+ const ggml_tensor * add = cgraph->nodes[node_idx];
+ if (cgraph->nodes[node_idx + 1]->src[0] != add) {
+ return false;
+ }
+
+ // the fused kernel does not broadcast the add
+ if (add->src[0]->type != GGML_TYPE_F32 || add->src[1]->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32 ||
+ !ggml_are_same_shape(add->src[0], add) || !ggml_are_same_shape(add->src[1], add)) {
+ return false;
+ }
+
+ return ggml_sycl_can_fuse(cgraph, node_idx + 1, { GGML_OP_RMS_NORM, GGML_OP_MUL }, {});
+ }
+
if (!ggml_can_fuse(cgraph, node_idx, ops)) {
return false;
}
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index 5b87fa024..3de5f1bd5 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -6708,6 +6708,12 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc
continue;
}
}
+ if (node->op == GGML_OP_ADD &&
+ ggml_sycl_can_fuse(cgraph, i, { GGML_OP_ADD, GGML_OP_RMS_NORM, GGML_OP_MUL }, {}) &&
+ ggml_sycl_op_add_rms_norm_fused(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2])) {
+ i += 2;
+ continue;
+ }
if (node->op == GGML_OP_RMS_NORM &&
ggml_sycl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) {
ggml_sycl_op_rms_norm_fused_add(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]);
diff --git a/ggml/src/ggml-sycl/norm.cpp b/ggml/src/ggml-sycl/norm.cpp
index 36576e9c2..0cb30aa35 100644
--- a/ggml/src/ggml-sycl/norm.cpp
+++ b/ggml/src/ggml-sycl/norm.cpp
@@ -1,4 +1,6 @@
#include "norm.hpp"
+
+#include "fusion.hpp"
#include "ggml-sycl/common.hpp"
#include "ggml-sycl/presets.hpp"
@@ -188,9 +190,28 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols,
float tmp = 0.0f; // partial sum for thread in warp
- for (int col = tid; col < ncols; col += block_size) {
- const float xi = x[col * src_stride_col];
- tmp += xi * xi;
+ // float4 path for contiguous, 16-byte aligned rows (decode), views and odd lengths stay strided
+ bool contig = src_stride_col == 1 && dst_stride_col == 1 && ncols % 4 == 0 && (uintptr_t) x % 16 == 0 &&
+ (uintptr_t) dst % 16 == 0;
+ if constexpr (do_multiply) {
+ contig = contig && (uintptr_t) mul % 16 == 0;
+ }
+ if constexpr (do_add) {
+ contig = contig && (uintptr_t) add % 16 == 0;
+ }
+ const int n4 = ncols / 4;
+
+ if (contig) {
+ const sycl::float4 * x4 = reinterpret_cast<const sycl::float4 *>(x);
+ for (int i = tid; i < n4; i += block_size) {
+ const sycl::float4 v = x4[i];
+ tmp += sycl::dot(v, v);
+ }
+ } else {
+ for (int col = tid; col < ncols; col += block_size) {
+ const float xi = x[col * src_stride_col];
+ tmp += xi * xi;
+ }
}
// sum up partial sums
@@ -216,17 +237,37 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols,
const float mean = tmp / ncols;
const float scale = sycl::rsqrt(mean + eps);
- for (int col = tid; col < ncols; col += block_size) {
- if constexpr (do_multiply && do_add) {
- dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col] + add[col];
- } else if constexpr (do_multiply) {
- dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col];
- } else {
- // folded epilogue of a fused GGML_OP_SCALE consumer (qwen35 GDN l2 norms);
- // the explicit temporary keeps the float evaluation order identical to
- // running rms_norm and scale as two separate kernels
- const float v = scale * x[col * src_stride_col];
- dst[col * dst_stride_col] = v * scale_mul;
+ if (contig) {
+ const sycl::float4 * x4 = reinterpret_cast<const sycl::float4 *>(x);
+ sycl::float4 * d4 = reinterpret_cast<sycl::float4 *>(dst);
+ const sycl::float4 * m4 = reinterpret_cast<const sycl::float4 *>(mul);
+ const sycl::float4 * a4 = reinterpret_cast<const sycl::float4 *>(add);
+ for (int i = tid; i < n4; i += block_size) {
+ const sycl::float4 v = x4[i];
+ if constexpr (do_multiply && do_add) {
+ const sycl::float4 m = m4[i];
+ const sycl::float4 a = a4[i];
+ d4[i] = scale * v * m + a;
+ } else if constexpr (do_multiply) {
+ const sycl::float4 m = m4[i];
+ d4[i] = scale * v * m;
+ } else {
+ d4[i] = scale * v * scale_mul;
+ }
+ }
+ } else {
+ for (int col = tid; col < ncols; col += block_size) {
+ if constexpr (do_multiply && do_add) {
+ dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col] + add[col];
+ } else if constexpr (do_multiply) {
+ dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col];
+ } else {
+ // folded epilogue of a fused GGML_OP_SCALE consumer (qwen35 GDN l2 norms);
+ // the explicit temporary keeps the float evaluation order identical to
+ // running rms_norm and scale as two separate kernels
+ const float v = scale * x[col * src_stride_col];
+ dst[col * dst_stride_col] = v * scale_mul;
+ }
}
}
}
@@ -918,6 +959,127 @@ void ggml_sycl_op_rms_norm_fused_add(ggml_backend_sycl_context & ctx, ggml_tenso
add_s01, add_s02, add_s03, add_nrows, add_nchannels, add_nsamples, eps, main_stream, ctx.device);
}
+// ADD(a, b) -> RMS_NORM -> MUL(w) in one launch. The sum (read by a later residual add) and
+// normed * w are written after a barrier, as mul->data may alias a.
+static void add_rms_norm_mul_f32(const float * a,
+ const float * b,
+ float * sum,
+ const float * mul,
+ float * dst,
+ const int ncols,
+ const float eps,
+ const sycl::nd_item<3> & item_ct1,
+ float * s_sum,
+ const int block_size) {
+ const int row = item_ct1.get_group(2);
+ const size_t off = (size_t) row * ncols;
+ const int tid = item_ct1.get_local_id(2);
+ const int n4 = ncols / 4;
+
+ const sycl::float4 * a4 = reinterpret_cast<const sycl::float4 *>(a + off);
+ const sycl::float4 * b4 = reinterpret_cast<const sycl::float4 *>(b + off);
+
+ sycl::float4 v[2];
+ float tmp = 0.0f;
+#pragma unroll
+ for (int k = 0; k < 2; ++k) {
+ const int i = tid + k * block_size;
+ if (i < n4) {
+ const sycl::float4 va = a4[i];
+ const sycl::float4 vb = b4[i];
+ v[k] = va + vb;
+ tmp += sycl::dot(v[k], v[k]);
+ }
+ }
+
+ const int nwarps = block_size / WARP_SIZE;
+ tmp = warp_reduce_sum(tmp, item_ct1);
+ if (block_size > WARP_SIZE) {
+ const auto sub_group = item_ct1.get_sub_group();
+ if (sub_group.get_local_linear_id() == 0) {
+ s_sum[sub_group.get_group_linear_id()] = tmp;
+ }
+ item_ct1.barrier(sycl::access::fence_space::local_space);
+ const size_t nreduce = ceil_div(nwarps, WARP_SIZE);
+ tmp = 0.0f;
+ for (size_t i = 0; i < nreduce; i += 1) {
+ tmp += s_sum[sub_group.get_local_linear_id() + i * WARP_SIZE];
+ }
+ tmp = warp_reduce_sum(tmp, item_ct1);
+ }
+
+ const float mean = tmp / ncols;
+ const float scale = sycl::rsqrt(mean + eps);
+
+ sycl::float4 * s4 = reinterpret_cast<sycl::float4 *>(sum + off);
+ sycl::float4 * d4 = reinterpret_cast<sycl::float4 *>(dst + off);
+ const sycl::float4 * m4 = reinterpret_cast<const sycl::float4 *>(mul);
+#pragma unroll
+ for (int k = 0; k < 2; ++k) {
+ const int i = tid + k * block_size;
+ if (i < n4) {
+ const sycl::float4 m = m4[i];
+ const sycl::float4 u = scale * v[k] * m;
+ s4[i] = v[k];
+ d4[i] = u;
+ }
+ }
+}
+
+static void add_rms_norm_mul_f32_sycl(const float * a,
+ const float * b,
+ float * sum,
+ const float * mul,
+ float * dst,
+ const int ncols,
+ const int nrows,
+ const float eps,
+ queue_ptr stream,
+ int device) {
+ const int work_group_size = ggml_sycl_info().max_work_group_sizes[device];
+ GGML_ASSERT(work_group_size % (WARP_SIZE * WARP_SIZE) == 0);
+ const sycl::range<3> global_dims(1, 1, nrows);
+ const sycl::range<3> block_dims(1, 1, work_group_size);
+ stream->submit([&](sycl::handler & cgh) {
+ sycl::local_accessor<float, 1> s_sum(sycl::range<1>(work_group_size / WARP_SIZE), cgh);
+ cgh.parallel_for(sycl::nd_range<3>(global_dims * block_dims, block_dims),
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+ add_rms_norm_mul_f32(a, b, sum, mul, dst, ncols, eps, item_ct1, get_pointer(s_sum),
+ work_group_size);
+ });
+ });
+}
+
+bool ggml_sycl_op_add_rms_norm_fused(ggml_backend_sycl_context & ctx,
+ ggml_tensor * add,
+ ggml_tensor * rms,
+ ggml_tensor * mul) {
+ ggml_tensor * a = add->src[0];
+ ggml_tensor * b = add->src[1];
+ ggml_tensor * w = mul->src[0] == rms ? mul->src[1] : mul->src[0];
+ const int64_t ncols = add->ne[0];
+ const int wg = ggml_sycl_info().max_work_group_sizes[ctx.device];
+ // whole rows of float4, at most two per work-item, from 16-byte aligned contiguous tensors
+ if (add->ne[2] != 1 || add->ne[3] != 1 || ncols < 1024 || ncols % 4 != 0 || ncols > 8 * (int64_t) wg ||
+ w->ne[1] != 1) {
+ return false;
+ }
+ for (const ggml_tensor * t : { a, b, add, mul, w }) {
+ if (!ggml_is_contiguous(t) || (uintptr_t) t->data % 16 != 0) {
+ return false;
+ }
+ }
+ float eps = 0.0f;
+ memcpy(&eps, rms->op_params, sizeof(float));
+ if (!(eps >= 0.0f)) {
+ return false;
+ }
+ add_rms_norm_mul_f32_sycl((const float *) a->data, (const float *) b->data, (float *) add->data,
+ (const float *) w->data, (float *) mul->data, (int) ncols, (int) add->ne[1], eps,
+ ctx.stream(), ctx.device);
+ return true;
+}
+
void ggml_sycl_op_rms_norm_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
diff --git a/ggml/src/ggml-sycl/norm.hpp b/ggml/src/ggml-sycl/norm.hpp
index fb667ac6b..59a9ffd6f 100644
--- a/ggml/src/ggml-sycl/norm.hpp
+++ b/ggml/src/ggml-sycl/norm.hpp
@@ -25,6 +25,11 @@ void ggml_sycl_op_rms_norm_scale_fused(ggml_backend_sycl_context& ctx, ggml_tens
void ggml_sycl_op_rms_norm_fused_add(ggml_backend_sycl_context& ctx, ggml_tensor* dst, ggml_tensor* mul_tensor, ggml_tensor* add_tensor);
+bool ggml_sycl_op_add_rms_norm_fused(ggml_backend_sycl_context & ctx,
+ ggml_tensor * add,
+ ggml_tensor * rms,
+ ggml_tensor * mul);
+
void ggml_sycl_op_rms_norm_back(ggml_backend_sycl_context& ctx, ggml_tensor* dst);
void ggml_sycl_op_group_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst);
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 11bfa42a9..fbf5c12d3 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -4351,6 +4351,52 @@ struct test_add_rms_norm : public test_case {
}
};
+// GGML_OP_ADD + GGML_OP_RMS_NORM + GGML_OP_MUL (fused operation)
+struct test_add_rms_norm_mul : public test_case {
+ const std::array<int64_t, 4> ne;
+ const float eps;
+
+ ggml_tensor * sum = nullptr;
+ ggml_tensor * out = nullptr;
+
+ std::string op_desc(ggml_tensor * t) override {
+ GGML_UNUSED(t);
+ return "ADD_RMS_NORM_MUL";
+ }
+
+ bool run_whole_graph() override { return true; }
+
+ std::vector<ggml_tensor *> fusion_test_nodes() override { return { sum, out }; }
+
+ std::string vars() override { return VARS_TO_STR2(ne, eps); }
+
+ test_add_rms_norm_mul(std::array<int64_t, 4> ne = { 4096, 1, 1, 1 }, float eps = 1e-6f) : ne(ne), eps(eps) {}
+
+ ggml_tensor * build_graph(ggml_context * ctx) override {
+ ggml_tensor * a = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data());
+ ggml_tensor * b = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data());
+ ggml_tensor * w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]);
+ ggml_set_name(a, "a");
+ ggml_set_name(b, "b");
+ ggml_set_name(w, "w");
+
+ sum = ggml_add(ctx, a, b);
+ ggml_set_name(sum, "sum");
+ ggml_tensor * normed = ggml_mul(ctx, ggml_rms_norm(ctx, sum, eps), w);
+ ggml_set_name(normed, "normed");
+
+ out = ggml_add(ctx, normed, sum);
+ ggml_set_name(out, "out");
+ return out;
+ }
+
+ void initialize_tensors(ggml_context * ctx) override {
+ for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
+ init_tensor_uniform(t, -10.f, 10.f);
+ }
+ }
+};
+
// GGML_OP_UNARY(RELU) + GGML_OP_SQR (fused operation)
struct test_relu_sqr : public test_case {
const ggml_type type;
@@ -11801,6 +11847,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
+ // decode-shaped residual rows: below, at and above the fused path's limits, odd row counts, batches
+ for (int64_t n : { 1020, 1024, 4096, 5120, 8192, 8196, 16384 }) {
+ for (int64_t rows : { 1, 3, 4 }) {
+ test_cases.emplace_back(new test_add_rms_norm_mul({ n, rows, 1, 1 }, 1e-6f));
+ }
+ test_cases.emplace_back(new test_add_rms_norm_mul({ n, 2, 3, 1 }, 1e-6f));
+ }
for (uint32_t n : {64, 1025}) {
test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, false, false));
test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, true, false));