Commit a818a46f3 for llama.cpp

commit a818a46f3df4b6dee7f59f174283deaeae083c4b
Author: Clemens Wasser <clemens.wasser@gmail.com>
Date:   Sat Nov 21 13:20:22 2026 +0100

    sycl: add fused ADD, RMS_NORM, MUL and vectorize rms norm (#31753)

    * sycl: vectorize contiguous rms norm

    * sycl: add fused ADD, RMS_NORM, MUL

    * tests: test fused ADD, RMS_NORM, MUL at decode widths

diff --git a/ggml/src/ggml-sycl/fusion.cpp b/ggml/src/ggml-sycl/fusion.cpp
index 9b553829b..9ea6af6b0 100644
--- a/ggml/src/ggml-sycl/fusion.cpp
+++ b/ggml/src/ggml-sycl/fusion.cpp
@@ -121,6 +121,28 @@ bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializ
         return ggml_sycl_should_fuse_mul_mat_glu(gate, up, glu);
     }

+    // the residual sum stays materialised (a later residual add reads it), so ggml_can_fuse cannot
+    // express this either: the add and the mul are the subgraph's outputs
+    if (ops.size() == 3 && ops.begin()[0] == GGML_OP_ADD && ops.begin()[1] == GGML_OP_RMS_NORM &&
+        ops.begin()[2] == GGML_OP_MUL) {
+        if (!ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx, node_idx + 2 })) {
+            return false;
+        }
+
+        const ggml_tensor * add = cgraph->nodes[node_idx];
+        if (cgraph->nodes[node_idx + 1]->src[0] != add) {
+            return false;
+        }
+
+        // the fused kernel does not broadcast the add
+        if (add->src[0]->type != GGML_TYPE_F32 || add->src[1]->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32 ||
+            !ggml_are_same_shape(add->src[0], add) || !ggml_are_same_shape(add->src[1], add)) {
+            return false;
+        }
+
+        return ggml_sycl_can_fuse(cgraph, node_idx + 1, { GGML_OP_RMS_NORM, GGML_OP_MUL }, {});
+    }
+
     if (!ggml_can_fuse(cgraph, node_idx, ops)) {
         return false;
     }
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index 5b87fa024..3de5f1bd5 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -6708,6 +6708,12 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc
                 continue;
             }
         }
+        if (node->op == GGML_OP_ADD &&
+            ggml_sycl_can_fuse(cgraph, i, { GGML_OP_ADD, GGML_OP_RMS_NORM, GGML_OP_MUL }, {}) &&
+            ggml_sycl_op_add_rms_norm_fused(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2])) {
+            i += 2;
+            continue;
+        }
         if (node->op == GGML_OP_RMS_NORM &&
             ggml_sycl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) {
             ggml_sycl_op_rms_norm_fused_add(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]);
diff --git a/ggml/src/ggml-sycl/norm.cpp b/ggml/src/ggml-sycl/norm.cpp
index 36576e9c2..0cb30aa35 100644
--- a/ggml/src/ggml-sycl/norm.cpp
+++ b/ggml/src/ggml-sycl/norm.cpp
@@ -1,4 +1,6 @@
 #include "norm.hpp"
+
+#include "fusion.hpp"
 #include "ggml-sycl/common.hpp"
 #include "ggml-sycl/presets.hpp"

@@ -188,9 +190,28 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols,

     float tmp = 0.0f; // partial sum for thread in warp

-    for (int col = tid; col < ncols; col += block_size) {
-        const float xi = x[col * src_stride_col];
-        tmp += xi * xi;
+    // float4 path for contiguous, 16-byte aligned rows (decode), views and odd lengths stay strided
+    bool contig = src_stride_col == 1 && dst_stride_col == 1 && ncols % 4 == 0 && (uintptr_t) x % 16 == 0 &&
+                  (uintptr_t) dst % 16 == 0;
+    if constexpr (do_multiply) {
+        contig = contig && (uintptr_t) mul % 16 == 0;
+    }
+    if constexpr (do_add) {
+        contig = contig && (uintptr_t) add % 16 == 0;
+    }
+    const int n4 = ncols / 4;
+
+    if (contig) {
+        const sycl::float4 * x4 = reinterpret_cast<const sycl::float4 *>(x);
+        for (int i = tid; i < n4; i += block_size) {
+            const sycl::float4 v = x4[i];
+            tmp += sycl::dot(v, v);
+        }
+    } else {
+        for (int col = tid; col < ncols; col += block_size) {
+            const float xi = x[col * src_stride_col];
+            tmp += xi * xi;
+        }
     }

     // sum up partial sums
@@ -216,17 +237,37 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols,
     const float mean = tmp / ncols;
     const float scale = sycl::rsqrt(mean + eps);

-    for (int col = tid; col < ncols; col += block_size) {
-        if constexpr (do_multiply && do_add) {
-            dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col] + add[col];
-        } else if constexpr (do_multiply) {
-            dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col];
-        } else {
-            // folded epilogue of a fused GGML_OP_SCALE consumer (qwen35 GDN l2 norms);
-            // the explicit temporary keeps the float evaluation order identical to
-            // running rms_norm and scale as two separate kernels
-            const float v = scale * x[col * src_stride_col];
-            dst[col * dst_stride_col] = v * scale_mul;
+    if (contig) {
+        const sycl::float4 * x4 = reinterpret_cast<const sycl::float4 *>(x);
+        sycl::float4 *       d4 = reinterpret_cast<sycl::float4 *>(dst);
+        const sycl::float4 * m4 = reinterpret_cast<const sycl::float4 *>(mul);
+        const sycl::float4 * a4 = reinterpret_cast<const sycl::float4 *>(add);
+        for (int i = tid; i < n4; i += block_size) {
+            const sycl::float4 v = x4[i];
+            if constexpr (do_multiply && do_add) {
+                const sycl::float4 m = m4[i];
+                const sycl::float4 a = a4[i];
+                d4[i]                = scale * v * m + a;
+            } else if constexpr (do_multiply) {
+                const sycl::float4 m = m4[i];
+                d4[i]                = scale * v * m;
+            } else {
+                d4[i] = scale * v * scale_mul;
+            }
+        }
+    } else {
+        for (int col = tid; col < ncols; col += block_size) {
+            if constexpr (do_multiply && do_add) {
+                dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col] + add[col];
+            } else if constexpr (do_multiply) {
+                dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col];
+            } else {
+                // folded epilogue of a fused GGML_OP_SCALE consumer (qwen35 GDN l2 norms);
+                // the explicit temporary keeps the float evaluation order identical to
+                // running rms_norm and scale as two separate kernels
+                const float v             = scale * x[col * src_stride_col];
+                dst[col * dst_stride_col] = v * scale_mul;
+            }
         }
     }
 }
@@ -918,6 +959,127 @@ void ggml_sycl_op_rms_norm_fused_add(ggml_backend_sycl_context & ctx, ggml_tenso
         add_s01, add_s02, add_s03, add_nrows, add_nchannels, add_nsamples, eps, main_stream, ctx.device);
 }

+// ADD(a, b) -> RMS_NORM -> MUL(w) in one launch. The sum (read by a later residual add) and
+// normed * w are written after a barrier, as mul->data may alias a.
+static void add_rms_norm_mul_f32(const float *            a,
+                                 const float *            b,
+                                 float *                  sum,
+                                 const float *            mul,
+                                 float *                  dst,
+                                 const int                ncols,
+                                 const float              eps,
+                                 const sycl::nd_item<3> & item_ct1,
+                                 float *                  s_sum,
+                                 const int                block_size) {
+    const int    row = item_ct1.get_group(2);
+    const size_t off = (size_t) row * ncols;
+    const int    tid = item_ct1.get_local_id(2);
+    const int    n4  = ncols / 4;
+
+    const sycl::float4 * a4 = reinterpret_cast<const sycl::float4 *>(a + off);
+    const sycl::float4 * b4 = reinterpret_cast<const sycl::float4 *>(b + off);
+
+    sycl::float4 v[2];
+    float        tmp = 0.0f;
+#pragma unroll
+    for (int k = 0; k < 2; ++k) {
+        const int i = tid + k * block_size;
+        if (i < n4) {
+            const sycl::float4 va = a4[i];
+            const sycl::float4 vb = b4[i];
+            v[k]                  = va + vb;
+            tmp += sycl::dot(v[k], v[k]);
+        }
+    }
+
+    const int nwarps = block_size / WARP_SIZE;
+    tmp              = warp_reduce_sum(tmp, item_ct1);
+    if (block_size > WARP_SIZE) {
+        const auto sub_group = item_ct1.get_sub_group();
+        if (sub_group.get_local_linear_id() == 0) {
+            s_sum[sub_group.get_group_linear_id()] = tmp;
+        }
+        item_ct1.barrier(sycl::access::fence_space::local_space);
+        const size_t nreduce = ceil_div(nwarps, WARP_SIZE);
+        tmp                  = 0.0f;
+        for (size_t i = 0; i < nreduce; i += 1) {
+            tmp += s_sum[sub_group.get_local_linear_id() + i * WARP_SIZE];
+        }
+        tmp = warp_reduce_sum(tmp, item_ct1);
+    }
+
+    const float mean  = tmp / ncols;
+    const float scale = sycl::rsqrt(mean + eps);
+
+    sycl::float4 *       s4 = reinterpret_cast<sycl::float4 *>(sum + off);
+    sycl::float4 *       d4 = reinterpret_cast<sycl::float4 *>(dst + off);
+    const sycl::float4 * m4 = reinterpret_cast<const sycl::float4 *>(mul);
+#pragma unroll
+    for (int k = 0; k < 2; ++k) {
+        const int i = tid + k * block_size;
+        if (i < n4) {
+            const sycl::float4 m = m4[i];
+            const sycl::float4 u = scale * v[k] * m;
+            s4[i]                = v[k];
+            d4[i]                = u;
+        }
+    }
+}
+
+static void add_rms_norm_mul_f32_sycl(const float * a,
+                                      const float * b,
+                                      float *       sum,
+                                      const float * mul,
+                                      float *       dst,
+                                      const int     ncols,
+                                      const int     nrows,
+                                      const float   eps,
+                                      queue_ptr     stream,
+                                      int           device) {
+    const int work_group_size = ggml_sycl_info().max_work_group_sizes[device];
+    GGML_ASSERT(work_group_size % (WARP_SIZE * WARP_SIZE) == 0);
+    const sycl::range<3> global_dims(1, 1, nrows);
+    const sycl::range<3> block_dims(1, 1, work_group_size);
+    stream->submit([&](sycl::handler & cgh) {
+        sycl::local_accessor<float, 1> s_sum(sycl::range<1>(work_group_size / WARP_SIZE), cgh);
+        cgh.parallel_for(sycl::nd_range<3>(global_dims * block_dims, block_dims),
+                         [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                             add_rms_norm_mul_f32(a, b, sum, mul, dst, ncols, eps, item_ct1, get_pointer(s_sum),
+                                                  work_group_size);
+                         });
+    });
+}
+
+bool ggml_sycl_op_add_rms_norm_fused(ggml_backend_sycl_context & ctx,
+                                     ggml_tensor *               add,
+                                     ggml_tensor *               rms,
+                                     ggml_tensor *               mul) {
+    ggml_tensor * a     = add->src[0];
+    ggml_tensor * b     = add->src[1];
+    ggml_tensor * w     = mul->src[0] == rms ? mul->src[1] : mul->src[0];
+    const int64_t ncols = add->ne[0];
+    const int     wg    = ggml_sycl_info().max_work_group_sizes[ctx.device];
+    // whole rows of float4, at most two per work-item, from 16-byte aligned contiguous tensors
+    if (add->ne[2] != 1 || add->ne[3] != 1 || ncols < 1024 || ncols % 4 != 0 || ncols > 8 * (int64_t) wg ||
+        w->ne[1] != 1) {
+        return false;
+    }
+    for (const ggml_tensor * t : { a, b, add, mul, w }) {
+        if (!ggml_is_contiguous(t) || (uintptr_t) t->data % 16 != 0) {
+            return false;
+        }
+    }
+    float eps = 0.0f;
+    memcpy(&eps, rms->op_params, sizeof(float));
+    if (!(eps >= 0.0f)) {
+        return false;
+    }
+    add_rms_norm_mul_f32_sycl((const float *) a->data, (const float *) b->data, (float *) add->data,
+                              (const float *) w->data, (float *) mul->data, (int) ncols, (int) add->ne[1], eps,
+                              ctx.stream(), ctx.device);
+    return true;
+}
+
 void ggml_sycl_op_rms_norm_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
     scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);

diff --git a/ggml/src/ggml-sycl/norm.hpp b/ggml/src/ggml-sycl/norm.hpp
index fb667ac6b..59a9ffd6f 100644
--- a/ggml/src/ggml-sycl/norm.hpp
+++ b/ggml/src/ggml-sycl/norm.hpp
@@ -25,6 +25,11 @@ void ggml_sycl_op_rms_norm_scale_fused(ggml_backend_sycl_context& ctx, ggml_tens

 void ggml_sycl_op_rms_norm_fused_add(ggml_backend_sycl_context& ctx, ggml_tensor* dst, ggml_tensor* mul_tensor, ggml_tensor* add_tensor);

+bool ggml_sycl_op_add_rms_norm_fused(ggml_backend_sycl_context & ctx,
+                                     ggml_tensor *               add,
+                                     ggml_tensor *               rms,
+                                     ggml_tensor *               mul);
+
 void ggml_sycl_op_rms_norm_back(ggml_backend_sycl_context& ctx, ggml_tensor* dst);

 void ggml_sycl_op_group_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst);
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 11bfa42a9..fbf5c12d3 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -4351,6 +4351,52 @@ struct test_add_rms_norm : public test_case {
     }
 };

+// GGML_OP_ADD + GGML_OP_RMS_NORM + GGML_OP_MUL (fused operation)
+struct test_add_rms_norm_mul : public test_case {
+    const std::array<int64_t, 4> ne;
+    const float                  eps;
+
+    ggml_tensor * sum = nullptr;
+    ggml_tensor * out = nullptr;
+
+    std::string op_desc(ggml_tensor * t) override {
+        GGML_UNUSED(t);
+        return "ADD_RMS_NORM_MUL";
+    }
+
+    bool run_whole_graph() override { return true; }
+
+    std::vector<ggml_tensor *> fusion_test_nodes() override { return { sum, out }; }
+
+    std::string vars() override { return VARS_TO_STR2(ne, eps); }
+
+    test_add_rms_norm_mul(std::array<int64_t, 4> ne = { 4096, 1, 1, 1 }, float eps = 1e-6f) : ne(ne), eps(eps) {}
+
+    ggml_tensor * build_graph(ggml_context * ctx) override {
+        ggml_tensor * a = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data());
+        ggml_tensor * b = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data());
+        ggml_tensor * w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]);
+        ggml_set_name(a, "a");
+        ggml_set_name(b, "b");
+        ggml_set_name(w, "w");
+
+        sum = ggml_add(ctx, a, b);
+        ggml_set_name(sum, "sum");
+        ggml_tensor * normed = ggml_mul(ctx, ggml_rms_norm(ctx, sum, eps), w);
+        ggml_set_name(normed, "normed");
+
+        out = ggml_add(ctx, normed, sum);
+        ggml_set_name(out, "out");
+        return out;
+    }
+
+    void initialize_tensors(ggml_context * ctx) override {
+        for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
+            init_tensor_uniform(t, -10.f, 10.f);
+        }
+    }
+};
+
 // GGML_OP_UNARY(RELU) + GGML_OP_SQR (fused operation)
 struct test_relu_sqr : public test_case {
     const ggml_type type;
@@ -11801,6 +11847,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
         }
     }

+    // decode-shaped residual rows: below, at and above the fused path's limits, odd row counts, batches
+    for (int64_t n : { 1020, 1024, 4096, 5120, 8192, 8196, 16384 }) {
+        for (int64_t rows : { 1, 3, 4 }) {
+            test_cases.emplace_back(new test_add_rms_norm_mul({ n, rows, 1, 1 }, 1e-6f));
+        }
+        test_cases.emplace_back(new test_add_rms_norm_mul({ n, 2, 3, 1 }, 1e-6f));
+    }
     for (uint32_t n : {64, 1025}) {
         test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, false, false));
         test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, true, false));