Commit a83d68b56 for llama.cpp
commit a83d68b562b19fba1f8836edc93eb76d9ca1ce25
Author: Tarek Dakhran <tarek@liquid.ai>
Date: Fri Nov 20 19:55:44 2026 +0100
metal : allow the norm chain in either operand of the fused MUL/ADD (#32300)
Assisted-by: Claude Opus 5.5
diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggml-metal-fusion.cpp
index 6766b4bd9..7b55a301e 100644
--- a/ggml/src/ggml-metal/ggml-metal-fusion.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-fusion.cpp
@@ -135,17 +135,15 @@ static bool ggml_metal_fusion_check_norm(
}
for (int j = 1; j < (int) fusion->ops.size(); j++) {
- // the fused MUL/ADD must read the previous node as src0
- if (nodes[j]->src[0] != nodes[j - 1]) {
- return false;
- }
+ // MUL/ADD are commutative, the previous node can be either operand
+ const ggml_tensor * src = nodes[j]->src[0] == nodes[j - 1] ? nodes[j]->src[1] : nodes[j]->src[0];
// the weight/bias must have the same row width as the norm input
- if (nodes[j]->src[1]->ne[0] != nodes[0]->ne[0]) {
+ if (src->ne[0] != nodes[0]->ne[0]) {
return false;
}
- if (!ggml_is_contiguous_rows(nodes[j]->src[1])) {
+ if (!ggml_is_contiguous_rows(src)) {
return false;
}
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp
index 1ae7f3d5d..a7a1ef61e 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp
@@ -4698,16 +4698,17 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) {
for (int i = 1; i < n_fuse; i++) {
const ggml_tensor * fn = ctx->node(idx + i);
+ const ggml_tensor * fs = fn->src[0] == ctx->node(idx + i - 1) ? fn->src[1] : fn->src[0];
- bid_fuse[i - 1] = ggml_metal_get_buffer_id(fn->src[1]);
+ bid_fuse[i - 1] = ggml_metal_get_buffer_id(fs);
- args.nef1[i] = fn->src[1]->ne[1];
- args.nef2[i] = fn->src[1]->ne[2];
- args.nef3[i] = fn->src[1]->ne[3];
+ args.nef1[i] = fs->ne[1];
+ args.nef2[i] = fs->ne[2];
+ args.nef3[i] = fs->ne[3];
- args.nbf1[i] = fn->src[1]->nb[1];
- args.nbf2[i] = fn->src[1]->nb[2];
- args.nbf3[i] = fn->src[1]->nb[3];
+ args.nbf1[i] = fs->nb[1];
+ args.nbf2[i] = fs->nb[2];
+ args.nbf3[i] = fs->nb[3];
}
if (debug_fusion > 1) {
diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv
index 295740c7c..b939d6683 100644
--- a/tests/fusion/MTL.csv
+++ b/tests/fusion/MTL.csv
@@ -63,7 +63,7 @@ deepseek4 ,0 ,any ,MUL+ADD , 8
deepseek4 ,0 ,any ,RMS_NORM+MUL , 20
deepseek4 ,0 ,any ,UNARY+SQRT+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 4
deepseek41 ,0 ,any ,MUL+ADD , 12
-deepseek41 ,0 ,any ,RMS_NORM+MUL , 29
+deepseek41 ,0 ,any ,RMS_NORM+MUL , 31
deepseek41 ,0 ,any ,UNARY+SQRT+ADD+ARGSORT+GET_ROWS+SUM_ROWS+SCALE+DIV, 6
domynedge ,0 ,any ,RMS_NORM+MUL , 8
domynedge ,0 ,any ,RMS_NORM+MUL+ADD , 3
@@ -101,8 +101,8 @@ gemma2 ,0 ,any ,RMS_NORM+MUL , 5
gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4
gemma3 ,0 ,any ,RMS_NORM+MUL , 9
gemma3 ,0 ,any ,RMS_NORM+MUL+ADD , 4
-gemma4 ,0 ,any ,RMS_NORM+MUL , 47
-gemma4 ,0 ,any ,RMS_NORM+MUL+ADD , 20
+gemma4 ,0 ,any ,RMS_NORM+MUL , 37
+gemma4 ,0 ,any ,RMS_NORM+MUL+ADD , 30
glm-dsa ,0 ,any ,ADD+ADD , 1
glm-dsa ,0 ,any ,MUL+ADD , 2
glm-dsa ,0 ,any ,NORM+MUL+ADD , 2
@@ -258,7 +258,8 @@ muse-glimmer ,0 ,any ,RMS_NORM+MUL , 10
muse-glimmer ,0 ,any ,RMS_NORM+MUL+ADD , 3
muse-spark ,0 ,prefill ,MUL+ADD , 2
muse-spark ,0 ,decode ,MUL+ADD , 4
-muse-spark ,0 ,any ,RMS_NORM+MUL , 14
+muse-spark ,0 ,any ,RMS_NORM+MUL , 11
+muse-spark ,0 ,any ,RMS_NORM+MUL+ADD , 3
muse-spark ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
nanbeige ,0 ,any ,RMS_NORM+MUL , 5
nemotron ,0 ,any ,NORM+MUL+ADD , 5
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 42e356ae0..11bfa42a9 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -4135,6 +4135,7 @@ struct test_rms_norm_mul_add : public test_case {
const bool weight_broadcast;
const ggml_type type_w; // type of b and d, GGML_TYPE_COUNT -> same as a
const ggml_type type_add; // type of c, GGML_TYPE_COUNT -> same as b
+ const bool swap; // the norm chain is the second operand of MUL and ADD
std::string op_desc(ggml_tensor * t) override {
GGML_UNUSED(t);
@@ -4151,6 +4152,9 @@ struct test_rms_norm_mul_add : public test_case {
if (type_add != GGML_TYPE_COUNT) {
s += "," + VAR_TO_STR(type_add);
}
+ if (swap) {
+ s += "," + VAR_TO_STR(swap);
+ }
return s;
}
@@ -4158,9 +4162,10 @@ struct test_rms_norm_mul_add : public test_case {
std::array<int64_t, 4> ne = {64, 5, 4, 3},
float eps = 1e-6f, bool broadcast = false, bool multi_add = false, bool post_mul = false,
bool alias_rms_input = false, bool weight_broadcast = false, ggml_type type_w = GGML_TYPE_COUNT,
- ggml_type type_add = GGML_TYPE_COUNT)
+ ggml_type type_add = GGML_TYPE_COUNT, bool swap = false)
: type(type), ne(ne), eps(eps), broadcast(broadcast), multi_add(multi_add), post_mul(post_mul),
- alias_rms_input(alias_rms_input), weight_broadcast(weight_broadcast), type_w(type_w), type_add(type_add) {}
+ alias_rms_input(alias_rms_input), weight_broadcast(weight_broadcast), type_w(type_w), type_add(type_add),
+ swap(swap) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
std::array<int64_t, 4> broadcast_dims = {ne[0]*2, ne[1]*3, ne[2]*3, ne[3]*4};
@@ -4191,8 +4196,9 @@ struct test_rms_norm_mul_add : public test_case {
if (multi_add) {
a = ggml_add(ctx, ggml_add(ctx, a, b), c);
}
- ggml_tensor * mul = ggml_mul(ctx, ggml_rms_norm(ctx, a, eps), b);
- ggml_tensor * out = alias_rms_input ? ggml_add_inplace(ctx, a, mul) : ggml_add(ctx, mul, c);
+ ggml_tensor * norm = ggml_rms_norm(ctx, a, eps);
+ ggml_tensor * mul = swap ? ggml_mul(ctx, b, norm) : ggml_mul(ctx, norm, b);
+ ggml_tensor * out = alias_rms_input ? ggml_add_inplace(ctx, a, mul) : swap ? ggml_add(ctx, c, mul) : ggml_add(ctx, mul, c);
if (post_mul) {
out = ggml_mul(ctx, out, d);
}
@@ -11771,6 +11777,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
for (uint32_t n : { 64, 1025 }) {
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false));
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true));
+ test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false, false, false, false, false, GGML_TYPE_COUNT, GGML_TYPE_COUNT, true));
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_BF16, { n, 5, 4, 3 }, eps, false, false, false, false, false, GGML_TYPE_F32));
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_BF16, { n, 5, 4, 3 }, eps, true, false, false, false, false, GGML_TYPE_F32));
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_BF16, { n, 5, 4, 3 }, eps, true, false, false, false, false, GGML_TYPE_F32, GGML_TYPE_BF16));