Commit ad50351f5 for llama.cpp
commit ad50351f5092436f312d98d25689207df25f4802
Author: gianni-cor <gianfranco.cordella@tether.io>
Date: Sun Nov 22 12:33:13 2026 +0100
ggml : add SIGMOID backward pass (#32126)
* ggml : add SIGMOID backward pass
Sigmoid is used in Qwen3.5 hybrid models and had no backward pass.
Bump RPC protocol patch version for the new op.
Signed-off-by: gianni-cor <gianfranco.cordella@tether.io>
* replace assert with ggml_assert
Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
* ggml : keep sigmoid backward checks in release builds
Assisted-by: Codex
* ggml : preserve existing operation IDs when adding SIGMOID_BACK
Assisted-by: Codex
---------
Signed-off-by: gianni-cor <gianfranco.cordella@tether.io>
Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
diff --git a/ggml/include/ggml-rpc.h b/ggml/include/ggml-rpc.h
index 168ee774e..062d5004e 100644
--- a/ggml/include/ggml-rpc.h
+++ b/ggml/include/ggml-rpc.h
@@ -8,10 +8,10 @@ extern "C" {
#define RPC_PROTO_MAJOR_VERSION 10
#define RPC_PROTO_MINOR_VERSION 0
-#define RPC_PROTO_PATCH_VERSION 1
+#define RPC_PROTO_PATCH_VERSION 2
#ifdef __cplusplus
-static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
+static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
#endif
#define GGML_RPC_MAX_SERVERS 16
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h
index 7c5b1e552..2f466e2e4 100644
--- a/ggml/include/ggml.h
+++ b/ggml/include/ggml.h
@@ -433,6 +433,8 @@ extern "C" {
GGML_OP_SLEEP,
+ GGML_OP_SIGMOID_BACK,
+
GGML_OP_COUNT,
};
@@ -1038,6 +1040,13 @@ extern "C" {
struct ggml_tensor * a,
struct ggml_tensor * b);
+ // a - dy
+ // b - x
+ GGML_API struct ggml_tensor * ggml_sigmoid_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
// hardswish(x) = x * relu6(x + 3) / 6
GGML_API struct ggml_tensor * ggml_hardswish(
struct ggml_context * ctx,
diff --git a/ggml/src/ggml-alloc.c b/ggml/src/ggml-alloc.c
index 65d2bea27..90082d861 100644
--- a/ggml/src/ggml-alloc.c
+++ b/ggml/src/ggml-alloc.c
@@ -35,6 +35,7 @@ bool ggml_op_can_inplace(enum ggml_op op) {
case GGML_OP_ROPE:
case GGML_OP_ROPE_BACK:
case GGML_OP_SILU_BACK:
+ case GGML_OP_SIGMOID_BACK:
case GGML_OP_RMS_NORM:
case GGML_OP_RMS_NORM_BACK:
case GGML_OP_CLAMP:
diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
index c4e37ed46..54d85b597 100644
--- a/ggml/src/ggml-backend-meta.cpp
+++ b/ggml/src/ggml-backend-meta.cpp
@@ -971,7 +971,8 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
case GGML_OP_CONCAT: {
split_state = handle_concat(src_ss);
} break;
- case GGML_OP_SILU_BACK: {
+ case GGML_OP_SILU_BACK:
+ case GGML_OP_SIGMOID_BACK: {
split_state = handle_generic(src_ss, /*scalar_only =*/ false);
} break;
case GGML_OP_NORM:
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
index 33193a5af..ba075ff8d 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -1884,6 +1884,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
{
ggml_compute_forward_silu_back(params, tensor);
} break;
+ case GGML_OP_SIGMOID_BACK:
+ {
+ ggml_compute_forward_sigmoid_back(params, tensor);
+ } break;
case GGML_OP_NORM:
{
ggml_compute_forward_norm(params, tensor);
@@ -2392,6 +2396,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
}
break;
case GGML_OP_SILU_BACK:
+ case GGML_OP_SIGMOID_BACK:
case GGML_OP_MUL:
case GGML_OP_DIV:
case GGML_OP_NORM:
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index cd8134740..f984f13a6 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -2898,6 +2898,118 @@ void ggml_compute_forward_silu_back(
}
}
+
+// ggml_compute_forward_sigmoid_back
+
+static void ggml_compute_forward_sigmoid_back_f32(
+ const ggml_compute_params * params,
+ ggml_tensor * dst) {
+
+ const ggml_tensor * grad = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_is_contiguous_1(grad));
+ GGML_ASSERT(ggml_is_contiguous_1(src1));
+ GGML_ASSERT(ggml_is_contiguous_1(dst));
+ GGML_ASSERT(ggml_are_same_shape(src1, dst));
+ GGML_ASSERT(ggml_are_same_shape(src1, grad));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nc = src1->ne[0];
+ const int nr = ggml_nrows(src1);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ ggml_vec_sigmoid_backward_f32(nc,
+ (float *) ((char *) dst->data + i1*( dst->nb[1])),
+ (float *) ((char *) src1->data + i1*(src1->nb[1])),
+ (float *) ((char *) grad->data + i1*(grad->nb[1])));
+
+#ifndef NDEBUG
+ for (int k = 0; k < nc; k++) {
+ const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];
+ GGML_UNUSED(x);
+ assert(!isnan(x));
+ assert(!isinf(x));
+ }
+#endif // NDEBUG
+ }
+}
+
+static void ggml_compute_forward_sigmoid_back_f16(
+ const ggml_compute_params * params,
+ ggml_tensor * dst) {
+
+ const ggml_tensor * grad = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_is_contiguous_1(grad));
+ GGML_ASSERT(ggml_is_contiguous_1(src1));
+ GGML_ASSERT(ggml_is_contiguous_1(dst));
+ GGML_ASSERT(ggml_are_same_shape(src1, dst));
+ GGML_ASSERT(ggml_are_same_shape(src1, grad));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nc = src1->ne[0];
+ const int nr = ggml_nrows(src1);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ ggml_vec_sigmoid_backward_f16(nc,
+ (ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])),
+ (ggml_fp16_t *) ((char *) src1->data + i1*(src1->nb[1])),
+ (ggml_fp16_t *) ((char *) grad->data + i1*(grad->nb[1])));
+
+#ifndef NDEBUG
+ for (int k = 0; k < nc; k++) {
+ const float x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];
+ const float v = GGML_CPU_FP16_TO_FP32(x);
+ GGML_UNUSED(v);
+ assert(!isnan(v));
+ assert(!isinf(v));
+ }
+#endif // NDEBUG
+ }
+}
+
+void ggml_compute_forward_sigmoid_back(
+ const ggml_compute_params * params,
+ ggml_tensor * dst) {
+
+ const ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_sigmoid_back_f32(params, dst);
+ } break;
+ case GGML_TYPE_F16:
+ {
+ ggml_compute_forward_sigmoid_back_f16(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
// ggml_compute_forward_reglu
// row loop shared by the BF16 GLU ops
diff --git a/ggml/src/ggml-cpu/ops.h b/ggml/src/ggml-cpu/ops.h
index 3a4fa0707..22017df70 100644
--- a/ggml/src/ggml-cpu/ops.h
+++ b/ggml/src/ggml-cpu/ops.h
@@ -41,6 +41,7 @@ void ggml_compute_forward_repeat(const struct ggml_compute_params * params, stru
void ggml_compute_forward_repeat_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_concat(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_silu_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+void ggml_compute_forward_sigmoid_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_rms_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_rms_norm_mul_fused(const struct ggml_compute_params * params, struct ggml_tensor * dst_rms_norm, struct ggml_tensor * dst_mul);
diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h
index 4aac5e966..0b8ac5d51 100644
--- a/ggml/src/ggml-cpu/vec.h
+++ b/ggml/src/ggml-cpu/vec.h
@@ -1536,6 +1536,29 @@ inline static void ggml_vec_silu_backward_f16(const int n, ggml_fp16_t * dx, con
}
}
+inline static float ggml_sigmoid_backward_f32(float x, float dy) {
+ const float s = 1.0f/(1.0f + expf(-x));
+ return dy*s*(1.0f - s);
+}
+
+inline static ggml_fp16_t ggml_sigmoid_backward_f16(ggml_fp16_t x, ggml_fp16_t dy) {
+ const float v = GGML_CPU_FP16_TO_FP32(x);
+ const float s = 1.0f/(1.0f + expf(-v));
+ return GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(dy)*s*(1.0f - s));
+}
+
+inline static void ggml_vec_sigmoid_backward_f32(const int n, float * dx, const float * x, const float * dy) {
+ for (int i = 0; i < n; ++i) {
+ dx[i] = ggml_sigmoid_backward_f32(x[i], dy[i]);
+ }
+}
+
+inline static void ggml_vec_sigmoid_backward_f16(const int n, ggml_fp16_t * dx, const ggml_fp16_t * x, const ggml_fp16_t * dy) {
+ for (int i = 0; i < n; ++i) {
+ dx[i] = ggml_sigmoid_backward_f16(x[i], dy[i]);
+ }
+}
+
inline static void ggml_vec_reglu_f32 (const int n, float * y, const float * x, const float * g) {
for (int i = 0; i < n; ++i) {
y[i] = (x[i] > 0.f) ? x[i] * g[i] : 0.f;
diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c
index b47c5addf..3389dcd63 100644
--- a/ggml/src/ggml.c
+++ b/ggml/src/ggml.c
@@ -1116,9 +1116,11 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
"GLU",
"SLEEP",
+
+ "SIGMOID_BACK",
};
-static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT != 100");
+static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT != 101");
static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
"none",
@@ -1231,9 +1233,11 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
"glu(x)",
"sleep(x)",
+
+ "sigmoid_back(x)",
};
-static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT != 100");
+static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT != 101");
static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");
@@ -2866,6 +2870,21 @@ struct ggml_tensor * ggml_silu_back(
return result;
}
+// ggml_sigmoid_back
+
+struct ggml_tensor * ggml_sigmoid_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b) {
+ struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
+
+ result->op = GGML_OP_SIGMOID_BACK;
+ result->src[0] = a;
+ result->src[1] = b;
+
+ return result;
+}
+
// ggml hardswish
struct ggml_tensor * ggml_hardswish(
@@ -7148,6 +7167,11 @@ static void ggml_compute_backward(
ggml_add_or_set(ctx, cgraph, isrc0, ggml_silu_back(ctx, grad, src0));
}
} break;
+ case GGML_UNARY_OP_SIGMOID: {
+ if (src0_needs_grads) {
+ ggml_add_or_set(ctx, cgraph, isrc0, ggml_sigmoid_back(ctx, grad, src0));
+ }
+ } break;
case GGML_UNARY_OP_EXP: {
if (src0_needs_grads) {
ggml_add_or_set(ctx, cgraph, isrc0, ggml_mul(ctx, tensor, grad));
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 9b8d7015c..91ff4c86a 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -2176,7 +2176,7 @@ struct test_unary : public test_case {
ggml_tensor * build_graph(ggml_context * ctx) override {
const bool grad_supported = op == GGML_UNARY_OP_ABS || op == GGML_UNARY_OP_SGN || op == GGML_UNARY_OP_NEG ||
op == GGML_UNARY_OP_STEP || op == GGML_UNARY_OP_RELU || op == GGML_UNARY_OP_SILU ||
- op == GGML_UNARY_OP_EXPM1 || op == GGML_UNARY_OP_SOFTPLUS;
+ op == GGML_UNARY_OP_EXPM1 || op == GGML_UNARY_OP_SOFTPLUS || op == GGML_UNARY_OP_SIGMOID;
ggml_tensor * a;
if (v & 1) {
@@ -2218,6 +2218,12 @@ struct test_unary : public test_case {
max = 10.f;
}
+ // saturated sigmoid inputs give zero analytic and numeric gradients, which the grad check cannot compare
+ if (mode == MODE_GRAD && op == GGML_UNARY_OP_SIGMOID) {
+ min = -2.f;
+ max = 2.f;
+ }
+
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
// test extended range of values to check for NaNs in GELU
init_tensor_uniform(t, min, max);
@@ -2225,7 +2231,16 @@ struct test_unary : public test_case {
}
float grad_eps() override {
- return 15.0f;
+ return op == GGML_UNARY_OP_SIGMOID ? 0.25f : 15.0f;
+ }
+
+ bool grad_precise() override {
+ return op == GGML_UNARY_OP_SIGMOID;
+ }
+
+ // f32 rounding of a large summed loss exceeds the tolerance, so only the small sigmoid case is grad checked
+ int64_t grad_nmax() override {
+ return op == GGML_UNARY_OP_SIGMOID ? 64 : test_case::grad_nmax();
}
std::vector<float> grad_expect() override {
@@ -3898,6 +3913,37 @@ struct test_silu_back : public test_case {
}
};
+// GGML_OP_SIGMOID_BACK
+struct test_sigmoid_back : public test_case {
+ const ggml_type type;
+ const std::array<int64_t, 4> ne;
+
+ std::string vars() override {
+ return VARS_TO_STR2(type, ne);
+ }
+
+ test_sigmoid_back(ggml_type type = GGML_TYPE_F32,
+ std::array<int64_t, 4> ne = {64, 5, 4, 3})
+ : type(type), ne(ne) {}
+
+ ggml_tensor * build_graph(ggml_context * ctx) override {
+ ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data());
+ ggml_set_name(a, "a");
+
+ ggml_tensor * grad = ggml_new_tensor(ctx, type, 4, ne.data());
+ ggml_set_name(grad, "grad");
+
+ ggml_tensor * out = ggml_sigmoid_back(ctx, a, grad);
+ ggml_set_name(out, "out");
+
+ return out;
+ }
+
+ bool grad_precise() override {
+ return true;
+ }
+};
+
// GGML_OP_NORM
struct test_norm : public test_case {
const ggml_type type;
@@ -10781,6 +10827,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
}
+ test_cases.emplace_back(new test_unary(GGML_UNARY_OP_SIGMOID, GGML_TYPE_F32, { 16, 4, 1, 1 }));
// fused relu + sqr (squared ReLU)
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
@@ -11761,6 +11808,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_scale(GGML_TYPE_F16, {10, 10, 10, 10}, 2.0f, 1.0f, false, 1)); // view test
test_cases.emplace_back(new test_softcap(GGML_TYPE_F32, {10, 10, 10, 10}, 50.0f));
test_cases.emplace_back(new test_silu_back());
+ test_cases.emplace_back(new test_sigmoid_back());
for (float eps : { 0.0f, 1e-6f, 1e-4f, 1e-1f, 10.f }) {
for (uint32_t n : { 64, 1025 }) {