Commit 5a638aedb for llama.cpp
commit 5a638aedb7c4352376d1e565a9a420b85f604628
Author: cqderek <cqiang@qti.qualcomm.com>
Date: Thu Oct 15 15:07:45 2026 +0800
cpu: add F16 SWIGLU_OAI reference implementation and validation (#29200)
* cpu: add F16 SWIGLU_OAI reference implementation and F16 activation op test coverage
Adds ggml_compute_forward_swiglu_oai_f16() to the CPU backend, mirroring
the existing F32 path, and extends test-backend-ops with F16 test cases
for SWIGLU_OAI so downstream backends(e.g. Hexagon) can be CPU-diffed
for correctness.
* ggml-openvino: fix f16 SWIGLU_OAI type mismatch in alpha/one constants
translate_glu_swiglu_oai creates alpha_const and one as f32 constants
unconditionally, so f16 inputs hit a type mismatch in Swish/Add on GPU
("Swish inputs must have the same type but they are: f16 and f32").
translate_glu_swiglu_clamp already upcasts to f32 and converts back at
the end; apply the same pattern here.
---------
Co-authored-by: ravi9 <ravi.panchumarthy@intel.com>
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index c7e7c47a7..3b39683d5 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -3548,6 +3548,73 @@ static void ggml_compute_forward_swiglu_oai_f32(
}
}
+static void ggml_compute_forward_swiglu_oai_f16(
+ const ggml_compute_params * params,
+ ggml_tensor * dst) {
+
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+ char * src0_d = (char *) src0->data;
+ char * src1_d = (char *) (src1 ? src1->data : src0->data);
+ const size_t src0_o = src0->nb[1];
+ const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
+
+ GGML_ASSERT(ggml_is_contiguous_1(src0));
+ GGML_ASSERT(ggml_is_contiguous_1(dst));
+
+ if (src1) {
+ GGML_ASSERT(ggml_is_contiguous_1(src1));
+ GGML_ASSERT(src0->type == src1->type);
+ }
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
+ const int nr = ggml_nrows(src0);
+
+ GGML_ASSERT(dst->ne[0] == nc);
+ GGML_ASSERT(ggml_nrows(dst) == nr);
+
+ const int32_t swapped = ggml_get_op_params_i32(dst, 1);
+ const float alpha = ggml_get_op_params_f32(dst, 2);
+ const float limit = ggml_get_op_params_f32(dst, 3);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);
+ ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);
+ ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1]));
+
+ if (!src1) {
+ src0_p += swapped ? nc : 0;
+ src1_p += swapped ? 0 : nc;
+ }
+
+ for (int k = 0; k < nc; k++) {
+ const float x = std::min(GGML_CPU_FP16_TO_FP32(src0_p[k]), limit);
+ const float y = std::clamp(GGML_CPU_FP16_TO_FP32(src1_p[k]), -limit, limit);
+ const float out_glu = x / (1.f + expf(alpha * (-x)));
+ dst_p[k] = GGML_CPU_FP32_TO_FP16(out_glu * (y + 1.f));
+ }
+
+#ifndef NDEBUG
+ for (int k = 0; k < nc; k++) {
+ const float x = GGML_CPU_FP16_TO_FP32(dst_p[k]);
+ GGML_UNUSED(x);
+ assert(!isnan(x));
+ assert(!isinf(x));
+ }
+#endif // NDEBUG
+ }
+}
+
static void ggml_compute_forward_swiglu_oai(
const ggml_compute_params * params,
ggml_tensor * dst) {
@@ -3559,6 +3626,10 @@ static void ggml_compute_forward_swiglu_oai(
{
ggml_compute_forward_swiglu_oai_f32(params, dst);
} break;
+ case GGML_TYPE_F16:
+ {
+ ggml_compute_forward_swiglu_oai_f16(params, dst);
+ } break;
default:
{
GGML_ABORT("fatal error");
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 9fe618f85..7c45d0d1f 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -5474,6 +5474,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
(ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_OAI || ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_CLAMP)) {
return false;
}
+ if (op->src[0]->type == GGML_TYPE_F16 && ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_OAI) {
+ return false;
+ }
return ggml_is_contiguous_1(op->src[0]);
default:
return false;
diff --git a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp
index 7eea81d96..2c206a597 100644
--- a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp
@@ -75,6 +75,15 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) {
const float alpha = reinterpret_cast<const float *>(params)[2];
const float limit = reinterpret_cast<const float *>(params)[3];
+ // Compute in f32: alpha_const/one are f32, and Swish/Add need matching input types.
+ auto output_type = context.get_output_type();
+ if (src0.get_element_type() != ov::element::f32) {
+ src0 = std::make_shared<ov::op::v0::Convert>(src0, ov::element::f32);
+ }
+ if (src1.get_element_type() != ov::element::f32) {
+ src1 = std::make_shared<ov::op::v0::Convert>(src1, ov::element::f32);
+ }
+
auto gate = std::make_shared<ov::op::v0::Clamp>(src0, -std::numeric_limits<float>::infinity(), limit);
auto alpha_const = ov::op::v0::Constant::create(ov::element::f32, {}, {alpha});
auto out_glu = std::make_shared<ov::op::v4::Swish>(gate, alpha_const);
@@ -82,7 +91,10 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) {
auto up = std::make_shared<ov::op::v0::Clamp>(src1, -limit, limit);
auto one = ov::op::v0::Constant::create(ov::element::f32, {}, {1.0f});
auto up_plus_one = std::make_shared<ov::op::v1::Add>(up, one);
- auto res = std::make_shared<ov::op::v1::Multiply>(out_glu, up_plus_one);
+ ov::Output<ov::Node> res = std::make_shared<ov::op::v1::Multiply>(out_glu, up_plus_one);
+ if (res.get_element_type() != output_type) {
+ res = std::make_shared<ov::op::v0::Convert>(res, output_type);
+ }
return rename_outputs_with_suffix({res}, context.get_name());
}
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index e61823084..93b30d8d2 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -2308,6 +2308,12 @@ struct test_swiglu_oai : public test_case {
init_tensor_uniform(t, -150.f, 150.f);
}
}
+
+ double max_nmse_err() override {
+ // F16 epsilon ~ 9.7e-4: relax tolerance vs the F32 default to absorb the
+ // natural roundoff drift from the exp()/clamp() chain computed in F16.
+ return type == GGML_TYPE_F16 ? 5e-6 : 1e-7;
+ }
};
struct test_swiglu_clamp : public test_case {
@@ -2365,6 +2371,12 @@ struct test_swiglu_clamp : public test_case {
init_tensor_uniform(t, -150.f, 150.f);
}
}
+
+ double max_nmse_err() override {
+ // F16 epsilon ~ 9.7e-4: relax tolerance vs the F32 default to absorb the
+ // natural roundoff drift from the sigmoid()/clamp() chain computed in F16.
+ return type == GGML_TYPE_F16 ? 5e-6 : 1e-7;
+ }
};
// GGML_OP_GET_ROWS
@@ -9747,10 +9759,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
- for (int v : {0, 1}) {
- for (float alpha : {.5f, 1.702f}) {
- for (float limit : {2.0f, 7.0f}) {
- test_cases.emplace_back(new test_swiglu_oai(GGML_TYPE_F32, { 128, 2, 2, 2 }, v, alpha, limit));
+ for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
+ for (int v : {0, 1}) {
+ for (float alpha : {.5f, 1.702f}) {
+ for (float limit : {2.0f, 7.0f}) {
+ test_cases.emplace_back(new test_swiglu_oai(type, { 128, 2, 2, 2 }, v, alpha, limit));
+ }
}
}
}