Commit 59fc5a1ca for llama.cpp
commit 59fc5a1ca3842241dd53617ae2ae030c1a015061
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Sat Sep 19 11:27:30 2026 +0300
metal : support qwen4exp hc ops (#29000)
Add support for the new DSV4 HC op variants used by qwen4exp:
- hc_pre with per-element sigmoid gate (gated variant)
- hc_post with identity mixing (comb == nullptr)
Assisted-by: pi:llama.cpp/Qwen3.8-27B
diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp
index b510cb957..0dcfad3af 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp
@@ -496,14 +496,29 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexe
return res;
}
-ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_metal_library_t lib, ggml_op op) {
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_metal_library_t lib, const ggml_tensor * op) {
const char * name = nullptr;
- switch (op) {
- case GGML_OP_DSV4_HC_COMB: name = "kernel_dsv4_hc_comb_f32"; break;
- case GGML_OP_DSV4_HC_PRE: name = "kernel_dsv4_hc_pre_f32"; break;
- case GGML_OP_DSV4_HC_POST: name = "kernel_dsv4_hc_post_f32"; break;
- default: GGML_ABORT("fatal error");
+ switch (op->op) {
+ case GGML_OP_DSV4_HC_COMB:
+ name = "kernel_dsv4_hc_comb_f32";
+ break;
+ case GGML_OP_DSV4_HC_PRE:
+ if (ggml_get_op_params_i32(op, 1) != 0) {
+ name = "kernel_dsv4_hc_pre_gated_f32";
+ } else {
+ name = "kernel_dsv4_hc_pre_f32";
+ }
+ break;
+ case GGML_OP_DSV4_HC_POST:
+ if (op->src[3]) {
+ name = "kernel_dsv4_hc_post_f32";
+ } else {
+ name = "kernel_dsv4_hc_post_nocomb_f32";
+ }
+ break;
+ default:
+ GGML_ABORT("fatal error");
}
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h
index f6243ffbd..0514f9ef0 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.h
+++ b/ggml/src/ggml-metal/ggml-metal-device.h
@@ -126,7 +126,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_ad
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_tri (ggml_metal_library_t lib, const struct ggml_tensor * op);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max (ggml_metal_library_t lib, const struct ggml_tensor * op);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexer (ggml_metal_library_t lib, const struct ggml_tensor * op);
-struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc (ggml_metal_library_t lib, enum ggml_op op);
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc (ggml_metal_library_t lib, const struct ggml_tensor * op);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv (ggml_metal_library_t lib, const struct ggml_tensor * op);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched (ggml_metal_library_t lib, const struct ggml_tensor * op, int ssm_conv_bs);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tail);
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
index c734c8e13..952d1c0a6 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.m
+++ b/ggml/src/ggml-metal/ggml-metal-device.m
@@ -1802,8 +1802,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
op->src[1]->type == GGML_TYPE_F32 &&
op->type == GGML_TYPE_F32 &&
op->src[0]->ne[1] == 4 &&
- op->src[1]->ne[0] == 4 &&
- op->src[1]->ne[2] == 1 &&
ggml_is_contiguous_rows(op->src[0]) &&
ggml_is_contiguous_rows(op->src[1]);
case GGML_OP_DSV4_HC_POST:
@@ -1811,17 +1809,15 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
op->src[0]->type == GGML_TYPE_F32 &&
op->src[1]->type == GGML_TYPE_F32 &&
op->src[2]->type == GGML_TYPE_F32 &&
- op->src[3] != NULL &&
- op->src[3]->type == GGML_TYPE_F32 &&
+ (op->src[3] == NULL || op->src[3]->type == GGML_TYPE_F32) &&
op->type == GGML_TYPE_F32 &&
op->src[1]->ne[1] == 4 &&
op->src[2]->ne[0] == 4 &&
- op->src[3]->ne[0] == 4 &&
- op->src[3]->ne[1] == 4 &&
+ (op->src[3] == NULL || (op->src[3]->ne[0] == 4 && op->src[3]->ne[1] == 4)) &&
ggml_is_contiguous_rows(op->src[0]) &&
ggml_is_contiguous_rows(op->src[1]) &&
ggml_is_contiguous_rows(op->src[2]) &&
- ggml_is_contiguous_rows(op->src[3]);
+ (op->src[3] == NULL || ggml_is_contiguous_rows(op->src[3]));
case GGML_OP_SSM_SCAN:
return has_simdgroup_reduction;
case GGML_OP_SSM_CONV:
diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h
index 7a2c65aaa..d84ca937b 100644
--- a/ggml/src/ggml-metal/ggml-metal-impl.h
+++ b/ggml/src/ggml-metal/ggml-metal-impl.h
@@ -1283,8 +1283,10 @@ typedef struct {
uint64_t nb_x2;
uint64_t nb_w0;
uint64_t nb_w1;
+ uint64_t nb_w2;
uint64_t nb_d0;
uint64_t nb_d1;
+ float scale;
} ggml_metal_kargs_dsv4_hc_pre;
typedef struct {
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp
index cc1bebfaa..77c399bdb 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp
@@ -1405,7 +1405,7 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) {
ggml_tensor * op = ctx->node(idx);
ggml_metal_encoder_t enc = ctx->enc;
- auto pipeline = ggml_metal_library_get_pipeline_dsv4_hc(ctx->lib, op->op);
+ auto pipeline = ggml_metal_library_get_pipeline_dsv4_hc(ctx->lib, op);
ggml_metal_encoder_set_pipeline(enc, pipeline);
@@ -1467,8 +1467,10 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) {
/*.nb_x2 =*/ x->nb[2],
/*.nb_w0 =*/ weights->nb[0],
/*.nb_w1 =*/ weights->nb[1],
+ /*.nb_w2 =*/ weights->nb[2],
/*.nb_d0 =*/ op->nb[0],
/*.nb_d1 =*/ op->nb[1],
+ /*.scale =*/ ggml_get_op_params_f32(op, 0),
};
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
@@ -1491,7 +1493,6 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) {
GGML_ASSERT(x->type == GGML_TYPE_F32);
GGML_ASSERT(residual->type == GGML_TYPE_F32);
GGML_ASSERT(post->type == GGML_TYPE_F32);
- GGML_ASSERT(comb->type == GGML_TYPE_F32);
GGML_ASSERT(op->type == GGML_TYPE_F32);
GGML_ASSERT(residual->ne[1] == 4);
@@ -1505,9 +1506,9 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) {
/*.nb_r2 =*/ residual->nb[2],
/*.nb_p0 =*/ post->nb[0],
/*.nb_p1 =*/ post->nb[1],
- /*.nb_c0 =*/ comb->nb[0],
- /*.nb_c1 =*/ comb->nb[1],
- /*.nb_c2 =*/ comb->nb[2],
+ /*.nb_c0 =*/ comb ? comb->nb[0] : 0,
+ /*.nb_c1 =*/ comb ? comb->nb[1] : 0,
+ /*.nb_c2 =*/ comb ? comb->nb[2] : 0,
/*.nb_d0 =*/ op->nb[0],
/*.nb_d1 =*/ op->nb[1],
/*.nb_d2 =*/ op->nb[2],
@@ -1517,8 +1518,12 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) {
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(x), 1);
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(residual), 2);
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(post), 3);
- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(comb), 4);
- ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5);
+ if (comb) {
+ ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(comb), 4);
+ ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5);
+ } else {
+ ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 4);
+ }
const int n_tiles = (args.n_embd + 31)/32;
const int nsg = std::min(4, n_tiles);
diff --git a/ggml/src/ggml-metal/kernels/misc.metal b/ggml/src/ggml-metal/kernels/misc.metal
index 11104b4d8..15a18e04a 100644
--- a/ggml/src/ggml-metal/kernels/misc.metal
+++ b/ggml/src/ggml-metal/kernels/misc.metal
@@ -531,7 +531,73 @@ kernel void kernel_dsv4_hc_pre_f32(
result = fma(*(device const float *) (xb + ih*args.nb_x1), w[ih], result);
}
- *(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = result;
+ *(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = args.scale*result;
+}
+
+kernel void kernel_dsv4_hc_pre_gated_f32(
+ constant ggml_metal_kargs_dsv4_hc_pre & args,
+ device const char * x,
+ device const char * gate,
+ device char * dst,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]],
+ ushort3 ntg[[threads_per_threadgroup]]) {
+ constexpr ushort hc = 4;
+
+ const int it = tgpig.y;
+ const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg;
+
+ if (i0 >= args.n_embd) {
+ return;
+ }
+
+ device const char * xb = x + i0*args.nb_x0 + it*args.nb_x2;
+ device const char * gb = gate + i0*args.nb_w0 + it*args.nb_w2;
+ float result = 0.0f;
+ FOR_UNROLL (ushort ih = 0; ih < hc; ++ih) {
+ const float g = 1.0f/(1.0f + exp(-*(device const float *) (gb + ih*args.nb_w1)));
+ result = fma(*(device const float *) (xb + ih*args.nb_x1), g, result);
+ }
+
+ *(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = args.scale*result;
+}
+
+kernel void kernel_dsv4_hc_post_nocomb_f32(
+ constant ggml_metal_kargs_dsv4_hc_post & args,
+ device const char * x,
+ device const char * residual,
+ device const char * post,
+ device char * dst,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]],
+ ushort3 ntg[[threads_per_threadgroup]]) {
+ constexpr ushort hc = 4;
+
+ const int it = tgpig.y;
+ const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg;
+
+ float post_lane = 0.0f;
+ if (tiisg < hc) {
+ post_lane = *(device const float *) (post + tiisg*args.nb_p0 + it*args.nb_p1);
+ }
+
+ float post_reg[hc];
+ FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) {
+ post_reg[idst] = simd_shuffle(post_lane, idst);
+ }
+
+ if (i0 >= args.n_embd) {
+ return;
+ }
+
+ const float xv = *(device const float *) (x + i0*args.nb_x0 + it*args.nb_x1);
+ device const char * rb = residual + i0*args.nb_r0 + it*args.nb_r2;
+ FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) {
+ const float rv = *(device const float *) (rb + idst*args.nb_r1);
+ *(device float *) (dst + i0*args.nb_d0 + idst*args.nb_d1 + it*args.nb_d2) = xv*post_reg[idst] + rv;
+ }
}
kernel void kernel_dsv4_hc_post_f32(