Commit 19e28a277 for llama.cpp
commit 19e28a27702117d8f2eb16b825b9a308111f67d9
Author: cqderek <cqiang@qti.qualcomm.com>
Date: Wed Sep 30 05:59:13 2026 +0800
Hexagon f16 activation ops (#29209)
* hexagon: add F16 support for activation ops (SILU/GELU/GELU_QUICK/GEGLU/SWIGLU)
Widens ggml_hexagon_supported_activations() to accept F16 (src0/dst/src1
must agree on type), and adds F16 per-thread worker functions in
act-ops.c mirroring the existing F32 workers, backed by new HVX f16
kernels (hvx_sigmoid_f16_aa, hvx_tanh_f16_aa, hvx_mul_mul_f16_aa,
hvx_min_scalar_f16 family).
SILU, GELU, GELU_QUICK, GEGLU, and SWIGLU are verified correct on-device
(QRD8850) via test-backend-ops CPU-diffed correctness tests. SWIGLU_OAI's
F16 path is code-complete and builds clean on host + all 4 DSP arch
variants (v73/v75/v79/v81), but has no F16 test-case coverage in
test-backend-ops and is therefore unverified on-device in this change.
* hex-ops: align macros
* hex-ops: minor formatting
---------
Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index 161eee248..4b913947d 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -6037,10 +6037,10 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session
const struct ggml_tensor * src1 = op->src[1];
const struct ggml_tensor * dst = op;
- if (src0->type != GGML_TYPE_F32) {
+ if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) {
return false;
}
- if (dst->type != GGML_TYPE_F32) {
+ if (dst->type != src0->type) {
return false;
}
@@ -6052,7 +6052,7 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session
}
if (src1) {
- if (src1->type != GGML_TYPE_F32) {
+ if (src1->type != src0->type) {
return false;
}
if (!ggml_are_same_shape(src0, src1)) {
@@ -6637,12 +6637,12 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
case GGML_OP_GLU:
switch (ggml_get_glu_op(t)) {
- case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU;
- case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI;
+ case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU;
+ case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI;
case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP;
- case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU;
- case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK;
- case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF;
+ case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU;
+ case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK;
+ case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF;
default: break;
}
break;
diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.c
index 28c581df4..59a0575c4 100644
--- a/ggml/src/ggml-hexagon/htp/act-ops.c
+++ b/ggml/src/ggml-hexagon/htp/act-ops.c
@@ -55,16 +55,22 @@
struct htp_act_context;
-typedef void (*glu_compute_fn_t)(const float * restrict src0,
- const float * restrict src1,
- float * restrict dst,
- const uint32_t num_rows,
- const struct htp_act_context * actx);
+typedef void (*glu_compute_f32_fn_t)(const float * restrict src0,
+ const float * restrict src1,
+ float * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_act_context * actx);
+
+typedef void (*glu_compute_f16_fn_t)(const _Float16 * restrict src0,
+ const _Float16 * restrict src1,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_act_context * actx);
struct htp_act_context {
struct htp_ops_context * octx;
- glu_compute_fn_t compute;
+ void * compute;
const char * op_str;
// Precomputed values
@@ -392,6 +398,58 @@ static inline void hvx_geglu_erf_f32_aa(uint8_t * restrict dst, const uint8_t *
}
}
+static inline void hvx_geglu_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src0 % 128 == 0);
+ assert((unsigned long) src1 % 128 == 0);
+
+ HVX_Vector * restrict vdst = (HVX_Vector *) dst;
+ const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0;
+ const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1;
+
+ const uint32_t epv = 128 / sizeof(_Float16);
+ const uint32_t nvec = n / epv;
+ const uint32_t nloe = n % epv;
+
+ const HVX_Vector v_coef_a = hvx_vec_splat_f16((_Float16) GELU_COEF_A);
+ const HVX_Vector v_sqrt_2_pi = hvx_vec_splat_f16((_Float16) SQRT_2_OVER_PI);
+ const HVX_Vector v_one = hvx_vec_splat_f16((_Float16) 1.0f);
+ const HVX_Vector v_half = hvx_vec_splat_f16((_Float16) 0.5f);
+
+ uint32_t i = 0;
+
+ _Pragma("unroll(4)")
+ for (; i < nvec; i++) {
+ HVX_Vector x = vsrc0[i];
+ HVX_Vector g = vsrc1[i];
+
+ // gelu(x) = 0.5*x*(1 + tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x)))
+ HVX_Vector x2 = hvx_vec_mul_f16_f16(x, x);
+ HVX_Vector coef = hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(x2, v_coef_a), v_one);
+ HVX_Vector inner = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, coef), v_sqrt_2_pi);
+
+ HVX_Vector t = hvx_vec_add_f16_f16(hvx_vec_tanh_f16(inner), v_one);
+
+ HVX_Vector gelu_x = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, t), v_half);
+ vdst[i] = hvx_vec_mul_f16_f16(gelu_x, g);
+ }
+
+ if (nloe) {
+ HVX_Vector x = vsrc0[i];
+ HVX_Vector g = vsrc1[i];
+
+ HVX_Vector x2 = hvx_vec_mul_f16_f16(x, x);
+ HVX_Vector coef = hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(x2, v_coef_a), v_one);
+ HVX_Vector inner = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, coef), v_sqrt_2_pi);
+
+ HVX_Vector t = hvx_vec_add_f16_f16(hvx_vec_tanh_f16(inner), v_one);
+
+ HVX_Vector gelu_x = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, t), v_half);
+ HVX_Vector res = hvx_vec_mul_f16_f16(gelu_x, g);
+ hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(_Float16), res);
+ }
+}
+
// geglu(x, g) = gelu(x) * g
static void geglu_f32(const float * restrict src0,
const float * restrict src1,
@@ -443,100 +501,189 @@ static void geglu_erf_f32(const float * restrict src0,
}
}
-static void glu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {
- struct htp_act_context * actx = (struct htp_act_context *) data;
- htp_act_preamble;
-
- struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL;
-
- size_t src0_row_size = actx->src0_row_size;
- size_t src1_row_size = actx->src1_row_size;
- size_t dst_row_size = actx->dst_row_size;
-
- size_t src0_row_stride = actx->src0_row_stride;
- size_t src1_row_stride = actx->src1_row_stride;
-
- const uint32_t src0_nrows = actx->src0_nrows;
- const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread;
+// swiglu(x) = x1 * sigmoid(x0)
+static void swiglu_f16(const _Float16 * restrict src0,
+ const _Float16 * restrict src1,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_act_context * actx) {
+ htp_glu_op_preamble;
- const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows);
+ for (uint32_t ib = 0; ib < num_rows; ib++) {
+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);
+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);
- /* no work for this thread */
- if (src0_start_row >= src0_end_row) {
- return;
+ hvx_sigmoid_f16_aa(dst_ptr, src0_ptr, nc);
+ hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc);
}
+}
- const dma_addr_t data_src0 = actx->data_src0;
- const dma_addr_t data_src1 = actx->data_src1;
- const dma_addr_t data_dst = actx->data_dst;
-
- const size_t src0_row_size_aligned = actx->src0_row_size_aligned;
- const size_t src1_row_size_aligned = actx->src1_row_size_aligned;
- const size_t dst_row_size_aligned = actx->dst_row_size_aligned;
-
- uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread);
- uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread);
- uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread);
+// out = x * sigmoid(alpha * x) * (clamp(y, -limit, limit) + 1.f)
+static void swiglu_oai_f16(const _Float16 * restrict src0,
+ const _Float16 * restrict src1,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_act_context * actx) {
+ htp_glu_op_preamble;
+ const _Float16 alpha = (_Float16) ((const float *) (actx->octx->op_params))[2];
+ const _Float16 limit = (_Float16) ((const float *) (actx->octx->op_params))[3];
- size_t src0_spad_half_size = actx->src0_spad_half_size;
- size_t src1_spad_half_size = actx->src1_spad_half_size;
- size_t dst_spad_half_size = actx->dst_spad_half_size;
+ for (uint32_t ib = 0; ib < num_rows; ib++) {
+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);
+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);
- const int BLOCK = actx->block;
- if (BLOCK == 0) {
- FARF(ERROR, "%s : VTCM reservation %zu is too small, needed %zu\n",
- actx->op_str, actx->vtcm_src0_size_per_thread, src0_row_size_aligned);
- return;
+ // x (src0_ptr) = std::min(src0_p[k], limit);
+ hvx_min_scalar_f16((uint8_t *) src0_ptr, src0_ptr, limit, nc);
+ // y1 (src1_ptr) = std::clamp(src1_p[k], -limit, limit);
+ hvx_clamp_scalar_f16((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc);
+ // y (src1_ptr) = y1 + 1.f
+ hvx_add_scalar_f16((uint8_t *) src1_ptr, src1_ptr, (_Float16) 1.0f, nc);
+ // x1 (dst_ptr) = alpha * x
+ hvx_mul_scalar_f16(dst_ptr, src0_ptr, alpha, nc);
+ // x2 (dst_ptr) = sigmoid(x1) = 1/(1+exp(-x1))
+ hvx_sigmoid_f16_aa(dst_ptr, dst_ptr, nc);
+ // out = x * sigmoid(alpha * x) * (y + 1.f)
+ hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc);
}
+}
- dma_queue * dma_q = actx->octx->ctx->dma[ith];
- glu_compute_fn_t compute = actx->compute;
-
- for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {
- const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
-
- /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */
- dma_queue_push(dma_q,
- dma_make_data(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),
- dst_row_size, dst_row_size_aligned, dst_row_size, 0);
+static void swiglu_clamp_f16(const _Float16 * restrict src0,
+ const _Float16 * restrict src1,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_act_context * actx) {
+ htp_glu_op_preamble;
+ const _Float16 limit = (_Float16) ((const float *) (actx->octx->op_params))[3];
- dma_queue_push(dma_q,
- dma_make_data(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)),
- src0_row_size_aligned, src0_row_stride, src0_row_size, block_size);
+ for (uint32_t ib = 0; ib < num_rows; ib++) {
+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);
+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);
- dma_queue_push(dma_q,
- dma_make_data(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)),
- src1_row_size_aligned, src1_row_stride, src1_row_size, block_size);
+ hvx_min_scalar_f16((uint8_t *) src0_ptr, src0_ptr, limit, nc);
+ hvx_clamp_scalar_f16((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc);
+ hvx_sigmoid_f16_aa(dst_ptr, src0_ptr, nc);
+ hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc);
}
+}
- for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {
- const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
-
- float * dst_spad = (float *) dma_queue_pop(dma_q).src;
- float * src0_spad = (float *) dma_queue_pop(dma_q).dst;
- float * src1_spad = (float *) dma_queue_pop(dma_q).dst;
+// geglu(x, g) = gelu(x) * g, gelu via tanh approximation
+static void geglu_f16(const _Float16 * restrict src0,
+ const _Float16 * restrict src1,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_act_context * actx) {
+ htp_glu_op_preamble;
- htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir);
- compute(src0_spad, src1_spad, dst_spad, block_size, actx);
- htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir);
+ for (uint32_t ib = 0; ib < num_rows; ib++) {
+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);
+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);
- dma_queue_push(dma_q, dma_make_data(data_dst + (ir * dst_row_size), dst_spad),
- dst_row_size, dst_row_size_aligned, dst_row_size, block_size);
+ hvx_geglu_f16_aa(dst_ptr, src0_ptr, src1_ptr, nc);
+ }
+}
- /* prefetch N+2 loop iteration if any */
- const uint32_t pref_block = (ir + BLOCK * 2);
- if (pref_block < src0_end_row) {
- const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);
- dma_queue_push(dma_q, dma_make_data(src0_spad, data_src0 + (pref_block * src0_row_stride)),
- src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size);
- dma_queue_push(dma_q, dma_make_data(src1_spad, data_src1 + (pref_block * src1_row_stride)),
- src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size);
- }
+#define DEFINE_GLU_PER_THREAD(TYPE, SUFFIX) \
+ static void glu_##SUFFIX##_per_thread(unsigned int nth, unsigned int ith, void * data) { \
+ struct htp_act_context * actx = (struct htp_act_context *) data; \
+ htp_act_preamble; \
+ \
+ struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \
+ \
+ size_t src0_row_size = actx->src0_row_size; \
+ size_t src1_row_size = actx->src1_row_size; \
+ size_t dst_row_size = actx->dst_row_size; \
+ \
+ size_t src0_row_stride = actx->src0_row_stride; \
+ size_t src1_row_stride = actx->src1_row_stride; \
+ \
+ const uint32_t src0_nrows = actx->src0_nrows; \
+ const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \
+ \
+ const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith; \
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows); \
+ \
+ /* no work for this thread */ \
+ if (src0_start_row >= src0_end_row) { \
+ return; \
+ } \
+ \
+ const dma_addr_t data_src0 = actx->data_src0; \
+ const dma_addr_t data_src1 = actx->data_src1; \
+ const dma_addr_t data_dst = actx->data_dst; \
+ \
+ const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \
+ const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \
+ const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \
+ \
+ uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \
+ uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \
+ uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \
+ \
+ size_t src0_spad_half_size = actx->src0_spad_half_size; \
+ size_t src1_spad_half_size = actx->src1_spad_half_size; \
+ size_t dst_spad_half_size = actx->dst_spad_half_size; \
+ \
+ const int BLOCK = actx->block; \
+ if (BLOCK == 0) { \
+ FARF(ERROR, "%s : VTCM reservation %zu is too small, needed %zu\n", \
+ actx->op_str, actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \
+ return; \
+ } \
+ \
+ dma_queue * dma_q = actx->octx->ctx->dma[ith]; \
+ glu_compute_##SUFFIX##_fn_t compute = (glu_compute_##SUFFIX##_fn_t) actx->compute; \
+ \
+ for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \
+ const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \
+ \
+ /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ \
+ dma_queue_push(dma_q, \
+ dma_make_data(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), \
+ dst_row_size, dst_row_size_aligned, dst_row_size, 0); \
+ \
+ dma_queue_push(dma_q, \
+ dma_make_data(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), \
+ src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); \
+ \
+ dma_queue_push(dma_q, \
+ dma_make_data(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), \
+ src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); \
+ } \
+ \
+ for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { \
+ const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \
+ \
+ TYPE * dst_spad = (TYPE *) dma_queue_pop(dma_q).src; \
+ TYPE * src0_spad = (TYPE *) dma_queue_pop(dma_q).dst; \
+ TYPE * src1_spad = (TYPE *) dma_queue_pop(dma_q).dst; \
+ \
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \
+ compute(src0_spad, src1_spad, dst_spad, block_size, actx); \
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \
+ \
+ dma_queue_push(dma_q, dma_make_data(data_dst + (ir * dst_row_size), dst_spad), \
+ dst_row_size, dst_row_size_aligned, dst_row_size, block_size); \
+ \
+ /* prefetch N+2 loop iteration if any */ \
+ const uint32_t pref_block = (ir + BLOCK * 2); \
+ if (pref_block < src0_end_row) { \
+ const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); \
+ dma_queue_push(dma_q, dma_make_data(src0_spad, data_src0 + (pref_block * src0_row_stride)), \
+ src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); \
+ dma_queue_push(dma_q, dma_make_data(src1_spad, data_src1 + (pref_block * src1_row_stride)), \
+ src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); \
+ } \
+ } \
+ \
+ dma_queue_flush(dma_q); \
}
- dma_queue_flush(dma_q);
-}
+DEFINE_GLU_PER_THREAD(float, f32)
+DEFINE_GLU_PER_THREAD(_Float16, f16)
static int execute_op_activations_f32(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0];
@@ -548,8 +695,8 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
- glu_compute_fn_t compute_fn = NULL;
- const char * op_type = NULL;
+ glu_compute_f32_fn_t compute_fn = NULL;
+ const char * op_type = NULL;
switch (octx->op) {
case HTP_OP_GLU_SWIGLU:
@@ -581,6 +728,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
compute_fn = geglu_erf_f32;
op_type = "geglu-erf-f32";
break;
+
default:
FARF(ERROR, "Unsupported activations Op %u\n", octx->op);
return HTP_STATUS_NO_SUPPORT;
@@ -643,7 +791,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
// Prepare context
struct htp_act_context actx;
actx.octx = octx;
- actx.compute = compute_fn;
+ actx.compute = (void *) compute_fn;
actx.op_str = op_type;
actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
@@ -708,11 +856,174 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
+static int execute_op_activations_f16(struct htp_ops_context * octx) {
+ const struct htp_tensor * src0 = octx->src[0];
+ const struct htp_tensor * src1 = octx->src[1];
+ const struct htp_tensor * dst = octx->dst;
+
+ if ((dst->ne[0] * SIZEOF_FP16) != dst->nb[1]) {
+ FARF(ERROR, "Non-contiguous dst is not supported at this time \n");
+ return HTP_STATUS_NO_SUPPORT;
+ }
+
+ glu_compute_f16_fn_t compute_fn = NULL;
+ const char * op_type = NULL;
+
+ switch (octx->op) {
+ case HTP_OP_GLU_SWIGLU:
+ compute_fn = swiglu_f16;
+ op_type = "swiglu-f16";
+ break;
+
+ case HTP_OP_GLU_SWIGLU_OAI:
+ compute_fn = swiglu_oai_f16;
+ op_type = "swiglu-oai-f16";
+ break;
+
+ case HTP_OP_GLU_SWIGLU_CLAMP:
+ compute_fn = swiglu_clamp_f16;
+ op_type = "swiglu-clamp-f16";
+ break;
+
+ case HTP_OP_GLU_GEGLU:
+ compute_fn = geglu_f16;
+ op_type = "geglu-f16";
+ break;
+
+ default:
+ FARF(ERROR, "Unsupported activations Op %u\n", octx->op);
+ return HTP_STATUS_NO_SUPPORT;
+ }
+
+ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
+ const size_t dst_row_size = dst->ne[0] * SIZEOF_FP16;
+
+ uint32_t row_start = 0;
+ uint32_t nrows = src0_nrows;
+
+ if (octx->ctx->mdev.count > 1) {
+ uint32_t rows_per_chunk = 0;
+ htp_tensor_mdev_rows_per_chunk(dst, sizeof(_Float16), (uint32_t) dst_row_size, &rows_per_chunk);
+ const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
+ row_start = range.start;
+ nrows = range.count;
+ }
+
+ if (nrows == 0) {
+ return HTP_STATUS_OK;
+ }
+
+ const uint32_t n_threads = octx->n_threads;
+
+ // row_size = bytes of useful data per row (what the kernel touches / what DMA copies).
+ // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src).
+ const size_t nc_bytes = dst_row_size;
+ const size_t src0_row_size = nc_bytes;
+ const size_t src1_row_size = nc_bytes;
+ const size_t src0_row_stride = src0->nb[1];
+ const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1];
+
+ const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);
+ const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);
+ const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);
+
+ struct htp_act_vtcm_layout L;
+ htp_act_vtcm_layout_build(&L, src0_row_size_aligned, src1_row_size_aligned, dst_row_size_aligned, n_threads,
+ octx->ctx->vtcm_size);
+
+ // Make sure the reserved vtcm size is sufficient
+ if (L.vtcm_row_per_thread == 0) {
+ FARF(ERROR, "act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,
+ (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads);
+ return HTP_STATUS_VTCM_TOO_SMALL;
+ }
+
+ if (src1) {
+ FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n",
+ op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2],
+ src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads,
+ L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads);
+ } else {
+ FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type,
+ src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
+ L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads);
+ }
+
+ // Prepare context
+ struct htp_act_context actx;
+ actx.octx = octx;
+ actx.compute = (void *) compute_fn;
+ actx.op_str = op_type;
+
+ actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
+
+ actx.src0_row_size = src0_row_size;
+ actx.src1_row_size = src1_row_size;
+ actx.dst_row_size = dst_row_size;
+
+ actx.src0_row_size_aligned = src0_row_size_aligned;
+ actx.src1_row_size_aligned = src1_row_size_aligned;
+ actx.dst_row_size_aligned = dst_row_size_aligned;
+
+ actx.src0_row_stride = src0_row_stride;
+ actx.src1_row_stride = src1_row_stride;
+
+ uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;
+ actx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);
+ actx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);
+ actx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);
+
+ actx.vtcm_src0_size_per_thread = L.src0_bytes_per_thread;
+ actx.vtcm_src1_size_per_thread = L.src1_bytes_per_thread;
+ actx.vtcm_dst_size_per_thread = L.dst_bytes_per_thread;
+
+ actx.src0_spad_half_size = L.src0_bytes_per_thread / 2;
+ actx.src1_spad_half_size = L.src1_bytes_per_thread / 2;
+ actx.dst_spad_half_size = L.dst_bytes_per_thread / 2;
+
+ actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned;
+ actx.src0_nrows = nrows;
+ actx.row_start = row_start;
+
+ actx.nc = dst->ne[0];
+
+ // Addresses and GLU logic
+ dma_addr_t data_src0 = src0->data;
+ dma_addr_t data_src1 = src1 ? src1->data : 0;
+
+ if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU ||
+ octx->op == HTP_OP_GLU_SWIGLU_OAI ||
+ octx->op == HTP_OP_GLU_SWIGLU_CLAMP ||
+ octx->op == HTP_OP_GLU_GEGLU)) {
+ const int32_t swapped = octx->op_params[1];
+ data_src1 = data_src0;
+ actx.src1_row_size = actx.src0_row_size;
+
+ size_t nc_in_bytes = actx.nc * SIZEOF_FP16;
+ if (swapped) {
+ data_src0 += nc_in_bytes;
+ } else {
+ data_src1 += nc_in_bytes;
+ }
+ }
+
+ actx.data_src0 = data_src0;
+ actx.data_src1 = data_src1;
+ actx.data_dst = dst->data;
+
+ work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f16_per_thread, &actx, n_threads);
+ return HTP_STATUS_OK;
+}
+
+
int op_activations(struct htp_ops_context * octx) {
switch (octx->src[0]->type) {
case HTP_TYPE_F32:
return execute_op_activations_f32(octx);
+ case HTP_TYPE_F16:
+ return execute_op_activations_f16(octx);
+
default:
return HTP_STATUS_NO_SUPPORT;
}
diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h
index c9a776f62..ab2064af3 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-arith.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h
@@ -164,6 +164,37 @@ static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * re
}
}
+static inline void hvx_mul_mul_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint8_t * restrict src2, const uint32_t num_elems) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src0 % 128 == 0);
+ assert((unsigned long) src1 % 128 == 0);
+ assert((unsigned long) src2 % 128 == 0);
+
+ HVX_Vector * restrict vdst = (HVX_Vector *) dst;
+ HVX_Vector * restrict vsrc0 = (HVX_Vector *) src0;
+ HVX_Vector * restrict vsrc1 = (HVX_Vector *) src1;
+ HVX_Vector * restrict vsrc2 = (HVX_Vector *) src2;
+
+ const uint32_t elem_size = sizeof(_Float16);
+ const uint32_t epv = 128 / elem_size;
+ const uint32_t nvec = num_elems / epv;
+ const uint32_t nloe = num_elems % epv;
+
+ uint32_t i = 0;
+
+ _Pragma("unroll(4)")
+ for (; i < nvec; i++) {
+ HVX_Vector v1 = HVX_OP_MUL_F16(vsrc0[i], vsrc1[i]);
+ vdst[i] = HVX_OP_MUL_F16(v1, vsrc2[i]);
+ }
+
+ if (nloe) {
+ HVX_Vector v1 = HVX_OP_MUL_F16(vsrc0[i], vsrc1[i]);
+ HVX_Vector v2 = HVX_OP_MUL_F16(v1, vsrc2[i]);
+ hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v2);
+ }
+}
+
// Scalar Operations
#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \
@@ -348,6 +379,44 @@ static inline void hvx_max_scalar_f32(uint8_t * restrict dst, const uint8_t * re
}
}
+#define HVX_OP_MIN_SCALAR_F16(v) Q6_Vhf_vmin_VhfVhf(val_vec, v)
+
+static inline void hvx_min_scalar_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
+ const HVX_Vector val_vec = hvx_vec_splat_f16(val);
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src % 128 == 0);
+ hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_MIN_SCALAR_F16);
+}
+
+static inline void hvx_min_scalar_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
+ const HVX_Vector val_vec = hvx_vec_splat_f16(val);
+ assert((unsigned long) dst % 128 == 0);
+ hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_MIN_SCALAR_F16);
+}
+
+static inline void hvx_min_scalar_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
+ const HVX_Vector val_vec = hvx_vec_splat_f16(val);
+ assert((unsigned long) src % 128 == 0);
+ hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_MIN_SCALAR_F16);
+}
+
+static inline void hvx_min_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
+ const HVX_Vector val_vec = hvx_vec_splat_f16(val);
+ hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_MIN_SCALAR_F16);
+}
+
+static inline void hvx_min_scalar_f16(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, const int num_elems) {
+ if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) {
+ hvx_min_scalar_f16_aa(dst, src, val, num_elems);
+ } else if (hex_is_aligned((void *) dst, 128)) {
+ hvx_min_scalar_f16_au(dst, src, val, num_elems);
+ } else if (hex_is_aligned((void *) src, 128)) {
+ hvx_min_scalar_f16_ua(dst, src, val, num_elems);
+ } else {
+ hvx_min_scalar_f16_uu(dst, src, val, num_elems);
+ }
+}
+
// CLAMP Scalar variants
#define HVX_OP_CLAMP_SCALAR(v) \
@@ -767,10 +836,12 @@ static inline void hvx_sqr_f16(uint8_t * restrict dst, const uint8_t * restrict
#undef hvx_scalar_loop_body
#undef HVX_OP_MIN_SCALAR
#undef HVX_OP_MAX_SCALAR
+#undef HVX_OP_MIN_SCALAR_F16
#undef HVX_OP_CLAMP_SCALAR
#undef HVX_OP_CLAMP_SCALAR_F16
#undef HVX_OP_LEAKY_RELU_SCALAR
#undef DEFINE_HVX_BINARY_OP_VARIANTS
+#undef DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS
#undef HVX_BINARY_DISPATCHER
#undef UNUSED
diff --git a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h
index 552017309..b7c2d7409 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h
@@ -178,4 +178,58 @@ static inline HVX_Vector hvx_vec_tanh_f16(HVX_Vector x) {
return hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(sig2x, v_two), v_neg_one);
}
+#define hvx_sigmoid_f16_loop_body(dst_type, src_type, vec_store) \
+ do { \
+ dst_type * restrict vdst = (dst_type *) dst; \
+ src_type * restrict vsrc = (src_type *) src; \
+ \
+ const uint32_t epv = 128 / sizeof(_Float16); \
+ const uint32_t nvec = n / epv; \
+ const uint32_t nloe = n % epv; \
+ \
+ uint32_t i = 0; \
+ \
+ _Pragma("unroll(4)") \
+ for (; i < nvec; i++) { \
+ vdst[i] = hvx_vec_fast_sigmoid_f16(vsrc[i]); \
+ } \
+ if (nloe) { \
+ HVX_Vector tmp = hvx_vec_fast_sigmoid_f16(vsrc[i]); \
+ vec_store((void *) &vdst[i], nloe * sizeof(_Float16), tmp); \
+ } \
+ } while(0)
+
+#define hvx_tanh_f16_loop_body(dst_type, src_type, vec_store) \
+ do { \
+ dst_type * restrict vdst = (dst_type *) dst; \
+ src_type * restrict vsrc = (src_type *) src; \
+ \
+ const uint32_t epv = 128 / sizeof(_Float16); \
+ const uint32_t nvec = n / epv; \
+ const uint32_t nloe = n % epv; \
+ \
+ uint32_t i = 0; \
+ \
+ _Pragma("unroll(4)") \
+ for (; i < nvec; i++) { \
+ vdst[i] = hvx_vec_tanh_f16(vsrc[i]); \
+ } \
+ if (nloe) { \
+ HVX_Vector tmp = hvx_vec_tanh_f16(vsrc[i]); \
+ vec_store((void *) &vdst[i], nloe * sizeof(_Float16), tmp); \
+ } \
+ } while(0)
+
+static inline void hvx_sigmoid_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src % 128 == 0);
+ hvx_sigmoid_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
+}
+
+static inline void hvx_tanh_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src % 128 == 0);
+ hvx_tanh_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
+}
+
#endif /* HVX_SIGMOID_H */