Commit 0ae0e1976 for llama.cpp
commit 0ae0e1976670fac25f4e0ed864df30021accc764
Author: kurquhar <kurquhar@qti.qualcomm.com>
Date: Fri Oct 23 14:58:44 2026 -0700
hexagon: add SGN op (#30346)
* hexagon : add SGN support
Assisted-by: OpenCode
* hexagon : optimize SGN with sign bit
Assisted-by: OpenCode
* hex-sgn: add f16
* hex-unary: update SGN to use generic unary_colapse and relax colapse checks to apply to more conditions
* hex-unary: improve dims colapsing and run it in the host-side solver
---------
Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp
index 71040945d..589a8accf 100644
--- a/ggml/src/ggml-hexagon/htp-kparams.cpp
+++ b/ggml/src/ggml-hexagon/htp-kparams.cpp
@@ -73,6 +73,7 @@ htp_op_code htp_op_remap(const struct ggml_tensor * t) {
case GGML_UNARY_OP_RELU: return HTP_OP_UNARY_RELU;
case GGML_UNARY_OP_STEP: return HTP_OP_UNARY_STEP;
case GGML_UNARY_OP_CEIL: return HTP_OP_UNARY_CEIL;
+ case GGML_UNARY_OP_SGN: return HTP_OP_UNARY_SGN;
default: break;
}
break;
@@ -1392,6 +1393,52 @@ static bool htp_kparams_solve_binary(
return true;
}
+// collapse contiguous dims of elementwise ops so that rows fill at least one HVX vector
+static void htp_unary_collapse_dims(struct ggml_tensor * src, struct ggml_tensor * dst, size_t elem_size) {
+ if (src->nb[0] != elem_size || dst->nb[0] != elem_size) {
+ return;
+ }
+
+ while (src->ne[1] > 1 || src->ne[2] > 1 || src->ne[3] > 1) {
+ const size_t row_size = (size_t) src->ne[0] * elem_size;
+ const bool rows_contig = src->nb[1] == row_size && dst->nb[1] == (size_t) dst->ne[0] * elem_size;
+ const bool can_merge = (src->ne[1] == 1 && dst->ne[1] == 1) ||
+ (rows_contig && (src->ne[0] == 1 || (row_size < 128 && row_size * src->ne[1] <= 2048)));
+ if (!can_merge) {
+ break;
+ }
+ src->ne[0] *= src->ne[1];
+ src->ne[1] = src->ne[2];
+ src->ne[2] = src->ne[3];
+ src->ne[3] = 1;
+ dst->ne[0] *= dst->ne[1];
+ dst->ne[1] = dst->ne[2];
+ dst->ne[2] = dst->ne[3];
+ dst->ne[3] = 1;
+ src->nb[1] = src->nb[2];
+ src->nb[2] = src->nb[3];
+ dst->nb[1] = dst->nb[2];
+ dst->nb[2] = dst->nb[3];
+ }
+
+ // ne1 too big to merge in full: merge it in chunks of k rows (k divides ne1)
+ const size_t row_size = (size_t) src->ne[0] * elem_size;
+ if (src->ne[1] > 1 && row_size < 128 && src->nb[1] == row_size && dst->nb[1] == row_size) {
+ int64_t k = 2048 / row_size;
+ while (k > 1 && src->ne[1] % k != 0) {
+ k--;
+ }
+ if (k > 1) {
+ src->ne[0] *= k;
+ src->ne[1] /= k;
+ src->nb[1] *= k;
+ dst->ne[0] *= k;
+ dst->ne[1] /= k;
+ dst->nb[1] *= k;
+ }
+ }
+}
+
static bool htp_kparams_solve_unary_impl(
const struct ggml_hexagon_session * sess,
uint32_t op,
@@ -1435,6 +1482,7 @@ static bool htp_kparams_solve_unary_impl(
case HTP_OP_UNARY_ABS:
case HTP_OP_UNARY_STEP:
case HTP_OP_UNARY_CEIL:
+ case HTP_OP_UNARY_SGN:
break;
default:
return false;
@@ -1451,9 +1499,14 @@ static bool htp_kparams_solve_unary_impl(
struct ggml_tensor src0_c = *src0;
struct ggml_tensor dst_c = *dst;
if (htp_unary_is_elementwise(op)) {
- HTP_UNARY_COLLAPSE_DIMS(src0_c.ne, src0_c.nb, dst_c.ne, dst_c.nb, elem_size);
+ htp_unary_collapse_dims(&src0_c, &dst_c, elem_size);
src0 = &src0_c;
dst = &dst_c;
+ for (int i = 0; i < HTP_OP_MAX_DIMS; i++) {
+ kparams->ne[i] = (uint32_t) src0->ne[i];
+ kparams->src0_nb[i] = (uint32_t) src0->nb[i];
+ kparams->dst_nb[i] = (uint32_t) dst->nb[i];
+ }
}
const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type);
diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h
index 50addf414..f3b2d6b73 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ops.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h
@@ -113,6 +113,7 @@ enum htp_op_code {
HTP_OP_GLU_REGLU,
HTP_OP_GLU_SIGMOID,
HTP_OP_UNARY_CEIL,
+ HTP_OP_UNARY_SGN,
HTP_OP_INVALID
};
diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h
index 60da71fef..ec0de8e19 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-arith.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h
@@ -677,6 +677,56 @@ static inline void hvx_step_f32_aa(uint8_t * restrict dst, const uint8_t * restr
}
}
+static inline void hvx_sgn_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src % 128 == 0);
+
+ HVX_Vector * restrict vdst = (HVX_Vector *) dst;
+ HVX_Vector * restrict vsrc = (HVX_Vector *) src;
+
+ const uint32_t elem_size = sizeof(float);
+ const uint32_t epv = 128 / elem_size;
+ const uint32_t nvec = n / epv;
+ const uint32_t nloe = n % epv;
+
+ uint32_t i = 0;
+
+ _Pragma("unroll(4)")
+ for (; i < nvec; i++) {
+ vdst[i] = hvx_vec_sgn_f32(vsrc[i]);
+ }
+ if (nloe) {
+ HVX_Vector v = hvx_vec_sgn_f32(vsrc[i]);
+ hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
+ }
+}
+
+static inline void hvx_sgn_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src % 128 == 0);
+
+ HVX_Vector * restrict vdst = (HVX_Vector *) dst;
+ HVX_Vector * restrict vsrc = (HVX_Vector *) src;
+
+ const uint32_t elem_size = sizeof(_Float16);
+ const uint32_t epv = 128 / elem_size;
+ const uint32_t nvec = n / epv;
+ const uint32_t nloe = n % epv;
+
+ uint32_t i = 0;
+
+ _Pragma("unroll(4)")
+ for (; i < nvec; i++) {
+ HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
+ vdst[i] = hvx_vec_f32_to_f16(hvx_vec_sgn_f32(Q6_V_lo_W(p)), hvx_vec_sgn_f32(Q6_V_hi_W(p)));
+ }
+ if (nloe) {
+ HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
+ HVX_Vector v = hvx_vec_f32_to_f16(hvx_vec_sgn_f32(Q6_V_lo_W(p)), hvx_vec_sgn_f32(Q6_V_hi_W(p)));
+ hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
+ }
+}
+
static inline void hvx_step_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
diff --git a/ggml/src/ggml-hexagon/htp/hvx-base.h b/ggml/src/ggml-hexagon/htp/hvx-base.h
index 20e0d6e66..e87a30d14 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-base.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-base.h
@@ -118,6 +118,17 @@ static inline HVX_Vector hvx_vec_step_f32(HVX_Vector v) {
return Q6_V_vmux_QVV(q, one, zero);
}
+static inline HVX_Vector hvx_vec_sgn_f32(HVX_Vector v) {
+ const HVX_Vector zero = Q6_V_vzero();
+ const HVX_Vector one = hvx_vec_splat_f32(1.0f);
+ const HVX_Vector sign_mask = Q6_V_vsplat_R(0x80000000);
+ const HVX_Vector mag = hvx_vec_abs_f32(v);
+ const HVX_Vector signed_one = Q6_V_vor_VV(one, Q6_V_vand_VV(v, sign_mask));
+ // The ordered magnitude comparison maps NaN and both signed zeros to zero.
+ const HVX_VectorPred nonzero = Q6_Q_vcmp_gt_VsfVsf(mag, zero);
+ return Q6_V_vmux_QVV(nonzero, signed_one, zero);
+}
+
static inline HVX_Vector hvx_vec_step_f16(HVX_Vector v) {
const HVX_Vector zero = Q6_V_vzero();
const HVX_Vector one = hvx_vec_splat_f16((_Float16) 1.0f);
diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c
index b22c3c944..d1a4afcd6 100644
--- a/ggml/src/ggml-hexagon/htp/main.c
+++ b/ggml/src/ggml-hexagon/htp/main.c
@@ -946,6 +946,7 @@ static int execute_op(struct htp_ops_context * octx) {
case HTP_OP_UNARY_RELU:
case HTP_OP_UNARY_STEP:
case HTP_OP_UNARY_CEIL:
+ case HTP_OP_UNARY_SGN:
case HTP_OP_L2_NORM:
case HTP_OP_HADAMARD:
return op_unary(octx);
diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c
index 3a69337df..194569ab5 100644
--- a/ggml/src/ggml-hexagon/htp/unary-ops.c
+++ b/ggml/src/ggml-hexagon/htp/unary-ops.c
@@ -440,6 +440,20 @@ static void ceil_f16(const void * restrict src,
}
}
+static void sgn_f16(const void * restrict src,
+ void * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+ hvx_sgn_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
+ }
+}
+
static void l2_norm_f16(const void * restrict src,
void * restrict dst,
const uint32_t num_rows,
@@ -976,6 +990,20 @@ static void ceil_f32(const void * restrict src,
}
}
+static void sgn_f32(const void * restrict src,
+ void * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+ hvx_sgn_f32_aa(dst_local, src_local, ne0);
+ }
+}
+
static void log_f32(const void * restrict src,
void * restrict dst,
const uint32_t num_rows,
@@ -1094,6 +1122,11 @@ static void tile_ceil_f32(void * restrict dst, const void * restrict src, uint32
hvx_ceil_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw);
}
+static void tile_sgn_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) {
+ (void) uctx;
+ hvx_sgn_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw);
+}
+
static void tri_apply_tile_f32(const void * restrict src, void * restrict dst,
uint32_t tile_elems, uint32_t col_start, uint32_t i01,
uint32_t ne0, int32_t ttype) {
@@ -1738,6 +1771,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
case HTP_OP_UNARY_RELU: op_type = "relu-f32"; break;
case HTP_OP_UNARY_STEP: op_type = is_f16 ? "step-f16" : "step-f32"; break;
case HTP_OP_UNARY_CEIL: op_type = is_f16 ? "ceil-f16" : "ceil-f32"; break;
+ case HTP_OP_UNARY_SGN: op_type = is_f16 ? "sgn-f16" : "sgn-f32"; break;
case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break;
case HTP_OP_TRI: op_type = "tri-f32"; break;
case HTP_OP_HADAMARD: op_type = is_f16 ? "hadamard-f16" : "hadamard-f32"; break;
@@ -1752,9 +1786,9 @@ static int execute_op_unary(struct htp_ops_context * octx) {
return HTP_STATUS_INVAL_PARAMS;
}
- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const size_t src0_elem_size = is_f16 ? sizeof(_Float16) : sizeof(float);
const size_t dst_elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float);
+ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const size_t src0_data_row_size = src0->ne[0] * src0_elem_size;
const size_t dst_data_row_size = dst->ne[0] * dst_elem_size;
@@ -1872,6 +1906,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
case HTP_OP_UNARY_RELU: compute_func = (void *) tile_relu_f32; break;
case HTP_OP_UNARY_STEP: compute_func = (void *) tile_step_f32; break;
case HTP_OP_UNARY_CEIL: compute_func = (void *) tile_ceil_f32; break;
+ case HTP_OP_UNARY_SGN: compute_func = (void *) tile_sgn_f32; break;
case HTP_OP_TRI:
task_func = unary_thread_tiled_tri_f32;
compute_func = (void *) tri_apply_tile_f32;
@@ -1892,6 +1927,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
case HTP_OP_UNARY_LOG: compute_func = (void *) log_f16; break;
case HTP_OP_UNARY_STEP: compute_func = (void *) step_f16; break;
case HTP_OP_UNARY_CEIL: compute_func = (void *) ceil_f16; break;
+ case HTP_OP_UNARY_SGN: compute_func = (void *) sgn_f16; break;
case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f16; break;
default: break;
}
@@ -1923,6 +1959,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
case HTP_OP_UNARY_RELU: compute_func = (void *) relu_f32; break;
case HTP_OP_UNARY_STEP: compute_func = (void *) step_f32; break;
case HTP_OP_UNARY_CEIL: compute_func = (void *) ceil_f32; break;
+ case HTP_OP_UNARY_SGN: compute_func = (void *) sgn_f32; break;
case HTP_OP_L2_NORM: compute_func = (void *) l2_norm_f32; break;
case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f32; break;
case HTP_OP_TRI:
@@ -1952,10 +1989,14 @@ int op_unary(struct htp_ops_context * octx) {
return execute_op_unary(octx);
}
+ const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params;
+
struct htp_tensor src0_c = *octx->src[0];
struct htp_tensor dst_c = *octx->dst;
- const size_t elem_size = (src0_c.type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float);
- HTP_UNARY_COLLAPSE_DIMS(src0_c.ne, src0_c.nb, dst_c.ne, dst_c.nb, elem_size);
+ memcpy(src0_c.ne, kparams->ne, sizeof(src0_c.ne));
+ memcpy(dst_c.ne, kparams->ne, sizeof(dst_c.ne));
+ memcpy(src0_c.nb, kparams->src0_nb, sizeof(src0_c.nb));
+ memcpy(dst_c.nb, kparams->dst_nb, sizeof(dst_c.nb));
const struct htp_tensor * orig_src0 = octx->src[0];
const struct htp_tensor * orig_dst = octx->dst;
diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h
index 93b9e0e75..b824544ff 100644
--- a/ggml/src/ggml-hexagon/htp/unary-ops.h
+++ b/ggml/src/ggml-hexagon/htp/unary-ops.h
@@ -24,6 +24,11 @@ struct htp_unary_kernel_params {
struct fastdiv_values div_ne02;
struct fastdiv_values div_ne012;
struct fastdiv_values div_tpr;
+
+ // collapsed shape of elementwise ops
+ uint32_t ne[HTP_OP_MAX_DIMS];
+ uint32_t src0_nb[HTP_OP_MAX_DIMS];
+ uint32_t dst_nb[HTP_OP_MAX_DIMS];
};
#if defined(__cplusplus)
@@ -56,6 +61,7 @@ static inline bool htp_op_is_unary(uint32_t opcode) {
case HTP_OP_UNARY_RELU:
case HTP_OP_UNARY_STEP:
case HTP_OP_UNARY_CEIL:
+ case HTP_OP_UNARY_SGN:
case HTP_OP_L2_NORM:
case HTP_OP_TRI:
case HTP_OP_HADAMARD:
@@ -79,31 +85,6 @@ static inline bool htp_unary_is_elementwise(uint32_t opcode) {
}
}
-#define HTP_UNARY_COLLAPSE_DIMS(src_ne, src_nb, dst_ne, dst_nb, elem_size) \
- do { \
- while (((src_ne)[1] > 1 || (src_ne)[2] > 1 || (src_ne)[3] > 1) && \
- (size_t) (src_nb)[0] == (elem_size) && \
- (size_t) (dst_nb)[0] == (elem_size) && \
- (((src_ne)[1] == 1 && (dst_ne)[1] == 1) || \
- ((size_t) (src_nb)[1] == (size_t) (src_ne)[0] * (size_t) (src_nb)[0] && \
- (size_t) (dst_nb)[1] == (size_t) (dst_ne)[0] * (size_t) (dst_nb)[0] && \
- ((src_ne)[0] == 1 || \
- (uint64_t) (src_ne)[0] * (uint64_t) (src_ne)[1] * (elem_size) <= 128)))) { \
- (src_ne)[0] *= (src_ne)[1]; \
- (src_ne)[1] = (src_ne)[2]; \
- (src_ne)[2] = (src_ne)[3]; \
- (src_ne)[3] = 1; \
- (dst_ne)[0] *= (dst_ne)[1]; \
- (dst_ne)[1] = (dst_ne)[2]; \
- (dst_ne)[2] = (dst_ne)[3]; \
- (dst_ne)[3] = 1; \
- (src_nb)[1] = (src_nb)[2]; \
- (src_nb)[2] = (src_nb)[3]; \
- (dst_nb)[1] = (dst_nb)[2]; \
- (dst_nb)[2] = (dst_nb)[3]; \
- } \
- } while (0)
-
struct htp_unary_vtcm_layout {
size_t total_bytes;
size_t off_src0;