Commit 848bbda90 for llama.cpp
commit 848bbda9041decd87904f7eb5a2990863ecce03b
Author: Todor Boinovski <todorb@qti.qualcomm.com>
Date: Sat Oct 17 19:17:33 2026 -0700
hexagon: add support for hadamard MUL_MAT and GET_ROW for flat and tiled Q1_0 and Q2_0 (#30627)
* hex-hadamard: add support for hadamard ops
* hex-mm: reduce pkt counts in the q8_1 quantizer and vec_dot_q1
* hex-q1/q2: add support for flat and tiled GET_ROWS for Q1_0 and Q2_0
---------
Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index 6e714ff3b..605c23c68 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -5359,10 +5359,38 @@ static void ggml_hexagon_precompute_fused_mmidnx_params(
kparams->n_weights = n_weights;
}
+static bool ggml_hexagon_op_mul_mat_use_fwht(const struct ggml_tensor * dst) {
+ if (dst->op != GGML_OP_MUL_MAT || ggml_get_op_params_i32(dst, 1) != GGML_HINT_SRC0_IS_HADAMARD) {
+ return false;
+ }
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ const int64_t n = src1->ne[0];
+ if (n < 32 || (n & (n - 1)) != 0) {
+ return false;
+ }
+
+ return src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 &&
+ (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) &&
+ ggml_is_contiguous_rows(src1) && ggml_is_contiguous(dst) &&
+ ggml_are_same_shape(src1, dst);
+}
+
static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];
+ if (ggml_get_op_params_i32(dst, 1) == GGML_HINT_SRC0_IS_HADAMARD) {
+ if (!ggml_hexagon_op_mul_mat_use_fwht(dst)) {
+ return false;
+ }
+ struct htp_unary_kernel_params ukparams;
+ ggml_hexagon_precompute_unary_params(sess, HTP_OP_HADAMARD, src1, nullptr, dst, &ukparams);
+ return ukparams.vtcm_row_per_thread > 0 && (size_t) ukparams.vtcm_size <= sess->vtcm_size;
+ }
+
if (dst->type != GGML_TYPE_F32) {
return false;
}
@@ -5921,8 +5949,9 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *
const struct ggml_tensor * src1 = op->src[1]; // indices
const struct ggml_tensor * dst = op;
- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16 &&
- src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) {
+ if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16 && src0->type != GGML_TYPE_I32 &&
+ src0->type != GGML_TYPE_Q1_0 && src0->type != GGML_TYPE_Q2_0 && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 &&
+ src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K ) {
return false;
}
@@ -6666,7 +6695,11 @@ static void ggml_backend_hexagon_free(ggml_backend_t backend) {
static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
switch (t->op) {
case GGML_OP_FLASH_ATTN_EXT: return HTP_OP_FLASH_ATTN_EXT;
- case GGML_OP_MUL_MAT: return HTP_OP_MUL_MAT;
+ case GGML_OP_MUL_MAT:
+ if (ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD) {
+ return HTP_OP_HADAMARD;
+ }
+ return HTP_OP_MUL_MAT;
case GGML_OP_MUL_MAT_ID: return HTP_OP_MUL_MAT_ID;
case GGML_OP_MUL: return HTP_OP_MUL;
case GGML_OP_ADD: return HTP_OP_ADD;
@@ -6795,6 +6828,7 @@ static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const st
static bool is_mergeable_mul_mat(const ggml_tensor * t) {
if (t->op != GGML_OP_MUL_MAT) return false;
+ if (ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD) return false;
const ggml_tensor * src0 = t->src[0];
const ggml_tensor * src1 = t->src[1];
@@ -6897,7 +6931,9 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
if (graph->nodes[i]->op == GGML_OP_RMS_NORM && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {
extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE;
- } else if (graph->nodes[i]->op == GGML_OP_MUL_MAT || graph->nodes[i]->op == GGML_OP_MUL_MAT_ID) {
+ } else if ((graph->nodes[i]->op == GGML_OP_MUL_MAT &&
+ ggml_get_op_params_i32(graph->nodes[i], 1) != GGML_HINT_SRC0_IS_HADAMARD) ||
+ graph->nodes[i]->op == GGML_OP_MUL_MAT_ID) {
if ((i + 1 < graph->n_nodes && graph->nodes[i + 1]->op == GGML_OP_ADD && ggml_can_fuse(graph, i, { graph->nodes[i]->op, GGML_OP_ADD })) ||
ggml_node_has_n_uses(graph, i, 1)) {
extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE;
diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h
index d62ce1d00..57cd5f208 100644
--- a/ggml/src/ggml-hexagon/htp-opnode.h
+++ b/ggml/src/ggml-hexagon/htp-opnode.h
@@ -47,13 +47,19 @@ struct htp_opnode {
void init(ggml_tensor * node) {
this->node = node;
if (this->node) {
- this->name = ggml_op_desc(this->node);
-
- // Build inputs (preserving optional nullptrs)
- int n_inputs = n_active_src(this->node);
- this->inputs.resize(n_inputs, nullptr);
- for (int i = 0; i < n_inputs; i++) {
- this->inputs[i] = this->node->src[i];
+ if (this->node->op == GGML_OP_MUL_MAT &&
+ ggml_get_op_params_i32(this->node, 1) == GGML_HINT_SRC0_IS_HADAMARD) {
+ this->name = "MUL_MAT_HADAMARD";
+ this->inputs.push_back(this->node->src[1]);
+ } else {
+ this->name = ggml_op_desc(this->node);
+
+ // Build inputs (preserving optional nullptrs)
+ int n_inputs = n_active_src(this->node);
+ this->inputs.resize(n_inputs, nullptr);
+ for (int i = 0; i < n_inputs; i++) {
+ this->inputs[i] = this->node->src[i];
+ }
}
// Build outputs
@@ -124,6 +130,10 @@ struct htp_opnode {
bool stackable() const {
switch (this->op()) {
case GGML_OP_MUL_MAT:
+ if (ggml_get_op_params_i32(this->node, 1) == GGML_HINT_SRC0_IS_HADAMARD) {
+ return false;
+ }
+ return ggml_is_quantized(this->src0()->type);
case GGML_OP_MUL_MAT_ID:
return ggml_is_quantized(this->src0()->type);
default:
diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c
index 27948c37a..78eb1bb50 100644
--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c
+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c
@@ -288,7 +288,7 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat
}
const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);
- const uint32_t n_k_tiles = ne00 / HTP_TENSOR_TILE_N_COLS;
+ const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(octx->src[0]->type);
const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout;
uint8_t * src_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread;
uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_thread;
@@ -450,7 +450,9 @@ int op_get_rows(struct htp_ops_context * octx) {
break;
case HTP_GET_ROWS_KERNEL_TILED:
switch (octx->src[0]->type) {
- case HTP_TYPE_F16: grctx.compute_tiled = hvx_dequantize_row_tiled_f16_f32; break;
+ case HTP_TYPE_F16: grctx.compute_tiled = hvx_dequantize_row_tiled_f16_f32; break;
+ case HTP_TYPE_Q1_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q1_0_f32; break;
+ case HTP_TYPE_Q2_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q2_0_f32; break;
case HTP_TYPE_Q8_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q8_0_f32; break;
case HTP_TYPE_Q4_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_0_f32; break;
case HTP_TYPE_Q4_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_k_f32; break;
@@ -466,6 +468,8 @@ int op_get_rows(struct htp_ops_context * octx) {
switch (octx->src[0]->type) {
case HTP_TYPE_F16: grctx.compute_flat = hvx_dequantize_row_flat_f16_f32; break;
case HTP_TYPE_BF16: grctx.compute_flat = hvx_dequantize_row_flat_bf16_f32; break;
+ case HTP_TYPE_Q1_0: grctx.compute_flat = hvx_dequantize_row_flat_q1_0_f32; break;
+ case HTP_TYPE_Q2_0: grctx.compute_flat = hvx_dequantize_row_flat_q2_0_f32; break;
case HTP_TYPE_Q8_0: grctx.compute_flat = hvx_dequantize_row_flat_q8_0_f32; break;
case HTP_TYPE_Q4_0: grctx.compute_flat = hvx_dequantize_row_flat_q4_0_f32; break;
case HTP_TYPE_Q4_K: grctx.compute_flat = hvx_dequantize_row_flat_q4_k_f32; break;
diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.h
index f44e58d16..94f3c15f2 100644
--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h
+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h
@@ -64,7 +64,7 @@ static inline void htp_get_rows_vtcm_layout_build(
size_t src0_row_size = 0;
if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) {
const size_t tile_stride = htp_tensor_get_tile_padded_size(type);
- const uint32_t n_k_tiles = ne00 / HTP_TENSOR_TILE_N_COLS;
+ const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(type);
src0_row_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride;
} else {
src0_row_size = htp_tensor_get_row_size(type, ne00);
diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h
index 3fc558119..9b4b86d3b 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ops.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h
@@ -114,6 +114,7 @@ enum htp_op_code {
HTP_OP_POOL_1D,
HTP_OP_UNARY_GELU_QUICK,
HTP_OP_IM2COL_3D,
+ HTP_OP_HADAMARD,
HTP_OP_INVALID
};
diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h
index 842001c85..9b2e13133 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h
@@ -29,29 +29,27 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_
HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero);
HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero);
- HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero);
- HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero);
-
HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero);
HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero);
- HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero);
- HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero);
- HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf)));
- HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf)));
-
- HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf)));
- HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf)));
+ HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf)));
+ HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf)));
+ HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0
+ HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16);
+ HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf);
+ vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf));
- HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0
- HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0
- HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16);
- HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16);
+ HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero);
+ HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero);
+ HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero);
+ HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero);
- HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf);
+ HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf)));
+ HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf)));
+ HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0
+ HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16);
HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf);
- vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf));
- vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf));
+ vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf));
HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf);
HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf);
@@ -81,25 +79,10 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_
HVX_Vector voff01_hf = hvx_vec_f32_to_f16(voff0_sf, voff1_sf);
HVX_Vector voff23_hf = hvx_vec_f32_to_f16(voff2_sf, voff3_sf);
- HVX_Vector r_scale[4] = {
- hvx_vec_repl_f16(vd01_hf),
- hvx_vec_repl_f16(Q6_V_vror_VR(vd01_hf, 64)),
- hvx_vec_repl_f16(vd23_hf),
- hvx_vec_repl_f16(Q6_V_vror_VR(vd23_hf, 64)),
- };
- HVX_Vector r_offset[4] = {
- hvx_vec_repl_f16(voff01_hf),
- hvx_vec_repl_f16(Q6_V_vror_VR(voff01_hf, 64)),
- hvx_vec_repl_f16(voff23_hf),
- hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 64)),
- };
+ int32_t __attribute__((aligned(128))) ws[32];
if (sums16) {
// |sum of 16 int8| <= 2032 fits a halfword
- int32_t __attribute__((aligned(128))) ws[32];
*(HVX_Vector *) ws = v_sums;
- for (int b = 0; b < 4; b++) {
- r_offset[b] = Q6_V_vsplat_R(((uint32_t) ws[8 * b + 4] << 16) | ((uint32_t) ws[8 * b] & 0xFFFF));
- }
}
static const uint8_t __attribute__((aligned(128))) repl[128] = {
@@ -114,8 +97,15 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_
};
HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl;
+ #pragma unroll
for (int b = 0; b < 4; b++) {
HVX_Vector v_act = Q6_V_vror_VR(vx_i8, b * 32);
+ HVX_Vector vd_b = (b < 2) ? vd01_hf : vd23_hf;
+ HVX_Vector voff_b = (b < 2) ? voff01_hf : voff23_hf;
+ HVX_Vector r_scale = hvx_vec_repl_f16((b & 1) ? Q6_V_vror_VR(vd_b, 64) : vd_b);
+ HVX_Vector r_offset = sums16
+ ? Q6_V_vsplat_R(((uint32_t) ws[8 * b + 4] << 16) | ((uint32_t) ws[8 * b] & 0xFFFF))
+ : hvx_vec_repl_f16((b & 1) ? Q6_V_vror_VR(voff_b, 64) : voff_b);
HVX_Vector r0 = Q6_V_vdelta_VV(v_act, v_repl_ctrl);
HVX_Vector r1 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 4), v_repl_ctrl);
@@ -135,8 +125,8 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_
dst[5] = r5;
dst[6] = r6;
dst[7] = r7;
- dst[8] = r_scale[b];
- dst[9] = r_offset[b];
+ dst[8] = r_scale;
+ dst[9] = r_offset;
}
}
@@ -932,7 +922,7 @@ static void tiled_vec_dot_q1_0_32x1(const uint32_t n, float * restrict s, const
const uint8_t * restrict tile_ptr = vx;
const uint8_t * restrict y_q = vy;
- HVX_Vector v_sum_float = Q6_V_vzero();
+ HVX_Vector v_sum_qf = Q6_V_vzero();
HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1);
HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1);
@@ -951,9 +941,10 @@ static void tiled_vec_dot_q1_0_32x1(const uint32_t n, float * restrict s, const
}
HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4]));
- v_sum_float = hvx_vec_add_f32_f32(v_sum_float, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf));
+ v_sum_qf = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf));
}
+ HVX_Vector v_sum_float = Q6_Vsf_equals_Vqf32(v_sum_qf);
if (sz) {
hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz)));
} else {
@@ -966,16 +957,16 @@ static void tiled_vec_dot_q1_0_32x2(const uint32_t n, float * restrict s0, float
const uint8_t * restrict y0_q = vy0;
const uint8_t * restrict y1_q = vy1;
- HVX_Vector v_sum_float_c0 = Q6_V_vzero();
- HVX_Vector v_sum_float_c1 = Q6_V_vzero();
- HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1);
- HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1);
+ HVX_Vector v_sum_qf_c0 = Q6_V_vzero();
+ HVX_Vector v_sum_qf_c1 = Q6_V_vzero();
+ HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1);
+ HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1);
const uint32_t n_k_tiles = n / HTP_TENSOR_TILE_K_Q1_0;
for (uint32_t kt = 0; kt < n_k_tiles; kt++) {
const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * htp_tensor_get_tile_padded_size(HTP_TYPE_Q1_0));
- HVX_Vector v_tile_c0 = Q6_V_vzero();
- HVX_Vector v_tile_c1 = Q6_V_vzero();
+ HVX_Vector v_tile_qf_c0 = Q6_V_vzero();
+ HVX_Vector v_tile_qf_c1 = Q6_V_vzero();
for (int quarter = 0; quarter < 4; quarter++) {
const uint32_t act_tile = 4 * kt + quarter;
@@ -987,15 +978,17 @@ static void tiled_vec_dot_q1_0_32x2(const uint32_t n, float * restrict s0, float
HVX_Vector v_da_sf_c0 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act0[8]));
HVX_Vector v_da_sf_c1 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act1[8]));
- v_tile_c0 = hvx_vec_add_f32_f32(v_tile_c0, hvx_vec_mul_f32_f32(v_sum_sf_c0, v_da_sf_c0));
- v_tile_c1 = hvx_vec_add_f32_f32(v_tile_c1, hvx_vec_mul_f32_f32(v_sum_sf_c1, v_da_sf_c1));
+ v_tile_qf_c0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c0, Q6_Vqf32_vmpy_VsfVsf(v_sum_sf_c0, v_da_sf_c0));
+ v_tile_qf_c1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c1, Q6_Vqf32_vmpy_VsfVsf(v_sum_sf_c1, v_da_sf_c1));
}
HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4]));
- v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_mul_f32_f32(v_tile_c0, v_dw_sf));
- v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_mul_f32_f32(v_tile_c1, v_dw_sf));
+ v_sum_qf_c0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf_c0, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf_c0), v_dw_sf));
+ v_sum_qf_c1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf_c1, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf_c1), v_dw_sf));
}
+ HVX_Vector v_sum_float_c0 = Q6_Vsf_equals_Vqf32(v_sum_qf_c0);
+ HVX_Vector v_sum_float_c1 = Q6_Vsf_equals_Vqf32(v_sum_qf_c1);
if (sz0) {
hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0)));
} else {
diff --git a/ggml/src/ggml-hexagon/htp/hvx-quant.h b/ggml/src/ggml-hexagon/htp/hvx-quant.h
index b7837967a..b0ca6ecf6 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-quant.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-quant.h
@@ -191,6 +191,64 @@ static inline void hvx_dequantize_row_flat_bf16_f32(float * restrict dst_ptr, co
}
}
+static inline void hvx_dequantize_row_flat_q1_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
+ static const uint32_t __attribute__((aligned(128))) bit_mask[32] = {
+ 1u << 0, 1u << 1, 1u << 2, 1u << 3, 1u << 4, 1u << 5, 1u << 6, 1u << 7,
+ 1u << 8, 1u << 9, 1u << 10, 1u << 11, 1u << 12, 1u << 13, 1u << 14, 1u << 15,
+ 1u << 16, 1u << 17, 1u << 18, 1u << 19, 1u << 20, 1u << 21, 1u << 22, 1u << 23,
+ 1u << 24, 1u << 25, 1u << 26, 1u << 27, 1u << 28, 1u << 29, 1u << 30, 1u << 31,
+ };
+ const int nb = n / QK1_0;
+ const block_q1_0 * src = (const block_q1_0 *) src_ptr;
+ const HVX_Vector v_bit_mask = *(const HVX_Vector *) bit_mask;
+ const HVX_Vector v_zero = Q6_V_vzero();
+ const HVX_Vector v_sign_bit = Q6_V_vsplat_R(0x80000000);
+
+ for (int i = 0; i < nb; i++) {
+ HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
+ HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(vd_f16));
+ HVX_Vector neg_vd = Q6_V_vxor_VV(vd, v_sign_bit);
+ HVX_Vector v_qs = *(const HVX_UVector *) src[i].qs;
+
+ float * block_dst = dst_ptr + i * QK1_0;
+ #pragma unroll
+ for (int q = 0; q < 4; q++) {
+ HVX_Vector v_bits = hvx_vec_repl_u32(q == 0 ? v_qs : Q6_V_vror_VR(v_qs, 4 * q));
+ HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(v_bits, v_bit_mask), v_zero);
+ hvx_vmem(block_dst + q * 32) = Q6_V_vmux_QVV(q_zero, neg_vd, vd);
+ }
+ }
+}
+
+static inline void hvx_dequantize_row_flat_q2_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
+ const int nb = n / QK2_0;
+ const block_q2_0 * src = (const block_q2_0 *) src_ptr;
+ const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
+ const HVX_Vector v_one = Q6_Vb_vsplat_R(1);
+
+ for (int i = 0; i < nb; i++) {
+ HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
+ HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(vd_f16));
+
+ HVX_Vector vq = *(const HVX_UVector *) src[i].qs;
+ HVX_Vector v0 = Q6_V_vand_VV(vq, mask_03);
+ HVX_Vector v1 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vq, 2), mask_03);
+ HVX_Vector v2 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vq, 4), mask_03);
+ HVX_Vector v3 = Q6_Vub_vlsr_VubR(vq, 6);
+
+ HVX_Vector v01 = Q6_V_lo_W(Q6_W_vshuff_VVR(v1, v0, -1));
+ HVX_Vector v23 = Q6_V_lo_W(Q6_W_vshuff_VVR(v3, v2, -1));
+ HVX_Vector v64 = Q6_Vb_vsub_VbVb(Q6_V_lo_W(Q6_W_vshuff_VVR(v23, v01, -2)), v_one);
+
+ HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(v64);
+ HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
+
+ float * block_dst = dst_ptr + i * QK2_0;
+ hvx_vmem(block_dst + 0) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), vd);
+ hvx_vmem(block_dst + 32) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(p32)), vd);
+ }
+}
+
static inline void hvx_dequantize_row_flat_q4_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
const int nb = n / QK4_0;
const block_q4_0 * src = (const block_q4_0 *) src_ptr;
@@ -603,4 +661,66 @@ static inline void hvx_dequantize_row_tiled_f16_f32(float * dst_spad, const uint
}
}
+static inline void hvx_dequantize_row_tiled_q1_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
+ static const uint32_t __attribute__((aligned(128))) bit_mask[32] = {
+ 1u << 0, 1u << 8, 1u << 16, 1u << 24,
+ 1u << 1, 1u << 9, 1u << 17, 1u << 25,
+ 1u << 2, 1u << 10, 1u << 18, 1u << 26,
+ 1u << 3, 1u << 11, 1u << 19, 1u << 27,
+ 1u << 4, 1u << 12, 1u << 20, 1u << 28,
+ 1u << 5, 1u << 13, 1u << 21, 1u << 29,
+ 1u << 6, 1u << 14, 1u << 22, 1u << 30,
+ 1u << 7, 1u << 15, 1u << 23, 1u << 31,
+ };
+ const HVX_Vector v_bit_mask = *(const HVX_Vector *) bit_mask;
+ const HVX_Vector v_zero = Q6_V_vzero();
+ const HVX_Vector v_sign_bit = Q6_V_vsplat_R(0x80000000);
+
+ for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
+ const uint8_t * tile = src_spad + k_tile * tile_stride;
+ float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_K_Q1_0;
+
+ const HVX_Vector scales = hvx_vmem(tile + 512);
+ const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
+ const HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
+ const HVX_Vector neg_vd = Q6_V_vxor_VV(vd, v_sign_bit);
+
+ #pragma unroll
+ for (int q = 0; q < 4; ++q) {
+ const HVX_Vector v_bits = hvx_vec_repl_u32(Q6_V_vror_VR(hvx_vmem(tile + q * VLEN), 4 * row));
+ const HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(v_bits, v_bit_mask), v_zero);
+ hvx_vmem(dst + q * 32) = Q6_V_vmux_QVV(q_zero, neg_vd, vd);
+ }
+ }
+}
+
+static inline void hvx_dequantize_row_tiled_q2_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
+ const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
+ const HVX_Vector v_one = Q6_Vb_vsplat_R(1);
+
+ for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
+ const uint8_t * tile = src_spad + k_tile * tile_stride;
+ float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_K_Q2_0;
+
+ const HVX_Vector v_packed = hvx_dequantize_tiled_f16_gather4(tile, 4 * row);
+ const HVX_Vector p0 = Q6_V_vand_VV(v_packed, mask_03);
+ const HVX_Vector p1 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_packed, 2), mask_03);
+ const HVX_Vector p2 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_packed, 4), mask_03);
+ const HVX_Vector p3 = Q6_Vub_vlsr_VubR(v_packed, 6);
+
+ const HVX_Vector p01 = Q6_V_lo_W(Q6_W_vshuff_VVR(p1, p0, -4));
+ const HVX_Vector p23 = Q6_V_lo_W(Q6_W_vshuff_VVR(p3, p2, -4));
+ const HVX_Vector vq = Q6_Vb_vsub_VbVb(Q6_V_lo_W(Q6_W_vshuff_VVR(p23, p01, -8)), v_one);
+
+ const HVX_Vector scales = hvx_vmem(tile + 512);
+ const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
+ const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
+
+ const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
+ const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
+ hvx_vmem(dst + 0) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
+ hvx_vmem(dst + 32) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(p32)), scale);
+ }
+}
+
#endif // HVX_QUANT_H
diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c
index 2374add3f..b72ef2b75 100644
--- a/ggml/src/ggml-hexagon/htp/main.c
+++ b/ggml/src/ggml-hexagon/htp/main.c
@@ -940,6 +940,7 @@ static int execute_op(struct htp_ops_context * octx) {
case HTP_OP_UNARY_RELU:
case HTP_OP_UNARY_STEP:
case HTP_OP_L2_NORM:
+ case HTP_OP_HADAMARD:
return op_unary(octx);
case HTP_OP_GLU_SWIGLU:
diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c
index 37df31ddb..ea3d92ad0 100644
--- a/ggml/src/ggml-hexagon/htp/unary-ops.c
+++ b/ggml/src/ggml-hexagon/htp/unary-ops.c
@@ -751,6 +751,143 @@ static void l2_norm_f32(const void * restrict src,
}
}
+static inline HVX_VectorPair hvx_fwht_stage_pair(HVX_Vector v0, HVX_Vector v1, int32_t b) {
+ HVX_VectorPair p = Q6_W_vshuff_VVR(v1, v0, b);
+ HVX_Vector s = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(p), Q6_V_hi_W(p));
+ HVX_Vector d = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_V_lo_W(p), Q6_V_hi_W(p));
+ return Q6_W_vshuff_VVR(d, s, b);
+}
+
+static inline HVX_VectorPair hvx_fwht32_pair(HVX_Vector v0_sf, HVX_Vector v1_sf, HVX_Vector v_scale) {
+ HVX_Vector v0 = Q6_Vqf32_vmpy_VsfVsf(v0_sf, v_scale);
+ HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(v1_sf, v_scale);
+ HVX_VectorPair p;
+ p = hvx_fwht_stage_pair(v0, v1, 4);
+ p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 8);
+ p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 16);
+ p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 32);
+ p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 64);
+ return p;
+}
+
+static inline void hvx_fwht_store32(HVX_Vector * restrict v_dst, HVX_VectorPair p) {
+ v_dst[0] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p));
+}
+
+static inline void hvx_fwht_store64(HVX_Vector * restrict v_dst, HVX_VectorPair p) {
+ HVX_Vector u = Q6_V_lo_W(p);
+ HVX_Vector v = Q6_V_hi_W(p);
+ v_dst[0] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(u, v));
+ v_dst[1] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(u, v));
+}
+
+static inline void hvx_fwht_tail_f32(HVX_Vector * restrict v_dst, uint32_t nvec) {
+ uint32_t len_vec = 2;
+ for (; len_vec * 2 < nvec; len_vec <<= 2) {
+ const uint32_t step = len_vec * 4;
+ for (uint32_t i = 0; i < nvec; i += step) {
+ for (uint32_t j = 0; j < len_vec; j++) {
+ HVX_Vector a0 = v_dst[i + j];
+ HVX_Vector a1 = v_dst[i + j + len_vec];
+ HVX_Vector a2 = v_dst[i + j + 2 * len_vec];
+ HVX_Vector a3 = v_dst[i + j + 3 * len_vec];
+
+ HVX_Vector t0 = Q6_Vqf32_vadd_VsfVsf(a0, a1);
+ HVX_Vector t1 = Q6_Vqf32_vsub_VsfVsf(a0, a1);
+ HVX_Vector t2 = Q6_Vqf32_vadd_VsfVsf(a2, a3);
+ HVX_Vector t3 = Q6_Vqf32_vsub_VsfVsf(a2, a3);
+
+ v_dst[i + j] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(t0, t2));
+ v_dst[i + j + len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(t1, t3));
+ v_dst[i + j + 2 * len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(t0, t2));
+ v_dst[i + j + 3 * len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(t1, t3));
+ }
+ }
+ }
+
+ if (len_vec < nvec) {
+ for (uint32_t i = 0; i < nvec; i += 2 * len_vec) {
+ for (uint32_t j = 0; j < len_vec; j++) {
+ HVX_Vector u = v_dst[i + j];
+ HVX_Vector v = v_dst[i + len_vec + j];
+ v_dst[i + j] = hvx_vec_add_f32_f32(u, v);
+ v_dst[i + len_vec + j] = hvx_vec_sub_f32_f32(u, v);
+ }
+ }
+ }
+}
+
+static inline void hvx_fwht_row_f32(const uint8_t * restrict src, uint8_t * restrict dst, uint32_t n, HVX_Vector v_scale) {
+ const HVX_Vector * restrict v_src = (const HVX_Vector *) src;
+ HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
+ const uint32_t nvec = n / VLEN_FP32;
+
+ if (nvec == 1) {
+ HVX_VectorPair p = hvx_fwht32_pair(v_src[0], v_src[0], v_scale);
+ hvx_fwht_store32(v_dst, p);
+ return;
+ }
+
+ for (uint32_t i = 0; i < nvec; i += 2) {
+ HVX_VectorPair p = hvx_fwht32_pair(v_src[i], v_src[i + 1], v_scale);
+ hvx_fwht_store64(&v_dst[i], p);
+ }
+
+ hvx_fwht_tail_f32(v_dst, nvec);
+}
+
+static inline void hvx_fwht_row_f16(const uint8_t * restrict src, uint8_t * restrict dst, uint32_t n, HVX_Vector v_scale) {
+ const HVX_Vector * restrict v_src = (const HVX_Vector *) src;
+ HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
+ const uint32_t nvec = n / VLEN_FP32;
+
+ if (nvec == 1) {
+ HVX_Vector v0 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_src[0]));
+ HVX_VectorPair p = hvx_fwht32_pair(v0, v0, v_scale);
+ hvx_fwht_store32(v_dst, p);
+ return;
+ }
+
+ const uint32_t nvec_f16 = n / VLEN_FP16;
+ for (uint32_t i = 0; i < nvec_f16; i++) {
+ HVX_VectorPair f32p = hvx_vec_f16_to_f32(v_src[i]);
+ HVX_VectorPair p = hvx_fwht32_pair(Q6_V_lo_W(f32p), Q6_V_hi_W(f32p), v_scale);
+ hvx_fwht_store64(&v_dst[2 * i], p);
+ }
+
+ hvx_fwht_tail_f32(v_dst, nvec);
+}
+
+static void hadamard_f32(const void * restrict src,
+ void * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+ const HVX_Vector v_scale = hvx_vec_splat_f32(1.0f / sqrtf((float) ne0));
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_row = (const uint8_t *) src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_row = (uint8_t *) dst + (ir * dst_row_size_aligned);
+
+ hvx_fwht_row_f32(src_row, dst_row, ne0, v_scale);
+ }
+}
+
+static void hadamard_f16(const void * restrict src,
+ void * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+ const HVX_Vector v_scale = hvx_vec_splat_f32(1.0f / sqrtf((float) ne0));
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_row = (const uint8_t *) src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_row = (uint8_t *) dst + (ir * dst_row_size_aligned);
+
+ hvx_fwht_row_f16(src_row, dst_row, ne0, v_scale);
+ }
+}
+
static void tanh_f32(const void * restrict src,
void * restrict dst,
const uint32_t num_rows,
@@ -1644,6 +1781,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
case HTP_OP_UNARY_STEP: op_type = is_f16 ? "step-f16" : "step-f32"; break;
case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break;
case HTP_OP_TRI: op_type = "tri-f32"; break;
+ case HTP_OP_HADAMARD: op_type = is_f16 ? "hadamard-f16" : "hadamard-f32"; break;
default:
FARF(ERROR, "Unsupported unary Op %u\n", octx->op);
return HTP_STATUS_NO_SUPPORT;
@@ -1663,6 +1801,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
case HTP_OP_UNARY_ABS:
case HTP_OP_UNARY_LOG:
case HTP_OP_UNARY_STEP:
+ case HTP_OP_HADAMARD:
break;
default:
FARF(ERROR, "unary-%s: not supported for F16\n", op_type);
@@ -1677,16 +1816,17 @@ static int execute_op_unary(struct htp_ops_context * octx) {
}
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const size_t elem_size = is_f16 ? sizeof(_Float16) : sizeof(float);
- const size_t src0_data_row_size = src0->ne[0] * elem_size;
- const size_t dst_data_row_size = dst->ne[0] * elem_size;
+ const size_t src0_elem_size = is_f16 ? sizeof(_Float16) : sizeof(float);
+ const size_t dst_elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float);
+ const size_t src0_data_row_size = src0->ne[0] * src0_elem_size;
+ const size_t dst_data_row_size = dst->ne[0] * dst_elem_size;
uint32_t row_start = 0;
uint32_t nrows = src0_nrows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
- htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk);
+ htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) dst_elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
@@ -1813,6 +1953,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
case HTP_OP_UNARY_ABS: compute_func = (void *) abs_f16; break;
case HTP_OP_UNARY_LOG: compute_func = (void *) log_f16; break;
case HTP_OP_UNARY_STEP: compute_func = (void *) step_f16; break;
+ case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f16; break;
default: break;
}
} else {
@@ -1843,6 +1984,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
case HTP_OP_UNARY_RELU: compute_func = (void *) relu_f32; break;
case HTP_OP_UNARY_STEP: compute_func = (void *) step_f32; break;
case HTP_OP_L2_NORM: compute_func = (void *) l2_norm_f32; break;
+ case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f32; break;
case HTP_OP_TRI:
task_func = unary_thread_tri_f32;
compute_func = (void *) tri_f32;
diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h
index 7e5779e5a..902028635 100644
--- a/ggml/src/ggml-hexagon/htp/unary-ops.h
+++ b/ggml/src/ggml-hexagon/htp/unary-ops.h
@@ -64,6 +64,7 @@ static inline bool htp_op_is_unary(uint32_t opcode) {
case HTP_OP_UNARY_STEP:
case HTP_OP_L2_NORM:
case HTP_OP_TRI:
+ case HTP_OP_HADAMARD:
return true;
default:
return false;
@@ -94,8 +95,9 @@ static inline void htp_unary_vtcm_layout_build(
uint32_t * out_col_tile,
uint32_t * out_vtcm_row_per_thread
) {
+ const size_t dst_elem_size = (op == HTP_OP_HADAMARD) ? sizeof(float) : elem_size;
const size_t src0_data_row_size = ne00 * elem_size;
- const size_t dst_data_row_size = ne10 * elem_size;
+ const size_t dst_data_row_size = ne10 * dst_elem_size;
const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);
const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128);
@@ -131,7 +133,8 @@ static inline void htp_unary_vtcm_layout_build(
}
const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM ||
- op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM);
+ op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM ||
+ op == HTP_OP_HADAMARD);
// The tiled fallback path below only has F32 task functions (unary_task_f32_tiled_*);
// F16 has no tiled kernels, so it must stay on the row-block path like reduction ops.
// NOTE: if F16 ends up with vtcm_row_per_thread == 0 here (row too large for the VTCM