Commit 601dbd62 for whisper.cpp
commit 601dbd626944239d4d5db1f960c69c8495b32338
Author: Max Krasnyansky <maxk@qti.qualcomm.com>
Date: Sun Oct 25 02:11:48 2026 -0700
hexagon: generic dims collapse/reshape for unary, binary and matmul ops (llama/31071)
* hex-tools: support for --compare option in the profile.py
* hex-unary: improve collapse logic to also do splits if nrows is small
* hex-colapse: handle all colapsing at the host and make it generic
* hex-colapse: remove leftover unary helper
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index c7e8ea85..45e6ae90 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -2397,7 +2397,7 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
}
}
- GGML_ASSERT(htp_kparams_solve(sess, node.node, node.kernel_params));
+ GGML_ASSERT(htp_kparams_solve(sess, node));
if (node.opcode == HTP_OP_CPY && node.kparams<htp_copy_kernel_params>()->total_elems == 0) {
continue;
}
diff --git a/ggml/src/ggml-hexagon/htp-fusion.cpp b/ggml/src/ggml-hexagon/htp-fusion.cpp
index 65aacf73..ba2dd641 100644
--- a/ggml/src/ggml-hexagon/htp-fusion.cpp
+++ b/ggml/src/ggml-hexagon/htp-fusion.cpp
@@ -192,15 +192,20 @@ static bool fuse_mul_mat_add(const ggml_hexagon_session * sess, htp_opnode & pre
if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &src2)) return false;
if (src2->type != GGML_TYPE_F32) return false;
- const ggml_tensor * src0 = prev.src0();
- const ggml_tensor * src1 = prev.src1();
- const ggml_tensor * dst = cur.dst();
- if (!ggml_are_same_shape(prev.dst(), dst)) return false;
+ const ggml_tensor * src0 = prev.node->src[0];
+ const ggml_tensor * src1 = prev.node->src[1];
+ const ggml_tensor * dst = cur.node;
+ if (!ggml_are_same_shape(prev.node, dst)) return false;
+
+ const bool prev_collapsed = htp_matmul_can_collapse(src0, src1, prev.node);
+ const bool cur_collapsed = htp_matmul_add_can_collapse(src0, src1, src2, dst);
+ if (prev_collapsed != cur_collapsed) {
+ return false;
+ }
const auto * orig_kparams = prev.kparams<htp_mm_kernel_params>();
struct htp_mm_kernel_params kparams;
- if (!htp_kparams_solve_matmul_add(sess, src0, src1, src2, dst, &kparams) ||
- orig_kparams->collapse != kparams.collapse) {
+ if (!htp_kparams_solve_matmul_add(sess, src0, src1, src2, dst, &kparams)) {
return false;
}
@@ -222,6 +227,7 @@ static bool fuse_mul_mat_add(const ggml_hexagon_session * sess, htp_opnode & pre
prev.opcode = HTP_OP_MUL_MAT_ADD;
prev.inputs = { src0, src1, src2 };
prev.outputs = { dst };
+ htp_opnode_collapse(sess, prev);
prev.fused.push_back(cur.node);
prev.set_kparams(kparams);
return true;
@@ -233,36 +239,31 @@ static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev
const uint32_t curr_n = (uint32_t) prev.outputs.size();
if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 1 >= HTP_OP_MAX_INPUTS) return false;
- const ggml_tensor * w0 = prev.inputs[0];
- const ggml_tensor * x = prev.inputs[curr_n];
- const ggml_tensor * w_in = cur.src0();
- const ggml_tensor * x_in = cur.src1();
- const ggml_tensor * d_in = cur.dst();
+ const ggml_tensor * w0 = prev.inputs[0];
+ const ggml_tensor * x_orig = prev.node->src[1];
+ const ggml_tensor * w_in = cur.node->src[0];
+ const ggml_tensor * x_in = cur.node->src[1];
+ const ggml_tensor * d_in = cur.node;
auto * orig_kparams = prev.kparams<htp_mm_kernel_params>();
const auto * cur_kparams = cur.kparams<htp_mm_kernel_params>();
- if (x_in != x || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] ||
+ if (x_in != x_orig || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] ||
w0->ne[2] != 1 || w0->ne[3] != 1 || w_in->ne[2] != 1 || w_in->ne[3] != 1 ||
htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) ||
orig_kparams->n_hmx != cur_kparams->n_hmx) {
return false;
}
- const bool collapse = orig_kparams->collapse && htp_matmul_can_collapse(w_in, x, d_in);
- if ((x->ne[2] > 1 || x->ne[3] > 1) && !collapse) return false;
+ const bool collapse = htp_matmul_can_collapse(w0, x_orig, prev.node) && htp_matmul_can_collapse(w_in, x_in, d_in);
+ if ((x_orig->ne[2] > 1 || x_orig->ne[3] > 1) && !collapse) return false;
if (prev.opcode == HTP_OP_MUL_MAT_NX) {
orig_kparams->n_weights = curr_n + 1;
} else {
struct htp_mm_kernel_params kparams;
- if (collapse) {
- const ggml_tensor x_collapsed = htp_tensor_collapse_rows(x);
- if (!htp_kparams_solve_mmnx(sess, w0, &x_collapsed, curr_n + 1, &kparams)) return false;
- kparams.collapse = 1;
- } else if (!htp_kparams_solve_mmnx(sess, w0, x, curr_n + 1, &kparams)) {
- return false;
- }
+ const ggml_tensor x_collapsed = collapse ? htp_tensor_collapse_rows(x_orig) : *x_orig;
+ if (!htp_kparams_solve_mmnx(sess, w0, &x_collapsed, curr_n + 1, &kparams)) return false;
if (!is_supported_mul_mat_nx_kernel(w0, &kparams)) return false;
prev.set_kparams(kparams);
@@ -270,7 +271,7 @@ static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev
}
prev.inputs.insert(prev.inputs.begin() + curr_n, w_in);
- prev.outputs.push_back(d_in);
+ prev.outputs.push_back(collapse ? prev.add_dummy(htp_tensor_collapse_rows(d_in)) : d_in);
prev.fused.push_back(cur.node);
return true;
}
@@ -415,15 +416,13 @@ static bool fuse_unary_mul_glu(const ggml_hexagon_session * sess, htp_opnode & p
const ggml_tensor * dst = cur.dst();
if (htp_inputs_overlap_unsafe(prev, dst, up)) return false;
- struct htp_glu_kernel_params kparams;
- if (!htp_kparams_solve_unary_mul_glu(sess, glu_op, gate, up, dst, &kparams)) return false;
-
prev.opcode = glu_op;
prev.inputs = { gate, up };
prev.outputs = { dst };
+ if (!htp_kparams_solve(sess, prev)) return false;
+
prev.fused.push_back(cur.node);
memset(prev.params, 0, sizeof(prev.params));
- prev.set_kparams(kparams);
return true;
}
diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp
index a8f01a4c..b8d1aee3 100644
--- a/ggml/src/ggml-hexagon/htp-kparams.cpp
+++ b/ggml/src/ggml-hexagon/htp-kparams.cpp
@@ -1,6 +1,7 @@
#define _USE_MATH_DEFINES
#include "htp-kparams.h"
+#include "htp-opnode.h"
#include "ggml-impl.h"
#include <algorithm>
@@ -1023,14 +1024,12 @@ static bool htp_kparams_solve_matmul(
if (htp_matmul_can_collapse(src0, src1, dst)) {
const ggml_tensor src1_collapsed = htp_tensor_collapse_rows(src1);
const ggml_tensor dst_collapsed = htp_tensor_collapse_rows(dst);
- const bool ok = htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, 0, 0, kparams);
- kparams->collapse = 1;
- return ok;
+ return htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, 0, 0, kparams);
}
return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, 0, 0, kparams);
}
-static bool htp_matmul_add_can_collapse(
+bool htp_matmul_add_can_collapse(
const struct ggml_tensor * src0,
const struct ggml_tensor * src1,
const struct ggml_tensor * src2,
@@ -1068,9 +1067,7 @@ bool htp_kparams_solve_matmul_add(
: (src2 ? *src2 : ggml_tensor{});
const struct ggml_tensor * p_src2 = src2 ? &src2_collapsed : nullptr;
const size_t src2_size = p_src2 ? hex_round_up(ggml_nbytes(p_src2), 128) : 0;
- const bool ok = htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, p_src2 ? p_src2->nb[1] : 0, src2_size, kparams);
- kparams->collapse = 1;
- return ok;
+ return htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, p_src2 ? p_src2->nb[1] : 0, src2_size, kparams);
}
const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0;
return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, kparams);
@@ -1405,47 +1402,103 @@ static bool htp_kparams_solve_binary(
}
// collapse contiguous dims of elementwise ops so that rows fill at least one HVX vector
-static void htp_unary_collapse_dims(struct ggml_tensor * src, struct ggml_tensor * dst, size_t elem_size) {
- if (src->nb[0] != elem_size || dst->nb[0] != elem_size) {
+static void htp_collapse_dims(struct ggml_tensor ** tensors, size_t n_tensors, size_t elem_size, uint32_t n_threads) {
+ if (n_tensors == 0 || !tensors[0]) {
return;
}
+ for (size_t t = 0; t < n_tensors; t++) {
+ if (!tensors[t] || tensors[t]->nb[0] != elem_size) {
+ return;
+ }
+ if (t > 0 && !ggml_are_same_shape(tensors[0], tensors[t])) {
+ return;
+ }
+ }
+
+ struct ggml_tensor * t0 = tensors[0];
- while (src->ne[1] > 1 || src->ne[2] > 1 || src->ne[3] > 1) {
- const size_t row_size = (size_t) src->ne[0] * elem_size;
- const bool rows_contig = src->nb[1] == row_size && dst->nb[1] == (size_t) dst->ne[0] * elem_size;
- const bool can_merge = (src->ne[1] == 1 && dst->ne[1] == 1) ||
- (rows_contig && (src->ne[0] == 1 || (row_size < 128 && row_size * src->ne[1] <= 2048)));
+ while (t0->ne[1] > 1 || t0->ne[2] > 1 || t0->ne[3] > 1) {
+ const size_t row_size = (size_t) t0->ne[0] * elem_size;
+ bool rows_contig = true;
+ for (size_t t = 0; t < n_tensors; t++) {
+ if (tensors[t]->nb[1] != (size_t) tensors[t]->ne[0] * elem_size) {
+ rows_contig = false;
+ break;
+ }
+ }
+ bool all_ne1_one = true;
+ for (size_t t = 0; t < n_tensors; t++) {
+ if (tensors[t]->ne[1] != 1) {
+ all_ne1_one = false;
+ break;
+ }
+ }
+ const bool can_merge = all_ne1_one ||
+ (rows_contig && (t0->ne[0] == 1 || (row_size < 128 && row_size * t0->ne[1] <= 2048)));
if (!can_merge) {
break;
}
- src->ne[0] *= src->ne[1];
- src->ne[1] = src->ne[2];
- src->ne[2] = src->ne[3];
- src->ne[3] = 1;
- dst->ne[0] *= dst->ne[1];
- dst->ne[1] = dst->ne[2];
- dst->ne[2] = dst->ne[3];
- dst->ne[3] = 1;
- src->nb[1] = src->nb[2];
- src->nb[2] = src->nb[3];
- dst->nb[1] = dst->nb[2];
- dst->nb[2] = dst->nb[3];
+ for (size_t t = 0; t < n_tensors; t++) {
+ tensors[t]->ne[0] *= tensors[t]->ne[1];
+ tensors[t]->ne[1] = tensors[t]->ne[2];
+ tensors[t]->ne[2] = tensors[t]->ne[3];
+ tensors[t]->ne[3] = 1;
+ tensors[t]->nb[1] = tensors[t]->nb[2];
+ tensors[t]->nb[2] = tensors[t]->nb[3];
+ tensors[t]->nb[3] = tensors[t]->nb[2] * tensors[t]->ne[2];
+ }
}
// ne1 too big to merge in full: merge it in chunks of k rows (k divides ne1)
- const size_t row_size = (size_t) src->ne[0] * elem_size;
- if (src->ne[1] > 1 && row_size < 128 && src->nb[1] == row_size && dst->nb[1] == row_size) {
+ const size_t row_size = (size_t) t0->ne[0] * elem_size;
+ bool rows_contig = true;
+ for (size_t t = 0; t < n_tensors; t++) {
+ if (tensors[t]->nb[1] != row_size) {
+ rows_contig = false;
+ break;
+ }
+ }
+ if (t0->ne[1] > 1 && row_size < 128 && rows_contig) {
int64_t k = 2048 / row_size;
- while (k > 1 && src->ne[1] % k != 0) {
+ while (k > 1 && t0->ne[1] % k != 0) {
k--;
}
if (k > 1) {
- src->ne[0] *= k;
- src->ne[1] /= k;
- src->nb[1] *= k;
- dst->ne[0] *= k;
- dst->ne[1] /= k;
- dst->nb[1] *= k;
+ for (size_t t = 0; t < n_tensors; t++) {
+ tensors[t]->ne[0] *= k;
+ tensors[t]->ne[1] /= k;
+ tensors[t]->nb[1] *= k;
+ tensors[t]->nb[2] = tensors[t]->nb[1] * tensors[t]->ne[1];
+ tensors[t]->nb[3] = tensors[t]->nb[2] * tensors[t]->ne[2];
+ }
+ }
+ }
+
+ // nrows too small to saturate threads: split ne0 into k rows (k divides ne0)
+ const size_t row_size_curr = (size_t) t0->ne[0] * elem_size;
+ const uint32_t nrows = (uint32_t) (t0->ne[1] * t0->ne[2] * t0->ne[3]);
+ rows_contig = true;
+ for (size_t t = 0; t < n_tensors; t++) {
+ if (tensors[t]->ne[1] != 1 && tensors[t]->nb[1] != row_size_curr) {
+ rows_contig = false;
+ break;
+ }
+ }
+ if (nrows < n_threads && rows_contig && row_size_curr >= 4096) {
+ const size_t vec_elems = 128 / elem_size;
+ int64_t k = std::max<int64_t>(row_size_curr / 16384, (n_threads * 4 + nrows - 1) / nrows);
+ k = std::min<int64_t>(k, row_size_curr / 2048);
+ while (k > 1 && t0->ne[0] % (k * vec_elems) != 0) {
+ k--;
+ }
+ if (k > 1) {
+ for (size_t t = 0; t < n_tensors; t++) {
+ tensors[t]->ne[0] /= k;
+ tensors[t]->ne[1] *= k;
+ tensors[t]->nb[1] = tensors[t]->ne[0] * elem_size;
+ tensors[t]->nb[2] = tensors[t]->nb[1] * tensors[t]->ne[1];
+ tensors[t]->nb[3] = tensors[t]->nb[2] * tensors[t]->ne[2];
+ }
}
}
}
@@ -1507,19 +1560,6 @@ static bool htp_kparams_solve_unary_impl(
const size_t elem_size = ggml_type_size(src0->type);
- struct ggml_tensor src0_c = *src0;
- struct ggml_tensor dst_c = *dst;
- if (htp_unary_is_elementwise(op)) {
- htp_unary_collapse_dims(&src0_c, &dst_c, elem_size);
- src0 = &src0_c;
- dst = &dst_c;
- for (int i = 0; i < HTP_OP_MAX_DIMS; i++) {
- kparams->ne[i] = (uint32_t) src0->ne[i];
- kparams->src0_nb[i] = (uint32_t) src0->nb[i];
- kparams->dst_nb[i] = (uint32_t) dst->nb[i];
- }
- }
-
const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type);
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
@@ -1533,9 +1573,14 @@ static bool htp_kparams_solve_unary_impl(
const bool broadcast_weight = (op == HTP_OP_RMS_NORM_MUL) && (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1);
kparams->broadcast_weight = broadcast_weight;
- const bool src0_contig = ggml_is_contiguous_1(src0);
- const bool src1_contig = (op == HTP_OP_RMS_NORM_MUL && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true;
- const uint32_t max_block = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1];
+ const bool src0_contig = ggml_is_contiguous_1(src0);
+ const bool src1_contig = (op == HTP_OP_RMS_NORM_MUL && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true;
+ const uint32_t src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
+ const uint32_t min_block_rows = (uint32_t) (std::max<size_t>(1, 2048 / dst_data_row_size));
+ uint32_t max_block = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1];
+ if (src0_nrows_per_thread >= 2 * min_block_rows) {
+ max_block = (std::min)(max_block, (std::max)(min_block_rows, src0_nrows_per_thread / 4));
+ }
kparams->src0_contig = src0_contig ? 1 : 0;
kparams->src1_contig = src1_contig ? 1 : 0;
@@ -2000,11 +2045,17 @@ bool htp_kparams_solve_unary_mul_glu(
const size_t vtcm_size = htp_session_vtcm_size(sess);
const size_t row_size_aligned = hex_round_up(dst_row_size, 128);
- const uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * 3 * row_size_aligned));
+ uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * 3 * row_size_aligned));
if (vtcm_row_per_thread < 2) {
return false;
}
+ const uint32_t src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
+ if (src0_nrows_per_thread >= 2 && vtcm_row_per_thread > src0_nrows_per_thread) {
+ uint32_t target_rpt = (src0_nrows_per_thread >= 4) ? (src0_nrows_per_thread / 2) & ~1u : 2;
+ vtcm_row_per_thread = (std::max)(2u, (std::min)(vtcm_row_per_thread, target_rpt));
+ }
+
struct htp_glu_vtcm_layout L;
htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, vtcm_row_per_thread);
@@ -3246,12 +3297,131 @@ static inline bool htp_kparams_invoke(
return SolveFn(sess, op, kparams ? (KParams *) kparams : &tmp);
}
+void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & node) {
+ const uint32_t n_threads = htp_session_n_threads(sess);
+
+ if (htp_op_is_unary(node.opcode) && htp_unary_is_elementwise(node.opcode)) {
+ if (node.inputs.empty() || !node.inputs[0] || node.outputs.empty() || !node.outputs[0]) {
+ return;
+ }
+ if (!ggml_are_same_shape(node.inputs[0], node.outputs[0])) {
+ return;
+ }
+ const size_t elem_size = ggml_type_size(node.inputs[0]->type);
+ struct ggml_tensor src0_c = *node.inputs[0];
+ struct ggml_tensor dst_c = *node.outputs[0];
+ struct ggml_tensor * tensors[2] = { &src0_c, &dst_c };
+ htp_collapse_dims(tensors, 2, elem_size, n_threads);
+ if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) {
+ node.inputs[0] = node.add_dummy(src0_c);
+ node.outputs[0] = node.add_dummy(dst_c);
+ }
+ return;
+ }
+
+ if (htp_op_is_glu(node.opcode)) {
+ if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) {
+ return;
+ }
+ if (!ggml_are_same_shape(node.inputs[0], node.outputs[0]) ||
+ !ggml_are_same_shape(node.inputs[1], node.outputs[0])) {
+ return;
+ }
+ const size_t elem_size = ggml_type_size(node.inputs[0]->type);
+ struct ggml_tensor src0_c = *node.inputs[0];
+ struct ggml_tensor src1_c = *node.inputs[1];
+ struct ggml_tensor dst_c = *node.outputs[0];
+ struct ggml_tensor * tensors[3] = { &src0_c, &src1_c, &dst_c };
+ htp_collapse_dims(tensors, 3, elem_size, n_threads);
+ if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) {
+ node.inputs[0] = node.add_dummy(src0_c);
+ node.inputs[1] = node.add_dummy(src1_c);
+ node.outputs[0] = node.add_dummy(dst_c);
+ }
+ return;
+ }
+
+ if (node.opcode == HTP_OP_MUL_MAT) {
+ if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) {
+ return;
+ }
+ const struct ggml_tensor * src0 = node.inputs[0];
+ const struct ggml_tensor * src1 = node.inputs[1];
+ const struct ggml_tensor * dst = node.outputs[0];
+ if (htp_matmul_can_collapse(src0, src1, dst)) {
+ node.inputs[1] = node.add_dummy(htp_tensor_collapse_rows(src1));
+ node.outputs[0] = node.add_dummy(htp_tensor_collapse_rows(dst));
+ }
+ return;
+ }
+
+ if (node.opcode == HTP_OP_MUL_MAT_ADD) {
+ if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) {
+ return;
+ }
+ const struct ggml_tensor * src0 = node.inputs[0];
+ const struct ggml_tensor * src1 = node.inputs[1];
+ const struct ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr;
+ const struct ggml_tensor * dst = node.outputs[0];
+ if (htp_matmul_add_can_collapse(src0, src1, src2, dst)) {
+ node.inputs[1] = node.add_dummy(htp_tensor_collapse_rows(src1));
+ node.outputs[0] = node.add_dummy(htp_tensor_collapse_rows(dst));
+ if (src2 && (src2->ne[1] * src2->ne[2] * src2->ne[3] > 1)) {
+ node.inputs[2] = node.add_dummy(htp_tensor_collapse_rows(src2));
+ }
+ }
+ return;
+ }
+}
+
+bool htp_kparams_solve(
+ const struct ggml_hexagon_session * sess,
+ htp_opnode & node
+) {
+ htp_opnode_collapse(sess, node);
+
+ const ggml_tensor * src0 = node.inputs.empty() ? nullptr : node.inputs[0];
+ const ggml_tensor * src1 = node.inputs.size() > 1 ? node.inputs[1] : nullptr;
+ const ggml_tensor * dst = node.dst();
+
+ if (htp_op_is_unary(node.opcode)) {
+ return htp_kparams_solve_unary_impl(sess, node.opcode, src0, src1, dst, node.kparams<htp_unary_kernel_params>());
+ }
+ if (htp_op_is_glu(node.opcode)) {
+ return htp_kparams_solve_unary_mul_glu(sess, node.opcode, src0, src1, dst, node.kparams<htp_glu_kernel_params>());
+ }
+ if (node.opcode == HTP_OP_MUL_MAT) {
+ if (!htp_matmul_check_types(src0, src1, dst)) {
+ return false;
+ }
+ return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, 0, 0, node.kparams<htp_mm_kernel_params>());
+ }
+ if (node.opcode == HTP_OP_MUL_MAT_ADD) {
+ const ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr;
+ const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0;
+ return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, node.kparams<htp_mm_kernel_params>());
+ }
+
+ return htp_kparams_solve(sess, node.node, node.kernel_params);
+}
+
bool htp_kparams_solve(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * op,
void * kparams
) {
const htp_op_code opcode = htp_op_remap(op);
+ if (htp_op_is_unary(opcode) || htp_op_is_glu(opcode)) {
+ htp_opnode tmp(opcode, op);
+ if (!htp_kparams_solve(sess, tmp)) {
+ return false;
+ }
+ if (kparams) {
+ memcpy(kparams, tmp.kernel_params, sizeof(tmp.kernel_params));
+ }
+ return true;
+ }
+
switch (opcode) {
case HTP_OP_MUL_MAT:
case HTP_OP_MUL_MAT_ID:
diff --git a/ggml/src/ggml-hexagon/htp-kparams.h b/ggml/src/ggml-hexagon/htp-kparams.h
index 0871209e..0bde658d 100644
--- a/ggml/src/ggml-hexagon/htp-kparams.h
+++ b/ggml/src/ggml-hexagon/htp-kparams.h
@@ -34,6 +34,7 @@
bool htp_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride);
bool htp_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst);
+bool htp_matmul_add_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * src2, const struct ggml_tensor * dst);
ggml_tensor htp_tensor_collapse_rows(const struct ggml_tensor * t);
template <typename... Args>
@@ -45,6 +46,18 @@ static inline bool htp_kparams_check(Args &&... args) {
return SolveFn(std::forward<Args>(args)..., &kparams);
}
+struct htp_opnode;
+
+bool htp_kparams_solve(
+ const struct ggml_hexagon_session * sess,
+ struct htp_opnode & node
+);
+
+void htp_opnode_collapse(
+ const struct ggml_hexagon_session * sess,
+ struct htp_opnode & node
+);
+
bool htp_kparams_solve(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * op,
diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c
index eb40fa64..784bb37b 100644
--- a/ggml/src/ggml-hexagon/htp/matmul-ops.c
+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c
@@ -3775,16 +3775,6 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k
return HTP_STATUS_OK;
}
-static inline void htp_mm_tensor_collapse_rows(struct htp_tensor * c, const struct htp_tensor * t, uint32_t stride) {
- *c = *t;
- c->ne[1] = t->ne[1] * t->ne[2] * t->ne[3];
- c->ne[2] = 1;
- c->ne[3] = 1;
- c->nb[1] = stride;
- c->nb[2] = c->nb[1] * c->ne[1];
- c->nb[3] = c->nb[2];
-}
-
static int op_matmul_impl(struct htp_ops_context * octx) {
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
@@ -3801,36 +3791,6 @@ static int op_matmul_impl(struct htp_ops_context * octx) {
}
int op_matmul(struct htp_ops_context * octx) {
- const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
-
- if (kparams->collapse) {
- const struct htp_tensor * act = octx->src[1];
- const struct htp_tensor * dst = octx->dst;
- const uint32_t s_act = (act->ne[1] > 1) ? act->nb[1] : ((act->ne[2] > 1) ? act->nb[2] : act->nb[3]);
- const uint32_t sd = (dst->ne[1] > 1) ? dst->nb[1] : ((dst->ne[2] > 1) ? dst->nb[2] : dst->nb[3]);
- struct htp_tensor act_collapsed, dst_collapsed;
- htp_mm_tensor_collapse_rows(&act_collapsed, act, s_act);
- htp_mm_tensor_collapse_rows(&dst_collapsed, dst, sd);
- octx->src[1] = &act_collapsed;
- octx->dst = &dst_collapsed;
-
- struct htp_tensor src2_collapsed;
- const struct htp_tensor * src2 = octx->src[2];
- if (src2 && (src2->ne[1] * src2->ne[2] * src2->ne[3] > 1)) {
- const uint32_t s2 = (src2->ne[1] > 1) ? src2->nb[1] : ((src2->ne[2] > 1) ? src2->nb[2] : src2->nb[3]);
- htp_mm_tensor_collapse_rows(&src2_collapsed, src2, s2);
- octx->src[2] = &src2_collapsed;
- }
-
- const int status = op_matmul_impl(octx);
- octx->src[1] = act;
- octx->dst = dst;
- if (src2) {
- octx->src[2] = src2;
- }
- return status;
- }
-
return op_matmul_impl(octx);
}
@@ -4562,35 +4522,5 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) {
}
int op_matmul_nx(struct htp_ops_context * octx) {
- const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
-
- if (kparams->collapse) {
- const uint32_t n_weights = kparams->n_weights;
- const struct htp_tensor * act = octx->src[n_weights];
- const uint32_t s1 = (act->ne[1] > 1) ? act->nb[1] : ((act->ne[2] > 1) ? act->nb[2] : act->nb[3]);
- struct htp_tensor act_collapsed;
- struct htp_tensor dsts_collapsed[HTP_OP_MAX_OUTPUTS];
- const struct htp_tensor * orig_dsts[HTP_OP_MAX_OUTPUTS];
-
- htp_mm_tensor_collapse_rows(&act_collapsed, act, s1);
- octx->src[n_weights] = &act_collapsed;
-
- for (uint32_t p = 0; p < n_weights; p++) {
- orig_dsts[p] = octx->dsts[p];
- const struct htp_tensor * d = octx->dsts[p];
- const uint32_t sd = (d->ne[1] > 1) ? d->nb[1] : ((d->ne[2] > 1) ? d->nb[2] : d->nb[3]);
- htp_mm_tensor_collapse_rows(&dsts_collapsed[p], d, sd);
- octx->dsts[p] = &dsts_collapsed[p];
- }
-
- const int status = op_matmul_nx_impl(octx);
-
- octx->src[n_weights] = act;
- for (uint32_t p = 0; p < n_weights; p++) {
- octx->dsts[p] = orig_dsts[p];
- }
- return status;
- }
-
return op_matmul_nx_impl(octx);
}
diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h
index 474c2dbc..4971ed95 100644
--- a/ggml/src/ggml-hexagon/htp/matmul-ops.h
+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h
@@ -61,7 +61,7 @@ enum htp_mm_kernel_type {
struct htp_mm_kernel_params {
uint8_t kernel_type; // enum htp_mm_kernel_type
uint8_t pipeline; // 1 = pipelined execution, 0 = standard
- uint8_t collapse; // 1 = collapse outer dims into 2D, 0 = standard
+ uint8_t unused; // reserved
uint8_t n_hmx; // 1 = use HMX, 0 = use HVX
uint8_t n_threads; // Number of threads to spawn
diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c
index 194569ab..3b63ac67 100644
--- a/ggml/src/ggml-hexagon/htp/unary-ops.c
+++ b/ggml/src/ggml-hexagon/htp/unary-ops.c
@@ -1984,29 +1984,8 @@ static int execute_op_unary(struct htp_ops_context * octx) {
int op_unary(struct htp_ops_context * octx) {
switch (octx->src[0]->type) {
case HTP_TYPE_F32:
- case HTP_TYPE_F16: {
- if (!htp_unary_is_elementwise(octx->op)) {
- return execute_op_unary(octx);
- }
-
- const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params;
-
- struct htp_tensor src0_c = *octx->src[0];
- struct htp_tensor dst_c = *octx->dst;
- memcpy(src0_c.ne, kparams->ne, sizeof(src0_c.ne));
- memcpy(dst_c.ne, kparams->ne, sizeof(dst_c.ne));
- memcpy(src0_c.nb, kparams->src0_nb, sizeof(src0_c.nb));
- memcpy(dst_c.nb, kparams->dst_nb, sizeof(dst_c.nb));
-
- const struct htp_tensor * orig_src0 = octx->src[0];
- const struct htp_tensor * orig_dst = octx->dst;
- octx->src[0] = &src0_c;
- octx->dst = &dst_c;
- const int status = execute_op_unary(octx);
- octx->src[0] = orig_src0;
- octx->dst = orig_dst;
- return status;
- }
+ case HTP_TYPE_F16:
+ return execute_op_unary(octx);
default:
return HTP_STATUS_NO_SUPPORT;
diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h
index b824544f..713e40d5 100644
--- a/ggml/src/ggml-hexagon/htp/unary-ops.h
+++ b/ggml/src/ggml-hexagon/htp/unary-ops.h
@@ -24,11 +24,6 @@ struct htp_unary_kernel_params {
struct fastdiv_values div_ne02;
struct fastdiv_values div_ne012;
struct fastdiv_values div_tpr;
-
- // collapsed shape of elementwise ops
- uint32_t ne[HTP_OP_MAX_DIMS];
- uint32_t src0_nb[HTP_OP_MAX_DIMS];
- uint32_t dst_nb[HTP_OP_MAX_DIMS];
};
#if defined(__cplusplus)
@@ -72,6 +67,9 @@ static inline bool htp_op_is_unary(uint32_t opcode) {
}
static inline bool htp_unary_is_elementwise(uint32_t opcode) {
+ if (!htp_op_is_unary(opcode)) {
+ return false;
+ }
switch (opcode) {
case HTP_OP_NORM:
case HTP_OP_RMS_NORM: