Commit 20a51f24a for llama.cpp
commit 20a51f24a8523b290dcc1fc4fe9ce8fbdef9de13
Author: Todor Boinovski <tboinovski@gmail.com>
Date: Tue Nov 3 08:26:18 2026 -0800
hexagon: add DSV4_HC_PRE and DSV4_HC_POST ops (#30953)
* hexagon: add support for DSV4_HC_PRE
* hexagon: hsv4-hc-pre plain/gated split kernel and l2 prefetch optimization
* hex-dsv4-hc-pre: double-buffered DMA for prefill
* hex-dsv4-hc-pre: refactor DMA block setup
* hex-dsv4-hc-pre: precompute kernel params
* hex-dsv4-hc-pre: support for n_hc != 4
* hexagon: add support for DSV4_HC_POST
* hex-dsv4-hc-pre: refactor gated PRE prefill DMA to dst writeback after each block
* hex-dsv4-hc-pre: remove in-loop vector spills and redundant device-side checks
* hex-run: add support for test-llama-archs tool
* hex-dsv4: refactoring and optimizations
Remove non-dma paths.
Clean up DMA pipelines and dma_addr_t casts.
Add dims collapsing support.
Tested with test-llama-archs -a qwen4ext and test-backend-ops
---------
Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp
index e1d3385e2..f5c4ad7cb 100644
--- a/ggml/src/ggml-hexagon/htp-kparams.cpp
+++ b/ggml/src/ggml-hexagon/htp-kparams.cpp
@@ -53,6 +53,8 @@ htp_op_code htp_op_remap(const struct ggml_tensor * t) {
case GGML_OP_FILL: return HTP_OP_FILL;
case GGML_OP_DIAG: return HTP_OP_DIAG;
case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI;
+ case GGML_OP_DSV4_HC_PRE: return HTP_OP_DSV4_HC_PRE;
+ case GGML_OP_DSV4_HC_POST: return HTP_OP_DSV4_HC_POST;
case GGML_OP_TRI: return HTP_OP_TRI;
case GGML_OP_PAD: return HTP_OP_PAD;
case GGML_OP_IM2COL: return HTP_OP_IM2COL;
@@ -4141,6 +4143,382 @@ static bool htp_kparams_solve_conv2d_dw(
return true;
}
+static bool htp_dsv4_hc_pre_can_collapse(
+ const struct ggml_tensor * x,
+ const struct ggml_tensor * weights,
+ const struct ggml_tensor * dst
+) {
+ if (!x || !weights || !dst) {
+ return false;
+ }
+ if (x->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (x->nb[0] != sizeof(float) || weights->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) {
+ return false;
+ }
+ if (x->ne[1] != 4 || dst->ne[0] != x->ne[0]) {
+ return false;
+ }
+
+ const bool gated = ggml_get_op_params_i32(dst, 1) != 0;
+ const int64_t x_tokens = x->ne[2] * x->ne[3];
+ const int64_t d_tokens = dst->ne[1] * dst->ne[2] * dst->ne[3];
+ const int64_t w_tokens = gated ? (weights->ne[2] * weights->ne[3]) : (weights->ne[1] * weights->ne[2] * weights->ne[3]);
+
+ if (x_tokens != d_tokens || x_tokens != w_tokens) {
+ return false;
+ }
+
+ if (x->ne[3] == 1 && dst->ne[2] == 1 && dst->ne[3] == 1 &&
+ (gated ? (weights->ne[3] == 1) : (weights->ne[2] == 1 && weights->ne[3] == 1))) {
+ return false;
+ }
+
+ if (x->ne[3] > 1 && x->nb[3] != x->ne[2] * x->nb[2]) return false;
+
+ if (dst->ne[2] > 1 && dst->nb[2] != dst->ne[1] * dst->nb[1]) return false;
+ if (dst->ne[3] > 1 && dst->nb[3] != dst->ne[2] * dst->nb[2]) return false;
+
+ if (gated) {
+ if (weights->ne[3] > 1 && weights->nb[3] != weights->ne[2] * weights->nb[2]) return false;
+ } else {
+ if (weights->ne[2] > 1 && weights->nb[2] != weights->ne[1] * weights->nb[1]) return false;
+ if (weights->ne[3] > 1 && weights->nb[3] != weights->ne[2] * weights->nb[2]) return false;
+ }
+
+ return true;
+}
+
+static void htp_dsv4_hc_pre_collapse_tensors(
+ const struct ggml_tensor * x,
+ const struct ggml_tensor * weights,
+ const struct ggml_tensor * dst,
+ struct ggml_tensor * x_c,
+ struct ggml_tensor * w_c,
+ struct ggml_tensor * d_c
+) {
+ const int64_t n_tokens = x->ne[2] * x->ne[3];
+ const bool gated = ggml_get_op_params_i32(dst, 1) != 0;
+
+ *x_c = *x;
+ x_c->ne[2] = n_tokens;
+ x_c->ne[3] = 1;
+ x_c->nb[3] = x_c->ne[2] * x_c->nb[2];
+
+ *d_c = *dst;
+ d_c->ne[1] = n_tokens;
+ d_c->ne[2] = 1;
+ d_c->ne[3] = 1;
+ d_c->nb[2] = d_c->ne[1] * d_c->nb[1];
+ d_c->nb[3] = d_c->nb[2];
+
+ *w_c = *weights;
+ if (gated) {
+ w_c->ne[2] = n_tokens;
+ w_c->ne[3] = 1;
+ w_c->nb[3] = w_c->ne[2] * w_c->nb[2];
+ } else {
+ w_c->ne[1] = n_tokens;
+ w_c->ne[2] = 1;
+ w_c->ne[3] = 1;
+ w_c->nb[2] = w_c->ne[1] * w_c->nb[1];
+ w_c->nb[3] = w_c->nb[2];
+ }
+}
+
+static bool htp_kparams_solve_dsv4_hc_pre_impl(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * x,
+ const struct ggml_tensor * weights,
+ const struct ggml_tensor * dst,
+ struct htp_dsv4_hc_pre_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ if (x->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (x->ne[1] != 4 || x->ne[3] != 1) {
+ return false;
+ }
+ if (x->ne[0] % HTP_DSV4_HC_CHUNK_ELEMS != 0) {
+ return false;
+ }
+ if (dst->ne[0] != x->ne[0] || dst->ne[1] != x->ne[2] || dst->ne[2] != 1 || dst->ne[3] != 1) {
+ return false;
+ }
+ if (x->nb[0] != sizeof(float) || weights->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) {
+ return false;
+ }
+ if (ggml_is_permuted(x) || ggml_is_permuted(weights) || ggml_is_permuted(dst)) {
+ return false;
+ }
+
+ const bool gated = ggml_get_op_params_i32(dst, 1) != 0;
+ if (gated) {
+ if (weights->ne[0] != x->ne[0] || weights->ne[1] != x->ne[1] || weights->ne[2] != x->ne[2] || weights->ne[3] != 1) {
+ return false;
+ }
+ } else {
+ if (weights->ne[0] != x->ne[1] || weights->ne[1] != x->ne[2] || weights->ne[2] != 1 || weights->ne[3] != 1) {
+ return false;
+ }
+ }
+
+ const uint32_t n_tokens = (uint32_t) x->ne[2];
+ kparams->kernel_type = gated ? HTP_DSV4_HC_PRE_KERNEL_GATED : HTP_DSV4_HC_PRE_KERNEL_PLAIN;
+
+ kparams->n_chunks = ((uint32_t) x->ne[0] + HTP_DSV4_HC_CHUNK_ELEMS - 1) / HTP_DSV4_HC_CHUNK_ELEMS;
+ kparams->total_jobs = n_tokens * kparams->n_chunks;
+ if (kparams->total_jobs == 0) {
+ return true;
+ }
+ kparams->n_threads = (std::min)(htp_session_n_threads(sess), kparams->total_jobs);
+ kparams->jobs_per_thread = (kparams->total_jobs + kparams->n_threads - 1) / kparams->n_threads;
+
+ const size_t vtcm_budget = htp_session_vtcm_size(sess);
+ const uint32_t n_rows = gated ? HTP_DSV4_HC_PRE_GATED_ROWS : HTP_DSV4_HC_PRE_PLAIN_ROWS;
+ uint32_t block_chunks = (std::min)((uint32_t) HTP_DSV4_HC_MAX_CHUNKS, kparams->n_chunks);
+ while (block_chunks > 16) {
+ const uint32_t stride = block_chunks * 128;
+ const uint32_t buf_sz = n_rows * stride;
+ if (2 * buf_sz * kparams->n_threads <= vtcm_budget) {
+ break;
+ }
+ block_chunks /= 2;
+ }
+
+ kparams->dma_block_chunks = block_chunks;
+ kparams->dma_row_stride = block_chunks * 128;
+ kparams->dma_buffer_size = n_rows * kparams->dma_row_stride;
+ kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_threads;
+ if (kparams->vtcm_size > vtcm_budget) {
+ return false;
+ }
+
+ kparams->div_n_chunks = init_fastdiv_values(kparams->n_chunks);
+ kparams->div_n_threads = init_fastdiv_values(kparams->n_threads);
+
+ return true;
+}
+
+static inline bool htp_kparams_solve_dsv4_hc_pre(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_dsv4_hc_pre_kernel_params * kparams
+) {
+ if (htp_dsv4_hc_pre_can_collapse(op->src[0], op->src[1], op)) {
+ struct ggml_tensor x_c, w_c, d_c;
+ htp_dsv4_hc_pre_collapse_tensors(op->src[0], op->src[1], op, &x_c, &w_c, &d_c);
+ return htp_kparams_solve_dsv4_hc_pre_impl(sess, &x_c, &w_c, &d_c, kparams);
+ }
+ return htp_kparams_solve_dsv4_hc_pre_impl(sess, op->src[0], op->src[1], op, kparams);
+}
+
+static bool htp_dsv4_hc_post_can_collapse(
+ const struct ggml_tensor * x,
+ const struct ggml_tensor * residual,
+ const struct ggml_tensor * post,
+ const struct ggml_tensor * comb,
+ const struct ggml_tensor * dst
+) {
+ if (!x || !residual || !post || !dst) {
+ return false;
+ }
+ if (x->type != GGML_TYPE_F32 || residual->type != GGML_TYPE_F32 || post->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (comb && comb->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (x->nb[0] != sizeof(float) || residual->nb[0] != sizeof(float) || post->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) {
+ return false;
+ }
+ if (comb && comb->nb[0] != sizeof(float)) {
+ return false;
+ }
+ if (residual->ne[0] != x->ne[0] || residual->ne[1] != 4 || dst->ne[0] != x->ne[0] || dst->ne[1] != 4 || post->ne[0] != 4) {
+ return false;
+ }
+ if (comb && (comb->ne[0] != 4 || comb->ne[1] != 4)) {
+ return false;
+ }
+
+ const int64_t x_tokens = x->ne[1] * x->ne[2] * x->ne[3];
+ const int64_t r_tokens = residual->ne[2] * residual->ne[3];
+ const int64_t p_tokens = post->ne[1] * post->ne[2] * post->ne[3];
+ const int64_t d_tokens = dst->ne[2] * dst->ne[3];
+ if (x_tokens != r_tokens || x_tokens != p_tokens || x_tokens != d_tokens) {
+ return false;
+ }
+ if (comb && (comb->ne[2] * comb->ne[3] != x_tokens)) {
+ return false;
+ }
+
+ if (x->ne[2] == 1 && x->ne[3] == 1 &&
+ residual->ne[3] == 1 &&
+ post->ne[2] == 1 && post->ne[3] == 1 &&
+ dst->ne[3] == 1 &&
+ (!comb || comb->ne[3] == 1)) {
+ return false;
+ }
+
+ if (x->ne[2] > 1 && x->nb[2] != x->ne[1] * x->nb[1]) return false;
+ if (x->ne[3] > 1 && x->nb[3] != x->ne[2] * x->nb[2]) return false;
+
+ if (residual->ne[3] > 1 && residual->nb[3] != residual->ne[2] * residual->nb[2]) return false;
+ if (dst->ne[3] > 1 && dst->nb[3] != dst->ne[2] * dst->nb[2]) return false;
+
+ if (post->ne[2] > 1 && post->nb[2] != post->ne[1] * post->nb[1]) return false;
+ if (post->ne[3] > 1 && post->nb[3] != post->ne[2] * post->nb[2]) return false;
+
+ if (comb && comb->ne[3] > 1 && comb->nb[3] != comb->ne[2] * comb->nb[2]) return false;
+
+ return true;
+}
+
+static void htp_dsv4_hc_post_collapse_tensors(
+ const struct ggml_tensor * x,
+ const struct ggml_tensor * residual,
+ const struct ggml_tensor * post,
+ const struct ggml_tensor * comb,
+ const struct ggml_tensor * dst,
+ struct ggml_tensor * x_c,
+ struct ggml_tensor * r_c,
+ struct ggml_tensor * p_c,
+ struct ggml_tensor * c_c,
+ struct ggml_tensor * d_c
+) {
+ const int64_t n_tokens = x->ne[1] * x->ne[2] * x->ne[3];
+
+ *x_c = *x;
+ x_c->ne[1] = n_tokens;
+ x_c->ne[2] = 1;
+ x_c->ne[3] = 1;
+ x_c->nb[2] = x_c->ne[1] * x_c->nb[1];
+ x_c->nb[3] = x_c->nb[2];
+
+ *r_c = *residual;
+ r_c->ne[2] = n_tokens;
+ r_c->ne[3] = 1;
+ r_c->nb[3] = r_c->ne[2] * r_c->nb[2];
+
+ *d_c = *dst;
+ d_c->ne[2] = n_tokens;
+ d_c->ne[3] = 1;
+ d_c->nb[3] = d_c->ne[2] * d_c->nb[2];
+
+ *p_c = *post;
+ p_c->ne[1] = n_tokens;
+ p_c->ne[2] = 1;
+ p_c->ne[3] = 1;
+ p_c->nb[2] = p_c->ne[1] * p_c->nb[1];
+ p_c->nb[3] = p_c->nb[2];
+
+ if (comb && c_c) {
+ *c_c = *comb;
+ c_c->ne[2] = n_tokens;
+ c_c->ne[3] = 1;
+ c_c->nb[3] = c_c->ne[2] * c_c->nb[2];
+ }
+}
+
+static bool htp_kparams_solve_dsv4_hc_post_impl(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * x,
+ const struct ggml_tensor * residual,
+ const struct ggml_tensor * post,
+ const struct ggml_tensor * comb,
+ const struct ggml_tensor * dst,
+ struct htp_dsv4_hc_post_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ if (x->type != GGML_TYPE_F32 || residual->type != GGML_TYPE_F32 || post->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (comb && comb->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (x->ne[2] != 1 || x->ne[3] != 1) {
+ return false;
+ }
+ if (x->ne[0] % HTP_DSV4_HC_CHUNK_ELEMS != 0) {
+ return false;
+ }
+ if (residual->ne[0] != x->ne[0] || residual->ne[1] != 4 || residual->ne[2] != x->ne[1] || residual->ne[3] != 1) {
+ return false;
+ }
+ if (post->ne[0] != 4 || post->ne[1] != x->ne[1] || post->ne[2] != 1 || post->ne[3] != 1) {
+ return false;
+ }
+ if (comb && (comb->ne[0] != 4 || comb->ne[1] != 4 || comb->ne[2] != x->ne[1] || comb->ne[3] != 1)) {
+ return false;
+ }
+ if (dst->ne[0] != x->ne[0] || dst->ne[1] != 4 || dst->ne[2] != x->ne[1] || dst->ne[3] != 1) {
+ return false;
+ }
+ if (x->nb[0] != sizeof(float) || residual->nb[0] != sizeof(float) || post->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) {
+ return false;
+ }
+ if (comb && comb->nb[0] != sizeof(float)) {
+ return false;
+ }
+ if (ggml_is_permuted(x) || ggml_is_permuted(residual) || ggml_is_permuted(post) || ggml_is_permuted(dst) || (comb && ggml_is_permuted(comb))) {
+ return false;
+ }
+
+ const uint32_t n_tokens = (uint32_t) x->ne[1];
+
+ kparams->has_comb = comb != nullptr;
+ kparams->kernel_type = kparams->has_comb ? HTP_DSV4_HC_POST_KERNEL_COMB : HTP_DSV4_HC_POST_KERNEL_IDENTITY;
+ kparams->n_chunks = ((uint32_t) x->ne[0] + HTP_DSV4_HC_CHUNK_ELEMS - 1) / HTP_DSV4_HC_CHUNK_ELEMS;
+ kparams->total_jobs = n_tokens * kparams->n_chunks;
+ if (kparams->total_jobs == 0) {
+ return true;
+ }
+ kparams->n_threads = (std::min)(htp_session_n_threads(sess), kparams->total_jobs);
+ kparams->jobs_per_thread = (kparams->total_jobs + kparams->n_threads - 1) / kparams->n_threads;
+
+ const size_t vtcm_budget = htp_session_vtcm_size(sess);
+ uint32_t block_chunks = (std::min)((uint32_t) HTP_DSV4_HC_MAX_CHUNKS, kparams->n_chunks);
+ while (block_chunks > 16) {
+ const uint32_t stride = block_chunks * 128;
+ const uint32_t buf_sz = HTP_DSV4_HC_POST_ROWS * stride;
+ if (2 * buf_sz * kparams->n_threads <= vtcm_budget) {
+ break;
+ }
+ block_chunks /= 2;
+ }
+
+ kparams->dma_block_chunks = block_chunks;
+ kparams->dma_row_stride = block_chunks * 128;
+ kparams->dma_buffer_size = HTP_DSV4_HC_POST_ROWS * kparams->dma_row_stride;
+ kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_threads;
+ if (kparams->vtcm_size > vtcm_budget) {
+ return false;
+ }
+
+ kparams->div_n_chunks = init_fastdiv_values(kparams->n_chunks);
+ kparams->div_n_threads = init_fastdiv_values(kparams->n_threads);
+
+ return true;
+}
+
+static inline bool htp_kparams_solve_dsv4_hc_post(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_dsv4_hc_post_kernel_params * kparams
+) {
+ if (htp_dsv4_hc_post_can_collapse(op->src[0], op->src[1], op->src[2], op->src[3], op)) {
+ struct ggml_tensor x_c, r_c, p_c, c_c, d_c;
+ htp_dsv4_hc_post_collapse_tensors(op->src[0], op->src[1], op->src[2], op->src[3], op, &x_c, &r_c, &p_c, op->src[3] ? &c_c : nullptr, &d_c);
+ return htp_kparams_solve_dsv4_hc_post_impl(sess, &x_c, &r_c, &p_c, op->src[3] ? &c_c : nullptr, &d_c, kparams);
+ }
+ return htp_kparams_solve_dsv4_hc_post_impl(sess, op->src[0], op->src[1], op->src[2], op->src[3], op, kparams);
+}
+
template <auto SolveFn>
static inline bool htp_kparams_invoke(
const struct ggml_hexagon_session * sess,
@@ -4252,6 +4630,48 @@ void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode &
}
return;
}
+
+ if (node.opcode == HTP_OP_DSV4_HC_PRE) {
+ if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) {
+ return;
+ }
+ const struct ggml_tensor * x = node.inputs[0];
+ const struct ggml_tensor * weights = node.inputs[1];
+ const struct ggml_tensor * dst = node.outputs[0];
+
+ if (htp_dsv4_hc_pre_can_collapse(x, weights, dst)) {
+ struct ggml_tensor x_c, w_c, d_c;
+ htp_dsv4_hc_pre_collapse_tensors(x, weights, dst, &x_c, &w_c, &d_c);
+ node.inputs[0] = node.add_dummy(x_c);
+ node.inputs[1] = node.add_dummy(w_c);
+ node.outputs[0] = node.add_dummy(d_c);
+ }
+ return;
+ }
+
+ if (node.opcode == HTP_OP_DSV4_HC_POST) {
+ if (node.inputs.size() < 3 || !node.inputs[0] || !node.inputs[1] || !node.inputs[2] || node.outputs.empty() || !node.outputs[0]) {
+ return;
+ }
+ const struct ggml_tensor * x = node.inputs[0];
+ const struct ggml_tensor * residual = node.inputs[1];
+ const struct ggml_tensor * post = node.inputs[2];
+ const struct ggml_tensor * comb = node.inputs.size() > 3 ? node.inputs[3] : nullptr;
+ const struct ggml_tensor * dst = node.outputs[0];
+
+ if (htp_dsv4_hc_post_can_collapse(x, residual, post, comb, dst)) {
+ struct ggml_tensor x_c, r_c, p_c, c_c, d_c;
+ htp_dsv4_hc_post_collapse_tensors(x, residual, post, comb, dst, &x_c, &r_c, &p_c, comb ? &c_c : nullptr, &d_c);
+ node.inputs[0] = node.add_dummy(x_c);
+ node.inputs[1] = node.add_dummy(r_c);
+ node.inputs[2] = node.add_dummy(p_c);
+ if (comb) {
+ node.inputs[3] = node.add_dummy(c_c);
+ }
+ node.outputs[0] = node.add_dummy(d_c);
+ }
+ return;
+ }
}
bool htp_kparams_solve(
@@ -4284,6 +4704,14 @@ bool htp_kparams_solve(
if (node.opcode == HTP_OP_UPSCALE) {
return htp_kparams_solve_upscale(sess, src0, dst, node.kparams<htp_upscale_kernel_params>());
}
+ if (node.opcode == HTP_OP_DSV4_HC_PRE) {
+ return htp_kparams_solve_dsv4_hc_pre_impl(sess, src0, src1, dst, node.kparams<htp_dsv4_hc_pre_kernel_params>());
+ }
+ if (node.opcode == HTP_OP_DSV4_HC_POST) {
+ const ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr;
+ const ggml_tensor * src3 = node.inputs.size() > 3 ? node.inputs[3] : nullptr;
+ return htp_kparams_solve_dsv4_hc_post_impl(sess, src0, src1, src2, src3, dst, node.kparams<htp_dsv4_hc_post_kernel_params>());
+ }
return htp_kparams_solve(sess, node.node, node.kernel_params);
}
@@ -4391,6 +4819,12 @@ bool htp_kparams_solve(
case HTP_OP_SOLVE_TRI:
return htp_kparams_invoke<htp_kparams_solve_solve_tri>(sess, op, kparams);
+ case HTP_OP_DSV4_HC_PRE:
+ return htp_kparams_invoke<htp_kparams_solve_dsv4_hc_pre>(sess, op, kparams);
+
+ case HTP_OP_DSV4_HC_POST:
+ return htp_kparams_invoke<htp_kparams_solve_dsv4_hc_post>(sess, op, kparams);
+
default:
if (htp_op_is_unary(opcode)) {
return htp_kparams_invoke<htp_kparams_solve_unary>(sess, op, kparams);
diff --git a/ggml/src/ggml-hexagon/htp-kparams.h b/ggml/src/ggml-hexagon/htp-kparams.h
index b757f430b..9f2896eb5 100644
--- a/ggml/src/ggml-hexagon/htp-kparams.h
+++ b/ggml/src/ggml-hexagon/htp-kparams.h
@@ -11,6 +11,7 @@
#include "htp/cpy-ops.h"
#include "htp/cumsum-ops.h"
#include "htp/diag-ops.h"
+#include "htp/dsv4-hc-ops.h"
#include "htp/fill-ops.h"
#include "htp/flash-attn-ops.h"
#include "htp/gated-delta-net-ops.h"
diff --git a/ggml/src/ggml-hexagon/htp-opnode.cpp b/ggml/src/ggml-hexagon/htp-opnode.cpp
index d4e5150c0..36e7a3f38 100644
--- a/ggml/src/ggml-hexagon/htp-opnode.cpp
+++ b/ggml/src/ggml-hexagon/htp-opnode.cpp
@@ -5,6 +5,7 @@
#include "htp/binary-ops.h"
#include "htp/cumsum-ops.h"
#include "htp/diag-ops.h"
+#include "htp/dsv4-hc-ops.h"
#include "htp/fill-ops.h"
#include "htp/flash-attn-ops.h"
#include "htp/gated-delta-net-ops.h"
@@ -185,6 +186,16 @@ static std::string format_kernel_params(const htp_opnode & node) {
const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent";
snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar",
(unsigned int) kparams->vtcm_size);
+ } else if (node.opcode == HTP_OP_DSV4_HC_PRE) {
+ const auto * kparams = node.kparams<htp_dsv4_hc_pre_kernel_params>();
+ const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_PRE_KERNEL_GATED ? "gated" : "plain";
+ snprintf(str, max_size, "%s nth %u vtcm %u", ktype_str,
+ (unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
+ } else if (node.opcode == HTP_OP_DSV4_HC_POST) {
+ const auto * kparams = node.kparams<htp_dsv4_hc_post_kernel_params>();
+ const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_POST_KERNEL_COMB ? "comb" : "identity";
+ snprintf(str, max_size, "%s nth %u vtcm %u", ktype_str,
+ (unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID ||
node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {
const auto * kparams = node.kparams<htp_binary_kernel_params>();
diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt
index b5d6d55da..7c7c5be17 100644
--- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt
+++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt
@@ -49,6 +49,7 @@ add_library(${HTP_LIB} SHARED
pool-ops.c
upscale-ops.c
allreduce-ops.c
+ dsv4-hc-ops.c
)
target_compile_definitions(${HTP_LIB} PRIVATE
diff --git a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c
new file mode 100644
index 000000000..62f12c55d
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c
@@ -0,0 +1,675 @@
+#pragma clang diagnostic ignored "-Wunused-variable"
+#pragma clang diagnostic ignored "-Wunused-function"
+
+#include <HAP_farf.h>
+
+#include <stdint.h>
+#include <string.h>
+#include <math.h>
+
+#include "dma-queue.h"
+#include "hex-common.h"
+#include "hex-profile.h"
+#include "htp-ctx.h"
+#include "htp-tensor.h"
+#include "hvx-utils.h"
+#include "dsv4-hc-ops.h"
+
+// ---------------------------------------------------------------------------
+// DSV4_HC_PRE
+// ---------------------------------------------------------------------------
+
+// hvx_vec_fast_sigmoid_f32_guard_2it with fewer live constants and predicates
+static inline HVX_Vector dsv4_hc_sigmoid_f32(HVX_Vector v, HVX_Vector max_exp, HVX_Vector min_exp) {
+ v = Q6_Vsf_vmin_VsfVsf(Q6_Vsf_vmax_VsfVsf(v, min_exp), max_exp);
+ v = Q6_Vqf32_vmpy_VsfVsf(v, hvx_vec_splat_f32(0.5f * 1.442695022f));
+
+ HVX_Vector in_int = Q6_Vw_equals_Vsf(Q6_Vsf_equals_Vqf32(v));
+ HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int));
+ HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x);
+
+ HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2));
+ v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F));
+
+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1));
+ v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx);
+ v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x);
+
+ HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1));
+ v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24);
+
+ HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1));
+ HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4));
+
+ // Newton-Raphson with 2 iterations
+ HVX_Vector r = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5);
+ for (int i = 0; i < 2; ++i) {
+ const HVX_Vector rv = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r, v5));
+ r = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vadd_VsfVsf(r, r), Q6_Vqf32_vmpy_VsfVsf(r, rv)));
+ }
+
+ return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, r));
+}
+
+struct dsv4_hc_pre_context {
+ struct htp_ops_context * octx;
+ const struct htp_dsv4_hc_pre_kernel_params * kparams;
+ uint32_t token_start;
+ uint32_t n_chunks;
+ uint32_t total_jobs;
+ uint32_t jobs_per_thread;
+ float scale;
+ uint8_t * vtcm;
+ uint32_t dma_block_chunks;
+ uint32_t dma_row_stride;
+ uint32_t dma_buffer_size;
+};
+
+struct dsv4_hc_block {
+ uint32_t it;
+ uint32_t chunk;
+ uint32_t count;
+};
+
+static inline struct dsv4_hc_block dsv4_hc_pre_block_make(
+ const struct dsv4_hc_pre_context * dctx,
+ uint32_t job,
+ uint32_t end) {
+ const uint32_t local_token = fastdiv(job, &dctx->kparams->div_n_chunks);
+ const uint32_t chunk = job - local_token*dctx->n_chunks;
+ const uint32_t count = hex_smin(dctx->dma_block_chunks, hex_smin(dctx->n_chunks - chunk, end - job));
+ return (struct dsv4_hc_block) {
+ .it = dctx->token_start + local_token,
+ .chunk = chunk,
+ .count = count,
+ };
+}
+
+static inline void dsv4_hc_pre_push_plain(
+ dma_queue * dma_q,
+ const struct dsv4_hc_pre_context * dctx,
+ const struct dsv4_hc_block * block,
+ uint8_t * spad) {
+ const struct htp_tensor * x = dctx->octx->src[0];
+ const struct htp_tensor * weights = dctx->octx->src[1];
+ const uint32_t row_stride = dctx->dma_row_stride;
+ const uint32_t row_size = block->count * VLEN;
+
+ dma_queue_push(dma_q, dma_make_data(spad, x->data + (block->it*x->nb[2] + block->chunk*VLEN)), row_stride, x->nb[1], row_size, 4);
+ dma_queue_push(dma_q, dma_make_data(spad + 5*row_stride, weights->data + (block->it*weights->nb[1])), 16, 16, 16, 1);
+}
+
+static inline void dsv4_hc_pre_push_gated(
+ dma_queue * dma_q,
+ const struct dsv4_hc_pre_context * dctx,
+ const struct dsv4_hc_block * block,
+ uint8_t * spad) {
+ const struct htp_tensor * x = dctx->octx->src[0];
+ const struct htp_tensor * g = dctx->octx->src[1];
+ const uint32_t row_stride = dctx->dma_row_stride;
+ const uint32_t row_size = block->count * VLEN;
+
+ dma_queue_push(dma_q, dma_make_data(spad, x->data + (block->it*x->nb[2] + block->chunk*VLEN)), row_stride, x->nb[1], row_size, 4);
+ dma_queue_push(dma_q, dma_make_data(spad + 4*row_stride, g->data + (block->it*g->nb[2] + block->chunk*VLEN)), row_stride, g->nb[1], row_size, 4);
+}
+
+static __attribute__((noinline)) void dsv4_hc_pre_plain_block_f32(
+ const struct dsv4_hc_pre_context * dctx,
+ const struct dsv4_hc_block * block,
+ const uint8_t * x_spad,
+ uint8_t * d_spad,
+ const float * wb) {
+ const uint32_t row_stride = dctx->dma_row_stride;
+ const HVX_Vector scale_v = hvx_vec_splat_f32(dctx->scale);
+ const HVX_Vector w0 = hvx_vec_splat_f32(wb[0]);
+ const HVX_Vector w1 = hvx_vec_splat_f32(wb[1]);
+ const HVX_Vector w2 = hvx_vec_splat_f32(wb[2]);
+ const HVX_Vector w3 = hvx_vec_splat_f32(wb[3]);
+
+ for (uint32_t c = 0; c < block->count; ++c) {
+ const uint8_t * xb = x_spad + c*VLEN;
+
+ HVX_Vector x0 = *(const HVX_Vector *) xb;
+ HVX_Vector x1 = *(const HVX_Vector *) (xb + row_stride);
+ HVX_Vector x2 = *(const HVX_Vector *) (xb + 2*row_stride);
+ HVX_Vector x3 = *(const HVX_Vector *) (xb + 3*row_stride);
+
+ HVX_Vector acc = hvx_vec_mul_f32_f32(x0, w0);
+ acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(x1, w1));
+ acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(x2, w2));
+ acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(x3, w3));
+
+ *(HVX_Vector *) (d_spad + c*VLEN) = hvx_vec_mul_f32_f32(acc, scale_v);
+ }
+}
+
+static __attribute__((noinline)) void dsv4_hc_pre_gated_block_f32(
+ const struct dsv4_hc_pre_context * dctx,
+ const struct dsv4_hc_block * block,
+ const uint8_t * x_spad,
+ const uint8_t * g_spad,
+ uint8_t * d_spad) {
+ const uint32_t row_stride = dctx->dma_row_stride;
+ const HVX_Vector scale_v = hvx_vec_splat_f32(dctx->scale);
+ const HVX_Vector max_exp = hvx_vec_splat_f32(87.0f);
+ const HVX_Vector min_exp = hvx_vec_splat_f32(-87.0f);
+
+ for (uint32_t c = 0; c < block->count; ++c) {
+ const uint8_t * xb = x_spad + c*VLEN;
+ const uint8_t * gb = g_spad + c*VLEN;
+
+ HVX_Vector xv = *(const HVX_Vector *) xb;
+ HVX_Vector gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) gb, max_exp, min_exp);
+ HVX_Vector acc = hvx_vec_mul_f32_f32(xv, gv);
+
+ xv = *(const HVX_Vector *) (xb + row_stride);
+ gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) (gb + row_stride), max_exp, min_exp);
+ acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(xv, gv));
+
+ xv = *(const HVX_Vector *) (xb + 2*row_stride);
+ gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) (gb + 2*row_stride), max_exp, min_exp);
+ acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(xv, gv));
+
+ xv = *(const HVX_Vector *) (xb + 3*row_stride);
+ gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) (gb + 3*row_stride), max_exp, min_exp);
+ acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(xv, gv));
+
+ *(HVX_Vector *) (d_spad + c*VLEN) = hvx_vec_mul_f32_f32(acc, scale_v);
+ }
+}
+
+static void dsv4_hc_pre_thread_plain_f32(unsigned int nth, unsigned int ith, void * data) {
+ struct dsv4_hc_pre_context * dctx = (struct dsv4_hc_pre_context *) data;
+ struct htp_ops_context * octx = dctx->octx;
+
+ const struct htp_tensor * dst = octx->dst;
+ const uint64_t d_data = dst->data;
+
+ const uint32_t row_stride = dctx->dma_row_stride;
+ const uint32_t buffer_size = dctx->dma_buffer_size;
+ const uint32_t start = dctx->jobs_per_thread * ith;
+ const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs);
+ if (start >= end) {
+ return;
+ }
+
+ uint8_t * spad[2] = {
+ dctx->vtcm + 2*buffer_size*ith,
+ dctx->vtcm + 2*buffer_size*ith + buffer_size,
+ };
+
+ dma_queue * dma_q = octx->ctx->dma[ith];
+ uint32_t prefetch_job = start;
+ for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) {
+ const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, prefetch_job, end);
+ // empty dst writeback keeps pop order dst, x, weights in loop
+ dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 4*row_stride), block.count*VLEN, block.count*VLEN, 0, 0);
+ dsv4_hc_pre_push_plain(dma_q, dctx, &block, spad[i]);
+ prefetch_job += block.count;
+ }
+
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+
+ uint32_t job = start;
+ uint32_t curr = 0;
+ while (job < end) {
+ const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, job, end);
+
+ dma_queue_pop(dma_q);
+ dma_queue_pop(dma_q);
+ dma_queue_pop(dma_q);
+
+ const float * wb = (const float *) (spad[curr] + 5*row_stride);
+ uint8_t * d_spad = spad[curr] + 4*row_stride;
+
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it);
+ dsv4_hc_pre_plain_block_f32(dctx, &block, spad[curr], d_spad, wb);
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it);
+
+ const uint64_t d_dst = d_data + (block.it*dst->nb[1] + block.chunk*VLEN);
+ dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), block.count*VLEN, block.count*VLEN, block.count*VLEN, 1);
+
+ if (prefetch_job < end) {
+ const struct dsv4_hc_block next = dsv4_hc_pre_block_make(dctx, prefetch_job, end);
+ dsv4_hc_pre_push_plain(dma_q, dctx, &next, spad[curr]);
+ prefetch_job += next.count;
+ }
+
+ job += block.count;
+ curr ^= 1;
+ }
+
+ dma_queue_flush(dma_q);
+
+ (void) nth;
+}
+
+static void dsv4_hc_pre_thread_gated_f32(unsigned int nth, unsigned int ith, void * data) {
+ struct dsv4_hc_pre_context * dctx = (struct dsv4_hc_pre_context *) data;
+ struct htp_ops_context * octx = dctx->octx;
+
+ const struct htp_tensor * dst = octx->dst;
+ const uint64_t d_data = dst->data;
+
+ const uint32_t row_stride = dctx->dma_row_stride;
+ const uint32_t buffer_size = dctx->dma_buffer_size;
+ const uint32_t start = dctx->jobs_per_thread * ith;
+ const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs);
+ if (start >= end) {
+ return;
+ }
+
+ uint8_t * spad[2] = {
+ dctx->vtcm + 2*buffer_size*ith,
+ dctx->vtcm + 2*buffer_size*ith + buffer_size,
+ };
+
+ dma_queue * dma_q = octx->ctx->dma[ith];
+ uint32_t prefetch_job = start;
+ for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) {
+ const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, prefetch_job, end);
+ // empty dst writeback keeps pop order dst, x, g in loop
+ dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 8*row_stride), block.count*VLEN, block.count*VLEN, 0, 0);
+ dsv4_hc_pre_push_gated(dma_q, dctx, &block, spad[i]);
+ prefetch_job += block.count;
+ }
+
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+
+ uint32_t job = start;
+ uint32_t curr = 0;
+ while (job < end) {
+ const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, job, end);
+
+ dma_queue_pop(dma_q);
+ dma_queue_pop(dma_q);
+ dma_queue_pop(dma_q);
+
+ uint8_t * d_spad = spad[curr] + 8*row_stride;
+
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it);
+ dsv4_hc_pre_gated_block_f32(dctx, &block, spad[curr], spad[curr] + 4*row_stride, d_spad);
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it);
+
+ const uint64_t d_dst = d_data + (block.it*dst->nb[1] + block.chunk*VLEN);
+ dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), block.count*VLEN, block.count*VLEN, block.count*VLEN, 1);
+
+ if (prefetch_job < end) {
+ const struct dsv4_hc_block next = dsv4_hc_pre_block_make(dctx, prefetch_job, end);
+ dsv4_hc_pre_push_gated(dma_q, dctx, &next, spad[curr]);
+ prefetch_job += next.count;
+ }
+
+ job += block.count;
+ curr ^= 1;
+ }
+
+ dma_queue_flush(dma_q);
+
+ (void) nth;
+}
+
+int op_dsv4_hc_pre(struct htp_ops_context * octx) {
+ const struct htp_tensor * x = octx->src[0];
+ const struct htp_tensor * weights = octx->src[1];
+ const struct htp_tensor * dst = octx->dst;
+
+ const struct htp_dsv4_hc_pre_kernel_params * kparams =
+ (const struct htp_dsv4_hc_pre_kernel_params *) octx->kernel_params;
+
+ const uint32_t n_threads = kparams->n_threads;
+ if (!htp_ops_context_set_n_threads_div(octx, n_threads, kparams->div_n_threads)) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
+ uint32_t token_start = 0;
+ uint32_t n_tokens = x->ne[2];
+ uint32_t total_jobs = kparams->total_jobs;
+ uint32_t jobs_per_thread = kparams->jobs_per_thread;
+ if (octx->ctx->mdev.count > 1) {
+ const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0);
+ const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_tokens, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
+ token_start = range.start;
+ n_tokens = range.count;
+ total_jobs = n_tokens * kparams->n_chunks;
+ jobs_per_thread = fastdiv(total_jobs + n_threads - 1, &kparams->div_n_threads);
+ }
+
+ if (n_tokens == 0) {
+ return HTP_STATUS_OK;
+ }
+
+ float scale;
+ memcpy(&scale, octx->op_params, sizeof(scale));
+
+ struct dsv4_hc_pre_context dctx = {
+ .octx = octx,
+ .kparams = kparams,
+ .token_start = token_start,
+ .n_chunks = kparams->n_chunks,
+ .total_jobs = total_jobs,
+ .jobs_per_thread = jobs_per_thread,
+ .scale = scale,
+ .vtcm = (uint8_t *) octx->ctx->vtcm_base,
+ .dma_block_chunks = kparams->dma_block_chunks,
+ .dma_row_stride = kparams->dma_row_stride,
+ .dma_buffer_size = kparams->dma_buffer_size,
+ };
+
+ work_queue_func_t worker = NULL;
+ switch (kparams->kernel_type) {
+ case HTP_DSV4_HC_PRE_KERNEL_PLAIN: worker = dsv4_hc_pre_thread_plain_f32; break;
+ case HTP_DSV4_HC_PRE_KERNEL_GATED: worker = dsv4_hc_pre_thread_gated_f32; break;
+ default: return HTP_STATUS_INVAL_PARAMS;
+ }
+
+ FARF(HIGH, "dsv4-hc-pre: (%ux%ux%u) -> (%ux%u) : ktype %u tokens %u:%u nth %u\n",
+ x->ne[0], x->ne[1], x->ne[2], dst->ne[0], dst->ne[1], kparams->kernel_type,
+ token_start, token_start + n_tokens, n_threads);
+
+ work_queue_run(octx->ctx->work_queue, worker, &dctx, n_threads);
+ return HTP_STATUS_OK;
+}
+
+// ---------------------------------------------------------------------------
+// DSV4_HC_POST
+// ---------------------------------------------------------------------------
+
+struct dsv4_hc_post_context {
+ struct htp_ops_context * octx;
+ const struct htp_dsv4_hc_post_kernel_params * kparams;
+ uint32_t token_start;
+ uint32_t total_jobs;
+ uint32_t jobs_per_thread;
+ uint8_t * vtcm;
+};
+
+static inline struct dsv4_hc_block dsv4_hc_post_block_make(
+ const struct dsv4_hc_post_context * dctx,
+ uint32_t job,
+ uint32_t end,
+ uint32_t block_chunks) {
+ const uint32_t local_token = fastdiv(job, &dctx->kparams->div_n_chunks);
+ const uint32_t chunk = job - local_token*dctx->kparams->n_chunks;
+ const uint32_t count = hex_smin(block_chunks, hex_smin(dctx->kparams->n_chunks - chunk, end - job));
+ return (struct dsv4_hc_block) {
+ .it = dctx->token_start + local_token,
+ .chunk = chunk,
+ .count = count,
+ };
+}
+
+static __attribute__((noinline)) void dsv4_hc_post_identity_block_f32(
+ const uint8_t * xb,
+ const uint8_t * rb,
+ size_t r_stride,
+ uint8_t * db,
+ size_t d_stride,
+ uint32_t count,
+ const float * p) {
+ for (uint32_t id = 0; id < 4; ++id) {
+ const HVX_Vector pv = hvx_vec_splat_f32(p[id]);
+ const uint8_t * rr = rb + id*r_stride;
+ uint8_t * dd = db + id*d_stride;
+
+ for (uint32_t i = 0; i < count; ++i) {
+ const HVX_Vector xv = *(const HVX_Vector *) (xb + i*VLEN);
+ const HVX_Vector rv = *(const HVX_Vector *) (rr + i*VLEN);
+ *(HVX_Vector *) (dd + i*VLEN) = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(xv, pv), rv);
+ }
+ }
+}
+
+static __attribute__((noinline)) void dsv4_hc_post_comb_block_f32(
+ const uint8_t * xb,
+ const uint8_t * rb,
+ size_t r_stride,
+ uint8_t * db,
+ size_t d_stride,
+ uint32_t count,
+ const float * p,
+ const float * c) {
+ for (uint32_t id = 0; id < 4; ++id) {
+ const HVX_Vector pv = hvx_vec_splat_f32(p[id]);
+ const HVX_Vector c0 = hvx_vec_splat_f32(c[id]);
+ const HVX_Vector c1 = hvx_vec_splat_f32(c[4 + id]);
+ const HVX_Vector c2 = hvx_vec_splat_f32(c[8 + id]);
+ const HVX_Vector c3 = hvx_vec_splat_f32(c[12 + id]);
+ uint8_t * dd = db + id*d_stride;
+
+ for (uint32_t i = 0; i < count; ++i) {
+ const uint8_t * r = rb + i*VLEN;
+ HVX_Vector acc = hvx_vec_mul_f32_f32(*(const HVX_Vector *) (xb + i*VLEN), pv);
+ acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) r, c0));
+ acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) (r + r_stride), c1));
+ acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) (r + 2*r_stride), c2));
+ acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) (r + 3*r_stride), c3));
+ *(HVX_Vector *) (dd + i*VLEN) = acc;
+ }
+ }
+}
+
+static inline void dsv4_hc_post_push_identity(
+ dma_queue * dma_q,
+ const struct dsv4_hc_post_context * dctx,
+ const struct dsv4_hc_block * block,
+ uint8_t * spad) {
+ const struct htp_tensor * x = dctx->octx->src[0];
+ const struct htp_tensor * residual = dctx->octx->src[1];
+ const struct htp_tensor * post = dctx->octx->src[2];
+ const uint32_t row_stride = dctx->kparams->dma_row_stride;
+ const uint32_t row_size = block->count * VLEN;
+
+ dma_queue_push(dma_q, dma_make_data(spad, residual->data + (block->it*residual->nb[2] + block->chunk*VLEN)),
+ row_stride, residual->nb[1], row_size, 4);
+ dma_queue_push(dma_q, dma_make_data(spad + 4*row_stride, x->data + (block->it*x->nb[1] + block->chunk*VLEN)),
+ row_stride, row_size, row_size, 1);
+ dma_queue_push(dma_q, dma_make_data(spad + 9*row_stride, post->data + (block->it*post->nb[1])),
+ 16, 16, 16, 1);
+}
+
+static inline void dsv4_hc_post_push_comb(
+ dma_queue * dma_q,
+ const struct dsv4_hc_post_context * dctx,
+ const struct dsv4_hc_block * block,
+ uint8_t * spad) {
+ const struct htp_tensor * comb = dctx->octx->src[3];
+ const uint32_t row_stride = dctx->kparams->dma_row_stride;
+
+ dsv4_hc_post_push_identity(dma_q, dctx, block, spad);
+ dma_queue_push(dma_q, dma_make_data(spad + 9*row_stride + 64, comb->data + (block->it*comb->nb[2])),
+ 16, comb->nb[1], 16, 4);
+}
+
+static void dsv4_hc_post_thread_identity_f32(unsigned int nth, unsigned int ith, void * data) {
+ struct dsv4_hc_post_context * dctx = (struct dsv4_hc_post_context *) data;
+ struct htp_ops_context * octx = dctx->octx;
+
+ const struct htp_tensor * dst = octx->dst;
+ const uint64_t d_data = dst->data;
+
+ const uint32_t block_chunks = dctx->kparams->dma_block_chunks;
+ const uint32_t row_stride = dctx->kparams->dma_row_stride;
+ const uint32_t buffer_size = dctx->kparams->dma_buffer_size;
+ const uint32_t start = dctx->jobs_per_thread * ith;
+ const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs);
+ if (start >= end) {
+ return;
+ }
+
+ uint8_t * spad[2] = {
+ dctx->vtcm + 2*buffer_size*ith,
+ dctx->vtcm + 2*buffer_size*ith + buffer_size,
+ };
+
+ dma_queue * dma_q = octx->ctx->dma[ith];
+ uint32_t prefetch_job = start;
+ for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) {
+ const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks);
+ // empty dst writeback keeps pop order dst, residual, x, post in loop
+ dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 5*row_stride), dst->nb[1], row_stride, 0, 0);
+ dsv4_hc_post_push_identity(dma_q, dctx, &block, spad[i]);
+ prefetch_job += block.count;
+ }
+
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+
+ uint32_t job = start;
+ uint32_t curr = 0;
+ while (job < end) {
+ const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, job, end, block_chunks);
+
+ dma_queue_pop(dma_q);
+ dma_queue_pop(dma_q);
+ dma_queue_pop(dma_q);
+ dma_queue_pop(dma_q);
+
+ const float * p = (const float *) (spad[curr] + 9*row_stride);
+ uint8_t * d_spad = spad[curr] + 5*row_stride;
+
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it);
+ dsv4_hc_post_identity_block_f32(spad[curr] + 4*row_stride, spad[curr], row_stride, d_spad, row_stride, block.count, p);
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it);
+
+ const uint64_t d_dst = d_data + (block.it*dst->nb[2] + block.chunk*VLEN);
+ dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), dst->nb[1], row_stride, block.count*VLEN, 4);
+
+ if (prefetch_job < end) {
+ const struct dsv4_hc_block next = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks);
+ dsv4_hc_post_push_identity(dma_q, dctx, &next, spad[curr]);
+ prefetch_job += next.count;
+ }
+
+ job += block.count;
+ curr ^= 1;
+ }
+
+ dma_queue_flush(dma_q);
+
+ (void) nth;
+}
+
+static void dsv4_hc_post_thread_comb_f32(unsigned int nth, unsigned int ith, void * data) {
+ struct dsv4_hc_post_context * dctx = (struct dsv4_hc_post_context *) data;
+ struct htp_ops_context * octx = dctx->octx;
+
+ const struct htp_tensor * dst = octx->dst;
+ const uint64_t d_data = dst->data;
+
+ const uint32_t block_chunks = dctx->kparams->dma_block_chunks;
+ const uint32_t row_stride = dctx->kparams->dma_row_stride;
+ const uint32_t buffer_size = dctx->kparams->dma_buffer_size;
+ const uint32_t start = dctx->jobs_per_thread * ith;
+ const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs);
+ if (start >= end) {
+ return;
+ }
+
+ uint8_t * spad[2] = {
+ dctx->vtcm + 2*buffer_size*ith,
+ dctx->vtcm + 2*buffer_size*ith + buffer_size,
+ };
+
+ dma_queue * dma_q = octx->ctx->dma[ith];
+ uint32_t prefetch_job = start;
+ for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) {
+ const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks);
+ // empty dst writeback keeps pop order dst, residual, x, post, comb in loop
+ dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 5*row_stride), dst->nb[1], row_stride, 0, 0);
+ dsv4_hc_post_push_comb(dma_q, dctx, &block, spad[i]);
+ prefetch_job += block.count;
+ }
+
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+
+ uint32_t job = start;
+ uint32_t curr = 0;
+ while (job < end) {
+ const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, job, end, block_chunks);
+
+ dma_queue_pop(dma_q);
+ dma_queue_pop(dma_q);
+ dma_queue_pop(dma_q);
+ dma_queue_pop(dma_q);
+ dma_queue_pop(dma_q);
+
+ const float * p = (const float *) (spad[curr] + 9*row_stride);
+ const float * c = (const float *) (spad[curr] + 9*row_stride + 64);
+ uint8_t * d_spad = spad[curr] + 5*row_stride;
+
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it);
+ dsv4_hc_post_comb_block_f32(spad[curr] + 4*row_stride, spad[curr], row_stride, d_spad, row_stride, block.count, p, c);
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it);
+
+ const uint64_t d_dst = d_data + (block.it*dst->nb[2] + block.chunk*VLEN);
+ dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), dst->nb[1], row_stride, block.count*VLEN, 4);
+
+ if (prefetch_job < end) {
+ const struct dsv4_hc_block next = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks);
+ dsv4_hc_post_push_comb(dma_q, dctx, &next, spad[curr]);
+ prefetch_job += next.count;
+ }
+
+ job += block.count;
+ curr ^= 1;
+ }
+
+ dma_queue_flush(dma_q);
+
+ (void) nth;
+}
+
+int op_dsv4_hc_post(struct htp_ops_context * octx) {
+ const struct htp_tensor * x = octx->src[0];
+ const struct htp_tensor * residual = octx->src[1];
+ const struct htp_tensor * dst = octx->dst;
+
+ const struct htp_dsv4_hc_post_kernel_params * kparams =
+ (const struct htp_dsv4_hc_post_kernel_params *) octx->kernel_params;
+
+ const uint32_t n_threads = kparams->n_threads;
+ if (!htp_ops_context_set_n_threads_div(octx, n_threads, kparams->div_n_threads)) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
+ uint32_t token_start = 0;
+ uint32_t n_tokens = x->ne[1];
+ uint32_t total_jobs = kparams->total_jobs;
+ uint32_t jobs_per_thread = kparams->jobs_per_thread;
+ if (octx->ctx->mdev.count > 1) {
+ const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0);
+ const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_tokens, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
+ token_start = range.start;
+ n_tokens = range.count;
+ total_jobs = n_tokens * kparams->n_chunks;
+ jobs_per_thread = fastdiv(total_jobs + n_threads - 1, &kparams->div_n_threads);
+ }
+
+ if (n_tokens == 0) {
+ return HTP_STATUS_OK;
+ }
+
+ struct dsv4_hc_post_context dctx = {
+ .octx = octx,
+ .kparams = kparams,
+ .token_start = token_start,
+ .total_jobs = total_jobs,
+ .jobs_per_thread = jobs_per_thread,
+ .vtcm = (uint8_t *) octx->ctx->vtcm_base,
+ };
+
+ work_queue_func_t worker = NULL;
+ switch (kparams->kernel_type) {
+ case HTP_DSV4_HC_POST_KERNEL_IDENTITY: worker = dsv4_hc_post_thread_identity_f32; break;
+ case HTP_DSV4_HC_POST_KERNEL_COMB: worker = dsv4_hc_post_thread_comb_f32; break;
+ default: return HTP_STATUS_INVAL_PARAMS;
+ }
+
+ FARF(HIGH, "dsv4-hc-post: (%ux%u) (%ux%ux%u) -> (%ux%ux%u) : ktype %u tokens %u:%u nth %u\n",
+ x->ne[0], x->ne[1], residual->ne[0], residual->ne[1], residual->ne[2],
+ dst->ne[0], dst->ne[1], dst->ne[2], kparams->kernel_type,
+ token_start, token_start + n_tokens, n_threads);
+
+ work_queue_run(octx->ctx->work_queue, worker, &dctx, n_threads);
+ return HTP_STATUS_OK;
+}
diff --git a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h
new file mode 100644
index 000000000..a0c01aa91
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h
@@ -0,0 +1,77 @@
+#ifndef HTP_DSV4_HC_OPS_H
+#define HTP_DSV4_HC_OPS_H
+
+#include <stdint.h>
+
+#include "hex-fastdiv.h"
+#include "htp-ops.h"
+
+#define HTP_DSV4_HC_CHUNK_ELEMS 32
+#define HTP_DSV4_HC_MAX_CHUNKS 128
+#define HTP_DSV4_HC_PRE_GATED_ROWS 9
+#define HTP_DSV4_HC_PRE_PLAIN_ROWS 6
+
+enum htp_dsv4_hc_pre_kernel_type {
+ HTP_DSV4_HC_PRE_KERNEL_PLAIN = 0,
+ HTP_DSV4_HC_PRE_KERNEL_GATED = 1,
+};
+
+struct htp_dsv4_hc_pre_kernel_params {
+ uint32_t kernel_type;
+ uint32_t n_threads;
+ uint32_t n_chunks;
+ uint32_t total_jobs;
+ uint32_t jobs_per_thread;
+
+ uint32_t dma_block_chunks;
+ uint32_t dma_row_stride;
+ uint32_t dma_buffer_size;
+ uint32_t vtcm_size;
+
+ struct fastdiv_values div_n_chunks;
+ struct fastdiv_values div_n_threads;
+};
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_dsv4_hc_pre_kernel_params) <= 128, "htp_dsv4_hc_pre_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_dsv4_hc_pre_kernel_params) <= 128, "htp_dsv4_hc_pre_kernel_params is too large for kernel_params blob");
+#endif
+
+// dma buffer holds residual rows 0..3, x at row 4, dst rows 5..8, post/comb at row 9
+#define HTP_DSV4_HC_POST_ROWS 10
+
+enum htp_dsv4_hc_post_kernel_type {
+ HTP_DSV4_HC_POST_KERNEL_IDENTITY = 0,
+ HTP_DSV4_HC_POST_KERNEL_COMB = 1,
+};
+
+struct htp_dsv4_hc_post_kernel_params {
+ uint32_t kernel_type;
+ uint32_t n_threads;
+ uint32_t n_chunks;
+ uint32_t total_jobs;
+ uint32_t jobs_per_thread;
+ uint32_t has_comb;
+
+ uint32_t dma_block_chunks;
+ uint32_t dma_row_stride;
+ uint32_t dma_buffer_size;
+ uint32_t vtcm_size;
+
+ struct fastdiv_values div_n_chunks;
+ struct fastdiv_values div_n_threads;
+};
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_dsv4_hc_post_kernel_params) <= 128, "htp_dsv4_hc_post_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_dsv4_hc_post_kernel_params) <= 128, "htp_dsv4_hc_post_kernel_params is too large for kernel_params blob");
+#endif
+
+struct htp_ops_context;
+
+int op_dsv4_hc_pre(struct htp_ops_context * octx);
+int op_dsv4_hc_post(struct htp_ops_context * octx);
+
+#endif // HTP_DSV4_HC_OPS_H
diff --git a/ggml/src/ggml-hexagon/htp/glu-ops.c b/ggml/src/ggml-hexagon/htp/glu-ops.c
index b4edddf3f..dc621bf46 100644
--- a/ggml/src/ggml-hexagon/htp/glu-ops.c
+++ b/ggml/src/ggml-hexagon/htp/glu-ops.c
@@ -186,53 +186,6 @@ static void swiglu_clamp_f32(const float * restrict src0,
static const float GELU_COEF_A = 0.044715f;
static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
-static inline HVX_Vector hvx_vec_fast_sigmoid_f32_2it(HVX_Vector v) {
- v = Q6_Vqf32_vmpy_VsfVsf(v, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F));
- v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), Q6_V_vsplat_R(FAST_SIGMOID_C3));
-
- HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v));
- HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int));
- HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x);
-
- HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2));
- v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F));
-
- HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1));
- v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx);
- v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x);
-
- HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1));
- v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24);
-
- HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1));
- HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4));
-
- // Newton-Raphson with 2 iterations
- HVX_Vector two_sf = hvx_vec_splat_f32(2.0f);
- HVX_Vector i_sf = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5);
- HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf(
- i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5)))));
- r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32(
- r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5))));
- HVX_Vector res = Q6_Vsf_equals_Vqf32(r_qf);
-
- res = Q6_Vqf32_vmpy_VsfVsf(v3, res);
-
- return Q6_Vsf_equals_Vqf32(res);
-}
-
-static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard_2it(HVX_Vector v,
- HVX_Vector one,
- HVX_Vector max_exp,
- HVX_Vector min_exp) {
- const HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(max_exp, v);
- const HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(v, min_exp);
-
- HVX_Vector out = hvx_vec_fast_sigmoid_f32_2it(v);
- out = Q6_V_vmux_QVV(pred_max, out, one);
- return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero());
-}
-
static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src0 % 128 == 0);
diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h
index 0c455141d..4e2386bfa 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ctx.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h
@@ -131,21 +131,34 @@ static inline void htp_ops_context_reset(struct htp_ops_context * octx, struct h
octx->n_threads_div = ctx->n_threads_div;
}
-static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) {
+static inline bool htp_ops_context_set_n_threads_div(
+ struct htp_ops_context * octx,
+ uint32_t n_threads,
+ struct fastdiv_values n_threads_div) {
if (n_threads == 0 || n_threads > octx->ctx->n_threads) {
return false;
}
- if (n_threads != octx->n_threads) {
- octx->n_threads = n_threads;
- octx->n_threads_div = n_threads == octx->ctx->n_threads
- ? octx->ctx->n_threads_div
- : init_fastdiv_values(n_threads);
- }
+ octx->n_threads = n_threads;
+ octx->n_threads_div = n_threads_div;
return true;
}
+static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) {
+ if (n_threads == 0 || n_threads > octx->ctx->n_threads) {
+ return false;
+ }
+ if (n_threads == octx->n_threads) {
+ return true;
+ }
+
+ const struct fastdiv_values n_threads_div = n_threads == octx->ctx->n_threads
+ ? octx->ctx->n_threads_div
+ : init_fastdiv_values(n_threads);
+ return htp_ops_context_set_n_threads_div(octx, n_threads, n_threads_div);
+}
+
static inline void htp_ops_context_set_status(struct htp_ops_context * octx, int status) {
if (status > HTP_STATUS_OK && octx->status == HTP_STATUS_OK) {
octx->status = status;
diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h
index b8df01830..1e132f877 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ops.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h
@@ -117,6 +117,8 @@ enum htp_op_code {
HTP_OP_UNARY_CEIL,
HTP_OP_UNARY_SGN,
HTP_OP_UPSCALE,
+ HTP_OP_DSV4_HC_PRE,
+ HTP_OP_DSV4_HC_POST,
HTP_OP_INVALID
};
diff --git a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h
index d4d67ff2f..3f253a0b0 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h
@@ -52,6 +52,53 @@ static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard(HVX_Vector v,
return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero());
}
+static inline HVX_Vector hvx_vec_fast_sigmoid_f32_2it(HVX_Vector v) {
+ v = Q6_Vqf32_vmpy_VsfVsf(v, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F));
+ v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), Q6_V_vsplat_R(FAST_SIGMOID_C3));
+
+ HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v));
+ HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int));
+ HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x);
+
+ HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2));
+ v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F));
+
+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1));
+ v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx);
+ v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x);
+
+ HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1));
+ v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24);
+
+ HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1));
+ HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4));
+
+ // Newton-Raphson with 2 iterations
+ HVX_Vector two_sf = hvx_vec_splat_f32(2.0f);
+ HVX_Vector i_sf = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5);
+ HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf(
+ i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5)))));
+ r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32(
+ r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5))));
+ HVX_Vector res = Q6_Vsf_equals_Vqf32(r_qf);
+
+ res = Q6_Vqf32_vmpy_VsfVsf(v3, res);
+
+ return Q6_Vsf_equals_Vqf32(res);
+}
+
+static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard_2it(HVX_Vector v,
+ HVX_Vector one,
+ HVX_Vector max_exp,
+ HVX_Vector min_exp) {
+ const HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(max_exp, v);
+ const HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(v, min_exp);
+
+ HVX_Vector out = hvx_vec_fast_sigmoid_f32_2it(v);
+ out = Q6_V_vmux_QVV(pred_max, out, one);
+ return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero());
+}
+
static inline HVX_Vector hvx_vec_tanh_f32(HVX_Vector x) {
// tanh(x) = 2 * sigmoid(2x) - 1
HVX_Vector two = hvx_vec_splat_f32(2.0f);
diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c
index 78adc9a49..4c82c684d 100644
--- a/ggml/src/ggml-hexagon/htp/main.c
+++ b/ggml/src/ggml-hexagon/htp/main.c
@@ -34,6 +34,7 @@
#include "work-queue.h"
#include "hex-profile.h"
#include "allreduce-ops.h"
+#include "dsv4-hc-ops.h"
#include "htp-fence.h"
#define HMX_QUEUE_CAPACITY 128
@@ -1040,6 +1041,12 @@ static int execute_op(struct htp_ops_context * octx) {
case HTP_OP_POOL_1D:
return op_pool_1d(octx);
+ case HTP_OP_DSV4_HC_PRE:
+ return op_dsv4_hc_pre(octx);
+
+ case HTP_OP_DSV4_HC_POST:
+ return op_dsv4_hc_post(octx);
+
case HTP_OP_CONCAT:
return op_concat(octx);
diff --git a/scripts/snapdragon/run.py b/scripts/snapdragon/run.py
index 99b43bb2b..bef6d4e50 100755
--- a/scripts/snapdragon/run.py
+++ b/scripts/snapdragon/run.py
@@ -177,11 +177,17 @@ def device_arg_from_devices(devices):
def normalize_cmd_device_args(cmd_args):
+ exec_name = os.path.basename(cmd_args[0]) if cmd_args else ""
+ if exec_name.endswith(".exe"):
+ exec_name = exec_name[:-4]
+
for i, arg in enumerate(cmd_args):
if arg == "--device" and i + 1 < len(cmd_args):
cmd_args[i + 1] = device_arg_from_devices(cmd_args[i + 1])
elif arg.startswith("--device="):
cmd_args[i] = "--device=" + device_arg_from_devices(arg.split("=", 1)[1])
+ elif exec_name in ("test-backend-ops", "test-llama-archs") and arg in ("-b", "--backend") and i + 1 < len(cmd_args):
+ cmd_args[i + 1] = device_arg_from_devices(cmd_args[i + 1])
def main():
@@ -299,6 +305,10 @@ def main():
env_vars[env_name] = str(opt_val)
# Resolve and filter devices (HTP vs OpenCL)
+ exec_name = os.path.basename(cmd_args[0]) if cmd_args else ""
+ if exec_name.endswith(".exe"):
+ exec_name = exec_name[:-4]
+
device_in_cmd = None
for i, arg in enumerate(cmd_args):
if arg == "--device" and i + 1 < len(cmd_args):
@@ -307,6 +317,9 @@ def main():
elif arg.startswith("--device="):
device_in_cmd = arg.split("=", 1)[1]
break
+ elif exec_name in ("test-backend-ops", "test-llama-archs") and arg in ("-b", "--backend") and i + 1 < len(cmd_args):
+ device_in_cmd = cmd_args[i + 1]
+ break
if args.devices is not None:
devices_val = args.devices
@@ -402,7 +415,7 @@ def main():
# Resolve executable path
executable = cmd_args[0]
- known_binaries = ["llama-cli", "llama-bench", "llama-completion", "llama-mtmd-cli", "test-backend-ops"]
+ known_binaries = ["llama-cli", "llama-bench", "llama-completion", "llama-mtmd-cli", "test-backend-ops", "test-llama-archs"]
if executable in known_binaries:
if target_type in ("android", "linux"):
resolved_exec = f"./bin/{executable}"
@@ -419,22 +432,23 @@ def main():
basename = basename[:-4]
device_val = None
- if basename == "test-backend-ops":
- for i in range(len(cmd_args)):
- if cmd_args[i] in ("-p", "--params") and i + 1 < len(cmd_args):
- val = cmd_args[i + 1]
- new_val = ""
- for j, char in enumerate(val):
- if char in ('[', ']'):
- if j > 0 and val[j - 1] == '\\':
- new_val += char
+ if basename in ("test-backend-ops", "test-llama-archs"):
+ if basename == "test-backend-ops":
+ for i in range(len(cmd_args)):
+ if cmd_args[i] in ("-p", "--params") and i + 1 < len(cmd_args):
+ val = cmd_args[i + 1]
+ new_val = ""
+ for j, char in enumerate(val):
+ if char in ('[', ']'):
+ if j > 0 and val[j - 1] == '\\':
+ new_val += char
+ else:
+ new_val += '\\' + char
else:
- new_val += '\\' + char
- else:
- new_val += char
- cmd_args[i + 1] = new_val
+ new_val += char
+ cmd_args[i + 1] = new_val
- has_b = any(arg == "-b" for arg in cmd_args)
+ has_b = any(arg in ("-b", "--backend") for arg in cmd_args)
if not has_b:
if args.devices:
device_val = device_arg_from_devices(args.devices)
@@ -462,9 +476,13 @@ def main():
or args.hex_profile is not None
or args.hex_optrace is not None
)
- if verbose_trigger and basename in ("llama-cli", "llama-completion", "llama-bench", "llama-server", "llama-mtmd-cli"):
- if "-v" not in cmd_args and "--verbose" not in cmd_args:
- cmd_args.append("-v")
+ if verbose_trigger:
+ if basename == "test-llama-archs":
+ if "-v" not in cmd_args:
+ cmd_args += ["-v", "5"]
+ elif basename in ("llama-cli", "llama-completion", "llama-bench", "llama-server", "llama-mtmd-cli"):
+ if "-v" not in cmd_args and "--verbose" not in cmd_args:
+ cmd_args.append("-v")
# Inject defaults for llama-cli, llama-completion, and llama-server if not overridden by the user
if basename in ("llama-cli", "llama-completion", "llama-server"):
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 68b25c876..ae910f294 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -10234,6 +10234,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_dsv4_hc_post(31, 17, true, true));
test_cases.emplace_back(new test_dsv4_hc_post(2560, 21, true, true));
test_cases.emplace_back(new test_dsv4_hc_post(31, 17, false, true));
+ test_cases.emplace_back(new test_dsv4_hc_post(4096, 1)); // DeepSeek-V4-Flash decode
+ test_cases.emplace_back(new test_dsv4_hc_post(2560, 1, true, true)); // Qwen4-Exp decode
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
// blocks with extra cells and scores (k-pool)
@@ -12771,6 +12773,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
// delta-net conv input: conv state followed by the transposed qkv of the ubatch
test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {3, 8192, 1, 1}, 512, 0, 16));
+ // DSV4_HC_PRE: n_tokens 1 is decode, 128 and 512 are prompt batches
+ for (int64_t n_tokens : {1, 128, 512}) {
+ test_cases.emplace_back(new test_dsv4_hc_pre(4096, 4, n_tokens)); // DeepSeek-V4-Flash
+ test_cases.emplace_back(new test_dsv4_hc_pre(4096, 4, n_tokens, true)); // synthetic, no gated model uses this width
+ test_cases.emplace_back(new test_dsv4_hc_pre(2560, 4, n_tokens, true)); // Qwen4-Exp (Qwen3.8-Flash-Next)
+ }
+
+ // DSV4_HC_POST: n_tokens 1 is decode, 128 and 512 are prompt batches
+ for (int64_t n_tokens : {1, 128, 512}) {
+ test_cases.emplace_back(new test_dsv4_hc_post(4096, n_tokens)); // DeepSeek-V4-Flash
+ test_cases.emplace_back(new test_dsv4_hc_post(7168, n_tokens)); // DeepSeek-V4-Pro
+ test_cases.emplace_back(new test_dsv4_hc_post(2560, n_tokens, true, true)); // Qwen4-Exp, with the post gate chain
+ test_cases.emplace_back(new test_dsv4_hc_post(2560, n_tokens, true)); // synthetic, Qwen4-Exp kernel without the gate chain
+ }
+
// SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands
// note: same bytes either way, so a backend that indexes them differently shows it here
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) {