Commit e9a2a44fe for llama.cpp
commit e9a2a44fe92e4c171fa5e2007f542a2ed7d095b6
Author: Max Krasnyansky <maxk@qti.qualcomm.com>
Date: Sat Nov 7 15:03:31 2026 -0800
hexagon: n_hvx and n_threads cleanup (#31857)
* hex-nhvx: apply nhvx updates to the latest master
* hex-nhvx: set N_HVX and N_THREADS based on the arch to avoid overallocation
* hex-build: add support for enabling HTP_DEBUG via build.py
* hex-trace: move tracing state/helpers into new htp-trace.h header
* hex-dcache: cleanup dcache #defines
* hex-hnvx: minimize the diff by reflowing the macros
* hex-ci: fix python type/linter warnings
* hex-nhvx: consitent logging
diff --git a/docs/backend/snapdragon/developer.md b/docs/backend/snapdragon/developer.md
index 363be41ba..95b6127a4 100644
--- a/docs/backend/snapdragon/developer.md
+++ b/docs/backend/snapdragon/developer.md
@@ -255,7 +255,7 @@ Writing high-performance operators for Hexagon requires following specific guide
### Tracing Instrumentation
- All kernels must include trace events for performance profiling and timeline visualization in Perfetto
- ([`hex-profile.h`](../../../ggml/src/ggml-hexagon/htp/hex-profile.h)).
+ ([`htp-trace.h`](../../../ggml/src/ggml-hexagon/htp/htp-trace.h)).
- Surround compute sections with `htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) info)` and
`htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) info)`.
- Use specific event types for major phases:
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index 9b2600c66..185aa4d46 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -242,10 +242,11 @@ static void ggml_hexagon_dump_batch_prof(const char * sess_name, const htp_opbat
char evt_str[256] = "----";
if (opt_profile == 3) {
- snprintf(evt_str, sizeof(evt_str), "evt-cnt %u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u",
+ static_assert(HTP_OP_MAX_NTHREADS == 10, "current implementation assumes 10 trace counters");
+ snprintf(evt_str, sizeof(evt_str), "evt-cnt %u,%u,%u,%u,%u,%u,%u,%u,%u,%u",
rsp.n_traces[0], rsp.n_traces[1], rsp.n_traces[2], rsp.n_traces[3],
rsp.n_traces[4], rsp.n_traces[5], rsp.n_traces[6], rsp.n_traces[7],
- rsp.n_traces[8], rsp.n_traces[9], rsp.n_traces[10]);
+ rsp.n_traces[8], rsp.n_traces[9]);
}
GGML_LOG_DEBUG("ggml-hex: %s profile-op OPBATCH|----|n-ops %u|%s|----|----|usec %u cycles %llu start %llu mhz %.1f\n",
@@ -253,16 +254,11 @@ static void ggml_hexagon_dump_batch_prof(const char * sess_name, const htp_opbat
}
static void ggml_hexagon_dump_trace_events(const char * sess_name, const htp_opbatch_rsp & rsp,
- const htp_trace_desc * trace_events, uint32_t n_traces) {
+ const htp_trace_desc * trace_events, uint32_t n_threads, uint32_t n_traces) {
if (opt_profile == 3 && trace_events) {
- uint32_t valid_cnt[HTP_MAX_NTHREADS + 1] = {0};
- for (uint32_t t = 0; t <= HTP_MAX_NTHREADS; t++) {
- uint32_t count = rsp.n_traces[t];
- valid_cnt[t] = count > n_traces ? n_traces : count;
- }
-
- for (uint32_t t = 0; t <= HTP_MAX_NTHREADS; t++) {
- for (uint32_t idx = 0; idx < valid_cnt[t]; idx++) {
+ for (uint32_t t = 0; t < n_threads; t++) {
+ uint32_t count = std::min(rsp.n_traces[t], n_traces);
+ for (uint32_t idx = 0; idx < count; idx++) {
const auto & e = trace_events[t * n_traces + idx];
bool is_stop = (e.info & 0x8000) != 0;
uint16_t info = e.info & 0x7FFF;
@@ -393,7 +389,7 @@ struct ggml_hexagon_session {
const char * htp_session_name(const struct ggml_hexagon_session * sess) { return sess->c_name(); }
size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess) { return sess->vtcm_size; }
-uint32_t htp_session_n_threads(const struct ggml_hexagon_session * sess) { return sess->n_threads; }
+uint32_t htp_session_n_hvx(const struct ggml_hexagon_session * sess) { return sess->n_hvx; }
uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess) { return sess->n_hmx; }
uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess) { return sess->mdev.count; }
@@ -1547,7 +1543,7 @@ struct ggml_hexagon_opqueue {
size_t tr_size = 0;
if (opt_profile == 3) {
- tr_size = (HTP_MAX_NTHREADS + 1) * opt_optrace * sizeof(htp_trace_desc);
+ tr_size = sess->n_threads * opt_optrace * sizeof(htp_trace_desc);
}
shm_blk_size = sizeof(htp_buf_desc) * n_bufs +
@@ -1602,7 +1598,7 @@ struct ggml_hexagon_opqueue {
size_t tr_size = 0;
if (opt_profile == 3) {
req.n_traces = opt_optrace;
- tr_size = (HTP_MAX_NTHREADS + 1) * req.n_traces * sizeof(htp_trace_desc);
+ tr_size = shm_buf->sess->n_threads * req.n_traces * sizeof(htp_trace_desc);
} else {
req.n_traces = 0;
}
@@ -1657,7 +1653,7 @@ struct ggml_hexagon_opqueue {
uint32_t n_traces = 0;
if (opt_profile == 3) {
n_traces = opt_optrace;
- tr_size = (HTP_MAX_NTHREADS + 1) * n_traces * sizeof(htp_trace_desc);
+ tr_size = shm_buf->sess->n_threads * n_traces * sizeof(htp_trace_desc);
}
const size_t m_size = b_size + t_size + o_size + p_size + tr_size;
@@ -1686,7 +1682,7 @@ struct ggml_hexagon_opqueue {
ggml_hexagon_dump_op_prof(shm_buf->sess->c_name(), ops[i], pd[i]);
}
- ggml_hexagon_dump_trace_events(shm_buf->sess->c_name(), rsp, trace_events, n_traces);
+ ggml_hexagon_dump_trace_events(shm_buf->sess->c_name(), rsp, trace_events, shm_buf->sess->n_threads, n_traces);
}
}
};
@@ -2162,11 +2158,11 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
unsigned long long hw_vtcm_size = 0;
int hw_err = htp_iface_hwinfo(this->handle, &hw_n_threads, &hw_n_hvx, &hw_n_hmx, &hw_vtcm_size);
if (hw_err == 0) {
- const uint32_t max_n_threads = (std::min)((uint32_t) HTP_MAX_NTHREADS, (uint32_t) hw_n_threads);
- this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_threads) : max_n_threads;
- this->n_hvx = this->n_threads;
- this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0;
- this->vtcm_size = (uint64_t)hw_vtcm_size;
+ this->n_threads = (std::min)((uint32_t) HTP_OP_MAX_NTHREADS, (uint32_t) hw_n_threads);
+ const uint32_t max_n_hvx = (std::min)({this->n_threads - 1, (uint32_t) hw_n_hvx, (uint32_t) HTP_OP_MAX_NHVX});
+ this->n_hvx = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_hvx) : max_n_hvx;
+ this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0;
+ this->vtcm_size = (uint64_t)hw_vtcm_size;
if (opt_vtcm_size > 0) {
if (opt_vtcm_size > hw_vtcm_size) {
GGML_LOG_WARN("ggml-hex: %s VTCM override %zu exceeds HW size %llu\n", this->c_name(), opt_vtcm_size, hw_vtcm_size);
@@ -2178,11 +2174,11 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
(unsigned long long)(this->vtcm_size / (1024 * 1024)));
} else {
GGML_LOG_WARN("ggml-hex: %s failed to query hwinfo (0x%x), using defaults\n", this->c_name(), hw_err);
- const uint32_t default_n_threads = (std::min)(8u, (uint32_t) HTP_MAX_NTHREADS);
- this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads;
- this->n_hvx = this->n_threads;
- this->n_hmx = (opt_nhmx != 0) ? 1 : 0;
- this->vtcm_size = opt_vtcm_size > 0 ? opt_vtcm_size : 8 * 1024 * 1024;
+ this->n_threads = (std::min)(8u, (uint32_t) HTP_OP_MAX_NTHREADS);
+ const uint32_t max_n_hvx = (std::min)(this->n_threads - 1, (uint32_t) HTP_OP_MAX_NHVX);
+ this->n_hvx = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_hvx) : (std::min)(4u, max_n_hvx);
+ this->n_hmx = (opt_nhmx != 0) ? 1 : 0;
+ this->vtcm_size = opt_vtcm_size > 0 ? opt_vtcm_size : 8 * 1024 * 1024;
}
}
@@ -2251,7 +2247,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem);
// Start dspqueue/opbatch processing
- err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, opt_vtcm_size, this->max_vmem,
+ err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_hvx, opt_nhmx, opt_vtcm_size, this->max_vmem,
(uint32_t) opt_power_mode);
if (err != 0) {
GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err);
diff --git a/ggml/src/ggml-hexagon/htp-common.h b/ggml/src/ggml-hexagon/htp-common.h
index 815a021b8..4e9e54415 100644
--- a/ggml/src/ggml-hexagon/htp-common.h
+++ b/ggml/src/ggml-hexagon/htp-common.h
@@ -97,7 +97,7 @@ struct ggml_hexagon_session;
const char * htp_session_name(const struct ggml_hexagon_session * sess);
size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess);
-uint32_t htp_session_n_threads(const struct ggml_hexagon_session * sess);
+uint32_t htp_session_n_hvx(const struct ggml_hexagon_session * sess);
uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess);
uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess);
diff --git a/ggml/src/ggml-hexagon/htp-fusion.cpp b/ggml/src/ggml-hexagon/htp-fusion.cpp
index 7a3bdd46b..eb0862669 100644
--- a/ggml/src/ggml-hexagon/htp-fusion.cpp
+++ b/ggml/src/ggml-hexagon/htp-fusion.cpp
@@ -239,11 +239,11 @@ static bool fuse_mul_mat_add(const ggml_hexagon_session * sess, htp_opnode & pre
if (kparams.n_hmx > 0 && orig_kparams->n_hmx > 0) {
if (kparams.m_chunk < orig_kparams->m_chunk ||
kparams.n_chunk < orig_kparams->n_chunk ||
- kparams.n_act_threads < orig_kparams->n_act_threads) {
- HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, th %d->%d)\n",
+ kparams.n_act_hvx < orig_kparams->n_act_hvx) {
+ HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, hvx %d->%d)\n",
htp_session_name(sess), orig_kparams->m_chunk, kparams.m_chunk,
orig_kparams->n_chunk, kparams.n_chunk,
- orig_kparams->n_act_threads, kparams.n_act_threads);
+ orig_kparams->n_act_hvx, kparams.n_act_hvx);
return false;
}
}
@@ -546,8 +546,8 @@ static bool fuse_binary_nx(const ggml_hexagon_session * sess, htp_opnode & prev,
const uint32_t total_elems = (uint32_t) ggml_nelements(src0);
const uint32_t elem_size = (uint32_t) ggml_type_size(src0->type);
- const uint32_t max_threads = htp_session_n_threads(sess);
- if (!htp_binary_solve_chunked(&L, &kparams, total_elems, elem_size, max_threads, n_srcs, false, vtcm_size)) {
+ const uint32_t max_hvx = htp_session_n_hvx(sess);
+ if (!htp_binary_solve_chunked(&L, &kparams, total_elems, elem_size, max_hvx, n_srcs, false, vtcm_size)) {
return false;
}
}
diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp
index 49b673e85..76923d26b 100644
--- a/ggml/src/ggml-hexagon/htp-kparams.cpp
+++ b/ggml/src/ggml-hexagon/htp-kparams.cpp
@@ -154,7 +154,7 @@ bool htp_kparams_solve_allreduce(
kparams->n_ranks = (int32_t) n_ranks;
kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0;
- const uint32_t max_threads = htp_session_n_threads(sess);
+ const uint32_t max_hvx = htp_session_n_hvx(sess);
const size_t vtcm_size = htp_session_vtcm_size(sess);
const uint32_t nelem = (uint32_t) ggml_nelements(dst);
@@ -208,27 +208,27 @@ bool htp_kparams_solve_allreduce(
if (use_1d) {
const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem;
- const uint32_t n_threads = (std::min)(max_threads, (std::max)(1u, rank_nelem / 128));
- kparams->n_threads = n_threads;
- const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast);
+ const uint32_t n_hvx = (std::min)(max_hvx, (std::max)(1u, rank_nelem / 128));
+ kparams->n_hvx = n_hvx;
+ const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_hvx, has_add, is_row_bcast);
uint32_t block_elems = 65536;
- if (block_elems > rank_nelem / n_threads && rank_nelem / n_threads > 128) {
- block_elems = hex_round_up(rank_nelem / (n_threads * 2), 128);
+ if (block_elems > rank_nelem / n_hvx && rank_nelem / n_hvx > 128) {
+ block_elems = hex_round_up(rank_nelem / (n_hvx * 2), 128);
}
block_elems = (std::max)(128u, block_elems);
kparams->block_elems = block_elems;
- kparams->vtcm_size_per_thread = 2 * block_elems * elem_size;
- kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;
+ kparams->vtcm_size_per_hvx = 2 * block_elems * elem_size;
+ kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx;
while ((size_t) kparams->vtcm_size > vtcm_size && block_elems > 128) {
const size_t max_bytes_per_buf = vtcm_size / (n_vtcm_buffers * 2);
block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128);
if (block_elems < 128) break;
kparams->block_elems = block_elems;
- kparams->vtcm_size_per_thread = 2 * block_elems * elem_size;
- kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;
+ kparams->vtcm_size_per_hvx = 2 * block_elems * elem_size;
+ kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx;
}
if (vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) {
@@ -237,33 +237,33 @@ bool htp_kparams_solve_allreduce(
return false;
}
- kparams->elems_per_thread = hex_round_up((rank_nelem + n_threads - 1) / n_threads, block_elems);
+ kparams->elems_per_hvx = hex_round_up((rank_nelem + n_hvx - 1) / n_hvx, block_elems);
kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_1D;
return true;
} else {
const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem;
- const uint32_t n_threads = (std::min)(max_threads, (std::max)(1u, rank_nrows));
- kparams->n_threads = n_threads;
- const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast);
+ const uint32_t n_hvx = (std::min)(max_hvx, (std::max)(1u, rank_nrows));
+ kparams->n_hvx = n_hvx;
+ const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_hvx, has_add, is_row_bcast);
const uint32_t row_bytes = ne0 * elem_size;
const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128);
kparams->row_size_aligned = row_size_aligned;
- const uint32_t nrows_per_thread = (rank_nrows + n_threads - 1) / n_threads;
- uint32_t block_rows = (std::min)(128u, nrows_per_thread);
+ const uint32_t nrows_per_hvx = (rank_nrows + n_hvx - 1) / n_hvx;
+ uint32_t block_rows = (std::min)(128u, nrows_per_hvx);
block_rows = (std::max)(1u, block_rows);
kparams->block_elems = block_rows;
- kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned);
- kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;
+ kparams->vtcm_size_per_hvx = 2 * (block_rows * row_size_aligned);
+ kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx;
while ((size_t) kparams->vtcm_size > vtcm_size && block_rows > 1) {
const size_t max_rows_per_buf = vtcm_size / (n_vtcm_buffers * 2 * row_size_aligned);
block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf);
kparams->block_elems = block_rows;
- kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned);
- kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;
+ kparams->vtcm_size_per_hvx = 2 * (block_rows * row_size_aligned);
+ kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx;
if (max_rows_per_buf == 0) break;
}
@@ -273,7 +273,7 @@ bool htp_kparams_solve_allreduce(
return false;
}
- kparams->elems_per_thread = nrows_per_thread;
+ kparams->elems_per_hvx = nrows_per_hvx;
kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_2D;
return true;
}
@@ -313,12 +313,12 @@ static bool htp_flash_attn_is_hmx_eligible(
return true;
}
- const uint32_t n_threads = htp_session_n_threads(sess);
+ const uint32_t n_hvx = htp_session_n_hvx(sess);
const uint32_t n_head = q->ne[2];
const uint32_t n_kv_heads = k->ne[2];
const uint32_t G = n_kv_heads > 0 ? n_head / n_kv_heads : 1;
const uint32_t S = k->ne[1];
- const uint32_t total_threads = n_threads * htp_session_mdev_count(sess);
+ const uint32_t total_threads = n_hvx * htp_session_mdev_count(sess);
if (DK > 256 || (DK == 256 && (DV > 256 || n_kv_heads < total_threads))) {
return true;
@@ -326,7 +326,7 @@ static bool htp_flash_attn_is_hmx_eligible(
const bool is_tile_aligned = (G > 0 && (32 % G == 0));
if (!is_tile_aligned) {
- if (n_threads >= 6) {
+ if (n_hvx >= 6) {
return false;
}
return S >= 1024;
@@ -336,9 +336,9 @@ static bool htp_flash_attn_is_hmx_eligible(
if (DK == 256) {
s_cross = 4096;
} else if (DK <= 64) {
- s_cross = (n_threads >= 8) ? 2048 : ((n_threads >= 6) ? 768 : 512);
+ s_cross = (n_hvx >= 8) ? 2048 : ((n_hvx >= 6) ? 768 : 512);
} else {
- s_cross = (n_threads >= 8) ? 1024 : 512;
+ s_cross = (n_hvx >= 8) ? 1024 : 512;
}
return S >= s_cross;
@@ -382,7 +382,7 @@ static bool htp_kparams_solve_flash_attn(
return false;
}
- const uint32_t n_threads = htp_session_n_threads(sess);
+ const uint32_t n_hvx = htp_session_n_hvx(sess);
const size_t vtcm_size = htp_session_vtcm_size(sess);
const uint32_t neq0 = q->ne[0];
@@ -434,7 +434,7 @@ static bool htp_kparams_solve_flash_attn(
const uint32_t DK_pad = hex_round_up(DK, 64);
const uint32_t DV_pad = hex_round_up(DV, 64);
size_t Br = 0, Bc = 0;
- int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, vtcm_size, n_threads, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head, fa_team_size);
+ int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, vtcm_size, n_hvx, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head, fa_team_size);
if (ret == 0) {
kparams->kernel_type = HTP_FA_KERNEL_HMX;
if (logit_softcap == 0.0f) {
@@ -447,10 +447,10 @@ static bool htp_kparams_solve_flash_attn(
kparams->Br = Br;
kparams->Bc = Bc;
kparams->n_kv_blocks = (nek1 + Bc - 1) / Bc;
- kparams->n_threads = (kparams->n_kv_blocks >= 3 && n_threads >= 2) ? n_threads : 1;
+ kparams->n_hvx = (kparams->n_kv_blocks >= 3 && n_hvx >= 2) ? n_hvx : 1;
kparams->u.hmx.g_br = hex_align_up(G * Br, 32);
- kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && n_threads >= 2) ? 1 : 0;
- kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK_pad, DV_pad, Br, Bc, kparams->n_threads, kparams->u.hmx.pipeline != 0, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head);
+ kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && n_hvx >= 2) ? 1 : 0;
+ kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK_pad, DV_pad, Br, Bc, kparams->n_hvx, kparams->u.hmx.pipeline != 0, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head);
kparams->u.hmx.mask_broadcast = (mask != nullptr && mask->ne[2] == 1) ? 1 : 0;
kparams->u.hmx.div_G = init_fastdiv_values(G);
if (mask) {
@@ -465,13 +465,13 @@ static bool htp_kparams_solve_flash_attn(
kparams->Br = 1;
kparams->Bc = 64;
kparams->n_kv_blocks = (k->ne[1] + 64 - 1) / 64;
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
const size_t size_q_row_padded = hex_round_up(q->ne[0] * ((q->type == GGML_TYPE_F32) ? 4 : 2), 128);
const size_t size_k_row_padded = hex_round_up(k->ne[0] * 2, 128);
const size_t size_v_row_padded = hex_round_up(v->ne[0] * 2, 128);
- kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, (q->type == GGML_TYPE_F32), mask != nullptr, sinks != nullptr, n_head, n_threads);
+ kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, (q->type == GGML_TYPE_F32), mask != nullptr, sinks != nullptr, n_head, n_hvx);
if ((size_t) kparams->vtcm_size > vtcm_size) {
HEX_VERBOSE("ggml-hex: skip flash_attn_ext because VTCM needed (%d) > budget (%zu)\n",
kparams->vtcm_size, vtcm_size);
@@ -554,7 +554,7 @@ static bool htp_kparams_solve_gated_delta_net(
const uint32_t rq3 = (uint32_t) (n_seqs / q->ne[3]);
const uint32_t rk3 = (uint32_t) (n_seqs / k->ne[3]);
const uint32_t total_rows = (uint32_t) (H * n_seqs);
- const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), total_rows);
+ const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), total_rows);
const size_t vtcm_size = htp_session_vtcm_size(sess);
const bool can_use_hmx = (opt_gdn_select >= 2) &&
@@ -568,7 +568,7 @@ static bool htp_kparams_solve_gated_delta_net(
struct htp_gdn_vtcm_layout hvx_layout;
uint32_t n_heads_batch = 1;
- if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout, (uint32_t) S_v, HTP_GDN_CHUNK_SIZE, total_rows, vtcm_size, n_threads, true, &n_heads_batch)) {
+ if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout, (uint32_t) S_v, HTP_GDN_CHUNK_SIZE, total_rows, vtcm_size, n_hvx, true, &n_heads_batch)) {
kparams->kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED;
kparams->pipeline = hmx_layout.pipeline ? 1 : 0;
kparams->chunk_size = HTP_GDN_CHUNK_SIZE;
@@ -576,7 +576,7 @@ static bool htp_kparams_solve_gated_delta_net(
kparams->n_heads_batch = (uint16_t) n_heads_batch;
kparams->vtcm_size = (uint32_t) hmx_layout.total_bytes;
} else {
- htp_gdn_vtcm_layout_build(&hvx_layout, (uint32_t) S_v, n_threads);
+ htp_gdn_vtcm_layout_build(&hvx_layout, (uint32_t) S_v, n_hvx);
if (hvx_layout.total_bytes > vtcm_size) {
return false;
}
@@ -586,7 +586,7 @@ static bool htp_kparams_solve_gated_delta_net(
kparams->vtcm_size = (uint32_t) hvx_layout.total_bytes;
}
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->S_v = (uint32_t) S_v;
kparams->H = (uint32_t) H;
kparams->n_tokens = (uint32_t) n_tokens;
@@ -677,7 +677,7 @@ static bool htp_kparams_solve_hmx_mm(
) {
const bool is_tiled = ggml_hexagon_tensor_will_tile(src0);
const int aligned_tile_size = is_tiled ? htp_tensor_get_tile_padded_size(wtype) : 0;
- const int n_threads = (int) htp_session_n_threads(sess);
+ const int n_hvx = (int) htp_session_n_hvx(sess);
const uint32_t mdev_count = htp_session_mdev_count(sess);
const int ne10 = act->ne[0];
@@ -699,10 +699,10 @@ static bool htp_kparams_solve_hmx_mm(
size_t n_chunk = 0;
size_t vtcm_size = 0;
bool use_grouped = false;
- int act_threads_selected = 0;
+ int act_hvx_selected = 0;
if (is_batched_val && wtype == GGML_TYPE_F16 && !is_tiled && group_size > 1) {
- if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, m_for_solver, group_size, n_threads, pipeline, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) {
+ if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, m_for_solver, group_size, n_hvx, pipeline, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_hvx_selected, &vtcm_size)) {
use_grouped = true;
}
}
@@ -719,7 +719,7 @@ static bool htp_kparams_solve_hmx_mm(
m_id_rows = hex_round_up(std::max(m_per_expert, 32), 32);
}
const uint32_t cost_m = is_matmul_id ? (uint32_t) m_id_rows : (uint32_t) m_for_solver;
- if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, m_for_solver_padded, cost_m, n_threads, pipeline, is_matmul_id, aligned_tile_size, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) {
+ if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, m_for_solver_padded, cost_m, n_hvx, pipeline, is_matmul_id, aligned_tile_size, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_hvx_selected, &vtcm_size)) {
return false;
}
}
@@ -728,13 +728,13 @@ static bool htp_kparams_solve_hmx_mm(
kparams->pipeline = pipeline ? 1 : 0;
kparams->m_chunk = m_chunk;
kparams->n_chunk = n_chunk;
- kparams->n_threads = n_threads;
- kparams->n_act_threads = act_threads_selected;
+ kparams->n_hvx = n_hvx;
+ kparams->n_act_hvx = act_hvx_selected;
kparams->tile_size = is_tiled ? htp_tensor_get_tile_size(wtype) : 0;
kparams->aligned_tile_size = aligned_tile_size;
kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
kparams->vtcm_size = vtcm_size;
- kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected);
+ kparams->div_n_act_hvx = init_fastdiv_values(act_hvx_selected);
kparams->div_ne00_padded = init_fastdiv_values(ne00_padded);
kparams->vtcm_bias_size = (int32_t) src2_size;
kparams->kernel_type = (is_batched && !is_matmul_id) ? HTP_MM_KERNEL_HMX_BATCHED : HTP_MM_KERNEL_HMX_2D;
@@ -764,9 +764,9 @@ static void htp_kparams_solve_hvx_mm(
return;
}
- const uint32_t n_threads = htp_session_n_threads(sess);
+ const uint32_t n_hvx = htp_session_n_hvx(sess);
kparams->n_hmx = 0;
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
GGML_UNUSED(ne02);
GGML_UNUSED(ne03);
@@ -780,7 +780,7 @@ static void htp_kparams_solve_hvx_mm(
const bool k_align = (ne10 % htp_tensor_get_tile_k(wtype) == 0);
if (is_matmul_id) {
- kparams->kernel_type = (src1_nrows < (int) n_threads)
+ kparams->kernel_type = (src1_nrows < (int) n_hvx)
? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW;
kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
@@ -789,7 +789,7 @@ static void htp_kparams_solve_hvx_mm(
uint32_t best_n_prefetch = 2;
for (uint32_t d = max_prefetch; d >= 2; d /= 2) {
htp_mm_hvx_vtcm_layout_build(
- &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_threads,
+ &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_hvx,
0, src0->nb[1], kparams->act_row_size, 0, d, true, false
);
if (L.total_bytes <= vtcm_budget) {
@@ -808,7 +808,7 @@ static void htp_kparams_solve_hvx_mm(
} else {
if (k_align && opt_mm_select >= 1) {
kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
- kparams->kernel_type = (src1_nrows < (int) n_threads)
+ kparams->kernel_type = (src1_nrows < (int) n_hvx)
? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW;
struct htp_mm_hvx_vtcm_layout L;
@@ -816,7 +816,7 @@ static void htp_kparams_solve_hvx_mm(
uint32_t best_n_prefetch = 2;
for (uint32_t d = max_prefetch; d >= 2; d /= 2) {
htp_mm_hvx_vtcm_layout_build(
- &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_threads,
+ &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_hvx,
dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, d, false, false
);
if (L.total_bytes <= vtcm_budget) {
@@ -827,7 +827,7 @@ static void htp_kparams_solve_hvx_mm(
uint32_t m_chunk = 0;
if (htp_mm_hvx_solve_vtcm_params(
- kparams->kernel_type, wtype, ne10, src1_nrows, n_threads,
+ kparams->kernel_type, wtype, ne10, src1_nrows, n_hvx,
dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, best_n_prefetch, vtcm_budget,
&L, &m_chunk)) {
kparams->n_prefetch = best_n_prefetch;
@@ -857,7 +857,7 @@ static void htp_kparams_solve_hvx_mm(
struct htp_mm_hvx_vtcm_layout L;
uint32_t m_chunk = 0;
if (htp_mm_hvx_solve_vtcm_params(
- ktype, wtype, ne10, src1_nrows, n_threads,
+ ktype, wtype, ne10, src1_nrows, n_hvx,
dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget,
&L, &m_chunk)) {
kparams->kernel_type = ktype;
@@ -1093,12 +1093,12 @@ bool htp_kparams_solve_mmnx(
int32_t n_weights,
struct htp_mm_kernel_params * kparams
) {
- const uint32_t n_threads = htp_session_n_threads(sess);
+ const uint32_t n_hvx = htp_session_n_hvx(sess);
const uint32_t n_hmx = htp_session_n_hmx(sess);
const size_t vtcm_budget = htp_session_vtcm_size(sess);
memset(kparams, 0, sizeof(*kparams));
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
const int ne00 = src0->ne[0];
const int ne01 = src0->ne[1];
@@ -1140,7 +1140,7 @@ bool htp_kparams_solve_mmnx(
for (uint32_t d = max_prefetch; d >= 2; d /= 2) {
struct htp_mm_hvx_vtcm_layout L;
htp_mm_hvx_vtcm_layout_build(
- &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_threads,
+ &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_hvx,
0, src0_row_size, act_row_size, 0, d, false, true
);
if (L.total_bytes <= vtcm_budget) {
@@ -1151,7 +1151,7 @@ bool htp_kparams_solve_mmnx(
struct htp_mm_hvx_vtcm_layout L;
htp_mm_hvx_vtcm_layout_build(
- &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_threads,
+ &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_hvx,
0, src0_row_size, act_row_size, 0, best_n_prefetch, false, true
);
@@ -1244,7 +1244,7 @@ static bool htp_kparams_solve_binary(
}
}
- const uint32_t max_threads_sess = htp_session_n_threads(sess);
+ const uint32_t max_hvx_sess = htp_session_n_hvx(sess);
const size_t vtcm_size = htp_session_vtcm_size(sess);
const size_t elem_size = ggml_type_size(src0->type);
@@ -1278,14 +1278,14 @@ static bool htp_kparams_solve_binary(
const uint32_t total_elems = (uint32_t) ggml_nelements(src0);
const uint32_t max_chunk_elems = 32768 / elem_size;
const uint32_t min_chunk_elems = 256;
- const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_threads_sess) - 1) / (2 * max_threads_sess), 32);
+ const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_hvx_sess) - 1) / (2 * max_hvx_sess), 32);
const uint32_t chunk_size = (std::min)(max_chunk_elems, (std::max)(target_chunk_elems, min_chunk_elems));
const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128);
const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size;
- const uint32_t n_threads = (total_chunks >= 2) ? (std::min)(max_threads_sess, total_chunks) : 1;
+ const uint32_t n_hvx = (total_chunks >= 2) ? (std::min)(max_hvx_sess, total_chunks) : 1;
kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED;
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->n_srcs = 2;
kparams->rows_per_buffer = 1;
kparams->src0_row_size_aligned = src0_row_size_aligned;
@@ -1346,11 +1346,11 @@ static bool htp_kparams_solve_binary(
}
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size));
- const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread);
+ const uint32_t min_rows_per_hvx = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size));
+ const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx);
kparams->kernel_type = kernel_type;
- kparams->n_threads = (std::min)(max_threads_sess, max_threads);
+ kparams->n_hvx = (std::min)(max_hvx_sess, max_hvx);
kparams->n_srcs = 2;
kparams->src0_row_size_aligned = src0_row_size_aligned;
kparams->src1_row_size_aligned = src1_row_size_aligned;
@@ -1403,7 +1403,7 @@ static bool htp_kparams_solve_binary(
if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) {
const uint32_t total_elems = (uint32_t) ggml_nelements(src0);
return htp_binary_solve_chunked(&L, kparams, total_elems, (uint32_t) elem_size,
- max_threads_sess, 2, is_scalar_broadcast, vtcm_size);
+ max_hvx_sess, 2, is_scalar_broadcast, vtcm_size);
}
return false;
}
@@ -1415,7 +1415,7 @@ static bool htp_kparams_solve_binary(
}
// collapse contiguous dims of elementwise ops so that rows fill at least one HVX vector
-static void htp_collapse_dims(struct ggml_tensor ** tensors, size_t n_tensors, size_t elem_size, uint32_t n_threads) {
+static void htp_collapse_dims(struct ggml_tensor ** tensors, size_t n_tensors, size_t elem_size, uint32_t n_hvx) {
if (n_tensors == 0 || !tensors[0]) {
return;
}
@@ -1497,9 +1497,9 @@ static void htp_collapse_dims(struct ggml_tensor ** tensors, size_t n_tensors, s
break;
}
}
- if (nrows < n_threads && rows_contig && row_size_curr >= 4096) {
+ if (nrows < n_hvx && rows_contig && row_size_curr >= 4096) {
const size_t vec_elems = 128 / elem_size;
- int64_t k = std::max<int64_t>(row_size_curr / 16384, (n_threads * 4 + nrows - 1) / nrows);
+ int64_t k = std::max<int64_t>(row_size_curr / 16384, (n_hvx * 4 + nrows - 1) / nrows);
k = std::min<int64_t>(k, row_size_curr / 2048);
while (k > 1 && t0->ne[0] % (k * vec_elems) != 0) {
k--;
@@ -1576,12 +1576,12 @@ static bool htp_kparams_solve_unary_impl(
const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type);
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_data_row_size - 1) / dst_data_row_size));
- const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread);
- const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+ const uint32_t min_rows_per_hvx = (uint32_t) (std::max<size_t>(1, (128 + dst_data_row_size - 1) / dst_data_row_size));
+ const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx);
+ const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx);
const size_t vtcm_size = htp_session_vtcm_size(sess);
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
const bool has_weight = (op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_RMS_NORM_MUL_ADD);
const bool broadcast_weight = has_weight && (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1);
@@ -1590,13 +1590,13 @@ static bool htp_kparams_solve_unary_impl(
}
kparams->broadcast_weight = broadcast_weight;
- const bool src0_contig = ggml_is_contiguous_1(src0);
- const bool src1_contig = (has_weight && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true;
- const uint32_t src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
- const uint32_t min_block_rows = (uint32_t) (std::max<size_t>(1, 2048 / dst_data_row_size));
- uint32_t max_block = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1];
- if (src0_nrows_per_thread >= 2 * min_block_rows) {
- max_block = (std::min)(max_block, (std::max)(min_block_rows, src0_nrows_per_thread / 4));
+ const bool src0_contig = ggml_is_contiguous_1(src0);
+ const bool src1_contig = (has_weight && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true;
+ const uint32_t src0_nrows_per_hvx = (src0_nrows + n_hvx - 1) / n_hvx;
+ const uint32_t min_block_rows = (uint32_t) (std::max<size_t>(1, 2048 / dst_data_row_size));
+ uint32_t max_block = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1];
+ if (src0_nrows_per_hvx >= 2 * min_block_rows) {
+ max_block = (std::min)(max_block, (std::max)(min_block_rows, src0_nrows_per_hvx / 4));
}
kparams->src0_contig = src0_contig ? 1 : 0;
@@ -1604,16 +1604,16 @@ static bool htp_kparams_solve_unary_impl(
struct htp_unary_vtcm_layout L;
uint32_t col_tile = 0;
- uint32_t vtcm_row_per_thread = 0;
+ uint32_t vtcm_row_per_hvx = 0;
htp_unary_solve_layout(&L, op, src0->ne[0], dst->ne[0],
has_weight ? src1->ne[0] : 0,
- max_block, broadcast_weight, n_threads, vtcm_size, elem_size,
- &col_tile, &vtcm_row_per_thread);
+ max_block, broadcast_weight, n_hvx, vtcm_size, elem_size,
+ &col_tile, &vtcm_row_per_hvx);
kparams->col_tile = col_tile;
kparams->vtcm_size = L.total_bytes;
- kparams->block = col_tile ? 0 : vtcm_row_per_thread;
+ kparams->block = col_tile ? 0 : vtcm_row_per_hvx;
const uint32_t tiles_per_row = col_tile > 0 ? (src0->ne[0] + col_tile - 1) / col_tile : 1;
kparams->tiles_per_row = tiles_per_row;
@@ -1622,7 +1622,7 @@ static bool htp_kparams_solve_unary_impl(
kparams->div_ne012 = init_fastdiv_values(src0->ne[1] * src0->ne[2]);
kparams->div_tpr = init_fastdiv_values(tiles_per_row);
- return (col_tile > 0 || vtcm_row_per_thread > 0) && L.total_bytes <= vtcm_size;
+ return (col_tile > 0 || vtcm_row_per_hvx > 0) && L.total_bytes <= vtcm_size;
}
static bool htp_kparams_solve_unary(
@@ -1710,22 +1710,22 @@ bool htp_kparams_solve_repeat_get_rows(
}
const uint32_t total_tasks = nr;
- const uint32_t min_tasks_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst->nb[1] - 1) / dst->nb[1]));
- const uint32_t max_threads = (std::max)(1u, total_tasks / min_tasks_per_thread);
+ const uint32_t min_tasks_per_hvx = (uint32_t) (std::max<size_t>(1, (128 + dst->nb[1] - 1) / dst->nb[1]));
+ const uint32_t max_hvx = (std::max)(1u, total_tasks / min_tasks_per_hvx);
const size_t vtcm_size = htp_session_vtcm_size(sess);
- kparams->n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+ kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx);
struct htp_get_rows_vtcm_layout vtcm_layout = {};
- while (kparams->n_threads > 0) {
- htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, kparams->n_threads);
+ while (kparams->n_hvx > 0) {
+ htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, kparams->n_hvx);
if (vtcm_layout.total_bytes <= vtcm_size) {
break;
}
- --kparams->n_threads;
+ --kparams->n_hvx;
}
- if (kparams->n_threads == 0 && total_tasks > 0) {
+ if (kparams->n_hvx == 0 && total_tasks > 0) {
htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, 1);
}
@@ -1740,7 +1740,7 @@ bool htp_kparams_solve_repeat_get_rows(
if (total_tasks == 0) {
return true;
}
- return kparams->n_threads > 0 && (size_t) kparams->vtcm_size <= vtcm_size;
+ return kparams->n_hvx > 0 && (size_t) kparams->vtcm_size <= vtcm_size;
}
static bool htp_kparams_solve_get_rows(
@@ -1777,11 +1777,11 @@ static bool htp_kparams_solve_set_rows(
const uint32_t nr = src0->ne[1];
- kparams->n_threads = (std::min)(htp_session_n_threads(sess), nr);
+ kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), nr);
kparams->total_tasks = nr;
struct htp_set_rows_vtcm_layout vtcm_layout;
- htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_threads);
+ htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_hvx);
kparams->vtcm_size = vtcm_layout.total_bytes;
return vtcm_layout.total_bytes <= htp_session_vtcm_size(sess);
@@ -1819,9 +1819,9 @@ static bool htp_kparams_solve_softmax(
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const size_t dst_data_row_size = op->ne[0] * ggml_type_size(op->type);
- const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_data_row_size - 1) / dst_data_row_size));
- const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread);
- const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+ const uint32_t min_rows_per_hvx = (uint32_t) (std::max<size_t>(1, (128 + dst_data_row_size - 1) / dst_data_row_size));
+ const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx);
+ const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx);
float scale = 1.0f;
float max_bias = 0.0f;
@@ -1851,12 +1851,12 @@ static bool htp_kparams_solve_softmax(
const uint32_t ne10 = src1 ? src1->ne[0] : 1;
struct htp_softmax_vtcm_layout layout;
- htp_softmax_vtcm_layout_build(&layout, ne00, ne10, kparams->use_src1 != 0, kparams->use_f16 != 0, n_threads);
+ htp_softmax_vtcm_layout_build(&layout, ne00, ne10, kparams->use_src1 != 0, kparams->use_f16 != 0, n_hvx);
if (layout.total_bytes > htp_session_vtcm_size(sess)) {
return false;
}
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->vtcm_size = (uint32_t) layout.total_bytes;
if (!kparams->use_src1) {
@@ -1962,16 +1962,16 @@ static bool htp_kparams_solve_rope(
return false;
}
- const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), src0_nrows);
+ const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), src0_nrows);
const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0;
struct htp_rope_vtcm_layout layout;
- htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads, n_freq_factors);
+ htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_hvx, n_freq_factors);
if (layout.total_bytes > htp_session_vtcm_size(sess)) {
return false;
}
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->vtcm_size = (uint32_t) layout.total_bytes;
kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]);
kparams->div_ne1 = init_fastdiv_values(dst->ne[1]);
@@ -2005,26 +2005,26 @@ static bool htp_kparams_solve_sum_rows(
const uint32_t ne00 = src0->ne[0];
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const uint32_t max_threads_sess = htp_session_n_threads(sess);
+ const uint32_t max_hvx_sess = htp_session_n_hvx(sess);
if (op->op == GGML_OP_SUM) {
const uint32_t total_elems = (uint32_t) ggml_nelements(src0);
- const uint32_t n_threads = (total_elems >= 1024) ? (std::min)(max_threads_sess, (uint32_t) HTP_MAX_NTHREADS) : 1;
- const uint32_t raw_chunk = (total_elems + n_threads - 1) / n_threads;
+ const uint32_t n_hvx = (total_elems >= 1024) ? max_hvx_sess : 1;
+ const uint32_t raw_chunk = (total_elems + n_hvx - 1) / n_hvx;
- kparams->n_threads = n_threads;
- kparams->elems_per_thread = hex_round_up(raw_chunk, 32);
+ kparams->n_hvx = n_hvx;
+ kparams->elems_per_hvx = hex_round_up(raw_chunk, 32);
} else if (op->op == GGML_OP_ARGMAX && src0_nrows <= 1) {
- const uint32_t n_threads = (ne00 >= 1024) ? (std::min)(max_threads_sess, (uint32_t) HTP_MAX_NTHREADS) : 1;
- const uint32_t raw_chunk = (ne00 + n_threads - 1) / n_threads;
+ const uint32_t n_hvx = (ne00 >= 1024) ? max_hvx_sess : 1;
+ const uint32_t raw_chunk = (ne00 + n_hvx - 1) / n_hvx;
- kparams->n_threads = n_threads;
- kparams->elems_per_thread = hex_round_up(raw_chunk, 32);
+ kparams->n_hvx = n_hvx;
+ kparams->elems_per_hvx = hex_round_up(raw_chunk, 32);
} else {
const uint32_t min_rpt = (src0->nb[1] < 1024) ? 32 : 1;
- const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rpt);
+ const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rpt);
- kparams->n_threads = (std::min)(max_threads_sess, max_threads);
+ kparams->n_hvx = (std::min)(max_hvx_sess, max_hvx);
}
return true;
@@ -2065,28 +2065,28 @@ bool htp_kparams_solve_unary_mul_glu(
const size_t dst_row_size = dst->ne[0] * elem_size;
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size));
- const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread);
- const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+ const uint32_t min_rows_per_hvx = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size));
+ const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx);
+ const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx);
const size_t vtcm_size = htp_session_vtcm_size(sess);
const size_t row_size_aligned = hex_round_up(dst_row_size, 128);
- uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * 3 * row_size_aligned));
- if (vtcm_row_per_thread < 2) {
+ uint32_t vtcm_row_per_hvx = (uint32_t) (vtcm_size / (n_hvx * 3 * row_size_aligned));
+ if (vtcm_row_per_hvx < 2) {
return false;
}
- const uint32_t src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
- if (src0_nrows_per_thread >= 2 && vtcm_row_per_thread > src0_nrows_per_thread) {
- uint32_t target_rpt = (src0_nrows_per_thread >= 4) ? (src0_nrows_per_thread / 2) & ~1u : 2;
- vtcm_row_per_thread = (std::max)(2u, (std::min)(vtcm_row_per_thread, target_rpt));
+ const uint32_t src0_nrows_per_hvx = (src0_nrows + n_hvx - 1) / n_hvx;
+ if (src0_nrows_per_hvx >= 2 && vtcm_row_per_hvx > src0_nrows_per_hvx) {
+ uint32_t target_rpt = (src0_nrows_per_hvx >= 4) ? (src0_nrows_per_hvx / 2) & ~1u : 2;
+ vtcm_row_per_hvx = (std::max)(2u, (std::min)(vtcm_row_per_hvx, target_rpt));
}
struct htp_glu_vtcm_layout L;
- htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, vtcm_row_per_thread);
+ htp_glu_vtcm_layout_build(&L, dst_row_size, n_hvx, vtcm_row_per_hvx);
- kparams->n_threads = n_threads;
- kparams->vtcm_row_per_thread = L.vtcm_row_per_thread;
+ kparams->n_hvx = n_hvx;
+ kparams->vtcm_row_per_hvx = L.vtcm_row_per_hvx;
kparams->vtcm_size = (uint32_t) L.total_bytes;
return true;
@@ -2119,11 +2119,11 @@ static bool htp_kparams_solve_repeat(
const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3];
const size_t dst_row_size = dst->ne[0] * ggml_type_size(dst->type);
- const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size));
- const uint32_t max_threads = (std::max)(1u, total_dst_rows / min_rows_per_thread);
- const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+ const uint32_t min_rows_per_hvx = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size));
+ const uint32_t max_hvx = (std::max)(1u, total_dst_rows / min_rows_per_hvx);
+ const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx);
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->nr0 = dst->ne[0] / src0->ne[0];
kparams->nr1 = dst->ne[1] / src0->ne[1];
kparams->nr2 = dst->ne[2] / src0->ne[2];
@@ -2186,31 +2186,31 @@ static bool htp_kparams_solve_ssm_conv(
const uint32_t n_s = (uint32_t) n_s_64;
const uint32_t ncs = (uint32_t) src0->ne[0];
- const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), (d_inner + 31) / 32);
+ const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), (d_inner + 31) / 32);
const size_t vtcm_size = htp_session_vtcm_size(sess);
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->d_conv = d_conv;
kparams->d_inner = d_inner;
kparams->n_t = n_t;
kparams->n_s = n_s;
- const uint32_t raw_rpt = (d_inner + n_threads - 1) / n_threads;
- const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, 32);
+ const uint32_t raw_rpt = (d_inner + n_hvx - 1) / n_hvx;
+ const uint32_t d_inner_per_hvx = hex_round_up(raw_rpt, 32);
if (n_t == 1) {
- kparams->d_inner_tile = d_inner_per_thread;
+ kparams->d_inner_tile = d_inner_per_hvx;
} else {
- const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128) + 128;
- const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128);
- const uint32_t vtcm_src1_per_thread = src1_raw_bytes + src1_T_bytes;
+ const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_hvx * d_conv * sizeof(float), 128) + 128;
+ const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_hvx * sizeof(float), 128);
+ const uint32_t vtcm_src1_per_hvx = src1_raw_bytes + src1_T_bytes;
- const size_t vtcm_budget = (vtcm_size > 0 ? vtcm_size / n_threads : (1024 * 1024));
+ const size_t vtcm_budget = (vtcm_size > 0 ? vtcm_size / n_hvx : (1024 * 1024));
const uint32_t src0_block_T = hex_round_up(ncs * 32 * sizeof(float), 128);
- const size_t fixed_bytes = vtcm_src1_per_thread + src0_block_T;
+ const size_t fixed_bytes = vtcm_src1_per_hvx + src0_block_T;
const size_t avail_for_tiles = vtcm_budget > fixed_bytes ? vtcm_budget - fixed_bytes : (128 * 1024);
- uint32_t target_max_tile = hex_round_up((d_inner_per_thread + 3) / 4, 32);
+ uint32_t target_max_tile = hex_round_up((d_inner_per_hvx + 3) / 4, 32);
target_max_tile = (std::max)(target_max_tile, 32u);
target_max_tile = (std::min)(target_max_tile, 128u);
@@ -2223,14 +2223,14 @@ static bool htp_kparams_solve_ssm_conv(
if (d_inner_tile > target_max_tile) {
d_inner_tile = target_max_tile;
}
- if (d_inner_tile > d_inner_per_thread) {
- d_inner_tile = d_inner_per_thread;
+ if (d_inner_tile > d_inner_per_hvx) {
+ d_inner_tile = d_inner_per_hvx;
}
kparams->d_inner_tile = d_inner_tile;
}
struct htp_ssm_conv_vtcm_layout L;
- htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_thread, d_conv, n_t, kparams->d_inner_tile, n_threads);
+ htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_hvx, d_conv, n_t, kparams->d_inner_tile, n_hvx);
kparams->vtcm_size = (uint32_t) L.total_bytes;
kparams->div_d_inner_tile = init_fastdiv_values(kparams->d_inner_tile);
@@ -2256,7 +2256,7 @@ static bool htp_kparams_solve_sort(
const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const uint32_t ne00 = src0->ne[0];
const uint32_t k = dst->ne[0];
- const uint32_t n_threads_max = htp_session_n_threads(sess) > 0 ? htp_session_n_threads(sess) : 4;
+ const uint32_t n_hvx_max = htp_session_n_hvx(sess) > 0 ? htp_session_n_hvx(sess) : 4;
const size_t vtcm_budget = htp_session_vtcm_size(sess) > 0 ? htp_session_vtcm_size(sess) : (8 * 1024 * 1024);
int32_t order = GGML_SORT_ORDER_DESC;
@@ -2265,11 +2265,11 @@ static bool htp_kparams_solve_sort(
}
struct htp_sort_vtcm_layout layout;
- if (!htp_sort_solve_layout(&layout, ne00, total_rows, k, n_threads_max, vtcm_budget, is_top_k)) {
+ if (!htp_sort_solve_layout(&layout, ne00, total_rows, k, n_hvx_max, vtcm_budget, is_top_k)) {
return false;
}
- kparams->n_threads = layout.n_threads;
+ kparams->n_hvx = layout.n_hvx;
kparams->order = (uint32_t) order;
kparams->is_top_k = is_top_k ? 1 : 0;
kparams->chunk_elems = layout.chunk_elems;
@@ -2347,11 +2347,11 @@ static bool htp_kparams_solve_pool_2d(
const uint32_t planes = (uint32_t) (is_pool_1d
? (uint64_t) src0->ne[1] * (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3]
: (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3]);
- const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), planes > 0 ? planes : 1);
+ const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), planes > 0 ? planes : 1);
const uint32_t planes_per_chunk = (dst_plane_bytes > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_plane_bytes, HEX_L2_LINE_SIZE)) : 1;
struct htp_pool_vtcm_layout layout;
- if (!htp_pool_solve_layout(&layout, src_x, src_y, dst_x, dst_y, n_threads, htp_session_vtcm_size(sess))) {
+ if (!htp_pool_solve_layout(&layout, src_x, src_y, dst_x, dst_y, n_hvx, htp_session_vtcm_size(sess))) {
return false;
}
@@ -2365,7 +2365,7 @@ static bool htp_kparams_solve_pool_2d(
kparams->stride_y = (uint32_t) stride_y;
kparams->pad_x = pad_x;
kparams->pad_y = pad_y;
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->planes = planes;
kparams->planes_per_chunk = planes_per_chunk;
kparams->pool_op = (uint32_t) params[0];
@@ -2471,18 +2471,18 @@ bool htp_kparams_solve_upscale(
const float invscale1 = antialias ? (1.0f / filter_radius1) : 0.0f;
const uint32_t total_slices = dst_z * dst_w;
- uint32_t n_threads = htp_session_n_threads(sess);
+ uint32_t n_hvx = htp_session_n_hvx(sess);
struct htp_upscale_vtcm_layout layout;
uint32_t dst_nrows_per_chunk = 0;
- while (n_threads > 0) {
- if (htp_upscale_solve_layout(&layout, src_x, src_y, dst_x, dst_y, inv_sf1, filter_radius1, n_threads,
+ while (n_hvx > 0) {
+ if (htp_upscale_solve_layout(&layout, src_x, src_y, dst_x, dst_y, inv_sf1, filter_radius1, n_hvx,
htp_session_vtcm_size(sess), &dst_nrows_per_chunk)) {
break;
}
- n_threads--;
+ n_hvx--;
}
- if (n_threads == 0) {
+ if (n_hvx == 0) {
return false;
}
@@ -2509,8 +2509,8 @@ bool htp_kparams_solve_upscale(
const uint32_t chunks_per_slice = (dst_y + dst_nrows_per_chunk - 1) / dst_nrows_per_chunk;
const uint32_t total_chunks = total_slices * chunks_per_slice;
- n_threads = (std::min)(n_threads, total_chunks > 0 ? total_chunks : 1);
- htp_upscale_vtcm_layout_build(&layout, src_x, src_y, dst_x, dst_y, dst_nrows_per_chunk, inv_sf1, filter_radius1, n_threads);
+ n_hvx = (std::min)(n_hvx, total_chunks > 0 ? total_chunks : 1);
+ htp_upscale_vtcm_layout_build(&layout, src_x, src_y, dst_x, dst_y, dst_nrows_per_chunk, inv_sf1, filter_radius1, n_hvx);
const uint32_t slice_dst_bytes = dst_x * dst_y * sizeof(float);
const uint32_t chunk_dst_bytes = dst_x * dst_nrows_per_chunk * sizeof(float);
@@ -2565,7 +2565,7 @@ bool htp_kparams_solve_upscale(
kparams->invscale0 = invscale0;
kparams->invscale1 = invscale1;
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->chunks_per_group = chunks_per_group;
kparams->dst_nrows_per_chunk = dst_nrows_per_chunk;
@@ -2609,7 +2609,7 @@ static bool htp_kparams_solve_concat(
}
const uint32_t type_size = ggml_type_size(dst->type);
- const uint32_t n_threads = htp_session_n_threads(sess);
+ const uint32_t n_hvx = htp_session_n_hvx(sess);
const uint32_t mdev_count = htp_session_mdev_count(sess);
const size_t vtcm_size = htp_session_vtcm_size(sess);
@@ -2643,10 +2643,10 @@ static bool htp_kparams_solve_concat(
(uint32_t) src0->ne[0], (uint32_t) src0->nb[1],
(uint32_t) src1->ne[0], (uint32_t) src1->nb[1], b_dense,
type_size, total_rows,
- n_threads, mdev_count, vtcm_size)) {
+ n_hvx, mdev_count, vtcm_size)) {
kparams->kernel_type = HTP_CONCAT_KERNEL_GATHER_ROWS;
- kparams->n_threads = (uint8_t) n_threads;
- kparams->vtcm_size = kparams->u.gather.slice * n_threads;
+ kparams->n_hvx = (uint8_t) n_hvx;
+ kparams->vtcm_size = kparams->u.gather.slice * n_hvx;
return true;
}
}
@@ -2654,7 +2654,7 @@ static bool htp_kparams_solve_concat(
const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size);
if (dma_strides_ok) {
kparams->kernel_type = HTP_CONCAT_KERNEL_REGULAR;
- kparams->n_threads = 1;
+ kparams->n_hvx = 1;
return true;
}
@@ -2665,7 +2665,7 @@ static bool htp_kparams_solve_concat(
if (dim == 0 && is_src1_transposed && !is_src0_transposed && transposed_rows_ok &&
(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16)) {
- const uint32_t nth = n_threads > 0 ? n_threads : 8;
+ const uint32_t nth = n_hvx > 0 ? n_hvx : 8;
struct htp_concat_transposed_vtcm_layout layout;
htp_concat_transposed_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src1->ne[0], type_size, nth);
@@ -2674,7 +2674,7 @@ static bool htp_kparams_solve_concat(
}
kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED;
- kparams->n_threads = nth;
+ kparams->n_hvx = nth;
kparams->vtcm_size = layout.total_bytes;
kparams->u.transposed.div_ne2 = init_fastdiv_values((uint32_t) dst->ne[2]);
return true;
@@ -2777,14 +2777,14 @@ static bool htp_kparams_solve_cpy_transpose(
m++;
}
- const uint32_t n_threads = htp_session_n_threads(sess);
+ const uint32_t n_hvx = htp_session_n_hvx(sess);
const size_t vtcm_size = htp_session_vtcm_size(sess);
uint64_t vtcm = 0;
for (uint32_t blk_bytes = HTP_COPY_TRANSPOSE_BLK_BYTES; ; blk_bytes /= 2) {
if (!htp_copy_transpose_layout_build(&tp, blk_bytes)) {
return false;
}
- vtcm = (uint64_t) n_threads * 2 * (tp.in_buf_size + tp.out_buf_size);
+ vtcm = (uint64_t) n_hvx * 2 * (tp.in_buf_size + tp.out_buf_size);
if (vtcm <= vtcm_size) {
break;
}
@@ -2800,7 +2800,7 @@ static bool htp_kparams_solve_cpy_transpose(
tp.div_ne2 = init_fastdiv_values(tp.ne[2]);
kparams->kernel_type = HTP_COPY_KERNEL_TRANSPOSE;
- kparams->n_threads = (uint8_t) n_threads;
+ kparams->n_hvx = (uint8_t) n_hvx;
kparams->vtcm_size = (uint32_t) vtcm;
kparams->u.transpose = tp;
return true;
@@ -2872,7 +2872,7 @@ static bool htp_kparams_solve_cpy(
const bool src_is_contiguous = ggml_is_contiguous(src0);
const bool dst_is_contiguous = ggml_is_contiguous(dst);
- const uint32_t n_threads = htp_session_n_threads(sess);
+ const uint32_t n_hvx = htp_session_n_hvx(sess);
const uint32_t mdev_count = htp_session_mdev_count(sess);
const size_t vtcm_size = htp_session_vtcm_size(sess);
@@ -2895,10 +2895,10 @@ static bool htp_kparams_solve_cpy(
(uint32_t) src0->ne[0], (uint32_t) src0->nb[1],
0, 0, false,
src_type_size, total_rows,
- n_threads, mdev_count, vtcm_size)) {
+ n_hvx, mdev_count, vtcm_size)) {
kparams->kernel_type = HTP_COPY_KERNEL_GATHER_ROWS;
- kparams->n_threads = (uint8_t) n_threads;
- kparams->vtcm_size = kparams->u.gather.slice * n_threads;
+ kparams->n_hvx = (uint8_t) n_hvx;
+ kparams->vtcm_size = kparams->u.gather.slice * n_hvx;
return true;
}
}
@@ -2914,7 +2914,7 @@ static bool htp_kparams_solve_cpy(
}
kparams->kernel_type = HTP_COPY_KERNEL_RESHAPE;
- kparams->n_threads = n_threads > 0 ? (uint8_t) n_threads : 4;
+ kparams->n_hvx = n_hvx > 0 ? (uint8_t) n_hvx : 4;
kparams->u.reshape.div_ne0 = init_fastdiv_values((uint32_t) dst->ne[0]);
kparams->u.reshape.div_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[1] * dst->ne[0]));
kparams->u.reshape.div_ne2_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[2] * dst->ne[1] * dst->ne[0]));
@@ -2936,7 +2936,7 @@ static bool htp_kparams_solve_cpy(
return false;
}
- const uint32_t nth = n_threads > 0 ? n_threads : 4;
+ const uint32_t nth = n_hvx > 0 ? n_hvx : 4;
struct htp_copy_convert_vtcm_layout layout;
htp_copy_convert_solve_layout(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1], src_type_size, dst_type_size, nth);
@@ -2946,7 +2946,7 @@ static bool htp_kparams_solve_cpy(
kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_CONVERT;
kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
- kparams->n_threads = (uint8_t) nth;
+ kparams->n_hvx = (uint8_t) nth;
kparams->vtcm_size = (uint32_t) layout.total_bytes;
kparams->u.convert.blk_rows = layout.blk_rows;
kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);
@@ -3046,8 +3046,8 @@ static bool htp_kparams_solve_im2col(
is_patchembed = is_patchembed && (s1 == (int32_t) KH && p1 == 0 && d1 == 1);
}
- const uint32_t max_threads = (sess && htp_session_n_threads(sess) > 0) ? htp_session_n_threads(sess) : 8;
- const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads;
+ const uint32_t max_hvx = (sess && htp_session_n_hvx(sess) > 0) ? htp_session_n_hvx(sess) : 8;
+ const uint32_t n_hvx = (total_rows < max_hvx) ? total_rows : max_hvx;
const uint32_t vtcm_size = (sess && htp_session_vtcm_size(sess) > 0) ? (uint32_t) htp_session_vtcm_size(sess) : 0;
const uint32_t row_size = (uint32_t) dst->nb[2];
const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;
@@ -3060,12 +3060,12 @@ static bool htp_kparams_solve_im2col(
const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads,
+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_hvx,
htp_im2col_scratch_bytes(HTP_IM2COL_KERNEL_PATCHEMBED_DMA, KH, patch_stride));
if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;
- kparams->n_threads = (uint8_t) n_threads;
+ kparams->n_hvx = (uint8_t) n_hvx;
kparams->vtcm_size = (uint32_t) layout.total_bytes;
kparams->src_row_bytes = src_row_bytes;
kparams->dst_row_bytes = dst_row_bytes;
@@ -3113,7 +3113,7 @@ static bool htp_kparams_solve_im2col(
// threads work on output-col blocks, so a single output row (1D) can use them all
const uint32_t nbpr = (OW + owb - 1) / owb;
const uint64_t nblocks = (uint64_t) total_rows * nbpr;
- const uint32_t blk_threads = (uint32_t) (nblocks < max_threads ? nblocks : max_threads);
+ const uint32_t blk_threads = (uint32_t) (nblocks < max_hvx ? nblocks : max_hvx);
struct htp_im2col_vtcm_layout layout;
htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, blk_threads,
@@ -3121,7 +3121,7 @@ static bool htp_kparams_solve_im2col(
if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA;
- kparams->n_threads = (uint8_t) blk_threads;
+ kparams->n_hvx = (uint8_t) blk_threads;
kparams->vtcm_size = (uint32_t) layout.total_bytes;
kparams->src_row_bytes = src_row_bytes;
kparams->dst_row_bytes = dst_row_bytes;
@@ -3232,8 +3232,8 @@ static bool htp_kparams_solve_im2col_3d(
p0 == 0 && p1 == 0 && p2 == 0 &&
d0 == 1 && d1 == 1 && d2 == 1);
- const uint32_t max_threads = (sess && htp_session_n_threads(sess) > 0) ? htp_session_n_threads(sess) : 8;
- const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads;
+ const uint32_t max_hvx = (sess && htp_session_n_hvx(sess) > 0) ? htp_session_n_hvx(sess) : 8;
+ const uint32_t n_hvx = (total_rows < max_hvx) ? total_rows : max_hvx;
const uint32_t vtcm_size = (sess && htp_session_vtcm_size(sess) > 0) ? (uint32_t) htp_session_vtcm_size(sess) : 0;
const uint32_t row_size = (uint32_t) dst->nb[2];
const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;
@@ -3246,11 +3246,11 @@ static bool htp_kparams_solve_im2col_3d(
const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads, 256);
+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_hvx, 256);
if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;
- kparams->n_threads = (uint8_t) n_threads;
+ kparams->n_hvx = (uint8_t) n_hvx;
kparams->vtcm_size = (uint32_t) layout.total_bytes;
kparams->src_row_bytes = src_row_bytes;
kparams->dst_row_bytes = dst_row_bytes;
@@ -3296,11 +3296,11 @@ static bool htp_kparams_solve_im2col_3d(
const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads, 256);
+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_hvx, 256);
if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA;
- kparams->n_threads = (uint8_t) n_threads;
+ kparams->n_hvx = (uint8_t) n_hvx;
kparams->vtcm_size = (uint32_t) layout.total_bytes;
kparams->src_row_bytes = src_row_bytes;
kparams->dst_row_bytes = dst_row_bytes;
@@ -3355,16 +3355,16 @@ static bool htp_kparams_solve_pad(
const uint32_t type_size = (uint32_t) ggml_type_size(src0->type);
const uint32_t total_dst_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);
- const uint32_t max_threads = (std::max)(1u, total_dst_rows);
- const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+ const uint32_t max_hvx = (std::max)(1u, total_dst_rows);
+ const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx);
struct htp_pad_vtcm_layout layout;
- htp_pad_vtcm_layout_build(&layout, (uint32_t) src0->ne[0] * type_size, (uint32_t) dst->ne[0] * type_size, n_threads);
+ htp_pad_vtcm_layout_build(&layout, (uint32_t) src0->ne[0] * type_size, (uint32_t) dst->ne[0] * type_size, n_hvx);
const bool use_dma = (src0->nb[0] == type_size) && (src0->ne[0] >= 512) &&
(htp_session_vtcm_size(sess) >= layout.total_bytes);
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->total_rows = total_dst_rows;
kparams->use_dma = use_dma ? 1 : 0;
kparams->elem_size = type_size;
@@ -3407,15 +3407,15 @@ static bool htp_kparams_solve_cumsum(
}
const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
- const uint32_t max_threads = (std::max)(1u, total_rows);
- const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+ const uint32_t max_hvx = (std::max)(1u, total_rows);
+ const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx);
struct htp_cumsum_vtcm_layout layout;
- htp_cumsum_vtcm_layout_build(&layout, (uint32_t) src0->nb[1], (uint32_t) dst->nb[1], n_threads);
+ htp_cumsum_vtcm_layout_build(&layout, (uint32_t) src0->nb[1], (uint32_t) dst->nb[1], n_hvx);
const bool use_dma = (htp_session_vtcm_size(sess) >= layout.total_bytes);
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->total_rows = total_rows;
kparams->use_dma = use_dma ? 1 : 0;
kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0;
@@ -3425,17 +3425,17 @@ static bool htp_kparams_solve_cumsum(
// Shrink until the layout fits: halve tile_cols down to 1024 columns, then drop threads. False below the VTCM floor.
static bool htp_top_k_mask_search(struct htp_top_k_mask_vtcm_layout * L, const struct htp_top_k_mask_shape * S, uint32_t kernel_type,
- size_t vtcm_size, uint32_t * n_threads, uint32_t * tile_cols) {
+ size_t vtcm_size, uint32_t * n_hvx, uint32_t * tile_cols) {
const uint32_t min_cols = 1024;
while (true) {
- htp_top_k_mask_vtcm_layout_build(L, S, kernel_type, *n_threads, *tile_cols);
+ htp_top_k_mask_vtcm_layout_build(L, S, kernel_type, *n_hvx, *tile_cols);
if (L->total_bytes <= vtcm_size) {
return true;
}
if (*tile_cols > min_cols) {
*tile_cols = (std::max)(min_cols, (uint32_t) hex_round_up(*tile_cols / 2, 64));
- } else if (*n_threads > 1) {
- (*n_threads)--;
+ } else if (*n_hvx > 1) {
+ (*n_hvx)--;
} else {
return false;
}
@@ -3514,11 +3514,11 @@ static bool htp_kparams_solve_top_k_mask(
}
// PICKS for one row with a resident mask when its single row tile fits, ROWS otherwise
- const uint32_t sess_threads = (std::max)(1u, htp_session_n_threads(sess));
+ const uint32_t sess_threads = (std::max)(1u, htp_session_n_hvx(sess));
const size_t vtcm_size = htp_session_vtcm_size(sess);
uint32_t kernel_type = HTP_TOP_K_MASK_KERNEL_ROWS;
- uint32_t n_threads = sess_threads;
+ uint32_t n_hvx = sess_threads;
uint32_t tile_cols = 0;
struct htp_top_k_mask_vtcm_layout L;
@@ -3526,14 +3526,14 @@ static bool htp_kparams_solve_top_k_mask(
if (total_rows == 1 && S.mask_resident && sess_threads > 1) {
kernel_type = HTP_TOP_K_MASK_KERNEL_PICKS;
tile_cols = hex_round_up(S.n_kv, 64);
- htp_top_k_mask_vtcm_layout_build(&L, &S, kernel_type, n_threads, tile_cols);
+ htp_top_k_mask_vtcm_layout_build(&L, &S, kernel_type, n_hvx, tile_cols);
found = L.total_bytes <= vtcm_size;
}
if (!found) {
kernel_type = HTP_TOP_K_MASK_KERNEL_ROWS;
- n_threads = (std::max)(1u, (std::min)(sess_threads, total_rows));
+ n_hvx = (std::max)(1u, (std::min)(sess_threads, total_rows));
tile_cols = hex_round_up(S.n_kv, 64);
- found = htp_top_k_mask_search(&L, &S, kernel_type, vtcm_size, &n_threads, &tile_cols);
+ found = htp_top_k_mask_search(&L, &S, kernel_type, vtcm_size, &n_hvx, &tile_cols);
}
if (!found) {
return false;
@@ -3548,7 +3548,7 @@ static bool htp_kparams_solve_top_k_mask(
}
const auto units = [](uint64_t bytes) { return (uint16_t) (bytes / HTP_TOP_K_MASK_UNIT); };
- kparams->n_threads = (uint8_t) n_threads;
+ kparams->n_hvx = (uint8_t) n_hvx;
kparams->kernel_type = (uint8_t) kernel_type;
kparams->flags = (dst->type == GGML_TYPE_F16 ? HTP_TOP_K_MASK_F16 : 0) |
(score ? HTP_TOP_K_MASK_HAS_SCORE : 0) |
@@ -3581,8 +3581,8 @@ static bool htp_kparams_solve_top_k_mask(
if (kernel_type == HTP_TOP_K_MASK_KERNEL_PICKS) {
// per thread slices of the static copies and of the flag clear, a multiple of the unit
- const auto slice = [&](uint64_t n) { return units(hex_round_up((n + n_threads - 1) / n_threads, HTP_TOP_K_MASK_UNIT)); };
- kparams->u.picks.picks_per_thread = (uint32_t) hex_round_up((S.n_top + n_threads - 1) / n_threads, 32);
+ const auto slice = [&](uint64_t n) { return units(hex_round_up((n + n_hvx - 1) / n_hvx, HTP_TOP_K_MASK_UNIT)); };
+ kparams->u.picks.picks_per_hvx = (uint32_t) hex_round_up((S.n_top + n_hvx - 1) / n_hvx, 32);
kparams->u.picks.blocks_slice_u = slice(L.blocks_bytes);
kparams->u.picks.top_slice_u = slice((uint64_t) S.n_top * 4);
kparams->u.picks.extra_slice_u = slice((uint64_t) S.n_extra * 4);
@@ -3633,12 +3633,12 @@ static bool htp_kparams_solve_fill(
const uint32_t nb1 = (uint32_t) dst->nb[1];
const uint32_t total_rows = ne1 * ne2 * ne3;
- const uint32_t max_threads = (std::max)(1u, total_rows);
- const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+ const uint32_t max_hvx = (std::max)(1u, total_rows);
+ const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx);
const uint32_t elem_size = (uint32_t) ggml_type_size(dst->type);
const uint32_t rows_per_chunk = (nb1 > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(nb1, HEX_L2_LINE_SIZE)) : 1;
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->total_rows = total_rows;
kparams->elem_size = elem_size;
kparams->kernel_type = ggml_is_contiguous(dst) ? HTP_FILL_KERNEL_CONTIGUOUS : HTP_FILL_KERNEL_STRIDED;
@@ -3685,10 +3685,10 @@ static bool htp_kparams_solve_roll(
}
const uint32_t total_rows = ne1 * ne2 * ne3;
- const uint32_t max_threads = (std::max)(1u, total_rows);
- const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+ const uint32_t max_hvx = (std::max)(1u, total_rows);
+ const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx);
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->total_rows = total_rows;
kparams->i0_src0 = i0_src0;
kparams->n0 = ne0 - i0_src0;
@@ -3731,8 +3731,8 @@ static bool htp_kparams_solve_diag(
}
const uint32_t total_batches = (uint32_t) (src0->ne[2] * src0->ne[3]);
- const uint32_t max_threads = (std::max)(1u, total_batches);
- const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+ const uint32_t max_hvx = (std::max)(1u, total_batches);
+ const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx);
const uint32_t dst_batch_size = (uint32_t) (dst->ne[1] * dst->nb[1]);
bool can_split = (dst->ne[0] == 1 || dst->nb[0] == sizeof(float)) && !ggml_is_permuted(dst);
@@ -3753,11 +3753,11 @@ static bool htp_kparams_solve_diag(
const uint32_t dst_row_size = (uint32_t) (dst->ne[0] * sizeof(float));
struct htp_diag_vtcm_layout layout;
- htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_threads);
+ htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_hvx);
const bool use_dma = (htp_session_vtcm_size(sess) >= layout.total_bytes);
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->total_batches = total_batches;
kparams->use_dma = use_dma ? 1 : 0;
kparams->can_split = can_split ? 1 : 0;
@@ -3788,19 +3788,19 @@ static bool htp_kparams_solve_solve_tri(
return false;
}
- const uint32_t max_threads_sess = htp_session_n_threads(sess);
+ const uint32_t max_hvx_sess = htp_session_n_hvx(sess);
const uint32_t k = (uint32_t) src1->ne[0];
const uint32_t col_block = 32;
const uint32_t k_chunks = (k + col_block - 1) / col_block;
const uint32_t total_batches = (uint32_t) (src0->ne[2] * src0->ne[3]);
- const bool batched = total_batches >= max_threads_sess;
+ const bool batched = total_batches >= max_hvx_sess;
const uint32_t total_jobs = batched ? total_batches : (total_batches * k_chunks);
- const uint32_t max_threads = (std::max)(1u, total_jobs);
- const uint32_t n_threads = (std::min)(max_threads_sess, max_threads);
+ const uint32_t max_hvx = (std::max)(1u, total_jobs);
+ const uint32_t n_hvx = (std::min)(max_hvx_sess, max_hvx);
const uint32_t batch_size = (uint32_t) dst->nb[2];
const uint32_t batches_per_chunk = (batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(batch_size, HEX_L2_LINE_SIZE)) : 1;
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->total_batches = total_batches;
kparams->total_jobs = total_jobs;
kparams->k_chunks = k_chunks;
@@ -3877,7 +3877,7 @@ static bool htp_kparams_conv2d_win(
const uint32_t halo = moff / 32 + 1;
const uint32_t n_rt = (Cout + 31) / 32;
const uint64_t n_q = (uint64_t) (N * IHS - (IHS - OH) - 1) * SR + OW;
- const uint32_t nth = kp->n_threads;
+ const uint32_t nth = kp->n_hvx;
const size_t tile = 2048;
if ((uint64_t) N * IHS * SR + moff + 64 >= (1ull << 31)) {
@@ -3900,14 +3900,14 @@ static bool htp_kparams_conv2d_win(
}
// enough packing units for all threads
const uint32_t kpw2 = es == 2 ? 64 : 32;
- while (!fold && w_cs > 2 && n_rt * ((Cin + 31) / 32) * (32 / w_cs) < kp->n_threads) {
+ while (!fold && w_cs > 2 && n_rt * ((Cin + 31) / 32) * (32 / w_cs) < kp->n_hvx) {
w_cs /= 2;
}
wraw = (std::min)(wraw, (size_t) 32 * ((w_cs * T + kpw2 - 1) / kpw2) * 128);
wscr = 3 * wraw + 64 * 128 + 128;
} else {
w_cs = fold ? KW : (std::min)(T, 16u);
- while (!fold && w_cs > 1 && n_rt * ((Cin + 31) / 32) * ((T + w_cs - 1) / w_cs) < kp->n_threads) {
+ while (!fold && w_cs > 1 && n_rt * ((Cin + 31) / 32) * ((T + w_cs - 1) / w_cs) < kp->n_hvx) {
w_cs = (w_cs + 1) / 2;
}
wscr = 2 * (size_t) w_cs * 32 * 128;
@@ -3959,7 +3959,7 @@ static bool htp_kparams_conv2d_win(
}
}
- uint32_t n_threads = nth;
+ uint32_t n_hvx = nth;
// Keep the fold schedule, where output traffic can dominate input packing.
if (!fold && rt_block < n_rt && ct > 0 && nth > 0) {
const uint32_t old_rt = rt_block;
@@ -4022,7 +4022,7 @@ static bool htp_kparams_conv2d_win(
rt_block = rb;
ct = candidate;
seg_b = sb;
- n_threads = nt;
+ n_hvx = nt;
stage = input_stage;
thr = scratch;
act = input;
@@ -4047,7 +4047,7 @@ static bool htp_kparams_conv2d_win(
const uint32_t n_cs = (32 + CS - 1) / CS;
kp->kernel_type = (uint8_t) HTP_CONV2D_KERNEL_HMX_WINDOW;
- kp->n_threads = (uint8_t) n_threads;
+ kp->n_hvx = (uint8_t) n_hvx;
kp->cwhn = cwhn ? 1 : 0;
kp->w_kmajor = !w_rows ? 1 : 0;
kp->n_row_tiles = n_rt;
@@ -4165,7 +4165,7 @@ static bool htp_kparams_conv2d_tiled(
const uint32_t n_owt = (g.OW + 31) / 32;
const uint32_t n_ct = g.N * g.OH * n_owt;
const uint32_t wk = (std::min)(k_pad, 512u);
- const uint32_t max_threads = htp_session_n_threads(sess);
+ const uint32_t max_hvx = htp_session_n_hvx(sess);
const size_t tile = 2048;
uint32_t rt_block = n_rt;
@@ -4184,7 +4184,7 @@ static bool htp_kparams_conv2d_tiled(
// Keep the kernel resident to avoid regenerating the input for each rt block.
// Balance patch work per thread and chunk overhead before reducing rt.
for (uint32_t rb = n_rt; rb > 0 && ct == 0; rb = rb > 1 ? (rb + 1) / 2 : 0) {
- for (uint32_t nt = max_threads; nt > 0; nt /= 2) {
+ for (uint32_t nt = max_hvx; nt > 0; nt /= 2) {
for (uint32_t sg = (std::min)(8u, n_owt); sg > 0; --sg) {
// Keep the unused final stride lane for full-vector loads during packing.
const uint64_t wb = (uint64_t) 32 * sg * g.s0 + (uint64_t) (g.KW - 1) * g.d0;
@@ -4219,11 +4219,11 @@ static bool htp_kparams_conv2d_tiled(
if (candidate == 0 || (i > 0 && candidate == candidates[0])) {
continue;
}
- const uint32_t per_thread = (candidate + nt - 1) / nt;
- if (sg > per_thread) {
+ const uint32_t per_hvx = (candidate + nt - 1) / nt;
+ if (sg > per_hvx) {
continue;
}
- const size_t work = (size_t) (n_ct / candidate) * per_thread + (n_ct % candidate + nt - 1) / nt;
+ const size_t work = (size_t) (n_ct / candidate) * per_hvx + (n_ct % candidate + nt - 1) / nt;
const size_t chunks = n_ct / candidate + (n_ct % candidate != 0);
const size_t cost = work * n_kt + c_chunk * chunks;
if (cost < best_cost) {
@@ -4267,7 +4267,7 @@ static bool htp_kparams_conv2d_tiled(
}
kp->kernel_type = (uint8_t) HTP_CONV2D_KERNEL_HMX_TILED;
- kp->n_threads = (uint8_t) nth;
+ kp->n_hvx = (uint8_t) nth;
kp->cwhn = cwhn ? 1 : 0;
kp->w_kmajor = kmajor ? 1 : 0;
kp->n_row_tiles = n_rt;
@@ -4303,7 +4303,7 @@ static bool htp_kparams_solve_conv2d_unchecked(
const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);
kparams->total_rows = total_rows;
- kparams->n_threads = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_threads(sess), total_rows);
+ kparams->n_hvx = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_hvx(sess), total_rows);
if (htp_kparams_conv2d_win(sess, dst, kparams)) {
return true;
@@ -4448,10 +4448,10 @@ static bool htp_kparams_solve_conv2d_dw(
const uint32_t w_size = (uint32_t) (kw * kh * c * ggml_type_size(src0->type));
- uint32_t max_threads_sess = (uint32_t) htp_session_n_threads(sess);
+ uint32_t max_hvx_sess = (uint32_t) htp_session_n_hvx(sess);
uint32_t oh_chunk = oh_u;
- uint32_t n_threads = max_threads_sess;
+ uint32_t n_hvx = max_hvx_sess;
struct htp_conv2d_dw_vtcm_layout layout;
while (true) {
@@ -4459,7 +4459,7 @@ static bool htp_kparams_solve_conv2d_dw(
const uint32_t max_src_bytes = (max_ih_chunk + pad_rows) * row_src_bytes + 128;
const uint32_t max_dst_bytes = oh_chunk * row_dst_bytes + 128;
- htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_threads);
+ htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_hvx);
if (layout.total_bytes <= vtcm_budget && layout.total_bytes > 0) {
break;
@@ -4467,8 +4467,8 @@ static bool htp_kparams_solve_conv2d_dw(
if (oh_chunk > 1) {
oh_chunk = (oh_chunk + 1) / 2;
- } else if (n_threads > 1) {
- n_threads--;
+ } else if (n_hvx > 1) {
+ n_hvx--;
} else {
return false;
}
@@ -4476,18 +4476,18 @@ static bool htp_kparams_solve_conv2d_dw(
const uint32_t n_chunks = (oh_u + oh_chunk - 1) / oh_chunk;
const uint32_t total_tasks = cwhn ? (n * n_chunks) : (c * n * n_chunks);
- n_threads = (std::min)(n_threads, total_tasks);
- if (n_threads == 0) {
- n_threads = 1;
+ n_hvx = (std::min)(n_hvx, total_tasks);
+ if (n_hvx == 0) {
+ n_hvx = 1;
}
const uint32_t max_ih_chunk = (std::min)(h, (oh_chunk - 1) * (uint32_t) s1 + (kh - 1) * (uint32_t) d1 + 1);
const uint32_t max_src_bytes = (max_ih_chunk + pad_rows) * row_src_bytes + 128;
const uint32_t max_dst_bytes = oh_chunk * row_dst_bytes + 128;
- htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_threads);
+ htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_hvx);
kparams->kernel_type = kernel_type;
- kparams->n_threads = (uint8_t) n_threads;
+ kparams->n_hvx = (uint8_t) n_hvx;
kparams->cwhn = cwhn ? 1 : 0;
kparams->is_f16 = (src0->type == GGML_TYPE_F16) ? 1 : 0;
kparams->total_tasks = total_tasks;
@@ -4566,7 +4566,7 @@ static bool htp_kparams_solve_conv3d(
return false;
}
kparams->total_rows = (uint32_t) total_rows;
- kparams->n_threads = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_threads(sess), (uint32_t) total_rows);
+ kparams->n_hvx = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_hvx(sess), (uint32_t) total_rows);
return htp_kparams_conv2d_tiled(sess, op, kparams);
}
@@ -4702,8 +4702,8 @@ static bool htp_kparams_solve_dsv4_hc_pre_impl(
if (kparams->total_jobs == 0) {
return true;
}
- kparams->n_threads = (std::min)(htp_session_n_threads(sess), kparams->total_jobs);
- kparams->jobs_per_thread = (kparams->total_jobs + kparams->n_threads - 1) / kparams->n_threads;
+ kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), kparams->total_jobs);
+ kparams->jobs_per_hvx = (kparams->total_jobs + kparams->n_hvx - 1) / kparams->n_hvx;
const size_t vtcm_budget = htp_session_vtcm_size(sess);
const uint32_t n_rows = gated ? HTP_DSV4_HC_PRE_GATED_ROWS : HTP_DSV4_HC_PRE_PLAIN_ROWS;
@@ -4711,7 +4711,7 @@ static bool htp_kparams_solve_dsv4_hc_pre_impl(
while (block_chunks > 16) {
const uint32_t stride = block_chunks * 128;
const uint32_t buf_sz = n_rows * stride;
- if (2 * buf_sz * kparams->n_threads <= vtcm_budget) {
+ if (2 * buf_sz * kparams->n_hvx <= vtcm_budget) {
break;
}
block_chunks /= 2;
@@ -4720,13 +4720,13 @@ static bool htp_kparams_solve_dsv4_hc_pre_impl(
kparams->dma_block_chunks = block_chunks;
kparams->dma_row_stride = block_chunks * 128;
kparams->dma_buffer_size = n_rows * kparams->dma_row_stride;
- kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_threads;
+ kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_hvx;
if (kparams->vtcm_size > vtcm_budget) {
return false;
}
kparams->div_n_chunks = init_fastdiv_values(kparams->n_chunks);
- kparams->div_n_threads = init_fastdiv_values(kparams->n_threads);
+ kparams->div_n_hvx = init_fastdiv_values(kparams->n_hvx);
return true;
}
@@ -4906,15 +4906,15 @@ static bool htp_kparams_solve_dsv4_hc_post_impl(
if (kparams->total_jobs == 0) {
return true;
}
- kparams->n_threads = (std::min)(htp_session_n_threads(sess), kparams->total_jobs);
- kparams->jobs_per_thread = (kparams->total_jobs + kparams->n_threads - 1) / kparams->n_threads;
+ kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), kparams->total_jobs);
+ kparams->jobs_per_hvx = (kparams->total_jobs + kparams->n_hvx - 1) / kparams->n_hvx;
const size_t vtcm_budget = htp_session_vtcm_size(sess);
uint32_t block_chunks = (std::min)((uint32_t) HTP_DSV4_HC_MAX_CHUNKS, kparams->n_chunks);
while (block_chunks > 16) {
const uint32_t stride = block_chunks * 128;
const uint32_t buf_sz = HTP_DSV4_HC_POST_ROWS * stride;
- if (2 * buf_sz * kparams->n_threads <= vtcm_budget) {
+ if (2 * buf_sz * kparams->n_hvx <= vtcm_budget) {
break;
}
block_chunks /= 2;
@@ -4923,13 +4923,13 @@ static bool htp_kparams_solve_dsv4_hc_post_impl(
kparams->dma_block_chunks = block_chunks;
kparams->dma_row_stride = block_chunks * 128;
kparams->dma_buffer_size = HTP_DSV4_HC_POST_ROWS * kparams->dma_row_stride;
- kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_threads;
+ kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_hvx;
if (kparams->vtcm_size > vtcm_budget) {
return false;
}
kparams->div_n_chunks = init_fastdiv_values(kparams->n_chunks);
- kparams->div_n_threads = init_fastdiv_values(kparams->n_threads);
+ kparams->div_n_hvx = init_fastdiv_values(kparams->n_hvx);
return true;
}
@@ -5072,12 +5072,12 @@ static bool htp_kparams_solve_lightning_indexer_impl(
kparams->div_m_ne3 = init_fastdiv_values((uint32_t) m->ne[3]);
if (opt_li_select >= 2 && htp_session_n_hmx(sess) > 0 && D % HMX_FP16_TILE_N_COLS == 0) {
- const uint32_t n_threads = htp_session_n_threads(sess);
+ const uint32_t n_hvx = htp_session_n_hvx(sess);
const uint32_t n_stream = q->ne[3];
const uint32_t n_dot = D / HMX_FP16_TILE_N_COLS;
auto vtcm_usage = [&](uint32_t n_tok, uint32_t kv_block, uint32_t n_qsets) {
- return htp_lid_hmx_compute_vtcm_usage(D, n_head, n_tok, kv_block, n_qsets, n_threads, k_row_size);
+ return htp_lid_hmx_compute_vtcm_usage(D, n_head, n_tok, kv_block, n_qsets, n_hvx, k_row_size);
};
uint32_t n_tok = (std::min)(n_tokens, (uint32_t) HTP_LID_HMX_TOKENS_MAX);
@@ -5118,10 +5118,10 @@ static bool htp_kparams_solve_lightning_indexer_impl(
if (vtcm_size <= htp_session_vtcm_size(sess)) {
const uint32_t n_tt = (n_tokens + n_tok - 1) / n_tok;
kparams->kernel_type = HTP_LID_KERNEL_HMX;
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->n_tokens_tile = n_tok;
kparams->n_kv_blocks = (n_rows + kv_block - 1) / kv_block;
- kparams->blocks_per_thread = 0;
+ kparams->blocks_per_hvx = 0;
kparams->kv_block = kv_block;
kparams->n_qsets = n_qsets;
kparams->n_qbuf = n_qbuf;
@@ -5135,22 +5135,22 @@ static bool htp_kparams_solve_lightning_indexer_impl(
}
}
- const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), n_kv_blocks);
+ const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), n_kv_blocks);
uint32_t n_tok = (std::min)((uint32_t) q->ne[2], (uint32_t) HTP_LID_TOKENS_MAX);
- while (n_tok > 1 && htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_threads, k_row_size) > htp_session_vtcm_size(sess)) {
+ while (n_tok > 1 && htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_hvx, k_row_size) > htp_session_vtcm_size(sess)) {
n_tok /= 2;
}
- const size_t vtcm_size = htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_threads, k_row_size);
+ const size_t vtcm_size = htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_hvx, k_row_size);
if (vtcm_size > htp_session_vtcm_size(sess)) {
return false;
}
kparams->kernel_type = HTP_LID_KERNEL_HVX;
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->n_tokens_tile = n_tok;
kparams->n_kv_blocks = n_kv_blocks;
- kparams->blocks_per_thread = (n_kv_blocks + n_threads - 1) / n_threads;
+ kparams->blocks_per_hvx = (n_kv_blocks + n_hvx - 1) / n_hvx;
kparams->kv_block = HTP_LID_KV_BLOCK;
kparams->n_qsets = 1;
kparams->n_qbuf = 1;
@@ -5186,7 +5186,7 @@ static inline bool htp_kparams_invoke(
}
void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & node) {
- const uint32_t n_threads = htp_session_n_threads(sess);
+ const uint32_t n_hvx = htp_session_n_hvx(sess);
if (htp_op_is_unary(node.opcode) && htp_unary_is_elementwise(node.opcode)) {
if (node.inputs.empty() || !node.inputs[0] || node.outputs.empty() || !node.outputs[0]) {
@@ -5199,7 +5199,7 @@ void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode &
struct ggml_tensor src0_c = *node.inputs[0];
struct ggml_tensor dst_c = *node.outputs[0];
struct ggml_tensor * tensors[2] = { &src0_c, &dst_c };
- htp_collapse_dims(tensors, 2, elem_size, n_threads);
+ htp_collapse_dims(tensors, 2, elem_size, n_hvx);
if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) {
node.inputs[0] = node.add_dummy(src0_c);
node.outputs[0] = node.add_dummy(dst_c);
@@ -5220,7 +5220,7 @@ void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode &
struct ggml_tensor src1_c = *node.inputs[1];
struct ggml_tensor dst_c = *node.outputs[0];
struct ggml_tensor * tensors[3] = { &src0_c, &src1_c, &dst_c };
- htp_collapse_dims(tensors, 3, elem_size, n_threads);
+ htp_collapse_dims(tensors, 3, elem_size, n_hvx);
if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) {
node.inputs[0] = node.add_dummy(src0_c);
node.inputs[1] = node.add_dummy(src1_c);
diff --git a/ggml/src/ggml-hexagon/htp-opnode.cpp b/ggml/src/ggml-hexagon/htp-opnode.cpp
index a4efa8503..8dffedc4b 100644
--- a/ggml/src/ggml-hexagon/htp-opnode.cpp
+++ b/ggml/src/ggml-hexagon/htp-opnode.cpp
@@ -157,7 +157,7 @@ static std::string format_kernel_params(const htp_opnode & node) {
} else if (type == HTP_MM_KERNEL_HVX_QUANT_ROW || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {
path = "hvx-tiled";
}
- snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);
+ snprintf(str, max_size, "%s nhvx %u vtcm %u", path, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) {
const auto * kparams = node.kparams<htp_fa_kernel_params>();
const char * path = "unknown";
@@ -167,10 +167,11 @@ static std::string format_kernel_params(const htp_opnode & node) {
} else if (type == HTP_FA_KERNEL_HVX) {
path = "hvx";
}
- snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);
+ snprintf(str, max_size, "%s nhvx %u vtcm %u", path, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size);
} else if (htp_op_is_unary(node.opcode)) {
const auto * kparams = node.kparams<htp_unary_kernel_params>();
- snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size);
+ snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->col_tile ? "wide-row" : "row-block",
+ (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) {
snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]);
} else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) {
@@ -179,41 +180,43 @@ static std::string format_kernel_params(const htp_opnode & node) {
snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]);
} else if (node.opcode == HTP_OP_SSM_CONV) {
const auto * kparams = node.kparams<htp_ssm_conv_kernel_params>();
- snprintf(str, max_size, "%s vtcm %d", kparams->n_t == 1 ? "decode" : "prefill", (int) kparams->vtcm_size);
+ snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->n_t == 1 ? "decode" : "prefill",
+ (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_SOFTMAX) {
const auto * kparams = node.kparams<htp_softmax_kernel_params>();
- snprintf(str, max_size, "k%d nth %d vtcm %d", (int) kparams->kernel_type, (int) kparams->n_threads, (int) kparams->vtcm_size);
+ snprintf(str, max_size, "k%d nhvx %u vtcm %u", (int) kparams->kernel_type,
+ (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_GATED_DELTA_NET) {
const auto * kparams = node.kparams<htp_gdn_kernel_params>();
const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent";
- snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar",
- (unsigned int) kparams->vtcm_size);
+ snprintf(str, max_size, "%s-%s nhvx %u vtcm %u", path, kparams->kda ? "kda" : "scalar",
+ (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_DSV4_HC_PRE) {
const auto * kparams = node.kparams<htp_dsv4_hc_pre_kernel_params>();
const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_PRE_KERNEL_GATED ? "gated" : "plain";
- snprintf(str, max_size, "%s nth %u vtcm %u", ktype_str,
- (unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
+ snprintf(str, max_size, "%s nhvx %u vtcm %u", ktype_str,
+ (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_DSV4_HC_POST) {
const auto * kparams = node.kparams<htp_dsv4_hc_post_kernel_params>();
const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_POST_KERNEL_COMB ? "comb" : "identity";
- snprintf(str, max_size, "%s nth %u vtcm %u", ktype_str,
- (unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
+ snprintf(str, max_size, "%s nhvx %u vtcm %u", ktype_str,
+ (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_LIGHTNING_INDEXER) {
const auto * kparams = node.kparams<htp_lid_kernel_params>();
- snprintf(str, max_size, "%s tok %u bc %u nth %u vtcm %u",
+ snprintf(str, max_size, "%s nhvx %u tok %u bc %u vtcm %u",
kparams->kernel_type == HTP_LID_KERNEL_HMX ? "hmx" : "hvx",
- (unsigned int) kparams->n_tokens_tile, (unsigned int) kparams->kv_block,
- (unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
+ (unsigned int) kparams->n_hvx, (unsigned int) kparams->n_tokens_tile,
+ (unsigned int) kparams->kv_block, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID ||
node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {
const auto * kparams = node.kparams<htp_binary_kernel_params>();
- snprintf(str, max_size, "vtcm %u", (unsigned int) kparams->vtcm_size);
+ snprintf(str, max_size, "nhvx %u vtcm %u", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_ARGSORT || node.opcode == HTP_OP_TOP_K) {
const auto * kparams = node.kparams<htp_sort_kernel_params>();
- snprintf(str, max_size, "%s nth %d nchk %d chk %d vtcm %d",
+ snprintf(str, max_size, "%s nhvx %u nchk %u chk %u vtcm %u",
node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort",
- (int) kparams->n_threads, (int) kparams->n_chunks,
- (int) kparams->chunk_elems, (int) kparams->vtcm_size);
+ (unsigned int) kparams->n_hvx, (unsigned int) kparams->n_chunks,
+ (unsigned int) kparams->chunk_elems, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_GET_ROWS) {
const auto * kparams = node.kparams<htp_get_rows_kernel_params>();
const char * ktype_str = "unknown";
@@ -222,16 +225,16 @@ static std::string format_kernel_params(const htp_opnode & node) {
case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break;
case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break;
}
- snprintf(str, max_size, "%s%s vtcm %u", ktype_str, kparams->n_threads > 1 ? "-multi" : "", (unsigned int) kparams->vtcm_size);
+ snprintf(str, max_size, "%s nhvx %u vtcm %u", ktype_str, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size);
} else if (htp_op_is_glu(node.opcode)) {
const auto * kparams = node.kparams<htp_glu_kernel_params>();
- snprintf(str, max_size, "nth %d vtcm %u", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
+ snprintf(str, max_size, "nhvx %u vtcm %u", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_PAD) {
const auto * kparams = node.kparams<htp_pad_kernel_params>();
- snprintf(str, max_size, "k%d nth %d vtcm %u", (int) kparams->kernel_type, (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
+ snprintf(str, max_size, "k%d nhvx %u vtcm %u", (int) kparams->kernel_type, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_CUMSUM) {
const auto * kparams = node.kparams<htp_cumsum_kernel_params>();
- snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
+ snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->use_dma ? "dma" : "hvx", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_TOP_K_MASK) {
const auto * kparams = node.kparams<htp_top_k_mask_kernel_params>();
const char * ktype_str = "?";
@@ -239,19 +242,19 @@ static std::string format_kernel_params(const htp_opnode & node) {
case HTP_TOP_K_MASK_KERNEL_ROWS: ktype_str = "rows"; break;
case HTP_TOP_K_MASK_KERNEL_PICKS: ktype_str = "picks"; break;
}
- snprintf(str, max_size, "%s nth %d tile %u x%u vtcm %u", ktype_str, (int) kparams->n_threads, (unsigned int) kparams->tile_cols, (unsigned int) kparams->n_col_tiles, (unsigned int) kparams->vtcm_size);
+ snprintf(str, max_size, "%s nhvx %u tile %u x%u vtcm %u", ktype_str, (unsigned int) kparams->n_hvx, (unsigned int) kparams->tile_cols, (unsigned int) kparams->n_col_tiles, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_FILL) {
const auto * kparams = node.kparams<htp_fill_kernel_params>();
- snprintf(str, max_size, "%s nth %d", kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS ? "contig" : "strided", (int) kparams->n_threads);
+ snprintf(str, max_size, "%s nhvx %u", kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS ? "contig" : "strided", (unsigned int) kparams->n_hvx);
} else if (node.opcode == HTP_OP_ROLL) {
const auto * kparams = node.kparams<htp_roll_kernel_params>();
- snprintf(str, max_size, "k%d nth %d", (int) kparams->kernel_type, (int) kparams->n_threads);
+ snprintf(str, max_size, "k%d nhvx %u", (int) kparams->kernel_type, (unsigned int) kparams->n_hvx);
} else if (node.opcode == HTP_OP_DIAG) {
const auto * kparams = node.kparams<htp_diag_kernel_params>();
- snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
+ snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->use_dma ? "dma" : "hvx", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_SOLVE_TRI) {
const auto * kparams = node.kparams<htp_solve_tri_kernel_params>();
- snprintf(str, max_size, "%s nth %d", kparams->batched ? "batched" : "chunked", (int) kparams->n_threads);
+ snprintf(str, max_size, "%s nhvx %u", kparams->batched ? "batched" : "chunked", (unsigned int) kparams->n_hvx);
} else if (node.opcode == HTP_OP_UPSCALE) {
const auto * kparams = node.kparams<htp_upscale_kernel_params>();
const char * ktype_str = "unknown";
@@ -264,7 +267,7 @@ static std::string format_kernel_params(const htp_opnode & node) {
case HTP_UPSCALE_KERNEL_BILINEAR_AA: ktype_str = "bilinear-aa"; break;
case HTP_UPSCALE_KERNEL_BICUBIC: ktype_str = "bicubic"; break;
}
- snprintf(str, max_size, "%s nth %d chk %d vtcm %u", ktype_str, (int) kparams->n_threads, (int) kparams->dst_nrows_per_chunk, (unsigned int) kparams->vtcm_size);
+ snprintf(str, max_size, "%s nhvx %u chk %u vtcm %u", ktype_str, (unsigned int) kparams->n_hvx, (unsigned int) kparams->dst_nrows_per_chunk, (unsigned int) kparams->vtcm_size);
} else {
snprintf(str, max_size, "----");
}
diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.c b/ggml/src/ggml-hexagon/htp/allreduce-ops.c
index 7fcc4e866..1b95917ae 100644
--- a/ggml/src/ggml-hexagon/htp/allreduce-ops.c
+++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.c
@@ -15,7 +15,7 @@
#include "hvx-utils.h"
#include "htp-tensor.h"
#include "dma-queue.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "allreduce-ops.h"
#include "htp-fence.h"
@@ -29,9 +29,9 @@ struct htp_allreduce_context {
uint32_t row_size_aligned;
uint32_t rank_elem_start;
uint32_t rank_nelem;
- uint32_t elems_per_thread;
+ uint32_t elems_per_hvx;
uint32_t block_elems;
- uint32_t vtcm_size_per_thread;
+ uint32_t vtcm_size_per_hvx;
bool is_row_bcast;
uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS];
uint8_t * dst_spad_base;
@@ -39,7 +39,8 @@ struct htp_allreduce_context {
};
#define DEFINE_ALLREDUCE_THREAD_DMA_1D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD) \
-static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \
+static void allreduce_thread_dma_1d_##SUFFIX(const htp_thread_t * th, unsigned int nth, unsigned int ith, \
+ void * data) { \
struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \
struct htp_ops_context * octx = actx->octx; \
\
@@ -47,19 +48,19 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith,
const uint32_t n_dsts = actx->n_dsts; \
const uint32_t block_elems = actx->block_elems; \
\
- const uint32_t dr = actx->elems_per_thread; \
+ const uint32_t dr = actx->elems_per_hvx; \
const uint32_t ir0 = actx->rank_elem_start + dr * ith; \
const uint32_t ir1 = MIN(ir0 + dr, actx->rank_elem_start + actx->rank_nelem); \
if (ir0 >= ir1) return; \
\
- struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
- dma_queue * dma_q = octx->ctx->dma[ith]; \
+ struct htp_thread_trace * tr = th->trace; \
+ dma_queue * dma_q = th->dma_q; \
\
- const size_t vtcm_thread_offset = ith * actx->vtcm_size_per_thread; \
- uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_thread_offset; \
- uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + vtcm_thread_offset) : NULL; \
+ const size_t vtcm_hvx_offset = ith * actx->vtcm_size_per_hvx; \
+ uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_hvx_offset; \
+ uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + vtcm_hvx_offset) : NULL; \
\
- const size_t spad_half = actx->vtcm_size_per_thread / 2; \
+ const size_t spad_half = actx->vtcm_size_per_hvx / 2; \
uint32_t ir_prefetch = ir0; \
int spad_idx = 0; \
\
@@ -72,7 +73,7 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith,
dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 0); \
} \
for (uint32_t s = 0; s < n_ranks; s++) { \
- uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + spad_idx * spad_half; \
+ uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + spad_idx * spad_half; \
const dma_addr_t s_ddr = octx->src[s]->data + ir_prefetch * sizeof(TYPE); \
dma_queue_push(dma_q, dma_make_data(s_spad, s_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \
} \
@@ -99,13 +100,13 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith,
dma_queue_pop(dma_q); \
} \
uint8_t * d_spad = dst_spad_base + comp_spad_idx * spad_half; \
- uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_thread_offset + comp_spad_idx * spad_half; \
- uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_thread_offset + comp_spad_idx * spad_half; \
+ uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_hvx_offset + comp_spad_idx * spad_half; \
+ uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_hvx_offset + comp_spad_idx * spad_half; \
uint8_t * r_spad = HAS_ADD ? (res_spad_base + comp_spad_idx * spad_half) : NULL; \
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \
HVX_ADD_FN(d_spad, s0_spad, s1_spad, cur_elems); \
for (uint32_t s = 2; s < n_ranks; s++) { \
- uint8_t * ss_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \
+ uint8_t * ss_spad = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half; \
HVX_ADD_FN(d_spad, d_spad, ss_spad, cur_elems); \
} \
if (HAS_ADD) { \
@@ -120,7 +121,7 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith,
uint32_t next_elems = MIN(block_elems, ir1 - ir_prefetch); \
size_t next_bytes = next_elems * sizeof(TYPE); \
for (uint32_t s = 0; s < n_ranks; s++) { \
- uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \
+ uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half; \
const dma_addr_t s_next = octx->src[s]->data + ir_prefetch * sizeof(TYPE); \
dma_queue_push(dma_q, dma_make_data(s_spad, s_next), next_bytes, next_bytes, next_bytes, 1); \
} \
@@ -143,7 +144,7 @@ DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f16, __fp16, hvx_add_f16_aaa, 1)
DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f32, float, hvx_add_f32_aaa, 1)
#define DEFINE_ALLREDUCE_THREAD_DMA_2D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD, IS_ROW_BCAST) \
-static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \
+static void allreduce_thread_dma_2d_##SUFFIX(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { \
struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \
struct htp_ops_context * octx = actx->octx; \
\
@@ -154,19 +155,19 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith,
const uint32_t row_size_aligned = actx->row_size_aligned; \
const uint32_t row_bytes = ne0 * sizeof(TYPE); \
\
- const uint32_t dr = actx->elems_per_thread; \
+ const uint32_t dr = actx->elems_per_hvx; \
const uint32_t r0 = actx->rank_elem_start + dr * ith; \
const uint32_t r1 = MIN(r0 + dr, actx->rank_elem_start + actx->rank_nelem); \
if (r0 >= r1) return; \
\
- struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
- dma_queue * dma_q = octx->ctx->dma[ith]; \
+ struct htp_thread_trace * tr = th->trace; \
+ dma_queue * dma_q = th->dma_q; \
\
- const size_t vtcm_thread_offset = ith * actx->vtcm_size_per_thread; \
- uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_thread_offset; \
- uint8_t * res_spad_base = HAS_ADD ? (IS_ROW_BCAST ? actx->res_spad_base : (actx->res_spad_base + vtcm_thread_offset)) : NULL; \
+ const size_t vtcm_hvx_offset = ith * actx->vtcm_size_per_hvx; \
+ uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_hvx_offset; \
+ uint8_t * res_spad_base = HAS_ADD ? (IS_ROW_BCAST ? actx->res_spad_base : (actx->res_spad_base + vtcm_hvx_offset)) : NULL; \
\
- const size_t spad_half = actx->vtcm_size_per_thread / 2; \
+ const size_t spad_half = actx->vtcm_size_per_hvx / 2; \
uint32_t r_prefetch = r0; \
int spad_idx = 0; \
\
@@ -178,7 +179,7 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith,
dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, 0); \
} \
for (uint32_t s = 0; s < n_ranks; s++) { \
- uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + spad_idx * spad_half; \
+ uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + spad_idx * spad_half; \
const dma_addr_t s_ddr = octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \
dma_queue_push(dma_q, dma_make_data(s_spad, s_ddr), row_size_aligned, octx->src[s]->nb[1], row_bytes, cur_rows); \
} \
@@ -204,8 +205,8 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith,
dma_queue_pop(dma_q); \
} \
uint8_t * d_spad = dst_spad_base + comp_spad_idx * spad_half; \
- uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_thread_offset + comp_spad_idx * spad_half; \
- uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_thread_offset + comp_spad_idx * spad_half; \
+ uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_hvx_offset + comp_spad_idx * spad_half; \
+ uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_hvx_offset + comp_spad_idx * spad_half; \
uint8_t * r_spad = (HAS_ADD && !IS_ROW_BCAST) ? (res_spad_base + comp_spad_idx * spad_half) : NULL; \
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
for (uint32_t row = 0; row < cur_rows; row++) { \
@@ -214,7 +215,7 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith,
const uint8_t * s1_row = s1_spad + row * row_size_aligned; \
HVX_ADD_FN(d_row, s0_row, s1_row, ne0); \
for (uint32_t s = 2; s < n_ranks; s++) { \
- const uint8_t * ss_row = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half + row * row_size_aligned; \
+ const uint8_t * ss_row = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half + row * row_size_aligned; \
HVX_ADD_FN(d_row, d_row, ss_row, ne0); \
} \
if (HAS_ADD) { \
@@ -230,7 +231,7 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith,
if (r_prefetch < r1) { \
uint32_t next_rows = MIN(block_rows, r1 - r_prefetch); \
for (uint32_t s = 0; s < n_ranks; s++) { \
- uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \
+ uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half; \
const dma_addr_t s_next = octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \
dma_queue_push(dma_q, dma_make_data(s_spad, s_next), row_size_aligned, octx->src[s]->nb[1], row_bytes, next_rows); \
} \
@@ -259,19 +260,19 @@ static int validate_allreduce(
const struct htp_allreduce_kernel_params * kparams,
uint32_t n_ranks
) {
- if (kparams->n_threads == 0 || (uint32_t) kparams->n_threads > octx->ctx->n_threads) {
+ if (kparams->n_hvx == 0 || (uint32_t) kparams->n_hvx > octx->ctx->n_hvx) {
return HTP_STATUS_INVAL_PARAMS;
}
- octx->n_threads = (uint32_t) kparams->n_threads;
+ octx->n_hvx = (uint32_t) kparams->n_hvx;
- if (kparams->vtcm_size_per_thread <= 0 || kparams->vtcm_size <= 0) {
+ if (kparams->vtcm_size_per_hvx <= 0 || kparams->vtcm_size <= 0) {
return HTP_STATUS_INVAL_PARAMS;
}
const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD);
const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(
- n_ranks, octx->n_threads, has_add, kparams->is_row_bcast != 0);
- const size_t vtcm_size = n_vtcm_buffers * (size_t) kparams->vtcm_size_per_thread;
+ n_ranks, octx->n_hvx, has_add, kparams->is_row_bcast != 0);
+ const size_t vtcm_size = n_vtcm_buffers * (size_t) kparams->vtcm_size_per_hvx;
if (vtcm_size != (size_t) kparams->vtcm_size) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -360,10 +361,10 @@ int op_allreduce(struct htp_ops_context * octx) {
// 2. Multi-threaded Reduction across assigned rank chunk
if (nelem > 0) {
- const uint32_t n_threads = (uint32_t) kparams->n_threads;
+ const uint32_t n_hvx = (uint32_t) kparams->n_hvx;
const uint32_t block_elems = (uint32_t) kparams->block_elems;
- const uint32_t elems_per_thread = (uint32_t) kparams->elems_per_thread;
- const uint32_t vtcm_size_per_thread = (uint32_t) kparams->vtcm_size_per_thread;
+ const uint32_t elems_per_hvx = (uint32_t) kparams->elems_per_hvx;
+ const uint32_t vtcm_size_per_hvx = (uint32_t) kparams->vtcm_size_per_hvx;
struct htp_allreduce_context actx;
actx.octx = octx;
@@ -375,9 +376,9 @@ int op_allreduce(struct htp_ops_context * octx) {
actx.row_size_aligned = (uint32_t) kparams->row_size_aligned;
actx.rank_elem_start = (uint32_t) kparams->rank_elem_start;
actx.rank_nelem = (uint32_t) kparams->rank_nelem;
- actx.elems_per_thread = elems_per_thread;
+ actx.elems_per_hvx = elems_per_hvx;
actx.block_elems = block_elems;
- actx.vtcm_size_per_thread = vtcm_size_per_thread;
+ actx.vtcm_size_per_hvx = vtcm_size_per_hvx;
actx.is_row_bcast = (kparams->is_row_bcast != 0);
work_queue_func_t reduce_fun = NULL;
@@ -409,13 +410,13 @@ int op_allreduce(struct htp_ops_context * octx) {
uint8_t * vtcm_ptr = (uint8_t *) octx->ctx->vtcm_base;
for (uint32_t s = 0; s < n_ranks; s++) {
actx.src_spad_base[s] = vtcm_ptr;
- vtcm_ptr += n_threads * vtcm_size_per_thread;
+ vtcm_ptr += n_hvx * vtcm_size_per_hvx;
}
actx.dst_spad_base = vtcm_ptr;
- vtcm_ptr += n_threads * vtcm_size_per_thread;
+ vtcm_ptr += n_hvx * vtcm_size_per_hvx;
if (has_add) {
actx.res_spad_base = vtcm_ptr;
- vtcm_ptr += (actx.is_row_bcast ? 1 : n_threads) * vtcm_size_per_thread;
+ vtcm_ptr += (actx.is_row_bcast ? 1 : n_hvx) * vtcm_size_per_hvx;
}
if (has_add && actx.is_row_bcast) {
@@ -426,7 +427,7 @@ int op_allreduce(struct htp_ops_context * octx) {
dma_queue_pop(dma_q);
}
- work_queue_run(octx->ctx->work_queue, reduce_fun, &actx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, reduce_fun, &actx, n_hvx);
}
// 4. Exit Barrier: Synchronize all ranks after writing
diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.h b/ggml/src/ggml-hexagon/htp/allreduce-ops.h
index 1264c1cbb..1fb486cb6 100644
--- a/ggml/src/ggml-hexagon/htp/allreduce-ops.h
+++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.h
@@ -24,21 +24,21 @@ enum htp_allreduce_mode {
static inline size_t htp_allreduce_vtcm_buffer_count(
uint32_t n_ranks,
- uint32_t n_threads,
+ uint32_t n_hvx,
bool has_add,
bool is_row_bcast
) {
- return (size_t) (n_ranks + 1) * n_threads + (has_add ? (is_row_bcast ? 1 : n_threads) : 0);
+ return (size_t) (n_ranks + 1) * n_hvx + (has_add ? (is_row_bcast ? 1 : n_hvx) : 0);
}
struct htp_allreduce_kernel_params {
uint8_t kernel_type;
int32_t rank;
int32_t n_ranks;
- int32_t n_threads;
+ int32_t n_hvx;
int32_t block_elems; // 1D: block_elems, 2D: block_rows
- int32_t elems_per_thread; // 1D: nelem_per_thread, 2D: nrows_per_thread
- int32_t vtcm_size_per_thread;
+ int32_t elems_per_hvx; // 1D: nelem_per_hvx, 2D: nrows_per_hvx
+ int32_t vtcm_size_per_hvx;
int32_t vtcm_size;
int32_t ne0;
int32_t ne1;
diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.c b/ggml/src/ggml-hexagon/htp/argsort-ops.c
index 9351a73f2..af2605954 100644
--- a/ggml/src/ggml-hexagon/htp/argsort-ops.c
+++ b/ggml/src/ggml-hexagon/htp/argsort-ops.c
@@ -463,10 +463,10 @@ struct htp_sort_chunk_ctx {
size_t phase1_slot_size;
float * merge_values;
int32_t * merge_indices;
- uint32_t real_count[HTP_MAX_NTHREADS];
+ uint32_t real_count[HTP_OP_MAX_NHVX];
};
-static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) {
+static void htp_sort_chunk_job(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
struct htp_sort_chunk_ctx * cctx = (struct htp_sort_chunk_ctx *) data;
struct htp_ops_context * octx = cctx->octx;
const struct htp_sort_kernel_params * kparams = cctx->kparams;
@@ -487,7 +487,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) {
float * merge_values = cctx->merge_values;
int32_t * merge_indices = cctx->merge_indices;
- struct htp_thread_trace * tr = &octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
const dma_addr_t src_addr = src0->data + (size_t) chunk_base * sizeof(float);
if (kparams->is_top_k) {
@@ -503,7 +503,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) {
}
if (local_k == 1 && ne00 >= 128*1024) {
- dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float));
+ dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float));
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
float max_value;
@@ -515,7 +515,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) {
}
if (local_k > 1 && local_k <= 64 && chunk_elems > 1024) {
- dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float));
+ dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float));
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
top_k_scan(values_buf, real_count, local_k, &merge_values[i * local_k], &merge_indices[i * local_k]);
@@ -526,7 +526,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) {
return;
}
- dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float));
+ dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float));
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
if (chunk_elems > real_count) {
@@ -542,7 +542,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) {
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
} else {
if (real_count > 0) {
- dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float));
+ dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float));
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
init_indices_ramp(indices_buf, real_count, chunk_base);
@@ -562,7 +562,7 @@ struct htp_argsort_merge_l1_ctx {
bool asc;
};
-static void htp_argsort_merge_l1_job(unsigned int n, unsigned int i, void * data) {
+static void htp_argsort_merge_l1_job(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
struct htp_argsort_merge_l1_ctx * mctx = (struct htp_argsort_merge_l1_ctx *) data;
struct htp_sort_chunk_ctx * cctx = mctx->cctx;
const struct htp_sort_kernel_params * kparams = cctx->kparams;
@@ -585,7 +585,7 @@ static void htp_argsort_merge_l1_job(unsigned int n, unsigned int i, void * data
float * out_val = cctx->merge_values + out_offset;
int32_t * out_idx = cctx->merge_indices + out_offset;
- struct htp_thread_trace * tr = &cctx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
merge_runs(val0, idx0, count0, val1, idx1, count1, out_val, out_idx, mctx->asc);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
@@ -596,7 +596,7 @@ struct htp_argsort_merge_l2_ctx {
bool asc;
};
-static void htp_argsort_merge_l2_job(unsigned int n, unsigned int i, void * data) {
+static void htp_argsort_merge_l2_job(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
struct htp_argsort_merge_l2_ctx * mctx = (struct htp_argsort_merge_l2_ctx *) data;
struct htp_sort_chunk_ctx * cctx = mctx->cctx;
const struct htp_sort_kernel_params * kparams = cctx->kparams;
@@ -619,7 +619,7 @@ static void htp_argsort_merge_l2_job(unsigned int n, unsigned int i, void * data
float * out_val = (float *) slot_base;
int32_t * out_idx = (int32_t *) (slot_base + 4 * chunk_elems * sizeof(float));
- struct htp_thread_trace * tr = &cctx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
merge_runs(val0, idx0, count0, val1, idx1, count1, out_val, out_idx, mctx->asc);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
@@ -632,7 +632,7 @@ struct htp_sort_multi_row_ctx {
size_t phase1_slot_size;
uint32_t row_start;
uint32_t row_end;
- uint32_t nrows_per_thread;
+ uint32_t nrows_per_hvx;
};
struct htp_tensor_row_coord {
@@ -721,7 +721,7 @@ static inline void compute_row_sort(
}
}
-static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data) {
+static void htp_sort_multi_row_job(const htp_thread_t * th, unsigned int n, unsigned int ith, void * data) {
struct htp_sort_multi_row_ctx * mctx = (struct htp_sort_multi_row_ctx *) data;
struct htp_ops_context * octx = mctx->octx;
const struct htp_sort_kernel_params * kparams = mctx->kparams;
@@ -742,21 +742,21 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data
uint32_t row_start = mctx->row_start;
uint32_t row_end = mctx->row_end;
- uint32_t r_start = row_start + ith * mctx->nrows_per_thread;
- uint32_t r_end = MIN(r_start + mctx->nrows_per_thread, row_end);
+ uint32_t r_start = row_start + ith * mctx->nrows_per_hvx;
+ uint32_t r_end = MIN(r_start + mctx->nrows_per_hvx, row_end);
if (r_start >= r_end) return;
size_t slot_size = mctx->phase1_slot_size;
size_t spad_offset = ith * slot_size * (size_t) kparams->n_slots;
- uint8_t * thread_spad = mctx->phase1_base + spad_offset;
+ uint8_t * hvx_spad = mctx->phase1_base + spad_offset;
uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;
- float * values_buf = (float *) thread_spad;
- int32_t * indices_buf = (int32_t *) (thread_spad + chunk_elems * sizeof(float));
+ float * values_buf = (float *) hvx_spad;
+ int32_t * indices_buf = (int32_t *) (hvx_spad + chunk_elems * sizeof(float));
- dma_queue * dma_q = octx->ctx->dma[ith];
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ dma_queue * dma_q = th->dma_q;
+ struct htp_thread_trace * tr = th->trace;
bool is_top_k = kparams->is_top_k != 0;
uint32_t k = dst->ne[0];
@@ -768,7 +768,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data
if (kparams->n_slots == 2) {
struct htp_tensor_row_coord prime_coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02);
for (uint32_t r = r_start, spad_idx = 0; r < r_end && spad_idx < 2; r++, spad_idx++) {
- uint8_t * cur_spad = thread_spad + spad_idx * slot_size;
+ uint8_t * cur_spad = hvx_spad + spad_idx * slot_size;
float * cur_values = (float *) cur_spad;
int32_t * cur_indices = (int32_t *) (cur_spad + chunk_elems * sizeof(float));
@@ -791,7 +791,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data
for (uint32_t r = r_start; r < r_end; r++) {
uint32_t cur_slot = (r - r_start) & 1;
- uint8_t * cur_spad = thread_spad + cur_slot * slot_size;
+ uint8_t * cur_spad = hvx_spad + cur_slot * slot_size;
float * cur_values = (float *) cur_spad;
int32_t * cur_indices = (int32_t *) (cur_spad + chunk_elems * sizeof(float));
@@ -876,7 +876,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {
const struct htp_sort_kernel_params * kparams = (const struct htp_sort_kernel_params *) octx->kernel_params;
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -886,7 +886,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {
(uint32_t) kparams->chunk_elems,
(uint32_t) kparams->n_chunks,
(uint32_t) kparams->merge_elems,
- (uint32_t) kparams->n_threads,
+ (uint32_t) kparams->n_hvx,
(uint32_t) kparams->n_slots);
if (octx->ctx->vtcm_size < layout.total_bytes) {
@@ -909,7 +909,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {
cctx.merge_indices = merge_indices;
memset(cctx.real_count, 0, sizeof(cctx.real_count));
- work_queue_run(octx->ctx->work_queue, htp_sort_chunk_job, &cctx, (uint32_t) kparams->n_chunks);
+ work_queue_run(octx->ctx->hvx_queue, htp_sort_chunk_job, &cctx, (uint32_t) kparams->n_chunks);
if (is_top_k) {
uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;
@@ -952,10 +952,10 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {
bool asc = kparams->order == GGML_SORT_ORDER_ASC;
if (kparams->n_chunks == 8) {
struct htp_argsort_merge_l1_ctx l1ctx = { &cctx, asc };
- work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l1_job, &l1ctx, 4);
+ work_queue_run(octx->ctx->hvx_queue, htp_argsort_merge_l1_job, &l1ctx, 4);
struct htp_argsort_merge_l2_ctx l2ctx = { &cctx, asc };
- work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l2_job, &l2ctx, 2);
+ work_queue_run(octx->ctx->hvx_queue, htp_argsort_merge_l2_job, &l2ctx, 2);
size_t slot_size = layout.phase1_slot_size;
uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;
@@ -976,7 +976,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {
dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t));
} else if (kparams->n_chunks == 4) {
struct htp_argsort_merge_l1_ctx l1ctx = { &cctx, asc };
- work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l1_job, &l1ctx, 2);
+ work_queue_run(octx->ctx->hvx_queue, htp_argsort_merge_l1_job, &l1ctx, 2);
uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;
uint32_t n01 = cctx.real_count[0] + cctx.real_count[1];
@@ -1031,13 +1031,13 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {
if (rows_per_chunk == 0) {
rows_per_chunk = 1;
}
- uint32_t dr = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div);
+ uint32_t dr = fastdiv(nrows + octx->n_hvx - 1, &octx->n_hvx_div);
if (rows_per_chunk > 1) {
dr = hex_round_up(dr, rows_per_chunk);
}
- mctx.nrows_per_thread = dr;
+ mctx.nrows_per_hvx = dr;
- work_queue_run(octx->ctx->work_queue, htp_sort_multi_row_job, &mctx, octx->n_threads);
+ work_queue_run(octx->ctx->hvx_queue, htp_sort_multi_row_job, &mctx, octx->n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.h b/ggml/src/ggml-hexagon/htp/argsort-ops.h
index f74189fba..d1cb8087b 100644
--- a/ggml/src/ggml-hexagon/htp/argsort-ops.h
+++ b/ggml/src/ggml-hexagon/htp/argsort-ops.h
@@ -10,7 +10,7 @@
#include "htp-vtcm.h"
struct htp_sort_kernel_params {
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t order; // GGML_SORT_ORDER_ASC (0) or GGML_SORT_ORDER_DESC (1)
uint32_t is_top_k; // 1 if TOP_K, 0 if ARGSORT
uint32_t chunk_elems;
@@ -31,7 +31,7 @@ struct htp_sort_vtcm_layout {
uint32_t chunk_elems;
uint32_t n_chunks;
uint32_t merge_elems;
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t n_slots;
};
@@ -41,14 +41,14 @@ static inline void htp_sort_vtcm_layout_build(
uint32_t chunk_elems,
uint32_t n_chunks,
uint32_t merge_elems,
- uint32_t n_threads,
+ uint32_t n_hvx,
uint32_t n_slots) {
memset(L, 0, sizeof(*L));
L->chunk_elems = chunk_elems;
L->n_chunks = n_chunks;
L->merge_elems = merge_elems;
- L->n_threads = n_threads;
+ L->n_hvx = n_hvx;
L->n_slots = n_slots;
const size_t values_size = ((chunk_elems * sizeof(float)) + 127) & ~127;
@@ -57,7 +57,7 @@ static inline void htp_sort_vtcm_layout_build(
size_t off = 0;
if (total_rows > 1) {
- VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_slots * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_slots * n_hvx);
} else {
VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_chunks * n_slots);
const size_t merge_values_size = ((merge_elems * sizeof(float)) + 127) & ~127;
@@ -73,14 +73,14 @@ static inline bool htp_sort_solve_layout(
uint32_t ne00,
uint32_t total_rows,
uint32_t k,
- uint32_t n_threads_max,
+ uint32_t n_hvx_max,
size_t vtcm_budget,
bool is_top_k) {
memset(layout, 0, sizeof(*layout));
if (total_rows > 1) {
- uint32_t n_threads = total_rows < n_threads_max ? total_rows : n_threads_max;
+ uint32_t n_hvx = total_rows < n_hvx_max ? total_rows : n_hvx_max;
uint32_t n_vec = (ne00 + 31) / 32;
uint32_t n_vec_pow2 = 1;
@@ -96,13 +96,13 @@ static inline bool htp_sort_solve_layout(
n_slots = 1;
}
- size_t spad_per_thread = spad_per_slot * n_slots;
+ size_t spad_per_hvx = spad_per_slot * n_slots;
- while (n_threads > 1 && (spad_per_thread * n_threads) > vtcm_budget) {
- n_threads--;
+ while (n_hvx > 1 && (spad_per_hvx * n_hvx) > vtcm_budget) {
+ n_hvx--;
}
- htp_sort_vtcm_layout_build(layout, total_rows, ne00_padded, 1, 0, n_threads, n_slots);
+ htp_sort_vtcm_layout_build(layout, total_rows, ne00_padded, 1, 0, n_hvx, n_slots);
return layout->total_bytes <= vtcm_budget;
}
@@ -112,7 +112,7 @@ static inline bool htp_sort_solve_layout(
uint32_t n_chunks = 1;
if (ne00 > 1024) {
- while (n_chunks * 2 <= n_threads_max && n_chunks * 2 <= n_vec_pow2) {
+ while (n_chunks * 2 <= n_hvx_max && n_chunks * 2 <= n_vec_pow2) {
n_chunks *= 2;
}
}
diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.c
index 48f461e46..a5951556b 100644
--- a/ggml/src/ggml-hexagon/htp/binary-ops.c
+++ b/ggml/src/ggml-hexagon/htp/binary-ops.c
@@ -14,7 +14,7 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "binary-ops.h"
#include "htp-ctx.h"
#include "htp-ops.h"
@@ -39,7 +39,7 @@ struct htp_binary_context {
struct fastdiv_values src1_dim3_div; // ne13
uint32_t block_max;
- uint32_t nrows_per_thread;
+ uint32_t nrows_per_hvx;
uint32_t total_rows;
uint32_t row_start;
uint32_t n_srcs;
@@ -260,7 +260,7 @@ static void compute_add_id_f32(
}
// 1. Scalar src1 in VTCM via DMA (ne10 == 1)
-static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data) {
+static void binary_thread_scalar(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_binary_context * bctx = (struct htp_binary_context *) data;
struct htp_ops_context * octx = bctx->octx;
htp_binary_preamble;
@@ -270,21 +270,21 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data
const size_t row_stride = (size_t) ne11 * elem_size;
const uint32_t row_size_bytes = bctx->row_size_bytes;
- const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;
- const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);
+ const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith;
+ const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows);
if (start_row >= end_row) return;
FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)",
ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);
const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout;
- uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread);
- uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);
+ uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx);
+ uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx);
size_t src0_spad_half = layout->src0_spad_half_size;
size_t dst_spad_half = layout->dst_spad_half_size;
const void * s1_table = VTCM_LAYOUT_PTR(const void, bctx->vtcm_base, layout->off_src1);
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
uint32_t ir_prefetch = start_row;
int spad_idx = 0;
@@ -309,7 +309,7 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data
spad_idx ^= 1;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
compute_scalar_t compute = (compute_scalar_t) bctx->compute;
for (uint32_t ir = start_row; ir < end_row; ) {
@@ -358,29 +358,29 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data
}
// 2. Vector Same Shape (ne1x == ne0x) or Simple Broadcast
-static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, void * data) {
+static void binary_thread_vector_same_shape(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_binary_context * bctx = (struct htp_binary_context *) data;
struct htp_ops_context * octx = bctx->octx;
htp_binary_preamble;
const uint32_t row_size_bytes = bctx->row_size_bytes;
- const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;
- const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);
+ const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith;
+ const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows);
if (start_row >= end_row) return;
FARF(HIGH, "binary-same-shape: %d/%d (%u:%u) row-size %u (%u)",
ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);
const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout;
- uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread);
- uint8_t * src1_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_thread);
- uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);
+ uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx);
+ uint8_t * src1_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_hvx);
+ uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx);
size_t src0_spad_half = layout->src0_spad_half_size;
size_t src1_spad_half = layout->src1_spad_half_size;
size_t dst_spad_half = layout->dst_spad_half_size;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
const uint32_t n_srcs = bctx->n_srcs;
uint32_t ir_prefetch = start_row;
int spad_idx = 0;
@@ -415,7 +415,7 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith,
spad_idx ^= 1;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
compute_same_shape_t compute = (compute_same_shape_t) bctx->compute;
for (uint32_t ir = start_row; ir < end_row; ) {
@@ -472,27 +472,27 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith,
}
// 3. Row Broadcast (ne11 == 1, ne12 == 1, single row src1)
-static void binary_thread_vector_row_broadcast(unsigned int nth, unsigned int ith, void * data) {
+static void binary_thread_vector_row_broadcast(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_binary_context * bctx = (struct htp_binary_context *) data;
struct htp_ops_context * octx = bctx->octx;
htp_binary_preamble;
const uint32_t row_size_bytes = bctx->row_size_bytes;
- const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;
- const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);
+ const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith;
+ const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows);
if (start_row >= end_row) return;
FARF(HIGH, "binary-row-bcast: %d/%d (%u:%u) row-size %u (%u)",
ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);
const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout;
- uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread);
- uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);
+ uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx);
+ uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx);
size_t src0_spad_half = layout->src0_spad_half_size;
size_t dst_spad_half = layout->dst_spad_half_size;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
uint32_t ir_prefetch = start_row;
int spad_idx = 0;
@@ -517,7 +517,7 @@ static void binary_thread_vector_row_broadcast(unsigned int nth, unsigned int it
spad_idx ^= 1;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
compute_row_bcast_t compute = (compute_row_bcast_t) bctx->compute;
for (uint32_t ir = start_row; ir < end_row; ) {
@@ -555,22 +555,22 @@ static void binary_thread_vector_row_broadcast(unsigned int nth, unsigned int it
}
// 4. Vector Complex (ne10 == ne00, complex broadcast)
-static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, void * data) {
+static void binary_thread_vector_complex(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_binary_context * bctx = (struct htp_binary_context *) data;
struct htp_ops_context * octx = bctx->octx;
htp_binary_preamble;
const uint32_t row_size_bytes = bctx->row_size_bytes;
- const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;
- const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);
+ const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith;
+ const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows);
if (start_row >= end_row) return;
FARF(HIGH, "binary-complex: %d/%d (%u:%u) row-size %u (%u)",
ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);
const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout;
- uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread);
- uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);
+ uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx);
+ uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx);
size_t src0_spad_half = layout->src0_spad_half_size;
size_t dst_spad_half = layout->dst_spad_half_size;
const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1);
@@ -580,7 +580,7 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi
const size_t vtcm_plane_stride = (size_t) ne11 * vtcm_row_stride;
const size_t vtcm_vol_stride = (size_t) ne12 * vtcm_plane_stride;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
uint32_t ir_prefetch = start_row;
int spad_idx = 0;
@@ -603,7 +603,7 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi
spad_idx ^= 1;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
compute_complex_t compute = (compute_complex_t) bctx->compute;
for (uint32_t ir = start_row; ir < end_row; ) {
@@ -646,19 +646,19 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi
}
// 5. Element Repeat (ne10 != ne00)
-static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, void * data) {
+static void binary_thread_element_repeat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_binary_context * bctx = (struct htp_binary_context *) data;
struct htp_ops_context * octx = bctx->octx;
htp_binary_preamble;
const uint32_t row_size_bytes = bctx->row_size_bytes;
- const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;
- const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);
+ const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith;
+ const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows);
if (start_row >= end_row) return;
const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout;
- uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread);
- uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);
+ uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx);
+ uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx);
size_t src0_spad_half = layout->src0_spad_half_size;
size_t dst_spad_half = layout->dst_spad_half_size;
const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1);
@@ -671,7 +671,7 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi
FARF(HIGH, "binary-repeat: %d/%d (%u:%u) row-size %u (%u)",
ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
uint32_t ir_prefetch = start_row;
int spad_idx = 0;
@@ -694,7 +694,7 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi
spad_idx ^= 1;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
compute_repeat_t compute = (compute_repeat_t) bctx->compute;
for (uint32_t ir = start_row; ir < end_row; ) {
@@ -737,7 +737,7 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi
}
// 6. ADD_ID (src1 gathered via src2 indices)
-static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * data) {
+static void binary_thread_add_id_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_binary_context * bctx = (struct htp_binary_context *) data;
struct htp_ops_context * octx = bctx->octx;
@@ -760,19 +760,19 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void *
const uint32_t nb3 = dst->nb[3];
const uint32_t row_size_bytes = bctx->row_size_bytes;
- const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;
- const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);
+ const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith;
+ const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows);
if (start_row >= end_row) return;
const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout;
- uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread);
- uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);
+ uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx);
+ uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx);
const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1);
const int32_t * vtcm_src2 = VTCM_LAYOUT_PTR(const int32_t, bctx->vtcm_base, layout->off_src2);
size_t src0_spad_half = layout->src0_spad_half_size;
size_t dst_spad_half = layout->dst_spad_half_size;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
uint32_t ir_prefetch = start_row;
int spad_idx = 0;
@@ -795,7 +795,7 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void *
spad_idx ^= 1;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
for (uint32_t ir = start_row; ir < end_row; ) {
uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);
@@ -870,7 +870,7 @@ struct binary_chunked_context {
uint32_t elem_start;
uint32_t nelem;
uint32_t chunk_size;
- uint32_t chunks_per_thread;
+ uint32_t chunks_per_hvx;
uint32_t total_chunks;
uint32_t n_srcs;
bool is_scalar;
@@ -881,15 +881,15 @@ struct binary_chunked_context {
compute_binary_scalar_chunked_f16_t compute_scalar_f16;
};
-static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * data) {
+static void binary_thread_chunked(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct binary_chunked_context * ctx = (struct binary_chunked_context *) data;
struct htp_ops_context * octx = ctx->octx;
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
- const uint32_t start_chunk = ctx->chunks_per_thread * ith;
- const uint32_t end_chunk = MIN(start_chunk + ctx->chunks_per_thread, ctx->total_chunks);
+ const uint32_t start_chunk = ctx->chunks_per_hvx * ith;
+ const uint32_t end_chunk = MIN(start_chunk + ctx->chunks_per_hvx, ctx->total_chunks);
if (start_chunk >= end_chunk) {
return;
}
@@ -903,11 +903,11 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat
ith, nth, start_chunk, end_chunk, ctx->total_chunks, ctx->nelem);
const struct htp_binary_vtcm_layout * layout = &ctx->vtcm_layout;
- uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread);
- uint8_t * src1_spad_base = ctx->is_scalar ? NULL : (VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_thread));
- uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);
+ uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx);
+ uint8_t * src1_spad_base = ctx->is_scalar ? NULL : (VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_hvx));
+ uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx);
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
const uint32_t n_srcs = ctx->n_srcs;
uint32_t prefetch_chunk = start_chunk;
int spad_idx = 0;
@@ -939,7 +939,7 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat
spad_idx ^= 1;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
for (uint32_t c = start_chunk; c < end_chunk; c++) {
const uint32_t c_start = ctx->elem_start + c * chunk_size;
@@ -1091,7 +1091,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -1103,8 +1103,8 @@ static int execute_op_binary(struct htp_ops_context * octx) {
const uint32_t chunk_size = kparams->chunk_size;
const uint32_t total_chunks = (nelem + chunk_size - 1) / chunk_size;
- const uint32_t n_threads = (total_chunks >= 2) ? MIN(octx->n_threads, total_chunks) : 1;
- const uint32_t chunks_per_thread = (total_chunks + n_threads - 1) / n_threads;
+ const uint32_t n_hvx = (total_chunks >= 2) ? MIN(octx->n_hvx, total_chunks) : 1;
+ const uint32_t chunks_per_hvx = (total_chunks + n_hvx - 1) / n_hvx;
float scalar_f32 = 0.0f;
_Float16 scalar_f16 = 0;
@@ -1128,7 +1128,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {
.elem_start = elem_start,
.nelem = nelem,
.chunk_size = chunk_size,
- .chunks_per_thread = chunks_per_thread,
+ .chunks_per_hvx = chunks_per_hvx,
.total_chunks = total_chunks,
.n_srcs = n_srcs,
.is_scalar = is_scalar,
@@ -1139,7 +1139,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {
.compute_scalar_f16 = compute_scalar_f16,
};
- work_queue_run(octx->ctx->work_queue, binary_thread_chunked, &cctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, binary_thread_chunked, &cctx, n_hvx);
return HTP_STATUS_OK;
}
@@ -1159,11 +1159,11 @@ static int execute_op_binary(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
const size_t src0_row_size_aligned = kparams->src0_row_size_aligned;
const size_t src1_row_size_aligned = kparams->src1_row_size_aligned;
const size_t dst_row_size_aligned = kparams->dst_row_size_aligned;
@@ -1176,7 +1176,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {
return HTP_STATUS_VTCM_TOO_SMALL;
}
- worker_callback_t worker_func = NULL;
+ work_queue_func_t worker_func = NULL;
void * compute_func = NULL;
switch (kparams->kernel_type) {
@@ -1336,7 +1336,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {
}
bctx.octx = octx;
- bctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
+ bctx.nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div);
bctx.total_rows = nrows;
bctx.row_start = row_start;
bctx.n_srcs = n_srcs;
@@ -1375,7 +1375,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {
}
}
- work_queue_run(octx->ctx->work_queue, worker_func, &bctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, worker_func, &bctx, n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.h b/ggml/src/ggml-hexagon/htp/binary-ops.h
index 8c5e683e8..6f849396c 100644
--- a/ggml/src/ggml-hexagon/htp/binary-ops.h
+++ b/ggml/src/ggml-hexagon/htp/binary-ops.h
@@ -30,7 +30,7 @@ struct htp_binary_kernel_params {
uint8_t is_scalar;
uint8_t pad[2];
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t rows_per_buffer;
uint32_t src0_row_size_aligned;
@@ -86,9 +86,9 @@ struct htp_binary_vtcm_layout {
size_t off_src2;
size_t off_dst;
- size_t src0_bytes_per_thread;
- size_t src1_bytes_per_thread;
- size_t dst_bytes_per_thread;
+ size_t src0_bytes_per_hvx;
+ size_t src1_bytes_per_hvx;
+ size_t dst_bytes_per_hvx;
size_t src0_spad_half_size;
size_t src1_spad_half_size;
@@ -106,8 +106,8 @@ static inline void htp_binary_vtcm_layout_build(
) {
memset(L, 0, sizeof(*L));
- const uint32_t n_threads = kparams->n_threads;
- if (n_threads == 0) {
+ const uint32_t n_hvx = kparams->n_hvx;
+ if (n_hvx == 0) {
return;
}
@@ -119,9 +119,9 @@ static inline void htp_binary_vtcm_layout_build(
return;
}
- L->src0_bytes_per_thread = 2 * chunk_bytes;
- L->src1_bytes_per_thread = kparams->is_scalar ? 0 : (2 * (n_srcs - 1) * chunk_bytes);
- L->dst_bytes_per_thread = 2 * chunk_bytes;
+ L->src0_bytes_per_hvx = 2 * chunk_bytes;
+ L->src1_bytes_per_hvx = kparams->is_scalar ? 0 : (2 * (n_srcs - 1) * chunk_bytes);
+ L->dst_bytes_per_hvx = 2 * chunk_bytes;
L->src0_spad_half_size = chunk_bytes;
L->src1_spad_half_size = kparams->is_scalar ? 0 : chunk_bytes;
@@ -130,9 +130,9 @@ static inline void htp_binary_vtcm_layout_build(
L->rows_per_buffer = 1;
L->src1_size = 0;
- const size_t src0_total = n_threads * L->src0_bytes_per_thread;
- const size_t src1_total = kparams->is_scalar ? 128 : (n_threads * L->src1_bytes_per_thread);
- const size_t dst_total = n_threads * L->dst_bytes_per_thread;
+ const size_t src0_total = n_hvx * L->src0_bytes_per_hvx;
+ const size_t src1_total = kparams->is_scalar ? 128 : (n_hvx * L->src1_bytes_per_hvx);
+ const size_t dst_total = n_hvx * L->dst_bytes_per_hvx;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src0, src0_total);
@@ -158,7 +158,7 @@ static inline void htp_binary_vtcm_layout_build(
const size_t rows_per_buffer = (kparams->rows_per_buffer > 0)
? kparams->rows_per_buffer
- : (vtcm_size - static_vtcm_size) / (n_threads * spad_row_total);
+ : (vtcm_size - static_vtcm_size) / (n_hvx * spad_row_total);
if (rows_per_buffer == 0) {
return;
}
@@ -167,24 +167,24 @@ static inline void htp_binary_vtcm_layout_build(
L->src1_size = kparams->src1_size;
L->src2_size = kparams->src2_size;
- L->src0_bytes_per_thread = rows_per_buffer * 2 * kparams->src0_row_size_aligned;
- L->dst_bytes_per_thread = rows_per_buffer * 2 * kparams->dst_row_size_aligned;
- L->src1_bytes_per_thread = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE)
+ L->src0_bytes_per_hvx = rows_per_buffer * 2 * kparams->src0_row_size_aligned;
+ L->dst_bytes_per_hvx = rows_per_buffer * 2 * kparams->dst_row_size_aligned;
+ L->src1_bytes_per_hvx = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE)
? rows_per_buffer * 2 * (n_srcs - 1) * kparams->src1_row_size_aligned
: 0;
- L->src0_spad_half_size = L->src0_bytes_per_thread / 2;
+ L->src0_spad_half_size = L->src0_bytes_per_hvx / 2;
L->src1_spad_half_size = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE)
? rows_per_buffer * kparams->src1_row_size_aligned
: 0;
- L->dst_spad_half_size = L->dst_bytes_per_thread / 2;
+ L->dst_spad_half_size = L->dst_bytes_per_hvx / 2;
- const size_t src0_total = n_threads * L->src0_bytes_per_thread;
+ const size_t src0_total = n_hvx * L->src0_bytes_per_hvx;
const size_t src1_total = (kparams->src1_size > 0)
? kparams->src1_size
- : n_threads * L->src1_bytes_per_thread;
+ : n_hvx * L->src1_bytes_per_hvx;
const size_t src2_total = kparams->src2_size;
- const size_t dst_total = n_threads * L->dst_bytes_per_thread;
+ const size_t dst_total = n_hvx * L->dst_bytes_per_hvx;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src0, src0_total);
@@ -200,15 +200,15 @@ static inline bool htp_binary_solve_chunked(
struct htp_binary_kernel_params * kparams,
uint32_t total_elems,
uint32_t elem_size,
- uint32_t max_threads,
+ uint32_t max_hvx,
uint32_t n_srcs,
bool is_scalar,
size_t vtcm_size
) {
const size_t n_bufs = is_scalar ? 2 : (size_t) n_srcs + 1;
const size_t scalar_bytes = is_scalar ? 128 : 0;
- const size_t budget_per_buf = (max_threads > 0 && vtcm_size > scalar_bytes)
- ? hex_align_down((vtcm_size - scalar_bytes) / (2 * max_threads * n_bufs), 128)
+ const size_t budget_per_buf = (max_hvx > 0 && vtcm_size > scalar_bytes)
+ ? hex_align_down((vtcm_size - scalar_bytes) / (2 * max_hvx * n_bufs), 128)
: 0;
const uint32_t max_chunk_bytes = (budget_per_buf > 0 && budget_per_buf < 32768) ? (uint32_t) budget_per_buf : 32768u;
const uint32_t max_chunk_elems = max_chunk_bytes / elem_size;
@@ -217,16 +217,16 @@ static inline bool htp_binary_solve_chunked(
}
const uint32_t min_chunk_elems = 256 < max_chunk_elems ? 256 : max_chunk_elems;
- const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_threads) - 1) / (2 * max_threads), 32);
+ const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_hvx) - 1) / (2 * max_hvx), 32);
const uint32_t chunk_size = (target_chunk_elems < min_chunk_elems) ? min_chunk_elems
: (target_chunk_elems > max_chunk_elems) ? max_chunk_elems
: target_chunk_elems;
const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128);
const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size;
- const uint32_t n_threads = (total_chunks >= 2) ? (max_threads < total_chunks ? max_threads : total_chunks) : 1;
+ const uint32_t n_hvx = (total_chunks >= 2) ? (max_hvx < total_chunks ? max_hvx : total_chunks) : 1;
kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED;
- kparams->n_threads = n_threads;
+ kparams->n_hvx = n_hvx;
kparams->n_srcs = (uint8_t) n_srcs;
kparams->rows_per_buffer = 1;
kparams->src1_row_size_aligned = is_scalar ? 0 : kparams->src0_row_size_aligned;
diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c
index 8f3e199c2..a374a8fd6 100644
--- a/ggml/src/ggml-hexagon/htp/concat-ops.c
+++ b/ggml/src/ggml-hexagon/htp/concat-ops.c
@@ -3,7 +3,7 @@
#include "hex-common.h"
#include "dma-copy.h"
#include "hex-fastdiv.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "hexagon_protos.h"
#include "hexagon_types.h"
#include "htp-ctx.h"
@@ -21,11 +21,11 @@ struct htp_concat_context {
struct htp_ops_context * octx;
uint8_t * spad0_base;
uint8_t * spad1_base;
- uint32_t spad0_size_per_thread;
- uint32_t spad1_size_per_thread;
+ uint32_t spad0_size_per_hvx;
+ uint32_t spad1_size_per_hvx;
uint32_t row_start;
uint32_t nrows;
- uint32_t nrows_per_thread;
+ uint32_t nrows_per_hvx;
uint32_t nplanes;
struct fastdiv_values div_ne2;
};
@@ -36,7 +36,7 @@ static inline dma_addr_t concat_plane_addr(const struct htp_tensor * t, uint32_t
return t->data + i2 * t->nb[2] + i3 * t->nb[3];
}
-static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * data) {
+static void concat_2d_f32_transposed(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_concat_context * cctx = (struct htp_concat_context *) data;
struct htp_ops_context * octx = cctx->octx;
@@ -48,12 +48,12 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void *
const uint32_t src1_ne0 = src1->ne[0];
// dst rows [row_begin, row_end) of this thread; a dst row is a src0 row followed by a src1 column
- const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_thread;
- const uint32_t row_end = MIN(row_begin + cctx->nrows_per_thread, cctx->row_start + cctx->nrows);
+ const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_hvx;
+ const uint32_t row_end = MIN(row_begin + cctx->nrows_per_hvx, cctx->row_start + cctx->nrows);
if (row_begin >= row_end) return;
- dma_queue * dma_q = octx->ctx->dma[ith];
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ dma_queue * dma_q = th->dma_q;
+ struct htp_thread_trace * tr = th->trace;
const struct fastdiv_values * div_ne2 = &cctx->div_ne2;
const uint32_t ne2 = dst->ne[2];
@@ -72,8 +72,8 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void *
const uint32_t spad0_stride = src0_row_padded + block_cols_padded * sizeof(float);
// two slots per scratchpad: tile b gathers in slot b&1 while dst[b-1] and src1[b+1] DMA in the other
- uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread;
- uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread;
+ uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_hvx;
+ uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_hvx;
const uint32_t spad0_slot = block_rows * spad0_stride;
const uint32_t spad1_slot = block_cols_padded * spad1_stride;
@@ -153,7 +153,7 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void *
dma_queue_flush(dma_q);
}
-static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * data) {
+static void concat_2d_f16_transposed(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_concat_context * cctx = (struct htp_concat_context *) data;
struct htp_ops_context * octx = cctx->octx;
@@ -165,12 +165,12 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void *
const uint32_t src1_ne0 = src1->ne[0];
// dst rows [row_begin, row_end) of this thread; a dst row is a src0 row followed by a src1 column
- const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_thread;
- const uint32_t row_end = MIN(row_begin + cctx->nrows_per_thread, cctx->row_start + cctx->nrows);
+ const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_hvx;
+ const uint32_t row_end = MIN(row_begin + cctx->nrows_per_hvx, cctx->row_start + cctx->nrows);
if (row_begin >= row_end) return;
- dma_queue * dma_q = octx->ctx->dma[ith];
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ dma_queue * dma_q = th->dma_q;
+ struct htp_thread_trace * tr = th->trace;
const struct fastdiv_values * div_ne2 = &cctx->div_ne2;
const uint32_t ne2 = dst->ne[2];
@@ -189,8 +189,8 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void *
const uint32_t spad0_stride = src0_row_padded + block_cols_padded * sizeof(__fp16);
// two slots per scratchpad: tile b gathers in slot b&1 while dst[b-1] and src1[b+1] DMA in the other
- uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread;
- uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread;
+ uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_hvx;
+ uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_hvx;
const uint32_t spad0_slot = block_rows * spad0_stride;
const uint32_t spad1_slot = block_cols_padded * spad1_stride;
@@ -311,7 +311,7 @@ static int concat_regular(struct htp_ops_context * octx, int dim, uint32_t type_
}
static int concat_transposed(struct htp_ops_context * octx, const struct htp_concat_kernel_params * kparams, uint32_t type_size) {
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -334,10 +334,10 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con
return HTP_STATUS_VTCM_TOO_SMALL;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
struct htp_concat_transposed_vtcm_layout layout;
- htp_concat_transposed_vtcm_layout_build(&layout, octx->src[0]->ne[0], octx->src[1]->ne[0], type_size, n_threads);
+ htp_concat_transposed_vtcm_layout_build(&layout, octx->src[0]->ne[0], octx->src[1]->ne[0], type_size, n_hvx);
uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
@@ -345,16 +345,16 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con
cctx.octx = octx;
cctx.spad0_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad0);
cctx.spad1_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad1);
- cctx.spad0_size_per_thread = layout.src0_spad_size_per_thread;
- cctx.spad1_size_per_thread = layout.src1_spad_size_per_thread;
+ cctx.spad0_size_per_hvx = layout.src0_spad_size_per_hvx;
+ cctx.spad1_size_per_hvx = layout.src1_spad_size_per_hvx;
cctx.row_start = row_start;
cctx.nrows = nrows;
cctx.nplanes = dst->ne[2] * dst->ne[3];
cctx.div_ne2 = kparams->u.transposed.div_ne2;
- cctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
+ cctx.nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div);
work_queue_func_t worker_func = (type_size == 4) ? concat_2d_f32_transposed : concat_2d_f16_transposed;
- work_queue_run(octx->ctx->work_queue, worker_func, &cctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, worker_func, &cctx, n_hvx);
return HTP_STATUS_OK;
}
@@ -377,7 +377,7 @@ int op_concat(struct htp_ops_context * octx) {
break;
case HTP_CONCAT_KERNEL_GATHER_ROWS:
- status = htp_gather_rows(octx, src0, src1, &kparams->u.gather, kparams->n_threads);
+ status = htp_gather_rows(octx, src0, src1, &kparams->u.gather, kparams->n_hvx);
break;
default:
diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.h b/ggml/src/ggml-hexagon/htp/concat-ops.h
index 1f0744434..580fe0e15 100644
--- a/ggml/src/ggml-hexagon/htp/concat-ops.h
+++ b/ggml/src/ggml-hexagon/htp/concat-ops.h
@@ -21,7 +21,7 @@ struct htp_concat_transposed_params {
struct htp_concat_kernel_params {
uint8_t kernel_type;
uint8_t dim;
- uint8_t n_threads;
+ uint8_t n_hvx;
uint8_t pad;
uint32_t vtcm_size;
@@ -39,8 +39,8 @@ _Static_assert(sizeof(struct htp_concat_kernel_params) <= 128, "htp_concat_kerne
#endif
struct htp_concat_transposed_vtcm_layout {
- uint32_t src0_spad_size_per_thread;
- uint32_t src1_spad_size_per_thread;
+ uint32_t src0_spad_size_per_hvx;
+ uint32_t src1_spad_size_per_hvx;
size_t off_spad0;
size_t off_spad1;
uint32_t total_bytes;
@@ -51,7 +51,7 @@ static inline void htp_concat_transposed_vtcm_layout_build(
uint32_t src0_ne0,
uint32_t src1_ne0,
uint32_t type_size,
- uint32_t n_threads) {
+ uint32_t n_hvx) {
// src1_ne0 is processed in chunks of block_cols tokens, so VTCM does not grow with the batch
uint32_t block_rows = 128;
@@ -62,12 +62,12 @@ static inline void htp_concat_transposed_vtcm_layout_build(
uint32_t spad0_row_bytes = hex_round_up(src0_ne0 * type_size, 128) + block_cols_padded * type_size;
// double-buffered: two slots per thread
- L->src0_spad_size_per_thread = 2 * block_rows * spad0_row_bytes;
- L->src1_spad_size_per_thread = 2 * block_cols_padded * spad1_stride;
+ L->src0_spad_size_per_hvx = 2 * block_rows * spad0_row_bytes;
+ L->src1_spad_size_per_hvx = 2 * block_cols_padded * spad1_stride;
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_spad0, (size_t) n_threads * L->src0_spad_size_per_thread);
- VTCM_LAYOUT_ALLOC(off, off_spad1, (size_t) n_threads * L->src1_spad_size_per_thread);
+ VTCM_LAYOUT_ALLOC(off, off_spad0, (size_t) n_hvx * L->src0_spad_size_per_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_spad1, (size_t) n_hvx * L->src1_spad_size_per_hvx);
L->total_bytes = (uint32_t) off;
}
diff --git a/ggml/src/ggml-hexagon/htp/conv2d-ops.c b/ggml/src/ggml-hexagon/htp/conv2d-ops.c
index 6f35cb19b..67e8ec67b 100644
--- a/ggml/src/ggml-hexagon/htp/conv2d-ops.c
+++ b/ggml/src/ggml-hexagon/htp/conv2d-ops.c
@@ -12,7 +12,7 @@
#include "dma-queue.h"
#include "hex-common.h"
#include "hex-fastdiv.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
@@ -100,7 +100,7 @@ static void conv2d_mdev_rows(const struct htp_ops_context * octx, uint32_t total
}
}
-static void conv2d_hmx_worker(void * data) {
+static void conv2d_hmx_worker(const htp_thread_t * th, void * data) {
struct htp_conv2d_hmx_task * task = (struct htp_conv2d_hmx_task *) data;
const size_t dot_stride = task->n_dot_tiles * HMX_FP16_TILE_N_ELMS;
@@ -205,7 +205,7 @@ static inline uint32_t conv2d_hmx_chunk_range(const struct htp_conv2d_hmx_contex
return cs;
}
-static void conv2d_hmx_wpack_thread(unsigned int nth, unsigned int ith, void * data) {
+static void conv2d_hmx_wpack_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_conv2d_hmx_context * hc = (const struct htp_conv2d_hmx_context *) data;
struct htp_ops_context * octx = hc->octx;
const struct htp_conv2d_kernel_params * kp = hc->kp;
@@ -218,7 +218,7 @@ static void conv2d_hmx_wpack_thread(unsigned int nth, unsigned int ith, void * d
const bool is_f16 = kernel->type == HTP_TYPE_F16;
const uint32_t es = is_f16 ? sizeof(__fp16) : sizeof(float);
- dma_queue * dmaq = octx->ctx->dma[ith];
+ dma_queue * dmaq = th->dma_q;
uint8_t * thr = hc->thr_base + (size_t) ith * kp->thr_size;
const size_t w_buf_size = (size_t) WK * 32 * sizeof(__fp16);
const size_t w_raw_size = (size_t) WK * 32 * sizeof(float);
@@ -232,7 +232,7 @@ static void conv2d_hmx_wpack_thread(unsigned int nth, unsigned int ith, void * d
const HVX_Vector zero = Q6_V_vzero();
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
for (uint32_t rt = ith; rt < hc->n_rt; rt += nth) {
const uint32_t co0 = (hc->rt0 + rt) * 32;
@@ -704,7 +704,7 @@ static inline bool conv2d_wpack_kmajor_next_task(struct conv2d_wpack_kmajor_iter
}
// Kernel stored as [k][cout] (cout contiguous): DMA k rows of 64 cout, transpose into two row tiles
-static void conv2d_hmx_wpack_kmajor_thread(unsigned int nth, unsigned int ith, void * data) {
+static void conv2d_hmx_wpack_kmajor_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_conv2d_hmx_context * hc = (const struct htp_conv2d_hmx_context *) data;
struct htp_ops_context * octx = hc->octx;
const struct htp_conv2d_kernel_params * kp = hc->kp;
@@ -721,7 +721,7 @@ static void conv2d_hmx_wpack_kmajor_thread(unsigned int nth, unsigned int ith, v
const uint32_t es = is_f16 ? sizeof(__fp16) : sizeof(float);
const uint32_t n_cb = (hc->n_rt + 1) / 2;
- dma_queue * dmaq = octx->ctx->dma[ith];
+ dma_queue * dmaq = th->dma_q;
const size_t w_raw_size = (size_t) WK * 64 * sizeof(float);
uint8_t * wraws[2] = { hc->thr_base + (size_t) ith * kp->thr_size,
hc->thr_base + (size_t) ith * kp->thr_size + w_raw_size };
@@ -729,7 +729,7 @@ static void conv2d_hmx_wpack_kmajor_thread(unsigned int nth, unsigned int ith, v
HVX_Vector * scratch = vin + 32;
HVX_Vector * sink = scratch + 32;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
struct conv2d_wpack_kmajor_iter it = {
.cb = 0,
@@ -778,7 +778,7 @@ static void conv2d_hmx_wpack_kmajor_thread(unsigned int nth, unsigned int ith, v
}
}
-static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t ith) {
+static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, const htp_thread_t * th, uint32_t ith) {
struct htp_ops_context * octx = hc->octx;
const struct htp_conv2d_kernel_params * kp = hc->kp;
const struct htp_tensor * kernel = octx->src[0];
@@ -790,7 +790,7 @@ static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t i
const uint32_t d0 = (uint32_t) hc->g->d0;
const uint32_t n_vec = kp->tiled.n_k_tiles * (HMX_FP16_TILE_N_ELMS / 64);
- dma_queue * dmaq = octx->ctx->dma[ith];
+ dma_queue * dmaq = th->dma_q;
uint8_t * thr = hc->thr_base + (size_t) ith * kp->thr_size;
float * stages[2] = {
(float *) thr,
@@ -808,7 +808,7 @@ static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t i
uint32_t t0, t1;
const uint32_t cs = conv2d_hmx_chunk_range(hc, hc->pack_chunk, hc->pack_tpt, ith, &t0, &t1);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
if (t0 >= t1) {
return;
@@ -866,7 +866,7 @@ static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t i
}
}
-static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, uint32_t ith) {
+static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, const htp_thread_t * th, uint32_t ith) {
struct htp_ops_context * octx = hc->octx;
const struct htp_conv2d_kernel_params * kp = hc->kp;
const struct htp_tensor * dst = octx->dst;
@@ -877,7 +877,7 @@ static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, uint32_t
const uint32_t n_owt = kp->tiled.n_ow_tiles;
const uint32_t OS = kp->tiled.seg_tiles * 32;
- dma_queue * dmaq = octx->ctx->dma[ith];
+ dma_queue * dmaq = th->dma_q;
uint8_t * thr = hc->thr_base + (size_t) ith * kp->thr_size + (size_t) (kp->tiled.n_stage > 1 ? 2 : 1) * kp->stage_size;
float * ostage[2] = {
(float *) thr,
@@ -889,7 +889,7 @@ static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, uint32_t
const uint32_t cs = conv2d_hmx_chunk_range(hc, hc->unpack_chunk, hc->unpack_tpt, ith, &t0, &t1);
const uint32_t nct = MIN(cs + kp->ct_chunk, hc->t_end) - cs;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
if (t0 >= t1) {
return;
@@ -945,11 +945,11 @@ static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, uint32_t
}
}
-static void conv2d_hmx_phase_thread(unsigned int nth, unsigned int ith, void * data) {
+static void conv2d_hmx_phase_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_conv2d_hmx_context * hc = (const struct htp_conv2d_hmx_context *) data;
if (hc->unpack_chunk >= 0) {
- conv2d_hmx_unpack(hc, ith);
+ conv2d_hmx_unpack(hc, th, ith);
}
// weights of this rt block are packed before its first chunk
if (hc->pack_chunk == 0) {
@@ -960,7 +960,7 @@ static void conv2d_hmx_phase_thread(unsigned int nth, unsigned int ith, void * d
}
}
if (hc->pack_chunk >= 0) {
- conv2d_hmx_pack(hc, ith);
+ conv2d_hmx_pack(hc, th, ith);
}
}
@@ -978,7 +978,7 @@ static void conv2d_hmx_run_phase(struct htp_conv2d_hmx_context * hc, int32_t pac
hc->unpack_chunk = unpack;
hc->pack_tpt = conv2d_hmx_tpt(hc, pack, nth);
hc->unpack_tpt = conv2d_hmx_tpt(hc, unpack, nth);
- work_queue_run(hc->octx->ctx->work_queue, conv2d_hmx_phase_thread, hc, nth);
+ work_queue_run(hc->octx->ctx->hvx_queue, conv2d_hmx_phase_thread, hc, nth);
}
static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv2d_kernel_params * kp) {
@@ -997,7 +997,7 @@ static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv
const size_t out_size = (size_t) kp->rt_block * kp->ct_chunk * tile;
struct htp_conv2d_vtcm_layout layout;
- htp_conv2d_vtcm_layout_build(&layout, kp->rt_block, w_size, act_size, out_size, kp->thr_size, kp->n_threads);
+ htp_conv2d_vtcm_layout_build(&layout, kp->rt_block, w_size, act_size, out_size, kp->thr_size, kp->n_hvx);
uint8_t * base = (uint8_t *) octx->ctx->vtcm_base;
@@ -1015,7 +1015,7 @@ static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv
.t_end = (row_base + n_rows) * kp->tiled.n_ow_tiles,
};
- const uint32_t nth = kp->n_threads;
+ const uint32_t nth = kp->n_hvx;
const int32_t n_chunks = (int32_t) ((hc.t_end - hc.t_base + kp->ct_chunk - 1) / kp->ct_chunk);
struct htp_conv2d_hmx_task task;
@@ -1023,7 +1023,7 @@ static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv
for (uint32_t rt0 = 0; rt0 < kp->n_row_tiles; rt0 += kp->rt_block) {
hc.rt0 = rt0;
hc.n_rt = MIN(kp->rt_block, kp->n_row_tiles - rt0);
- work_queue_run(octx->ctx->work_queue, kp->w_kmajor ? conv2d_hmx_wpack_kmajor_thread : conv2d_hmx_wpack_thread, &hc, nth);
+ work_queue_run(octx->ctx->hvx_queue, kp->w_kmajor ? conv2d_hmx_wpack_kmajor_thread : conv2d_hmx_wpack_thread, &hc, nth);
conv2d_hmx_run_phase(&hc, 0, -1, nth);
@@ -1110,7 +1110,7 @@ static inline uint32_t conv2d_hmx_win_act_off(uint32_t u) {
return ((u >> 5) << 11) | ((u & 30) << 6) | ((u & 1) << 1);
}
-static void conv2d_hmx_win_worker(void * data) {
+static void conv2d_hmx_win_worker(const htp_thread_t * th, void * data) {
const struct htp_conv2d_win_task * j = (const struct htp_conv2d_win_task *) data;
const uint32_t * taps = j->taps;
@@ -1327,7 +1327,7 @@ static inline void conv2d_hmx_win_wpack_kmajor_compute(const struct htp_conv2d_w
}
// Cout-contiguous kernel: DMA w[t][c][32 cout] rows, tile vector p = (row 2p, row 2p+1) interleaved per cout
-static void conv2d_hmx_win_wpack_kmajor(const struct htp_conv2d_win_context * wc, unsigned int nth, unsigned int ith) {
+static void conv2d_hmx_win_wpack_kmajor(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) {
struct htp_ops_context * octx = wc->octx;
const struct htp_conv2d_kernel_params * kp = wc->kp;
const struct htp_tensor * kernel = octx->src[0];
@@ -1342,13 +1342,13 @@ static void conv2d_hmx_win_wpack_kmajor(const struct htp_conv2d_win_context * wc
const bool is_f16 = kernel->type == HTP_TYPE_F16;
const uint32_t es = is_f16 ? sizeof(__fp16) : sizeof(float);
- dma_queue * dmaq = octx->ctx->dma[ith];
+ dma_queue * dmaq = th->dma_q;
const size_t w_raw_size = (size_t) TG * 32 * VLEN;
uint8_t * raws[2] = { wc->thr_base + (size_t) ith * kp->thr_size,
wc->thr_base + (size_t) ith * kp->thr_size + w_raw_size };
const HVX_Vector zero = Q6_V_vzero();
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const uint32_t n_tg = (T + TG - 1) / TG;
const uint32_t n_units = wc->n_rt * n_cc * n_tg;
@@ -1491,7 +1491,7 @@ static void conv2d_hmx_win_wrows_pack(const struct htp_conv2d_win_context * wc,
}
}
-static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc, unsigned int nth, unsigned int ith) {
+static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) {
struct htp_ops_context * octx = wc->octx;
const struct htp_conv2d_kernel_params * kp = wc->kp;
const struct htp_tensor * kernel = octx->src[0];
@@ -1501,7 +1501,7 @@ static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc,
const uint32_t kpw = kernel->type == HTP_TYPE_F16 ? 64 : 32; // k per vector
const size_t wraw = (size_t) 32 * ((CS * T + kpw - 1) / kpw) * VLEN;
- dma_queue * dmaq = octx->ctx->dma[ith];
+ dma_queue * dmaq = th->dma_q;
uint8_t * raws[2] = { wc->thr_base + (size_t) ith * kp->thr_size, wc->thr_base + (size_t) ith * kp->thr_size + wraw };
uint8_t * xr = raws[1] + wraw;
HVX_Vector * vin = (HVX_Vector *) (xr + wraw);
@@ -1510,7 +1510,7 @@ static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc,
const uint32_t n_cs = (32 + CS - 1) / CS;
const uint32_t n_units = wc->n_rt * kp->window.n_cc * n_cs;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
// one DMA per unit: push(N), push(N + 1), pop(N), pack(N), push(N + 2), ...
for (uint32_t k = 0, unit = ith; k < 2 && unit < n_units; k++, unit += nth) {
@@ -1531,13 +1531,13 @@ static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc,
}
}
-static void conv2d_hmx_win_wpack_thread(unsigned int nth, unsigned int ith, void * data) {
+static void conv2d_hmx_win_wpack_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_conv2d_win_context * wc = (const struct htp_conv2d_win_context *) data;
if (wc->kp->w_kmajor) {
- conv2d_hmx_win_wpack_kmajor(wc, nth, ith);
+ conv2d_hmx_win_wpack_kmajor(wc, th, nth, ith);
} else {
- conv2d_hmx_win_wpack_rows(wc, nth, ith);
+ conv2d_hmx_win_wpack_rows(wc, th, nth, ith);
}
}
@@ -1983,7 +1983,7 @@ static void conv2d_hmx_win_croutons_fold_cwhn(const struct htp_conv2d_win_contex
}
}
-static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, unsigned int nth, unsigned int ith) {
+static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) {
struct htp_ops_context * octx = wc->octx;
const struct htp_conv2d_kernel_params * kp = wc->kp;
@@ -1994,7 +1994,7 @@ static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, unsign
const uint32_t b0 = MIN(ith * bpt, ncb);
const uint32_t b1 = MIN(b0 + bpt, ncb);
- dma_queue * dmaq = octx->ctx->dma[ith];
+ dma_queue * dmaq = th->dma_q;
uint8_t * thr = wc->thr_base + (size_t) ith * kp->thr_size;
float * stages[2] = { (float *) thr, (float *) (thr + kp->stage_size) };
__fp16 * act = wc->act[wc->pack_chunk & 1];
@@ -2002,7 +2002,7 @@ static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, unsign
uint32_t (*stage_fn)(const struct htp_conv2d_win_context *, dma_queue *, float *, uint32_t, uint32_t) =
kp->window.fold ? conv2d_hmx_win_stage_fold : conv2d_hmx_win_stage;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const uint32_t SB = kp->window.seg_b;
@@ -2041,7 +2041,7 @@ static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, unsign
}
}
-static void conv2d_hmx_win_unpack(const struct htp_conv2d_win_context * wc, unsigned int nth, unsigned int ith) {
+static void conv2d_hmx_win_unpack(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) {
struct htp_ops_context * octx = wc->octx;
const struct htp_conv2d_kernel_params * kp = wc->kp;
const struct htp_tensor * dst = octx->dst;
@@ -2060,12 +2060,12 @@ static void conv2d_hmx_win_unpack(const struct htp_conv2d_win_context * wc, unsi
const uint32_t i0 = MIN(ith * tpt, nt);
const uint32_t i1 = MIN(i0 + tpt, nt);
- dma_queue * dmaq = octx->ctx->dma[ith];
+ dma_queue * dmaq = th->dma_q;
uint8_t * thr = wc->thr_base + (size_t) ith * kp->thr_size + 2 * (size_t) kp->stage_size;
float * obuf = (float *) thr;
__fp16 * out = wc->out[wc->unpack_chunk & 1];
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
// nd[bi] = DMAs pushed from obuf bi, popped before it is reused HTP_CONV2D_WIN_OBUF steps later
uint32_t nd[HTP_CONV2D_WIN_OBUF] = { 0 };
@@ -2154,11 +2154,11 @@ static void conv2d_hmx_win_unpack(const struct htp_conv2d_win_context * wc, unsi
}
// per-thread callback for one pipeline phase
-static void conv2d_hmx_win_phase_thread(unsigned int nth, unsigned int ith, void * data) {
+static void conv2d_hmx_win_phase_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_conv2d_win_context * wc = (const struct htp_conv2d_win_context *) data;
if (wc->unpack_chunk >= 0) {
- conv2d_hmx_win_unpack(wc, nth, ith);
+ conv2d_hmx_win_unpack(wc, th, nth, ith);
}
// weights of this rt block are packed before its first chunk
if (wc->pack_chunk == 0) {
@@ -2170,12 +2170,12 @@ static void conv2d_hmx_win_phase_thread(unsigned int nth, unsigned int ith, void
}
}
if (wc->pack_chunk >= 0) {
- conv2d_hmx_win_pack(wc, nth, ith);
+ conv2d_hmx_win_pack(wc, th, nth, ith);
}
}
// zero out 2 act buffer regions in VTCM before packing
-static void conv2d_hmx_win_zero_thread(unsigned int nth, unsigned int ith, void * data) {
+static void conv2d_hmx_win_zero_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_conv2d_win_context * wc = (const struct htp_conv2d_win_context *) data;
const size_t n = 2 * wc->layout.act_size / sizeof(float);
@@ -2183,7 +2183,7 @@ static void conv2d_hmx_win_zero_thread(unsigned int nth, unsigned int ith, void
const size_t i0 = MIN((size_t) ith * per, n);
const size_t i1 = MIN(i0 + per, n);
- struct htp_thread_trace * tr = &wc->octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const uint16_t block = (uint16_t) (i0 / VLEN_FP32);
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, block);
if (i1 > i0) {
@@ -2195,7 +2195,7 @@ static void conv2d_hmx_win_zero_thread(unsigned int nth, unsigned int ith, void
static void conv2d_hmx_win_run_phase(struct htp_conv2d_win_context * wc, int32_t pack, int32_t unpack) {
wc->pack_chunk = pack;
wc->unpack_chunk = unpack;
- work_queue_run(wc->octx->ctx->work_queue, conv2d_hmx_win_phase_thread, wc, wc->kp->n_threads);
+ work_queue_run(wc->octx->ctx->hvx_queue, conv2d_hmx_win_phase_thread, wc, wc->kp->n_hvx);
}
@@ -2252,7 +2252,7 @@ static int conv2d_hmx_win(struct htp_ops_context * octx, const struct htp_conv2d
const size_t out_size = (size_t) kp->rt_block * kp->ct_chunk * tile;
struct htp_conv2d_vtcm_layout layout;
- htp_conv2d_vtcm_layout_build(&layout, kp->rt_block, w_size, act_size, out_size, kp->thr_size, kp->n_threads);
+ htp_conv2d_vtcm_layout_build(&layout, kp->rt_block, w_size, act_size, out_size, kp->thr_size, kp->n_hvx);
uint8_t * base = (uint8_t *) octx->ctx->vtcm_base;
@@ -2275,7 +2275,7 @@ static int conv2d_hmx_win(struct htp_ops_context * octx, const struct htp_conv2d
const uint32_t n_ch = octx->src[1]->ne[2] * (kp->window.fold ? KW : 1);
if (!kp->cwhn && n_ch <= 2 * CONV2D_WIN_SCAT_MAX) {
wctx.n_scat = (n_ch + 1) / 2;
- work_queue_run(octx->ctx->work_queue, conv2d_hmx_win_zero_thread, &wctx, kp->n_threads);
+ work_queue_run(octx->ctx->hvx_queue, conv2d_hmx_win_zero_thread, &wctx, kp->n_hvx);
}
const int32_t n_chunks = (int32_t) ((wctx.t_end - wctx.t_base + kp->ct_chunk - 1) / kp->ct_chunk);
@@ -2307,7 +2307,7 @@ static int conv2d_hmx_win(struct htp_ops_context * octx, const struct htp_conv2d
for (uint32_t rt0 = 0; rt0 < kp->n_row_tiles; rt0 += kp->rt_block) {
wctx.rt0 = rt0;
wctx.n_rt = MIN(kp->rt_block, kp->n_row_tiles - rt0);
- work_queue_run(octx->ctx->work_queue, conv2d_hmx_win_wpack_thread, &wctx, kp->n_threads);
+ work_queue_run(octx->ctx->hvx_queue, conv2d_hmx_win_wpack_thread, &wctx, kp->n_hvx);
conv2d_hmx_win_run_phase(&wctx, 0, -1);
@@ -3103,10 +3103,10 @@ struct conv2d_dw_dma_context {
conv2d_dw_compute_fn_t compute_fn;
uint32_t task_start;
uint32_t task_count;
- uint32_t tasks_per_thread;
+ uint32_t tasks_per_hvx;
};
-static void conv2d_dw_dma_thread(unsigned int nth, unsigned int ith, void * data) {
+static void conv2d_dw_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct conv2d_dw_dma_context * ctx = (const struct conv2d_dw_dma_context *) data;
const struct htp_ops_context * octx = ctx->octx;
const struct htp_conv2d_dw_kernel_params * kp = ctx->kp;
@@ -3115,24 +3115,24 @@ static void conv2d_dw_dma_thread(unsigned int nth, unsigned int ith, void * data
const struct htp_tensor * dst = octx->dst;
conv2d_dw_compute_fn_t compute_fn = ctx->compute_fn;
- const uint32_t first = ctx->task_start + ith * ctx->tasks_per_thread;
- const uint32_t last = MIN(first + ctx->tasks_per_thread, ctx->task_start + ctx->task_count);
+ const uint32_t first = ctx->task_start + ith * ctx->tasks_per_hvx;
+ const uint32_t last = MIN(first + ctx->tasks_per_hvx, ctx->task_start + ctx->task_count);
if (first >= last) {
return;
}
(void) nth;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
- dma_queue * dma_q = octx->ctx->dma[ith];
+ struct htp_thread_trace * tr = th->trace;
+ dma_queue * dma_q = th->dma_q;
const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size;
const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size;
const uint32_t pad_top_stride = kp->pad_top_stride;
uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
- uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread;
- uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread;
+ uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_hvx;
+ uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_hvx;
float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) };
float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) };
@@ -3213,7 +3213,7 @@ int op_conv_2d_dw(struct htp_ops_context * octx) {
const uint32_t w_size = src0->size;
struct htp_conv2d_dw_vtcm_layout layout;
- htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, kp->n_threads);
+ htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, kp->n_hvx);
uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
void * w_vtcm = VTCM_LAYOUT_PTR(void, vtcm_base, layout.off_w);
@@ -3240,8 +3240,8 @@ int op_conv_2d_dw(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t nth = MIN(kp->n_threads, task_count);
- const uint32_t tasks_per_thread = (task_count + nth - 1) / nth;
+ const uint32_t nth = MIN(kp->n_hvx, task_count);
+ const uint32_t tasks_per_hvx = (task_count + nth - 1) / nth;
struct conv2d_dw_dma_context ctx = {
.octx = octx,
@@ -3251,9 +3251,9 @@ int op_conv_2d_dw(struct htp_ops_context * octx) {
.compute_fn = conv2d_dw_kernels[kp->kernel_type][kp->is_f16 ? 1 : 0],
.task_start = task_start,
.task_count = task_count,
- .tasks_per_thread = tasks_per_thread,
+ .tasks_per_hvx = tasks_per_hvx,
};
- work_queue_run(octx->ctx->work_queue, conv2d_dw_dma_thread, &ctx, nth);
+ work_queue_run(octx->ctx->hvx_queue, conv2d_dw_dma_thread, &ctx, nth);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/conv2d-ops.h b/ggml/src/ggml-hexagon/htp/conv2d-ops.h
index aefed269e..29dfa812f 100644
--- a/ggml/src/ggml-hexagon/htp/conv2d-ops.h
+++ b/ggml/src/ggml-hexagon/htp/conv2d-ops.h
@@ -60,7 +60,7 @@ struct htp_conv2d_window_params {
struct htp_conv2d_kernel_params {
uint8_t kernel_type;
- uint8_t n_threads;
+ uint8_t n_hvx;
uint8_t cwhn;
uint8_t w_kmajor;
@@ -98,7 +98,7 @@ enum htp_conv2d_dw_kernel_type {
struct htp_conv2d_dw_kernel_params {
uint8_t kernel_type;
- uint8_t n_threads;
+ uint8_t n_hvx;
uint8_t cwhn;
uint8_t is_f16;
@@ -140,8 +140,8 @@ struct htp_conv2d_dw_vtcm_layout {
size_t off_dst;
size_t src_spad_half_size;
size_t dst_spad_half_size;
- size_t src_bytes_per_thread;
- size_t dst_bytes_per_thread;
+ size_t src_bytes_per_hvx;
+ size_t dst_bytes_per_hvx;
};
static inline void htp_conv2d_dw_vtcm_layout_build(
@@ -149,17 +149,17 @@ static inline void htp_conv2d_dw_vtcm_layout_build(
uint32_t max_src_bytes,
uint32_t max_dst_bytes,
uint32_t w_size,
- uint32_t n_threads
+ uint32_t n_hvx
) {
L->src_spad_half_size = hex_round_up(max_src_bytes + 256, 128);
L->dst_spad_half_size = hex_round_up(max_dst_bytes + 128, 128);
- L->src_bytes_per_thread = 2 * L->src_spad_half_size;
- L->dst_bytes_per_thread = 2 * L->dst_spad_half_size;
+ L->src_bytes_per_hvx = 2 * L->src_spad_half_size;
+ L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_w, hex_round_up(w_size, 128));
- VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx);
L->total_bytes = off;
}
@@ -181,7 +181,7 @@ static inline void htp_conv2d_vtcm_layout_build(
size_t act_size,
size_t out_size,
size_t thr_size,
- uint32_t n_threads
+ uint32_t n_hvx
) {
const size_t sc = hex_round_up((size_t) rt_block * HTP_CONV2D_HMX_SCALES_SIZE, 2048);
size_t off = 0;
@@ -191,7 +191,7 @@ static inline void htp_conv2d_vtcm_layout_build(
VTCM_LAYOUT_ALLOC(off, off_act[1], act_size);
VTCM_LAYOUT_ALLOC(off, off_out[0], out_size);
VTCM_LAYOUT_ALLOC(off, off_out[1], out_size);
- VTCM_LAYOUT_ALLOC(off, off_thr, (size_t) n_threads * thr_size);
+ VTCM_LAYOUT_ALLOC(off, off_thr, (size_t) n_hvx * thr_size);
L->act_size = act_size;
L->total_bytes = off;
}
diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c
index 59f406c7a..cdf4d8ec6 100644
--- a/ggml/src/ggml-hexagon/htp/cpy-ops.c
+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c
@@ -26,11 +26,11 @@ struct htp_copy_context {
uint32_t row_start;
uint32_t nrows;
- uint32_t src0_nrows_per_thread;
+ uint32_t src0_nrows_per_hvx;
uint32_t elem_start;
uint32_t nelem;
- uint32_t elem_per_thread;
+ uint32_t elem_per_hvx;
struct htp_copy_convert_vtcm_layout layout;
uint8_t * vtcm_src0;
@@ -38,7 +38,7 @@ struct htp_copy_context {
uint32_t tile_start;
uint32_t ntiles;
- uint32_t tiles_per_thread;
+ uint32_t tiles_per_hvx;
};
#define cpy_preamble \
@@ -66,16 +66,17 @@ struct htp_copy_context {
const uint32_t nb3 = dst->nb[3];
#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \
-static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \
+static void cpy_thread_##NAME##_reshape(const htp_thread_t * th, unsigned int nth, unsigned int ith, \
+ void * data) { \
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
struct htp_ops_context * octx = ct->octx; \
cpy_preamble; \
- const uint32_t th_nelem = ct->elem_per_thread; \
+ const uint32_t th_nelem = ct->elem_per_hvx; \
const uint32_t th_start = ct->elem_start + ith * th_nelem; \
const uint32_t th_end = MIN(th_start + th_nelem, ct->elem_start + ct->nelem); \
if (th_start >= th_end) return; \
\
- dma_queue * dma_q = octx->ctx->dma[ith]; \
+ dma_queue * dma_q = th->dma_q; \
\
const uint32_t ne01_ne00 = ne01 * ne00; \
const uint32_t ne02_ne01_ne00 = ne02 * ne01_ne00; \
@@ -215,19 +216,19 @@ static inline void cpy_transpose_push_in(dma_queue * q, const struct htp_copy_tr
}
// Pipelined tile worker: DMA in -> vgather transpose -> DMA out
-static void cpy_thread_transpose(unsigned int nth, unsigned int ith, void * data) {
+static void cpy_thread_transpose(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_copy_context * ct = (struct htp_copy_context *) data;
struct htp_ops_context * octx = ct->octx;
const struct htp_copy_transpose_params * tp = &ct->kparams->u.transpose;
- const uint32_t t0 = ct->tile_start + ith * ct->tiles_per_thread;
- const uint32_t t1 = MIN(t0 + ct->tiles_per_thread, ct->tile_start + ct->ntiles);
+ const uint32_t t0 = ct->tile_start + ith * ct->tiles_per_hvx;
+ const uint32_t t1 = MIN(t0 + ct->tiles_per_hvx, ct->tile_start + ct->ntiles);
if (t0 >= t1) {
return;
}
- dma_queue * dma_q = octx->ctx->dma[ith];
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ dma_queue * dma_q = th->dma_q;
+ struct htp_thread_trace * tr = th->trace;
const dma_addr_t src = octx->src[0]->data;
const dma_addr_t dst = octx->dst->data;
@@ -332,18 +333,19 @@ static void cpy_thread_transpose(unsigned int nth, unsigned int ith, void * data
}
#define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC) \
-static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \
+static void cpy_thread_##NAME##_sameshape(const htp_thread_t * th, unsigned int nth, \
+ unsigned int ith, void * data) { \
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
struct htp_ops_context * octx = ct->octx; \
cpy_preamble; \
\
- const uint32_t dr = ct->src0_nrows_per_thread; \
+ const uint32_t dr = ct->src0_nrows_per_hvx; \
const uint32_t ir0 = ct->row_start + dr * ith; \
const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \
if (ir0 >= ir1) return; \
\
- dma_queue * dma_q = octx->ctx->dma[ith]; \
- struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
+ dma_queue * dma_q = th->dma_q; \
+ struct htp_thread_trace * tr = th->trace; \
\
const struct htp_copy_convert_params * cvt = &ct->kparams->u.convert; \
const struct htp_copy_convert_vtcm_layout * L = &ct->layout; \
@@ -352,8 +354,8 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
const uint32_t src0_row_stride = L->src0_row_stride; \
const uint32_t dst_row_stride = L->dst_row_stride; \
const uint32_t blk_rows = L->blk_rows; \
- uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * L->spad0_size_per_thread; \
- uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * L->spad1_size_per_thread; \
+ uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * L->spad0_size_per_hvx; \
+ uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * L->spad1_size_per_hvx; \
const uint32_t src0_row_size = ne00 * ct->kparams->src0_type_size; \
const uint32_t dst_row_size = ne00 * ct->kparams->dst_type_size; \
\
@@ -510,7 +512,7 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -532,17 +534,17 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp
return HTP_STATUS_VTCM_TOO_SMALL;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
const struct htp_copy_convert_params * cvt = &kparams->u.convert;
struct htp_copy_context ct;
- htp_copy_convert_vtcm_layout_build(&ct.layout, src0->ne[0], kparams->src0_type_size, kparams->dst_type_size, cvt->blk_rows, n_threads);
+ htp_copy_convert_vtcm_layout_build(&ct.layout, src0->ne[0], kparams->src0_type_size, kparams->dst_type_size, cvt->blk_rows, n_hvx);
ct.octx = octx;
ct.kparams = kparams;
ct.row_start = row_start;
ct.nrows = nrows;
- ct.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
+ ct.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div);
uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
ct.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, ct.layout.off_src0);
@@ -561,7 +563,7 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp
return HTP_STATUS_NO_SUPPORT;
}
- work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, copy_fun, &ct, n_hvx);
return HTP_STATUS_OK;
}
@@ -569,7 +571,7 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -587,14 +589,14 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
struct htp_copy_context ct;
ct.octx = octx;
ct.kparams = kparams;
ct.elem_start = elem_start;
ct.nelem = nelem;
- ct.elem_per_thread = fastdiv(nelem + n_threads - 1, &octx->n_threads_div);
+ ct.elem_per_hvx = fastdiv(nelem + n_hvx - 1, &octx->n_hvx_div);
work_queue_func_t copy_fun = NULL;
switch (src0->type) {
@@ -604,12 +606,12 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern
default: return HTP_STATUS_NO_SUPPORT;
}
- work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, copy_fun, &ct, n_hvx);
return HTP_STATUS_OK;
}
static int cpy_transpose(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) {
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -633,20 +635,20 @@ static int cpy_transpose(struct htp_ops_context * octx, const struct htp_copy_ke
return HTP_STATUS_VTCM_TOO_SMALL;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
struct htp_copy_context ct;
- ct.octx = octx;
- ct.kparams = kparams;
- ct.tile_start = tile_start;
- ct.ntiles = ntiles;
- ct.tiles_per_thread = fastdiv(ntiles + n_threads - 1, &octx->n_threads_div);
+ ct.octx = octx;
+ ct.kparams = kparams;
+ ct.tile_start = tile_start;
+ ct.ntiles = ntiles;
+ ct.tiles_per_hvx = fastdiv(ntiles + n_hvx - 1, &octx->n_hvx_div);
uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
ct.vtcm_src0 = vtcm_base;
- ct.vtcm_dst = vtcm_base + (size_t) n_threads * 2 * tp->in_buf_size;
+ ct.vtcm_dst = vtcm_base + (size_t) n_hvx * 2 * tp->in_buf_size;
- work_queue_run(octx->ctx->work_queue, cpy_thread_transpose, &ct, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, cpy_thread_transpose, &ct, n_hvx);
return HTP_STATUS_OK;
}
@@ -674,7 +676,7 @@ int op_cpy(struct htp_ops_context * octx) {
status = cpy_transpose(octx, kparams);
break;
case HTP_COPY_KERNEL_GATHER_ROWS:
- status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_threads);
+ status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_hvx);
break;
default:
status = HTP_STATUS_NO_SUPPORT;
diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.h b/ggml/src/ggml-hexagon/htp/cpy-ops.h
index 6bb96ca44..1ea50ca1d 100644
--- a/ggml/src/ggml-hexagon/htp/cpy-ops.h
+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.h
@@ -60,7 +60,7 @@ struct htp_copy_kernel_params {
uint8_t kernel_type;
uint8_t src0_type_size;
uint8_t dst_type_size;
- uint8_t n_threads;
+ uint8_t n_hvx;
uint32_t total_elems;
uint32_t total_rows;
@@ -88,8 +88,8 @@ struct htp_copy_convert_vtcm_layout {
uint32_t blk_rows;
uint32_t src0_buf_size;
uint32_t dst_buf_size;
- uint32_t spad0_size_per_thread;
- uint32_t spad1_size_per_thread;
+ uint32_t spad0_size_per_hvx;
+ uint32_t spad1_size_per_hvx;
};
static inline void htp_copy_convert_vtcm_layout_build(
@@ -98,7 +98,7 @@ static inline void htp_copy_convert_vtcm_layout_build(
uint32_t src_type_size,
uint32_t dst_type_size,
uint32_t blk_rows,
- uint32_t n_threads) {
+ uint32_t n_hvx) {
const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256);
const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256);
@@ -108,12 +108,12 @@ static inline void htp_copy_convert_vtcm_layout_build(
L->blk_rows = blk_rows;
L->src0_buf_size = blk_rows * src0_row_stride;
L->dst_buf_size = blk_rows * dst_row_stride;
- L->spad0_size_per_thread = 2 * L->src0_buf_size;
- L->spad1_size_per_thread = 2 * L->dst_buf_size;
+ L->spad0_size_per_hvx = 2 * L->src0_buf_size;
+ L->spad1_size_per_hvx = 2 * L->dst_buf_size;
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_src0, (size_t) n_threads * L->spad0_size_per_thread);
- VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) n_threads * L->spad1_size_per_thread);
+ VTCM_LAYOUT_ALLOC(off, off_src0, (size_t) n_hvx * L->spad0_size_per_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) n_hvx * L->spad1_size_per_hvx);
L->total_bytes = off;
}
@@ -124,7 +124,7 @@ static inline void htp_copy_convert_solve_layout(
uint32_t ne01,
uint32_t src_type_size,
uint32_t dst_type_size,
- uint32_t n_threads) {
+ uint32_t n_hvx) {
const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256);
const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256);
@@ -138,7 +138,7 @@ static inline void htp_copy_convert_solve_layout(
blk_rows = 1;
}
- htp_copy_convert_vtcm_layout_build(layout, ne00, src_type_size, dst_type_size, blk_rows, n_threads);
+ htp_copy_convert_vtcm_layout_build(layout, ne00, src_type_size, dst_type_size, blk_rows, n_hvx);
}
#define HTP_COPY_TRANSPOSE_BLK_BYTES (64 * 1024)
diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.c b/ggml/src/ggml-hexagon/htp/cumsum-ops.c
index 36fc81276..292a05455 100644
--- a/ggml/src/ggml-hexagon/htp/cumsum-ops.c
+++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.c
@@ -8,7 +8,7 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
@@ -48,7 +48,7 @@ struct htp_cumsum_context {
size_t dst_row_size;
size_t src_row_size_aligned;
size_t dst_row_size_aligned;
- uint32_t rows_per_thread;
+ uint32_t rows_per_hvx;
uint32_t total_rows;
uint32_t row_start;
};
@@ -57,7 +57,7 @@ struct htp_cumsum_context {
struct htp_cumsum_context * cctx = (struct htp_cumsum_context *) data; \
struct htp_ops_context * octx = cctx->octx; \
htp_cumsum_tensors_preamble; \
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
// ---------------------------------------------------------------------------
// HVX prefix scan helpers
@@ -118,11 +118,11 @@ static inline void hvx_cumsum_row_f32(const float * restrict src, float * restri
// Per thread worker: Double-buffered DMA
// ---------------------------------------------------------------------------
-static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) {
+static void cumsum_thread_f32_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
htp_cumsum_preamble;
- const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith;
- const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows);
+ const uint32_t ir0 = cctx->row_start + cctx->rows_per_hvx * ith;
+ const uint32_t ir1 = MIN(ir0 + cctx->rows_per_hvx, cctx->row_start + cctx->total_rows);
if (ir0 >= ir1) {
return;
@@ -151,7 +151,7 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat
src_row_size_aligned, src_row_size, src_row_size, 1);
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
for (uint32_t ir = ir0; ir < ir1; ir++) {
float * dst_spad_row = (float *) dma_queue_pop(dma_q).src;
@@ -184,16 +184,16 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat
// Per thread worker: Direct HVX (no DMA)
// ---------------------------------------------------------------------------
-static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) {
+static void cumsum_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
htp_cumsum_preamble;
const uint8_t * src_data = (const uint8_t *) src0->data;
uint8_t * dst_data = (uint8_t *) dst->data;
- const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith;
- const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows);
+ const uint32_t ir0 = cctx->row_start + cctx->rows_per_hvx * ith;
+ const uint32_t ir1 = MIN(ir0 + cctx->rows_per_hvx, cctx->row_start + cctx->total_rows);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);
for (uint32_t ir = ir0; ir < ir1; ir++) {
@@ -234,11 +234,11 @@ int op_cumsum_f32(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {
+ if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) {
return HTP_STATUS_INVAL_PARAMS;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {
return HTP_STATUS_NO_SUPPORT;
@@ -246,12 +246,12 @@ int op_cumsum_f32(struct htp_ops_context * octx) {
struct htp_cumsum_vtcm_layout layout = { 0 };
if (kparams->use_dma) {
- htp_cumsum_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_threads);
+ htp_cumsum_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_hvx);
- octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread;
- octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread;
- octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread;
- octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread;
+ octx->src0_spad.size_per_hvx = layout.src0_bytes_per_hvx;
+ octx->dst_spad.size_per_hvx = layout.dst_bytes_per_hvx;
+ octx->src0_spad.size = n_hvx * layout.src0_bytes_per_hvx;
+ octx->dst_spad.size = n_hvx * layout.dst_bytes_per_hvx;
octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0);
octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst);
octx->src0_spad.src = NULL;
@@ -264,15 +264,15 @@ int op_cumsum_f32(struct htp_ops_context * octx) {
.dst_row_size = dst_row_size,
.src_row_size_aligned = layout.src_row_size_aligned,
.dst_row_size_aligned = layout.dst_row_size_aligned,
- .rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
+ .rows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div),
.total_rows = nrows,
.row_start = row_start,
};
if (kparams->use_dma) {
- work_queue_run(octx->ctx->work_queue, cumsum_thread_f32_dma, &cctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, cumsum_thread_f32_dma, &cctx, n_hvx);
} else {
- work_queue_run(octx->ctx->work_queue, cumsum_thread_f32, &cctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, cumsum_thread_f32, &cctx, n_hvx);
}
return HTP_STATUS_OK;
diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.h b/ggml/src/ggml-hexagon/htp/cumsum-ops.h
index 6e1ac316f..48f4053ed 100644
--- a/ggml/src/ggml-hexagon/htp/cumsum-ops.h
+++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.h
@@ -11,30 +11,30 @@ struct htp_cumsum_vtcm_layout {
size_t off_dst;
size_t src_row_size_aligned;
size_t dst_row_size_aligned;
- size_t src0_bytes_per_thread;
- size_t dst_bytes_per_thread;
+ size_t src0_bytes_per_hvx;
+ size_t dst_bytes_per_hvx;
};
static inline void htp_cumsum_vtcm_layout_build(
struct htp_cumsum_vtcm_layout * L,
size_t src_row_size,
size_t dst_row_size,
- uint32_t n_threads
+ uint32_t n_hvx
) {
L->src_row_size_aligned = hex_round_up(src_row_size, 128);
L->dst_row_size_aligned = hex_round_up(dst_row_size, 128);
- L->src0_bytes_per_thread = 2 * L->src_row_size_aligned;
- L->dst_bytes_per_thread = 2 * L->dst_row_size_aligned;
+ L->src0_bytes_per_hvx = 2 * L->src_row_size_aligned;
+ L->dst_bytes_per_hvx = 2 * L->dst_row_size_aligned;
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx);
L->total_bytes = off;
}
struct htp_cumsum_kernel_params {
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t total_rows;
uint32_t use_dma;
uint32_t vtcm_size;
diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.c b/ggml/src/ggml-hexagon/htp/diag-ops.c
index 7f03e8550..ea1a50316 100644
--- a/ggml/src/ggml-hexagon/htp/diag-ops.c
+++ b/ggml/src/ggml-hexagon/htp/diag-ops.c
@@ -6,7 +6,7 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
@@ -40,7 +40,7 @@ struct htp_diag_context {
size_t dst_row_size;
size_t src_batch_size_aligned;
size_t dst_row_size_aligned;
- uint32_t batches_per_thread;
+ uint32_t batches_per_hvx;
uint32_t total_batches;
uint32_t batch_start;
};
@@ -60,12 +60,12 @@ static inline void hvx_diag_row_f32(const float * restrict src, float * restrict
// Per thread worker: DMA src fetch, compute in VTCM, DMA dst writeback
// ---------------------------------------------------------------------------
-static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) {
+static void diag_thread_f32_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
htp_diag_preamble;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
- const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith;
- const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches);
+ const uint32_t ib0 = dctx->batch_start + dctx->batches_per_hvx * ith;
+ const uint32_t ib1 = MIN(ib0 + dctx->batches_per_hvx, dctx->batch_start + dctx->total_batches);
if (ib0 >= ib1) {
return;
@@ -83,7 +83,7 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data)
uint8_t * src_spad = octx->src0_spad.data + (ith * src_batch_size_aligned);
uint8_t * dst_spad = octx->dst_spad.data + (ith * dst_row_size_aligned);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
for (uint32_t ib = ib0; ib < ib1; ib++) {
const uint32_t i3 = fastdiv(ib, &dctx->kparams->div_ne02);
@@ -124,16 +124,16 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data)
// Per thread worker: Direct HVX (no DMA)
// ---------------------------------------------------------------------------
-static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) {
+static void diag_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
htp_diag_preamble;
const uint8_t * src_data = (const uint8_t *) src0->data;
uint8_t * dst_data = (uint8_t *) dst->data;
- const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith;
- const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches);
+ const uint32_t ib0 = dctx->batch_start + dctx->batches_per_hvx * ith;
+ const uint32_t ib1 = MIN(ib0 + dctx->batches_per_hvx, dctx->batch_start + dctx->total_batches);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0);
for (uint32_t ib = ib0; ib < ib1; ib++) {
@@ -177,11 +177,11 @@ int op_diag_f32(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nbatches))) {
+ if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nbatches))) {
return HTP_STATUS_INVAL_PARAMS;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {
return HTP_STATUS_NO_SUPPORT;
@@ -192,12 +192,12 @@ int op_diag_f32(struct htp_ops_context * octx) {
struct htp_diag_vtcm_layout layout = { 0 };
if (kparams->use_dma) {
- htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_threads);
+ htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_hvx);
- octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread;
- octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread;
- octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread;
- octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread;
+ octx->src0_spad.size_per_hvx = layout.src0_bytes_per_hvx;
+ octx->dst_spad.size_per_hvx = layout.dst_bytes_per_hvx;
+ octx->src0_spad.size = n_hvx * layout.src0_bytes_per_hvx;
+ octx->dst_spad.size = n_hvx * layout.dst_bytes_per_hvx;
octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0);
octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst);
octx->src0_spad.src = NULL;
@@ -211,15 +211,15 @@ int op_diag_f32(struct htp_ops_context * octx) {
.dst_row_size = dst_row_size,
.src_batch_size_aligned = layout.src_batch_size_aligned,
.dst_row_size_aligned = layout.dst_row_size_aligned,
- .batches_per_thread = fastdiv(nbatches + n_threads - 1, &octx->n_threads_div),
+ .batches_per_hvx = fastdiv(nbatches + n_hvx - 1, &octx->n_hvx_div),
.total_batches = nbatches,
.batch_start = batch_start,
};
if (kparams->use_dma) {
- work_queue_run(octx->ctx->work_queue, diag_thread_f32_dma, &dctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, diag_thread_f32_dma, &dctx, n_hvx);
} else {
- work_queue_run(octx->ctx->work_queue, diag_thread_f32, &dctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, diag_thread_f32, &dctx, n_hvx);
}
return HTP_STATUS_OK;
diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.h b/ggml/src/ggml-hexagon/htp/diag-ops.h
index 983bc0242..609a061c5 100644
--- a/ggml/src/ggml-hexagon/htp/diag-ops.h
+++ b/ggml/src/ggml-hexagon/htp/diag-ops.h
@@ -12,30 +12,30 @@ struct htp_diag_vtcm_layout {
size_t off_dst;
size_t src_batch_size_aligned;
size_t dst_row_size_aligned;
- size_t src0_bytes_per_thread;
- size_t dst_bytes_per_thread;
+ size_t src0_bytes_per_hvx;
+ size_t dst_bytes_per_hvx;
};
static inline void htp_diag_vtcm_layout_build(
struct htp_diag_vtcm_layout * L,
size_t src_batch_size,
size_t dst_row_size,
- uint32_t n_threads
+ uint32_t n_hvx
) {
L->src_batch_size_aligned = hex_round_up(src_batch_size, 128);
L->dst_row_size_aligned = hex_round_up(dst_row_size, 128);
- L->src0_bytes_per_thread = L->src_batch_size_aligned;
- L->dst_bytes_per_thread = L->dst_row_size_aligned;
+ L->src0_bytes_per_hvx = L->src_batch_size_aligned;
+ L->dst_bytes_per_hvx = L->dst_row_size_aligned;
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx);
L->total_bytes = off;
}
struct htp_diag_kernel_params {
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t total_batches;
uint32_t use_dma;
uint32_t can_split;
diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.h
index 678fc61d7..28d06119b 100644
--- a/ggml/src/ggml-hexagon/htp/dma-queue.h
+++ b/ggml/src/ggml-hexagon/htp/dma-queue.h
@@ -9,7 +9,7 @@
#include <string.h>
#include "hex-utils.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#ifdef __cplusplus
extern "C" {
diff --git a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c
index 62f12c55d..b7123923c 100644
--- a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c
+++ b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c
@@ -9,7 +9,7 @@
#include "dma-queue.h"
#include "hex-common.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "htp-ctx.h"
#include "htp-tensor.h"
#include "hvx-utils.h"
@@ -57,7 +57,7 @@ struct dsv4_hc_pre_context {
uint32_t token_start;
uint32_t n_chunks;
uint32_t total_jobs;
- uint32_t jobs_per_thread;
+ uint32_t jobs_per_hvx;
float scale;
uint8_t * vtcm;
uint32_t dma_block_chunks;
@@ -178,7 +178,7 @@ static __attribute__((noinline)) void dsv4_hc_pre_gated_block_f32(
}
}
-static void dsv4_hc_pre_thread_plain_f32(unsigned int nth, unsigned int ith, void * data) {
+static void dsv4_hc_pre_thread_plain_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct dsv4_hc_pre_context * dctx = (struct dsv4_hc_pre_context *) data;
struct htp_ops_context * octx = dctx->octx;
@@ -187,8 +187,8 @@ static void dsv4_hc_pre_thread_plain_f32(unsigned int nth, unsigned int ith, voi
const uint32_t row_stride = dctx->dma_row_stride;
const uint32_t buffer_size = dctx->dma_buffer_size;
- const uint32_t start = dctx->jobs_per_thread * ith;
- const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs);
+ const uint32_t start = dctx->jobs_per_hvx * ith;
+ const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs);
if (start >= end) {
return;
}
@@ -198,7 +198,7 @@ static void dsv4_hc_pre_thread_plain_f32(unsigned int nth, unsigned int ith, voi
dctx->vtcm + 2*buffer_size*ith + buffer_size,
};
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
uint32_t prefetch_job = start;
for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) {
const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, prefetch_job, end);
@@ -208,7 +208,7 @@ static void dsv4_hc_pre_thread_plain_f32(unsigned int nth, unsigned int ith, voi
prefetch_job += block.count;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
uint32_t job = start;
uint32_t curr = 0;
@@ -244,7 +244,7 @@ static void dsv4_hc_pre_thread_plain_f32(unsigned int nth, unsigned int ith, voi
(void) nth;
}
-static void dsv4_hc_pre_thread_gated_f32(unsigned int nth, unsigned int ith, void * data) {
+static void dsv4_hc_pre_thread_gated_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct dsv4_hc_pre_context * dctx = (struct dsv4_hc_pre_context *) data;
struct htp_ops_context * octx = dctx->octx;
@@ -253,8 +253,8 @@ static void dsv4_hc_pre_thread_gated_f32(unsigned int nth, unsigned int ith, voi
const uint32_t row_stride = dctx->dma_row_stride;
const uint32_t buffer_size = dctx->dma_buffer_size;
- const uint32_t start = dctx->jobs_per_thread * ith;
- const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs);
+ const uint32_t start = dctx->jobs_per_hvx * ith;
+ const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs);
if (start >= end) {
return;
}
@@ -264,7 +264,7 @@ static void dsv4_hc_pre_thread_gated_f32(unsigned int nth, unsigned int ith, voi
dctx->vtcm + 2*buffer_size*ith + buffer_size,
};
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
uint32_t prefetch_job = start;
for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) {
const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, prefetch_job, end);
@@ -274,7 +274,7 @@ static void dsv4_hc_pre_thread_gated_f32(unsigned int nth, unsigned int ith, voi
prefetch_job += block.count;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
uint32_t job = start;
uint32_t curr = 0;
@@ -317,22 +317,22 @@ int op_dsv4_hc_pre(struct htp_ops_context * octx) {
const struct htp_dsv4_hc_pre_kernel_params * kparams =
(const struct htp_dsv4_hc_pre_kernel_params *) octx->kernel_params;
- const uint32_t n_threads = kparams->n_threads;
- if (!htp_ops_context_set_n_threads_div(octx, n_threads, kparams->div_n_threads)) {
+ const uint32_t n_hvx = kparams->n_hvx;
+ if (!htp_ops_context_set_n_hvx_div(octx, n_hvx, kparams->div_n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
uint32_t token_start = 0;
uint32_t n_tokens = x->ne[2];
uint32_t total_jobs = kparams->total_jobs;
- uint32_t jobs_per_thread = kparams->jobs_per_thread;
+ uint32_t jobs_per_hvx = kparams->jobs_per_hvx;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_tokens, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
token_start = range.start;
n_tokens = range.count;
total_jobs = n_tokens * kparams->n_chunks;
- jobs_per_thread = fastdiv(total_jobs + n_threads - 1, &kparams->div_n_threads);
+ jobs_per_hvx = fastdiv(total_jobs + n_hvx - 1, &kparams->div_n_hvx);
}
if (n_tokens == 0) {
@@ -348,7 +348,7 @@ int op_dsv4_hc_pre(struct htp_ops_context * octx) {
.token_start = token_start,
.n_chunks = kparams->n_chunks,
.total_jobs = total_jobs,
- .jobs_per_thread = jobs_per_thread,
+ .jobs_per_hvx = jobs_per_hvx,
.scale = scale,
.vtcm = (uint8_t *) octx->ctx->vtcm_base,
.dma_block_chunks = kparams->dma_block_chunks,
@@ -363,11 +363,11 @@ int op_dsv4_hc_pre(struct htp_ops_context * octx) {
default: return HTP_STATUS_INVAL_PARAMS;
}
- FARF(HIGH, "dsv4-hc-pre: (%ux%ux%u) -> (%ux%u) : ktype %u tokens %u:%u nth %u\n",
+ FARF(HIGH, "dsv4-hc-pre: (%ux%ux%u) -> (%ux%u) : ktype %u tokens %u:%u nhvx %u\n",
x->ne[0], x->ne[1], x->ne[2], dst->ne[0], dst->ne[1], kparams->kernel_type,
- token_start, token_start + n_tokens, n_threads);
+ token_start, token_start + n_tokens, n_hvx);
- work_queue_run(octx->ctx->work_queue, worker, &dctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, worker, &dctx, n_hvx);
return HTP_STATUS_OK;
}
@@ -380,7 +380,7 @@ struct dsv4_hc_post_context {
const struct htp_dsv4_hc_post_kernel_params * kparams;
uint32_t token_start;
uint32_t total_jobs;
- uint32_t jobs_per_thread;
+ uint32_t jobs_per_hvx;
uint8_t * vtcm;
};
@@ -481,7 +481,7 @@ static inline void dsv4_hc_post_push_comb(
16, comb->nb[1], 16, 4);
}
-static void dsv4_hc_post_thread_identity_f32(unsigned int nth, unsigned int ith, void * data) {
+static void dsv4_hc_post_thread_identity_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct dsv4_hc_post_context * dctx = (struct dsv4_hc_post_context *) data;
struct htp_ops_context * octx = dctx->octx;
@@ -491,8 +491,8 @@ static void dsv4_hc_post_thread_identity_f32(unsigned int nth, unsigned int ith,
const uint32_t block_chunks = dctx->kparams->dma_block_chunks;
const uint32_t row_stride = dctx->kparams->dma_row_stride;
const uint32_t buffer_size = dctx->kparams->dma_buffer_size;
- const uint32_t start = dctx->jobs_per_thread * ith;
- const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs);
+ const uint32_t start = dctx->jobs_per_hvx * ith;
+ const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs);
if (start >= end) {
return;
}
@@ -502,7 +502,7 @@ static void dsv4_hc_post_thread_identity_f32(unsigned int nth, unsigned int ith,
dctx->vtcm + 2*buffer_size*ith + buffer_size,
};
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
uint32_t prefetch_job = start;
for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) {
const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks);
@@ -512,7 +512,7 @@ static void dsv4_hc_post_thread_identity_f32(unsigned int nth, unsigned int ith,
prefetch_job += block.count;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
uint32_t job = start;
uint32_t curr = 0;
@@ -549,7 +549,7 @@ static void dsv4_hc_post_thread_identity_f32(unsigned int nth, unsigned int ith,
(void) nth;
}
-static void dsv4_hc_post_thread_comb_f32(unsigned int nth, unsigned int ith, void * data) {
+static void dsv4_hc_post_thread_comb_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct dsv4_hc_post_context * dctx = (struct dsv4_hc_post_context *) data;
struct htp_ops_context * octx = dctx->octx;
@@ -559,8 +559,8 @@ static void dsv4_hc_post_thread_comb_f32(unsigned int nth, unsigned int ith, voi
const uint32_t block_chunks = dctx->kparams->dma_block_chunks;
const uint32_t row_stride = dctx->kparams->dma_row_stride;
const uint32_t buffer_size = dctx->kparams->dma_buffer_size;
- const uint32_t start = dctx->jobs_per_thread * ith;
- const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs);
+ const uint32_t start = dctx->jobs_per_hvx * ith;
+ const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs);
if (start >= end) {
return;
}
@@ -570,7 +570,7 @@ static void dsv4_hc_post_thread_comb_f32(unsigned int nth, unsigned int ith, voi
dctx->vtcm + 2*buffer_size*ith + buffer_size,
};
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
uint32_t prefetch_job = start;
for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) {
const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks);
@@ -580,7 +580,7 @@ static void dsv4_hc_post_thread_comb_f32(unsigned int nth, unsigned int ith, voi
prefetch_job += block.count;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
uint32_t job = start;
uint32_t curr = 0;
@@ -627,22 +627,22 @@ int op_dsv4_hc_post(struct htp_ops_context * octx) {
const struct htp_dsv4_hc_post_kernel_params * kparams =
(const struct htp_dsv4_hc_post_kernel_params *) octx->kernel_params;
- const uint32_t n_threads = kparams->n_threads;
- if (!htp_ops_context_set_n_threads_div(octx, n_threads, kparams->div_n_threads)) {
+ const uint32_t n_hvx = kparams->n_hvx;
+ if (!htp_ops_context_set_n_hvx_div(octx, n_hvx, kparams->div_n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
uint32_t token_start = 0;
uint32_t n_tokens = x->ne[1];
uint32_t total_jobs = kparams->total_jobs;
- uint32_t jobs_per_thread = kparams->jobs_per_thread;
+ uint32_t jobs_per_hvx = kparams->jobs_per_hvx;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_tokens, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
token_start = range.start;
n_tokens = range.count;
total_jobs = n_tokens * kparams->n_chunks;
- jobs_per_thread = fastdiv(total_jobs + n_threads - 1, &kparams->div_n_threads);
+ jobs_per_hvx = fastdiv(total_jobs + n_hvx - 1, &kparams->div_n_hvx);
}
if (n_tokens == 0) {
@@ -654,7 +654,7 @@ int op_dsv4_hc_post(struct htp_ops_context * octx) {
.kparams = kparams,
.token_start = token_start,
.total_jobs = total_jobs,
- .jobs_per_thread = jobs_per_thread,
+ .jobs_per_hvx = jobs_per_hvx,
.vtcm = (uint8_t *) octx->ctx->vtcm_base,
};
@@ -665,11 +665,11 @@ int op_dsv4_hc_post(struct htp_ops_context * octx) {
default: return HTP_STATUS_INVAL_PARAMS;
}
- FARF(HIGH, "dsv4-hc-post: (%ux%u) (%ux%ux%u) -> (%ux%ux%u) : ktype %u tokens %u:%u nth %u\n",
+ FARF(HIGH, "dsv4-hc-post: (%ux%u) (%ux%ux%u) -> (%ux%ux%u) : ktype %u tokens %u:%u nhvx %u\n",
x->ne[0], x->ne[1], residual->ne[0], residual->ne[1], residual->ne[2],
dst->ne[0], dst->ne[1], dst->ne[2], kparams->kernel_type,
- token_start, token_start + n_tokens, n_threads);
+ token_start, token_start + n_tokens, n_hvx);
- work_queue_run(octx->ctx->work_queue, worker, &dctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, worker, &dctx, n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h
index a0c01aa91..58a3d0d3c 100644
--- a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h
+++ b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h
@@ -18,10 +18,10 @@ enum htp_dsv4_hc_pre_kernel_type {
struct htp_dsv4_hc_pre_kernel_params {
uint32_t kernel_type;
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t n_chunks;
uint32_t total_jobs;
- uint32_t jobs_per_thread;
+ uint32_t jobs_per_hvx;
uint32_t dma_block_chunks;
uint32_t dma_row_stride;
@@ -29,7 +29,7 @@ struct htp_dsv4_hc_pre_kernel_params {
uint32_t vtcm_size;
struct fastdiv_values div_n_chunks;
- struct fastdiv_values div_n_threads;
+ struct fastdiv_values div_n_hvx;
};
#if defined(__cplusplus)
@@ -48,10 +48,10 @@ enum htp_dsv4_hc_post_kernel_type {
struct htp_dsv4_hc_post_kernel_params {
uint32_t kernel_type;
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t n_chunks;
uint32_t total_jobs;
- uint32_t jobs_per_thread;
+ uint32_t jobs_per_hvx;
uint32_t has_comb;
uint32_t dma_block_chunks;
@@ -60,7 +60,7 @@ struct htp_dsv4_hc_post_kernel_params {
uint32_t vtcm_size;
struct fastdiv_values div_n_chunks;
- struct fastdiv_values div_n_threads;
+ struct fastdiv_values div_n_hvx;
};
#if defined(__cplusplus)
diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.c b/ggml/src/ggml-hexagon/htp/fill-ops.c
index 2cab31b5f..620263063 100644
--- a/ggml/src/ggml-hexagon/htp/fill-ops.c
+++ b/ggml/src/ggml-hexagon/htp/fill-ops.c
@@ -6,7 +6,7 @@
#include <string.h>
#include "hex-common.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "hvx-copy.h"
#include "hvx-utils.h"
@@ -36,27 +36,27 @@
struct htp_fill_context {
struct htp_ops_context * octx;
const struct htp_fill_kernel_params * kparams;
- uint32_t nrows_per_thread;
+ uint32_t nrows_per_hvx;
uint32_t total_rows; // ne1 * ne2 * ne3
uint32_t row_start;
HVX_Vector splat_vec;
uint32_t elem_size;
};
-static void fill_thread_contiguous(unsigned int nth, unsigned int ith, void * data) {
+static void fill_thread_contiguous(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_fill_context * fctx = (const struct htp_fill_context *) data;
struct htp_ops_context * octx = fctx->octx;
fill_preamble;
// Parallelise over the flat row index spanning ne1*ne2*ne3
- const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith;
- const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows);
+ const uint32_t ir0 = fctx->row_start + fctx->nrows_per_hvx * ith;
+ const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_hvx, fctx->row_start + fctx->total_rows);
if (ir0 >= ir1) {
return;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);
const uint32_t elem_start = ir0 * ne0;
@@ -68,21 +68,21 @@ static void fill_thread_contiguous(unsigned int nth, unsigned int ith, void * da
FARF(HIGH, "fill-contig %u/%u: rows %u:%u\n", ith, nth, ir0, ir1);
}
-static void fill_thread_strided(unsigned int nth, unsigned int ith, void * data) {
+static void fill_thread_strided(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_fill_context * fctx = (const struct htp_fill_context *) data;
struct htp_ops_context * octx = fctx->octx;
const struct htp_fill_kernel_params * kparams = fctx->kparams;
fill_preamble;
// Parallelise over the flat row index spanning ne1*ne2*ne3
- const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith;
- const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows);
+ const uint32_t ir0 = fctx->row_start + fctx->nrows_per_hvx * ith;
+ const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_hvx, fctx->row_start + fctx->total_rows);
if (ir0 >= ir1) {
return;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);
for (uint32_t ir = ir0; ir < ir1; ++ir) {
@@ -120,11 +120,11 @@ int op_fill(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {
+ if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) {
return HTP_STATUS_INVAL_PARAMS;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
FARF(HIGH, "fill: (%ux%ux%ux%u) type=%u ktype=%u\n",
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], dst->type, kparams->kernel_type);
@@ -135,7 +135,7 @@ int op_fill(struct htp_ops_context * octx) {
struct htp_fill_context fctx = {
.octx = octx,
.kparams = kparams,
- .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
+ .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div),
.total_rows = nrows,
.row_start = row_start,
.elem_size = kparams->elem_size,
@@ -156,7 +156,7 @@ int op_fill(struct htp_ops_context * octx) {
? fill_thread_contiguous
: fill_thread_strided;
- work_queue_run(octx->ctx->work_queue, worker, &fctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, worker, &fctx, n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.h b/ggml/src/ggml-hexagon/htp/fill-ops.h
index d62f99751..06fe08209 100644
--- a/ggml/src/ggml-hexagon/htp/fill-ops.h
+++ b/ggml/src/ggml-hexagon/htp/fill-ops.h
@@ -11,7 +11,7 @@ enum htp_fill_kernel_type {
struct htp_fill_kernel_params {
uint8_t kernel_type;
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t total_rows;
uint32_t elem_size;
uint32_t rows_per_chunk;
diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c
index ee258172b..0b7fc2dae 100644
--- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c
+++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c
@@ -14,7 +14,7 @@
#include "dma-queue.h"
#include "hex-fastdiv.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "hmx-queue.h"
#include "hmx-utils.h"
#include "hvx-utils.h"
@@ -44,7 +44,7 @@
struct hvx_fa_ddr_spad {
__fp16 slopes[512];
uint16_t active_blocks[FA_MAX_KV_BLOCKS];
- uint16_t thread_active_blocks[HTP_MAX_NTHREADS][FA_MAX_KV_BLOCKS] __attribute__((aligned(128)));
+ uint16_t thread_active_blocks[HTP_MAX_NHVX][FA_MAX_KV_BLOCKS] __attribute__((aligned(128)));
};
struct hmx_fa_ddr_spad {
@@ -95,7 +95,7 @@ struct htp_fa_context {
size_t size_m_block;
uint32_t qrows;
- uint32_t qrows_per_thread;
+ uint32_t qrows_per_hvx;
uint32_t qrow_start;
bool is_q_fp32;
@@ -111,7 +111,7 @@ struct htp_fa_context {
float * spad_sinks;
struct hvx_fa_ddr_spad * ddr;
- dma_cache_dm * m_cache; // [n_threads]
+ dma_cache_dm * m_cache; // [n_hvx]
const struct htp_tensor * k;
const struct htp_tensor * v;
@@ -122,8 +122,8 @@ struct hmx_fa_context {
const struct htp_tensor * sinks; // attention sinks (src[4]), NULL if absent
const struct htp_tensor * k;
const struct htp_tensor * v;
- bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_threads >= 2
- uint32_t n_threads;
+ bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_hvx >= 2
+ uint32_t n_hvx;
// Op parameters
__fp16 scale;
@@ -171,7 +171,7 @@ struct hmx_fa_context {
HVX_Vector * vtcm_l_vec; // Row sum [g_br]
HVX_Vector * vtcm_s_rowmax; // Softmax intermediate [g_br]
HVX_Vector * vtcm_p_rowsum; // Softmax intermediate [g_br]
- HVX_Vector * vtcm_row_bufs; // Per-thread softmax row scratch [n_threads][2][Bc/64]
+ HVX_Vector * vtcm_row_bufs; // Per-HVX softmax row scratch [n_hvx][2][Bc/64]
uint8_t * vtcm_hmx_scales_id; // HMX output scales (identity)
uint8_t * vtcm_hmx_scales_qk; // HMX output scales (qk_scale)
__fp16 * vtcm_mask_buf; // VTCM mask buffer [Br * m_line], DMA'd per KV block
@@ -260,7 +260,7 @@ static uint32_t fa_scan_active_blocks(
return n_active;
}
-static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * data) {
+static void flash_attn_ext_f16_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_fa_context * factx = (struct htp_fa_context *) data;
const struct htp_ops_context * octx = factx->octx;
const struct htp_tensor * q = octx->src[0];
@@ -306,15 +306,15 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
const uint32_t nb3 = dst->nb[3];
// total rows in q
- const uint32_t dr = factx->qrows_per_thread;
+ const uint32_t dr = factx->qrows_per_hvx;
const uint32_t ir0 = factx->qrow_start + dr * ith;
const uint32_t ir1 = MIN(ir0 + dr, factx->qrow_start + factx->qrows);
if (ir0 >= ir1) return;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
const uint32_t DK = nek0;
const uint32_t DV = nev0;
@@ -774,7 +774,7 @@ typedef struct {
size_t buf_idx;
} fa_k_int_args_t;
-static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) {
+static void fa_k_interleave_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
fa_k_int_args_t * args = (fa_k_int_args_t *) data;
struct hmx_fa_context * factx = args->factx;
@@ -787,7 +787,7 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data)
return;
}
- struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start));
if (factx->k->type == HTP_TYPE_Q8_0) {
for (uint32_t r = start; r < end; ++r) {
@@ -801,18 +801,14 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data)
}
static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_rows, size_t src_stride, void * curr_k, uint32_t kv_start, size_t buf_idx) {
- work_queue_t wp = factx->octx->ctx->work_queue;
+ work_queue_t wp = factx->octx->ctx->hvx_queue;
uint32_t n = 1;
- if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) {
- n = factx->n_threads;
+ if (factx->n_hvx > 1 && kv_rows >= factx->n_hvx * 2) {
+ n = factx->n_hvx;
}
uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2);
fa_k_int_args_t args = { factx, kv_rows, src_stride, curr_k, kv_start, rows_per_t, buf_idx };
- if (n > 1) {
- work_queue_run(wp, fa_k_interleave_thread, &args, n);
- } else {
- fa_k_interleave_thread(1, 0, &args);
- }
+ work_queue_run(wp, fa_k_interleave_thread, &args, n);
}
typedef struct {
@@ -826,7 +822,7 @@ typedef struct {
uint32_t rows_per_t;
} fa_v_int_args_t;
-static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) {
+static void fa_v_interleave_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
fa_v_int_args_t * args = (fa_v_int_args_t *) data;
struct hmx_fa_context * factx = args->factx;
@@ -841,7 +837,7 @@ static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data)
__fp16 * v_tiles_dst = (__fp16 *) args->v_tiles_dst;
- struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start));
if (factx->v->type == HTP_TYPE_Q8_0) {
for (uint32_t r = start; r < end; ++r) {
@@ -861,18 +857,14 @@ static void fa_phase_v_interleave(struct hmx_fa_context * factx,
void * v_tiles_dst,
size_t n_col_tiles,
uint32_t kv_start) {
- work_queue_t wp = factx->octx->ctx->work_queue;
+ work_queue_t wp = factx->octx->ctx->hvx_queue;
uint32_t n = 1;
- if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) {
- n = factx->n_threads;
+ if (factx->n_hvx > 1 && kv_rows >= factx->n_hvx * 2) {
+ n = factx->n_hvx;
}
uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2);
fa_v_int_args_t args = { factx, kv_rows, src_stride, v_src, v_tiles_dst, n_col_tiles, kv_start, rows_per_t };
- if (n > 1) {
- work_queue_run(wp, fa_v_interleave_thread, &args, n);
- } else {
- fa_v_interleave_thread(1, 0, &args);
- }
+ work_queue_run(wp, fa_v_interleave_thread, &args, n);
}
typedef struct {
@@ -888,7 +880,7 @@ typedef struct {
atomic_uint barrier;
} fa_q_load_args_t;
-static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) {
+static void fa_q_load_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
fa_q_load_args_t * args = (fa_q_load_args_t *) data;
struct hmx_fa_context * factx = args->factx;
@@ -902,7 +894,7 @@ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) {
const size_t start = (size_t) i * rows_per_t;
const size_t end = hex_smin(start + rows_per_t, factx->g_br);
- struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start));
// Parallel initialization of per-block state
@@ -1010,10 +1002,10 @@ static void fa_phase_q_load(struct hmx_fa_context * factx,
uint32_t kv_head,
uint32_t ib3,
size_t n_rows_g) {
- work_queue_t wp = factx->octx->ctx->work_queue;
+ work_queue_t wp = factx->octx->ctx->hvx_queue;
uint32_t n = 1;
- if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) {
- n = factx->n_threads;
+ if (factx->n_hvx > 1 && n_rows_g >= (size_t) (factx->n_hvx * 2)) {
+ n = factx->n_hvx;
}
size_t rows_per_t = hex_align_up(hmx_ceil_div(factx->g_br, n), 2);
const uint32_t n_rows_q = hex_smin(factx->Br, factx->neq1 - q_start);
@@ -1028,11 +1020,7 @@ static void fa_phase_q_load(struct hmx_fa_context * factx,
args.n_rows_q = n_rows_q;
args.q_transposed = q->nb[1] < q->nb[2];
atomic_init(&args.barrier, n);
- if (n > 1) {
- work_queue_run(wp, fa_q_load_thread, &args, n);
- } else {
- fa_q_load_thread(1, 0, &args);
- }
+ work_queue_run(wp, fa_q_load_thread, &args, n);
}
typedef struct {
@@ -1046,7 +1034,7 @@ typedef struct {
size_t rows_per_t;
} fa_o_store_args_t;
-static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) {
+static void fa_o_store_thread_f32(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
fa_o_store_args_t * args = (fa_o_store_args_t *) data;
struct hmx_fa_context * factx = args->factx;
@@ -1062,7 +1050,7 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) {
return;
}
- struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));
const struct htp_tensor * dst = args->dst;
@@ -1113,7 +1101,7 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) {
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));
}
-static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) {
+static void fa_o_store_thread_f16(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
fa_o_store_args_t * args = (fa_o_store_args_t *) data;
struct hmx_fa_context * factx = args->factx;
@@ -1128,7 +1116,7 @@ static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) {
return;
}
- struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));
const struct htp_tensor * dst = args->dst;
@@ -1190,19 +1178,15 @@ static void fa_phase_o_store(struct hmx_fa_context * factx,
uint32_t kv_head,
uint32_t ib3,
size_t n_rows_g) {
- work_queue_t wp = factx->octx->ctx->work_queue;
+ work_queue_t wp = factx->octx->ctx->hvx_queue;
uint32_t n = 1;
- if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) {
- n = factx->n_threads;
+ if (factx->n_hvx > 1 && n_rows_g >= (size_t) (factx->n_hvx * 2)) {
+ n = factx->n_hvx;
}
size_t rows_per_t = hmx_ceil_div(n_rows_g, n);
fa_o_store_args_t args = { factx, dst, o_tile_src, q_start, kv_head, ib3, n_rows_g, rows_per_t };
- worker_callback_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16;
- if (n > 1) {
- work_queue_run(wp, store_fn, &args, n);
- } else {
- store_fn(1, 0, &args);
- }
+ work_queue_func_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16;
+ work_queue_run(wp, store_fn, &args, n);
}
typedef struct {
@@ -1229,7 +1213,7 @@ typedef struct {
} fa_softmax_args_t;
static inline void fa_softmax_impl(
- unsigned int n, unsigned int i, void * data,
+ const htp_thread_t * th, unsigned int n, unsigned int i, void * data,
const bool has_mask,
const bool mask_broadcast,
const bool is_g1,
@@ -1259,7 +1243,7 @@ static inline void fa_softmax_impl(
return;
}
- struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64));
// Per-thread row scratch: thread i uses bufs at offset i * 2 * stride
@@ -1619,8 +1603,8 @@ static inline void fa_softmax_impl(
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64));
}
-static void fa_softmax_thread_nomask(unsigned int n, unsigned int i, void * data) {
- fa_softmax_impl(n, i, data,
+static void fa_softmax_thread_nomask(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
+ fa_softmax_impl(th, n, i, data,
/*has_mask=*/false,
/*mask_broadcast=*/false,
/*is_g1=*/false,
@@ -1628,8 +1612,8 @@ static void fa_softmax_thread_nomask(unsigned int n, unsigned int i, void * data
/*has_softcap=*/false);
}
-static void fa_softmax_thread_mask_broadcast_g1(unsigned int n, unsigned int i, void * data) {
- fa_softmax_impl(n, i, data,
+static void fa_softmax_thread_mask_broadcast_g1(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
+ fa_softmax_impl(th, n, i, data,
/*has_mask=*/true,
/*mask_broadcast=*/true,
/*is_g1=*/true,
@@ -1637,8 +1621,8 @@ static void fa_softmax_thread_mask_broadcast_g1(unsigned int n, unsigned int i,
/*has_softcap=*/false);
}
-static void fa_softmax_thread_mask_broadcast_gn(unsigned int n, unsigned int i, void * data) {
- fa_softmax_impl(n, i, data,
+static void fa_softmax_thread_mask_broadcast_gn(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
+ fa_softmax_impl(th, n, i, data,
/*has_mask=*/true,
/*mask_broadcast=*/true,
/*is_g1=*/false,
@@ -1646,7 +1630,7 @@ static void fa_softmax_thread_mask_broadcast_gn(unsigned int n, unsigned int i,
/*has_softcap=*/false);
}
-static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) {
+static void fa_softmax_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
fa_softmax_args_t * args = (fa_softmax_args_t *) data;
struct hmx_fa_context * factx = args->factx;
@@ -1656,7 +1640,7 @@ static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) {
const bool has_alibi = args->has_alibi;
const bool has_softcap = factx->has_softcap;
- fa_softmax_impl(n, i, data, has_mask, mask_broadcast, is_g1, has_alibi, has_softcap);
+ fa_softmax_impl(th, n, i, data, has_mask, mask_broadcast, is_g1, has_alibi, has_softcap);
}
static inline HVX_Vector hvx_fa_load_sinks(const float * sinks_data, size_t idx, size_t count) {
@@ -1742,10 +1726,10 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx,
fa_softmax_args_t * sargs,
size_t n_row_tiles,
size_t n_row_tiles_g_br) {
- work_queue_t wp = factx->octx->ctx->work_queue;
+ work_queue_t wp = factx->octx->ctx->hvx_queue;
const size_t n_row_vec_cnt = hmx_ceil_div(sargs->n_rows_g, 64);
- worker_callback_t softmax_fn = fa_softmax_thread;
+ work_queue_func_t softmax_fn = fa_softmax_thread;
if (sargs->mask == NULL && !factx->has_softcap && !sargs->has_alibi) {
softmax_fn = fa_softmax_thread_nomask;
} else if (sargs->mask != NULL && factx->mask_broadcast && !factx->has_softcap && !sargs->has_alibi) {
@@ -1756,13 +1740,12 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx,
}
}
- if (factx->n_threads > 1 && n_row_vec_cnt >= 2) {
- uint32_t n_use = (uint32_t) hex_smin((size_t) factx->n_threads, n_row_vec_cnt);
+ uint32_t n_use = 1;
+ if (factx->n_hvx > 1 && n_row_vec_cnt >= 2) {
+ n_use = (uint32_t) hex_smin((size_t) factx->n_hvx, n_row_vec_cnt);
sargs->thread_div = init_fastdiv_values(n_use);
- work_queue_run(wp, softmax_fn, sargs, n_use);
- } else {
- softmax_fn(1, 0, sargs);
}
+ work_queue_run(wp, softmax_fn, sargs, n_use);
}
// ============================================================================
@@ -1780,7 +1763,7 @@ typedef struct {
uint8_t * hmx_scales;
} hmx_fa_qk_job_t;
-static void hmx_fa_qk_dot_worker(void * data) {
+static void hmx_fa_qk_dot_worker(const htp_thread_t * th, void * data) {
hmx_fa_qk_job_t * job = (hmx_fa_qk_job_t *) data;
const size_t n_row_tiles = job->n_row_tiles;
const size_t n_col_tiles = job->n_col_tiles;
@@ -1822,7 +1805,7 @@ typedef struct {
size_t DV;
} hmx_fa_o_update_job_t;
-static void hmx_fa_o_update_worker(void * data) {
+static void hmx_fa_o_update_worker(const htp_thread_t * th, void * data) {
hmx_fa_o_update_job_t * job = (hmx_fa_o_update_job_t *) data;
const size_t n_row_tiles = job->n_row_tiles;
const size_t n_col_tiles = job->n_col_tiles;
@@ -1867,7 +1850,7 @@ typedef struct {
size_t DV;
} hmx_fa_o_norm_job_t;
-static void hmx_fa_o_norm_worker(void * data) {
+static void hmx_fa_o_norm_worker(const htp_thread_t * th, void * data) {
hmx_fa_o_norm_job_t * job = (hmx_fa_o_norm_job_t *) data;
const size_t n_row_tiles = job->n_row_tiles;
const size_t n_row_tiles_g_br = job->n_row_tiles_g_br;
@@ -1992,7 +1975,6 @@ static inline void fa_prefetch_block(dma_queue * dma_q, const struct htp_tensor
int hmx_flash_attn_ext(struct htp_ops_context * octx) {
struct htp_thread_trace * tr_hvx = &octx->ctx->trace[0];
- struct htp_thread_trace * tr_hmx = &octx->ctx->trace[HTP_MAX_NTHREADS];
const struct htp_tensor * q = octx->src[0];
const struct htp_tensor * k = octx->src[1];
const struct htp_tensor * v = octx->src[2];
@@ -2037,7 +2019,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
factx.sinks = octx->src[4]; // NULL if this op has no attention sinks
factx.k = k;
factx.v = v;
- factx.n_threads = kparams->n_threads;
+ factx.n_hvx = kparams->n_hvx;
factx.DK = DK;
factx.DV = DV;
factx.DK_pad = DK_pad;
@@ -2077,7 +2059,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
const uint32_t Bc = factx.Bc;
const uint32_t g_br = factx.g_br;
const bool pipeline = factx.pipeline;
- const uint32_t n_threads = factx.n_threads;
+ const uint32_t n_hvx = factx.n_hvx;
const uint32_t G = factx.G;
// Multi-device: prefer head-parallel partitioning (each core owns a disjoint head
@@ -2147,7 +2129,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
// Build the VTCM layout once (shared with the host estimator) and place every
// scratch buffer at its computed offset. Padded head dims size the HMX tiles.
struct hmx_fa_vtcm_layout L;
- hmx_fa_vtcm_layout_build(&L, G, DK_pad, DV_pad, Br, Bc, n_threads, pipeline, factx.is_q_fp32, factx.sinks != NULL, factx.n_heads);
+ hmx_fa_vtcm_layout_build(&L, G, DK_pad, DV_pad, Br, Bc, n_hvx, pipeline, factx.is_q_fp32, factx.sinks != NULL, factx.n_heads);
if (L.total_bytes > ctx->vtcm_size) {
return HTP_STATUS_VTCM_TOO_SMALL;
@@ -2681,7 +2663,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -2776,16 +2758,16 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
factx.qrows = qrows;
factx.qrow_start = qrow_start;
- factx.qrows_per_thread = fastdiv(qrows + n_threads - 1, &octx->n_threads_div);
+ factx.qrows_per_hvx = fastdiv(qrows + n_hvx - 1, &octx->n_hvx_div);
const bool has_mask = (mask != NULL);
const bool has_sinks = (octx->src[4] != NULL);
struct hvx_fa_vtcm_layout L;
- hvx_fa_vtcm_layout_build(&L, k->ne[0], v->ne[0], factx.is_q_fp32, has_mask, has_sinks, n_head, n_threads);
+ hvx_fa_vtcm_layout_build(&L, k->ne[0], v->ne[0], factx.is_q_fp32, has_mask, has_sinks, n_head, n_hvx);
if (L.total_bytes > octx->ctx->vtcm_size) {
return HTP_STATUS_VTCM_TOO_SMALL;
@@ -2822,7 +2804,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
dma_queue_pop(dma_q);
}
- work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, flash_attn_ext_f16_thread, &factx, n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h
index f2654f0a8..70cf3e41a 100644
--- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h
+++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h
@@ -46,7 +46,7 @@ struct htp_fa_kernel_params {
uint8_t kernel_type; // enum htp_fa_kernel_type
uint8_t head_split; // 1 = partition by KV heads in multicore, 0 = token partition
uint8_t flags; // HTP_FA_FLAG_*
- uint8_t n_threads; // Number of threads to run
+ uint8_t n_hvx; // Number of threads to run
// Common parameters
uint16_t Br;
@@ -155,7 +155,7 @@ struct hmx_fa_vtcm_layout {
static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L,
size_t gqa_factor, size_t DK, size_t DV,
- size_t Br, size_t Bc, size_t n_threads,
+ size_t Br, size_t Bc, size_t n_hvx,
bool pipeline, bool is_q_fp32,
bool has_sinks, size_t n_heads) {
const size_t g_br = hex_align_up(gqa_factor * Br, HMX_FP16_TILE_N_ROWS);
@@ -207,7 +207,7 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L,
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_p_tiles[1], s_tile_size, pipeline);
VTCM_LAYOUT_ALLOC(off_group_b, off_s_rowmax, col_vec_size);
VTCM_LAYOUT_ALLOC(off_group_b, off_p_rowsum, col_vec_size);
- VTCM_LAYOUT_ALLOC(off_group_b, off_row_bufs, row_vec_size * 2 * n_threads);
+ VTCM_LAYOUT_ALLOC(off_group_b, off_row_bufs, row_vec_size * 2 * n_hvx);
const size_t group_b_size = off_group_b - off_group_b_c;
@@ -251,9 +251,9 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L,
}
// Exact VTCM usage for a given (gqa_factor, DK, DV, Br, Bc) configuration.
-static inline size_t hmx_fa_compute_vtcm_usage(size_t gqa_factor, size_t DK, size_t DV, size_t Br, size_t Bc, size_t n_threads, bool pipeline, bool is_q_fp32, bool has_sinks, size_t n_heads) {
+static inline size_t hmx_fa_compute_vtcm_usage(size_t gqa_factor, size_t DK, size_t DV, size_t Br, size_t Bc, size_t n_hvx, bool pipeline, bool is_q_fp32, bool has_sinks, size_t n_heads) {
struct hmx_fa_vtcm_layout L;
- hmx_fa_vtcm_layout_build(&L, gqa_factor, DK, DV, Br, Bc, n_threads, pipeline, is_q_fp32, has_sinks, n_heads);
+ hmx_fa_vtcm_layout_build(&L, gqa_factor, DK, DV, Br, Bc, n_hvx, pipeline, is_q_fp32, has_sinks, n_heads);
return L.total_bytes;
}
@@ -282,7 +282,7 @@ static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L,
size_t DK, size_t DV,
bool is_q_fp32, bool has_mask,
bool has_sinks, size_t n_heads,
- size_t n_threads) {
+ size_t n_hvx) {
const size_t size_q_row_padded = hex_round_up(DK * (is_q_fp32 ? 4 : 2), 128);
const size_t size_k_row_padded = hex_round_up(DK * sizeof(__fp16), 128);
const size_t size_v_row_padded = hex_round_up(DV * sizeof(__fp16), 128);
@@ -296,11 +296,11 @@ static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L,
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_q, size_q_block * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_k, size_k_block * 2 * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_v, size_v_block * 2 * n_threads);
- VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_m, size_m_block * HVX_FA_DMA_CACHE_SIZE * n_threads, has_mask);
- VTCM_LAYOUT_ALLOC(off, off_a, size_vkq_acc * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_q, size_q_block * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_k, size_k_block * 2 * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_v, size_v_block * 2 * n_hvx);
+ VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_m, size_m_block * HVX_FA_DMA_CACHE_SIZE * n_hvx, has_mask);
+ VTCM_LAYOUT_ALLOC(off, off_a, size_vkq_acc * n_hvx);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_sinks, size_sinks, has_sinks);
L->size_q_block = size_q_block;
@@ -312,9 +312,9 @@ static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L,
L->total_bytes = off;
}
-static inline size_t hvx_fa_compute_vtcm_usage(size_t DK, size_t DV, bool is_q_fp32, bool has_mask, bool has_sinks, size_t n_heads, size_t n_threads) {
+static inline size_t hvx_fa_compute_vtcm_usage(size_t DK, size_t DV, bool is_q_fp32, bool has_mask, bool has_sinks, size_t n_heads, size_t n_hvx) {
struct hvx_fa_vtcm_layout L;
- hvx_fa_vtcm_layout_build(&L, DK, DV, is_q_fp32, has_mask, has_sinks, n_heads, n_threads);
+ hvx_fa_vtcm_layout_build(&L, DK, DV, is_q_fp32, has_mask, has_sinks, n_heads, n_hvx);
return L.total_bytes;
}
@@ -329,7 +329,7 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out,
size_t qo_len,
size_t kv_len,
size_t vtcm_budget,
- size_t n_threads,
+ size_t n_hvx,
bool is_q_fp32,
bool has_sinks,
size_t n_heads,
@@ -337,7 +337,7 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out,
const size_t T = HMX_FP16_TILE_N_ROWS; // 32
const size_t br_unit = hmx_ceil_div(T, gqa_factor);
const size_t bc_unit = HMX_FP16_TILE_N_COLS * 2; // 64
- const bool can_pipeline = (kv_len >= FA_MIN_KV_BLOCKS * bc_unit && n_threads >= 2);
+ const bool can_pipeline = (kv_len >= FA_MIN_KV_BLOCKS * bc_unit && n_hvx >= 2);
// Br_max: largest Br aligned to br_unit that does not exceed qo_len.
const size_t Br_max = qo_len >= br_unit ? hex_align_down(qo_len, br_unit) : br_unit;
@@ -361,15 +361,15 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out,
for (size_t Br = Br_max; Br >= br_unit; Br -= br_unit) {
// Try all Bc candidates from Bc_limit down to bc_unit
for (size_t Bc = Bc_limit; Bc >= bc_unit; Bc -= bc_unit) {
- size_t vtcm_needed = hmx_fa_compute_vtcm_usage(gqa_factor, DK, DV, Br, Bc, n_threads, can_pipeline, is_q_fp32, has_sinks, n_heads);
+ size_t vtcm_needed = hmx_fa_compute_vtcm_usage(gqa_factor, DK, DV, Br, Bc, n_hvx, can_pipeline, is_q_fp32, has_sinks, n_heads);
if (vtcm_needed <= vtcm_budget) {
// This Bc fits for this Br!
const size_t q_blocks = (qo_len + Br - 1) / Br;
const size_t kv_blocks = (kv_len + Bc - 1) / Bc;
- const size_t actual_threads = (kv_blocks >= 3 && n_threads >= 2) ? n_threads : 1;
+ const size_t actual_hvx = (kv_blocks >= 3 && n_hvx >= 2) ? n_hvx : 1;
const size_t n_rows_g = Br * gqa_factor;
const size_t n_row_vec_cnt = (n_rows_g + 63) / 64;
- const size_t n_use = n_row_vec_cnt < actual_threads ? n_row_vec_cnt : actual_threads;
+ const size_t n_use = n_row_vec_cnt < actual_hvx ? n_row_vec_cnt : actual_hvx;
const size_t vecs_per_t = n_use > 0 ? (n_row_vec_cnt + n_use - 1) / n_use : 1;
const size_t c_iter_actual = c_iter_base + c_softmax * vecs_per_t;
diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
index a70f41185..0280f99a4 100644
--- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
+++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
@@ -748,7 +748,7 @@ static inline void gdn_step_scalar_f32(
}
}
-static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, void * data) {
+static void gated_delta_net_f32_pp_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_gdn_context * gctx = (struct htp_gdn_context *) data;
struct htp_ops_context * octx = gctx->octx;
const struct htp_gdn_kernel_params * kparams = gctx->kparams;
@@ -777,10 +777,10 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo
float * dst_base = (float *) (uintptr_t) dst->data;
float * state_out_base = dst_cache ? (float *) (uintptr_t) dst_cache->data : (dst_base + S_v * H * n_tokens * n_seqs);
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
const struct htp_gdn_vtcm_layout * layout = &gctx->layout;
float * s_work[2];
- s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_thread * ith);
+ s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_hvx * ith);
s_work[1] = s_work[0] + layout->state_aligned / sizeof(float);
const struct fastdiv_values * fd_H = &kparams->div_H;
@@ -817,7 +817,7 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo
spad_idx ^= 1;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
int curr_spad_idx = 0;
for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) {
@@ -892,7 +892,7 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo
dma_queue_flush(dma_q);
}
-static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, void * data) {
+static void gated_delta_net_f32_tg_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_gdn_context * gctx = (struct htp_gdn_context *) data;
struct htp_ops_context * octx = gctx->octx;
const struct htp_gdn_kernel_params * kparams = gctx->kparams;
@@ -918,10 +918,10 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo
const float scale = kparams->scale;
float * dst_base = (float *) (uintptr_t) dst->data;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
const struct htp_gdn_vtcm_layout * layout = &gctx->layout;
float * s_work[2];
- s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_thread * ith);
+ s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_hvx * ith);
s_work[1] = s_work[0] + layout->state_aligned / sizeof(float);
const struct fastdiv_values * fd_H = &kparams->div_H;
@@ -957,7 +957,7 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo
spad_idx ^= 1;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
int curr_spad_idx = 0;
for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) {
@@ -1031,7 +1031,7 @@ struct htp_gdn_hmx_gemm_task {
uint8_t * hmx_scales;
};
-static void htp_gdn_hmx_gemm_worker(void * data) {
+static void htp_gdn_hmx_gemm_worker(const htp_thread_t * th, void * data) {
struct htp_gdn_hmx_gemm_task * task = (struct htp_gdn_hmx_gemm_task *) data;
asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)task->hmx_scales));
@@ -1623,7 +1623,7 @@ static inline void gdn_init_head_ptrs(
head->rows_inv = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_inv + h * row_vecs_sz);
head->rows_a = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_a + h * row_vecs_sz);
- head->vtcm_m = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_thread_scratch + h * (64 * 128));
+ head->vtcm_m = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_hvx_scratch + h * (64 * 128));
head->vtcm_tmp = head->vtcm_m + 32;
head->iv1 = base_iv1 + h;
@@ -1654,10 +1654,10 @@ struct htp_gdn_batch_context {
const struct htp_gdn_kernel_params * kparams;
};
-static void gdn_hvx_init_state_worker(unsigned int n, unsigned int i, void * data) {
+static void gdn_hvx_init_state_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
(void) n;
struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data;
- struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0);
struct htp_gdn_head_ptrs * head = &bctx->heads[i];
@@ -1690,10 +1690,10 @@ static inline HVX_Vector hvx_splat_last_f32(HVX_Vector v) {
return hvx_vec_repl4(Q6_V_vror_VR(v, 124));
}
-static void gdn_hvx_phase1a_worker(unsigned int n, unsigned int i, void * data) {
+static void gdn_hvx_phase1a_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
(void) n;
struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data;
- struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
const uint16_t info = (uint16_t) bctx->c;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info);
@@ -1763,10 +1763,10 @@ static void gdn_hvx_phase1a_worker(unsigned int n, unsigned int i, void * data)
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info);
}
-static void gdn_hvx_phase1b_worker(unsigned int n, unsigned int i, void * data) {
+static void gdn_hvx_phase1b_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
(void) n;
struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data;
- struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
const uint16_t info = (uint16_t) bctx->c;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info);
@@ -1860,10 +1860,10 @@ static void gdn_hvx_phase1b_worker(unsigned int n, unsigned int i, void * data)
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info);
}
-static void gdn_hvx_phase2_worker(unsigned int n, unsigned int i, void * data) {
+static void gdn_hvx_phase2_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
(void) n;
struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data;
- struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
const uint16_t info = (uint16_t) bctx->c;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_SOLVE, info);
@@ -1884,10 +1884,10 @@ static void gdn_hvx_phase2_worker(unsigned int n, unsigned int i, void * data) {
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_SOLVE, info);
}
-static void gdn_hvx_phase3_worker(unsigned int n, unsigned int i, void * data) {
+static void gdn_hvx_phase3_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
(void) n;
struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data;
- struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
const uint16_t info = (uint16_t) bctx->c;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_V_PREP, info);
@@ -1928,10 +1928,10 @@ static void gdn_hvx_phase3_worker(unsigned int n, unsigned int i, void * data) {
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_V_PREP, info);
}
-static void gdn_hvx_phase4_worker(unsigned int n, unsigned int i, void * data) {
+static void gdn_hvx_phase4_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
(void) n;
struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data;
- struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
const uint16_t info = (uint16_t) bctx->c;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_D_PREP, info);
@@ -1959,10 +1959,10 @@ static void gdn_hvx_phase4_worker(unsigned int n, unsigned int i, void * data) {
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_D_PREP, info);
}
-static void gdn_hvx_phase5_worker(unsigned int n, unsigned int i, void * data) {
+static void gdn_hvx_phase5_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
(void) n;
struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data;
- struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
const uint16_t info = (uint16_t) bctx->c;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_OUT, info);
@@ -1984,10 +1984,10 @@ static void gdn_hvx_phase5_worker(unsigned int n, unsigned int i, void * data) {
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_OUT, info);
}
-static void gdn_hvx_phase6_worker(unsigned int n, unsigned int i, void * data) {
+static void gdn_hvx_phase6_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
(void) n;
struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data;
- struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i];
+ struct htp_thread_trace * tr = th->trace;
const uint16_t info = (uint16_t) bctx->c;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_STATE, info);
@@ -2040,7 +2040,7 @@ static int gated_delta_net_f32_hmx_chunked(
const uint32_t n_sv_tiles = S_v / 32;
struct htp_gdn_hmx_vtcm_layout L;
- htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, kparams->n_heads_batch, kparams->n_threads, kparams->pipeline != 0);
+ htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, kparams->n_heads_batch, kparams->n_hvx, kparams->pipeline != 0);
if (L.total_bytes > octx->ctx->vtcm_size) {
return HTP_STATUS_VTCM_TOO_SMALL;
@@ -2062,7 +2062,7 @@ static int gated_delta_net_f32_hmx_chunked(
hmx_queue_t hmx_q = octx->ctx->hmx_queue;
dma_queue * dma_q = octx->ctx->dma[0];
- work_queue_t wp = octx->ctx->work_queue;
+ work_queue_t wp = octx->ctx->hvx_queue;
struct htp_gdn_head_ptrs heads[8];
struct htp_gdn_hmx_gemm_task gemm_tasks[8][9];
@@ -2286,7 +2286,7 @@ int op_gated_delta_net(struct htp_ops_context * octx) {
}
const struct htp_gdn_kernel_params * kparams = (const struct htp_gdn_kernel_params *) octx->kernel_params;
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -2321,7 +2321,7 @@ int op_gated_delta_net(struct htp_ops_context * octx) {
return gated_delta_net_f32_hmx_chunked(octx, kparams, row_start, nrows);
}
- const uint32_t n_threads = (nrows < octx->n_threads) ? nrows : octx->n_threads;
+ const uint32_t n_hvx = (nrows < octx->n_hvx) ? nrows : octx->n_hvx;
struct htp_gdn_context gctx;
gctx.octx = octx;
@@ -2329,7 +2329,7 @@ int op_gated_delta_net(struct htp_ops_context * octx) {
gctx.row_start = row_start;
gctx.nrows = nrows;
- htp_gdn_vtcm_layout_build(&gctx.layout, S_v, n_threads);
+ htp_gdn_vtcm_layout_build(&gctx.layout, S_v, n_hvx);
gctx.vtcm_base = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, gctx.layout.off_state);
if (gctx.layout.total_bytes > octx->ctx->vtcm_size) {
@@ -2337,18 +2337,18 @@ int op_gated_delta_net(struct htp_ops_context * octx) {
}
FARF(HIGH, "gated-delta-net-f32: q(%ux%ux%ux%u) k(%ux%ux%ux%u) v(%ux%ux%ux%u) state(%ux%ux%ux%u) -> (%ux%ux%ux%u) : "
- "vtcm-size %zu n_threads %u\n",
+ "vtcm-size %zu n_hvx %u\n",
octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3],
octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3],
octx->src[2]->ne[0], octx->src[2]->ne[1], octx->src[2]->ne[2], octx->src[2]->ne[3],
octx->src[5]->ne[0], octx->src[5]->ne[1], octx->src[5]->ne[2], octx->src[5]->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
- gctx.layout.total_bytes, n_threads);
+ gctx.layout.total_bytes, n_hvx);
if (n_tokens == 1) {
- work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_tg_thread, &gctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, gated_delta_net_f32_tg_thread, &gctx, n_hvx);
} else {
- work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_pp_thread, &gctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, gated_delta_net_f32_pp_thread, &gctx, n_hvx);
}
return HTP_STATUS_OK;
diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h
index 0d8818588..aff1eb645 100644
--- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h
+++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h
@@ -29,7 +29,7 @@ struct htp_gdn_kernel_params {
uint16_t n_chunks;
uint16_t n_heads_batch;
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t S_v;
uint32_t H;
uint32_t n_tokens;
@@ -59,7 +59,7 @@ _Static_assert(sizeof(struct htp_gdn_kernel_params) <= 128, "htp_gdn_kernel_para
struct htp_gdn_vtcm_layout {
size_t state_aligned;
- size_t bytes_per_thread;
+ size_t bytes_per_hvx;
size_t off_state;
size_t total_bytes;
};
@@ -67,13 +67,13 @@ struct htp_gdn_vtcm_layout {
static inline void htp_gdn_vtcm_layout_build(
struct htp_gdn_vtcm_layout * L,
uint32_t S_v,
- uint32_t n_threads
+ uint32_t n_hvx
) {
size_t state_bytes = (size_t) S_v * S_v * sizeof(float);
L->state_aligned = hex_round_up(state_bytes, 128);
- L->bytes_per_thread = 2 * L->state_aligned;
+ L->bytes_per_hvx = 2 * L->state_aligned;
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_state, L->bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_state, L->bytes_per_hvx * n_hvx);
L->total_bytes = off;
}
@@ -128,7 +128,7 @@ struct htp_gdn_hmx_vtcm_layout {
size_t off_rows_inv;
size_t off_rows_a;
- size_t off_thread_scratch;
+ size_t off_hvx_scratch;
size_t off_scales_1;
size_t state_f32_bytes;
@@ -140,7 +140,7 @@ struct htp_gdn_hmx_vtcm_layout {
size_t tile_64x64_bytes;
uint32_t n_heads_batch;
- uint32_t n_threads;
+ uint32_t n_hvx;
bool pipeline;
size_t total_bytes;
};
@@ -150,16 +150,16 @@ static inline void htp_gdn_hmx_vtcm_layout_build(
uint32_t S_v,
uint32_t chunk_size,
uint32_t n_heads_batch,
- uint32_t n_threads,
+ uint32_t n_hvx,
bool pipeline
) {
memset(L, 0, sizeof(*L));
L->n_heads_batch = n_heads_batch;
- L->n_threads = n_threads;
+ L->n_hvx = n_hvx;
L->pipeline = pipeline;
const size_t bh = (size_t) n_heads_batch;
- const size_t nth = (size_t) (n_threads > 0 ? n_threads : 1);
+ const size_t nth = (size_t) (n_hvx > 0 ? n_hvx : 1);
const size_t state_f32_sz = hex_round_up(S_v * S_v * sizeof(float), 2048);
const size_t state_f16_sz = hex_round_up(S_v * S_v * sizeof(__fp16), 2048);
@@ -248,9 +248,9 @@ static inline void htp_gdn_hmx_vtcm_layout_build(
VTCM_LAYOUT_ALLOC(off, off_rows_inv, bh * row_vecs_sz);
VTCM_LAYOUT_ALLOC(off, off_rows_a, bh * row_vecs_sz);
- const size_t thread_scratch_sz = 64 * 128;
+ const size_t hvx_scratch_sz = 64 * 128;
off = hex_align_up(off, HMX_FP16_TILE_SIZE);
- VTCM_LAYOUT_ALLOC(off, off_thread_scratch, nth * thread_scratch_sz);
+ VTCM_LAYOUT_ALLOC(off, off_hvx_scratch, nth * hvx_scratch_sz);
off = hex_align_up(off, HMX_FP16_TILE_SIZE);
VTCM_LAYOUT_ALLOC(off, off_scales_1, HMX_FP16_TILE_SIZE);
@@ -263,7 +263,7 @@ static inline bool htp_gdn_hmx_solve_layout(
uint32_t chunk_size,
uint32_t total_rows,
size_t vtcm_budget,
- uint32_t n_threads,
+ uint32_t n_hvx,
bool pipeline,
uint32_t * n_heads_batch_out
) {
@@ -271,8 +271,8 @@ static inline bool htp_gdn_hmx_solve_layout(
if (max_batch > total_rows) {
max_batch = total_rows;
}
- if (max_batch > n_threads) {
- max_batch = n_threads;
+ if (max_batch > n_hvx) {
+ max_batch = n_hvx;
}
static const uint32_t candidates[] = { 8, 6, 4, 2, 1 };
for (size_t i = 0; i < sizeof(candidates) / sizeof(candidates[0]); ++i) {
@@ -281,7 +281,7 @@ static inline bool htp_gdn_hmx_solve_layout(
continue;
}
struct htp_gdn_hmx_vtcm_layout L;
- htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, bh, n_threads, pipeline);
+ htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, bh, n_hvx, pipeline);
if (L.total_bytes <= vtcm_budget) {
*layout_out = L;
*n_heads_batch_out = bh;
@@ -289,7 +289,7 @@ static inline bool htp_gdn_hmx_solve_layout(
}
}
if (pipeline) {
- return htp_gdn_hmx_solve_layout(layout_out, S_v, chunk_size, total_rows, vtcm_budget, n_threads, false, n_heads_batch_out);
+ return htp_gdn_hmx_solve_layout(layout_out, S_v, chunk_size, total_rows, vtcm_budget, n_hvx, false, n_heads_batch_out);
}
return false;
}
diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.c b/ggml/src/ggml-hexagon/htp/gather-rows.c
index 1e88445f1..02804db27 100644
--- a/ggml/src/ggml-hexagon/htp/gather-rows.c
+++ b/ggml/src/ggml-hexagon/htp/gather-rows.c
@@ -3,7 +3,7 @@
#include <assert.h>
#include "hex-common.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "hvx-utils.h"
#include "dma-queue.h"
#include "htp-ctx.h"
@@ -23,7 +23,7 @@ struct gather_rows_task {
dma_addr_t dst;
uint32_t r_start;
uint32_t rows;
- uint32_t rows_per_thread;
+ uint32_t rows_per_hvx;
const struct htp_gather_rows_params * kparams;
gather_rows_compute_fn_t compute;
};
@@ -190,21 +190,21 @@ static void gather_rows_compute_f16_vgn(
}
}
-static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * data) {
+static void gather_rows_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
const struct gather_rows_task * task = (const struct gather_rows_task *) data;
const struct htp_gather_rows_params * kparams = task->kparams;
const struct htp_tensor * a = task->a;
const struct htp_tensor * b = task->b;
- const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows);
+ const uint32_t r_offset = MIN(ith * task->rows_per_hvx, task->rows);
const uint32_t r_beg = task->r_start + r_offset;
- const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows);
+ const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_hvx, task->rows);
if (r_beg >= r_end) {
return;
}
- struct htp_thread_trace * tr = &task->octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
struct htp_gather_rows_vtcm_layout layout;
htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 4);
@@ -227,7 +227,7 @@ static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * da
gather_rows_init_tables_f32(tab, inc, kparams, a, b);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
- dma_queue * q = task->octx->ctx->dma[ith];
+ dma_queue * q = th->dma_q;
const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0);
const uint32_t nr_max = kparams->nr_max;
const uint32_t nr0 = MIN(nr_max, r_end - r_beg);
@@ -271,21 +271,21 @@ static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * da
dma_queue_flush(q);
}
-static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * data) {
+static void gather_rows_thread_f16(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
const struct gather_rows_task * task = (const struct gather_rows_task *) data;
const struct htp_gather_rows_params * kparams = task->kparams;
const struct htp_tensor * a = task->a;
const struct htp_tensor * b = task->b;
- const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows);
+ const uint32_t r_offset = MIN(ith * task->rows_per_hvx, task->rows);
const uint32_t r_beg = task->r_start + r_offset;
- const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows);
+ const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_hvx, task->rows);
if (r_beg >= r_end) {
return;
}
- struct htp_thread_trace * tr = &task->octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
struct htp_gather_rows_vtcm_layout layout;
htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 2);
@@ -308,7 +308,7 @@ static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * da
gather_rows_init_tables_f16(tab, inc, kparams, a, b);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
- dma_queue * q = task->octx->ctx->dma[ith];
+ dma_queue * q = th->dma_q;
const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0);
const uint32_t nr_max = kparams->nr_max;
@@ -357,13 +357,13 @@ int htp_gather_rows(
const struct htp_tensor * a,
const struct htp_tensor * b,
const struct htp_gather_rows_params * kparams,
- uint32_t n_threads) {
+ uint32_t n_hvx) {
- if (!htp_ops_context_set_n_threads(octx, n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
- assert(kparams->slice * octx->n_threads <= octx->ctx->vtcm_size);
+ assert(kparams->slice * octx->n_hvx <= octx->ctx->vtcm_size);
const uint32_t total_rows = kparams->total_rows;
uint32_t r_start = 0;
@@ -389,7 +389,7 @@ int htp_gather_rows(
task.r_start = r_start;
task.rows = r_count;
task.kparams = kparams;
- task.rows_per_thread = kparams->rows_per_thread;
+ task.rows_per_hvx = kparams->rows_per_hvx;
if (elem_size == 2) {
task.compute = (kparams->vg == 1) ? gather_rows_compute_f16_vg1 : gather_rows_compute_f16_vgn;
@@ -398,6 +398,6 @@ int htp_gather_rows(
}
work_queue_func_t worker_func = (elem_size == 2) ? gather_rows_thread_f16 : gather_rows_thread_f32;
- work_queue_run(octx->ctx->work_queue, worker_func, &task, octx->n_threads);
+ work_queue_run(octx->ctx->hvx_queue, worker_func, &task, octx->n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.h b/ggml/src/ggml-hexagon/htp/gather-rows.h
index c08ef4d38..9ca215ac3 100644
--- a/ggml/src/ggml-hexagon/htp/gather-rows.h
+++ b/ggml/src/ggml-hexagon/htp/gather-rows.h
@@ -22,7 +22,7 @@ struct htp_gather_rows_params {
uint32_t region_size;
uint32_t slice;
uint32_t total_rows;
- uint32_t rows_per_thread;
+ uint32_t rows_per_hvx;
uint8_t b_dense;
uint8_t pad[3];
};
@@ -35,7 +35,7 @@ struct htp_gather_rows_vtcm_layout {
size_t off_out0;
size_t off_region1;
size_t off_out1;
- size_t total_per_thread;
+ size_t total_per_hvx;
};
static inline void htp_gather_rows_vtcm_layout_build(
@@ -58,7 +58,7 @@ static inline void htp_gather_rows_vtcm_layout_build(
VTCM_LAYOUT_ALLOC(off, off_out0, out_aligned);
VTCM_LAYOUT_ALLOC(off, off_region1, region_aligned);
VTCM_LAYOUT_ALLOC(off, off_out1, out_aligned);
- L->total_per_thread = off;
+ L->total_per_hvx = off;
}
static inline bool htp_gather_rows_solve_layout(
@@ -67,7 +67,7 @@ static inline bool htp_gather_rows_solve_layout(
uint32_t nb, uint32_t b_nb1, bool b_dense,
uint32_t elem_size,
uint32_t total_rows,
- uint32_t n_threads,
+ uint32_t n_hvx,
uint32_t mdev_count,
uint32_t vtcm_size) {
@@ -86,13 +86,13 @@ static inline bool htp_gather_rows_solve_layout(
const uint32_t rg = v_lanes / g;
const uint32_t vg = ne / g;
- const uint32_t slice = (vtcm_size / n_threads) & ~127u;
+ const uint32_t slice = (vtcm_size / n_hvx) & ~127u;
const uint32_t total_chunks = total_rows / rg;
const uint32_t chunks_per_dev = (total_chunks + mdev_count - 1) / mdev_count;
const uint32_t max_rows_per_dev = (mdev_count > 1 && total_chunks >= mdev_count)
? (chunks_per_dev * rg + (total_rows % rg))
: total_rows;
- const uint32_t rows_per_thread = hex_round_up((max_rows_per_dev + n_threads - 1) / n_threads, rg);
+ const uint32_t rows_per_hvx = hex_round_up((max_rows_per_dev + n_hvx - 1) / n_hvx, rg);
const uint32_t fixed = 3 * vg * 128;
const uint32_t per_row = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0) + ne * elem_size;
@@ -101,7 +101,7 @@ static inline bool htp_gather_rows_solve_layout(
}
const uint32_t avail_per_buf = (slice - fixed) / 2;
- uint32_t nr_max = MIN(avail_per_buf / per_row, rows_per_thread);
+ uint32_t nr_max = MIN(avail_per_buf / per_row, rows_per_hvx);
if (elem_size == 2) {
const uint32_t per_row_region = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0);
if (per_row_region > 0) {
@@ -132,7 +132,7 @@ static inline bool htp_gather_rows_solve_layout(
struct htp_gather_rows_vtcm_layout layout;
htp_gather_rows_vtcm_layout_build(&layout, vg, region_size, nr_max, ne, elem_size);
- if ((elem_size != 2 || region_size <= 32768) && layout.total_per_thread <= slice) {
+ if ((elem_size != 2 || region_size <= 32768) && layout.total_per_hvx <= slice) {
break;
}
@@ -154,7 +154,7 @@ static inline bool htp_gather_rows_solve_layout(
kparams->region_size = region_size;
kparams->slice = slice;
kparams->total_rows = total_rows;
- kparams->rows_per_thread = rows_per_thread;
+ kparams->rows_per_hvx = rows_per_hvx;
kparams->b_dense = b_dense ? 1 : 0;
kparams->pad[0] = 0;
kparams->pad[1] = 0;
@@ -183,7 +183,7 @@ int htp_gather_rows(
const struct htp_tensor * a,
const struct htp_tensor * b,
const struct htp_gather_rows_params * kparams,
- uint32_t n_threads);
+ uint32_t n_hvx);
#endif
#endif // HTP_GATHER_ROWS_H
diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c
index 39acdfda6..83e85f005 100644
--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c
+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c
@@ -37,7 +37,7 @@ struct get_rows_context {
get_rows_tiled_compute_fn_t compute_tiled;
uint32_t task_start;
uint32_t tasks;
- uint32_t tasks_per_thread;
+ uint32_t tasks_per_hvx;
uint32_t tile_size;
uint32_t tile_stride;
uint32_t src0_row_size;
@@ -95,20 +95,20 @@ static __attribute__((noinline)) uint32_t get_rows_read_index_vtcm(
return (uint32_t) Q6_R_vextract_VR(v, off);
}
-static void get_rows_thread_sametype(unsigned int nth, unsigned int ith, void * data) {
+static void get_rows_thread_sametype(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct get_rows_context * grctx = (struct get_rows_context *) data;
struct htp_ops_context * octx = grctx->octx;
const struct htp_get_rows_kernel_params * kparams = grctx->kparams;
get_rows_preamble;
- const uint32_t dr = grctx->tasks_per_thread;
+ const uint32_t dr = grctx->tasks_per_hvx;
const uint32_t ir0 = grctx->task_start + dr * ith;
if (ir0 >= grctx->task_start + grctx->tasks) {
return;
}
const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);
const uint32_t row_size_bytes = grctx->src0_row_size;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
for (uint32_t i = ir0; i < ir1; ++i) {
const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11);
@@ -174,23 +174,23 @@ static inline struct get_rows_flat_task get_rows_flat_calc_task(
return task;
}
-static void get_rows_thread_flat(unsigned int nth, unsigned int ith, void * data) {
+static void get_rows_thread_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct get_rows_context * grctx = (struct get_rows_context *) data;
struct htp_ops_context * octx = grctx->octx;
get_rows_preamble;
- const uint32_t dr = grctx->tasks_per_thread;
+ const uint32_t dr = grctx->tasks_per_hvx;
const uint32_t ir0 = grctx->task_start + dr * ith;
if (ir0 >= grctx->task_start + grctx->tasks) {
return;
}
const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);
- dma_queue * dma_q = octx->ctx->dma[ith];
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ dma_queue * dma_q = th->dma_q;
+ struct htp_thread_trace * tr = th->trace;
const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout;
- uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread;
- uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_thread;
+ uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_hvx;
+ uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_hvx;
const uint32_t src0_bytes = grctx->src0_row_size;
const uint32_t dst_bytes = grctx->dst_row_size;
get_rows_flat_compute_fn_t compute = grctx->compute_flat;
@@ -276,12 +276,12 @@ static inline struct get_rows_tiled_task get_rows_tiled_calc_task(
return task;
}
-static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * data) {
+static void get_rows_thread_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct get_rows_context * grctx = (struct get_rows_context *) data;
struct htp_ops_context * octx = grctx->octx;
get_rows_preamble;
- const uint32_t dr = grctx->tasks_per_thread;
+ const uint32_t dr = grctx->tasks_per_hvx;
const uint32_t ir0 = grctx->task_start + dr * ith;
if (ir0 >= grctx->task_start + grctx->tasks) {
return;
@@ -290,10 +290,10 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat
const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);
const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(octx->src[0]->type);
const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout;
- uint8_t * src_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread;
- uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_thread;
- dma_queue * dma_q = octx->ctx->dma[ith];
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ uint8_t * src_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_hvx;
+ uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_hvx;
+ dma_queue * dma_q = th->dma_q;
+ struct htp_thread_trace * tr = th->trace;
const uint32_t tile_size = grctx->tile_size;
const uint32_t tile_stride = grctx->tile_stride;
@@ -414,11 +414,11 @@ int op_get_rows(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
struct get_rows_context grctx;
grctx.octx = octx;
@@ -428,14 +428,14 @@ int op_get_rows(struct htp_ops_context * octx) {
grctx.compute_tiled = NULL;
grctx.task_start = task_start;
grctx.tasks = tasks;
- grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div);
+ grctx.tasks_per_hvx = fastdiv(tasks + n_hvx - 1, &octx->n_hvx_div);
grctx.tile_size = 0;
grctx.tile_stride = 0;
grctx.src0_row_size = src0_row_size;
grctx.dst_row_size = dst_row_size;
grctx.idx_type = octx->src[1]->type;
- htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, total_tasks, grctx.idx_type, n_threads);
+ htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, total_tasks, grctx.idx_type, n_hvx);
if (grctx.vtcm_layout.total_bytes > octx->ctx->vtcm_size) {
FARF(ERROR, "get-rows: VTCM reservation %zu is too small, needed %zu\n",
@@ -497,14 +497,14 @@ int op_get_rows(struct htp_ops_context * octx) {
grctx.read_index = get_rows_read_index_vtcm;
}
- FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-threads %d\n",
+ FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-hvx %d\n",
octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3],
octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3],
octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3],
- grctx.vtcm_layout.src0_bytes_per_thread * n_threads,
- grctx.vtcm_layout.dst_bytes_per_thread * n_threads,
- kparams->kernel_type, n_threads);
+ grctx.vtcm_layout.src0_bytes_per_hvx * n_hvx,
+ grctx.vtcm_layout.dst_bytes_per_hvx * n_hvx,
+ kparams->kernel_type, n_hvx);
- work_queue_run(octx->ctx->work_queue, q_func, &grctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, q_func, &grctx, n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.h
index 4634be2e2..986966f11 100644
--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h
+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h
@@ -16,7 +16,7 @@ enum htp_get_rows_kernel_type {
struct htp_get_rows_kernel_params {
uint8_t kernel_type;
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t total_tasks;
uint32_t vtcm_size;
@@ -34,8 +34,8 @@ struct htp_get_rows_vtcm_layout {
size_t off_src0;
size_t off_dst;
- size_t src0_bytes_per_thread;
- size_t dst_bytes_per_thread;
+ size_t src0_bytes_per_hvx;
+ size_t dst_bytes_per_hvx;
size_t src0_spad_half_size;
size_t dst_spad_half_size;
@@ -48,7 +48,7 @@ static inline void htp_get_rows_vtcm_layout_build(
uint32_t ne00,
uint32_t total_tasks,
int idx_type,
- uint32_t n_threads) {
+ uint32_t n_hvx) {
memset(L, 0, sizeof(*L));
@@ -71,14 +71,14 @@ static inline void htp_get_rows_vtcm_layout_build(
L->src0_spad_half_size = (src0_row_size + 255) & ~255;
L->dst_spad_half_size = (ne00 * sizeof(float) + 255) & ~255;
- L->src0_bytes_per_thread = 2 * L->src0_spad_half_size;
- L->dst_bytes_per_thread = 2 * L->dst_spad_half_size;
+ L->src0_bytes_per_hvx = 2 * L->src0_spad_half_size;
+ L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size;
}
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src1, src1_alloc_bytes);
- VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx);
L->total_bytes = off;
}
diff --git a/ggml/src/ggml-hexagon/htp/glu-ops.c b/ggml/src/ggml-hexagon/htp/glu-ops.c
index dc621bf46..223c1829f 100644
--- a/ggml/src/ggml-hexagon/htp/glu-ops.c
+++ b/ggml/src/ggml-hexagon/htp/glu-ops.c
@@ -96,7 +96,7 @@ struct htp_glu_context {
uint32_t block;
uint32_t src0_nrows;
- uint32_t src0_nrows_per_thread;
+ uint32_t src0_nrows_per_hvx;
uint32_t row_start;
int nc;
@@ -104,9 +104,9 @@ struct htp_glu_context {
uint8_t * vtcm_src1;
uint8_t * vtcm_dst;
- size_t vtcm_src0_size_per_thread;
- size_t vtcm_src1_size_per_thread;
- size_t vtcm_dst_size_per_thread;
+ size_t vtcm_src0_size_per_hvx;
+ size_t vtcm_src1_size_per_hvx;
+ size_t vtcm_dst_size_per_hvx;
};
#define htp_glu_op_preamble \
@@ -722,11 +722,11 @@ static void sigglu_f16(const _Float16 * restrict src0,
}
#define DEFINE_GLU_PER_THREAD(TYPE, SUFFIX) \
- static void glu_##SUFFIX##_per_thread(unsigned int nth, unsigned int ith, void * data) { \
+ static void glu_##SUFFIX##_per_hvx(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { \
struct htp_glu_context * gctx = (struct htp_glu_context *) data; \
htp_glu_preamble; \
\
- struct htp_thread_trace * tr = gctx->octx->ctx ? &gctx->octx->ctx->trace[ith] : NULL; \
+ struct htp_thread_trace * tr = th->trace; \
\
size_t src0_row_size = gctx->src0_row_size; \
size_t src1_row_size = gctx->src1_row_size; \
@@ -736,10 +736,10 @@ static void sigglu_f16(const _Float16 * restrict src0,
size_t src1_row_stride = gctx->src1_row_stride; \
\
const uint32_t src0_nrows = gctx->src0_nrows; \
- const uint32_t src0_nrows_per_thread = gctx->src0_nrows_per_thread; \
+ const uint32_t src0_nrows_per_hvx = gctx->src0_nrows_per_hvx; \
\
- const uint32_t src0_start_row = gctx->row_start + src0_nrows_per_thread * ith; \
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, gctx->row_start + src0_nrows); \
+ const uint32_t src0_start_row = gctx->row_start + src0_nrows_per_hvx * ith; \
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, gctx->row_start + src0_nrows); \
\
/* no work for this thread */ \
if (src0_start_row >= src0_end_row) { \
@@ -754,9 +754,9 @@ static void sigglu_f16(const _Float16 * restrict src0,
const size_t src1_row_size_aligned = gctx->src1_row_size_aligned; \
const size_t dst_row_size_aligned = gctx->dst_row_size_aligned; \
\
- uint8_t * restrict src0_spad_data = gctx->vtcm_src0 + (ith * gctx->vtcm_src0_size_per_thread); \
- uint8_t * restrict src1_spad_data = gctx->vtcm_src1 + (ith * gctx->vtcm_src1_size_per_thread); \
- uint8_t * restrict dst_spad_data = gctx->vtcm_dst + (ith * gctx->vtcm_dst_size_per_thread); \
+ uint8_t * restrict src0_spad_data = gctx->vtcm_src0 + (ith * gctx->vtcm_src0_size_per_hvx); \
+ uint8_t * restrict src1_spad_data = gctx->vtcm_src1 + (ith * gctx->vtcm_src1_size_per_hvx); \
+ uint8_t * restrict dst_spad_data = gctx->vtcm_dst + (ith * gctx->vtcm_dst_size_per_hvx); \
\
size_t src0_spad_half_size = gctx->src0_spad_half_size; \
size_t src1_spad_half_size = gctx->src1_spad_half_size; \
@@ -764,7 +764,7 @@ static void sigglu_f16(const _Float16 * restrict src0,
\
const int BLOCK = gctx->block; \
\
- dma_queue * dma_q = gctx->octx->ctx->dma[ith]; \
+ dma_queue * dma_q = th->dma_q; \
glu_compute_##SUFFIX##_fn_t compute = (glu_compute_##SUFFIX##_fn_t) gctx->compute; \
\
for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \
@@ -871,7 +871,7 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) {
const struct htp_glu_kernel_params * kparams = (const struct htp_glu_kernel_params *) octx->kernel_params;
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -893,7 +893,7 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
// row_size = bytes of useful data per row (what the kernel touches / what DMA copies).
// row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src).
@@ -904,28 +904,28 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) {
const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1];
struct htp_glu_vtcm_layout L;
- htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, kparams->vtcm_row_per_thread);
+ htp_glu_vtcm_layout_build(&L, dst_row_size, n_hvx, kparams->vtcm_row_per_hvx);
const size_t src0_row_size_aligned = L.row_size_aligned;
const size_t src1_row_size_aligned = L.row_size_aligned;
const size_t dst_row_size_aligned = L.row_size_aligned;
// Make sure the reserved vtcm size is sufficient
- if (kparams->vtcm_row_per_thread == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) {
+ if (kparams->vtcm_row_per_hvx == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) {
FARF(ERROR, "glu-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,
- (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads);
+ (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_hvx);
return HTP_STATUS_VTCM_TOO_SMALL;
}
if (src1) {
FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n",
op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2],
- src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads,
- L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads);
+ src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_hvx * n_hvx,
+ L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx);
} else {
FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type,
src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
- L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads);
+ L.src0_bytes_per_hvx * n_hvx, L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx);
}
// Prepare context
@@ -934,7 +934,7 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) {
gctx.compute = (void *) compute_fn;
gctx.op_str = op_type;
- gctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
+ gctx.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div);
gctx.src0_row_size = src0_row_size;
gctx.src1_row_size = src1_row_size;
@@ -952,13 +952,13 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) {
gctx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);
gctx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);
- gctx.vtcm_src0_size_per_thread = (uint32_t) L.src0_bytes_per_thread;
- gctx.vtcm_src1_size_per_thread = (uint32_t) L.src1_bytes_per_thread;
- gctx.vtcm_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread;
+ gctx.vtcm_src0_size_per_hvx = (uint32_t) L.src0_bytes_per_hvx;
+ gctx.vtcm_src1_size_per_hvx = (uint32_t) L.src1_bytes_per_hvx;
+ gctx.vtcm_dst_size_per_hvx = (uint32_t) L.dst_bytes_per_hvx;
- gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_thread >> 1;
- gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_thread >> 1;
- gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_thread >> 1;
+ gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_hvx >> 1;
+ gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_hvx >> 1;
+ gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_hvx >> 1;
gctx.block = L.block;
gctx.src0_nrows = nrows;
@@ -987,7 +987,7 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) {
gctx.data_src1 = data_src1;
gctx.data_dst = dst->data;
- work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f32_per_thread, &gctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, (work_queue_func_t)glu_f32_per_hvx, &gctx, n_hvx);
return HTP_STATUS_OK;
}
@@ -1037,7 +1037,7 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) {
const struct htp_glu_kernel_params * kparams = (const struct htp_glu_kernel_params *) octx->kernel_params;
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -1059,7 +1059,7 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
// row_size = bytes of useful data per row (what the kernel touches / what DMA copies).
// row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src).
@@ -1070,28 +1070,28 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) {
const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1];
struct htp_glu_vtcm_layout L;
- htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, kparams->vtcm_row_per_thread);
+ htp_glu_vtcm_layout_build(&L, dst_row_size, n_hvx, kparams->vtcm_row_per_hvx);
const size_t src0_row_size_aligned = L.row_size_aligned;
const size_t src1_row_size_aligned = L.row_size_aligned;
const size_t dst_row_size_aligned = L.row_size_aligned;
// Make sure the reserved vtcm size is sufficient
- if (kparams->vtcm_row_per_thread == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) {
+ if (kparams->vtcm_row_per_hvx == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) {
FARF(ERROR, "glu-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,
- (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads);
+ (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_hvx);
return HTP_STATUS_VTCM_TOO_SMALL;
}
if (src1) {
FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n",
op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2],
- src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads,
- L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads);
+ src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_hvx * n_hvx,
+ L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx);
} else {
FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type,
src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
- L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads);
+ L.src0_bytes_per_hvx * n_hvx, L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx);
}
// Prepare context
@@ -1100,7 +1100,7 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) {
gctx.compute = (void *) compute_fn;
gctx.op_str = op_type;
- gctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
+ gctx.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div);
gctx.src0_row_size = src0_row_size;
gctx.src1_row_size = src1_row_size;
@@ -1118,13 +1118,13 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) {
gctx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);
gctx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);
- gctx.vtcm_src0_size_per_thread = (uint32_t) L.src0_bytes_per_thread;
- gctx.vtcm_src1_size_per_thread = (uint32_t) L.src1_bytes_per_thread;
- gctx.vtcm_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread;
+ gctx.vtcm_src0_size_per_hvx = (uint32_t) L.src0_bytes_per_hvx;
+ gctx.vtcm_src1_size_per_hvx = (uint32_t) L.src1_bytes_per_hvx;
+ gctx.vtcm_dst_size_per_hvx = (uint32_t) L.dst_bytes_per_hvx;
- gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_thread >> 1;
- gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_thread >> 1;
- gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_thread >> 1;
+ gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_hvx >> 1;
+ gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_hvx >> 1;
+ gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_hvx >> 1;
gctx.block = L.block;
gctx.src0_nrows = nrows;
@@ -1153,7 +1153,7 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) {
gctx.data_src1 = data_src1;
gctx.data_dst = dst->data;
- work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f16_per_thread, &gctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, (work_queue_func_t)glu_f16_per_hvx, &gctx, n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/glu-ops.h b/ggml/src/ggml-hexagon/htp/glu-ops.h
index 41b3857b1..5e248f7d4 100644
--- a/ggml/src/ggml-hexagon/htp/glu-ops.h
+++ b/ggml/src/ggml-hexagon/htp/glu-ops.h
@@ -7,8 +7,8 @@
#include <stdint.h>
struct htp_glu_kernel_params {
- uint32_t n_threads;
- uint32_t vtcm_row_per_thread;
+ uint32_t n_hvx;
+ uint32_t vtcm_row_per_hvx;
uint32_t vtcm_size;
};
@@ -41,34 +41,34 @@ struct htp_glu_vtcm_layout {
size_t off_dst;
size_t row_size_aligned;
- size_t src0_bytes_per_thread;
- size_t src1_bytes_per_thread;
- size_t dst_bytes_per_thread;
+ size_t src0_bytes_per_hvx;
+ size_t src1_bytes_per_hvx;
+ size_t dst_bytes_per_hvx;
- uint32_t vtcm_row_per_thread;
+ uint32_t vtcm_row_per_hvx;
uint32_t block;
};
static inline void htp_glu_vtcm_layout_build(
struct htp_glu_vtcm_layout * L,
size_t dst_row_size,
- uint32_t n_threads,
- uint32_t vtcm_row_per_thread
+ uint32_t n_hvx,
+ uint32_t vtcm_row_per_hvx
) {
const size_t row_size_aligned = hex_round_up(dst_row_size, 128);
L->row_size_aligned = row_size_aligned;
- L->vtcm_row_per_thread = vtcm_row_per_thread;
- L->block = vtcm_row_per_thread >> 1;
+ L->vtcm_row_per_hvx = vtcm_row_per_hvx;
+ L->block = vtcm_row_per_hvx >> 1;
- L->src0_bytes_per_thread = row_size_aligned * vtcm_row_per_thread;
- L->src1_bytes_per_thread = row_size_aligned * vtcm_row_per_thread;
- L->dst_bytes_per_thread = row_size_aligned * vtcm_row_per_thread;
+ L->src0_bytes_per_hvx = row_size_aligned * vtcm_row_per_hvx;
+ L->src1_bytes_per_hvx = row_size_aligned * vtcm_row_per_hvx;
+ L->dst_bytes_per_hvx = row_size_aligned * vtcm_row_per_hvx;
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx);
L->total_bytes = off;
}
diff --git a/ggml/src/ggml-hexagon/htp/hex-profile.h b/ggml/src/ggml-hexagon/htp/hex-profile.h
deleted file mode 100644
index 5c7a1a488..000000000
--- a/ggml/src/ggml-hexagon/htp/hex-profile.h
+++ /dev/null
@@ -1,66 +0,0 @@
-#ifndef HEX_PROFILE_H
-#define HEX_PROFILE_H
-
-#include <stdbool.h>
-#include <stdint.h>
-#include <qurt.h>
-
-#include "hex-utils.h"
-#include "htp-ops.h"
-
-#define HTP_TRACE_EVT_START 0
-#define HTP_TRACE_EVT_STOP 1
-
-#ifndef HEX_NUM_PMU_COUNTERS
-#define HEX_NUM_PMU_COUNTERS 8
-#endif
-
-static inline void hex_get_pmu(uint32_t counters[]) {
-#if __HVX_ARCH__ >= 79
- asm volatile("%0 = upmucnt0" : "=r"(counters[0]));
- asm volatile("%0 = upmucnt1" : "=r"(counters[1]));
- asm volatile("%0 = upmucnt2" : "=r"(counters[2]));
- asm volatile("%0 = upmucnt3" : "=r"(counters[3]));
- asm volatile("%0 = upmucnt4" : "=r"(counters[4]));
- asm volatile("%0 = upmucnt5" : "=r"(counters[5]));
- asm volatile("%0 = upmucnt6" : "=r"(counters[6]));
- asm volatile("%0 = upmucnt7" : "=r"(counters[7]));
-#else
- counters[0] = qurt_pmu_get(QURT_PMUCNT0);
- counters[1] = qurt_pmu_get(QURT_PMUCNT1);
- counters[2] = qurt_pmu_get(QURT_PMUCNT2);
- counters[3] = qurt_pmu_get(QURT_PMUCNT3);
- counters[4] = qurt_pmu_get(QURT_PMUCNT4);
- counters[5] = qurt_pmu_get(QURT_PMUCNT5);
- counters[6] = qurt_pmu_get(QURT_PMUCNT6);
- counters[7] = qurt_pmu_get(QURT_PMUCNT7);
-#endif
-}
-
-struct htp_thread_trace {
- uint32_t count;
- uint32_t max_events;
- struct htp_trace_desc * events;
-};
-
-static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) {
- if (!tr->max_events) {
- return;
- }
- uint32_t i = tr->count++;
- if (i < tr->max_events) {
- tr->events[i].id = id;
- tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0);
- tr->events[i].cycles = (uint32_t) hex_get_cycles();
- }
-}
-
-static inline void htp_trace_event_start(struct htp_thread_trace * tr, uint16_t id, uint16_t info) {
- htp_trace_event(tr, id, info, HTP_TRACE_EVT_START);
-}
-
-static inline void htp_trace_event_stop(struct htp_thread_trace * tr, uint16_t id, uint16_t info) {
- htp_trace_event(tr, id, info, HTP_TRACE_EVT_STOP);
-}
-
-#endif /* HEX_PROFILE_H */
diff --git a/ggml/src/ggml-hexagon/htp/hex-utils.h b/ggml/src/ggml-hexagon/htp/hex-utils.h
index abb44da25..cb3093dec 100644
--- a/ggml/src/ggml-hexagon/htp/hex-utils.h
+++ b/ggml/src/ggml-hexagon/htp/hex-utils.h
@@ -25,6 +25,32 @@ static inline uint64_t hex_get_pktcnt() {
return pktcnt;
}
+#ifndef HEX_NUM_PMU_COUNTERS
+#define HEX_NUM_PMU_COUNTERS 8
+#endif
+
+static inline void hex_get_pmu(uint32_t counters[]) {
+#if __HVX_ARCH__ >= 79
+ asm volatile("%0 = upmucnt0" : "=r"(counters[0]));
+ asm volatile("%0 = upmucnt1" : "=r"(counters[1]));
+ asm volatile("%0 = upmucnt2" : "=r"(counters[2]));
+ asm volatile("%0 = upmucnt3" : "=r"(counters[3]));
+ asm volatile("%0 = upmucnt4" : "=r"(counters[4]));
+ asm volatile("%0 = upmucnt5" : "=r"(counters[5]));
+ asm volatile("%0 = upmucnt6" : "=r"(counters[6]));
+ asm volatile("%0 = upmucnt7" : "=r"(counters[7]));
+#else
+ counters[0] = qurt_pmu_get(QURT_PMUCNT0);
+ counters[1] = qurt_pmu_get(QURT_PMUCNT1);
+ counters[2] = qurt_pmu_get(QURT_PMUCNT2);
+ counters[3] = qurt_pmu_get(QURT_PMUCNT3);
+ counters[4] = qurt_pmu_get(QURT_PMUCNT4);
+ counters[5] = qurt_pmu_get(QURT_PMUCNT5);
+ counters[6] = qurt_pmu_get(QURT_PMUCNT6);
+ counters[7] = qurt_pmu_get(QURT_PMUCNT7);
+#endif
+}
+
static inline void hex_l2fetch(const void * p, uint32_t width, uint32_t stride, uint32_t height) {
const uint64_t control = Q6_P_combine_RR(stride, Q6_R_combine_RlRl(width, height));
Q6_l2fetch_AP((void *) p, control);
@@ -37,11 +63,6 @@ static inline void hex_l2fetch_block(const void * addr, size_t size) {
hex_l2fetch(addr, width, width, height);
}
-#define HEX_L2_LINE_SIZE 128
-#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (chunks per thread)
-#define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024)
-#define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024)
-
static inline void hex_l2flush(void * addr, size_t size) {
if (size == 0) return;
const uint32_t s = ((uint32_t) addr) & ~(HEX_L2_LINE_SIZE - 1);
diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h
index 65f90c17a..590167492 100644
--- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h
+++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h
@@ -18,8 +18,6 @@ typedef struct tiled_dequantize_state tiled_dequantize_state_t;
typedef void (*tiled_dequant_task_fn_t)(const tiled_dequantize_state_t * state, uint32_t start_tile, uint32_t end_tile);
struct tiled_dequantize_state {
- struct htp_context * ctx;
- struct htp_thread_trace * traces;
__fp16 * dst;
const uint8_t * src;
tiled_dequant_task_fn_t task_fn;
@@ -1235,7 +1233,6 @@ typedef struct {
uint32_t dst_stride; // DDR row stride
uint32_t src2_stride; // DDR row stride for residual
uint32_t dst_cols; // Actual output columns
- struct htp_thread_trace * traces;
} output_transfer_task_state_t;
// activations : fp32 -> fp16
diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.c b/ggml/src/ggml-hexagon/htp/hmx-queue.c
index c369d3dd2..e40a28bed 100644
--- a/ggml/src/ggml-hexagon/htp/hmx-queue.c
+++ b/ggml/src/ggml-hexagon/htp/hmx-queue.c
@@ -46,9 +46,9 @@ static inline void hmx_queue_process(hmx_queue_t q, bool* killed) {
case HMX_QUEUE_WAKEUP: hmx_lock(q); break;
default:
hmx_lock(q);
- htp_trace_event_start(q->trace, HTP_TRACE_EVT_HMX_COMP, ir);
- d->func(d->data);
- htp_trace_event_stop(q->trace, HTP_TRACE_EVT_HMX_COMP, ir);
+ htp_trace_event_start(q->th.trace, HTP_TRACE_EVT_HMX_COMP, ir);
+ d->func(&q->th, d->data);
+ htp_trace_event_stop(q->th.trace, HTP_TRACE_EVT_HMX_COMP, ir);
break;
}
@@ -63,7 +63,7 @@ static inline void hmx_queue_process(hmx_queue_t q, bool* killed) {
static void hmx_queue_thread(void * arg) {
hmx_queue_t q = (hmx_queue_t) arg;
- FARF(HIGH, "hmx-queue-thread: started");
+ FARF(HIGH, "hmx-queue-thread (tid %u): started", q->th.tid);
bool killed = false;
@@ -90,7 +90,7 @@ static void hmx_queue_thread(void * arg) {
hmx_queue_process(q, &killed);
}
- FARF(HIGH, "hmx-queue-thread: stopped");
+ FARF(HIGH, "hmx-queue-thread (tid %u): stopped", q->th.tid);
}
size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size) {
@@ -105,7 +105,7 @@ size_t hmx_queue_alignof(void) {
return HEX_L2_LINE_SIZE;
}
-hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace) {
+hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, dma_queue_t * dma, struct htp_thread_trace * trace) {
capacity = hex_ceil_pow2(capacity);
size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE);
size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE);
@@ -126,7 +126,7 @@ hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uin
q->stack = block;
memset(q->stack, 0, stack_size);
- q->trace = trace;
+ htp_thread_init(&q->th, dma, trace);
// Match caller thread priority (same pattern as worker-pool.c).
int prio = qurt_thread_get_priority(qurt_thread_get_id());
diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.h b/ggml/src/ggml-hexagon/htp/hmx-queue.h
index c2b1859a2..efefb1455 100644
--- a/ggml/src/ggml-hexagon/htp/hmx-queue.h
+++ b/ggml/src/ggml-hexagon/htp/hmx-queue.h
@@ -11,7 +11,8 @@
#include <HAP_farf.h>
#include "hex-utils.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
+#include "htp-thread.h"
#ifdef __cplusplus
extern "C" {
@@ -23,7 +24,7 @@ extern "C" {
#define HMX_QUEUE_POLL_COUNT 1
#endif
-typedef void (*hmx_queue_func)(void *);
+typedef void (*hmx_queue_func)(const htp_thread_t * th, void *);
// Dummy funcs used as signals
enum hmx_queue_signal {
@@ -49,10 +50,10 @@ struct hmx_queue_s {
atomic_uint seqn; // incremented for all pushes, used with futex
qurt_thread_t thread;
+ htp_thread_t th;
void * stack;
uint32_t hap_rctx;
bool hmx_locked;
- struct htp_thread_trace * trace;
bool external_mem; // memory owned externally
};
@@ -60,7 +61,7 @@ typedef struct hmx_queue_s * hmx_queue_t;
size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size);
size_t hmx_queue_alignof(void);
-hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace);
+hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, dma_queue_t * dma, struct htp_thread_trace * trace);
void hmx_queue_free(hmx_queue_t q);
static inline struct hmx_queue_desc hmx_queue_make_desc(hmx_queue_func func, void * data) {
diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h
index 5c35cd988..721fa2162 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ctx.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h
@@ -5,7 +5,7 @@
#include "hmx-queue.h"
#include "htp-dcache.h"
#include "htp-ops.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "work-queue.h"
#include "hex-fastdiv.h"
@@ -17,10 +17,6 @@
#include <stdbool.h>
#include <string.h>
-#ifndef HTP_MAX_NTHREADS
-#define HTP_MAX_NTHREADS 10
-#endif
-
#define HTP_MAX_MMAPS 64
// Memory mapping
@@ -37,7 +33,7 @@ struct htp_spad {
uint8_t * data; // pointer to an area in vtcm
uint32_t stride; // stride used inside this spad
uint32_t size; // total size
- uint32_t size_per_thread; // size per thread
+ uint32_t size_per_hvx; // size per hvx
};
struct htp_mdev_group {
@@ -71,14 +67,14 @@ struct htp_ops_context {
struct htp_spad dst_spad;
uint32_t flags;
- uint32_t n_threads;
- struct fastdiv_values n_threads_div;
+ uint32_t n_hvx;
+ struct fastdiv_values n_hvx_div;
int status;
// Must stay last: not cleared by htp_ops_context_reset(), ops init the caches they use
union {
- dma_cache_dm dma_cache_dm[HTP_MAX_NTHREADS];
- dma_cache_fa dma_cache_fa[HTP_MAX_NTHREADS];
+ dma_cache_dm dma_cache_dm[HTP_MAX_NHVX];
+ dma_cache_fa dma_cache_fa[HTP_MAX_NHVX];
} __attribute__((aligned(128)));
};
@@ -88,12 +84,13 @@ struct htp_context {
struct htp_mmap mmap[HTP_MAX_MMAPS];
dma_queue_t dma[HTP_MAX_NTHREADS];
- struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1];
- work_queue_t work_queue;
+ struct htp_thread_trace trace[HTP_MAX_NTHREADS];
+ work_queue_t hvx_queue;
hmx_queue_t hmx_queue;
uint32_t n_threads;
- struct fastdiv_values n_threads_div;
+ uint32_t n_hvx;
+ struct fastdiv_values n_hvx_div;
int thread_id;
int thread_prio;
@@ -126,37 +123,37 @@ struct htp_context {
static inline void htp_ops_context_reset(struct htp_ops_context * octx, struct htp_context * ctx) {
memset(octx, 0, offsetof(struct htp_ops_context, dma_cache_dm));
- octx->ctx = ctx;
- octx->n_threads = ctx->n_threads;
- octx->n_threads_div = ctx->n_threads_div;
+ octx->ctx = ctx;
+ octx->n_hvx = ctx->n_hvx;
+ octx->n_hvx_div = ctx->n_hvx_div;
}
-static inline bool htp_ops_context_set_n_threads_div(
+static inline bool htp_ops_context_set_n_hvx_div(
struct htp_ops_context * octx,
- uint32_t n_threads,
- struct fastdiv_values n_threads_div) {
- if (n_threads == 0 || n_threads > octx->ctx->n_threads) {
+ uint32_t n_hvx,
+ struct fastdiv_values n_hvx_div) {
+ if (n_hvx == 0 || n_hvx > octx->ctx->n_hvx) {
return false;
}
- octx->n_threads = n_threads;
- octx->n_threads_div = n_threads_div;
+ octx->n_hvx = n_hvx;
+ octx->n_hvx_div = n_hvx_div;
return true;
}
-static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) {
- if (n_threads == 0 || n_threads > octx->ctx->n_threads) {
+static inline bool htp_ops_context_set_n_hvx(struct htp_ops_context * octx, uint32_t n_hvx) {
+ if (n_hvx == 0 || n_hvx > octx->ctx->n_hvx) {
return false;
}
- if (n_threads == octx->n_threads) {
+ if (n_hvx == octx->n_hvx) {
return true;
}
- const struct fastdiv_values n_threads_div = n_threads == octx->ctx->n_threads
- ? octx->ctx->n_threads_div
- : init_fastdiv_values(n_threads);
- return htp_ops_context_set_n_threads_div(octx, n_threads, n_threads_div);
+ const struct fastdiv_values n_hvx_div = n_hvx == octx->ctx->n_hvx
+ ? octx->ctx->n_hvx_div
+ : init_fastdiv_values(n_hvx);
+ return htp_ops_context_set_n_hvx_div(octx, n_hvx, n_hvx_div);
}
static inline void htp_ops_context_set_status(struct htp_ops_context * octx, int status) {
diff --git a/ggml/src/ggml-hexagon/htp/htp-dcache.c b/ggml/src/ggml-hexagon/htp/htp-dcache.c
index 840b1edb7..52fc8e403 100644
--- a/ggml/src/ggml-hexagon/htp/htp-dcache.c
+++ b/ggml/src/ggml-hexagon/htp/htp-dcache.c
@@ -7,7 +7,7 @@
#include "hex-common.h"
#include "hex-utils.h"
#include "hex-fastdiv.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "htp-ctx.h"
#include "work-queue.h"
@@ -15,15 +15,14 @@ struct l2flush_range {
uint32_t start; // line-aligned start address
uint32_t end; // line-aligned end address
uint32_t block_first; // global block index of this range's first block
- uint32_t n_blocks; // number of HEX_L2_BLOCK_SIZE chunks (last may be partial)
+ uint32_t n_blocks; // number of HTP_DCACHE_BLOCK_SIZE chunks (last may be partial)
};
struct l2flush_multi_task {
- struct htp_thread_trace * trace;
struct l2flush_range ranges[HTP_MAX_DIRTY_RANGES];
uint32_t n_ranges;
uint32_t total_blocks;
- uint32_t blocks_per_thread;
+ uint32_t blocks_per_hvx;
};
static void flush_all_dcache(struct htp_context * ctx) {
@@ -35,12 +34,12 @@ static void flush_all_dcache(struct htp_context * ctx) {
memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges));
}
-static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) {
+static void l2flush_multi_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
struct l2flush_multi_task * task = (struct l2flush_multi_task *) data;
(void) n;
- const uint32_t gb_first = i * task->blocks_per_thread;
- uint32_t gb_last = gb_first + task->blocks_per_thread;
+ const uint32_t gb_first = i * task->blocks_per_hvx;
+ uint32_t gb_last = gb_first + task->blocks_per_hvx;
if (gb_last > task->total_blocks) {
gb_last = task->total_blocks;
}
@@ -48,7 +47,7 @@ static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) {
return;
}
- struct htp_thread_trace * tr = &task->trace[i];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, gb_first);
for (uint32_t r = 0; r < task->n_ranges; r++) {
@@ -62,8 +61,8 @@ static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) {
continue;
}
- const uint32_t s = rg->start + (lo - rb_first) * HEX_L2_BLOCK_SIZE;
- uint32_t e = rg->start + (hi - rb_first) * HEX_L2_BLOCK_SIZE;
+ const uint32_t s = rg->start + (lo - rb_first) * HTP_DCACHE_BLOCK_SIZE;
+ uint32_t e = rg->start + (hi - rb_first) * HTP_DCACHE_BLOCK_SIZE;
if (e > rg->end) {
e = rg->end;
}
@@ -163,7 +162,7 @@ void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_te
total_evict_size += r->end - r->start;
}
- if (total_evict_size > HEX_L2_FLUSH_ALL_THRESHOLD) {
+ if (total_evict_size > HTP_DCACHE_FLUSH_ALL_THRESHOLD) {
flush_all_dcache(ctx);
for (uint32_t i = 0; i < n_pending; i++) {
struct htp_dirty_range * r = &ctx->dirty_ranges[i];
@@ -174,9 +173,8 @@ void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_te
return;
}
- if (total_evict_size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1 && n_evict <= HTP_MAX_DIRTY_RANGES) {
+ if (total_evict_size > HTP_DCACHE_FLUSH_WQ_THRESHOLD && ctx->n_hvx > 1 && n_evict <= HTP_MAX_DIRTY_RANGES) {
struct l2flush_multi_task task;
- task.trace = ctx->trace;
task.n_ranges = n_evict;
uint32_t block_acc = 0;
@@ -188,14 +186,14 @@ void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_te
rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE);
rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE);
rg->block_first = block_acc;
- rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE;
+ rg->n_blocks = (rg->end - rg->start + HTP_DCACHE_BLOCK_SIZE - 1) / HTP_DCACHE_BLOCK_SIZE;
block_acc += rg->n_blocks;
}
- task.total_blocks = block_acc;
- task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div);
+ task.total_blocks = block_acc;
+ task.blocks_per_hvx = fastdiv(block_acc + ctx->n_hvx - 1, &ctx->n_hvx_div);
- work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads);
+ work_queue_run(ctx->hvx_queue, l2flush_multi_worker, &task, ctx->n_hvx);
} else {
struct htp_thread_trace * tr = &ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0);
@@ -270,9 +268,8 @@ static inline bool is_tensor_dirty(struct htp_context * ctx, const struct htp_te
}
static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_range * ranges, uint32_t n_ranges, uint64_t total_dirty) {
- if (total_dirty >= HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) {
+ if (total_dirty >= HTP_DCACHE_FLUSH_WQ_THRESHOLD && ctx->n_hvx > 1) {
struct l2flush_multi_task task;
- task.trace = ctx->trace;
task.n_ranges = n_ranges;
uint32_t block_acc = 0;
@@ -282,14 +279,14 @@ static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_
rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE);
rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE);
rg->block_first = block_acc;
- rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE;
+ rg->n_blocks = (rg->end - rg->start + HTP_DCACHE_BLOCK_SIZE - 1) / HTP_DCACHE_BLOCK_SIZE;
block_acc += rg->n_blocks;
}
- task.total_blocks = block_acc;
- task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div);
+ task.total_blocks = block_acc;
+ task.blocks_per_hvx = fastdiv(block_acc + ctx->n_hvx - 1, &ctx->n_hvx_div);
- work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads);
+ work_queue_run(ctx->hvx_queue, l2flush_multi_worker, &task, ctx->n_hvx);
} else {
struct htp_thread_trace * tr = &ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0);
@@ -319,7 +316,7 @@ void htp_dcache_flush_dirty_ranges(struct htp_context * ctx) {
return;
}
- if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) {
+ if (total_dirty > HTP_DCACHE_FLUSH_ALL_THRESHOLD) {
flush_all_dcache(ctx);
return;
}
@@ -348,7 +345,7 @@ void htp_dcache_flush_dirty_tensors(struct htp_context * ctx, const struct htp_t
return;
}
- if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) {
+ if (total_dirty > HTP_DCACHE_FLUSH_ALL_THRESHOLD) {
flush_all_dcache(ctx);
return;
}
diff --git a/ggml/src/ggml-hexagon/htp/htp-dcache.h b/ggml/src/ggml-hexagon/htp/htp-dcache.h
index 1996ab258..c1c37573c 100644
--- a/ggml/src/ggml-hexagon/htp/htp-dcache.h
+++ b/ggml/src/ggml-hexagon/htp/htp-dcache.h
@@ -2,9 +2,13 @@
#define HTP_DCACHE_H
#include <stdint.h>
+#include "hex-common.h"
#include "htp-ops.h"
-#define HTP_MAX_DIRTY_RANGES 64
+#define HTP_MAX_DIRTY_RANGES 64
+#define HTP_DCACHE_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (chunks per thread)
+#define HTP_DCACHE_FLUSH_WQ_THRESHOLD (4 * 1024)
+#define HTP_DCACHE_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024)
struct htp_dirty_range {
uint32_t start;
diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h
index a3ff51ebe..7c4b97307 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ops.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h
@@ -127,14 +127,17 @@ enum htp_op_code {
HTP_OP_INVALID
};
-#define HTP_OP_MAX_DIMS 4 // aka GGML_MAX_DIMS
-#define HTP_OP_MAX_INPUTS 10 // aka GGML_MAX_SRCS
-#define HTP_OP_MAX_OUTPUTS 4
-#define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS
+#define HTP_OP_MAX_DIMS 4 // aka GGML_MAX_DIMS
+#define HTP_OP_MAX_INPUTS 10 // aka GGML_MAX_SRCS
+#define HTP_OP_MAX_OUTPUTS 4
+#define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS
#define HTP_OP_MAX_KERN_PARAMS 32
-#define HTP_OP_MAX_BUFS 64
-#define HTP_OP_MAX_TENSORS 65534 // must stay under 64K (uint16)
+#define HTP_OP_MAX_BUFS 64
+#define HTP_OP_MAX_TENSORS 65534 // must stay under 64K (uint16)
+
+#define HTP_OP_MAX_NTHREADS 10
+#define HTP_OP_MAX_NHVX 8
#define HTP_FENCE_TIMEOUT (1000000000ULL)
#define HTP_FENCE_SLOT_SIZE 128
@@ -191,8 +194,20 @@ struct htp_op_desc {
uint16_t pad[2]; // padding to align to 64 bits
};
-#ifndef HTP_MAX_NTHREADS
+#ifdef __HVX_ARCH__
+#if __HVX_ARCH__ <= 73
+#define HTP_MAX_NTHREADS 6
+#define HTP_MAX_NHVX 4
+#elif __HVX_ARCH__ <= 75
+#define HTP_MAX_NTHREADS 8
+#define HTP_MAX_NHVX 6
+#else
#define HTP_MAX_NTHREADS 10
+#define HTP_MAX_NHVX 8
+#endif
+
+_Static_assert(HTP_MAX_NTHREADS <= HTP_OP_MAX_NTHREADS, "HTP_MAX_NTHREADS exceeds HTP_OP_MAX_NTHREADS");
+_Static_assert(HTP_MAX_NHVX <= HTP_OP_MAX_NHVX, "HTP_MAX_NHVX exceeds HTP_OP_MAX_NHVX");
#endif
#define HTP_TRACE_MAX_EVENTS 256
@@ -285,7 +300,7 @@ struct htp_opbatch_rsp {
uint32_t n_tensors; // Number of tensors
uint32_t n_ops; // Number of op profile descriptors
uint32_t usecs; // Number of usec
- uint32_t n_traces[HTP_MAX_NTHREADS + 1];
+ uint32_t n_traces[HTP_OP_MAX_NTHREADS];
// struct htp_prof_desc profs[]; -- dspqueue buf 0
};
diff --git a/ggml/src/ggml-hexagon/htp/htp-thread.h b/ggml/src/ggml-hexagon/htp/htp-thread.h
new file mode 100644
index 000000000..e4ca874de
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/htp-thread.h
@@ -0,0 +1,33 @@
+#ifndef HTP_THREAD_H
+#define HTP_THREAD_H
+
+#include <stdatomic.h>
+#include <stdint.h>
+
+#include "dma-queue.h"
+#include "htp-trace.h"
+
+typedef struct {
+ unsigned int tid;
+ dma_queue * dma_q;
+ struct htp_thread_trace * trace;
+} htp_thread_t;
+
+extern atomic_uint htp_thread_count;
+
+static inline void htp_thread_reset(void) {
+ atomic_store_explicit(&htp_thread_count, 0, memory_order_relaxed);
+}
+
+static inline unsigned int htp_thread_alloc_id(void) {
+ return atomic_fetch_add_explicit(&htp_thread_count, 1, memory_order_relaxed);
+}
+
+static inline void htp_thread_init(htp_thread_t * th, dma_queue_t * dma, struct htp_thread_trace * trace) {
+ unsigned int tid = htp_thread_alloc_id();
+ th->tid = tid;
+ th->dma_q = dma ? dma[tid] : NULL;
+ th->trace = trace ? &trace[tid] : NULL;
+}
+
+#endif /* HTP_THREAD_H */
diff --git a/ggml/src/ggml-hexagon/htp/htp-trace.h b/ggml/src/ggml-hexagon/htp/htp-trace.h
new file mode 100644
index 000000000..66bd9110d
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/htp-trace.h
@@ -0,0 +1,38 @@
+#ifndef HTP_TRACE_H
+#define HTP_TRACE_H
+
+#include <stdint.h>
+
+#include "hex-utils.h"
+#include "htp-ops.h"
+
+#define HTP_TRACE_EVT_START 0
+#define HTP_TRACE_EVT_STOP 1
+
+struct htp_thread_trace {
+ uint32_t count;
+ uint32_t max_events;
+ struct htp_trace_desc * events;
+};
+
+static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) {
+ if (!tr->max_events) {
+ return;
+ }
+ uint32_t i = tr->count++;
+ if (i < tr->max_events) {
+ tr->events[i].id = id;
+ tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0);
+ tr->events[i].cycles = (uint32_t) hex_get_cycles();
+ }
+}
+
+static inline void htp_trace_event_start(struct htp_thread_trace * tr, uint16_t id, uint16_t info) {
+ htp_trace_event(tr, id, info, HTP_TRACE_EVT_START);
+}
+
+static inline void htp_trace_event_stop(struct htp_thread_trace * tr, uint16_t id, uint16_t info) {
+ htp_trace_event(tr, id, info, HTP_TRACE_EVT_STOP);
+}
+
+#endif /* HTP_TRACE_H */
diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.c b/ggml/src/ggml-hexagon/htp/im2col-ops.c
index 9e662694b..602ee1598 100644
--- a/ggml/src/ggml-hexagon/htp/im2col-ops.c
+++ b/ggml/src/ggml-hexagon/htp/im2col-ops.c
@@ -15,7 +15,7 @@
#include "htp-ops.h"
#include "hvx-utils.h"
#include "dma-queue.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "htp-vtcm.h"
#include "htp-tensor.h"
#include "im2col-ops.h"
@@ -25,16 +25,16 @@ struct htp_im2col_context {
uint32_t pe_row_base; // first output row index assigned to this dev (DMA path)
uint32_t pe_nrows; // number of output rows assigned to this dev (DMA path)
- uint32_t pe_rows_per_thread; // output rows per worker
- uint32_t pe_blocks_per_thread; // output-col blocks per worker (blocked DMA path)
+ uint32_t pe_rows_per_hvx; // output rows per worker
+ uint32_t pe_blocks_per_hvx; // output-col blocks per worker (blocked DMA path)
uint32_t pe_src_row_bytes; // one output row's source, rounded 256
uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256
// Patch-embed DMA path VTCM ping-pong.
uint8_t * pe_vtcm_src; // base of the 2x src buffers region
uint8_t * pe_vtcm_dst; // base of the 2x dst buffers region
- uint32_t pe_src_size_per_thread; // 2 * pe_src_row_bytes
- uint32_t pe_dst_size_per_thread; // 2 * pe_dst_row_bytes
+ uint32_t pe_src_size_per_hvx; // 2 * pe_src_row_bytes
+ uint32_t pe_dst_size_per_hvx; // 2 * pe_dst_row_bytes
uint32_t pe_owb; // output-col block size
uint32_t pe_wb; // staged source window width
@@ -50,10 +50,10 @@ struct htp_im2col_context {
uint32_t patch_stride;
// Per-thread initial coordinates (removes division from thread prologue)
- uint32_t thread_in[HTP_MAX_NTHREADS];
- uint32_t thread_iod[HTP_MAX_NTHREADS];
- uint32_t thread_ioh[HTP_MAX_NTHREADS];
- uint32_t thread_c0[HTP_MAX_NTHREADS];
+ uint32_t thread_in[HTP_MAX_NHVX];
+ uint32_t thread_iod[HTP_MAX_NHVX];
+ uint32_t thread_ioh[HTP_MAX_NHVX];
+ uint32_t thread_c0[HTP_MAX_NHVX];
void * compute;
uint32_t dst_elem;
@@ -321,11 +321,11 @@ static void FNAME(
IM2COL_GATHER_COMPUTE_BODY(im2col_gather_compute_f16, uint16_t, hvx_copy_f16_f32_uu)
IM2COL_GATHER_COMPUTE_BODY(im2col_gather_compute_f32, float, hvx_copy_f32_uu)
-static void im2col_blocked_dma_thread(unsigned int nth, unsigned int ith, void * data) {
+static void im2col_blocked_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct htp_im2col_context * ictx = (struct htp_im2col_context *) data;
struct htp_ops_context * octx = ictx->octx;
- struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * restrict tr = th->trace;
const uint32_t OW = ictx->OW;
const uint32_t KH = ictx->KH;
const uint32_t owb = ictx->pe_owb;
@@ -334,16 +334,16 @@ static void im2col_blocked_dma_thread(unsigned int nth, unsigned int ith, void *
const uint32_t dst_elem = ictx->dst_elem;
im2col_blocked_compute_fn_t compute = (im2col_blocked_compute_fn_t) ictx->compute;
const dma_addr_t dst_data = octx->dst->data;
- dma_queue * dma_q = octx->ctx->dma[ith];
- uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread;
- uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread;
+ dma_queue * dma_q = th->dma_q;
+ uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx;
+ uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx;
float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) };
void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes };
float * scratch = (float *) (srcb_base + 2 * ictx->pe_src_row_bytes);
// work is split over output-col blocks, so a single output row (1D) still uses every thread
const uint32_t blk_end_max = ictx->pe_nrows * nbpr;
- const uint32_t blk_start = ictx->pe_blocks_per_thread * ith;
- const uint32_t blk_end = MIN(blk_start + ictx->pe_blocks_per_thread, blk_end_max);
+ const uint32_t blk_start = ictx->pe_blocks_per_hvx * ith;
+ const uint32_t blk_end = MIN(blk_start + ictx->pe_blocks_per_hvx, blk_end_max);
if (blk_start >= blk_end) {
return;
}
@@ -510,24 +510,24 @@ static void im2col_patchembed_compute_f32(
}
}
-static void im2col_patchembed_dma_thread(unsigned int nth, unsigned int ith, void * data) {
+static void im2col_patchembed_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct htp_im2col_context * ictx = (struct htp_im2col_context *) data;
struct htp_ops_context * octx = ictx->octx;
- struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * restrict tr = th->trace;
const uint32_t OW = ictx->OW;
const uint32_t patch_stride = ictx->patch_stride;
const uint32_t dst_elem = ictx->dst_elem;
im2col_patchembed_compute_fn_t compute = (im2col_patchembed_compute_fn_t) ictx->compute;
const dma_addr_t dst_data = octx->dst->data;
- dma_queue * dma_q = octx->ctx->dma[ith];
- uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread;
- uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread;
+ dma_queue * dma_q = th->dma_q;
+ uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx;
+ uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx;
float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) };
void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes };
const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows;
- const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith;
- const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max);
+ const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_hvx * ith;
+ const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_hvx, row_end_max);
if (row_start >= row_end) {
return;
}
@@ -718,11 +718,11 @@ static void FNAME(const struct htp_im2col_context * ictx,
IM2COL_3D_BLOCKED_COMPUTE_BODY(im2col_3d_blocked_compute_f16, uint16_t, hvx_copy_f16_f32_uu, hvx_splat_u16_u)
IM2COL_3D_BLOCKED_COMPUTE_BODY(im2col_3d_blocked_compute_f32, float, hvx_copy_f32_uu, hvx_splat_f32_u)
-static void im2col_3d_blocked_dma_thread(unsigned int nth, unsigned int ith, void * data) {
+static void im2col_3d_blocked_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct htp_im2col_context * ictx = (struct htp_im2col_context *) data;
struct htp_ops_context * octx = ictx->octx;
- struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * restrict tr = th->trace;
const struct htp_tensor * restrict dst = octx->dst;
const uint32_t OW = ictx->OW;
const uint32_t KD = ictx->KD, KH = ictx->KH;
@@ -732,15 +732,15 @@ static void im2col_3d_blocked_dma_thread(unsigned int nth, unsigned int ith, voi
const uint32_t nbpr = ictx->pe_nbpr;
const uint32_t dst_elem = ictx->dst_elem;
const dma_addr_t dst_data = dst->data;
- dma_queue * dma_q = octx->ctx->dma[ith];
- uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread;
- uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread;
+ dma_queue * dma_q = th->dma_q;
+ uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx;
+ uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx;
float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) };
void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes };
float * scratch = (float *) (srcb_base + 2 * ictx->pe_src_row_bytes);
const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows;
- const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith;
- const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max);
+ const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_hvx * ith;
+ const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_hvx, row_end_max);
if (row_start >= row_end) {
return;
}
@@ -860,28 +860,28 @@ static void im2col_3d_patchembed_compute_f32(
}
}
-static void im2col_3d_patchembed_dma_thread(unsigned int nth, unsigned int ith, void * data) {
+static void im2col_3d_patchembed_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct htp_im2col_context * ictx = (struct htp_im2col_context *) data;
struct htp_ops_context * octx = ictx->octx;
- struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * restrict tr = th->trace;
const struct htp_tensor * restrict dst = octx->dst;
const uint32_t KD = ictx->KD;
const uint32_t OW = ictx->OW;
const uint32_t patch_stride = ictx->patch_stride;
const uint32_t dst_elem = ictx->dst_elem;
const dma_addr_t dst_data = dst->data;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows;
- const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith;
- const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max);
+ const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_hvx * ith;
+ const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_hvx, row_end_max);
if (row_start >= row_end) {
return;
}
const uint32_t nrows_local = row_end - row_start;
- uint8_t * src_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread;
- uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread;
+ uint8_t * src_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx;
+ uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx;
float * srcb2[2] = { (float *) src_base, (float *) (src_base + ictx->pe_src_row_bytes) };
void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes };
struct im2col_cursor cur = {
@@ -948,29 +948,29 @@ int op_im2col(struct htp_ops_context * octx) {
// the blocked path splits work over output-col blocks, so a single output row (1D) can use every thread
const bool blocked = kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA;
const uint32_t nwork = blocked ? nrows * kparams->nbpr : nrows;
- if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nwork))) {
+ if (!htp_ops_context_set_n_hvx(octx, MIN((uint32_t) kparams->n_hvx, nwork))) {
return HTP_STATUS_INVAL_PARAMS;
}
struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads,
+ htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_hvx,
htp_im2col_scratch_bytes(kparams->kernel_type, kparams->KH, kparams->patch_stride));
struct htp_im2col_context ictx = { 0 };
uint8_t * const base = octx->ctx->vtcm_base;
- const uint32_t pth = octx->n_threads;
+ const uint32_t pth = octx->n_hvx;
ictx.octx = octx;
ictx.pe_row_base = row_base;
ictx.pe_nrows = nrows;
- ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div);
- ictx.pe_blocks_per_thread = fastdiv(nwork + pth - 1, &octx->n_threads_div);
+ ictx.pe_rows_per_hvx = fastdiv(nrows + pth - 1, &octx->n_hvx_div);
+ ictx.pe_blocks_per_hvx = fastdiv(nwork + pth - 1, &octx->n_hvx_div);
ictx.pe_src_row_bytes = kparams->src_row_bytes;
ictx.pe_dst_row_bytes = kparams->dst_row_bytes;
ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src);
ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst);
- ictx.pe_src_size_per_thread = layout.src_bytes_per_thread;
- ictx.pe_dst_size_per_thread = layout.dst_bytes_per_thread;
+ ictx.pe_src_size_per_hvx = layout.src_bytes_per_hvx;
+ ictx.pe_dst_size_per_hvx = layout.dst_bytes_per_hvx;
ictx.pe_owb = kparams->owb;
ictx.pe_wb = kparams->wb;
ictx.pe_nbpr = kparams->nbpr;
@@ -990,10 +990,10 @@ int op_im2col(struct htp_ops_context * octx) {
// a thread starts at a row, or at an output-col block (possibly mid-row) on the blocked path
for (uint32_t ith = 0; ith < pth; ith++) {
- uint32_t r = row_base + ictx.pe_rows_per_thread * ith;
+ uint32_t r = row_base + ictx.pe_rows_per_hvx * ith;
uint32_t c0 = 0;
if (blocked) {
- const uint32_t blk = ictx.pe_blocks_per_thread * ith;
+ const uint32_t blk = ictx.pe_blocks_per_hvx * ith;
const uint32_t q = fastdiv(blk, &kparams->div_nbpr);
r = row_base + q;
c0 = (blk - q * kparams->nbpr) * kparams->owb;
@@ -1009,7 +1009,7 @@ int op_im2col(struct htp_ops_context * octx) {
if (kparams->kernel_type == HTP_IM2COL_KERNEL_PATCHEMBED_DMA) {
ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_patchembed_compute_f16
: (void *) im2col_patchembed_compute_f32;
- work_queue_run(octx->ctx->work_queue, im2col_patchembed_dma_thread, &ictx, pth);
+ work_queue_run(octx->ctx->hvx_queue, im2col_patchembed_dma_thread, &ictx, pth);
return HTP_STATUS_OK;
} else if (blocked) {
if (kparams->KH == 1) {
@@ -1019,7 +1019,7 @@ int op_im2col(struct htp_ops_context * octx) {
ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_blocked_compute_f16
: (void *) im2col_blocked_compute_f32;
}
- work_queue_run(octx->ctx->work_queue, im2col_blocked_dma_thread, &ictx, pth);
+ work_queue_run(octx->ctx->hvx_queue, im2col_blocked_dma_thread, &ictx, pth);
return HTP_STATUS_OK;
}
@@ -1049,27 +1049,27 @@ int op_im2col_3d(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nrows))) {
+ if (!htp_ops_context_set_n_hvx(octx, MIN((uint32_t) kparams->n_hvx, nrows))) {
return HTP_STATUS_INVAL_PARAMS;
}
struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads, 256);
+ htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_hvx, 256);
struct htp_im2col_context ictx = { 0 };
uint8_t * const base = octx->ctx->vtcm_base;
- const uint32_t pth = octx->n_threads;
+ const uint32_t pth = octx->n_hvx;
ictx.octx = octx;
ictx.pe_row_base = row_base;
ictx.pe_nrows = nrows;
- ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div);
+ ictx.pe_rows_per_hvx = fastdiv(nrows + pth - 1, &octx->n_hvx_div);
ictx.pe_src_row_bytes = kparams->src_row_bytes;
ictx.pe_dst_row_bytes = kparams->dst_row_bytes;
ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src);
ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst);
- ictx.pe_src_size_per_thread = layout.src_bytes_per_thread;
- ictx.pe_dst_size_per_thread = layout.dst_bytes_per_thread;
+ ictx.pe_src_size_per_hvx = layout.src_bytes_per_hvx;
+ ictx.pe_dst_size_per_hvx = layout.dst_bytes_per_hvx;
ictx.pe_owb = kparams->owb;
ictx.pe_wb = kparams->wb;
ictx.pe_nbpr = kparams->nbpr;
@@ -1087,7 +1087,7 @@ int op_im2col_3d(struct htp_ops_context * octx) {
ictx.patch_stride = kparams->patch_stride;
for (uint32_t ith = 0; ith < pth; ith++) {
- const uint32_t rem = row_base + ictx.pe_rows_per_thread * ith;
+ const uint32_t rem = row_base + ictx.pe_rows_per_hvx * ith;
const uint32_t q_oh = fastdiv(rem, &kparams->div_OH);
ictx.thread_ioh[ith] = rem - q_oh * kparams->OH;
const uint32_t in = fastdiv(q_oh, &kparams->div_OD);
@@ -1100,12 +1100,12 @@ int op_im2col_3d(struct htp_ops_context * octx) {
if (kparams->kernel_type == HTP_IM2COL_KERNEL_PATCHEMBED_DMA) {
ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_3d_patchembed_compute_f16
: (void *) im2col_3d_patchembed_compute_f32;
- work_queue_run(octx->ctx->work_queue, im2col_3d_patchembed_dma_thread, &ictx, pth);
+ work_queue_run(octx->ctx->hvx_queue, im2col_3d_patchembed_dma_thread, &ictx, pth);
return HTP_STATUS_OK;
} else if (kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA) {
ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_3d_blocked_compute_f16
: (void *) im2col_3d_blocked_compute_f32;
- work_queue_run(octx->ctx->work_queue, im2col_3d_blocked_dma_thread, &ictx, pth);
+ work_queue_run(octx->ctx->hvx_queue, im2col_3d_blocked_dma_thread, &ictx, pth);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.h b/ggml/src/ggml-hexagon/htp/im2col-ops.h
index b34e6a09d..23497d01d 100644
--- a/ggml/src/ggml-hexagon/htp/im2col-ops.h
+++ b/ggml/src/ggml-hexagon/htp/im2col-ops.h
@@ -16,28 +16,28 @@ struct htp_im2col_vtcm_layout {
size_t total_bytes;
size_t off_src;
size_t off_dst;
- uint32_t src_bytes_per_thread;
- uint32_t dst_bytes_per_thread;
+ uint32_t src_bytes_per_hvx;
+ uint32_t dst_bytes_per_hvx;
};
-// scratch_bytes: per-thread VTCM after the two src buffers (at least 256)
+// scratch_bytes: per-hvx VTCM after the two src buffers (at least 256)
static inline void htp_im2col_vtcm_layout_build(
struct htp_im2col_vtcm_layout * L,
uint32_t src_row_bytes,
uint32_t dst_row_bytes,
- uint32_t n_threads,
+ uint32_t n_hvx,
uint32_t scratch_bytes) {
- L->src_bytes_per_thread = 2 * src_row_bytes + (scratch_bytes > 256 ? scratch_bytes : 256);
- L->dst_bytes_per_thread = 2 * dst_row_bytes;
+ L->src_bytes_per_hvx = 2 * src_row_bytes + (scratch_bytes > 256 ? scratch_bytes : 256);
+ L->dst_bytes_per_hvx = 2 * dst_row_bytes;
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_src, (size_t) L->src_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) L->dst_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_src, (size_t) L->src_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) L->dst_bytes_per_hvx * n_hvx);
L->total_bytes = off;
}
-// per-thread scratch: the KH == 1 blocked path keeps its offset table and one gathered row there,
+// per-hvx scratch: the KH == 1 blocked path keeps its offset table and one gathered row there,
// both padded to whole vectors
static inline uint32_t htp_im2col_scratch_bytes(uint32_t kernel_type, uint32_t KH, uint32_t patch_stride) {
if (kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA && KH == 1) {
@@ -48,7 +48,7 @@ static inline uint32_t htp_im2col_scratch_bytes(uint32_t kernel_type, uint32_t K
struct htp_im2col_kernel_params {
uint8_t kernel_type;
- uint8_t n_threads;
+ uint8_t n_hvx;
uint32_t vtcm_size;
uint32_t src_row_bytes;
diff --git a/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c
index c8a42eb4e..5d60c506e 100644
--- a/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c
+++ b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c
@@ -6,7 +6,7 @@
#include <string.h>
#include "dma-queue.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "hvx-utils.h"
#include "hvx-copy.h"
#include "htp-vtcm.h"
@@ -30,7 +30,7 @@ struct htp_lid_context {
const struct htp_lid_kernel_params * kp;
uint32_t n_blocks;
- uint32_t blocks_per_thread;
+ uint32_t blocks_per_hvx;
// current stream and token tile
uint32_t is;
@@ -72,7 +72,7 @@ static inline void lid_prep_push_step(dma_queue * dma_q, const struct htp_lid_co
// Splat every Q element (f16) and every head weight (f32) into a full vector.
// Work is split over (token, head) pairs.
-static void lid_prep_thread(unsigned int nth, unsigned int ith, void * data) {
+static void lid_prep_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_lid_context * lctx = (struct htp_lid_context *) data;
const struct htp_ops_context * octx = lctx->octx;
const struct htp_tensor * q = octx->src[0];
@@ -83,9 +83,9 @@ static void lid_prep_thread(unsigned int nth, unsigned int ith, void * data) {
const uint32_t D = q->ne[0];
const uint32_t n_head = q->ne[1];
- // nth is min(n_threads, n_items): the same chunk as a split over n_threads
+ // nth is min(n_hvx, n_items): the same chunk as a split over n_hvx
const uint32_t n_items = lctx->nt * n_head;
- const uint32_t chunk = fastdiv(n_items + octx->n_threads - 1, &octx->n_threads_div);
+ const uint32_t chunk = fastdiv(n_items + octx->n_hvx - 1, &octx->n_hvx_div);
const uint32_t i0 = ith * chunk;
const uint32_t i1 = MIN(i0 + chunk, n_items);
@@ -93,8 +93,8 @@ static void lid_prep_thread(unsigned int nth, unsigned int ith, void * data) {
return;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
- dma_queue * dma_q = octx->ctx->dma[ith];
+ struct htp_thread_trace * tr = th->trace;
+ dma_queue * dma_q = th->dma_q;
const uint32_t w_row_size = hex_round_up(n_head * sizeof(float), 128);
const size_t qs_size = n_head * L->size_qs_row;
@@ -506,7 +506,7 @@ static inline void lid_compute_row2(float * restrict o0,
}
// Each thread owns a range of kv blocks and streams K / mask in, output out (HTP_LID_N_BUF buffers)
-static void lid_compute_thread(unsigned int nth, unsigned int ith, void * data) {
+static void lid_compute_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_lid_context * lctx = (struct htp_lid_context *) data;
const struct htp_ops_context * octx = lctx->octx;
const struct htp_tensor * q = octx->src[0];
@@ -514,8 +514,8 @@ static void lid_compute_thread(unsigned int nth, unsigned int ith, void * data)
const struct htp_tensor * m = octx->src[3];
const struct htp_tensor * dst = octx->dst;
- const uint32_t b0 = lctx->blocks_per_thread * ith;
- const uint32_t b1 = MIN(b0 + lctx->blocks_per_thread, lctx->n_blocks);
+ const uint32_t b0 = lctx->blocks_per_hvx * ith;
+ const uint32_t b1 = MIN(b0 + lctx->blocks_per_hvx, lctx->n_blocks);
if (b0 >= b1) {
return;
@@ -556,8 +556,8 @@ static void lid_compute_thread(unsigned int nth, unsigned int ith, void * data)
HVX_Vector * sc = kt_t1 + HTP_LID_KV_BLOCK;
const HVX_Vector g_offs = hvx_vec_gather_offsets_h(L->size_k_raw_row);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
- dma_queue * dma_q = octx->ctx->dma[ith];
+ struct htp_thread_trace * tr = th->trace;
+ dma_queue * dma_q = th->dma_q;
for (uint32_t b = b0, i = 0; b < b1 && i < HTP_LID_N_BUF; ++b, ++i) {
const uint32_t kv0 = b * HTP_LID_KV_BLOCK;
@@ -796,7 +796,7 @@ static inline void lid_hmx_dot(const __fp16 * a, const __fp16 * b, uint32_t n) {
}
// Per segment: channel tile outer, row tile inner, one bias load per channel tile. S holds [n_rt][n_sg] tiles.
-static void lid_hmx_worker(void * data) {
+static void lid_hmx_worker(const htp_thread_t * th, void * data) {
const lid_hmx_job_t * job = (const lid_hmx_job_t *) data;
const size_t dot_stride = (size_t) job->n_dot * HMX_FP16_TILE_N_ELMS;
@@ -837,7 +837,7 @@ static void lid_hmx_worker(void * data) {
// Thread slices: K rows by row pairs of the padded block, mask / output rows by token.
// Block 0 is prepped together with Q: its K rows go to the threads without Q work.
-// K rows are split only in stages on all n_threads threads.
+// K rows are split only in stages on all n_hvx threads.
static inline void lid_hmx_rows(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g, uint32_t n_rows,
uint32_t * r0, uint32_t * r1) {
uint32_t per;
@@ -849,22 +849,22 @@ static inline void lid_hmx_rows(const struct htp_lid_hmx_context * hctx, uint32_
ith -= hctx->n_qthr;
per = (n_rows + nth - hctx->n_qthr - 1) / (nth - hctx->n_qthr);
} else {
- per = fastdiv(n_rows + hctx->octx->n_threads - 1, &hctx->octx->n_threads_div);
+ per = fastdiv(n_rows + hctx->octx->n_hvx - 1, &hctx->octx->n_hvx_div);
}
per = hex_round_up(per, 2);
*r0 = MIN(ith * per, n_rows);
*r1 = MIN(*r0 + per, n_rows);
}
-// Split over n_threads; a stage on one thread has one token, which this split also gives to thread 0
+// Split over n_hvx; a stage on one thread has one token, which this split also gives to thread 0
static inline void lid_hmx_toks(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nt, uint32_t * t0, uint32_t * t1) {
- const uint32_t per = fastdiv(nt + hctx->octx->n_threads - 1, &hctx->octx->n_threads_div);
+ const uint32_t per = fastdiv(nt + hctx->octx->n_hvx - 1, &hctx->octx->n_hvx_div);
*t0 = MIN(ith * per, nt);
*t1 = MIN((ith + 1) * per, nt);
}
// Push (or only count, push == false) the K row and mask DMAs of block g for this thread, one per segment
-static uint32_t lid_hmx_in(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g, bool push) {
+static uint32_t lid_hmx_in(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth, uint32_t g, bool push) {
const struct htp_ops_context * octx = hctx->octx;
const struct htp_tensor * k = octx->src[1];
const struct htp_tensor * m = octx->src[3];
@@ -878,7 +878,7 @@ static uint32_t lid_hmx_in(const struct htp_lid_hmx_context * hctx, uint32_t ith
lid_hmx_rows(hctx, ith, nth, g, n_rows, &r0, &r1);
lid_hmx_toks(hctx, ith, hctx->nt_run, &ta, &tb);
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
const uint32_t k_row_size = hctx->kp->k_row_size;
uint32_t n = 0;
@@ -907,8 +907,8 @@ static uint32_t lid_hmx_in(const struct htp_lid_hmx_context * hctx, uint32_t ith
return n;
}
-static inline uint32_t lid_hmx_push_in(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g) {
- return lid_hmx_in(hctx, ith, nth, g, true);
+static inline uint32_t lid_hmx_push_in(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth, uint32_t g) {
+ return lid_hmx_in(hctx, th, ith, nth, g, true);
}
// Row pair (a, b) quants of two blocks (64 values each, bytes 0..63) to the two tile vectors, scaled by s0, s1
@@ -1066,7 +1066,7 @@ static __attribute__((noinline)) void lid_hmx_prep_k(const struct htp_lid_hmx_co
}
// O column of each token (64 kv per gather) + mask -> f32 rows, then one DMA out per segment
-static __attribute__((noinline)) void lid_hmx_extract(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g) {
+static __attribute__((noinline)) void lid_hmx_extract(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth, uint32_t g) {
const struct htp_ops_context * octx = hctx->octx;
const struct htp_tensor * dst = octx->dst;
const struct htp_lid_hmx_vtcm_layout * L = &hctx->L;
@@ -1081,7 +1081,7 @@ static __attribute__((noinline)) void lid_hmx_extract(const struct htp_lid_hmx_c
uint32_t n_rows;
const uint32_t n_seg = lid_hmx_segs(hctx, g, seg, &n_rows);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) g);
const uint32_t n_u = hctx->n_u_run;
@@ -1138,7 +1138,7 @@ static __attribute__((noinline)) void lid_hmx_extract(const struct htp_lid_hmx_c
}
const struct lid_hmx_set S = hctx->sets[seg[i].set];
- dma_queue_push(octx->ctx->dma[ith],
+ dma_queue_push(th->dma_q,
dma_make_data(dst->data + S.is * dst->nb[3] + (S.t0 + ta) * dst->nb[1] + seg[i].kv0 * sizeof(float),
o_buf + ta * L->size_o_row + seg[i].x0 * sizeof(float)),
dst->nb[1], L->size_o_row, seg[i].nv * sizeof(float), tb - ta);
@@ -1159,15 +1159,15 @@ static inline uint32_t lid_hmx_n_out(const struct htp_lid_hmx_context * hctx, ui
return lid_hmx_segs(hctx, g, seg, &n_rows);
}
-static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth);
+static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth);
// Pipeline stage g (HMX runs block g meanwhile): prep block g + 1, extract block g - 1
-static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth) {
+static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth) {
const int32_t g = hctx->g;
const int32_t n = (int32_t) hctx->n_blocks;
- struct htp_thread_trace * tr = &hctx->octx->ctx->trace[ith];
- dma_queue * dma_q = hctx->octx->ctx->dma[ith];
+ struct htp_thread_trace * tr = th->trace;
+ dma_queue * dma_q = th->dma_q;
// FIFO order per thread: OUT(g - 3), IN(g + 1), OUT(g - 2). When this stage preps next run Q, also pop
// OUT(g - 2) so the queue is empty for the Q DMAs; else leave it in flight for one more stage.
@@ -1178,7 +1178,7 @@ static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, uint32_t ith,
n_pop += lid_hmx_n_out(hctx, ith, nth, g - 3);
}
if (g >= 0 && g + 1 < n) {
- n_pop += lid_hmx_in(hctx, ith, nth, g + 1, false);
+ n_pop += lid_hmx_in(hctx, th, ith, nth, g + 1, false);
}
if (lag == 2 && g - 2 >= 0) {
n_pop += lid_hmx_n_out(hctx, ith, nth, g - 2);
@@ -1189,11 +1189,11 @@ static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, uint32_t ith,
if (hctx->nx_on && g >= 0) {
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) g);
- lid_hmx_q_chunk(hctx, ith, nth);
+ lid_hmx_q_chunk(hctx, th, ith, nth);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) g);
}
if (g + 2 < n) {
- lid_hmx_push_in(hctx, ith, nth, g + 2);
+ lid_hmx_push_in(hctx, th, ith, nth, g + 2);
}
if (g + 1 < n) {
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (g + 1));
@@ -1201,7 +1201,7 @@ static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, uint32_t ith,
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (g + 1));
}
if (g - 1 >= 0) {
- lid_hmx_extract(hctx, ith, nth, g - 1);
+ lid_hmx_extract(hctx, th, ith, nth, g - 1);
}
if (g == n) {
uint32_t n_tail = lid_hmx_n_out(hctx, ith, nth, n - 1);
@@ -1214,8 +1214,8 @@ static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, uint32_t ith,
}
}
-static void lid_hmx_stage_thread(unsigned int nth, unsigned int ith, void * data) {
- lid_hmx_stage((const struct htp_lid_hmx_context *) data, ith, nth);
+static void lid_hmx_stage_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
+ lid_hmx_stage((const struct htp_lid_hmx_context *) data, th, ith, nth);
}
// Push the Q rows of channel tile c of set S, one DMA per token. Returns the number of DMA descriptors.
@@ -1361,19 +1361,19 @@ static inline uint32_t lid_hmx_q_chunk_push(const struct htp_lid_hmx_context * h
}
// Stage g share of the next run's Q prep (the DMA queue of the thread is empty here)
-static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth) {
+static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth) {
const struct htp_ops_context * octx = hctx->octx;
const struct htp_lid_hmx_vtcm_layout * L = &hctx->L;
const uint32_t n_tot = hctx->nx_n_run * L->n_ch;
const uint32_t a = fastdiv(hctx->g * n_tot, &hctx->nx_stg_div);
const uint32_t b = fastdiv((hctx->g + 1) * n_tot, &hctx->nx_stg_div);
- const uint32_t per = fastdiv(b - a + octx->n_threads - 1, &octx->n_threads_div);
+ const uint32_t per = fastdiv(b - a + octx->n_hvx - 1, &octx->n_hvx_div);
const uint32_t gc0 = MIN(a + ith * per, b);
const uint32_t gc1 = MIN(gc0 + per, b);
const uint32_t qb = hctx->qbuf ^ 1;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
uint8_t * qs[2];
qs[0] = hctx->qs + ith * L->size_qs;
qs[1] = qs[0] + 32 * L->size_qs_row;
@@ -1422,7 +1422,7 @@ static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_c
// Q rows to weight tiles, per channel scales |w| and G tiles for all sets of the run, then stage -1 (prep block 0).
// Q work goes to the first threads (thread 0 starts first), the K rows of block 0 to the others.
-static void lid_hmx_qprep_thread(unsigned int nth, unsigned int ith, void * data) {
+static void lid_hmx_qprep_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_lid_hmx_context * hctx = (struct htp_lid_hmx_context *) data;
const struct htp_ops_context * octx = hctx->octx;
const struct htp_tensor * q = octx->src[0];
@@ -1431,11 +1431,11 @@ static void lid_hmx_qprep_thread(unsigned int nth, unsigned int ith, void * data
const uint32_t nh = q->ne[1];
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
- dma_queue * dma_q = octx->ctx->dma[ith];
+ struct htp_thread_trace * tr = th->trace;
+ dma_queue * dma_q = th->dma_q;
const uint32_t n_tot = hctx->n_run * L->n_ch;
- const uint32_t per = fastdiv(n_tot + octx->n_threads - 1, &octx->n_threads_div);
+ const uint32_t per = fastdiv(n_tot + octx->n_hvx - 1, &octx->n_hvx_div);
const uint32_t gc0 = MIN(ith * per, n_tot);
const uint32_t gc1 = MIN(gc0 + per, n_tot);
@@ -1470,7 +1470,7 @@ static void lid_hmx_qprep_thread(unsigned int nth, unsigned int ith, void * data
n_dma += lid_hmx_push_q(hctx, dma_q, qs, &S, first - set * L->n_ch);
}
- uint32_t n_in = lid_hmx_push_in(hctx, ith, nth, 0);
+ uint32_t n_in = lid_hmx_push_in(hctx, th, ith, nth, 0);
for (uint32_t i = 0; i < n_dma; ++i) {
dma_queue_pop(dma_q);
@@ -1509,18 +1509,18 @@ static void lid_hmx_qprep_thread(unsigned int nth, unsigned int ith, void * data
dma_queue_pop(dma_q);
}
- lid_hmx_stage(hctx, ith, nth);
+ lid_hmx_stage(hctx, th, ith, nth);
}
// Run start when its Q was prepped in the previous run: block 0 K and mask, then stage -1
-static void lid_hmx_first_thread(unsigned int nth, unsigned int ith, void * data) {
+static void lid_hmx_first_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_lid_hmx_context * hctx = (const struct htp_lid_hmx_context *) data;
- const uint32_t n_in = lid_hmx_push_in(hctx, ith, nth, 0);
+ const uint32_t n_in = lid_hmx_push_in(hctx, th, ith, nth, 0);
for (uint32_t i = 0; i < n_in; ++i) {
- dma_queue_pop(hctx->octx->ctx->dma[ith]);
+ dma_queue_pop(th->dma_q);
}
- lid_hmx_stage(hctx, ith, nth);
+ lid_hmx_stage(hctx, th, ith, nth);
}
static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kernel_params * kparams) {
@@ -1529,7 +1529,7 @@ static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kerne
const uint32_t D = q->ne[0];
const uint32_t n_head = q->ne[1];
const uint32_t n_stream = q->ne[3];
- const uint32_t n_threads = kparams->n_threads;
+ const uint32_t n_hvx = kparams->n_hvx;
const uint32_t n_qsets = kparams->n_qsets;
struct htp_lid_hmx_context hctx;
@@ -1541,7 +1541,7 @@ static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kerne
hctx.n_tile = kparams->n_tokens_tile;
hctx.n_tt = kparams->n_tt;
hctx.n_qsets = n_qsets;
- htp_lid_hmx_vtcm_layout_build(&hctx.L, D, n_head, hctx.n_tile, hctx.Bc, n_qsets * kparams->n_qbuf, n_threads,
+ htp_lid_hmx_vtcm_layout_build(&hctx.L, D, n_head, hctx.n_tile, hctx.Bc, n_qsets * kparams->n_qbuf, n_hvx,
kparams->k_row_size);
uint8_t * const base = octx->ctx->vtcm_base;
@@ -1588,7 +1588,7 @@ static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kerne
const int32_t n = (int32_t) hctx.n_blocks;
const uint32_t n_tot = hctx.n_run * L->n_ch;
- const uint32_t per = fastdiv(n_tot + n_threads - 1, &octx->n_threads_div);
+ const uint32_t per = fastdiv(n_tot + n_hvx - 1, &octx->n_hvx_div);
hctx.n_qthr = (n_tot + per - 1) / per;
hctx.g = -1;
@@ -1597,9 +1597,9 @@ static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kerne
if (q_ready) {
hctx.n_qthr = 0;
hctx.nx_on = false;
- work_queue_run(octx->ctx->work_queue, lid_hmx_first_thread, &hctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, lid_hmx_first_thread, &hctx, n_hvx);
} else {
- work_queue_run(octx->ctx->work_queue, lid_hmx_qprep_thread, &hctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, lid_hmx_qprep_thread, &hctx, n_hvx);
}
// with two Q buffers the stages of this run prep the Q of the next one
@@ -1644,7 +1644,7 @@ static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kerne
// stage g has work only if there is a next block, a previous one or next run Q work
hctx.g = g;
if (g + 1 < n || g >= 1 || hctx.nx_on) {
- work_queue_run(octx->ctx->work_queue, lid_hmx_stage_thread, &hctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, lid_hmx_stage_thread, &hctx, n_hvx);
}
hmx_queue_pop(hmx_q);
@@ -1652,8 +1652,8 @@ static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kerne
// last stage only extracts; with one token only thread 0 has work (same token split for nth 1)
hctx.g = n;
- const uint32_t n_last = hctx.nt_run == 1 && !hctx.nx_on ? 1 : n_threads;
- work_queue_run(octx->ctx->work_queue, lid_hmx_stage_thread, &hctx, n_last);
+ const uint32_t n_last = hctx.nt_run == 1 && !hctx.nx_on ? 1 : n_hvx;
+ work_queue_run(octx->ctx->hvx_queue, lid_hmx_stage_thread, &hctx, n_last);
}
return HTP_STATUS_OK;
@@ -1674,7 +1674,7 @@ int op_lightning_indexer(struct htp_ops_context * octx) {
return HTP_STATUS_VTCM_TOO_SMALL;
}
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -1686,9 +1686,9 @@ int op_lightning_indexer(struct htp_ops_context * octx) {
lctx.octx = octx;
lctx.kp = kparams;
lctx.n_blocks = kparams->n_kv_blocks;
- lctx.blocks_per_thread = kparams->blocks_per_thread;
+ lctx.blocks_per_hvx = kparams->blocks_per_hvx;
- htp_lid_vtcm_layout_build(&lctx.L, q->ne[0], q->ne[1], kparams->n_tokens_tile, kparams->n_threads, kparams->k_row_size);
+ htp_lid_vtcm_layout_build(&lctx.L, q->ne[0], q->ne[1], kparams->n_tokens_tile, kparams->n_hvx, kparams->k_row_size);
uint8_t * const base = octx->ctx->vtcm_base;
@@ -1703,7 +1703,7 @@ int op_lightning_indexer(struct htp_ops_context * octx) {
const uint32_t n_tokens = q->ne[2];
const uint32_t n_stream = q->ne[3];
const uint32_t n_tile = kparams->n_tokens_tile;
- const uint32_t n_threads = kparams->n_threads;
+ const uint32_t n_hvx = kparams->n_hvx;
for (uint32_t is = 0; is < n_stream; ++is) {
lctx.is = is;
@@ -1713,8 +1713,8 @@ int op_lightning_indexer(struct htp_ops_context * octx) {
lctx.t0 = t0;
lctx.nt = MIN(n_tile, n_tokens - t0);
- work_queue_run(octx->ctx->work_queue, lid_prep_thread, &lctx, MIN(n_threads, lctx.nt * q->ne[1]));
- work_queue_run(octx->ctx->work_queue, lid_compute_thread, &lctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, lid_prep_thread, &lctx, MIN(n_hvx, lctx.nt * q->ne[1]));
+ work_queue_run(octx->ctx->hvx_queue, lid_compute_thread, &lctx, n_hvx);
}
}
diff --git a/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h
index 6571db32b..14b3503f6 100644
--- a/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h
+++ b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h
@@ -34,10 +34,10 @@ enum htp_lid_kernel_type {
struct htp_lid_kernel_params {
uint8_t kernel_type; // enum htp_lid_kernel_type
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t n_tokens_tile; // tokens per Q tile
uint32_t n_kv_blocks; // HVX: per stream, HMX: per run of n_qsets sets
- uint32_t blocks_per_thread;
+ uint32_t blocks_per_hvx;
uint32_t vtcm_size;
uint32_t k_row_size; // K row bytes
uint32_t kv_block; // HMX: kv rows per block (multiple of 64)
@@ -61,7 +61,7 @@ _Static_assert(sizeof(struct htp_lid_kernel_params) <= 128, "htp_lid_kernel_para
// VTCM layout shared by host (fit check) and device (buffer offsets).
// Shared regions: Q splats (one f16 vector per q element) and head weight splats for the current token tile.
-// Per-thread regions: Q prep staging, K / mask / output buffers, two transposed K blocks and transpose scratch.
+// Per-HVX regions: Q prep staging, K / mask / output buffers, two transposed K blocks and transpose scratch.
struct htp_lid_vtcm_layout {
size_t off_q;
size_t off_w;
@@ -87,7 +87,7 @@ struct htp_lid_vtcm_layout {
size_t total_bytes;
};
-static inline void htp_lid_vtcm_layout_build(struct htp_lid_vtcm_layout * L, size_t D, size_t n_head, size_t n_tok, size_t n_threads, size_t k_row_size) {
+static inline void htp_lid_vtcm_layout_build(struct htp_lid_vtcm_layout * L, size_t D, size_t n_head, size_t n_tok, size_t n_hvx, size_t k_row_size) {
L->size_q_row = hex_round_up(D * sizeof(uint16_t), 128);
L->size_qs_row = hex_round_up(D * sizeof(float), 128);
L->size_k_row = hex_round_up(D * sizeof(uint16_t), 128);
@@ -104,18 +104,18 @@ static inline void htp_lid_vtcm_layout_build(struct htp_lid_vtcm_layout * L, siz
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_q, n_tok * n_head * D * 128);
VTCM_LAYOUT_ALLOC(off, off_w, n_tok * n_head * 128);
- VTCM_LAYOUT_ALLOC(off, off_qs, n_threads * L->size_qs);
- VTCM_LAYOUT_ALLOC(off, off_k, n_threads * HTP_LID_N_BUF * L->size_k_buf);
- VTCM_LAYOUT_ALLOC(off, off_kt, n_threads * L->size_kt);
- VTCM_LAYOUT_ALLOC(off, off_m, n_threads * HTP_LID_N_BUF * L->size_m_buf);
- VTCM_LAYOUT_ALLOC(off, off_o, n_threads * HTP_LID_N_BUF * L->size_o_buf);
+ VTCM_LAYOUT_ALLOC(off, off_qs, n_hvx * L->size_qs);
+ VTCM_LAYOUT_ALLOC(off, off_k, n_hvx * HTP_LID_N_BUF * L->size_k_buf);
+ VTCM_LAYOUT_ALLOC(off, off_kt, n_hvx * L->size_kt);
+ VTCM_LAYOUT_ALLOC(off, off_m, n_hvx * HTP_LID_N_BUF * L->size_m_buf);
+ VTCM_LAYOUT_ALLOC(off, off_o, n_hvx * HTP_LID_N_BUF * L->size_o_buf);
L->total_bytes = off;
}
-static inline size_t htp_lid_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t n_threads, size_t k_row_size) {
+static inline size_t htp_lid_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t n_hvx, size_t k_row_size) {
struct htp_lid_vtcm_layout L;
- htp_lid_vtcm_layout_build(&L, D, n_head, n_tok, n_threads, k_row_size);
+ htp_lid_vtcm_layout_build(&L, D, n_head, n_tok, n_hvx, k_row_size);
return L.total_bytes;
}
@@ -156,7 +156,7 @@ struct htp_lid_hmx_vtcm_layout {
};
static inline void htp_lid_hmx_vtcm_layout_build(struct htp_lid_hmx_vtcm_layout * L,
- size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_threads, size_t k_row_size) {
+ size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_hvx, size_t k_row_size) {
L->n_ch = (uint32_t) ((n_tok * n_head + 31) / 32);
L->n_u = (uint32_t) ((n_tok + 31) / 32);
L->n_dot = (uint32_t) (D / 32);
@@ -186,7 +186,7 @@ static inline void htp_lid_hmx_vtcm_layout_build(struct htp_lid_hmx_vtcm_layout
VTCM_LAYOUT_ALLOC(off, off_sc, n_qsets * L->n_ch * 256);
VTCM_LAYOUT_ALLOC(off, off_sc_id, 256);
VTCM_LAYOUT_ALLOC(off, off_w, n_qsets * L->size_w);
- VTCM_LAYOUT_ALLOC(off, off_qs, n_threads * L->size_qs);
+ VTCM_LAYOUT_ALLOC(off, off_qs, n_hvx * L->size_qs);
for (int i = 0; i < 2; i++) {
VTCM_LAYOUT_ALLOC(off, off_k_raw[i], Bc * L->size_k_raw_row);
}
@@ -200,9 +200,9 @@ static inline void htp_lid_hmx_vtcm_layout_build(struct htp_lid_hmx_vtcm_layout
L->total_bytes = off;
}
-static inline size_t htp_lid_hmx_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_threads, size_t k_row_size) {
+static inline size_t htp_lid_hmx_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_hvx, size_t k_row_size) {
struct htp_lid_hmx_vtcm_layout L;
- htp_lid_hmx_vtcm_layout_build(&L, D, n_head, n_tok, Bc, n_qsets, n_threads, k_row_size);
+ htp_lid_hmx_vtcm_layout_build(&L, D, n_head, n_tok, Bc, n_qsets, n_hvx, k_row_size);
return L.total_bytes;
}
diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c
index 1424f1213..ad457d1a6 100644
--- a/ggml/src/ggml-hexagon/htp/main.c
+++ b/ggml/src/ggml-hexagon/htp/main.c
@@ -20,6 +20,7 @@
#include <string.h>
#include <stdatomic.h>
+#include "htp-thread.h"
#include "hex-utils.h"
#include "dma-queue.h"
#include "hmx-queue.h"
@@ -32,7 +33,7 @@
#include "htp-ops.h"
#include "htp_iface.h"
#include "work-queue.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "allreduce-ops.h"
#include "dsv4-hc-ops.h"
#include "htp-fence.h"
@@ -43,8 +44,7 @@
#define WORK_QUEUE_STACK_SIZE 16384
#define MAIN_THREAD_STACK_SIZE 32768
-_Static_assert(WORK_QUEUE_MAX_N_THREADS >= HTP_MAX_NTHREADS,
- "work-queue thread cap must be >= HTP_MAX_NTHREADS");
+atomic_uint htp_thread_count = 0;
struct htp_handle {
struct htp_context * ctx;
@@ -491,17 +491,23 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp
qurt_sysenv_max_hthreads_t hw_threads;
qurt_sysenv_get_max_hw_threads(&hw_threads);
uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF;
+ uint32_t hw_nth = hw_threads.max_hthreads;
+ if (hw_nth > HTP_MAX_NTHREADS) {
+ hw_nth = HTP_MAX_NTHREADS;
+ }
if (n_hvx == 0) {
n_hvx = hw_nhvx;
}
- if (n_hvx > hw_threads.max_hthreads) {
- n_hvx = hw_threads.max_hthreads;
+ if (n_hvx > HTP_MAX_NHVX) {
+ n_hvx = HTP_MAX_NHVX;
}
- if (n_hvx > HTP_MAX_NTHREADS) {
- n_hvx = HTP_MAX_NTHREADS;
+ if (n_hvx > hw_nth - 1) {
+ n_hvx = hw_nth - 1;
}
+ uint32_t n_threads = hw_nth;
+
// layout segments of our contiguous block
// 1. htp_context : sits at the base (block is 4K-aligned via memalign below)
@@ -520,13 +526,13 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp
size_t wq_align = work_queue_alignof();
size_t offset_wq = hex_align_up(offset, wq_align);
size_t size_wq = work_queue_sizeof(n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE);
- offset = offset_wq + size_wq;
+ offset = offset_wq + size_wq;
// 4. dma_queue
size_t dma_align = dma_queue_alignof();
size_t offset_dma = hex_align_up(offset, dma_align);
size_t size_dma = 0;
- for (uint32_t i = 0; i < n_hvx; i++) {
+ for (uint32_t i = 0; i < n_threads; i++) {
size_dma = hex_align_up(size_dma, dma_queue_alignof());
size_dma += dma_queue_sizeof(256);
}
@@ -657,16 +663,9 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp
}
#endif
- ctx->hmx_enabled = n_hmx;
- ctx->hmx_queue = NULL;
- if (n_hmx) {
- void * hmx_ptr = (void *) ((uintptr_t) block + offset_hmx);
- ctx->hmx_queue = hmx_queue_init(hmx_ptr, HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE, ctx->vtcm_rctx, &ctx->trace[HTP_MAX_NTHREADS]);
- }
- FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx);
-
- ctx->n_threads = n_hvx;
- ctx->n_threads_div = init_fastdiv_values(ctx->n_threads);
+ ctx->n_threads = n_threads;
+ ctx->n_hvx = n_hvx;
+ ctx->n_hvx_div = init_fastdiv_values(ctx->n_hvx);
// Initialize DMA queues
uint8_t * dma_ptr_curr = (uint8_t *) ((uintptr_t) block + offset_dma);
@@ -681,8 +680,18 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp
ctx->ddr_spad_size = HTP_DDR_SPAD_SIZE;
ctx->ddr_spad_base = memalign(128, ctx->ddr_spad_size);
+ htp_thread_reset();
+
void * wq_ptr = (void *) ((uintptr_t) block + offset_wq);
- ctx->work_queue = work_queue_init(wq_ptr, n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE);
+ ctx->hvx_queue = work_queue_init(wq_ptr, n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE, ctx->dma, ctx->trace);
+
+ ctx->hmx_enabled = n_hmx;
+ ctx->hmx_queue = NULL;
+ if (n_hmx) {
+ void * hmx_ptr = (void *) ((uintptr_t) block + offset_hmx);
+ ctx->hmx_queue = hmx_queue_init(hmx_ptr, HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE, ctx->vtcm_rctx, ctx->dma, ctx->trace);
+ }
+ FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx);
ctx->main_stack = NULL;
ctx->main_thread = 0;
@@ -708,7 +717,7 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp
}
FARF(HIGH, "session %u started: n-hvx %u vtcm-size %zu vtcm-rctx %u n-threads %u thread-id %d thread-prio %d \n",
- sess_id, hw_nhvx, ctx->vtcm_size, ctx->vtcm_rctx, ctx->n_threads, ctx->thread_id, ctx->thread_prio);
+ sess_id, ctx->n_hvx, ctx->vtcm_size, ctx->vtcm_rctx, ctx->n_threads, ctx->thread_id, ctx->thread_prio);
return AEE_SUCCESS;
}
@@ -733,7 +742,7 @@ AEEResult htp_iface_stop(remote_handle64 handle) {
return err;
}
- work_queue_free(ctx->work_queue);
+ work_queue_free(ctx->hvx_queue);
for (int i = 0; i < ctx->n_threads; i++) {
dma_queue_free(ctx->dma[i]);
@@ -769,16 +778,20 @@ AEEResult htp_iface_hwinfo(remote_handle64 handle, uint32_t * n_threads, uint32_
qurt_sysenv_get_max_hw_threads(&hw_threads);
uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF;
+ uint32_t n_threads_val = hw_threads.max_hthreads;
+ if (n_threads_val > HTP_MAX_NTHREADS) {
+ n_threads_val = HTP_MAX_NTHREADS;
+ }
+
uint32_t n_hvx_val = hw_nhvx;
- if (n_hvx_val > hw_threads.max_hthreads) {
- n_hvx_val = hw_threads.max_hthreads;
+ if (n_hvx_val > HTP_MAX_NHVX) {
+ n_hvx_val = HTP_MAX_NHVX;
}
- if (n_hvx_val > HTP_MAX_NTHREADS) {
- n_hvx_val = HTP_MAX_NTHREADS;
+ if (n_hvx_val > n_threads_val - 1) {
+ n_hvx_val = n_threads_val - 1;
}
- // for now we force n_threads == n_hvx
- *n_threads = n_hvx_val;
+ *n_threads = n_threads_val;
*n_hvx = n_hvx_val;
*n_hmx = 1;
@@ -1227,10 +1240,10 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs
struct htp_tensor * tens, uint32_t idx, struct htp_op_desc * op) {
memcpy(octx->op_params, op->params, sizeof(octx->op_params));
memcpy(octx->kernel_params, op->kernel_params, sizeof(octx->kernel_params));
- octx->flags = op->flags;
- octx->op = op->opcode;
- octx->n_threads = octx->ctx->n_threads;
- octx->n_threads_div = octx->ctx->n_threads_div;
+ octx->flags = op->flags;
+ octx->op = op->opcode;
+ octx->n_hvx = octx->ctx->n_hvx;
+ octx->n_hvx_div = octx->ctx->n_hvx_div;
FARF(HIGH, "proc-op #%u: opcode %u flags 0x%x", idx, octx->op, octx->flags);
@@ -1292,7 +1305,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
const uint32_t t_size = sizeof(struct htp_tensor) * n_tens;
const uint32_t o_size = sizeof(struct htp_op_desc) * n_ops;
const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops;
- const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req->n_traces * sizeof(struct htp_trace_desc);
+ const uint32_t tr_size = ctx->n_threads * req->n_traces * sizeof(struct htp_trace_desc);
if (dbuf->size < b_size + t_size + o_size + p_size + tr_size) {
FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf->size, b_size + t_size + o_size + p_size + tr_size);
@@ -1315,7 +1328,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
memset(ctx->trace, 0, sizeof(ctx->trace));
if (ctx->profiler == HTP_PROF_TRACE) {
struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size);
- for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
+ for (uint32_t t = 0; t < ctx->n_threads; t++) {
ctx->trace[t].events = &trace_events[t * req->n_traces];
ctx->trace[t].max_events = req->n_traces;
}
@@ -1339,7 +1352,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
mdev_group_init(ctx, req);
- work_queue_wakeup(ctx->work_queue);
+ work_queue_wakeup(ctx->hvx_queue);
if (ctx->hmx_queue) {
hmx_queue_wakeup(ctx->hmx_queue);
}
@@ -1372,7 +1385,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
hmx_queue_suspend(ctx->hmx_queue);
hmx_queue_flush(ctx->hmx_queue);
}
- work_queue_suspend(ctx->work_queue);
+ work_queue_suspend(ctx->hvx_queue);
// Flush remaining dirty tensors at the end of the batch
htp_trace_event_start(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0);
@@ -1395,7 +1408,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
rsp.cycles_stop = batch_prof.cycles_stop;
if (ctx->profiler == HTP_PROF_TRACE) {
- for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
+ for (uint32_t t = 0; t < ctx->n_threads; t++) {
rsp.n_traces[t] = ctx->trace[t].count;
}
}
diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c
index b4b943058..fb15310cf 100644
--- a/ggml/src/ggml-hexagon/htp/matmul-ops.c
+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c
@@ -100,7 +100,7 @@ struct htp_mm_context {
const float * restrict sz0, const float * restrict sz1);
// Precomputed values
- uint32_t src0_nrows_per_thread;
+ uint32_t src0_nrows_per_hvx;
uint32_t src0_row_start;
uint32_t src0_row_end;
uint32_t src0_row_size_padded;
@@ -115,12 +115,12 @@ struct htp_mm_context {
struct fastdiv_values mm_div_ne11;
// Precomputed block-parallel quantization values
- uint32_t quant_ib_first[WORK_QUEUE_MAX_N_THREADS];
- uint32_t quant_ib_last[WORK_QUEUE_MAX_N_THREADS];
- uint32_t quant_r[WORK_QUEUE_MAX_N_THREADS];
- uint32_t quant_c[WORK_QUEUE_MAX_N_THREADS];
+ uint32_t quant_ib_first[HTP_MAX_NHVX];
+ uint32_t quant_ib_last[HTP_MAX_NHVX];
+ uint32_t quant_r[HTP_MAX_NHVX];
+ uint32_t quant_c[HTP_MAX_NHVX];
uint32_t n_quant_tasks;
- uint32_t n_quant_rows_per_thread;
+ uint32_t n_quant_rows_per_hvx;
// Fields for scattered mapping & HMX support in MUL_MAT_ID
const uint32_t * matrix_row_counts;
@@ -140,21 +140,21 @@ struct htp_mm_context {
uint32_t vtcm_act_raw_stride;
// Cached thread offsets/sizes
- uint32_t vtcm_src0_size_per_thread;
- uint32_t vtcm_act_size_per_thread;
- uint32_t vtcm_dst_size_per_thread;
+ uint32_t vtcm_src0_size_per_hvx;
+ uint32_t vtcm_act_size_per_hvx;
+ uint32_t vtcm_dst_size_per_hvx;
};
static int htp_mm_init_context(
struct htp_ops_context * octx,
const struct htp_mm_kernel_params * kparams
) {
- if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, (uint32_t) kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
if (kparams->n_hmx) {
- if (kparams->n_act_threads <= 0 || kparams->n_act_threads > (int32_t) octx->n_threads) {
+ if (kparams->n_act_hvx <= 0 || kparams->n_act_hvx > (int32_t) octx->n_hvx) {
return HTP_STATUS_INVAL_PARAMS;
}
}
@@ -236,8 +236,8 @@ static const uint8_t __attribute__((aligned(VLEN))) kvalues_mxfp4_lut[] = {
#define htp_matmul_preamble \
struct htp_mm_context * mmctx = data; \
struct htp_ops_context * octx = mmctx->octx; \
- dma_queue * dma_q = octx->ctx->dma[ith]; \
- uint32_t src0_nrows_per_thread = mmctx->src0_nrows_per_thread; \
+ dma_queue * dma_q = th->dma_q; \
+ uint32_t src0_nrows_per_hvx = mmctx->src0_nrows_per_hvx; \
htp_matmul_tensors_preamble;
@@ -248,17 +248,17 @@ static const uint8_t __attribute__((aligned(VLEN))) kvalues_mxfp4_lut[] = {
#include "hvx-mm-kernels-float.h"
#include "hmx-mm-kernels-tiled.h"
-static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
+static void hvx_mm_2d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
htp_matmul_preamble;
const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start;
const uint32_t act_nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13);
const uint32_t cur_m_start = mmctx->cur_m_start;
- const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end);
+ const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
const uint32_t n_prefetch = kparams->n_prefetch;
@@ -269,8 +269,8 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
const size_t act_stride = mmctx->vtcm_act_stride;
const size_t src2_stride = src2 ? ((src2->ne[1] == 1) ? 0 : src2->nb[1]) : 0;
- uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith;
- uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith;
+ uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith;
+ uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith;
uint8_t * restrict act_data = mmctx->vtcm_act;
const dma_addr_t src0_row = src0->data;
@@ -347,15 +347,15 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
}
}
-static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
+static void hvx_mv_2d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
htp_matmul_preamble;
const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start;
- const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end);
+ const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
const uint32_t n_prefetch = kparams->n_prefetch;
@@ -365,8 +365,8 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
const size_t act_row_size = nb11;
const size_t act_stride = mmctx->vtcm_act_stride;
- uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith;
- uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith;
+ uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith;
+ uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith;
uint8_t * act_data = mmctx->vtcm_act;
float * tmp = (float *) vtcm_dst_ptr;
@@ -441,7 +441,7 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
}
}
-static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
+static void hvx_mm_nx_2d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_mm_context * mmctx = data;
struct htp_ops_context * octx = mmctx->octx;
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
@@ -452,10 +452,10 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data)
const uint32_t act_nrows = act->ne[1] * act->ne[2] * act->ne[3];
const size_t act_stride = mmctx->vtcm_act_stride;
- uint8_t * restrict vtcm_weight_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith;
+ uint8_t * restrict vtcm_weight_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith;
uint8_t * restrict act_data = mmctx->vtcm_act;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const uint32_t n_prefetch = kparams->n_prefetch;
assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0);
@@ -463,7 +463,7 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data)
const struct htp_tensor * restrict src_w = octx->src[widx];
const struct htp_tensor * restrict dst = octx->dsts[widx];
if (!src_w || !dst) continue;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type);
const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type);
@@ -487,11 +487,11 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data)
}
const uint32_t nrows = src0_end_row - src0_start_row;
- uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div);
- src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32);
+ uint32_t src0_nrows_per_hvx = fastdiv(nrows + nth - 1, &octx->n_hvx_div);
+ src0_nrows_per_hvx = hex_round_up(src0_nrows_per_hvx, 32);
- const uint32_t start_row = src0_start_row + src0_nrows_per_thread * ith;
- const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_end_row);
+ const uint32_t start_row = src0_start_row + src0_nrows_per_hvx * ith;
+ const uint32_t end_row = MIN(start_row + src0_nrows_per_hvx, src0_end_row);
if (start_row >= end_row) continue;
uint32_t ct_start = start_row / 32;
@@ -612,25 +612,25 @@ static void hvx_mm_transfer_act_dma(
}
#define QUANTIZE_IMPL(name, log_name, kernel_fn, dst_row_size_expr) \
-static void name(unsigned int nth, unsigned int ith, void * data) { \
+static void name(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { \
(void) nth; \
struct htp_mm_context * mmctx = data; \
struct htp_ops_context * octx = mmctx->octx; \
const struct htp_tensor * src = mmctx->act; \
const uint32_t ne0 = src->ne[0]; \
const uint32_t nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : mmctx->act_nrows; \
- const uint32_t nrows_per_thread = mmctx->n_quant_rows_per_thread; \
+ const uint32_t nrows_per_hvx = mmctx->n_quant_rows_per_hvx; \
\
- const uint32_t ir_first = nrows_per_thread * ith; \
+ const uint32_t ir_first = nrows_per_hvx * ith; \
if (ir_first >= nrows) { \
return; \
} \
\
- struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
+ struct htp_thread_trace * tr = th->trace; \
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \
\
uint8_t * restrict dst = mmctx->vtcm_act; \
- const uint32_t ir_last = MIN(ir_first + nrows_per_thread, nrows); \
+ const uint32_t ir_last = MIN(ir_first + nrows_per_hvx, nrows); \
const size_t raw_row_size = mmctx->vtcm_act_raw_stride; \
const size_t dst_row_size = (dst_row_size_expr); \
\
@@ -650,14 +650,14 @@ QUANTIZE_IMPL(quantize_f32_f32, "quantize-f32-f32", quanti
QUANTIZE_IMPL(quantize_f32_f16, "quantize-f32-f16", quantize_f32_f16_kernel, mmctx->vtcm_act_stride)
QUANTIZE_IMPL(quantize_f16_f16, "quantize-f16-f16", quantize_f16_f16_kernel, mmctx->vtcm_act_stride)
-static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, void * data) {
+static void quantize_f32_q8_0_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct htp_mm_context * mmctx = data;
if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) {
return;
}
struct htp_ops_context * octx = mmctx->octx;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);
const struct htp_tensor * src = mmctx->act;
@@ -678,14 +678,14 @@ static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, vo
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);
}
-static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, void * data) {
+static void quantize_f32_q8_1_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct htp_mm_context * mmctx = data;
if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) {
return;
}
struct htp_ops_context * octx = mmctx->octx;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);
const struct htp_tensor * src = mmctx->act;
@@ -706,14 +706,14 @@ static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, vo
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);
}
-static void quantize_f32_q8_1_s16_tiled_block(unsigned int nth, unsigned int ith, void * data) {
+static void quantize_f32_q8_1_s16_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct htp_mm_context * mmctx = data;
if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) {
return;
}
struct htp_ops_context * octx = mmctx->octx;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);
const struct htp_tensor * src = mmctx->act;
@@ -734,14 +734,14 @@ static void quantize_f32_q8_1_s16_tiled_block(unsigned int nth, unsigned int ith
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);
}
-static void quantize_f32_f16_tiled_block(unsigned int nth, unsigned int ith, void * data) {
+static void quantize_f32_f16_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct htp_mm_context * mmctx = data;
if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) {
return;
}
struct htp_ops_context * octx = mmctx->octx;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);
const struct htp_tensor * src = mmctx->act;
@@ -762,14 +762,14 @@ static void quantize_f32_f16_tiled_block(unsigned int nth, unsigned int ith, voi
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);
}
-static void quantize_f16_f16_tiled_block(unsigned int nth, unsigned int ith, void * data) {
+static void quantize_f16_f16_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct htp_mm_context * mmctx = data;
if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) {
return;
}
struct htp_ops_context * octx = mmctx->octx;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);
const struct htp_tensor * src = mmctx->act;
@@ -791,7 +791,7 @@ static void quantize_f16_f16_tiled_block(unsigned int nth, unsigned int ith, voi
}
// q8_1 for weight types with offsets (q8_1_s16 for Q2_K), f16_tiled for F16, otherwise q8_0
-static inline worker_callback_t htp_mm_act_quant_row_func(int weight_type, int act_type) {
+static inline work_queue_func_t htp_mm_act_quant_row_func(int weight_type, int act_type) {
if (weight_type == HTP_TYPE_F16) {
return (act_type == HTP_TYPE_F16) ? quantize_f16_f16_tiled : quantize_f32_f16_tiled;
}
@@ -801,7 +801,7 @@ static inline worker_callback_t htp_mm_act_quant_row_func(int weight_type, int a
return htp_mm_weight_has_offset(weight_type) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;
}
-static inline worker_callback_t htp_mm_act_quant_block_func(int weight_type, int act_type) {
+static inline work_queue_func_t htp_mm_act_quant_block_func(int weight_type, int act_type) {
if (weight_type == HTP_TYPE_F16) {
return (act_type == HTP_TYPE_F16) ? quantize_f16_f16_tiled_block : quantize_f32_f16_tiled_block;
}
@@ -811,17 +811,17 @@ static inline worker_callback_t htp_mm_act_quant_block_func(int weight_type, int
return htp_mm_weight_has_offset(weight_type) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;
}
-static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) {
+static void hvx_mm_4d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
htp_matmul_preamble;
const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start;
const uint32_t cur_m_rows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13);
const uint32_t cur_m_start = mmctx->cur_m_start;
- const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end);
+ const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
const uint32_t n_prefetch = kparams->n_prefetch;
@@ -830,7 +830,7 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) {
const size_t dst_row_size = nb1;
const size_t act_stride = mmctx->vtcm_act_stride;
- uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith;
+ uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith;
uint8_t * restrict act_data = mmctx->vtcm_act;
const uint32_t tile_size = htp_tensor_get_tile_size(src0->type);
@@ -935,7 +935,7 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) {
}
}
-static void hvx_mm_2d_flat(unsigned int nth, unsigned int ith, void * data) {
+static void hvx_mm_2d_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
htp_matmul_preamble;
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
@@ -947,11 +947,11 @@ static void hvx_mm_2d_flat(unsigned int nth, unsigned int ith, void * data) {
const uint32_t act_nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : mmctx->act_nrows;
const uint32_t cur_m_start = mmctx->cur_m_start;
- const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end);
+ const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end);
const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const size_t dst_row_size = nb1;
const size_t src0_row_size = nb01;
@@ -961,8 +961,8 @@ static void hvx_mm_2d_flat(unsigned int nth, unsigned int ith, void * data) {
const size_t act_stride = mmctx->vtcm_act_stride;
// Per-thread VTCMs for all tensors
- uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith;
- uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith;
+ uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith;
+ uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith;
uint8_t * restrict act_data = mmctx->vtcm_act;
const dma_addr_t src0_row = src0->data;
@@ -1038,15 +1038,15 @@ static void hvx_mm_2d_flat(unsigned int nth, unsigned int ith, void * data) {
}
}
-static void hvx_mv_2d_flat(unsigned int nth, unsigned int ith, void * data) {
+static void hvx_mv_2d_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
htp_matmul_preamble;
const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start;
- const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end);
+ const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const size_t dst_row_size = nb1;
const size_t src0_row_size = nb01;
@@ -1056,8 +1056,8 @@ static void hvx_mv_2d_flat(unsigned int nth, unsigned int ith, void * data) {
const size_t act_stride = mmctx->vtcm_act_stride;
// Per-thread VTCMs for all tensors
- uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith;
- uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith;
+ uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith;
+ uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith;
uint8_t * act_data = mmctx->vtcm_act;
float * tmp = (float *) vtcm_dst_ptr;
@@ -1143,7 +1143,7 @@ static void hvx_mv_2d_flat(unsigned int nth, unsigned int ith, void * data) {
}
}
-static void hvx_mm_4d_flat(unsigned int nth, unsigned int ith, void * data) {
+static void hvx_mm_4d_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
htp_matmul_preamble;
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
@@ -1155,18 +1155,18 @@ static void hvx_mm_4d_flat(unsigned int nth, unsigned int ith, void * data) {
const uint32_t cur_m_rows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13);
const uint32_t cur_m_start = mmctx->cur_m_start;
- const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end);
+ const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end);
const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const size_t dst_row_size = nb1;
const size_t src0_row_size = nb01;
const size_t src0_stride = mmctx->vtcm_src0_stride;
const size_t act_stride = mmctx->vtcm_act_stride;
- uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith;
+ uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith;
uint8_t * restrict act_data = mmctx->vtcm_act;
@@ -1292,22 +1292,22 @@ static void hvx_mm_4d_flat(unsigned int nth, unsigned int ith, void * data) {
#define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id) * mmctx->mapping_stride + (i1)]
-static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) {
+static void hvx_mm_id_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
htp_matmul_preamble;
const struct htp_tensor * restrict ids = octx->src[2];
const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert
const uint32_t act_nrows = ne11;
- const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end);
+ const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end);
if (src0_start_row >= src0_end_row) {
return;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
const uint32_t n_prefetch = kparams->n_prefetch;
@@ -1325,7 +1325,7 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) {
const size_t act_stride = mmctx->vtcm_act_stride;
// Per-thread VTCMs for all tensors
- uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith;
+ uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith;
uint8_t * restrict act_data = mmctx->vtcm_act;
for (uint32_t cur_a = 0; cur_a < n_as; ++cur_a) {
@@ -1380,21 +1380,21 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) {
}
}
-static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) {
+static void hvx_mv_id_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
htp_matmul_preamble;
const struct htp_tensor * restrict ids = octx->src[2];
const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert
- const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end);
+ const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end);
if (src0_start_row >= src0_end_row) {
return;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
const uint32_t n_prefetch = kparams->n_prefetch;
@@ -1409,7 +1409,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) {
const uint32_t n_ids = ne02; // num experts
// Per-thread VTCMs for all tensors
- uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith;
+ uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith;
uint8_t * restrict act_data = mmctx->vtcm_act;
for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) { // for each expert
@@ -1457,7 +1457,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) {
}
}
-static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) {
+static void hvx_mv_id_nx_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_mm_context * mmctx = (struct htp_mm_context *) data;
struct htp_ops_context * octx = mmctx->octx;
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
@@ -1467,7 +1467,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
const struct htp_tensor * restrict ids = octx->src[n_weights + 1];
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const uint32_t n_prefetch = kparams->n_prefetch;
assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0);
@@ -1475,7 +1475,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
const uint32_t n_aids = ids->ne[0];
const uint32_t n_ids = src0->ne[2];
- uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith;
+ uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith;
uint8_t * restrict act_data = mmctx->vtcm_act;
for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) {
@@ -1487,7 +1487,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
const struct htp_tensor * restrict src_w = octx->src[p];
const struct htp_tensor * restrict dst = octx->dsts[p];
if (!src_w || !dst) continue;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
const uint32_t ne01 = src_w->ne[1];
uint32_t start_row = 0;
@@ -1500,11 +1500,11 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
}
const uint32_t nrows = end_row - start_row;
- uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div);
- src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32);
+ uint32_t src0_nrows_per_hvx = fastdiv(nrows + nth - 1, &octx->n_hvx_div);
+ src0_nrows_per_hvx = hex_round_up(src0_nrows_per_hvx, 32);
- const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row);
+ const uint32_t src0_start_row = start_row + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, end_row);
if (src0_start_row >= src0_end_row) continue;
const dma_addr_t src0_row = src_w->data + eid * src_w->nb[2];
@@ -1546,7 +1546,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
}
}
-static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) {
+static void hvx_mm_id_nx_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_mm_context * mmctx = (struct htp_mm_context *) data;
struct htp_ops_context * octx = mmctx->octx;
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
@@ -1556,7 +1556,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
const struct htp_tensor * restrict ids = octx->src[n_weights + 1];
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const uint32_t n_prefetch = kparams->n_prefetch;
assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0);
@@ -1568,7 +1568,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
const size_t act_stride = mmctx->vtcm_act_stride;
- uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith;
+ uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith;
uint8_t * restrict act_data = mmctx->vtcm_act;
for (uint32_t cur_a = 0; cur_a < n_as; ++cur_a) {
@@ -1579,7 +1579,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
const struct htp_tensor * restrict src_w = octx->src[p];
const struct htp_tensor * restrict dst = octx->dsts[p];
if (!src_w || !dst) continue;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
const uint32_t ne01 = src_w->ne[1];
uint32_t start_row = 0;
@@ -1592,11 +1592,11 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
}
const uint32_t nrows = end_row - start_row;
- uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div);
- src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32);
+ uint32_t src0_nrows_per_hvx = fastdiv(nrows + nth - 1, &octx->n_hvx_div);
+ src0_nrows_per_hvx = hex_round_up(src0_nrows_per_hvx, 32);
- const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row);
+ const uint32_t src0_start_row = start_row + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, end_row);
if (src0_start_row >= src0_end_row) continue;
const dma_addr_t src0_row = src_w->data + cur_a * src_w->nb[2];
@@ -1751,12 +1751,12 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
const bool is_tiled = htp_tensor_is_tiled(src0);
- // Compute src0_nrows_per_thread
- mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div);
+ // Compute src0_nrows_per_hvx
+ mmctx->src0_nrows_per_hvx = fastdiv(nrows + octx->n_hvx - 1, &octx->n_hvx_div);
if (is_tiled) {
- mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32);
+ mmctx->src0_nrows_per_hvx = hex_round_up(mmctx->src0_nrows_per_hvx, 32);
} else {
- mmctx->src0_nrows_per_thread += (mmctx->src0_nrows_per_thread & 1); // round up to even
+ mmctx->src0_nrows_per_hvx += (mmctx->src0_nrows_per_hvx & 1); // round up to even
}
const size_t src0_row_size = nb01;
@@ -1765,8 +1765,8 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128);
- worker_callback_t quant_task_func;
- worker_callback_t matmul_job_func;
+ work_queue_func_t quant_task_func;
+ work_queue_func_t matmul_job_func;
uint32_t n_quant_tasks = 1;
const bool is_batched = (ne12 > 1 || ne13 > 1 || ne02 > 1 || ne03 > 1);
if (is_batched) {
@@ -1821,8 +1821,8 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
const uint32_t nb = (ne10 + qk - 1) / qk;
const uint32_t total_nb = act_nrows * nb;
- if (act_nrows < octx->n_threads && !is_batched) {
- n_quant_tasks = MIN(total_nb, octx->n_threads);
+ if (act_nrows < octx->n_hvx && !is_batched) {
+ n_quant_tasks = MIN(total_nb, octx->n_hvx);
quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type);
for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) {
uint32_t ib_first = (total_nb * ith) / n_quant_tasks;
@@ -1833,7 +1833,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
mmctx->quant_c[ith] = ib_first % nb;
}
} else {
- n_quant_tasks = MIN(act_nrows, octx->n_threads);
+ n_quant_tasks = MIN(act_nrows, octx->n_hvx);
quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type);
}
act_row_size = kparams->act_row_size;
@@ -1846,7 +1846,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
const uint32_t m_layout_rows = m_chunk;
struct htp_mm_hvx_vtcm_layout L;
- htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, m_layout_rows, octx->n_threads,
+ htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, m_layout_rows, octx->n_hvx,
dst_row_size, src0_row_size, act_row_size, src2 ? src2->nb[1] : 0, kparams->n_prefetch, false, false);
if (kparams->kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM ||
@@ -1854,13 +1854,13 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
kparams->kernel_type == HTP_MM_KERNEL_HVX_BF16_F32_VTCM ||
kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW ||
kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {
- mmctx->vtcm_act_size_per_thread = L.act_bytes;
+ mmctx->vtcm_act_size_per_hvx = L.act_bytes;
} else {
- mmctx->vtcm_act_size_per_thread = fastdiv(L.act_bytes, &octx->n_threads_div);
+ mmctx->vtcm_act_size_per_hvx = fastdiv(L.act_bytes, &octx->n_hvx_div);
}
- mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div);
- mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div);
+ mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div);
+ mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div);
const size_t vtcm_size = L.total_bytes;
@@ -1910,8 +1910,8 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
const uint32_t total_nb = cur_m_rows * nb;
uint32_t quant_tasks;
work_queue_func_t q_func;
- if (cur_m_rows < octx->n_threads && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW)) {
- quant_tasks = MIN(total_nb, octx->n_threads);
+ if (cur_m_rows < octx->n_hvx && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW)) {
+ quant_tasks = MIN(total_nb, octx->n_hvx);
q_func = htp_mm_act_quant_block_func(src0->type, act->type);
for (uint32_t ith = 0; ith < quant_tasks; ++ith) {
uint32_t ib_first = (total_nb * ith) / quant_tasks;
@@ -1922,17 +1922,17 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
mmctx->quant_c[ith] = ib_first % nb;
}
} else {
- quant_tasks = MIN(cur_m_rows, octx->n_threads);
+ quant_tasks = MIN(cur_m_rows, octx->n_hvx);
q_func = quant_task_func;
- mmctx->n_quant_rows_per_thread = (cur_m_rows + quant_tasks - 1) / quant_tasks;
+ mmctx->n_quant_rows_per_hvx = (cur_m_rows + quant_tasks - 1) / quant_tasks;
}
mmctx->n_quant_tasks = quant_tasks;
- work_queue_run(octx->ctx->work_queue, q_func, mmctx, quant_tasks);
+ work_queue_run(octx->ctx->hvx_queue, q_func, mmctx, quant_tasks);
} else {
hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act, mmctx->vtcm_act_stride, m_start, cur_m_rows);
}
- work_queue_run(octx->ctx->work_queue, matmul_job_func, mmctx, octx->n_threads);
+ work_queue_run(octx->ctx->hvx_queue, matmul_job_func, mmctx, octx->n_hvx);
}
} else {
mmctx->cur_m_start = 0;
@@ -1940,22 +1940,22 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
if (need_quant) {
hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows);
- mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks;
+ mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks;
mmctx->n_quant_tasks = n_quant_tasks;
- work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks);
+ work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks);
} else {
hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act, mmctx->vtcm_act_stride, 0, act_nrows);
}
- work_queue_run(octx->ctx->work_queue, matmul_job_func, mmctx, octx->n_threads);
+ work_queue_run(octx->ctx->hvx_queue, matmul_job_func, mmctx, octx->n_hvx);
}
return HTP_STATUS_OK;
}
-static void dequantize_tiled_worker_loop(unsigned int n, unsigned int i, void * data) {
+static void dequantize_tiled_worker_loop(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) {
tiled_dequantize_state_t * state = (tiled_dequantize_state_t *) data;
- struct htp_thread_trace * tr = &state->traces[i];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i);
for (unsigned int task_id = i; task_id < (unsigned int) state->n_tasks; task_id += n) {
int start = task_id * state->n_tiles_per_task;
@@ -1965,10 +1965,10 @@ static void dequantize_tiled_worker_loop(unsigned int n, unsigned int i, void *
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i);
}
-static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void *data) {
+static void transfer_output_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) {
output_transfer_task_state_t *st = (output_transfer_task_state_t *) data;
- struct htp_thread_trace * tr = &st->traces[i];
+ struct htp_thread_trace * tr = th->trace;
int start_chunk_idx = i * st->n_chunks_per_task;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, start_chunk_idx);
@@ -1986,8 +1986,6 @@ static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void
}
typedef struct {
- struct htp_context * ctx;
- struct htp_thread_trace * traces;
__fp16 * dst;
dma_addr_t act_dma_addr;
float * vtcm_f32_act;
@@ -1997,15 +1995,13 @@ typedef struct {
uint32_t k_block;
uint32_t k_stride;
uint32_t k_valid;
- size_t vtcm_f32_act_bytes_per_thread;
+ size_t vtcm_f32_act_bytes_per_hvx;
uint32_t dma_step_rows;
uint32_t dma_step_rows_shift;
uint32_t act_elem_size; // 4=F32 src1, 2=F16 src1
} activation_transfer_task_state_t;
typedef struct {
- struct htp_context * ctx;
- struct htp_thread_trace * traces;
__fp16 * dst;
dma_addr_t act_dma_addr;
float * vtcm_f32_act;
@@ -2014,7 +2010,7 @@ typedef struct {
uint32_t k_stride;
uint32_t k_valid;
uint32_t n_col_chunks;
- struct fastdiv_values n_threads_div;
+ struct fastdiv_values n_hvx_div;
size_t vtcm_f32_act_bytes;
uint32_t dma_step_rows;
uint32_t dma_step_rows_shift;
@@ -2088,16 +2084,16 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk(
}
}
-static void transfer_activation_chunk_col_chunk_worker_fn(unsigned int n, unsigned int i, void *data) {
+static void transfer_activation_chunk_col_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) {
activation_transfer_col_chunk_state_t *st = (activation_transfer_col_chunk_state_t *) data;
- struct htp_thread_trace * tr = &st->traces[i];
+ struct htp_thread_trace * tr = th->trace;
uint32_t n_blocks = st->k_block / 32;
- uint32_t b_first = fastdiv(n_blocks * i, &st->n_threads_div);
- uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_threads_div);
+ uint32_t b_first = fastdiv(n_blocks * i, &st->n_hvx_div);
+ uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_hvx_div);
uint32_t c_first = b_first * 32;
- uint32_t c_last = b_last * 32;
- uint32_t c_len = c_last - c_first;
+ uint32_t c_last = b_last * 32;
+ uint32_t c_len = c_last - c_first;
if (c_len == 0) {
return;
@@ -2110,11 +2106,11 @@ static void transfer_activation_chunk_col_chunk_worker_fn(unsigned int n, unsign
__fp16 *dst = st->dst;
- size_t thread_scratch_bytes = hex_align_down(fastdiv(st->vtcm_f32_act_bytes, &st->n_threads_div), 128);
+ size_t thread_scratch_bytes = hex_align_down(fastdiv(st->vtcm_f32_act_bytes, &st->n_hvx_div), 128);
float *thread_f32_act = (float *)((char *)st->vtcm_f32_act + i * thread_scratch_bytes);
transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk(
- st->ctx->dma[i], dst, st->act_dma_addr, st->n_rows, st->k_block, st->k_stride, k_chunk_valid,
+ th->dma_q, dst, st->act_dma_addr, st->n_rows, st->k_block, st->k_stride, k_chunk_valid,
c_first, c_len, thread_f32_act, tr, st->dma_step_rows, st->dma_step_rows_shift
);
}
@@ -2202,10 +2198,10 @@ static void transfer_activation_chunk_to_fp16_dma_pipelined(
}
}
-static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i, void *data) {
+static void transfer_activation_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) {
activation_transfer_task_state_t *st = (activation_transfer_task_state_t *) data;
- struct htp_thread_trace * tr = &st->traces[i];
+ struct htp_thread_trace * tr = th->trace;
for (unsigned int task_id = i; task_id < (unsigned int)st->n_tasks; task_id += n) {
int chunk_idx = task_id * st->n_chunks_per_task;
@@ -2214,15 +2210,14 @@ static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i,
__fp16 *dst = st->dst + chunk_idx * st->k_block;
const dma_addr_t act_dma_addr = st->act_dma_addr + (size_t) chunk_idx * st->k_stride * st->act_elem_size;
- uint8_t *thread_act = (uint8_t *) st->vtcm_f32_act + i * st->vtcm_f32_act_bytes_per_thread;
+ uint8_t *thread_act = (uint8_t *) st->vtcm_f32_act + i * st->vtcm_f32_act_bytes_per_hvx;
transfer_activation_chunk_to_fp16_dma_pipelined(
- st->ctx->dma[i], dst, act_dma_addr, chunk_size, st->k_block, st->k_stride, st->k_valid, thread_act, st->act_elem_size, tr, st->dma_step_rows, st->dma_step_rows_shift
+ th->dma_q, dst, act_dma_addr, chunk_size, st->k_block, st->k_stride, st->k_valid, thread_act, st->act_elem_size, tr, st->dma_step_rows, st->dma_step_rows_shift
);
}
}
typedef struct {
- struct htp_thread_trace * traces;
const struct mmid_row_mapping * matrix_rows;
__fp16 * dst;
const float * src;
@@ -2242,7 +2237,6 @@ typedef struct {
} activation_transfer_gathered_task_state_t;
typedef struct {
- struct htp_thread_trace * traces;
const struct mmid_row_mapping * matrix_rows;
const __fp16 * vtcm_src;
float * dst;
@@ -2258,9 +2252,9 @@ typedef struct {
uint32_t cne1;
} output_transfer_scattered_task_state_t;
-static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigned int i, void *data) {
+static void transfer_activation_chunk_gathered_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) {
activation_transfer_gathered_task_state_t *st = data;
- struct htp_thread_trace * tr = &st->traces[i];
+ struct htp_thread_trace * tr = th->trace;
int chunk_idx = i;
int chunk_size = st->n_chunks_per_task;
int vtcm_start_row = chunk_idx * chunk_size;
@@ -2276,9 +2270,9 @@ static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigne
}
}
-static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, unsigned int i, void *data) {
+static void transfer_activation_chunk_gathered_worker_flat_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) {
activation_transfer_gathered_task_state_t *st = data;
- struct htp_thread_trace * tr = &st->traces[i];
+ struct htp_thread_trace * tr = th->trace;
int chunk_idx = i;
int chunk_size = st->n_chunks_per_task;
int vtcm_start_row = chunk_idx * chunk_size;
@@ -2294,9 +2288,9 @@ static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, un
}
}
-static void transfer_output_chunk_scattered_worker_fn(unsigned int n, unsigned int i, void *data) {
+static void transfer_output_chunk_scattered_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) {
output_transfer_scattered_task_state_t *st = data;
- struct htp_thread_trace * tr = &st->traces[i];
+ struct htp_thread_trace * tr = th->trace;
int chunk_idx = i;
int chunk_size = st->n_chunks_per_task;
int vtcm_start_row = chunk_idx * chunk_size;
@@ -2341,7 +2335,7 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles(
int n_cols, int k_block,
size_t row_stride, int weight_type,
int n_k_tiles, struct fastdiv_values n_k_tiles_div,
- tiled_dequant_task_fn_t dequant_worker_fn, int n_threads) {
+ tiled_dequant_task_fn_t dequant_worker_fn, int n_hvx) {
if (!dequant_worker_fn) {
return;
@@ -2353,7 +2347,7 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles(
const uint32_t n_weight_tiles = k_block / htp_tensor_get_tile_k(weight_type);
const size_t n_tot_tiles = n_col_tiles * n_weight_tiles;
- size_t n_tiles_per_task = (n_threads == 1) ? n_tot_tiles : hmx_ceil_div(n_tot_tiles, n_threads);
+ size_t n_tiles_per_task = (n_hvx == 1) ? n_tot_tiles : hmx_ceil_div(n_tot_tiles, n_hvx);
tiled_dequantize_state_t state;
state.n_tasks = (n_tot_tiles + n_tiles_per_task - 1) / n_tiles_per_task;
@@ -2369,23 +2363,15 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles(
state.n_k_tiles = n_k_tiles;
state.n_weight_tiles = n_weight_tiles;
state.n_k_tiles_div = n_k_tiles_div;
- state.traces = ctx->trace;
- state.ctx = ctx;
state.tile_size = htp_tensor_get_tile_size(weight_type);
state.aligned_tile_size = htp_tensor_get_tile_padded_size(weight_type);
- if (state.n_tasks == 1 || n_threads == 1) {
- dequantize_tiled_worker_loop(1, 0, &state);
- } else {
- int n_tasks = hex_smin((int) state.n_tasks, n_threads);
- worker_pool_run_func(ctx->worker_pool, dequantize_tiled_worker_loop, &state, n_tasks);
- }
+ int n_tasks = hex_smin((int) state.n_tasks, n_hvx);
+ work_queue_run(ctx->hvx_queue, dequantize_tiled_worker_loop, &state, n_tasks);
}
typedef struct {
- struct htp_context * ctx;
- struct htp_thread_trace * traces;
float * dst;
const __fp16 * vtcm_src;
const float * src2;
@@ -2394,19 +2380,19 @@ typedef struct {
uint32_t dst_stride;
uint32_t src2_stride;
uint32_t dst_cols;
- struct fastdiv_values n_threads_div;
+ struct fastdiv_values n_hvx_div;
} output_transfer_col_chunk_state_t;
-static void transfer_output_chunk_col_chunk_worker_fn(unsigned int n, unsigned int i, void *data) {
+static void transfer_output_chunk_col_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) {
(void) n;
output_transfer_col_chunk_state_t *st = (output_transfer_col_chunk_state_t *) data;
- struct htp_thread_trace * tr = &st->traces[i];
+ struct htp_thread_trace * tr = th->trace;
// Round up: the last block is partial when N is not 32-aligned. Its pad
// columns are dropped by the dst_cols clamp inside the store.
uint32_t n_blocks = hmx_ceil_div(st->n_cols, 32);
- uint32_t b_first = fastdiv(n_blocks * i, &st->n_threads_div);
- uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_threads_div);
+ uint32_t b_first = fastdiv(n_blocks * i, &st->n_hvx_div);
+ uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_hvx_div);
uint32_t c_first = b_first * 32;
uint32_t c_last = b_last * 32;
uint32_t c_len = c_last - c_first;
@@ -2431,12 +2417,12 @@ static void transfer_output_chunk_col_chunk_worker_fn(unsigned int n, unsigned i
}
static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, const float *src2, const __fp16 *vtcm_src,
- int n_rows, int n_cols, int dst_stride, uint32_t src2_stride, int dst_cols, int n_threads) {
+ int n_rows, int n_cols, int dst_stride, uint32_t src2_stride, int dst_cols, int n_hvx) {
if (n_rows <= 0) return;
uint32_t n_blocks = hmx_ceil_div((uint32_t) n_cols, 32);
- if (n_threads > 1 && n_blocks >= (uint32_t)n_threads) {
- struct fastdiv_values n_threads_div = (n_threads == (int)ctx->n_threads) ? ctx->n_threads_div : init_fastdiv_values(n_threads);
+ if (n_hvx > 1 && n_blocks >= (uint32_t)n_hvx) {
+ struct fastdiv_values n_hvx_div = (n_hvx == (int)ctx->n_hvx) ? ctx->n_hvx_div : init_fastdiv_values(n_hvx);
output_transfer_col_chunk_state_t col_state;
col_state.dst = dst;
col_state.src2 = src2;
@@ -2446,22 +2432,20 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst,
col_state.dst_stride = (uint32_t)dst_stride;
col_state.src2_stride = src2_stride;
col_state.dst_cols = (uint32_t)dst_cols;
- col_state.n_threads_div = n_threads_div;
- col_state.traces = ctx->trace;
- col_state.ctx = ctx;
+ col_state.n_hvx_div = n_hvx_div;
- worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_col_chunk_worker_fn, &col_state, n_threads);
+ work_queue_run(ctx->hvx_queue, transfer_output_chunk_col_chunk_worker_fn, &col_state, n_hvx);
return;
}
size_t n_tot_chunks = n_rows;
- size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : hmx_ceil_div(n_rows, n_threads);
+ size_t n_chunks_per_task = (n_hvx == 1) ? n_tot_chunks : hmx_ceil_div(n_rows, n_hvx);
n_chunks_per_task = hex_align_up(n_chunks_per_task, 2);
- int actual_threads = hmx_ceil_div(n_rows, n_chunks_per_task);
+ int actual_hvx = hmx_ceil_div(n_rows, n_chunks_per_task);
output_transfer_task_state_t state;
- state.n_tasks = actual_threads;
+ state.n_tasks = actual_hvx;
state.n_tot_chunks = n_tot_chunks;
state.n_chunks_per_task = n_chunks_per_task;
state.dst = dst;
@@ -2471,13 +2455,8 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst,
state.dst_stride = dst_stride;
state.src2_stride = src2_stride;
state.dst_cols = dst_cols;
- state.traces = ctx->trace;
- if (actual_threads <= 1) {
- transfer_output_chunk_worker_fn(1, 0, &state);
- } else {
- worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_worker_fn, &state, actual_threads);
- }
+ work_queue_run(ctx->hvx_queue, transfer_output_chunk_worker_fn, &state, actual_hvx);
}
struct activation_transfer_params {
@@ -2487,8 +2466,8 @@ struct activation_transfer_params {
int n_rows;
int k_block;
int k_stride;
- int n_threads;
- const struct fastdiv_values * act_threads_div;
+ int n_hvx;
+ const struct fastdiv_values * act_hvx_div;
const struct fastdiv_values * k_div;
int k_valid;
float * vtcm_f32_act;
@@ -2503,8 +2482,8 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_
int n_rows = params->n_rows;
int k_block = params->k_block;
int k_stride = params->k_stride;
- int n_threads = params->n_threads;
- const struct fastdiv_values * act_threads_div = params->act_threads_div;
+ int n_hvx = params->n_hvx;
+ const struct fastdiv_values * act_hvx_div = params->act_hvx_div;
const struct fastdiv_values * k_div = params->k_div;
int k_valid = params->k_valid;
float * vtcm_f32_act = params->vtcm_f32_act;
@@ -2517,12 +2496,12 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_
}
const size_t n_tasks = (n_rows + 31) >> 5;
- if (act_elem_size == sizeof(float) && n_threads > 1 && k_block > 32 && n_tasks < (size_t) n_threads) {
+ if (act_elem_size == sizeof(float) && n_hvx > 1 && k_block > 32 && n_tasks < (size_t) n_hvx) {
// Calculate step rows parameters for column-chunked dma pipelining
uint32_t dma_step_rows = 2;
uint32_t dma_step_rows_shift = 1;
if (vtcm_f32_act_bytes > 0) {
- size_t thread_scratch_bytes = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_threads_div), 128);
+ size_t thread_scratch_bytes = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_hvx_div), 128);
size_t thread_scratch_elements = thread_scratch_bytes / sizeof(float);
size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div);
if (dma_step_rows_max >= 4) {
@@ -2538,26 +2517,24 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_
col_state.k_block = k_block;
col_state.k_stride = k_stride;
col_state.k_valid = k_valid;
- col_state.n_col_chunks = n_threads;
- col_state.n_threads_div = *act_threads_div;
+ col_state.n_col_chunks = n_hvx;
+ col_state.n_hvx_div = *act_hvx_div;
col_state.vtcm_f32_act = vtcm_f32_act;
col_state.vtcm_f32_act_bytes = vtcm_f32_act_bytes;
- col_state.traces = ctx->trace;
- col_state.ctx = ctx;
col_state.dma_step_rows = dma_step_rows;
col_state.dma_step_rows_shift = dma_step_rows_shift;
- worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_col_chunk_worker_fn, &col_state, n_threads);
+ work_queue_run(ctx->hvx_queue, transfer_activation_chunk_col_chunk_worker_fn, &col_state, n_hvx);
return;
}
assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0 && k_stride % HTP_MM_HMX_TILE_N_COLS == 0);
size_t n_tot_chunks = n_rows;
- size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : 32; // must be multiple of 32 to ensure correct destination address
+ size_t n_chunks_per_task = (n_hvx == 1) ? n_tot_chunks : 32; // must be multiple of 32 to ensure correct destination address
activation_transfer_task_state_t state;
- state.n_tasks = (n_threads == 1) ? 1 : hmx_ceil_div(n_tot_chunks, 32);
+ state.n_tasks = (n_hvx == 1) ? 1 : hmx_ceil_div(n_tot_chunks, 32);
state.n_tot_chunks = n_tot_chunks;
state.n_chunks_per_task = n_chunks_per_task;
state.dst = dst;
@@ -2565,17 +2542,15 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_
state.k_block = k_block;
state.k_stride = k_stride;
state.k_valid = k_valid;
- state.traces = ctx->trace;
- state.ctx = ctx;
state.vtcm_f32_act = vtcm_f32_act;
- state.act_elem_size = act_elem_size;
+ state.act_elem_size = act_elem_size;
- state.vtcm_f32_act_bytes_per_thread = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_threads_div), 128);
+ state.vtcm_f32_act_bytes_per_hvx = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_hvx_div), 128);
uint32_t dma_step_rows = 2;
uint32_t dma_step_rows_shift = 1;
- if (state.vtcm_f32_act_bytes_per_thread > 0) {
- size_t thread_scratch_elements = state.vtcm_f32_act_bytes_per_thread / sizeof(float);
+ if (state.vtcm_f32_act_bytes_per_hvx > 0) {
+ size_t thread_scratch_elements = state.vtcm_f32_act_bytes_per_hvx / sizeof(float);
size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div);
if (dma_step_rows_max >= 4) {
dma_step_rows = 4;
@@ -2585,12 +2560,8 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_
state.dma_step_rows = dma_step_rows;
state.dma_step_rows_shift = dma_step_rows_shift;
- int active_threads = hex_smin(n_threads, (int)state.n_tasks);
- if (state.n_tasks == 1 || n_threads == 1) {
- transfer_activation_chunk_worker_fn(1, 0, &state);
- } else {
- worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_worker_fn, &state, active_threads);
- }
+ int active_hvx = hex_smin(n_hvx, (int)state.n_tasks);
+ work_queue_run(ctx->hvx_queue, transfer_activation_chunk_worker_fn, &state, active_hvx);
}
// --- Async HMX matmul job (for pipeline overlap) ---
@@ -2604,7 +2575,7 @@ typedef struct {
uint32_t n_dot_tiles;
} hmx_matmul_job_t;
-static void hmx_matmul_worker_fn(void * data) {
+static void hmx_matmul_worker_fn(const htp_thread_t * th, void * data) {
hmx_matmul_job_t * job = (hmx_matmul_job_t *) data;
FARF(HIGH, "hmx-mm-job: n_row_tiles %u n_col_tiles %u n_dot_tiles %u", job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles);
core_dot_chunk_fp16(job->output, job->activation, job->weight, job->scales, job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles);
@@ -2646,9 +2617,9 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
int m_chunk,
int n_chunk,
int pipeline,
- int n_threads,
- int act_threads,
- const struct fastdiv_values * act_threads_div,
+ int n_hvx,
+ int act_hvx,
+ const struct fastdiv_values * act_hvx_div,
const struct fastdiv_values * k_div,
int tile_size,
int aligned_tile_size,
@@ -2686,7 +2657,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
size_t vtcm_used = vtcm_size;
struct htp_mm_hmx_vtcm_layout L;
- htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_threads, aligned_tile_size, src2_bytes);
+ htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_hvx, aligned_tile_size, src2_bytes);
vtcm_used = L.total_bytes;
if (vtcm_used > vtcm_budget) {
@@ -2751,8 +2722,8 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
.n_rows = (int) n_rows,
.k_block = k,
.k_stride = act_stride,
- .n_threads = act_threads,
- .act_threads_div = act_threads_div,
+ .n_hvx = act_hvx,
+ .act_hvx_div = act_hvx_div,
.k_div = k_div,
.k_valid = k_valid,
.vtcm_f32_act = vtcm_f32_act,
@@ -2789,7 +2760,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
dequantize_tiled_weight_chunk_to_fp16_tiles(
ctx, vtcm_weight_bufs[i % 4], curr_raw,
n_cols, k, row_stride, weight_type,
- n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads);
+ n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx);
// 3. push A_{i+2} (if i+2 < n_chunk_cnt)
if (i + 2 < n_chunk_cnt) {
@@ -2814,7 +2785,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc_prev) : NULL;
int chunk_dst_cols = dst_cols - (int)nc_prev;
if (chunk_dst_cols > 0) {
- transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, src2_stride, chunk_dst_cols, n_threads);
+ transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, src2_stride, chunk_dst_cols, n_hvx);
}
}
}
@@ -2827,7 +2798,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc_last) : NULL;
int chunk_dst_cols = dst_cols - (int)nc_last;
if (chunk_dst_cols > 0) {
- transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, src2_stride, chunk_dst_cols, n_threads);
+ transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, src2_stride, chunk_dst_cols, n_hvx);
}
}
} else {
@@ -2846,8 +2817,8 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
.n_rows = (int) n_rows,
.k_block = k,
.k_stride = act_stride,
- .n_threads = act_threads,
- .act_threads_div = act_threads_div,
+ .n_hvx = act_hvx,
+ .act_hvx_div = act_hvx_div,
.k_div = k_div,
.k_valid = k_valid,
.vtcm_f32_act = vtcm_f32_act,
@@ -2876,7 +2847,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
dequantize_tiled_weight_chunk_to_fp16_tiles(
ctx, vtcm_weight_bufs[i % 2], curr_raw,
n_cols, k, row_stride, weight_type,
- n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads);
+ n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx);
// Start weight DMA for the next chunk early
const size_t nc_next = nc + n_chunk_n_cols;
@@ -2896,7 +2867,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc) : NULL;
int chunk_dst_cols = dst_cols - (int)nc;
if (chunk_dst_cols > 0) {
- transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output, n_rows, n_cols, dst_stride, src2_stride, chunk_dst_cols, n_threads);
+ transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output, n_rows, n_cols, dst_stride, src2_stride, chunk_dst_cols, n_hvx);
}
}
}
@@ -2947,16 +2918,16 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
const int m_chunk_n_rows = kparams->m_chunk;
const int n_chunk_n_cols = kparams->n_chunk;
const int pipeline = kparams->pipeline;
- const int n_threads = octx->n_threads;
- const int act_threads = kparams->n_act_threads;
- const struct fastdiv_values * act_threads_div = &kparams->div_n_act_threads;
+ const int n_hvx = octx->n_hvx;
+ const int act_hvx = kparams->n_act_hvx;
+ const struct fastdiv_values * act_hvx_div = &kparams->div_n_act_hvx;
const struct fastdiv_values * k_div = &kparams->div_ne00_padded;
const uint32_t dma_dst_stride = is_tiled ? aligned_tile_size : row_stride;
const uint32_t dma_width_bytes = is_tiled ? tile_size : row_stride;
struct htp_mm_hmx_vtcm_layout L;
- htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_threads, aligned_tile_size, 0);
+ htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_hvx, aligned_tile_size, 0);
if (L.total_bytes > vtcm_budget) {
FARF(ERROR, "hmx-mm-nx-2d: VTCM overflow: used %zu budget %zu, m %d k %d mc %d nc %d",
@@ -3034,8 +3005,8 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
.n_rows = (int) n_rows,
.k_block = k,
.k_stride = act_stride,
- .n_threads = act_threads,
- .act_threads_div = act_threads_div,
+ .n_hvx = act_hvx,
+ .act_hvx_div = act_hvx_div,
.k_div = k_div,
.k_valid = k_valid,
.vtcm_f32_act = vtcm_f32_act,
@@ -3085,7 +3056,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
dequantize_tiled_weight_chunk_to_fp16_tiles(
ctx, vtcm_weight_bufs[i % 4], curr_raw,
n_cols, k, row_stride, weight_type,
- n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads);
+ n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx);
if (i + 2 < n_chunk_cnt) {
const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_weight_tiles : n_cols_p2;
@@ -3106,7 +3077,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
float *output_chunk = dst_ptr + (mr * dst_stride + nc_prev);
int chunk_dst_cols = dst_cols - (int)nc_prev;
if (chunk_dst_cols > 0) {
- transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, 0, chunk_dst_cols, n_threads);
+ transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, 0, chunk_dst_cols, n_hvx);
}
}
}
@@ -3117,7 +3088,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
float *output_chunk = dst_ptr + (mr * dst_stride + nc_last);
int chunk_dst_cols = dst_cols - (int)nc_last;
if (chunk_dst_cols > 0) {
- transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, 0, chunk_dst_cols, n_threads);
+ transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, 0, chunk_dst_cols, n_hvx);
}
}
}
@@ -3136,8 +3107,8 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
.n_rows = (int) n_rows,
.k_block = k,
.k_stride = act_stride,
- .n_threads = act_threads,
- .act_threads_div = act_threads_div,
+ .n_hvx = act_hvx,
+ .act_hvx_div = act_hvx_div,
.k_div = k_div,
.k_valid = k_valid,
.vtcm_f32_act = vtcm_f32_act,
@@ -3180,7 +3151,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
dequantize_tiled_weight_chunk_to_fp16_tiles(
ctx, vtcm_weight_bufs[i % 2], curr_raw,
n_cols, k, row_stride, weight_type,
- n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads);
+ n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx);
const size_t nc_next = nc + n_chunk_n_cols;
if (nc_next < n) {
@@ -3196,7 +3167,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
float *output_chunk = dst_ptr + (mr * dst_stride + nc);
int chunk_dst_cols = dst_cols - (int)nc;
if (chunk_dst_cols > 0) {
- transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output, n_rows, n_cols, dst_stride, 0, chunk_dst_cols, n_threads);
+ transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output, n_rows, n_cols, dst_stride, 0, chunk_dst_cols, n_hvx);
}
}
}
@@ -3228,8 +3199,8 @@ static inline float *hmx_mm_dst_batch_ptr(const hmx_mm_batched_params_t *params,
static int hmx_mm_batched_simple(struct htp_context *ctx,
const hmx_mm_batched_params_t *params,
- int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size,
- const struct fastdiv_values * act_threads_div, const struct fastdiv_values * k_div) {
+ int m_chunk, int n_chunk, int pipeline, int n_hvx, int act_hvx, int vtcm_size,
+ const struct fastdiv_values * act_hvx_div, const struct fastdiv_values * k_div) {
int ret = 0;
for (int b3 = 0; b3 < params->ne13 && ret == 0; ++b3) {
for (int b2 = 0; b2 < params->ne12 && ret == 0; ++b2) {
@@ -3243,16 +3214,16 @@ static int hmx_mm_batched_simple(struct htp_context *ctx,
params->m, params->k, params->n,
params->act_stride, params->act_elem_size, params->weight_stride,
params->weight_type, params->k, params->dst_stride, params->bias_stride, params->dst_cols,
- m_chunk, n_chunk, pipeline, n_threads, act_threads,
- act_threads_div, k_div, params->tile_size, params->aligned_tile_size, vtcm_size);
+ m_chunk, n_chunk, pipeline, n_hvx, act_hvx,
+ act_hvx_div, k_div, params->tile_size, params->aligned_tile_size, vtcm_size);
}
}
return ret;
}
static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t *params,
- int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads,
- const struct fastdiv_values * act_threads_div,
+ int m_chunk, int n_chunk, int pipeline, int n_hvx, int act_hvx,
+ const struct fastdiv_values * act_hvx_div,
const struct fastdiv_values * k_div,
int vtcm_size) {
const size_t row_stride = htp_tensor_get_row_size(params->weight_type, params->k);
@@ -3272,7 +3243,7 @@ static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t
// Grouped path is only valid for flat F16 if group_size > 1 and it fits within VTCM budget.
bool run_grouped = (params->weight_type == HTP_TYPE_F16 && params->tile_size == 0 && group_size > 1 && (size_t)vtcm_size <= vtcm_budget);
if (!run_grouped) {
- return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div);
+ return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_hvx, act_hvx, vtcm_size, act_hvx_div, k_div);
}
struct htp_thread_trace * tr = &ctx->trace[0];
@@ -3282,12 +3253,12 @@ static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t
size_t n_chunk_n_cols = n_chunk;
struct htp_mm_hmx_vtcm_layout L;
- htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, HTP_TYPE_F16, params->k, m_chunk_n_rows, n_chunk_n_cols, group_size, false, act_threads, 0, params->bias_bytes);
+ htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, HTP_TYPE_F16, params->k, m_chunk_n_rows, n_chunk_n_cols, group_size, false, act_hvx, 0, params->bias_bytes);
if (L.total_bytes > vtcm_budget) {
FARF(HIGH, "%s: grouped layout overflowed VTCM, falling back to simple batched loop", __func__);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
- return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div);
+ return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_hvx, act_hvx, vtcm_size, act_hvx_div, k_div);
}
uint8_t * const base = (uint8_t *) ctx->vtcm_base;
@@ -3345,8 +3316,8 @@ static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t
.n_rows = (int) n_rows,
.k_block = params->k,
.k_stride = params->act_stride,
- .n_threads = act_threads,
- .act_threads_div = act_threads_div,
+ .n_hvx = act_hvx,
+ .act_hvx_div = act_hvx_div,
.k_div = k_div,
.k_valid = params->k,
.vtcm_f32_act = vtcm_f32_act,
@@ -3405,7 +3376,7 @@ static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t
int chunk_dst_cols = params->dst_cols - (int)nc;
if (chunk_dst_cols > 0) {
transfer_output_chunk_threaded(ctx, output, bias_chunk, vtcm_output, (int) n_rows, (int) n_cols,
- params->dst_stride, params->bias_stride, chunk_dst_cols, n_threads);
+ params->dst_stride, params->bias_stride, chunk_dst_cols, n_hvx);
}
}
}
@@ -3431,20 +3402,20 @@ static void transfer_activation_chunk_gathered_threaded(
size_t nb11,
size_t nb12,
int cne1,
- int n_threads,
+ int n_hvx,
int k_valid) {
if (n_rows <= 0) return;
- int chunks_per_thread = hmx_ceil_div(n_rows, n_threads);
- chunks_per_thread = hex_align_up(chunks_per_thread, 2);
+ int chunks_per_hvx = hmx_ceil_div(n_rows, n_hvx);
+ chunks_per_hvx = hex_align_up(chunks_per_hvx, 2);
- int actual_threads = hmx_ceil_div(n_rows, chunks_per_thread);
+ int actual_hvx = hmx_ceil_div(n_rows, chunks_per_hvx);
activation_transfer_gathered_task_state_t state = {
.dst = dst,
.src = src,
- .n_tasks = actual_threads,
+ .n_tasks = actual_hvx,
.n_tot_chunks = n_rows,
- .n_chunks_per_task = chunks_per_thread,
+ .n_chunks_per_task = chunks_per_hvx,
.k_block = k_block,
.matrix_rows = matrix_rows,
.cur_a = cur_a,
@@ -3456,17 +3427,12 @@ static void transfer_activation_chunk_gathered_threaded(
.start_row = start_row,
.cne1 = cne1,
.k_valid = k_valid,
- .traces = ctx->trace,
};
- worker_callback_t worker_fn = ne11 == 1 ? transfer_activation_chunk_gathered_worker_flat_fn :
+ work_queue_func_t worker_fn = ne11 == 1 ? transfer_activation_chunk_gathered_worker_flat_fn :
transfer_activation_chunk_gathered_worker_fn;
- if (actual_threads <= 1) {
- worker_fn(1, 0, &state);
- } else {
- worker_pool_run_func(ctx->worker_pool, worker_fn, &state, actual_threads);
- }
+ work_queue_run(ctx->hvx_queue, worker_fn, &state, actual_hvx);
}
static void transfer_output_chunk_scattered_threaded(
@@ -3482,19 +3448,19 @@ static void transfer_output_chunk_scattered_threaded(
size_t dst_nb1,
size_t dst_nb2,
int cne1,
- int n_threads) {
+ int n_hvx) {
if (n_rows <= 0) return;
- int chunks_per_thread = hmx_ceil_div(n_rows, n_threads);
- chunks_per_thread = hex_align_up(chunks_per_thread, 2);
+ int chunks_per_hvx = hmx_ceil_div(n_rows, n_hvx);
+ chunks_per_hvx = hex_align_up(chunks_per_hvx, 2);
- int actual_threads = hmx_ceil_div(n_rows, chunks_per_thread);
+ int actual_hvx = hmx_ceil_div(n_rows, chunks_per_hvx);
output_transfer_scattered_task_state_t state = {
.vtcm_src = vtcm_src,
.dst = dst,
- .n_tasks = actual_threads,
+ .n_tasks = actual_hvx,
.n_tot_chunks = n_rows,
- .n_chunks_per_task = chunks_per_thread,
+ .n_chunks_per_task = chunks_per_hvx,
.n_cols = n_cols,
.matrix_rows = matrix_rows,
.cur_a = cur_a,
@@ -3503,14 +3469,9 @@ static void transfer_output_chunk_scattered_threaded(
.dst_nb2 = dst_nb2,
.start_row = start_row,
.cne1 = cne1,
- .traces = ctx->trace,
};
- if (actual_threads <= 1) {
- transfer_output_chunk_scattered_worker_fn(1, 0, &state);
- } else {
- worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_scattered_worker_fn, &state, actual_threads);
- }
+ work_queue_run(ctx->hvx_queue, transfer_output_chunk_scattered_worker_fn, &state, actual_hvx);
}
static int hmx_mm_id_2d_f32(struct htp_context *ctx,
@@ -3532,7 +3493,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
int mapping_stride,
int m_start,
int m_end,
- int n_threads) {
+ int n_hvx) {
struct htp_thread_trace * tr = &ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);
@@ -3614,7 +3575,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
transfer_activation_chunk_gathered_threaded(
ctx, vtcm_f16_act, activation, (int) mr, (int) n_rows, k,
- matrix_rows, cur_a, mapping_stride, ne11, act_nb1, act_nb2, cne1, n_threads, k_valid);
+ matrix_rows, cur_a, mapping_stride, ne11, act_nb1, act_nb2, cne1, n_hvx, k_valid);
// A0: Pre-fetch the first weight chunk (nc = 0)
if (n > 0) {
@@ -3636,7 +3597,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
dequantize_tiled_weight_chunk_to_fp16_tiles(
ctx, vtcm_weight_bufs[i % 2], curr_raw,
n_cols, k, row_stride, weight_type,
- n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads
+ n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx
);
// Start weight DMA for the next chunk early
@@ -3656,7 +3617,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
// D: Output Store
transfer_output_chunk_scattered_threaded(
ctx, dst + nc, vtcm_output, (int) mr, (int) n_rows, (int) n_cols,
- matrix_rows, cur_a, mapping_stride, dst_nb1, dst_nb2, cne1, n_threads);
+ matrix_rows, cur_a, mapping_stride, dst_nb1, dst_nb2, cne1, n_hvx);
}
}
@@ -3710,7 +3671,7 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k
const dma_addr_t act_addr = act->data + (size_t) m_start * act_stride * act_elem_size;
int ret = -1;
- const int n_threads = kparams->n_threads;
+ const int n_hvx = kparams->n_hvx;
if (kparams->kernel_type == HTP_MM_KERNEL_HMX_BATCHED) {
hmx_mm_batched_params_t batch_params = {
.dst = dst_ptr,
@@ -3750,9 +3711,9 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k
};
ret = hmx_mm_batched(octx->ctx, &batch_params,
kparams->m_chunk, kparams->n_chunk,
- kparams->pipeline, n_threads,
- kparams->n_act_threads,
- &kparams->div_n_act_threads,
+ kparams->pipeline, n_hvx,
+ kparams->n_act_hvx,
+ &kparams->div_n_act_hvx,
&kparams->div_ne00_padded,
kparams->vtcm_size);
} else {
@@ -3761,9 +3722,9 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k
act_addr, src0->data,
m_rows, k, n, act_stride, act_elem_size, (int) src0->nb[1], (int) src0->type, (int) act->ne[0],
dst_stride, src2_stride, (int)dst->ne[0],
- kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads,
- kparams->n_act_threads,
- &kparams->div_n_act_threads,
+ kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_hvx,
+ kparams->n_act_hvx,
+ &kparams->div_n_act_hvx,
&kparams->div_ne00_padded,
kparams->tile_size, kparams->aligned_tile_size, kparams->vtcm_size
);
@@ -3849,7 +3810,7 @@ static int hmx_mm_op_matmul_id(
(int) src0->nb[1], (int) src0->type,
kparams->tile_size, kparams->aligned_tile_size,
matrix_rows, cur_a, mmctx->mapping_stride,
- m_start, m_end, (int) octx->n_threads);
+ m_start, m_end, (int) octx->n_hvx);
if (ret != 0) {
FARF(ERROR, "HMX matmul failed for expert %u, error %d\n", cur_a, ret);
return HTP_STATUS_NO_SUPPORT;
@@ -3881,8 +3842,8 @@ static int hvx_mm_matmul_id(
work_queue_func_t quant_task_func;
uint32_t n_quant_tasks = 1;
- if (act_nrows < octx->n_threads) {
- n_quant_tasks = MIN(total_nb, octx->n_threads);
+ if (act_nrows < octx->n_hvx) {
+ n_quant_tasks = MIN(total_nb, octx->n_hvx);
quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type);
for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) {
uint32_t ib_first = (total_nb * ith) / n_quant_tasks;
@@ -3893,13 +3854,13 @@ static int hvx_mm_matmul_id(
mmctx->quant_c[ith] = ib_first % nb;
}
} else {
- n_quant_tasks = MIN(act_nrows, octx->n_threads);
+ n_quant_tasks = MIN(act_nrows, octx->n_hvx);
quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type);
}
size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, ne10);
struct htp_mm_hvx_vtcm_layout L;
- htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, act_nrows, octx->n_threads,
+ htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, act_nrows, octx->n_hvx,
0, src0_row_size, act_row_size, 0, kparams->n_prefetch, true, false);
const size_t vtcm_size = L.total_bytes;
@@ -3929,9 +3890,9 @@ static int hvx_mm_matmul_id(
mmctx->vtcm_act_stride = act_row_size;
mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(ne10);
- mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div);
- mmctx->vtcm_act_size_per_thread = L.act_bytes;
- mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div);
+ mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div);
+ mmctx->vtcm_act_size_per_hvx = L.act_bytes;
+ mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div);
mmctx->cur_m_start = 0;
mmctx->cur_m_rows = act_nrows;
@@ -3940,11 +3901,11 @@ static int hvx_mm_matmul_id(
hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows);
- mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks;
+ mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks;
mmctx->n_quant_tasks = n_quant_tasks;
- work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks);
+ work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks);
- work_queue_run(octx->ctx->work_queue, hvx_mmid_task_func, mmctx, octx->n_threads);
+ work_queue_run(octx->ctx->hvx_queue, hvx_mmid_task_func, mmctx, octx->n_hvx);
return HTP_STATUS_OK;
}
@@ -4013,7 +3974,7 @@ static int hmx_mm_op_matmul_id_nx(
(int) src_w->nb[1], (int) src_w->type,
kparams->tile_size, kparams->aligned_tile_size,
matrix_rows, cur_a, mmctx->mapping_stride,
- m_start, m_end, (int) octx->n_threads);
+ m_start, m_end, (int) octx->n_hvx);
if (ret != 0) {
FARF(ERROR, "HMX matmul ID NX failed for expert %u weight %u, error %d\n", cur_a, p, ret);
return HTP_STATUS_NO_SUPPORT;
@@ -4048,8 +4009,8 @@ static int hvx_mm_matmul_id_nx(
work_queue_func_t quant_task_func;
uint32_t n_quant_tasks = 1;
- if (act_nrows < octx->n_threads) {
- n_quant_tasks = MIN(total_nb, octx->n_threads);
+ if (act_nrows < octx->n_hvx) {
+ n_quant_tasks = MIN(total_nb, octx->n_hvx);
quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type);
for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) {
uint32_t ib_first = (total_nb * ith) / n_quant_tasks;
@@ -4060,13 +4021,13 @@ static int hvx_mm_matmul_id_nx(
mmctx->quant_c[ith] = ib_first % nb;
}
} else {
- n_quant_tasks = MIN(act_nrows, octx->n_threads);
+ n_quant_tasks = MIN(act_nrows, octx->n_hvx);
quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type);
}
size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, act->ne[0]);
struct htp_mm_hvx_vtcm_layout L;
- htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_threads,
+ htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_hvx,
0, src0_row_size, act_row_size, 0, kparams->n_prefetch, true, false);
const size_t vtcm_size = L.total_bytes;
@@ -4087,27 +4048,27 @@ static int hvx_mm_matmul_id_nx(
mmctx->vtcm_act_stride = act_row_size;
mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(act->ne[0]);
- mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div);
- mmctx->vtcm_act_size_per_thread = L.act_bytes;
- mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div);
+ mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div);
+ mmctx->vtcm_act_size_per_hvx = L.act_bytes;
+ mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div);
mmctx->cur_m_start = 0;
mmctx->cur_m_rows = act_nrows;
- FARF(HIGH, "matmul-id-nx: src0 %d:%d:%d type %s nrows %u, act %d:%d:%d nrows %u, vtcm %zu/%zu, threads %d\n",
+ FARF(HIGH, "matmul-id-nx: src0 %d:%d:%d type %s nrows %u, act %d:%d:%d nrows %u, vtcm %zu/%zu, nhvx %d\n",
src0->ne[0], src0->ne[1], src0->ne[2], mmctx->type, src0->ne[1],
act->ne[0], act->ne[1], act->ne[2], act_nrows,
- L.total_bytes, octx->ctx->vtcm_size, octx->n_threads);
+ L.total_bytes, octx->ctx->vtcm_size, octx->n_hvx);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows);
- mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks;
+ mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks;
mmctx->n_quant_tasks = n_quant_tasks;
- work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks);
+ work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks);
- work_queue_run(octx->ctx->work_queue, hvx_mmid_task_func, mmctx, octx->n_threads);
+ work_queue_run(octx->ctx->hvx_queue, hvx_mmid_task_func, mmctx, octx->n_hvx);
return HTP_STATUS_OK;
}
@@ -4386,8 +4347,8 @@ int op_matmul_id(struct htp_ops_context * octx) {
mmctx->src0_row_start = src0_row_start;
mmctx->src0_row_end = src0_row_end;
- mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div);
- mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32);
+ mmctx->src0_nrows_per_hvx = fastdiv(nrows + octx->n_hvx - 1, &octx->n_hvx_div);
+ mmctx->src0_nrows_per_hvx = hex_round_up(mmctx->src0_nrows_per_hvx, 32);
if (hvx_mm_init_tiled_vec_dot(mmctx, src0->type) == 0) {
s = hvx_mm_matmul_id(octx, mmctx, act_nrows > 1 ? hvx_mm_id_tiled : hvx_mv_id_tiled);
@@ -4530,10 +4491,10 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) {
const uint32_t nb = (act->ne[0] + qk - 1) / qk;
const uint32_t total_nb = act_nrows * nb;
- worker_callback_t quant_task_func;
+ work_queue_func_t quant_task_func;
uint32_t n_quant_tasks = 1;
- if (act_nrows < octx->n_threads) {
- n_quant_tasks = MIN(total_nb, octx->n_threads);
+ if (act_nrows < octx->n_hvx) {
+ n_quant_tasks = MIN(total_nb, octx->n_hvx);
quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type);
for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) {
uint32_t ib_first = (total_nb * ith) / n_quant_tasks;
@@ -4544,14 +4505,14 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) {
mmctx->quant_c[ith] = ib_first % nb;
}
} else {
- n_quant_tasks = MIN(act_nrows, octx->n_threads);
+ n_quant_tasks = MIN(act_nrows, octx->n_hvx);
quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type);
}
const size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, act->ne[0]);
struct htp_mm_hvx_vtcm_layout L;
- htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_threads,
+ htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_hvx,
0, src0_row_size, act_row_size, 0, kparams->n_prefetch, false, true);
const size_t vtcm_size = L.total_bytes;
@@ -4572,19 +4533,19 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) {
mmctx->vtcm_act_stride = act_row_size;
mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(act->ne[0]);
- mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div);
- mmctx->vtcm_act_size_per_thread = L.act_bytes;
- mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div);
+ mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div);
+ mmctx->vtcm_act_size_per_hvx = L.act_bytes;
+ mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows);
- mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks;
+ mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks;
mmctx->n_quant_tasks = n_quant_tasks;
- work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks);
+ work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks);
- work_queue_run(octx->ctx->work_queue, hvx_mm_nx_2d_tiled, mmctx, octx->n_threads);
+ work_queue_run(octx->ctx->hvx_queue, hvx_mm_nx_2d_tiled, mmctx, octx->n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h
index 4971ed958..0c16fe7b8 100644
--- a/ggml/src/ggml-hexagon/htp/matmul-ops.h
+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h
@@ -64,8 +64,8 @@ struct htp_mm_kernel_params {
uint8_t unused; // reserved
uint8_t n_hmx; // 1 = use HMX, 0 = use HVX
- uint8_t n_threads; // Number of threads to spawn
- uint8_t n_act_threads; // Number of threads for activation preparation
+ uint8_t n_hvx; // Number of threads to spawn
+ uint8_t n_act_hvx; // Number of threads for activation preparation
uint8_t n_prefetch; // Prefetch lookahead buffers/rows in VTCM
uint8_t n_weights; // Number of weights for fused NX
@@ -83,7 +83,7 @@ struct htp_mm_kernel_params {
struct fastdiv_values div_r2;
struct fastdiv_values div_r3;
struct fastdiv_values div_ne12;
- struct fastdiv_values div_n_act_threads;
+ struct fastdiv_values div_n_act_hvx;
struct fastdiv_values div_ne00_padded;
};
@@ -310,7 +310,7 @@ static inline void htp_mm_hmx_vtcm_layout_build(
size_t nc,
uint32_t group_size,
bool pipeline,
- uint32_t act_threads,
+ uint32_t act_hvx,
uint32_t aligned_tile_size,
size_t bias_size
) {
@@ -323,7 +323,7 @@ static inline void htp_mm_hmx_vtcm_layout_build(
const size_t activation_area_size = hex_align_up(group_size * act_head_stride * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE);
const size_t output_area_size = hex_align_up(group_size * mc * nc * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE);
const size_t scratch_area_size = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE);
- const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128);
+ const size_t min_f32_size = hex_align_up(act_hvx * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128);
// Group A: Permanent activation tiles and scales
size_t off_group_a = 0;
@@ -343,7 +343,7 @@ static inline void htp_mm_hmx_vtcm_layout_build(
const size_t group_b_size = off_group_b - off_group_a;
// Group C: Activation prep temporary buffer (overlaps Group B, starting at off_group_a)
- const size_t max_f32_size = act_threads * 64 * k * sizeof(float);
+ const size_t max_f32_size = act_hvx * 64 * k * sizeof(float);
const size_t act_f32_size = hex_align_up(hex_smin(max_f32_size, hex_smax(min_f32_size, group_b_size)), 128);
size_t off_group_c = off_group_a;
VTCM_LAYOUT_ALLOC(off_group_c, off_act_f32, act_f32_size);
@@ -367,7 +367,7 @@ static inline void htp_mm_hmx_vtcm_layout_build(
const size_t vec_dot_size = k * sizeof(uint16_t);
const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype);
- const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128);
+ const size_t min_f32_size = hex_align_up(act_hvx * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128);
const size_t weight_area_size = is_quant
? hex_align_up((nc / 32) * n_weight_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE)
: hex_align_up(nc * row_stride, HTP_MM_HMX_TILE_SIZE);
@@ -395,7 +395,7 @@ static inline void htp_mm_hmx_vtcm_layout_build(
const size_t group_b_size = off_group_b - off_group_a;
// Group C: Activation prep temporary buffer (overlaps Group B, starting at off_group_a)
- const size_t max_f32_size = act_threads * 64 * k * sizeof(float);
+ const size_t max_f32_size = act_hvx * 64 * k * sizeof(float);
const size_t act_f32_size = hex_align_up(hex_smin(max_f32_size, hex_smax(min_f32_size, group_b_size)), 128);
size_t off_group_c = off_group_a;
VTCM_LAYOUT_ALLOC(off_group_c, off_act_f32, act_f32_size);
@@ -423,7 +423,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(
int wtype,
uint32_t ne10, // k
uint32_t act_nrows, // m_total
- uint32_t n_threads,
+ uint32_t n_hvx,
size_t dst_row_size,
size_t src0_row_size,
size_t act_row_size,
@@ -443,13 +443,13 @@ static inline void htp_mm_hvx_vtcm_layout_build(
uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype);
uint32_t tile_row_size = n_k_tiles * aligned_tile_size;
- size_t weight_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128);
+ size_t weight_sz_per_hvx = hex_round_up(n_prefetch * tile_row_size, 128);
size_t tiled_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
size_t q_act_sz = hex_round_up(tiled_act_row_size * act_nrows, 128);
size_t raw_row_size = htp_mm_act_raw_row_size(ne10);
- src0_sz = weight_sz_per_thread * n_threads; // shared single-weight prefetch buffer
+ src0_sz = weight_sz_per_hvx * n_hvx; // shared single-weight prefetch buffer
act_sz = q_act_sz; // quantized activation buffer
bias_sz = 0;
dst_sz = 0;
@@ -459,13 +459,13 @@ static inline void htp_mm_hvx_vtcm_layout_build(
const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
const uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype);
const uint32_t tile_row_size = n_k_tiles * aligned_tile_size;
- size_t src0_sz_per_thread = htp_mm_round_up(n_prefetch * tile_row_size, 256);
+ size_t src0_sz_per_hvx = htp_mm_round_up(n_prefetch * tile_row_size, 256);
act_sz = htp_mm_round_up(act_row_size_tiled * act_nrows, 256);
size_t raw_row_size = htp_mm_act_raw_row_size(ne10);
- src0_sz = src0_sz_per_thread * n_threads;
+ src0_sz = src0_sz_per_hvx * n_hvx;
dst_sz = 0;
bias_sz = 0;
act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128);
@@ -477,8 +477,8 @@ static inline void htp_mm_hvx_vtcm_layout_build(
case HTP_MM_KERNEL_HVX_F16_F16_VTCM: {
size_t f16_act_row_size = htp_mm_round_up(ne10 * 2, 128);
act_sz = htp_mm_round_up(f16_act_row_size * act_nrows, 256);
- src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads;
- dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0;
+ src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_hvx;
+ dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_hvx : 0;
act_raw_sz = hex_round_up(hex_round_up(ne10 * sizeof(float), 128) * act_nrows, 128);
break;
}
@@ -486,8 +486,8 @@ static inline void htp_mm_hvx_vtcm_layout_build(
case HTP_MM_KERNEL_HVX_BF16_F32_VTCM: {
size_t f32_act_row_size = htp_mm_round_up(ne10 * 4, 128);
act_sz = htp_mm_round_up(f32_act_row_size * act_nrows, 256);
- src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads;
- dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0;
+ src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_hvx;
+ dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_hvx : 0;
act_raw_sz = 0;
break;
}
@@ -499,11 +499,11 @@ static inline void htp_mm_hvx_vtcm_layout_build(
uint32_t tile_row_size = n_weight_tiles * aligned_tile_size;
size_t tiled_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256);
- src0_sz = tiled_vtcm_size * n_threads;
+ src0_sz = tiled_vtcm_size * n_hvx;
act_sz = htp_mm_round_up(q_act_row_size * act_nrows, 256);
- size_t dst_slice_per_thread = (dst_nrows > 0 && act_nrows == 1) ? htp_mm_round_up((dst_row_size + n_threads - 1) / n_threads, 128) : 0;
- dst_sz = dst_slice_per_thread * n_threads;
+ size_t dst_slice_per_hvx = (dst_nrows > 0 && act_nrows == 1) ? htp_mm_round_up((dst_row_size + n_hvx - 1) / n_hvx, 128) : 0;
+ dst_sz = dst_slice_per_hvx * n_hvx;
size_t raw_row_size = htp_mm_act_raw_row_size(ne10);
act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128);
break;
@@ -542,7 +542,7 @@ static inline bool htp_mm_hvx_solve_vtcm_params(
int wtype,
uint32_t ne10,
uint32_t act_nrows,
- uint32_t n_threads,
+ uint32_t n_hvx,
size_t dst_row_size,
size_t src0_row_size,
size_t act_row_size,
@@ -554,7 +554,7 @@ static inline bool htp_mm_hvx_solve_vtcm_params(
) {
struct htp_mm_hvx_vtcm_layout L;
htp_mm_hvx_vtcm_layout_build(
- &L, kernel_type, wtype, ne10, act_nrows, n_threads,
+ &L, kernel_type, wtype, ne10, act_nrows, n_hvx,
dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false
);
@@ -599,14 +599,14 @@ static inline bool htp_mm_hvx_solve_vtcm_params(
}
htp_mm_hvx_vtcm_layout_build(
- &L, kernel_type, wtype, ne10, m_chunk, n_threads,
+ &L, kernel_type, wtype, ne10, m_chunk, n_hvx,
dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false
);
while (m_chunk > 2 && L.total_bytes > vtcm_budget) {
m_chunk -= 2;
htp_mm_hvx_vtcm_layout_build(
- &L, kernel_type, wtype, ne10, m_chunk, n_threads,
+ &L, kernel_type, wtype, ne10, m_chunk, n_hvx,
dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false
);
}
@@ -621,18 +621,18 @@ static inline bool htp_mm_hvx_solve_vtcm_params(
}
static inline size_t htp_mm_hmx_get_2d_vtcm_size(
- int wtype, uint32_t k, size_t mc, size_t nc, bool pipeline, uint32_t act_threads, uint32_t aligned_tile_size, size_t bias_size
+ int wtype, uint32_t k, size_t mc, size_t nc, bool pipeline, uint32_t act_hvx, uint32_t aligned_tile_size, size_t bias_size
) {
struct htp_mm_hmx_vtcm_layout L;
- htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, wtype, k, mc, nc, 1, pipeline, act_threads, aligned_tile_size, bias_size);
+ htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, wtype, k, mc, nc, 1, pipeline, act_hvx, aligned_tile_size, bias_size);
return L.total_bytes;
}
static inline size_t htp_mm_hmx_get_batched_vtcm_size(
- int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool pipeline, uint32_t act_threads, size_t bias_size) {
+ int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool pipeline, uint32_t act_hvx, size_t bias_size) {
(void)pipeline;
struct htp_mm_hmx_vtcm_layout L;
- htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, wtype, k, mc, nc, group_size, false, act_threads, 0, bias_size);
+ htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, wtype, k, mc, nc, group_size, false, act_hvx, 0, bias_size);
return L.total_bytes;
}
@@ -642,23 +642,23 @@ static inline bool htp_mm_hmx_solve_batched_params(
uint32_t ne01_padded,
uint32_t ne11,
uint32_t group_size,
- int n_threads,
+ int n_hvx,
bool pipeline,
size_t bias_size,
size_t vtcm_budget,
size_t * m_chunk_out,
size_t * n_chunk_out,
- int * act_threads_out,
+ int * act_hvx_out,
size_t * vtcm_size_out
) {
size_t best_mblocks = SIZE_MAX;
- int best_act_threads = 0;
+ int best_act_hvx = 0;
size_t best_m_chunk = 0;
size_t best_n_chunk = 0;
size_t best_vtcm_size = 0;
- int act_threads = n_threads;
- while (act_threads >= 1) {
+ int act_hvx = n_hvx;
+ while (act_hvx >= 1) {
size_t group_overhead = htp_mm_hmx_get_batched_overhead() + (bias_size > 0 ? hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE) : 0);
size_t group_size_per_n, group_size_per_m, group_size_per_mn;
htp_mm_hmx_get_batched_chunk_costs(k, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn);
@@ -670,30 +670,30 @@ static inline bool htp_mm_hmx_solve_batched_params(
if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded,
(size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT,
&m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) {
- size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, pipeline, act_threads, bias_size);
+ size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, pipeline, act_hvx, bias_size);
if (exact_size <= vtcm_budget) {
size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate;
- if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) {
+ if (mblocks < best_mblocks || (mblocks == best_mblocks && act_hvx > best_act_hvx)) {
best_mblocks = mblocks;
- best_act_threads = act_threads;
+ best_act_hvx = act_hvx;
best_m_chunk = m_chunk_candidate;
best_n_chunk = n_chunk_candidate;
best_vtcm_size = exact_size;
}
}
}
- if (act_threads == 1) {
- act_threads = 0;
+ if (act_hvx == 1) {
+ act_hvx = 0;
} else {
- act_threads /= 2;
+ act_hvx /= 2;
}
}
- if (best_act_threads > 0) {
+ if (best_act_hvx > 0) {
*m_chunk_out = best_m_chunk;
*n_chunk_out = best_n_chunk;
*vtcm_size_out = best_vtcm_size;
- *act_threads_out = best_act_threads;
+ *act_hvx_out = best_act_hvx;
return true;
}
return false;
@@ -706,7 +706,7 @@ static inline bool htp_mm_hmx_solve_2d_params(
uint32_t ne01_padded,
uint32_t ne11_padded,
uint32_t m_for_cost,
- int n_threads,
+ int n_hvx,
bool pipeline,
bool is_matmul_id,
uint32_t aligned_tile_size,
@@ -714,19 +714,19 @@ static inline bool htp_mm_hmx_solve_2d_params(
size_t vtcm_budget,
size_t * m_chunk_out,
size_t * n_chunk_out,
- int * act_threads_out,
+ int * act_hvx_out,
size_t * vtcm_size_out
) {
size_t best_mblocks = SIZE_MAX;
- int best_act_threads = 0;
+ int best_act_hvx = 0;
size_t best_m_chunk = 0;
size_t best_n_chunk = 0;
size_t best_vtcm_size = 0;
const int m_for_chunks = is_matmul_id ? hex_align_up(m_id_rows, 32) : ne11_padded;
- int act_threads = n_threads;
- while (act_threads >= 1) {
+ int act_hvx = n_hvx;
+ while (act_hvx >= 1) {
size_t simple_2d_overhead = htp_mm_hmx_get_2d_overhead(pipeline, is_matmul_id) + (bias_size > 0 ? hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE) : 0);
size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn;
htp_mm_hmx_get_2d_chunk_costs(wtype, k, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn);
@@ -738,30 +738,30 @@ static inline bool htp_mm_hmx_solve_2d_params(
if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded,
(size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT,
&m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) {
- size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size, bias_size);
+ size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_hvx, aligned_tile_size, bias_size);
if (exact_size <= vtcm_budget) {
size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate;
- if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) {
+ if (mblocks < best_mblocks || (mblocks == best_mblocks && act_hvx > best_act_hvx)) {
best_mblocks = mblocks;
- best_act_threads = act_threads;
+ best_act_hvx = act_hvx;
best_m_chunk = m_chunk_candidate;
best_n_chunk = n_chunk_candidate;
best_vtcm_size = exact_size;
}
}
}
- if (act_threads == 1) {
- act_threads = 0;
+ if (act_hvx == 1) {
+ act_hvx = 0;
} else {
- act_threads /= 2;
+ act_hvx /= 2;
}
}
- if (best_act_threads > 0) {
+ if (best_act_hvx > 0) {
*m_chunk_out = best_m_chunk;
*n_chunk_out = best_n_chunk;
*vtcm_size_out = best_vtcm_size;
- *act_threads_out = best_act_threads;
+ *act_hvx_out = best_act_hvx;
return true;
}
return false;
diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.c b/ggml/src/ggml-hexagon/htp/pad-ops.c
index cda6df318..b77e95a90 100644
--- a/ggml/src/ggml-hexagon/htp/pad-ops.c
+++ b/ggml/src/ggml-hexagon/htp/pad-ops.c
@@ -13,7 +13,7 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
@@ -100,7 +100,7 @@ struct htp_pad_context {
int32_t lp2, rp2;
int32_t lp3, rp3;
- uint32_t nrows_per_thread;
+ uint32_t nrows_per_hvx;
uint32_t total_dst_rows;
uint32_t row_start;
@@ -136,8 +136,8 @@ struct htp_pad_context {
\
const size_t type_size = pctx->type_size; \
\
- const uint32_t row_start = pctx->row_start + pctx->nrows_per_thread * ith; \
- const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->row_start + pctx->total_dst_rows);
+ const uint32_t row_start = pctx->row_start + pctx->nrows_per_hvx * ith; \
+ const uint32_t row_end = MIN(row_start + pctx->nrows_per_hvx, pctx->row_start + pctx->total_dst_rows);
#define htp_pad_dma_preamble \
@@ -146,21 +146,21 @@ struct htp_pad_context {
const size_t dst_row_size = pctx->dst_row_size; \
const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \
\
- uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; \
- uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; \
+ uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_hvx; \
+ uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_hvx; \
\
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
// ---------------------------------------------------------------------------
// HVX vectorized PAD kernel
// ---------------------------------------------------------------------------
-static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * data) {
+static void pad_job_per_hvx_hvx(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
struct htp_ops_context * octx = pctx->octx;
htp_pad_preamble;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
@@ -213,7 +213,7 @@ static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * da
// HVX + DMA PAD kernel - aligned, double-buffered
// ---------------------------------------------------------------------------
-static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void * data) {
+static void pad_job_per_hvx_hvx_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
struct htp_ops_context * octx = pctx->octx;
htp_pad_preamble;
@@ -252,7 +252,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void
// Main loop: pop completed DMAs, compute in VTCM with aligned HVX ops,
// push dst DMA and prefetch src for the next+1 row.
// -----------------------------------------------------------------------
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
for (uint32_t ir = row_start; ir < row_end; ir++) {
uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma_q).src;
@@ -318,12 +318,12 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void
// HVX circular PAD kernel
// ---------------------------------------------------------------------------
-static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, void * data) {
+static void pad_job_per_hvx_hvx_circular(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
struct htp_ops_context * octx = pctx->octx;
htp_pad_preamble;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
@@ -387,7 +387,7 @@ static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith,
// HVX + DMA circular PAD kernel - aligned, double-buffered
// ---------------------------------------------------------------------------
-static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int ith, void * data) {
+static void pad_job_per_hvx_hvx_circular_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
struct htp_ops_context * octx = pctx->octx;
htp_pad_preamble;
@@ -417,7 +417,7 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i
// Main loop: pop completed DMAs, assemble circular row in VTCM with
// aligned HVX ops, push dst DMA and prefetch src for the next+1 row.
// -----------------------------------------------------------------------
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
for (uint32_t ir = row_start; ir < row_end; ir++) {
uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma_q).src;
@@ -521,11 +521,11 @@ int op_pad(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {
+ if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) {
return HTP_STATUS_INVAL_PARAMS;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {
return HTP_STATUS_NO_SUPPORT;
@@ -533,12 +533,12 @@ int op_pad(struct htp_ops_context * octx) {
struct htp_pad_vtcm_layout layout = { 0 };
if (kparams->use_dma) {
- htp_pad_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_threads);
+ htp_pad_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_hvx);
- octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread;
- octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread;
- octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread;
- octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread;
+ octx->src0_spad.size_per_hvx = layout.src0_bytes_per_hvx;
+ octx->dst_spad.size_per_hvx = layout.dst_bytes_per_hvx;
+ octx->src0_spad.size = n_hvx * layout.src0_bytes_per_hvx;
+ octx->dst_spad.size = n_hvx * layout.dst_bytes_per_hvx;
octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0);
octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst);
octx->src0_spad.src = NULL;
@@ -552,7 +552,7 @@ int op_pad(struct htp_ops_context * octx) {
.lp1 = lp1, .rp1 = rp1,
.lp2 = lp2, .rp2 = rp2,
.lp3 = lp3, .rp3 = rp3,
- .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
+ .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div),
.total_dst_rows = nrows,
.row_start = row_start,
.type_size = type_size,
@@ -571,16 +571,16 @@ int op_pad(struct htp_ops_context * octx) {
switch (kparams->kernel_type) {
case HTP_PAD_KERNEL_HVX_CIRCULAR_DMA:
- work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx_circular_dma, &pctx, n_hvx);
break;
case HTP_PAD_KERNEL_HVX_CIRCULAR:
- work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular, &pctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx_circular, &pctx, n_hvx);
break;
case HTP_PAD_KERNEL_HVX_DMA:
- work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_dma, &pctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx_dma, &pctx, n_hvx);
break;
default:
- work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx, &pctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx, &pctx, n_hvx);
break;
}
diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.h b/ggml/src/ggml-hexagon/htp/pad-ops.h
index 5b8557538..15357e5f6 100644
--- a/ggml/src/ggml-hexagon/htp/pad-ops.h
+++ b/ggml/src/ggml-hexagon/htp/pad-ops.h
@@ -19,31 +19,31 @@ struct htp_pad_vtcm_layout {
size_t off_dst;
size_t src_row_size_aligned;
size_t dst_row_size_aligned;
- size_t src0_bytes_per_thread;
- size_t dst_bytes_per_thread;
+ size_t src0_bytes_per_hvx;
+ size_t dst_bytes_per_hvx;
};
static inline void htp_pad_vtcm_layout_build(
struct htp_pad_vtcm_layout * L,
size_t src_row_size,
size_t dst_row_size,
- uint32_t n_threads
+ uint32_t n_hvx
) {
L->src_row_size_aligned = hex_round_up(src_row_size, 128);
L->dst_row_size_aligned = hex_round_up(dst_row_size, 128);
- L->src0_bytes_per_thread = 2 * L->src_row_size_aligned;
- L->dst_bytes_per_thread = 2 * L->dst_row_size_aligned;
+ L->src0_bytes_per_hvx = 2 * L->src_row_size_aligned;
+ L->dst_bytes_per_hvx = 2 * L->dst_row_size_aligned;
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx);
L->total_bytes = off;
}
struct htp_pad_kernel_params {
uint8_t kernel_type;
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t total_rows;
uint32_t use_dma;
uint32_t elem_size;
diff --git a/ggml/src/ggml-hexagon/htp/pool-ops.c b/ggml/src/ggml-hexagon/htp/pool-ops.c
index b41c7bed2..12cd8ea90 100644
--- a/ggml/src/ggml-hexagon/htp/pool-ops.c
+++ b/ggml/src/ggml-hexagon/htp/pool-ops.c
@@ -5,7 +5,7 @@
#include "hex-common.h"
#include "dma-queue.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
@@ -249,29 +249,29 @@ struct pool_2d_context {
const struct htp_pool_2d_kernel_params * kparams;
struct htp_pool_vtcm_layout layout;
pool_plane_fn_t pool_plane;
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t plane_start;
uint32_t plane_count;
- uint32_t planes_per_thread;
+ uint32_t planes_per_hvx;
};
-static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) {
+static void pool_2d_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct pool_2d_context * ctx = (struct pool_2d_context *) data;
const struct htp_pool_2d_kernel_params * p = ctx->kparams;
const struct htp_pool_vtcm_layout * L = &ctx->layout;
const struct htp_tensor * src0 = ctx->octx->src[0];
const struct htp_tensor * dst = ctx->octx->dst;
pool_plane_fn_t pool_plane = ctx->pool_plane;
- const uint32_t planes_per_thread = ctx->planes_per_thread;
- const uint32_t first = ctx->plane_start + ith * planes_per_thread;
- const uint32_t last = MIN(first + planes_per_thread, ctx->plane_start + ctx->plane_count);
+ const uint32_t planes_per_hvx = ctx->planes_per_hvx;
+ const uint32_t first = ctx->plane_start + ith * planes_per_hvx;
+ const uint32_t last = MIN(first + planes_per_hvx, ctx->plane_start + ctx->plane_count);
if (first >= last) {
return;
}
- struct htp_thread_trace * tr = &ctx->octx->ctx->trace[ith];
- dma_queue * dma_queue = ctx->octx->ctx->dma[ith];
+ struct htp_thread_trace * tr = th->trace;
+ dma_queue * dma_queue = th->dma_q;
const uint32_t src_plane_bytes = L->src_plane_bytes;
const uint32_t dst_plane_bytes = L->dst_plane_bytes;
@@ -279,8 +279,8 @@ static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) {
const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size;
uint8_t * vtcm_base = (uint8_t *) ctx->octx->ctx->vtcm_base;
- uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread;
- uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread;
+ uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_hvx;
+ uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_hvx;
float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) };
float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) };
@@ -355,12 +355,12 @@ int op_pool_2d(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = MIN(p->n_threads, plane_count);
- if (!htp_ops_context_set_n_threads(octx, n_threads)) {
+ const uint32_t n_hvx = MIN(p->n_hvx, plane_count);
+ if (!htp_ops_context_set_n_hvx(octx, n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
- const uint32_t planes_per_thread = fastdiv(plane_count + n_threads - 1, &octx->n_threads_div);
+ const uint32_t planes_per_hvx = fastdiv(plane_count + n_hvx - 1, &octx->n_hvx_div);
pool_plane_fn_t pool_plane;
switch (p->kernel_type) {
@@ -375,13 +375,13 @@ int op_pool_2d(struct htp_ops_context * octx) {
.octx = octx,
.kparams = p,
.pool_plane = pool_plane,
- .n_threads = n_threads,
+ .n_hvx = n_hvx,
.plane_start = plane_start,
.plane_count = plane_count,
- .planes_per_thread = planes_per_thread,
+ .planes_per_hvx = planes_per_hvx,
};
- htp_pool_vtcm_layout_build(&ctx.layout, p->src_x, p->src_y, p->dst_x, p->dst_y, n_threads);
- work_queue_run(octx->ctx->work_queue, pool_2d_thread, &ctx, n_threads);
+ htp_pool_vtcm_layout_build(&ctx.layout, p->src_x, p->src_y, p->dst_x, p->dst_y, n_hvx);
+ work_queue_run(octx->ctx->hvx_queue, pool_2d_thread, &ctx, n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/pool-ops.h b/ggml/src/ggml-hexagon/htp/pool-ops.h
index 178e5d00f..e113bb9dd 100644
--- a/ggml/src/ggml-hexagon/htp/pool-ops.h
+++ b/ggml/src/ggml-hexagon/htp/pool-ops.h
@@ -29,7 +29,7 @@ struct htp_pool_2d_kernel_params {
uint32_t stride_y;
int32_t pad_x;
int32_t pad_y;
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t planes;
uint32_t planes_per_chunk;
uint32_t pool_op;
@@ -51,8 +51,8 @@ struct htp_pool_vtcm_layout {
size_t off_dst;
uint32_t src_plane_bytes;
uint32_t dst_plane_bytes;
- size_t src_bytes_per_thread;
- size_t dst_bytes_per_thread;
+ size_t src_bytes_per_hvx;
+ size_t dst_bytes_per_hvx;
size_t src_spad_half_size;
size_t dst_spad_half_size;
};
@@ -63,19 +63,19 @@ static inline void htp_pool_vtcm_layout_build(
uint32_t src_y,
uint32_t dst_x,
uint32_t dst_y,
- uint32_t n_threads
+ uint32_t n_hvx
) {
// Full-plane double buffering (256 bytes guard space for vector loads)
L->src_plane_bytes = src_x * src_y * sizeof(float);
L->dst_plane_bytes = dst_x * dst_y * sizeof(float);
L->src_spad_half_size = hex_round_up(L->src_plane_bytes + 256, 128);
L->dst_spad_half_size = hex_round_up(L->dst_plane_bytes, 128);
- L->src_bytes_per_thread = 2 * L->src_spad_half_size;
- L->dst_bytes_per_thread = 2 * L->dst_spad_half_size;
+ L->src_bytes_per_hvx = 2 * L->src_spad_half_size;
+ L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size;
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx);
L->total_bytes = off;
}
@@ -85,10 +85,10 @@ static inline bool htp_pool_solve_layout(
uint32_t src_y,
uint32_t dst_x,
uint32_t dst_y,
- uint32_t n_threads,
+ uint32_t n_hvx,
size_t vtcm_budget
) {
- htp_pool_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, n_threads);
+ htp_pool_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, n_hvx);
return layout->total_bytes <= vtcm_budget;
}
diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.c b/ggml/src/ggml-hexagon/htp/repeat-ops.c
index 827ef24c5..a9e4d5c86 100644
--- a/ggml/src/ggml-hexagon/htp/repeat-ops.c
+++ b/ggml/src/ggml-hexagon/htp/repeat-ops.c
@@ -13,7 +13,7 @@
#include "ggml-common.h"
#include "htp-ctx.h"
#include "hex-common.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "repeat-ops.h"
@@ -22,14 +22,14 @@ struct htp_repeat_context {
struct htp_ops_context * octx;
const struct htp_repeat_kernel_params * kparams;
- uint32_t nrows_per_thread;
+ uint32_t nrows_per_hvx;
uint32_t total_dst_rows; // ne1 * ne2 * ne3
uint32_t row_start;
size_t type_size;
};
-static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * data) {
+static void repeat_job_per_hvx(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_repeat_context * rctx = (const struct htp_repeat_context *) data;
struct htp_ops_context * octx = rctx->octx;
const struct htp_repeat_kernel_params * kparams = rctx->kparams;
@@ -60,13 +60,13 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat
const size_t row_bytes = ne00 * rctx->type_size;
- const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith;
- const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->total_dst_rows);
+ const uint32_t row_start = rctx->row_start + rctx->nrows_per_hvx * ith;
+ const uint32_t row_end = MIN(row_start + rctx->nrows_per_hvx, rctx->row_start + rctx->total_dst_rows);
if (row_start >= row_end) {
return;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
@@ -119,7 +119,7 @@ int op_repeat(struct htp_ops_context * octx) {
const struct htp_repeat_kernel_params * kparams =
(const struct htp_repeat_kernel_params *) octx->kernel_params;
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -141,12 +141,12 @@ int op_repeat(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
struct htp_repeat_context rctx = {
.octx = octx,
.kparams = kparams,
- .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
+ .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div),
.total_dst_rows = nrows,
.row_start = row_start,
.type_size = type_size,
@@ -157,7 +157,7 @@ int op_repeat(struct htp_ops_context * octx) {
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
kparams->nr0, kparams->nr1, kparams->nr2, kparams->nr3);
- work_queue_run(octx->ctx->work_queue, repeat_job_per_thread, &rctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, repeat_job_per_hvx, &rctx, n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.h b/ggml/src/ggml-hexagon/htp/repeat-ops.h
index 47759e037..1561994d4 100644
--- a/ggml/src/ggml-hexagon/htp/repeat-ops.h
+++ b/ggml/src/ggml-hexagon/htp/repeat-ops.h
@@ -5,7 +5,7 @@
#include <stdint.h>
struct htp_repeat_kernel_params {
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t nr0;
uint32_t nr1;
uint32_t nr2;
diff --git a/ggml/src/ggml-hexagon/htp/roll-ops.c b/ggml/src/ggml-hexagon/htp/roll-ops.c
index 77ded0fe9..aa7a0ad4b 100644
--- a/ggml/src/ggml-hexagon/htp/roll-ops.c
+++ b/ggml/src/ggml-hexagon/htp/roll-ops.c
@@ -14,7 +14,7 @@
#include "ggml-common.h"
#include "htp-ctx.h"
#include "hex-common.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "roll-ops.h"
@@ -25,7 +25,7 @@ struct htp_roll_context {
uint32_t row_start;
uint32_t nrows;
- uint32_t nrows_per_thread;
+ uint32_t nrows_per_hvx;
};
static inline uint32_t htp_roll_wrap(int32_t i, uint32_t ne) {
@@ -199,19 +199,19 @@ static int roll_dma_f32_strided(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
-static void roll_thread_f32(unsigned int nth, unsigned int ith, void * data) {
+static void roll_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_roll_context * rctx = (struct htp_roll_context *) data;
struct htp_ops_context * octx = rctx->octx;
htp_roll_preamble;
- const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith;
- const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->nrows);
+ const uint32_t row_start = rctx->row_start + rctx->nrows_per_hvx * ith;
+ const uint32_t row_end = MIN(row_start + rctx->nrows_per_hvx, rctx->row_start + rctx->nrows);
if (row_start >= row_end) {
return;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
for (uint32_t row = row_start; row < row_end; row++) {
@@ -275,20 +275,20 @@ int execute_op_roll_f32(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
- if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {
+ if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) {
return HTP_STATUS_INVAL_PARAMS;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
struct htp_roll_context rctx = {
.octx = octx,
.kparams = kparams,
.row_start = row_start,
.nrows = nrows,
- .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
+ .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div),
};
- work_queue_run(octx->ctx->work_queue, roll_thread_f32, &rctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, roll_thread_f32, &rctx, n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/roll-ops.h b/ggml/src/ggml-hexagon/htp/roll-ops.h
index 2646aaa4c..f98fb8ea9 100644
--- a/ggml/src/ggml-hexagon/htp/roll-ops.h
+++ b/ggml/src/ggml-hexagon/htp/roll-ops.h
@@ -12,7 +12,7 @@ enum htp_roll_kernel_type {
struct htp_roll_kernel_params {
uint8_t kernel_type;
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t total_rows;
uint32_t i0_src0;
uint32_t n0;
diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c
index 8c55ac3af..1c73de10b 100644
--- a/ggml/src/ggml-hexagon/htp/rope-ops.c
+++ b/ggml/src/ggml-hexagon/htp/rope-ops.c
@@ -66,12 +66,12 @@ struct htp_rope_context {
float theta_powers[32];
float corr_dims[2];
- uint32_t src0_nrows_per_thread;
+ uint32_t src0_nrows_per_hvx;
struct htp_ops_context * octx;
uint8_t * vtcm_base;
- size_t spad_per_thread;
+ size_t spad_per_hvx;
size_t theta_cache_offset;
size_t src0_row_size;
@@ -532,7 +532,7 @@ static void inline rope_neox_f32_inplace(struct htp_rope_context * rctx, uint8_t
}
}
-static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {
+static void rope_job_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_rope_context * rctx = (struct htp_rope_context *) data;
struct htp_ops_context * octx = rctx->octx;
@@ -544,10 +544,10 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {
htp_rope_preamble;
const uint32_t src0_nrows = rctx->nrows;
- const uint32_t src0_nrows_per_thread = rctx->src0_nrows_per_thread;
+ const uint32_t src0_nrows_per_hvx = rctx->src0_nrows_per_hvx;
- const uint32_t src0_start_row = rctx->row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, rctx->row_start + src0_nrows);
+ const uint32_t src0_start_row = rctx->row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, rctx->row_start + src0_nrows);
// no work for this thread
if (src0_start_row >= src0_end_row) {
@@ -560,12 +560,12 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {
const bool is_vision = (mode == HTP_ROPE_TYPE_VISION);
// VTCM setup
- uint8_t * src0_spad_base = rctx->vtcm_base + (ith * rctx->spad_per_thread);
+ uint8_t * src0_spad_base = rctx->vtcm_base + (ith * rctx->spad_per_hvx);
float * theta_cache = (float *) (src0_spad_base);
src0_spad_base = src0_spad_base + rctx->theta_cache_offset;
- dma_queue * dma_q = octx->ctx->dma[ith];
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ dma_queue * dma_q = th->dma_q;
+ struct htp_thread_trace * tr = th->trace;
const int32_t * pos = (const int32_t *) (uintptr_t) src1->data;
const float * freq_factors = rctx->freq_factors;
@@ -701,7 +701,7 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
const struct htp_tensor * dst = octx->dst;
const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params;
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
assert(octx->ctx->vtcm_size >= kparams->vtcm_size);
@@ -729,7 +729,7 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
const uint32_t ne0 = dst->ne[0];
const size_t src0_row_size = src0->ne[0] * sizeof(float);
@@ -738,14 +738,14 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
const size_t dst_row_stride = dst->nb[1];
struct htp_rope_vtcm_layout layout;
- htp_rope_vtcm_layout_build(&layout, src0->ne[0], kparams->n_threads, src2 ? src2->ne[0] : 0);
+ htp_rope_vtcm_layout_build(&layout, src0->ne[0], kparams->n_hvx, src2 ? src2->ne[0] : 0);
struct htp_rope_context rctx;
memset(&rctx, 0, sizeof(struct htp_rope_context));
rctx.octx = octx;
rctx.vtcm_base = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_spad);
- rctx.spad_per_thread = layout.bytes_per_thread;
+ rctx.spad_per_hvx = layout.bytes_per_hvx;
rctx.theta_cache_offset = layout.theta_cache_size_aligned;
if (src2) {
@@ -790,14 +790,14 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
rctx.src0_nrows = nrows;
rctx.nrows = nrows;
rctx.row_start = row_start;
- rctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
+ rctx.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div);
rctx.div_ne2_ne1 = kparams->div_ne2_ne1;
rctx.div_ne1 = kparams->div_ne1;
FARF(HIGH, "rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\n", rctx.src0_nrows, rctx.n_dims, ne0,
rctx.ext_factor, rctx.theta_scale, rctx.attn_factor);
- work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, rope_job_f32, &rctx, n_hvx);
return err;
}
diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.h b/ggml/src/ggml-hexagon/htp/rope-ops.h
index 0c28ef826..a21b85c06 100644
--- a/ggml/src/ggml-hexagon/htp/rope-ops.h
+++ b/ggml/src/ggml-hexagon/htp/rope-ops.h
@@ -10,7 +10,7 @@
#define HTP_ROPE_SPAD_NROWS (HTP_ROPE_SPAD_BLOCK * HTP_ROPE_SPAD_NSLOTS)
struct htp_rope_kernel_params {
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t vtcm_size;
struct fastdiv_values div_ne2_ne1;
@@ -27,7 +27,7 @@ struct htp_rope_vtcm_layout {
size_t total_bytes;
size_t off_spad;
size_t off_freq_factors;
- size_t bytes_per_thread;
+ size_t bytes_per_hvx;
size_t theta_cache_size_aligned;
size_t src0_row_size_aligned;
size_t freq_factors_size_aligned;
@@ -36,7 +36,7 @@ struct htp_rope_vtcm_layout {
static inline void htp_rope_vtcm_layout_build(
struct htp_rope_vtcm_layout * L,
uint32_t ne00,
- uint32_t n_threads,
+ uint32_t n_hvx,
uint32_t n_freq_factors
) {
const size_t src0_row_size = ne00 * sizeof(float);
@@ -47,10 +47,10 @@ static inline void htp_rope_vtcm_layout_build(
L->src0_row_size_aligned = src0_row_size_aligned;
L->theta_cache_size_aligned = theta_cache_size_aligned;
L->freq_factors_size_aligned = freq_factors_size_aligned;
- L->bytes_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned;
+ L->bytes_per_hvx = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned;
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_spad, L->bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_spad, L->bytes_per_hvx * n_hvx);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_freq_factors, freq_factors_size_aligned, n_freq_factors > 0);
L->total_bytes = off;
}
diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.c b/ggml/src/ggml-hexagon/htp/set-rows-ops.c
index 191cf0c6a..add91408f 100644
--- a/ggml/src/ggml-hexagon/htp/set-rows-ops.c
+++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.c
@@ -61,32 +61,33 @@ struct set_rows_context {
uint8_t * vtcm_base;
uint32_t task_start;
uint32_t tasks;
- uint32_t tasks_per_thread;
+ uint32_t tasks_per_hvx;
};
#define SET_ROWS_THREAD_DMA_FN(TYPE_NAME, IDX_TYPE, COMPUTE_EXPR) \
-static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \
+static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(const htp_thread_t * th, unsigned int nth, \
+ unsigned int ith, void *data) { \
struct set_rows_context * srctx = (struct set_rows_context *)data; \
struct htp_ops_context * octx = srctx->octx; \
const struct htp_set_rows_kernel_params * kparams = srctx->kparams; \
set_rows_preamble; \
- struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
- const uint32_t dr = srctx->tasks_per_thread; \
+ struct htp_thread_trace * tr = th->trace; \
+ const uint32_t dr = srctx->tasks_per_hvx; \
const uint32_t ir0 = srctx->task_start + dr * ith; \
if (ir0 >= srctx->task_start + srctx->tasks) { \
return; \
} \
const uint32_t ir1 = MIN(ir0 + dr, srctx->task_start + srctx->tasks); \
- dma_queue * dma_q = octx->ctx->dma[ith]; \
+ dma_queue * dma_q = th->dma_q; \
const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \
uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, srctx->vtcm_base, vtcm_layout->off_src0) + \
- ith * vtcm_layout->src0_bytes_per_thread; \
+ ith * vtcm_layout->src0_bytes_per_hvx; \
uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, srctx->vtcm_base, vtcm_layout->off_dst) + \
- ith * vtcm_layout->dst_bytes_per_thread; \
+ ith * vtcm_layout->dst_bytes_per_hvx; \
const uint32_t src0_row_size = ne00 * sizeof(float); \
const uint32_t dst_row_size = htp_tensor_get_row_size(octx->dst->type, ne00); \
- const uint32_t nrows_per_thread = ir1 - ir0; \
- const uint32_t total_steps = ne03 * ne02 * nrows_per_thread; \
+ const uint32_t nrows_per_hvx = ir1 - ir0; \
+ const uint32_t total_steps = ne03 * ne02 * nrows_per_hvx; \
uint32_t pi_step = 0; \
uint32_t pi02 = 0; \
uint32_t pi03 = 0; \
@@ -102,7 +103,7 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig
src0_data), \
vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \
pi_step++; \
- if (pi_step == nrows_per_thread) { \
+ if (pi_step == nrows_per_hvx) { \
pi_step = 0; \
pi02++; \
if (pi02 == ne02) { \
@@ -147,7 +148,7 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig
dma_make_data(src_spad, psrc0_data), \
vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \
pi_step++; \
- if (pi_step == nrows_per_thread) { \
+ if (pi_step == nrows_per_hvx) { \
pi_step = 0; \
pi02++; \
if (pi02 == ne02) { \
@@ -157,7 +158,7 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig
} \
} \
ci_step++; \
- if (ci_step == nrows_per_thread) { \
+ if (ci_step == nrows_per_hvx) { \
ci_step = 0; \
ci02++; \
ci11_base++; \
@@ -211,11 +212,11 @@ int op_set_rows(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, (uint32_t) kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
// l2fetch the src1 (indices) tensor in the main thread
hex_l2fetch_block((const void *)octx->src[1]->data, octx->src[1]->ne[3] * octx->src[1]->nb[3]);
@@ -225,9 +226,9 @@ int op_set_rows(struct htp_ops_context * octx) {
srctx.kparams = kparams;
srctx.task_start = task_start;
srctx.tasks = tasks;
- srctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div);
+ srctx.tasks_per_hvx = fastdiv(tasks + n_hvx - 1, &octx->n_hvx_div);
- htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, n_threads);
+ htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, n_hvx);
srctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base;
work_queue_func_t q_func = NULL;
@@ -240,15 +241,15 @@ int op_set_rows(struct htp_ops_context * octx) {
default: return HTP_STATUS_NO_SUPPORT;
}
- FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-threads %d\n",
+ FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-hvx %d\n",
octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3],
octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3],
octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3],
- srctx.vtcm_layout.src0_bytes_per_thread * n_threads,
- srctx.vtcm_layout.dst_bytes_per_thread * n_threads,
- n_threads);
+ srctx.vtcm_layout.src0_bytes_per_hvx * n_hvx,
+ srctx.vtcm_layout.dst_bytes_per_hvx * n_hvx,
+ n_hvx);
- work_queue_run(octx->ctx->work_queue, q_func, &srctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, q_func, &srctx, n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.h b/ggml/src/ggml-hexagon/htp/set-rows-ops.h
index b9cb6cc90..85086b08e 100644
--- a/ggml/src/ggml-hexagon/htp/set-rows-ops.h
+++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.h
@@ -6,7 +6,7 @@
#include "htp-vtcm.h"
struct htp_set_rows_kernel_params {
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t total_tasks;
uint32_t vtcm_size;
};
@@ -16,8 +16,8 @@ struct htp_set_rows_vtcm_layout {
size_t off_src0;
size_t off_dst;
- size_t src0_bytes_per_thread;
- size_t dst_bytes_per_thread;
+ size_t src0_bytes_per_hvx;
+ size_t dst_bytes_per_hvx;
size_t src0_spad_half_size;
size_t dst_spad_half_size;
@@ -27,7 +27,7 @@ static inline void htp_set_rows_vtcm_layout_build(
struct htp_set_rows_vtcm_layout * L,
int dst_type,
uint32_t ne00,
- uint32_t n_threads) {
+ uint32_t n_hvx) {
size_t src0_row_size = ne00 * sizeof(float);
size_t dst_row_size = htp_tensor_get_row_size(dst_type, ne00);
@@ -38,12 +38,12 @@ static inline void htp_set_rows_vtcm_layout_build(
L->src0_spad_half_size = src0_row_size_aligned;
L->dst_spad_half_size = dst_row_size_aligned;
- L->src0_bytes_per_thread = src0_row_size_aligned * 2;
- L->dst_bytes_per_thread = dst_row_size_aligned * 2;
+ L->src0_bytes_per_hvx = src0_row_size_aligned * 2;
+ L->dst_bytes_per_hvx = dst_row_size_aligned * 2;
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx);
L->total_bytes = off;
}
diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.c b/ggml/src/ggml-hexagon/htp/softmax-ops.c
index d65c3fae3..22aebcd5e 100644
--- a/ggml/src/ggml-hexagon/htp/softmax-ops.c
+++ b/ggml/src/ggml-hexagon/htp/softmax-ops.c
@@ -13,7 +13,7 @@
#include "hvx-utils.h"
#include "hex-fastdiv.h"
#include "hex-common.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
@@ -38,9 +38,9 @@ struct htp_softmax_context {
uint8_t * vtcm_src1;
uint8_t * vtcm_dst;
- uint32_t vtcm_src0_size_per_thread;
- uint32_t vtcm_src1_size_per_thread;
- uint32_t vtcm_dst_size_per_thread;
+ uint32_t vtcm_src0_size_per_hvx;
+ uint32_t vtcm_src1_size_per_hvx;
+ uint32_t vtcm_dst_size_per_hvx;
uint32_t src0_spad_half_size;
uint32_t src1_spad_half_size;
@@ -65,7 +65,7 @@ struct htp_softmax_context {
struct fastdiv_values div_ne12;
struct fastdiv_values div_ne13;
- uint32_t src0_nrows_per_thread;
+ uint32_t src0_nrows_per_hvx;
uint32_t row_start;
uint32_t nrows;
@@ -287,19 +287,19 @@ static const softmax_compute_fn_t softmax_kernels[HTP_SOFTMAX_KERNEL_COUNT] = {
[HTP_SOFTMAX_KERNEL_MASK_F16] = compute_fast_softmax_f32_mask_f16,
};
-static void softmax_thread_dma(unsigned int nth, unsigned int ith, void * data) {
+static void softmax_thread_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
const struct htp_softmax_context * smctx = (const struct htp_softmax_context *) data;
struct htp_ops_context * octx = smctx->octx;
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const uint32_t src0_nrows = smctx->nrows;
- const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread;
+ const uint32_t src0_nrows_per_hvx = smctx->src0_nrows_per_hvx;
- const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows);
+ const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, smctx->row_start + src0_nrows);
if (src0_start_row >= src0_end_row) {
return;
@@ -311,13 +311,13 @@ static void softmax_thread_dma(unsigned int nth, unsigned int ith, void * data)
const size_t src0_row_size = src0->ne[0] * sizeof(float);
const size_t dst_row_size = src0->ne[0] * sizeof(float);
- uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_thread);
- uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_thread);
+ uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_hvx);
+ uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_hvx);
const size_t src0_vtcm_half = smctx->src0_spad_half_size;
const size_t dst_vtcm_half = smctx->dst_spad_half_size;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
for (uint32_t r = src0_start_row, idx = 0; r < src0_end_row && idx < 2; r++, idx++) {
dma_addr_t cur_dst = data_dst + r * dst_row_size;
@@ -357,20 +357,20 @@ static void softmax_thread_dma(unsigned int nth, unsigned int ith, void * data)
dma_queue_flush(dma_q);
}
-static void softmax_thread_mask_dma(unsigned int nth, unsigned int ith, void * data) {
+static void softmax_thread_mask_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
const struct htp_softmax_context * smctx = (const struct htp_softmax_context *) data;
struct htp_ops_context * octx = smctx->octx;
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
const uint32_t src0_nrows = smctx->nrows;
- const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread;
+ const uint32_t src0_nrows_per_hvx = smctx->src0_nrows_per_hvx;
- const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows);
+ const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, smctx->row_start + src0_nrows);
if (src0_start_row >= src0_end_row) {
return;
@@ -384,9 +384,9 @@ static void softmax_thread_mask_dma(unsigned int nth, unsigned int ith, void * d
const size_t dst_row_size = src0->ne[0] * sizeof(float);
const size_t mask_row_size = smctx->use_f16 ? (src1->ne[0] * sizeof(__fp16)) : (src1->ne[0] * sizeof(float));
- uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_thread);
- uint8_t * src1_vtcm_base = smctx->vtcm_src1 + (ith * smctx->vtcm_src1_size_per_thread);
- uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_thread);
+ uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_hvx);
+ uint8_t * src1_vtcm_base = smctx->vtcm_src1 + (ith * smctx->vtcm_src1_size_per_hvx);
+ uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_hvx);
const size_t src0_vtcm_half = smctx->src0_spad_half_size;
const size_t src1_vtcm_half = smctx->src1_spad_half_size;
@@ -408,7 +408,7 @@ static void softmax_thread_mask_dma(unsigned int nth, unsigned int ith, void * d
const struct fastdiv_values * div_ne12 = &smctx->div_ne12;
const struct fastdiv_values * div_ne13 = &smctx->div_ne13;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
for (uint32_t r = src0_start_row, idx = 0; r < src0_end_row && idx < 2; r++, idx++) {
dma_addr_t cur_dst = data_dst + r * dst_row_size;
@@ -492,7 +492,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) {
const struct htp_softmax_kernel_params * kparams =
(const struct htp_softmax_kernel_params *) octx->kernel_params;
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -520,8 +520,8 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) {
src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
- if (nrows < octx->n_threads) {
- htp_ops_context_set_n_threads(octx, nrows ? nrows : 1);
+ if (nrows < octx->n_hvx) {
+ htp_ops_context_set_n_hvx(octx, nrows ? nrows : 1);
}
}
@@ -529,7 +529,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
uint8_t * const vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
struct htp_softmax_vtcm_layout layout;
@@ -538,7 +538,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) {
kparams->use_src1 ? octx->src[1]->ne[0] : 0,
kparams->use_src1 != 0,
kparams->use_f16 != 0,
- kparams->n_threads);
+ kparams->n_hvx);
struct htp_softmax_context smctx = {
.octx = octx,
@@ -553,9 +553,9 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) {
.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_dst),
.vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, vtcm_base, layout.off_src1, kparams->use_src1),
- .vtcm_src0_size_per_thread = (uint32_t) layout.src0_bytes_per_thread,
- .vtcm_src1_size_per_thread = (uint32_t) layout.src1_bytes_per_thread,
- .vtcm_dst_size_per_thread = (uint32_t) layout.dst_bytes_per_thread,
+ .vtcm_src0_size_per_hvx = (uint32_t) layout.src0_bytes_per_hvx,
+ .vtcm_src1_size_per_hvx = (uint32_t) layout.src1_bytes_per_hvx,
+ .vtcm_dst_size_per_hvx = (uint32_t) layout.dst_bytes_per_hvx,
.src0_spad_half_size = (uint32_t) layout.src0_spad_half_size,
.src1_spad_half_size = (uint32_t) layout.src1_spad_half_size,
@@ -580,7 +580,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) {
.div_ne12 = kparams->div_ne12,
.div_ne13 = kparams->div_ne13,
- .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
+ .src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div),
.row_start = row_start,
.nrows = nrows,
};
@@ -596,7 +596,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) {
}
work_queue_func_t task_func = kparams->use_src1 ? softmax_thread_mask_dma : softmax_thread_dma;
- work_queue_run(octx->ctx->work_queue, task_func, &smctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, task_func, &smctx, n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.h b/ggml/src/ggml-hexagon/htp/softmax-ops.h
index ef6567306..9e5f9c759 100644
--- a/ggml/src/ggml-hexagon/htp/softmax-ops.h
+++ b/ggml/src/ggml-hexagon/htp/softmax-ops.h
@@ -18,7 +18,7 @@ enum htp_softmax_kernel_type {
struct htp_softmax_kernel_params {
uint8_t kernel_type;
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t vtcm_size;
uint32_t n_head;
@@ -49,9 +49,9 @@ struct htp_softmax_vtcm_layout {
size_t off_dst;
size_t off_src1;
- size_t src0_bytes_per_thread;
- size_t dst_bytes_per_thread;
- size_t src1_bytes_per_thread;
+ size_t src0_bytes_per_hvx;
+ size_t dst_bytes_per_hvx;
+ size_t src1_bytes_per_hvx;
size_t src0_spad_half_size;
size_t dst_spad_half_size;
@@ -64,7 +64,7 @@ static inline void htp_softmax_vtcm_layout_build(
uint32_t ne10,
bool use_src1,
bool use_f16,
- uint32_t n_threads
+ uint32_t n_hvx
) {
size_t src0_row_size = ne00 * sizeof(float);
size_t dst_row_size = ne00 * sizeof(float);
@@ -79,14 +79,14 @@ static inline void htp_softmax_vtcm_layout_build(
L->src1_spad_half_size = src1_row_size_aligned;
// Double buffering: 2 half-buffers per thread
- L->src0_bytes_per_thread = src0_row_size_aligned * 2;
- L->dst_bytes_per_thread = dst_row_size_aligned * 2;
- L->src1_bytes_per_thread = src1_row_size_aligned * 2;
+ L->src0_bytes_per_hvx = src0_row_size_aligned * 2;
+ L->dst_bytes_per_hvx = dst_row_size_aligned * 2;
+ L->src1_bytes_per_hvx = src1_row_size_aligned * 2;
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes_per_thread * n_threads, use_src1);
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes_per_hvx * n_hvx, use_src1);
L->total_bytes = off;
}
diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c
index 628ca65e8..73eff927e 100644
--- a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c
+++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c
@@ -4,7 +4,7 @@
#include <string.h>
#include "hex-common.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
@@ -18,7 +18,7 @@
struct htp_solve_tri_context {
struct htp_ops_context * octx;
const struct htp_solve_tri_kernel_params * kparams;
- uint32_t jobs_per_thread;
+ uint32_t jobs_per_hvx;
uint32_t total_jobs;
uint32_t job_start;
uint32_t k_chunks;
@@ -79,7 +79,7 @@ static inline void solve_tri_row_hvx(const float * A_row,
}
// Batch-level thread: each job is one full batch.
-static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void * data) {
+static void solve_tri_batch_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_solve_tri_context * sctx = (struct htp_solve_tri_context *) data;
struct htp_ops_context * octx = sctx->octx;
const struct htp_solve_tri_kernel_params * kparams = sctx->kparams;
@@ -96,10 +96,10 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void
const uint32_t col_block = VLEN_FP32;
const uint32_t k_full = k & ~(col_block - 1);
- const uint32_t start_batch = sctx->job_start + sctx->jobs_per_thread * ith;
- const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs);
+ const uint32_t start_batch = sctx->job_start + sctx->jobs_per_hvx * ith;
+ const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_hvx, sctx->job_start + sctx->total_jobs);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_batch);
for (uint32_t batch = start_batch; batch < end_batch; ++batch) {
@@ -141,7 +141,7 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void
}
// Chunk-level thread: each job is one (batch, col_chunk) pair.
-static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void * data) {
+static void solve_tri_chunk_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_solve_tri_context * sctx = (struct htp_solve_tri_context *) data;
struct htp_ops_context * octx = sctx->octx;
const struct htp_solve_tri_kernel_params * kparams = sctx->kparams;
@@ -155,10 +155,10 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void
const uint32_t ne02 = src0->ne[2];
- const uint32_t start_job = sctx->job_start + sctx->jobs_per_thread * ith;
- const uint32_t end_job = MIN(start_job + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs);
+ const uint32_t start_job = sctx->job_start + sctx->jobs_per_hvx * ith;
+ const uint32_t end_job = MIN(start_job + sctx->jobs_per_hvx, sctx->job_start + sctx->total_jobs);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_job);
for (uint32_t job = start_job; job < end_job; ++job) {
@@ -233,23 +233,23 @@ int op_solve_tri(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, njobs))) {
+ if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, njobs))) {
return HTP_STATUS_INVAL_PARAMS;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
struct htp_solve_tri_context sctx = {
.octx = octx,
.kparams = kparams,
- .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div),
+ .jobs_per_hvx = fastdiv(njobs + n_hvx - 1, &octx->n_hvx_div),
.total_jobs = njobs,
.job_start = job_start,
.k_chunks = k_chunks,
.col_block = col_block,
};
- work_queue_run(octx->ctx->work_queue, solve_tri_batch_thread_f32, &sctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, solve_tri_batch_thread_f32, &sctx, n_hvx);
} else {
const uint32_t total_jobs = kparams->total_jobs;
@@ -267,23 +267,23 @@ int op_solve_tri(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, njobs))) {
+ if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, njobs))) {
return HTP_STATUS_INVAL_PARAMS;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
struct htp_solve_tri_context sctx = {
.octx = octx,
.kparams = kparams,
- .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div),
+ .jobs_per_hvx = fastdiv(njobs + n_hvx - 1, &octx->n_hvx_div),
.total_jobs = njobs,
.job_start = job_start,
.k_chunks = k_chunks,
.col_block = col_block,
};
- work_queue_run(octx->ctx->work_queue, solve_tri_chunk_thread_f32, &sctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, solve_tri_chunk_thread_f32, &sctx, n_hvx);
}
return HTP_STATUS_OK;
diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.h b/ggml/src/ggml-hexagon/htp/solve-tri-ops.h
index 327415814..6a46cfdc4 100644
--- a/ggml/src/ggml-hexagon/htp/solve-tri-ops.h
+++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.h
@@ -5,7 +5,7 @@
#include <stdint.h>
struct htp_solve_tri_kernel_params {
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t total_batches;
uint32_t total_jobs;
uint32_t k_chunks;
diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.c b/ggml/src/ggml-hexagon/htp/ssm-conv.c
index 4b06b18f2..d75304fef 100644
--- a/ggml/src/ggml-hexagon/htp/ssm-conv.c
+++ b/ggml/src/ggml-hexagon/htp/ssm-conv.c
@@ -15,7 +15,7 @@
#include "ggml-common.h"
#include "htp-ctx.h"
#include "dma-queue.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "hvx-utils.h"
@@ -24,7 +24,7 @@
struct htp_ssm_conv_context {
struct htp_ops_context * octx;
const struct htp_ssm_conv_kernel_params * kparams;
- uint32_t nrows_per_thread;
+ uint32_t nrows_per_hvx;
uint32_t d_inner_tile;
uint32_t row_start;
uint32_t nrows;
@@ -80,8 +80,8 @@ static inline void hvx_transpose_32x32_f32(HVX_Vector m[32]) {
}
// HVX deinterleave for d_conv == 4: channel-major raw VTCM -> tap-major T VTCM
-static inline void hvx_ssm_conv_unpack_to_T_4(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride) {
- for (uint32_t cb = 0; cb < d_inner_per_thread; cb += VLEN_FP32) {
+static inline void hvx_ssm_conv_unpack_to_T_4(const float * raw, float * T, uint32_t d_inner_per_hvx, uint32_t d_inner_stride) {
+ for (uint32_t cb = 0; cb < d_inner_per_hvx; cb += VLEN_FP32) {
HVX_Vector v0 = *(const HVX_Vector *)(raw + (cb + 0) * 4);
HVX_Vector v1 = *(const HVX_Vector *)(raw + (cb + 8) * 4);
HVX_Vector v2 = *(const HVX_Vector *)(raw + (cb + 16) * 4);
@@ -101,15 +101,15 @@ static inline void hvx_ssm_conv_unpack_to_T_4(const float * raw, float * T, uint
}
// HVX transpose for general d_conv <= 32: channel-major raw VTCM -> tap-major T VTCM
-static inline void hvx_ssm_conv_unpack_to_T_gen(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride, uint32_t d_conv) {
+static inline void hvx_ssm_conv_unpack_to_T_gen(const float * raw, float * T, uint32_t d_inner_per_hvx, uint32_t d_inner_stride, uint32_t d_conv) {
uint32_t __attribute__((aligned(VLEN))) mask_buf[VLEN_FP32] = { 0 };
for (uint32_t j = 0; j < d_conv; ++j) {
mask_buf[j] = 0xFFFFFFFF;
}
const HVX_Vector mask = *(const HVX_Vector *) mask_buf;
- for (uint32_t cb = 0; cb < d_inner_per_thread; cb += VLEN_FP32) {
- const uint32_t cb_n = MIN(VLEN_FP32, d_inner_per_thread - cb);
+ for (uint32_t cb = 0; cb < d_inner_per_hvx; cb += VLEN_FP32) {
+ const uint32_t cb_n = MIN(VLEN_FP32, d_inner_per_hvx - cb);
HVX_Vector sub[32];
for (uint32_t r = 0; r < cb_n; ++r) {
const float * ch_ptr = raw + (cb + r) * d_conv;
@@ -127,11 +127,11 @@ static inline void hvx_ssm_conv_unpack_to_T_gen(const float * raw, float * T, ui
}
}
-static inline void hvx_ssm_conv_unpack_to_T(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride, uint32_t d_conv) {
- if (d_conv == 4 && (d_inner_per_thread % VLEN_FP32 == 0)) {
- hvx_ssm_conv_unpack_to_T_4(raw, T, d_inner_per_thread, d_inner_stride);
+static inline void hvx_ssm_conv_unpack_to_T(const float * raw, float * T, uint32_t d_inner_per_hvx, uint32_t d_inner_stride, uint32_t d_conv) {
+ if (d_conv == 4 && (d_inner_per_hvx % VLEN_FP32 == 0)) {
+ hvx_ssm_conv_unpack_to_T_4(raw, T, d_inner_per_hvx, d_inner_stride);
} else {
- hvx_ssm_conv_unpack_to_T_gen(raw, T, d_inner_per_thread, d_inner_stride, d_conv);
+ hvx_ssm_conv_unpack_to_T_gen(raw, T, d_inner_per_hvx, d_inner_stride, d_conv);
}
}
@@ -186,7 +186,7 @@ static inline void hvx_ssm_conv_transpose_block(const float * raw_block,
}
// Single-token decode worker (n_t == 1)
-static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void * data) {
+static void ssm_conv_thread_f32_decode(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data;
struct htp_ops_context * octx = scctx->octx;
const struct htp_ssm_conv_kernel_params * kparams = scctx->kparams;
@@ -195,14 +195,14 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void
const struct htp_tensor * restrict src1 = octx->src[1];
const struct htp_tensor * restrict dst = octx->dst;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
const uint32_t d_conv = kparams->d_conv;
const uint32_t d_inner = kparams->d_inner;
const uint32_t n_s = kparams->n_s;
const bool apply_silu = kparams->silu != 0;
- const uint32_t dr = scctx->nrows_per_thread;
+ const uint32_t dr = scctx->nrows_per_hvx;
const uint32_t ir0 = scctx->row_start + dr * ith;
const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows);
@@ -210,19 +210,19 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void
return;
}
- const uint32_t d_inner_per_thread = ir1 - ir0;
- const uint32_t d_inner_stride = hex_round_up(d_inner_per_thread, VLEN_FP32);
+ const uint32_t d_inner_per_hvx = ir1 - ir0;
+ const uint32_t d_inner_stride = hex_round_up(d_inner_per_hvx, VLEN_FP32);
const uint32_t d_inner_tile = scctx->d_inner_tile;
const size_t src0_stride_inner_bytes = src0->nb[1];
const size_t src0_stride_seq_bytes = src0->nb[2];
const size_t dst_stride_seq_bytes = dst->nb[2];
- uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread;
- uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread;
- uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread;
+ uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_hvx;
+ uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_hvx;
+ uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_hvx;
- const size_t weight_bytes = (size_t) d_inner_per_thread * d_conv * sizeof(float);
+ const size_t weight_bytes = (size_t) d_inner_per_hvx * d_conv * sizeof(float);
const size_t weight_raw_size = hex_round_up(weight_bytes, 128);
float * src1_raw = (float *) src1_spad_base;
@@ -236,12 +236,12 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void
float * dst_tile[2] = { (float *) dst_spad_base, (float *) (dst_spad_base + dst_tile_bytes) };
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
// raw_dot keeps both operands in the DMA layout, so src1 needs no prep pass
- const bool raw_dot = (d_conv == 4) && (d_inner_per_thread % VLEN_FP32 == 0);
+ const bool raw_dot = (d_conv == 4) && (d_inner_per_hvx % VLEN_FP32 == 0);
- const uint32_t n_tiles = fastdiv(d_inner_per_thread + d_inner_tile - 1, &kparams->div_d_inner_tile);
+ const uint32_t n_tiles = fastdiv(d_inner_per_hvx + d_inner_tile - 1, &kparams->div_d_inner_tile);
const uint32_t n_chunks = n_s * n_tiles;
const size_t row_bytes = d_conv * sizeof(float);
@@ -250,14 +250,14 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void
#define SSM_CONV_DECODE_PUSH_FETCH(c) \
do { \
- const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off_fetch); \
+ const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off_fetch); \
const dma_addr_t fetch_ddr = src0->data + s_fetch * src0_stride_seq_bytes + \
(ir0 + tile_off_fetch) * src0_stride_inner_bytes; \
dma_queue_push(dma_q, \
dma_make_data((uint8_t *) src0_tile_raw[(c) & 1], fetch_ddr), \
row_bytes, src0_stride_inner_bytes, row_bytes, cur_tile_n); \
tile_off_fetch += d_inner_tile; \
- if (tile_off_fetch >= d_inner_per_thread) { \
+ if (tile_off_fetch >= d_inner_per_hvx) { \
tile_off_fetch = 0; \
s_fetch++; \
} \
@@ -277,7 +277,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void
if (!raw_dot) {
// Unpack/transpose src1_raw into src1_T {d_conv, d_inner_stride}
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0);
- hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_thread, d_inner_stride, d_conv);
+ hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_hvx, d_inner_stride, d_conv);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0);
}
@@ -285,7 +285,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void
uint32_t tile_off = 0;
for (uint32_t c = 0; c < n_chunks; ++c) {
- const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off);
+ const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off);
if (c >= 2) {
dma_queue_pop(dma_q); // writeback of chunk c-2, frees dst_tile[c & 1]
@@ -328,7 +328,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void
}
tile_off += d_inner_tile;
- if (tile_off >= d_inner_per_thread) {
+ if (tile_off >= d_inner_per_hvx) {
tile_off = 0;
i3++;
}
@@ -347,7 +347,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void
}
// Multi-token prefill worker (n_t > 1)
-static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void * data) {
+static void ssm_conv_thread_f32_prefill(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data;
struct htp_ops_context * octx = scctx->octx;
const struct htp_ssm_conv_kernel_params * kparams = scctx->kparams;
@@ -356,7 +356,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
const struct htp_tensor * restrict src1 = octx->src[1];
const struct htp_tensor * restrict dst = octx->dst;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
const uint32_t d_conv = kparams->d_conv;
const uint32_t d_inner = kparams->d_inner;
@@ -365,7 +365,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
const uint32_t ncs = src0->ne[0];
const bool apply_silu = kparams->silu != 0;
- const uint32_t dr = scctx->nrows_per_thread;
+ const uint32_t dr = scctx->nrows_per_hvx;
const uint32_t ir0 = scctx->row_start + dr * ith;
const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows);
@@ -373,8 +373,8 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
return;
}
- const uint32_t d_inner_per_thread = ir1 - ir0;
- const uint32_t d_inner_stride = hex_round_up(d_inner_per_thread, VLEN_FP32);
+ const uint32_t d_inner_per_hvx = ir1 - ir0;
+ const uint32_t d_inner_stride = hex_round_up(d_inner_per_hvx, VLEN_FP32);
const uint32_t d_inner_tile = scctx->d_inner_tile;
const size_t src0_stride_inner_bytes = src0->nb[1];
@@ -382,11 +382,11 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
const size_t dst_stride_token_bytes = dst->nb[1];
const size_t dst_stride_seq_bytes = dst->nb[2];
- uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread;
- uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread;
- uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread;
+ uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_hvx;
+ uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_hvx;
+ uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_hvx;
- const size_t weight_bytes = (size_t) d_inner_per_thread * d_conv * sizeof(float);
+ const size_t weight_bytes = (size_t) d_inner_per_hvx * d_conv * sizeof(float);
const size_t weight_raw_size = hex_round_up(weight_bytes, 128);
float * src1_raw = (float *) src1_spad_base;
@@ -403,9 +403,9 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
float * dst_tile[2] = { (float *) dst_spad_base, (float *) (dst_spad_base + dst_tile_bytes) };
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
- const uint32_t n_tiles = fastdiv(d_inner_per_thread + d_inner_tile - 1, &kparams->div_d_inner_tile);
+ const uint32_t n_tiles = fastdiv(d_inner_per_hvx + d_inner_tile - 1, &kparams->div_d_inner_tile);
const uint32_t n_chunks = n_s * n_tiles;
const size_t row_bytes = ncs * sizeof(float);
@@ -416,14 +416,14 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
// Two fetches run ahead, so the queue order is F0 F1 W0 F2 W1 ... and pops follow it.
#define SSM_CONV_PUSH_FETCH(c) \
do { \
- const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off_fetch); \
+ const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off_fetch); \
const dma_addr_t fetch_ddr = src0->data + s_fetch * src0_stride_seq_bytes + \
(ir0 + tile_off_fetch) * src0_stride_inner_bytes; \
dma_queue_push(dma_q, \
dma_make_data((uint8_t *) src0_tile_raw[(c) & 1], fetch_ddr), \
raw_stride_bytes, src0_stride_inner_bytes, row_bytes, cur_tile_n); \
tile_off_fetch += d_inner_tile; \
- if (tile_off_fetch >= d_inner_per_thread) { \
+ if (tile_off_fetch >= d_inner_per_hvx) { \
tile_off_fetch = 0; \
s_fetch++; \
} \
@@ -442,7 +442,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
// Unpack/transpose src1_raw into src1_T {d_conv, d_inner_stride}
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0);
- hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_thread, d_inner_stride, d_conv);
+ hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_hvx, d_inner_stride, d_conv);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0);
const uint32_t C_TILE = VLEN_FP32;
@@ -454,7 +454,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
uint32_t tile_off = 0;
for (uint32_t c = 0; c < n_chunks; ++c) {
- const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off);
+ const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off);
const float * restrict raw = src0_tile_raw[c & 1];
float * restrict out = dst_tile[c & 1];
@@ -533,7 +533,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
}
tile_off += d_inner_tile;
- if (tile_off >= d_inner_per_thread) {
+ if (tile_off >= d_inner_per_hvx) {
tile_off = 0;
i3++;
}
@@ -558,7 +558,7 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) {
const struct htp_ssm_conv_kernel_params * kparams = (const struct htp_ssm_conv_kernel_params *) octx->kernel_params;
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -586,18 +586,18 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) {
return HTP_STATUS_VTCM_TOO_SMALL;
}
- const uint32_t n_threads = octx->n_threads;
- const uint32_t raw_rpt = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
- const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, VLEN_FP32);
+ const uint32_t n_hvx = octx->n_hvx;
+ const uint32_t raw_rpt = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div);
+ const uint32_t d_inner_per_hvx = hex_round_up(raw_rpt, VLEN_FP32);
struct htp_ssm_conv_vtcm_layout L;
- htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_thread, kparams->d_conv, kparams->n_t, kparams->d_inner_tile, n_threads);
+ htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_hvx, kparams->d_conv, kparams->n_t, kparams->d_inner_tile, n_hvx);
uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;
- octx->src0_spad.size_per_thread = L.src0_bytes_per_thread;
- octx->src1_spad.size_per_thread = L.src1_bytes_per_thread;
- octx->dst_spad.size_per_thread = L.dst_bytes_per_thread;
+ octx->src0_spad.size_per_hvx = L.src0_bytes_per_hvx;
+ octx->src1_spad.size_per_hvx = L.src1_bytes_per_hvx;
+ octx->dst_spad.size_per_hvx = L.dst_bytes_per_hvx;
octx->src0_spad.size = L.src0_bytes;
octx->src1_spad.size = L.src1_bytes;
@@ -613,7 +613,7 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) {
struct htp_ssm_conv_context scctx = {
.octx = octx,
.kparams = kparams,
- .nrows_per_thread = d_inner_per_thread,
+ .nrows_per_hvx = d_inner_per_hvx,
.d_inner_tile = kparams->d_inner_tile,
.row_start = row_start,
.nrows = nrows,
@@ -626,9 +626,9 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) {
kparams->n_t == 1 ? "decode" : "prefill");
if (kparams->n_t == 1) {
- work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_decode, &scctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, ssm_conv_thread_f32_decode, &scctx, n_hvx);
} else {
- work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_prefill, &scctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, ssm_conv_thread_f32_prefill, &scctx, n_hvx);
}
return HTP_STATUS_OK;
diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.h b/ggml/src/ggml-hexagon/htp/ssm-conv.h
index cd4180857..7c3f3041f 100644
--- a/ggml/src/ggml-hexagon/htp/ssm-conv.h
+++ b/ggml/src/ggml-hexagon/htp/ssm-conv.h
@@ -9,7 +9,7 @@
#include "htp-vtcm.h"
struct htp_ssm_conv_kernel_params {
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t d_conv;
uint32_t d_inner;
uint32_t n_t;
@@ -27,9 +27,9 @@ struct htp_ssm_conv_vtcm_layout {
size_t off_src0;
size_t off_src1;
size_t off_dst;
- size_t src0_bytes_per_thread;
- size_t src1_bytes_per_thread;
- size_t dst_bytes_per_thread;
+ size_t src0_bytes_per_hvx;
+ size_t src1_bytes_per_hvx;
+ size_t dst_bytes_per_hvx;
size_t src0_bytes;
size_t src1_bytes;
size_t dst_bytes;
@@ -37,34 +37,34 @@ struct htp_ssm_conv_vtcm_layout {
static inline void htp_ssm_conv_vtcm_layout_build(
struct htp_ssm_conv_vtcm_layout * L,
- uint32_t d_inner_per_thread,
+ uint32_t d_inner_per_hvx,
uint32_t d_conv,
uint32_t n_t,
uint32_t d_inner_tile,
- uint32_t n_threads
+ uint32_t n_hvx
) {
- const size_t src1_raw_bytes = hex_round_up((size_t) d_inner_per_thread * d_conv * sizeof(float), 128) + 128;
- const size_t src1_T_bytes = hex_round_up((size_t) d_conv * d_inner_per_thread * sizeof(float), 128);
+ const size_t src1_raw_bytes = hex_round_up((size_t) d_inner_per_hvx * d_conv * sizeof(float), 128) + 128;
+ const size_t src1_T_bytes = hex_round_up((size_t) d_conv * d_inner_per_hvx * sizeof(float), 128);
- L->src1_bytes_per_thread = src1_raw_bytes + src1_T_bytes;
+ L->src1_bytes_per_hvx = src1_raw_bytes + src1_T_bytes;
if (n_t == 1) {
const size_t src0_tile_raw_bytes = hex_round_up((size_t) d_inner_tile * d_conv * sizeof(float), 128);
const size_t src0_T_bytes = hex_round_up((size_t) d_conv * d_inner_tile * sizeof(float), 128);
- L->src0_bytes_per_thread = 2 * src0_tile_raw_bytes + src0_T_bytes;
- L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * sizeof(float), 128);
+ L->src0_bytes_per_hvx = 2 * src0_tile_raw_bytes + src0_T_bytes;
+ L->dst_bytes_per_hvx = 2 * hex_round_up((size_t) d_inner_tile * sizeof(float), 128);
} else {
const uint32_t ncs = d_conv - 1 + n_t;
const size_t raw_stride_bytes = ncs * sizeof(float) + sizeof(float);
const size_t src0_block_T = hex_round_up((size_t) ncs * 32 * sizeof(float), 128);
const size_t src0_tile_raw = hex_round_up((size_t) d_inner_tile * raw_stride_bytes, 128);
- L->src0_bytes_per_thread = 2 * src0_tile_raw + src0_block_T;
- L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * n_t * sizeof(float), 128);
+ L->src0_bytes_per_hvx = 2 * src0_tile_raw + src0_block_T;
+ L->dst_bytes_per_hvx = 2 * hex_round_up((size_t) d_inner_tile * n_t * sizeof(float), 128);
}
- L->src0_bytes = L->src0_bytes_per_thread * n_threads;
- L->src1_bytes = L->src1_bytes_per_thread * n_threads;
- L->dst_bytes = L->dst_bytes_per_thread * n_threads;
+ L->src0_bytes = L->src0_bytes_per_hvx * n_hvx;
+ L->src1_bytes = L->src1_bytes_per_hvx * n_hvx;
+ L->dst_bytes = L->dst_bytes_per_hvx * n_hvx;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes);
diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c
index 3d931ed19..b3f8a7090 100644
--- a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c
+++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c
@@ -14,7 +14,7 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
@@ -51,18 +51,18 @@ struct sum_rows_context {
uint32_t ne00;
size_t src_stride;
size_t dst_stride;
- uint32_t rows_per_thread;
+ uint32_t rows_per_hvx;
uint32_t total_rows;
};
-static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) {
+static void sum_rows_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void *data) {
const struct sum_rows_context * smctx = (const struct sum_rows_context *) data;
- const uint32_t rows_per_thread = smctx->rows_per_thread;
+ const uint32_t rows_per_hvx = smctx->rows_per_hvx;
const uint32_t total_rows = smctx->total_rows;
- const uint32_t start_row = rows_per_thread * ith;
- const uint32_t end_row = MIN(start_row + rows_per_thread, total_rows);
+ const uint32_t start_row = rows_per_hvx * ith;
+ const uint32_t end_row = MIN(start_row + rows_per_hvx, total_rows);
if (start_row >= end_row) {
return;
@@ -78,7 +78,7 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data)
// Calculate actual number of rows for this thread
const uint32_t n_rows = end_row - start_row;
- struct htp_thread_trace * tr = &smctx->octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row);
for (uint32_t ir = 0; ir < n_rows; ir++) {
@@ -109,7 +109,7 @@ int op_sum_rows(struct htp_ops_context * octx) {
const struct htp_sum_rows_kernel_params * kparams =
(const struct htp_sum_rows_kernel_params *) octx->kernel_params;
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -131,8 +131,8 @@ int op_sum_rows(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
- const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
+ const uint32_t n_hvx = octx->n_hvx;
+ const uint32_t rows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div);
struct sum_rows_context smctx = {
.octx = octx,
@@ -141,11 +141,11 @@ int op_sum_rows(struct htp_ops_context * octx) {
.ne00 = ne00,
.src_stride = nb01,
.dst_stride = nb1,
- .rows_per_thread = rows_per_thread,
+ .rows_per_hvx = rows_per_hvx,
.total_rows = nrows,
};
- work_queue_run(octx->ctx->work_queue, sum_rows_thread_f32, &smctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, sum_rows_thread_f32, &smctx, n_hvx);
return HTP_STATUS_OK;
}
@@ -153,15 +153,15 @@ int op_sum_rows(struct htp_ops_context * octx) {
struct sum_context {
struct htp_ops_context * octx;
const float * src_data;
- float partial_sums[HTP_MAX_NTHREADS];
+ float partial_sums[HTP_MAX_NHVX];
uint32_t total_elems;
- uint32_t elems_per_thread;
+ uint32_t elems_per_hvx;
};
-static void sum_thread_f32(unsigned int nth, unsigned int ith, void * data) {
+static void sum_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct sum_context * sctx = (struct sum_context *) data;
- const uint32_t start = sctx->elems_per_thread * ith;
- const uint32_t end = MIN(start + sctx->elems_per_thread, sctx->total_elems);
+ const uint32_t start = sctx->elems_per_hvx * ith;
+ const uint32_t end = MIN(start + sctx->elems_per_hvx, sctx->total_elems);
if (start >= end) {
sctx->partial_sums[ith] = 0.0f;
@@ -171,7 +171,7 @@ static void sum_thread_f32(unsigned int nth, unsigned int ith, void * data) {
const uint32_t n = end - start;
const float * src = sctx->src_data + start;
- struct htp_thread_trace * tr = &sctx->octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start);
hex_l2fetch_block((const void *) src, n * sizeof(float));
@@ -196,7 +196,7 @@ int op_sum(struct htp_ops_context * octx) {
const struct htp_sum_rows_kernel_params * kparams =
(const struct htp_sum_rows_kernel_params *) octx->kernel_params;
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -210,20 +210,20 @@ int op_sum(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
- const uint32_t elems_per_thread = kparams->elems_per_thread;
+ const uint32_t n_hvx = octx->n_hvx;
+ const uint32_t elems_per_hvx = kparams->elems_per_hvx;
struct sum_context sctx = {
.octx = octx,
.src_data = (const float *) src0->data,
.total_elems = total_elems,
- .elems_per_thread = elems_per_thread,
+ .elems_per_hvx = elems_per_hvx,
};
- work_queue_run(octx->ctx->work_queue, sum_thread_f32, &sctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, sum_thread_f32, &sctx, n_hvx);
float sum = 0.0f;
- for (uint32_t i = 0; i < n_threads; i++) {
+ for (uint32_t i = 0; i < n_hvx; i++) {
sum += sctx.partial_sums[i];
}
((float *) dst->data)[0] = sum;
@@ -250,17 +250,17 @@ struct argmax_context {
uint32_t dst_stride;
uint32_t row_start;
uint32_t nrows;
- uint32_t rows_per_thread;
- uint32_t elems_per_thread;
+ uint32_t rows_per_hvx;
+ uint32_t elems_per_hvx;
- float partial_max[HTP_MAX_NTHREADS];
- int32_t partial_idx[HTP_MAX_NTHREADS];
+ float partial_max[HTP_MAX_NHVX];
+ int32_t partial_idx[HTP_MAX_NHVX];
};
-static void argmax_thread_single_row(unsigned int nth, unsigned int ith, void * data) {
+static void argmax_thread_single_row(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct argmax_context * actx = (struct argmax_context *) data;
- const uint32_t start = actx->elems_per_thread * ith;
- const uint32_t end = MIN(start + actx->elems_per_thread, actx->ne00);
+ const uint32_t start = actx->elems_per_hvx * ith;
+ const uint32_t end = MIN(start + actx->elems_per_hvx, actx->ne00);
if (start >= end) {
actx->partial_max[ith] = -INFINITY;
@@ -271,7 +271,7 @@ static void argmax_thread_single_row(unsigned int nth, unsigned int ith, void *
const uint32_t n = end - start;
const float * src = actx->src_data + start;
- struct htp_thread_trace * tr = &actx->octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start);
hex_l2fetch_block((const void *) src, n * sizeof(float));
@@ -286,16 +286,16 @@ static void argmax_thread_single_row(unsigned int nth, unsigned int ith, void *
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start);
}
-static void argmax_thread_multi_row(unsigned int nth, unsigned int ith, void * data) {
+static void argmax_thread_multi_row(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct argmax_context * actx = (struct argmax_context *) data;
- const uint32_t r0 = actx->row_start + actx->rows_per_thread * ith;
- const uint32_t r1 = MIN(r0 + actx->rows_per_thread, actx->row_start + actx->nrows);
+ const uint32_t r0 = actx->row_start + actx->rows_per_hvx * ith;
+ const uint32_t r1 = MIN(r0 + actx->rows_per_hvx, actx->row_start + actx->nrows);
if (r0 >= r1) {
return;
}
- struct htp_thread_trace * tr = &actx->octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r0);
for (uint32_t r = r0; r < r1; r++) {
@@ -329,7 +329,7 @@ int op_argmax(struct htp_ops_context * octx) {
const struct htp_sum_rows_kernel_params * kparams =
(const struct htp_sum_rows_kernel_params *) octx->kernel_params;
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -350,8 +350,8 @@ int op_argmax(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
- const uint32_t elems_per_thread = kparams->elems_per_thread;
+ const uint32_t n_hvx = octx->n_hvx;
+ const uint32_t elems_per_hvx = kparams->elems_per_hvx;
struct argmax_context actx = {
.octx = octx,
@@ -362,15 +362,15 @@ int op_argmax(struct htp_ops_context * octx) {
.dst_stride = dst->nb[0] > 0 ? (uint32_t) dst->nb[0] : (uint32_t) sizeof(int32_t),
.row_start = 0,
.nrows = 1,
- .rows_per_thread = 1,
- .elems_per_thread = elems_per_thread,
+ .rows_per_hvx = 1,
+ .elems_per_hvx = elems_per_hvx,
};
- work_queue_run(octx->ctx->work_queue, argmax_thread_single_row, &actx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, argmax_thread_single_row, &actx, n_hvx);
float best_val = actx.partial_max[0];
int32_t best_idx = actx.partial_idx[0];
- for (uint32_t i = 1; i < n_threads; i++) {
+ for (uint32_t i = 1; i < n_hvx; i++) {
if (actx.partial_max[i] > best_val) {
best_val = actx.partial_max[i];
best_idx = actx.partial_idx[i];
@@ -397,8 +397,8 @@ int op_argmax(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
- const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
+ const uint32_t n_hvx = octx->n_hvx;
+ const uint32_t rows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div);
struct argmax_context actx = {
.octx = octx,
@@ -409,10 +409,10 @@ int op_argmax(struct htp_ops_context * octx) {
.dst_stride = dst->nb[0] > 0 ? (uint32_t) dst->nb[0] : (uint32_t) sizeof(int32_t),
.row_start = row_start,
.nrows = nrows,
- .rows_per_thread = rows_per_thread,
- .elems_per_thread = 0,
+ .rows_per_hvx = rows_per_hvx,
+ .elems_per_hvx = 0,
};
- work_queue_run(octx->ctx->work_queue, argmax_thread_multi_row, &actx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, argmax_thread_multi_row, &actx, n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.h b/ggml/src/ggml-hexagon/htp/sum-rows-ops.h
index cc9ddd483..0b8acf030 100644
--- a/ggml/src/ggml-hexagon/htp/sum-rows-ops.h
+++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.h
@@ -4,8 +4,8 @@
#include <stdint.h>
struct htp_sum_rows_kernel_params {
- uint32_t n_threads;
- uint32_t elems_per_thread;
+ uint32_t n_hvx;
+ uint32_t elems_per_hvx;
};
#if defined(__cplusplus)
diff --git a/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c
index 215f01ced..9c4dc583c 100644
--- a/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c
+++ b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c
@@ -9,7 +9,7 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
@@ -25,11 +25,11 @@ struct htp_top_k_mask_context {
struct htp_ops_context * octx;
const struct htp_top_k_mask_kernel_params * kparams;
- uint32_t rows_per_thread;
+ uint32_t rows_per_hvx;
uint32_t row0, row1; // rows this device handles
uint32_t off_extra, off_score; // parts inside one row input slot
uint32_t col0, ncols_out; // PICKS: columns this device writes back
- atomic_uint barrier; // PICKS: arrivals, barrier k waits for k * n_threads
+ atomic_uint barrier; // PICKS: arrivals, barrier k waits for k * n_hvx
uint8_t * blocks;
uint8_t * mask;
@@ -348,7 +348,7 @@ static inline void top_k_mask_next(uint32_t * r, uint32_t * ct, uint32_t n_col_t
// ROWS: the thread count follows the device rows (mdev), so the slice of an n byte copy is sized here
static inline uint32_t top_k_mask_slice(const struct htp_ops_context * octx, unsigned int nth, uint32_t n) {
- return hex_round_up(fastdiv(n + nth - 1, &octx->n_threads_div), HTP_TOP_K_MASK_UNIT);
+ return hex_round_up(fastdiv(n + nth - 1, &octx->n_hvx_div), HTP_TOP_K_MASK_UNIT);
}
// Push this thread's slice (128B aligned) of a contiguous n byte copy. Returns the push count.
@@ -363,12 +363,12 @@ static inline uint32_t top_k_mask_push_slice(dma_queue * dma_q, unsigned int ith
}
// Pass 1 (ROWS): load the blocks table and the resident mask row, split across threads. Fill the tiles of this thread with -inf meanwhile.
-static void top_k_mask_preload(unsigned int nth, unsigned int ith, void * data) {
+static void top_k_mask_preload(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_top_k_mask_context * tctx = (struct htp_top_k_mask_context *) data;
const struct htp_top_k_mask_kernel_params * kp = tctx->kparams;
struct htp_ops_context * octx = tctx->octx;
- dma_queue * dma_q = octx->ctx->dma[ith];
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ dma_queue * dma_q = th->dma_q;
+ struct htp_thread_trace * tr = th->trace;
uint32_t n_pushed = 0;
@@ -393,12 +393,12 @@ static void top_k_mask_preload(unsigned int nth, unsigned int ith, void * data)
// Pass 2 (ROWS): depth-2 DMA pipeline over the items of rows [row_first, row_last), item = (row, col tile).
// The row inputs alternate between 2 slots by row.
-static void top_k_mask_run_items(const struct htp_top_k_mask_context * tctx, unsigned int ith, uint32_t row_first, uint32_t row_last) {
+static void top_k_mask_run_items(const struct htp_top_k_mask_context * tctx, const htp_thread_t * th, unsigned int ith, uint32_t row_first, uint32_t row_last) {
struct htp_ops_context * octx = tctx->octx;
const struct htp_top_k_mask_kernel_params * kp = tctx->kparams;
const struct htp_tensor * dst = octx->dst;
- dma_queue * dma_q = octx->ctx->dma[ith];
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ dma_queue * dma_q = th->dma_q;
+ struct htp_thread_trace * tr = th->trace;
const bool is_f16 = kp->flags & HTP_TOP_K_MASK_F16;
const uint32_t elem_size = kp->elem_size;
@@ -480,16 +480,16 @@ static void top_k_mask_run_items(const struct htp_top_k_mask_context * tctx, uns
}
// Pass 2 (ROWS): whole rows split across threads, the col tiles of a row in order
-static void top_k_mask_thread_rows(unsigned int nth, unsigned int ith, void * data) {
+static void top_k_mask_thread_rows(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
const struct htp_top_k_mask_context * tctx = (const struct htp_top_k_mask_context *) data;
- const uint32_t ir0 = tctx->row0 + ith * tctx->rows_per_thread;
- const uint32_t ir1 = MIN(ir0 + tctx->rows_per_thread, tctx->row1);
+ const uint32_t ir0 = tctx->row0 + ith * tctx->rows_per_hvx;
+ const uint32_t ir1 = MIN(ir0 + tctx->rows_per_hvx, tctx->row1);
if (ir0 >= ir1) {
return;
}
- top_k_mask_run_items(tctx, ith, ir0, ir1);
+ top_k_mask_run_items(tctx, th, ith, ir0, ir1);
}
static inline void top_k_mask_barrier(atomic_uint * b, uint32_t target, struct htp_thread_trace * tr, uint16_t info) {
@@ -501,15 +501,15 @@ static inline void top_k_mask_barrier(atomic_uint * b, uint32_t target, struct h
htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, info);
}
-// PICKS, one pass over n_virt = kp->n_threads virtual workers (host slices); real worker ith runs the virtual workers ith, ith + nth, ...
+// PICKS, one pass over n_virt = kp->n_hvx virtual workers (host slices); real worker ith runs the virtual workers ith, ith + nth, ...
// A: shared preload and a slice of the flag clear per virtual worker, the mask slice of this worker into the row tile;
// barrier; B: scatter keep flags of the picks of each virtual worker; barrier; C: select mask or -inf in this worker's slice and write it back
-static void top_k_mask_thread_picks(unsigned int nth, unsigned int ith, void * data) {
+static void top_k_mask_thread_picks(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct htp_top_k_mask_context * tctx = (struct htp_top_k_mask_context *) data;
const struct htp_top_k_mask_kernel_params * kp = tctx->kparams;
struct htp_ops_context * octx = tctx->octx;
- dma_queue * dma_q = octx->ctx->dma[ith];
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ dma_queue * dma_q = th->dma_q;
+ struct htp_thread_trace * tr = th->trace;
const struct htp_tensor * top_k = octx->src[1];
const struct htp_tensor * score = octx->src[2];
@@ -520,13 +520,13 @@ static void top_k_mask_thread_picks(unsigned int nth, unsigned int ith, void * d
uint8_t * out = tctx->tiles;
uint8_t * flags = tctx->tiles + tile_bytes;
- const uint32_t n_virt = kp->n_threads;
+ const uint32_t n_virt = kp->n_hvx;
// this worker's slice of the device columns: the mask goes straight into the row tile, it is popped after the scan.
// The device share of the columns depends on mdev, so this slice is sized here.
const uint32_t out_b0 = tctx->col0 * kp->elem_size;
const uint32_t out_nb = tctx->ncols_out * kp->elem_size;
- const uint32_t wslice = hex_round_up(fastdiv(out_nb + nth - 1, &octx->n_threads_div), 128);
+ const uint32_t wslice = hex_round_up(fastdiv(out_nb + nth - 1, &octx->n_hvx_div), 128);
const uint32_t wb0 = out_b0 + ith * wslice;
const uint32_t wnb = ith * wslice < out_nb ? MIN(wslice, out_nb - ith * wslice) : 0;
@@ -567,8 +567,8 @@ static void top_k_mask_thread_picks(unsigned int nth, unsigned int ith, void * d
HVX_Vector * tmp = (HVX_Vector *) (tctx->scratch + ith * HTP_TOP_K_MASK_SCRATCH_BYTES);
for (uint32_t v = ith; v < n_virt; v += nth) {
- const uint32_t j0 = MIN(v * kp->u.picks.picks_per_thread, kp->n_top);
- const uint32_t j1 = MIN(j0 + kp->u.picks.picks_per_thread, kp->n_top);
+ const uint32_t j0 = MIN(v * kp->u.picks.picks_per_hvx, kp->n_top);
+ const uint32_t j1 = MIN(j0 + kp->u.picks.picks_per_hvx, kp->n_top);
const bool do_extra = v == n_virt - 1;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) v);
@@ -624,12 +624,12 @@ int op_top_k_mask(struct htp_ops_context * octx) {
}
}
- // fewer workers than kparams->n_threads always fit the VTCM layout: ROWS runs one row range per worker, PICKS loops over the host slices
- const uint32_t max_threads = MIN((uint32_t) kp->n_threads, octx->ctx->n_threads);
- if (!htp_ops_context_set_n_threads(octx, picks ? max_threads : MIN(max_threads, dev_rows))) {
+ // fewer workers than kparams->n_hvx always fit the VTCM layout: ROWS runs one row range per worker, PICKS loops over the host slices
+ const uint32_t max_hvx = MIN((uint32_t) kp->n_hvx, octx->ctx->n_hvx);
+ if (!htp_ops_context_set_n_hvx(octx, picks ? max_hvx : MIN(max_hvx, dev_rows))) {
return HTP_STATUS_INVAL_PARAMS;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
uint8_t * vtcm = octx->ctx->vtcm_base;
@@ -652,14 +652,14 @@ int op_top_k_mask(struct htp_ops_context * octx) {
if (picks) {
atomic_init(&tctx.barrier, 0);
- work_queue_run(octx->ctx->work_queue, top_k_mask_thread_picks, &tctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, top_k_mask_thread_picks, &tctx, n_hvx);
return HTP_STATUS_OK;
}
- tctx.rows_per_thread = fastdiv(dev_rows + n_threads - 1, &octx->n_threads_div);
+ tctx.rows_per_hvx = fastdiv(dev_rows + n_hvx - 1, &octx->n_hvx_div);
- work_queue_run(octx->ctx->work_queue, top_k_mask_preload, &tctx, n_threads);
- work_queue_run(octx->ctx->work_queue, top_k_mask_thread_rows, &tctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, top_k_mask_preload, &tctx, n_hvx);
+ work_queue_run(octx->ctx->hvx_queue, top_k_mask_thread_rows, &tctx, n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h
index ba1be96b8..9dcdfaa48 100644
--- a/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h
+++ b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h
@@ -53,7 +53,7 @@ static inline uint64_t htp_top_k_mask_rowin_off_score(uint32_t n_top, uint32_t n
// the caller compares total_bytes against the VTCM size
static inline void htp_top_k_mask_vtcm_layout_build(struct htp_top_k_mask_vtcm_layout * L,
- const struct htp_top_k_mask_shape * S, uint32_t kernel_type, uint32_t n_threads, uint32_t tile_cols) {
+ const struct htp_top_k_mask_shape * S, uint32_t kernel_type, uint32_t n_hvx, uint32_t tile_cols) {
memset(L, 0, sizeof(*L));
// PICKS streams the mask straight into its row tile
@@ -67,10 +67,10 @@ static inline void htp_top_k_mask_vtcm_layout_build(struct htp_top_k_mask_vtcm_l
const uint64_t tile_bytes = htp_top_k_mask_align128((uint64_t) tile_cols * S->elem_size);
// ROWS streams row inputs into 2 slots per thread. PICKS: one row, a row tile and a flag tile.
- const uint64_t rowin_total = (picks ? 1 : 2 * (uint64_t) n_threads) * rowin_bytes;
- const uint64_t tiles_total = picks ? 2 * tile_bytes : 2 * (uint64_t) n_threads * tile_bytes;
+ const uint64_t rowin_total = (picks ? 1 : 2 * (uint64_t) n_hvx) * rowin_bytes;
+ const uint64_t tiles_total = picks ? 2 * tile_bytes : 2 * (uint64_t) n_hvx * tile_bytes;
const uint64_t mtiles_total = S->mask_resident ? 0 : tiles_total;
- const uint64_t scratch = (uint64_t) n_threads * HTP_TOP_K_MASK_SCRATCH_BYTES;
+ const uint64_t scratch = (uint64_t) n_hvx * HTP_TOP_K_MASK_SCRATCH_BYTES;
size_t off = 0;
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_blocks, (size_t) blocks_bytes, blocks_bytes > 0);
@@ -90,9 +90,9 @@ static inline void htp_top_k_mask_vtcm_layout_build(struct htp_top_k_mask_vtcm_l
// Set by the host, used as is by the NPU: no tensor shape, stride or layout is derived on the NPU.
// Sizes of one slot or tile, the slot parts and the VTCM offsets are in HTP_TOP_K_MASK_UNIT bytes as u16.
// Row strides: byte offset of flat row r of top_k, extra, score and dst is r * stride (the host requires row flattenable inputs).
-// PICKS splits the work in n_threads virtual workers; fewer real workers loop over them.
+// PICKS splits the work in n_hvx virtual workers; fewer real workers loop over them.
struct htp_top_k_mask_kernel_params {
- uint8_t n_threads, kernel_type, flags, elem_size;
+ uint8_t n_hvx, kernel_type, flags, elem_size;
uint32_t vtcm_size;
uint32_t n_kv, n_top, n_pick, n_extra;
uint32_t ninf; // -inf cell pattern of elem_size
@@ -116,7 +116,7 @@ struct htp_top_k_mask_kernel_params {
} m;
} rows;
struct {
- uint32_t picks_per_thread; // per virtual worker, multiple of the lanes of one vector
+ uint32_t picks_per_hvx; // per virtual worker, multiple of the lanes of one vector
uint16_t blocks_slice_u, top_slice_u, extra_slice_u, score_slice_u, flags_slice_u; // per virtual worker slices
} picks;
} u;
diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c
index 3a6840169..19ade22f3 100644
--- a/ggml/src/ggml-hexagon/htp/unary-ops.c
+++ b/ggml/src/ggml-hexagon/htp/unary-ops.c
@@ -21,7 +21,7 @@
#include "htp-ops.h"
#include "htp-tensor.h"
#include "htp-vtcm.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
struct htp_unary_context;
@@ -87,7 +87,7 @@ struct htp_unary_context {
uint32_t block;
uint32_t src0_nrows;
- uint32_t src0_nrows_per_thread;
+ uint32_t src0_nrows_per_hvx;
uint32_t row_start;
uint32_t nc;
uint32_t col_tile; // tiled mode
@@ -98,10 +98,10 @@ struct htp_unary_context {
uint8_t * vtcm_src2;
uint8_t * vtcm_dst;
- size_t vtcm_src0_size_per_thread;
- size_t vtcm_src1_size_per_thread;
- size_t vtcm_src2_size_per_thread;
- size_t vtcm_dst_size_per_thread;
+ size_t vtcm_src0_size_per_hvx;
+ size_t vtcm_src1_size_per_hvx;
+ size_t vtcm_src2_size_per_hvx;
+ size_t vtcm_dst_size_per_hvx;
};
// Convert flat row index to DDR byte offset using the tensor's actual strides.
@@ -1233,25 +1233,25 @@ static void tri_apply_tile_f32(const void * restrict src, void * restrict dst,
}
// 1. Standard row-block unary task (F32 and F16).
-static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * data) {
+static void unary_thread_row_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
const struct htp_unary_context * uctx = (const struct htp_unary_context *) data;
struct htp_ops_context * octx = uctx->octx;
const struct htp_tensor * src = octx->src[0];
const struct htp_tensor * dst = octx->dst;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_unary_preamble;
- const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread;
+ const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx;
const size_t src0_data_row_size = uctx->src0_data_row_size;
const size_t dst_data_row_size = uctx->dst_data_row_size;
const size_t src0_row_size_aligned = uctx->src0_row_size_aligned;
const size_t dst_row_size_aligned = uctx->dst_row_size_aligned;
const uint32_t src0_nrows = uctx->src0_nrows;
- const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows);
+ const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows);
if (src0_start_row >= src0_end_row) {
return;
@@ -1260,8 +1260,8 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da
const dma_addr_t data_src = uctx->data_src0;
const dma_addr_t data_dst = uctx->data_dst;
- uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread);
- uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread);
+ uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx);
+ uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx);
const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size;
const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size;
@@ -1273,7 +1273,7 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da
const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02;
const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) {
const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01);
@@ -1330,25 +1330,25 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da
}
// 2. RMS_NORM_MUL row-block task with weight buffer.
-static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, void * data) {
+static void unary_thread_rms_norm_mul_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
const struct htp_unary_context * uctx = (const struct htp_unary_context *) data;
struct htp_ops_context * octx = uctx->octx;
const struct htp_tensor * src = octx->src[0];
const struct htp_tensor * dst = octx->dst;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_unary_preamble;
- const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread;
+ const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx;
const size_t src0_data_row_size = uctx->src0_data_row_size;
const size_t dst_data_row_size = uctx->dst_data_row_size;
const size_t src0_row_size_aligned = uctx->src0_row_size_aligned;
const size_t dst_row_size_aligned = uctx->dst_row_size_aligned;
const uint32_t src0_nrows = uctx->src0_nrows;
- const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows);
+ const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows);
if (src0_start_row >= src0_end_row) {
return;
@@ -1363,9 +1363,9 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo
const uint32_t nb12 = src1->nb[2];
const uint32_t nb13 = src1->nb[3];
- uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread);
- uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_thread)) : NULL;
- uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread);
+ uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx);
+ uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_hvx)) : NULL;
+ uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx);
const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size;
const size_t src1_vtcm_half_size = uctx->src1_vtcm_half_size;
@@ -1380,7 +1380,7 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo
const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02;
const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
if (uctx->broadcast_weight) {
dma_queue_push(dma_q, dma_make_data(src1_vtcm_data, data_src1),
@@ -1464,25 +1464,25 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo
}
// 2b. RMS_NORM_MUL_ADD row-block task with broadcast weight and residual add.
-static void unary_thread_rms_norm_mul_add_f32(unsigned int nth, unsigned int ith, void * data) {
+static void unary_thread_rms_norm_mul_add_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
const struct htp_unary_context * uctx = (const struct htp_unary_context *) data;
struct htp_ops_context * octx = uctx->octx;
const struct htp_tensor * src = octx->src[0];
const struct htp_tensor * dst = octx->dst;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_unary_preamble;
- const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread;
+ const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx;
const size_t src0_data_row_size = uctx->src0_data_row_size;
const size_t dst_data_row_size = uctx->dst_data_row_size;
const size_t src0_row_size_aligned = uctx->src0_row_size_aligned;
const size_t dst_row_size_aligned = uctx->dst_row_size_aligned;
const uint32_t src0_nrows = uctx->src0_nrows;
- const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows);
+ const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows);
if (src0_start_row >= src0_end_row) {
return;
@@ -1493,10 +1493,10 @@ static void unary_thread_rms_norm_mul_add_f32(unsigned int nth, unsigned int ith
const dma_addr_t data_src2 = uctx->data_src2;
const dma_addr_t data_dst = uctx->data_dst;
- uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread);
- uint8_t * src1_vtcm_data = uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_thread);
- uint8_t * src2_vtcm_data = uctx->vtcm_src2 + (ith * uctx->vtcm_src2_size_per_thread);
- uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread);
+ uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx);
+ uint8_t * src1_vtcm_data = uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_hvx);
+ uint8_t * src2_vtcm_data = uctx->vtcm_src2 + (ith * uctx->vtcm_src2_size_per_hvx);
+ uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx);
const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size;
const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size;
@@ -1508,7 +1508,7 @@ static void unary_thread_rms_norm_mul_add_f32(unsigned int nth, unsigned int ith
const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02;
const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
dma_queue_push(dma_q, dma_make_data(src1_vtcm_data, data_src1),
uctx->src1_row_size_aligned, 0, uctx->src1_data_row_size, 1);
@@ -1576,25 +1576,25 @@ static void unary_thread_rms_norm_mul_add_f32(unsigned int nth, unsigned int ith
}
// 3. TRI row-block task with row index ir.
-static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data) {
+static void unary_thread_tri_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
const struct htp_unary_context * uctx = (const struct htp_unary_context *) data;
struct htp_ops_context * octx = uctx->octx;
const struct htp_tensor * src = octx->src[0];
const struct htp_tensor * dst = octx->dst;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_unary_preamble;
- const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread;
+ const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx;
const size_t src0_data_row_size = uctx->src0_data_row_size;
const size_t dst_data_row_size = uctx->dst_data_row_size;
const size_t src0_row_size_aligned = uctx->src0_row_size_aligned;
const size_t dst_row_size_aligned = uctx->dst_row_size_aligned;
const uint32_t src0_nrows = uctx->src0_nrows;
- const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows);
+ const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows);
if (src0_start_row >= src0_end_row) {
return;
@@ -1603,14 +1603,14 @@ static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data
const dma_addr_t data_src = uctx->data_src0;
const dma_addr_t data_dst = uctx->data_dst;
- uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread);
- uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread);
+ uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx);
+ uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx);
const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size;
const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size;
const uint32_t BLOCK = uctx->block;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) {
const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
@@ -1663,22 +1663,22 @@ static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data
}
// 4. Pointwise tiled unary task.
-static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data) {
+static void unary_thread_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
const struct htp_unary_context * uctx = (const struct htp_unary_context *) data;
struct htp_ops_context * octx = uctx->octx;
const struct htp_tensor * src = octx->src[0];
const struct htp_tensor * dst = octx->dst;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_unary_preamble;
- const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread;
+ const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx;
const uint32_t col_tile = uctx->col_tile;
const uint32_t src0_nrows = uctx->src0_nrows;
- const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows);
+ const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows);
if (src0_start_row >= src0_end_row) {
return;
@@ -1687,13 +1687,13 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data)
const dma_addr_t data_src = uctx->data_src0;
const dma_addr_t data_dst = uctx->data_dst;
- uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread);
- uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread);
+ uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx);
+ uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx);
const size_t src0_half = uctx->src0_vtcm_half_size;
const size_t dst_half = uctx->dst_vtcm_half_size;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01;
const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02;
@@ -1774,23 +1774,23 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data)
}
// 5. TRI tiled task.
-static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, void * data) {
+static void unary_thread_tiled_tri_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
(void) nth;
const struct htp_unary_context * uctx = (const struct htp_unary_context *) data;
struct htp_ops_context * octx = uctx->octx;
const struct htp_tensor * src = octx->src[0];
const struct htp_tensor * dst = octx->dst;
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ struct htp_thread_trace * tr = th->trace;
htp_unary_preamble;
- const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread;
+ const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx;
const int32_t * op_params = octx->op_params;
const uint32_t col_tile = uctx->col_tile;
const uint32_t src0_nrows = uctx->src0_nrows;
- const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows);
+ const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows);
if (src0_start_row >= src0_end_row) {
return;
@@ -1799,13 +1799,13 @@ static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, void
const dma_addr_t data_src = uctx->data_src0;
const dma_addr_t data_dst = uctx->data_dst;
- uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread);
- uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread);
+ uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx);
+ uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx);
const size_t src0_half = uctx->src0_vtcm_half_size;
const size_t dst_half = uctx->dst_vtcm_half_size;
- dma_queue * dma_q = octx->ctx->dma[ith];
+ dma_queue * dma_q = th->dma_q;
const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01;
const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr;
@@ -1928,7 +1928,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params;
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -1953,7 +1953,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_hvx = octx->n_hvx;
// Always 0 for F16 - htp_unary_solve_layout() keeps F16 on the row-block path,
// since only F32 has unary_task_f32_tiled_* kernels.
@@ -1984,17 +1984,17 @@ static int execute_op_unary(struct htp_ops_context * octx) {
struct htp_unary_vtcm_layout L;
htp_unary_vtcm_layout_build(&L, octx->op, src0_data_row_size, dst_data_row_size, src1_data_row_size,
- src0_elem_size, col_tile, kparams->block, broadcast_weight, kparams->n_threads);
+ src0_elem_size, col_tile, kparams->block, broadcast_weight, kparams->n_hvx);
FARF(HIGH, "%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type,
src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
- L.src0_bytes * n_threads, L.src1_bytes * n_threads, L.dst_bytes * n_threads);
+ L.src0_bytes * n_hvx, L.src1_bytes * n_hvx, L.dst_bytes * n_hvx);
uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;
struct htp_unary_context uctx = {
.octx = octx,
.kparams = kparams,
- .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
+ .src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div),
.src0_nrows = nrows,
.row_start = row_start,
@@ -2025,15 +2025,15 @@ static int execute_op_unary(struct htp_ops_context * octx) {
.vtcm_src2 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_src2, L.src2_bytes > 0),
.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst),
- .vtcm_src0_size_per_thread = L.src0_bytes,
- .vtcm_src1_size_per_thread = L.src1_bytes,
- .vtcm_src2_size_per_thread = L.src2_bytes,
- .vtcm_dst_size_per_thread = L.dst_bytes,
+ .vtcm_src0_size_per_hvx = L.src0_bytes,
+ .vtcm_src1_size_per_hvx = L.src1_bytes,
+ .vtcm_src2_size_per_hvx = L.src2_bytes,
+ .vtcm_dst_size_per_hvx = L.dst_bytes,
};
FARF(HIGH, "%s: %s mode (col_tile %u)\n", op_type, col_tile ? "tiled" : "row-block", col_tile);
- worker_callback_t task_func = NULL;
+ work_queue_func_t task_func = NULL;
void * compute_func = NULL;
if (col_tile) {
@@ -2132,7 +2132,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
}
uctx.compute = compute_func;
- work_queue_run(octx->ctx->work_queue, task_func, &uctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, task_func, &uctx, n_hvx);
return err;
}
diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h
index 8699a1403..e03b2ac73 100644
--- a/ggml/src/ggml-hexagon/htp/unary-ops.h
+++ b/ggml/src/ggml-hexagon/htp/unary-ops.h
@@ -8,7 +8,7 @@
// Op-specific struct for precomputed unary params
struct htp_unary_kernel_params {
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t col_tile;
uint32_t block;
uint32_t broadcast_weight;
@@ -112,7 +112,7 @@ static inline void htp_unary_vtcm_layout_build(
uint32_t col_tile,
uint32_t block,
bool broadcast_weight,
- uint32_t n_threads
+ uint32_t n_hvx
) {
const bool has_weight = (op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_RMS_NORM_MUL_ADD);
@@ -138,10 +138,10 @@ static inline void htp_unary_vtcm_layout_build(
}
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes * n_threads);
- VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes * n_threads, L->src1_bytes > 0);
- VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src2, L->src2_bytes * n_threads, L->src2_bytes > 0);
- VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes * n_hvx);
+ VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes * n_hvx, L->src1_bytes > 0);
+ VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src2, L->src2_bytes * n_hvx, L->src2_bytes > 0);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes * n_hvx);
L->total_bytes = off;
}
@@ -153,11 +153,11 @@ static inline void htp_unary_solve_layout(
uint32_t ne11,
uint32_t max_block,
bool broadcast_weight,
- uint32_t n_threads,
+ uint32_t n_hvx,
size_t vtcm_size,
size_t elem_size,
uint32_t * out_col_tile,
- uint32_t * out_vtcm_row_per_thread
+ uint32_t * out_vtcm_row_per_hvx
) {
const bool has_weight = (op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_RMS_NORM_MUL_ADD);
@@ -171,35 +171,35 @@ static inline void htp_unary_solve_layout(
const size_t src1_row_size_aligned = has_weight ? hex_round_up(src1_data_row_size, 128) : 0;
size_t vtcm_size_per_row = 0;
- size_t vtcm_row_per_thread = 0;
+ size_t vtcm_row_per_hvx = 0;
if (op == HTP_OP_RMS_NORM_MUL_ADD) {
- size_t src1_vtcm_total = n_threads * src1_row_size_aligned;
+ size_t src1_vtcm_total = n_hvx * src1_row_size_aligned;
size_t available_vtcm = (vtcm_size > src1_vtcm_total) ? (vtcm_size - src1_vtcm_total) : 0;
vtcm_size_per_row = 2 * (2 * src0_row_size_aligned + dst_row_size_aligned);
- vtcm_row_per_thread = available_vtcm / (n_threads * vtcm_size_per_row);
+ vtcm_row_per_hvx = available_vtcm / (n_hvx * vtcm_size_per_row);
} else if (op == HTP_OP_RMS_NORM_MUL) {
if (broadcast_weight) {
size_t available_vtcm = vtcm_size;
- size_t src1_vtcm_total = n_threads * src1_row_size_aligned;
+ size_t src1_vtcm_total = n_hvx * src1_row_size_aligned;
if (available_vtcm > src1_vtcm_total) {
available_vtcm -= src1_vtcm_total;
} else {
available_vtcm = 0;
}
vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned);
- vtcm_row_per_thread = available_vtcm / (n_threads * vtcm_size_per_row);
+ vtcm_row_per_hvx = available_vtcm / (n_hvx * vtcm_size_per_row);
} else {
vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned + src1_row_size_aligned);
- vtcm_row_per_thread = vtcm_size / (n_threads * vtcm_size_per_row);
+ vtcm_row_per_hvx = vtcm_size / (n_hvx * vtcm_size_per_row);
}
} else {
vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned);
- vtcm_row_per_thread = vtcm_size / (n_threads * vtcm_size_per_row);
+ vtcm_row_per_hvx = vtcm_size / (n_hvx * vtcm_size_per_row);
}
- if (max_block > 0 && vtcm_row_per_thread > max_block) {
- vtcm_row_per_thread = max_block;
+ if (max_block > 0 && vtcm_row_per_hvx > max_block) {
+ vtcm_row_per_hvx = max_block;
}
const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM ||
@@ -207,25 +207,25 @@ static inline void htp_unary_solve_layout(
op == HTP_OP_L2_NORM || op == HTP_OP_HADAMARD);
// The tiled fallback path below only has F32 task functions (unary_task_f32_tiled_*);
// F16 has no tiled kernels, so it must stay on the row-block path like reduction ops.
- // NOTE: if F16 ends up with vtcm_row_per_thread == 0 here (row too large for the VTCM
+ // NOTE: if F16 ends up with vtcm_row_per_hvx == 0 here (row too large for the VTCM
// budget), execute_op_unary() will see BLOCK == 0 and skip computation for that op
// (logged via FARF(ERROR, ...)) since there is no F16 tiled fallback. This is a known
// limitation; supporting it would require adding F16 tiled kernels.
const bool is_f16 = (elem_size == sizeof(_Float16));
uint32_t col_tile = 0;
- if (vtcm_row_per_thread == 0 && !is_reduction && !is_f16) {
- const size_t per_thread_budget = vtcm_size / n_threads;
- const size_t col_tile_bytes = hex_align_down(per_thread_budget / 4, 128);
+ if (vtcm_row_per_hvx == 0 && !is_reduction && !is_f16) {
+ const size_t per_hvx_budget = vtcm_size / n_hvx;
+ const size_t col_tile_bytes = hex_align_down(per_hvx_budget / 4, 128);
col_tile = (uint32_t) (col_tile_bytes / elem_size);
}
- const uint32_t block = col_tile ? 0 : (uint32_t) vtcm_row_per_thread;
+ const uint32_t block = col_tile ? 0 : (uint32_t) vtcm_row_per_hvx;
htp_unary_vtcm_layout_build(L, op, src0_data_row_size, dst_data_row_size, src1_data_row_size,
- elem_size, col_tile, block, broadcast_weight, n_threads);
+ elem_size, col_tile, block, broadcast_weight, n_hvx);
*out_col_tile = col_tile;
- *out_vtcm_row_per_thread = (uint32_t) vtcm_row_per_thread;
+ *out_vtcm_row_per_hvx = (uint32_t) vtcm_row_per_hvx;
}
#endif /* HTP_UNARY_OPS_H */
diff --git a/ggml/src/ggml-hexagon/htp/upscale-ops.c b/ggml/src/ggml-hexagon/htp/upscale-ops.c
index 62070cf5d..59ab7914a 100644
--- a/ggml/src/ggml-hexagon/htp/upscale-ops.c
+++ b/ggml/src/ggml-hexagon/htp/upscale-ops.c
@@ -6,7 +6,7 @@
#include "hex-common.h"
#include "dma-queue.h"
-#include "hex-profile.h"
+#include "htp-trace.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
@@ -306,7 +306,7 @@ struct upscale_context {
const void * lut_y;
uint32_t chunk_start;
uint32_t chunk_count;
- uint32_t chunks_per_thread;
+ uint32_t chunks_per_hvx;
};
static void compute_nearest_chunk(
@@ -756,31 +756,31 @@ static inline void dma_push_src_chunk(
}
}
-static void upscale_worker(unsigned int nth, unsigned int ith, void * data) {
+static void upscale_worker(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) {
struct upscale_context * ctx = (struct upscale_context *) data;
const struct htp_upscale_kernel_params * p = ctx->kparams;
const struct htp_upscale_vtcm_layout * L = &ctx->layout;
const struct htp_tensor * src0 = ctx->octx->src[0];
const struct htp_tensor * dst = ctx->octx->dst;
- const uint32_t first = ctx->chunk_start + ith * ctx->chunks_per_thread;
- const uint32_t last = MIN(first + ctx->chunks_per_thread, ctx->chunk_start + ctx->chunk_count);
+ const uint32_t first = ctx->chunk_start + ith * ctx->chunks_per_hvx;
+ const uint32_t last = MIN(first + ctx->chunks_per_hvx, ctx->chunk_start + ctx->chunk_count);
if (first >= last) {
return;
}
- struct htp_thread_trace * tr = &ctx->octx->ctx->trace[ith];
- dma_queue * dma_q = ctx->octx->ctx->dma[ith];
+ struct htp_thread_trace * tr = th->trace;
+ dma_queue * dma_q = th->dma_q;
const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size;
const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size;
uint8_t * vtcm_base = (uint8_t *) ctx->octx->ctx->vtcm_base;
- uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread;
- uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread;
- float * R_buf = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_r) + ith * (L->r_bytes_per_thread / sizeof(float));
- float * gather_tmp = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_gather) + ith * (L->gather_bytes_per_thread / sizeof(float));
+ uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_hvx;
+ uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_hvx;
+ float * R_buf = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_r) + ith * (L->r_bytes_per_hvx / sizeof(float));
+ float * gather_tmp = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_gather) + ith * (L->gather_bytes_per_hvx / sizeof(float));
const void * lut_y = ctx->lut_y;
float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) };
@@ -866,12 +866,12 @@ int op_upscale(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = MIN(p->n_threads, chunk_count);
- if (!htp_ops_context_set_n_threads(octx, n_threads)) {
+ const uint32_t n_hvx = MIN(p->n_hvx, chunk_count);
+ if (!htp_ops_context_set_n_hvx(octx, n_hvx)) {
return HTP_STATUS_INVAL_PARAMS;
}
- const uint32_t chunks_per_thread = fastdiv(chunk_count + n_threads - 1, &octx->n_threads_div);
+ const uint32_t chunks_per_hvx = fastdiv(chunk_count + n_hvx - 1, &octx->n_hvx_div);
size_t lut_x_size = 0;
size_t elem_y_sz = sizeof(uint32_t);
@@ -946,13 +946,13 @@ int op_upscale(struct htp_ops_context * octx) {
.lut_y = lut_y,
.chunk_start = chunk_start,
.chunk_count = chunk_count,
- .chunks_per_thread = chunks_per_thread,
+ .chunks_per_hvx = chunks_per_hvx,
};
htp_upscale_vtcm_layout_build(&ctx.layout, src_x, src_y, dst_x, dst_y,
- p->dst_nrows_per_chunk, p->inv_sf1, p->filter_radius1, n_threads);
+ p->dst_nrows_per_chunk, p->inv_sf1, p->filter_radius1, n_hvx);
- work_queue_run(octx->ctx->work_queue, upscale_worker, &ctx, n_threads);
+ work_queue_run(octx->ctx->hvx_queue, upscale_worker, &ctx, n_hvx);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/upscale-ops.h b/ggml/src/ggml-hexagon/htp/upscale-ops.h
index aff7854b5..592bf7f00 100644
--- a/ggml/src/ggml-hexagon/htp/upscale-ops.h
+++ b/ggml/src/ggml-hexagon/htp/upscale-ops.h
@@ -43,7 +43,7 @@ struct htp_upscale_kernel_params {
float invscale0;
float invscale1;
- uint32_t n_threads;
+ uint32_t n_hvx;
uint32_t chunks_per_group;
uint32_t dst_nrows_per_chunk;
@@ -103,10 +103,10 @@ struct htp_upscale_vtcm_layout {
size_t off_dst;
size_t off_r;
size_t off_gather;
- size_t src_bytes_per_thread;
- size_t dst_bytes_per_thread;
- size_t r_bytes_per_thread;
- size_t gather_bytes_per_thread;
+ size_t src_bytes_per_hvx;
+ size_t dst_bytes_per_hvx;
+ size_t r_bytes_per_hvx;
+ size_t gather_bytes_per_hvx;
size_t src_spad_half_size;
size_t dst_spad_half_size;
uint32_t max_src_chunk_rows;
@@ -122,7 +122,7 @@ static inline void htp_upscale_vtcm_layout_build(
uint32_t dst_chunk_rows,
float inv_sf1,
float filter_radius1,
- uint32_t n_threads
+ uint32_t n_hvx
) {
uint32_t max_src_rows;
if (dst_chunk_rows >= dst_y) {
@@ -142,16 +142,16 @@ static inline void htp_upscale_vtcm_layout_build(
L->dst_chunk_rows = dst_chunk_rows;
L->src_spad_half_size = hex_round_up(src_chunk_bytes + 256, 128);
L->dst_spad_half_size = hex_round_up(dst_chunk_bytes, 128);
- L->src_bytes_per_thread = 2 * L->src_spad_half_size;
- L->dst_bytes_per_thread = 2 * L->dst_spad_half_size;
- L->r_bytes_per_thread = hex_round_up(src_x * sizeof(float), 128);
- L->gather_bytes_per_thread = 256;
+ L->src_bytes_per_hvx = 2 * L->src_spad_half_size;
+ L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size;
+ L->r_bytes_per_hvx = hex_round_up(src_x * sizeof(float), 128);
+ L->gather_bytes_per_hvx = 256;
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_r, L->r_bytes_per_thread * n_threads);
- VTCM_LAYOUT_ALLOC(off, off_gather, L->gather_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_r, L->r_bytes_per_hvx * n_hvx);
+ VTCM_LAYOUT_ALLOC(off, off_gather, L->gather_bytes_per_hvx * n_hvx);
L->total_bytes = off;
}
@@ -163,13 +163,13 @@ static inline bool htp_upscale_solve_layout(
uint32_t dst_y,
float inv_sf1,
float filter_radius1,
- uint32_t n_threads,
+ uint32_t n_hvx,
size_t vtcm_budget,
uint32_t * out_dst_chunk_rows
) {
// 1. Try full plane
uint32_t chunk_rows = dst_y;
- htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads);
+ htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_hvx);
if (layout->total_bytes <= vtcm_budget) {
*out_dst_chunk_rows = chunk_rows;
return true;
@@ -179,7 +179,7 @@ static inline bool htp_upscale_solve_layout(
chunk_rows = 64;
while (chunk_rows > 1) {
if (chunk_rows < dst_y) {
- htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads);
+ htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_hvx);
if (layout->total_bytes <= vtcm_budget) {
*out_dst_chunk_rows = chunk_rows;
return true;
@@ -190,7 +190,7 @@ static inline bool htp_upscale_solve_layout(
// 3. Fallback to 1 row
chunk_rows = 1;
- htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads);
+ htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_hvx);
if (layout->total_bytes <= vtcm_budget) {
*out_dst_chunk_rows = chunk_rows;
return true;
diff --git a/ggml/src/ggml-hexagon/htp/work-queue.c b/ggml/src/ggml-hexagon/htp/work-queue.c
index 6c74445f7..5d274274a 100644
--- a/ggml/src/ggml-hexagon/htp/work-queue.c
+++ b/ggml/src/ggml-hexagon/htp/work-queue.c
@@ -16,8 +16,8 @@
// internal structure kept in thread-local storage per instance of work queue
typedef struct {
- work_queue_t queue;
- unsigned int id;
+ htp_thread_t th;
+ work_queue_t queue;
} worker_context_t;
struct work_queue_task_s {
@@ -35,15 +35,15 @@ struct work_queue_s {
uint32_t idx_mask;
uint32_t capacity;
- qurt_thread_t thread[WORK_QUEUE_MAX_N_THREADS]; // thread ID's of the workers
- worker_context_t context[WORK_QUEUE_MAX_N_THREADS]; // worker contexts
- void * stack[WORK_QUEUE_MAX_N_THREADS]; // thread stack pointers
- unsigned int n_threads; // total threads (workers + main)
- unsigned int n_workers; // number of active threads (just workers)
+ qurt_thread_t thread[HTP_MAX_NTHREADS]; // thread ID's of the workers
+ worker_context_t context[HTP_MAX_NTHREADS]; // worker contexts (0 = main, 1..n_workers = workers)
+ void * stack[HTP_MAX_NTHREADS]; // thread stack pointers
+ unsigned int n_threads; // total threads (workers + main)
+ unsigned int n_workers; // number of active threads (just workers)
- atomic_uint active; // workers are polling/active
- atomic_bool killed; // threads need to exit
- bool external_mem; // memory owned externally
+ atomic_uint active; // workers are polling/active
+ atomic_bool killed; // threads need to exit
+ bool external_mem; // memory owned externally
struct work_queue_task_s queue[] __attribute__((aligned(HEX_L2_LINE_SIZE)));
};
@@ -51,8 +51,9 @@ struct work_queue_s {
static void work_queue_thread(void * context) {
worker_context_t * me = (worker_context_t *) context;
work_queue_t q = me->queue;
+ unsigned int id = (unsigned int) (me - q->context);
- FARF(HIGH, "work-queue: thread %u started", me->id);
+ FARF(HIGH, "work-queue: thread %u (tid %u) started", id, me->th.tid);
unsigned int prev_seqn = 0;
@@ -62,6 +63,7 @@ static void work_queue_thread(void * context) {
if (atomic_load_explicit(&q->active, memory_order_relaxed)) {
hex_pause();
} else {
+ qurt_hvx_unlock();
qurt_futex_wait(&q->active, 0);
}
continue;
@@ -77,9 +79,8 @@ static void work_queue_thread(void * context) {
struct work_queue_task_s * task = &q->queue[ir];
unsigned int n = task->n_threads;
- unsigned int i = me->id;
- if (i < n) {
- task->func(n, i, task->data);
+ if (id < n) {
+ task->func(&me->th, n, id, task->data);
atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release);
} else {
@@ -92,10 +93,15 @@ static void work_queue_thread(void * context) {
}
}
- FARF(HIGH, "work-queue: thread %u stopped", me->id);
+ FARF(HIGH, "work-queue: thread %u (tid %u) stopped", id, me->th.tid);
}
-bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) {
+bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) {
+ if (n <= 1) {
+ func(&q->context[0].th, n, 0, data);
+ return true;
+ }
+
if (n > q->n_threads) {
FARF(ERROR, "work-queue: invalid number of jobs %u for n-threads %u", n, q->n_threads);
return false;
@@ -121,7 +127,7 @@ bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, u
atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release);
// main thread runs job #0
- func(n, 0, data);
+ func(&q->context[0].th, n, 0, data);
atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release);
@@ -148,7 +154,7 @@ size_t work_queue_alignof(void) {
return 4096;
}
-work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size) {
+work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size, dma_queue_t * dma, struct htp_thread_trace * trace) {
capacity = hex_ceil_pow2(capacity);
uint32_t n_workers = n_threads > 1 ? n_threads - 1 : 0;
unsigned char * mem_blob = (unsigned char *) ptr;
@@ -161,11 +167,14 @@ work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity,
q->external_mem = true;
q->capacity = capacity;
+ for (unsigned int i = 0; i < n_threads; i++) {
+ htp_thread_init(&q->context[i].th, dma, trace);
+ q->context[i].queue = q;
+ }
+
for (unsigned int i = 0; i < n_workers; i++) {
q->stack[i] = mem_blob; mem_blob += stack_size;
q->thread[i] = 0;
- q->context[i].id = i + 1;
- q->context[i].queue = q;
}
atomic_init(&q->idx_read, 0);
@@ -190,7 +199,7 @@ work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity,
qurt_thread_attr_set_stack_size(&attr, stack_size);
char thread_name[32];
- snprintf(thread_name, sizeof(thread_name), "work-queue:%u", i);
+ snprintf(thread_name, sizeof(thread_name), "work-queue:%u", i + 1);
qurt_thread_attr_set_name(&attr, thread_name);
// set up priority - by default, match the creating thread's prio
@@ -204,7 +213,7 @@ work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity,
qurt_thread_attr_set_priority(&attr, prio);
- int err = qurt_thread_create(&q->thread[i], &attr, work_queue_thread, (void *) &q->context[i]);
+ int err = qurt_thread_create(&q->thread[i], &attr, work_queue_thread, (void *) &q->context[i + 1]);
if (err) {
FARF(ERROR, "Could not launch worker threads!");
work_queue_free(q);
diff --git a/ggml/src/ggml-hexagon/htp/work-queue.h b/ggml/src/ggml-hexagon/htp/work-queue.h
index 09ca4b1f4..446b72c46 100644
--- a/ggml/src/ggml-hexagon/htp/work-queue.h
+++ b/ggml/src/ggml-hexagon/htp/work-queue.h
@@ -5,34 +5,21 @@
#include <stdint.h>
#include <stddef.h>
-typedef void (*work_queue_func_t)(unsigned int n, unsigned int i, void *);
+#include "htp-thread.h"
+
+typedef void (*work_queue_func_t)(const htp_thread_t * th, unsigned int n, unsigned int i, void *);
struct work_queue_s;
typedef struct work_queue_s * work_queue_t;
-#define WORK_QUEUE_MAX_N_THREADS 10
-
size_t work_queue_sizeof(uint32_t n_threads, uint32_t capacity, uint32_t stack_size);
size_t work_queue_alignof(void);
-work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size);
+work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size, dma_queue_t * dma, struct htp_thread_trace * trace);
void work_queue_free(work_queue_t q);
void work_queue_wakeup(work_queue_t q);
void work_queue_suspend(work_queue_t q);
-bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n);
-
-static inline bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) {
- if (n <= 1) {
- func(n, 0, data);
- return true;
- }
- return work_queue_run_async(q, func, data, n);
-}
-
-// Legacy compatibility
-typedef work_queue_func_t worker_callback_t;
-#define worker_pool_run_func work_queue_run
-#define worker_pool work_queue
+bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n);
#endif // #ifndef HTP_WORK_QUEUE_H
diff --git a/scripts/snapdragon/build.py b/scripts/snapdragon/build.py
index 05e515167..1bf35b08b 100755
--- a/scripts/snapdragon/build.py
+++ b/scripts/snapdragon/build.py
@@ -106,6 +106,7 @@ def main():
parser.add_argument("--no-docker", action="store_true", help="Build natively on the host instead of in a docker container")
parser.add_argument("--preset", help="Override the CMake preset to use")
parser.add_argument("--debug", action="store_true", help="Build in debug mode (uses -debug presets instead of -release)")
+ parser.add_argument("--hex-debug", "--hexagon-debug", action="store_true", help="Enable Hexagon HTP debug output (sets GGML_HEXAGON_HTP_DEBUG=ON)")
parser.add_argument("--hex-only", "--hexagon-only", action="store_true", help="Only build and push ggml-hexagon and libggml-htp libraries")
parser.add_argument("--hex-arch", choices=["v73", "v75", "v79", "v81", "v85", "73", "75", "79", "81", "85"], help="Specific Hexagon DSP architecture to build with --hex-only (default: all)")
@@ -197,13 +198,15 @@ def main():
else:
hex_targets = ["ggml-hexagon", "htp-v73", "htp-v75", "htp-v79", "htp-v81", "htp-v85"]
+ hex_debug_arg = f"-DGGML_HEXAGON_HTP_DEBUG={'ON' if args.hex_debug else 'OFF'}"
+
if args.no_docker:
# Native/local host build
logger.info("Running native/local CMake build...")
install_prefix = os.path.join(repo_root, install_dir, "llama.cpp")
# Configure
- configure_cmd = ["cmake", f"--preset={preset}", "-B", build_dir]
+ configure_cmd = ["cmake", f"--preset={preset}", hex_debug_arg, "-B", build_dir]
logger.info(f"+ {' '.join(configure_cmd)}")
res = subprocess.run(configure_cmd, cwd=repo_root)
if res.returncode != 0:
@@ -238,7 +241,7 @@ def main():
target_args = f"--target {' '.join(hex_targets)} " if args.hex_only else ""
build_sh_cmd = (
- f"cmake --preset {preset} -B /workspace/{build_dir} && "
+ f"cmake --preset {preset} {hex_debug_arg} -B /workspace/{build_dir} && "
f"cmake --build /workspace/{build_dir} {target_args}-j {jobs} && "
f"cmake --install /workspace/{build_dir} --prefix {install_prefix_container}"
)
@@ -261,6 +264,17 @@ def main():
logger.error("Docker-based build failed.")
sys.exit(res.returncode)
+ install_lib_dir = os.path.join(repo_root, install_dir, "llama.cpp", "lib")
+ if os.path.isdir(install_lib_dir):
+ if args.hex_debug:
+ farf_src = os.path.join(repo_root, "scripts", "snapdragon", "adb", "llama-cli.farf")
+ farf_apps = ["llama-server", "llama-cli", "llama-completion", "test-backend-ops", "test-llama-archs"]
+ for app in farf_apps:
+ shutil.copyfile(farf_src, os.path.join(install_lib_dir, f"{app}.farf"))
+ else:
+ for f in glob.glob(os.path.join(install_lib_dir, "*.farf")):
+ os.remove(f)
+
logger.info("\nBuild and installation completed successfully!")
# Push/deploy if requested
@@ -286,6 +300,9 @@ def main():
else:
hex_libs += glob.glob(os.path.join(lib_dir, "libggml-htp*"))
+ if args.hex_debug:
+ hex_libs += glob.glob(os.path.join(lib_dir, "*.farf"))
+
if not hex_libs:
logger.error(f"Error: no hexagon libraries found in {lib_dir} to push.")
sys.exit(1)
@@ -307,6 +324,10 @@ def main():
if res.returncode != 0:
logger.error("ADB mkdir failed.")
sys.exit(res.returncode)
+ if not args.hex_debug:
+ rm_farf_cmd = adb_cmd + ["shell", f"rm -f {shlex.quote(target_lib_dir)}/*.farf"]
+ logger.info(f"+ {' '.join(rm_farf_cmd)}")
+ subprocess.run(rm_farf_cmd)
for lib in hex_libs:
push_cmd = adb_cmd + ["push", lib, f"{target_lib_dir}/"]
logger.info(f"+ {' '.join(push_cmd)}")
@@ -383,6 +404,10 @@ def main():
if args.hex_only:
logger.info(f"\nDeploying Hexagon libraries to Linux device {ssh_host} via SCP...")
target_lib_dir = f"{target_dir}/lib"
+ if not args.hex_debug:
+ rm_farf_cmd = ["ssh", ssh_host, f"rm -f {target_lib_dir}/*.farf"]
+ logger.info(f"+ {' '.join(rm_farf_cmd)}")
+ subprocess.run(rm_farf_cmd)
deploy_cmd = ["scp"] + hex_libs + [f"{ssh_host}:{target_lib_dir}/"]
logger.info(f"+ {' '.join(deploy_cmd)}")
res = subprocess.run(deploy_cmd)
diff --git a/scripts/snapdragon/ggml-hexagon-align-macros.py b/scripts/snapdragon/ggml-hexagon-align-macros.py
index b64db3e65..dc1475c52 100755
--- a/scripts/snapdragon/ggml-hexagon-align-macros.py
+++ b/scripts/snapdragon/ggml-hexagon-align-macros.py
@@ -6,6 +6,7 @@ Usage:
align-macros.py [paths...] # Check and report misaligned macros
align-macros.py --diff [paths...] # Show unified diff of fixes
align-macros.py --fix [paths...] # Fix misaligned macros in-place
+ align-macros.py --fix --reflow false ... # Disable line reflowing in mostly-aligned macros
align-macros.py --fix --mode majority ... # Align to the dominant column
align-macros.py --fix --pad 2 ... # Align to (max_content_len + pad)
@@ -127,9 +128,227 @@ def is_macro_aligned(macro: MacroDef) -> bool:
return len(set(bs_cols)) == 1 and not has_trailing_ws
+def str_to_bool(val: str) -> bool:
+ if isinstance(val, bool):
+ return val
+ v = val.strip().lower()
+ if v in ("true", "1", "yes", "y", "on"):
+ return True
+ if v in ("false", "0", "no", "n", "off"):
+ return False
+ raise argparse.ArgumentTypeError(f"Boolean value expected, got '{val}'")
+
+
+def find_paren_comma_groups(s: str) -> List[Tuple[int, int, List[int]]]:
+ """Find parenthesized groups (open_idx, close_idx, comma_indices) ordered from outermost to innermost."""
+ stack: List[Tuple[int, List[int]]] = []
+ groups: List[Tuple[int, int, List[int]]] = []
+ in_str: Optional[str] = None
+ escape = False
+
+ for idx, ch in enumerate(s):
+ if escape:
+ escape = False
+ continue
+ if ch == "\\":
+ escape = True
+ continue
+ if in_str is not None:
+ if ch == in_str:
+ in_str = None
+ continue
+ if ch in ('"', "'"):
+ in_str = ch
+ continue
+ if ch == "(":
+ stack.append((idx, []))
+ elif ch == ")" and stack:
+ open_idx, commas = stack.pop()
+ if commas:
+ groups.append((open_idx, idx, commas))
+ elif ch == "," and stack:
+ stack[-1][1].append(idx)
+
+ groups.sort(key=lambda g: g[0])
+ return groups
+
+
+def find_top_level_op_splits(s: str) -> Tuple[Optional[int], List[int]]:
+ """Find split positions after binary operators (+, -, &&, ||) at paren depth 0."""
+ depth = 0
+ in_str: Optional[str] = None
+ escape = False
+ assign_idx: Optional[int] = None
+ splits: List[int] = []
+ n = len(s)
+ i = 0
+
+ while i < n:
+ ch = s[i]
+ if escape:
+ escape = False
+ i += 1
+ continue
+ if ch == "\\":
+ escape = True
+ i += 1
+ continue
+ if in_str is not None:
+ if ch == in_str:
+ in_str = None
+ i += 1
+ continue
+ if ch in ('"', "'"):
+ in_str = ch
+ i += 1
+ continue
+ if ch in "([{":
+ depth += 1
+ elif ch in ")]}":
+ depth = max(0, depth - 1)
+ elif depth == 0:
+ if s.startswith("&&", i) or s.startswith("||", i):
+ splits.append(i + 2)
+ i += 2
+ continue
+ if ch == "=" and (i == 0 or s[i - 1] not in "=!<>+-*/%&|^") and (i + 1 >= n or s[i + 1] != "="):
+ if assign_idx is None:
+ assign_idx = i
+ elif ch in "+-" and i > 0 and s[i - 1] == " " and i + 1 < n and s[i + 1] == " ":
+ splits.append(i + 1)
+ i += 1
+
+ return assign_idx, splits
+
+
+def reflow_c_line(content: str, max_len: int) -> Optional[List[str]]:
+ """Try to cleanly reflow a C statement/signature line so each piece has len <= max_len."""
+ if len(content) <= max_len:
+ return [content]
+
+ stripped = content.lstrip()
+ if not stripped or stripped.startswith("#") or "//" in content or "/*" in content:
+ return None
+
+ base_indent = content[:len(content) - len(stripped)]
+
+ # 1. Try splitting comma-separated arguments/parameters inside (...)
+ for open_idx, _, commas in find_paren_comma_groups(content):
+ pts = [0] + [c + 1 for c in commas] + [len(content)]
+ segs: List[str] = []
+ for idx in range(len(pts) - 1):
+ seg = content[pts[idx]:pts[idx + 1]].rstrip()
+ if idx > 0:
+ seg = seg.lstrip()
+ segs.append(seg)
+
+ for cont_indent in [" " * (open_idx + 1), base_indent + " ", base_indent + " "]:
+ lines = [segs[0]]
+ for seg in segs[1:]:
+ cand = f"{lines[-1]} {seg}"
+ if len(cand) <= max_len:
+ lines[-1] = cand
+ else:
+ lines.append(f"{cont_indent}{seg}")
+ if len(lines) > 1 and all(len(line) <= max_len for line in lines):
+ return lines
+
+ # 2. Try splitting after top-level binary operators (+, -, &&, ||) or assignment (=)
+ assign_idx, op_splits = find_top_level_op_splits(content)
+ if not op_splits and assign_idx is not None:
+ op_splits = [assign_idx + 1]
+ if op_splits:
+ pts = [0] + op_splits + [len(content)]
+ segs = []
+ for idx in range(len(pts) - 1):
+ seg = content[pts[idx]:pts[idx + 1]].rstrip()
+ if idx > 0:
+ seg = seg.lstrip()
+ segs.append(seg)
+
+ indents = []
+ if assign_idx is not None:
+ rhs_start = assign_idx + 1
+ while rhs_start < len(content) and content[rhs_start] == " ":
+ rhs_start += 1
+ indents.append(" " * rhs_start)
+ indents.extend([base_indent + " ", base_indent + " "])
+
+ for cont_indent in indents:
+ lines = [segs[0]]
+ for seg in segs[1:]:
+ cand = f"{lines[-1]} {seg}"
+ if len(cand) <= max_len:
+ lines[-1] = cand
+ else:
+ lines.append(f"{cont_indent}{seg}")
+ if len(lines) > 1 and all(len(line) <= max_len for line in lines):
+ return lines
+
+ return None
+
+
+def reflow_macro_if_needed(
+ macro: MacroDef,
+ pad: int,
+ dom_ratio_threshold: float = 0.80,
+ fit_ratio_threshold: float = 0.90,
+) -> Tuple[MacroDef, Optional[int]]:
+ """
+ If a macro is already mostly aligned at a dominant column (>= dom_ratio_threshold)
+ and >= fit_ratio_threshold of continuation lines fit within that column, reflow any
+ exceeding lines so the macro can keep its dominant column instead of shifting all lines.
+ """
+ cont_lines = [ml for ml in macro.lines if ml.bs_col is not None]
+ if not cont_lines:
+ return macro, None
+
+ counts = Counter(ml.bs_col for ml in cont_lines if ml.bs_col is not None)
+ dom_col, dom_count = sorted(counts.items(), key=lambda x: (-x[1], -x[0]))[0]
+ max_allowed_len = dom_col - pad
+ if max_allowed_len <= 0:
+ return macro, None
+
+ dom_ratio = dom_count / len(cont_lines)
+ fit_count = sum(1 for ml in cont_lines if len(ml.content) <= max_allowed_len)
+ fit_ratio = fit_count / len(cont_lines)
+
+ effective_dom_threshold = 0.70 if fit_count == len(cont_lines) else dom_ratio_threshold
+ if dom_ratio < effective_dom_threshold or fit_ratio < fit_ratio_threshold:
+ return macro, None
+
+ new_macro_lines: List[MacroLine] = []
+ for ml in macro.lines:
+ if ml.bs_col is None or len(ml.content) <= max_allowed_len:
+ new_macro_lines.append(ml)
+ else:
+ reflowed = reflow_c_line(ml.content, max_allowed_len)
+ if not reflowed:
+ return macro, None
+ nl = "\r\n" if ml.raw.endswith("\r\n") else "\n"
+ for piece in reflowed:
+ new_macro_lines.append(MacroLine(
+ line_num=ml.line_num,
+ raw=f"{piece} \\{nl}",
+ content=piece,
+ bs_col=dom_col,
+ trailing_ws=False,
+ ))
+
+ return MacroDef(
+ name=macro.name,
+ filepath=macro.filepath,
+ start_line=macro.start_line,
+ end_line=macro.end_line,
+ lines=new_macro_lines,
+ ), dom_col
+
+
def compute_target_column(macro: MacroDef, mode: str, pad: int, target_col: Optional[int]) -> int:
"""Determine the column where backslashes should be aligned."""
- max_content_len = max(len(ml.content) for ml in macro.lines)
+ cont_lines = [ml for ml in macro.lines if ml.bs_col is not None]
+ lines_for_len = cont_lines if cont_lines else macro.lines
+ max_content_len = max(len(ml.content) for ml in lines_for_len)
min_needed = max_content_len + pad
if target_col is not None:
@@ -181,6 +400,7 @@ def process_file(filepath: str, args: argparse.Namespace) -> Tuple[int, int, Opt
misaligned_count = 0
modified = False
new_file_lines = list(file_lines)
+ line_offset = 0
for macro in macros:
aligned = is_macro_aligned(macro)
@@ -188,8 +408,9 @@ def process_file(filepath: str, args: argparse.Namespace) -> Tuple[int, int, Opt
if not aligned:
misaligned_count += 1
- bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None]
- max_content = max(len(ml.content) for ml in macro.lines)
+ cont_lines = [ml for ml in macro.lines if ml.bs_col is not None]
+ bs_cols = [ml.bs_col for ml in cont_lines if ml.bs_col is not None]
+ max_content = max(len(ml.content) for ml in (cont_lines or macro.lines))
col_counts = Counter(bs_cols)
if not args.quiet:
@@ -200,16 +421,28 @@ def process_file(filepath: str, args: argparse.Namespace) -> Tuple[int, int, Opt
if trailing_ws_lines:
logger.warning(f" Warning: Trailing whitespace after backslash on line(s): {trailing_ws_lines}")
- target_col = compute_target_column(macro, args.mode, args.pad, args.target_col)
+ reflow_col: Optional[int] = None
+ target_macro = macro
+ if args.reflow and args.target_col is None:
+ target_macro, reflow_col = reflow_macro_if_needed(macro, args.pad)
+
+ if reflow_col is not None:
+ target_col = reflow_col
+ if not args.quiet and len(target_macro.lines) != len(macro.lines):
+ logger.info(f" -> Reflowed {len(target_macro.lines) - len(macro.lines)} line(s) to preserve column {target_col}")
+ else:
+ target_col = compute_target_column(target_macro, args.mode, args.pad, args.target_col)
+
if not args.quiet:
logger.info(f" -> Target alignment column: {target_col}")
- realigned = realign_macro_lines(macro, target_col)
+ realigned = realign_macro_lines(target_macro, target_col)
- start_idx = macro.start_line - 1
+ start_idx = macro.start_line - 1 + line_offset
end_idx = start_idx + len(macro.lines)
if new_file_lines[start_idx:end_idx] != realigned:
new_file_lines[start_idx:end_idx] = realigned
+ line_offset += len(realigned) - len(macro.lines)
modified = True
diff_text = None
@@ -258,6 +491,8 @@ def main():
parser.add_argument("--check", action="store_true", help="Exit with code 1 if misaligned macros exist")
parser.add_argument("--mode", choices=["min", "max", "majority"], default="min",
help="Alignment mode: 'min' (max_len + pad), 'max' (max existing col), 'majority' (dominant col)")
+ parser.add_argument("--reflow", type=str_to_bool, nargs="?", const=True, default=True,
+ help="Reflow long lines in mostly-aligned macros instead of shifting the whole macro (default: true)")
parser.add_argument("--pad", type=int, default=2, help="Spaces between longest line and backslash (default: 2)")
parser.add_argument("--target-col", type=int, default=None, help="Force alignment to an exact column")
parser.add_argument("--all", action="store_true", help="Realign all macros even if already aligned (default: only misaligned)")
diff --git a/scripts/snapdragon/ggml-hexagon-profile.py b/scripts/snapdragon/ggml-hexagon-profile.py
index 7c95ec8cb..f6a949e98 100755
--- a/scripts/snapdragon/ggml-hexagon-profile.py
+++ b/scripts/snapdragon/ggml-hexagon-profile.py
@@ -466,10 +466,11 @@ def print_bubbles_timeline(op):
logger.info(f" Avg Thread DMA IDLE : {avg_dma_idle:.1f}%")
logger.info("-" * 100)
+ hmx_threads = {e['thread'] for e in events if e['event'].startswith('HMX')}
logger.info(" Per-Thread Idle Analysis:")
for t in active_threads:
stats = thread_stats[t]
- thread_name = f"Thread {t:<2} (HVX)" if t != 10 else "Thread 10 (HMX)"
+ thread_name = f"Thread {t:<2} (HMX)" if t in hmx_threads else f"Thread {t:<2} (HVX)"
logger.info(f" {thread_name} -> Compute Idle: {stats['compute_idle_pct']:.1f}% | DMA Idle: {stats['dma_idle_pct']:.1f}%")
all_bubbles = []
@@ -531,8 +532,9 @@ def print_ascii_summary(op_name, dims, types, usec, cycles, events):
norm_evt = normalize_event_name(evt)
thread_totals[t][norm_evt] += dur
+ hmx_threads = {e['thread'] for e in events if e['event'].startswith('HMX')}
for t in sorted(thread_totals.keys()):
- thread_name = f"Thread {t} (HVX)" if t != 10 else "Thread 10 (HMX)"
+ thread_name = f"Thread {t} (HMX)" if t in hmx_threads else f"Thread {t} (HVX)"
sorted_evts = sorted(thread_totals[t].items(), key=lambda item: item[0])
evt_strs = []
diff --git a/scripts/snapdragon/ggml-hexagon-trace.py b/scripts/snapdragon/ggml-hexagon-trace.py
index 5241dca15..308591b83 100755
--- a/scripts/snapdragon/ggml-hexagon-trace.py
+++ b/scripts/snapdragon/ggml-hexagon-trace.py
@@ -433,7 +433,7 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path):
time_mappers = {dev: DeviceTimeMapper(dev, filtered_ops) for dev in unique_devices}
# Process events
- completed_events = []
+ completed_events: List[Dict[str, Any]] = []
if trace_events:
trace_events = sorted(trace_events, key=lambda e: e['unwrapped_cycles'])
@@ -516,6 +516,9 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path):
dur_ns = tm.dur_cycles_to_ns(e['start_cyc'], e['end_cyc'] - e['start_cyc'])
e['dur_ns'] = max(dur_ns, 1)
+ # Identify HMX threads dynamically from HMX* events
+ hmx_threads = {(e['device'], e['thread']) for e in completed_events if e['event'].startswith('HMX')}
+
# Allocate slots (sub-tracks) to prevent overlaps on same virtual track
active_slots = defaultdict(list)
for e in completed_events:
@@ -528,7 +531,7 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path):
norm_evt = normalize_event_name(evt, e['info'])
if norm_evt == "DMA":
track_key = (dev, t, "DMA")
- elif t == 10:
+ elif (dev, t) in hmx_threads:
track_key = (dev, t, "HMX")
else:
track_key = (dev, t, "HVX")
@@ -552,23 +555,24 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path):
evt = e['event']
slot = e['slot']
dev = e['device']
+ is_hmx = (dev, t) in hmx_threads
norm_evt = normalize_event_name(evt, e['info'])
if norm_evt == "DMA":
track_evt = "DMA"
evt_id = 1
- elif t == 10:
+ elif is_hmx:
track_evt = "HMX"
evt_id = 3
else:
track_evt = "HVX"
evt_id = 2
- t_sort = 1 if t == 10 else t + 2
+ t_sort = 1 if is_hmx else t + 2
dev_idx = device_to_idx[dev]
# Unique UUID for each sub-track
- if t == 10:
+ if is_hmx:
uuid = dev_idx * 10000000 + 20 # HMX thread track UUID
else:
uuid = int(dev_idx * 10000000 + t_sort * 1000000 + evt_id * 1000 + slot)
@@ -601,9 +605,9 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path):
hmx_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(hmx_track_uuid, parent_uuid=proc_uuid, thread=hmx_thread_desc))
write_trace_packet_to_file(f, hmx_packet)
- # Define Thread Tracks (T0, T1, ..., T9) for this device
+ # Define Thread Tracks for this device
dev_used_tracks = {uuid: val for uuid, val in used_tracks.items() if val[0] == dev}
- unique_threads = sorted(list(set(t for (_, t, _, _) in dev_used_tracks.values() if t != 10)))
+ unique_threads = sorted(list(set(t for (_, t, _, _) in dev_used_tracks.values() if (dev, t) not in hmx_threads)))
for t in unique_threads:
thread_uuid = dev_idx * 10000000 + 10 + t
thread_name = f"T{t}"
@@ -623,7 +627,7 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path):
for uuid in sorted(used_tracks.keys()):
dev, t, evt, slot = used_tracks[uuid]
dev_idx = device_to_idx[dev]
- if t == 10:
+ if (dev, t) in hmx_threads:
continue
name = f"T{t} {evt}"
rank = 0 if evt == "HVX" else 1