Commit 643b0cdac for llama.cpp
commit 643b0cdacb4ca13762a1c833558c890207b54ae2
Author: Max Krasnyansky <maxk@qti.qualcomm.com>
Date: Thu Oct 22 21:05:44 2026 -0700
hexagon: Op Fusion refactoring, optimizations and new fusions like `BINARY_NX` (back-to-back ADDs, etc) (#30843)
* hex-fusion: support for GELU+MUL, REPEAT+GET_ROWS and BINARY_NX (back-to-back ADD/MUL/...)
* hex-fusion: factor out fusion logic into separate .h and .cpp and update all related code
* hex-fusion: allow for RMS_NORM+MUL fusions for non-contig tensors
* hex-fusion: streamline fusion eligibility checks and cover missing corner cases
* hex-fusion: streamline fusion flow
* hex-opbatch: make n_tensors limit more scalable
* hex-bufs: adjust the trace buff sizing
* hex-glu: cleanup glu-ops interfaces and add kernel_params
* hex-kparams: clean up kparams handling in unary, ssm-conv, and other ops
* hex-kparams: update the rest of the ops PAD,DIAG,... to use kparams
* hex-unary: add support for collapsing dims
* hex-ops: cleanup redundant runtime checks
* hex-ops: consistent use of kernel_type in all kparams
* hex-fusion: remove redundant check in gdn+cpy
* hex-fusion: further streamlining of fusion rules and code
* hex-fusion: cleanup naming
* hex-opfmt: cleanup op formatting
* hex-fusion: bump fusion lookahead to 64
* hex-fusion: fuse SSM_CONV+SILU
* hex-fusion: fix hvx register spills in fused ssm_conv+silu
* hex-fusion: add more GLU fusions
* hex-fusion: remove duplicate checks and use ggml-common helpers
* hex-kparams: refactor kparams into htp-kparams.h/cpp
* hex-fusion: fix binary-nx fusion regression
* hex-kparams: another pass at cleaning up kparams (removing unused, etc)
* hex-kparams: streamling nameing and move _check_ wrappers into .h
* hex-kparams: templetize htp_kparams_check_<op> to reduce code bloat
* hex-ops: de-duplicate op dispatch switches
* hex-kparams: trim down exposed kparams calls
* hex-kparams: simplify type casting of kparams structs
* hex-fence: de-dup fence tensor construction
* hex-opbatch: simplify tensor shape init/checks
* hex-mm: cleanup leftovers of the original QK defines
* hex-fa: fix stack overflow on older npu firmware
* hex-ctx: move dma_caches into ops_ctx
diff --git a/ggml/src/ggml-hexagon/CMakeLists.txt b/ggml/src/ggml-hexagon/CMakeLists.txt
index 6ab520175..9444f7321 100644
--- a/ggml/src/ggml-hexagon/CMakeLists.txt
+++ b/ggml/src/ggml-hexagon/CMakeLists.txt
@@ -51,6 +51,13 @@ endif()
set(TARGET_NAME ggml-hexagon)
ggml_add_backend_library(${TARGET_NAME}
ggml-hexagon.cpp
+ htp-common.h
+ htp-kparams.cpp
+ htp-kparams.h
+ htp-opnode.cpp
+ htp-opnode.h
+ htp-fusion.cpp
+ htp-fusion.h
htp-repack.cpp
htp-repack.h
htp-drv.cpp
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index da6dba85e..0821bce16 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -26,10 +26,6 @@
#include <cmath>
#include <initializer_list>
-#ifndef M_LOG2E
-# define M_LOG2E 1.44269504088896340736
-#endif
-
#ifdef _WIN32
# define WIN32_LEAN_AND_MEAN
# ifndef NOMINMAX
@@ -59,25 +55,13 @@
#include "ggml-impl.h"
#include "ggml-quants.h"
#include "htp-opnode.h"
-#include "htp-ops.h"
-#include "htp/matmul-ops.h"
-#include "htp/binary-ops.h"
-#include "htp/flash-attn-ops.h"
-#include "htp/unary-ops.h"
-#include "htp/get-rows-ops.h"
-#include "htp/set-rows-ops.h"
-#include "htp/softmax-ops.h"
-#include "htp/pool-ops.h"
-#include "htp/rope-ops.h"
-#include "htp/ssm-conv.h"
-#include "htp/gated-delta-net-ops.h"
-#include "htp/argsort-ops.h"
-#include "htp/concat-ops.h"
-#include "htp/cpy-ops.h"
-#include "htp/im2col-ops.h"
+#include "htp/htp-ops.h"
#include "htp_iface.h"
#include "htp-drv.h"
#include "htp-repack.h"
+#include "htp-common.h"
+#include "htp-kparams.h"
+#include "htp-fusion.h"
using intvec = std::vector<int>;
using uintvec = std::vector<unsigned int>;
@@ -85,8 +69,6 @@ using u32vec = std::vector<uint32_t>;
#define GGML_HEXAGON_MAX_SESSIONS 16
-#define GGML_HEXAGON_FENCE_SLOT_SIZE 128
-
struct ggml_hexagon_device_config {
int physical_idx = 0;
int virtual_idx = 0;
@@ -99,17 +81,17 @@ struct ggml_hexagon_device_config {
static ggml_hexagon_device_config opt_device_configs[GGML_HEXAGON_MAX_SESSIONS];
-static int opt_arch = 0; // autodetect
-static size_t opt_ndev = 1;
-static size_t opt_nhvx = 0; // use all
-static int opt_nhmx = 1; // when set, enable HMX; when 0, use HVX only
-static size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for buffer mappings
-static int opt_etm = 0;
-static int opt_verbose = 0;
-static int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu)
-static bool opt_hostbuf = false;
-static bool opt_dma64 = false;
-static int opt_power_mode = HTP_POWER_MODE_BURST; // DCVS/HMX power profile requested at session start
+int opt_arch = 0; // autodetect
+size_t opt_ndev = 1;
+size_t opt_nhvx = 0; // use all
+int opt_nhmx = 1; // when set, enable HMX; when 0, use HVX only
+size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for buffer mappings
+int opt_etm = 0;
+int opt_verbose = 0;
+int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu)
+bool opt_hostbuf = false;
+bool opt_dma64 = false;
+int opt_power_mode = HTP_POWER_MODE_BURST; // DCVS/HMX power profile requested at session start
static const std::pair<const char *, int> htp_power_mode_names[] = {
{ "low_power_saver", HTP_POWER_MODE_LOW_POWER_SAVER },
@@ -145,49 +127,31 @@ static const char * htp_power_mode_to_str(int mode) {
return "unknown";
}
-static size_t opt_mbuf_dyn = 512ul * 1024 * 1024; // max dynamic (compute) buffer size
-static size_t opt_mbuf_static = 1ul * 1024 * 1024 * 1024; // max static (weight/KV) buffer size
-static size_t opt_mbuf_total = 0; // total buffer space limit (0 = unconstrained)
+size_t opt_mbuf_dyn = 512ul * 1024 * 1024; // max dynamic (compute) buffer size
+size_t opt_mbuf_static = 1ul * 1024 * 1024 * 1024; // max static (weight/KV) buffer size
+size_t opt_mbuf_total = 0; // total buffer space limit (0 = unconstrained)
-static int opt_mm_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)
-static int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)
-static int opt_fa_head_split = 1; // 1 = partition flash_attn by KV heads in multicore (default on), 0 = token-based (original)
-static int opt_gdn_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported)
-static int opt_ar_select = 2; // 2 = fused ALLREDUCE+ADD (default), 1 = unfused ALLREDUCE, 0 = fallback to CPY+FENCE
-static int opt_ar_scatter = 1; // 1 = reduce-scatter the fused ALLREDUCE+ADD (default), 0 = full reduction
+int opt_mm_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)
+int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)
+int opt_fa_head_split = 1; // 1 = partition flash_attn by KV heads in multicore (default on), 0 = token-based (original)
+int opt_gdn_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported)
+int opt_ar_select = 2; // 2 = fused ALLREDUCE+ADD (default), 1 = unfused ALLREDUCE, 0 = fallback to CPY+FENCE
+int opt_ar_scatter = 1; // 1 = reduce-scatter the fused ALLREDUCE+ADD (default), 0 = full reduction
// Default PMU events, if profiling with PMU (mode=2) is enabled
// See https://docs.qualcomm.com/doc/80-N2040-60/topic/pmu-events.html
// https://docs.qualcomm.com/doc/80-N2040-61/topic/hvx-pmu-events.html
static u32vec opt_pmu_evt { 0x3, 0x111, 0x100, 0x105, 0x240, 0x256, 0x7D, 0x8C };
-static int opt_opbatch = 1280; // max number of ops in a batch
-static int opt_opqueue = 8; // max number of pending batches
-static int opt_optrace = 0; // trace buffer size per thread (0 means default)
-static int opt_oppoll = 0; // polling for batch completions
-static int opt_opfusion = 1; // enable/disable op fusion
-static int opt_rt_repack = 1; // runtime repacking: 0 = disabled, 1 = quantized only (default), 2 = always
-
-enum ggml_hexagon_fusion_flags {
- GGML_HEXAGON_FUSE_ALLREDUCE_ADD = (1 << 1), // 2
- GGML_HEXAGON_FUSE_RMS_NORM_MUL = (1 << 2), // 4
- GGML_HEXAGON_FUSE_MUL_MAT_ADD = (1 << 3), // 8
- GGML_HEXAGON_FUSE_MUL_MAT_NX = (1 << 4), // 16
- GGML_HEXAGON_FUSE_MUL_MAT_ID_NX = (1 << 5), // 32
- GGML_HEXAGON_FUSE_GDN_CPY = (1 << 6), // 64
-};
-
-static inline bool ggml_hexagon_is_fusion_enabled(int flag) {
- if (opt_opfusion <= 0) return false;
- if (opt_opfusion == 1) return true; // 1 enables all
- return (opt_opfusion & flag) != 0;
-}
+int opt_opbatch = 2048; // max number of ops in a batch
+int opt_opqueue = 8; // max number of pending batches
+int opt_optrace = 0; // trace buffer size per thread (0 means default)
+int opt_oppoll = 0; // polling for batch completions
+int opt_opfusion = 1; // enable/disable op fusion
+int opt_rt_repack = 1; // runtime repacking: 0 = disabled, 1 = quantized only (default), 2 = always
static std::regex* opt_opfilter = NULL; // regex of ops to not claim
-#define HEX_VERBOSE(...) \
- if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__)
-
static const char * status_to_str(uint32_t status) {
switch (status) {
case HTP_STATUS_OK:
@@ -207,20 +171,18 @@ static const char * status_to_str(uint32_t status) {
// ** debug helpers
-static void ggml_hexagon_dump_op_exec(const std::string &sess_name, const htp_opnode & node, const uint32_t req_flags) {
+static void ggml_hexagon_dump_op_exec(const char * sess_name, const htp_opnode & node, const uint32_t req_flags) {
if (!opt_verbose) return;
- htp_opformat fmt(node);
- GGML_LOG_DEBUG("ggml-hex: %s execute-op %s|%s|%s|%s|%s|%s|%s|flags 0x%x\n", sess_name.c_str(),
- node.op_name().c_str(), fmt.names, fmt.dims, fmt.types, fmt.strides, fmt.buffs, fmt.kparams, req_flags);
+ GGML_LOG_DEBUG("ggml-hex: %s execute-op %s|flags 0x%x\n",
+ sess_name, htp_format_op(node, HTP_OPFMT_EXEC).c_str(), req_flags);
}
-static void ggml_hexagon_dump_op_supp(const std::string &sess_name, const struct ggml_tensor * op, bool supp) {
+static void ggml_hexagon_dump_op_supp(const char * sess_name, const struct ggml_tensor * op, bool supp) {
if (!opt_verbose) return;
- htp_opformat fmt(htp_opformat(htp_opnode(HTP_OP_INVALID, const_cast<ggml_tensor*>(op))));
- GGML_LOG_DEBUG("ggml-hex: %s supports-op %s|%s|%s|%s|%s|%s|%s\n", sess_name.c_str(),
- ggml_op_desc(op), fmt.names, fmt.dims, fmt.types, fmt.strides, fmt.buffs, supp ? "yes" : "no");
+ GGML_LOG_DEBUG("ggml-hex: %s supports-op %s|%s\n",
+ sess_name, htp_format_op(op, HTP_OPFMT_SUPP).c_str(), supp ? "yes" : "no");
}
static const char * htp_event_name(uint16_t id) {
@@ -253,7 +215,7 @@ static const char * htp_event_name(uint16_t id) {
}
}
-static void ggml_hexagon_dump_op_prof(const std::string &sess_name, const htp_opnode & node, const htp_prof_desc & pd) {
+static void ggml_hexagon_dump_op_prof(const char * sess_name, const htp_opnode & node, const htp_prof_desc & pd) {
if (!opt_profile) return;
uint32_t op_usec = pd.usecs;
@@ -267,13 +229,12 @@ static void ggml_hexagon_dump_op_prof(const std::string &sess_name, const htp_op
pmu[0], pmu[1], pmu[2], pmu[3], pmu[4], pmu[5], pmu[6], pmu[7]);
}
- htp_opformat fmt(node);
float mhz = op_usec > 0 ? (float) op_cycles / op_usec : 0.0f;
- GGML_LOG_DEBUG("ggml-hex: %s profile-op %s|%s|%s|%s|%s|%s|usec %u cycles %u start %u mhz %.1f%s\n", sess_name.c_str(),
- node.op_name().c_str(), fmt.names, fmt.dims, fmt.types, fmt.strides, fmt.kparams, op_usec, op_cycles, pd.cycles_start, mhz, pmu_str);
+ GGML_LOG_DEBUG("ggml-hex: %s profile-op %s|usec %u cycles %u start %u mhz %.1f%s\n",
+ sess_name, htp_format_op(node, HTP_OPFMT_PROF).c_str(), op_usec, op_cycles, pd.cycles_start, mhz, pmu_str);
}
-static void ggml_hexagon_dump_batch_prof(const std::string & sess_name, const htp_opbatch_rsp & rsp) {
+static void ggml_hexagon_dump_batch_prof(const char * sess_name, const htp_opbatch_rsp & rsp) {
uint64_t batch_cycles = rsp.cycles_stop - rsp.cycles_start;
float batch_mhz = rsp.usecs > 0 ? (float) batch_cycles / rsp.usecs : 0.0f;
@@ -286,10 +247,10 @@ static void ggml_hexagon_dump_batch_prof(const std::string & sess_name, const ht
}
GGML_LOG_DEBUG("ggml-hex: %s profile-op OPBATCH|----|n-ops %u|%s|----|----|usec %u cycles %llu start %llu mhz %.1f\n",
- sess_name.c_str(), rsp.n_ops, evt_str, rsp.usecs, (unsigned long long) batch_cycles, (unsigned long long) rsp.cycles_start, batch_mhz);
+ sess_name, rsp.n_ops, evt_str, rsp.usecs, (unsigned long long) batch_cycles, (unsigned long long) rsp.cycles_start, batch_mhz);
}
-static void ggml_hexagon_dump_trace_events(const std::string & sess_name, const htp_opbatch_rsp & rsp,
+static void ggml_hexagon_dump_trace_events(const char * sess_name, const htp_opbatch_rsp & rsp,
const htp_trace_desc * trace_events, uint32_t n_traces) {
if (opt_profile == 3 && trace_events) {
uint32_t valid_cnt[HTP_MAX_NTHREADS + 1] = {0};
@@ -304,237 +265,14 @@ static void ggml_hexagon_dump_trace_events(const std::string & sess_name, const
bool is_stop = (e.info & 0x8000) != 0;
uint16_t info = e.info & 0x7FFF;
GGML_LOG_DEBUG("ggml-hex: %s trace-evt %s: thread %u info %u %s %u\n",
- sess_name.c_str(), htp_event_name(e.id), t, info, is_stop ? "stop" : "start", e.cycles);
+ sess_name, htp_event_name(e.id), t, info, is_stop ? "stop" : "start", e.cycles);
}
}
}
}
-enum ggml_hexagon_tensor_flags {
- GGML_HEXAGON_TENSOR_TILED = (1 << 0),
- GGML_HEXAGON_TENSOR_WEIGHT = (1 << 1),
- GGML_HEXAGON_TENSOR_FENCE = (1 << 2),
- GGML_HEXAGON_TENSOR_FUSEABLE = (1 << 3),
-};
-
-struct ggml_hexagon_session;
-
-static void ggml_hexagon_precompute_matmul_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * dst,
- struct htp_mm_kernel_params * kparams
-);
-
-static void ggml_hexagon_precompute_fused_matmul_add_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * src2,
- const struct ggml_tensor * dst,
- struct htp_mm_kernel_params * kparams
-);
-
-static void ggml_hexagon_precompute_unary_params(
- const struct ggml_hexagon_session * sess,
- uint32_t op,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * dst,
- struct htp_unary_kernel_params * kparams
-);
-
-static bool ggml_hexagon_precompute_binary_params(
- const struct ggml_hexagon_session * sess,
- uint32_t op,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * dst,
- struct htp_binary_kernel_params * kparams,
- const struct ggml_tensor * src2 = nullptr
-);
-
-static void ggml_hexagon_precompute_get_rows_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * dst,
- struct htp_get_rows_kernel_params * kparams
-);
-
-static void ggml_hexagon_precompute_set_rows_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * dst,
- struct htp_set_rows_kernel_params * kparams
-);
-
-static void ggml_hexagon_precompute_softmax_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_softmax_kernel_params * kparams
-);
-
-static void ggml_hexagon_precompute_rope_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_rope_kernel_params * kparams
-);
-
-static void ggml_hexagon_precompute_ssm_conv_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * dst,
- struct htp_ssm_conv_kernel_params * kparams
-);
-
-static void ggml_hexagon_precompute_gated_delta_net_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_gdn_kernel_params * kparams
-);
-
-static void ggml_hexagon_precompute_sort_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- bool is_top_k,
- struct htp_sort_kernel_params * kparams
-);
-
-static void ggml_hexagon_precompute_pool_2d_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * dst,
- struct htp_pool_2d_kernel_params * kparams,
- bool is_pool_1d
-);
-
-static bool ggml_hexagon_precompute_concat_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_concat_kernel_params * kparams
-);
-
-static bool ggml_hexagon_precompute_cpy_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_copy_kernel_params * kparams
-);
-
-static bool ggml_hexagon_precompute_im2col_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_im2col_kernel_params * kparams
-);
-
-static bool ggml_hexagon_precompute_im2col_3d_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_im2col_kernel_params * kparams
-);
-
-static void ggml_hexagon_precompute_fused_mmnx_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * act,
- int32_t n_weights,
- struct htp_mm_kernel_params * kparams
-);
-
-static void ggml_hexagon_precompute_fused_mmidnx_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * act,
- const struct ggml_tensor * dst,
- int32_t n_weights,
- struct htp_mm_kernel_params * kparams
-);
-
-static bool ggml_hexagon_precompute_allreduce_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * dst,
- uint32_t rank,
- uint32_t n_ranks,
- bool has_add,
- bool is_row_bcast,
- bool is_shard_ok,
- struct htp_allreduce_kernel_params * kparams
-);
-
-static bool ggml_hexagon_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride);
-static bool ggml_hexagon_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst);
-static ggml_tensor ggml_hexagon_tensor_collapse_rows(const struct ggml_tensor * t);
-
-static bool mm_is_hmx_eligible(const ggml_tensor * t);
-static htp_op_code op_remap_to_htp(const ggml_tensor * t);
-static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams);
-static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams);
-static bool is_mergeable_mul_mat(const ggml_tensor * t);
-static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor * n2);
-static bool is_mergeable_mul_mat_id(const ggml_tensor * t);
-static bool is_mergeable_mul_mat_id_pair(const ggml_tensor * n1, const ggml_tensor * n2);
-
// ** backend sessions
-struct ggml_hexagon_tensor_extra {
- std::vector<uint8_t> shadow_buf;
- size_t shadow_size { 0 };
- uint32_t flags { 0 };
-};
-
-static inline bool ggml_hexagon_tensor_is_fuseable(const struct ggml_tensor * t) {
- if (!t->extra) return false;
- auto extra = (const struct ggml_hexagon_tensor_extra *) t->extra;
- return (extra->flags & GGML_HEXAGON_TENSOR_FUSEABLE) != 0;
-}
-
-static inline bool ggml_hexagon_can_rt_repack(const struct ggml_tensor * t) {
- if (opt_rt_repack == 0 || !htp_is_tiled_weight_tensor(t)) {
- return false;
- }
- if (t->buffer && ggml_backend_buft_is_host(t->buffer->buft)) {
- return false;
- }
- return opt_rt_repack > 1 || ggml_is_quantized(t->type);
-}
-
-static inline bool ggml_hexagon_tensor_is_tiled(const struct ggml_tensor * t) {
- const auto * extra = (const ggml_hexagon_tensor_extra *) t->extra;
- if (extra && (extra->flags & GGML_HEXAGON_TENSOR_TILED)) {
- return true;
- }
- return ggml_hexagon_can_rt_repack(t);
-}
-
-static inline bool ggml_hexagon_needs_repack(const struct ggml_tensor * t) {
- const auto * extra = (const ggml_hexagon_tensor_extra *) t->extra;
- if (extra && (extra->flags & GGML_HEXAGON_TENSOR_TILED)) {
- return false;
- }
- return ggml_hexagon_can_rt_repack(t);
-}
-
-static inline bool ggml_hexagon_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) {
- const uintptr_t a0 = (uintptr_t) a->data;
- const uintptr_t b0 = (uintptr_t) b->data;
- const uintptr_t a1 = a0 + ggml_nbytes(a);
- const uintptr_t b1 = b0 + ggml_nbytes(b);
-
- return a0 < b1 && b0 < a1;
-}
-
-static inline bool ggml_hexagon_can_row_partition(const struct ggml_tensor * t) {
- if (t->ne[1] > 1 && (t->nb[1] & 127) != 0) return false;
- if (t->ne[2] > 1 && (t->nb[2] & 127) != 0) return false;
- if (t->ne[3] > 1 && (t->nb[3] & 127) != 0) return false;
- return true;
-}
-
-struct htp_opnode;
-
struct ggml_hexagon_opbatch;
struct ggml_hexagon_opqueue;
struct ggml_hexagon_shared_buffer;
@@ -611,8 +349,13 @@ struct ggml_hexagon_session {
void allocate(const ggml_hexagon_device_config & config) noexcept(false);
void release() noexcept(true);
- uint8_t * alloc_fence(uint32_t n_slots = 1);
- void free_fence(void * ptr, uint32_t n_slots = 1);
+ uint8_t * alloc_fence(uint32_t n_slots = 1);
+ void free_fence(void * ptr, uint32_t n_slots = 1);
+ ggml_tensor make_fence_tensor(const volatile void * slot, uint32_t n_slots = 1) const;
+ uint32_t next_fence_seq() {
+ if (++fence_seq == 0) fence_seq = 1;
+ return fence_seq;
+ }
uint8_t * mdev_fence_slot = nullptr;
std::unordered_map<uint64_t, volatile uint32_t *> cpy_fence_slots;
@@ -646,6 +389,12 @@ struct ggml_hexagon_session {
}
};
+const char * htp_session_name(const struct ggml_hexagon_session * sess) { return sess->c_name(); }
+size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess) { return sess->vtcm_size; }
+uint32_t htp_session_n_threads(const struct ggml_hexagon_session * sess) { return sess->n_threads; }
+uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess) { return sess->n_hmx; }
+uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess) { return sess->mdev.count; }
+
// ** backend buffers
struct ggml_backend_hexagon_device_context {
@@ -712,7 +461,7 @@ struct ggml_hexagon_rpcmem_block {
struct ggml_hexagon_shared_buffer {
ggml_hexagon_session * sess;
std::shared_ptr<ggml_hexagon_rpcmem_block> mem;
- std::vector<ggml_hexagon_tensor_extra *> tensor_extra;
+ std::vector<htp_tensor_extra *> tensor_extra;
bool mapped;
bool pinned;
bool extended;
@@ -824,7 +573,7 @@ struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer {
ggml_hexagon_fence_buffer(ggml_hexagon_session * sess, ggml_backend_buffer_type_t buft, size_t size)
: ggml_hexagon_shared_buffer(sess, size, false /* pinned */),
- slot_count(size / GGML_HEXAGON_FENCE_SLOT_SIZE),
+ slot_count(size / HTP_FENCE_SLOT_SIZE),
slot_head(0) {
backend_buffer.buft = buft;
backend_buffer.context = static_cast<ggml_hexagon_shared_buffer *>(this);
@@ -837,21 +586,21 @@ struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer {
if (n_slots == 1 && !free_slots.empty()) {
uint32_t slot = free_slots.back();
free_slots.pop_back();
- ptr = base() + (size_t) slot * GGML_HEXAGON_FENCE_SLOT_SIZE;
+ ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE;
} else if (slot_head + n_slots <= slot_count) {
uint32_t slot = slot_head;
slot_head += n_slots;
- ptr = base() + (size_t) slot * GGML_HEXAGON_FENCE_SLOT_SIZE;
+ ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE;
}
if (ptr) {
- memset(ptr, 0, (size_t) n_slots * GGML_HEXAGON_FENCE_SLOT_SIZE);
+ memset(ptr, 0, (size_t) n_slots * HTP_FENCE_SLOT_SIZE);
}
return ptr;
}
void free_slot(void * ptr, uint32_t n_slots = 1) {
if (!ptr) return;
- uint32_t slot = ((uint8_t *) ptr - base()) / GGML_HEXAGON_FENCE_SLOT_SIZE;
+ uint32_t slot = ((uint8_t *) ptr - base()) / HTP_FENCE_SLOT_SIZE;
for (uint32_t i = 0; i < n_slots; i++) {
free_slots.push_back(slot + i);
}
@@ -870,6 +619,26 @@ inline void ggml_hexagon_session::free_fence(void * ptr, uint32_t n_slots) {
}
}
+inline ggml_tensor ggml_hexagon_session::make_fence_tensor(const volatile void * slot, uint32_t n_slots) const {
+ static htp_tensor_extra fence_extra { {}, 0, HTP_TENSOR_FENCE };
+
+ ggml_tensor t {};
+ t.buffer = &this->fence_buf->backend_buffer;
+ t.extra = &fence_extra;
+ t.data = const_cast<void *>(slot);
+ t.type = GGML_TYPE_I8;
+ t.ne[0] = HTP_FENCE_SLOT_SIZE;
+ t.ne[1] = (int64_t) n_slots;
+ t.ne[2] = 1;
+ t.ne[3] = 1;
+ t.nb[0] = 1;
+ t.nb[1] = HTP_FENCE_SLOT_SIZE;
+ t.nb[2] = t.nb[1] * t.ne[1];
+ t.nb[3] = t.nb[2];
+ t.op = GGML_OP_NONE;
+ return t;
+}
+
static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) {
auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(buffer->context);
sbuf->sess->unclone_buffer(sbuf);
@@ -888,7 +657,7 @@ static enum ggml_status ggml_backend_hexagon_buffer_init_tensor(ggml_backend_buf
HEX_VERBOSE("ggml-hex: %s init-tensor %s : base %p data %p nbytes %zu\n", sess->c_name(),
tensor->name, (void *) sbuf->base(), tensor->data, ggml_nbytes(tensor));
- auto extra = new ggml_hexagon_tensor_extra();
+ auto extra = new htp_tensor_extra();
sbuf->tensor_extra.push_back(extra);
tensor->extra = extra;
@@ -901,21 +670,21 @@ static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer,
const void * data,
size_t offset,
size_t size) {
- auto extra = (ggml_hexagon_tensor_extra *) tensor->extra;
+ auto extra = (htp_tensor_extra *) tensor->extra;
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
- extra->flags |= GGML_HEXAGON_TENSOR_WEIGHT;
+ extra->flags |= HTP_TENSOR_WEIGHT;
if (htp_is_tiled_weight_tensor(tensor)) {
- extra->flags |= GGML_HEXAGON_TENSOR_TILED;
+ extra->flags |= HTP_TENSOR_TILED;
}
}
HEX_VERBOSE("ggml-hex: %s set-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n",
sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags);
- if ((extra->flags & GGML_HEXAGON_TENSOR_TILED) == 0) {
+ if ((extra->flags & HTP_TENSOR_TILED) == 0) {
memcpy((char *) tensor->data + offset, data, size);
return;
}
@@ -944,14 +713,14 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,
void * data,
size_t offset,
size_t size) {
- auto extra = (ggml_hexagon_tensor_extra *) tensor->extra;
+ auto extra = (htp_tensor_extra *) tensor->extra;
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
HEX_VERBOSE("ggml-hex: %s get-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n",
sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags);
- if ((extra->flags & GGML_HEXAGON_TENSOR_TILED) == 0) {
+ if ((extra->flags & HTP_TENSOR_TILED) == 0) {
memcpy(data, (const char *) tensor->data + offset, size);
return;
}
@@ -978,21 +747,21 @@ static void ggml_backend_hexagon_buffer_set_tensor_2d(ggml_backend_buffer_t buff
size_t n_copies,
size_t stride_tensor,
size_t stride_data) {
- auto extra = (ggml_hexagon_tensor_extra *) tensor->extra;
+ auto extra = (htp_tensor_extra *) tensor->extra;
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
- extra->flags |= GGML_HEXAGON_TENSOR_WEIGHT;
+ extra->flags |= HTP_TENSOR_WEIGHT;
if (htp_is_tiled_weight_tensor(tensor)) {
- extra->flags |= GGML_HEXAGON_TENSOR_TILED;
+ extra->flags |= HTP_TENSOR_TILED;
}
}
HEX_VERBOSE("ggml-hex: %s set-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d flags 0x%x\n",
sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, extra->flags);
- if ((extra->flags & GGML_HEXAGON_TENSOR_TILED) == 0) {
+ if ((extra->flags & HTP_TENSOR_TILED) == 0) {
for (size_t i = 0; i < n_copies; i++) {
memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size);
}
@@ -1023,14 +792,14 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff
size_t n_copies,
size_t stride_tensor,
size_t stride_data) {
- auto extra = (ggml_hexagon_tensor_extra *) tensor->extra;
+ auto extra = (htp_tensor_extra *) tensor->extra;
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
HEX_VERBOSE("ggml-hex: %s get-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n",
sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage);
- if ((extra->flags & GGML_HEXAGON_TENSOR_TILED) == 0) {
+ if ((extra->flags & HTP_TENSOR_TILED) == 0) {
for (size_t i = 0; i < n_copies; i++) {
memcpy((uint8_t *)data + i * stride_data, (const uint8_t *)tensor->data + offset + i * stride_tensor, size);
}
@@ -1435,7 +1204,6 @@ struct ggml_hexagon_opbatch {
std::vector<htp_tensor> h_tens; // htp tensor descriptors
std::vector<htp_op_desc> h_ops; // htp op descriptors
- std::unordered_map<int, int> b_map; // buffer fd to index
std::unordered_map<const ggml_tensor*, int> t_map; // tensor ptr to index
std::unordered_multimap<void*, int> d_map; // tensor data to index
@@ -1455,7 +1223,6 @@ struct ggml_hexagon_opbatch {
n_ops = 0;
b_vmem = 0;
- b_map.clear();
t_map.clear();
d_map.clear();
ops.resize(n_ops_max);
@@ -1466,7 +1233,7 @@ struct ggml_hexagon_opbatch {
n_bufs_max = HTP_OP_MAX_BUFS;
n_ops_max = batch_size;
- n_tens_max = std::min<size_t>(n_ops_max + n_ops_max * HTP_OP_MAX_INPUTS, HTP_OP_MAX_TENSORS);
+ n_tens_max = std::min<size_t>(n_ops_max * 4 + HTP_OP_MAX_INPUTS + HTP_OP_MAX_OUTPUTS, HTP_OP_MAX_TENSORS);
b_vmem_max = max_vmem;
@@ -1476,7 +1243,6 @@ struct ggml_hexagon_opbatch {
h_tens.resize(n_tens_max);
h_ops.resize(n_ops_max);
- b_map.reserve(n_bufs_max);
t_map.reserve(n_tens_max);
d_map.reserve(n_tens_max);
@@ -1487,18 +1253,26 @@ struct ggml_hexagon_opbatch {
}
bool empty() const { return n_ops == 0; }
+ bool full() const { return n_ops >= n_ops_max; }
+
+ int find_buffer(int fd) const {
+ for (unsigned int i = 0; i < n_bufs; i++) {
+ if ((int) h_bufs[i].fd == fd) {
+ return (int) i;
+ }
+ }
+ return -1;
+ }
// add buffer and return its index
int add_buffer(ggml_hexagon_shared_buffer * sbuf) {
// Lookup by fd
- auto it = b_map.find(sbuf->fd());
- if (it != b_map.end()) { return it->second; }
+ int bi = find_buffer(sbuf->fd());
+ if (bi >= 0) { return bi; }
// Add new buffer to the batch
GGML_ASSERT(n_bufs < HTP_OP_MAX_BUFS);
- int bi = n_bufs++;
-
- b_map.insert({sbuf->fd(), bi});
+ bi = n_bufs++;
htp_buf_desc &b = h_bufs[bi];
b.base = (uint64_t) sbuf->base();
@@ -1515,62 +1289,16 @@ struct ggml_hexagon_opbatch {
return bi;
}
- bool same_shape(const htp_tensor * h, const ggml_tensor * t) const {
- auto extra = (ggml_hexagon_tensor_extra *) t->extra;
-
- const bool is_tiled = (extra->flags & GGML_HEXAGON_TENSOR_TILED) != 0;
- const int64_t ne0_padded = hex_round_up(t->ne[0], 32);
- const int64_t ne1_padded = hex_round_up(t->ne[1], 32);
-
- int64_t nb1 = is_tiled ? (int64_t) htp_tiled_row_size(t->type, ne0_padded) : t->nb[1];
- int64_t nb2 = is_tiled ? nb1 * ne1_padded : t->nb[2];
- int64_t nb3 = is_tiled ? nb2 * t->ne[2] : t->nb[3];
- int64_t nb0 = (is_tiled && (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_BF16)) ? sizeof(ggml_fp16_t) : t->nb[0];
- enum ggml_type expected_type = (is_tiled && (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : t->type;
-
- return (h->type == expected_type) &&
- (h->ne[0] == t->ne[0]) && (h->ne[1] == t->ne[1]) && (h->ne[2] == t->ne[2]) && (h->ne[3] == t->ne[3]) &&
- (h->nb[0] == nb0) && (h->nb[1] == nb1) && (h->nb[2] == nb2) && (h->nb[3] == nb3);
- }
-
- // add tensor and return its index
- int add_tensor(const ggml_tensor * t) {
- auto extra = (ggml_hexagon_tensor_extra *) t->extra;
- auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
-
- // First lookup by tensor data
- auto range = d_map.equal_range(t->data);
- for (auto it = range.first; it != range.second; ++it) {
- htp_tensor * h = &h_tens[it->second];
- if (same_shape(h, t)) { return it->second; }
- }
-
- // Lookup by tensor ptr
- auto it = t_map.find(t);
- if (it != t_map.end()) { return it->second; }
-
- // Add new tensor to the batch
- int ti = n_tens++;
- GGML_ASSERT(n_tens <= n_tens_max);
-
- t_map.insert({t, ti});
- d_map.insert({t->data, ti});
-
- uint64_t t_offset = (uint8_t *) t->data - sbuf->base();
- size_t t_size = ggml_nbytes(t);
+ static void init_tensor_shape(htp_tensor & h, const ggml_tensor * t) {
+ auto extra = (htp_tensor_extra *) t->extra;
- htp_tensor &h = h_tens[ti];
- h.bi = add_buffer(sbuf);
- h.ti = ti;
- h.data = t_offset;
h.type = t->type;
h.ne[0] = t->ne[0];
h.ne[1] = t->ne[1];
h.ne[2] = t->ne[2];
h.ne[3] = t->ne[3];
- const bool is_tiled = (extra->flags & GGML_HEXAGON_TENSOR_TILED) != 0;
- if (is_tiled) {
+ if (extra->flags & HTP_TENSOR_TILED) {
if (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_BF16) {
h.type = GGML_TYPE_F16;
h.nb[0] = sizeof(ggml_fp16_t);
@@ -1583,60 +1311,96 @@ struct ggml_hexagon_opbatch {
h.nb[2] = h.nb[1] * ne1_padded;
h.nb[3] = h.nb[2] * h.ne[2];
h.size = h.nb[3] * h.ne[3];
- t_size = h.size;
} else {
- h.size = t_size;
- h.nb[0] = t->nb[0]; h.nb[1] = t->nb[1]; h.nb[2] = t->nb[2]; h.nb[3] = t->nb[3];
+ h.nb[0] = t->nb[0];
+ h.nb[1] = t->nb[1];
+ h.nb[2] = t->nb[2];
+ h.nb[3] = t->nb[3];
+ h.size = ggml_nbytes(t);
}
- h.flags = 0;
- if ((extra->flags & GGML_HEXAGON_TENSOR_WEIGHT) != 0) {
- h.flags |= HTP_TENSOR_WEIGHT;
- }
- if ((extra->flags & GGML_HEXAGON_TENSOR_TILED) != 0) {
- h.flags |= HTP_TENSOR_TILED;
- }
- if ((extra->flags & GGML_HEXAGON_TENSOR_FENCE) != 0) {
- h.flags |= HTP_TENSOR_FENCE;
+ h.flags = extra->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_TILED | HTP_TENSOR_FENCE);
+ }
+
+ static bool same_shape(const htp_tensor & a, const htp_tensor & b) {
+ return (a.type == b.type) &&
+ (a.ne[0] == b.ne[0]) && (a.ne[1] == b.ne[1]) && (a.ne[2] == b.ne[2]) && (a.ne[3] == b.ne[3]) &&
+ (a.nb[0] == b.nb[0]) && (a.nb[1] == b.nb[1]) && (a.nb[2] == b.nb[2]) && (a.nb[3] == b.nb[3]);
+ }
+
+ // add tensor and return its index
+ int add_tensor(const ggml_tensor * t) {
+ // Lookup by tensor ptr
+ auto it = t_map.find(t);
+ if (it != t_map.end()) { return it->second; }
+
+ htp_tensor shape {};
+ init_tensor_shape(shape, t);
+
+ // Lookup by tensor data and shape
+ auto range = d_map.equal_range(t->data);
+ for (auto dit = range.first; dit != range.second; ++dit) {
+ if (same_shape(h_tens[dit->second], shape)) {
+ t_map.insert({t, dit->second});
+ return dit->second;
+ }
}
+ auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
+
+ // Add new tensor to the batch
+ int ti = n_tens++;
+ GGML_ASSERT(n_tens <= n_tens_max);
+
+ t_map.insert({t, ti});
+ d_map.insert({t->data, ti});
+
+ uint64_t t_offset = (uint8_t *) t->data - sbuf->base();
+
+ htp_tensor & h = h_tens[ti];
+ h = shape;
+ h.bi = add_buffer(sbuf);
+ h.ti = ti;
+ h.data = t_offset;
+
HEX_VERBOSE("ggml-hex: %s add-tensor #%u %s : bi %d data %p offset %zu size %zu flags 0x%x : %zu:%zu:%zu:%zu\n", sess->c_name(),
- ti, t->name, h.bi, (void*) t->data, (size_t) t_offset, t_size, h.flags,
+ ti, t->name, h.bi, (void*) t->data, (size_t) t_offset, (size_t) h.size, h.flags,
(size_t) h.ne[0], (size_t) h.ne[1], (size_t) h.ne[2], (size_t) h.ne[3]);
return ti;
}
- bool fit_op(const htp_opnode & node) const {
- if (n_ops >= n_ops_max) return false;
-
+ bool fit_tensors(const htp_opnode & node) const {
// check how much extras we will need
size_t extra_bufs = 0;
size_t extra_vmem = 0;
size_t extra_tens = 0;
+ const bool check_tens = (n_tens + node.inputs.size() + node.outputs.size()) > n_tens_max;
+
int seen_bufs[HTP_OP_MAX_BUFS];
int n_seen_bufs = 0;
- auto fit_tensor = [&](const ggml_tensor *t) {
+ auto fit_tensor = [&](const ggml_tensor * t) {
if (!t) return;
- if (!t_map.count(t)) {
+ if (check_tens) {
+ if (t_map.count(t)) return;
extra_tens++;
+ }
- auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
- int fd = sbuf->fd();
- if (!b_map.count(fd)) {
- for (int i = 0; i < n_seen_bufs; i++) {
- if (seen_bufs[i] == fd) return;
- }
- if (n_seen_bufs < HTP_OP_MAX_BUFS) {
- seen_bufs[n_seen_bufs++] = fd;
- }
- if (!sbuf->extended) {
- extra_vmem += sbuf->size();
- }
- extra_bufs += 1;
+ auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
+ int fd = sbuf->fd();
+ if (find_buffer(fd) < 0) {
+ for (int i = 0; i < n_seen_bufs; i++) {
+ if (seen_bufs[i] == fd) return;
}
+ if (n_seen_bufs < HTP_OP_MAX_BUFS) {
+ seen_bufs[n_seen_bufs++] = fd;
+ }
+ if (!sbuf->extended) {
+ extra_vmem += sbuf->size();
+ }
+ extra_bufs += 1;
}
};
@@ -1648,40 +1412,43 @@ struct ggml_hexagon_opbatch {
}
if ((extra_bufs + n_bufs) > n_bufs_max) return false;
- if ((extra_tens + n_tens) > n_tens_max) return false;
+ if (check_tens && (extra_tens + n_tens) > n_tens_max) return false;
if ((extra_vmem + b_vmem) > b_vmem_max) return false;
return true;
}
- // assumes that fit_op() was called first and returned true
- void add_op(const htp_opnode & node) {
- // Add new op
-
- unsigned int n = n_ops++;
- GGML_ASSERT(n_ops <= n_ops_max);
-
- ops[n] = node;
+ void sync_op_desc(unsigned int n) {
+ const htp_opnode & node = ops[n];
+ htp_op_desc & o = h_ops[n];
- htp_op_desc &o = h_ops[n];
- memcpy(o.params, node.node->op_params, sizeof(node.node->op_params));
- memcpy(o.kernel_params, node.kernel_params, sizeof(o.kernel_params));
+ memcpy(o.params, node.params, sizeof(o.params));
+ memcpy(o.kernel_params, node.kernel_params, sizeof(o.kernel_params));
o.opcode = node.opcode;
- o.flags = 0;
- ggml_hexagon_dump_op_exec(sess->c_name(), ops[n], o.flags);
-
- auto inputs = node.get_inputs();
- for (unsigned int i=0; i < HTP_OP_MAX_INPUTS; i++) {
- o.src[i] = (i < inputs.size() && inputs[i]) ? add_tensor(inputs[i]) : 0xffff;
+ const auto & inputs = node.get_inputs();
+ for (unsigned int i = 0; i < HTP_OP_MAX_INPUTS; i++) {
+ o.src[i] = (i < inputs.size() && inputs[i]) ? add_tensor(inputs[i]) : 0xffff;
}
- auto outputs = node.get_outputs();
- for (unsigned int i=0; i < HTP_OP_MAX_OUTPUTS; i++) {
+ const auto & outputs = node.get_outputs();
+ for (unsigned int i = 0; i < HTP_OP_MAX_OUTPUTS; i++) {
o.dst[i] = (i < outputs.size() && outputs[i]) ? add_tensor(outputs[i]) : 0xffff;
}
}
+ // caller must ensure the op fits
+ void add_op(const htp_opnode & node) {
+ unsigned int n = n_ops++;
+ GGML_ASSERT(n_ops <= n_ops_max);
+
+ ops[n] = node;
+ h_ops[n].flags = 0;
+
+ ggml_hexagon_dump_op_exec(sess->c_name(), ops[n], h_ops[n].flags);
+ sync_op_desc(n);
+ }
+
void sort_buffers() {
if (n_bufs <= 1) return;
@@ -1724,5184 +1491,866 @@ struct ggml_hexagon_opbatch {
}
}
- bool try_fuse_common(std::initializer_list<const ggml_tensor *> tensors) const {
- size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0;
-
- int seen_bufs[HTP_OP_MAX_BUFS];
- int n_seen_bufs = 0;
-
- for (const auto * t : tensors) {
- if (!t || t_map.count(t)) {
- continue;
- }
- extra_tens++;
- auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
- int fd = sbuf->fd();
- if (!b_map.count(fd)) {
- bool found = false;
- for (int i = 0; i < n_seen_bufs; i++) {
- if (seen_bufs[i] == fd) {
- found = true;
- break;
- }
- }
- if (!found) {
- if (n_seen_bufs < HTP_OP_MAX_BUFS) {
- seen_bufs[n_seen_bufs++] = fd;
- }
- if (!sbuf->extended) {
- extra_vmem += sbuf->size();
- }
- extra_bufs += 1;
- }
- }
+ bool try_fuse(const htp_opnode & node) {
+ if (n_ops == 0) {
+ return false;
}
- if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) {
+ htp_opnode & last_node = ops[n_ops - 1];
+ if (!htp_fusion_try_fuse(sess, last_node, node)) {
return false;
}
+ sync_op_desc(n_ops - 1);
+ HEX_VERBOSE("ggml-hex: %s fused %s (N=%zu, #%u)\n",
+ sess->c_name(), last_node.name().c_str(), last_node.fused.size() + 1, n_ops - 1);
return true;
}
+};
- bool try_fuse_common(const ggml_tensor * t1, const ggml_tensor * t2) const {
- return try_fuse_common({t1, t2});
- }
-
- bool try_fuse_allreduce_add(const htp_opnode & node) {
- if (n_ops == 0 || opt_ar_select != 2) return false;
- if (node.opcode != HTP_OP_ADD) return false;
-
- htp_opnode & last_node = ops[n_ops - 1];
- if (last_node.opcode != HTP_OP_ALLREDUCE) return false;
-
- auto * ar_kparams = (struct htp_allreduce_kernel_params *) last_node.kernel_params;
- const uint32_t rank = (uint32_t) ar_kparams->rank;
- const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks;
- const ggml_tensor * ar_local = last_node.inputs[rank];
- const ggml_tensor * add_src0 = node.src0();
- const ggml_tensor * add_src1 = node.src1();
- const ggml_tensor * add_dst = node.dst();
-
- if (!ggml_hexagon_tensor_is_fuseable(ar_local)) return false;
-
- const ggml_tensor * res_tensor;
- if (add_src0 == ar_local || add_src0->data == ar_local->data) {
- res_tensor = add_src1;
- } else if (add_src1 == ar_local || add_src1->data == ar_local->data) {
- res_tensor = add_src0;
- } else {
- return false;
- }
+struct ggml_hexagon_registry {
+ ggml_hexagon_registry(ggml_backend_reg_t reg);
+ ~ggml_hexagon_registry();
- if (ar_local->type != res_tensor->type) return false;
+ ggml_backend_device devices[GGML_HEXAGON_MAX_SESSIONS];
+};
- const bool is_same_shape = (ar_local->ne[0] == res_tensor->ne[0] && ar_local->ne[1] == res_tensor->ne[1] &&
- ar_local->ne[2] == res_tensor->ne[2] && ar_local->ne[3] == res_tensor->ne[3]);
- const bool is_row_bcast = !is_same_shape && (ar_local->ne[0] == res_tensor->ne[0] && res_tensor->ne[1] == 1 &&
- res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1);
+struct ggml_hexagon_opqueue {
+ // Shared buffer for storing batches
+ ggml_hexagon_shared_buffer *shm_buf;
+ size_t shm_blk_size;
+ size_t depth;
- if (!is_same_shape && !is_row_bcast) return false;
+ using opvec = std::vector<htp_opnode>;
- if (is_same_shape) {
- if (ar_local->nb[1] != res_tensor->nb[1] || ar_local->nb[2] != res_tensor->nb[2] ||
- ar_local->nb[3] != res_tensor->nb[3]) {
- return false;
- }
- if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(res_tensor)) {
- return false;
- }
- }
- if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(add_dst)) {
- return false;
- }
+ std::vector<opvec> op_cache; // per batch op cache
- // scatter is only valid for in-place add within max outputs
- const bool is_shard_ok = n_ranks <= HTP_OP_MAX_OUTPUTS && add_dst->data == ar_local->data;
+ ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) : depth(depth) {
+ size_t n_bufs = HTP_OP_MAX_BUFS;
+ size_t n_ops = batch_size;
+ size_t n_tensors = std::min<size_t>(n_ops * 4 + HTP_OP_MAX_INPUTS + HTP_OP_MAX_OUTPUTS, HTP_OP_MAX_TENSORS);
- struct htp_allreduce_kernel_params new_kparams;
- if (!ggml_hexagon_precompute_allreduce_params(
- sess, add_dst, (uint32_t) ar_kparams->rank, (uint32_t) ar_kparams->n_ranks, true, is_row_bcast, is_shard_ok, &new_kparams
- )) {
- HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: solver failed\n", sess->c_name());
- return false;
+ size_t tr_size = 0;
+ if (opt_profile == 3) {
+ tr_size = (HTP_MAX_NTHREADS + 1) * opt_optrace * sizeof(htp_trace_desc);
}
- const bool scatter_ok = new_kparams.n_dsts > 1;
- new_kparams.mode = scatter_ok ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL;
+ shm_blk_size = sizeof(htp_buf_desc) * n_bufs +
+ sizeof(htp_tensor) * n_tensors +
+ sizeof(htp_op_desc) * n_ops +
+ sizeof(htp_prof_desc) * n_ops +
+ tr_size;
- for (uint32_t r = 0; r < n_ranks; r++) {
- const ggml_tensor * ar_src = last_node.inputs[r];
- if (!ggml_hexagon_tensors_overlap(add_dst, ar_src)) {
- continue;
- }
- // in-place aliasing is safe under reduce-scatter since each rank writes disjoint shards
- if (scatter_ok && r == rank) {
- continue;
- }
- HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: dst overlaps allreduce src %u\n", sess->c_name(), r);
- return false;
- }
+ shm_buf = new ggml_hexagon_shared_buffer(sess, shm_blk_size * depth, true /* pinned */);
- if (!try_fuse_common(res_tensor, add_dst)) {
- return false;
- }
+ op_cache.resize(depth);
- last_node.opcode = HTP_OP_ALLREDUCE_ADD;
- last_node.name = "ALLREDUCE+ADD";
- last_node.inputs.push_back(res_tensor);
- last_node.outputs.clear();
- last_node.outputs.push_back(add_dst);
- last_node.fused.push_back(node.node);
- memcpy(last_node.kernel_params, &new_kparams, sizeof(new_kparams));
-
- htp_op_desc & o = h_ops[n_ops - 1];
- o.opcode = HTP_OP_ALLREDUCE_ADD;
- memcpy(o.kernel_params, &new_kparams, sizeof(new_kparams));
-
- o.src[2 * n_ranks] = add_tensor(res_tensor);
- if (new_kparams.mode == HTP_ALLREDUCE_SHARDED_FANOUT) {
- // fan out shard to all per-rank partial buffers
- GGML_ASSERT((uint32_t) new_kparams.n_dsts == n_ranks);
- for (uint32_t d = 0; d < n_ranks; d++) {
- GGML_ASSERT(o.src[d] != 0xffff);
- o.dst[d] = o.src[d];
- }
- for (uint32_t d = n_ranks; d < HTP_OP_MAX_OUTPUTS; d++) {
- o.dst[d] = 0xffff;
- }
- } else {
- o.dst[0] = add_tensor(add_dst);
- for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) {
- o.dst[d] = 0xffff;
- }
+ if (opt_verbose) {
+ GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n",
+ sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size);
}
-
- HEX_VERBOSE("ggml-hex: %s fused ALLREDUCE+ADD (#%u) mode=%d n_dsts=%d\n",
- sess->c_name(), n_ops - 1, (int) new_kparams.mode, (int) new_kparams.n_dsts);
- return true;
}
- bool try_fuse_rms_norm_mul(const htp_opnode & node) {
- if (n_ops == 0) return false;
- if (node.opcode != HTP_OP_MUL) return false;
+ ~ggml_hexagon_opqueue() {
+ delete shm_buf;
+ }
- htp_opnode & last_node = ops[n_ops - 1];
- if (last_node.opcode != HTP_OP_RMS_NORM) return false;
+ size_t shm_size() const { return shm_buf ? shm_buf->size() : 0; }
- const ggml_tensor * mul_src0 = node.src0();
- const ggml_tensor * mul_src1 = node.src1();
- const ggml_tensor * rms_out = last_node.dst();
+ // push new batch
+ bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, const ggml_hexagon_opbatch* op_batch, uint64_t seq) {
+ static_assert(sizeof(htp_opbatch_req) % 8 == 0, "sizeof(htp_opbatch_req) must be multiple of 8");
+ static_assert(sizeof(htp_opbatch_rsp) % 8 == 0, "sizeof(htp_opbatch_rsp) must be multiple of 8");
+ static_assert(sizeof(htp_buf_desc) % 8 == 0, "sizeof(htp_buf_desc) must be multiple of 8");
+ static_assert(sizeof(htp_tensor) % 8 == 0, "sizeof(htp_tensor) must be multiple of 8");
+ static_assert(sizeof(htp_op_desc) % 8 == 0, "sizeof(htp_op_desc) must be multiple of 8");
+ static_assert(sizeof(htp_prof_desc) % 8 == 0, "sizeof(htp_prof_desc) must be multiple of 8");
- if (!ggml_hexagon_tensor_is_fuseable(rms_out)) return false;
+ if (seq - shm_buf->sess->batch_rsp_seq > depth) { return false; }
- const ggml_tensor * weight;
- if (mul_src0 == rms_out || mul_src0->data == rms_out->data) {
- weight = mul_src1;
- } else if (mul_src1 == rms_out || mul_src1->data == rms_out->data) {
- weight = mul_src0;
- } else {
- return false;
- }
+ const uint32_t slot = (uint32_t) ((seq - 1) % depth);
- const ggml_tensor * src0 = last_node.src0();
+ req.seq = seq;
+ req.n_bufs = op_batch->n_bufs;
+ req.n_tensors = op_batch->n_tens;
+ req.n_ops = op_batch->n_ops;
- if (src0->ne[0] != weight->ne[0] || src0->ne[0] != node.dst()->ne[0]) {
- return false;
+ if (opt_profile) {
+ op_cache[slot].assign(op_batch->ops.begin(), op_batch->ops.begin() + req.n_ops);
}
- const bool is_row_bcast = (weight->ne[1] == 1 && weight->ne[2] == 1 && weight->ne[3] == 1);
- const bool is_same_shape = (src0->ne[0] == weight->ne[0] && src0->ne[1] == weight->ne[1] &&
- src0->ne[2] == weight->ne[2] && src0->ne[3] == weight->ne[3]);
- if (!is_row_bcast && !is_same_shape) return false;
+ const size_t b_size = sizeof(htp_buf_desc) * req.n_bufs;
+ const size_t t_size = sizeof(htp_tensor) * req.n_tensors;
+ const size_t o_size = sizeof(htp_op_desc) * req.n_ops;
+ const size_t p_size = sizeof(htp_prof_desc) * req.n_ops;
- if (!ggml_are_same_shape(src0, node.dst())) {
- return false;
- }
- if (ggml_is_contiguous(src0) != ggml_is_contiguous(node.dst())) {
- return false;
+ size_t tr_size = 0;
+ if (opt_profile == 3) {
+ req.n_traces = opt_optrace;
+ tr_size = (HTP_MAX_NTHREADS + 1) * req.n_traces * sizeof(htp_trace_desc);
+ } else {
+ req.n_traces = 0;
}
- struct htp_unary_kernel_params new_kparams;
- ggml_hexagon_precompute_unary_params(
- sess, HTP_OP_RMS_NORM_MUL, src0, weight, node.dst(), &new_kparams
- );
-
- if ((size_t) new_kparams.vtcm_size > sess->vtcm_size) {
- HEX_VERBOSE("ggml-hex: %s skip RMS_NORM_MUL fusion: VTCM needed (%d) > budget (%zu)\n",
- sess->c_name(), new_kparams.vtcm_size, sess->vtcm_size);
- return false;
- }
+ dbuf.ptr = shm_buf->base() + ((size_t) slot * shm_blk_size);
+ dbuf.fd = shm_buf->fd();
+ dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;
+ dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base();
+ dbuf.size = b_size + t_size + o_size + p_size + tr_size;
- if (!try_fuse_common(weight, node.dst())) {
- return false;
- }
+ GGML_ASSERT(dbuf.size <= shm_blk_size);
- last_node.opcode = HTP_OP_RMS_NORM_MUL;
- last_node.name = "RMS_NORM+MUL";
- last_node.inputs.clear();
- last_node.inputs.push_back(src0);
- last_node.inputs.push_back(weight);
- last_node.outputs.clear();
- last_node.outputs.push_back(node.dst());
- last_node.fused.push_back(node.node);
- memcpy(last_node.kernel_params, &new_kparams, sizeof(new_kparams));
-
- htp_op_desc & o = h_ops[n_ops - 1];
- o.opcode = HTP_OP_RMS_NORM_MUL;
- memcpy(o.kernel_params, &new_kparams, sizeof(new_kparams));
-
- o.src[0] = add_tensor(src0);
- o.src[1] = add_tensor(weight);
- for (uint32_t s = 2; s < HTP_OP_MAX_INPUTS; s++) {
- o.src[s] = 0xffff;
- }
- o.dst[0] = add_tensor(node.dst());
- for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) {
- o.dst[d] = 0xffff;
- }
+ uint8_t * m_ptr = (uint8_t*) dbuf.ptr;
+ uint8_t * b_ptr = m_ptr; m_ptr += b_size;
+ uint8_t * t_ptr = m_ptr; m_ptr += t_size;
+ uint8_t * o_ptr = m_ptr;
- HEX_VERBOSE("ggml-hex: %s fused RMS_NORM+MUL (#%u)\n", sess->c_name(), n_ops - 1);
- return true;
- }
-
- bool try_fuse_mul_mat_add(const htp_opnode & node) {
- if (n_ops == 0) return false;
- if (node.opcode != HTP_OP_ADD) return false;
-
- htp_opnode & last_node = ops[n_ops - 1];
- if (last_node.opcode != HTP_OP_MUL_MAT) return false;
-
- const ggml_tensor * add_src0 = node.src0();
- const ggml_tensor * add_src1 = node.src1();
- const ggml_tensor * mm_out = last_node.dst();
-
- if (!ggml_hexagon_tensor_is_fuseable(mm_out)) return false;
-
- const ggml_tensor * src2;
- if (add_src0 == mm_out || add_src0->data == mm_out->data) {
- src2 = add_src1;
- } else if (add_src1 == mm_out || add_src1->data == mm_out->data) {
- src2 = add_src0;
- } else {
- return false;
- }
-
- const ggml_tensor * src0 = last_node.src0();
- const ggml_tensor * src1 = last_node.src1();
-
- if (src2->type != GGML_TYPE_F32) return false;
-
- // the fused kernel writes the shape of the MUL_MAT output: the ADD must not broadcast it
- if (!ggml_are_same_shape(mm_out, node.dst())) return false;
-
- const struct htp_mm_kernel_params * orig_kparams = (const struct htp_mm_kernel_params *) last_node.kernel_params;
- struct htp_mm_kernel_params kparams;
- ggml_hexagon_precompute_fused_matmul_add_params(sess, src0, src1, src2, node.dst(), &kparams);
- if (kparams.kernel_type == HTP_MM_KERNEL_UNSUPPORTED) {
- return false;
- }
-
- if (orig_kparams->collapse != kparams.collapse) {
- return false;
- }
-
- const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3];
- const bool can_fuse = (kparams.n_hmx > 0) || (src1_nrows == 1);
- if (!can_fuse) return false;
+ memcpy(b_ptr, (void *) op_batch->h_bufs.data(), b_size);
+ memcpy(t_ptr, (void *) op_batch->h_tens.data(), t_size);
+ memcpy(o_ptr, (void *) op_batch->h_ops.data(), o_size);
- if ((size_t) kparams.vtcm_size > sess->vtcm_size) {
- HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: VTCM needed (%d) > budget (%zu)\n",
- sess->c_name(), kparams.vtcm_size, sess->vtcm_size);
- return false;
- }
+ HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%llu : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n",
+ shm_buf->sess->c_name(), (unsigned long long) req.seq, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem,
+ b_size, t_size, o_size, (size_t) dbuf.size);
- if (kparams.n_hmx > 0 && orig_kparams->n_hmx > 0) {
- if (kparams.m_chunk < orig_kparams->m_chunk ||
- kparams.n_chunk < orig_kparams->n_chunk ||
- kparams.n_act_threads < orig_kparams->n_act_threads) {
- HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, th %d->%d)\n",
- sess->c_name(), orig_kparams->m_chunk, kparams.m_chunk,
- orig_kparams->n_chunk, kparams.n_chunk,
- orig_kparams->n_act_threads, kparams.n_act_threads);
- return false;
+ if (opt_verbose > 1) {
+ htp_buf_desc *b = (htp_buf_desc*) b_ptr;
+ for (unsigned int i=0; i < req.n_bufs; i++) {
+ GGML_LOG_DEBUG("ggml-hex: %s htp-buf #%u : fd %d base %p size %zu\n", shm_buf->sess->c_name(), i,
+ b[i].fd, (void *) b[i].base, (size_t) b[i].size);
+ }
+ htp_tensor *t = (htp_tensor*) t_ptr;
+ for (unsigned int i=0; i < req.n_tensors; i++) {
+ GGML_LOG_DEBUG("ggml-hex: %s htp-tensor #%u : bi %u offset %llu size %u : %zu:%zu:%zu:%zu\n",
+ shm_buf->sess->c_name(), i, t[i].bi, (unsigned long long) t[i].data, t[i].size,
+ (size_t) t[i].ne[0], (size_t) t[i].ne[1], (size_t) t[i].ne[2], (size_t) t[i].ne[3]);
}
}
- if (!try_fuse_common(src2, node.dst())) {
- return false;
- }
-
- last_node.opcode = HTP_OP_MUL_MAT_ADD;
- last_node.name = "MUL_MAT+ADD";
- last_node.inputs.clear();
- last_node.inputs.push_back(src0);
- last_node.inputs.push_back(src1);
- last_node.inputs.push_back(src2);
- last_node.outputs.clear();
- last_node.outputs.push_back(node.dst());
- last_node.fused.push_back(node.node);
- memcpy(last_node.kernel_params, &kparams, sizeof(kparams));
-
- htp_op_desc & o = h_ops[n_ops - 1];
- o.opcode = HTP_OP_MUL_MAT_ADD;
- memcpy(o.kernel_params, &kparams, sizeof(kparams));
-
- o.src[0] = add_tensor(src0);
- o.src[1] = add_tensor(src1);
- o.src[2] = add_tensor(src2);
- for (uint32_t s = 3; s < HTP_OP_MAX_INPUTS; s++) {
- o.src[s] = 0xffff;
- }
- o.dst[0] = add_tensor(node.dst());
- for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) {
- o.dst[d] = 0xffff;
- }
-
- HEX_VERBOSE("ggml-hex: %s fused MUL_MAT+ADD (#%u)\n", sess->c_name(), n_ops - 1);
return true;
}
- bool try_fuse_mul_mat_nx(const htp_opnode & node) {
- if (n_ops == 0 || node.opcode != HTP_OP_MUL_MAT) return false;
- if (!is_mergeable_mul_mat(node.node)) return false;
-
- const ggml_tensor * w_in = node.src0();
- const ggml_tensor * x_in = node.src1();
- const ggml_tensor * d_in = node.dst();
+ void pop(htp_opbatch_rsp rsp, dspqueue_buffer dbuf) {
+ const uint32_t slot = (uint32_t) ((rsp.seq - 1) % depth);
- htp_opnode & last_node = ops[n_ops - 1];
+ const size_t b_size = sizeof(htp_buf_desc) * rsp.n_bufs;
+ const size_t t_size = sizeof(htp_tensor) * rsp.n_tensors;
+ const size_t o_size = sizeof(htp_op_desc) * rsp.n_ops;
+ const size_t p_size = sizeof(htp_prof_desc) * rsp.n_ops;
- // Case 1: last_node is already MUL_MAT_NX
- if (last_node.opcode == HTP_OP_MUL_MAT_NX) {
- const uint32_t curr_n = (uint32_t) last_node.outputs.size();
- if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 1 >= HTP_OP_MAX_INPUTS) {
- return false;
- }
+ size_t tr_size = 0;
+ uint32_t n_traces = 0;
+ if (opt_profile == 3) {
+ n_traces = opt_optrace;
+ tr_size = (HTP_MAX_NTHREADS + 1) * n_traces * sizeof(htp_trace_desc);
+ }
- const ggml_tensor * w0 = last_node.inputs[0];
- const ggml_tensor * x = last_node.inputs[curr_n];
+ const size_t m_size = b_size + t_size + o_size + p_size + tr_size;
+ GGML_ASSERT(m_size <= shm_blk_size);
- if (x_in != x || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] ||
- ggml_hexagon_tensor_is_tiled(w_in) != ggml_hexagon_tensor_is_tiled(w0)) {
- return false;
- }
- if (!last_node.fused.empty() && (mm_is_hmx_eligible(last_node.fused[0]) != mm_is_hmx_eligible(node.node))) {
- return false;
- }
+ HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%llu : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n",
+ shm_buf->sess->c_name(), (unsigned long long) rsp.seq, rsp.n_bufs, rsp.n_tensors, rsp.n_ops,
+ (size_t) dbuf.size, b_size, t_size, o_size);
- const struct htp_mm_kernel_params * orig_kparams = (const struct htp_mm_kernel_params *) last_node.kernel_params;
- const bool collapse = orig_kparams->collapse && ggml_hexagon_matmul_can_collapse(w_in, x, d_in);
- if (orig_kparams->collapse && !collapse) {
- return false;
- }
+ uint8_t * m_ptr = (uint8_t*) dbuf.ptr;
+ uint8_t * p_ptr = m_ptr + (b_size + t_size + o_size);
- struct htp_mm_kernel_params kparams;
- if (collapse) {
- const ggml_tensor x_collapsed = ggml_hexagon_tensor_collapse_rows(x);
- ggml_hexagon_precompute_fused_mmnx_params(sess, w0, &x_collapsed, curr_n + 1, &kparams);
- kparams.collapse = 1;
- } else {
- ggml_hexagon_precompute_fused_mmnx_params(sess, w0, x, curr_n + 1, &kparams);
- }
- if (!is_supported_mul_mat_nx_kernel(w0, &kparams)) {
- return false;
- }
- if ((size_t) kparams.vtcm_size > sess->vtcm_size) {
- HEX_VERBOSE("ggml-hex: %s skip NX fusion: VTCM needed (%d) > budget (%zu)\n",
- sess->c_name(), kparams.vtcm_size, sess->vtcm_size);
- return false;
- }
+ if (rsp.n_ops > 0 && opt_profile) {
+ auto & ops = op_cache[slot];
+ GGML_ASSERT(rsp.n_ops <= ops.size());
- if (!try_fuse_common(w_in, d_in)) {
- return false;
+ const htp_prof_desc * pd = (const htp_prof_desc *) p_ptr;
+ const htp_trace_desc * trace_events = nullptr;
+ if (opt_profile == 3) {
+ trace_events = (const htp_trace_desc *) (p_ptr + p_size);
}
- last_node.inputs[curr_n] = w_in;
- last_node.inputs.push_back(x);
- last_node.outputs.push_back(d_in);
- last_node.fused.push_back(node.node);
- memcpy(last_node.kernel_params, &kparams, sizeof(kparams));
+ ggml_hexagon_dump_batch_prof(shm_buf->sess->c_name(), rsp);
- htp_op_desc & o = h_ops[n_ops - 1];
- memcpy(o.kernel_params, &kparams, sizeof(kparams));
-
- for (uint32_t s = 0; s <= curr_n + 1; s++) {
- o.src[s] = add_tensor(last_node.inputs[s]);
- }
- for (uint32_t s = curr_n + 2; s < HTP_OP_MAX_INPUTS; s++) {
- o.src[s] = 0xffff;
- }
- for (uint32_t d = 0; d <= curr_n; d++) {
- o.dst[d] = add_tensor(last_node.outputs[d]);
- }
- for (uint32_t d = curr_n + 1; d < HTP_OP_MAX_OUTPUTS; d++) {
- o.dst[d] = 0xffff;
+ for (uint32_t i = 0; i < rsp.n_ops; i++) {
+ ggml_hexagon_dump_op_prof(shm_buf->sess->c_name(), ops[i], pd[i]);
}
- HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_NX (N=%u, #%u)\n", sess->c_name(), curr_n + 1, n_ops - 1);
- return true;
+ ggml_hexagon_dump_trace_events(shm_buf->sess->c_name(), rsp, trace_events, n_traces);
}
+ }
+};
- // Case 2: last_node is single MUL_MAT
- if (last_node.opcode == HTP_OP_MUL_MAT) {
- if (!is_mergeable_mul_mat_pair(last_node.node, node.node)) {
- return false;
- }
-
- const ggml_tensor * w0 = last_node.src0();
- const ggml_tensor * x = last_node.src1();
- const ggml_tensor * w1 = node.src0();
- const ggml_tensor * dst_0 = last_node.dst();
- const ggml_tensor * dst_1 = node.dst();
-
- const struct htp_mm_kernel_params * orig_kparams = (const struct htp_mm_kernel_params *) last_node.kernel_params;
- const bool collapse = orig_kparams->collapse && ggml_hexagon_matmul_can_collapse(w1, x, dst_1);
- if (orig_kparams->collapse && !collapse) {
- return false;
- }
+void ggml_hexagon_session::flush_peers() {
+ auto vpeers = std::move(virt_peers);
+ virt_peers.clear();
+ for (auto * peer : vpeers) {
+ peer->flush_sync();
+ }
- struct htp_mm_kernel_params kparams;
- if (collapse) {
- const ggml_tensor x_collapsed = ggml_hexagon_tensor_collapse_rows(x);
- ggml_hexagon_precompute_fused_mmnx_params(sess, w0, &x_collapsed, 2, &kparams);
- kparams.collapse = 1;
- } else {
- ggml_hexagon_precompute_fused_mmnx_params(sess, w0, x, 2, &kparams);
- }
- if (!is_supported_mul_mat_nx_kernel(w0, &kparams)) {
- return false;
- }
- if ((size_t) kparams.vtcm_size > sess->vtcm_size) {
- HEX_VERBOSE("ggml-hex: %s skip NX fusion: VTCM needed (%d) > budget (%zu)\n",
- sess->c_name(), kparams.vtcm_size, sess->vtcm_size);
- return false;
- }
+ auto ppeers = std::move(phys_peers);
+ phys_peers.clear();
+ for (auto * peer : ppeers) {
+ peer->flush_async();
+ }
- if (!try_fuse_common(w1, node.dst())) {
- return false;
- }
+ for (auto & sub : this->mdev.sessions) {
+ sub->flush_peers();
+ }
+}
- last_node.opcode = HTP_OP_MUL_MAT_NX;
- last_node.name = "MUL_MAT_NX";
- last_node.inputs.clear();
- last_node.inputs.push_back(w0);
- last_node.inputs.push_back(w1);
- last_node.inputs.push_back(x);
- last_node.outputs.clear();
- last_node.outputs.push_back(dst_0);
- last_node.outputs.push_back(dst_1);
- last_node.fused.push_back(node.node);
- memcpy(last_node.kernel_params, &kparams, sizeof(kparams));
-
- htp_op_desc & o = h_ops[n_ops - 1];
- o.opcode = HTP_OP_MUL_MAT_NX;
- memcpy(o.kernel_params, &kparams, sizeof(kparams));
-
- o.src[0] = add_tensor(w0);
- o.src[1] = add_tensor(w1);
- o.src[2] = add_tensor(x);
- for (uint32_t s = 3; s < HTP_OP_MAX_INPUTS; s++) {
- o.src[s] = 0xffff;
- }
- o.dst[0] = add_tensor(dst_0);
- o.dst[1] = add_tensor(dst_1);
- for (uint32_t d = 2; d < HTP_OP_MAX_OUTPUTS; d++) {
- o.dst[d] = 0xffff;
- }
+void ggml_hexagon_session::flush_async() {
+ flush_peers();
+ flush_batch();
+}
- HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_NX (N=2, #%u)\n", sess->c_name(), n_ops - 1);
- return true;
+void ggml_hexagon_session::flush_pending(bool all) {
+ for (auto & sub : this->mdev.sessions) {
+ sub->flush_pending(all);
+ if (sub->last_error > HTP_STATUS_OK) {
+ this->last_error = sub->last_error.load();
}
-
- return false;
}
- bool try_fuse_mul_mat_id_nx(const htp_opnode & node) {
- if (n_ops == 0 || node.opcode != HTP_OP_MUL_MAT_ID) return false;
- if (!is_mergeable_mul_mat_id(node.node)) return false;
-
- const ggml_tensor * w_in = node.src0();
- const ggml_tensor * x_in = node.src1();
- const ggml_tensor * ids_in = node.node->src[2];
- const ggml_tensor * d_in = node.dst();
-
- htp_opnode & last_node = ops[n_ops - 1];
-
- // Case 1: last_node is already MUL_MAT_ID_NX
- if (last_node.opcode == HTP_OP_MUL_MAT_ID_NX) {
- const uint32_t curr_n = (uint32_t) last_node.outputs.size();
- if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 2 >= HTP_OP_MAX_INPUTS) {
- return false;
- }
-
- const ggml_tensor * w0 = last_node.inputs[0];
- const ggml_tensor * x = last_node.inputs[curr_n];
- const ggml_tensor * ids = last_node.inputs[curr_n + 1];
-
- if (x_in != x || ids_in != ids || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] || w_in->ne[2] != w0->ne[2] ||
- ggml_hexagon_tensor_is_tiled(w_in) != ggml_hexagon_tensor_is_tiled(w0)) {
- return false;
- }
- if (!last_node.fused.empty() && (mm_is_hmx_eligible(last_node.fused[0]) != mm_is_hmx_eligible(node.node))) {
- return false;
- }
-
- struct htp_mm_kernel_params kparams;
- ggml_hexagon_precompute_fused_mmidnx_params(sess, w0, x, d_in, curr_n + 1, &kparams);
- if (!is_supported_mul_mat_id_nx_kernel(w0, &kparams)) {
- return false;
- }
- if ((size_t) kparams.vtcm_size > sess->vtcm_size) {
- HEX_VERBOSE("ggml-hex: %s skip ID NX fusion: VTCM needed (%d) > budget (%zu)\n",
- sess->c_name(), kparams.vtcm_size, sess->vtcm_size);
- return false;
- }
+ while (this->batch_rsp_seq < this->batch_req_seq) {
+ struct htp_opbatch_rsp rsp;
+ uint32_t rsp_size;
+ uint32_t flags;
- if (!try_fuse_common(w_in, d_in)) {
- return false;
- }
+ struct dspqueue_buffer dbuf;
+ uint32_t n_dbufs;
- last_node.inputs[curr_n] = w_in;
- last_node.inputs[curr_n + 1] = x;
- last_node.inputs.push_back(ids);
- last_node.outputs.push_back(d_in);
- last_node.fused.push_back(node.node);
- memcpy(last_node.kernel_params, &kparams, sizeof(kparams));
+ // Read response packet from queue
+ const uint32_t timeo = opt_oppoll ? 0 : DSPQUEUE_TIMEOUT;
- htp_op_desc & o = h_ops[n_ops - 1];
- memcpy(o.kernel_params, &kparams, sizeof(kparams));
+ int err = dspqueue_read(this->queue, &flags, 1, &n_dbufs, &dbuf, sizeof(rsp), &rsp_size, (uint8_t *) &rsp, timeo);
+ if (err == AEE_EEXPIRED || err == AEE_EWOULDBLOCK) {
+ continue;
+ }
- for (uint32_t s = 0; s <= curr_n + 2; s++) {
- o.src[s] = add_tensor(last_node.inputs[s]);
- }
- for (uint32_t s = curr_n + 3; s < HTP_OP_MAX_INPUTS; s++) {
- o.src[s] = 0xffff;
- }
- for (uint32_t d = 0; d <= curr_n; d++) {
- o.dst[d] = add_tensor(last_node.outputs[d]);
- }
- for (uint32_t d = curr_n + 1; d < HTP_OP_MAX_OUTPUTS; d++) {
- o.dst[d] = 0xffff;
- }
+ if (err != 0) {
+ GGML_ABORT("ggml-hex: dspqueue_read failed: 0x%08x\n", (unsigned) err);
+ }
- HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_ID_NX (N=%u, #%u)\n", sess->c_name(), curr_n + 1, n_ops - 1);
- return true;
+ // Basic sanity checks
+ if (rsp_size != sizeof(rsp) || n_dbufs != 1) {
+ GGML_ABORT("ggml-hex: %s dspcall : bad response : size %u dspbufs %u\n", this->c_name(), rsp_size, n_dbufs);
}
- // Case 2: last_node is single MUL_MAT_ID
- if (last_node.opcode == HTP_OP_MUL_MAT_ID) {
- if (!is_mergeable_mul_mat_id_pair(last_node.node, node.node)) {
- return false;
+ if (rsp.status > HTP_STATUS_OK) {
+ GGML_LOG_ERROR("ggml-hex: %s dspcall : dsp-rsp %s\n", this->c_name(), status_to_str(rsp.status));
+ this->last_error = rsp.status;
+ for (auto & sub : this->mdev.sessions) {
+ sub->last_error = rsp.status;
}
+ }
- const ggml_tensor * w0 = last_node.src0();
- const ggml_tensor * x = last_node.src1();
- const ggml_tensor * ids = last_node.node->src[2];
- const ggml_tensor * w1 = node.src0();
-
- struct htp_mm_kernel_params kparams;
- ggml_hexagon_precompute_fused_mmidnx_params(sess, w0, x, node.dst(), 2, &kparams);
- if (!is_supported_mul_mat_id_nx_kernel(w0, &kparams)) {
- return false;
- }
- if ((size_t) kparams.vtcm_size > sess->vtcm_size) {
- HEX_VERBOSE("ggml-hex: %s skip ID NX fusion: VTCM needed (%d) > budget (%zu)\n",
- sess->c_name(), kparams.vtcm_size, sess->vtcm_size);
- return false;
- }
+ op_queue->pop(rsp, dbuf);
- if (!try_fuse_common(w1, node.dst())) {
- return false;
- }
+ GGML_ASSERT(rsp.seq == this->batch_rsp_seq + 1);
+ this->batch_rsp_seq = rsp.seq;
- const ggml_tensor * dst_0 = last_node.dst();
- const ggml_tensor * dst_1 = node.dst();
-
- last_node.opcode = HTP_OP_MUL_MAT_ID_NX;
- last_node.name = "MUL_MAT_ID_NX";
- last_node.inputs.clear();
- last_node.inputs.push_back(w0);
- last_node.inputs.push_back(w1);
- last_node.inputs.push_back(x);
- last_node.inputs.push_back(ids);
- last_node.outputs.clear();
- last_node.outputs.push_back(dst_0);
- last_node.outputs.push_back(dst_1);
- last_node.fused.push_back(node.node);
- memcpy(last_node.kernel_params, &kparams, sizeof(kparams));
-
- htp_op_desc & o = h_ops[n_ops - 1];
- o.opcode = HTP_OP_MUL_MAT_ID_NX;
- memcpy(o.kernel_params, &kparams, sizeof(kparams));
-
- o.src[0] = add_tensor(w0);
- o.src[1] = add_tensor(w1);
- o.src[2] = add_tensor(x);
- o.src[3] = add_tensor(ids);
- for (uint32_t s = 4; s < HTP_OP_MAX_INPUTS; s++) {
- o.src[s] = 0xffff;
- }
- o.dst[0] = add_tensor(dst_0);
- o.dst[1] = add_tensor(dst_1);
- for (uint32_t d = 2; d < HTP_OP_MAX_OUTPUTS; d++) {
- o.dst[d] = 0xffff;
- }
+ if (!all) break;
+ }
+}
- HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_ID_NX (N=2, #%u)\n", sess->c_name(), n_ops - 1);
- return true;
- }
+void ggml_hexagon_session::flush_sync(bool all) {
+ flush_async();
+ flush_pending(all);
+}
- return false;
+void ggml_hexagon_session::start_batch() {
+ if (this->mdev.count > 1) {
+ enqueue_mdev_group();
}
+}
- bool try_fuse_gdn_cpy(const htp_opnode & node) {
- if (n_ops == 0 || node.opcode != HTP_OP_CPY) return false;
-
- htp_opnode & last_node = ops[n_ops - 1];
- if (last_node.opcode != HTP_OP_GATED_DELTA_NET) return false;
- if (last_node.outputs.size() != 1) return false;
+void ggml_hexagon_session::flush_batch(size_t min_ops) {
+ if (op_batch->n_ops < min_ops) { return; }
- const ggml_tensor * gdn_out = last_node.dst();
- const ggml_tensor * cpy_node = node.node;
- const ggml_tensor * cpy_src = node.src0();
- const ggml_tensor * cpy_dst = node.dst();
+ op_batch->sort_buffers();
- if (!cpy_src || !cpy_dst || !cpy_dst->data) return false;
- if (gdn_out->type != GGML_TYPE_F32 || cpy_src->type != GGML_TYPE_F32 || cpy_dst->type != GGML_TYPE_F32) return false;
- if ((gdn_out->flags & GGML_TENSOR_FLAG_OUTPUT) || (cpy_node->flags & GGML_TENSOR_FLAG_OUTPUT)) return false;
+ htp_opbatch_req req {};
+ dspqueue_buffer dbuf{};
- const ggml_tensor * v = last_node.node->src[2];
- if (!v) return false;
+ const uint64_t seq = ++this->batch_req_seq;
- const int64_t S_v = v->ne[0];
- const int64_t H = v->ne[1];
- const int64_t n_tokens = v->ne[2];
- const int64_t n_seqs = v->ne[3];
- const int64_t K = ggml_get_op_params_i32(last_node.node, 0);
- const size_t tail_off = (size_t) S_v * H * n_tokens * n_seqs * sizeof(float);
+ op_batch->update_mdev_group(this->mdev.idx);
- const int64_t D = S_v * S_v * H;
- const int64_t n_written = std::min<int64_t>(n_tokens, K);
+ if (!op_queue->push(req, dbuf, op_batch, seq)) {
+ flush_pending(false);
+ op_queue->push(req, dbuf, op_batch, seq);
+ }
- if (cpy_src->op != GGML_OP_VIEW || (cpy_src->view_src != gdn_out && cpy_src->view_src->data != gdn_out->data) ||
- cpy_src->view_offs != tail_off || !ggml_is_contiguous(cpy_src)) {
- return false;
- }
+ for (auto & sub : this->mdev.sessions) {
+ htp_opbatch_req sub_req {};
+ dspqueue_buffer sub_dbuf{};
- if (cpy_dst->ne[0] != D || cpy_dst->ne[1] != n_seqs || cpy_dst->nb[0] != sizeof(float)) {
- return false;
- }
- if (n_seqs > 1 && cpy_dst->nb[1] != (size_t) D * sizeof(float)) {
- return false;
- }
- if (n_written > 1) {
- if (cpy_dst->ne[2] != n_written || cpy_dst->nb[2] != (size_t) D * n_seqs * sizeof(float)) {
- return false;
- }
- }
+ sub->batch_req_seq = seq;
+ op_batch->update_mdev_group(sub->mdev.idx);
- if (!try_fuse_common({cpy_dst})) {
- return false;
+ if (!sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq)) {
+ sub->flush_pending(false);
+ sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq);
}
- last_node.name += "+CPY";
- last_node.outputs.push_back(cpy_dst);
- last_node.fused.push_back(node.node);
+ HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", sub->c_name(), sub_dbuf.ptr, sub_dbuf.size);
- htp_op_desc & o = h_ops[n_ops - 1];
- o.dst[1] = add_tensor(cpy_dst);
- for (uint32_t d = 2; d < HTP_OP_MAX_OUTPUTS; d++) {
- o.dst[d] = 0xffff;
+ int err = dspqueue_write(sub->queue, 0, 1, &sub_dbuf, sizeof(sub_req), (const uint8_t*) &sub_req, DSPQUEUE_TIMEOUT);
+ if (err != 0) {
+ GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", sub->c_name(), (unsigned) err);
}
-
- HEX_VERBOSE("ggml-hex: %s fused GATED_DELTA_NET+CPY (#%u)\n", sess->c_name(), n_ops - 1);
- return true;
}
- bool try_fuse(const htp_opnode & node) {
- if (!opt_opfusion) return false;
- if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_ALLREDUCE_ADD) && try_fuse_allreduce_add(node)) return true;
- if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_RMS_NORM_MUL) && try_fuse_rms_norm_mul(node)) return true;
- if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_ADD) && try_fuse_mul_mat_add(node)) return true;
- if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_NX) && try_fuse_mul_mat_nx(node)) return true;
- if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_ID_NX) && try_fuse_mul_mat_id_nx(node)) return true;
- if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_GDN_CPY) && try_fuse_gdn_cpy(node)) return true;
- return false;
+ HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", this->c_name(), dbuf.ptr, dbuf.size);
+
+ int err = dspqueue_write(this->queue, 0, 1, &dbuf, sizeof(req), (const uint8_t*) &req, DSPQUEUE_TIMEOUT);
+ if (err != 0) {
+ GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", this->c_name(), (unsigned) err);
}
-};
-struct ggml_hexagon_registry {
- ggml_hexagon_registry(ggml_backend_reg_t reg);
- ~ggml_hexagon_registry();
-
- ggml_backend_device devices[GGML_HEXAGON_MAX_SESSIONS];
-};
-
-struct ggml_hexagon_opqueue {
- // Shared buffer for storing batches
- ggml_hexagon_shared_buffer *shm_buf;
- size_t shm_blk_size;
- size_t depth;
-
- using opvec = std::vector<htp_opnode>;
-
- std::vector<opvec> op_cache; // per batch op cache
- std::vector<uint64_t> start_usec; // per batch start time
-
- ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) : depth(depth) {
- size_t n_bufs = HTP_OP_MAX_BUFS;
- size_t n_ops = batch_size;
- size_t n_tensors = n_ops * HTP_OP_MAX_OUTPUTS + n_ops * HTP_OP_MAX_INPUTS;
-
- size_t tr_size = 0;
- if (opt_profile == 3) {
- tr_size = (HTP_MAX_NTHREADS + 1) * opt_optrace * sizeof(htp_trace_desc);
- }
-
- shm_blk_size = sizeof(htp_buf_desc) * n_bufs +
- sizeof(htp_tensor) * n_tensors +
- sizeof(htp_op_desc) * n_ops +
- sizeof(htp_prof_desc) * n_ops +
- tr_size;
-
- shm_buf = new ggml_hexagon_shared_buffer(sess, shm_blk_size * depth, true /* pinned */);
-
- op_cache.resize(depth);
- start_usec.resize(depth, 0);
-
- if (opt_verbose) {
- GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n",
- sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size);
- }
- }
-
- ~ggml_hexagon_opqueue() {
- delete shm_buf;
- }
-
- size_t shm_size() const { return shm_buf ? shm_buf->size() : 0; }
-
- // push new batch
- bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, const ggml_hexagon_opbatch* op_batch, uint64_t seq) {
- static_assert(sizeof(htp_opbatch_req) % 8 == 0, "sizeof(htp_opbatch_req) must be multiple of 8");
- static_assert(sizeof(htp_opbatch_rsp) % 8 == 0, "sizeof(htp_opbatch_rsp) must be multiple of 8");
- static_assert(sizeof(htp_buf_desc) % 8 == 0, "sizeof(htp_buf_desc) must be multiple of 8");
- static_assert(sizeof(htp_tensor) % 8 == 0, "sizeof(htp_tensor) must be multiple of 8");
- static_assert(sizeof(htp_op_desc) % 8 == 0, "sizeof(htp_op_desc) must be multiple of 8");
- static_assert(sizeof(htp_prof_desc) % 8 == 0, "sizeof(htp_prof_desc) must be multiple of 8");
-
- if (seq - shm_buf->sess->batch_rsp_seq > depth) { return false; }
-
- const uint32_t slot = (uint32_t) ((seq - 1) % depth);
-
- req.seq = seq;
- req.n_bufs = op_batch->n_bufs;
- req.n_tensors = op_batch->n_tens;
- req.n_ops = op_batch->n_ops;
-
- op_cache[slot] = op_batch->ops;
- start_usec[slot] = ggml_time_us();
-
- const size_t b_size = sizeof(htp_buf_desc) * req.n_bufs;
- const size_t t_size = sizeof(htp_tensor) * req.n_tensors;
- const size_t o_size = sizeof(htp_op_desc) * req.n_ops;
- const size_t p_size = sizeof(htp_prof_desc) * req.n_ops;
-
- size_t tr_size = 0;
- if (opt_profile == 3) {
- req.n_traces = opt_optrace;
- tr_size = (HTP_MAX_NTHREADS + 1) * req.n_traces * sizeof(htp_trace_desc);
- } else {
- req.n_traces = 0;
- }
-
- dbuf.ptr = shm_buf->base() + ((size_t) slot * shm_blk_size);
- dbuf.fd = shm_buf->fd();
- dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;
- dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base();
- dbuf.size = b_size + t_size + o_size + p_size + tr_size;
-
- GGML_ASSERT(dbuf.size <= shm_blk_size);
-
- uint8_t * m_ptr = (uint8_t*) dbuf.ptr;
- uint8_t * b_ptr = m_ptr; m_ptr += b_size;
- uint8_t * t_ptr = m_ptr; m_ptr += t_size;
- uint8_t * o_ptr = m_ptr;
-
- memcpy(b_ptr, (void *) op_batch->h_bufs.data(), b_size);
- memcpy(t_ptr, (void *) op_batch->h_tens.data(), t_size);
- memcpy(o_ptr, (void *) op_batch->h_ops.data(), o_size);
-
- HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%llu : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n",
- shm_buf->sess->c_name(), (unsigned long long) req.seq, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem,
- b_size, t_size, o_size, (size_t) dbuf.size);
-
- if (opt_verbose > 1) {
- htp_buf_desc *b = (htp_buf_desc*) b_ptr;
- for (unsigned int i=0; i < req.n_bufs; i++) {
- GGML_LOG_DEBUG("ggml-hex: %s htp-buf #%u : fd %d base %p size %zu\n", shm_buf->sess->c_name(), i,
- b[i].fd, (void *) b[i].base, (size_t) b[i].size);
- }
- htp_tensor *t = (htp_tensor*) t_ptr;
- for (unsigned int i=0; i < req.n_tensors; i++) {
- GGML_LOG_DEBUG("ggml-hex: %s htp-tensor #%u : bi %u offset %llu size %u : %zu:%zu:%zu:%zu\n",
- shm_buf->sess->c_name(), i, t[i].bi, (unsigned long long) t[i].data, t[i].size,
- (size_t) t[i].ne[0], (size_t) t[i].ne[1], (size_t) t[i].ne[2], (size_t) t[i].ne[3]);
- }
- }
-
- return true;
- }
-
- void pop(htp_opbatch_rsp rsp, dspqueue_buffer dbuf) {
- const uint32_t slot = (uint32_t) ((rsp.seq - 1) % depth);
-
- const size_t b_size = sizeof(htp_buf_desc) * rsp.n_bufs;
- const size_t t_size = sizeof(htp_tensor) * rsp.n_tensors;
- const size_t o_size = sizeof(htp_op_desc) * rsp.n_ops;
- const size_t p_size = sizeof(htp_prof_desc) * rsp.n_ops;
-
- size_t tr_size = 0;
- uint32_t n_traces = 0;
- if (opt_profile == 3) {
- n_traces = opt_optrace;
- tr_size = (HTP_MAX_NTHREADS + 1) * n_traces * sizeof(htp_trace_desc);
- }
-
- const size_t m_size = b_size + t_size + o_size + p_size + tr_size;
- GGML_ASSERT(m_size <= shm_blk_size);
-
- HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%llu : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n",
- shm_buf->sess->c_name(), (unsigned long long) rsp.seq, rsp.n_bufs, rsp.n_tensors, rsp.n_ops,
- (size_t) dbuf.size, b_size, t_size, o_size);
-
- uint8_t * m_ptr = (uint8_t*) dbuf.ptr;
- uint8_t * p_ptr = m_ptr + (b_size + t_size + o_size);
-
- if (rsp.n_ops > 0) {
- auto & ops = op_cache[slot];
- GGML_ASSERT(rsp.n_ops <= ops.size());
-
- const htp_prof_desc * pd = (const htp_prof_desc *) p_ptr;
- const htp_trace_desc * trace_events = nullptr;
- if (opt_profile == 3) {
- trace_events = (const htp_trace_desc *) (p_ptr + p_size);
- }
-
- if (opt_profile) {
- ggml_hexagon_dump_batch_prof(shm_buf->sess->name, rsp);
- }
-
- for (uint32_t i = 0; i < rsp.n_ops; i++) {
- if (opt_profile) {
- ggml_hexagon_dump_op_prof(shm_buf->sess->name, ops[i], pd[i]);
- }
- }
-
- if (opt_profile) {
- ggml_hexagon_dump_trace_events(shm_buf->sess->name, rsp, trace_events, n_traces);
- }
- }
- }
-};
-
-void ggml_hexagon_session::flush_peers() {
- auto vpeers = std::move(virt_peers);
- virt_peers.clear();
- for (auto * peer : vpeers) {
- peer->flush_sync();
- }
-
- auto ppeers = std::move(phys_peers);
- phys_peers.clear();
- for (auto * peer : ppeers) {
- peer->flush_async();
- }
-
- for (auto & sub : this->mdev.sessions) {
- sub->flush_peers();
- }
-}
-
-void ggml_hexagon_session::flush_async() {
- flush_peers();
- flush_batch();
-}
-
-void ggml_hexagon_session::flush_pending(bool all) {
- for (auto & sub : this->mdev.sessions) {
- sub->flush_pending(all);
- if (sub->last_error > HTP_STATUS_OK) {
- this->last_error = sub->last_error.load();
- }
- }
-
- while (this->batch_rsp_seq < this->batch_req_seq) {
- struct htp_opbatch_rsp rsp;
- uint32_t rsp_size;
- uint32_t flags;
-
- struct dspqueue_buffer dbuf;
- uint32_t n_dbufs;
-
- // Read response packet from queue
- const uint32_t timeo = opt_oppoll ? 0 : DSPQUEUE_TIMEOUT;
-
- int err = dspqueue_read(this->queue, &flags, 1, &n_dbufs, &dbuf, sizeof(rsp), &rsp_size, (uint8_t *) &rsp, timeo);
- if (err == AEE_EEXPIRED || err == AEE_EWOULDBLOCK) {
- continue;
- }
-
- if (err != 0) {
- GGML_ABORT("ggml-hex: dspqueue_read failed: 0x%08x\n", (unsigned) err);
- }
-
- // Basic sanity checks
- if (rsp_size != sizeof(rsp) || n_dbufs != 1) {
- GGML_ABORT("ggml-hex: %s dspcall : bad response : size %u dspbufs %u\n", this->c_name(), rsp_size, n_dbufs);
- }
-
- if (rsp.status > HTP_STATUS_OK) {
- GGML_LOG_ERROR("ggml-hex: %s dspcall : dsp-rsp %s\n", this->c_name(), status_to_str(rsp.status));
- this->last_error = rsp.status;
- for (auto & sub : this->mdev.sessions) {
- sub->last_error = rsp.status;
- }
- }
-
- op_queue->pop(rsp, dbuf);
-
- GGML_ASSERT(rsp.seq == this->batch_rsp_seq + 1);
- this->batch_rsp_seq = rsp.seq;
-
- if (!all) break;
- }
-}
-
-void ggml_hexagon_session::flush_sync(bool all) {
- flush_async();
- flush_pending(all);
-}
-
-void ggml_hexagon_session::start_batch() {
- if (this->mdev.count > 1) {
- enqueue_mdev_group();
- }
-}
-
-void ggml_hexagon_session::flush_batch(size_t min_ops) {
- if (op_batch->n_ops < min_ops) { return; }
-
- op_batch->sort_buffers();
-
- htp_opbatch_req req {};
- dspqueue_buffer dbuf{};
-
- const uint64_t seq = ++this->batch_req_seq;
-
- op_batch->update_mdev_group(this->mdev.idx);
-
- if (!op_queue->push(req, dbuf, op_batch, seq)) {
- flush_pending(false);
- op_queue->push(req, dbuf, op_batch, seq);
- }
-
- for (auto & sub : this->mdev.sessions) {
- htp_opbatch_req sub_req {};
- dspqueue_buffer sub_dbuf{};
-
- sub->batch_req_seq = seq;
- op_batch->update_mdev_group(sub->mdev.idx);
-
- if (!sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq)) {
- sub->flush_pending(false);
- sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq);
- }
-
- HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", sub->c_name(), sub_dbuf.ptr, sub_dbuf.size);
-
- int err = dspqueue_write(sub->queue, 0, 1, &sub_dbuf, sizeof(sub_req), (const uint8_t*) &sub_req, DSPQUEUE_TIMEOUT);
- if (err != 0) {
- GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", sub->c_name(), (unsigned) err);
- }
- }
-
- HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", this->c_name(), dbuf.ptr, dbuf.size);
-
- int err = dspqueue_write(this->queue, 0, 1, &dbuf, sizeof(req), (const uint8_t*) &req, DSPQUEUE_TIMEOUT);
- if (err != 0) {
- GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", this->c_name(), (unsigned) err);
- }
-
- op_batch->reset();
-}
-
-void ggml_hexagon_session::enqueue_op(const htp_opnode & node) {
- auto clone_tensor_buffer = [this](const ggml_tensor * t) {
- auto sbuf = this->mmap_tensor(t);
- if (!sbuf) return;
- if (sbuf->sess != this) {
- this->clone_buffer(sbuf);
- }
- for (auto & sub : this->mdev.sessions) {
- sub->clone_buffer(sbuf);
- }
- };
-
- for (auto t : node.get_inputs()) {
- clone_tensor_buffer(t);
- }
- for (auto t : node.get_outputs()) {
- clone_tensor_buffer(t);
- }
-
- if (opt_opfusion && op_batch->try_fuse(node)) {
- return;
- }
-
- if (!op_batch->fit_op(node)) {
- flush_async();
- }
-
- if (op_batch->empty()) {
- start_batch();
- }
-
- if (!op_batch->fit_op(node)) {
- GGML_ABORT("ggml-hex: %s op does not fit into empty batch (vmem/tensor/buffer limit exceeded)\n",
- c_name());
- }
-
- op_batch->add_op(node);
-}
-
-void ggml_hexagon_session::enqueue_mdev_group() {
- htp_opnode group_node(HTP_OP_MDEV_GROUP);
-
- uint8_t * fence_slot = this->mdev_fence_slot;
-
- static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE };
- ggml_tensor dummy_t {};
- dummy_t.buffer = &this->fence_buf->backend_buffer;
- dummy_t.extra = &fence_extra;
- dummy_t.data = (void *) fence_slot;
- dummy_t.type = GGML_TYPE_I8;
- dummy_t.ne[0] = HTP_FENCE_SLOT_SIZE;
- dummy_t.ne[1] = (int64_t) this->mdev.count;
- dummy_t.ne[2] = 1;
- dummy_t.ne[3] = 1;
- dummy_t.nb[0] = 1;
- dummy_t.nb[1] = HTP_FENCE_SLOT_SIZE;
- dummy_t.nb[2] = dummy_t.nb[1] * dummy_t.ne[1];
- dummy_t.nb[3] = dummy_t.nb[2];
- dummy_t.op = GGML_OP_NONE;
- dummy_t.op_params[0] = (int32_t) this->mdev.idx;
-
- ggml_tensor * node = group_node.add_dummy(dummy_t);
- node->src[0] = node;
- group_node.init(node);
- group_node.outputs.clear();
- group_node.name = "MDEV_GROUP";
-
- for (auto & sub : this->mdev.sessions) {
- sub->clone_buffer(this->fence_buf);
- }
-
- op_batch->add_op(group_node);
-}
-
-void ggml_hexagon_session::enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor, uint32_t fence_seq) {
- const bool with_fence = sync_tensor != nullptr;
- htp_opnode cpy_node(with_fence ? HTP_OP_CPY_FENCE : HTP_OP_CPY);
-
- ggml_tensor* node = cpy_node.add_dummy(*dst);
- node->op = GGML_OP_CPY;
- node->src[0] = const_cast<ggml_tensor *>(src);
- node->src[1] = with_fence ? cpy_node.add_dummy(*sync_tensor) : nullptr;
- if (with_fence) {
- node->op_params[0] = (int32_t) fence_seq;
- }
-
- cpy_node.init(node);
- if (with_fence) {
- cpy_node.name = "CPY+FENCE";
- }
- const bool ok = ggml_hexagon_precompute_cpy_params(this, node, (struct htp_copy_kernel_params *) cpy_node.kernel_params);
- const auto * kparams = (const struct htp_copy_kernel_params *) cpy_node.kernel_params;
- if (ok && !with_fence && kparams->total_elems == 0) {
- return;
- }
- this->enqueue_op(cpy_node);
-}
-
-void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq, bool wait) {
- htp_opnode sync_node(HTP_OP_FENCE);
-
- ggml_tensor* node = sync_node.add_dummy(*sync_tensor);
- node->op = GGML_OP_NONE;
- node->src[0] = node;
- node->op_params[0] = (int32_t) fence_seq;
- node->op_params[1] = wait ? 0 : 1;
-
- sync_node.init(node);
- sync_node.name = wait ? "FENCE_WAIT" : "FENCE_SIGNAL";
- this->enqueue_op(sync_node);
-}
-
-static bool ggml_hexagon_precompute_allreduce_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * dst,
- uint32_t rank,
- uint32_t n_ranks,
- bool has_add,
- bool is_row_bcast,
- bool is_shard_ok,
- struct htp_allreduce_kernel_params * kparams
-) {
- memset(kparams, 0, sizeof(*kparams));
- kparams->rank = (int32_t) rank;
- kparams->n_ranks = (int32_t) n_ranks;
- kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0;
-
- const uint32_t nelem = (uint32_t) ggml_nelements(dst);
- const uint32_t elem_size = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);
- const bool is_contiguous = ggml_is_contiguous(dst);
-
- const uint32_t ne0 = (uint32_t) dst->ne[0];
- const uint32_t ne1 = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);
- kparams->ne0 = (int32_t) ne0;
- kparams->ne1 = (int32_t) ne1;
-
- const bool use_1d = is_contiguous && !(has_add && is_row_bcast && ne1 > 1);
-
- if (has_add) {
- // sharded reduce-scatter for contiguous in-place add
- if (opt_ar_scatter && is_shard_ok && use_1d && !is_row_bcast) {
- const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128);
- const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem);
- const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem);
- kparams->n_dsts = (int32_t) n_ranks;
- kparams->rank_elem_start = (int32_t) rank_elem_start;
- kparams->rank_nelem = (int32_t) (rank_elem_end - rank_elem_start);
- } else if (use_1d) {
- kparams->n_dsts = 1;
- kparams->rank_elem_start = 0;
- kparams->rank_nelem = (int32_t) nelem;
- } else {
- kparams->n_dsts = 1;
- kparams->rank_elem_start = 0;
- kparams->rank_nelem = (int32_t) ne1;
- }
- } else {
- kparams->n_dsts = (int32_t) n_ranks;
- if (use_1d) {
- const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128);
- const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem);
- const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem);
- const uint32_t rank_nelem = rank_elem_end - rank_elem_start;
- kparams->rank_elem_start = (int32_t) rank_elem_start;
- kparams->rank_nelem = (int32_t) rank_nelem;
- } else {
- const uint32_t rank_chunk_rows = (ne1 + n_ranks - 1) / n_ranks;
- const uint32_t rank_r0 = (std::min)(rank * rank_chunk_rows, ne1);
- const uint32_t rank_r1 = (std::min)(rank_r0 + rank_chunk_rows, ne1);
- const uint32_t rank_nrows = rank_r1 - rank_r0;
- kparams->rank_elem_start = (int32_t) rank_r0;
- kparams->rank_nelem = (int32_t) rank_nrows;
- }
- }
-
- kparams->mode = (kparams->n_dsts > 1) ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL;
-
- if (use_1d) {
- const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem;
- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nelem / 128));
- kparams->n_threads = n_threads;
- const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast);
-
- uint32_t block_elems = 65536;
- if (block_elems > rank_nelem / n_threads && rank_nelem / n_threads > 128) {
- block_elems = hex_round_up(rank_nelem / (n_threads * 2), 128);
- }
- block_elems = (std::max)(128u, block_elems);
-
- kparams->block_elems = block_elems;
- kparams->vtcm_size_per_thread = 2 * block_elems * elem_size;
- kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;
-
- while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_elems > 128) {
- const size_t max_bytes_per_buf = sess->vtcm_size / (n_vtcm_buffers * 2);
- block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128);
- if (block_elems < 128) break;
- kparams->block_elems = block_elems;
- kparams->vtcm_size_per_thread = 2 * block_elems * elem_size;
- kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;
- }
-
- if (sess->vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) {
- HEX_VERBOSE("ggml-hex: %s allreduce 1D solver failed to fit VTCM (%d > %zu)\n",
- sess->c_name(), kparams->vtcm_size, sess->vtcm_size);
- return false;
- }
-
- kparams->elems_per_thread = hex_round_up((rank_nelem + n_threads - 1) / n_threads, block_elems);
- kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_1D;
- return true;
- } else {
- const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem;
- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nrows));
- kparams->n_threads = n_threads;
- const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast);
-
- const uint32_t row_bytes = ne0 * elem_size;
- const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128);
- kparams->row_size_aligned = row_size_aligned;
-
- const uint32_t nrows_per_thread = (rank_nrows + n_threads - 1) / n_threads;
- uint32_t block_rows = (std::min)(128u, nrows_per_thread);
- block_rows = (std::max)(1u, block_rows);
- kparams->block_elems = block_rows;
-
- kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned);
- kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;
-
- while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_rows > 1) {
- const size_t max_rows_per_buf = sess->vtcm_size / (n_vtcm_buffers * 2 * row_size_aligned);
- block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf);
- kparams->block_elems = block_rows;
- kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned);
- kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;
- if (max_rows_per_buf == 0) break;
- }
-
- if (sess->vtcm_size < (size_t) kparams->vtcm_size || block_rows < 1) {
- HEX_VERBOSE("ggml-hex: %s allreduce 2D solver failed to fit VTCM (%d > %zu)\n",
- sess->c_name(), kparams->vtcm_size, sess->vtcm_size);
- return false;
- }
-
- kparams->elems_per_thread = nrows_per_thread;
- kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_2D;
- return true;
- }
-}
-
-void ggml_hexagon_session::enqueue_allreduce(
- const ggml_tensor * dst,
- const std::vector<const ggml_tensor *> & src_tensors,
- const std::vector<const ggml_tensor *> & sync_tensors,
- uint32_t rank,
- uint32_t n_ranks,
- uint32_t fence_seq_entry,
- uint32_t fence_seq_exit
-) {
- htp_opnode ar_node(HTP_OP_ALLREDUCE);
-
- ggml_tensor* node = ar_node.add_dummy(*dst);
- node->op = GGML_OP_NONE;
- node->op_params[0] = (int32_t) fence_seq_entry;
- node->op_params[1] = (int32_t) fence_seq_exit;
-
- ar_node.init(node);
-
- ar_node.inputs.clear();
- for (size_t i = 0; i < src_tensors.size(); i++) {
- ar_node.inputs.push_back(src_tensors[i]);
- }
- for (size_t i = 0; i < sync_tensors.size(); i++) {
- ar_node.inputs.push_back(ar_node.add_dummy(*sync_tensors[i]));
- }
-
- ar_node.outputs.clear();
- for (size_t i = 0; i < src_tensors.size(); i++) {
- ar_node.outputs.push_back(src_tensors[i]);
- }
-
- ggml_hexagon_precompute_allreduce_params(
- this, dst, rank, n_ranks, false, false, /*is_shard_ok=*/ false,
- (struct htp_allreduce_kernel_params *) ar_node.kernel_params);
-
- ar_node.name = "ALLREDUCE";
- this->enqueue_op(ar_node);
-}
-
-ggml_hexagon_shared_buffer * ggml_hexagon_session::mmap_tensor(const ggml_tensor * t) {
- if (!t) return nullptr;
-
- auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
- if (!sbuf->mapped) {
- const bool is_weight = ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS;
- const bool extended = opt_dma64 && is_weight;
- sbuf->mmap(extended);
- }
- return sbuf;
-}
-
-bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf)
-{
- GGML_ASSERT(sbuf && sbuf->mem);
- if (sbuf->sess == this) return true;
-
- auto mem = sbuf->mem;
- int fd = mem->fd;
-
- GGML_ASSERT(fd >= 0);
-
- if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return true;
-
- GGML_ASSERT(sbuf->mapped);
-
- HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(),
- sbuf->c_name(), sbuf->base(), sbuf->size(), fd);
-
- auto clone = std::make_unique<ggml_hexagon_shared_buffer>(this, *sbuf);
- try {
- clone->mmap(sbuf->extended);
- } catch (const std::exception & exc) {
- GGML_LOG_ERROR("ggml-hex: %s lazy mapping of buffer context failed: %s\n", this->c_name(), exc.what());
- return false;
- }
-
- this->cloned_buffers[fd] = std::move(clone);
- mem->mapped_clones.insert(this);
- return true;
-}
-
-void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbuf) {
- GGML_ASSERT(sbuf && sbuf->mem);
-
- auto mem = sbuf->mem;
- int fd = mem->fd;
-
- GGML_ASSERT(fd >= 0);
-
- auto it = this->cloned_buffers.find(fd);
- if (it != this->cloned_buffers.end()) {
- auto clone = std::move(it->second);
- this->cloned_buffers.erase(it);
- }
- mem->mapped_clones.erase(this);
-}
-
-void ggml_hexagon_session::unclone_buffer(const ggml_hexagon_shared_buffer * sbuf) {
- GGML_ASSERT(sbuf && sbuf->mem);
-
- auto mem = sbuf->mem;
- std::vector<ggml_hexagon_session *> sessions(mem->mapped_clones.begin(), mem->mapped_clones.end());
-
- for (auto * sess : sessions) {
- sess->release_buffer(sbuf);
- }
-}
-
-static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) {
- // Allocate a bunch pinned buffers till failure.
- // This is kind of expensive but handy for figuring out exactly how much we can mmap on a specific device.
- // Typically we're going to allocate all/most of these buffers anyway for the model weights.
-
- std::vector<ggml_hexagon_shared_buffer *> sbufs;
-
- const size_t MiB = 1024 * 1024;
- const size_t GiB = MiB * 1024;
-
- size_t vmem = 0;
- size_t step = 256u * MiB;
-
- try {
- sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB;
- sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB;
- sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB;
-
- while (1) {
- sbufs.push_back(new ggml_hexagon_shared_buffer(sess, step, true));
- vmem += step;
- }
- } catch (...) { }
-
- for (auto b : sbufs) { delete b; }
-
- return vmem - step; // backoff to account for overhead from internal mappings
-}
-
-void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) noexcept(false) {
- int phys_idx = config.physical_idx;
- int virt_idx = config.virtual_idx;
-
- this->valid_session = false;
- this->valid_handle = false;
- this->valid_queue = false;
- this->valid_iface = false;
-
- this->name = config.name;
- this->phys_idx = phys_idx;
- this->virt_idx = virt_idx;
- this->domain_id = config.domain_id;
- this->session_id = 0;
- this->batch_req_seq = 0;
- this->batch_rsp_seq = 0;
- this->last_error = HTP_STATUS_OK;
-
- GGML_LOG_DEBUG("ggml-hex: %s allocating new session : domain %u phys-idx %u virt-idx %u\n", this->name.c_str(), this->domain_id, phys_idx, virt_idx);
-
- if (config.domain_id < 0 || config.domain_name.empty()) {
- GGML_LOG_ERROR("ggml-hex: %s: invalid physical CDSP core %d\n", config.name.c_str(), config.physical_idx);
- throw std::runtime_error("ggml-hex: invalid physical CDSP core");
- }
-
- const std::string & dom_name = config.domain_name;
-
- // Create new session if virtual_idx > 0
- if (virt_idx > 0) {
- struct remote_rpc_reserve_new_session n {};
- n.domain_name_len = dom_name.size();
- n.domain_name = const_cast<char *>(dom_name.c_str());
- n.session_name = const_cast<char *>(this->name.c_str());
- n.session_name_len = this->name.size();
- n.session_id = virt_idx;
-
- int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n));
- if (err != AEE_SUCCESS) {
- GGML_LOG_ERROR("ggml-hex: %s failed to reserve new session (physical %d, virtual %d) : error 0x%x\n",
- this->c_name(), phys_idx, virt_idx, err);
- throw std::runtime_error("ggml-hex: remote_session_control(new-sess) failed (see log for details)");
- }
-
- // Save the IDs
- this->session_id = n.session_id;
- this->domain_id = n.effective_domain_id;
- this->valid_session = true;
- } else {
- struct remote_rpc_effective_domain_id eff {};
- eff.domain_name = const_cast<char *>(dom_name.c_str());
- eff.domain_name_len = dom_name.size();
- eff.session_id = 0;
-
- int err = remote_session_control(FASTRPC_GET_EFFECTIVE_DOMAIN_ID, (void *) &eff, sizeof(eff));
- if (err == AEE_SUCCESS) {
- this->domain_id = eff.effective_domain_id;
- } else {
- GGML_LOG_DEBUG("ggml-hex: %s FASTRPC_GET_EFFECTIVE_DOMAIN_ID returned 0x%x, using domain_id %d\n",
- this->name.c_str(), err, this->domain_id);
- }
- }
-
- // Enable unsigned modules
- {
- struct remote_rpc_control_unsigned_module u;
- u.domain = this->domain_id;
- u.enable = 1;
- int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u));
- if (err != AEE_SUCCESS) {
- GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err);
- throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)");
- }
- }
-
- char session_uri[256];
- {
- char htp_uri[256];
- snprintf(htp_uri, sizeof(htp_uri), "file:///libggml-htp-v%u.so?htp_iface_skel_handle_invoke&_modver=1.0", opt_arch);
-
- struct remote_rpc_get_uri u = {};
- u.session_id = this->session_id;
- u.domain_name = const_cast<char *>(dom_name.c_str());
- u.domain_name_len = dom_name.size();
- u.module_uri = const_cast<char *>(htp_uri);
- u.module_uri_len = strlen(htp_uri);
- u.uri = session_uri;
- u.uri_len = sizeof(session_uri);
-
- int err = remote_session_control(FASTRPC_GET_URI, (void *) &u, sizeof(u));
- if (err != AEE_SUCCESS) {
- snprintf(session_uri, sizeof(session_uri), "%s&_dom=%s&_session=%u",
- htp_uri, dom_name.c_str(), this->session_id);
-
- GGML_LOG_WARN("ggml-hex: %s failed to get URI (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n",
- this->c_name(), phys_idx, virt_idx, err, session_uri);
- }
- }
-
- // Open session
- int err = htp_iface_open(session_uri, &this->handle);
- if (err != AEE_SUCCESS) {
- GGML_LOG_ERROR("ggml-hex: %s failed to open session : uri %s error 0x%x\n", this->c_name(), session_uri, err);
- throw std::runtime_error("ggml-hex: failed to open session (see log for details)");
- }
-
- this->valid_handle = true;
-
- // Query HW info and resolve session options
- {
- unsigned int hw_n_threads = 0;
- unsigned int hw_n_hvx = 0;
- unsigned int hw_n_hmx = 0;
- unsigned long long hw_vtcm_size = 0;
- int hw_err = htp_iface_hwinfo(this->handle, &hw_n_threads, &hw_n_hvx, &hw_n_hmx, &hw_vtcm_size);
- if (hw_err == 0) {
- const uint32_t max_n_threads = (std::min)((uint32_t) HTP_MAX_NTHREADS, (uint32_t) hw_n_threads);
- this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_threads) : max_n_threads;
- this->n_hvx = this->n_threads;
- this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0;
- this->vtcm_size = (uint64_t)hw_vtcm_size;
- GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n",
- this->c_name(), this->n_threads, this->n_hvx, this->n_hmx,
- (unsigned long long)(this->vtcm_size / (1024 * 1024)));
- } else {
- GGML_LOG_WARN("ggml-hex: %s failed to query hwinfo (0x%x), using defaults\n", this->c_name(), hw_err);
- const uint32_t default_n_threads = (std::min)(8u, (uint32_t) HTP_MAX_NTHREADS);
- this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads;
- this->n_hvx = this->n_threads;
- this->n_hmx = (opt_nhmx != 0) ? 1 : 0;
- this->vtcm_size = 8 * 1024 * 1024;
- }
- }
-
- // Enable FastRPC QoS mode
- {
- struct remote_rpc_control_latency l;
- l.enable = 1;
-
- int err = remote_handle64_control(this->handle, DSPRPC_CONTROL_LATENCY, (void *) &l, sizeof(l));
- if (err != 0) {
- GGML_LOG_WARN("ggml-hex: failed to enable fastrpc QOS mode: 0x%08x\n", (unsigned) err);
- }
- }
-
- GGML_LOG_INFO("ggml-hex: %s new session : session-id %d domain-id %d uri %s handle 0x%lx\n", this->c_name(),
- this->session_id, this->domain_id, session_uri, (unsigned long) this->handle);
-
- const size_t req_q_size = (sizeof(htp_opbatch_req) * opt_opqueue * 2) + 1024;
- const size_t rsp_q_size = (sizeof(htp_opbatch_rsp) * opt_opqueue * 2) + 1024;
-
- // Now let's setup the DSP queue
- err = dspqueue_create(this->domain_id,
- 0, // Flags
- req_q_size, // Request queue size (in bytes)
- rsp_q_size, // Response queue size (in bytes)
- nullptr, // Read packet callback (we handle reads explicitly)
- nullptr, // Error callback (we handle errors during reads)
- (void *) this, // Callback context
- &queue);
- if (err != 0) {
- GGML_LOG_ERROR("ggml-hex: %s dspqueue_create failed: 0x%08x\n", this->name.c_str(), (unsigned) err);
- throw std::runtime_error("ggml-hex: failed to create dspqueue (see log for details)");
- }
-
- this->valid_queue = true;
-
- // Export queue for use on the DSP
- err = dspqueue_export(queue, &this->queue_id);
- if (err != 0) {
- GGML_LOG_ERROR("ggml-hex: dspqueue_export failed: 0x%08x\n", (unsigned) err);
- throw std::runtime_error("ggml-hex: dspqueue export failed (see log for details)");
- }
-
- if (opt_etm) {
- err = htp_iface_etm(this->handle, 1);
- if (err != 0) {
- GGML_LOG_ERROR("ggml-hex: failed to enable ETM tracing: 0x%08x\n", (unsigned) err);
- }
- }
-
- // Allocate buffers and state for op batching
- this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue);
-
- this->fence_buf = new ggml_hexagon_fence_buffer(this, &dev_ctx->fence_buffer_type, 64 * 1024);
- if (this->mdev.count > 1) {
- this->mdev_fence_slot = this->alloc_fence(this->mdev.count);
- }
-
- if (!opt_vmem) {
- opt_vmem = ggml_hexagon_measure_max_vmem(this);
- GGML_LOG_INFO("ggml-hex: %s measured max vmem %zu\n", this->c_name(), opt_vmem);
- }
- const size_t shm_size = this->op_queue->shm_size();
- this->max_vmem = (opt_vmem > shm_size) ? (opt_vmem - shm_size) : opt_vmem;
-
- this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem);
-
- // Start dspqueue/opbatch processing
- err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem, (uint32_t) opt_power_mode);
- if (err != 0) {
- GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err);
- throw std::runtime_error("ggml-hex: iface start failed (see log for details)");
- }
- this->valid_iface = true;
- GGML_LOG_INFO("ggml-hex: %s power mode: %s\n", this->c_name(), htp_power_mode_to_str(opt_power_mode));
-
- if (opt_profile) {
- htp_iface_pmu_conf pmu_conf{};
- std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events);
-
- err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf);
- if (err != 0) {
- GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err);
- }
- }
-}
-
-void ggml_hexagon_session::release() noexcept(true) {
- GGML_LOG_INFO("ggml-hex: releasing session: %s\n", this->name.c_str());
-
- this->mdev.sessions.clear();
-
- int err;
-
- if (this->valid_iface) {
- // Stop dspqueue/opbatch processing
- err = htp_iface_stop(this->handle);
- if (err != 0) {
- GGML_ABORT("ggml-hex: htp_iface_stop failed: 0x%08x\n", (unsigned) err);
- }
- }
-
- delete this->op_batch;
- delete this->op_queue;
- for (auto & it : this->cpy_fence_slots) {
- free_fence((void *) it.second, 1);
- }
- this->cpy_fence_slots.clear();
-
- if (this->fence_buf) {
- unclone_buffer(this->fence_buf);
- delete this->fence_buf;
- this->fence_buf = nullptr;
- }
- while (!this->cloned_buffers.empty()) {
- release_buffer(this->cloned_buffers.begin()->second.get());
- }
-
- if (opt_etm) {
- err = htp_iface_etm(this->handle, 0);
- if (err != 0) {
- GGML_LOG_ERROR("ggml-hex: warn : failed to disable ETM tracing: 0x%08x\n", (unsigned) err);
- }
- }
-
- if (opt_profile) {
- htp_iface_pmu_conf pmu_conf{};
- err = htp_iface_profiler(this->handle, 0, &pmu_conf);
- if (err != 0) {
- GGML_LOG_ERROR("ggml-hex: warn : failed to disable profiling: 0x%08x\n", (unsigned) err);
- }
- }
-
- if (this->valid_queue) {
- err = dspqueue_close(queue);
- if (err != 0) {
- GGML_ABORT("ggml-hex: dspqueue_close failed: 0x%08x\n", (unsigned) err);
- }
- }
-
- if (this->valid_handle) {
- htp_iface_close(this->handle);
- }
-}
-
-ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev, uint32_t mdev_idx, uint32_t mdev_count) noexcept(false) {
- this->dev = dev;
- this->dev_ctx = static_cast<ggml_backend_hexagon_device_context *>(dev->context);
- this->mdev.idx = mdev_idx;
- this->mdev.count = mdev_count > 0 ? mdev_count : (uint32_t) (1 + config.mdev_group.size());
- op_batch = nullptr;
- op_queue = nullptr;
- fence_buf = nullptr;
- fence_seq = ((uintptr_t)this) & 0xFFFF;
-
- try {
- allocate(config);
- if (this->mdev.idx == 0 && !config.mdev_group.empty()) {
- for (size_t i = 0; i < config.mdev_group.size(); i++) {
- this->mdev.sessions.push_back(std::make_unique<ggml_hexagon_session>(
- config.mdev_group[i], this->dev, (uint32_t) (i + 1), this->mdev.count));
- }
- }
- } catch (const std::exception & exc) {
- release();
- throw;
- }
-}
-
-ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) {
- release();
-}
-
-// ** backend interface
-
-static bool ggml_hexagon_flash_attn_is_hmx_eligible(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * q,
- const struct ggml_tensor * k,
- const struct ggml_tensor * v,
- const struct ggml_tensor * sinks
-) {
- if (sess->n_hmx == 0) {
- return false;
- }
-
- if (opt_fa_select < 2) {
- return false;
- }
-
- if ((k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0) ||
- (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_Q8_0)) {
- return false;
- }
-
- const uint32_t DK = q->ne[0];
- const uint32_t DV = v->ne[0];
-
- // Head dims that are not multiples of 64 are handled by internally padding to
- // DK_pad/DV_pad = round_up(.,64) and zero-filling the tail lanes.
- if (DK % 8 != 0 || DV % 8 != 0) {
- return false;
- }
-
- GGML_UNUSED(sinks);
-
- // Explicit force mode
- if (opt_fa_select > 2) {
- return true;
- }
-
- const uint32_t M = q->ne[1];
-
- // Prefill or batched decode
- if (M > 1) {
- return true;
- }
-
- const uint32_t n_head = q->ne[2];
- const uint32_t n_kv_heads = k->ne[2];
- const uint32_t G = n_kv_heads > 0 ? n_head / n_kv_heads : 1;
- const uint32_t S = k->ne[1];
- const uint32_t total_threads = sess->n_threads * sess->mdev.count;
-
- // Compute-bound head dim (for DK == 256 decode, HVX is faster when KV heads span all threads)
- if (DK > 256 || (DK == 256 && (DV > 256 || n_kv_heads < total_threads))) {
- return true;
- }
-
- // Tile alignment for 32-row HMX tiles
- const bool is_tile_aligned = (G > 0 && (32 % G == 0));
- if (!is_tile_aligned) {
- if (sess->n_threads >= 6) {
- return false;
- }
- return S >= 1024;
- }
-
- // Context depth crossover
- uint32_t s_cross = 512;
- if (DK == 256) {
- s_cross = 4096;
- } else if (DK <= 64) {
- s_cross = (sess->n_threads >= 8) ? 2048 : ((sess->n_threads >= 6) ? 768 : 512);
- } else {
- s_cross = (sess->n_threads >= 8) ? 1024 : 512;
- }
-
- return S >= s_cross;
-}
-
-static bool ggml_hexagon_precompute_flash_attn_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_fa_kernel_params * kparams
-) {
- if (opt_fa_select < 1) {
- return false;
- }
-
- memset(kparams, 0, sizeof(*kparams));
-
- const struct ggml_tensor * q = op->src[0];
- const struct ggml_tensor * k = op->src[1];
- const struct ggml_tensor * v = op->src[2];
- const struct ggml_tensor * mask = op->src[3];
-
- const uint32_t neq0 = q->ne[0]; // head_dim (DK)
- const uint32_t neq1 = q->ne[1]; // n_tokens
- const uint32_t neq2 = q->ne[2]; // n_heads
-
- const uint32_t nek1 = k->ne[1]; // kv_len
-
- const uint32_t nev0 = v->ne[0]; // head_dim (DV)
-
- const uint32_t DK = neq0;
- const uint32_t DV = nev0;
-
- const uint32_t n_kv_heads = k->ne[2];
- const uint32_t G = neq2 / n_kv_heads;
-
- float scale = 1.0f;
- float max_bias = 0.0f;
- float logit_softcap = 0.0f;
- memcpy(&scale, &op->op_params[0], sizeof(float));
- memcpy(&max_bias, &op->op_params[1], sizeof(float));
- memcpy(&logit_softcap, &op->op_params[2], sizeof(float));
-
- if (logit_softcap != 0.0f) {
- scale /= logit_softcap;
- }
-
- kparams->scale = scale;
- kparams->max_bias = max_bias;
- kparams->logit_softcap = logit_softcap;
-
- kparams->head_split = (opt_fa_head_split != 0) ? 1 : 0;
- kparams->flags = 0;
- kparams->G = G;
-
- const uint32_t n_head = q->ne[2];
- kparams->n_head_log2 = 1u << (uint32_t) std::floor(std::log2(n_head));
- kparams->m0 = std::pow(2.0f, -(max_bias) / kparams->n_head_log2);
- kparams->m1 = std::pow(2.0f, -(max_bias / 2.0f) / kparams->n_head_log2);
-
- // Check HMX eligibility
- const struct ggml_tensor * sinks = op->src[4];
- if (ggml_hexagon_flash_attn_is_hmx_eligible(sess, q, k, v, sinks)) {
- // HMX tiles head_dim in units of 64; when DK/DV are not 64-aligned the kernel
- // operates on padded dims with zero-filled tail lanes. VTCM budget and chunk-size
- // are sized for the padded tiles.
- const uint32_t DK_pad = hex_round_up(DK, 64);
- const uint32_t DV_pad = hex_round_up(DV, 64);
- size_t Br = 0, Bc = 0;
- int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, sess->vtcm_size, sess->n_threads, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head);
- if (ret == 0) {
- kparams->kernel_type = HTP_FA_KERNEL_HMX;
- if (logit_softcap == 0.0f) {
- kparams->scale = scale * (float) M_LOG2E;
- kparams->logit_softcap = 0.0f;
- } else {
- kparams->scale = scale;
- kparams->logit_softcap = logit_softcap * (float) M_LOG2E;
- }
- kparams->Br = Br;
- kparams->Bc = Bc;
- kparams->n_kv_blocks = (nek1 + Bc - 1) / Bc;
- kparams->n_threads = (kparams->n_kv_blocks >= 3 && sess->n_threads >= 2) ? sess->n_threads : 1;
-
- kparams->u.hmx.g_br = hex_align_up(G * Br, 32);
- kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && sess->n_threads >= 2) ? 1 : 0;
- kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK_pad, DV_pad, Br, Bc, kparams->n_threads, kparams->u.hmx.pipeline != 0, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head);
-
- const size_t row_vec_bytes = hex_align_up(Bc * sizeof(uint16_t), 256);
- kparams->u.hmx.row_buf_stride = row_vec_bytes / 128; // HVX vector is 128 bytes
-
- const size_t m_line_bytes = hex_align_up(Bc * sizeof(uint16_t), 128);
- kparams->u.hmx.mask_buf_row_stride = m_line_bytes / sizeof(uint16_t);
- kparams->u.hmx.mask_broadcast = (mask != nullptr && mask->ne[2] == 1) ? 1 : 0;
- kparams->u.hmx.div_G = init_fastdiv_values(G);
- if (mask) {
- kparams->src3_div2 = init_fastdiv_values(mask->ne[2]);
- kparams->src3_div3 = init_fastdiv_values(mask->ne[3]);
- }
-
- kparams->qrows = 0;
- kparams->qrows_per_thread = 0;
- return true;
- }
- }
-
- // Fallback to HVX
- kparams->kernel_type = HTP_FA_KERNEL_HVX;
- kparams->Br = 1;
- kparams->Bc = 64; // FLASH_ATTN_BLOCK_SIZE
- kparams->n_kv_blocks = (k->ne[1] + 64 - 1) / 64;
- kparams->n_threads = sess->n_threads;
-
- const size_t size_q_row_padded = hex_round_up(q->ne[0] * ((q->type == GGML_TYPE_F32) ? 4 : 2), 128);
- const size_t size_k_row_padded = hex_round_up(k->ne[0] * 2, 128);
- const size_t size_v_row_padded = hex_round_up(v->ne[0] * 2, 128);
-
- kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, (q->type == GGML_TYPE_F32), mask != nullptr, sinks != nullptr, n_head, sess->n_threads);
-
- kparams->u.hvx.size_q_row_padded = size_q_row_padded;
- kparams->u.hvx.size_k_row_padded = size_k_row_padded;
- kparams->u.hvx.size_v_row_padded = size_v_row_padded;
- kparams->u.hvx.src0_div21 = init_fastdiv_values(q->ne[2] * q->ne[1]);
- kparams->u.hvx.src0_div1 = init_fastdiv_values(q->ne[1]);
- kparams->broadcast_rk2 = init_fastdiv_values(q->ne[2]/k->ne[2]);
- kparams->broadcast_rk3 = init_fastdiv_values(q->ne[3]/k->ne[3]);
- kparams->broadcast_rv2 = init_fastdiv_values(q->ne[2]/v->ne[2]);
- kparams->broadcast_rv3 = init_fastdiv_values(q->ne[3]/v->ne[3]);
- if (mask) {
- kparams->src3_div2 = init_fastdiv_values(mask->ne[2]);
- kparams->src3_div3 = init_fastdiv_values(mask->ne[3]);
- }
-
- kparams->qrows = q->ne[1] * q->ne[2] * q->ne[3];
- kparams->qrows_per_thread = (kparams->qrows + sess->n_threads - 1) / sess->n_threads;
-
- return true;
-}
-
-static bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * src1 = op->src[1];
- const struct ggml_tensor * src2 = op->src[2];
- const struct ggml_tensor * src3 = op->src[3];
- const struct ggml_tensor * src4 = op->src[4];
- const struct ggml_tensor * dst = op;
-
- // Check for F16/Q8_0 support
- if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) ||
- (src1->type != GGML_TYPE_F16 && src1->type != GGML_TYPE_Q8_0) ||
- (src2->type != GGML_TYPE_F16 && src2->type != GGML_TYPE_Q8_0)) {
- return false;
- }
-
- if (src3 && src3->type != GGML_TYPE_F16) { // mask
- return false;
- }
-
- if (src4 && src4->type != GGML_TYPE_F32) { // sinks
- return false;
- }
-
- // For now we support F32 or F16 output as htp backend often converts output on the fly if needed,
- // but the op implementation writes to F16 or F32.
- // Let's assume dst can be F32 or F16.
- if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) {
- return false;
- }
-
- struct htp_fa_kernel_params kparams;
- if (!ggml_hexagon_precompute_flash_attn_params(sess, op, &kparams)) {
- return false;
- }
-
- if ((size_t) kparams.vtcm_size > sess->vtcm_size) {
- HEX_VERBOSE("ggml-hex: skip flash_attn_ext because VTCM needed (%d) > budget (%zu)\n",
- kparams.vtcm_size, sess->vtcm_size);
- return false;
- }
-
- return true;
-}
-
-static bool ggml_hexagon_supported_gated_delta_net(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- if (opt_gdn_select < 1) {
- return false;
- }
-
- const struct ggml_tensor * q = op->src[0];
- const struct ggml_tensor * k = op->src[1];
- const struct ggml_tensor * v = op->src[2];
- const struct ggml_tensor * g = op->src[3];
- const struct ggml_tensor * beta = op->src[4];
- const struct ggml_tensor * state = op->src[5];
- const struct ggml_tensor * dst = op;
-
- if (q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F32 || v->type != GGML_TYPE_F32 ||
- g->type != GGML_TYPE_F32 || beta->type != GGML_TYPE_F32 || state->type != GGML_TYPE_F32 ||
- dst->type != GGML_TYPE_F32) {
- return false;
- }
-
- if (!ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) || !ggml_is_contiguous_rows(v) ||
- !ggml_is_contiguous(g) || !ggml_is_contiguous(beta) || !ggml_is_contiguous(state) ||
- !ggml_is_contiguous(dst)) {
- return false;
- }
-
- const int64_t S_v = v->ne[0];
- const int64_t H = v->ne[1];
- const int64_t n_tokens = v->ne[2];
- const int64_t n_seqs = v->ne[3];
- const int64_t K = ggml_get_op_params_i32(op, 0);
-
- if (S_v <= 0 || S_v > 128 || H <= 0 || n_tokens <= 0 || n_seqs <= 0) {
- return false;
- }
- if (q->ne[0] != S_v || k->ne[0] != S_v || q->ne[1] <= 0 || k->ne[1] <= 0 ||
- q->ne[2] != n_tokens || k->ne[2] != n_tokens || q->ne[3] <= 0 || k->ne[3] <= 0 ||
- (n_seqs % q->ne[3]) != 0 || (n_seqs % k->ne[3]) != 0) {
- return false;
- }
- if ((g->ne[0] != 1 && g->ne[0] != S_v) || beta->ne[0] != 1) {
- return false;
- }
- // state holds s0 only [S_v, S_v, H, n_seqs]; K is op param 0.
- if (ggml_nelements(state) != S_v * S_v * H * n_seqs) {
- return false;
- }
- if (dst->ne[0] != S_v * H || dst->ne[1] != n_tokens * n_seqs + S_v * n_seqs * K) {
- return false;
- }
-
- const uint32_t total_rows = (uint32_t) (H * n_seqs);
- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, total_rows);
-
- const bool can_use_hmx = (opt_gdn_select >= 2) &&
- (sess->n_hmx > 0) &&
- (S_v % 64 == 0) &&
- (n_tokens >= HTP_GDN_MIN_TOKENS) &&
- (g->ne[0] == 1) &&
- (K == 1);
-
- if (can_use_hmx) {
- struct htp_gdn_hmx_vtcm_layout layout;
- uint32_t n_heads_batch = 0;
- if (!htp_gdn_hmx_solve_layout(&layout, (uint32_t) S_v, HTP_GDN_CHUNK_SIZE, total_rows, sess->vtcm_size, n_threads, true, &n_heads_batch)) {
- return false;
- }
- } else {
- struct htp_gdn_vtcm_layout layout;
- htp_gdn_vtcm_layout_build(&layout, (uint32_t) S_v, n_threads);
- if (layout.total_bytes > sess->vtcm_size) {
- return false;
- }
- }
-
- return true;
-}
-
-static bool ggml_hexagon_matmul_is_hmx_eligible(
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * dst,
- bool is_matmul_id,
- bool is_batched
-) {
- GGML_UNUSED(dst);
-
- if (src1->type != GGML_TYPE_F32 && (src1->type != GGML_TYPE_F16 || is_matmul_id)) {
- return false;
- }
-
- const int ne00 = src0->ne[0];
- const int ne01 = src0->ne[1];
- const int ne11 = src1->ne[1];
- const int ne12 = src1->ne[2];
- const int wtype = src0->type;
-
- // HMX weight tiles accept non-32-aligned N for non-matmul_id.
- if (ne01 % HTP_TENSOR_TILE_N_ROWS != 0 && is_matmul_id) {
- return false;
- }
-
- // HMX supports F16, F32, and repack quantized types.
- if (!htp_is_hmx_weight_type((ggml_type) wtype)) {
- return false;
- }
-
- // HMX paths require K aligned to HMX tile cols (32).
- if (ne00 % HTP_TENSOR_TILE_N_COLS != 0) {
- return false;
- }
-
- // Quantized and tiled HMX kernels only handle flat 2D matmul (or matmul_id wrapping flat 2D matmuls).
- const bool is_tiled = ggml_hexagon_tensor_is_tiled(src0);
- if (ggml_is_quantized((ggml_type) wtype) && !is_tiled) {
- return false;
- }
- if (!is_matmul_id && is_batched && is_tiled) {
- return false;
- }
-
- // HMX assumes contiguous row-major layout.
- if (src0->nb[0] > src0->nb[1] || src1->nb[0] > src1->nb[1]) {
- return false;
- }
-
- // M alignment: Use HMX when M > HTP_MM_HMX_MIN_NROWS.
- // For MUL_MAT_ID, src1 shape is [K, n_expert_used, n_tokens, 1], so n_tokens is ne12.
- const int m = is_matmul_id ? ne12 : ne11;
- if (m <= HTP_MM_HMX_MIN_NROWS) {
- return false;
- }
-
- return true;
-
- GGML_UNUSED(dst);
-}
-
-static bool ggml_hexagon_precompute_hmx_mm_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * act,
- const struct ggml_tensor * dst,
- int wtype,
- int ne00_padded,
- int ne01_padded,
- int ne02,
- int ne11,
- int ne12,
- int ne11_padded,
- bool is_matmul_id,
- bool is_batched,
- size_t src2_size,
- size_t vtcm_budget,
- struct htp_mm_kernel_params * kparams
-) {
- const bool is_tiled = ggml_hexagon_tensor_is_tiled(src0);
- const int aligned_tile_size = is_tiled ? htp_tensor_get_tile_padded_size(wtype) : 0;
- const int n_threads = (int)sess->n_threads;
- const int ne10 = act->ne[0];
-
- int m_for_solver = ne11;
- int m_for_solver_padded = ne11_padded;
- // matmul_id partitions by expert; regular matmul partitions M rows (ne11) across devices
- if (!is_matmul_id && sess->mdev.count > 1 && ((uint32_t) ne11 >= sess->mdev.count)) {
- // when dst is null, padded dims are used for estimate which are 128-byte aligned
- const bool dst_row_split = dst ? ggml_hexagon_can_row_partition(dst) : true;
- const bool act_row_split = ggml_hexagon_can_row_partition(act);
- if (dst_row_split && act_row_split) {
- m_for_solver = (ne11 + (int) sess->mdev.count - 1) / (int) sess->mdev.count;
- m_for_solver_padded = hex_round_up(std::max(m_for_solver, 32), 32);
- }
- }
- const bool pipeline = is_matmul_id ? false : htp_mm_hmx_pipeline(m_for_solver);
-
- const bool is_batched_val = is_matmul_id ? false : is_batched;
- const int group_size = (ne02 > 0 ? ne12 / ne02 : 1);
-
- size_t m_chunk = 0;
- size_t n_chunk = 0;
- size_t vtcm_size = 0;
- bool use_grouped = false;
- int act_threads_selected = 0;
-
- if (is_batched_val && wtype == GGML_TYPE_F16 && !is_tiled && group_size > 1) {
- // Try grouped path first
- if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, m_for_solver, group_size, n_threads, pipeline, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) {
- use_grouped = true;
- }
- }
-
- if (!use_grouped) {
- // Fallback to simple 2D path (group_size = 1)
- int m_id_rows = 0;
- if (dst && is_matmul_id) {
- const int n_experts = ne02 > 0 ? ne02 : 1;
- const size_t total_expert_rows = (size_t) dst->ne[1] * dst->ne[2];
- int m_per_expert = (int) ((total_expert_rows + n_experts - 1) / n_experts);
- if (sess->mdev.count > 1 && ggml_hexagon_can_row_partition(dst)) {
- m_per_expert = (m_per_expert + (int) sess->mdev.count - 1) / (int) sess->mdev.count;
- }
- m_id_rows = hex_round_up(std::max(m_per_expert, 32), 32);
- }
- const uint32_t cost_m = is_matmul_id ? (uint32_t) m_id_rows : (uint32_t) m_for_solver;
- if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, m_for_solver_padded, cost_m, n_threads, pipeline, is_matmul_id, aligned_tile_size, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) {
- return false;
- }
- }
-
- kparams->n_hmx = 1;
- kparams->pipeline = pipeline ? 1 : 0;
- kparams->m_chunk = m_chunk;
- kparams->n_chunk = n_chunk;
- kparams->n_threads = n_threads;
- kparams->n_act_threads = act_threads_selected;
- kparams->tile_size = is_tiled ? htp_tensor_get_tile_size(wtype) : 0;
- kparams->aligned_tile_size = aligned_tile_size;
- kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
- kparams->vtcm_size = vtcm_size;
- kparams->vtcm_src0_size = 0;
- kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected);
- kparams->div_ne00_padded = init_fastdiv_values(ne00_padded);
- kparams->vtcm_act_size = 0;
- kparams->vtcm_bias_size = (int32_t) src2_size;
- kparams->vtcm_dst_size = 0;
-
- if (is_batched && !is_matmul_id) {
- kparams->kernel_type = HTP_MM_KERNEL_HMX_BATCHED;
- } else {
- kparams->kernel_type = HTP_MM_KERNEL_HMX_2D;
- }
- return true;
-}
-
-static void ggml_hexagon_precompute_hvx_mm_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * dst,
- int wtype,
- bool is_tiled,
- int ne02,
- int ne03,
- int ne10,
- int ne11,
- int ne12,
- int ne13,
- bool is_matmul_id,
- const size_t src2_row_size,
- size_t vtcm_budget,
- struct htp_mm_kernel_params * kparams
-) {
- if (opt_mm_select < 1) {
- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
- return;
- }
-
- kparams->n_hmx = 0;
- kparams->n_threads = sess->n_threads;
-
- GGML_UNUSED(ne02);
- GGML_UNUSED(ne03);
-
- const int src1_nrows = ne11 * ne12 * ne13;
-
- if (is_tiled) {
- // Tiled HVX (quantized weights or tiled F16/F32/BF16)
- kparams->tile_size = htp_tensor_get_tile_size(wtype);
- kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
-
- const bool k_align = (ne10 % htp_tensor_get_tile_k(wtype) == 0);
-
- if (is_matmul_id) {
- kparams->kernel_type = (src1_nrows < (int) sess->n_threads)
- ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW;
- kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
-
- struct htp_mm_hvx_vtcm_layout L;
- uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;
- uint32_t best_n_prefetch = 2;
- for (uint32_t d = max_prefetch; d >= 2; d /= 2) {
- htp_mm_hvx_vtcm_layout_build(
- &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads,
- 0, src0->nb[1], kparams->act_row_size, 0, d, true, false
- );
- if (L.total_bytes <= vtcm_budget) {
- best_n_prefetch = d;
- break;
- }
- }
- if (L.total_bytes > vtcm_budget) {
- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
- return;
- }
- kparams->n_prefetch = best_n_prefetch;
- kparams->vtcm_size = L.total_bytes;
- kparams->vtcm_src0_size = L.src0_bytes;
- kparams->vtcm_act_size = L.act_bytes;
- kparams->vtcm_bias_size = 0;
- kparams->vtcm_dst_size = L.dst_bytes;
- return;
- } else {
- bool try_tiled = (k_align && opt_mm_select >= 1);
- if (try_tiled) {
- kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
- if (src1_nrows < (int) sess->n_threads) {
- kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_BLOCK;
- } else {
- kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW;
- }
-
- struct htp_mm_hvx_vtcm_layout L;
- uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;
- uint32_t best_n_prefetch = 2;
- for (uint32_t d = max_prefetch; d >= 2; d /= 2) {
- htp_mm_hvx_vtcm_layout_build(
- &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads,
- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, d, false, false
- );
- if (L.total_bytes <= vtcm_budget) {
- best_n_prefetch = d;
- break;
- }
- }
-
- uint32_t m_chunk = 0;
- if (htp_mm_hvx_solve_vtcm_params(
- kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads,
- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, best_n_prefetch, vtcm_budget,
- &L, &m_chunk)) {
- kparams->n_prefetch = best_n_prefetch;
- kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0;
- kparams->vtcm_size = L.total_bytes;
- kparams->vtcm_src0_size = L.src0_bytes;
- kparams->vtcm_act_size = L.act_bytes;
- kparams->vtcm_bias_size = L.bias_bytes;
- kparams->vtcm_dst_size = L.dst_bytes;
- return;
- }
- }
-
- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
- return;
- }
- } else if (is_matmul_id) {
- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
- return;
- } else if (wtype == GGML_TYPE_F16) {
- // Flat F16 HVX
- struct htp_mm_hvx_vtcm_layout L;
- uint32_t m_chunk = 0;
- if (htp_mm_hvx_solve_vtcm_params(
- HTP_MM_KERNEL_HVX_F16_F16_VTCM, wtype, ne10, src1_nrows, sess->n_threads,
- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget,
- &L, &m_chunk)) {
- kparams->kernel_type = HTP_MM_KERNEL_HVX_F16_F16_VTCM;
- kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0;
- kparams->act_row_size = hex_round_up(ne10 * 2, 128);
- kparams->vtcm_size = L.total_bytes;
- kparams->vtcm_src0_size = L.src0_bytes;
- kparams->vtcm_act_size = L.act_bytes;
- kparams->vtcm_bias_size = L.bias_bytes;
- kparams->vtcm_dst_size = L.dst_bytes;
- kparams->n_prefetch = 16;
- return;
- }
-
- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
- return;
- } else if (wtype == GGML_TYPE_F32) {
- // Flat F32 HVX
- struct htp_mm_hvx_vtcm_layout L;
- uint32_t m_chunk = 0;
- if (htp_mm_hvx_solve_vtcm_params(
- HTP_MM_KERNEL_HVX_F32_F32_VTCM, wtype, ne10, src1_nrows, sess->n_threads,
- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget,
- &L, &m_chunk)) {
- kparams->kernel_type = HTP_MM_KERNEL_HVX_F32_F32_VTCM;
- kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0;
- kparams->act_row_size = hex_round_up(ne10 * 4, 128);
- kparams->vtcm_size = L.total_bytes;
- kparams->vtcm_src0_size = L.src0_bytes;
- kparams->vtcm_act_size = L.act_bytes;
- kparams->vtcm_bias_size = L.bias_bytes;
- kparams->vtcm_dst_size = L.dst_bytes;
- kparams->n_prefetch = 16;
- return;
- }
-
- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
- return;
- } else if (wtype == GGML_TYPE_BF16) {
- // Flat BF16 HVX
- struct htp_mm_hvx_vtcm_layout L;
- uint32_t m_chunk = 0;
- if (htp_mm_hvx_solve_vtcm_params(
- HTP_MM_KERNEL_HVX_BF16_F32_VTCM, wtype, ne10, src1_nrows, sess->n_threads,
- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget,
- &L, &m_chunk)) {
- kparams->kernel_type = HTP_MM_KERNEL_HVX_BF16_F32_VTCM;
- kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0;
- kparams->act_row_size = hex_round_up(ne10 * 4, 128);
- kparams->vtcm_size = L.total_bytes;
- kparams->vtcm_src0_size = L.src0_bytes;
- kparams->vtcm_act_size = L.act_bytes;
- kparams->vtcm_bias_size = L.bias_bytes;
- kparams->vtcm_dst_size = L.dst_bytes;
- kparams->n_prefetch = 16;
- return;
- }
-
- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
- return;
- } else {
- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
- return;
- }
-}
-
-static void ggml_hexagon_precompute_matmul_params_impl(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * dst,
- const size_t src2_row_size,
- const size_t src2_size,
- struct htp_mm_kernel_params * kparams
-) {
- memset(kparams, 0, sizeof(*kparams));
-
- const int ne00 = src0->ne[0];
- const int ne01 = src0->ne[1];
- const int ne02 = src0->ne[2];
- const int ne03 = src0->ne[3];
-
- const int ne10 = src1->ne[0];
- const int ne11 = src1->ne[1];
- const int ne12 = src1->ne[2];
- const int ne13 = src1->ne[3];
-
- const bool is_tiled = ggml_hexagon_tensor_is_tiled(src0);
- const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;
- const int ne00_padded = is_tiled ? hex_round_up(ne00, htp_tensor_get_tile_k(wtype)) : ne00;
- const int ne11_padded = hex_round_up(ne11, 32);
- // VTCM has to hold whole 32-row weight tiles, so size for the rounded-up N
- // even when the tensor itself is ragged.
- const int ne01_tiled = hex_round_up(ne01, 32);
-
- const bool is_matmul_id = (dst->op == GGML_OP_MUL_MAT_ID);
- const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1);
-
- const size_t vtcm_budget = sess->vtcm_size;
-
- // Check HMX eligibility and try precomputing HMX parameters
- bool hmx_enabled = (sess->n_hmx > 0) && (opt_mm_select >= 2);
- if (hmx_enabled && ggml_hexagon_matmul_is_hmx_eligible(src0, src1, dst, is_matmul_id, is_batched)) {
- if (ggml_hexagon_precompute_hmx_mm_params(sess, src0, src1, dst, wtype, ne00_padded, ne01_tiled, ne02, ne11, ne12, ne11_padded, is_matmul_id, is_batched, src2_size, vtcm_budget, kparams)) {
- goto finalize;
- }
- }
-
- // Fallback to HVX parameter computation
- ggml_hexagon_precompute_hvx_mm_params(sess, src0, src1, dst, wtype, is_tiled, ne02, ne03, ne10, ne11, ne12, ne13, is_matmul_id, src2_row_size, vtcm_budget, kparams);
-
-finalize:
- kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11);
- kparams->div_ne1 = init_fastdiv_values(ne11);
- kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1);
- kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1);
- kparams->div_ne12 = init_fastdiv_values(ne12);
-}
-
-// The rows of dims 1..3 can be walked with one stride (size-1 dims skipped); returns that stride
-static bool ggml_hexagon_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride) {
- size_t s = 0, next = 0;
- for (int i = 1; i < GGML_MAX_DIMS; i++) {
- if (ne[i] == 1) continue;
- if (s == 0) { s = nb[i]; next = s * ne[i]; continue; }
- if (nb[i] != next) return false;
- next *= ne[i];
- }
- *stride = s ? s : nb[1];
- return true;
-}
-
-// A 2D weight applied to a batched activation whose rows are evenly strided is the same matmul over ne11 * ne12 * ne13 rows
-static bool ggml_hexagon_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst) {
- size_t s1, sd;
- return (dst->op == GGML_OP_MUL_MAT || dst->op == GGML_OP_ADD) &&
- src0->ne[2] == 1 && src0->ne[3] == 1 && src1->ne[2] * src1->ne[3] > 1 &&
- src1->nb[0] == ggml_type_size(src1->type) && ggml_hexagon_rows_stride(src1->ne, src1->nb, &s1) &&
- dst->nb[0] == ggml_type_size(dst->type) && ggml_hexagon_rows_stride(dst->ne, dst->nb, &sd);
-}
-
-static bool ggml_hexagon_matmul_add_can_collapse(
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * src2,
- const struct ggml_tensor * dst
-) {
- if (!ggml_hexagon_matmul_can_collapse(src0, src1, dst)) {
- return false;
- }
- if (!src2) {
- return true;
- }
- const int64_t src2_nrows = src2->ne[1] * src2->ne[2] * src2->ne[3];
- if (src2_nrows == 1) {
- return true;
- }
- size_t s2;
- return src2->nb[0] == ggml_type_size(src2->type) &&
- src2->ne[0] == dst->ne[0] &&
- src2->ne[1] == dst->ne[1] &&
- src2->ne[2] == dst->ne[2] &&
- src2->ne[3] == dst->ne[3] &&
- ggml_hexagon_rows_stride(src2->ne, src2->nb, &s2);
-}
-
-static ggml_tensor ggml_hexagon_tensor_collapse_rows(const struct ggml_tensor * t) {
- size_t stride = 0;
- ggml_hexagon_rows_stride(t->ne, t->nb, &stride);
- ggml_tensor c = *t;
- c.ne[1] = t->ne[1] * t->ne[2] * t->ne[3];
- c.ne[2] = 1;
- c.ne[3] = 1;
- c.nb[1] = stride;
- c.nb[2] = c.nb[1] * c.ne[1];
- c.nb[3] = c.nb[2];
- return c;
-}
-
-static void ggml_hexagon_precompute_matmul_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * dst,
- struct htp_mm_kernel_params * kparams
-) {
- if (ggml_hexagon_matmul_can_collapse(src0, src1, dst)) {
- const ggml_tensor src1_collapsed = ggml_hexagon_tensor_collapse_rows(src1);
- const ggml_tensor dst_collapsed = ggml_hexagon_tensor_collapse_rows(dst);
- ggml_hexagon_precompute_matmul_params_impl(sess, src0, &src1_collapsed, &dst_collapsed, 0, 0, kparams);
- kparams->collapse = 1;
- return;
- }
- ggml_hexagon_precompute_matmul_params_impl(sess, src0, src1, dst, 0, 0, kparams);
-}
-
-static void ggml_hexagon_precompute_fused_matmul_add_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * src2,
- const struct ggml_tensor * dst,
- struct htp_mm_kernel_params * kparams
-) {
- if (ggml_hexagon_matmul_add_can_collapse(src0, src1, src2, dst)) {
- const ggml_tensor src1_collapsed = ggml_hexagon_tensor_collapse_rows(src1);
- const ggml_tensor dst_collapsed = ggml_hexagon_tensor_collapse_rows(dst);
- const ggml_tensor src2_collapsed = (src2 && (src2->ne[1] * src2->ne[2] * src2->ne[3] > 1))
- ? ggml_hexagon_tensor_collapse_rows(src2)
- : (src2 ? *src2 : ggml_tensor{});
- const struct ggml_tensor * p_src2 = src2 ? &src2_collapsed : nullptr;
- const size_t src2_size = p_src2 ? hex_round_up(ggml_nbytes(p_src2), 128) : 0;
- ggml_hexagon_precompute_matmul_params_impl(sess, src0, &src1_collapsed, &dst_collapsed, p_src2 ? p_src2->nb[1] : 0, src2_size, kparams);
- kparams->collapse = 1;
- return;
- }
- const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0;
- ggml_hexagon_precompute_matmul_params_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, kparams);
-}
-
-static bool ggml_hexagon_precompute_binary_params(
- const struct ggml_hexagon_session * sess,
- uint32_t op,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * dst,
- struct htp_binary_kernel_params * kparams,
- const struct ggml_tensor * src2
-) {
- memset(kparams, 0, sizeof(*kparams));
-
- const size_t elem_size = ggml_type_size(src0->type);
- const size_t src0_row_size = src0->ne[0] * elem_size;
- const size_t src1_row_size = src1->ne[0] * elem_size;
- const size_t dst_row_size = dst->ne[0] * elem_size;
-
- const size_t src0_row_size_aligned = hex_round_up(src0_row_size, 128);
- const size_t src1_row_size_aligned = hex_round_up(src1_row_size, 128);
- const size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128);
-
- if (!htp_binary_strides_safe(src0, src1, dst, elem_size)) {
- return false;
- }
-
- const bool is_add_id = op == HTP_OP_ADD_ID;
- const bool is_scalar = !is_add_id && src1->ne[0] == 1;
- const bool is_row_bcast = !is_add_id && !is_scalar &&
- src1->ne[0] == src0->ne[0] &&
- (src0->ne[1] > 1 || src0->ne[2] > 1 || src0->ne[3] > 1) &&
- src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1;
- const bool is_same_shape = !is_add_id && !is_scalar &&
- src1->ne[0] == src0->ne[0] &&
- src1->ne[1] == src0->ne[1] &&
- (src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) &&
- (src1->ne[3] == src0->ne[3] || src1->ne[3] == 1);
- const bool is_complex = !is_add_id && !is_scalar && !is_same_shape && !is_row_bcast && (src1->ne[0] == src0->ne[0]);
- const bool is_contig = ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst);
- const bool is_scalar_broadcast = !is_add_id && (ggml_nelements(src1) == 1);
-
- if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) {
- const uint32_t total_elems = (uint32_t) ggml_nelements(src0);
- const uint32_t n_threads = sess->n_threads;
- const uint32_t max_chunk_elems = 32768 / elem_size;
- const uint32_t min_chunk_elems = 256;
- const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * n_threads) - 1) / (2 * n_threads), 32);
- const uint32_t chunk_size = (std::min)(max_chunk_elems, (std::max)(target_chunk_elems, min_chunk_elems));
- const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128);
-
- kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED;
- kparams->n_threads = n_threads;
- kparams->rows_per_buffer = 1;
- kparams->src0_row_size_aligned = src0_row_size_aligned;
- kparams->src1_row_size_aligned = is_scalar_broadcast ? 0 : src1_row_size_aligned;
- kparams->dst_row_size_aligned = dst_row_size_aligned;
- kparams->src1_size = 0;
- kparams->chunk_size = chunk_size;
- kparams->chunk_bytes = chunk_bytes;
- kparams->is_scalar = is_scalar_broadcast ? 1 : 0;
-
- struct htp_binary_vtcm_layout L;
- htp_binary_vtcm_layout_build(&L, kparams, sess->vtcm_size);
- if (L.total_bytes == 0 || L.total_bytes > sess->vtcm_size) {
- return false;
- }
-
- kparams->vtcm_size = L.total_bytes;
- return true;
- }
-
- enum htp_binary_kernel_type kernel_type;
- size_t src1_size = 0;
- size_t src2_size = 0;
-
- if (is_add_id) {
- GGML_ASSERT(src2 != nullptr);
- kernel_type = HTP_BINARY_KERNEL_ADD_ID;
- src1_size = hex_round_up(src1->ne[1] * src1_row_size_aligned, 128);
- src2_size = hex_round_up(ggml_nelements(src2) * sizeof(int32_t), 128);
- if (src1_size + src2_size > sess->vtcm_size) {
- return false;
- }
- } else if (is_row_bcast) {
- kernel_type = HTP_BINARY_KERNEL_ROW_BCAST;
- src1_size = src1_row_size_aligned;
- } else if (is_scalar) {
- src1_size = hex_round_up(ggml_nelements(src1) * elem_size, 128);
- if (src1_size > sess->vtcm_size) {
- return false;
- }
- kernel_type = HTP_BINARY_KERNEL_SCALAR;
- kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;
- } else if (is_same_shape) {
- kernel_type = HTP_BINARY_KERNEL_SAME_SHAPE;
- } else if (is_complex) {
- src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128);
- if (src1_size > sess->vtcm_size) {
- return false;
- }
- kernel_type = HTP_BINARY_KERNEL_COMPLEX;
- kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;
- } else {
- src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128);
- if (src1_size > sess->vtcm_size) {
- return false;
- }
- kernel_type = HTP_BINARY_KERNEL_REPEAT;
- kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;
- }
-
- kparams->kernel_type = kernel_type;
- kparams->n_threads = sess->n_threads;
- kparams->src0_row_size_aligned = src0_row_size_aligned;
- kparams->src1_row_size_aligned = src1_row_size_aligned;
- kparams->dst_row_size_aligned = dst_row_size_aligned;
- kparams->src1_size = src1_size;
- kparams->src2_size = src2_size;
-
- const bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]);
- const bool src1_contig_dim1 = (src1->nb[2] == src1->ne[1] * src1->nb[1]);
- const bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]);
-
- const bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]);
- const bool src1_contig_dim2 = (src1->nb[3] == src1->ne[2] * src1->nb[2]);
- const bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]);
-
- bool split_at_ne01 = false;
- if (src0->ne[2] > 1) {
- if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {
- split_at_ne01 = (src1->ne[2] != src0->ne[2]) || !src0_contig_dim1 || !src1_contig_dim1 || !dst_contig_dim1;
- } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {
- split_at_ne01 = !src0_contig_dim1 || !dst_contig_dim1;
- } else {
- split_at_ne01 = true;
- }
- }
- kparams->split_at_ne01 = split_at_ne01 ? 1 : 0;
-
- bool split_at_ne02 = false;
- if (src0->ne[3] > 1) {
- if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {
- split_at_ne02 = (src1->ne[3] != src0->ne[3]) || !src0_contig_dim2 || !src1_contig_dim2 || !dst_contig_dim2;
- } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {
- split_at_ne02 = !src0_contig_dim2 || !dst_contig_dim2;
- } else {
- split_at_ne02 = true;
- }
- }
- kparams->split_at_ne02 = split_at_ne02 ? 1 : 0;
-
- struct htp_binary_vtcm_layout L;
- htp_binary_vtcm_layout_build(&L, kparams, sess->vtcm_size);
- if (L.rows_per_buffer == 0 || L.total_bytes > sess->vtcm_size) {
- if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) {
- const uint32_t total_elems = (uint32_t) ggml_nelements(src0);
- const uint32_t n_threads = sess->n_threads;
- const uint32_t max_chunk_elems = 32768 / elem_size;
- const uint32_t min_chunk_elems = 256;
- const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * n_threads) - 1) / (2 * n_threads), 32);
- const uint32_t chunk_size = (std::min)(max_chunk_elems, (std::max)(target_chunk_elems, min_chunk_elems));
- const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128);
-
- kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED;
- kparams->n_threads = n_threads;
- kparams->rows_per_buffer = 1;
- kparams->src1_row_size_aligned = is_scalar_broadcast ? 0 : src1_row_size_aligned;
- kparams->src1_size = 0;
- kparams->chunk_size = chunk_size;
- kparams->chunk_bytes = chunk_bytes;
- kparams->is_scalar = is_scalar_broadcast ? 1 : 0;
-
- htp_binary_vtcm_layout_build(&L, kparams, sess->vtcm_size);
- if (L.total_bytes == 0 || L.total_bytes > sess->vtcm_size) {
- return false;
- }
-
- kparams->vtcm_size = L.total_bytes;
- return true;
- }
- return false;
- }
-
- kparams->rows_per_buffer = L.rows_per_buffer;
- kparams->vtcm_size = L.total_bytes;
-
- return true;
-}
-
-static void ggml_hexagon_precompute_unary_params(
- const struct ggml_hexagon_session * sess,
- uint32_t op,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * dst,
- struct htp_unary_kernel_params * kparams
-) {
- memset(kparams, 0, sizeof(*kparams));
-
- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const uint32_t n_threads = (std::min)((uint32_t)sess->n_threads, src0_nrows);
-
- kparams->n_threads = n_threads;
-
- const size_t elem_size = ggml_type_size(src0->type);
-
- const size_t src0_data_row_size = src0->ne[0] * elem_size;
- const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type);
-
- const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);
- const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128);
-
- kparams->src0_row_size_aligned = src0_row_size_aligned;
- kparams->dst_row_size_aligned = dst_row_size_aligned;
-
- size_t src1_data_row_size = 0;
- size_t src1_row_size_aligned = 0;
- bool broadcast_weight = false;
-
- if (op == HTP_OP_RMS_NORM_MUL) {
- GGML_ASSERT(src1 != nullptr);
- src1_data_row_size = src1->ne[0] * ggml_type_size(src1->type);
- src1_row_size_aligned = hex_round_up(src1_data_row_size, 128);
- broadcast_weight = (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1);
- }
-
- kparams->src1_row_size_aligned = src1_row_size_aligned;
- kparams->broadcast_weight = broadcast_weight;
-
- struct htp_unary_vtcm_layout L;
- uint32_t col_tile = 0;
- uint32_t vtcm_row_per_thread = 0;
-
- htp_unary_vtcm_layout_build(&L, op, src0->ne[0], dst->ne[0],
- op == HTP_OP_RMS_NORM_MUL ? src1->ne[0] : 0,
- broadcast_weight, n_threads, sess->vtcm_size, elem_size,
- &col_tile, &vtcm_row_per_thread);
-
- kparams->col_tile = col_tile;
- kparams->vtcm_row_per_thread = vtcm_row_per_thread;
- kparams->vtcm_size = L.total_bytes;
-
- kparams->vtcm_src0_size_per_thread = L.src0_bytes;
- kparams->vtcm_src1_size_per_thread = L.src1_bytes;
- kparams->vtcm_dst_size_per_thread = L.dst_bytes;
-
- kparams->vtcm_src0_size = L.src0_bytes * n_threads;
- kparams->vtcm_src1_size = L.src1_bytes * n_threads;
- kparams->vtcm_dst_size = L.dst_bytes * n_threads;
-
- kparams->block = col_tile ? 0 : ((L.src0_bytes / 2) / src0_row_size_aligned);
-
- const uint32_t tiles_per_row = col_tile > 0 ? (src0->ne[0] + col_tile - 1) / col_tile : 1;
- kparams->div_ne01 = init_fastdiv_values(src0->ne[1]);
- kparams->div_ne02 = init_fastdiv_values(src0->ne[2]);
- kparams->div_ne012 = init_fastdiv_values(src0->ne[1] * src0->ne[2]);
- kparams->div_tpr = init_fastdiv_values(tiles_per_row);
-}
-
-static void ggml_hexagon_precompute_get_rows_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * dst,
- struct htp_get_rows_kernel_params * kparams
-) {
- memset(kparams, 0, sizeof(*kparams));
-
- const uint32_t ne00 = src0->ne[0];
- const uint32_t ne02 = src0->ne[2];
- const uint32_t ne03 = src0->ne[3];
-
- const uint32_t ne10 = src1->ne[0];
- const uint32_t ne11 = src1->ne[1];
- const uint32_t ne12 = src1->ne[2];
- const uint32_t nr = ne10 * ne11 * ne12;
-
- const bool tiled = ggml_hexagon_tensor_is_tiled(src0);
- const int wtype = (tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;
-
- if (tiled) {
- kparams->kernel_type = HTP_GET_ROWS_KERNEL_TILED;
- } else if (src0->type == dst->type) {
- kparams->kernel_type = HTP_GET_ROWS_KERNEL_SAMETYPE;
- } else {
- kparams->kernel_type = HTP_GET_ROWS_KERNEL_FLAT;
- }
-
- const uint32_t total_tasks = nr;
-
- kparams->n_threads = (std::min)((uint32_t)sess->n_threads, total_tasks);
-
- struct htp_get_rows_vtcm_layout vtcm_layout = {};
- while (kparams->n_threads > 0) {
- htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, kparams->n_threads);
- if (vtcm_layout.total_bytes <= sess->vtcm_size) {
- break;
- }
- --kparams->n_threads;
- }
-
- if (kparams->n_threads == 0 && total_tasks > 0) {
- htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, 1);
- }
-
- kparams->vtcm_size = (total_tasks == 0) ? 0 : vtcm_layout.total_bytes;
- kparams->tasks_per_thread = kparams->n_threads > 0 ? (total_tasks + kparams->n_threads - 1) / kparams->n_threads : 0;
- kparams->total_tasks = total_tasks;
-
- kparams->div_ne10 = ne10 > 0 ? init_fastdiv_values(ne10) : fastdiv_values{0, 0};
- kparams->div_ne10_ne11 = (ne10 * ne11) > 0 ? init_fastdiv_values(ne10 * ne11) : fastdiv_values{0, 0};
- kparams->div_ne02 = ne02 > 0 ? init_fastdiv_values(ne02) : fastdiv_values{0, 0};
- kparams->div_ne03 = ne03 > 0 ? init_fastdiv_values(ne03) : fastdiv_values{0, 0};
-}
-
-static void ggml_hexagon_precompute_set_rows_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0, // values
- const struct ggml_tensor * src1, // indices
- const struct ggml_tensor * dst, // destination
- struct htp_set_rows_kernel_params * kparams
-) {
- memset(kparams, 0, sizeof(*kparams));
-
- const uint32_t nr = src0->ne[1];
-
- kparams->n_threads = (std::min)((uint32_t)sess->n_threads, nr);
- kparams->tasks_per_thread = (nr + kparams->n_threads - 1) / kparams->n_threads;
- kparams->total_tasks = nr;
-
- kparams->div_ne11 = init_fastdiv_values(src1->ne[1]);
- kparams->div_ne12 = init_fastdiv_values(src1->ne[2]);
- kparams->div_tasks_per_thread = init_fastdiv_values(kparams->tasks_per_thread);
- kparams->div_ne02 = init_fastdiv_values(src0->ne[2]);
-
- struct htp_set_rows_vtcm_layout vtcm_layout;
- htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_threads);
- kparams->vtcm_size = vtcm_layout.total_bytes;
-}
-
-static void ggml_hexagon_precompute_softmax_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_softmax_kernel_params * kparams
-) {
- memset(kparams, 0, sizeof(*kparams));
-
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * src1 = op->src[1];
-
- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows);
-
- float scale = 1.0f;
- float max_bias = 0.0f;
- memcpy(&scale, &op->op_params[0], sizeof(float));
- memcpy(&max_bias, &op->op_params[1], sizeof(float));
-
- kparams->scale = scale;
- kparams->max_bias = max_bias;
-
- const uint32_t n_head = src0->ne[2];
- const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));
- kparams->n_head = n_head;
- kparams->n_head_log2 = n_head_log2;
-
- if (max_bias > 0.0f && n_head_log2 > 0) {
- kparams->m0 = powf(2.0f, -(max_bias) / n_head_log2);
- kparams->m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
- } else {
- kparams->m0 = 1.0f;
- kparams->m1 = 1.0f;
- }
-
- kparams->use_src1 = (src1 != nullptr) ? 1 : 0;
- kparams->use_f16 = (src1 != nullptr && src1->type == GGML_TYPE_F16) ? 1 : 0;
-
- const uint32_t ne00 = src0->ne[0];
- const uint32_t ne10 = src1 ? src1->ne[0] : 1;
-
- struct htp_softmax_vtcm_layout layout;
- htp_softmax_vtcm_layout_build(&layout, ne00, ne10, kparams->use_src1 != 0, kparams->use_f16 != 0, n_threads);
-
- kparams->n_threads = n_threads;
- kparams->src0_nrows = src0_nrows;
- kparams->src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
- kparams->vtcm_size = (uint32_t) layout.total_bytes;
- kparams->vtcm_src0_size_per_thread = (uint32_t) layout.src0_bytes_per_thread;
- kparams->vtcm_src1_size_per_thread = (uint32_t) layout.src1_bytes_per_thread;
- kparams->vtcm_dst_size_per_thread = (uint32_t) layout.dst_bytes_per_thread;
- kparams->src0_row_size_aligned = (uint32_t) layout.src0_spad_half_size;
- kparams->src1_row_size_aligned = (uint32_t) layout.src1_spad_half_size;
- kparams->dst_row_size_aligned = (uint32_t) layout.dst_spad_half_size;
- kparams->src0_spad_half_size = (uint32_t) layout.src0_spad_half_size;
- kparams->src1_spad_half_size = (uint32_t) layout.src1_spad_half_size;
- kparams->dst_spad_half_size = (uint32_t) layout.dst_spad_half_size;
- if (!kparams->use_src1) {
- kparams->kernel_id = HTP_SOFTMAX_KERNEL_NOMASK;
- } else if (kparams->use_f16) {
- kparams->kernel_id = HTP_SOFTMAX_KERNEL_MASK_F16;
- } else {
- kparams->kernel_id = HTP_SOFTMAX_KERNEL_MASK_F32;
- }
-
- if (src0->ne[1] > 0) kparams->div_ne01 = init_fastdiv_values(src0->ne[1]);
- if (src0->ne[2] > 0) kparams->div_ne02 = init_fastdiv_values(src0->ne[2]);
- const uint32_t ne12 = src1 ? src1->ne[2] : 1;
- const uint32_t ne13 = src1 ? src1->ne[3] : 1;
- if (ne12 > 0) kparams->div_ne12 = init_fastdiv_values(ne12);
- if (ne13 > 0) kparams->div_ne13 = init_fastdiv_values(ne13);
-}
-
-static void ggml_hexagon_precompute_rope_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_rope_kernel_params * kparams
-) {
- memset(kparams, 0, sizeof(*kparams));
-
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * src2 = op->src[2];
- const struct ggml_tensor * dst = op;
-
- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows);
- const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0;
-
- struct htp_rope_vtcm_layout layout;
- htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads, n_freq_factors);
-
- kparams->n_threads = n_threads;
- kparams->src0_nrows = src0_nrows;
- kparams->src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
- kparams->vtcm_size = (uint32_t) layout.total_bytes;
- kparams->spad_per_thread = (uint32_t) layout.bytes_per_thread;
- kparams->theta_cache_offset = (uint32_t) layout.theta_cache_size_aligned;
- kparams->src0_row_size_aligned = (uint32_t) layout.src0_row_size_aligned;
- kparams->freq_factors_offset = (uint32_t) (layout.bytes_per_thread * n_threads);
- kparams->freq_factors_size = (uint32_t) layout.freq_factors_size_aligned;
-
- if (src0_nrows > 0) {
- kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]);
- kparams->div_ne1 = init_fastdiv_values(dst->ne[1]);
- }
-}
-
-static void ggml_hexagon_precompute_ssm_conv_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * src1,
- const struct ggml_tensor * dst,
- struct htp_ssm_conv_kernel_params * kparams
-) {
- memset(kparams, 0, sizeof(*kparams));
-
- const uint32_t d_conv = (uint32_t) src1->ne[0];
- const uint32_t d_inner = (uint32_t) src0->ne[1];
- const uint32_t n_t = (uint32_t) dst->ne[1];
- const uint32_t n_s = (uint32_t) dst->ne[2];
- const uint32_t ncs = (uint32_t) src0->ne[0];
-
- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (d_inner + 31) / 32);
-
- kparams->n_threads = n_threads;
- kparams->d_conv = d_conv;
- kparams->d_inner = d_inner;
- kparams->n_t = n_t;
- kparams->n_s = n_s;
-
- const uint32_t raw_rpt = (d_inner + n_threads - 1) / n_threads;
- const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, 32);
-
- const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128) + 128;
- const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128);
- const uint32_t vtcm_src1_per_thread = src1_raw_bytes + src1_T_bytes;
-
- uint32_t vtcm_src0_per_thread = 0;
- uint32_t vtcm_dst_per_thread = 0;
-
- if (n_t == 1) {
- kparams->d_inner_tile = d_inner_per_thread;
-
- const uint32_t src0_tile_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128);
- const uint32_t src0_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128);
- vtcm_src0_per_thread = 2 * src0_tile_raw_bytes + src0_T_bytes;
-
- const uint32_t dst_tile_bytes = hex_round_up(d_inner_per_thread * sizeof(float), 128);
- vtcm_dst_per_thread = 2 * dst_tile_bytes;
- } else {
- const size_t vtcm_budget = (sess->vtcm_size > 0 ? sess->vtcm_size / n_threads : (1024 * 1024));
-
- // the kernel double-buffers the raw src0 tile and the dst tile, and transposes
- // one 32-channel block at a time
- const uint32_t src0_block_T = hex_round_up(ncs * 32 * sizeof(float), 128);
- const size_t fixed_bytes = vtcm_src1_per_thread + src0_block_T;
- const size_t avail_for_tiles = vtcm_budget > fixed_bytes ? vtcm_budget - fixed_bytes : (128 * 1024);
-
- uint32_t target_max_tile = hex_round_up((d_inner_per_thread + 3) / 4, 32);
- target_max_tile = (std::max)(target_max_tile, 32u);
- target_max_tile = (std::min)(target_max_tile, 128u);
-
- uint32_t d_inner_tile = (uint32_t)(avail_for_tiles / (2 * (ncs + n_t) * sizeof(float)));
- d_inner_tile = (d_inner_tile / 32) * 32;
- if (d_inner_tile == 0) {
- d_inner_tile = 32;
- }
- if (d_inner_tile > target_max_tile) {
- d_inner_tile = target_max_tile;
- }
- if (d_inner_tile > d_inner_per_thread) {
- d_inner_tile = d_inner_per_thread;
- }
- kparams->d_inner_tile = d_inner_tile;
-
- const uint32_t src0_tile_raw = hex_round_up(d_inner_tile * ncs * sizeof(float), 128);
- vtcm_src0_per_thread = 2 * src0_tile_raw + src0_block_T;
-
- vtcm_dst_per_thread = 2 * hex_round_up(d_inner_tile * n_t * sizeof(float), 128);
- }
-
- kparams->vtcm_src0_size_per_thread = vtcm_src0_per_thread;
- kparams->vtcm_src1_size_per_thread = vtcm_src1_per_thread;
- kparams->vtcm_dst_size_per_thread = vtcm_dst_per_thread;
-
- kparams->vtcm_src0_size = vtcm_src0_per_thread * n_threads;
- kparams->vtcm_src1_size = vtcm_src1_per_thread * n_threads;
- kparams->vtcm_dst_size = vtcm_dst_per_thread * n_threads;
- kparams->vtcm_size = kparams->vtcm_src0_size + kparams->vtcm_src1_size + kparams->vtcm_dst_size;
-}
-
-static void ggml_hexagon_precompute_gated_delta_net_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_gdn_kernel_params * kparams
-) {
- memset(kparams, 0, sizeof(*kparams));
-
- const struct ggml_tensor * q = op->src[0];
- const struct ggml_tensor * k = op->src[1];
- const struct ggml_tensor * v = op->src[2];
- const struct ggml_tensor * g = op->src[3];
- const struct ggml_tensor * state = op->src[5];
-
- const uint32_t S_v = (uint32_t) v->ne[0];
- const uint32_t H = (uint32_t) v->ne[1];
- const uint32_t n_tokens = (uint32_t) v->ne[2];
- const uint32_t n_seqs = (uint32_t) v->ne[3];
- const uint32_t K = (uint32_t) ggml_get_op_params_i32(op, 0);
-
- const uint32_t rq3 = (uint32_t) (n_seqs / q->ne[3]);
- const uint32_t rk3 = (uint32_t) (n_seqs / k->ne[3]);
- const uint32_t total_rows = H * n_seqs;
- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, total_rows);
-
- const bool can_use_hmx = (opt_gdn_select >= 2) &&
- (sess->n_hmx > 0) &&
- (S_v % 64 == 0) &&
- (n_tokens >= HTP_GDN_MIN_TOKENS) &&
- (g->ne[0] == 1) &&
- (K == 1);
-
- struct htp_gdn_hmx_vtcm_layout hmx_layout;
- struct htp_gdn_vtcm_layout hvx_layout;
- uint32_t n_heads_batch = 1;
-
- if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout, S_v, HTP_GDN_CHUNK_SIZE, total_rows, sess->vtcm_size, n_threads, true, &n_heads_batch)) {
- kparams->kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED;
- kparams->pipeline = hmx_layout.pipeline ? 1 : 0;
- kparams->chunk_size = HTP_GDN_CHUNK_SIZE;
- kparams->n_chunks = (n_tokens + HTP_GDN_CHUNK_SIZE - 1) / HTP_GDN_CHUNK_SIZE;
- kparams->n_heads_batch = (uint16_t) n_heads_batch;
- kparams->vtcm_size = (uint32_t) hmx_layout.total_bytes;
- kparams->state_aligned = (uint32_t) hmx_layout.state_f32_bytes;
- kparams->vtcm_per_thread = (uint32_t) (hmx_layout.total_bytes / (n_threads > 0 ? n_threads : 1));
- } else {
- htp_gdn_vtcm_layout_build(&hvx_layout, S_v, n_threads);
- kparams->kernel_type = HTP_GDN_KERNEL_HVX_RECURRENT;
- kparams->pipeline = 0;
- kparams->n_heads_batch = 1;
- kparams->state_aligned = (uint32_t) hvx_layout.state_aligned;
- kparams->vtcm_per_thread = (uint32_t) hvx_layout.bytes_per_thread;
- kparams->vtcm_size = (uint32_t) hvx_layout.total_bytes;
- }
-
- kparams->n_threads = n_threads;
- kparams->S_v = S_v;
- kparams->H = H;
- kparams->n_tokens = n_tokens;
- kparams->n_seqs = n_seqs;
- kparams->K = K;
- kparams->total_rows = total_rows;
- kparams->rows_per_thread = (total_rows + kparams->n_threads - 1) / kparams->n_threads;
- kparams->kda = (g->ne[0] == S_v) ? 1 : 0;
- kparams->state_seq_stride = (uint32_t) (state->nb[3] / sizeof(float));
- kparams->state_size_per_snap = S_v * S_v * H * n_seqs;
- kparams->scale = 1.0f / sqrtf((float) S_v);
-
- if (H > 0) kparams->div_H = init_fastdiv_values(H);
- if (q->ne[1] > 0) kparams->div_q1 = init_fastdiv_values((uint32_t) q->ne[1]);
- if (k->ne[1] > 0) kparams->div_k1 = init_fastdiv_values((uint32_t) k->ne[1]);
- if (rq3 > 0) kparams->div_rq3 = init_fastdiv_values(rq3);
- if (rk3 > 0) kparams->div_rk3 = init_fastdiv_values(rk3);
- if (kparams->n_threads > 0) kparams->div_n_threads = init_fastdiv_values(kparams->n_threads);
-}
-
-static void ggml_hexagon_precompute_sort_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- bool is_top_k,
- struct htp_sort_kernel_params * kparams
-) {
- memset(kparams, 0, sizeof(*kparams));
-
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * dst = op;
-
- const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const uint32_t ne00 = src0->ne[0];
- const uint32_t k = dst->ne[0];
-
- int32_t order = GGML_SORT_ORDER_DESC;
- if (!is_top_k) {
- order = ((const int32_t *) op->op_params)[0];
- }
-
- const uint32_t n_threads_max = sess->n_threads > 0 ? sess->n_threads : 4;
- const size_t vtcm_budget = sess->vtcm_size > 0 ? sess->vtcm_size : (8 * 1024 * 1024);
-
- struct htp_sort_vtcm_layout layout;
- bool ok = htp_sort_solve_layout(&layout, ne00, total_rows, k, n_threads_max, vtcm_budget, is_top_k);
- GGML_ASSERT(ok);
-
- kparams->n_threads = (int32_t) layout.n_threads;
- kparams->total_rows = (int32_t) total_rows;
- kparams->row_start = 0;
- kparams->row_end = (int32_t) total_rows;
- kparams->ne00 = (int32_t) ne00;
- kparams->k = (int32_t) k;
- kparams->order = order;
- kparams->is_top_k = is_top_k ? 1 : 0;
- kparams->use_dma = 1;
- kparams->chunk_elems = (int32_t) layout.chunk_elems;
- kparams->n_chunks = (int32_t) layout.n_chunks;
- kparams->vtcm_size = (int32_t) layout.total_bytes;
- kparams->phase1_slot_size = (int32_t) layout.phase1_slot_size;
- kparams->merge_values_off = (int32_t) layout.merge_values_off;
- kparams->merge_indices_off = (int32_t) layout.merge_indices_off;
- kparams->merge_elems = (int32_t) layout.merge_elems;
- kparams->n_slots = (int32_t) layout.n_slots;
-}
-
-static void ggml_hexagon_pool_interior_range(
- uint32_t src_x, uint32_t dst_x, uint32_t kernel_x, uint32_t stride_x, int32_t pad_x,
- uint32_t * ox_lo, uint32_t * ox_hi) {
- const uint32_t lo_raw = ((uint32_t) pad_x + stride_x - 1) / stride_x;
- *ox_lo = (lo_raw < dst_x) ? lo_raw : dst_x;
-
- const int32_t numer_hi = (int32_t) src_x - (int32_t) kernel_x + pad_x;
- if (numer_hi < 0) {
- *ox_hi = 0;
- } else {
- const uint32_t hi_raw = (uint32_t) numer_hi / stride_x + 1;
- *ox_hi = (hi_raw < dst_x) ? hi_raw : dst_x;
- }
- if (*ox_hi < *ox_lo) {
- *ox_hi = *ox_lo;
- }
-}
-
-static bool ggml_hexagon_pool_shape_fits(
- const struct ggml_tensor * src0,
- const struct ggml_tensor * dst
-) {
- return ggml_nbytes(src0) <= UINT32_MAX - 256 && ggml_nbytes(dst) <= UINT32_MAX - 256;
-}
-
-static void ggml_hexagon_precompute_pool_2d_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0,
- const struct ggml_tensor * dst,
- struct htp_pool_2d_kernel_params * kparams,
- bool is_pool_1d
-) {
- memset(kparams, 0, sizeof(*kparams));
-
- const uint32_t src_x = (uint32_t) src0->ne[0];
- const uint32_t src_y = is_pool_1d ? 1 : (uint32_t) src0->ne[1];
- const uint32_t dst_x = (uint32_t) dst->ne[0];
- const uint32_t dst_y = is_pool_1d ? 1 : (uint32_t) dst->ne[1];
- const uint32_t src_plane_bytes = src_x * src_y * sizeof(float);
- const uint32_t dst_plane_bytes = dst_x * dst_y * sizeof(float);
- const uint32_t planes = (uint32_t) (is_pool_1d
- ? (uint64_t) src0->ne[1] * (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3]
- : (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3]);
- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, planes > 0 ? planes : 1);
-
- struct htp_pool_vtcm_layout layout;
- const bool ok = htp_pool_solve_layout(&layout, src_x, src_y, dst_x, dst_y, n_threads, sess->vtcm_size);
- GGML_ASSERT(ok);
-
- kparams->src_x = src_x;
- kparams->src_y = src_y;
- kparams->dst_x = dst_x;
- kparams->dst_y = dst_y;
- kparams->kernel_x = (uint32_t) ggml_get_op_params_i32(dst, 1);
- kparams->kernel_y = is_pool_1d ? 1 : (uint32_t) ggml_get_op_params_i32(dst, 2);
- kparams->stride_x = (uint32_t) ggml_get_op_params_i32(dst, is_pool_1d ? 2 : 3);
- kparams->stride_y = is_pool_1d ? 1 : (uint32_t) ggml_get_op_params_i32(dst, 4);
- kparams->pad_x = ggml_get_op_params_i32(dst, is_pool_1d ? 3 : 5);
- kparams->pad_y = is_pool_1d ? 0 : ggml_get_op_params_i32(dst, 6);
- kparams->src_plane_bytes = src_plane_bytes;
- kparams->dst_plane_bytes = dst_plane_bytes;
- kparams->src_plane_bytes_aligned = (uint32_t) layout.src_spad_half_size;
- kparams->dst_plane_bytes_aligned = (uint32_t) layout.dst_spad_half_size;
- kparams->n_threads = n_threads;
- kparams->planes = planes;
- kparams->pool_op = (uint32_t) ggml_get_op_params_i32(dst, 0);
- // Fast HVX path requires exact tiling and a supported kernel width.
- kparams->fast_path = (kparams->pad_x == 0 && kparams->pad_y == 0 &&
- kparams->stride_x == kparams->kernel_x && kparams->stride_y == kparams->kernel_y &&
- (kparams->kernel_x == 1 || kparams->kernel_x == 2)) ? 1 : 0;
- kparams->global_path = (kparams->pad_x == 0 && kparams->pad_y == 0 &&
- kparams->stride_x == kparams->kernel_x &&
- kparams->stride_y == kparams->kernel_y &&
- kparams->kernel_x == kparams->src_x &&
- kparams->kernel_y == kparams->src_y &&
- kparams->dst_x == 1 && kparams->dst_y == 1) ? 1 : 0;
- kparams->block_path = (kparams->pad_x == 0 && kparams->pad_y == 0 &&
- kparams->kernel_y == 1 && kparams->stride_y == 1 &&
- kparams->stride_x == kparams->kernel_x &&
- kparams->kernel_x >= 32) ? 1 : 0;
- kparams->avg_divide_count = (is_pool_1d && kparams->pool_op == GGML_OP_POOL_AVG) ? 1 : 0;
- ggml_hexagon_pool_interior_range(kparams->src_x, kparams->dst_x, kparams->kernel_x,
- kparams->stride_x, kparams->pad_x,
- &kparams->ox_lo, &kparams->ox_hi);
-
- const bool narrow_ok = (uint64_t) kparams->dst_x * kparams->kernel_x <= 32;
- kparams->narrow_path = (kparams->fast_path && narrow_ok) ? 1 : 0;
- kparams->inv_kernel_area = 1.0f / (float) (kparams->kernel_x * kparams->kernel_y);
-}
-
-static bool ggml_hexagon_precompute_concat_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_concat_kernel_params * kparams
-) {
- memset(kparams, 0, sizeof(*kparams));
- kparams->kernel_type = HTP_CONCAT_KERNEL_UNSUPPORTED;
-
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * src1 = op->src[1];
- const struct ggml_tensor * dst = op;
-
- if (!src0 || !src1 || !dst) {
- return false;
- }
-
- int dim = ((const int32_t *) op->op_params)[0];
- if (dim < 0 || dim >= GGML_MAX_DIMS) {
- return false;
- }
- kparams->dim = dim;
-
- if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_I32) {
- return false;
- }
- if (src0->type != dst->type || src1->type != dst->type) {
- return false;
- }
-
- const uint32_t type_size = ggml_type_size(dst->type);
-
- for (int d = 0; d < GGML_MAX_DIMS; d++) {
- const int64_t ne_d = (d == dim) ? src0->ne[d] + src1->ne[d] : src0->ne[d];
- if (dst->ne[d] != ne_d || (d != dim && src1->ne[d] != dst->ne[d])) {
- return false;
- }
- }
-
- const uint32_t max_gather_ne0 = 128 / type_size;
- const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);
- const bool b_dense = (src1->nb[0] == type_size);
- const bool gather_strides_ok =
- src0->nb[1] >= (size_t) src0->ne[0] * type_size &&
- (b_dense ? src1->nb[1] >= (size_t) src1->ne[0] * type_size
- : src1->nb[0] >= (size_t) src1->ne[1] * type_size);
-
- if (dim == 0 &&
- (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16) &&
- dst->ne[0] <= max_gather_ne0 &&
- dst->nb[0] == type_size && dst->nb[1] == (size_t) dst->ne[0] * type_size &&
- src0->nb[0] == type_size &&
- (src1->nb[0] == type_size || src1->nb[1] == type_size) &&
- gather_strides_ok &&
- htp_gather_rows_tensor_outer_contiguous(dst) &&
- htp_gather_rows_tensor_outer_contiguous(src0) &&
- htp_gather_rows_tensor_outer_contiguous(src1)) {
-
- if (htp_gather_rows_solve_layout(&kparams->u.gather,
- (uint32_t) src0->ne[0], (uint32_t) src0->nb[1],
- (uint32_t) src1->ne[0], (uint32_t) src1->nb[1], b_dense,
- type_size, total_rows,
- sess->n_threads, sess->mdev.count, sess->vtcm_size)) {
- kparams->kernel_type = HTP_CONCAT_KERNEL_GATHER_ROWS;
- kparams->n_threads = (uint8_t) sess->n_threads;
- kparams->vtcm_size = kparams->u.gather.slice * sess->n_threads;
- return true;
- }
- }
-
- const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size);
-
- if (dma_strides_ok) {
- kparams->kernel_type = HTP_CONCAT_KERNEL_REGULAR;
- kparams->n_threads = 1;
- return true;
- }
-
- const bool is_src1_transposed = (src1->nb[0] > src1->nb[1]);
- const bool is_src0_transposed = (src0->nb[0] > src0->nb[1]);
- const bool transposed_rows_ok = (src0->nb[0] == type_size && src1->nb[1] == type_size && dst->nb[0] == type_size);
-
- if (dim == 0 && is_src1_transposed && !is_src0_transposed && transposed_rows_ok &&
- (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16)) {
-
- const uint32_t n_threads = sess->n_threads > 0 ? (uint32_t) sess->n_threads : 8;
- struct htp_concat_transposed_vtcm_layout layout;
- htp_concat_transposed_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src1->ne[0], type_size, n_threads);
-
- if (sess->vtcm_size > 0 && layout.total_bytes > sess->vtcm_size) {
- return false;
- }
-
- kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED;
- kparams->n_threads = n_threads;
- kparams->vtcm_size = layout.total_bytes;
- kparams->u.transposed.spad0_size_per_thread = layout.src0_spad_size_per_thread;
- kparams->u.transposed.spad1_size_per_thread = layout.src1_spad_size_per_thread;
- return true;
- }
-
- return false;
-}
-
-static bool ggml_hexagon_precompute_cpy_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_copy_kernel_params * kparams
-) {
- memset(kparams, 0, sizeof(*kparams));
- kparams->kernel_type = HTP_COPY_KERNEL_UNSUPPORTED;
-
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * dst = op;
-
- if (!src0 || !dst) {
- return false;
- }
-
- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_I32) {
- return false;
- }
- if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_I32) {
- return false;
- }
-
- const int64_t nelem_src = ggml_nelements(src0);
- const int64_t nelem_dst = ggml_nelements(dst);
- if (nelem_src != nelem_dst || nelem_src < 0) {
- return false;
- }
-
- const uint32_t src_type_size = ggml_type_size(src0->type);
- const uint32_t dst_type_size = ggml_type_size(dst->type);
-
- kparams->src0_type_size = (uint8_t) src_type_size;
- kparams->dst_type_size = (uint8_t) dst_type_size;
- kparams->total_elems = (uint32_t) nelem_src;
-
- if (nelem_src == 0) {
- kparams->kernel_type = HTP_COPY_KERNEL_1D_CONTIG;
- return true;
- }
-
- if (nelem_src == 1) {
- if (src0->type == dst->type) {
- kparams->kernel_type = HTP_COPY_KERNEL_SCALAR;
- return true;
- }
- if ((src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) ||
- (src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32) ||
- (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) ||
- (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32)) {
- kparams->kernel_type = HTP_COPY_KERNEL_SCALAR;
- return true;
- }
- return false;
- }
-
- const bool sametype = (src0->type == dst->type);
-
- bool same_extents = true;
- for (int d = 0; d < GGML_MAX_DIMS; d++) {
- if (src0->ne[d] != dst->ne[d]) {
- same_extents = false;
- break;
- }
- }
-
- const bool transposed = (src0->nb[0] > src0->nb[1]) || (dst->nb[0] > dst->nb[1]) ||
- (src0->nb[0] != src_type_size) || (dst->nb[0] != dst_type_size) ||
- (src0->nb[1] < (size_t) src0->ne[0] * src_type_size) || (dst->nb[1] < (size_t) dst->ne[0] * dst_type_size);
- const bool sameshape = same_extents && !transposed;
-
- const bool src_is_contiguous = ggml_is_contiguous(src0);
- const bool dst_is_contiguous = ggml_is_contiguous(dst);
-
- if (sametype) {
- if (src_is_contiguous && dst_is_contiguous) {
- kparams->kernel_type = HTP_COPY_KERNEL_1D_CONTIG;
- return true;
- }
-
- const uint32_t max_gather_ne0 = 128 / src_type_size;
- const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
- if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) &&
- dst_is_contiguous && !src_is_contiguous &&
- src0->nb[0] == src_type_size && src0->ne[0] <= max_gather_ne0 &&
- src0->nb[1] >= (size_t) src0->ne[0] * src_type_size &&
- total_rows > 1 &&
- htp_gather_rows_tensor_outer_contiguous(src0)) {
-
- if (htp_gather_rows_solve_layout(&kparams->u.gather,
- (uint32_t) src0->ne[0], (uint32_t) src0->nb[1],
- 0, 0, false,
- src_type_size, total_rows,
- sess->n_threads, sess->mdev.count, sess->vtcm_size)) {
- kparams->kernel_type = HTP_COPY_KERNEL_GATHER_ROWS;
- kparams->n_threads = (uint8_t) sess->n_threads;
- kparams->vtcm_size = kparams->u.gather.slice * sess->n_threads;
- return true;
- }
- }
-
- if (sameshape) {
- kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE;
- kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
- return true;
- }
-
- kparams->kernel_type = HTP_COPY_KERNEL_RESHAPE;
- kparams->n_threads = sess->n_threads > 0 ? (uint8_t) sess->n_threads : 4;
- kparams->u.reshape.div_ne0 = init_fastdiv_values((uint32_t) dst->ne[0]);
- kparams->u.reshape.div_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[1] * dst->ne[0]));
- kparams->u.reshape.div_ne2_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[2] * dst->ne[1] * dst->ne[0]));
- kparams->u.reshape.div_ne00 = init_fastdiv_values((uint32_t) src0->ne[0]);
- kparams->u.reshape.div_ne01_ne00 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[0]));
- kparams->u.reshape.div_ne02_ne01_ne00 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1] * src0->ne[0]));
- return true;
- }
-
- if (!sameshape) {
- return false;
- }
-
- const bool valid_conversion = (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) ||
- (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) ||
- (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) ||
- (src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32);
- if (!valid_conversion) {
- return false;
- }
-
- const uint32_t n_threads = sess->n_threads > 0 ? (uint32_t) sess->n_threads : 4;
- struct htp_copy_convert_vtcm_layout layout;
- htp_copy_convert_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1], src_type_size, dst_type_size, n_threads);
-
- if (sess->vtcm_size > 0 && layout.total_bytes > sess->vtcm_size) {
- return false;
- }
-
- kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_CONVERT;
- kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
- kparams->n_threads = (uint8_t) n_threads;
- kparams->vtcm_size = layout.total_bytes;
- kparams->u.convert.src0_buf_size = layout.src0_buf_size;
- kparams->u.convert.dst_buf_size = layout.dst_buf_size;
- kparams->u.convert.spad0_size_per_thread = layout.spad0_size_per_thread;
- kparams->u.convert.spad1_size_per_thread = layout.spad1_size_per_thread;
- kparams->u.convert.src0_row_stride = layout.src0_row_stride;
- kparams->u.convert.dst_row_stride = layout.dst_row_stride;
- kparams->u.convert.blk_rows = layout.blk_rows;
- kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);
- kparams->u.convert.div_ne02_ne01 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1]));
-
- return true;
-}
-
-static void ggml_hexagon_precompute_fused_mmnx_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0, // W0
- const struct ggml_tensor * act, // x
- int32_t n_weights,
- struct htp_mm_kernel_params * kparams
-) {
- memset(kparams, 0, sizeof(*kparams));
- kparams->n_threads = sess->n_threads;
-
- const int ne00 = src0->ne[0];
- const int ne01 = src0->ne[1];
- const int ne02 = src0->ne[2];
- const int ne03 = src0->ne[3];
-
- const int ne10 = act->ne[0];
- const int ne11 = act->ne[1];
- const int ne12 = act->ne[2];
- const int ne13 = act->ne[3];
-
- const bool is_tiled = ggml_hexagon_tensor_is_tiled(src0);
- const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;
- const int ne00_padded = is_tiled ? hex_round_up(ne00, 32) : ne00;
- const int ne11_padded = hex_round_up(ne11, 32);
- const int ne01_tiled = hex_round_up(ne01, 32);
-
- const size_t vtcm_budget = sess->vtcm_size;
- const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1);
-
- bool hmx_enabled = (sess->n_hmx > 0) && (opt_mm_select >= 2);
- if (hmx_enabled && ggml_hexagon_matmul_is_hmx_eligible(src0, act, nullptr, false, is_batched)) {
- if (ggml_hexagon_precompute_hmx_mm_params(sess, src0, act, nullptr, wtype, ne00_padded, ne01_tiled, ne02, ne11, ne12, ne11_padded, false, is_batched, 0, vtcm_budget, kparams)) {
- kparams->n_weights = n_weights;
- goto finalize;
- }
- }
-
- if (!is_tiled || ne10 % 32 != 0) {
- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
- return;
- }
-
- {
- const int act_nrows = ne11 * ne12 * ne13;
- const size_t act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
- const size_t src0_row_size = src0->nb[1];
-
- const uint32_t max_prefetch = (act_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;
- uint32_t best_n_prefetch = 2;
- for (uint32_t d = max_prefetch; d >= 2; d /= 2) {
- struct htp_mm_hvx_vtcm_layout L;
- htp_mm_hvx_vtcm_layout_build(
- &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, sess->n_threads,
- 0, src0_row_size, act_row_size, 0, d, false, true
- );
- if (L.total_bytes <= sess->vtcm_size) {
- best_n_prefetch = d;
- break;
- }
- }
-
- struct htp_mm_hvx_vtcm_layout L;
- bool try_tiled = (opt_mm_select >= 1);
-
- // Test tiled first
- htp_mm_hvx_vtcm_layout_build(
- &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, sess->n_threads,
- 0, src0_row_size, act_row_size, 0, best_n_prefetch, false, true
- );
-
- if (try_tiled && L.total_bytes <= sess->vtcm_size) {
- kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW;
- kparams->tile_size = htp_tensor_get_tile_size(wtype);
- kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
- kparams->act_row_size = act_row_size;
- kparams->vtcm_src0_size = L.src0_bytes;
- kparams->vtcm_act_size = L.act_bytes;
- kparams->vtcm_bias_size = 0;
- kparams->vtcm_dst_size = L.dst_bytes;
- kparams->vtcm_size = L.total_bytes;
- kparams->n_prefetch = best_n_prefetch;
- kparams->n_weights = n_weights;
- } else {
- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
- return;
- }
- }
-
-finalize:
- kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11);
- kparams->div_ne1 = init_fastdiv_values(ne11);
- kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1);
- kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1);
- kparams->div_ne12 = init_fastdiv_values(ne12);
-}
-
-static void ggml_hexagon_precompute_fused_mmidnx_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0, // W0
- const struct ggml_tensor * act, // x
- const struct ggml_tensor * dst, // dst0
- int32_t n_weights,
- struct htp_mm_kernel_params * kparams
-) {
- ggml_hexagon_precompute_matmul_params_impl(sess, src0, act, dst, 0, 0, kparams);
- kparams->n_weights = n_weights;
-}
-
-static bool ggml_hexagon_op_mul_mat_use_fwht(const struct ggml_tensor * dst) {
- if (dst->op != GGML_OP_MUL_MAT || ggml_get_op_params_i32(dst, 1) != GGML_HINT_SRC0_IS_HADAMARD) {
- return false;
- }
-
- const struct ggml_tensor * src0 = dst->src[0];
- const struct ggml_tensor * src1 = dst->src[1];
-
- const int64_t n = src1->ne[0];
- if (n < 32 || (n & (n - 1)) != 0) {
- return false;
- }
-
- return src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 &&
- (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) &&
- ggml_is_contiguous_rows(src1) && ggml_is_contiguous(dst) &&
- ggml_are_same_shape(src1, dst);
-}
-
-static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) {
- const struct ggml_tensor * src0 = dst->src[0];
- const struct ggml_tensor * src1 = dst->src[1];
-
- if (ggml_get_op_params_i32(dst, 1) == GGML_HINT_SRC0_IS_HADAMARD) {
- if (!ggml_hexagon_op_mul_mat_use_fwht(dst)) {
- return false;
- }
- struct htp_unary_kernel_params ukparams;
- ggml_hexagon_precompute_unary_params(sess, HTP_OP_HADAMARD, src1, nullptr, dst, &ukparams);
- return ukparams.vtcm_row_per_thread > 0 && (size_t) ukparams.vtcm_size <= sess->vtcm_size;
- }
-
- if (dst->type != GGML_TYPE_F32) {
- return false;
- }
-
- if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) {
- return false;
- }
-
- switch (src0->type) {
- case GGML_TYPE_Q1_0:
- case GGML_TYPE_Q2_0:
- case GGML_TYPE_Q4_0:
- case GGML_TYPE_Q4_1:
- case GGML_TYPE_Q8_0:
- case GGML_TYPE_IQ4_NL:
- case GGML_TYPE_MXFP4:
- case GGML_TYPE_Q4_K:
- case GGML_TYPE_Q5_K:
- case GGML_TYPE_Q6_K:
- case GGML_TYPE_Q3_K:
- case GGML_TYPE_Q2_K: {
- if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) {
- return false;
- }
-
- const int64_t qk = ggml_blck_size(src0->type);
- if (src0->ne[0] % qk) {
- return false;
- }
-
- if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) {
- return false;
- }
- if (src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) {
- return false;
- }
- break;
- }
-
- case GGML_TYPE_F16:
- if (src0->nb[1] < src0->nb[0]) {
- return false;
- }
- if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) {
- return false;
- }
- if (src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) {
- return false;
- }
- break;
-
- case GGML_TYPE_F32:
- if (src1->type != GGML_TYPE_F32) {
- return false;
- }
- if (src0->nb[1] < src0->nb[0]) {
- return false;
- }
- if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) {
- return false;
- }
- if (src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) {
- return false;
- }
- break;
-
- case GGML_TYPE_BF16:
- if (src0->nb[1] < src0->nb[0]) {
- return false;
- }
- if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) {
- return false;
- }
- if (src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) {
- return false;
- }
- break;
-
- default:
- return false;
- }
-
- struct htp_mm_kernel_params kparams;
- ggml_hexagon_precompute_matmul_params(sess, src0, src1, dst, &kparams);
- if (kparams.kernel_type == HTP_MM_KERNEL_UNSUPPORTED || (size_t) kparams.vtcm_size > sess->vtcm_size) {
- HEX_VERBOSE("ggml-hex: %s supported MUL_MAT VTCM size needed (%d) > budget (%zu)\n", sess->c_name(), kparams.vtcm_size, sess->vtcm_size);
- return false;
- }
-
- return true;
-}
-
-static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * src1 = op->src[1];
- const struct ggml_tensor * src2 = op->src[2];
- const struct ggml_tensor * dst = op;
-
- if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32) {
- return false;
- }
-
- if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) {
- return false;
- }
-
- switch (src0->type) {
- case GGML_TYPE_Q1_0:
- case GGML_TYPE_Q2_0:
- case GGML_TYPE_Q4_0:
- case GGML_TYPE_Q4_1:
- case GGML_TYPE_Q8_0:
- case GGML_TYPE_IQ4_NL:
- case GGML_TYPE_MXFP4:
- case GGML_TYPE_Q4_K:
- case GGML_TYPE_Q5_K:
- case GGML_TYPE_Q6_K:
- case GGML_TYPE_Q3_K:
- case GGML_TYPE_Q2_K:
- if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) {
- return false;
- }
-
- if (src0->ne[0] % ggml_blck_size(src0->type) != 0) {
- return false;
- }
- break;
-
- case GGML_TYPE_F16:
- case GGML_TYPE_F32:
- case GGML_TYPE_BF16:
- if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0)) {
- return false;
- }
- if (src0->ne[0] % 32 != 0) {
- return false;
- }
- break;
-
- default:
- return false;
- }
-
- struct htp_mm_kernel_params kparams;
- ggml_hexagon_precompute_matmul_params(sess, src0, src1, dst, &kparams);
- if (kparams.kernel_type == HTP_MM_KERNEL_UNSUPPORTED || (size_t) kparams.vtcm_size > sess->vtcm_size) {
- HEX_VERBOSE("ggml-hex: %s supported MUL_MAT_ID VTCM size needed (%d) > budget (%zu)\n", sess->c_name(), kparams.vtcm_size, sess->vtcm_size);
- return false;
- }
-
- return true;
-}
-
-static bool ggml_hexagon_supported_binary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * src1 = op->src[1];
- const struct ggml_tensor * dst = op;
-
- if (src0->type == GGML_TYPE_F32) {
- if (src1->type != GGML_TYPE_F32) {
- return false;
- }
- if (dst->type != GGML_TYPE_F32) {
- return false;
- }
- }
- else if (src0->type == GGML_TYPE_F16) {
- if (src1->type != GGML_TYPE_F16) {
- return false;
- }
- if (dst->type != GGML_TYPE_F16) {
- return false;
- }
- }
- else {
- return false;
- }
-
- if (ggml_is_permuted(src0) || ggml_is_permuted(dst)) {
- return false;
- }
- if (!ggml_are_same_shape(src0, dst)) {
- return false;
- }
- if (!ggml_can_repeat(src1, src0) || ggml_is_permuted(src1)) {
- return false;
- }
-
- struct htp_binary_kernel_params kparams;
- return ggml_hexagon_precompute_binary_params(sess, op_remap_to_htp(op), src0, src1, dst, &kparams);
-}
-
-static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * src1 = op->src[1];
- const struct ggml_tensor * src2 = op->src[2];
- const struct ggml_tensor * dst = op;
-
- if (!src2) {
- return false;
- }
- if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32) {
- return false;
- }
- if (!ggml_are_same_shape(src0, dst)) {
- return false;
- }
- if (src1->ne[0] != src0->ne[0] || src1->ne[2] != 1 || src1->ne[3] != 1) {
- return false;
- }
- if (src2->ne[0] != src0->ne[1] || src2->ne[1] != src0->ne[2]) {
- return false;
- }
- if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float) || src2->nb[0] != sizeof(int32_t)) {
- return false;
- }
-
- // REVISIT: add support for non-contiguous tensors
- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {
- return false;
- }
- if (src2->nb[1] < src2->ne[0] * sizeof(int32_t)) {
- return false;
- }
-
- struct htp_binary_kernel_params kparams;
- return ggml_hexagon_precompute_binary_params(sess, HTP_OP_ADD_ID, src0, src1, dst, &kparams, src2);
-}
-
-static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * dst = op;
-
- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) {
- return false;
- }
- if (dst->type != src0->type) {
- return false;
- }
- if (!ggml_is_contiguous_rows(src0)) {
- return false;
- }
-
- // F16 device kernels only cover this explicit whitelist (must stay in sync with
- // the is_f16 whitelist in execute_op_unary(), unary-ops.c).
- if (src0->type == GGML_TYPE_F16) {
- switch (op->op) {
- case GGML_OP_NORM:
- case GGML_OP_RMS_NORM:
- case GGML_OP_L2_NORM:
- case GGML_OP_SCALE:
- case GGML_OP_CLAMP:
- case GGML_OP_SQR:
- case GGML_OP_SQRT:
- case GGML_OP_LOG:
- break;
- case GGML_OP_UNARY:
- if (ggml_get_unary_op(op) != GGML_UNARY_OP_ABS &&
- ggml_get_unary_op(op) != GGML_UNARY_OP_STEP) {
- return false;
- }
- break;
- default:
- return false;
- }
- }
-
- if (!ggml_are_same_shape(src0, dst)) {
- return false;
- }
-
- // dst must be contiguous; src0 may be non-contiguous
- if (!ggml_is_contiguous(dst)) {
- return false;
- }
-
- return true;
-
- GGML_UNUSED(sess);
-}
-
-static bool ggml_hexagon_supported_sum(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * dst = op;
-
- if (src0->type != GGML_TYPE_F32) {
- return false;
- }
- if (dst->type != GGML_TYPE_F32) {
- return false;
- }
-
- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {
- return false;
- }
-
- return true;
-
- GGML_UNUSED(sess);
-}
-
-static bool ggml_hexagon_supported_sum_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * dst = op;
-
- if (src0->type != GGML_TYPE_F32) {
- return false;
- }
- if (dst->type != GGML_TYPE_F32) {
- return false;
- }
-
- // TODO: add support for non-contigiuos tensors
- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {
- return false;
- }
-
- return true;
-
- GGML_UNUSED(sess);
-}
-
-static bool ggml_hexagon_supported_argmax(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * dst = op;
-
- if (src0->type != GGML_TYPE_F32) {
- return false;
- }
- if (dst->type != GGML_TYPE_I32) {
- return false;
- }
-
- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {
- return false;
- }
-
- return true;
-
- GGML_UNUSED(sess);
-}
-
-static bool ggml_hexagon_supported_pool_2d(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const int32_t * params = op->op_params;
-
- if (params[0] != GGML_OP_POOL_AVG && params[0] != GGML_OP_POOL_MAX) {
- return false;
- }
- if (src0->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) {
- return false;
- }
- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(op)) {
- return false;
- }
- if (!ggml_hexagon_pool_shape_fits(src0, op)) {
- return false;
- }
-
- const int32_t kernel_x = params[1];
- const int32_t kernel_y = params[2];
- const int32_t stride_x = params[3];
- const int32_t stride_y = params[4];
- const int32_t pad_x = params[5];
- const int32_t pad_y = params[6];
-
- // Keep invalid parameters on CPU, avoiding unsafe unsigned values in HTP.
- if (kernel_x <= 0 || kernel_y <= 0 || stride_x <= 0 || stride_y <= 0 ||
- pad_x < 0 || pad_y < 0) {
- return false;
- }
-
- const uint32_t planes = (uint32_t) (src0->ne[2] * src0->ne[3]);
- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, planes > 0 ? planes : 1);
-
- struct htp_pool_vtcm_layout layout;
- return htp_pool_solve_layout(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1],
- (uint32_t) op->ne[0], (uint32_t) op->ne[1],
- n_threads, sess->vtcm_size);
-}
-
-static bool ggml_hexagon_supported_pool_1d(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const int32_t * params = op->op_params;
-
- if (params[0] != GGML_OP_POOL_AVG && params[0] != GGML_OP_POOL_MAX) {
- return false;
- }
- if (src0->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) {
- return false;
- }
- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(op)) {
- return false;
- }
- if (!ggml_hexagon_pool_shape_fits(src0, op)) {
- return false;
- }
-
- const int32_t kernel = params[1];
- const int32_t stride = params[2];
- const int32_t pad = params[3];
- if (kernel <= 0 || stride <= 0 || pad < 0) {
- return false;
- }
-
- const uint32_t planes = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, planes > 0 ? planes : 1);
-
- struct htp_pool_vtcm_layout layout;
- return htp_pool_solve_layout(&layout, (uint32_t) src0->ne[0], 1, (uint32_t) op->ne[0], 1, n_threads, sess->vtcm_size);
-}
-
-static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * src1 = op->src[1];
- const struct ggml_tensor * dst = op;
-
- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) {
- return false;
- }
- if (dst->type != src0->type) {
- return false;
- }
-
- if (!ggml_is_contiguous_1(src0)) {
- return false;
- }
- if (!ggml_is_contiguous(dst)) {
- return false;
- }
-
- if (src1) {
- if (src1->type != src0->type) {
- return false;
- }
- if (!ggml_are_same_shape(src0, src1)) {
- return false;
- }
- if (!ggml_is_contiguous_1(src1)) {
- return false;
- }
- }
-
- return true;
-
- GGML_UNUSED(sess);
-}
-
-static bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * src1 = op->src[1];
- const struct ggml_tensor * src2 = op->src[2];
- const struct ggml_tensor * dst = op;
-
- if (src2) {
- return false; // FIXME: add support for sinks
- }
-
- if (src0->type != GGML_TYPE_F32) {
- return false;
- }
- if (dst->type != GGML_TYPE_F32) {
- return false;
- }
-
- if (src0->ne[2] > 512) {
- return false;
- }
-
- if (src1) {
- if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) {
- return false;
- }
- if (src0->ne[0] != src1->ne[0]) {
- return false;
- }
- if (src1->ne[1] < src0->ne[1]) {
- return false;
- }
- if (src0->ne[2] % src1->ne[2] != 0) {
- return false;
- }
- if (src0->ne[3] % src1->ne[3] != 0) {
- return false;
- }
- }
-
- if (src1) {
- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {
- return false;
- }
- } else {
- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {
- return false;
- }
- }
-
- // Reject non-HVX-aligned sizes when ne[0] > HVX_F32_LANES
- // The HVX softmax implementation has issues with tail handling for larger non-aligned sizes
- // Small sizes (ne[0] <= 32) work correctly with tail-only processing
- const int64_t ne0 = src0->ne[0];
- if (ne0 > 32 && (ne0 & (32 - 1)) != 0) {
- return false;
- }
-
- // HVX vector size constraints for softmax
- #define SOFTMAX_MAX_ROW_SIZE 131072 // 128K elements max for numerical precision
-
- // Reject very large row sizes to avoid numerical precision issues
- // Softmax accumulation over many elements can lead to precision loss
- if (ne0 > SOFTMAX_MAX_ROW_SIZE) {
- return false;
- }
-
- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows);
- struct htp_softmax_vtcm_layout layout;
- htp_softmax_vtcm_layout_build(&layout, src0->ne[0], src1 ? src1->ne[0] : 1, src1 != nullptr, src1 && src1->type == GGML_TYPE_F16, n_threads);
- if (layout.total_bytes > sess->vtcm_size) {
- return false;
- }
-
- return true;
-
- GGML_UNUSED(sess);
-}
-
-static bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0]; // values
- const struct ggml_tensor * src1 = op->src[1]; // indices
- const struct ggml_tensor * dst = op->src[2] ? op->src[2] : op;
-
- if (dst->type == GGML_TYPE_Q8_0 && src0->ne[0] < 32) {
- return false;
- }
-
- if (src0->type != GGML_TYPE_F32) {
- return false;
- }
-
- if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) {
- return false;
- }
-
- if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_Q8_0) {
- return false;
- }
-
- return true;
-
- GGML_UNUSED(sess);
-}
-
-static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0]; // values
- const struct ggml_tensor * src1 = op->src[1]; // indices
- const struct ggml_tensor * dst = op;
-
- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16 && src0->type != GGML_TYPE_I32 &&
- src0->type != GGML_TYPE_Q1_0 && src0->type != GGML_TYPE_Q2_0 && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 &&
- src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K ) {
- return false;
- }
-
- if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous_rows(src1) || !ggml_is_contiguous_rows(dst)) {
- return false;
- }
-
- if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) {
- return false;
- }
-
- if (src0->type == dst->type) {
- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->type != GGML_TYPE_F16) {
- return false;
- }
- } else if (src0->type == GGML_TYPE_I32) {
- return false;
- } else if (dst->type != GGML_TYPE_F32) {
- return false;
- }
-
- // Empty recurrent-state gathers are skipped at execution; do not split the graph for them.
- if (ggml_is_empty(op)) {
- return true;
- }
-
- struct htp_get_rows_kernel_params kparams;
- ggml_hexagon_precompute_get_rows_params(sess, src0, src1, dst, &kparams);
- if (kparams.n_threads == 0 || (size_t) kparams.vtcm_size > sess->vtcm_size) {
- return false;
- }
-
- return true;
-
- GGML_UNUSED(sess);
-}
-
-static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0]; // values
- const struct ggml_tensor * dst = op; // indices
-
- if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_I32) {
- return false;
- }
-
- const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const uint32_t n_threads_max = sess->n_threads > 0 ? sess->n_threads : 4;
- const size_t vtcm_budget = sess->vtcm_size > 0 ? sess->vtcm_size : (8 * 1024 * 1024);
-
- struct htp_sort_vtcm_layout layout;
- if (!htp_sort_solve_layout(&layout, src0->ne[0], total_rows, dst->ne[0], n_threads_max, vtcm_budget, false)) {
- return false;
- }
-
- return true;
-}
-
-static bool ggml_hexagon_supported_top_k(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0]; // values
- const struct ggml_tensor * dst = op; // indices
-
- if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_I32) {
- return false;
- }
-
- const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const uint32_t n_threads_max = sess->n_threads > 0 ? sess->n_threads : 4;
- const size_t vtcm_budget = sess->vtcm_size > 0 ? sess->vtcm_size : (8 * 1024 * 1024);
-
- struct htp_sort_vtcm_layout layout;
- if (!htp_sort_solve_layout(&layout, src0->ne[0], total_rows, dst->ne[0], n_threads_max, vtcm_budget, true)) {
- return false;
- }
-
- return true;
+ op_batch->reset();
}
-static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * src1 = op->src[1];
- const struct ggml_tensor * src2 = op->src[2];
- const struct ggml_tensor * dst = op;
-
- if (!ggml_are_same_shape(src0, dst)) {
- return false;
- }
-
- if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32) {
- return false;
- }
-
- if (src0->ne[0] <= 0) {
- return false;
- }
-
- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- if (src0_nrows == 0) {
- return false;
- }
-
- const int32_t * op_params = &op->op_params[0];
- const int n_dims = op_params[1];
- const int mode = op_params[2];
- const int n_offs = op_params[15];
-
- // llama probes weight placement with a dummy rope where every param is 0 (llama-model-loader.cpp).
- // Rejecting it puts rope_freqs on the CPU, which then splits the graph at every full-attention layer.
- if (n_dims < 0 || n_dims % 2 != 0) {
- return false;
- }
+void ggml_hexagon_session::enqueue_op(const htp_opnode & node) {
+ auto clone_tensor_buffer = [this](const ggml_tensor * t) {
+ auto sbuf = this->mmap_tensor(t);
+ if (!sbuf) return;
+ if (sbuf->sess != this) {
+ this->clone_buffer(sbuf);
+ }
+ for (auto & sub : this->mdev.sessions) {
+ sub->clone_buffer(sbuf);
+ }
+ };
- // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems)
- if (n_offs < 0 || (n_offs % 32 != 0) || (n_offs + n_dims > src0->ne[0])) {
- return false;
+ for (auto t : node.get_inputs()) {
+ clone_tensor_buffer(t);
}
-
- float freq_base;
- memcpy(&freq_base, op_params + 5, sizeof(float));
- if (freq_base < 0.0f) {
- return false;
+ for (auto t : node.get_outputs()) {
+ clone_tensor_buffer(t);
}
- if (mode != GGML_ROPE_TYPE_NORMAL &&
- mode != GGML_ROPE_TYPE_NEOX &&
- mode != GGML_ROPE_TYPE_MROPE &&
- mode != GGML_ROPE_TYPE_VISION &&
- mode != GGML_ROPE_TYPE_IMROPE) {
- return false;
+ const bool fits_tensors = op_batch->fit_tensors(node);
+ if (opt_opfusion && fits_tensors && op_batch->try_fuse(node)) {
+ return;
}
- const bool is_mrope = (mode & GGML_ROPE_TYPE_MROPE) != 0;
-
- // n_dims == ne0/2, so the rotation spans the full row
- if (mode == GGML_ROPE_TYPE_VISION) {
- if (n_dims != (int) (src0->ne[0] / 2) || n_offs != 0) {
- return false;
- }
+ if (op_batch->full() || !fits_tensors) {
+ flush_async();
}
- if (is_mrope) {
- const int32_t * sections = op_params + 11;
- if (sections[0] <= 0 && sections[1] <= 0 && sections[2] <= 0) {
- return false;
+ if (op_batch->empty()) {
+ start_batch();
+ if (!op_batch->fit_tensors(node)) {
+ GGML_ABORT("ggml-hex: %s op does not fit into empty batch (vmem/tensor/buffer limit exceeded)\n", c_name());
}
}
- const int64_t min_pos_len = (is_mrope || mode == GGML_ROPE_TYPE_VISION) ? src0->ne[2] * 4 : src0->ne[2];
- if (src1->ne[0] < min_pos_len || !ggml_is_contiguous(src1)) {
- return false;
- }
+ op_batch->add_op(node);
+}
- if (src2) {
- if (src2->type != GGML_TYPE_F32 || !ggml_is_contiguous(src2)) {
- return false;
- }
- if (src2->ne[0] < (n_dims / 2)) {
- return false;
- }
- }
+void ggml_hexagon_session::enqueue_mdev_group() {
+ htp_opnode group_node(HTP_OP_MDEV_GROUP);
- // src0/dst elements within a row must be contiguous (nb[0] == sizeof(float)).
- // nb[1] may exceed ne[0]*sizeof(float) when the tensor is a strided view of a larger one
- if (src0->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) {
- return false;
- }
- if (src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) {
- return false;
- }
+ ggml_tensor dummy_t = make_fence_tensor(this->mdev_fence_slot, this->mdev.count);
+ dummy_t.op_params[0] = (int32_t) this->mdev.idx;
- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows);
- const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0;
+ ggml_tensor * node = group_node.add_dummy(dummy_t);
+ node->src[0] = node;
+ group_node.init(node);
+ group_node.outputs.clear();
- struct htp_rope_vtcm_layout layout;
- htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads, n_freq_factors);
- if (layout.total_bytes > sess->vtcm_size) {
- return false;
+ for (auto & sub : this->mdev.sessions) {
+ sub->clone_buffer(this->fence_buf);
}
- return true;
+ op_batch->add_op(group_node);
}
-static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * src1 = op->src[1];
- const struct ggml_tensor * dst = op;
-
- // Only support FP32 for now
- if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
- return false;
- }
-
- // Check IO tensor shapes and dims
- if (src0->ne[3] != 1 || src1->ne[2] != 1 || src1->ne[3] != 1 || dst->ne[3] != 1) {
- return false; // src0 should be effectively 3D
- }
-
- const int d_conv = src1->ne[0];
- const int d_inner = src0->ne[1];
- const int n_t = dst->ne[1];
- const int n_s = dst->ne[2];
+void ggml_hexagon_session::enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor, uint32_t fence_seq) {
+ const bool with_fence = sync_tensor != nullptr;
+ htp_opnode cpy_node(with_fence ? HTP_OP_CPY_FENCE : HTP_OP_CPY);
- if (d_conv == 0 || d_conv > 32 || d_inner == 0) {
- return false;
- }
- if (src0->ne[0] != d_conv - 1 + n_t || src0->ne[1] != d_inner || src0->ne[2] != n_s) {
- return false;
- }
- if (src1->ne[0] != d_conv || src1->ne[1] != d_inner) {
- return false;
- }
- if (dst->ne[0] != d_inner || dst->ne[1] != n_t || dst->ne[2] != n_s) {
- return false;
- }
- if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) {
- return false;
- }
- if (src0->nb[1] != src0->ne[0] * sizeof(float) || src1->nb[1] != src1->ne[0] * sizeof(float)) {
- return false;
+ ggml_tensor* node = cpy_node.add_dummy(*dst);
+ node->op = GGML_OP_CPY;
+ node->src[0] = const_cast<ggml_tensor *>(src);
+ node->src[1] = with_fence ? cpy_node.add_dummy(*sync_tensor) : nullptr;
+ if (with_fence) {
+ node->op_params[0] = (int32_t) fence_seq;
}
- struct htp_ssm_conv_kernel_params kparams;
- ggml_hexagon_precompute_ssm_conv_params(sess, src0, src1, dst, &kparams);
- if ((size_t) kparams.vtcm_size > sess->vtcm_size) {
- return false;
+ cpy_node.init(node);
+ const bool ok = htp_kparams_solve(this, node, cpy_node.kernel_params);
+ if (ok && !with_fence && cpy_node.kparams<htp_copy_kernel_params>()->total_elems == 0) {
+ return;
}
-
- return true;
+ this->enqueue_op(cpy_node);
}
-static bool im2col_3d_calc_output_size(uint32_t input, uint32_t kernel, int32_t stride, int32_t padding,
- int32_t dilation, uint32_t * output) {
- if (stride <= 0 || padding < 0 || dilation <= 0 || kernel == 0) {
- return false;
- }
-
- const int64_t numerator = (int64_t) input + 2LL * padding - (int64_t) dilation * (kernel - 1) - 1;
- if (numerator < 0) {
- return false;
- }
+void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq, bool wait) {
+ htp_opnode sync_node(HTP_OP_FENCE);
- const uint64_t size = (uint64_t) (numerator / stride) + 1;
- if (size == 0 || size > UINT32_MAX) {
- return false;
- }
+ ggml_tensor* node = sync_node.add_dummy(*sync_tensor);
+ node->op = GGML_OP_NONE;
+ node->src[0] = node;
+ node->op_params[0] = (int32_t) fence_seq;
+ node->op_params[1] = wait ? 0 : 1;
- *output = (uint32_t) size;
- return true;
+ sync_node.init(node);
+ this->enqueue_op(sync_node);
}
-static bool ggml_hexagon_precompute_im2col_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_im2col_kernel_params * kparams
+void ggml_hexagon_session::enqueue_allreduce(
+ const ggml_tensor * dst,
+ const std::vector<const ggml_tensor *> & src_tensors,
+ const std::vector<const ggml_tensor *> & sync_tensors,
+ uint32_t rank,
+ uint32_t n_ranks,
+ uint32_t fence_seq_entry,
+ uint32_t fence_seq_exit
) {
- memset(kparams, 0, sizeof(*kparams));
- kparams->kernel_type = HTP_IM2COL_KERNEL_UNSUPPORTED;
+ htp_opnode ar_node(HTP_OP_ALLREDUCE);
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * src1 = op->src[1];
- const struct ggml_tensor * dst = op;
+ ggml_tensor* node = ar_node.add_dummy(*dst);
+ node->op = GGML_OP_NONE;
+ node->op_params[0] = (int32_t) fence_seq_entry;
+ node->op_params[1] = (int32_t) fence_seq_exit;
- if (!src0 || !src1 || !dst) {
- return false;
- }
+ ar_node.init(node);
- if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) {
- return false;
+ ar_node.inputs.clear();
+ for (size_t i = 0; i < src_tensors.size(); i++) {
+ ar_node.inputs.push_back(src_tensors[i]);
}
-
- if (src1->nb[0] != sizeof(float) || !ggml_is_contiguous(dst)) {
- return false;
+ for (size_t i = 0; i < sync_tensors.size(); i++) {
+ ar_node.inputs.push_back(ar_node.add_dummy(*sync_tensors[i]));
}
- const int32_t * op_params = (const int32_t *) op->op_params;
- const int32_t s0 = op_params[0];
- const int32_t s1 = op_params[1];
- const int32_t p0 = op_params[2];
- const int32_t p1 = op_params[3];
- const int32_t d0 = op_params[4];
- const int32_t d1 = op_params[5];
- const bool is_2D = (op_params[6] == 1);
-
- if (s0 <= 0 || d0 <= 0 || p0 < 0) {
- return false;
- }
- if (is_2D && (s1 <= 0 || d1 <= 0 || p1 < 0)) {
- return false;
+ ar_node.outputs.clear();
+ for (size_t i = 0; i < src_tensors.size(); i++) {
+ ar_node.outputs.push_back(src_tensors[i]);
}
- const uint32_t N = is_2D ? (uint32_t) src1->ne[3] : (uint32_t) src1->ne[2];
- const uint32_t IC = is_2D ? (uint32_t) src1->ne[2] : (uint32_t) src1->ne[1];
- const uint32_t IH = is_2D ? (uint32_t) src1->ne[1] : 1;
- const uint32_t IW = (uint32_t) src1->ne[0];
- const uint32_t KH = is_2D ? (uint32_t) src0->ne[1] : 1;
- const uint32_t KW = (uint32_t) src0->ne[0];
- const uint32_t OH = is_2D ? (uint32_t) dst->ne[2] : 1;
- const uint32_t OW = (uint32_t) dst->ne[1];
-
- if (KW == 0 || KH == 0 || IW == 0 || IH == 0 || IC == 0 || OW == 0 || OH == 0 || N == 0) {
- return false;
- }
+ htp_kparams_solve_allreduce(
+ this, dst, rank, n_ranks, false, false, /*is_shard_ok=*/ false,
+ ar_node.kparams<htp_allreduce_kernel_params>());
- const uint64_t patch_stride_64 = (uint64_t) IC * KH * KW;
- const uint64_t total_rows_64 = (uint64_t) N * OH;
- if (patch_stride_64 > UINT32_MAX || total_rows_64 > UINT32_MAX) {
- return false;
- }
- const uint32_t patch_stride = (uint32_t) patch_stride_64;
- const uint32_t total_rows = (uint32_t) total_rows_64;
+ this->enqueue_op(ar_node);
+}
- if (dst->ne[0] != patch_stride || dst->ne[1] != OW) {
- return false;
- }
- if (is_2D && dst->ne[2] != OH) {
- return false;
- }
+ggml_hexagon_shared_buffer * ggml_hexagon_session::mmap_tensor(const ggml_tensor * t) {
+ if (!t) return nullptr;
- const uint32_t dst_elem = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);
-
- bool is_patchembed = ggml_is_contiguous(src1) && (s0 == (int32_t) KW && p0 == 0 && d0 == 1);
- if (is_2D) {
- is_patchembed = is_patchembed && (s1 == (int32_t) KH && p1 == 0 && d1 == 1);
- }
-
- const uint32_t max_threads = (sess && sess->n_threads > 0) ? (uint32_t) sess->n_threads : 8;
- const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads;
- const uint32_t vtcm_size = (sess && sess->vtcm_size > 0) ? sess->vtcm_size : 0;
-
- if (is_patchembed) {
- const uint64_t src_row_bytes_raw = (uint64_t) IC * KH * IW * sizeof(float);
- const uint64_t dst_row_bytes_raw = (uint64_t) OW * patch_stride * dst_elem;
- if (src_row_bytes_raw <= UINT32_MAX && dst_row_bytes_raw <= UINT32_MAX) {
- const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);
- const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
-
- struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
-
- if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
- kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;
- kparams->is_2d = is_2D ? 1 : 0;
- kparams->is_3d = 0;
- kparams->n_threads = (uint8_t) n_threads;
- kparams->vtcm_size = layout.total_bytes;
- kparams->off_src = layout.off_src;
- kparams->off_dst = layout.off_dst;
- kparams->src_bytes_per_thread = layout.src_bytes_per_thread;
- kparams->dst_bytes_per_thread = layout.dst_bytes_per_thread;
- kparams->src_row_bytes = src_row_bytes;
- kparams->dst_row_bytes = dst_row_bytes;
- kparams->owb = OW;
- kparams->wb = IW;
- kparams->nbpr = 1;
- kparams->N = N;
- kparams->IC = IC;
- kparams->ID = 1;
- kparams->IH = IH;
- kparams->IW = IW;
- kparams->KD = 1;
- kparams->KH = KH;
- kparams->KW = KW;
- kparams->OD = 1;
- kparams->OH = OH;
- kparams->OW = OW;
- kparams->patch_stride = patch_stride;
- kparams->total_rows = total_rows;
- return true;
- }
- }
+ auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
+ if (!sbuf->mapped) {
+ const bool is_weight = ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS;
+ const bool extended = opt_dma64 && is_weight;
+ sbuf->mmap(extended);
}
+ return sbuf;
+}
- for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) {
- const uint64_t Wb = (uint64_t) (owb - 1) * s0 + (uint64_t) (KW - 1) * d0 + 1;
- if (Wb > UINT32_MAX) {
- if (owb == 1) break;
- continue;
- }
-
- const uint64_t src_row_bytes_raw = (uint64_t) IC * KH * Wb * sizeof(float);
- const uint64_t dst_row_bytes_raw = (uint64_t) owb * patch_stride * dst_elem;
- if (src_row_bytes_raw > UINT32_MAX || dst_row_bytes_raw > UINT32_MAX) {
- if (owb == 1) break;
- continue;
- }
-
- const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);
- const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
-
- struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
-
- if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
- kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA;
- kparams->is_2d = is_2D ? 1 : 0;
- kparams->is_3d = 0;
- kparams->n_threads = (uint8_t) n_threads;
- kparams->vtcm_size = layout.total_bytes;
- kparams->off_src = layout.off_src;
- kparams->off_dst = layout.off_dst;
- kparams->src_bytes_per_thread = layout.src_bytes_per_thread;
- kparams->dst_bytes_per_thread = layout.dst_bytes_per_thread;
- kparams->src_row_bytes = src_row_bytes;
- kparams->dst_row_bytes = dst_row_bytes;
- kparams->owb = owb;
- kparams->wb = (uint32_t) Wb;
- kparams->nbpr = (OW + owb - 1) / owb;
- kparams->N = N;
- kparams->IC = IC;
- kparams->ID = 1;
- kparams->IH = IH;
- kparams->IW = IW;
- kparams->KD = 1;
- kparams->KH = KH;
- kparams->KW = KW;
- kparams->OD = 1;
- kparams->OH = OH;
- kparams->OW = OW;
- kparams->patch_stride = patch_stride;
- kparams->total_rows = total_rows;
- return true;
- }
- if (owb == 1) break;
- }
+bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf)
+{
+ GGML_ASSERT(sbuf && sbuf->mem);
+ if (sbuf->sess == this) return true;
- return false;
-}
+ auto mem = sbuf->mem;
+ int fd = mem->fd;
-static bool ggml_hexagon_precompute_im2col_3d_params(
- const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * op,
- struct htp_im2col_kernel_params * kparams
-) {
- memset(kparams, 0, sizeof(*kparams));
- kparams->kernel_type = HTP_IM2COL_KERNEL_UNSUPPORTED;
+ GGML_ASSERT(fd >= 0);
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * src1 = op->src[1];
- const struct ggml_tensor * dst = op;
+ if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return true;
- if (!src0 || !src1 || !dst) {
- return false;
- }
+ GGML_ASSERT(sbuf->mapped);
- if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) {
- return false;
- }
+ HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(),
+ sbuf->c_name(), sbuf->base(), sbuf->size(), fd);
- if (src1->nb[0] != sizeof(float) || !ggml_is_contiguous(dst)) {
+ auto clone = std::make_unique<ggml_hexagon_shared_buffer>(this, *sbuf);
+ try {
+ clone->mmap(sbuf->extended);
+ } catch (const std::exception & exc) {
+ GGML_LOG_ERROR("ggml-hex: %s lazy mapping of buffer context failed: %s\n", this->c_name(), exc.what());
return false;
}
- const int32_t * params = (const int32_t *) op->op_params;
- const int32_t s0 = params[0], s1 = params[1], s2 = params[2];
- const int32_t p0 = params[3], p1 = params[4], p2 = params[5];
- const int32_t d0 = params[6], d1 = params[7], d2 = params[8];
- const int32_t ic = params[9];
+ this->cloned_buffers[fd] = std::move(clone);
+ mem->mapped_clones.insert(this);
+ return true;
+}
- if (s0 <= 0 || s1 <= 0 || s2 <= 0 ||
- p0 < 0 || p1 < 0 || p2 < 0 ||
- d0 <= 0 || d1 <= 0 || d2 <= 0 || ic <= 0) {
- return false;
- }
+void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbuf) {
+ GGML_ASSERT(sbuf && sbuf->mem);
- if (src1->ne[3] % ic != 0) {
- return false;
- }
+ auto mem = sbuf->mem;
+ int fd = mem->fd;
- const uint32_t IC = (uint32_t) ic;
- const uint32_t N = (uint32_t) (src1->ne[3] / IC);
- if (N == 0) {
- return false;
- }
+ GGML_ASSERT(fd >= 0);
- uint32_t OW, OH, OD;
- if (!im2col_3d_calc_output_size(src1->ne[0], src0->ne[0], s0, p0, d0, &OW) ||
- !im2col_3d_calc_output_size(src1->ne[1], src0->ne[1], s1, p1, d1, &OH) ||
- !im2col_3d_calc_output_size(src1->ne[2], src0->ne[2], s2, p2, d2, &OD)) {
- return false;
+ auto it = this->cloned_buffers.find(fd);
+ if (it != this->cloned_buffers.end()) {
+ auto clone = std::move(it->second);
+ this->cloned_buffers.erase(it);
}
+ mem->mapped_clones.erase(this);
+}
- const uint32_t KD = (uint32_t) src0->ne[2];
- const uint32_t KH = (uint32_t) src0->ne[1];
- const uint32_t KW = (uint32_t) src0->ne[0];
- const uint32_t ID = (uint32_t) src1->ne[2];
- const uint32_t IH = (uint32_t) src1->ne[1];
- const uint32_t IW = (uint32_t) src1->ne[0];
+void ggml_hexagon_session::unclone_buffer(const ggml_hexagon_shared_buffer * sbuf) {
+ GGML_ASSERT(sbuf && sbuf->mem);
- const uint64_t patch_stride_64 = (uint64_t) IC * KD * KH * KW;
- const uint64_t total_rows_64 = (uint64_t) N * OD * OH;
- const uint64_t dst_ne3_64 = (uint64_t) OD * N;
+ auto mem = sbuf->mem;
+ std::vector<ggml_hexagon_session *> sessions(mem->mapped_clones.begin(), mem->mapped_clones.end());
- if (patch_stride_64 > UINT32_MAX || total_rows_64 > UINT32_MAX || dst_ne3_64 > UINT32_MAX) {
- return false;
+ for (auto * sess : sessions) {
+ sess->release_buffer(sbuf);
}
- const uint32_t patch_stride = (uint32_t) patch_stride_64;
- const uint32_t total_rows = (uint32_t) total_rows_64;
- const uint32_t dst_ne3 = (uint32_t) dst_ne3_64;
+}
- if (dst->ne[0] != patch_stride || dst->ne[1] != OW || dst->ne[2] != OH || dst->ne[3] != dst_ne3) {
- return false;
- }
+static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) {
+ // Allocate a bunch pinned buffers till failure.
+ // This is kind of expensive but handy for figuring out exactly how much we can mmap on a specific device.
+ // Typically we're going to allocate all/most of these buffers anyway for the model weights.
- const uint32_t dst_elem = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);
-
- const bool is_patchembed = ggml_is_contiguous(src1) &&
- (s0 == (int32_t) KW && s1 == (int32_t) KH && s2 == (int32_t) KD &&
- p0 == 0 && p1 == 0 && p2 == 0 &&
- d0 == 1 && d1 == 1 && d2 == 1);
-
- const uint32_t max_threads = (sess && sess->n_threads > 0) ? (uint32_t) sess->n_threads : 8;
- const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads;
- const uint32_t vtcm_size = (sess && sess->vtcm_size > 0) ? sess->vtcm_size : 0;
-
- if (is_patchembed) {
- const uint64_t src_row_bytes_raw = (uint64_t) IC * KD * KH * IW * sizeof(float);
- const uint64_t dst_row_bytes_raw = (uint64_t) OW * patch_stride * dst_elem;
- if (src_row_bytes_raw <= UINT32_MAX && dst_row_bytes_raw <= UINT32_MAX) {
- const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);
- const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
-
- struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
-
- if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
- kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;
- kparams->is_2d = 0;
- kparams->is_3d = 1;
- kparams->n_threads = (uint8_t) n_threads;
- kparams->vtcm_size = layout.total_bytes;
- kparams->off_src = layout.off_src;
- kparams->off_dst = layout.off_dst;
- kparams->src_bytes_per_thread = layout.src_bytes_per_thread;
- kparams->dst_bytes_per_thread = layout.dst_bytes_per_thread;
- kparams->src_row_bytes = src_row_bytes;
- kparams->dst_row_bytes = dst_row_bytes;
- kparams->owb = OW;
- kparams->wb = IW;
- kparams->nbpr = 1;
- kparams->N = N;
- kparams->IC = IC;
- kparams->ID = ID;
- kparams->IH = IH;
- kparams->IW = IW;
- kparams->KD = KD;
- kparams->KH = KH;
- kparams->KW = KW;
- kparams->OD = OD;
- kparams->OH = OH;
- kparams->OW = OW;
- kparams->patch_stride = patch_stride;
- kparams->total_rows = total_rows;
- return true;
- }
- }
- }
+ std::vector<ggml_hexagon_shared_buffer *> sbufs;
- for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) {
- const uint64_t Wb = (uint64_t) (owb - 1) * s0 + (uint64_t) (KW - 1) * d0 + 1;
- if (Wb > UINT32_MAX) {
- if (owb == 1) break;
- continue;
- }
+ const size_t MiB = 1024 * 1024;
+ const size_t GiB = MiB * 1024;
- const uint64_t src_row_bytes_raw = (uint64_t) IC * KD * KH * Wb * sizeof(float);
- const uint64_t dst_row_bytes_raw = (uint64_t) owb * patch_stride * dst_elem;
- if (src_row_bytes_raw > UINT32_MAX || dst_row_bytes_raw > UINT32_MAX) {
- if (owb == 1) break;
- continue;
- }
+ size_t vmem = 0;
+ size_t step = 256u * MiB;
+
+ try {
+ sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB;
+ sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB;
+ sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB;
- const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);
- const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
-
- struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
-
- if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
- kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA;
- kparams->is_2d = 0;
- kparams->is_3d = 1;
- kparams->n_threads = (uint8_t) n_threads;
- kparams->vtcm_size = layout.total_bytes;
- kparams->off_src = layout.off_src;
- kparams->off_dst = layout.off_dst;
- kparams->src_bytes_per_thread = layout.src_bytes_per_thread;
- kparams->dst_bytes_per_thread = layout.dst_bytes_per_thread;
- kparams->src_row_bytes = src_row_bytes;
- kparams->dst_row_bytes = dst_row_bytes;
- kparams->owb = owb;
- kparams->wb = (uint32_t) Wb;
- kparams->nbpr = (OW + owb - 1) / owb;
- kparams->N = N;
- kparams->IC = IC;
- kparams->ID = ID;
- kparams->IH = IH;
- kparams->IW = IW;
- kparams->KD = KD;
- kparams->KH = KH;
- kparams->KW = KW;
- kparams->OD = OD;
- kparams->OH = OH;
- kparams->OW = OW;
- kparams->patch_stride = patch_stride;
- kparams->total_rows = total_rows;
- return true;
+ while (1) {
+ sbufs.push_back(new ggml_hexagon_shared_buffer(sess, step, true));
+ vmem += step;
}
- if (owb == 1) break;
- }
+ } catch (...) { }
- return false;
-}
+ for (auto b : sbufs) { delete b; }
-static bool ggml_hexagon_supported_im2col(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- struct htp_im2col_kernel_params kparams;
- return ggml_hexagon_precompute_im2col_params(sess, op, &kparams);
+ return vmem - step; // backoff to account for overhead from internal mappings
}
-static bool ggml_hexagon_supported_im2col_3d(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- struct htp_im2col_kernel_params kparams;
- return ggml_hexagon_precompute_im2col_3d_params(sess, op, &kparams);
-}
+void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) noexcept(false) {
+ int phys_idx = config.physical_idx;
+ int virt_idx = config.virtual_idx;
-static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * dst = op;
+ this->valid_session = false;
+ this->valid_handle = false;
+ this->valid_queue = false;
+ this->valid_iface = false;
- if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
- return false;
- }
+ this->name = config.name;
+ this->phys_idx = phys_idx;
+ this->virt_idx = virt_idx;
+ this->domain_id = config.domain_id;
+ this->session_id = 0;
+ this->batch_req_seq = 0;
+ this->batch_rsp_seq = 0;
+ this->last_error = HTP_STATUS_OK;
- const int32_t lp0 = ((const int32_t *) op->op_params)[0];
- const int32_t rp0 = ((const int32_t *) op->op_params)[1];
- const int32_t circular = ((const int32_t *) op->op_params)[8];
+ GGML_LOG_DEBUG("ggml-hex: %s allocating new session : domain %u phys-idx %u virt-idx %u\n", this->name.c_str(), this->domain_id, phys_idx, virt_idx);
- if (circular && (lp0 > src0->ne[0] || rp0 > src0->ne[0])) {
- return false;
+ if (config.domain_id < 0 || config.domain_name.empty()) {
+ GGML_LOG_ERROR("ggml-hex: %s: invalid physical CDSP core %d\n", config.name.c_str(), config.physical_idx);
+ throw std::runtime_error("ggml-hex: invalid physical CDSP core");
}
- return true;
+ const std::string & dom_name = config.domain_name;
- GGML_UNUSED(sess);
-}
+ // Create new session if virtual_idx > 0
+ if (virt_idx > 0) {
+ struct remote_rpc_reserve_new_session n {};
+ n.domain_name_len = dom_name.size();
+ n.domain_name = const_cast<char *>(dom_name.c_str());
+ n.session_name = const_cast<char *>(this->name.c_str());
+ n.session_name_len = this->name.size();
+ n.session_id = virt_idx;
-static bool ggml_hexagon_supported_cumsum(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * dst = op;
+ int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n));
+ if (err != AEE_SUCCESS) {
+ GGML_LOG_ERROR("ggml-hex: %s failed to reserve new session (physical %d, virtual %d) : error 0x%x\n",
+ this->c_name(), phys_idx, virt_idx, err);
+ throw std::runtime_error("ggml-hex: remote_session_control(new-sess) failed (see log for details)");
+ }
- if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
- return false;
+ // Save the IDs
+ this->session_id = n.session_id;
+ this->domain_id = n.effective_domain_id;
+ this->valid_session = true;
+ } else {
+ struct remote_rpc_effective_domain_id eff {};
+ eff.domain_name = const_cast<char *>(dom_name.c_str());
+ eff.domain_name_len = dom_name.size();
+ eff.session_id = 0;
+
+ int err = remote_session_control(FASTRPC_GET_EFFECTIVE_DOMAIN_ID, (void *) &eff, sizeof(eff));
+ if (err == AEE_SUCCESS) {
+ this->domain_id = eff.effective_domain_id;
+ } else {
+ GGML_LOG_DEBUG("ggml-hex: %s FASTRPC_GET_EFFECTIVE_DOMAIN_ID returned 0x%x, using domain_id %d\n",
+ this->name.c_str(), err, this->domain_id);
+ }
}
- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {
- return false;
+ // Enable unsigned modules
+ {
+ struct remote_rpc_control_unsigned_module u;
+ u.domain = this->domain_id;
+ u.enable = 1;
+ int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u));
+ if (err != AEE_SUCCESS) {
+ GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err);
+ throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)");
+ }
}
- return true;
+ char session_uri[256];
+ {
+ char htp_uri[256];
+ snprintf(htp_uri, sizeof(htp_uri), "file:///libggml-htp-v%u.so?htp_iface_skel_handle_invoke&_modver=1.0", opt_arch);
- GGML_UNUSED(sess);
-}
+ struct remote_rpc_get_uri u = {};
+ u.session_id = this->session_id;
+ u.domain_name = const_cast<char *>(dom_name.c_str());
+ u.domain_name_len = dom_name.size();
+ u.module_uri = const_cast<char *>(htp_uri);
+ u.module_uri_len = strlen(htp_uri);
+ u.uri = session_uri;
+ u.uri_len = sizeof(session_uri);
-static bool ggml_hexagon_supported_diag(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * dst = op;
+ int err = remote_session_control(FASTRPC_GET_URI, (void *) &u, sizeof(u));
+ if (err != AEE_SUCCESS) {
+ snprintf(session_uri, sizeof(session_uri), "%s&_dom=%s&_session=%u",
+ htp_uri, dom_name.c_str(), this->session_id);
- // diag only supports F32 currently
- if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
- return false;
+ GGML_LOG_WARN("ggml-hex: %s failed to get URI (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n",
+ this->c_name(), phys_idx, virt_idx, err, session_uri);
+ }
}
- // Input must have ne[1] == 1 (vector input)
- if (src0->ne[1] != 1) {
- return false;
+ // Open session
+ int err = htp_iface_open(session_uri, &this->handle);
+ if (err != AEE_SUCCESS) {
+ GGML_LOG_ERROR("ggml-hex: %s failed to open session : uri %s error 0x%x\n", this->c_name(), session_uri, err);
+ throw std::runtime_error("ggml-hex: failed to open session (see log for details)");
}
- // Output must be square in first two dimensions
- if (dst->ne[0] != dst->ne[1] || dst->ne[0] != src0->ne[0]) {
- return false;
+ this->valid_handle = true;
+
+ // Query HW info and resolve session options
+ {
+ unsigned int hw_n_threads = 0;
+ unsigned int hw_n_hvx = 0;
+ unsigned int hw_n_hmx = 0;
+ unsigned long long hw_vtcm_size = 0;
+ int hw_err = htp_iface_hwinfo(this->handle, &hw_n_threads, &hw_n_hvx, &hw_n_hmx, &hw_vtcm_size);
+ if (hw_err == 0) {
+ const uint32_t max_n_threads = (std::min)((uint32_t) HTP_MAX_NTHREADS, (uint32_t) hw_n_threads);
+ this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_threads) : max_n_threads;
+ this->n_hvx = this->n_threads;
+ this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0;
+ this->vtcm_size = (uint64_t)hw_vtcm_size;
+ GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n",
+ this->c_name(), this->n_threads, this->n_hvx, this->n_hmx,
+ (unsigned long long)(this->vtcm_size / (1024 * 1024)));
+ } else {
+ GGML_LOG_WARN("ggml-hex: %s failed to query hwinfo (0x%x), using defaults\n", this->c_name(), hw_err);
+ const uint32_t default_n_threads = (std::min)(8u, (uint32_t) HTP_MAX_NTHREADS);
+ this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads;
+ this->n_hvx = this->n_threads;
+ this->n_hmx = (opt_nhmx != 0) ? 1 : 0;
+ this->vtcm_size = 8 * 1024 * 1024;
+ }
}
- return true;
+ // Enable FastRPC QoS mode
+ {
+ struct remote_rpc_control_latency l;
+ l.enable = 1;
- GGML_UNUSED(sess);
-}
+ int err = remote_handle64_control(this->handle, DSPRPC_CONTROL_LATENCY, (void *) &l, sizeof(l));
+ if (err != 0) {
+ GGML_LOG_WARN("ggml-hex: failed to enable fastrpc QOS mode: 0x%08x\n", (unsigned) err);
+ }
+ }
-static bool ggml_hexagon_supported_solve_tri(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * src0 = op->src[0]; // A
- const struct ggml_tensor * src1 = op->src[1]; // B
- const struct ggml_tensor * dst = op; // X
+ GGML_LOG_INFO("ggml-hex: %s new session : session-id %d domain-id %d uri %s handle 0x%lx\n", this->c_name(),
+ this->session_id, this->domain_id, session_uri, (unsigned long) this->handle);
- if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
- return false;
- }
+ const size_t req_q_size = (sizeof(htp_opbatch_req) * opt_opqueue * 2) + 1024;
+ const size_t rsp_q_size = (sizeof(htp_opbatch_rsp) * opt_opqueue * 2) + 1024;
- if (src0->ne[0] != src0->ne[1]) {
- return false;
+ // Now let's setup the DSP queue
+ err = dspqueue_create(this->domain_id,
+ 0, // Flags
+ req_q_size, // Request queue size (in bytes)
+ rsp_q_size, // Response queue size (in bytes)
+ nullptr, // Read packet callback (we handle reads explicitly)
+ nullptr, // Error callback (we handle errors during reads)
+ (void *) this, // Callback context
+ &queue);
+ if (err != 0) {
+ GGML_LOG_ERROR("ggml-hex: %s dspqueue_create failed: 0x%08x\n", this->name.c_str(), (unsigned) err);
+ throw std::runtime_error("ggml-hex: failed to create dspqueue (see log for details)");
}
- if (src0->ne[1] != src1->ne[1]) {
- return false;
- }
+ this->valid_queue = true;
- if (src0->ne[2] != src1->ne[2] || src0->ne[3] != src1->ne[3]) {
- return false;
+ // Export queue for use on the DSP
+ err = dspqueue_export(queue, &this->queue_id);
+ if (err != 0) {
+ GGML_LOG_ERROR("ggml-hex: dspqueue_export failed: 0x%08x\n", (unsigned) err);
+ throw std::runtime_error("ggml-hex: dspqueue export failed (see log for details)");
}
- if (dst->ne[0] != src1->ne[0] || dst->ne[1] != src1->ne[1] || dst->ne[2] != src1->ne[2] || dst->ne[3] != src1->ne[3]) {
- return false;
+ if (opt_etm) {
+ err = htp_iface_etm(this->handle, 1);
+ if (err != 0) {
+ GGML_LOG_ERROR("ggml-hex: failed to enable ETM tracing: 0x%08x\n", (unsigned) err);
+ }
}
- return true;
-
- GGML_UNUSED(sess);
-}
-
-static bool ggml_hexagon_supported_tri(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
-
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * dst = op;
-
- if (src0->type != GGML_TYPE_F32) { return false; }
- if (dst->type != GGML_TYPE_F32) { return false; }
- if (!ggml_are_same_shape(src0, dst)) { return false; }
- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { return false; }
-
- return true;
+ // Allocate buffers and state for op batching
+ this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue);
- GGML_UNUSED(sess);
-}
+ this->fence_buf = new ggml_hexagon_fence_buffer(this, &dev_ctx->fence_buffer_type, 64 * 1024);
+ if (this->mdev.count > 1) {
+ this->mdev_fence_slot = this->alloc_fence(this->mdev.count);
+ }
-static const char * ggml_backend_hexagon_name(ggml_backend_t backend) {
- auto sess = static_cast<ggml_hexagon_session *>(backend->context);
- return sess->c_name();
-}
+ if (!opt_vmem) {
+ opt_vmem = ggml_hexagon_measure_max_vmem(this);
+ GGML_LOG_INFO("ggml-hex: %s measured max vmem %zu\n", this->c_name(), opt_vmem);
+ }
+ const size_t shm_size = this->op_queue->shm_size();
+ this->max_vmem = (opt_vmem > shm_size) ? (opt_vmem - shm_size) : opt_vmem;
-static void ggml_backend_hexagon_free(ggml_backend_t backend) {
- // we just need to delete the backend here
- // the sessions are allocated & freed as part of the registry
- delete backend;
-}
+ this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem);
-static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
- switch (t->op) {
- case GGML_OP_FLASH_ATTN_EXT: return HTP_OP_FLASH_ATTN_EXT;
- case GGML_OP_MUL_MAT:
- if (ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD) {
- return HTP_OP_HADAMARD;
- }
- return HTP_OP_MUL_MAT;
- case GGML_OP_MUL_MAT_ID: return HTP_OP_MUL_MAT_ID;
- case GGML_OP_MUL: return HTP_OP_MUL;
- case GGML_OP_ADD: return HTP_OP_ADD;
- case GGML_OP_ADD_ID: return HTP_OP_ADD_ID;
- case GGML_OP_SUB: return HTP_OP_SUB;
- case GGML_OP_DIV: return HTP_OP_DIV;
- case GGML_OP_DUP:
- case GGML_OP_CPY: return HTP_OP_CPY;
- case GGML_OP_CONT: return HTP_OP_CPY;
- case GGML_OP_GET_ROWS: return HTP_OP_GET_ROWS;
- case GGML_OP_SET_ROWS: return HTP_OP_SET_ROWS;
- case GGML_OP_SUM: return HTP_OP_SUM;
- case GGML_OP_SUM_ROWS: return HTP_OP_SUM_ROWS;
- case GGML_OP_POOL_2D: return HTP_OP_POOL_2D;
- case GGML_OP_POOL_1D: return HTP_OP_POOL_1D;
- case GGML_OP_ARGSORT: return HTP_OP_ARGSORT;
- case GGML_OP_TOP_K: return HTP_OP_TOP_K;
- case GGML_OP_ARGMAX: return HTP_OP_ARGMAX;
- case GGML_OP_NORM: return HTP_OP_NORM;
- case GGML_OP_L2_NORM: return HTP_OP_L2_NORM;
- case GGML_OP_RMS_NORM: return HTP_OP_RMS_NORM;
- case GGML_OP_CONCAT: return HTP_OP_CONCAT;
- case GGML_OP_SCALE: return HTP_OP_SCALE;
- case GGML_OP_CLAMP: return HTP_OP_CLAMP;
- case GGML_OP_LEAKY_RELU: return HTP_OP_LEAKY_RELU;
- case GGML_OP_SQR: return HTP_OP_SQR;
- case GGML_OP_SQRT: return HTP_OP_SQRT;
- case GGML_OP_LOG: return HTP_OP_UNARY_LOG;
- case GGML_OP_SOFT_MAX: return HTP_OP_SOFTMAX;
- case GGML_OP_SSM_CONV: return HTP_OP_SSM_CONV;
- case GGML_OP_GATED_DELTA_NET: return HTP_OP_GATED_DELTA_NET;
- case GGML_OP_ROPE: return HTP_OP_ROPE;
- case GGML_OP_REPEAT: return HTP_OP_REPEAT;
- case GGML_OP_CUMSUM: return HTP_OP_CUMSUM;
- case GGML_OP_FILL: return HTP_OP_FILL;
- case GGML_OP_DIAG: return HTP_OP_DIAG;
- case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI;
- case GGML_OP_TRI: return HTP_OP_TRI;
- case GGML_OP_PAD: return HTP_OP_PAD;
- case GGML_OP_IM2COL: return HTP_OP_IM2COL;
- case GGML_OP_IM2COL_3D: return HTP_OP_IM2COL_3D;
- case GGML_OP_ROLL: return HTP_OP_ROLL;
-
- case GGML_OP_UNARY:
- switch (ggml_get_unary_op(t)) {
- case GGML_UNARY_OP_SILU: return HTP_OP_UNARY_SILU;
- case GGML_UNARY_OP_GELU: return HTP_OP_UNARY_GELU;
- case GGML_UNARY_OP_GELU_QUICK: return HTP_OP_UNARY_GELU_QUICK;
- case GGML_UNARY_OP_GELU_ERF: return HTP_OP_UNARY_GELU_ERF;
- case GGML_UNARY_OP_SIGMOID: return HTP_OP_UNARY_SIGMOID;
- case GGML_UNARY_OP_NEG: return HTP_OP_UNARY_NEG;
- case GGML_UNARY_OP_EXP: return HTP_OP_UNARY_EXP;
- case GGML_UNARY_OP_SOFTPLUS: return HTP_OP_UNARY_SOFTPLUS;
- case GGML_UNARY_OP_TANH: return HTP_OP_UNARY_TANH;
- case GGML_UNARY_OP_ABS: return HTP_OP_UNARY_ABS;
- case GGML_UNARY_OP_RELU: return HTP_OP_UNARY_RELU;
- case GGML_UNARY_OP_STEP: return HTP_OP_UNARY_STEP;
- default:
- break;
- }
- break;
+ // Start dspqueue/opbatch processing
+ err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem, (uint32_t) opt_power_mode);
+ if (err != 0) {
+ GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err);
+ throw std::runtime_error("ggml-hex: iface start failed (see log for details)");
+ }
+ this->valid_iface = true;
+ GGML_LOG_INFO("ggml-hex: %s power mode: %s\n", this->c_name(), htp_power_mode_to_str(opt_power_mode));
- case GGML_OP_GLU:
- switch (ggml_get_glu_op(t)) {
- case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU;
- case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI;
- case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP;
- case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU;
- case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK;
- case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF;
- default: break;
- }
- break;
+ if (opt_profile) {
+ htp_iface_pmu_conf pmu_conf{};
+ std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events);
- default:
- GGML_ABORT("\nggml-hex: graph-compute %s is not supported\n", ggml_op_desc(t));
+ err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf);
+ if (err != 0) {
+ GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err);
+ }
}
- return HTP_OP_INVALID;
}
-static inline bool op_is_compute(ggml_tensor *node)
-{
- return !ggml_op_is_empty(node->op) && !ggml_is_empty(node) && (node->flags & GGML_TENSOR_FLAG_COMPUTE);
-}
+void ggml_hexagon_session::release() noexcept(true) {
+ GGML_LOG_INFO("ggml-hex: releasing session: %s\n", this->name.c_str());
-static bool mm_is_hmx_eligible(const ggml_tensor * t) {
- if (opt_nhmx == 0) { return false; }
+ this->mdev.sessions.clear();
- const ggml_tensor * src0 = t->src[0];
- const ggml_tensor * src1 = t->src[1];
+ int err;
- if (ggml_hexagon_matmul_can_collapse(src0, src1, t)) {
- const ggml_tensor src1_c = ggml_hexagon_tensor_collapse_rows(src1);
- return ggml_hexagon_matmul_is_hmx_eligible(src0, &src1_c, t, false, false);
+ if (this->valid_iface) {
+ // Stop dspqueue/opbatch processing
+ err = htp_iface_stop(this->handle);
+ if (err != 0) {
+ GGML_ABORT("ggml-hex: htp_iface_stop failed: 0x%08x\n", (unsigned) err);
+ }
}
- const bool is_matmul_id = (t->op == GGML_OP_MUL_MAT_ID);
- const bool is_batched = (src0->ne[2] * src0->ne[3] > 1 || src1->ne[2] * src1->ne[3] > 1);
-
- return ggml_hexagon_matmul_is_hmx_eligible(src0, src1, t, is_matmul_id, is_batched);
-}
-
-static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) {
- if (kparams->n_hmx) {
- return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;
+ delete this->op_batch;
+ delete this->op_queue;
+ for (auto & it : this->cpy_fence_slots) {
+ free_fence((void *) it.second, 1);
}
+ this->cpy_fence_slots.clear();
- if (!ggml_hexagon_tensor_is_tiled(src0)) {
- return false;
+ if (this->fence_buf) {
+ unclone_buffer(this->fence_buf);
+ delete this->fence_buf;
+ this->fence_buf = nullptr;
+ }
+ while (!this->cloned_buffers.empty()) {
+ release_buffer(this->cloned_buffers.begin()->second.get());
}
- return kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW;
-}
+ if (opt_etm) {
+ err = htp_iface_etm(this->handle, 0);
+ if (err != 0) {
+ GGML_LOG_ERROR("ggml-hex: warn : failed to disable ETM tracing: 0x%08x\n", (unsigned) err);
+ }
+ }
-static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) {
- if (kparams->n_hmx) {
- return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;
+ if (opt_profile) {
+ htp_iface_pmu_conf pmu_conf{};
+ err = htp_iface_profiler(this->handle, 0, &pmu_conf);
+ if (err != 0) {
+ GGML_LOG_ERROR("ggml-hex: warn : failed to disable profiling: 0x%08x\n", (unsigned) err);
+ }
}
- if (!ggml_hexagon_tensor_is_tiled(src0)) {
- return false;
+ if (this->valid_queue) {
+ err = dspqueue_close(queue);
+ if (err != 0) {
+ GGML_ABORT("ggml-hex: dspqueue_close failed: 0x%08x\n", (unsigned) err);
+ }
}
- return kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK;
+ if (this->valid_handle) {
+ htp_iface_close(this->handle);
+ }
}
-static bool is_mergeable_mul_mat(const ggml_tensor * t) {
- if (t->op != GGML_OP_MUL_MAT) return false;
- if (ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD) return false;
-
- const ggml_tensor * src0 = t->src[0];
- const ggml_tensor * src1 = t->src[1];
- if (src0->ne[2] != 1 || src0->ne[3] != 1) return false;
+ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev, uint32_t mdev_idx, uint32_t mdev_count) noexcept(false) {
+ this->dev = dev;
+ this->dev_ctx = static_cast<ggml_backend_hexagon_device_context *>(dev->context);
+ this->mdev.idx = mdev_idx;
+ this->mdev.count = mdev_count > 0 ? mdev_count : (uint32_t) (1 + config.mdev_group.size());
+ op_batch = nullptr;
+ op_queue = nullptr;
+ fence_buf = nullptr;
+ fence_seq = ((uintptr_t)this) & 0xFFFF;
- if (mm_is_hmx_eligible(t)) {
- return htp_is_hmx_weight_type(src0->type);
+ try {
+ allocate(config);
+ if (this->mdev.idx == 0 && !config.mdev_group.empty()) {
+ for (size_t i = 0; i < config.mdev_group.size(); i++) {
+ this->mdev.sessions.push_back(std::make_unique<ggml_hexagon_session>(
+ config.mdev_group[i], this->dev, (uint32_t) (i + 1), this->mdev.count));
+ }
+ }
+ } catch (const std::exception & exc) {
+ release();
+ throw;
}
-
- // HVX path requires F32 activations and tiled weights
- if (src1->type != GGML_TYPE_F32) return false;
-
- return ggml_hexagon_tensor_is_tiled(src0);
}
-static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor * n2) {
- if (!is_mergeable_mul_mat(n1) || !is_mergeable_mul_mat(n2)) {
- return false;
- }
- if (n1->src[1] != n2->src[1]) {
- return false;
- }
- if (n1->src[0]->ne[0] != n2->src[0]->ne[0]) {
- return false;
- }
- if (n1->src[0]->type != n2->src[0]->type) {
- return false;
- }
- if (ggml_hexagon_tensor_is_tiled(n1->src[0]) != ggml_hexagon_tensor_is_tiled(n2->src[0])) {
- return false;
- }
- if (mm_is_hmx_eligible(n1) != mm_is_hmx_eligible(n2)) {
- return false;
- }
- return true;
+ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) {
+ release();
}
-static bool is_mergeable_mul_mat_id(const ggml_tensor * t) {
- if (t->op != GGML_OP_MUL_MAT_ID) return false;
+// ** backend interface
- const ggml_tensor * src0 = t->src[0];
- if (mm_is_hmx_eligible(t)) {
- return htp_is_hmx_weight_type(src0->type);
- }
+static const char * ggml_backend_hexagon_name(ggml_backend_t backend) {
+ auto sess = static_cast<ggml_hexagon_session *>(backend->context);
+ return sess->c_name();
+}
- return ggml_hexagon_tensor_is_tiled(src0);
+static void ggml_backend_hexagon_free(ggml_backend_t backend) {
+ // we just need to delete the backend here
+ // the sessions are allocated & freed as part of the registry
+ delete backend;
}
-static bool is_mergeable_mul_mat_id_pair(const ggml_tensor * n1, const ggml_tensor * n2) {
- if (!is_mergeable_mul_mat_id(n1) || !is_mergeable_mul_mat_id(n2)) {
- return false;
- }
- if (n1->src[1] != n2->src[1]) {
- return false;
- }
- if (n1->src[2] != n2->src[2]) {
- return false;
- }
- if (n1->src[0]->ne[0] != n2->src[0]->ne[0]) {
- return false;
- }
- if (n1->src[0]->ne[2] != n2->src[0]->ne[2]) {
- return false;
- }
- if (n1->src[0]->type != n2->src[0]->type) {
- return false;
- }
- if (ggml_hexagon_tensor_is_tiled(n1->src[0]) != ggml_hexagon_tensor_is_tiled(n2->src[0])) {
- return false;
- }
- if (mm_is_hmx_eligible(n1) != mm_is_hmx_eligible(n2)) {
- return false;
- }
- return true;
+static inline bool op_is_compute(ggml_tensor *node)
+{
+ return !ggml_op_is_empty(node->op) && !ggml_is_empty(node) && (node->flags & GGML_TENSOR_FLAG_COMPUTE);
}
static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) {
@@ -6916,29 +2365,11 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
const std::vector<htp_opnode> * nodes_ptr = nullptr;
std::vector<htp_opnode> computed_nodes;
- // Check for cache hit
bool cache_hit = (graph->uid != 0 && sess->cached_uid == graph->uid);
if (cache_hit) {
nodes_ptr = &sess->cached_nodes;
} else {
- // Tag fusable tensors in graph
- for (int i = 0; i < graph->n_nodes; i++) {
- auto * extra = (ggml_hexagon_tensor_extra *) graph->nodes[i]->extra;
- if (!extra) continue;
-
- extra->flags &= ~GGML_HEXAGON_TENSOR_FUSEABLE;
-
- if (graph->nodes[i]->op == GGML_OP_RMS_NORM && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {
- extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE;
- } else if ((graph->nodes[i]->op == GGML_OP_MUL_MAT &&
- ggml_get_op_params_i32(graph->nodes[i], 1) != GGML_HINT_SRC0_IS_HADAMARD) ||
- graph->nodes[i]->op == GGML_OP_MUL_MAT_ID) {
- if ((i + 1 < graph->n_nodes && graph->nodes[i + 1]->op == GGML_OP_ADD && ggml_can_fuse(graph, i, { graph->nodes[i]->op, GGML_OP_ADD })) ||
- ggml_node_has_n_uses(graph, i, 1)) {
- extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE;
- }
- }
- }
+ htp_fusion_tag_graph(graph);
computed_nodes.reserve(graph->n_nodes);
@@ -6948,118 +2379,29 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
continue;
}
- htp_opnode node(HTP_OP_INVALID, n);
- node.opcode = op_remap_to_htp(n);
+ const htp_op_code opcode = htp_op_remap(n);
+ if (opcode == HTP_OP_INVALID) {
+ GGML_ABORT("\nggml-hex: graph-compute %s is not supported\n", ggml_op_desc(n));
+ }
+
+ htp_opnode node(opcode, n);
if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID || node.opcode == HTP_OP_GET_ROWS) {
ggml_tensor * src0 = node.node->src[0];
- if (ggml_hexagon_needs_repack(src0)) {
- if (opt_verbose) {
- char dims_buf[64];
- char strides_buf[64];
- htp_opformat fmt;
- fmt.format_tensor_dims(dims_buf, sizeof(dims_buf), src0);
- fmt.format_tensor_strides(strides_buf, sizeof(strides_buf), src0);
- GGML_LOG_DEBUG("ggml-hex: %s rt-repack %s|%s|%s|%s|size %zu\n",
- sess->c_name(), src0->name, dims_buf, ggml_type_name(src0->type), strides_buf, ggml_nbytes(src0));
- }
+ if (htp_needs_repack(src0)) {
+ HEX_VERBOSE("ggml-hex: %s rt-repack %s|size %zu\n", sess->c_name(), htp_format_tensor(src0, HTP_OPFMT_TENSOR).c_str(), ggml_nbytes(src0));
std::vector<uint8_t> raw(ggml_nbytes(src0));
memcpy(raw.data(), src0->data, raw.size());
htp_repack_tensor_tiled(src0, raw.data(), raw.size());
- auto * extra = (ggml_hexagon_tensor_extra *) src0->extra;
- extra->flags |= GGML_HEXAGON_TENSOR_TILED;
+ auto * extra = (htp_tensor_extra *) src0->extra;
+ extra->flags |= HTP_TENSOR_TILED;
}
}
- if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID) {
- ggml_hexagon_precompute_matmul_params(sess,
- node.node->src[0], node.node->src[1], node.node,
- (struct htp_mm_kernel_params *)node.kernel_params
- );
- } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {
- const ggml_tensor * src1 = node.node->src[1];
- const ggml_tensor * src2 = (node.opcode == HTP_OP_ADD_ID) ? node.node->src[2] : nullptr;
- GGML_ASSERT(ggml_hexagon_precompute_binary_params(sess,
- node.opcode, node.node->src[0], src1, node.node,
- (struct htp_binary_kernel_params *) node.kernel_params, src2));
- } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) {
- ggml_hexagon_precompute_flash_attn_params(sess,
- node.node,
- (struct htp_fa_kernel_params *)node.kernel_params
- );
- } else if (htp_op_is_unary(node.opcode)) {
- auto inputs = node.get_inputs();
- const struct ggml_tensor * src0 = inputs[0];
- const struct ggml_tensor * src1 = inputs.size() > 1 ? inputs[1] : nullptr;
- ggml_hexagon_precompute_unary_params(sess,
- node.opcode, src0, src1, node.dst(),
- (struct htp_unary_kernel_params *)node.kernel_params
- );
- } else if (node.opcode == HTP_OP_GET_ROWS) {
- ggml_hexagon_precompute_get_rows_params(sess,
- node.node->src[0], node.node->src[1], node.dst(),
- (struct htp_get_rows_kernel_params *)node.kernel_params
- );
- } else if (node.opcode == HTP_OP_SET_ROWS) {
- ggml_hexagon_precompute_set_rows_params(sess,
- node.node->src[0], node.node->src[1], node.dst(),
- (struct htp_set_rows_kernel_params *)node.kernel_params
- );
- } else if (node.opcode == HTP_OP_ROPE) {
- ggml_hexagon_precompute_rope_params(sess,
- node.node,
- (struct htp_rope_kernel_params *)node.kernel_params
- );
- } else if (node.opcode == HTP_OP_SSM_CONV) {
- ggml_hexagon_precompute_ssm_conv_params(sess,
- node.node->src[0], node.node->src[1], node.dst(),
- (struct htp_ssm_conv_kernel_params *)node.kernel_params
- );
- } else if (node.opcode == HTP_OP_SOFTMAX) {
- ggml_hexagon_precompute_softmax_params(sess,
- node.node,
- (struct htp_softmax_kernel_params *)node.kernel_params
- );
- } else if (node.opcode == HTP_OP_GATED_DELTA_NET) {
- ggml_hexagon_precompute_gated_delta_net_params(sess,
- node.node,
- (struct htp_gdn_kernel_params *)node.kernel_params
- );
- } else if (node.opcode == HTP_OP_ARGSORT || node.opcode == HTP_OP_TOP_K) {
- ggml_hexagon_precompute_sort_params(sess,
- node.node,
- node.opcode == HTP_OP_TOP_K,
- (struct htp_sort_kernel_params *) node.kernel_params
- );
- } else if (node.opcode == HTP_OP_POOL_2D || node.opcode == HTP_OP_POOL_1D) {
- ggml_hexagon_precompute_pool_2d_params(
- sess, node.node->src[0], node.dst(),
- (struct htp_pool_2d_kernel_params *)node.kernel_params,
- node.opcode == HTP_OP_POOL_1D);
- } else if (node.opcode == HTP_OP_CONCAT) {
- ggml_hexagon_precompute_concat_params(sess,
- node.node,
- (struct htp_concat_kernel_params *) node.kernel_params
- );
- } else if (node.opcode == HTP_OP_CPY || node.opcode == HTP_OP_CPY_FENCE) {
- const bool ok = ggml_hexagon_precompute_cpy_params(sess,
- node.node,
- (struct htp_copy_kernel_params *) node.kernel_params
- );
- const auto * kparams = (const struct htp_copy_kernel_params *) node.kernel_params;
- if (ok && node.opcode == HTP_OP_CPY && kparams->total_elems == 0) {
- continue;
- }
- } else if (node.opcode == HTP_OP_IM2COL) {
- ggml_hexagon_precompute_im2col_params(sess,
- node.node,
- (struct htp_im2col_kernel_params *) node.kernel_params
- );
- } else if (node.opcode == HTP_OP_IM2COL_3D) {
- ggml_hexagon_precompute_im2col_3d_params(sess,
- node.node,
- (struct htp_im2col_kernel_params *) node.kernel_params
- );
+ GGML_ASSERT(htp_kparams_solve(sess, node.node, node.kernel_params));
+ if (node.opcode == HTP_OP_CPY && node.kparams<htp_copy_kernel_params>()->total_elems == 0) {
+ continue;
}
+
computed_nodes.push_back(std::move(node));
}
@@ -7072,7 +2414,6 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
}
}
- // Queue and execute
for (const auto & node : *nodes_ptr) {
sess->enqueue_op(node);
}
@@ -7156,7 +2497,22 @@ static ggml_hexagon_mem_range ggml_hexagon_mem_range_from_tensor(const ggml_tens
return mr;
}
-static void ggml_hexagon_mem_ranges_add_node(ggml_hexagon_mem_ranges & mrs, const htp_opnode & node) {
+struct node_info {
+ ggml_tensor * node;
+ htp_opvec<ggml_tensor *, HTP_OP_MAX_FUSED> fused;
+
+ const ggml_tensor * dst() const { return fused.empty() ? node : fused.back(); }
+
+ bool is_empty() const {
+ return ggml_op_is_empty(node->op);
+ }
+
+ void add_fused(ggml_tensor * t) {
+ fused.push_back(t);
+ }
+};
+
+static void ggml_hexagon_mem_ranges_add_node(ggml_hexagon_mem_ranges & mrs, const node_info & node) {
if (node.is_empty()) return;
for (int i = 0; i < GGML_MAX_SRC; i++) {
@@ -7174,7 +2530,7 @@ static void ggml_hexagon_mem_ranges_add_node(ggml_hexagon_mem_ranges & mrs, cons
mrs.add(ggml_hexagon_mem_range_from_tensor(node.dst(), HEXAGON_MEM_RANGE_TYPE_DST));
}
-static bool ggml_hexagon_mem_ranges_check_node(const ggml_hexagon_mem_ranges & mrs, const htp_opnode & node) {
+static bool ggml_hexagon_mem_ranges_check_node(const ggml_hexagon_mem_ranges & mrs, const node_info & node) {
if (node.is_empty()) return true;
for (int i = 0; i < GGML_MAX_SRC; i++) {
@@ -7196,7 +2552,7 @@ static bool ggml_hexagon_mem_ranges_check_node(const ggml_hexagon_mem_ranges & m
return mrs.check(ggml_hexagon_mem_range_from_tensor(node.dst(), HEXAGON_MEM_RANGE_TYPE_DST));
}
-static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<htp_opnode> & nodes) {
+static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<node_info> & nodes) {
const int n = nodes.size();
std::vector<int> res;
@@ -7216,16 +2572,16 @@ static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<ht
const auto & node0 = nodes[i0];
- if (!node0.stackable()) {
+ if (!htp_fusion_can_stack(node0.node)) {
res.push_back(i0);
used[i0] = true;
continue;
}
// that many nodes forward to search for stackable nodes that can reuse VTCM
- constexpr int N_FORWARD = 16;
+ constexpr int N_FORWARD = 64;
- std::vector<int> stack;
+ htp_opvec<int, N_FORWARD> stack;
stack.push_back(i0);
mrs.reset();
@@ -7237,7 +2593,7 @@ static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<ht
const auto & node1 = nodes[i1];
- if (node1.stackable() && node1.same_input(node0) && ggml_hexagon_mem_ranges_check_node(mrs, node1)) {
+ if (htp_fusion_can_stack(node0.node, node1.node) && ggml_hexagon_mem_ranges_check_node(mrs, node1)) {
stack.push_back(i1);
} else {
ggml_hexagon_mem_ranges_add_node(mrs, node1);
@@ -7258,52 +2614,16 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr
const int n = gf->n_nodes;
- constexpr int MAX_FUSE = 16;
-
- enum ggml_op ops[MAX_FUSE];
-
- std::vector<htp_opnode> nodes;
+ std::vector<node_info> nodes;
nodes.reserve(gf->n_nodes);
// Pack nodes for reordering
for (int i = 0; i < n; i++) {
- htp_opnode node(HTP_OP_INVALID, gf->nodes[i]);
-
- // fuse only ops that start with these operations
- // can be expanded when needed
- if (node.op() == GGML_OP_ADD ||
- node.op() == GGML_OP_NORM ||
- node.op() == GGML_OP_RMS_NORM) {
- ops[0] = node.op();
-
- int f = i + 1;
- while (f < n && f < i + MAX_FUSE) {
- // conservatively allow fusing only these ops
- // can be expanded when needed
- if (gf->nodes[f]->op != GGML_OP_ADD &&
- gf->nodes[f]->op != GGML_OP_MUL &&
- gf->nodes[f]->op != GGML_OP_NORM &&
- gf->nodes[f]->op != GGML_OP_RMS_NORM) {
- break;
- }
- ops[f - i] = gf->nodes[f]->op;
- f++;
- }
-
- f -= i;
- for (; f > 1; f--) {
- if (ggml_can_fuse(gf, i, ops, f)) {
- break;
- }
- }
+ node_info node = { gf->nodes[i], {} };
- // add the fused tensors into the node info so we can unfuse them later
- for (int k = 1; k < f; k++) {
- ++i;
-
- // the .dst() becomes the last fused tensor
- node.add_fused(gf->nodes[i]);
- }
+ const int f = htp_fusion_max(gf, i);
+ for (int k = 1; k < f; k++) {
+ node.add_fused(gf->nodes[++i]);
}
nodes.push_back(std::move(node));
@@ -7349,29 +2669,12 @@ static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_
if (!sess_src->clone_buffer(sess_dst->fence_buf)) { return false; }
- if (++sess_dst->fence_seq == 0) sess_dst->fence_seq = 1;
- uint32_t fence_seq = sess_dst->fence_seq;
+ uint32_t fence_seq = sess_dst->next_fence_seq();
HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq 0x%x\n",
sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src), fence_seq);
- // dummy fence extra (must be static)
- static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE };
-
- ggml_tensor fence_tensor {};
- fence_tensor.buffer = &sess_dst->fence_buf->backend_buffer;
- fence_tensor.extra = &fence_extra;
- fence_tensor.data = (void *) fence_slot;
- fence_tensor.type = GGML_TYPE_I32;
- fence_tensor.ne[0] = 1;
- fence_tensor.ne[1] = 1;
- fence_tensor.ne[2] = 1;
- fence_tensor.ne[3] = 1;
- fence_tensor.nb[0] = sizeof(int32_t);
- fence_tensor.nb[1] = sizeof(int32_t);
- fence_tensor.nb[2] = sizeof(int32_t);
- fence_tensor.nb[3] = sizeof(int32_t);
- fence_tensor.op = GGML_OP_NONE;
+ ggml_tensor fence_tensor = sess_dst->make_fence_tensor(fence_slot);
sess_src->enqueue_cpy(src, dst, &fence_tensor, fence_seq);
sess_dst->enqueue_fence(&fence_tensor, fence_seq, /* wait = */ true);
@@ -7409,9 +2712,9 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg
ggml_backend_buffer_init_tensor(dst->buffer, dst);
}
- auto * dst_extra = static_cast<ggml_hexagon_tensor_extra *>(dst->extra);
- const auto * src_extra = static_cast<const ggml_hexagon_tensor_extra *>(src->extra);
- dst_extra->flags = src_extra->flags & ~GGML_HEXAGON_TENSOR_FUSEABLE;
+ auto * dst_extra = static_cast<htp_tensor_extra *>(dst->extra);
+ const auto * src_extra = static_cast<const htp_tensor_extra *>(src->extra);
+ dst_extra->flags = src_extra->flags & ~HTP_TENSOR_FUSEABLE;
auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context);
auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context);
@@ -7433,24 +2736,10 @@ static ggml_backend_event_t ggml_backend_hexagon_device_event_new(ggml_backend_d
auto sess = dev_ctx->session();
ggml_hexagon_event * hex_event = new ggml_hexagon_event();
- hex_event->fence_sess = sess;
- hex_event->sess = sess;
- hex_event->fence_slot = (volatile uint32_t *) sess->alloc_fence(1);
-
- static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE };
- hex_event->fence_tensor.buffer = &sess->fence_buf->backend_buffer;
- hex_event->fence_tensor.extra = &fence_extra;
- hex_event->fence_tensor.data = (void *) hex_event->fence_slot;
- hex_event->fence_tensor.type = GGML_TYPE_I32;
- hex_event->fence_tensor.ne[0] = 1;
- hex_event->fence_tensor.ne[1] = 1;
- hex_event->fence_tensor.ne[2] = 1;
- hex_event->fence_tensor.ne[3] = 1;
- hex_event->fence_tensor.nb[0] = sizeof(int32_t);
- hex_event->fence_tensor.nb[1] = sizeof(int32_t);
- hex_event->fence_tensor.nb[2] = sizeof(int32_t);
- hex_event->fence_tensor.nb[3] = sizeof(int32_t);
- hex_event->fence_tensor.op = GGML_OP_NONE;
+ hex_event->fence_sess = sess;
+ hex_event->sess = sess;
+ hex_event->fence_slot = (volatile uint32_t *) sess->alloc_fence(1);
+ hex_event->fence_tensor = sess->make_fence_tensor(hex_event->fence_slot);
HEX_VERBOSE("ggml-hex: %s event-new : event %p fence %p\n", ggml_backend_dev_name(dev), (void *)hex_event, (void *)hex_event->fence_slot);
@@ -7505,9 +2794,8 @@ static void ggml_backend_hexagon_event_record(ggml_backend_t backend, ggml_backe
auto sess = static_cast<ggml_hexagon_session *>(backend->context);
auto hex_event = static_cast<ggml_hexagon_event *>(event->context);
- if (++sess->fence_seq == 0) sess->fence_seq = 1;
hex_event->sess = sess;
- hex_event->seq = sess->fence_seq;
+ hex_event->seq = sess->next_fence_seq();
sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ false);
@@ -7686,80 +2974,6 @@ static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_host_buffer_ty
return &dev_ctx->host_buffer_type;
}
-static bool ggml_hexagon_supported_cpy(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- struct htp_copy_kernel_params kparams;
- return ggml_hexagon_precompute_cpy_params(sess, op, &kparams);
-}
-
-static bool ggml_hexagon_supported_cont(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- struct htp_copy_kernel_params kparams;
- return ggml_hexagon_precompute_cpy_params(sess, op, &kparams);
-}
-
-static bool ggml_hexagon_supported_repeat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- GGML_UNUSED(sess);
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * dst = op;
-
- // Support f32 and f16
- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) return false;
-
- // src and dst must be the same type
- if (src0->type != dst->type) return false;
-
- // dst dims must be multiples of src dims
- if (dst->ne[0] % src0->ne[0] != 0) return false;
- if (dst->ne[1] % src0->ne[1] != 0) return false;
- if (dst->ne[2] % src0->ne[2] != 0) return false;
- if (dst->ne[3] % src0->ne[3] != 0) return false;
-
- // require contiguous tensors (no transposition)
- if (ggml_is_transposed(src0) || ggml_is_transposed(dst)) return false;
-
- return true;
-}
-
-static bool ggml_hexagon_supported_concat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- struct htp_concat_kernel_params kparams;
- return ggml_hexagon_precompute_concat_params(sess, op, &kparams);
-}
-
-static bool ggml_hexagon_supported_fill(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const struct ggml_tensor * dst = op;
-
- if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) {
- return false;
- }
-
- return true;
- GGML_UNUSED(sess);
-}
-
-static bool ggml_hexagon_supported_roll(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- GGML_UNUSED(sess);
-
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * dst = op;
-
- if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
- return false;
- }
-
- if (!ggml_are_same_shape(src0, dst)) {
- return false;
- }
-
- if (src0->nb[0] != ggml_type_size(src0->type) || dst->nb[0] != ggml_type_size(dst->type)) {
- return false;
- }
-
- if (!ggml_is_contiguous(dst)) {
- return false;
- }
-
- return true;
-}
-
// IMPORTANT: supports_op runs before buffer allocation and before tensor data is populated.
// Never check buffer usage, buffer properties, or repacking state (is_tiled) here.
// Support must depend only on static tensor properties (shape, type, strides).
@@ -7782,194 +2996,12 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
supp = true;
break;
- case GGML_OP_MUL:
- case GGML_OP_ADD:
- case GGML_OP_SUB:
- case GGML_OP_DIV:
- supp = ggml_hexagon_supported_binary(sess, op);
- break;
-
- case GGML_OP_MUL_MAT:
- supp = ggml_hexagon_supported_mul_mat(sess, op);
- break;
-
- case GGML_OP_MUL_MAT_ID:
- supp = ggml_hexagon_supported_mul_mat_id(sess, op);
- break;
-
- case GGML_OP_ADD_ID:
- supp = ggml_hexagon_supported_add_id(sess, op);
- break;
-
- case GGML_OP_NORM:
- case GGML_OP_L2_NORM:
- case GGML_OP_RMS_NORM:
- case GGML_OP_SCALE:
- case GGML_OP_CLAMP:
- case GGML_OP_LEAKY_RELU:
- supp = ggml_hexagon_supported_unary(sess, op);
- break;
-
- case GGML_OP_SQR:
- case GGML_OP_SQRT:
- case GGML_OP_LOG:
- supp = ggml_hexagon_supported_unary(sess, op);
- break;
-
- case GGML_OP_SUM:
- supp = ggml_hexagon_supported_sum(sess, op);
- break;
-
- case GGML_OP_SUM_ROWS:
- supp = ggml_hexagon_supported_sum_rows(sess, op);
- break;
-
- case GGML_OP_ARGMAX:
- supp = ggml_hexagon_supported_argmax(sess, op);
- break;
-
- case GGML_OP_POOL_2D:
- supp = ggml_hexagon_supported_pool_2d(sess, op);
- break;
-
- case GGML_OP_POOL_1D:
- supp = ggml_hexagon_supported_pool_1d(sess, op);
- break;
-
- case GGML_OP_SOFT_MAX:
- supp = ggml_hexagon_supported_softmax(sess, op);
- break;
-
- case GGML_OP_UNARY:
- switch (ggml_get_unary_op(op)) {
- case GGML_UNARY_OP_NEG:
- case GGML_UNARY_OP_EXP:
- case GGML_UNARY_OP_SIGMOID:
- case GGML_UNARY_OP_SOFTPLUS:
- case GGML_UNARY_OP_TANH:
- case GGML_UNARY_OP_ABS:
- case GGML_UNARY_OP_SILU:
- case GGML_UNARY_OP_GELU:
- case GGML_UNARY_OP_GELU_QUICK:
- case GGML_UNARY_OP_GELU_ERF:
- case GGML_UNARY_OP_RELU:
- case GGML_UNARY_OP_STEP:
- supp = ggml_hexagon_supported_unary(sess, op);
- break;
- default:
- supp = false;
- break;
- }
- break;
-
- case GGML_OP_GLU:
- switch (ggml_get_glu_op(op)) {
- case GGML_GLU_OP_SWIGLU:
- case GGML_GLU_OP_SWIGLU_OAI:
- case GGML_GLU_OP_SWIGLU_CLAMP:
- case GGML_GLU_OP_GEGLU:
- supp = ggml_hexagon_supported_activations(sess, op);
- break;
- case GGML_GLU_OP_GEGLU_QUICK:
- case GGML_GLU_OP_GEGLU_ERF:
- supp = (op->src[0]->type == GGML_TYPE_F32) && ggml_hexagon_supported_activations(sess, op);
- break;
- default:
- supp = false;
- break;
- }
- break;
-
- case GGML_OP_ROPE:
- supp = ggml_hexagon_supported_rope(sess, op);
- break;
-
- case GGML_OP_FLASH_ATTN_EXT:
- supp = ggml_hexagon_supported_flash_attn_ext(sess, op);
- break;
-
- case GGML_OP_SET_ROWS:
- supp = ggml_hexagon_supported_set_rows(sess, op);
- break;
-
- case GGML_OP_GET_ROWS:
- supp = ggml_hexagon_supported_get_rows(sess, op);
- break;
-
- case GGML_OP_DUP:
- case GGML_OP_CPY:
- supp = ggml_hexagon_supported_cpy(sess, op);
- break;
-
- case GGML_OP_CONT:
- supp = ggml_hexagon_supported_cont(sess, op);
- break;
-
- case GGML_OP_REPEAT:
- supp = ggml_hexagon_supported_repeat(sess, op);
- break;
-
- case GGML_OP_ARGSORT:
- supp = ggml_hexagon_supported_argsort(sess, op);
- break;
-
- case GGML_OP_TOP_K:
- supp = ggml_hexagon_supported_top_k(sess, op);
- break;
-
- case GGML_OP_SSM_CONV:
- supp = ggml_hexagon_supported_ssm_conv(sess, op);
- break;
-
- case GGML_OP_IM2COL:
- supp = ggml_hexagon_supported_im2col(sess, op);
- break;
-
- case GGML_OP_IM2COL_3D:
- supp = ggml_hexagon_supported_im2col_3d(sess, op);
- break;
-
- case GGML_OP_GATED_DELTA_NET:
- supp = ggml_hexagon_supported_gated_delta_net(sess, op);
- break;
-
- case GGML_OP_CUMSUM:
- supp = ggml_hexagon_supported_cumsum(sess, op);
- break;
-
- case GGML_OP_CONCAT:
- supp = ggml_hexagon_supported_concat(sess, op);
- break;
-
- case GGML_OP_FILL:
- supp = ggml_hexagon_supported_fill(sess, op);
- break;
-
- case GGML_OP_DIAG:
- supp = ggml_hexagon_supported_diag(sess, op);
- break;
-
- case GGML_OP_SOLVE_TRI:
- supp = ggml_hexagon_supported_solve_tri(sess, op);
- break;
-
- case GGML_OP_TRI:
- supp = ggml_hexagon_supported_tri(sess, op);
- break;
-
- case GGML_OP_PAD:
- supp = ggml_hexagon_supported_pad(sess, op);
- break;
-
- case GGML_OP_ROLL:
- supp = ggml_hexagon_supported_roll(sess, op);
- break;
-
default:
+ supp = htp_kparams_check(sess, op);
break;
}
- ggml_hexagon_dump_op_supp(sess->name, op, supp);
+ ggml_hexagon_dump_op_supp(sess->c_name(), op, supp);
return supp;
}
@@ -8086,24 +3118,10 @@ static void * ggml_backend_hexagon_comm_init(ggml_backend_t * backends, size_t n
ctx->backends.assign(backends, backends + n_backends);
ctx->n_backends = n_backends;
- static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE };
for (size_t i = 0; i < n_backends; i++) {
auto sess_i = static_cast<ggml_hexagon_session *>(backends[i]->context);
- ctx->fence_slots[i] = (volatile uint32_t *) sess_i->alloc_fence(1);
- ctx->fence_tensors[i] = {};
- ctx->fence_tensors[i].buffer = &sess_i->fence_buf->backend_buffer;
- ctx->fence_tensors[i].extra = &fence_extra;
- ctx->fence_tensors[i].data = (void *) ctx->fence_slots[i];
- ctx->fence_tensors[i].type = GGML_TYPE_I32;
- ctx->fence_tensors[i].ne[0] = 4;
- ctx->fence_tensors[i].ne[1] = 1;
- ctx->fence_tensors[i].ne[2] = 1;
- ctx->fence_tensors[i].ne[3] = 1;
- ctx->fence_tensors[i].nb[0] = sizeof(int32_t);
- ctx->fence_tensors[i].nb[1] = sizeof(int32_t);
- ctx->fence_tensors[i].nb[2] = sizeof(int32_t);
- ctx->fence_tensors[i].nb[3] = sizeof(int32_t);
- ctx->fence_tensors[i].op = GGML_OP_NONE;
+ ctx->fence_slots[i] = (volatile uint32_t *) sess_i->alloc_fence(1);
+ ctx->fence_tensors[i] = sess_i->make_fence_tensor(ctx->fence_slots[i]);
}
return ctx;
@@ -8157,8 +3175,7 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct
for (size_t r = 0; r < n_backends; r++) {
auto sess = static_cast<ggml_hexagon_session *>(comm_ctx->backends[r]->context);
- struct htp_allreduce_kernel_params kparams;
- if (!ggml_hexagon_precompute_allreduce_params(sess, tensors[r], (uint32_t) r, (uint32_t) n_backends, false, false, /*is_shard_ok=*/ false, &kparams)) {
+ if (!htp_kparams_check<htp_kparams_solve_allreduce>(sess, tensors[r], (uint32_t) r, (uint32_t) n_backends, false, false, false)) {
return false;
}
}
@@ -8423,7 +3440,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
opt_verbose = str_verbose ? atoi(str_verbose) : 0;
opt_opbatch = str_opbatch ? strtoul(str_opbatch, NULL, 0) : opt_opbatch;
opt_opqueue = str_opqueue ? strtoul(str_opqueue, NULL, 0) : opt_opqueue;
- opt_optrace = str_optrace ? strtoul(str_optrace, NULL, 0) : (opt_opbatch * 256);
+ opt_optrace = str_optrace ? strtoul(str_optrace, NULL, 0) : (opt_opbatch * HTP_TRACE_MAX_EVENTS + 131072);
opt_oppoll = str_oppoll ? strtoul(str_oppoll, NULL, 0) : opt_oppoll;
opt_opfusion = str_opfusion ? atoi(str_opfusion) : opt_opfusion;
opt_rt_repack = str_rt_repack ? atoi(str_rt_repack) : opt_rt_repack;
diff --git a/ggml/src/ggml-hexagon/htp-common.h b/ggml/src/ggml-hexagon/htp-common.h
new file mode 100644
index 000000000..71a690a16
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp-common.h
@@ -0,0 +1,106 @@
+#ifndef HTP_COMMON_H
+#define HTP_COMMON_H
+
+#include "htp-opnode.h"
+#include "htp-repack.h"
+
+struct htp_tensor_extra {
+ std::vector<uint8_t> shadow_buf;
+ size_t shadow_size { 0 };
+ uint32_t flags { 0 };
+};
+
+extern int opt_arch;
+extern size_t opt_ndev;
+extern size_t opt_nhvx;
+extern int opt_nhmx;
+extern size_t opt_vmem;
+extern int opt_etm;
+extern int opt_verbose;
+extern int opt_profile;
+extern bool opt_hostbuf;
+extern bool opt_dma64;
+extern int opt_power_mode;
+
+extern size_t opt_mbuf_dyn;
+extern size_t opt_mbuf_static;
+extern size_t opt_mbuf_total;
+
+extern int opt_mm_select;
+extern int opt_fa_select;
+extern int opt_fa_head_split;
+extern int opt_gdn_select;
+extern int opt_ar_select;
+extern int opt_ar_scatter;
+
+extern int opt_opbatch;
+extern int opt_opqueue;
+extern int opt_optrace;
+extern int opt_oppoll;
+extern int opt_opfusion;
+extern int opt_rt_repack;
+
+#ifndef HEX_VERBOSE
+#define HEX_VERBOSE(...) \
+ if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__)
+#endif
+
+static inline bool htp_tensor_is_fuseable(const struct ggml_tensor * t) {
+ if (!t || !t->extra) return false;
+ const auto * extra = (const struct htp_tensor_extra *) t->extra;
+ return (extra->flags & HTP_TENSOR_FUSEABLE) != 0;
+}
+
+static inline bool htp_can_rt_repack(const struct ggml_tensor * t) {
+ if (opt_rt_repack == 0 || !htp_is_tiled_weight_tensor(t)) {
+ return false;
+ }
+ if (t->buffer && ggml_backend_buft_is_host(t->buffer->buft)) {
+ return false;
+ }
+ return opt_rt_repack > 1 || ggml_is_quantized(t->type);
+}
+
+static inline bool htp_tensor_is_tiled(const struct ggml_tensor * t) {
+ const auto * extra = (const struct htp_tensor_extra *) t->extra;
+ if (extra && (extra->flags & HTP_TENSOR_TILED)) {
+ return true;
+ }
+ return htp_can_rt_repack(t);
+}
+
+static inline bool htp_needs_repack(const struct ggml_tensor * t) {
+ const auto * extra = (const struct htp_tensor_extra *) t->extra;
+ if (extra && (extra->flags & HTP_TENSOR_TILED)) {
+ return false;
+ }
+ return htp_can_rt_repack(t);
+}
+
+static inline bool htp_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) {
+ const uintptr_t a0 = (uintptr_t) a->data;
+ const uintptr_t b0 = (uintptr_t) b->data;
+ const uintptr_t a1 = a0 + ggml_nbytes(a);
+ const uintptr_t b1 = b0 + ggml_nbytes(b);
+
+ return a0 < b1 && b0 < a1;
+}
+
+static inline bool htp_can_row_partition(const struct ggml_tensor * t) {
+ if (t->ne[1] > 1 && (t->nb[1] & 127) != 0) return false;
+ if (t->ne[2] > 1 && (t->nb[2] & 127) != 0) return false;
+ if (t->ne[3] > 1 && (t->nb[3] & 127) != 0) return false;
+ return true;
+}
+
+struct ggml_hexagon_session;
+
+const char * htp_session_name(const struct ggml_hexagon_session * sess);
+size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess);
+uint32_t htp_session_n_threads(const struct ggml_hexagon_session * sess);
+uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess);
+uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess);
+
+htp_op_code htp_op_remap(const struct ggml_tensor * t);
+
+#endif // HTP_COMMON_H
diff --git a/ggml/src/ggml-hexagon/htp-fusion.cpp b/ggml/src/ggml-hexagon/htp-fusion.cpp
new file mode 100644
index 000000000..65aacf73f
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp-fusion.cpp
@@ -0,0 +1,650 @@
+#include "htp-fusion.h"
+#include "htp-kparams.h"
+#include "ggml-impl.h"
+
+#include <algorithm>
+#include <cstring>
+
+static inline bool htp_same_tensor_data(const ggml_tensor * a, const ggml_tensor * b) {
+ return a && b && (a == b || a->data == b->data);
+}
+
+static inline bool htp_same_shape_and_contiguity(const ggml_tensor * a, const ggml_tensor * b) {
+ return ggml_are_same_shape(a, b) && (ggml_is_contiguous(a) == ggml_is_contiguous(b));
+}
+
+static inline bool htp_inplace_overlap_unsafe(const ggml_tensor * dst, const ggml_tensor * in) {
+ return htp_tensors_overlap(dst, in) && (dst->data != in->data || !ggml_are_same_stride(dst, in));
+}
+
+static inline bool htp_inputs_overlap_unsafe(const htp_opnode & prev, const ggml_tensor * dst, const ggml_tensor * extra = nullptr) {
+ if (extra && htp_inplace_overlap_unsafe(dst, extra)) return true;
+ for (const auto * in : prev.inputs) {
+ if (htp_inplace_overlap_unsafe(dst, in)) return true;
+ }
+ return false;
+}
+
+static inline bool htp_match_consumer(
+ const htp_opnode & prev,
+ const htp_opnode & cur,
+ bool commutative,
+ const ggml_tensor ** other_src = nullptr
+) {
+ if (prev.outputs.size() != 1) {
+ return false;
+ }
+ const ggml_tensor * prev_out = prev.dst();
+ if (!htp_tensor_is_fuseable(prev_out)) {
+ return false;
+ }
+ if (htp_same_tensor_data(cur.src0(), prev_out)) {
+ if (other_src) *other_src = cur.src1();
+ return true;
+ }
+ if (commutative && htp_same_tensor_data(cur.src1(), prev_out)) {
+ if (other_src) *other_src = cur.src0();
+ return true;
+ }
+ return false;
+}
+
+static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) {
+ if (kparams->n_hmx) {
+ return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;
+ }
+ return htp_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW);
+}
+
+static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) {
+ if (kparams->n_hmx) {
+ return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;
+ }
+ return htp_tensor_is_tiled(src0) &&
+ (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK);
+}
+
+// --- Fusion rules ---
+
+static bool fuse_allreduce_add(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
+ if (opt_ar_select != 2 || prev.opcode != HTP_OP_ALLREDUCE) return false;
+
+ const auto * ar_kparams = prev.kparams<htp_allreduce_kernel_params>();
+ const uint32_t rank = (uint32_t) ar_kparams->rank;
+ const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks;
+
+ const ggml_tensor * ar_local = prev.inputs[rank];
+ if (!htp_tensor_is_fuseable(ar_local)) return false;
+
+ const ggml_tensor * add_dst = cur.dst();
+ const ggml_tensor * res_tensor = nullptr;
+ if (htp_same_tensor_data(cur.src0(), ar_local)) {
+ res_tensor = cur.src1();
+ } else if (htp_same_tensor_data(cur.src1(), ar_local)) {
+ res_tensor = cur.src0();
+ } else {
+ return false;
+ }
+
+ if (ar_local->type != res_tensor->type) return false;
+
+ const bool is_same_shape = ggml_are_same_shape(ar_local, res_tensor);
+ const bool is_row_bcast = !is_same_shape && (ar_local->ne[0] == res_tensor->ne[0] &&
+ res_tensor->ne[1] == 1 && res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1);
+ if (!is_same_shape && !is_row_bcast) return false;
+
+ if (is_same_shape && !ggml_are_same_stride(ar_local, res_tensor)) {
+ return false;
+ }
+ if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(add_dst)) return false;
+
+ const bool is_shard_ok = n_ranks <= HTP_OP_MAX_OUTPUTS && add_dst->data == ar_local->data;
+
+ struct htp_allreduce_kernel_params kparams;
+ if (!htp_kparams_solve_allreduce(sess, add_dst, rank, n_ranks, true, is_row_bcast, is_shard_ok, &kparams)) {
+ HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: solver failed\n", htp_session_name(sess));
+ return false;
+ }
+
+ const bool scatter_ok = kparams.n_dsts > 1;
+ kparams.mode = scatter_ok ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL;
+
+ for (uint32_t r = 0; r < n_ranks; r++) {
+ const ggml_tensor * ar_src = prev.inputs[r];
+ if (!htp_tensors_overlap(add_dst, ar_src)) continue;
+ if (scatter_ok && r == rank) continue;
+ HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: dst overlaps allreduce src %u\n", htp_session_name(sess), r);
+ return false;
+ }
+
+ prev.opcode = HTP_OP_ALLREDUCE_ADD;
+ prev.inputs.push_back(res_tensor);
+ if (kparams.mode != HTP_ALLREDUCE_SHARDED_FANOUT) {
+ prev.outputs = { add_dst };
+ }
+ prev.fused.push_back(cur.node);
+ prev.set_kparams(kparams);
+ return true;
+}
+
+static bool fuse_rms_norm_mul(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
+ if (prev.opcode != HTP_OP_RMS_NORM) return false;
+
+ const ggml_tensor * weight = nullptr;
+ if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &weight)) return false;
+
+ const ggml_tensor * src0 = prev.src0();
+ const ggml_tensor * dst = cur.dst();
+ if (src0->type != GGML_TYPE_F32 || src0->ne[0] != weight->ne[0] || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false;
+ if (htp_inputs_overlap_unsafe(prev, dst, weight)) return false;
+
+ const bool is_row_bcast = (weight->ne[1] == 1 && weight->ne[2] == 1 && weight->ne[3] == 1);
+ const bool is_same_shape = ggml_are_same_shape(src0, weight);
+ if (!is_row_bcast && !is_same_shape) return false;
+
+ struct htp_unary_kernel_params kparams;
+ if (!htp_kparams_solve_rms_norm_mul(sess, src0, weight, dst, &kparams)) {
+ HEX_VERBOSE("ggml-hex: %s skip RMS_NORM_MUL fusion: VTCM needed (%d) > budget (%zu)\n",
+ htp_session_name(sess), kparams.vtcm_size, htp_session_vtcm_size(sess));
+ return false;
+ }
+
+ prev.opcode = HTP_OP_RMS_NORM_MUL;
+ prev.inputs = { src0, weight };
+ prev.outputs = { dst };
+ prev.fused.push_back(cur.node);
+ prev.set_kparams(kparams);
+ return true;
+}
+
+static bool fuse_rms_norm_scale(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
+ GGML_UNUSED(sess);
+ if (prev.opcode != HTP_OP_RMS_NORM && prev.opcode != HTP_OP_RMS_NORM_MUL) return false;
+ if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false;
+
+ const ggml_tensor * src0 = prev.src0();
+ const ggml_tensor * dst = cur.dst();
+ if (src0->type != GGML_TYPE_F32 || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false;
+ if (htp_inputs_overlap_unsafe(prev, dst)) return false;
+
+ float scale = 0.0f;
+ float bias = 0.0f;
+ memcpy(&scale, (const float *) cur.params + 0, sizeof(float));
+ memcpy(&bias, (const float *) cur.params + 1, sizeof(float));
+ if (bias != 0.0f || scale == 0.0f) return false;
+
+ float prev_scale = 0.0f;
+ memcpy(&prev_scale, &prev.params[1], sizeof(float));
+ if (prev_scale != 0.0f) {
+ scale *= prev_scale;
+ }
+ memcpy(&prev.params[1], &scale, sizeof(float));
+
+ prev.outputs[0] = dst;
+ prev.fused.push_back(cur.node);
+ return true;
+}
+
+static bool fuse_mul_mat_add(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
+ if (prev.opcode != HTP_OP_MUL_MAT) return false;
+
+ const ggml_tensor * src2 = nullptr;
+ if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &src2)) return false;
+ if (src2->type != GGML_TYPE_F32) return false;
+
+ const ggml_tensor * src0 = prev.src0();
+ const ggml_tensor * src1 = prev.src1();
+ const ggml_tensor * dst = cur.dst();
+ if (!ggml_are_same_shape(prev.dst(), dst)) return false;
+
+ const auto * orig_kparams = prev.kparams<htp_mm_kernel_params>();
+ struct htp_mm_kernel_params kparams;
+ if (!htp_kparams_solve_matmul_add(sess, src0, src1, src2, dst, &kparams) ||
+ orig_kparams->collapse != kparams.collapse) {
+ return false;
+ }
+
+ const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3];
+ if (kparams.n_hmx == 0 && src1_nrows != 1) return false;
+
+ if (kparams.n_hmx > 0 && orig_kparams->n_hmx > 0) {
+ if (kparams.m_chunk < orig_kparams->m_chunk ||
+ kparams.n_chunk < orig_kparams->n_chunk ||
+ kparams.n_act_threads < orig_kparams->n_act_threads) {
+ HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, th %d->%d)\n",
+ htp_session_name(sess), orig_kparams->m_chunk, kparams.m_chunk,
+ orig_kparams->n_chunk, kparams.n_chunk,
+ orig_kparams->n_act_threads, kparams.n_act_threads);
+ return false;
+ }
+ }
+
+ prev.opcode = HTP_OP_MUL_MAT_ADD;
+ prev.inputs = { src0, src1, src2 };
+ prev.outputs = { dst };
+ prev.fused.push_back(cur.node);
+ prev.set_kparams(kparams);
+ return true;
+}
+
+static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
+ if (prev.opcode != HTP_OP_MUL_MAT && prev.opcode != HTP_OP_MUL_MAT_NX) return false;
+
+ const uint32_t curr_n = (uint32_t) prev.outputs.size();
+ if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 1 >= HTP_OP_MAX_INPUTS) return false;
+
+ const ggml_tensor * w0 = prev.inputs[0];
+ const ggml_tensor * x = prev.inputs[curr_n];
+ const ggml_tensor * w_in = cur.src0();
+ const ggml_tensor * x_in = cur.src1();
+ const ggml_tensor * d_in = cur.dst();
+
+ auto * orig_kparams = prev.kparams<htp_mm_kernel_params>();
+ const auto * cur_kparams = cur.kparams<htp_mm_kernel_params>();
+
+ if (x_in != x || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] ||
+ w0->ne[2] != 1 || w0->ne[3] != 1 || w_in->ne[2] != 1 || w_in->ne[3] != 1 ||
+ htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) ||
+ orig_kparams->n_hmx != cur_kparams->n_hmx) {
+ return false;
+ }
+
+ const bool collapse = orig_kparams->collapse && htp_matmul_can_collapse(w_in, x, d_in);
+ if ((x->ne[2] > 1 || x->ne[3] > 1) && !collapse) return false;
+
+ if (prev.opcode == HTP_OP_MUL_MAT_NX) {
+ orig_kparams->n_weights = curr_n + 1;
+ } else {
+ struct htp_mm_kernel_params kparams;
+ if (collapse) {
+ const ggml_tensor x_collapsed = htp_tensor_collapse_rows(x);
+ if (!htp_kparams_solve_mmnx(sess, w0, &x_collapsed, curr_n + 1, &kparams)) return false;
+ kparams.collapse = 1;
+ } else if (!htp_kparams_solve_mmnx(sess, w0, x, curr_n + 1, &kparams)) {
+ return false;
+ }
+ if (!is_supported_mul_mat_nx_kernel(w0, &kparams)) return false;
+
+ prev.set_kparams(kparams);
+ prev.opcode = HTP_OP_MUL_MAT_NX;
+ }
+
+ prev.inputs.insert(prev.inputs.begin() + curr_n, w_in);
+ prev.outputs.push_back(d_in);
+ prev.fused.push_back(cur.node);
+ return true;
+}
+
+static bool fuse_mul_mat_id_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
+ if (prev.opcode != HTP_OP_MUL_MAT_ID && prev.opcode != HTP_OP_MUL_MAT_ID_NX) return false;
+
+ const uint32_t curr_n = (uint32_t) prev.outputs.size();
+ if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 2 >= HTP_OP_MAX_INPUTS) return false;
+
+ const ggml_tensor * w0 = prev.inputs[0];
+ const ggml_tensor * x = prev.inputs[curr_n];
+ const ggml_tensor * ids = prev.inputs[curr_n + 1];
+ const ggml_tensor * w_in = cur.src0();
+ const ggml_tensor * x_in = cur.src1();
+ const ggml_tensor * ids_in = cur.node->src[2];
+ const ggml_tensor * d_in = cur.dst();
+
+ auto * orig_kparams = prev.kparams<htp_mm_kernel_params>();
+ const auto * cur_kparams = cur.kparams<htp_mm_kernel_params>();
+
+ if (x_in != x || ids_in != ids || w_in->type != w0->type ||
+ w_in->ne[0] != w0->ne[0] || w_in->ne[2] != w0->ne[2] ||
+ htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) ||
+ orig_kparams->n_hmx != cur_kparams->n_hmx) {
+ return false;
+ }
+
+ struct htp_mm_kernel_params kparams;
+ if (d_in->nb[1] == prev.node->nb[1]) {
+ kparams = *orig_kparams;
+ kparams.n_weights = curr_n + 1;
+ } else if (!htp_kparams_solve_mmidnx(sess, w0, x, d_in, curr_n + 1, &kparams)) {
+ return false;
+ }
+ if (!is_supported_mul_mat_id_nx_kernel(w0, &kparams)) return false;
+
+ prev.opcode = HTP_OP_MUL_MAT_ID_NX;
+ prev.inputs.insert(prev.inputs.begin() + curr_n, w_in);
+ prev.outputs.push_back(d_in);
+ prev.fused.push_back(cur.node);
+ prev.set_kparams(kparams);
+ return true;
+}
+
+static bool fuse_gdn_cpy(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
+ GGML_UNUSED(sess);
+ if (prev.opcode != HTP_OP_GATED_DELTA_NET || prev.outputs.size() != 1) return false;
+
+ const ggml_tensor * gdn_out = prev.dst();
+ const ggml_tensor * cpy_node = cur.node;
+ const ggml_tensor * cpy_src = cur.src0();
+ const ggml_tensor * cpy_dst = cur.dst();
+
+ if (gdn_out->type != GGML_TYPE_F32 || cpy_src->type != GGML_TYPE_F32 || cpy_dst->type != GGML_TYPE_F32) return false;
+ if ((gdn_out->flags & GGML_TENSOR_FLAG_OUTPUT) || (cpy_node->flags & GGML_TENSOR_FLAG_OUTPUT)) return false;
+
+ const ggml_tensor * v = prev.node->src[2];
+ if (!v) return false;
+
+ const int64_t S_v = v->ne[0];
+ const int64_t H = v->ne[1];
+ const int64_t n_tokens = v->ne[2];
+ const int64_t n_seqs = v->ne[3];
+ const int64_t K = ggml_get_op_params_i32(prev.node, 0);
+ const size_t tail_off = (size_t) S_v * H * n_tokens * n_seqs * sizeof(float);
+ const int64_t D = S_v * S_v * H;
+ const int64_t n_written = std::min<int64_t>(n_tokens, K);
+
+ if (cpy_src->op != GGML_OP_VIEW || !htp_same_tensor_data(cpy_src->view_src, gdn_out) ||
+ cpy_src->view_offs != tail_off || !ggml_is_contiguous(cpy_src)) {
+ return false;
+ }
+
+ if (cpy_dst->ne[0] != D || cpy_dst->ne[1] != n_seqs || cpy_dst->ne[2] != n_written || !ggml_is_contiguous(cpy_dst)) {
+ return false;
+ }
+
+ prev.outputs.push_back(cpy_dst);
+ prev.fused.push_back(cur.node);
+ return true;
+}
+
+static bool fuse_ssm_conv_silu(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
+ GGML_UNUSED(sess);
+ if (prev.opcode != HTP_OP_SSM_CONV) return false;
+
+ auto * kparams = prev.kparams<htp_ssm_conv_kernel_params>();
+ if (kparams->silu) return false;
+ if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false;
+
+ const ggml_tensor * conv_dst = prev.dst();
+ const ggml_tensor * dst = cur.dst();
+ if (!ggml_are_same_layout(conv_dst, dst)) return false;
+ if (htp_tensors_overlap(dst, prev.src0()) || htp_tensors_overlap(dst, prev.src1())) return false;
+
+ kparams->silu = 1;
+ prev.outputs[0] = dst;
+ prev.fused.push_back(cur.node);
+ return true;
+}
+
+static bool fuse_repeat_get_rows(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
+ if (prev.opcode != HTP_OP_REPEAT) return false;
+ if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false;
+
+ const ggml_tensor * repeat_out = prev.dst();
+ const ggml_tensor * repeat_src0 = prev.src0();
+ const ggml_tensor * gr_src1 = cur.src1();
+ const ggml_tensor * gr_dst = cur.dst();
+
+ if (repeat_src0->ne[0] != repeat_out->ne[0] || repeat_src0->ne[1] != repeat_out->ne[1]) return false;
+ if (repeat_src0->type != repeat_out->type) return false;
+
+ struct htp_get_rows_kernel_params kparams;
+ if (!htp_kparams_solve_repeat_get_rows(sess, repeat_src0, gr_src1, gr_dst, &kparams) || kparams.vtcm_size == 0) return false;
+
+ prev.opcode = HTP_OP_GET_ROWS;
+ prev.inputs = { repeat_src0, gr_src1 };
+ prev.outputs = { gr_dst };
+ prev.fused.push_back(cur.node);
+ prev.set_kparams(kparams);
+ return true;
+}
+
+static bool fuse_unary_mul_glu(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
+ htp_op_code glu_op = HTP_OP_INVALID;
+ switch (prev.opcode) {
+ case HTP_OP_UNARY_SILU: glu_op = HTP_OP_GLU_SWIGLU; break;
+ case HTP_OP_UNARY_GELU: glu_op = HTP_OP_GLU_GEGLU; break;
+ case HTP_OP_UNARY_GELU_QUICK: glu_op = HTP_OP_GLU_GEGLU_QUICK; break;
+ case HTP_OP_UNARY_GELU_ERF: glu_op = HTP_OP_GLU_GEGLU_ERF; break;
+ case HTP_OP_UNARY_RELU: glu_op = HTP_OP_GLU_REGLU; break;
+ case HTP_OP_UNARY_SIGMOID: glu_op = HTP_OP_GLU_SIGMOID; break;
+ default: return false;
+ }
+
+ const ggml_tensor * up = nullptr;
+ if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &up)) return false;
+
+ const ggml_tensor * gate = prev.src0();
+ const ggml_tensor * dst = cur.dst();
+ if (htp_inputs_overlap_unsafe(prev, dst, up)) return false;
+
+ struct htp_glu_kernel_params kparams;
+ if (!htp_kparams_solve_unary_mul_glu(sess, glu_op, gate, up, dst, &kparams)) return false;
+
+ prev.opcode = glu_op;
+ prev.inputs = { gate, up };
+ prev.outputs = { dst };
+ prev.fused.push_back(cur.node);
+ memset(prev.params, 0, sizeof(prev.params));
+ prev.set_kparams(kparams);
+ return true;
+}
+
+static bool fuse_mul_mul_scalar(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
+ if (prev.opcode != HTP_OP_MUL || prev.inputs.size() != 2) return false;
+
+ const auto * k0 = prev.kparams<htp_binary_kernel_params>();
+ const auto * k1 = cur.kparams<htp_binary_kernel_params>();
+ if (k0->kernel_type != HTP_BINARY_KERNEL_SCALAR || k1->kernel_type != HTP_BINARY_KERNEL_SCALAR) return false;
+ if (k0->src1_strided || k1->src1_strided) return false;
+
+ const ggml_tensor * src2 = nullptr;
+ if (!htp_match_consumer(prev, cur, /*commutative=*/ false, &src2)) return false;
+
+ const ggml_tensor * src1 = prev.src1();
+ const ggml_tensor * dst = cur.dst();
+ if (!ggml_are_same_shape(src1, src2) || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false;
+ if (htp_inputs_overlap_unsafe(prev, dst, src2)) return false;
+
+ const size_t vtcm_size = htp_session_vtcm_size(sess);
+ struct htp_binary_kernel_params kparams = *k0;
+ kparams.src2_size = kparams.src1_size;
+ kparams.rows_per_buffer = 0;
+ if (kparams.src1_size > 65536 || kparams.src1_size + kparams.src2_size > vtcm_size) return false;
+
+ struct htp_binary_vtcm_layout L;
+ htp_binary_vtcm_layout_build(&L, &kparams, vtcm_size);
+ if (L.rows_per_buffer == 0 || L.total_bytes > vtcm_size) return false;
+
+ kparams.rows_per_buffer = L.rows_per_buffer;
+ kparams.vtcm_size = L.total_bytes;
+
+ prev.inputs.push_back(src2);
+ prev.outputs[0] = dst;
+ prev.fused.push_back(cur.node);
+ prev.set_kparams(kparams);
+ return true;
+}
+
+static bool fuse_binary_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
+ if (prev.opcode != cur.opcode) return false;
+
+ const uint32_t curr_n = (uint32_t) prev.inputs.size();
+ if (curr_n < 2 || curr_n >= HTP_OP_MAX_INPUTS) return false;
+
+ const auto * k0 = prev.kparams<htp_binary_kernel_params>();
+ const auto * k1 = cur.kparams<htp_binary_kernel_params>();
+ const bool k0_ok = (k0->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) ||
+ (k0->kernel_type == HTP_BINARY_KERNEL_CHUNKED && !k0->is_scalar);
+ const bool k1_ok = (k1->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) ||
+ (k1->kernel_type == HTP_BINARY_KERNEL_CHUNKED && !k1->is_scalar);
+ if (!k0_ok || !k1_ok) return false;
+
+ const bool is_comm = (cur.opcode == HTP_OP_ADD || cur.opcode == HTP_OP_MUL);
+ const ggml_tensor * new_src = nullptr;
+ if (!htp_match_consumer(prev, cur, is_comm, &new_src)) return false;
+
+ const ggml_tensor * src0 = prev.inputs[0];
+ const ggml_tensor * dst = cur.dst();
+ if (!ggml_are_same_shape(src0, prev.inputs[1]) ||
+ !ggml_are_same_shape(src0, new_src) ||
+ !htp_same_shape_and_contiguity(prev.dst(), dst)) {
+ return false;
+ }
+ if (htp_inputs_overlap_unsafe(prev, dst, new_src)) return false;
+
+ const size_t vtcm_size = htp_session_vtcm_size(sess);
+ struct htp_binary_kernel_params kparams = *k0;
+ struct htp_binary_vtcm_layout L;
+ memset(&L, 0, sizeof(L));
+
+ const uint32_t n_srcs = curr_n + 1;
+ if (k0->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE || k1->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {
+ if (k0->kernel_type != HTP_BINARY_KERNEL_SAME_SHAPE) {
+ kparams = *k1;
+ }
+ kparams.n_srcs = (uint8_t) n_srcs;
+ kparams.rows_per_buffer = 0;
+
+ auto check_split = [&](const ggml_tensor * t) {
+ if (src0->ne[2] > 1 && t->nb[2] != t->ne[1] * t->nb[1]) kparams.split_at_ne01 = 1;
+ if (src0->ne[3] > 1 && t->nb[3] != t->ne[2] * t->nb[2]) kparams.split_at_ne02 = 1;
+ };
+ for (const auto * in : prev.inputs) check_split(in);
+ check_split(new_src);
+ check_split(dst);
+
+ htp_binary_vtcm_layout_build(&L, &kparams, vtcm_size);
+ }
+
+ if (L.rows_per_buffer == 0 || L.total_bytes == 0 || L.total_bytes > vtcm_size) {
+ bool all_contig = ggml_is_contiguous(new_src) && ggml_is_contiguous(dst);
+ for (const auto * in : prev.inputs) {
+ all_contig = all_contig && ggml_is_contiguous(in);
+ }
+ if (!all_contig) return false;
+
+ const uint32_t total_elems = (uint32_t) ggml_nelements(src0);
+ const uint32_t elem_size = (uint32_t) ggml_type_size(src0->type);
+ const uint32_t max_threads = htp_session_n_threads(sess);
+ if (!htp_binary_solve_chunked(&L, &kparams, total_elems, elem_size, max_threads, n_srcs, false, vtcm_size)) {
+ return false;
+ }
+ }
+
+ kparams.rows_per_buffer = L.rows_per_buffer;
+ kparams.vtcm_size = (uint32_t) L.total_bytes;
+
+ prev.inputs.push_back(new_src);
+ prev.outputs[0] = dst;
+ prev.fused.push_back(cur.node);
+ prev.set_kparams(kparams);
+ return true;
+}
+
+bool htp_fusion_can_stack(const struct ggml_tensor * a) {
+ if (!a || !a->src[0] || !a->src[1]) return false;
+ switch (a->op) {
+ case GGML_OP_MUL_MAT:
+ if (!htp_fusion_enabled(HTP_FUSE_MUL_MAT_NX)) return false;
+ if (htp_is_hadamard(a)) return false;
+ if (a->src[0]->ne[2] != 1 || a->src[0]->ne[3] != 1) return false;
+ return ggml_is_quantized(a->src[0]->type);
+ case GGML_OP_MUL_MAT_ID:
+ if (!htp_fusion_enabled(HTP_FUSE_MUL_MAT_ID_NX)) return false;
+ return a->src[2] != nullptr && ggml_is_quantized(a->src[0]->type);
+ default:
+ return false;
+ }
+}
+
+bool htp_fusion_can_stack(const struct ggml_tensor * a, const struct ggml_tensor * b) {
+ if (!htp_fusion_can_stack(b) || a->op != b->op || a->src[1] != b->src[1]) {
+ return false;
+ }
+ if (a->op == GGML_OP_MUL_MAT_ID && a->src[2] != b->src[2]) {
+ return false;
+ }
+ const ggml_tensor * w0 = a->src[0];
+ const ggml_tensor * w1 = b->src[0];
+ return w0->type == w1->type && w0->ne[0] == w1->ne[0] && w0->ne[2] == w1->ne[2];
+}
+
+int htp_fusion_max(const struct ggml_cgraph * graph, int idx) {
+ const int n = graph->n_nodes;
+ const ggml_op op0 = graph->nodes[idx]->op;
+ if (op0 != GGML_OP_ADD && op0 != GGML_OP_NORM && op0 != GGML_OP_RMS_NORM) {
+ return 1;
+ }
+
+ enum ggml_op ops[HTP_OP_MAX_FUSED];
+ ops[0] = op0;
+
+ int f = idx + 1;
+ while (f < n && f < idx + HTP_OP_MAX_FUSED) {
+ const ggml_op op = graph->nodes[f]->op;
+ if (op != GGML_OP_ADD && op != GGML_OP_MUL && op != GGML_OP_SCALE &&
+ op != GGML_OP_NORM && op != GGML_OP_RMS_NORM) {
+ break;
+ }
+ ops[f - idx] = op;
+ f++;
+ }
+
+ f -= idx;
+ for (; f > 1; f--) {
+ if (ggml_can_fuse(graph, idx, ops, f)) {
+ break;
+ }
+ }
+ return f;
+}
+
+void htp_fusion_tag_graph(struct ggml_cgraph * graph) {
+ for (int i = 0; i < graph->n_nodes; i++) {
+ auto * extra = (htp_tensor_extra *) graph->nodes[i]->extra;
+ if (!extra) continue;
+
+ extra->flags &= ~HTP_TENSOR_FUSEABLE;
+
+ const ggml_op op = graph->nodes[i]->op;
+ if ((op == GGML_OP_RMS_NORM || op == GGML_OP_SCALE || op == GGML_OP_REPEAT ||
+ op == GGML_OP_UNARY || op == GGML_OP_ADD || op == GGML_OP_SUB ||
+ op == GGML_OP_MUL || op == GGML_OP_DIV || op == GGML_OP_MUL_MAT_ID ||
+ op == GGML_OP_SSM_CONV ||
+ (op == GGML_OP_MUL_MAT && !htp_is_hadamard(graph->nodes[i]))) &&
+ ggml_node_has_n_uses(graph, i, 1)) {
+ extra->flags |= HTP_TENSOR_FUSEABLE;
+ }
+ }
+}
+
+bool htp_fusion_try_fuse(const struct ggml_hexagon_session * sess, struct htp_opnode & prev, const struct htp_opnode & cur) {
+ if (!opt_opfusion) return false;
+
+ switch (cur.opcode) {
+ case HTP_OP_ADD:
+ return (htp_fusion_enabled(HTP_FUSE_ALLREDUCE_ADD) && fuse_allreduce_add(sess, prev, cur)) ||
+ (htp_fusion_enabled(HTP_FUSE_MUL_MAT_ADD) && fuse_mul_mat_add(sess, prev, cur)) ||
+ (htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur));
+ case HTP_OP_MUL:
+ return (htp_fusion_enabled(HTP_FUSE_RMS_NORM_MUL) && fuse_rms_norm_mul(sess, prev, cur)) ||
+ (htp_fusion_enabled(HTP_FUSE_UNARY_MUL_GLU) && fuse_unary_mul_glu(sess, prev, cur)) ||
+ (htp_fusion_enabled(HTP_FUSE_MUL_MUL_SCALAR) && fuse_mul_mul_scalar(sess, prev, cur)) ||
+ (htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur));
+ case HTP_OP_SUB:
+ case HTP_OP_DIV:
+ return htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur);
+ case HTP_OP_SCALE:
+ return htp_fusion_enabled(HTP_FUSE_RMS_NORM_SCALE) && fuse_rms_norm_scale(sess, prev, cur);
+ case HTP_OP_UNARY_SILU:
+ return htp_fusion_enabled(HTP_FUSE_SSM_CONV_SILU) && fuse_ssm_conv_silu(sess, prev, cur);
+ case HTP_OP_GET_ROWS:
+ return htp_fusion_enabled(HTP_FUSE_REPEAT_GET_ROWS) && fuse_repeat_get_rows(sess, prev, cur);
+ case HTP_OP_CPY:
+ return htp_fusion_enabled(HTP_FUSE_GDN_CPY) && fuse_gdn_cpy(sess, prev, cur);
+ case HTP_OP_MUL_MAT:
+ return htp_fusion_enabled(HTP_FUSE_MUL_MAT_NX) && fuse_mul_mat_nx(sess, prev, cur);
+ case HTP_OP_MUL_MAT_ID:
+ return htp_fusion_enabled(HTP_FUSE_MUL_MAT_ID_NX) && fuse_mul_mat_id_nx(sess, prev, cur);
+ default:
+ return false;
+ }
+}
diff --git a/ggml/src/ggml-hexagon/htp-fusion.h b/ggml/src/ggml-hexagon/htp-fusion.h
new file mode 100644
index 000000000..e794c1daa
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp-fusion.h
@@ -0,0 +1,33 @@
+#ifndef HTP_FUSION_H
+#define HTP_FUSION_H
+
+#include "htp-common.h"
+
+enum htp_fusion_flags {
+ HTP_FUSE_ALLREDUCE_ADD = (1 << 1), // 2
+ HTP_FUSE_RMS_NORM_MUL = (1 << 2), // 4
+ HTP_FUSE_MUL_MAT_ADD = (1 << 3), // 8
+ HTP_FUSE_MUL_MAT_NX = (1 << 4), // 16
+ HTP_FUSE_MUL_MAT_ID_NX = (1 << 5), // 32
+ HTP_FUSE_GDN_CPY = (1 << 6), // 64
+ HTP_FUSE_REPEAT_GET_ROWS = (1 << 7), // 128
+ HTP_FUSE_UNARY_MUL_GLU = (1 << 8), // 256
+ HTP_FUSE_MUL_MUL_SCALAR = (1 << 9), // 512
+ HTP_FUSE_BINARY_NX = (1 << 10), // 1024
+ HTP_FUSE_RMS_NORM_SCALE = (1 << 11), // 2048
+ HTP_FUSE_SSM_CONV_SILU = (1 << 12), // 4096
+};
+
+static inline bool htp_fusion_enabled(int flag) {
+ if (opt_opfusion <= 0) return false;
+ if (opt_opfusion == 1) return true; // 1 enables all
+ return (opt_opfusion & flag) != 0;
+}
+
+bool htp_fusion_can_stack(const struct ggml_tensor * a);
+bool htp_fusion_can_stack(const struct ggml_tensor * a, const struct ggml_tensor * b);
+int htp_fusion_max(const struct ggml_cgraph * graph, int idx);
+void htp_fusion_tag_graph(struct ggml_cgraph * graph);
+bool htp_fusion_try_fuse(const struct ggml_hexagon_session * sess, struct htp_opnode & prev, const struct htp_opnode & cur);
+
+#endif // HTP_FUSION_H
diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp
new file mode 100644
index 000000000..e886cd023
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp-kparams.cpp
@@ -0,0 +1,3278 @@
+#define _USE_MATH_DEFINES
+
+#include "htp-kparams.h"
+#include "ggml-impl.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+
+#ifndef M_LOG2E
+# define M_LOG2E 1.44269504088896340736
+#endif
+
+htp_op_code htp_op_remap(const struct ggml_tensor * t) {
+ switch (t->op) {
+ case GGML_OP_FLASH_ATTN_EXT: return HTP_OP_FLASH_ATTN_EXT;
+ case GGML_OP_MUL_MAT: return htp_is_hadamard(t) ? HTP_OP_HADAMARD : HTP_OP_MUL_MAT;
+ case GGML_OP_MUL_MAT_ID: return HTP_OP_MUL_MAT_ID;
+ case GGML_OP_MUL: return HTP_OP_MUL;
+ case GGML_OP_ADD: return HTP_OP_ADD;
+ case GGML_OP_ADD_ID: return HTP_OP_ADD_ID;
+ case GGML_OP_SUB: return HTP_OP_SUB;
+ case GGML_OP_DIV: return HTP_OP_DIV;
+ case GGML_OP_DUP:
+ case GGML_OP_CPY:
+ case GGML_OP_CONT: return HTP_OP_CPY;
+ case GGML_OP_GET_ROWS: return HTP_OP_GET_ROWS;
+ case GGML_OP_SET_ROWS: return HTP_OP_SET_ROWS;
+ case GGML_OP_SUM: return HTP_OP_SUM;
+ case GGML_OP_SUM_ROWS: return HTP_OP_SUM_ROWS;
+ case GGML_OP_POOL_2D: return HTP_OP_POOL_2D;
+ case GGML_OP_POOL_1D: return HTP_OP_POOL_1D;
+ case GGML_OP_ARGSORT: return HTP_OP_ARGSORT;
+ case GGML_OP_TOP_K: return HTP_OP_TOP_K;
+ case GGML_OP_ARGMAX: return HTP_OP_ARGMAX;
+ case GGML_OP_NORM: return HTP_OP_NORM;
+ case GGML_OP_L2_NORM: return HTP_OP_L2_NORM;
+ case GGML_OP_RMS_NORM: return HTP_OP_RMS_NORM;
+ case GGML_OP_CONCAT: return HTP_OP_CONCAT;
+ case GGML_OP_SCALE: return HTP_OP_SCALE;
+ case GGML_OP_CLAMP: return HTP_OP_CLAMP;
+ case GGML_OP_LEAKY_RELU: return HTP_OP_LEAKY_RELU;
+ case GGML_OP_SQR: return HTP_OP_SQR;
+ case GGML_OP_SQRT: return HTP_OP_SQRT;
+ case GGML_OP_LOG: return HTP_OP_UNARY_LOG;
+ case GGML_OP_SOFT_MAX: return HTP_OP_SOFTMAX;
+ case GGML_OP_SSM_CONV: return HTP_OP_SSM_CONV;
+ case GGML_OP_GATED_DELTA_NET: return HTP_OP_GATED_DELTA_NET;
+ case GGML_OP_ROPE: return HTP_OP_ROPE;
+ case GGML_OP_REPEAT: return HTP_OP_REPEAT;
+ case GGML_OP_CUMSUM: return HTP_OP_CUMSUM;
+ case GGML_OP_FILL: return HTP_OP_FILL;
+ case GGML_OP_DIAG: return HTP_OP_DIAG;
+ case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI;
+ case GGML_OP_TRI: return HTP_OP_TRI;
+ case GGML_OP_PAD: return HTP_OP_PAD;
+ case GGML_OP_IM2COL: return HTP_OP_IM2COL;
+ case GGML_OP_IM2COL_3D: return HTP_OP_IM2COL_3D;
+ case GGML_OP_ROLL: return HTP_OP_ROLL;
+
+ case GGML_OP_UNARY:
+ switch (ggml_get_unary_op(t)) {
+ case GGML_UNARY_OP_SILU: return HTP_OP_UNARY_SILU;
+ case GGML_UNARY_OP_GELU: return HTP_OP_UNARY_GELU;
+ case GGML_UNARY_OP_GELU_QUICK: return HTP_OP_UNARY_GELU_QUICK;
+ case GGML_UNARY_OP_GELU_ERF: return HTP_OP_UNARY_GELU_ERF;
+ case GGML_UNARY_OP_SIGMOID: return HTP_OP_UNARY_SIGMOID;
+ case GGML_UNARY_OP_NEG: return HTP_OP_UNARY_NEG;
+ case GGML_UNARY_OP_EXP: return HTP_OP_UNARY_EXP;
+ case GGML_UNARY_OP_SOFTPLUS: return HTP_OP_UNARY_SOFTPLUS;
+ case GGML_UNARY_OP_TANH: return HTP_OP_UNARY_TANH;
+ case GGML_UNARY_OP_ABS: return HTP_OP_UNARY_ABS;
+ case GGML_UNARY_OP_RELU: return HTP_OP_UNARY_RELU;
+ case GGML_UNARY_OP_STEP: return HTP_OP_UNARY_STEP;
+ default: break;
+ }
+ break;
+
+ case GGML_OP_GLU:
+ switch (ggml_get_glu_op(t)) {
+ case GGML_GLU_OP_REGLU: return HTP_OP_GLU_REGLU;
+ case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU;
+ case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI;
+ case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP;
+ case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU;
+ case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK;
+ case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF;
+ default: break;
+ }
+ break;
+
+ default:
+ break;
+ }
+ return HTP_OP_INVALID;
+}
+
+bool htp_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride) {
+ size_t s = 0, next = 0;
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
+ if (ne[i] == 1) continue;
+ if (s == 0) { s = nb[i]; next = s * ne[i]; continue; }
+ if (nb[i] != next) return false;
+ next *= ne[i];
+ }
+ *stride = s ? s : nb[1];
+ return true;
+}
+
+bool htp_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst) {
+ size_t s1, sd;
+ return (dst->op == GGML_OP_MUL_MAT || dst->op == GGML_OP_ADD) &&
+ src0->ne[2] == 1 && src0->ne[3] == 1 && src1->ne[2] * src1->ne[3] > 1 &&
+ src1->nb[0] == ggml_type_size(src1->type) && htp_rows_stride(src1->ne, src1->nb, &s1) &&
+ dst->nb[0] == ggml_type_size(dst->type) && htp_rows_stride(dst->ne, dst->nb, &sd);
+}
+
+ggml_tensor htp_tensor_collapse_rows(const struct ggml_tensor * t) {
+ size_t stride = 0;
+ htp_rows_stride(t->ne, t->nb, &stride);
+ ggml_tensor c = *t;
+ c.ne[1] = t->ne[1] * t->ne[2] * t->ne[3];
+ c.ne[2] = 1;
+ c.ne[3] = 1;
+ c.nb[1] = stride;
+ c.nb[2] = c.nb[1] * c.ne[1];
+ c.nb[3] = c.nb[2];
+ return c;
+}
+
+bool htp_kparams_solve_allreduce(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * dst,
+ uint32_t rank,
+ uint32_t n_ranks,
+ bool has_add,
+ bool is_row_bcast,
+ bool is_shard_ok,
+ struct htp_allreduce_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+ kparams->rank = (int32_t) rank;
+ kparams->n_ranks = (int32_t) n_ranks;
+ kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0;
+
+ const uint32_t max_threads = htp_session_n_threads(sess);
+ const size_t vtcm_size = htp_session_vtcm_size(sess);
+
+ const uint32_t nelem = (uint32_t) ggml_nelements(dst);
+ const uint32_t elem_size = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);
+ const bool is_contiguous = ggml_is_contiguous(dst);
+
+ const uint32_t ne0 = (uint32_t) dst->ne[0];
+ const uint32_t ne1 = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);
+ kparams->ne0 = (int32_t) ne0;
+ kparams->ne1 = (int32_t) ne1;
+
+ const bool use_1d = is_contiguous && !(has_add && is_row_bcast && ne1 > 1);
+
+ if (has_add) {
+ if (opt_ar_scatter && is_shard_ok && use_1d && !is_row_bcast) {
+ const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128);
+ const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem);
+ const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem);
+ kparams->n_dsts = (int32_t) n_ranks;
+ kparams->rank_elem_start = (int32_t) rank_elem_start;
+ kparams->rank_nelem = (int32_t) (rank_elem_end - rank_elem_start);
+ } else if (use_1d) {
+ kparams->n_dsts = 1;
+ kparams->rank_elem_start = 0;
+ kparams->rank_nelem = (int32_t) nelem;
+ } else {
+ kparams->n_dsts = 1;
+ kparams->rank_elem_start = 0;
+ kparams->rank_nelem = (int32_t) ne1;
+ }
+ } else {
+ kparams->n_dsts = (int32_t) n_ranks;
+ if (use_1d) {
+ const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128);
+ const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem);
+ const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem);
+ const uint32_t rank_nelem = rank_elem_end - rank_elem_start;
+ kparams->rank_elem_start = (int32_t) rank_elem_start;
+ kparams->rank_nelem = (int32_t) rank_nelem;
+ } else {
+ const uint32_t rank_chunk_rows = (ne1 + n_ranks - 1) / n_ranks;
+ const uint32_t rank_r0 = (std::min)(rank * rank_chunk_rows, ne1);
+ const uint32_t rank_r1 = (std::min)(rank_r0 + rank_chunk_rows, ne1);
+ const uint32_t rank_nrows = rank_r1 - rank_r0;
+ kparams->rank_elem_start = (int32_t) rank_r0;
+ kparams->rank_nelem = (int32_t) rank_nrows;
+ }
+ }
+
+ kparams->mode = (kparams->n_dsts > 1) ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL;
+
+ if (use_1d) {
+ const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem;
+ const uint32_t n_threads = (std::min)(max_threads, (std::max)(1u, rank_nelem / 128));
+ kparams->n_threads = n_threads;
+ const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast);
+
+ uint32_t block_elems = 65536;
+ if (block_elems > rank_nelem / n_threads && rank_nelem / n_threads > 128) {
+ block_elems = hex_round_up(rank_nelem / (n_threads * 2), 128);
+ }
+ block_elems = (std::max)(128u, block_elems);
+
+ kparams->block_elems = block_elems;
+ kparams->vtcm_size_per_thread = 2 * block_elems * elem_size;
+ kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;
+
+ while ((size_t) kparams->vtcm_size > vtcm_size && block_elems > 128) {
+ const size_t max_bytes_per_buf = vtcm_size / (n_vtcm_buffers * 2);
+ block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128);
+ if (block_elems < 128) break;
+ kparams->block_elems = block_elems;
+ kparams->vtcm_size_per_thread = 2 * block_elems * elem_size;
+ kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;
+ }
+
+ if (vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) {
+ HEX_VERBOSE("ggml-hex: %s allreduce 1D solver failed to fit VTCM (%d > %zu)\n",
+ htp_session_name(sess), kparams->vtcm_size, vtcm_size);
+ return false;
+ }
+
+ kparams->elems_per_thread = hex_round_up((rank_nelem + n_threads - 1) / n_threads, block_elems);
+ kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_1D;
+ return true;
+ } else {
+ const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem;
+ const uint32_t n_threads = (std::min)(max_threads, (std::max)(1u, rank_nrows));
+ kparams->n_threads = n_threads;
+ const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast);
+
+ const uint32_t row_bytes = ne0 * elem_size;
+ const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128);
+ kparams->row_size_aligned = row_size_aligned;
+
+ const uint32_t nrows_per_thread = (rank_nrows + n_threads - 1) / n_threads;
+ uint32_t block_rows = (std::min)(128u, nrows_per_thread);
+ block_rows = (std::max)(1u, block_rows);
+ kparams->block_elems = block_rows;
+
+ kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned);
+ kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;
+
+ while ((size_t) kparams->vtcm_size > vtcm_size && block_rows > 1) {
+ const size_t max_rows_per_buf = vtcm_size / (n_vtcm_buffers * 2 * row_size_aligned);
+ block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf);
+ kparams->block_elems = block_rows;
+ kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned);
+ kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;
+ if (max_rows_per_buf == 0) break;
+ }
+
+ if (vtcm_size < (size_t) kparams->vtcm_size || block_rows < 1) {
+ HEX_VERBOSE("ggml-hex: %s allreduce 2D solver failed to fit VTCM (%d > %zu)\n",
+ htp_session_name(sess), kparams->vtcm_size, vtcm_size);
+ return false;
+ }
+
+ kparams->elems_per_thread = nrows_per_thread;
+ kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_2D;
+ return true;
+ }
+}
+
+static bool htp_flash_attn_is_hmx_eligible(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * q,
+ const struct ggml_tensor * k,
+ const struct ggml_tensor * v,
+ const struct ggml_tensor * sinks
+) {
+ if (htp_session_n_hmx(sess) == 0 || opt_fa_select < 2) {
+ return false;
+ }
+
+ if ((k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0) ||
+ (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_Q8_0)) {
+ return false;
+ }
+
+ const uint32_t DK = q->ne[0];
+ const uint32_t DV = v->ne[0];
+
+ if (DK % 8 != 0 || DV % 8 != 0) {
+ return false;
+ }
+
+ GGML_UNUSED(sinks);
+
+ if (opt_fa_select > 2) {
+ return true;
+ }
+
+ const uint32_t M = q->ne[1];
+ if (M > 1) {
+ return true;
+ }
+
+ const uint32_t n_threads = htp_session_n_threads(sess);
+ const uint32_t n_head = q->ne[2];
+ const uint32_t n_kv_heads = k->ne[2];
+ const uint32_t G = n_kv_heads > 0 ? n_head / n_kv_heads : 1;
+ const uint32_t S = k->ne[1];
+ const uint32_t total_threads = n_threads * htp_session_mdev_count(sess);
+
+ if (DK > 256 || (DK == 256 && (DV > 256 || n_kv_heads < total_threads))) {
+ return true;
+ }
+
+ const bool is_tile_aligned = (G > 0 && (32 % G == 0));
+ if (!is_tile_aligned) {
+ if (n_threads >= 6) {
+ return false;
+ }
+ return S >= 1024;
+ }
+
+ uint32_t s_cross = 512;
+ if (DK == 256) {
+ s_cross = 4096;
+ } else if (DK <= 64) {
+ s_cross = (n_threads >= 8) ? 2048 : ((n_threads >= 6) ? 768 : 512);
+ } else {
+ s_cross = (n_threads >= 8) ? 1024 : 512;
+ }
+
+ return S >= s_cross;
+}
+
+static bool htp_kparams_solve_flash_attn(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_fa_kernel_params * kparams
+) {
+ if (opt_fa_select < 1) {
+ return false;
+ }
+
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * q = op->src[0];
+ const struct ggml_tensor * k = op->src[1];
+ const struct ggml_tensor * v = op->src[2];
+ const struct ggml_tensor * mask = op->src[3];
+ const struct ggml_tensor * sinks = op->src[4];
+ const struct ggml_tensor * dst = op;
+
+ if ((q->type != GGML_TYPE_F16 && q->type != GGML_TYPE_F32) ||
+ (k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0) ||
+ (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_Q8_0)) {
+ return false;
+ }
+ if (mask && mask->type != GGML_TYPE_F16) {
+ return false;
+ }
+ if (sinks && sinks->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) {
+ return false;
+ }
+
+ const uint32_t n_threads = htp_session_n_threads(sess);
+ const size_t vtcm_size = htp_session_vtcm_size(sess);
+
+ const uint32_t neq0 = q->ne[0];
+ const uint32_t neq1 = q->ne[1];
+ const uint32_t neq2 = q->ne[2];
+ const uint32_t nek1 = k->ne[1];
+ const uint32_t nev0 = v->ne[0];
+
+ const uint32_t DK = neq0;
+ const uint32_t DV = nev0;
+
+ const uint32_t n_kv_heads = k->ne[2];
+ const uint32_t G = neq2 / n_kv_heads;
+
+ float scale = 1.0f;
+ float max_bias = 0.0f;
+ float logit_softcap = 0.0f;
+ memcpy(&scale, &op->op_params[0], sizeof(float));
+ memcpy(&max_bias, &op->op_params[1], sizeof(float));
+ memcpy(&logit_softcap, &op->op_params[2], sizeof(float));
+
+ if (logit_softcap != 0.0f) {
+ scale /= logit_softcap;
+ }
+
+ kparams->scale = scale;
+ kparams->max_bias = max_bias;
+ kparams->logit_softcap = logit_softcap;
+ kparams->head_split = (opt_fa_head_split != 0) ? 1 : 0;
+ kparams->G = G;
+
+ const uint32_t n_head = q->ne[2];
+ kparams->n_head_log2 = 1u << (uint32_t) std::floor(std::log2(n_head));
+ kparams->m0 = std::pow(2.0f, -(max_bias) / kparams->n_head_log2);
+ kparams->m1 = std::pow(2.0f, -(max_bias / 2.0f) / kparams->n_head_log2);
+
+ if (htp_flash_attn_is_hmx_eligible(sess, q, k, v, sinks)) {
+ const uint32_t DK_pad = hex_round_up(DK, 64);
+ const uint32_t DV_pad = hex_round_up(DV, 64);
+ size_t Br = 0, Bc = 0;
+ int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, vtcm_size, n_threads, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head);
+ if (ret == 0) {
+ kparams->kernel_type = HTP_FA_KERNEL_HMX;
+ if (logit_softcap == 0.0f) {
+ kparams->scale = scale * (float) M_LOG2E;
+ kparams->logit_softcap = 0.0f;
+ } else {
+ kparams->scale = scale;
+ kparams->logit_softcap = logit_softcap * (float) M_LOG2E;
+ }
+ kparams->Br = Br;
+ kparams->Bc = Bc;
+ kparams->n_kv_blocks = (nek1 + Bc - 1) / Bc;
+ kparams->n_threads = (kparams->n_kv_blocks >= 3 && n_threads >= 2) ? n_threads : 1;
+ kparams->u.hmx.g_br = hex_align_up(G * Br, 32);
+ kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && n_threads >= 2) ? 1 : 0;
+ kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK_pad, DV_pad, Br, Bc, kparams->n_threads, kparams->u.hmx.pipeline != 0, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head);
+ kparams->u.hmx.mask_broadcast = (mask != nullptr && mask->ne[2] == 1) ? 1 : 0;
+ kparams->u.hmx.div_G = init_fastdiv_values(G);
+ if (mask) {
+ kparams->src3_div2 = init_fastdiv_values(mask->ne[2]);
+ kparams->src3_div3 = init_fastdiv_values(mask->ne[3]);
+ }
+ return true;
+ }
+ }
+
+ kparams->kernel_type = HTP_FA_KERNEL_HVX;
+ kparams->Br = 1;
+ kparams->Bc = 64;
+ kparams->n_kv_blocks = (k->ne[1] + 64 - 1) / 64;
+ kparams->n_threads = n_threads;
+
+ const size_t size_q_row_padded = hex_round_up(q->ne[0] * ((q->type == GGML_TYPE_F32) ? 4 : 2), 128);
+ const size_t size_k_row_padded = hex_round_up(k->ne[0] * 2, 128);
+ const size_t size_v_row_padded = hex_round_up(v->ne[0] * 2, 128);
+
+ kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, (q->type == GGML_TYPE_F32), mask != nullptr, sinks != nullptr, n_head, n_threads);
+ if ((size_t) kparams->vtcm_size > vtcm_size) {
+ HEX_VERBOSE("ggml-hex: skip flash_attn_ext because VTCM needed (%d) > budget (%zu)\n",
+ kparams->vtcm_size, vtcm_size);
+ return false;
+ }
+
+ kparams->u.hvx.size_q_row_padded = size_q_row_padded;
+ kparams->u.hvx.size_k_row_padded = size_k_row_padded;
+ kparams->u.hvx.size_v_row_padded = size_v_row_padded;
+ kparams->u.hvx.src0_div21 = init_fastdiv_values(q->ne[2] * q->ne[1]);
+ kparams->u.hvx.src0_div1 = init_fastdiv_values(q->ne[1]);
+ kparams->broadcast_rk2 = init_fastdiv_values(q->ne[2] / k->ne[2]);
+ kparams->broadcast_rk3 = init_fastdiv_values(q->ne[3] / k->ne[3]);
+ kparams->broadcast_rv2 = init_fastdiv_values(q->ne[2] / v->ne[2]);
+ kparams->broadcast_rv3 = init_fastdiv_values(q->ne[3] / v->ne[3]);
+ if (mask) {
+ kparams->src3_div2 = init_fastdiv_values(mask->ne[2]);
+ kparams->src3_div3 = init_fastdiv_values(mask->ne[3]);
+ }
+
+ return true;
+}
+
+static bool htp_kparams_solve_gated_delta_net(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_gdn_kernel_params * kparams
+) {
+ if (opt_gdn_select < 1) {
+ return false;
+ }
+
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * q = op->src[0];
+ const struct ggml_tensor * k = op->src[1];
+ const struct ggml_tensor * v = op->src[2];
+ const struct ggml_tensor * g = op->src[3];
+ const struct ggml_tensor * beta = op->src[4];
+ const struct ggml_tensor * state = op->src[5];
+ const struct ggml_tensor * dst = op;
+
+ if (q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F32 || v->type != GGML_TYPE_F32 ||
+ g->type != GGML_TYPE_F32 || beta->type != GGML_TYPE_F32 || state->type != GGML_TYPE_F32 ||
+ dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+
+ if (!ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) || !ggml_is_contiguous_rows(v) ||
+ !ggml_is_contiguous(g) || !ggml_is_contiguous(beta) || !ggml_is_contiguous(state) ||
+ !ggml_is_contiguous(dst)) {
+ return false;
+ }
+
+ const int64_t S_v = v->ne[0];
+ const int64_t H = v->ne[1];
+ const int64_t n_tokens = v->ne[2];
+ const int64_t n_seqs = v->ne[3];
+ const int64_t K = ggml_get_op_params_i32(op, 0);
+
+ if (S_v <= 0 || S_v > 128 || H <= 0 || n_tokens <= 0 || n_seqs <= 0) {
+ return false;
+ }
+ if (q->ne[0] != S_v || k->ne[0] != S_v || q->ne[1] <= 0 || k->ne[1] <= 0 ||
+ q->ne[2] != n_tokens || k->ne[2] != n_tokens || q->ne[3] <= 0 || k->ne[3] <= 0 ||
+ (n_seqs % q->ne[3]) != 0 || (n_seqs % k->ne[3]) != 0) {
+ return false;
+ }
+ if ((g->ne[0] != 1 && g->ne[0] != S_v) || beta->ne[0] != 1) {
+ return false;
+ }
+ if (state->ne[0] != S_v || state->ne[1] != S_v || state->ne[2] != H || state->ne[3] != n_seqs) {
+ return false;
+ }
+ if (dst->ne[0] != S_v * H || dst->ne[1] != n_tokens * n_seqs + S_v * n_seqs * K ||
+ dst->ne[2] != 1 || dst->ne[3] != 1) {
+ return false;
+ }
+
+ const uint32_t rq3 = (uint32_t) (n_seqs / q->ne[3]);
+ const uint32_t rk3 = (uint32_t) (n_seqs / k->ne[3]);
+ const uint32_t total_rows = (uint32_t) (H * n_seqs);
+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), total_rows);
+ const size_t vtcm_size = htp_session_vtcm_size(sess);
+
+ const bool can_use_hmx = (opt_gdn_select >= 2) &&
+ (htp_session_n_hmx(sess) > 0) &&
+ (S_v % 64 == 0) &&
+ (n_tokens >= HTP_GDN_MIN_TOKENS) &&
+ (g->ne[0] == 1) &&
+ (K == 1);
+
+ struct htp_gdn_hmx_vtcm_layout hmx_layout;
+ struct htp_gdn_vtcm_layout hvx_layout;
+ uint32_t n_heads_batch = 1;
+
+ if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout, (uint32_t) S_v, HTP_GDN_CHUNK_SIZE, total_rows, vtcm_size, n_threads, true, &n_heads_batch)) {
+ kparams->kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED;
+ kparams->pipeline = hmx_layout.pipeline ? 1 : 0;
+ kparams->chunk_size = HTP_GDN_CHUNK_SIZE;
+ kparams->n_chunks = ((uint32_t) n_tokens + HTP_GDN_CHUNK_SIZE - 1) / HTP_GDN_CHUNK_SIZE;
+ kparams->n_heads_batch = (uint16_t) n_heads_batch;
+ kparams->vtcm_size = (uint32_t) hmx_layout.total_bytes;
+ } else {
+ htp_gdn_vtcm_layout_build(&hvx_layout, (uint32_t) S_v, n_threads);
+ if (hvx_layout.total_bytes > vtcm_size) {
+ return false;
+ }
+ kparams->kernel_type = HTP_GDN_KERNEL_HVX_RECURRENT;
+ kparams->pipeline = 0;
+ kparams->n_heads_batch = 1;
+ kparams->vtcm_size = (uint32_t) hvx_layout.total_bytes;
+ }
+
+ kparams->n_threads = n_threads;
+ kparams->S_v = (uint32_t) S_v;
+ kparams->H = (uint32_t) H;
+ kparams->n_tokens = (uint32_t) n_tokens;
+ kparams->n_seqs = (uint32_t) n_seqs;
+ kparams->K = (uint32_t) K;
+ kparams->total_rows = total_rows;
+ kparams->kda = (g->ne[0] == S_v) ? 1 : 0;
+ kparams->state_seq_stride = (uint32_t) (state->nb[3] / sizeof(float));
+ kparams->state_size_per_snap = (uint32_t) (S_v * S_v * H * n_seqs);
+ kparams->scale = 1.0f / sqrtf((float) S_v);
+
+ if (H > 0) kparams->div_H = init_fastdiv_values((uint32_t) H);
+ if (q->ne[1] > 0) kparams->div_q1 = init_fastdiv_values((uint32_t) q->ne[1]);
+ if (k->ne[1] > 0) kparams->div_k1 = init_fastdiv_values((uint32_t) k->ne[1]);
+ if (rq3 > 0) kparams->div_rq3 = init_fastdiv_values(rq3);
+ if (rk3 > 0) kparams->div_rk3 = init_fastdiv_values(rk3);
+ return true;
+}
+
+static bool htp_matmul_is_hmx_eligible(
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * dst,
+ bool is_matmul_id,
+ bool is_batched
+) {
+ GGML_UNUSED(dst);
+
+ if (src1->type != GGML_TYPE_F32 && (src1->type != GGML_TYPE_F16 || is_matmul_id)) {
+ return false;
+ }
+
+ const int ne00 = src0->ne[0];
+ const int ne01 = src0->ne[1];
+ const int ne11 = src1->ne[1];
+ const int ne12 = src1->ne[2];
+ const int wtype = src0->type;
+
+ if (ne01 % HTP_TENSOR_TILE_N_ROWS != 0 && is_matmul_id) {
+ return false;
+ }
+
+ if (!htp_is_hmx_weight_type((ggml_type) wtype)) {
+ return false;
+ }
+
+ if (ne00 % HTP_TENSOR_TILE_N_COLS != 0) {
+ return false;
+ }
+
+ const bool is_tiled = htp_tensor_is_tiled(src0);
+ if (ggml_is_quantized((ggml_type) wtype) && !is_tiled) {
+ return false;
+ }
+ if (!is_matmul_id && is_batched && is_tiled) {
+ return false;
+ }
+
+ if (src0->nb[0] > src0->nb[1] || src1->nb[0] > src1->nb[1]) {
+ return false;
+ }
+
+ const int m = is_matmul_id ? ne12 : ne11;
+ if (m <= HTP_MM_HMX_MIN_NROWS) {
+ return false;
+ }
+
+ return true;
+}
+
+static bool htp_kparams_solve_hmx_mm(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * act,
+ const struct ggml_tensor * dst,
+ int wtype,
+ int ne00_padded,
+ int ne01_padded,
+ int ne02,
+ int ne11,
+ int ne12,
+ int ne11_padded,
+ bool is_matmul_id,
+ bool is_batched,
+ size_t src2_size,
+ size_t vtcm_budget,
+ struct htp_mm_kernel_params * kparams
+) {
+ const bool is_tiled = htp_tensor_is_tiled(src0);
+ const int aligned_tile_size = is_tiled ? htp_tensor_get_tile_padded_size(wtype) : 0;
+ const int n_threads = (int) htp_session_n_threads(sess);
+ const uint32_t mdev_count = htp_session_mdev_count(sess);
+ const int ne10 = act->ne[0];
+
+ int m_for_solver = ne11;
+ int m_for_solver_padded = ne11_padded;
+ if (!is_matmul_id && mdev_count > 1 && ((uint32_t) ne11 >= mdev_count)) {
+ const bool dst_row_split = dst ? htp_can_row_partition(dst) : true;
+ const bool act_row_split = htp_can_row_partition(act);
+ if (dst_row_split && act_row_split) {
+ m_for_solver = (ne11 + (int) mdev_count - 1) / (int) mdev_count;
+ m_for_solver_padded = hex_round_up(std::max(m_for_solver, 32), 32);
+ }
+ }
+ const bool pipeline = is_matmul_id ? false : htp_mm_hmx_pipeline(m_for_solver);
+ const bool is_batched_val = is_matmul_id ? false : is_batched;
+ const int group_size = (ne02 > 0 ? ne12 / ne02 : 1);
+
+ size_t m_chunk = 0;
+ size_t n_chunk = 0;
+ size_t vtcm_size = 0;
+ bool use_grouped = false;
+ int act_threads_selected = 0;
+
+ if (is_batched_val && wtype == GGML_TYPE_F16 && !is_tiled && group_size > 1) {
+ if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, m_for_solver, group_size, n_threads, pipeline, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) {
+ use_grouped = true;
+ }
+ }
+
+ if (!use_grouped) {
+ int m_id_rows = 0;
+ if (dst && is_matmul_id) {
+ const int n_experts = ne02 > 0 ? ne02 : 1;
+ const size_t total_expert_rows = (size_t) dst->ne[1] * dst->ne[2];
+ int m_per_expert = (int) ((total_expert_rows + n_experts - 1) / n_experts);
+ if (mdev_count > 1 && htp_can_row_partition(dst)) {
+ m_per_expert = (m_per_expert + (int) mdev_count - 1) / (int) mdev_count;
+ }
+ m_id_rows = hex_round_up(std::max(m_per_expert, 32), 32);
+ }
+ const uint32_t cost_m = is_matmul_id ? (uint32_t) m_id_rows : (uint32_t) m_for_solver;
+ if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, m_for_solver_padded, cost_m, n_threads, pipeline, is_matmul_id, aligned_tile_size, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) {
+ return false;
+ }
+ }
+
+ kparams->n_hmx = 1;
+ kparams->pipeline = pipeline ? 1 : 0;
+ kparams->m_chunk = m_chunk;
+ kparams->n_chunk = n_chunk;
+ kparams->n_threads = n_threads;
+ kparams->n_act_threads = act_threads_selected;
+ kparams->tile_size = is_tiled ? htp_tensor_get_tile_size(wtype) : 0;
+ kparams->aligned_tile_size = aligned_tile_size;
+ kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
+ kparams->vtcm_size = vtcm_size;
+ kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected);
+ kparams->div_ne00_padded = init_fastdiv_values(ne00_padded);
+ kparams->vtcm_bias_size = (int32_t) src2_size;
+ kparams->kernel_type = (is_batched && !is_matmul_id) ? HTP_MM_KERNEL_HMX_BATCHED : HTP_MM_KERNEL_HMX_2D;
+ return true;
+}
+
+static void htp_kparams_solve_hvx_mm(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * dst,
+ int wtype,
+ bool is_tiled,
+ int ne02,
+ int ne03,
+ int ne10,
+ int ne11,
+ int ne12,
+ int ne13,
+ bool is_matmul_id,
+ const size_t src2_row_size,
+ size_t vtcm_budget,
+ struct htp_mm_kernel_params * kparams
+) {
+ if (opt_mm_select < 1) {
+ kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
+ return;
+ }
+
+ const uint32_t n_threads = htp_session_n_threads(sess);
+ kparams->n_hmx = 0;
+ kparams->n_threads = n_threads;
+
+ GGML_UNUSED(ne02);
+ GGML_UNUSED(ne03);
+
+ const int src1_nrows = ne11 * ne12 * ne13;
+
+ if (is_tiled) {
+ kparams->tile_size = htp_tensor_get_tile_size(wtype);
+ kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
+
+ const bool k_align = (ne10 % htp_tensor_get_tile_k(wtype) == 0);
+
+ if (is_matmul_id) {
+ kparams->kernel_type = (src1_nrows < (int) n_threads)
+ ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW;
+ kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
+
+ struct htp_mm_hvx_vtcm_layout L;
+ uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;
+ uint32_t best_n_prefetch = 2;
+ for (uint32_t d = max_prefetch; d >= 2; d /= 2) {
+ htp_mm_hvx_vtcm_layout_build(
+ &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_threads,
+ 0, src0->nb[1], kparams->act_row_size, 0, d, true, false
+ );
+ if (L.total_bytes <= vtcm_budget) {
+ best_n_prefetch = d;
+ break;
+ }
+ }
+ if (L.total_bytes > vtcm_budget) {
+ kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
+ return;
+ }
+ kparams->n_prefetch = best_n_prefetch;
+ kparams->vtcm_size = L.total_bytes;
+ kparams->vtcm_bias_size = 0;
+ return;
+ } else {
+ if (k_align && opt_mm_select >= 1) {
+ kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
+ kparams->kernel_type = (src1_nrows < (int) n_threads)
+ ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW;
+
+ struct htp_mm_hvx_vtcm_layout L;
+ uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;
+ uint32_t best_n_prefetch = 2;
+ for (uint32_t d = max_prefetch; d >= 2; d /= 2) {
+ htp_mm_hvx_vtcm_layout_build(
+ &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_threads,
+ dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, d, false, false
+ );
+ if (L.total_bytes <= vtcm_budget) {
+ best_n_prefetch = d;
+ break;
+ }
+ }
+
+ uint32_t m_chunk = 0;
+ if (htp_mm_hvx_solve_vtcm_params(
+ kparams->kernel_type, wtype, ne10, src1_nrows, n_threads,
+ dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, best_n_prefetch, vtcm_budget,
+ &L, &m_chunk)) {
+ kparams->n_prefetch = best_n_prefetch;
+ kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0;
+ kparams->vtcm_size = L.total_bytes;
+ kparams->vtcm_bias_size = L.bias_bytes;
+ return;
+ }
+ }
+
+ kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
+ return;
+ }
+ } else if (is_matmul_id) {
+ kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
+ return;
+ } else if (wtype == GGML_TYPE_F16 || wtype == GGML_TYPE_F32 || wtype == GGML_TYPE_BF16) {
+ int32_t ktype = HTP_MM_KERNEL_HVX_F16_F16_VTCM;
+ int32_t act_elem = 2;
+ if (wtype == GGML_TYPE_F32) {
+ ktype = HTP_MM_KERNEL_HVX_F32_F32_VTCM;
+ act_elem = 4;
+ } else if (wtype == GGML_TYPE_BF16) {
+ ktype = HTP_MM_KERNEL_HVX_BF16_F32_VTCM;
+ act_elem = 4;
+ }
+ struct htp_mm_hvx_vtcm_layout L;
+ uint32_t m_chunk = 0;
+ if (htp_mm_hvx_solve_vtcm_params(
+ ktype, wtype, ne10, src1_nrows, n_threads,
+ dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget,
+ &L, &m_chunk)) {
+ kparams->kernel_type = ktype;
+ kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0;
+ kparams->act_row_size = hex_round_up(ne10 * act_elem, 128);
+ kparams->vtcm_size = L.total_bytes;
+ kparams->vtcm_bias_size = L.bias_bytes;
+ kparams->n_prefetch = 16;
+ return;
+ }
+ }
+
+ kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
+}
+
+static bool htp_kparams_solve_matmul_impl(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * dst,
+ const size_t src2_row_size,
+ const size_t src2_size,
+ struct htp_mm_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const int ne00 = src0->ne[0];
+ const int ne01 = src0->ne[1];
+ const int ne02 = src0->ne[2];
+ const int ne03 = src0->ne[3];
+
+ const int ne10 = src1->ne[0];
+ const int ne11 = src1->ne[1];
+ const int ne12 = src1->ne[2];
+ const int ne13 = src1->ne[3];
+
+ const bool is_tiled = htp_tensor_is_tiled(src0);
+ const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;
+ const int ne00_padded = is_tiled ? hex_round_up(ne00, htp_tensor_get_tile_k(wtype)) : ne00;
+ const int ne11_padded = hex_round_up(ne11, 32);
+ const int ne01_tiled = hex_round_up(ne01, 32);
+
+ const bool is_matmul_id = (dst->op == GGML_OP_MUL_MAT_ID);
+ const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1);
+ const size_t vtcm_budget = htp_session_vtcm_size(sess);
+
+ const bool hmx_enabled = (htp_session_n_hmx(sess) > 0) && (opt_mm_select >= 2);
+ if (hmx_enabled && htp_matmul_is_hmx_eligible(src0, src1, dst, is_matmul_id, is_batched)) {
+ if (htp_kparams_solve_hmx_mm(sess, src0, src1, dst, wtype, ne00_padded, ne01_tiled, ne02, ne11, ne12, ne11_padded, is_matmul_id, is_batched, src2_size, vtcm_budget, kparams)) {
+ goto finalize;
+ }
+ }
+
+ htp_kparams_solve_hvx_mm(sess, src0, src1, dst, wtype, is_tiled, ne02, ne03, ne10, ne11, ne12, ne13, is_matmul_id, src2_row_size, vtcm_budget, kparams);
+
+finalize:
+ kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11);
+ kparams->div_ne1 = init_fastdiv_values(ne11);
+ kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1);
+ kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1);
+ kparams->div_ne12 = init_fastdiv_values(ne12);
+
+ if (kparams->kernel_type == HTP_MM_KERNEL_UNSUPPORTED || (size_t) kparams->vtcm_size > vtcm_budget) {
+ HEX_VERBOSE("ggml-hex: %s supported %s VTCM size needed (%d) > budget (%zu)\n",
+ htp_session_name(sess), is_matmul_id ? "MUL_MAT_ID" : "MUL_MAT", kparams->vtcm_size, vtcm_budget);
+ return false;
+ }
+ return true;
+}
+
+static bool htp_matmul_check_types(
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * dst
+) {
+ const bool is_matmul_id = (dst->op == GGML_OP_MUL_MAT_ID);
+
+ if (dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+
+ if (is_matmul_id) {
+ const struct ggml_tensor * src2 = dst->src[2];
+ if (src1->type != GGML_TYPE_F32 || !src2 || src2->type != GGML_TYPE_I32) {
+ return false;
+ }
+ if (ggml_get_op_params_i32(dst, 3) == GGML_PREC_F32) {
+ return false;
+ }
+ switch (src0->type) {
+ case GGML_TYPE_Q1_0:
+ case GGML_TYPE_Q2_0:
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_MXFP4:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q2_K:
+ if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) {
+ return false;
+ }
+ if (src0->ne[0] % ggml_blck_size(src0->type) != 0) {
+ return false;
+ }
+ return true;
+
+ case GGML_TYPE_F16:
+ case GGML_TYPE_F32:
+ case GGML_TYPE_BF16:
+ if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0)) {
+ return false;
+ }
+ return src0->ne[0] % 32 == 0;
+
+ default:
+ return false;
+ }
+ }
+
+ if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) {
+ return false;
+ }
+ if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3] ||
+ src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) {
+ return false;
+ }
+
+ switch (src0->type) {
+ case GGML_TYPE_Q1_0:
+ case GGML_TYPE_Q2_0:
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_MXFP4:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q2_K:
+ if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) {
+ return false;
+ }
+ return src0->ne[0] % ggml_blck_size(src0->type) == 0;
+
+ case GGML_TYPE_F32:
+ if (src1->type != GGML_TYPE_F32) {
+ return false;
+ }
+ return src0->nb[1] >= src0->nb[0];
+
+ case GGML_TYPE_F16:
+ case GGML_TYPE_BF16:
+ return src0->nb[1] >= src0->nb[0];
+
+ default:
+ return false;
+ }
+}
+
+static bool htp_kparams_solve_matmul(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_mm_kernel_params * kparams
+) {
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * src1 = op->src[1];
+ const struct ggml_tensor * dst = op;
+
+ if (!htp_matmul_check_types(src0, src1, dst)) {
+ return false;
+ }
+ if (htp_matmul_can_collapse(src0, src1, dst)) {
+ const ggml_tensor src1_collapsed = htp_tensor_collapse_rows(src1);
+ const ggml_tensor dst_collapsed = htp_tensor_collapse_rows(dst);
+ const bool ok = htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, 0, 0, kparams);
+ kparams->collapse = 1;
+ return ok;
+ }
+ return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, 0, 0, kparams);
+}
+
+static bool htp_matmul_add_can_collapse(
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * src2,
+ const struct ggml_tensor * dst
+) {
+ if (!htp_matmul_can_collapse(src0, src1, dst)) {
+ return false;
+ }
+ if (!src2) {
+ return true;
+ }
+ const int64_t src2_nrows = src2->ne[1] * src2->ne[2] * src2->ne[3];
+ if (src2_nrows == 1) {
+ return true;
+ }
+ size_t s2;
+ return src2->nb[0] == ggml_type_size(src2->type) &&
+ ggml_are_same_shape(src2, dst) &&
+ htp_rows_stride(src2->ne, src2->nb, &s2);
+}
+
+bool htp_kparams_solve_matmul_add(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * src2,
+ const struct ggml_tensor * dst,
+ struct htp_mm_kernel_params * kparams
+) {
+ if (htp_matmul_add_can_collapse(src0, src1, src2, dst)) {
+ const ggml_tensor src1_collapsed = htp_tensor_collapse_rows(src1);
+ const ggml_tensor dst_collapsed = htp_tensor_collapse_rows(dst);
+ const ggml_tensor src2_collapsed = (src2 && (src2->ne[1] * src2->ne[2] * src2->ne[3] > 1))
+ ? htp_tensor_collapse_rows(src2)
+ : (src2 ? *src2 : ggml_tensor{});
+ const struct ggml_tensor * p_src2 = src2 ? &src2_collapsed : nullptr;
+ const size_t src2_size = p_src2 ? hex_round_up(ggml_nbytes(p_src2), 128) : 0;
+ const bool ok = htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, p_src2 ? p_src2->nb[1] : 0, src2_size, kparams);
+ kparams->collapse = 1;
+ return ok;
+ }
+ const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0;
+ return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, kparams);
+}
+
+bool htp_kparams_solve_mmnx(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * act,
+ int32_t n_weights,
+ struct htp_mm_kernel_params * kparams
+) {
+ const uint32_t n_threads = htp_session_n_threads(sess);
+ const uint32_t n_hmx = htp_session_n_hmx(sess);
+ const size_t vtcm_budget = htp_session_vtcm_size(sess);
+
+ memset(kparams, 0, sizeof(*kparams));
+ kparams->n_threads = n_threads;
+
+ const int ne00 = src0->ne[0];
+ const int ne01 = src0->ne[1];
+ const int ne02 = src0->ne[2];
+ const int ne03 = src0->ne[3];
+
+ const int ne10 = act->ne[0];
+ const int ne11 = act->ne[1];
+ const int ne12 = act->ne[2];
+ const int ne13 = act->ne[3];
+
+ const bool is_tiled = htp_tensor_is_tiled(src0);
+ const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;
+ const int ne00_padded = is_tiled ? hex_round_up(ne00, 32) : ne00;
+ const int ne11_padded = hex_round_up(ne11, 32);
+ const int ne01_tiled = hex_round_up(ne01, 32);
+ const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1);
+
+ const bool hmx_enabled = (n_hmx > 0) && (opt_mm_select >= 2);
+ if (hmx_enabled && htp_matmul_is_hmx_eligible(src0, act, nullptr, false, is_batched)) {
+ if (htp_kparams_solve_hmx_mm(sess, src0, act, nullptr, wtype, ne00_padded, ne01_tiled, ne02, ne11, ne12, ne11_padded, false, is_batched, 0, vtcm_budget, kparams)) {
+ kparams->n_weights = n_weights;
+ goto finalize;
+ }
+ }
+
+ if (!is_tiled || act->type != GGML_TYPE_F32 || ne10 % 32 != 0) {
+ kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
+ return false;
+ }
+
+ {
+ const int act_nrows = ne11 * ne12 * ne13;
+ const size_t act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
+ const size_t src0_row_size = src0->nb[1];
+
+ const uint32_t max_prefetch = (act_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;
+ uint32_t best_n_prefetch = 2;
+ for (uint32_t d = max_prefetch; d >= 2; d /= 2) {
+ struct htp_mm_hvx_vtcm_layout L;
+ htp_mm_hvx_vtcm_layout_build(
+ &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_threads,
+ 0, src0_row_size, act_row_size, 0, d, false, true
+ );
+ if (L.total_bytes <= vtcm_budget) {
+ best_n_prefetch = d;
+ break;
+ }
+ }
+
+ struct htp_mm_hvx_vtcm_layout L;
+ htp_mm_hvx_vtcm_layout_build(
+ &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_threads,
+ 0, src0_row_size, act_row_size, 0, best_n_prefetch, false, true
+ );
+
+ if (opt_mm_select >= 1 && L.total_bytes <= vtcm_budget) {
+ kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW;
+ kparams->tile_size = htp_tensor_get_tile_size(wtype);
+ kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
+ kparams->act_row_size = act_row_size;
+ kparams->vtcm_bias_size = 0;
+ kparams->vtcm_size = L.total_bytes;
+ kparams->n_prefetch = best_n_prefetch;
+ kparams->n_weights = n_weights;
+ } else {
+ kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;
+ return false;
+ }
+ }
+
+finalize:
+ kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11);
+ kparams->div_ne1 = init_fastdiv_values(ne11);
+ kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1);
+ kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1);
+ kparams->div_ne12 = init_fastdiv_values(ne12);
+ return (size_t) kparams->vtcm_size <= vtcm_budget;
+}
+
+bool htp_kparams_solve_mmidnx(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * act,
+ const struct ggml_tensor * dst,
+ int32_t n_weights,
+ struct htp_mm_kernel_params * kparams
+) {
+ const bool ok = htp_kparams_solve_matmul_impl(sess, src0, act, dst, 0, 0, kparams);
+ kparams->n_weights = n_weights;
+ return ok;
+}
+
+static bool htp_kparams_solve_binary(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op_tensor,
+ struct htp_binary_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const uint32_t op = htp_op_remap(op_tensor);
+ const struct ggml_tensor * src0 = op_tensor->src[0];
+ const struct ggml_tensor * src1 = op_tensor->src[1];
+ const struct ggml_tensor * src2 = (op == HTP_OP_ADD_ID) ? op_tensor->src[2] : nullptr;
+ const struct ggml_tensor * dst = op_tensor;
+
+ const bool is_add_id = (op == HTP_OP_ADD_ID);
+ if (is_add_id) {
+ if (!src2) {
+ return false;
+ }
+ if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32) {
+ return false;
+ }
+ if (!ggml_are_same_shape(src0, dst)) {
+ return false;
+ }
+ if (src1->ne[0] != src0->ne[0] || src1->ne[2] != 1 || src1->ne[3] != 1) {
+ return false;
+ }
+ if (src2->ne[0] != src0->ne[1] || src2->ne[1] != src0->ne[2]) {
+ return false;
+ }
+ if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float) || src2->nb[0] != sizeof(int32_t)) {
+ return false;
+ }
+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {
+ return false;
+ }
+ if (src2->nb[1] < src2->ne[0] * sizeof(int32_t)) {
+ return false;
+ }
+ } else {
+ if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) ||
+ src1->type != src0->type || dst->type != src0->type) {
+ return false;
+ }
+ if (ggml_is_permuted(src0) || ggml_is_permuted(dst) || !ggml_are_same_shape(src0, dst)) {
+ return false;
+ }
+ if (!ggml_can_repeat(src1, src0) || ggml_is_permuted(src1)) {
+ return false;
+ }
+ }
+
+ const uint32_t max_threads_sess = htp_session_n_threads(sess);
+ const size_t vtcm_size = htp_session_vtcm_size(sess);
+
+ const size_t elem_size = ggml_type_size(src0->type);
+ const size_t src0_row_size = src0->ne[0] * elem_size;
+ const size_t src1_row_size = src1->ne[0] * elem_size;
+ const size_t dst_row_size = dst->ne[0] * elem_size;
+
+ const size_t src0_row_size_aligned = hex_round_up(src0_row_size, 128);
+ const size_t src1_row_size_aligned = hex_round_up(src1_row_size, 128);
+ const size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128);
+
+ if (!htp_binary_strides_safe(src0, src1, dst, elem_size)) {
+ return false;
+ }
+
+ const bool is_scalar = !is_add_id && src1->ne[0] == 1;
+ const bool is_row_bcast = !is_add_id && !is_scalar &&
+ src1->ne[0] == src0->ne[0] &&
+ (src0->ne[1] > 1 || src0->ne[2] > 1 || src0->ne[3] > 1) &&
+ src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1;
+ const bool is_same_shape = !is_add_id && !is_scalar &&
+ src1->ne[0] == src0->ne[0] &&
+ src1->ne[1] == src0->ne[1] &&
+ (src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) &&
+ (src1->ne[3] == src0->ne[3] || src1->ne[3] == 1);
+ const bool is_complex = !is_add_id && !is_scalar && !is_same_shape && !is_row_bcast && (src1->ne[0] == src0->ne[0]);
+ const bool is_contig = ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst);
+ const bool is_scalar_broadcast = !is_add_id && (ggml_nelements(src1) == 1);
+
+ if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) {
+ const uint32_t total_elems = (uint32_t) ggml_nelements(src0);
+ const uint32_t max_chunk_elems = 32768 / elem_size;
+ const uint32_t min_chunk_elems = 256;
+ const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_threads_sess) - 1) / (2 * max_threads_sess), 32);
+ const uint32_t chunk_size = (std::min)(max_chunk_elems, (std::max)(target_chunk_elems, min_chunk_elems));
+ const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128);
+ const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size;
+ const uint32_t n_threads = (total_chunks >= 2) ? (std::min)(max_threads_sess, total_chunks) : 1;
+
+ kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED;
+ kparams->n_threads = n_threads;
+ kparams->n_srcs = 2;
+ kparams->rows_per_buffer = 1;
+ kparams->src0_row_size_aligned = src0_row_size_aligned;
+ kparams->src1_row_size_aligned = is_scalar_broadcast ? 0 : src1_row_size_aligned;
+ kparams->dst_row_size_aligned = dst_row_size_aligned;
+ kparams->src1_size = 0;
+ kparams->chunk_size = chunk_size;
+ kparams->chunk_bytes = chunk_bytes;
+ kparams->is_scalar = is_scalar_broadcast ? 1 : 0;
+
+ struct htp_binary_vtcm_layout L;
+ htp_binary_vtcm_layout_build(&L, kparams, vtcm_size);
+ if (L.total_bytes == 0 || L.total_bytes > vtcm_size) {
+ return false;
+ }
+
+ kparams->vtcm_size = L.total_bytes;
+ return true;
+ }
+
+ enum htp_binary_kernel_type kernel_type;
+ size_t src1_size = 0;
+ size_t src2_size = 0;
+
+ if (is_add_id) {
+ kernel_type = HTP_BINARY_KERNEL_ADD_ID;
+ src1_size = hex_round_up(src1->ne[1] * src1_row_size_aligned, 128);
+ src2_size = hex_round_up(ggml_nelements(src2) * sizeof(int32_t), 128);
+ if (src1_size + src2_size > vtcm_size) {
+ return false;
+ }
+ } else if (is_row_bcast) {
+ kernel_type = HTP_BINARY_KERNEL_ROW_BCAST;
+ src1_size = src1_row_size_aligned;
+ } else if (is_scalar) {
+ src1_size = hex_round_up(ggml_nelements(src1) * elem_size, 128);
+ if (src1_size > vtcm_size) {
+ return false;
+ }
+ kernel_type = HTP_BINARY_KERNEL_SCALAR;
+ kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;
+ } else if (is_same_shape) {
+ kernel_type = HTP_BINARY_KERNEL_SAME_SHAPE;
+ } else if (is_complex) {
+ src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128);
+ if (src1_size > vtcm_size) {
+ return false;
+ }
+ kernel_type = HTP_BINARY_KERNEL_COMPLEX;
+ kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;
+ } else {
+ src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128);
+ if (src1_size > vtcm_size) {
+ return false;
+ }
+ kernel_type = HTP_BINARY_KERNEL_REPEAT;
+ kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;
+ }
+
+ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
+ const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size));
+ const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread);
+
+ kparams->kernel_type = kernel_type;
+ kparams->n_threads = (std::min)(max_threads_sess, max_threads);
+ kparams->n_srcs = 2;
+ kparams->src0_row_size_aligned = src0_row_size_aligned;
+ kparams->src1_row_size_aligned = src1_row_size_aligned;
+ kparams->dst_row_size_aligned = dst_row_size_aligned;
+ kparams->src1_size = src1_size;
+ kparams->src2_size = src2_size;
+
+ const bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]);
+ const bool src1_contig_dim1 = (src1->nb[2] == src1->ne[1] * src1->nb[1]);
+ const bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]);
+
+ const bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]);
+ const bool src1_contig_dim2 = (src1->nb[3] == src1->ne[2] * src1->nb[2]);
+ const bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]);
+
+ bool split_at_ne01 = false;
+ if (src0->ne[2] > 1) {
+ if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {
+ split_at_ne01 = (src1->ne[2] != src0->ne[2]) || !src0_contig_dim1 || !src1_contig_dim1 || !dst_contig_dim1;
+ } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {
+ split_at_ne01 = !src0_contig_dim1 || !dst_contig_dim1;
+ } else {
+ split_at_ne01 = true;
+ }
+ }
+ kparams->split_at_ne01 = split_at_ne01 ? 1 : 0;
+
+ bool split_at_ne02 = false;
+ if (src0->ne[3] > 1) {
+ if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {
+ split_at_ne02 = (src1->ne[3] != src0->ne[3]) || !src0_contig_dim2 || !src1_contig_dim2 || !dst_contig_dim2;
+ } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {
+ split_at_ne02 = !src0_contig_dim2 || !dst_contig_dim2;
+ } else {
+ split_at_ne02 = true;
+ }
+ }
+ kparams->split_at_ne02 = split_at_ne02 ? 1 : 0;
+
+ kparams->div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);
+ kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]);
+ kparams->div_ne012 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[2]));
+ kparams->div_ne11 = init_fastdiv_values((uint32_t) src1->ne[1]);
+ kparams->div_ne12 = init_fastdiv_values((uint32_t) src1->ne[2]);
+ kparams->div_ne13 = init_fastdiv_values((uint32_t) src1->ne[3]);
+
+ struct htp_binary_vtcm_layout L;
+ htp_binary_vtcm_layout_build(&L, kparams, vtcm_size);
+ if (L.rows_per_buffer == 0 || L.total_bytes > vtcm_size) {
+ if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) {
+ const uint32_t total_elems = (uint32_t) ggml_nelements(src0);
+ return htp_binary_solve_chunked(&L, kparams, total_elems, (uint32_t) elem_size,
+ max_threads_sess, 2, is_scalar_broadcast, vtcm_size);
+ }
+ return false;
+ }
+
+ kparams->rows_per_buffer = L.rows_per_buffer;
+ kparams->vtcm_size = L.total_bytes;
+
+ return true;
+}
+
+static bool htp_kparams_solve_unary_impl(
+ const struct ggml_hexagon_session * sess,
+ uint32_t op,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * dst,
+ struct htp_unary_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ if (!htp_op_is_unary((htp_op_code) op)) {
+ return false;
+ }
+
+ if (op == HTP_OP_HADAMARD) {
+ const int64_t n = src0->ne[0];
+ if (n < 32 || (n & (n - 1)) != 0) {
+ return false;
+ }
+ if (!dst->src[0] || dst->src[0]->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 ||
+ (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16)) {
+ return false;
+ }
+ } else {
+ if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || dst->type != src0->type) {
+ return false;
+ }
+ if (op == HTP_OP_TRI && (src0->type != GGML_TYPE_F32 || !ggml_is_contiguous(src0))) {
+ return false;
+ }
+ if (src0->type == GGML_TYPE_F16) {
+ switch (op) {
+ case HTP_OP_NORM:
+ case HTP_OP_RMS_NORM:
+ case HTP_OP_L2_NORM:
+ case HTP_OP_SCALE:
+ case HTP_OP_CLAMP:
+ case HTP_OP_SQR:
+ case HTP_OP_SQRT:
+ case HTP_OP_UNARY_LOG:
+ case HTP_OP_UNARY_ABS:
+ case HTP_OP_UNARY_STEP:
+ break;
+ default:
+ return false;
+ }
+ }
+ }
+
+ if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous(dst) || !ggml_are_same_shape(src0, dst)) {
+ return false;
+ }
+
+ const size_t elem_size = ggml_type_size(src0->type);
+
+ struct ggml_tensor src0_c = *src0;
+ struct ggml_tensor dst_c = *dst;
+ if (htp_unary_is_elementwise(op)) {
+ HTP_UNARY_COLLAPSE_DIMS(src0_c.ne, src0_c.nb, dst_c.ne, dst_c.nb, elem_size);
+ src0 = &src0_c;
+ dst = &dst_c;
+ }
+
+ const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type);
+
+ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
+ const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_data_row_size - 1) / dst_data_row_size));
+ const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread);
+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+ const size_t vtcm_size = htp_session_vtcm_size(sess);
+
+ kparams->n_threads = n_threads;
+
+ const bool broadcast_weight = (op == HTP_OP_RMS_NORM_MUL) && (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1);
+ kparams->broadcast_weight = broadcast_weight;
+
+ const bool src0_contig = ggml_is_contiguous_1(src0);
+ const bool src1_contig = (op == HTP_OP_RMS_NORM_MUL && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true;
+ const uint32_t max_block = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1];
+
+ kparams->src0_contig = src0_contig ? 1 : 0;
+ kparams->src1_contig = src1_contig ? 1 : 0;
+
+ struct htp_unary_vtcm_layout L;
+ uint32_t col_tile = 0;
+ uint32_t vtcm_row_per_thread = 0;
+
+ htp_unary_solve_layout(&L, op, src0->ne[0], dst->ne[0],
+ op == HTP_OP_RMS_NORM_MUL ? src1->ne[0] : 0,
+ max_block, broadcast_weight, n_threads, vtcm_size, elem_size,
+ &col_tile, &vtcm_row_per_thread);
+
+ kparams->col_tile = col_tile;
+ kparams->vtcm_size = L.total_bytes;
+ kparams->block = col_tile ? 0 : vtcm_row_per_thread;
+
+ const uint32_t tiles_per_row = col_tile > 0 ? (src0->ne[0] + col_tile - 1) / col_tile : 1;
+ kparams->tiles_per_row = tiles_per_row;
+ kparams->div_ne01 = init_fastdiv_values(src0->ne[1]);
+ kparams->div_ne02 = init_fastdiv_values(src0->ne[2]);
+ kparams->div_ne012 = init_fastdiv_values(src0->ne[1] * src0->ne[2]);
+ kparams->div_tpr = init_fastdiv_values(tiles_per_row);
+
+ return (col_tile > 0 || vtcm_row_per_thread > 0) && L.total_bytes <= vtcm_size;
+}
+
+static bool htp_kparams_solve_unary(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_unary_kernel_params * kparams
+) {
+ const htp_op_code opcode = htp_op_remap(op);
+ const struct ggml_tensor * src0 = (opcode == HTP_OP_HADAMARD) ? op->src[1] : op->src[0];
+ const struct ggml_tensor * src1 = (opcode == HTP_OP_HADAMARD) ? nullptr : op->src[1];
+ return htp_kparams_solve_unary_impl(sess, opcode, src0, src1, op, kparams);
+}
+
+bool htp_kparams_solve_rms_norm_mul(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * dst,
+ struct htp_unary_kernel_params * kparams
+) {
+ return htp_kparams_solve_unary_impl(sess, HTP_OP_RMS_NORM_MUL, src0, src1, dst, kparams);
+}
+
+bool htp_kparams_solve_repeat_get_rows(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * dst,
+ struct htp_get_rows_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16 && src0->type != GGML_TYPE_I32 &&
+ src0->type != GGML_TYPE_Q1_0 && src0->type != GGML_TYPE_Q2_0 && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 &&
+ src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K) {
+ return false;
+ }
+
+ if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous_rows(src1) || !ggml_is_contiguous_rows(dst)) {
+ return false;
+ }
+
+ if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) {
+ return false;
+ }
+
+ if (src0->type == dst->type) {
+ if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->type != GGML_TYPE_F16) {
+ return false;
+ }
+ } else if (src0->type == GGML_TYPE_I32) {
+ return false;
+ } else if (dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+
+ const uint32_t ne00 = src0->ne[0];
+ const uint32_t ne02 = src0->ne[2];
+ const uint32_t ne03 = src0->ne[3];
+
+ const uint32_t ne10 = src1->ne[0];
+ const uint32_t ne11 = src1->ne[1];
+ const uint32_t ne12 = src1->ne[2];
+ const uint32_t nr = ne10 * ne11 * ne12;
+
+ const bool tiled = htp_tensor_is_tiled(src0);
+ const int wtype = (tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;
+
+ if (tiled) {
+ kparams->kernel_type = HTP_GET_ROWS_KERNEL_TILED;
+ } else if (src0->type == dst->type) {
+ kparams->kernel_type = HTP_GET_ROWS_KERNEL_SAMETYPE;
+ } else {
+ kparams->kernel_type = HTP_GET_ROWS_KERNEL_FLAT;
+ }
+
+ const uint32_t total_tasks = nr;
+ const uint32_t min_tasks_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst->nb[1] - 1) / dst->nb[1]));
+ const uint32_t max_threads = (std::max)(1u, total_tasks / min_tasks_per_thread);
+ const size_t vtcm_size = htp_session_vtcm_size(sess);
+
+ kparams->n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+
+ struct htp_get_rows_vtcm_layout vtcm_layout = {};
+ while (kparams->n_threads > 0) {
+ htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, kparams->n_threads);
+ if (vtcm_layout.total_bytes <= vtcm_size) {
+ break;
+ }
+ --kparams->n_threads;
+ }
+
+ if (kparams->n_threads == 0 && total_tasks > 0) {
+ htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, 1);
+ }
+
+ kparams->vtcm_size = (total_tasks == 0) ? 0 : vtcm_layout.total_bytes;
+ kparams->total_tasks = total_tasks;
+
+ kparams->div_ne10 = ne10 > 0 ? init_fastdiv_values(ne10) : fastdiv_values{0, 0};
+ kparams->div_ne10_ne11 = (ne10 * ne11) > 0 ? init_fastdiv_values(ne10 * ne11) : fastdiv_values{0, 0};
+ kparams->div_ne02 = ne02 > 0 ? init_fastdiv_values(ne02) : fastdiv_values{0, 0};
+ kparams->div_ne03 = ne03 > 0 ? init_fastdiv_values(ne03) : fastdiv_values{0, 0};
+
+ if (total_tasks == 0) {
+ return true;
+ }
+ return kparams->n_threads > 0 && (size_t) kparams->vtcm_size <= vtcm_size;
+}
+
+static bool htp_kparams_solve_get_rows(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_get_rows_kernel_params * kparams
+) {
+ return htp_kparams_solve_repeat_get_rows(sess, op->src[0], op->src[1], op, kparams);
+}
+
+static bool htp_kparams_solve_set_rows(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_set_rows_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * src1 = op->src[1];
+ const struct ggml_tensor * dst = op->src[2] ? op->src[2] : op;
+
+ if (dst->type == GGML_TYPE_Q8_0 && src0->ne[0] < 32) {
+ return false;
+ }
+ if (src0->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) {
+ return false;
+ }
+ if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_Q8_0) {
+ return false;
+ }
+
+ const uint32_t nr = src0->ne[1];
+
+ kparams->n_threads = (std::min)(htp_session_n_threads(sess), nr);
+ kparams->total_tasks = nr;
+
+ struct htp_set_rows_vtcm_layout vtcm_layout;
+ htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_threads);
+ kparams->vtcm_size = vtcm_layout.total_bytes;
+
+ return vtcm_layout.total_bytes <= htp_session_vtcm_size(sess);
+}
+
+static bool htp_kparams_solve_softmax(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_softmax_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * src1 = op->src[1];
+ const struct ggml_tensor * src2 = op->src[2];
+ const struct ggml_tensor * dst = op;
+
+ if (src2) {
+ return false;
+ }
+ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src0->ne[2] > 512) {
+ return false;
+ }
+ if (src1) {
+ if ((src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) ||
+ src0->ne[0] != src1->ne[0] || src1->ne[1] < src0->ne[1] ||
+ src0->ne[2] % src1->ne[2] != 0 || src0->ne[3] % src1->ne[3] != 0 ||
+ !ggml_is_contiguous(src1)) {
+ return false;
+ }
+ }
+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {
+ return false;
+ }
+
+ const int64_t ne0 = src0->ne[0];
+ if ((ne0 > 32 && (ne0 & (32 - 1)) != 0) || ne0 > 131072) {
+ return false;
+ }
+
+ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
+ const size_t dst_data_row_size = op->ne[0] * ggml_type_size(op->type);
+ const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_data_row_size - 1) / dst_data_row_size));
+ const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread);
+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+
+ float scale = 1.0f;
+ float max_bias = 0.0f;
+ memcpy(&scale, &op->op_params[0], sizeof(float));
+ memcpy(&max_bias, &op->op_params[1], sizeof(float));
+
+ kparams->scale = scale;
+ kparams->max_bias = max_bias;
+
+ const uint32_t n_head = src0->ne[2];
+ const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));
+ kparams->n_head = n_head;
+ kparams->n_head_log2 = n_head_log2;
+
+ if (max_bias > 0.0f && n_head_log2 > 0) {
+ kparams->m0 = powf(2.0f, -(max_bias) / n_head_log2);
+ kparams->m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
+ } else {
+ kparams->m0 = 1.0f;
+ kparams->m1 = 1.0f;
+ }
+
+ kparams->use_src1 = (src1 != nullptr) ? 1 : 0;
+ kparams->use_f16 = (src1 != nullptr && src1->type == GGML_TYPE_F16) ? 1 : 0;
+
+ const uint32_t ne00 = src0->ne[0];
+ const uint32_t ne10 = src1 ? src1->ne[0] : 1;
+
+ struct htp_softmax_vtcm_layout layout;
+ htp_softmax_vtcm_layout_build(&layout, ne00, ne10, kparams->use_src1 != 0, kparams->use_f16 != 0, n_threads);
+ if (layout.total_bytes > htp_session_vtcm_size(sess)) {
+ return false;
+ }
+
+ kparams->n_threads = n_threads;
+ kparams->vtcm_size = (uint32_t) layout.total_bytes;
+
+ if (!kparams->use_src1) {
+ kparams->kernel_type = HTP_SOFTMAX_KERNEL_NOMASK;
+ } else if (kparams->use_f16) {
+ kparams->kernel_type = HTP_SOFTMAX_KERNEL_MASK_F16;
+ } else {
+ kparams->kernel_type = HTP_SOFTMAX_KERNEL_MASK_F32;
+ }
+
+ if (src0->ne[1] > 0) kparams->div_ne01 = init_fastdiv_values(src0->ne[1]);
+ if (src0->ne[2] > 0) kparams->div_ne02 = init_fastdiv_values(src0->ne[2]);
+ const uint32_t ne12 = src1 ? src1->ne[2] : 1;
+ const uint32_t ne13 = src1 ? src1->ne[3] : 1;
+ if (ne12 > 0) kparams->div_ne12 = init_fastdiv_values(ne12);
+ if (ne13 > 0) kparams->div_ne13 = init_fastdiv_values(ne13);
+
+ return true;
+}
+
+static bool htp_kparams_solve_rope(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_rope_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * src1 = op->src[1];
+ const struct ggml_tensor * src2 = op->src[2];
+ const struct ggml_tensor * dst = op;
+
+ if (!ggml_are_same_shape(src0, dst)) {
+ return false;
+ }
+ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32) {
+ return false;
+ }
+ if (src0->ne[0] <= 0) {
+ return false;
+ }
+
+ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
+ if (src0_nrows == 0) {
+ return false;
+ }
+
+ const int32_t * op_params = &op->op_params[0];
+ const int n_dims = op_params[1];
+ const int mode = op_params[2];
+ const int n_offs = op_params[15];
+
+ if (n_dims < 0 || n_dims % 2 != 0) {
+ return false;
+ }
+ if (n_offs < 0 || (n_offs % 32 != 0) || (n_offs + n_dims > src0->ne[0])) {
+ return false;
+ }
+
+ float freq_base;
+ memcpy(&freq_base, op_params + 5, sizeof(float));
+ if (freq_base < 0.0f) {
+ return false;
+ }
+
+ if (mode != GGML_ROPE_TYPE_NORMAL &&
+ mode != GGML_ROPE_TYPE_NEOX &&
+ mode != GGML_ROPE_TYPE_MROPE &&
+ mode != GGML_ROPE_TYPE_VISION &&
+ mode != GGML_ROPE_TYPE_IMROPE) {
+ return false;
+ }
+
+ const bool is_mrope = (mode & GGML_ROPE_TYPE_MROPE) != 0;
+
+ if (mode == GGML_ROPE_TYPE_VISION && (n_dims != (int) (src0->ne[0] / 2) || n_offs != 0)) {
+ return false;
+ }
+ if (is_mrope) {
+ const int32_t * sections = op_params + 11;
+ if (sections[0] <= 0 && sections[1] <= 0 && sections[2] <= 0) {
+ return false;
+ }
+ }
+
+ const int64_t min_pos_len = (is_mrope || mode == GGML_ROPE_TYPE_VISION) ? src0->ne[2] * 4 : src0->ne[2];
+ if (src1->ne[0] < min_pos_len || !ggml_is_contiguous(src1)) {
+ return false;
+ }
+
+ if (src2) {
+ if (src2->type != GGML_TYPE_F32 || !ggml_is_contiguous(src2) || src2->ne[0] < (n_dims / 2)) {
+ return false;
+ }
+ }
+
+ if (src0->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float) ||
+ src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) {
+ return false;
+ }
+
+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), src0_nrows);
+ const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0;
+
+ struct htp_rope_vtcm_layout layout;
+ htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads, n_freq_factors);
+ if (layout.total_bytes > htp_session_vtcm_size(sess)) {
+ return false;
+ }
+
+ kparams->n_threads = n_threads;
+ kparams->vtcm_size = (uint32_t) layout.total_bytes;
+ kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]);
+ kparams->div_ne1 = init_fastdiv_values(dst->ne[1]);
+
+ return true;
+}
+
+static bool htp_kparams_solve_sum_rows(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_sum_rows_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * dst = op;
+
+ if (src0->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (op->op == GGML_OP_ARGMAX) {
+ if (dst->type != GGML_TYPE_I32) {
+ return false;
+ }
+ } else if (dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {
+ return false;
+ }
+
+ const uint32_t ne00 = src0->ne[0];
+ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
+ const uint32_t max_threads_sess = htp_session_n_threads(sess);
+
+ if (op->op == GGML_OP_SUM) {
+ const uint32_t total_elems = (uint32_t) ggml_nelements(src0);
+ const uint32_t n_threads = (total_elems >= 1024) ? (std::min)(max_threads_sess, (uint32_t) HTP_MAX_NTHREADS) : 1;
+ const uint32_t raw_chunk = (total_elems + n_threads - 1) / n_threads;
+
+ kparams->n_threads = n_threads;
+ kparams->elems_per_thread = hex_round_up(raw_chunk, 32);
+ } else if (op->op == GGML_OP_ARGMAX && src0_nrows <= 1) {
+ const uint32_t n_threads = (ne00 >= 1024) ? (std::min)(max_threads_sess, (uint32_t) HTP_MAX_NTHREADS) : 1;
+ const uint32_t raw_chunk = (ne00 + n_threads - 1) / n_threads;
+
+ kparams->n_threads = n_threads;
+ kparams->elems_per_thread = hex_round_up(raw_chunk, 32);
+ } else {
+ const uint32_t min_rpt = (src0->nb[1] < 1024) ? 32 : 1;
+ const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rpt);
+
+ kparams->n_threads = (std::min)(max_threads_sess, max_threads);
+ }
+
+ return true;
+}
+
+bool htp_kparams_solve_unary_mul_glu(
+ const struct ggml_hexagon_session * sess,
+ uint32_t op,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * dst,
+ struct htp_glu_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ if (!htp_op_is_glu((htp_op_code) op)) {
+ return false;
+ }
+ if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || dst->type != src0->type) {
+ return false;
+ }
+ if ((op == HTP_OP_GLU_GEGLU_QUICK || op == HTP_OP_GLU_GEGLU_ERF) && src0->type != GGML_TYPE_F32) {
+ return false;
+ }
+
+ const size_t elem_size = ggml_type_size(src0->type);
+ const auto is_valid_src = [elem_size](const struct ggml_tensor * t) {
+ return ggml_is_contiguous_1(t) && t->nb[0] == elem_size &&
+ (t->ne[1] <= 1 || t->nb[1] >= (size_t) t->ne[0] * elem_size);
+ };
+
+ if (!is_valid_src(src0) || !ggml_is_contiguous(dst)) {
+ return false;
+ }
+ if (src1 && (src1->type != src0->type || !ggml_are_same_shape(src0, src1) || !is_valid_src(src1))) {
+ return false;
+ }
+
+ const size_t dst_row_size = dst->ne[0] * elem_size;
+ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
+ const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size));
+ const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread);
+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+ const size_t vtcm_size = htp_session_vtcm_size(sess);
+
+ const size_t row_size_aligned = hex_round_up(dst_row_size, 128);
+ const uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * 3 * row_size_aligned));
+ if (vtcm_row_per_thread < 2) {
+ return false;
+ }
+
+ struct htp_glu_vtcm_layout L;
+ htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, vtcm_row_per_thread);
+
+ kparams->n_threads = n_threads;
+ kparams->vtcm_row_per_thread = L.vtcm_row_per_thread;
+ kparams->vtcm_size = (uint32_t) L.total_bytes;
+
+ return true;
+}
+
+static bool htp_kparams_solve_glu(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_glu_kernel_params * kparams
+) {
+ return htp_kparams_solve_unary_mul_glu(sess, htp_op_remap(op), op->src[0], op->src[1], op, kparams);
+}
+
+static bool htp_kparams_solve_repeat(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_repeat_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * dst = op;
+
+ if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || src0->type != dst->type) {
+ return false;
+ }
+ if (!ggml_can_repeat(src0, dst) || ggml_is_transposed(src0) || ggml_is_transposed(dst)) {
+ return false;
+ }
+
+ const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3];
+ const size_t dst_row_size = dst->ne[0] * ggml_type_size(dst->type);
+ const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size));
+ const uint32_t max_threads = (std::max)(1u, total_dst_rows / min_rows_per_thread);
+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+
+ kparams->n_threads = n_threads;
+ kparams->nr0 = dst->ne[0] / src0->ne[0];
+ kparams->nr1 = dst->ne[1] / src0->ne[1];
+ kparams->nr2 = dst->ne[2] / src0->ne[2];
+ kparams->nr3 = dst->ne[3] / src0->ne[3];
+
+ kparams->div_ne1 = init_fastdiv_values(dst->ne[1]);
+ kparams->div_ne12 = init_fastdiv_values(dst->ne[1] * dst->ne[2]);
+ kparams->div_ne01 = init_fastdiv_values(src0->ne[1]);
+ kparams->div_ne02 = init_fastdiv_values(src0->ne[2]);
+ kparams->div_ne03 = init_fastdiv_values(src0->ne[3]);
+
+ return true;
+}
+
+static bool htp_kparams_solve_ssm_conv(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_ssm_conv_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * src1 = op->src[1];
+ const struct ggml_tensor * dst = op;
+
+ if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (src0->ne[3] != 1 || src1->ne[2] != 1 || src1->ne[3] != 1 || dst->ne[3] != 1) {
+ return false;
+ }
+
+ const int64_t d_conv_64 = src1->ne[0];
+ const int64_t d_inner_64 = src0->ne[1];
+ const int64_t n_t_64 = dst->ne[1];
+ const int64_t n_s_64 = dst->ne[2];
+
+ if (d_conv_64 <= 0 || d_conv_64 > 32 || d_inner_64 <= 0) {
+ return false;
+ }
+ if (src0->ne[0] != d_conv_64 - 1 + n_t_64 || src0->ne[1] != d_inner_64 || src0->ne[2] != n_s_64) {
+ return false;
+ }
+ if (src1->ne[0] != d_conv_64 || src1->ne[1] != d_inner_64) {
+ return false;
+ }
+ if (dst->ne[0] != d_inner_64 || dst->ne[1] != n_t_64 || dst->ne[2] != n_s_64) {
+ return false;
+ }
+ if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) {
+ return false;
+ }
+ if (src0->nb[1] != src0->ne[0] * sizeof(float) || src1->nb[1] != src1->ne[0] * sizeof(float)) {
+ return false;
+ }
+
+ const uint32_t d_conv = (uint32_t) d_conv_64;
+ const uint32_t d_inner = (uint32_t) d_inner_64;
+ const uint32_t n_t = (uint32_t) n_t_64;
+ const uint32_t n_s = (uint32_t) n_s_64;
+ const uint32_t ncs = (uint32_t) src0->ne[0];
+
+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), (d_inner + 31) / 32);
+ const size_t vtcm_size = htp_session_vtcm_size(sess);
+
+ kparams->n_threads = n_threads;
+ kparams->d_conv = d_conv;
+ kparams->d_inner = d_inner;
+ kparams->n_t = n_t;
+ kparams->n_s = n_s;
+
+ const uint32_t raw_rpt = (d_inner + n_threads - 1) / n_threads;
+ const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, 32);
+
+ if (n_t == 1) {
+ kparams->d_inner_tile = d_inner_per_thread;
+ } else {
+ const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128) + 128;
+ const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128);
+ const uint32_t vtcm_src1_per_thread = src1_raw_bytes + src1_T_bytes;
+
+ const size_t vtcm_budget = (vtcm_size > 0 ? vtcm_size / n_threads : (1024 * 1024));
+ const uint32_t src0_block_T = hex_round_up(ncs * 32 * sizeof(float), 128);
+ const size_t fixed_bytes = vtcm_src1_per_thread + src0_block_T;
+ const size_t avail_for_tiles = vtcm_budget > fixed_bytes ? vtcm_budget - fixed_bytes : (128 * 1024);
+
+ uint32_t target_max_tile = hex_round_up((d_inner_per_thread + 3) / 4, 32);
+ target_max_tile = (std::max)(target_max_tile, 32u);
+ target_max_tile = (std::min)(target_max_tile, 128u);
+
+ uint32_t d_inner_tile = (uint32_t) (avail_for_tiles / (2 * (ncs + n_t) * sizeof(float)));
+ d_inner_tile = (d_inner_tile / 32) * 32;
+ if (d_inner_tile == 0) {
+ d_inner_tile = 32;
+ }
+ if (d_inner_tile > target_max_tile) {
+ d_inner_tile = target_max_tile;
+ }
+ if (d_inner_tile > d_inner_per_thread) {
+ d_inner_tile = d_inner_per_thread;
+ }
+ kparams->d_inner_tile = d_inner_tile;
+ }
+
+ struct htp_ssm_conv_vtcm_layout L;
+ htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_thread, d_conv, n_t, kparams->d_inner_tile, n_threads);
+
+ kparams->vtcm_size = (uint32_t) L.total_bytes;
+ kparams->div_d_inner_tile = init_fastdiv_values(kparams->d_inner_tile);
+
+ return L.total_bytes <= vtcm_size;
+}
+
+static bool htp_kparams_solve_sort(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_sort_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * dst = op;
+ const bool is_top_k = (op->op == GGML_OP_TOP_K);
+
+ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_I32) {
+ return false;
+ }
+
+ const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];
+ const uint32_t ne00 = src0->ne[0];
+ const uint32_t k = dst->ne[0];
+ const uint32_t n_threads_max = htp_session_n_threads(sess) > 0 ? htp_session_n_threads(sess) : 4;
+ const size_t vtcm_budget = htp_session_vtcm_size(sess) > 0 ? htp_session_vtcm_size(sess) : (8 * 1024 * 1024);
+
+ int32_t order = GGML_SORT_ORDER_DESC;
+ if (!is_top_k) {
+ order = ((const int32_t *) op->op_params)[0];
+ }
+
+ struct htp_sort_vtcm_layout layout;
+ if (!htp_sort_solve_layout(&layout, ne00, total_rows, k, n_threads_max, vtcm_budget, is_top_k)) {
+ return false;
+ }
+
+ kparams->n_threads = layout.n_threads;
+ kparams->order = (uint32_t) order;
+ kparams->is_top_k = is_top_k ? 1 : 0;
+ kparams->chunk_elems = layout.chunk_elems;
+ kparams->n_chunks = layout.n_chunks;
+ kparams->vtcm_size = (uint32_t) layout.total_bytes;
+ kparams->merge_elems = layout.merge_elems;
+ kparams->n_slots = layout.n_slots;
+ kparams->div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);
+ kparams->div_ne01_ne02 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[2]));
+
+ return true;
+}
+
+static void htp_pool_interior_range(
+ uint32_t src_x, uint32_t dst_x, uint32_t kernel_x, uint32_t stride_x, int32_t pad_x,
+ uint32_t * ox_lo, uint32_t * ox_hi
+) {
+ const uint32_t lo_raw = ((uint32_t) pad_x + stride_x - 1) / stride_x;
+ *ox_lo = (lo_raw < dst_x) ? lo_raw : dst_x;
+
+ const int32_t numer_hi = (int32_t) src_x - (int32_t) kernel_x + pad_x;
+ if (numer_hi < 0) {
+ *ox_hi = 0;
+ } else {
+ const uint32_t hi_raw = (uint32_t) numer_hi / stride_x + 1;
+ *ox_hi = (hi_raw < dst_x) ? hi_raw : dst_x;
+ }
+ if (*ox_hi < *ox_lo) {
+ *ox_hi = *ox_lo;
+ }
+}
+
+static bool htp_kparams_solve_pool_2d(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_pool_2d_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * dst = op;
+ const bool is_pool_1d = (op->op == GGML_OP_POOL_1D);
+
+ const int32_t * params = dst->op_params;
+ if (params[0] != GGML_OP_POOL_AVG && params[0] != GGML_OP_POOL_MAX) {
+ return false;
+ }
+ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {
+ return false;
+ }
+ if (ggml_nbytes(src0) > UINT32_MAX - 256 || ggml_nbytes(dst) > UINT32_MAX - 256) {
+ return false;
+ }
+
+ const int32_t kernel_x = params[1];
+ const int32_t kernel_y = is_pool_1d ? 1 : params[2];
+ const int32_t stride_x = params[is_pool_1d ? 2 : 3];
+ const int32_t stride_y = is_pool_1d ? 1 : params[4];
+ const int32_t pad_x = params[is_pool_1d ? 3 : 5];
+ const int32_t pad_y = is_pool_1d ? 0 : params[6];
+
+ if (kernel_x <= 0 || kernel_y <= 0 || stride_x <= 0 || stride_y <= 0 ||
+ pad_x < 0 || pad_y < 0) {
+ return false;
+ }
+
+ const uint32_t src_x = (uint32_t) src0->ne[0];
+ const uint32_t src_y = is_pool_1d ? 1 : (uint32_t) src0->ne[1];
+ const uint32_t dst_x = (uint32_t) dst->ne[0];
+ const uint32_t dst_y = is_pool_1d ? 1 : (uint32_t) dst->ne[1];
+ const uint32_t dst_plane_bytes = dst_x * dst_y * sizeof(float);
+ const uint32_t planes = (uint32_t) (is_pool_1d
+ ? (uint64_t) src0->ne[1] * (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3]
+ : (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3]);
+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), planes > 0 ? planes : 1);
+ const uint32_t planes_per_chunk = (dst_plane_bytes > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_plane_bytes, HEX_L2_LINE_SIZE)) : 1;
+
+ struct htp_pool_vtcm_layout layout;
+ if (!htp_pool_solve_layout(&layout, src_x, src_y, dst_x, dst_y, n_threads, htp_session_vtcm_size(sess))) {
+ return false;
+ }
+
+ kparams->src_x = src_x;
+ kparams->src_y = src_y;
+ kparams->dst_x = dst_x;
+ kparams->dst_y = dst_y;
+ kparams->kernel_x = (uint32_t) kernel_x;
+ kparams->kernel_y = (uint32_t) kernel_y;
+ kparams->stride_x = (uint32_t) stride_x;
+ kparams->stride_y = (uint32_t) stride_y;
+ kparams->pad_x = pad_x;
+ kparams->pad_y = pad_y;
+ kparams->n_threads = n_threads;
+ kparams->planes = planes;
+ kparams->planes_per_chunk = planes_per_chunk;
+ kparams->pool_op = (uint32_t) params[0];
+
+ const bool exact_tiling = (kparams->pad_x == 0 && kparams->pad_y == 0 &&
+ kparams->stride_x == kparams->kernel_x && kparams->stride_y == kparams->kernel_y);
+ const bool fast_ok = exact_tiling && (kparams->kernel_x == 1 || kparams->kernel_x == 2);
+ const bool narrow_ok = fast_ok && ((uint64_t) kparams->dst_x * kparams->kernel_x <= 32);
+ const bool global_ok = exact_tiling &&
+ kparams->kernel_x == kparams->src_x && kparams->kernel_y == kparams->src_y &&
+ kparams->dst_x == 1 && kparams->dst_y == 1;
+ const bool block_ok = exact_tiling &&
+ kparams->kernel_y == 1 && kparams->stride_y == 1 &&
+ kparams->kernel_x >= 32;
+
+ if (global_ok) {
+ kparams->kernel_type = HTP_POOL_KERNEL_GLOBAL;
+ } else if (block_ok) {
+ kparams->kernel_type = HTP_POOL_KERNEL_BLOCK;
+ } else if (narrow_ok) {
+ kparams->kernel_type = HTP_POOL_KERNEL_HVX_NARROW;
+ } else if (fast_ok) {
+ kparams->kernel_type = HTP_POOL_KERNEL_HVX;
+ } else {
+ kparams->kernel_type = HTP_POOL_KERNEL_GENERAL;
+ }
+
+ kparams->avg_divide_count = (is_pool_1d && kparams->pool_op == GGML_OP_POOL_AVG) ? 1 : 0;
+ htp_pool_interior_range(kparams->src_x, kparams->dst_x, kparams->kernel_x,
+ kparams->stride_x, kparams->pad_x,
+ &kparams->ox_lo, &kparams->ox_hi);
+ kparams->inv_kernel_area = 1.0f / (float) (kparams->kernel_x * kparams->kernel_y);
+
+ return true;
+}
+
+static bool htp_kparams_solve_concat(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_concat_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+ kparams->kernel_type = HTP_CONCAT_KERNEL_UNSUPPORTED;
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * src1 = op->src[1];
+ const struct ggml_tensor * dst = op;
+
+ if (!src0 || !src1 || !dst) {
+ return false;
+ }
+
+ int dim = ((const int32_t *) op->op_params)[0];
+ if (dim < 0 || dim >= GGML_MAX_DIMS) {
+ return false;
+ }
+ kparams->dim = dim;
+
+ if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_I32) {
+ return false;
+ }
+ if (src0->type != dst->type || src1->type != dst->type) {
+ return false;
+ }
+
+ const uint32_t type_size = ggml_type_size(dst->type);
+ const uint32_t n_threads = htp_session_n_threads(sess);
+ const uint32_t mdev_count = htp_session_mdev_count(sess);
+ const size_t vtcm_size = htp_session_vtcm_size(sess);
+
+ for (int d = 0; d < GGML_MAX_DIMS; d++) {
+ const int64_t ne_d = (d == dim) ? src0->ne[d] + src1->ne[d] : src0->ne[d];
+ if (dst->ne[d] != ne_d || (d != dim && src1->ne[d] != dst->ne[d])) {
+ return false;
+ }
+ }
+
+ const uint32_t max_gather_ne0 = 128 / type_size;
+ const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);
+ const bool b_dense = (src1->nb[0] == type_size);
+ const bool gather_strides_ok =
+ src0->nb[1] >= (size_t) src0->ne[0] * type_size &&
+ (b_dense ? src1->nb[1] >= (size_t) src1->ne[0] * type_size
+ : src1->nb[0] >= (size_t) src1->ne[1] * type_size);
+
+ if (dim == 0 &&
+ (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16) &&
+ dst->ne[0] <= max_gather_ne0 &&
+ dst->nb[0] == type_size && dst->nb[1] == (size_t) dst->ne[0] * type_size &&
+ src0->nb[0] == type_size &&
+ (src1->nb[0] == type_size || src1->nb[1] == type_size) &&
+ gather_strides_ok &&
+ htp_gather_rows_tensor_outer_contiguous(dst) &&
+ htp_gather_rows_tensor_outer_contiguous(src0) &&
+ htp_gather_rows_tensor_outer_contiguous(src1)) {
+
+ if (htp_gather_rows_solve_layout(&kparams->u.gather,
+ (uint32_t) src0->ne[0], (uint32_t) src0->nb[1],
+ (uint32_t) src1->ne[0], (uint32_t) src1->nb[1], b_dense,
+ type_size, total_rows,
+ n_threads, mdev_count, vtcm_size)) {
+ kparams->kernel_type = HTP_CONCAT_KERNEL_GATHER_ROWS;
+ kparams->n_threads = (uint8_t) n_threads;
+ kparams->vtcm_size = kparams->u.gather.slice * n_threads;
+ return true;
+ }
+ }
+
+ const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size);
+ if (dma_strides_ok) {
+ kparams->kernel_type = HTP_CONCAT_KERNEL_REGULAR;
+ kparams->n_threads = 1;
+ return true;
+ }
+
+ const bool is_src1_transposed = (src1->nb[0] > src1->nb[1]);
+ const bool is_src0_transposed = (src0->nb[0] > src0->nb[1]);
+ const bool transposed_rows_ok = (src0->nb[0] == type_size && src1->nb[1] == type_size && dst->nb[0] == type_size);
+
+ if (dim == 0 && is_src1_transposed && !is_src0_transposed && transposed_rows_ok &&
+ (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16)) {
+
+ const uint32_t nth = n_threads > 0 ? n_threads : 8;
+ struct htp_concat_transposed_vtcm_layout layout;
+ htp_concat_transposed_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src1->ne[0], type_size, nth);
+
+ if (vtcm_size > 0 && layout.total_bytes > vtcm_size) {
+ return false;
+ }
+
+ kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED;
+ kparams->n_threads = nth;
+ kparams->vtcm_size = layout.total_bytes;
+ kparams->u.transposed.div_ne2 = init_fastdiv_values((uint32_t) dst->ne[2]);
+ return true;
+ }
+
+ return false;
+}
+
+static bool htp_kparams_solve_cpy(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_copy_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+ kparams->kernel_type = HTP_COPY_KERNEL_UNSUPPORTED;
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * dst = op;
+
+ if (!src0 || !dst) {
+ return false;
+ }
+
+ if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_I32) {
+ return false;
+ }
+ if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_I32) {
+ return false;
+ }
+
+ const int64_t nelem_src = ggml_nelements(src0);
+ const int64_t nelem_dst = ggml_nelements(dst);
+ if (nelem_src != nelem_dst || nelem_src < 0) {
+ return false;
+ }
+
+ const uint32_t src_type_size = ggml_type_size(src0->type);
+ const uint32_t dst_type_size = ggml_type_size(dst->type);
+
+ kparams->src0_type_size = (uint8_t) src_type_size;
+ kparams->dst_type_size = (uint8_t) dst_type_size;
+ kparams->total_elems = (uint32_t) nelem_src;
+
+ if (nelem_src == 0) {
+ kparams->kernel_type = HTP_COPY_KERNEL_1D_CONTIG;
+ return true;
+ }
+
+ if (nelem_src == 1) {
+ if (src0->type == dst->type) {
+ kparams->kernel_type = HTP_COPY_KERNEL_SCALAR;
+ return true;
+ }
+ if ((src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) ||
+ (src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32) ||
+ (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) ||
+ (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32)) {
+ kparams->kernel_type = HTP_COPY_KERNEL_SCALAR;
+ return true;
+ }
+ return false;
+ }
+
+ const bool sametype = (src0->type == dst->type);
+ const bool same_extents = ggml_are_same_shape(src0, dst);
+
+ const bool transposed = (src0->nb[0] > src0->nb[1]) || (dst->nb[0] > dst->nb[1]) ||
+ (src0->nb[0] != src_type_size) || (dst->nb[0] != dst_type_size) ||
+ (src0->nb[1] < (size_t) src0->ne[0] * src_type_size) || (dst->nb[1] < (size_t) dst->ne[0] * dst_type_size);
+ const bool sameshape = same_extents && !transposed;
+
+ const bool src_is_contiguous = ggml_is_contiguous(src0);
+ const bool dst_is_contiguous = ggml_is_contiguous(dst);
+
+ const uint32_t n_threads = htp_session_n_threads(sess);
+ const uint32_t mdev_count = htp_session_mdev_count(sess);
+ const size_t vtcm_size = htp_session_vtcm_size(sess);
+
+ if (sametype) {
+ if (src_is_contiguous && dst_is_contiguous) {
+ kparams->kernel_type = HTP_COPY_KERNEL_1D_CONTIG;
+ return true;
+ }
+
+ const uint32_t max_gather_ne0 = 128 / src_type_size;
+ const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
+ if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) &&
+ dst_is_contiguous && !src_is_contiguous &&
+ src0->nb[0] == src_type_size && src0->ne[0] <= max_gather_ne0 &&
+ src0->nb[1] >= (size_t) src0->ne[0] * src_type_size &&
+ total_rows > 1 &&
+ htp_gather_rows_tensor_outer_contiguous(src0)) {
+
+ if (htp_gather_rows_solve_layout(&kparams->u.gather,
+ (uint32_t) src0->ne[0], (uint32_t) src0->nb[1],
+ 0, 0, false,
+ src_type_size, total_rows,
+ n_threads, mdev_count, vtcm_size)) {
+ kparams->kernel_type = HTP_COPY_KERNEL_GATHER_ROWS;
+ kparams->n_threads = (uint8_t) n_threads;
+ kparams->vtcm_size = kparams->u.gather.slice * n_threads;
+ return true;
+ }
+ }
+
+ if (sameshape) {
+ kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE;
+ kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
+ return true;
+ }
+
+ kparams->kernel_type = HTP_COPY_KERNEL_RESHAPE;
+ kparams->n_threads = n_threads > 0 ? (uint8_t) n_threads : 4;
+ kparams->u.reshape.div_ne0 = init_fastdiv_values((uint32_t) dst->ne[0]);
+ kparams->u.reshape.div_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[1] * dst->ne[0]));
+ kparams->u.reshape.div_ne2_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[2] * dst->ne[1] * dst->ne[0]));
+ kparams->u.reshape.div_ne00 = init_fastdiv_values((uint32_t) src0->ne[0]);
+ kparams->u.reshape.div_ne01_ne00 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[0]));
+ kparams->u.reshape.div_ne02_ne01_ne00 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1] * src0->ne[0]));
+ return true;
+ }
+
+ if (!sameshape) {
+ return false;
+ }
+
+ const bool valid_conversion = (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) ||
+ (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) ||
+ (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) ||
+ (src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32);
+ if (!valid_conversion) {
+ return false;
+ }
+
+ const uint32_t nth = n_threads > 0 ? n_threads : 4;
+ struct htp_copy_convert_vtcm_layout layout;
+ htp_copy_convert_solve_layout(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1], src_type_size, dst_type_size, nth);
+
+ if (vtcm_size > 0 && layout.total_bytes > vtcm_size) {
+ return false;
+ }
+
+ kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_CONVERT;
+ kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
+ kparams->n_threads = (uint8_t) nth;
+ kparams->vtcm_size = (uint32_t) layout.total_bytes;
+ kparams->u.convert.blk_rows = layout.blk_rows;
+ kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);
+ kparams->u.convert.div_ne02_ne01 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1]));
+
+ return true;
+}
+
+static bool im2col_3d_calc_output_size(
+ uint32_t input, uint32_t kernel, int32_t stride, int32_t padding, int32_t dilation, uint32_t * output
+) {
+ if (stride <= 0 || padding < 0 || dilation <= 0 || kernel == 0) {
+ return false;
+ }
+
+ const int64_t numerator = (int64_t) input + 2LL * padding - (int64_t) dilation * (kernel - 1) - 1;
+ if (numerator < 0) {
+ return false;
+ }
+
+ const uint64_t size = (uint64_t) (numerator / stride) + 1;
+ if (size == 0 || size > UINT32_MAX) {
+ return false;
+ }
+
+ *output = (uint32_t) size;
+ return true;
+}
+
+static bool htp_kparams_solve_im2col(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_im2col_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+ kparams->kernel_type = HTP_IM2COL_KERNEL_UNSUPPORTED;
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * src1 = op->src[1];
+ const struct ggml_tensor * dst = op;
+
+ if (!src0 || !src1 || !dst) {
+ return false;
+ }
+ if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) {
+ return false;
+ }
+ if (src1->nb[0] != sizeof(float) || !ggml_is_contiguous(dst)) {
+ return false;
+ }
+
+ const int32_t * op_params = (const int32_t *) op->op_params;
+ const int32_t s0 = op_params[0];
+ const int32_t s1 = op_params[1];
+ const int32_t p0 = op_params[2];
+ const int32_t p1 = op_params[3];
+ const int32_t d0 = op_params[4];
+ const int32_t d1 = op_params[5];
+ const bool is_2D = (op_params[6] == 1);
+
+ if (s0 <= 0 || d0 <= 0 || p0 < 0) {
+ return false;
+ }
+ if (is_2D && (s1 <= 0 || d1 <= 0 || p1 < 0)) {
+ return false;
+ }
+
+ const uint32_t N = is_2D ? (uint32_t) src1->ne[3] : (uint32_t) src1->ne[2];
+ const uint32_t IC = is_2D ? (uint32_t) src1->ne[2] : (uint32_t) src1->ne[1];
+ const uint32_t IH = is_2D ? (uint32_t) src1->ne[1] : 1;
+ const uint32_t IW = (uint32_t) src1->ne[0];
+ const uint32_t KH = is_2D ? (uint32_t) src0->ne[1] : 1;
+ const uint32_t KW = (uint32_t) src0->ne[0];
+ const uint32_t OH = is_2D ? (uint32_t) dst->ne[2] : 1;
+ const uint32_t OW = (uint32_t) dst->ne[1];
+
+ if (KW == 0 || KH == 0 || IW == 0 || IH == 0 || IC == 0 || OW == 0 || OH == 0 || N == 0) {
+ return false;
+ }
+
+ const uint64_t patch_stride_64 = (uint64_t) IC * KH * KW;
+ const uint64_t total_rows_64 = (uint64_t) N * OH;
+ if (patch_stride_64 > UINT32_MAX || total_rows_64 > UINT32_MAX) {
+ return false;
+ }
+ const uint32_t patch_stride = (uint32_t) patch_stride_64;
+ const uint32_t total_rows = (uint32_t) total_rows_64;
+
+ if (dst->ne[0] != patch_stride || dst->ne[1] != OW || (is_2D && dst->ne[2] != OH)) {
+ return false;
+ }
+
+ const uint32_t dst_elem = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);
+
+ bool is_patchembed = ggml_is_contiguous(src1) && (s0 == (int32_t) KW && p0 == 0 && d0 == 1);
+ if (is_2D) {
+ is_patchembed = is_patchembed && (s1 == (int32_t) KH && p1 == 0 && d1 == 1);
+ }
+
+ const uint32_t max_threads = (sess && htp_session_n_threads(sess) > 0) ? htp_session_n_threads(sess) : 8;
+ const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads;
+ const uint32_t vtcm_size = (sess && htp_session_vtcm_size(sess) > 0) ? (uint32_t) htp_session_vtcm_size(sess) : 0;
+ const uint32_t row_size = (uint32_t) dst->nb[2];
+ const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;
+
+ if (is_patchembed) {
+ const uint64_t src_row_bytes_raw = (uint64_t) IC * KH * IW * sizeof(float);
+ const uint64_t dst_row_bytes_raw = (uint64_t) OW * patch_stride * dst_elem;
+ if (src_row_bytes_raw <= UINT32_MAX && dst_row_bytes_raw <= UINT32_MAX) {
+ const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);
+ const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
+
+ struct htp_im2col_vtcm_layout layout;
+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
+
+ if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
+ kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;
+ kparams->n_threads = (uint8_t) n_threads;
+ kparams->vtcm_size = (uint32_t) layout.total_bytes;
+ kparams->src_row_bytes = src_row_bytes;
+ kparams->dst_row_bytes = dst_row_bytes;
+ kparams->owb = OW;
+ kparams->wb = IW;
+ kparams->nbpr = 1;
+ kparams->N = N;
+ kparams->IC = IC;
+ kparams->ID = 1;
+ kparams->IH = IH;
+ kparams->IW = IW;
+ kparams->KD = 1;
+ kparams->KH = KH;
+ kparams->KW = KW;
+ kparams->OD = 1;
+ kparams->OH = OH;
+ kparams->OW = OW;
+ kparams->patch_stride = patch_stride;
+ kparams->total_rows = total_rows;
+ kparams->rows_per_chunk = rows_per_chunk;
+ kparams->div_OH = init_fastdiv_values(OH);
+ kparams->div_OD = init_fastdiv_values(1);
+ return true;
+ }
+ }
+ }
+
+ for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) {
+ const uint64_t Wb = (uint64_t) (owb - 1) * s0 + (uint64_t) (KW - 1) * d0 + 1;
+ if (Wb > UINT32_MAX) {
+ if (owb == 1) break;
+ continue;
+ }
+
+ const uint64_t src_row_bytes_raw = (uint64_t) IC * KH * Wb * sizeof(float);
+ const uint64_t dst_row_bytes_raw = (uint64_t) owb * patch_stride * dst_elem;
+ if (src_row_bytes_raw > UINT32_MAX || dst_row_bytes_raw > UINT32_MAX) {
+ if (owb == 1) break;
+ continue;
+ }
+
+ const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);
+ const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
+
+ struct htp_im2col_vtcm_layout layout;
+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
+
+ if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
+ kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA;
+ kparams->n_threads = (uint8_t) n_threads;
+ kparams->vtcm_size = (uint32_t) layout.total_bytes;
+ kparams->src_row_bytes = src_row_bytes;
+ kparams->dst_row_bytes = dst_row_bytes;
+ kparams->owb = owb;
+ kparams->wb = (uint32_t) Wb;
+ kparams->nbpr = (OW + owb - 1) / owb;
+ kparams->N = N;
+ kparams->IC = IC;
+ kparams->ID = 1;
+ kparams->IH = IH;
+ kparams->IW = IW;
+ kparams->KD = 1;
+ kparams->KH = KH;
+ kparams->KW = KW;
+ kparams->OD = 1;
+ kparams->OH = OH;
+ kparams->OW = OW;
+ kparams->patch_stride = patch_stride;
+ kparams->total_rows = total_rows;
+ kparams->rows_per_chunk = rows_per_chunk;
+ kparams->div_OH = init_fastdiv_values(OH);
+ kparams->div_OD = init_fastdiv_values(1);
+ return true;
+ }
+ if (owb == 1) break;
+ }
+
+ return false;
+}
+
+static bool htp_kparams_solve_im2col_3d(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_im2col_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+ kparams->kernel_type = HTP_IM2COL_KERNEL_UNSUPPORTED;
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * src1 = op->src[1];
+ const struct ggml_tensor * dst = op;
+
+ if (!src0 || !src1 || !dst) {
+ return false;
+ }
+ if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) {
+ return false;
+ }
+ if (src1->nb[0] != sizeof(float) || !ggml_is_contiguous(dst)) {
+ return false;
+ }
+
+ const int32_t * params = (const int32_t *) op->op_params;
+ const int32_t s0 = params[0], s1 = params[1], s2 = params[2];
+ const int32_t p0 = params[3], p1 = params[4], p2 = params[5];
+ const int32_t d0 = params[6], d1 = params[7], d2 = params[8];
+ const int32_t ic = params[9];
+
+ if (s0 <= 0 || s1 <= 0 || s2 <= 0 ||
+ p0 < 0 || p1 < 0 || p2 < 0 ||
+ d0 <= 0 || d1 <= 0 || d2 <= 0 || ic <= 0) {
+ return false;
+ }
+ if (src1->ne[3] % ic != 0) {
+ return false;
+ }
+
+ const uint32_t IC = (uint32_t) ic;
+ const uint32_t N = (uint32_t) (src1->ne[3] / IC);
+ if (N == 0) {
+ return false;
+ }
+
+ uint32_t OW, OH, OD;
+ if (!im2col_3d_calc_output_size(src1->ne[0], src0->ne[0], s0, p0, d0, &OW) ||
+ !im2col_3d_calc_output_size(src1->ne[1], src0->ne[1], s1, p1, d1, &OH) ||
+ !im2col_3d_calc_output_size(src1->ne[2], src0->ne[2], s2, p2, d2, &OD)) {
+ return false;
+ }
+
+ const uint32_t KD = (uint32_t) src0->ne[2];
+ const uint32_t KH = (uint32_t) src0->ne[1];
+ const uint32_t KW = (uint32_t) src0->ne[0];
+ const uint32_t ID = (uint32_t) src1->ne[2];
+ const uint32_t IH = (uint32_t) src1->ne[1];
+ const uint32_t IW = (uint32_t) src1->ne[0];
+
+ const uint64_t patch_stride_64 = (uint64_t) IC * KD * KH * KW;
+ const uint64_t total_rows_64 = (uint64_t) N * OD * OH;
+ const uint64_t dst_ne3_64 = (uint64_t) OD * N;
+
+ if (patch_stride_64 > UINT32_MAX || total_rows_64 > UINT32_MAX || dst_ne3_64 > UINT32_MAX) {
+ return false;
+ }
+ const uint32_t patch_stride = (uint32_t) patch_stride_64;
+ const uint32_t total_rows = (uint32_t) total_rows_64;
+ const uint32_t dst_ne3 = (uint32_t) dst_ne3_64;
+
+ if (dst->ne[0] != patch_stride || dst->ne[1] != OW || dst->ne[2] != OH || dst->ne[3] != dst_ne3) {
+ return false;
+ }
+
+ const uint32_t dst_elem = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);
+
+ const bool is_patchembed = ggml_is_contiguous(src1) &&
+ (s0 == (int32_t) KW && s1 == (int32_t) KH && s2 == (int32_t) KD &&
+ p0 == 0 && p1 == 0 && p2 == 0 &&
+ d0 == 1 && d1 == 1 && d2 == 1);
+
+ const uint32_t max_threads = (sess && htp_session_n_threads(sess) > 0) ? htp_session_n_threads(sess) : 8;
+ const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads;
+ const uint32_t vtcm_size = (sess && htp_session_vtcm_size(sess) > 0) ? (uint32_t) htp_session_vtcm_size(sess) : 0;
+ const uint32_t row_size = (uint32_t) dst->nb[2];
+ const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;
+
+ if (is_patchembed) {
+ const uint64_t src_row_bytes_raw = (uint64_t) IC * KD * KH * IW * sizeof(float);
+ const uint64_t dst_row_bytes_raw = (uint64_t) OW * patch_stride * dst_elem;
+ if (src_row_bytes_raw <= UINT32_MAX && dst_row_bytes_raw <= UINT32_MAX) {
+ const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);
+ const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
+
+ struct htp_im2col_vtcm_layout layout;
+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
+
+ if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
+ kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;
+ kparams->n_threads = (uint8_t) n_threads;
+ kparams->vtcm_size = (uint32_t) layout.total_bytes;
+ kparams->src_row_bytes = src_row_bytes;
+ kparams->dst_row_bytes = dst_row_bytes;
+ kparams->owb = OW;
+ kparams->wb = IW;
+ kparams->nbpr = 1;
+ kparams->N = N;
+ kparams->IC = IC;
+ kparams->ID = ID;
+ kparams->IH = IH;
+ kparams->IW = IW;
+ kparams->KD = KD;
+ kparams->KH = KH;
+ kparams->KW = KW;
+ kparams->OD = OD;
+ kparams->OH = OH;
+ kparams->OW = OW;
+ kparams->patch_stride = patch_stride;
+ kparams->total_rows = total_rows;
+ kparams->rows_per_chunk = rows_per_chunk;
+ kparams->div_OH = init_fastdiv_values(OH);
+ kparams->div_OD = init_fastdiv_values(OD);
+ return true;
+ }
+ }
+ }
+
+ for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) {
+ const uint64_t Wb = (uint64_t) (owb - 1) * s0 + (uint64_t) (KW - 1) * d0 + 1;
+ if (Wb > UINT32_MAX) {
+ if (owb == 1) break;
+ continue;
+ }
+
+ const uint64_t src_row_bytes_raw = (uint64_t) IC * KD * KH * Wb * sizeof(float);
+ const uint64_t dst_row_bytes_raw = (uint64_t) owb * patch_stride * dst_elem;
+ if (src_row_bytes_raw > UINT32_MAX || dst_row_bytes_raw > UINT32_MAX) {
+ if (owb == 1) break;
+ continue;
+ }
+
+ const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);
+ const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
+
+ struct htp_im2col_vtcm_layout layout;
+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
+
+ if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
+ kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA;
+ kparams->n_threads = (uint8_t) n_threads;
+ kparams->vtcm_size = (uint32_t) layout.total_bytes;
+ kparams->src_row_bytes = src_row_bytes;
+ kparams->dst_row_bytes = dst_row_bytes;
+ kparams->owb = owb;
+ kparams->wb = (uint32_t) Wb;
+ kparams->nbpr = (OW + owb - 1) / owb;
+ kparams->N = N;
+ kparams->IC = IC;
+ kparams->ID = ID;
+ kparams->IH = IH;
+ kparams->IW = IW;
+ kparams->KD = KD;
+ kparams->KH = KH;
+ kparams->KW = KW;
+ kparams->OD = OD;
+ kparams->OH = OH;
+ kparams->OW = OW;
+ kparams->patch_stride = patch_stride;
+ kparams->total_rows = total_rows;
+ kparams->rows_per_chunk = rows_per_chunk;
+ kparams->div_OH = init_fastdiv_values(OH);
+ kparams->div_OD = init_fastdiv_values(OD);
+ return true;
+ }
+ if (owb == 1) break;
+ }
+
+ return false;
+}
+
+static bool htp_kparams_solve_pad(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_pad_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * dst = op;
+
+ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+
+ const int32_t lp0 = ((const int32_t *) op->op_params)[0];
+ const int32_t rp0 = ((const int32_t *) op->op_params)[1];
+ const bool circular = (((const int32_t *) op->op_params)[8] != 0);
+
+ if (circular && (lp0 > src0->ne[0] || rp0 > src0->ne[0])) {
+ return false;
+ }
+
+ const uint32_t type_size = (uint32_t) ggml_type_size(src0->type);
+ const uint32_t total_dst_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);
+ const uint32_t max_threads = (std::max)(1u, total_dst_rows);
+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+
+ struct htp_pad_vtcm_layout layout;
+ htp_pad_vtcm_layout_build(&layout, (uint32_t) src0->ne[0] * type_size, (uint32_t) dst->ne[0] * type_size, n_threads);
+
+ const bool use_dma = (src0->nb[0] == type_size) && (src0->ne[0] >= 512) &&
+ (htp_session_vtcm_size(sess) >= layout.total_bytes);
+
+ kparams->n_threads = n_threads;
+ kparams->total_rows = total_dst_rows;
+ kparams->use_dma = use_dma ? 1 : 0;
+ kparams->elem_size = type_size;
+ kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0;
+
+ if (circular && use_dma) {
+ kparams->kernel_type = HTP_PAD_KERNEL_HVX_CIRCULAR_DMA;
+ } else if (circular) {
+ kparams->kernel_type = HTP_PAD_KERNEL_HVX_CIRCULAR;
+ } else if (use_dma) {
+ kparams->kernel_type = HTP_PAD_KERNEL_HVX_DMA;
+ } else {
+ kparams->kernel_type = HTP_PAD_KERNEL_HVX;
+ }
+
+ kparams->div_ne1 = init_fastdiv_values((uint32_t) dst->ne[1]);
+ kparams->div_ne1_ne2 = init_fastdiv_values((uint32_t) (dst->ne[1] * dst->ne[2]));
+ kparams->div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);
+ kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]);
+ kparams->div_ne03 = init_fastdiv_values((uint32_t) src0->ne[3]);
+
+ return true;
+}
+
+static bool htp_kparams_solve_cumsum(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_cumsum_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * dst = op;
+
+ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {
+ return false;
+ }
+
+ const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
+ const uint32_t max_threads = (std::max)(1u, total_rows);
+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+
+ struct htp_cumsum_vtcm_layout layout;
+ htp_cumsum_vtcm_layout_build(&layout, (uint32_t) src0->nb[1], (uint32_t) dst->nb[1], n_threads);
+
+ const bool use_dma = (htp_session_vtcm_size(sess) >= layout.total_bytes);
+
+ kparams->n_threads = n_threads;
+ kparams->total_rows = total_rows;
+ kparams->use_dma = use_dma ? 1 : 0;
+ kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0;
+
+ return true;
+}
+
+static bool htp_kparams_solve_fill(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_fill_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * dst = op;
+
+ if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) {
+ return false;
+ }
+ if (!ggml_is_contiguous_rows(dst) || ggml_is_permuted(dst)) {
+ return false;
+ }
+
+ const uint32_t ne1 = (uint32_t) dst->ne[1];
+ const uint32_t ne2 = (uint32_t) dst->ne[2];
+ const uint32_t ne3 = (uint32_t) dst->ne[3];
+ const uint32_t nb1 = (uint32_t) dst->nb[1];
+
+ const uint32_t total_rows = ne1 * ne2 * ne3;
+ const uint32_t max_threads = (std::max)(1u, total_rows);
+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+ const uint32_t elem_size = (uint32_t) ggml_type_size(dst->type);
+ const uint32_t rows_per_chunk = (nb1 > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(nb1, HEX_L2_LINE_SIZE)) : 1;
+
+ kparams->n_threads = n_threads;
+ kparams->total_rows = total_rows;
+ kparams->elem_size = elem_size;
+ kparams->kernel_type = ggml_is_contiguous(dst) ? HTP_FILL_KERNEL_CONTIGUOUS : HTP_FILL_KERNEL_STRIDED;
+ kparams->rows_per_chunk = rows_per_chunk;
+ kparams->div_ne1 = init_fastdiv_values(ne1);
+ kparams->div_ne1_ne2 = init_fastdiv_values(ne1 * ne2);
+
+ return true;
+}
+
+static bool htp_kparams_solve_roll(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_roll_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * dst = op;
+
+ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (!ggml_are_same_shape(src0, dst)) {
+ return false;
+ }
+ if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous(dst)) {
+ return false;
+ }
+
+ const uint32_t ne0 = (uint32_t) dst->ne[0];
+ const uint32_t ne1 = (uint32_t) dst->ne[1];
+ const uint32_t ne2 = (uint32_t) dst->ne[2];
+ const uint32_t ne3 = (uint32_t) dst->ne[3];
+
+ const int32_t s0 = ((const int32_t *) op->op_params)[0];
+ uint32_t i0_src0 = 0;
+ if (-s0 < 0) {
+ i0_src0 = (uint32_t) (-s0 + (int32_t) ne0);
+ } else if ((uint32_t) (-s0) >= ne0) {
+ i0_src0 = (uint32_t) (-s0) - ne0;
+ } else {
+ i0_src0 = (uint32_t) (-s0);
+ }
+
+ const uint32_t total_rows = ne1 * ne2 * ne3;
+ const uint32_t max_threads = (std::max)(1u, total_rows);
+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+
+ kparams->n_threads = n_threads;
+ kparams->total_rows = total_rows;
+ kparams->i0_src0 = i0_src0;
+ kparams->n0 = ne0 - i0_src0;
+
+ if (htp_session_mdev_count(sess) <= 1) {
+ if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) {
+ kparams->kernel_type = HTP_ROLL_KERNEL_DMA_CONTIGUOUS;
+ } else {
+ kparams->kernel_type = HTP_ROLL_KERNEL_DMA_STRIDED;
+ }
+ } else {
+ kparams->kernel_type = HTP_ROLL_KERNEL_HVX;
+ }
+
+ kparams->div_ne1 = init_fastdiv_values(ne1);
+ kparams->div_ne2_ne1 = init_fastdiv_values(ne2 * ne1);
+
+ return true;
+}
+
+static bool htp_kparams_solve_diag(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_diag_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * dst = op;
+
+ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (src0->ne[1] != 1 || dst->ne[0] != dst->ne[1] || dst->ne[0] != src0->ne[0]) {
+ return false;
+ }
+ if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous_rows(dst) ||
+ ggml_is_permuted(src0) || ggml_is_permuted(dst)) {
+ return false;
+ }
+
+ const uint32_t total_batches = (uint32_t) (src0->ne[2] * src0->ne[3]);
+ const uint32_t max_threads = (std::max)(1u, total_batches);
+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);
+ const uint32_t dst_batch_size = (uint32_t) (dst->ne[1] * dst->nb[1]);
+
+ bool can_split = (dst->ne[0] == 1 || dst->nb[0] == sizeof(float)) && !ggml_is_permuted(dst);
+ uint32_t batches_per_chunk = 1;
+ if (can_split) {
+ if (dst->ne[2] > 1 && (dst->nb[2] & (HEX_L2_LINE_SIZE - 1)) == 0 &&
+ (dst->ne[3] <= 1 || (dst->nb[3] & (HEX_L2_LINE_SIZE - 1)) == 0)) {
+ batches_per_chunk = 1;
+ } else if (dst->nb[2] == dst_batch_size &&
+ (dst->ne[3] <= 1 || dst->nb[3] == dst->nb[2] * dst->ne[2])) {
+ batches_per_chunk = (dst_batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_batch_size, HEX_L2_LINE_SIZE)) : 1;
+ } else {
+ can_split = false;
+ }
+ }
+
+ const uint32_t src_batch_size = (uint32_t) (src0->ne[0] * sizeof(float));
+ const uint32_t dst_row_size = (uint32_t) (dst->ne[0] * sizeof(float));
+
+ struct htp_diag_vtcm_layout layout;
+ htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_threads);
+
+ const bool use_dma = (htp_session_vtcm_size(sess) >= layout.total_bytes);
+
+ kparams->n_threads = n_threads;
+ kparams->total_batches = total_batches;
+ kparams->use_dma = use_dma ? 1 : 0;
+ kparams->can_split = can_split ? 1 : 0;
+ kparams->batches_per_chunk = batches_per_chunk;
+ kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0;
+ kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]);
+
+ return true;
+}
+
+static bool htp_kparams_solve_solve_tri(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_solve_tri_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * src1 = op->src[1];
+ const struct ggml_tensor * dst = op;
+
+ if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (src0->ne[0] != src0->ne[1] || src0->ne[1] != src1->ne[1] ||
+ src0->ne[2] != src1->ne[2] || src0->ne[3] != src1->ne[3] ||
+ !ggml_are_same_shape(src1, dst)) {
+ return false;
+ }
+
+ const uint32_t max_threads_sess = htp_session_n_threads(sess);
+ const uint32_t k = (uint32_t) src1->ne[0];
+ const uint32_t col_block = 32;
+ const uint32_t k_chunks = (k + col_block - 1) / col_block;
+ const uint32_t total_batches = (uint32_t) (src0->ne[2] * src0->ne[3]);
+ const bool batched = total_batches >= max_threads_sess;
+ const uint32_t total_jobs = batched ? total_batches : (total_batches * k_chunks);
+ const uint32_t max_threads = (std::max)(1u, total_jobs);
+ const uint32_t n_threads = (std::min)(max_threads_sess, max_threads);
+ const uint32_t batch_size = (uint32_t) dst->nb[2];
+ const uint32_t batches_per_chunk = (batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(batch_size, HEX_L2_LINE_SIZE)) : 1;
+
+ kparams->n_threads = n_threads;
+ kparams->total_batches = total_batches;
+ kparams->total_jobs = total_jobs;
+ kparams->k_chunks = k_chunks;
+ kparams->col_block = col_block;
+ kparams->batched = batched ? 1 : 0;
+ kparams->batches_per_chunk = batches_per_chunk;
+ kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]);
+ kparams->div_k_chunks = init_fastdiv_values(k_chunks > 0 ? k_chunks : 1);
+
+ return true;
+}
+
+template <auto SolveFn>
+static inline bool htp_kparams_invoke(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ void * kparams
+) {
+ using KParams = decltype(htp_kparams_type(SolveFn));
+ KParams tmp;
+ return SolveFn(sess, op, kparams ? (KParams *) kparams : &tmp);
+}
+
+bool htp_kparams_solve(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ void * kparams
+) {
+ const htp_op_code opcode = htp_op_remap(op);
+ switch (opcode) {
+ case HTP_OP_MUL_MAT:
+ case HTP_OP_MUL_MAT_ID:
+ return htp_kparams_invoke<htp_kparams_solve_matmul>(sess, op, kparams);
+
+ case HTP_OP_MUL:
+ case HTP_OP_ADD:
+ case HTP_OP_ADD_ID:
+ case HTP_OP_SUB:
+ case HTP_OP_DIV:
+ return htp_kparams_invoke<htp_kparams_solve_binary>(sess, op, kparams);
+
+ case HTP_OP_FLASH_ATTN_EXT:
+ return htp_kparams_invoke<htp_kparams_solve_flash_attn>(sess, op, kparams);
+
+ case HTP_OP_GET_ROWS:
+ return htp_kparams_invoke<htp_kparams_solve_get_rows>(sess, op, kparams);
+
+ case HTP_OP_SET_ROWS:
+ return htp_kparams_invoke<htp_kparams_solve_set_rows>(sess, op, kparams);
+
+ case HTP_OP_ROPE:
+ return htp_kparams_invoke<htp_kparams_solve_rope>(sess, op, kparams);
+
+ case HTP_OP_SSM_CONV:
+ return htp_kparams_invoke<htp_kparams_solve_ssm_conv>(sess, op, kparams);
+
+ case HTP_OP_SOFTMAX:
+ return htp_kparams_invoke<htp_kparams_solve_softmax>(sess, op, kparams);
+
+ case HTP_OP_GATED_DELTA_NET:
+ return htp_kparams_invoke<htp_kparams_solve_gated_delta_net>(sess, op, kparams);
+
+ case HTP_OP_ARGSORT:
+ case HTP_OP_TOP_K:
+ return htp_kparams_invoke<htp_kparams_solve_sort>(sess, op, kparams);
+
+ case HTP_OP_POOL_2D:
+ case HTP_OP_POOL_1D:
+ return htp_kparams_invoke<htp_kparams_solve_pool_2d>(sess, op, kparams);
+
+ case HTP_OP_CONCAT:
+ return htp_kparams_invoke<htp_kparams_solve_concat>(sess, op, kparams);
+
+ case HTP_OP_CPY:
+ case HTP_OP_CPY_FENCE:
+ return htp_kparams_invoke<htp_kparams_solve_cpy>(sess, op, kparams);
+
+ case HTP_OP_IM2COL:
+ return htp_kparams_invoke<htp_kparams_solve_im2col>(sess, op, kparams);
+
+ case HTP_OP_IM2COL_3D:
+ return htp_kparams_invoke<htp_kparams_solve_im2col_3d>(sess, op, kparams);
+
+ case HTP_OP_SUM_ROWS:
+ case HTP_OP_SUM:
+ case HTP_OP_ARGMAX:
+ return htp_kparams_invoke<htp_kparams_solve_sum_rows>(sess, op, kparams);
+
+ case HTP_OP_REPEAT:
+ return htp_kparams_invoke<htp_kparams_solve_repeat>(sess, op, kparams);
+
+ case HTP_OP_PAD:
+ return htp_kparams_invoke<htp_kparams_solve_pad>(sess, op, kparams);
+
+ case HTP_OP_CUMSUM:
+ return htp_kparams_invoke<htp_kparams_solve_cumsum>(sess, op, kparams);
+
+ case HTP_OP_FILL:
+ return htp_kparams_invoke<htp_kparams_solve_fill>(sess, op, kparams);
+
+ case HTP_OP_ROLL:
+ return htp_kparams_invoke<htp_kparams_solve_roll>(sess, op, kparams);
+
+ case HTP_OP_DIAG:
+ return htp_kparams_invoke<htp_kparams_solve_diag>(sess, op, kparams);
+
+ case HTP_OP_SOLVE_TRI:
+ return htp_kparams_invoke<htp_kparams_solve_solve_tri>(sess, op, kparams);
+
+ default:
+ if (htp_op_is_unary(opcode)) {
+ return htp_kparams_invoke<htp_kparams_solve_unary>(sess, op, kparams);
+ }
+ if (htp_op_is_glu(opcode)) {
+ return htp_kparams_invoke<htp_kparams_solve_glu>(sess, op, kparams);
+ }
+ return false;
+ }
+}
diff --git a/ggml/src/ggml-hexagon/htp-kparams.h b/ggml/src/ggml-hexagon/htp-kparams.h
new file mode 100644
index 000000000..0871209e7
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp-kparams.h
@@ -0,0 +1,123 @@
+#ifndef HTP_KPARAMS_H
+#define HTP_KPARAMS_H
+
+#include "htp-common.h"
+
+#include "htp/allreduce-ops.h"
+#include "htp/argsort-ops.h"
+#include "htp/binary-ops.h"
+#include "htp/concat-ops.h"
+#include "htp/cpy-ops.h"
+#include "htp/cumsum-ops.h"
+#include "htp/diag-ops.h"
+#include "htp/fill-ops.h"
+#include "htp/flash-attn-ops.h"
+#include "htp/gated-delta-net-ops.h"
+#include "htp/get-rows-ops.h"
+#include "htp/glu-ops.h"
+#include "htp/im2col-ops.h"
+#include "htp/matmul-ops.h"
+#include "htp/pad-ops.h"
+#include "htp/pool-ops.h"
+#include "htp/repeat-ops.h"
+#include "htp/roll-ops.h"
+#include "htp/rope-ops.h"
+#include "htp/set-rows-ops.h"
+#include "htp/softmax-ops.h"
+#include "htp/solve-tri-ops.h"
+#include "htp/ssm-conv.h"
+#include "htp/sum-rows-ops.h"
+#include "htp/unary-ops.h"
+
+#include <type_traits>
+#include <utility>
+
+bool htp_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride);
+bool htp_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst);
+ggml_tensor htp_tensor_collapse_rows(const struct ggml_tensor * t);
+
+template <typename... Args>
+auto htp_kparams_type(bool (*)(Args...)) -> std::remove_pointer_t<std::remove_reference_t<decltype((std::declval<Args>(), ...))>>;
+
+template <auto SolveFn, typename... Args>
+static inline bool htp_kparams_check(Args &&... args) {
+ decltype(htp_kparams_type(SolveFn)) kparams;
+ return SolveFn(std::forward<Args>(args)..., &kparams);
+}
+
+bool htp_kparams_solve(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ void * kparams = nullptr
+);
+
+static inline bool htp_kparams_check(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op
+) {
+ return htp_kparams_solve(sess, op, nullptr);
+}
+
+bool htp_kparams_solve_allreduce(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * dst,
+ uint32_t rank,
+ uint32_t n_ranks,
+ bool has_add,
+ bool is_row_bcast,
+ bool is_shard_ok,
+ struct htp_allreduce_kernel_params * kparams
+);
+
+bool htp_kparams_solve_rms_norm_mul(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * dst,
+ struct htp_unary_kernel_params * kparams
+);
+
+bool htp_kparams_solve_matmul_add(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * src2,
+ const struct ggml_tensor * dst,
+ struct htp_mm_kernel_params * kparams
+);
+
+bool htp_kparams_solve_mmnx(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * act,
+ int32_t n_weights,
+ struct htp_mm_kernel_params * kparams
+);
+
+bool htp_kparams_solve_mmidnx(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * act,
+ const struct ggml_tensor * dst,
+ int32_t n_weights,
+ struct htp_mm_kernel_params * kparams
+);
+
+bool htp_kparams_solve_repeat_get_rows(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * dst,
+ struct htp_get_rows_kernel_params * kparams
+);
+
+bool htp_kparams_solve_unary_mul_glu(
+ const struct ggml_hexagon_session * sess,
+ uint32_t op,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * dst,
+ struct htp_glu_kernel_params * kparams
+);
+
+#endif // HTP_KPARAMS_H
diff --git a/ggml/src/ggml-hexagon/htp-opnode.cpp b/ggml/src/ggml-hexagon/htp-opnode.cpp
new file mode 100644
index 000000000..3509ffeee
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp-opnode.cpp
@@ -0,0 +1,260 @@
+#include "htp-opnode.h"
+
+#include "htp/allreduce-ops.h"
+#include "htp/argsort-ops.h"
+#include "htp/binary-ops.h"
+#include "htp/cumsum-ops.h"
+#include "htp/diag-ops.h"
+#include "htp/fill-ops.h"
+#include "htp/flash-attn-ops.h"
+#include "htp/gated-delta-net-ops.h"
+#include "htp/get-rows-ops.h"
+#include "htp/glu-ops.h"
+#include "htp/matmul-ops.h"
+#include "htp/pad-ops.h"
+#include "htp/repeat-ops.h"
+#include "htp/roll-ops.h"
+#include "htp/softmax-ops.h"
+#include "htp/solve-tri-ops.h"
+#include "htp/ssm-conv.h"
+#include "htp/sum-rows-ops.h"
+#include "htp/unary-ops.h"
+
+#include <stdio.h>
+
+std::string htp_opnode::name() const {
+ switch (opcode) {
+ case HTP_OP_MDEV_GROUP: return "MDEV_GROUP";
+ case HTP_OP_CPY_FENCE: return "CPY+FENCE";
+ case HTP_OP_FENCE: return (node && node->op_params[1] == 0) ? "FENCE_WAIT" : "FENCE_SIGNAL";
+ case HTP_OP_ALLREDUCE: return "ALLREDUCE";
+ case HTP_OP_ALLREDUCE_ADD: return "ALLREDUCE+ADD";
+ case HTP_OP_HADAMARD: return "MUL_MAT_HADAMARD";
+ case HTP_OP_MUL_MAT_NX: return "MUL_MAT_NX";
+ case HTP_OP_MUL_MAT_ID_NX: return "MUL_MAT_ID_NX";
+ case HTP_OP_ADD:
+ case HTP_OP_SUB:
+ case HTP_OP_MUL:
+ case HTP_OP_DIV:
+ if (!fused.empty()) {
+ const auto * kp = kparams<htp_binary_kernel_params>();
+ if (kp->kernel_type != HTP_BINARY_KERNEL_SCALAR) {
+ switch (opcode) {
+ case HTP_OP_ADD: return "ADD_NX";
+ case HTP_OP_SUB: return "SUB_NX";
+ case HTP_OP_MUL: return "MUL_NX";
+ case HTP_OP_DIV: return "DIV_NX";
+ default: break;
+ }
+ }
+ }
+ break;
+ default:
+ break;
+ }
+
+ if (!node) {
+ return "";
+ }
+ if (htp_is_hadamard(node)) {
+ return "MUL_MAT_HADAMARD";
+ }
+
+ std::string res = ggml_op_desc(node);
+ for (const auto * t : fused) {
+ res += "+";
+ res += ggml_op_desc(t);
+ }
+ return res;
+}
+
+static const char * format_tensor_name(const struct ggml_tensor * t) {
+ return t ? t->name : "NONE";
+}
+
+static std::string format_tensor_dims(const struct ggml_tensor * t) {
+ if (!t) {
+ return "NONE";
+ }
+ char buf[64];
+ if (t->ne[2] == 1 && t->ne[3] == 1) {
+ snprintf(buf, sizeof(buf), "%d:%d", (int) t->ne[0], (int) t->ne[1]);
+ } else {
+ snprintf(buf, sizeof(buf), "%d:%d:%d:%d", (int) t->ne[0], (int) t->ne[1], (int) t->ne[2], (int) t->ne[3]);
+ }
+ return buf;
+}
+
+static const char * format_tensor_type(const struct ggml_tensor * t) {
+ return t ? ggml_type_name(t->type) : "NONE";
+}
+
+static std::string format_tensor_strides(const struct ggml_tensor * t) {
+ if (!t) {
+ return "NONE";
+ }
+ char buf[64];
+ const char * c = ggml_is_contiguous(t) ? "" : "!";
+ if (t->ne[2] == 1 && t->ne[3] == 1) {
+ snprintf(buf, sizeof(buf), "%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], c);
+ } else {
+ snprintf(buf, sizeof(buf), "%zu:%zu:%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], (size_t) t->nb[2], (size_t) t->nb[3], c);
+ }
+ return buf;
+}
+
+static const char * format_tensor_buff(const struct ggml_tensor * t) {
+ if (t && t->buffer) {
+ return ggml_backend_buffer_name(t->buffer);
+ }
+ return "NONE";
+}
+
+template <typename Fn>
+static std::string format_op_tensors(const htp_opnode & node, Fn fn) {
+ std::string res;
+ const auto & inputs = node.get_inputs();
+ const auto & outputs = node.get_outputs();
+
+ if (!inputs.empty()) {
+ for (size_t i = 0; i < inputs.size(); i++) {
+ if (i > 0) res += " x ";
+ res += fn(inputs[i]);
+ }
+ res += " -> ";
+ }
+
+ if (outputs.empty()) {
+ res += fn(node.dst());
+ } else {
+ for (size_t i = 0; i < outputs.size(); i++) {
+ if (i > 0) res += " x ";
+ res += fn(outputs[i]);
+ }
+ }
+ return res;
+}
+
+static std::string format_kernel_params(const htp_opnode & node) {
+ char str[128];
+ const size_t max_size = sizeof(str);
+
+ if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID ||
+ node.opcode == HTP_OP_MUL_MAT_NX || node.opcode == HTP_OP_MUL_MAT_ID_NX ||
+ node.opcode == HTP_OP_MUL_MAT_ADD) {
+ const auto * kparams = node.kparams<htp_mm_kernel_params>();
+ const char * path = "unknown";
+ int32_t type = kparams->kernel_type;
+ if (type == HTP_MM_KERNEL_HMX_2D || type == HTP_MM_KERNEL_HMX_BATCHED) {
+ path = (kparams->tile_size > 0) ? "hmx-tiled" : "hmx-flat";
+ } else if (type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || type == HTP_MM_KERNEL_HVX_F32_F32_VTCM ||
+ type == HTP_MM_KERNEL_HVX_BF16_F32_VTCM) {
+ path = "hvx-flat";
+ } else if (type == HTP_MM_KERNEL_HVX_QUANT_ROW || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {
+ path = "hvx-tiled";
+ }
+ snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);
+ } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) {
+ const auto * kparams = node.kparams<htp_fa_kernel_params>();
+ const char * path = "unknown";
+ int32_t type = kparams->kernel_type;
+ if (type == HTP_FA_KERNEL_HMX) {
+ path = kparams->u.hmx.pipeline ? "hmx-pipe" : "hmx-seq";
+ } else if (type == HTP_FA_KERNEL_HVX) {
+ path = "hvx";
+ }
+ snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);
+ } else if (htp_op_is_unary(node.opcode)) {
+ const auto * kparams = node.kparams<htp_unary_kernel_params>();
+ snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size);
+ } else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) {
+ snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]);
+ } else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) {
+ snprintf(str, max_size, "seq 0x%x", (uint32_t) node.node->op_params[0]);
+ } else if (node.opcode == HTP_OP_ALLREDUCE && node.node) {
+ snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]);
+ } else if (node.opcode == HTP_OP_SSM_CONV) {
+ const auto * kparams = node.kparams<htp_ssm_conv_kernel_params>();
+ snprintf(str, max_size, "%s vtcm %d", kparams->n_t == 1 ? "decode" : "prefill", (int) kparams->vtcm_size);
+ } else if (node.opcode == HTP_OP_SOFTMAX) {
+ const auto * kparams = node.kparams<htp_softmax_kernel_params>();
+ snprintf(str, max_size, "k%d nth %d vtcm %d", (int) kparams->kernel_type, (int) kparams->n_threads, (int) kparams->vtcm_size);
+ } else if (node.opcode == HTP_OP_GATED_DELTA_NET) {
+ const auto * kparams = node.kparams<htp_gdn_kernel_params>();
+ const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent";
+ snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar",
+ (unsigned int) kparams->vtcm_size);
+ } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID ||
+ node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {
+ const auto * kparams = node.kparams<htp_binary_kernel_params>();
+ snprintf(str, max_size, "vtcm %u", (unsigned int) kparams->vtcm_size);
+ } else if (node.opcode == HTP_OP_ARGSORT || node.opcode == HTP_OP_TOP_K) {
+ const auto * kparams = node.kparams<htp_sort_kernel_params>();
+ snprintf(str, max_size, "%s nth %d nchk %d chk %d vtcm %d",
+ node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort",
+ (int) kparams->n_threads, (int) kparams->n_chunks,
+ (int) kparams->chunk_elems, (int) kparams->vtcm_size);
+ } else if (node.opcode == HTP_OP_GET_ROWS) {
+ const auto * kparams = node.kparams<htp_get_rows_kernel_params>();
+ const char * ktype_str = "unknown";
+ switch (kparams->kernel_type) {
+ case HTP_GET_ROWS_KERNEL_SAMETYPE: ktype_str = "sametype"; break;
+ case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break;
+ case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break;
+ }
+ snprintf(str, max_size, "%s%s vtcm %u", ktype_str, kparams->n_threads > 1 ? "-multi" : "", (unsigned int) kparams->vtcm_size);
+ } else if (htp_op_is_glu(node.opcode)) {
+ const auto * kparams = node.kparams<htp_glu_kernel_params>();
+ snprintf(str, max_size, "nth %d vtcm %u", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
+ } else if (node.opcode == HTP_OP_PAD) {
+ const auto * kparams = node.kparams<htp_pad_kernel_params>();
+ snprintf(str, max_size, "k%d nth %d vtcm %u", (int) kparams->kernel_type, (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
+ } else if (node.opcode == HTP_OP_CUMSUM) {
+ const auto * kparams = node.kparams<htp_cumsum_kernel_params>();
+ snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
+ } else if (node.opcode == HTP_OP_FILL) {
+ const auto * kparams = node.kparams<htp_fill_kernel_params>();
+ snprintf(str, max_size, "%s nth %d", kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS ? "contig" : "strided", (int) kparams->n_threads);
+ } else if (node.opcode == HTP_OP_ROLL) {
+ const auto * kparams = node.kparams<htp_roll_kernel_params>();
+ snprintf(str, max_size, "k%d nth %d", (int) kparams->kernel_type, (int) kparams->n_threads);
+ } else if (node.opcode == HTP_OP_DIAG) {
+ const auto * kparams = node.kparams<htp_diag_kernel_params>();
+ snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
+ } else if (node.opcode == HTP_OP_SOLVE_TRI) {
+ const auto * kparams = node.kparams<htp_solve_tri_kernel_params>();
+ snprintf(str, max_size, "%s nth %d", kparams->batched ? "batched" : "chunked", (int) kparams->n_threads);
+ } else {
+ snprintf(str, max_size, "----");
+ }
+ return str;
+}
+
+static void append_section(std::string & res, const std::string & sec) {
+ if (!res.empty()) {
+ res += "|";
+ }
+ res += sec;
+}
+
+std::string htp_format_tensor(const ggml_tensor * t, uint32_t flags) {
+ std::string res;
+ if (flags & HTP_OPFMT_NAMES) append_section(res, format_tensor_name(t));
+ if (flags & HTP_OPFMT_DIMS) append_section(res, format_tensor_dims(t));
+ if (flags & HTP_OPFMT_TYPES) append_section(res, format_tensor_type(t));
+ if (flags & HTP_OPFMT_STRIDES) append_section(res, format_tensor_strides(t));
+ if (flags & HTP_OPFMT_BUFFS) append_section(res, format_tensor_buff(t));
+ return res;
+}
+
+std::string htp_format_op(const htp_opnode & node, uint32_t flags) {
+ std::string res;
+ if (flags & HTP_OPFMT_OP) append_section(res, node.name());
+ if (flags & HTP_OPFMT_NAMES) append_section(res, format_op_tensors(node, format_tensor_name));
+ if (flags & HTP_OPFMT_DIMS) append_section(res, format_op_tensors(node, format_tensor_dims));
+ if (flags & HTP_OPFMT_TYPES) append_section(res, format_op_tensors(node, format_tensor_type));
+ if (flags & HTP_OPFMT_STRIDES) append_section(res, format_op_tensors(node, format_tensor_strides));
+ if (flags & HTP_OPFMT_BUFFS) append_section(res, format_op_tensors(node, format_tensor_buff));
+ if (flags & HTP_OPFMT_KPARAMS) append_section(res, format_kernel_params(node));
+ return res;
+}
diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h
index 57cd5f208..a30c5b600 100644
--- a/ggml/src/ggml-hexagon/htp-opnode.h
+++ b/ggml/src/ggml-hexagon/htp-opnode.h
@@ -1,38 +1,74 @@
#ifndef HTP_OPNODE_H
#define HTP_OPNODE_H
-#define GGML_COMMON_IMPL_CPP
#include "ggml-backend-impl.h"
-#include "ggml-common.h"
+#include "ggml-impl.h"
+#include "htp/htp-ops.h"
-#include <algorithm>
+#include <cstring>
+#include <initializer_list>
+#include <memory>
#include <string>
#include <vector>
-#include <memory>
-#include <stdio.h>
-#include "htp-ops.h"
-#include "htp/matmul-ops.h"
-#include "htp/flash-attn-ops.h"
-#include "htp/unary-ops.h"
-#include "htp/binary-ops.h"
-#include "htp/allreduce-ops.h"
-#include "htp/ssm-conv.h"
-#include "htp/gated-delta-net-ops.h"
-#include "htp/softmax-ops.h"
-#include "htp/argsort-ops.h"
-#include "htp/get-rows-ops.h"
+
+#define HTP_OP_MAX_FUSED 16
+
+// local std::inplace_vector replacement
+template <typename T, size_t MAX_N>
+struct htp_opvec {
+ T data[MAX_N] {};
+ size_t n = 0;
+
+ htp_opvec() = default;
+ htp_opvec(std::initializer_list<T> init) {
+ GGML_ASSERT(init.size() <= MAX_N);
+ for (const auto & v : init) {
+ data[n++] = v;
+ }
+ }
+
+ size_t size() const { return n; }
+ bool empty() const { return n == 0; }
+ void clear() { n = 0; }
+ T * begin() { return data; }
+ T * end() { return data + n; }
+ const T * begin() const { return data; }
+ const T * end() const { return data + n; }
+ T & back() { return data[n - 1]; }
+ const T & back() const { return data[n - 1]; }
+ T & operator[](size_t i) { return data[i]; }
+ const T & operator[](size_t i) const { return data[i]; }
+
+ void push_back(T val) {
+ GGML_ASSERT(n < MAX_N);
+ data[n++] = val;
+ }
+
+ void insert(T * pos, T val) {
+ GGML_ASSERT(n < MAX_N);
+ for (T * p = data + n; p > pos; --p) {
+ *p = *(p - 1);
+ }
+ *pos = val;
+ n++;
+ }
+};
+
+static inline bool htp_is_hadamard(const struct ggml_tensor * t) {
+ return t && t->op == GGML_OP_MUL_MAT && ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD;
+}
struct htp_opnode {
- ggml_tensor * node { nullptr };
- htp_op_code opcode { HTP_OP_INVALID };
- int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS] {0};
+ const ggml_tensor * node { nullptr };
+ htp_op_code opcode { HTP_OP_INVALID };
+ int32_t params[HTP_OP_MAX_PARAMS] {0};
+ int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS] {0};
- std::vector<ggml_tensor *> fused;
- std::vector<std::shared_ptr<ggml_tensor>> dummy;
+ std::vector<std::shared_ptr<ggml_tensor>> dummy;
- std::vector<const ggml_tensor *> inputs;
- std::vector<const ggml_tensor *> outputs;
- std::string name;
+ htp_opvec<const ggml_tensor *, HTP_OP_MAX_FUSED> fused;
+ htp_opvec<const ggml_tensor *, HTP_OP_MAX_INPUTS> inputs;
+ htp_opvec<const ggml_tensor *, HTP_OP_MAX_OUTPUTS> outputs;
int n_active_src(const ggml_tensor * t) const {
if (!t) return 0;
@@ -44,34 +80,36 @@ struct htp_opnode {
return 0;
}
- void init(ggml_tensor * node) {
+ void init(const ggml_tensor * node) {
+ static_assert(sizeof(this->params) == sizeof(this->node->op_params), "params size mismatch");
+
this->node = node;
- if (this->node) {
- if (this->node->op == GGML_OP_MUL_MAT &&
- ggml_get_op_params_i32(this->node, 1) == GGML_HINT_SRC0_IS_HADAMARD) {
- this->name = "MUL_MAT_HADAMARD";
- this->inputs.push_back(this->node->src[1]);
- } else {
- this->name = ggml_op_desc(this->node);
-
- // Build inputs (preserving optional nullptrs)
- int n_inputs = n_active_src(this->node);
- this->inputs.resize(n_inputs, nullptr);
- for (int i = 0; i < n_inputs; i++) {
- this->inputs[i] = this->node->src[i];
- }
- }
+ if (!node) return;
+
+ memcpy(this->params, this->node->op_params, sizeof(this->params));
- // Build outputs
- this->outputs.push_back(this->dst());
+ // Build inputs (preserving optional nullptrs)
+ if (htp_is_hadamard(this->node)) {
+ this->inputs.push_back(this->node->src[1]);
+ } else {
+ int n_inputs = n_active_src(this->node);
+ for (int i = 0; i < n_inputs; i++) {
+ this->inputs.push_back(this->node->src[i]);
+ }
}
+
+ // Build outputs
+ this->outputs.push_back(this->dst());
}
- htp_opnode(htp_op_code opcode = HTP_OP_INVALID, ggml_tensor * node = nullptr) : opcode(opcode) {
+ htp_opnode(htp_op_code opcode = HTP_OP_INVALID, const ggml_tensor * node = nullptr) : opcode(opcode) {
+ init(node);
+ }
+
+ htp_opnode(const ggml_tensor * node) : opcode(HTP_OP_INVALID) {
init(node);
}
- ggml_op op() const { return node->op; }
const ggml_tensor * src0() const { return node->src[0]; }
const ggml_tensor * src1() const { return node->src[1]; }
const ggml_tensor * dst() const { return outputs.empty() ? node : outputs.back(); }
@@ -81,343 +119,54 @@ struct htp_opnode {
return dummy.back().get();
}
- void add_fused(ggml_tensor * t, bool extra_dst = false) {
- fused.push_back(t);
-
- name += "+";
- name += ggml_op_desc(t);
-
- if (extra_dst) {
- outputs.push_back(t);
- } else {
- outputs.clear();
- outputs.push_back(t);
- }
-
- // Remove the newly fused intermediate output tensor t from inputs (if it was there)
- inputs.erase(std::remove(inputs.begin(), inputs.end(), t), inputs.end());
-
- // Append new inputs from t, preserving middle nullptrs
- int n_inputs = n_active_src(t);
- for (int i = 0; i < n_inputs; i++) {
- const auto * src = t->src[i];
- if (!src) {
- inputs.push_back(nullptr);
- } else if (src != node &&
- std::find(fused.begin(), fused.end(), src) == fused.end() &&
- std::find(inputs.begin(), inputs.end(), src) == inputs.end()) {
- inputs.push_back(src);
- }
- }
- }
-
- const std::vector<const ggml_tensor *> & get_inputs() const {
+ const htp_opvec<const ggml_tensor *, HTP_OP_MAX_INPUTS> & get_inputs() const {
return inputs;
}
- const std::vector<const ggml_tensor *> & get_outputs() const {
+ const htp_opvec<const ggml_tensor *, HTP_OP_MAX_OUTPUTS> & get_outputs() const {
return outputs;
}
- std::string op_name() const {
- return name;
+ template <typename T>
+ T * kparams() {
+ static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS");
+ static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t");
+ return reinterpret_cast<T *>(kernel_params);
}
- bool is_empty() const {
- return ggml_op_is_empty(node->op);
+ template <typename T>
+ const T * kparams() const {
+ static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS");
+ static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t");
+ return reinterpret_cast<const T *>(kernel_params);
}
- bool stackable() const {
- switch (this->op()) {
- case GGML_OP_MUL_MAT:
- if (ggml_get_op_params_i32(this->node, 1) == GGML_HINT_SRC0_IS_HADAMARD) {
- return false;
- }
- return ggml_is_quantized(this->src0()->type);
- case GGML_OP_MUL_MAT_ID:
- return ggml_is_quantized(this->src0()->type);
- default:
- return false;
- }
+ template <typename T>
+ void set_kparams(const T & kp) {
+ static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS");
+ static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t");
+ memcpy(kernel_params, &kp, sizeof(T));
}
- bool same_input(const htp_opnode& n) const {
- return n.src1() == this->src1();
- }
+ std::string name() const;
};
-struct htp_opformat {
- char strides[64 * GGML_MAX_SRC];
- char dims[64 * GGML_MAX_SRC];
- char types[16 * GGML_MAX_SRC];
- char buffs[64 * GGML_MAX_SRC];
- char names[64 * GGML_MAX_SRC];
- char kparams[128];
-
- int format_tensor_dims(char * str, size_t max_size, const struct ggml_tensor * t) {
- if (!t) {
- return snprintf(str, max_size, "NONE");
- }
- if (t->ne[2] == 1 && t->ne[3] == 1) {
- return snprintf(str, max_size, "%d:%d", (int) t->ne[0], (int) t->ne[1]);
- } else {
- return snprintf(str, max_size, "%d:%d:%d:%d", (int) t->ne[0], (int) t->ne[1], (int) t->ne[2], (int) t->ne[3]);
- }
- }
-
- void format_op_dims(char * str, size_t max_size, const htp_opnode & node) {
- char * p = str;
- char * p_end = str + max_size;
- auto inputs = node.get_inputs();
-
- if (!inputs.empty()) {
- p += std::min((size_t)format_tensor_dims(p, p_end - p, inputs[0]), (size_t)(p_end - p));
-
- for (size_t i = 1; i < inputs.size(); i++) {
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));
- }
- if (p < p_end) {
- p += std::min((size_t)format_tensor_dims(p, p_end - p, inputs[i]), (size_t)(p_end - p));
- }
- }
-
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));
- }
- }
-
- char self[64];
- format_tensor_dims(self, sizeof(self), node.dst());
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, "%s", self), (size_t)(p_end - p));
- }
- }
-
- int format_tensor_strides(char * str, size_t max_size, const struct ggml_tensor * t) {
- if (!t) {
- return snprintf(str, max_size, "NONE");
- }
- const char * c = ggml_is_contiguous(t) ? "" : "!";
-
- if (t->ne[2] == 1 && t->ne[3] == 1) {
- return snprintf(str, max_size, "%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], c);
- } else {
- return snprintf(str, max_size, "%zu:%zu:%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], (size_t) t->nb[2], (size_t) t->nb[3], c);
- }
- }
-
- void format_op_strides(char * str, size_t max_size, const htp_opnode & node) {
- char * p = str;
- char * p_end = str + max_size;
- auto inputs = node.get_inputs();
-
- if (!inputs.empty()) {
- p += std::min((size_t)format_tensor_strides(p, p_end - p, inputs[0]), (size_t)(p_end - p));
-
- for (size_t i = 1; i < inputs.size(); i++) {
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));
- }
- if (p < p_end) {
- p += std::min((size_t)format_tensor_strides(p, p_end - p, inputs[i]), (size_t)(p_end - p));
- }
- }
-
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));
- }
- }
-
- char self[64];
- format_tensor_strides(self, sizeof(self), node.dst());
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, "%s", self), (size_t)(p_end - p));
- }
- }
-
- void format_op_types(char * str, size_t max_size, const htp_opnode & node) {
- char * p = str;
- char * p_end = str + max_size;
- auto inputs = node.get_inputs();
-
- if (!inputs.empty()) {
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[0] ? ggml_type_name(inputs[0]->type) : "NONE"), (size_t)(p_end - p));
- }
-
- for (size_t i = 1; i < inputs.size(); i++) {
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));
- }
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[i] ? ggml_type_name(inputs[i]->type) : "NONE"), (size_t)(p_end - p));
- }
- }
-
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));
- }
- }
-
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, "%s", ggml_type_name(node.dst()->type)), (size_t)(p_end - p));
- }
- }
-
- const char * tensor_buff_name(const struct ggml_tensor * t) {
- if (t && t->buffer) {
- return ggml_backend_buffer_name(t->buffer);
- }
- return "NONE";
- }
-
- void format_op_buffs(char * str, size_t max_size, const htp_opnode & node) {
- char * p = str;
- char * p_end = str + max_size;
- auto inputs = node.get_inputs();
-
- if (!inputs.empty()) {
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, "%s", tensor_buff_name(inputs[0])), (size_t)(p_end - p));
- }
-
- for (size_t i = 1; i < inputs.size(); i++) {
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));
- }
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, "%s", tensor_buff_name(inputs[i])), (size_t)(p_end - p));
- }
- }
-
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));
- }
- }
-
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, "%s", tensor_buff_name(node.dst())), (size_t)(p_end - p));
- }
- }
-
- void format_op_names(char * str, size_t max_size, const htp_opnode & node) {
- char * p = str;
- char * p_end = str + max_size;
- auto inputs = node.get_inputs();
-
- if (!inputs.empty()) {
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[0] ? inputs[0]->name : "NONE"), (size_t)(p_end - p));
- }
-
- for (size_t i = 1; i < inputs.size(); i++) {
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));
- }
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[i] ? inputs[i]->name : "NONE"), (size_t)(p_end - p));
- }
- }
-
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));
- }
- }
-
- if (p < p_end) {
- p += std::min((size_t)snprintf(p, p_end - p, "%s", node.dst()->name), (size_t)(p_end - p));
- }
- }
- void format_kernel_params(char * str, size_t max_size, const htp_opnode & node) {
- if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID ||
- node.opcode == HTP_OP_MUL_MAT_NX || node.opcode == HTP_OP_MUL_MAT_ID_NX ||
- node.opcode == HTP_OP_MUL_MAT_ADD) {
- const auto * kparams = (const struct htp_mm_kernel_params *) node.kernel_params;
- const char * path = "unknown";
- int32_t type = kparams->kernel_type;
- if (type == HTP_MM_KERNEL_HMX_2D || type == HTP_MM_KERNEL_HMX_BATCHED) {
- path = (kparams->tile_size > 0) ? "hmx-tiled" : "hmx-flat";
- } else if (type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || type == HTP_MM_KERNEL_HVX_F32_F32_VTCM ||
- type == HTP_MM_KERNEL_HVX_BF16_F32_VTCM) {
- path = "hvx-flat";
- } else if (type == HTP_MM_KERNEL_HVX_QUANT_ROW || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {
- path = "hvx-tiled";
- }
- snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);
- } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) {
- const auto * kparams = (const struct htp_fa_kernel_params *) node.kernel_params;
- const char * path = "unknown";
- int32_t type = kparams->kernel_type;
- if (type == HTP_FA_KERNEL_HMX) {
- path = kparams->u.hmx.pipeline ? "hmx-pipe" : "hmx-seq";
- } else if (type == HTP_FA_KERNEL_HVX) {
- path = "hvx";
- }
- snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);
- } else if (htp_op_is_unary(node.opcode)) {
- const auto * kparams = (const struct htp_unary_kernel_params *) node.kernel_params;
- snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size);
- } else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) {
- snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]);
- } else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) {
- snprintf(str, max_size, "seq 0x%x", (uint32_t) node.node->op_params[0]);
- } else if (node.opcode == HTP_OP_ALLREDUCE && node.node) {
- snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]);
- } else if (node.opcode == HTP_OP_SSM_CONV) {
- const auto * kparams = (const struct htp_ssm_conv_kernel_params *) node.kernel_params;
- snprintf(str, max_size, "%s vtcm %d", kparams->n_t == 1 ? "decode" : "prefill", (int) kparams->vtcm_size);
- } else if (node.opcode == HTP_OP_SOFTMAX) {
- const auto * kparams = (const struct htp_softmax_kernel_params *) node.kernel_params;
- snprintf(str, max_size, "k%d nth %d vtcm %d", (int) kparams->kernel_id, (int) kparams->n_threads, (int) kparams->vtcm_size);
- } else if (node.opcode == HTP_OP_GATED_DELTA_NET) {
- const auto * kparams = (const struct htp_gdn_kernel_params *) node.kernel_params;
- const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent";
- snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar",
- (unsigned int) (kparams->vtcm_size ? kparams->vtcm_size : kparams->vtcm_per_thread * kparams->n_threads));
- } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID ||
- node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {
- const auto * kparams = (const struct htp_binary_kernel_params *) node.kernel_params;
- snprintf(str, max_size, "vtcm %u", (unsigned int) kparams->vtcm_size);
- } else if (node.opcode == HTP_OP_ARGSORT || node.opcode == HTP_OP_TOP_K) {
- const auto * kparams = (const struct htp_sort_kernel_params *) node.kernel_params;
- snprintf(str, max_size, "%s nth %d nchk %d chk %d vtcm %d",
- node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort",
- (int) kparams->n_threads, (int) kparams->n_chunks,
- (int) kparams->chunk_elems, (int) kparams->vtcm_size);
- } else if (node.opcode == HTP_OP_GET_ROWS) {
- const auto * kparams = (const struct htp_get_rows_kernel_params *) node.kernel_params;
- const char * ktype_str = "unknown";
- switch (kparams->kernel_type) {
- case HTP_GET_ROWS_KERNEL_SAMETYPE: ktype_str = "sametype"; break;
- case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break;
- case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break;
- }
- snprintf(str, max_size, "%s%s vtcm %u", ktype_str, kparams->n_threads > 1 ? "-multi" : "", (unsigned int) kparams->vtcm_size);
- } else {
- snprintf(str, max_size, "----");
- }
-
- }
-
- void format(const htp_opnode & node) {
- format_op_dims(dims, sizeof(dims), node);
- format_op_strides(strides, sizeof(strides), node);
- format_op_types(types, sizeof(types), node);
- format_op_buffs(buffs, sizeof(buffs), node);
- format_op_names(names, sizeof(names), node);
- format_kernel_params(kparams, sizeof(kparams), node);
- }
-
- htp_opformat() {
- strides[0] = '\0';
- dims[0] = '\0';
- types[0] = '\0';
- buffs[0] = '\0';
- names[0] = '\0';
- kparams[0] = '\0';
- }
- htp_opformat(const htp_opnode & node) { format(node); }
+enum htp_opformat_flags {
+ HTP_OPFMT_OP = (1 << 0),
+ HTP_OPFMT_NAMES = (1 << 1),
+ HTP_OPFMT_DIMS = (1 << 2),
+ HTP_OPFMT_TYPES = (1 << 3),
+ HTP_OPFMT_STRIDES = (1 << 4),
+ HTP_OPFMT_BUFFS = (1 << 5),
+ HTP_OPFMT_KPARAMS = (1 << 6),
+
+ HTP_OPFMT_EXEC = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_BUFFS | HTP_OPFMT_KPARAMS,
+ HTP_OPFMT_SUPP = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_BUFFS,
+ HTP_OPFMT_PROF = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_KPARAMS,
+ HTP_OPFMT_TENSOR = HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES,
};
+std::string htp_format_op(const htp_opnode & node, uint32_t flags = HTP_OPFMT_EXEC);
+std::string htp_format_tensor(const ggml_tensor * t, uint32_t flags = HTP_OPFMT_TENSOR);
+
#endif // HTP_OPNODE_H
diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt
index 28fae4fe8..2edd653ce 100644
--- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt
+++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt
@@ -28,7 +28,7 @@ add_library(${HTP_LIB} SHARED
unary-ops.c
sum-rows-ops.c
softmax-ops.c
- act-ops.c
+ glu-ops.c
rope-ops.c
set-rows-ops.c
get-rows-ops.c
diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.h b/ggml/src/ggml-hexagon/htp/allreduce-ops.h
index 54ef60378..1264c1cbb 100644
--- a/ggml/src/ggml-hexagon/htp/allreduce-ops.h
+++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.h
@@ -32,6 +32,7 @@ static inline size_t htp_allreduce_vtcm_buffer_count(
}
struct htp_allreduce_kernel_params {
+ uint8_t kernel_type;
int32_t rank;
int32_t n_ranks;
int32_t n_threads;
@@ -39,7 +40,6 @@ struct htp_allreduce_kernel_params {
int32_t elems_per_thread; // 1D: nelem_per_thread, 2D: nrows_per_thread
int32_t vtcm_size_per_thread;
int32_t vtcm_size;
- int32_t kernel_type;
int32_t ne0;
int32_t ne1;
int32_t row_size_aligned;
@@ -50,6 +50,12 @@ struct htp_allreduce_kernel_params {
int32_t mode; // enum htp_allreduce_mode: FULL or SHARDED_FANOUT
};
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_allreduce_kernel_params) <= 128, "htp_allreduce_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_allreduce_kernel_params) <= 128, "htp_allreduce_kernel_params is too large for kernel_params blob");
+#endif
+
#ifdef __cplusplus
}
#endif
diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.c b/ggml/src/ggml-hexagon/htp/argsort-ops.c
index 6f24d95f9..7aa7ca8e4 100644
--- a/ggml/src/ggml-hexagon/htp/argsort-ops.c
+++ b/ggml/src/ggml-hexagon/htp/argsort-ops.c
@@ -453,7 +453,10 @@ static void merge_runs(
struct htp_sort_chunk_ctx {
struct htp_ops_context * octx;
const struct htp_sort_kernel_params * kparams;
- uint8_t * vtcm_base;
+ uint8_t * phase1_base;
+ size_t phase1_slot_size;
+ float * merge_values;
+ int32_t * merge_indices;
uint32_t real_count[HTP_MAX_NTHREADS];
};
@@ -462,26 +465,27 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) {
struct htp_ops_context * octx = cctx->octx;
const struct htp_sort_kernel_params * kparams = cctx->kparams;
const struct htp_tensor * src0 = octx->src[0];
+ const struct htp_tensor * dst = octx->dst;
uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;
- uint32_t ne00 = (uint32_t) kparams->ne00;
+ uint32_t ne00 = src0->ne[0];
uint32_t chunk_base = i * chunk_elems;
uint32_t real_count = (chunk_base < ne00) ? MIN(chunk_elems, ne00 - chunk_base) : 0;
cctx->real_count[i] = real_count;
- uint8_t * spad = cctx->vtcm_base + (size_t) kparams->phase1_slot_size * i;
+ uint8_t * spad = cctx->phase1_base + cctx->phase1_slot_size * i;
float * values_buf = (float *) spad;
int32_t * indices_buf = (int32_t *) (spad + chunk_elems * sizeof(float));
- float * merge_values = (float *) (cctx->vtcm_base + kparams->merge_values_off);
- int32_t * merge_indices = (int32_t *) (cctx->vtcm_base + kparams->merge_indices_off);
+ float * merge_values = cctx->merge_values;
+ int32_t * merge_indices = cctx->merge_indices;
struct htp_thread_trace * tr = &octx->ctx->trace[i];
const dma_addr_t src_addr = src0->data + (size_t) chunk_base * sizeof(float);
if (kparams->is_top_k) {
- uint32_t k = (uint32_t) kparams->k;
+ uint32_t k = dst->ne[0];
uint32_t local_k = MIN(k, chunk_elems);
if (real_count == 0) {
@@ -559,17 +563,17 @@ static void htp_argsort_merge_l1_job(unsigned int n, unsigned int i, void * data
uint32_t count0 = cctx->real_count[c0];
uint32_t count1 = cctx->real_count[c1];
- size_t slot_size = (size_t) kparams->phase1_slot_size;
+ size_t slot_size = cctx->phase1_slot_size;
uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;
- const float * val0 = (const float *) (cctx->vtcm_base + slot_size * c0);
- const int32_t * idx0 = (const int32_t *) (cctx->vtcm_base + slot_size * c0 + chunk_elems * sizeof(float));
- const float * val1 = (const float *) (cctx->vtcm_base + slot_size * c1);
- const int32_t * idx1 = (const int32_t *) (cctx->vtcm_base + slot_size * c1 + chunk_elems * sizeof(float));
+ const float * val0 = (const float *) (cctx->phase1_base + slot_size * c0);
+ const int32_t * idx0 = (const int32_t *) (cctx->phase1_base + slot_size * c0 + chunk_elems * sizeof(float));
+ const float * val1 = (const float *) (cctx->phase1_base + slot_size * c1);
+ const int32_t * idx1 = (const int32_t *) (cctx->phase1_base + slot_size * c1 + chunk_elems * sizeof(float));
uint32_t out_offset = (2 * i) * chunk_elems;
- float * out_val = (float *) (cctx->vtcm_base + kparams->merge_values_off) + out_offset;
- int32_t * out_idx = (int32_t *) (cctx->vtcm_base + kparams->merge_indices_off) + out_offset;
+ float * out_val = cctx->merge_values + out_offset;
+ int32_t * out_idx = cctx->merge_indices + out_offset;
struct htp_thread_trace * tr = &cctx->octx->ctx->trace[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
@@ -587,21 +591,21 @@ static void htp_argsort_merge_l2_job(unsigned int n, unsigned int i, void * data
struct htp_sort_chunk_ctx * cctx = mctx->cctx;
const struct htp_sort_kernel_params * kparams = cctx->kparams;
- size_t slot_size = (size_t) kparams->phase1_slot_size;
+ size_t slot_size = cctx->phase1_slot_size;
uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;
uint32_t off0 = (4 * i) * chunk_elems;
uint32_t off1 = (4 * i + 2) * chunk_elems;
- const float * val0 = (const float *) (cctx->vtcm_base + kparams->merge_values_off) + off0;
- const int32_t * idx0 = (const int32_t *) (cctx->vtcm_base + kparams->merge_indices_off) + off0;
- const float * val1 = (const float *) (cctx->vtcm_base + kparams->merge_values_off) + off1;
- const int32_t * idx1 = (const int32_t *) (cctx->vtcm_base + kparams->merge_indices_off) + off1;
+ const float * val0 = cctx->merge_values + off0;
+ const int32_t * idx0 = cctx->merge_indices + off0;
+ const float * val1 = cctx->merge_values + off1;
+ const int32_t * idx1 = cctx->merge_indices + off1;
uint32_t count0 = cctx->real_count[4*i + 0] + cctx->real_count[4*i + 1];
uint32_t count1 = cctx->real_count[4*i + 2] + cctx->real_count[4*i + 3];
- uint8_t * slot_base = cctx->vtcm_base + (4 * i) * slot_size;
+ uint8_t * slot_base = cctx->phase1_base + (4 * i) * slot_size;
float * out_val = (float *) slot_base;
int32_t * out_idx = (int32_t *) (slot_base + 4 * chunk_elems * sizeof(float));
@@ -614,7 +618,10 @@ static void htp_argsort_merge_l2_job(unsigned int n, unsigned int i, void * data
struct htp_sort_multi_row_ctx {
struct htp_ops_context * octx;
const struct htp_sort_kernel_params * kparams;
- uint8_t * vtcm_base;
+ uint8_t * phase1_base;
+ size_t phase1_slot_size;
+ uint32_t row_start;
+ uint32_t row_end;
uint32_t nrows_per_thread;
};
@@ -624,12 +631,18 @@ struct htp_tensor_row_coord {
uint32_t i03;
};
-static inline struct htp_tensor_row_coord init_row_coord(uint32_t r, uint32_t ne01, uint32_t ne02) {
+static inline struct htp_tensor_row_coord init_row_coord(
+ uint32_t r,
+ uint32_t ne01,
+ uint32_t ne02,
+ const struct fastdiv_values * div_ne01,
+ const struct fastdiv_values * div_ne01_ne02
+) {
uint32_t ne01_ne02 = ne01 * ne02;
- uint32_t i03 = (ne01_ne02 > 0) ? (r / ne01_ne02) : 0;
- uint32_t rem = (ne01_ne02 > 0) ? (r % ne01_ne02) : 0;
- uint32_t i02 = (ne01 > 0) ? (rem / ne01) : 0;
- uint32_t i01 = (ne01 > 0) ? (rem % ne01) : 0;
+ uint32_t i03 = (ne01_ne02 > 0) ? fastdiv(r, div_ne01_ne02) : 0;
+ uint32_t rem = (ne01_ne02 > 0) ? (r - i03 * ne01_ne02) : 0;
+ uint32_t i02 = (ne01 > 0) ? fastdiv(rem, div_ne01) : 0;
+ uint32_t i01 = (ne01 > 0) ? (rem - i02 * ne01) : 0;
return (struct htp_tensor_row_coord){ i01, i02, i03 };
}
@@ -701,7 +714,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
- uint32_t ne00 = (uint32_t) kparams->ne00;
+ uint32_t ne00 = src0->ne[0];
uint32_t ne01 = src0->ne[1];
uint32_t ne02 = src0->ne[2];
@@ -713,16 +726,16 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data
uint32_t nb2 = dst->nb[2];
uint32_t nb3 = dst->nb[3];
- uint32_t row_start = (uint32_t) kparams->row_start;
- uint32_t row_end = (uint32_t) kparams->row_end;
+ uint32_t row_start = mctx->row_start;
+ uint32_t row_end = mctx->row_end;
uint32_t r_start = row_start + ith * mctx->nrows_per_thread;
uint32_t r_end = MIN(r_start + mctx->nrows_per_thread, row_end);
if (r_start >= r_end) return;
- size_t spad_offset = ith * (size_t) (kparams->phase1_slot_size * kparams->n_slots);
- uint8_t * thread_spad = mctx->vtcm_base + spad_offset;
- size_t slot_size = (size_t) kparams->phase1_slot_size;
+ size_t slot_size = mctx->phase1_slot_size;
+ size_t spad_offset = ith * slot_size * (size_t) kparams->n_slots;
+ uint8_t * thread_spad = mctx->phase1_base + spad_offset;
uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;
float * values_buf = (float *) thread_spad;
@@ -732,14 +745,14 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
bool is_top_k = kparams->is_top_k != 0;
- uint32_t k = (uint32_t) kparams->k;
+ uint32_t k = dst->ne[0];
enum ggml_sort_order order = (enum ggml_sort_order) kparams->order;
const uint32_t src_bytes = ne00 * sizeof(float);
const uint32_t dst_bytes = (is_top_k ? k : ne00) * sizeof(int32_t);
if (kparams->n_slots == 2) {
- struct htp_tensor_row_coord prime_coord = init_row_coord(r_start, ne01, ne02);
+ struct htp_tensor_row_coord prime_coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02);
for (uint32_t r = r_start, spad_idx = 0; r < r_end && spad_idx < 2; r++, spad_idx++) {
uint8_t * cur_spad = thread_spad + spad_idx * slot_size;
float * cur_values = (float *) cur_spad;
@@ -759,8 +772,8 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data
step_row_coord(&prime_coord, ne01, ne02);
}
- struct htp_tensor_row_coord dst_coord = init_row_coord(r_start, ne01, ne02);
- struct htp_tensor_row_coord next_src_coord = init_row_coord(r_start + 2, ne01, ne02);
+ struct htp_tensor_row_coord dst_coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02);
+ struct htp_tensor_row_coord next_src_coord = init_row_coord(r_start + 2, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02);
for (uint32_t r = r_start; r < r_end; r++) {
uint32_t cur_slot = (r - r_start) & 1;
@@ -796,7 +809,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data
dma_queue_flush(dma_q);
} else {
- struct htp_tensor_row_coord coord = init_row_coord(r_start, ne01, ne02);
+ struct htp_tensor_row_coord coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02);
for (uint32_t r = r_start; r < r_end; r++) {
dma_addr_t src_addr = src0->data + coord_offset(coord, nb01, nb02, nb03);
dma_addr_t dst_addr = dst->data + coord_offset(coord, nb1, nb2, nb3);
@@ -847,52 +860,39 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {
uint32_t ne00 = src0->ne[0];
uint32_t k = dst->ne[0];
- struct htp_sort_kernel_params kparams_local;
const struct htp_sort_kernel_params * kparams = (const struct htp_sort_kernel_params *) octx->kernel_params;
- if (kparams->n_threads == 0) {
- struct htp_sort_vtcm_layout layout;
- if (!htp_sort_solve_layout(&layout, ne00, total_rows, k, octx->n_threads, octx->ctx->vtcm_size, is_top_k)) {
- return HTP_STATUS_VTCM_TOO_SMALL;
- }
- memset(&kparams_local, 0, sizeof(kparams_local));
- kparams_local.n_threads = (int32_t) layout.n_threads;
- kparams_local.total_rows = (int32_t) total_rows;
- kparams_local.row_start = (int32_t) row_start;
- kparams_local.row_end = (int32_t) row_end;
- kparams_local.ne00 = (int32_t) ne00;
- kparams_local.k = (int32_t) k;
- kparams_local.order = is_top_k ? GGML_SORT_ORDER_DESC : (int32_t) octx->op_params[0];
- kparams_local.is_top_k = is_top_k ? 1 : 0;
- kparams_local.use_dma = 1;
- kparams_local.chunk_elems = (int32_t) layout.chunk_elems;
- kparams_local.n_chunks = (int32_t) layout.n_chunks;
- kparams_local.vtcm_size = (int32_t) layout.total_bytes;
- kparams_local.phase1_slot_size = (int32_t) layout.phase1_slot_size;
- kparams_local.merge_values_off = (int32_t) layout.merge_values_off;
- kparams_local.merge_indices_off = (int32_t) layout.merge_indices_off;
- kparams_local.merge_elems = (int32_t) layout.merge_elems;
- kparams_local.n_slots = (int32_t) layout.n_slots;
- kparams = &kparams_local;
- } else if (octx->ctx->mdev.count > 1) {
- memcpy(&kparams_local, kparams, sizeof(kparams_local));
- kparams_local.row_start = (int32_t) row_start;
- kparams_local.row_end = (int32_t) row_end;
- kparams = &kparams_local;
+ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ return HTP_STATUS_INVAL_PARAMS;
}
- if (octx->ctx->vtcm_size < (size_t) kparams->vtcm_size) {
- FARF(ERROR, "sort: VTCM size too small. Needed %d, have %zu", kparams->vtcm_size, octx->ctx->vtcm_size);
+ struct htp_sort_vtcm_layout layout;
+ htp_sort_vtcm_layout_build(&layout,
+ total_rows,
+ (uint32_t) kparams->chunk_elems,
+ (uint32_t) kparams->n_chunks,
+ (uint32_t) kparams->merge_elems,
+ (uint32_t) kparams->n_threads,
+ (uint32_t) kparams->n_slots);
+
+ if (octx->ctx->vtcm_size < layout.total_bytes) {
+ FARF(ERROR, "sort: VTCM size too small. Needed %zu, have %zu", layout.total_bytes, octx->ctx->vtcm_size);
return HTP_STATUS_VTCM_TOO_SMALL;
}
- uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
+ uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
+ uint8_t * phase1_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_phase1);
+ float * merge_values = VTCM_LAYOUT_PTR(float, vtcm_base, layout.off_merge_values);
+ int32_t * merge_indices = VTCM_LAYOUT_PTR(int32_t, vtcm_base, layout.off_merge_indices);
if (total_rows == 1 && kparams->n_chunks > 1) {
struct htp_sort_chunk_ctx cctx;
- cctx.octx = octx;
- cctx.kparams = kparams;
- cctx.vtcm_base = vtcm_base;
+ cctx.octx = octx;
+ cctx.kparams = kparams;
+ cctx.phase1_base = phase1_base;
+ cctx.phase1_slot_size = layout.phase1_slot_size;
+ cctx.merge_values = merge_values;
+ cctx.merge_indices = merge_indices;
memset(cctx.real_count, 0, sizeof(cctx.real_count));
work_queue_run(octx->ctx->work_queue, htp_sort_chunk_job, &cctx, (uint32_t) kparams->n_chunks);
@@ -902,8 +902,8 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {
uint32_t local_k = MIN(k, chunk_elems);
if (k == 1 && ne00 >= 128*1024) {
- float * cand_vals = (float *) (vtcm_base + kparams->merge_values_off);
- int32_t * cand_idxs = (int32_t *) (vtcm_base + kparams->merge_indices_off);
+ float * cand_vals = merge_values;
+ int32_t * cand_idxs = merge_indices;
float best_val = cand_vals[0];
int32_t best_idx = cand_idxs[0];
for (int32_t c = 1; c < kparams->n_chunks; c++) {
@@ -918,8 +918,8 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {
}
uint32_t total_candidates = (uint32_t) kparams->n_chunks * local_k;
- float * cand_vals = (float *) (vtcm_base + kparams->merge_values_off);
- int32_t * cand_idxs = (int32_t *) (vtcm_base + kparams->merge_indices_off);
+ float * cand_vals = merge_values;
+ int32_t * cand_idxs = merge_indices;
uint32_t merge_elems = (uint32_t) kparams->merge_elems;
if (merge_elems > total_candidates) {
@@ -943,18 +943,18 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {
struct htp_argsort_merge_l2_ctx l2ctx = { &cctx, asc };
work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l2_job, &l2ctx, 2);
- size_t slot_size = (size_t) kparams->phase1_slot_size;
+ size_t slot_size = layout.phase1_slot_size;
uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;
- const float * val0 = (const float *) vtcm_base;
- const int32_t * idx0 = (const int32_t *) (vtcm_base + 4 * chunk_elems * sizeof(float));
- const float * val1 = (const float *) (vtcm_base + 4 * slot_size);
- const int32_t * idx1 = (const int32_t *) (vtcm_base + 4 * slot_size + 4 * chunk_elems * sizeof(float));
+ const float * val0 = (const float *) phase1_base;
+ const int32_t * idx0 = (const int32_t *) (phase1_base + 4 * chunk_elems * sizeof(float));
+ const float * val1 = (const float *) (phase1_base + 4 * slot_size);
+ const int32_t * idx1 = (const int32_t *) (phase1_base + 4 * slot_size + 4 * chunk_elems * sizeof(float));
uint32_t n01 = cctx.real_count[0] + cctx.real_count[1] + cctx.real_count[2] + cctx.real_count[3];
uint32_t n23 = cctx.real_count[4] + cctx.real_count[5] + cctx.real_count[6] + cctx.real_count[7];
- int32_t * final_idx = (int32_t *) (vtcm_base + kparams->merge_indices_off);
+ int32_t * final_idx = merge_indices;
struct htp_thread_trace * tr = &octx->ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0);
merge_runs(val0, idx0, n01, val1, idx1, n23, NULL, final_idx, asc);
@@ -968,27 +968,27 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {
uint32_t n01 = cctx.real_count[0] + cctx.real_count[1];
uint32_t n23 = cctx.real_count[2] + cctx.real_count[3];
- const float * val01 = (const float *) (vtcm_base + kparams->merge_values_off);
- const int32_t * idx01 = (const int32_t *) (vtcm_base + kparams->merge_indices_off);
- const float * val23 = (const float *) (vtcm_base + kparams->merge_values_off) + 2 * chunk_elems;
- const int32_t * idx23 = (const int32_t *) (vtcm_base + kparams->merge_indices_off) + 2 * chunk_elems;
+ const float * val01 = merge_values;
+ const int32_t * idx01 = merge_indices;
+ const float * val23 = merge_values + 2 * chunk_elems;
+ const int32_t * idx23 = merge_indices + 2 * chunk_elems;
- int32_t * final_idx = (int32_t *) vtcm_base;
+ int32_t * final_idx = (int32_t *) phase1_base;
struct htp_thread_trace * tr = &octx->ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0);
merge_runs(val01, idx01, n01, val23, idx23, n23, NULL, final_idx, asc);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, 0);
dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t));
} else if (kparams->n_chunks == 2) {
- size_t slot_size = (size_t) kparams->phase1_slot_size;
+ size_t slot_size = layout.phase1_slot_size;
uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;
- const float * val0 = (const float *) (vtcm_base);
- const int32_t * idx0 = (const int32_t *) (vtcm_base + chunk_elems * sizeof(float));
- const float * val1 = (const float *) (vtcm_base + slot_size);
- const int32_t * idx1 = (const int32_t *) (vtcm_base + slot_size + chunk_elems * sizeof(float));
+ const float * val0 = (const float *) (phase1_base);
+ const int32_t * idx0 = (const int32_t *) (phase1_base + chunk_elems * sizeof(float));
+ const float * val1 = (const float *) (phase1_base + slot_size);
+ const int32_t * idx1 = (const int32_t *) (phase1_base + slot_size + chunk_elems * sizeof(float));
- int32_t * final_idx = (int32_t *) (vtcm_base + kparams->merge_indices_off);
+ int32_t * final_idx = merge_indices;
struct htp_thread_trace * tr = &octx->ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0);
merge_runs(val0, idx0, cctx.real_count[0], val1, idx1, cctx.real_count[1], NULL, final_idx, asc);
@@ -996,7 +996,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {
dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t));
} else {
uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;
- int32_t * final_idx = (int32_t *) (vtcm_base + chunk_elems * sizeof(float));
+ int32_t * final_idx = (int32_t *) (phase1_base + chunk_elems * sizeof(float));
dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t));
}
}
@@ -1006,7 +1006,10 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {
struct htp_sort_multi_row_ctx mctx;
mctx.octx = octx;
mctx.kparams = kparams;
- mctx.vtcm_base = vtcm_base;
+ mctx.phase1_base = phase1_base;
+ mctx.phase1_slot_size = layout.phase1_slot_size;
+ mctx.row_start = row_start;
+ mctx.row_end = row_end;
uint32_t dst_row_bytes = (uint32_t) dst_row_size;
uint32_t rows_per_chunk = 0;
@@ -1014,13 +1017,13 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {
if (rows_per_chunk == 0) {
rows_per_chunk = 1;
}
- uint32_t dr = (nrows + (uint32_t) kparams->n_threads - 1) / (uint32_t) kparams->n_threads;
+ uint32_t dr = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div);
if (rows_per_chunk > 1) {
dr = hex_round_up(dr, rows_per_chunk);
}
mctx.nrows_per_thread = dr;
- work_queue_run(octx->ctx->work_queue, htp_sort_multi_row_job, &mctx, (uint32_t) kparams->n_threads);
+ work_queue_run(octx->ctx->work_queue, htp_sort_multi_row_job, &mctx, octx->n_threads);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.h b/ggml/src/ggml-hexagon/htp/argsort-ops.h
index 05320a2e9..f74189fba 100644
--- a/ggml/src/ggml-hexagon/htp/argsort-ops.h
+++ b/ggml/src/ggml-hexagon/htp/argsort-ops.h
@@ -7,33 +7,27 @@
#include <string.h>
#include "hex-fastdiv.h"
+#include "htp-vtcm.h"
struct htp_sort_kernel_params {
- int32_t n_threads;
- int32_t total_rows;
- int32_t row_start;
- int32_t row_end;
- int32_t ne00;
- int32_t k;
- int32_t order; // GGML_SORT_ORDER_ASC (0) or GGML_SORT_ORDER_DESC (1)
- int32_t is_top_k; // 1 if TOP_K, 0 if ARGSORT
- int32_t use_dma; // 1 if DMA enabled
- int32_t chunk_elems;
- int32_t n_chunks;
- int32_t vtcm_size;
- int32_t phase1_slot_size;
- int32_t merge_values_off;
- int32_t merge_indices_off;
- int32_t merge_elems;
- int32_t n_slots;
- int32_t pad[15];
+ uint32_t n_threads;
+ uint32_t order; // GGML_SORT_ORDER_ASC (0) or GGML_SORT_ORDER_DESC (1)
+ uint32_t is_top_k; // 1 if TOP_K, 0 if ARGSORT
+ uint32_t chunk_elems;
+ uint32_t n_chunks;
+ uint32_t vtcm_size;
+ uint32_t merge_elems;
+ uint32_t n_slots;
+ struct fastdiv_values div_ne01;
+ struct fastdiv_values div_ne01_ne02;
};
struct htp_sort_vtcm_layout {
size_t total_bytes;
size_t phase1_slot_size;
- size_t merge_values_off;
- size_t merge_indices_off;
+ size_t off_phase1;
+ size_t off_merge_values;
+ size_t off_merge_indices;
uint32_t chunk_elems;
uint32_t n_chunks;
uint32_t merge_elems;
@@ -41,6 +35,39 @@ struct htp_sort_vtcm_layout {
uint32_t n_slots;
};
+static inline void htp_sort_vtcm_layout_build(
+ struct htp_sort_vtcm_layout * L,
+ uint32_t total_rows,
+ uint32_t chunk_elems,
+ uint32_t n_chunks,
+ uint32_t merge_elems,
+ uint32_t n_threads,
+ uint32_t n_slots) {
+
+ memset(L, 0, sizeof(*L));
+ L->chunk_elems = chunk_elems;
+ L->n_chunks = n_chunks;
+ L->merge_elems = merge_elems;
+ L->n_threads = n_threads;
+ L->n_slots = n_slots;
+
+ const size_t values_size = ((chunk_elems * sizeof(float)) + 127) & ~127;
+ const size_t indices_size = ((chunk_elems * sizeof(int32_t)) + 127) & ~127;
+ L->phase1_slot_size = ((values_size + indices_size) + 255) & ~255;
+
+ size_t off = 0;
+ if (total_rows > 1) {
+ VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_slots * n_threads);
+ } else {
+ VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_chunks * n_slots);
+ const size_t merge_values_size = ((merge_elems * sizeof(float)) + 127) & ~127;
+ const size_t merge_indices_size = ((merge_elems * sizeof(int32_t)) + 127) & ~127;
+ VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_merge_values, merge_values_size, n_chunks > 1);
+ VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_merge_indices, merge_indices_size, n_chunks > 1);
+ }
+ L->total_bytes = off;
+}
+
static inline bool htp_sort_solve_layout(
struct htp_sort_vtcm_layout * layout,
uint32_t ne00,
@@ -75,18 +102,8 @@ static inline bool htp_sort_solve_layout(
n_threads--;
}
- size_t total_bytes = spad_per_thread * n_threads;
- if (total_bytes > vtcm_budget) {
- return false;
- }
-
- layout->total_bytes = total_bytes;
- layout->phase1_slot_size = spad_per_slot;
- layout->chunk_elems = ne00_padded;
- layout->n_chunks = 1;
- layout->n_threads = n_threads;
- layout->n_slots = n_slots;
- return true;
+ htp_sort_vtcm_layout_build(layout, total_rows, ne00_padded, 1, 0, n_threads, n_slots);
+ return layout->total_bytes <= vtcm_budget;
}
uint32_t n_vec = (ne00 + 31) / 32;
@@ -102,17 +119,7 @@ static inline bool htp_sort_solve_layout(
uint32_t chunk_n_vec = n_vec_pow2 / n_chunks;
uint32_t chunk_elems = chunk_n_vec * 32;
-
- size_t phase1_values_size = ((chunk_elems * sizeof(float)) + 127) & ~127;
- size_t phase1_indices_size = ((chunk_elems * sizeof(int32_t)) + 127) & ~127;
- size_t phase1_slot_size = ((phase1_values_size + phase1_indices_size) + 255) & ~255;
- size_t phase1_total_size = phase1_slot_size * n_chunks;
-
- size_t merge_values_size = 0;
- size_t merge_indices_size = 0;
- size_t merge_values_off = phase1_total_size;
- size_t merge_indices_off = 0;
- uint32_t merge_elems = 0;
+ uint32_t merge_elems = 0;
if (n_chunks > 1) {
if (is_top_k) {
@@ -125,47 +132,25 @@ static inline bool htp_sort_solve_layout(
} else {
merge_elems = n_vec_pow2 * 32;
}
- merge_values_size = ((merge_elems * sizeof(float)) + 127) & ~127;
- merge_indices_size = ((merge_elems * sizeof(int32_t)) + 127) & ~127;
- merge_indices_off = merge_values_off + merge_values_size;
}
- size_t total_bytes = phase1_total_size + merge_values_size + merge_indices_size;
+ htp_sort_vtcm_layout_build(layout, 1, chunk_elems, n_chunks, merge_elems, n_chunks, 1);
- if (total_bytes > vtcm_budget && n_chunks > 1) {
- n_chunks = 1;
+ if (layout->total_bytes > vtcm_budget && n_chunks > 1) {
+ n_chunks = 1;
chunk_elems = n_vec_pow2 * 32;
- phase1_values_size = ((chunk_elems * sizeof(float)) + 127) & ~127;
- phase1_indices_size = ((chunk_elems * sizeof(int32_t)) + 127) & ~127;
- phase1_slot_size = ((phase1_values_size + phase1_indices_size) + 255) & ~255;
- phase1_total_size = phase1_slot_size;
- merge_values_size = 0;
- merge_indices_size = 0;
- merge_values_off = phase1_total_size;
- merge_indices_off = 0;
- merge_elems = 0;
- total_bytes = phase1_total_size;
+ merge_elems = 0;
+ htp_sort_vtcm_layout_build(layout, 1, chunk_elems, n_chunks, merge_elems, 1, 1);
}
- if (total_bytes > vtcm_budget) {
+ if (layout->total_bytes > vtcm_budget) {
return false;
}
- uint32_t n_slots = 1;
- if (n_chunks == 1 && phase1_slot_size * 2 <= vtcm_budget) {
- n_slots = 2;
- total_bytes = phase1_slot_size * 2;
+ if (n_chunks == 1 && layout->phase1_slot_size * 2 <= vtcm_budget) {
+ htp_sort_vtcm_layout_build(layout, 1, chunk_elems, 1, 0, 1, 2);
}
- layout->total_bytes = total_bytes;
- layout->phase1_slot_size = phase1_slot_size;
- layout->merge_values_off = merge_values_off;
- layout->merge_indices_off = merge_indices_off;
- layout->chunk_elems = chunk_elems;
- layout->n_chunks = n_chunks;
- layout->merge_elems = merge_elems;
- layout->n_threads = n_chunks;
- layout->n_slots = n_slots;
return true;
}
diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.c
index 78ce4bb63..48f461e46 100644
--- a/ggml/src/ggml-hexagon/htp/binary-ops.c
+++ b/ggml/src/ggml-hexagon/htp/binary-ops.c
@@ -42,6 +42,7 @@ struct htp_binary_context {
uint32_t nrows_per_thread;
uint32_t total_rows;
uint32_t row_start;
+ uint32_t n_srcs;
size_t src0_row_size_aligned;
size_t src1_row_size_aligned;
size_t dst_row_size_aligned;
@@ -380,6 +381,7 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith,
size_t dst_spad_half = layout->dst_spad_half_size;
dma_queue * dma_q = octx->ctx->dma[ith];
+ const uint32_t n_srcs = bctx->n_srcs;
uint32_t ir_prefetch = start_row;
int spad_idx = 0;
@@ -391,12 +393,7 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith,
i02 = fastdiv(rem, &bctx->src0_dim1_div);
i01 = rem - i02 * ne01;
- uint32_t i13 = (ne13 == 1) ? 0 : i03;
- uint32_t i12 = (ne12 == 1) ? 0 : i02;
- uint32_t i11 = (ne11 == 1) ? 0 : i01;
-
dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01;
- dma_addr_t src1_curr = src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11;
dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;
uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half;
@@ -405,7 +402,15 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith,
dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0);
dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size);
- dma_queue_push(dma_q, dma_make_data(s1_spad, src1_curr), bctx->src1_row_size_aligned, nb11, row_size_bytes, current_block_size);
+ for (uint32_t s = 1; s < n_srcs; s++) {
+ const struct htp_tensor * src_s = octx->src[s];
+ uint32_t is3 = (src_s->ne[3] == 1) ? 0 : i03;
+ uint32_t is2 = (src_s->ne[2] == 1) ? 0 : i02;
+ uint32_t is1 = (src_s->ne[1] == 1) ? 0 : i01;
+ dma_addr_t s_curr = src_s->data + is3 * src_s->nb[3] + is2 * src_s->nb[2] + is1 * src_s->nb[1];
+ uint8_t * s_spad = s1_spad + (s - 1) * 2 * src1_spad_half;
+ dma_queue_push(dma_q, dma_make_data(s_spad, s_curr), bctx->src1_row_size_aligned, src_s->nb[1], row_size_bytes, current_block_size);
+ }
ir_prefetch += current_block_size;
spad_idx ^= 1;
}
@@ -422,6 +427,11 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith,
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
compute(d_spad, s0_spad, s1_spad, current_block_size,
bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, bctx->src1_row_size_aligned, ne00);
+ for (uint32_t s = 2; s < n_srcs; s++) {
+ uint8_t * ss_spad = (uint8_t *) dma_queue_pop(dma_q).dst;
+ compute(d_spad, d_spad, ss_spad, current_block_size,
+ bctx->dst_row_size_aligned, bctx->dst_row_size_aligned, bctx->src1_row_size_aligned, ne00);
+ }
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
uint32_t i03, i02, i01, rem;
@@ -440,15 +450,18 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith,
p02 = fastdiv(prem, &bctx->src0_dim1_div);
p01 = prem - p02 * ne01;
- uint32_t p13 = (ne13 == 1) ? 0 : p03;
- uint32_t p12 = (ne12 == 1) ? 0 : p02;
- uint32_t p11 = (ne11 == 1) ? 0 : p01;
-
dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01;
- dma_addr_t s1_next = src1->data + p13 * nb13 + p12 * nb12 + p11 * nb11;
-
dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size);
- dma_queue_push(dma_q, dma_make_data(s1_spad, s1_next), bctx->src1_row_size_aligned, nb11, row_size_bytes, next_block_size);
+
+ for (uint32_t s = 1; s < n_srcs; s++) {
+ const struct htp_tensor * src_s = octx->src[s];
+ uint32_t ps3 = (src_s->ne[3] == 1) ? 0 : p03;
+ uint32_t ps2 = (src_s->ne[2] == 1) ? 0 : p02;
+ uint32_t ps1 = (src_s->ne[1] == 1) ? 0 : p01;
+ dma_addr_t s_next = src_s->data + ps3 * src_s->nb[3] + ps2 * src_s->nb[2] + ps1 * src_s->nb[1];
+ uint8_t * s_spad = s1_spad + (s - 1) * 2 * src1_spad_half;
+ dma_queue_push(dma_q, dma_make_data(s_spad, s_next), bctx->src1_row_size_aligned, src_s->nb[1], row_size_bytes, next_block_size);
+ }
ir_prefetch += next_block_size;
}
@@ -859,6 +872,7 @@ struct binary_chunked_context {
uint32_t chunk_size;
uint32_t chunks_per_thread;
uint32_t total_chunks;
+ uint32_t n_srcs;
bool is_scalar;
float scalar_f32;
_Float16 scalar_f16;
@@ -872,7 +886,6 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat
struct binary_chunked_context * ctx = (struct binary_chunked_context *) data;
struct htp_ops_context * octx = ctx->octx;
const struct htp_tensor * src0 = octx->src[0];
- const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
const uint32_t start_chunk = ctx->chunks_per_thread * ith;
@@ -895,6 +908,7 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat
uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);
dma_queue * dma_q = octx->ctx->dma[ith];
+ const uint32_t n_srcs = ctx->n_srcs;
uint32_t prefetch_chunk = start_chunk;
int spad_idx = 0;
@@ -913,9 +927,12 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat
dma_queue_push(dma_q, dma_make_data(d_curr, d_spad), chunk_bytes, chunk_bytes, cur_bytes, 0);
dma_queue_push(dma_q, dma_make_data(s0_spad, s0_curr), chunk_bytes, chunk_bytes, cur_bytes, 1);
if (!ctx->is_scalar) {
- dma_addr_t s1_curr = src1->data + (size_t) c_start * elem_size;
uint8_t * s1_spad = src1_spad_base + spad_idx * chunk_bytes;
- dma_queue_push(dma_q, dma_make_data(s1_spad, s1_curr), chunk_bytes, chunk_bytes, cur_bytes, 1);
+ for (uint32_t s = 1; s < n_srcs; s++) {
+ dma_addr_t s_curr = octx->src[s]->data + (size_t) c_start * elem_size;
+ uint8_t * s_spad = s1_spad + (s - 1) * 2 * chunk_bytes;
+ dma_queue_push(dma_q, dma_make_data(s_spad, s_curr), chunk_bytes, chunk_bytes, cur_bytes, 1);
+ }
}
prefetch_chunk++;
@@ -943,6 +960,10 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat
}
} else {
ctx->compute(d_spad, s0_spad, s1_spad, cur_elems);
+ for (uint32_t s = 2; s < n_srcs; s++) {
+ uint8_t * s_spad = (uint8_t *) dma_queue_pop(dma_q).dst;
+ ctx->compute(d_spad, d_spad, s_spad, cur_elems);
+ }
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) c);
@@ -958,8 +979,11 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat
dma_addr_t s0_next = src0->data + (size_t) pc_start * elem_size;
dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), chunk_bytes, chunk_bytes, p_bytes, 1);
if (!ctx->is_scalar) {
- dma_addr_t s1_next = src1->data + (size_t) pc_start * elem_size;
- dma_queue_push(dma_q, dma_make_data(s1_spad, s1_next), chunk_bytes, chunk_bytes, p_bytes, 1);
+ for (uint32_t s = 1; s < n_srcs; s++) {
+ dma_addr_t s_next = octx->src[s]->data + (size_t) pc_start * elem_size;
+ uint8_t * s_spad = s1_spad + (s - 1) * 2 * chunk_bytes;
+ dma_queue_push(dma_q, dma_make_data(s_spad, s_next), chunk_bytes, chunk_bytes, p_bytes, 1);
+ }
}
prefetch_chunk++;
@@ -974,6 +998,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
const struct htp_binary_kernel_params * kparams = (const struct htp_binary_kernel_params *) octx->kernel_params;
+ const uint32_t n_srcs = kparams->n_srcs > 2 ? kparams->n_srcs : 2;
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
@@ -985,7 +1010,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {
const size_t dst_row_size = dst->ne[0] * elem_size;
if (kparams->kernel_type == HTP_BINARY_KERNEL_CHUNKED) {
- const bool is_scalar = kparams->is_scalar || (src1->ne[0] == 1 && src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1);
+ const bool is_scalar = kparams->is_scalar;
compute_binary_chunked_t compute = NULL;
compute_binary_scalar_chunked_f32_t compute_scalar_f32 = NULL;
@@ -1045,12 +1070,16 @@ static int execute_op_binary(struct htp_ops_context * octx) {
const uint32_t elems_per_line = (src0_type == HTP_TYPE_F32) ? 32 : 64;
if (octx->ctx->mdev.count > 1) {
- const bool can_split = htp_tensor_mdev_data_aligned(dst) &&
- htp_tensor_mdev_data_aligned(src0) &&
- (is_scalar || htp_tensor_mdev_data_aligned(src1)) &&
- htp_tensor_is_contiguous(dst, elem_size) &&
- htp_tensor_is_contiguous(src0, elem_size) &&
- (is_scalar || htp_tensor_is_contiguous(src1, elem_size));
+ bool can_split = htp_tensor_mdev_data_aligned(dst) &&
+ htp_tensor_mdev_data_aligned(src0) &&
+ (is_scalar || htp_tensor_mdev_data_aligned(src1)) &&
+ htp_tensor_is_contiguous(dst, elem_size) &&
+ htp_tensor_is_contiguous(src0, elem_size) &&
+ (is_scalar || htp_tensor_is_contiguous(src1, elem_size));
+ for (uint32_t s = 2; can_split && s < n_srcs; s++) {
+ can_split = htp_tensor_mdev_data_aligned(octx->src[s]) &&
+ htp_tensor_is_contiguous(octx->src[s], elem_size);
+ }
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
total_elems, can_split ? elems_per_line : 0,
octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
@@ -1072,7 +1101,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {
return HTP_STATUS_VTCM_TOO_SMALL;
}
- const uint32_t chunk_size = kparams->chunk_size > 0 ? kparams->chunk_size : (32768 / elem_size);
+ const uint32_t chunk_size = kparams->chunk_size;
const uint32_t total_chunks = (nelem + chunk_size - 1) / chunk_size;
const uint32_t n_threads = (total_chunks >= 2) ? MIN(octx->n_threads, total_chunks) : 1;
const uint32_t chunks_per_thread = (total_chunks + n_threads - 1) / n_threads;
@@ -1101,6 +1130,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {
.chunk_size = chunk_size,
.chunks_per_thread = chunks_per_thread,
.total_chunks = total_chunks,
+ .n_srcs = n_srcs,
.is_scalar = is_scalar,
.scalar_f32 = scalar_f32,
.scalar_f16 = scalar_f16,
@@ -1146,65 +1176,6 @@ static int execute_op_binary(struct htp_ops_context * octx) {
return HTP_STATUS_VTCM_TOO_SMALL;
}
- dma_queue * dma_q = octx->ctx->dma[0];
- uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src1);
- if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {
- dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[0] * elem_size, 1);
- } else if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR ||
- kparams->kernel_type == HTP_BINARY_KERNEL_REPEAT ||
- kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX) {
- if (!kparams->src1_strided) {
- const size_t src1_bytes = (size_t) src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3] * elem_size;
- dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1_bytes, 1);
- } else {
- const uint32_t ne11 = src1->ne[1];
- const uint32_t ne12 = src1->ne[2];
- const uint32_t ne13 = src1->ne[3];
- const size_t row_bytes = src1->ne[0] * elem_size;
- for (uint32_t i13 = 0; i13 < ne13; i13++) {
- for (uint32_t i12 = 0; i12 < ne12; i12++) {
- dma_addr_t src_addr = src1->data + i13 * src1->nb[3] + i12 * src1->nb[2];
- uint8_t * dst_ptr = vtcm_src1 + (i13 * ne12 * ne11 + i12 * ne11) * row_bytes;
- while (!dma_queue_push(dma_q, dma_make_data(dst_ptr, src_addr),
- row_bytes, src1->nb[1], row_bytes, ne11)) {
- dma_queue_pop(dma_q);
- }
- }
- }
- }
- } else if (kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) {
- const struct htp_tensor * src2 = octx->src[2];
- uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2);
- dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data),
- kparams->src1_row_size_aligned, src1->nb[1],
- src1->ne[0] * elem_size, src1->ne[1]);
- const size_t src2_row_bytes = src2->ne[0] * sizeof(int32_t);
- dma_queue_push(dma_q, dma_make_data(vtcm_src2, src2->data),
- src2_row_bytes, src2->nb[1],
- src2_row_bytes, src2->ne[1]);
- }
-
- bctx.octx = octx;
- bctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
- bctx.total_rows = nrows;
- bctx.row_start = row_start;
- bctx.block_max = bctx.vtcm_layout.rows_per_buffer;
- bctx.src0_row_size_aligned = src0_row_size_aligned;
- bctx.src1_row_size_aligned = src1_row_size_aligned;
- bctx.dst_row_size_aligned = dst_row_size_aligned;
- bctx.row_size_bytes = src0_row_size;
-
- bctx.src0_dim1_div = init_fastdiv_values(src0->ne[1]);
- bctx.src0_dim2_div = init_fastdiv_values(src0->ne[2]);
- bctx.src0_dim12_div = init_fastdiv_values(src0->ne[1] * src0->ne[2]);
-
- bctx.src1_dim1_div = init_fastdiv_values(src1->ne[1]);
- bctx.src1_dim2_div = init_fastdiv_values(src1->ne[2]);
- bctx.src1_dim3_div = init_fastdiv_values(src1->ne[3]);
-
- bctx.split_at_ne01 = kparams->split_at_ne01;
- bctx.split_at_ne02 = kparams->split_at_ne02;
-
worker_callback_t worker_func = NULL;
void * compute_func = NULL;
@@ -1322,7 +1293,70 @@ static int execute_op_binary(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
- bctx.compute = compute_func;
+ dma_queue * dma_q = octx->ctx->dma[0];
+ uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src1);
+ if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {
+ dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[0] * elem_size, 1);
+ } else if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR ||
+ kparams->kernel_type == HTP_BINARY_KERNEL_REPEAT ||
+ kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX) {
+ if (!kparams->src1_strided) {
+ const size_t src1_bytes = (size_t) src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3] * elem_size;
+ dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1_bytes, 1);
+ if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR && octx->src[2] != NULL) {
+ uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2);
+ dma_queue_push(dma_q, dma_make_data(vtcm_src2, octx->src[2]->data), bctx.vtcm_layout.src2_size, 0, src1_bytes, 1);
+ }
+ } else {
+ const uint32_t ne11 = src1->ne[1];
+ const uint32_t ne12 = src1->ne[2];
+ const uint32_t ne13 = src1->ne[3];
+ const size_t row_bytes = src1->ne[0] * elem_size;
+ for (uint32_t i13 = 0; i13 < ne13; i13++) {
+ for (uint32_t i12 = 0; i12 < ne12; i12++) {
+ dma_addr_t src_addr = src1->data + i13 * src1->nb[3] + i12 * src1->nb[2];
+ uint8_t * dst_ptr = vtcm_src1 + (i13 * ne12 * ne11 + i12 * ne11) * row_bytes;
+ while (!dma_queue_push(dma_q, dma_make_data(dst_ptr, src_addr),
+ row_bytes, src1->nb[1], row_bytes, ne11)) {
+ dma_queue_pop(dma_q);
+ }
+ }
+ }
+ }
+ } else if (kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) {
+ const struct htp_tensor * src2 = octx->src[2];
+ uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2);
+ dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data),
+ kparams->src1_row_size_aligned, src1->nb[1],
+ src1->ne[0] * elem_size, src1->ne[1]);
+ const size_t src2_row_bytes = src2->ne[0] * sizeof(int32_t);
+ dma_queue_push(dma_q, dma_make_data(vtcm_src2, src2->data),
+ src2_row_bytes, src2->nb[1],
+ src2_row_bytes, src2->ne[1]);
+ }
+
+ bctx.octx = octx;
+ bctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
+ bctx.total_rows = nrows;
+ bctx.row_start = row_start;
+ bctx.n_srcs = n_srcs;
+ bctx.block_max = bctx.vtcm_layout.rows_per_buffer;
+ bctx.src0_row_size_aligned = src0_row_size_aligned;
+ bctx.src1_row_size_aligned = src1_row_size_aligned;
+ bctx.dst_row_size_aligned = dst_row_size_aligned;
+ bctx.row_size_bytes = src0_row_size;
+
+ bctx.src0_dim1_div = kparams->div_ne01;
+ bctx.src0_dim2_div = kparams->div_ne02;
+ bctx.src0_dim12_div = kparams->div_ne012;
+
+ bctx.src1_dim1_div = kparams->div_ne11;
+ bctx.src1_dim2_div = kparams->div_ne12;
+ bctx.src1_dim3_div = kparams->div_ne13;
+
+ bctx.split_at_ne01 = kparams->split_at_ne01;
+ bctx.split_at_ne02 = kparams->split_at_ne02;
+ bctx.compute = compute_func;
if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST ||
kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR ||
@@ -1330,6 +1364,15 @@ static int execute_op_binary(struct htp_ops_context * octx) {
kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX ||
kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) {
dma_queue_flush(dma_q);
+ if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR && octx->src[2] != NULL && octx->op == HTP_OP_MUL) {
+ uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2);
+ const uint32_t n_scalars = src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3];
+ if (src0_type == HTP_TYPE_F32) {
+ hvx_mul_f32_aaa(vtcm_src1, vtcm_src1, vtcm_src2, n_scalars);
+ } else if (src0_type == HTP_TYPE_F16) {
+ hvx_mul_f16_aaa(vtcm_src1, vtcm_src1, vtcm_src2, n_scalars);
+ }
+ }
}
work_queue_run(octx->ctx->work_queue, worker_func, &bctx, n_threads);
diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.h b/ggml/src/ggml-hexagon/htp/binary-ops.h
index db3d51489..8c5e683e8 100644
--- a/ggml/src/ggml-hexagon/htp/binary-ops.h
+++ b/ggml/src/ggml-hexagon/htp/binary-ops.h
@@ -6,6 +6,7 @@
#include <string.h>
#include "hex-common.h"
+#include "hex-fastdiv.h"
#include "htp-ops.h"
#include "htp-vtcm.h"
@@ -20,7 +21,15 @@ enum htp_binary_kernel_type {
};
struct htp_binary_kernel_params {
- uint32_t kernel_type;
+ uint8_t kernel_type;
+ uint8_t split_at_ne01;
+ uint8_t split_at_ne02;
+ uint8_t src1_strided;
+
+ uint8_t n_srcs;
+ uint8_t is_scalar;
+ uint8_t pad[2];
+
uint32_t n_threads;
uint32_t rows_per_buffer;
@@ -34,12 +43,14 @@ struct htp_binary_kernel_params {
uint32_t chunk_size;
uint32_t chunk_bytes;
- uint32_t is_scalar;
- uint8_t split_at_ne01;
- uint8_t split_at_ne02;
- uint8_t src1_strided;
- uint8_t pad;
+ struct fastdiv_values div_ne01;
+ struct fastdiv_values div_ne02;
+ struct fastdiv_values div_ne012;
+
+ struct fastdiv_values div_ne11;
+ struct fastdiv_values div_ne12;
+ struct fastdiv_values div_ne13;
};
#if defined(__cplusplus)
@@ -100,6 +111,8 @@ static inline void htp_binary_vtcm_layout_build(
return;
}
+ const uint32_t n_srcs = kparams->n_srcs > 2 ? kparams->n_srcs : 2;
+
if (kparams->kernel_type == HTP_BINARY_KERNEL_CHUNKED) {
const size_t chunk_bytes = kparams->chunk_bytes;
if (chunk_bytes == 0) {
@@ -107,7 +120,7 @@ static inline void htp_binary_vtcm_layout_build(
}
L->src0_bytes_per_thread = 2 * chunk_bytes;
- L->src1_bytes_per_thread = kparams->is_scalar ? 0 : (2 * chunk_bytes);
+ L->src1_bytes_per_thread = kparams->is_scalar ? 0 : (2 * (n_srcs - 1) * chunk_bytes);
L->dst_bytes_per_thread = 2 * chunk_bytes;
L->src0_spad_half_size = chunk_bytes;
@@ -135,7 +148,7 @@ static inline void htp_binary_vtcm_layout_build(
}
const size_t spad_row_total = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE)
- ? 2 * (kparams->src0_row_size_aligned + kparams->src1_row_size_aligned + kparams->dst_row_size_aligned)
+ ? 2 * (kparams->src0_row_size_aligned + (n_srcs - 1) * kparams->src1_row_size_aligned + kparams->dst_row_size_aligned)
: 2 * (kparams->src0_row_size_aligned + kparams->dst_row_size_aligned);
const size_t static_vtcm_size = kparams->src1_size + kparams->src2_size;
@@ -143,7 +156,9 @@ static inline void htp_binary_vtcm_layout_build(
return;
}
- const size_t rows_per_buffer = (vtcm_size - static_vtcm_size) / (n_threads * spad_row_total);
+ const size_t rows_per_buffer = (kparams->rows_per_buffer > 0)
+ ? kparams->rows_per_buffer
+ : (vtcm_size - static_vtcm_size) / (n_threads * spad_row_total);
if (rows_per_buffer == 0) {
return;
}
@@ -155,11 +170,13 @@ static inline void htp_binary_vtcm_layout_build(
L->src0_bytes_per_thread = rows_per_buffer * 2 * kparams->src0_row_size_aligned;
L->dst_bytes_per_thread = rows_per_buffer * 2 * kparams->dst_row_size_aligned;
L->src1_bytes_per_thread = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE)
- ? rows_per_buffer * 2 * kparams->src1_row_size_aligned
+ ? rows_per_buffer * 2 * (n_srcs - 1) * kparams->src1_row_size_aligned
: 0;
L->src0_spad_half_size = L->src0_bytes_per_thread / 2;
- L->src1_spad_half_size = L->src1_bytes_per_thread / 2;
+ L->src1_spad_half_size = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE)
+ ? rows_per_buffer * kparams->src1_row_size_aligned
+ : 0;
L->dst_spad_half_size = L->dst_bytes_per_thread / 2;
const size_t src0_total = n_threads * L->src0_bytes_per_thread;
@@ -178,4 +195,53 @@ static inline void htp_binary_vtcm_layout_build(
L->total_bytes = off;
}
+static inline bool htp_binary_solve_chunked(
+ struct htp_binary_vtcm_layout * L,
+ struct htp_binary_kernel_params * kparams,
+ uint32_t total_elems,
+ uint32_t elem_size,
+ uint32_t max_threads,
+ uint32_t n_srcs,
+ bool is_scalar,
+ size_t vtcm_size
+) {
+ const size_t n_bufs = is_scalar ? 2 : (size_t) n_srcs + 1;
+ const size_t scalar_bytes = is_scalar ? 128 : 0;
+ const size_t budget_per_buf = (max_threads > 0 && vtcm_size > scalar_bytes)
+ ? hex_align_down((vtcm_size - scalar_bytes) / (2 * max_threads * n_bufs), 128)
+ : 0;
+ const uint32_t max_chunk_bytes = (budget_per_buf > 0 && budget_per_buf < 32768) ? (uint32_t) budget_per_buf : 32768u;
+ const uint32_t max_chunk_elems = max_chunk_bytes / elem_size;
+ if (max_chunk_elems == 0) {
+ return false;
+ }
+
+ const uint32_t min_chunk_elems = 256 < max_chunk_elems ? 256 : max_chunk_elems;
+ const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_threads) - 1) / (2 * max_threads), 32);
+ const uint32_t chunk_size = (target_chunk_elems < min_chunk_elems) ? min_chunk_elems
+ : (target_chunk_elems > max_chunk_elems) ? max_chunk_elems
+ : target_chunk_elems;
+ const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128);
+ const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size;
+ const uint32_t n_threads = (total_chunks >= 2) ? (max_threads < total_chunks ? max_threads : total_chunks) : 1;
+
+ kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED;
+ kparams->n_threads = n_threads;
+ kparams->n_srcs = (uint8_t) n_srcs;
+ kparams->rows_per_buffer = 1;
+ kparams->src1_row_size_aligned = is_scalar ? 0 : kparams->src0_row_size_aligned;
+ kparams->src1_size = 0;
+ kparams->chunk_size = chunk_size;
+ kparams->chunk_bytes = chunk_bytes;
+ kparams->is_scalar = is_scalar ? 1 : 0;
+
+ htp_binary_vtcm_layout_build(L, kparams, vtcm_size);
+ if (L->total_bytes == 0 || L->total_bytes > vtcm_size) {
+ return false;
+ }
+
+ kparams->vtcm_size = (uint32_t) L->total_bytes;
+ return true;
+}
+
#endif
diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c
index 40fa09151..20369884b 100644
--- a/ggml/src/ggml-hexagon/htp/concat-ops.c
+++ b/ggml/src/ggml-hexagon/htp/concat-ops.c
@@ -294,22 +294,21 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con
const uint32_t n_threads = octx->n_threads;
- // layout precomputed on host; kept for reference:
- // struct htp_concat_transposed_vtcm_layout layout;
- // htp_concat_transposed_vtcm_layout_build(&layout, octx->src[0]->ne[0], octx->src[1]->ne[0], type_size, n_threads);
+ struct htp_concat_transposed_vtcm_layout layout;
+ htp_concat_transposed_vtcm_layout_build(&layout, octx->src[0]->ne[0], octx->src[1]->ne[0], type_size, n_threads);
uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
struct htp_concat_context cctx;
cctx.octx = octx;
- cctx.spad0_base = vtcm_base;
- cctx.spad1_base = vtcm_base + n_threads * kparams->u.transposed.spad0_size_per_thread;
- cctx.spad0_size_per_thread = kparams->u.transposed.spad0_size_per_thread;
- cctx.spad1_size_per_thread = kparams->u.transposed.spad1_size_per_thread;
+ cctx.spad0_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad0);
+ cctx.spad1_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad1);
+ cctx.spad0_size_per_thread = layout.src0_spad_size_per_thread;
+ cctx.spad1_size_per_thread = layout.src1_spad_size_per_thread;
cctx.row_start = row_start;
cctx.nrows = nrows;
cctx.nplanes = dst->ne[2] * dst->ne[3];
- cctx.div_ne2 = init_fastdiv_values(dst->ne[2]);
+ cctx.div_ne2 = kparams->u.transposed.div_ne2;
cctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
work_queue_func_t worker_func = (type_size == 4) ? concat_2d_f32_transposed : concat_2d_f16_transposed;
diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.h b/ggml/src/ggml-hexagon/htp/concat-ops.h
index 4bfba9894..a5d93284f 100644
--- a/ggml/src/ggml-hexagon/htp/concat-ops.h
+++ b/ggml/src/ggml-hexagon/htp/concat-ops.h
@@ -2,6 +2,8 @@
#define HTP_CONCAT_OPS_H
#include "hex-common.h"
+#include "hex-fastdiv.h"
+#include "htp-vtcm.h"
#include "gather-rows.h"
#include <stdint.h>
@@ -13,8 +15,7 @@ enum htp_concat_kernel_type {
};
struct htp_concat_transposed_params {
- uint32_t spad0_size_per_thread;
- uint32_t spad1_size_per_thread;
+ struct fastdiv_values div_ne2;
};
struct htp_concat_kernel_params {
@@ -40,11 +41,13 @@ _Static_assert(sizeof(struct htp_concat_kernel_params) <= 128, "htp_concat_kerne
struct htp_concat_transposed_vtcm_layout {
uint32_t src0_spad_size_per_thread;
uint32_t src1_spad_size_per_thread;
+ size_t off_spad0;
+ size_t off_spad1;
uint32_t total_bytes;
};
static inline void htp_concat_transposed_vtcm_layout_build(
- struct htp_concat_transposed_vtcm_layout * layout,
+ struct htp_concat_transposed_vtcm_layout * L,
uint32_t src0_ne0,
uint32_t src1_ne0,
uint32_t type_size,
@@ -55,9 +58,13 @@ static inline void htp_concat_transposed_vtcm_layout_build(
uint32_t src1_ne0_padded = hex_round_up(src1_ne0, block_i);
uint32_t spad0_row_bytes = hex_round_up(src0_ne0 * type_size, 128) + src1_ne0_padded * type_size;
- layout->src0_spad_size_per_thread = block_i * spad0_row_bytes;
- layout->src1_spad_size_per_thread = src1_ne0_padded * spad1_stride;
- layout->total_bytes = n_threads * (layout->src0_spad_size_per_thread + layout->src1_spad_size_per_thread);
+ L->src0_spad_size_per_thread = block_i * spad0_row_bytes;
+ L->src1_spad_size_per_thread = src1_ne0_padded * spad1_stride;
+
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_spad0, (size_t) n_threads * L->src0_spad_size_per_thread);
+ VTCM_LAYOUT_ALLOC(off, off_spad1, (size_t) n_threads * L->src1_spad_size_per_thread);
+ L->total_bytes = (uint32_t) off;
}
#endif // HTP_CONCAT_OPS_H
diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c
index 63ebb38e6..fce45c2d4 100644
--- a/ggml/src/ggml-hexagon/htp/cpy-ops.c
+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c
@@ -32,6 +32,7 @@ struct htp_copy_context {
uint32_t nelem;
uint32_t elem_per_thread;
+ struct htp_copy_convert_vtcm_layout layout;
uint8_t * vtcm_src0;
uint8_t * vtcm_dst;
};
@@ -176,13 +177,14 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
\
const struct htp_copy_convert_params * cvt = &ct->kparams->u.convert; \
- const uint32_t src0_buf_size = cvt->src0_buf_size; \
- const uint32_t dst_buf_size = cvt->dst_buf_size; \
- const uint32_t src0_row_stride = cvt->src0_row_stride; \
- const uint32_t dst_row_stride = cvt->dst_row_stride; \
- const uint32_t blk_rows = cvt->blk_rows; \
- uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * cvt->spad0_size_per_thread; \
- uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * cvt->spad1_size_per_thread; \
+ const struct htp_copy_convert_vtcm_layout * L = &ct->layout; \
+ const uint32_t src0_buf_size = L->src0_buf_size; \
+ const uint32_t dst_buf_size = L->dst_buf_size; \
+ const uint32_t src0_row_stride = L->src0_row_stride; \
+ const uint32_t dst_row_stride = L->dst_row_stride; \
+ const uint32_t blk_rows = L->blk_rows; \
+ uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * L->spad0_size_per_thread; \
+ uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * L->spad1_size_per_thread; \
const uint32_t src0_row_size = ne00 * ct->kparams->src0_type_size; \
const uint32_t dst_row_size = ne00 * ct->kparams->dst_type_size; \
\
@@ -254,17 +256,16 @@ static int cpy_scalar(struct htp_ops_context * octx, const struct htp_copy_kerne
}
dma_queue * dma_q = octx->ctx->dma[0];
- dma_addr_t s_vtcm = (dma_addr_t)(uintptr_t) octx->ctx->vtcm_base;
- dma_addr_t d_vtcm = s_vtcm + VLEN;
+ uint8_t * s_ptr = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, 0);
+ uint8_t * d_ptr = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, VLEN);
+ dma_addr_t s_vtcm = (dma_addr_t)(uintptr_t) s_ptr;
+ dma_addr_t d_vtcm = (dma_addr_t)(uintptr_t) d_ptr;
const uint32_t s_size = kparams->src0_type_size;
const uint32_t d_size = kparams->dst_type_size;
dma_queue_push(dma_q, dma_make_data(s_vtcm, src0->data), s_size, s_size, s_size, 1);
dma_queue_pop(dma_q);
- uint8_t * s_ptr = (uint8_t *) octx->ctx->vtcm_base;
- uint8_t * d_ptr = s_ptr + VLEN;
-
const HVX_Vector v_src = hvx_vmem(s_ptr);
HVX_Vector v_dst;
@@ -366,6 +367,8 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp
const struct htp_copy_convert_params * cvt = &kparams->u.convert;
struct htp_copy_context ct;
+ htp_copy_convert_vtcm_layout_build(&ct.layout, src0->ne[0], kparams->src0_type_size, kparams->dst_type_size, cvt->blk_rows, n_threads);
+
ct.octx = octx;
ct.kparams = kparams;
ct.row_start = row_start;
@@ -373,8 +376,8 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp
ct.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
- ct.vtcm_src0 = vtcm_base;
- ct.vtcm_dst = vtcm_base + (size_t) n_threads * cvt->spad0_size_per_thread;
+ ct.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, ct.layout.off_src0);
+ ct.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, ct.layout.off_dst);
work_queue_func_t copy_fun = NULL;
if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) {
diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.h b/ggml/src/ggml-hexagon/htp/cpy-ops.h
index 16695c2ab..c3b97c0ca 100644
--- a/ggml/src/ggml-hexagon/htp/cpy-ops.h
+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.h
@@ -3,6 +3,7 @@
#include "hex-common.h"
#include "hex-fastdiv.h"
+#include "htp-vtcm.h"
#include "gather-rows.h"
#include <stdint.h>
@@ -17,12 +18,6 @@ enum htp_copy_kernel_type {
};
struct htp_copy_convert_params {
- uint32_t src0_buf_size;
- uint32_t dst_buf_size;
- uint32_t spad0_size_per_thread;
- uint32_t spad1_size_per_thread;
- uint32_t src0_row_stride; // row pitch in the VTCM buffers
- uint32_t dst_row_stride;
uint32_t blk_rows; // rows moved per DMA descriptor
struct fastdiv_values div_ne01;
struct fastdiv_values div_ne02_ne01;
@@ -60,6 +55,9 @@ struct htp_copy_kernel_params {
#define HTP_COPY_VTCM_BLK_BYTES (32 * 1024)
struct htp_copy_convert_vtcm_layout {
+ size_t total_bytes;
+ size_t off_src0;
+ size_t off_dst;
uint32_t src0_row_stride;
uint32_t dst_row_stride;
uint32_t blk_rows;
@@ -67,11 +65,35 @@ struct htp_copy_convert_vtcm_layout {
uint32_t dst_buf_size;
uint32_t spad0_size_per_thread;
uint32_t spad1_size_per_thread;
- uint32_t total_bytes;
};
-// ne01 bounds a block: rows of one block share the dim-1 stride.
static inline void htp_copy_convert_vtcm_layout_build(
+ struct htp_copy_convert_vtcm_layout * L,
+ uint32_t ne00,
+ uint32_t src_type_size,
+ uint32_t dst_type_size,
+ uint32_t blk_rows,
+ uint32_t n_threads) {
+
+ const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256);
+ const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256);
+
+ L->src0_row_stride = src0_row_stride;
+ L->dst_row_stride = dst_row_stride;
+ L->blk_rows = blk_rows;
+ L->src0_buf_size = blk_rows * src0_row_stride;
+ L->dst_buf_size = blk_rows * dst_row_stride;
+ L->spad0_size_per_thread = 2 * L->src0_buf_size;
+ L->spad1_size_per_thread = 2 * L->dst_buf_size;
+
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_src0, (size_t) n_threads * L->spad0_size_per_thread);
+ VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) n_threads * L->spad1_size_per_thread);
+ L->total_bytes = off;
+}
+
+// ne01 bounds a block: rows of one block share the dim-1 stride.
+static inline void htp_copy_convert_solve_layout(
struct htp_copy_convert_vtcm_layout * layout,
uint32_t ne00,
uint32_t ne01,
@@ -79,10 +101,10 @@ static inline void htp_copy_convert_vtcm_layout_build(
uint32_t dst_type_size,
uint32_t n_threads) {
- layout->src0_row_stride = hex_round_up(ne00 * src_type_size, 256);
- layout->dst_row_stride = hex_round_up(ne00 * dst_type_size, 256);
+ const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256);
+ const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256);
- const uint32_t max_row = layout->src0_row_stride > layout->dst_row_stride ? layout->src0_row_stride : layout->dst_row_stride;
+ const uint32_t max_row = src0_row_stride > dst_row_stride ? src0_row_stride : dst_row_stride;
uint32_t blk_rows = HTP_COPY_VTCM_BLK_BYTES / max_row;
if (blk_rows > ne01) {
blk_rows = ne01;
@@ -90,13 +112,8 @@ static inline void htp_copy_convert_vtcm_layout_build(
if (blk_rows == 0) {
blk_rows = 1;
}
- layout->blk_rows = blk_rows;
- layout->src0_buf_size = blk_rows * layout->src0_row_stride;
- layout->dst_buf_size = blk_rows * layout->dst_row_stride;
- layout->spad0_size_per_thread = 2 * layout->src0_buf_size;
- layout->spad1_size_per_thread = 2 * layout->dst_buf_size;
- layout->total_bytes = n_threads * (layout->spad0_size_per_thread + layout->spad1_size_per_thread);
+ htp_copy_convert_vtcm_layout_build(layout, ne00, src_type_size, dst_type_size, blk_rows, n_threads);
}
#if defined(__cplusplus)
diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.c b/ggml/src/ggml-hexagon/htp/cumsum-ops.c
index eaab7d7e5..36fc81276 100644
--- a/ggml/src/ggml-hexagon/htp/cumsum-ops.c
+++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.c
@@ -12,9 +12,11 @@
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
+#include "htp-vtcm.h"
#include "hvx-types.h"
#include "hvx-utils.h"
#include "dma-queue.h"
+#include "cumsum-ops.h"
#define htp_cumsum_tensors_preamble \
const struct htp_tensor * restrict src0 = octx->src[0]; \
@@ -89,8 +91,8 @@ static inline HVX_Vector hvx_splat_last_f32(HVX_Vector v) {
}
static inline void hvx_cumsum_row_f32(const float * restrict src, float * restrict dst, uint32_t n) {
- const uint32_t nvec = n / VLEN_FP32;
- const uint32_t nloe = n % VLEN_FP32;
+ const uint32_t nvec = n >> 5;
+ const uint32_t nloe = n & (VLEN_FP32 - 1);
HVX_Vector carry = Q6_V_vsplat_R(0);
@@ -210,16 +212,19 @@ static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) {
int op_cumsum_f32(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
+ const struct htp_cumsum_kernel_params * kparams =
+ (const struct htp_cumsum_kernel_params *) octx->kernel_params;
- const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const size_t dst_data_row_size = dst->ne[0] * sizeof(float);
+ const uint32_t total_rows = kparams->total_rows;
+ const size_t src_row_size = src0->nb[1];
+ const size_t dst_row_size = dst->nb[1];
uint32_t row_start = 0;
uint32_t nrows = total_rows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
- htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk);
+ htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
@@ -229,43 +234,45 @@ int op_cumsum_f32(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
-
- const size_t src_row_size = src0->nb[1];
- const size_t dst_row_size = dst->nb[1];
- const size_t src_row_size_aligned = hex_round_up(src_row_size, VLEN);
- const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);
-
- // 2 ping-pong buffers per thread for src and dst
- const size_t spad_per_thread = 2 * (src_row_size_aligned + dst_row_size_aligned);
+ if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
- octx->src0_spad.size_per_thread = src_row_size_aligned * 2;
- octx->dst_spad.size_per_thread = dst_row_size_aligned * 2;
+ const uint32_t n_threads = octx->n_threads;
- octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread;
- octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread;
+ if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {
+ return HTP_STATUS_NO_SUPPORT;
+ }
- octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL;
- octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->dst_spad.src = NULL;
+ struct htp_cumsum_vtcm_layout layout = { 0 };
+ if (kparams->use_dma) {
+ htp_cumsum_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_threads);
+
+ octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread;
+ octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread;
+ octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread;
+ octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread;
+ octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0);
+ octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst);
+ octx->src0_spad.src = NULL;
+ octx->dst_spad.src = NULL;
+ }
struct htp_cumsum_context cctx = {
.octx = octx,
.src_row_size = src_row_size,
.dst_row_size = dst_row_size,
- .src_row_size_aligned = src_row_size_aligned,
- .dst_row_size_aligned = dst_row_size_aligned,
+ .src_row_size_aligned = layout.src_row_size_aligned,
+ .dst_row_size_aligned = layout.dst_row_size_aligned,
.rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
.total_rows = nrows,
.row_start = row_start,
};
- if (octx->ctx->vtcm_size < spad_per_thread * n_threads) {
- if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {
- return HTP_STATUS_NO_SUPPORT;
- }
- work_queue_run(octx->ctx->work_queue, cumsum_thread_f32, &cctx, n_threads);
- } else {
+ if (kparams->use_dma) {
work_queue_run(octx->ctx->work_queue, cumsum_thread_f32_dma, &cctx, n_threads);
+ } else {
+ work_queue_run(octx->ctx->work_queue, cumsum_thread_f32, &cctx, n_threads);
}
return HTP_STATUS_OK;
diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.h b/ggml/src/ggml-hexagon/htp/cumsum-ops.h
new file mode 100644
index 000000000..6e1ac316f
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.h
@@ -0,0 +1,49 @@
+#ifndef HTP_CUMSUM_OPS_H
+#define HTP_CUMSUM_OPS_H
+
+#include "hex-common.h"
+#include "htp-vtcm.h"
+#include <stdint.h>
+
+struct htp_cumsum_vtcm_layout {
+ size_t total_bytes;
+ size_t off_src0;
+ size_t off_dst;
+ size_t src_row_size_aligned;
+ size_t dst_row_size_aligned;
+ size_t src0_bytes_per_thread;
+ size_t dst_bytes_per_thread;
+};
+
+static inline void htp_cumsum_vtcm_layout_build(
+ struct htp_cumsum_vtcm_layout * L,
+ size_t src_row_size,
+ size_t dst_row_size,
+ uint32_t n_threads
+) {
+ L->src_row_size_aligned = hex_round_up(src_row_size, 128);
+ L->dst_row_size_aligned = hex_round_up(dst_row_size, 128);
+ L->src0_bytes_per_thread = 2 * L->src_row_size_aligned;
+ L->dst_bytes_per_thread = 2 * L->dst_row_size_aligned;
+
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+
+ L->total_bytes = off;
+}
+
+struct htp_cumsum_kernel_params {
+ uint32_t n_threads;
+ uint32_t total_rows;
+ uint32_t use_dma;
+ uint32_t vtcm_size;
+};
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_cumsum_kernel_params) <= 128, "htp_cumsum_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_cumsum_kernel_params) <= 128, "htp_cumsum_kernel_params is too large for kernel_params blob");
+#endif
+
+#endif // HTP_CUMSUM_OPS_H
diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.c b/ggml/src/ggml-hexagon/htp/diag-ops.c
index 162214d3e..7f03e8550 100644
--- a/ggml/src/ggml-hexagon/htp/diag-ops.c
+++ b/ggml/src/ggml-hexagon/htp/diag-ops.c
@@ -10,10 +10,12 @@
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
+#include "htp-vtcm.h"
#include "hvx-types.h"
#include "hex-utils.h"
#include "hvx-copy.h"
#include "dma-queue.h"
+#include "diag-ops.h"
#define htp_diag_tensors_preamble \
const struct htp_tensor * restrict src0 = octx->src[0]; \
@@ -33,6 +35,7 @@
struct htp_diag_context {
struct htp_ops_context * octx;
+ const struct htp_diag_kernel_params * kparams;
size_t src_batch_size;
size_t dst_row_size;
size_t src_batch_size_aligned;
@@ -83,8 +86,8 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data)
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ib = ib0; ib < ib1; ib++) {
- const uint32_t i3 = ib / ne02;
- const uint32_t i2 = ib % ne02;
+ const uint32_t i3 = fastdiv(ib, &dctx->kparams->div_ne02);
+ const uint32_t i2 = ib - i3 * ne02;
const dma_addr_t src_batch = src_data + i3 * nb03 + i2 * nb02;
@@ -134,8 +137,8 @@ static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) {
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0);
for (uint32_t ib = ib0; ib < ib1; ib++) {
- const uint32_t i3 = ib / ne02;
- const uint32_t i2 = ib % ne02;
+ const uint32_t i3 = fastdiv(ib, &dctx->kparams->div_ne02);
+ const uint32_t i2 = ib - i3 * ne02;
const float * restrict src_batch = (const float *)(src_data + i3 * nb03 + i2 * nb02);
@@ -155,29 +158,17 @@ static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) {
int op_diag_f32(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
+ const struct htp_diag_kernel_params * kparams =
+ (const struct htp_diag_kernel_params *) octx->kernel_params;
- const uint32_t total_batches = src0->ne[2] * src0->ne[3];
- const size_t dst_batch_size = dst->ne[1] * dst->nb[1];
+ const uint32_t total_batches = kparams->total_batches;
uint32_t batch_start = 0;
uint32_t nbatches = total_batches;
if (octx->ctx->mdev.count > 1) {
- bool can_split = htp_tensor_mdev_data_aligned(dst) && (dst->ne[0] == 1 || dst->nb[0] == sizeof(float)) && !htp_tensor_is_permuted(dst);
- uint32_t batches_per_chunk = 1;
- if (can_split) {
- if (dst->ne[2] > 1 && (dst->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 &&
- (dst->ne[3] <= 1 || (dst->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0)) {
- batches_per_chunk = 1;
- } else if (dst->nb[2] == dst_batch_size &&
- (dst->ne[3] <= 1 || dst->nb[3] == dst->nb[2] * dst->ne[2])) {
- batches_per_chunk = (dst_batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_batch_size, HEX_L2_LINE_SIZE)) : 1;
- } else {
- can_split = false;
- }
- }
-
- const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, can_split ? batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
+ const bool can_split = (kparams->can_split != 0) && htp_tensor_mdev_data_aligned(dst);
+ const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, can_split ? kparams->batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
batch_start = range.start;
nbatches = range.count;
}
@@ -186,43 +177,49 @@ int op_diag_f32(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
-
- const size_t src_batch_size = src0->ne[0] * sizeof(float);
- const size_t dst_row_size = dst->ne[0] * sizeof(float);
- const size_t src_batch_size_aligned = hex_round_up(src_batch_size, VLEN);
- const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);
-
- // 1 src buffer + 1 dst row buffer per thread
- const size_t spad_per_thread = src_batch_size_aligned + dst_row_size_aligned;
+ if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nbatches))) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
- octx->src0_spad.size_per_thread = src_batch_size_aligned;
- octx->dst_spad.size_per_thread = dst_row_size_aligned;
+ const uint32_t n_threads = octx->n_threads;
- octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread;
- octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread;
+ if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {
+ return HTP_STATUS_NO_SUPPORT;
+ }
- octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL;
- octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->dst_spad.src = NULL;
+ const size_t src_batch_size = src0->ne[0] * sizeof(float);
+ const size_t dst_row_size = dst->ne[0] * sizeof(float);
+
+ struct htp_diag_vtcm_layout layout = { 0 };
+ if (kparams->use_dma) {
+ htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_threads);
+
+ octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread;
+ octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread;
+ octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread;
+ octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread;
+ octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0);
+ octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst);
+ octx->src0_spad.src = NULL;
+ octx->dst_spad.src = NULL;
+ }
struct htp_diag_context dctx = {
.octx = octx,
+ .kparams = kparams,
.src_batch_size = src_batch_size,
.dst_row_size = dst_row_size,
- .src_batch_size_aligned = src_batch_size_aligned,
- .dst_row_size_aligned = dst_row_size_aligned,
+ .src_batch_size_aligned = layout.src_batch_size_aligned,
+ .dst_row_size_aligned = layout.dst_row_size_aligned,
.batches_per_thread = fastdiv(nbatches + n_threads - 1, &octx->n_threads_div),
.total_batches = nbatches,
.batch_start = batch_start,
};
- if (octx->ctx->vtcm_size < spad_per_thread * n_threads) {
- if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {
- return HTP_STATUS_NO_SUPPORT;
- }
- work_queue_run(octx->ctx->work_queue, diag_thread_f32, &dctx, n_threads);
- } else {
+ if (kparams->use_dma) {
work_queue_run(octx->ctx->work_queue, diag_thread_f32_dma, &dctx, n_threads);
+ } else {
+ work_queue_run(octx->ctx->work_queue, diag_thread_f32, &dctx, n_threads);
}
return HTP_STATUS_OK;
diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.h b/ggml/src/ggml-hexagon/htp/diag-ops.h
new file mode 100644
index 000000000..983bc0242
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/diag-ops.h
@@ -0,0 +1,54 @@
+#ifndef HTP_DIAG_OPS_H
+#define HTP_DIAG_OPS_H
+
+#include "hex-common.h"
+#include "hex-fastdiv.h"
+#include "htp-vtcm.h"
+#include <stdint.h>
+
+struct htp_diag_vtcm_layout {
+ size_t total_bytes;
+ size_t off_src0;
+ size_t off_dst;
+ size_t src_batch_size_aligned;
+ size_t dst_row_size_aligned;
+ size_t src0_bytes_per_thread;
+ size_t dst_bytes_per_thread;
+};
+
+static inline void htp_diag_vtcm_layout_build(
+ struct htp_diag_vtcm_layout * L,
+ size_t src_batch_size,
+ size_t dst_row_size,
+ uint32_t n_threads
+) {
+ L->src_batch_size_aligned = hex_round_up(src_batch_size, 128);
+ L->dst_row_size_aligned = hex_round_up(dst_row_size, 128);
+ L->src0_bytes_per_thread = L->src_batch_size_aligned;
+ L->dst_bytes_per_thread = L->dst_row_size_aligned;
+
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+
+ L->total_bytes = off;
+}
+
+struct htp_diag_kernel_params {
+ uint32_t n_threads;
+ uint32_t total_batches;
+ uint32_t use_dma;
+ uint32_t can_split;
+ uint32_t batches_per_chunk;
+ uint32_t vtcm_size;
+
+ struct fastdiv_values div_ne02;
+};
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_diag_kernel_params) <= 128, "htp_diag_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_diag_kernel_params) <= 128, "htp_diag_kernel_params is too large for kernel_params blob");
+#endif
+
+#endif // HTP_DIAG_OPS_H
diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.h
index 9b774bdd9..05f37305d 100644
--- a/ggml/src/ggml-hexagon/htp/dma-queue.h
+++ b/ggml/src/ggml-hexagon/htp/dma-queue.h
@@ -436,10 +436,10 @@ static inline void dma_sync_write(dma_queue * dma_q, dma_addr_t dst, const void
}
}
-#define DMA_CACHE_MAX_SIZE 256U
+#define DMA_CACHE_MAX_SIZE 128U
// Fully assoc LRU cache
-typedef struct {
+typedef struct __attribute__((aligned(128))) {
uint8_t *base;
uint32_t line_size;
uint32_t capacity;
@@ -485,7 +485,7 @@ static inline bool dma_cache_fa_push(dma_queue *q, dma_cache_fa *c, dma_addr_t s
}
// Direct mapped cache
-typedef struct {
+typedef struct __attribute__((aligned(128))) {
uint8_t *base;
uint32_t line_size;
uint32_t capacity;
diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.c b/ggml/src/ggml-hexagon/htp/fill-ops.c
index 212104a23..2cab31b5f 100644
--- a/ggml/src/ggml-hexagon/htp/fill-ops.c
+++ b/ggml/src/ggml-hexagon/htp/fill-ops.c
@@ -16,6 +16,7 @@
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
+#include "fill-ops.h"
// ggml op_params layout for FILL:
// op_params[0] (as float) - the scalar fill value
@@ -30,21 +31,19 @@
\
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
- const uint32_t nb3 = dst->nb[3]; \
- \
- const uint32_t nr = ne1 * ne2 * ne3;
+ const uint32_t nb3 = dst->nb[3];
struct htp_fill_context {
struct htp_ops_context * octx;
+ const struct htp_fill_kernel_params * kparams;
uint32_t nrows_per_thread;
uint32_t total_rows; // ne1 * ne2 * ne3
uint32_t row_start;
- bool opt_path;
HVX_Vector splat_vec;
uint32_t elem_size;
};
-static void fill_thread(unsigned int nth, unsigned int ith, void * data) {
+static void fill_thread_contiguous(unsigned int nth, unsigned int ith, void * data) {
const struct htp_fill_context * fctx = (const struct htp_fill_context *) data;
struct htp_ops_context * octx = fctx->octx;
fill_preamble;
@@ -60,45 +59,59 @@ static void fill_thread(unsigned int nth, unsigned int ith, void * data) {
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);
- if (fctx->opt_path) {
- // Opt path: tensor is fully contiguous, treat as flat array
- const uint32_t elem_start = ir0 * ne0;
- const uint32_t elem_end = ir1 * ne0;
- uint8_t * dst_ptr = (uint8_t *) dst->data + elem_start * fctx->elem_size;
- hvx_splat_u(dst_ptr, fctx->splat_vec, elem_end - elem_start, fctx->elem_size);
- } else {
- // Non-contiguous path: must respect strides
- for (uint32_t ir = ir0; ir < ir1; ++ir) {
- const uint32_t i1 = ir % ne1;
- const uint32_t i2 = (ir / ne1) % ne2;
- const uint32_t i3 = ir / (ne1 * ne2);
- uint8_t * dst_ptr = (uint8_t *) dst->data + i1*nb1 + i2*nb2 + i3*nb3;
- hvx_splat_u(dst_ptr, fctx->splat_vec, ne0, fctx->elem_size);
- }
- }
+ const uint32_t elem_start = ir0 * ne0;
+ const uint32_t elem_end = ir1 * ne0;
+ uint8_t * dst_ptr = (uint8_t *) dst->data + elem_start * fctx->elem_size;
+ hvx_splat_u(dst_ptr, fctx->splat_vec, elem_end - elem_start, fctx->elem_size);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1);
- FARF(HIGH, "fill %u/%u: rows %u:%u\n", ith, nth, ir0, ir1);
+ FARF(HIGH, "fill-contig %u/%u: rows %u:%u\n", ith, nth, ir0, ir1);
}
-int op_fill(struct htp_ops_context * octx) {
+static void fill_thread_strided(unsigned int nth, unsigned int ith, void * data) {
+ const struct htp_fill_context * fctx = (const struct htp_fill_context *) data;
+ struct htp_ops_context * octx = fctx->octx;
+ const struct htp_fill_kernel_params * kparams = fctx->kparams;
fill_preamble;
- if (dst->type != HTP_TYPE_F32 && dst->type != HTP_TYPE_F16) {
- return HTP_STATUS_NO_SUPPORT;
+ // Parallelise over the flat row index spanning ne1*ne2*ne3
+ const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith;
+ const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows);
+
+ if (ir0 >= ir1) {
+ return;
+ }
+
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);
+
+ for (uint32_t ir = ir0; ir < ir1; ++ir) {
+ const uint32_t i3 = fastdiv(ir, &kparams->div_ne1_ne2);
+ const uint32_t rem = ir - i3 * (ne1 * ne2);
+ const uint32_t i2 = fastdiv(rem, &kparams->div_ne1);
+ const uint32_t i1 = rem - i2 * ne1;
+ uint8_t * dst_ptr = (uint8_t *) dst->data + i1*nb1 + i2*nb2 + i3*nb3;
+ hvx_splat_u(dst_ptr, fctx->splat_vec, ne0, fctx->elem_size);
}
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1);
+ FARF(HIGH, "fill-strided %u/%u: rows %u:%u\n", ith, nth, ir0, ir1);
+}
+
+int op_fill(struct htp_ops_context * octx) {
+ const struct htp_tensor * dst = octx->dst;
+ const struct htp_fill_kernel_params * kparams =
+ (const struct htp_fill_kernel_params *) octx->kernel_params;
+
if (htp_tensor_is_extended(dst)) {
return HTP_STATUS_NO_SUPPORT;
}
uint32_t row_start = 0;
- uint32_t nrows = nr;
+ uint32_t nrows = kparams->total_rows;
if (octx->ctx->mdev.count > 1) {
- const uint32_t row_size = nb1;
- const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;
- const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(nr, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
+ const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
@@ -107,40 +120,43 @@ int op_fill(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- // nr = ne1*ne2*ne3 (flat row count across all outer dims); parallelise over it.
- const uint32_t n_threads = octx->n_threads;
+ if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
- // Optimize if fully contiguous: skip stride arithmetic, treat as flat array
- const bool opt_path = (nb2 == nb1 * ne1) && (nb3 == nb2 * ne2);
+ const uint32_t n_threads = octx->n_threads;
- FARF(HIGH, "fill: (%ux%ux%ux%u) type=%u opt=%d\n",
- dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], dst->type, (int) opt_path);
+ FARF(HIGH, "fill: (%ux%ux%ux%u) type=%u ktype=%u\n",
+ dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], dst->type, kparams->kernel_type);
float val_f32 = 0.f;
memcpy(&val_f32, &octx->op_params[0], sizeof(float));
struct htp_fill_context fctx = {
.octx = octx,
+ .kparams = kparams,
.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
.total_rows = nrows,
.row_start = row_start,
- .opt_path = opt_path,
+ .elem_size = kparams->elem_size,
};
switch (dst->type) {
case HTP_TYPE_F32:
fctx.splat_vec = hvx_vec_splat_f32(val_f32);
- fctx.elem_size = sizeof(float);
break;
case HTP_TYPE_F16:
fctx.splat_vec = hvx_vec_splat_f16((_Float16) val_f32);
- fctx.elem_size = sizeof(_Float16);
break;
default:
return HTP_STATUS_NO_SUPPORT;
}
- work_queue_run(octx->ctx->work_queue, fill_thread, &fctx, n_threads);
+ work_queue_func_t worker = (kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS)
+ ? fill_thread_contiguous
+ : fill_thread_strided;
+
+ work_queue_run(octx->ctx->work_queue, worker, &fctx, n_threads);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.h b/ggml/src/ggml-hexagon/htp/fill-ops.h
new file mode 100644
index 000000000..d62f99751
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/fill-ops.h
@@ -0,0 +1,29 @@
+#ifndef HTP_FILL_OPS_H
+#define HTP_FILL_OPS_H
+
+#include "hex-fastdiv.h"
+#include <stdint.h>
+
+enum htp_fill_kernel_type {
+ HTP_FILL_KERNEL_CONTIGUOUS = 0,
+ HTP_FILL_KERNEL_STRIDED = 1,
+};
+
+struct htp_fill_kernel_params {
+ uint8_t kernel_type;
+ uint32_t n_threads;
+ uint32_t total_rows;
+ uint32_t elem_size;
+ uint32_t rows_per_chunk;
+
+ struct fastdiv_values div_ne1;
+ struct fastdiv_values div_ne1_ne2;
+};
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_fill_kernel_params) <= 128, "htp_fill_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_fill_kernel_params) <= 128, "htp_fill_kernel_params is too large for kernel_params blob");
+#endif
+
+#endif // HTP_FILL_OPS_H
diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c
index 2cbd16e8a..fc5664185 100644
--- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c
+++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c
@@ -39,6 +39,26 @@
// Must be multiple of 32
#define FLASH_ATTN_BLOCK_SIZE (32 * 2)
+// Large scratch lives in ctx->ddr_spad to keep stack usage low.
+// Ops can run on the dspqueue callback thread, which has a small stack.
+struct hvx_fa_ddr_spad {
+ __fp16 slopes[512];
+ uint16_t active_blocks[FA_MAX_KV_BLOCKS];
+ uint16_t thread_active_blocks[HTP_MAX_NTHREADS][FA_MAX_KV_BLOCKS] __attribute__((aligned(128)));
+};
+
+struct hmx_fa_ddr_spad {
+ uint16_t active_blocks[FA_MAX_KV_BLOCKS];
+ __fp16 slopes[] __attribute__((aligned(128))); // [g_br]
+};
+
+static inline void * fa_get_ddr_spad(const struct htp_ops_context * octx, size_t size) {
+ if (!octx->ctx->ddr_spad_base || octx->ctx->ddr_spad_size < size) {
+ return NULL;
+ }
+ return octx->ctx->ddr_spad_base;
+}
+
struct htp_fa_context {
const struct htp_ops_context * octx;
@@ -61,7 +81,6 @@ struct htp_fa_context {
uint32_t n_head_log2;
float m0;
float m1;
- __fp16 slopes[512];
uint32_t n_blocks;
uint32_t n_active_blocks;
@@ -90,7 +109,9 @@ struct htp_fa_context {
uint8_t * spad_m;
uint8_t * spad_a;
float * spad_sinks;
- uint16_t active_blocks[FA_MAX_KV_BLOCKS];
+
+ struct hvx_fa_ddr_spad * ddr;
+ dma_cache_dm * m_cache; // [n_threads]
const struct htp_tensor * k;
const struct htp_tensor * v;
@@ -156,6 +177,7 @@ struct hmx_fa_context {
__fp16 * vtcm_mask_buf; // VTCM mask buffer [Br * m_line], DMA'd per KV block
__fp16 * vtcm_slopes; // ALiBi slopes [g_br]
float * vtcm_sinks; // Attention sinks
+ __fp16 * ddr_slopes; // ALiBi slopes staging buffer in DDR [g_br]
size_t row_buf_stride; // HVX vectors per row buffer (Bc/64)
size_t mask_buf_row_stride; // elements (__fp16) per row in mask buffer
size_t q_tile_bytes;
@@ -164,7 +186,7 @@ struct hmx_fa_context {
size_t col_vec_bytes;
size_t d_tile_bytes;
bool mask_broadcast; // true when mask->ne[2] == 1 (head-independent, single 2D DMA)
- dma_cache_fa m_cache;
+ dma_cache_fa * m_cache;
};
// Scan mask rows [q_start, q_start + n_rows_q) and build compact list of KV blocks with any entry != -inf.
@@ -308,11 +330,11 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
uint8_t * spad_m = factx->spad_m + (mask ? factx->size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * ith;
uint8_t * spad_a = factx->spad_a + factx->size_vkq_acc * ith;
- dma_cache_dm m_cache;
- dma_cache_dm_init(&m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE);
+ dma_cache_dm * m_cache = &factx->m_cache[ith];
+ dma_cache_dm_init(m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE);
- uint16_t thread_active_blocks[FA_MAX_KV_BLOCKS];
- const uint16_t * active_blocks = factx->shared_active_blocks ? factx->active_blocks : thread_active_blocks;
+ uint16_t * thread_active_blocks = factx->ddr->thread_active_blocks[ith];
+ const uint16_t * active_blocks = factx->shared_active_blocks ? factx->ddr->active_blocks : thread_active_blocks;
uint32_t n_active_blocks = factx->n_active_blocks;
dma_addr_t prev_mp_base = 0;
@@ -385,7 +407,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
const uint32_t r_iq1 = (r - r_iq3*neq2*neq1 - r_iq2 * neq1);
heads[g] = r_iq2;
- const __fp16 slope = factx->slopes[r_iq2];
+ const __fp16 slope = factx->ddr->slopes[r_iq2];
slope_vecs[g] = hvx_vec_splat_f16(slope);
S_vec[g] = hvx_vec_splat_f32(0.0f);
@@ -422,7 +444,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
// Mask
if (mask) {
const dma_addr_t m_src = mp_base + ic_start * sizeof(__fp16);
- dma_cache_dm_push(dma_q, &m_cache, ib, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1);
+ dma_cache_dm_push(dma_q, m_cache, ib, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1);
}
}
@@ -682,7 +704,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
// Mask
if (mask) {
const dma_addr_t m_src = mp_base + next_ic_start * sizeof(__fp16);
- dma_cache_dm_push(dma_q, &m_cache, next_ib, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1);
+ dma_cache_dm_push(dma_q, m_cache, next_ib, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1);
}
}
} // end for g
@@ -1887,7 +1909,7 @@ static __attribute__((noinline)) void fa_compute_slopes(
const float m0 = factx->m0;
const float m1 = factx->m1;
- __fp16 temp_slopes[512] __attribute__((aligned(128)));
+ __fp16 * temp_slopes = factx->ddr_slopes;
if (G <= 32) {
// Fast path: Compute G unique slope values in vector registers
HVX_Vector v_val = hvx_alibi_slopes(kv_head, G, n_head_log2, m0, m1);
@@ -1905,14 +1927,13 @@ static __attribute__((noinline)) void fa_compute_slopes(
}
}
- // Allocate stack buffer to avoid scalar writes to VTCM (which generates L2 misses)
- __fp16 local_slopes[n_rows_g] __attribute__((aligned(128)));
- for (size_t r = 0; r < n_rows_g; ++r) {
- local_slopes[r] = temp_slopes[fastmodulo(r, G, &factx->div_G)];
+ // Replicate the G values in DDR to avoid scalar writes to VTCM (which generates L2 misses)
+ for (size_t r = G; r < n_rows_g; ++r) {
+ temp_slopes[r] = temp_slopes[r - G];
}
// Copy to VTCM slopes using HVX block copy (both are aligned to 128 bytes)
- hvx_copy_f16_aa((uint8_t *)slopes, (const uint8_t *)local_slopes, n_rows_g);
+ hvx_copy_f16_aa((uint8_t *)slopes, (const uint8_t *)temp_slopes, n_rows_g);
}
static void fa_push_mask_dma_gqa(
@@ -1958,7 +1979,7 @@ static inline void fa_prefetch_block(dma_queue * dma_q, const struct htp_tensor
if (mask) {
if (__builtin_expect(factx->mask_broadcast, true)) {
const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + prefetch_start * sizeof(__fp16);
- dma_cache_fa_push(dma_q, &factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q);
+ dma_cache_fa_push(dma_q, factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q);
} else {
fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, prefetch_start, kv_head, G, m_line_bytes, prefetch_rows, n_rows_q, factx);
}
@@ -2151,7 +2172,14 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
const size_t m_line_bytes = L.m_line_bytes; // used by the mask DMAs in the KV loop
- dma_cache_fa_init(&factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE);
+ struct hmx_fa_ddr_spad * ddr = (struct hmx_fa_ddr_spad *) fa_get_ddr_spad(octx, sizeof(struct hmx_fa_ddr_spad) + factx.g_br * sizeof(__fp16));
+ if (!ddr) {
+ return HTP_STATUS_INTERNAL_ERR;
+ }
+
+ factx.ddr_slopes = ddr->slopes;
+ factx.m_cache = &octx->dma_cache_fa[0];
+ dma_cache_fa_init(factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE);
// Head-dim padding: the K/V DMA staging buffers and the flat-Q buffer are laid out
// with padded row strides (size_{k,v,q}_row_padded, covering D_pad columns) but the
@@ -2210,7 +2238,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
const size_t g_br_actual = hex_align_up(n_rows_g, HMX_FP16_TILE_N_ROWS);
const size_t n_row_tiles = g_br_actual / HMX_FP16_TILE_N_ROWS;
- uint16_t active_blocks[FA_MAX_KV_BLOCKS];
+ uint16_t * active_blocks = ddr->active_blocks;
uint32_t n_active_blocks = factx.n_kv_blocks;
if (mask && factx.mask_broadcast && factx.n_kv_blocks > 1) {
const size_t scratch_bytes = factx.k_tile_bytes * (factx.pipeline ? 2 : 1);
@@ -2252,7 +2280,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
if (factx.pipeline && mask) {
if (__builtin_expect(factx.mask_broadcast, true)) {
const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + b0_start * sizeof(__fp16);
- dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], b0_rows * sizeof(__fp16), n_rows_q);
+ dma_cache_fa_push(dma_q, factx.m_cache, ms_src, m_line_bytes, mask->nb[1], b0_rows * sizeof(__fp16), n_rows_q);
} else {
fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, b0_start, kv_head, G, m_line_bytes, b0_rows, n_rows_q, &factx);
}
@@ -2453,7 +2481,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
if (mask) {
if (__builtin_expect(factx.mask_broadcast, true)) {
const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + kv_start * sizeof(__fp16);
- dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q);
+ dma_cache_fa_push(dma_q, factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q);
} else {
fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, kv_start, kv_head, G, m_line_bytes, kv_rows, n_rows_q, &factx);
}
@@ -2577,7 +2605,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
if (factx.pipeline && mask) {
if (__builtin_expect(factx.mask_broadcast, true)) {
const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + b0_start * sizeof(__fp16);
- dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], b0_rows * sizeof(__fp16), n_rows_q);
+ dma_cache_fa_push(dma_q, factx.m_cache, ms_src, m_line_bytes, mask->nb[1], b0_rows * sizeof(__fp16), n_rows_q);
} else {
fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, b0_start, next_kv_head, G, m_line_bytes, b0_rows, n_rows_q, &factx);
}
@@ -2680,8 +2708,15 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
if (n_head > 512) {
return HTP_STATUS_NO_SUPPORT;
}
+
+ factx.ddr = (struct hvx_fa_ddr_spad *) fa_get_ddr_spad(octx, sizeof(struct hvx_fa_ddr_spad));
+ if (!factx.ddr) {
+ return HTP_STATUS_INTERNAL_ERR;
+ }
+ factx.m_cache = octx->dma_cache_dm;
+
for (uint32_t h = 0; h < n_head; ++h) {
- factx.slopes[h] = (__fp16) ((kparams->max_bias > 0.0f) ? alibi_slope(h, factx.n_head_log2, factx.m0, factx.m1) : 1.0f);
+ factx.ddr->slopes[h] = (__fp16) ((kparams->max_bias > 0.0f) ? alibi_slope(h, factx.n_head_log2, factx.m0, factx.m1) : 1.0f);
}
// total rows in q
@@ -2754,7 +2789,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
if (has_mask && neq1 == 1 && neq3 == 1 && mask->ne[2] == 1 && factx.n_blocks > 1 && factx.n_blocks <= FA_MAX_KV_BLOCKS) {
factx.n_active_blocks = fa_scan_active_blocks(
octx->ctx->dma[0], mask, 0, 1, 0, 0, k->ne[1], FLASH_ATTN_BLOCK_SIZE,
- factx.n_blocks, factx.spad_k, factx.size_k_block * 2, factx.active_blocks);
+ factx.n_blocks, factx.spad_k, factx.size_k_block * 2, factx.ddr->active_blocks);
factx.shared_active_blocks = true;
}
diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h
index 5f6a1e3e6..a3d380501 100644
--- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h
+++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h
@@ -35,7 +35,6 @@ enum htp_fa_kernel_type {
struct htp_fa_kernel_params {
uint8_t kernel_type; // enum htp_fa_kernel_type
uint8_t head_split; // 1 = partition by KV heads in multicore, 0 = token partition
- uint8_t flags; // reserved
uint8_t n_threads; // Number of threads to run
// Common parameters
@@ -49,9 +48,6 @@ struct htp_fa_kernel_params {
float logit_softcap;
uint32_t vtcm_size;
- uint32_t qrows;
- uint32_t qrows_per_thread;
- uint32_t qrow_start;
float m0;
float m1;
uint32_t n_head_log2;
@@ -67,8 +63,6 @@ struct htp_fa_kernel_params {
union {
struct {
uint32_t g_br;
- uint32_t row_buf_stride;
- uint32_t mask_buf_row_stride;
int32_t mask_broadcast;
int32_t pipeline;
struct fastdiv_values div_G;
@@ -85,6 +79,8 @@ struct htp_fa_kernel_params {
#if defined(__cplusplus)
static_assert(sizeof(struct htp_fa_kernel_params) <= 128, "htp_fa_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_fa_kernel_params) <= 128, "htp_fa_kernel_params is too large for kernel_params blob");
#endif
// VTCM region layout for the HMX flash-attention kernel.
diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
index 1dd828db7..5252ac352 100644
--- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
+++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
@@ -2270,46 +2270,9 @@ static int gated_delta_net_f32_hmx_chunked(
}
int op_gated_delta_net(struct htp_ops_context * octx) {
- const struct htp_tensor * q = octx->src[0];
- const struct htp_tensor * k = octx->src[1];
- const struct htp_tensor * v = octx->src[2];
- const struct htp_tensor * g = octx->src[3];
- const struct htp_tensor * beta = octx->src[4];
- const struct htp_tensor * state = octx->src[5];
- const struct htp_tensor * dst = octx->dst;
-
- if (q->type != HTP_TYPE_F32 || k->type != HTP_TYPE_F32 || v->type != HTP_TYPE_F32 ||
- g->type != HTP_TYPE_F32 || beta->type != HTP_TYPE_F32 || state->type != HTP_TYPE_F32 ||
- dst->type != HTP_TYPE_F32) {
- return HTP_STATUS_NO_SUPPORT;
- }
-
- const uint32_t S_v = v->ne[0];
- const uint32_t H = v->ne[1];
- const uint32_t n_tokens = v->ne[2];
- const uint32_t n_seqs = v->ne[3];
- const uint32_t K = octx->op_params[0];
-
- if (S_v == 0 || S_v > HTP_GDN_MAX_SV || H == 0 || n_tokens == 0 || n_seqs == 0) {
- return HTP_STATUS_NO_SUPPORT;
- }
- if ((g->ne[0] != 1 && g->ne[0] != S_v) || beta->ne[0] != 1) {
- return HTP_STATUS_NO_SUPPORT;
- }
- if (q->ne[0] != S_v || k->ne[0] != S_v || q->ne[1] == 0 || k->ne[1] == 0 ||
- q->ne[2] != n_tokens || k->ne[2] != n_tokens || q->ne[3] == 0 || k->ne[3] == 0 ||
- (n_seqs % q->ne[3]) != 0 || (n_seqs % k->ne[3]) != 0) {
- return HTP_STATUS_NO_SUPPORT;
- }
- // state holds s0 only: [S_v, S_v, H, n_seqs]
- if (state->ne[0] != S_v || state->ne[1] != S_v || state->ne[2] != H || state->ne[3] != n_seqs) {
- return HTP_STATUS_NO_SUPPORT;
- }
- if (dst->ne[0] != S_v * H || dst->ne[1] != n_tokens * n_seqs + S_v * n_seqs * K) {
- return HTP_STATUS_NO_SUPPORT;
- }
+ const struct htp_tensor * dst = octx->dst;
- for (int i = 0; i < 5; i++) {
+ for (int i = 0; i < 6; i++) {
if (htp_tensor_is_extended(octx->src[i])) {
return HTP_STATUS_NO_SUPPORT;
}
@@ -2325,68 +2288,12 @@ int op_gated_delta_net(struct htp_ops_context * octx) {
}
const struct htp_gdn_kernel_params * kparams = (const struct htp_gdn_kernel_params *) octx->kernel_params;
- struct htp_gdn_kernel_params kparams_local;
- if (!kparams || kparams->S_v == 0) {
- const uint32_t rq3 = n_seqs / q->ne[3];
- const uint32_t rk3 = n_seqs / k->ne[3];
- const uint32_t total_rows = H * n_seqs;
- uint32_t n_threads = (total_rows < octx->n_threads) ? total_rows : octx->n_threads;
- if (n_threads == 0) {
- n_threads = 1;
- }
-
- memset(&kparams_local, 0, sizeof(kparams_local));
- kparams_local.n_threads = n_threads;
- kparams_local.S_v = S_v;
- kparams_local.H = H;
- kparams_local.n_tokens = n_tokens;
- kparams_local.n_seqs = n_seqs;
- kparams_local.K = K;
- kparams_local.total_rows = total_rows;
- kparams_local.rows_per_thread = (total_rows + n_threads - 1) / n_threads;
- const bool can_use_hmx = (octx->ctx->hmx_enabled) &&
- (S_v % 64 == 0) &&
- (n_tokens >= HTP_GDN_MIN_TOKENS) &&
- (g->ne[0] == 1) &&
- (K == 1);
-
- struct htp_gdn_hmx_vtcm_layout hmx_layout_local;
- struct htp_gdn_vtcm_layout hvx_layout_local;
- uint32_t n_heads_batch = 1;
-
- if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout_local, S_v, HTP_GDN_CHUNK_SIZE, total_rows, octx->ctx->vtcm_size, n_threads, true, &n_heads_batch)) {
- kparams_local.kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED;
- kparams_local.pipeline = hmx_layout_local.pipeline ? 1 : 0;
- kparams_local.chunk_size = HTP_GDN_CHUNK_SIZE;
- kparams_local.n_chunks = (n_tokens + HTP_GDN_CHUNK_SIZE - 1) / HTP_GDN_CHUNK_SIZE;
- kparams_local.n_heads_batch = (uint16_t) n_heads_batch;
- kparams_local.vtcm_size = (uint32_t) hmx_layout_local.total_bytes;
- kparams_local.state_aligned = (uint32_t) hmx_layout_local.state_f32_bytes;
- kparams_local.vtcm_per_thread = (uint32_t) (hmx_layout_local.total_bytes / (n_threads > 0 ? n_threads : 1));
- } else {
- htp_gdn_vtcm_layout_build(&hvx_layout_local, S_v, n_threads);
- kparams_local.kernel_type = HTP_GDN_KERNEL_HVX_RECURRENT;
- kparams_local.pipeline = 0;
- kparams_local.n_heads_batch = 1;
- kparams_local.state_aligned = (uint32_t) hvx_layout_local.state_aligned;
- kparams_local.vtcm_per_thread = (uint32_t) hvx_layout_local.bytes_per_thread;
- kparams_local.vtcm_size = (uint32_t) hvx_layout_local.total_bytes;
- }
- kparams_local.kda = (g->ne[0] == S_v) ? 1 : 0;
- kparams_local.scale = 1.0f / sqrtf((float) S_v);
- kparams_local.state_seq_stride = (uint32_t) (state->nb[3] / sizeof(float));
- kparams_local.state_size_per_snap = S_v * S_v * H * n_seqs;
-
- kparams_local.div_H = init_fastdiv_values(H);
- kparams_local.div_q1 = init_fastdiv_values(q->ne[1]);
- kparams_local.div_k1 = init_fastdiv_values(k->ne[1]);
- kparams_local.div_rq3 = init_fastdiv_values(rq3);
- kparams_local.div_rk3 = init_fastdiv_values(rk3);
- kparams_local.div_n_threads = init_fastdiv_values(n_threads);
-
- kparams = &kparams_local;
+ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ return HTP_STATUS_INVAL_PARAMS;
}
+ const uint32_t S_v = kparams->S_v;
+ const uint32_t n_tokens = kparams->n_tokens;
const uint32_t total_rows = kparams->total_rows;
uint32_t row_start = 0;
uint32_t nrows = total_rows;
@@ -2416,16 +2323,16 @@ int op_gated_delta_net(struct htp_ops_context * octx) {
return gated_delta_net_f32_hmx_chunked(octx, kparams, row_start, nrows);
}
- const uint32_t n_threads = (nrows < kparams->n_threads) ? nrows : kparams->n_threads;
+ const uint32_t n_threads = (nrows < octx->n_threads) ? nrows : octx->n_threads;
struct htp_gdn_context gctx;
gctx.octx = octx;
gctx.kparams = kparams;
gctx.row_start = row_start;
gctx.nrows = nrows;
- gctx.vtcm_base = octx->ctx->vtcm_base;
htp_gdn_vtcm_layout_build(&gctx.layout, S_v, n_threads);
+ gctx.vtcm_base = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, gctx.layout.off_state);
if (gctx.layout.total_bytes > octx->ctx->vtcm_size) {
return HTP_STATUS_VTCM_TOO_SMALL;
@@ -2433,10 +2340,10 @@ int op_gated_delta_net(struct htp_ops_context * octx) {
FARF(HIGH, "gated-delta-net-f32: q(%ux%ux%ux%u) k(%ux%ux%ux%u) v(%ux%ux%ux%u) state(%ux%ux%ux%u) -> (%ux%ux%ux%u) : "
"vtcm-size %zu n_threads %u\n",
- q->ne[0], q->ne[1], q->ne[2], q->ne[3],
- k->ne[0], k->ne[1], k->ne[2], k->ne[3],
- v->ne[0], v->ne[1], v->ne[2], v->ne[3],
- state->ne[0], state->ne[1], state->ne[2], state->ne[3],
+ octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3],
+ octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3],
+ octx->src[2]->ne[0], octx->src[2]->ne[1], octx->src[2]->ne[2], octx->src[2]->ne[3],
+ octx->src[5]->ne[0], octx->src[5]->ne[1], octx->src[5]->ne[2], octx->src[5]->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
gctx.layout.total_bytes, n_threads);
diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h
index 32fb7d24b..0625c4d44 100644
--- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h
+++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h
@@ -37,13 +37,7 @@ struct htp_gdn_kernel_params {
uint32_t K;
uint32_t total_rows;
- uint32_t row_start;
- uint32_t nrows;
- uint32_t rows_per_thread;
-
uint32_t kda;
- uint32_t state_aligned;
- uint32_t vtcm_per_thread;
uint32_t vtcm_size;
uint32_t state_seq_stride;
uint32_t state_size_per_snap;
@@ -55,7 +49,6 @@ struct htp_gdn_kernel_params {
struct fastdiv_values div_k1;
struct fastdiv_values div_rq3;
struct fastdiv_values div_rk3;
- struct fastdiv_values div_n_threads;
};
#if defined(__cplusplus)
@@ -67,18 +60,21 @@ _Static_assert(sizeof(struct htp_gdn_kernel_params) <= 128, "htp_gdn_kernel_para
struct htp_gdn_vtcm_layout {
size_t state_aligned;
size_t bytes_per_thread;
+ size_t off_state;
size_t total_bytes;
};
static inline void htp_gdn_vtcm_layout_build(
- struct htp_gdn_vtcm_layout * layout,
+ struct htp_gdn_vtcm_layout * L,
uint32_t S_v,
uint32_t n_threads
) {
size_t state_bytes = (size_t) S_v * S_v * sizeof(float);
- layout->state_aligned = hex_round_up(state_bytes, 128);
- layout->bytes_per_thread = 2 * layout->state_aligned;
- layout->total_bytes = layout->bytes_per_thread * n_threads;
+ L->state_aligned = hex_round_up(state_bytes, 128);
+ L->bytes_per_thread = 2 * L->state_aligned;
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_state, L->bytes_per_thread * n_threads);
+ L->total_bytes = off;
}
struct htp_gdn_hmx_vtcm_layout {
diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.c b/ggml/src/ggml-hexagon/htp/gather-rows.c
index 1db48e59c..1e88445f1 100644
--- a/ggml/src/ggml-hexagon/htp/gather-rows.c
+++ b/ggml/src/ggml-hexagon/htp/gather-rows.c
@@ -206,14 +206,22 @@ static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * da
struct htp_thread_trace * tr = &task->octx->ctx->trace[ith];
- uint8_t * base = task->octx->ctx->vtcm_base + ith * kparams->slice;
- HVX_Vector * tab = (HVX_Vector *) base;
- HVX_Vector * inc = tab + kparams->vg;
- HVX_Vector * cur = inc + kparams->vg;
- uint8_t * region_base = (uint8_t *) (cur + kparams->vg);
-
- uint8_t * region[2] = { region_base, region_base + kparams->buf_stride };
- uint8_t * out[2] = { region[0] + kparams->out_off, region[1] + kparams->out_off };
+ struct htp_gather_rows_vtcm_layout layout;
+ htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 4);
+
+ uint8_t * base = VTCM_LAYOUT_PTR(uint8_t, task->octx->ctx->vtcm_base, (size_t) ith * kparams->slice);
+ HVX_Vector * tab = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_tab);
+ HVX_Vector * inc = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_inc);
+ HVX_Vector * cur = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_cur);
+
+ uint8_t * region[2] = {
+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region0),
+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region1),
+ };
+ uint8_t * out[2] = {
+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out0),
+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out1),
+ };
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
gather_rows_init_tables_f32(tab, inc, kparams, a, b);
@@ -279,14 +287,22 @@ static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * da
struct htp_thread_trace * tr = &task->octx->ctx->trace[ith];
- uint8_t * base = task->octx->ctx->vtcm_base + ith * kparams->slice;
- HVX_Vector * tab = (HVX_Vector *) base;
- HVX_Vector * inc = tab + kparams->vg;
- HVX_Vector * cur = inc + kparams->vg;
- uint8_t * region_base = (uint8_t *) (cur + kparams->vg);
-
- uint8_t * region[2] = { region_base, region_base + kparams->buf_stride };
- uint8_t * out[2] = { region[0] + kparams->out_off, region[1] + kparams->out_off };
+ struct htp_gather_rows_vtcm_layout layout;
+ htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 2);
+
+ uint8_t * base = VTCM_LAYOUT_PTR(uint8_t, task->octx->ctx->vtcm_base, (size_t) ith * kparams->slice);
+ HVX_Vector * tab = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_tab);
+ HVX_Vector * inc = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_inc);
+ HVX_Vector * cur = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_cur);
+
+ uint8_t * region[2] = {
+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region0),
+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region1),
+ };
+ uint8_t * out[2] = {
+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out0),
+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out1),
+ };
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
gather_rows_init_tables_f16(tab, inc, kparams, a, b);
diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.h b/ggml/src/ggml-hexagon/htp/gather-rows.h
index 3bbb1c635..c08ef4d38 100644
--- a/ggml/src/ggml-hexagon/htp/gather-rows.h
+++ b/ggml/src/ggml-hexagon/htp/gather-rows.h
@@ -2,6 +2,7 @@
#define HTP_GATHER_ROWS_H
#include "hex-common.h"
+#include "htp-vtcm.h"
#include <stdint.h>
#include <stdbool.h>
@@ -19,8 +20,6 @@ struct htp_gather_rows_params {
uint32_t v1;
uint32_t span1;
uint32_t region_size;
- uint32_t out_off;
- uint32_t buf_stride;
uint32_t slice;
uint32_t total_rows;
uint32_t rows_per_thread;
@@ -28,6 +27,40 @@ struct htp_gather_rows_params {
uint8_t pad[3];
};
+struct htp_gather_rows_vtcm_layout {
+ size_t off_tab;
+ size_t off_inc;
+ size_t off_cur;
+ size_t off_region0;
+ size_t off_out0;
+ size_t off_region1;
+ size_t off_out1;
+ size_t total_per_thread;
+};
+
+static inline void htp_gather_rows_vtcm_layout_build(
+ struct htp_gather_rows_vtcm_layout * L,
+ uint32_t vg,
+ uint32_t region_size,
+ uint32_t nr_max,
+ uint32_t ne,
+ uint32_t elem_size) {
+
+ const size_t vec_bytes = (size_t) vg * 128;
+ const size_t region_aligned = hex_round_up(region_size, 128);
+ const size_t out_aligned = hex_round_up(region_aligned + (size_t) nr_max * ne * elem_size, 128) - region_aligned;
+
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_tab, vec_bytes);
+ VTCM_LAYOUT_ALLOC(off, off_inc, vec_bytes);
+ VTCM_LAYOUT_ALLOC(off, off_cur, vec_bytes);
+ VTCM_LAYOUT_ALLOC(off, off_region0, region_aligned);
+ VTCM_LAYOUT_ALLOC(off, off_out0, out_aligned);
+ VTCM_LAYOUT_ALLOC(off, off_region1, region_aligned);
+ VTCM_LAYOUT_ALLOC(off, off_out1, out_aligned);
+ L->total_per_thread = off;
+}
+
static inline bool htp_gather_rows_solve_layout(
struct htp_gather_rows_params * kparams,
uint32_t na, uint32_t a_nb1,
@@ -80,8 +113,6 @@ static inline bool htp_gather_rows_solve_layout(
uint32_t v1 = 0;
uint32_t span1 = 0;
uint32_t region_size = 0;
- uint32_t out_off = 0;
- uint32_t buf_stride = 0;
while (nr_max > 0) {
v1 = hex_round_up((nr_max - 1) * a_nb1 + na * elem_size, 128);
@@ -98,10 +129,10 @@ static inline bool htp_gather_rows_solve_layout(
region_size = v1;
}
- out_off = hex_round_up(region_size, 128);
- buf_stride = hex_round_up(out_off + nr_max * ne * elem_size, 128);
+ struct htp_gather_rows_vtcm_layout layout;
+ htp_gather_rows_vtcm_layout_build(&layout, vg, region_size, nr_max, ne, elem_size);
- if ((elem_size != 2 || region_size <= 32768) && fixed + 2 * buf_stride <= slice) {
+ if ((elem_size != 2 || region_size <= 32768) && layout.total_per_thread <= slice) {
break;
}
@@ -121,8 +152,6 @@ static inline bool htp_gather_rows_solve_layout(
kparams->v1 = v1;
kparams->span1 = span1;
kparams->region_size = region_size;
- kparams->out_off = out_off;
- kparams->buf_stride = buf_stride;
kparams->slice = slice;
kparams->total_rows = total_rows;
kparams->rows_per_thread = rows_per_thread;
diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c
index 78eb1bb50..39acdfda6 100644
--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c
+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c
@@ -404,7 +404,7 @@ int op_get_rows(struct htp_ops_context * octx) {
if (octx->ctx->mdev.count > 1) {
uint32_t tasks_per_chunk = 1;
- htp_tensor_mdev_rows_per_chunk(dst, dst_row_size / dst->ne[0], dst_row_size, &tasks_per_chunk);
+ htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) dst->nb[0], dst_row_size, &tasks_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, tasks_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
task_start = range.start;
tasks = range.count;
@@ -414,7 +414,7 @@ int op_get_rows(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) {
+ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -428,7 +428,7 @@ int op_get_rows(struct htp_ops_context * octx) {
grctx.compute_tiled = NULL;
grctx.task_start = task_start;
grctx.tasks = tasks;
- grctx.tasks_per_thread = octx->ctx->mdev.count == 1 ? kparams->tasks_per_thread : fastdiv(tasks + n_threads - 1, &octx->n_threads_div);
+ grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div);
grctx.tile_size = 0;
grctx.tile_stride = 0;
grctx.src0_row_size = src0_row_size;
diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.h
index 94f3c15f2..4634be2e2 100644
--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h
+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h
@@ -15,11 +15,10 @@ enum htp_get_rows_kernel_type {
};
struct htp_get_rows_kernel_params {
- int32_t n_threads;
- int32_t kernel_type;
- int32_t total_tasks;
- int32_t tasks_per_thread;
- int32_t vtcm_size;
+ uint8_t kernel_type;
+ uint32_t n_threads;
+ uint32_t total_tasks;
+ uint32_t vtcm_size;
// Fastdiv helpers
struct fastdiv_values div_ne10;
diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/glu-ops.c
similarity index 69%
rename from ggml/src/ggml-hexagon/htp/act-ops.c
rename to ggml/src/ggml-hexagon/htp/glu-ops.c
index 59a0575c4..b4edddf3f 100644
--- a/ggml/src/ggml-hexagon/htp/act-ops.c
+++ b/ggml/src/ggml-hexagon/htp/glu-ops.c
@@ -17,11 +17,12 @@
#include "hex-common.h"
#include "htp-tensor.h"
#include "htp-vtcm.h"
+#include "glu-ops.h"
-#define htp_act_preamble \
- const struct htp_tensor * src0 = actx->octx->src[0]; \
- const struct htp_tensor * src1 = actx->octx->src[1]; \
- const struct htp_tensor * dst = actx->octx->dst; \
+#define htp_glu_preamble \
+ const struct htp_tensor * src0 = gctx->octx->src[0]; \
+ const struct htp_tensor * src1 = gctx->octx->src[1]; \
+ const struct htp_tensor * dst = gctx->octx->dst; \
\
const uint32_t ne00 = src0->ne[0]; \
const uint32_t ne01 = src0->ne[1]; \
@@ -53,21 +54,21 @@
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3];
-struct htp_act_context;
+struct htp_glu_context;
typedef void (*glu_compute_f32_fn_t)(const float * restrict src0,
const float * restrict src1,
float * restrict dst,
const uint32_t num_rows,
- const struct htp_act_context * actx);
+ const struct htp_glu_context * gctx);
typedef void (*glu_compute_f16_fn_t)(const _Float16 * restrict src0,
const _Float16 * restrict src1,
_Float16 * restrict dst,
const uint32_t num_rows,
- const struct htp_act_context * actx);
+ const struct htp_glu_context * gctx);
-struct htp_act_context {
+struct htp_glu_context {
struct htp_ops_context * octx;
void * compute;
@@ -108,53 +109,18 @@ struct htp_act_context {
size_t vtcm_dst_size_per_thread;
};
-struct htp_act_vtcm_layout {
- size_t total_bytes;
- size_t off_src0;
- size_t off_src1;
- size_t off_dst;
-
- size_t src0_bytes_per_thread;
- size_t src1_bytes_per_thread;
- size_t dst_bytes_per_thread;
-
- uint32_t vtcm_row_per_thread;
-};
-
-static inline void htp_act_vtcm_layout_build(struct htp_act_vtcm_layout * L,
- size_t src0_row_size_aligned,
- size_t src1_row_size_aligned,
- size_t dst_row_size_aligned,
- uint32_t n_threads,
- size_t vtcm_size) {
- const size_t spad_size_per_row = src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned;
- const uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * spad_size_per_row));
-
- L->vtcm_row_per_thread = vtcm_row_per_thread;
-
- L->src0_bytes_per_thread = src0_row_size_aligned * vtcm_row_per_thread;
- L->src1_bytes_per_thread = src1_row_size_aligned * vtcm_row_per_thread;
- L->dst_bytes_per_thread = dst_row_size_aligned * vtcm_row_per_thread;
-
- L->off_src0 = 0;
- L->off_src1 = L->off_src0 + L->src0_bytes_per_thread * n_threads;
- L->off_dst = L->off_src1 + L->src1_bytes_per_thread * n_threads;
-
- L->total_bytes = L->off_dst + L->dst_bytes_per_thread * n_threads;
-}
-
#define htp_glu_op_preamble \
- const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \
- const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \
- const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \
- const int nc = actx->nc;
+ const size_t src0_row_size_aligned = gctx->src0_row_size_aligned; \
+ const size_t src1_row_size_aligned = gctx->src1_row_size_aligned; \
+ const size_t dst_row_size_aligned = gctx->dst_row_size_aligned; \
+ const int nc = gctx->nc;
// swiglu(x) = x1 * sigmoid(x0)
static void swiglu_f32(const float * restrict src0,
const float * restrict src1,
float * restrict dst,
const uint32_t num_rows,
- const struct htp_act_context * actx) {
+ const struct htp_glu_context * gctx) {
htp_glu_op_preamble;
for (uint32_t ib = 0; ib < num_rows; ib++) {
@@ -172,10 +138,10 @@ static void swiglu_oai_f32(const float * restrict src0,
const float * restrict src1,
float * restrict dst,
const uint32_t num_rows,
- const struct htp_act_context * actx) {
+ const struct htp_glu_context * gctx) {
htp_glu_op_preamble;
- const float alpha = ((const float *) (actx->octx->op_params))[2];
- const float limit = ((const float *) (actx->octx->op_params))[3];
+ const float alpha = ((const float *) (gctx->octx->op_params))[2];
+ const float limit = ((const float *) (gctx->octx->op_params))[3];
for (uint32_t ib = 0; ib < num_rows; ib++) {
const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
@@ -201,9 +167,9 @@ static void swiglu_clamp_f32(const float * restrict src0,
const float * restrict src1,
float * restrict dst,
const uint32_t num_rows,
- const struct htp_act_context * actx) {
+ const struct htp_glu_context * gctx) {
htp_glu_op_preamble;
- const float limit = ((const float *) (actx->octx->op_params))[3];
+ const float limit = ((const float *) (gctx->octx->op_params))[3];
for (uint32_t ib = 0; ib < num_rows; ib++) {
const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
@@ -455,7 +421,7 @@ static void geglu_f32(const float * restrict src0,
const float * restrict src1,
float * restrict dst,
const uint32_t num_rows,
- const struct htp_act_context * actx) {
+ const struct htp_glu_context * gctx) {
htp_glu_op_preamble;
for (uint32_t ib = 0; ib < num_rows; ib++) {
@@ -472,7 +438,7 @@ static void geglu_quick_f32(const float * restrict src0,
const float * restrict src1,
float * restrict dst,
const uint32_t num_rows,
- const struct htp_act_context * actx) {
+ const struct htp_glu_context * gctx) {
htp_glu_op_preamble;
for (uint32_t ib = 0; ib < num_rows; ib++) {
@@ -489,7 +455,7 @@ static void geglu_erf_f32(const float * restrict src0,
const float * restrict src1,
float * restrict dst,
const uint32_t num_rows,
- const struct htp_act_context * actx) {
+ const struct htp_glu_context * gctx) {
htp_glu_op_preamble;
for (uint32_t ib = 0; ib < num_rows; ib++) {
@@ -501,12 +467,124 @@ static void geglu_erf_f32(const float * restrict src0,
}
}
+static inline void hvx_reglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src0 % 128 == 0);
+ assert((unsigned long) src1 % 128 == 0);
+
+ HVX_Vector * restrict vdst = (HVX_Vector *) dst;
+ const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0;
+ const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1;
+
+ const uint32_t epv = 128 / sizeof(float);
+ const uint32_t nvec = n / epv;
+ const uint32_t nloe = n % epv;
+
+ const HVX_Vector v_zero = Q6_V_vzero();
+
+ uint32_t i = 0;
+
+ _Pragma("unroll(4)")
+ for (; i < nvec; i++) {
+ HVX_Vector x = Q6_Vsf_vmax_VsfVsf(vsrc0[i], v_zero);
+ vdst[i] = hvx_vec_mul_f32_f32(x, vsrc1[i]);
+ }
+
+ if (nloe) {
+ HVX_Vector x = Q6_Vsf_vmax_VsfVsf(vsrc0[i], v_zero);
+ HVX_Vector res = hvx_vec_mul_f32_f32(x, vsrc1[i]);
+ hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(float), res);
+ }
+}
+
+// reglu(x, g) = relu(x) * g
+static void reglu_f32(const float * restrict src0,
+ const float * restrict src1,
+ float * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_glu_context * gctx) {
+ htp_glu_op_preamble;
+
+ for (uint32_t ib = 0; ib < num_rows; ib++) {
+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);
+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);
+
+ hvx_reglu_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc);
+ }
+}
+
+static __attribute__((noinline)) void hvx_sigglu_f32_vec(HVX_Vector * restrict vdst,
+ const HVX_Vector * restrict vsrc0,
+ const HVX_Vector * restrict vsrc1,
+ uint32_t nvec) {
+ const HVX_Vector v_one = hvx_vec_splat_f32(1.0f);
+ const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f);
+ const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f);
+
+ for (uint32_t i = 0; i < nvec; i++) {
+ HVX_Vector sig = hvx_vec_fast_sigmoid_f32_guard(vsrc0[i], v_one, v_max_exp, v_min_exp);
+ vdst[i] = hvx_vec_mul_f32_f32(sig, vsrc1[i]);
+ }
+}
+
+static __attribute__((noinline)) void hvx_sigglu_f32_tail(HVX_Vector * restrict vdst,
+ const HVX_Vector * restrict vsrc0,
+ const HVX_Vector * restrict vsrc1,
+ uint32_t nloe) {
+ const HVX_Vector v_one = hvx_vec_splat_f32(1.0f);
+ const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f);
+ const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f);
+
+ HVX_Vector sig = hvx_vec_fast_sigmoid_f32_guard(vsrc0[0], v_one, v_max_exp, v_min_exp);
+ HVX_Vector res = hvx_vec_mul_f32_f32(sig, vsrc1[0]);
+ hvx_vec_store_a((void *) vdst, nloe * sizeof(float), res);
+}
+
+static inline void hvx_sigglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src0 % 128 == 0);
+ assert((unsigned long) src1 % 128 == 0);
+
+ HVX_Vector * restrict vdst = (HVX_Vector *) dst;
+ const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0;
+ const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1;
+
+ const uint32_t epv = 128 / sizeof(float);
+ const uint32_t nvec = n / epv;
+ const uint32_t nloe = n % epv;
+
+ if (nvec) {
+ hvx_sigglu_f32_vec(vdst, vsrc0, vsrc1, nvec);
+ }
+ if (nloe) {
+ hvx_sigglu_f32_tail(vdst + nvec, vsrc0 + nvec, vsrc1 + nvec, nloe);
+ }
+}
+
+// sigglu(x, g) = sigmoid(x) * g
+static void sigglu_f32(const float * restrict src0,
+ const float * restrict src1,
+ float * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_glu_context * gctx) {
+ htp_glu_op_preamble;
+
+ for (uint32_t ib = 0; ib < num_rows; ib++) {
+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);
+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);
+
+ hvx_sigglu_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc);
+ }
+}
+
// swiglu(x) = x1 * sigmoid(x0)
static void swiglu_f16(const _Float16 * restrict src0,
const _Float16 * restrict src1,
_Float16 * restrict dst,
const uint32_t num_rows,
- const struct htp_act_context * actx) {
+ const struct htp_glu_context * gctx) {
htp_glu_op_preamble;
for (uint32_t ib = 0; ib < num_rows; ib++) {
@@ -524,10 +602,10 @@ static void swiglu_oai_f16(const _Float16 * restrict src0,
const _Float16 * restrict src1,
_Float16 * restrict dst,
const uint32_t num_rows,
- const struct htp_act_context * actx) {
+ const struct htp_glu_context * gctx) {
htp_glu_op_preamble;
- const _Float16 alpha = (_Float16) ((const float *) (actx->octx->op_params))[2];
- const _Float16 limit = (_Float16) ((const float *) (actx->octx->op_params))[3];
+ const _Float16 alpha = (_Float16) ((const float *) (gctx->octx->op_params))[2];
+ const _Float16 limit = (_Float16) ((const float *) (gctx->octx->op_params))[3];
for (uint32_t ib = 0; ib < num_rows; ib++) {
const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
@@ -553,9 +631,9 @@ static void swiglu_clamp_f16(const _Float16 * restrict src0,
const _Float16 * restrict src1,
_Float16 * restrict dst,
const uint32_t num_rows,
- const struct htp_act_context * actx) {
+ const struct htp_glu_context * gctx) {
htp_glu_op_preamble;
- const _Float16 limit = (_Float16) ((const float *) (actx->octx->op_params))[3];
+ const _Float16 limit = (_Float16) ((const float *) (gctx->octx->op_params))[3];
for (uint32_t ib = 0; ib < num_rows; ib++) {
const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
@@ -574,7 +652,7 @@ static void geglu_f16(const _Float16 * restrict src0,
const _Float16 * restrict src1,
_Float16 * restrict dst,
const uint32_t num_rows,
- const struct htp_act_context * actx) {
+ const struct htp_glu_context * gctx) {
htp_glu_op_preamble;
for (uint32_t ib = 0; ib < num_rows; ib++) {
@@ -586,56 +664,155 @@ static void geglu_f16(const _Float16 * restrict src0,
}
}
+static inline void hvx_reglu_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src0 % 128 == 0);
+ assert((unsigned long) src1 % 128 == 0);
+
+ HVX_Vector * restrict vdst = (HVX_Vector *) dst;
+ const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0;
+ const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1;
+
+ const uint32_t epv = 128 / sizeof(_Float16);
+ const uint32_t nvec = n / epv;
+ const uint32_t nloe = n % epv;
+
+ const HVX_Vector v_zero = Q6_V_vzero();
+
+ uint32_t i = 0;
+
+ _Pragma("unroll(4)")
+ for (; i < nvec; i++) {
+ HVX_Vector x = Q6_Vhf_vmax_VhfVhf(vsrc0[i], v_zero);
+ vdst[i] = hvx_vec_mul_f16_f16(x, vsrc1[i]);
+ }
+
+ if (nloe) {
+ HVX_Vector x = Q6_Vhf_vmax_VhfVhf(vsrc0[i], v_zero);
+ HVX_Vector res = hvx_vec_mul_f16_f16(x, vsrc1[i]);
+ hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(_Float16), res);
+ }
+}
+
+// reglu(x, g) = relu(x) * g
+static void reglu_f16(const _Float16 * restrict src0,
+ const _Float16 * restrict src1,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_glu_context * gctx) {
+ htp_glu_op_preamble;
+
+ for (uint32_t ib = 0; ib < num_rows; ib++) {
+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);
+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);
+
+ hvx_reglu_f16_aa(dst_ptr, src0_ptr, src1_ptr, nc);
+ }
+}
+
+static __attribute__((noinline)) void hvx_sigglu_f16_vec(HVX_Vector * restrict vdst,
+ const HVX_Vector * restrict vsrc0,
+ const HVX_Vector * restrict vsrc1,
+ uint32_t nvec) {
+ for (uint32_t i = 0; i < nvec; i++) {
+ HVX_Vector sig = hvx_vec_fast_sigmoid_f16(vsrc0[i]);
+ vdst[i] = hvx_vec_mul_f16_f16(sig, vsrc1[i]);
+ }
+}
+
+static __attribute__((noinline)) void hvx_sigglu_f16_tail(HVX_Vector * restrict vdst,
+ const HVX_Vector * restrict vsrc0,
+ const HVX_Vector * restrict vsrc1,
+ uint32_t nloe) {
+ HVX_Vector sig = hvx_vec_fast_sigmoid_f16(vsrc0[0]);
+ HVX_Vector res = hvx_vec_mul_f16_f16(sig, vsrc1[0]);
+ hvx_vec_store_a((void *) vdst, nloe * sizeof(_Float16), res);
+}
+
+static inline void hvx_sigglu_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src0 % 128 == 0);
+ assert((unsigned long) src1 % 128 == 0);
+
+ HVX_Vector * restrict vdst = (HVX_Vector *) dst;
+ const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0;
+ const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1;
+
+ const uint32_t epv = 128 / sizeof(_Float16);
+ const uint32_t nvec = n / epv;
+ const uint32_t nloe = n % epv;
+
+ if (nvec) {
+ hvx_sigglu_f16_vec(vdst, vsrc0, vsrc1, nvec);
+ }
+ if (nloe) {
+ hvx_sigglu_f16_tail(vdst + nvec, vsrc0 + nvec, vsrc1 + nvec, nloe);
+ }
+}
+
+// sigglu(x, g) = sigmoid(x) * g
+static void sigglu_f16(const _Float16 * restrict src0,
+ const _Float16 * restrict src1,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_glu_context * gctx) {
+ htp_glu_op_preamble;
+
+ for (uint32_t ib = 0; ib < num_rows; ib++) {
+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);
+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);
+
+ hvx_sigglu_f16_aa(dst_ptr, src0_ptr, src1_ptr, nc);
+ }
+}
+
#define DEFINE_GLU_PER_THREAD(TYPE, SUFFIX) \
static void glu_##SUFFIX##_per_thread(unsigned int nth, unsigned int ith, void * data) { \
- struct htp_act_context * actx = (struct htp_act_context *) data; \
- htp_act_preamble; \
+ struct htp_glu_context * gctx = (struct htp_glu_context *) data; \
+ htp_glu_preamble; \
\
- struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \
+ struct htp_thread_trace * tr = gctx->octx->ctx ? &gctx->octx->ctx->trace[ith] : NULL; \
\
- size_t src0_row_size = actx->src0_row_size; \
- size_t src1_row_size = actx->src1_row_size; \
- size_t dst_row_size = actx->dst_row_size; \
+ size_t src0_row_size = gctx->src0_row_size; \
+ size_t src1_row_size = gctx->src1_row_size; \
+ size_t dst_row_size = gctx->dst_row_size; \
\
- size_t src0_row_stride = actx->src0_row_stride; \
- size_t src1_row_stride = actx->src1_row_stride; \
+ size_t src0_row_stride = gctx->src0_row_stride; \
+ size_t src1_row_stride = gctx->src1_row_stride; \
\
- const uint32_t src0_nrows = actx->src0_nrows; \
- const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \
+ const uint32_t src0_nrows = gctx->src0_nrows; \
+ const uint32_t src0_nrows_per_thread = gctx->src0_nrows_per_thread; \
\
- const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith; \
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows); \
+ const uint32_t src0_start_row = gctx->row_start + src0_nrows_per_thread * ith; \
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, gctx->row_start + src0_nrows); \
\
/* no work for this thread */ \
if (src0_start_row >= src0_end_row) { \
return; \
} \
\
- const dma_addr_t data_src0 = actx->data_src0; \
- const dma_addr_t data_src1 = actx->data_src1; \
- const dma_addr_t data_dst = actx->data_dst; \
+ const dma_addr_t data_src0 = gctx->data_src0; \
+ const dma_addr_t data_src1 = gctx->data_src1; \
+ const dma_addr_t data_dst = gctx->data_dst; \
\
- const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \
- const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \
- const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \
+ const size_t src0_row_size_aligned = gctx->src0_row_size_aligned; \
+ const size_t src1_row_size_aligned = gctx->src1_row_size_aligned; \
+ const size_t dst_row_size_aligned = gctx->dst_row_size_aligned; \
\
- uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \
- uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \
- uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \
+ uint8_t * restrict src0_spad_data = gctx->vtcm_src0 + (ith * gctx->vtcm_src0_size_per_thread); \
+ uint8_t * restrict src1_spad_data = gctx->vtcm_src1 + (ith * gctx->vtcm_src1_size_per_thread); \
+ uint8_t * restrict dst_spad_data = gctx->vtcm_dst + (ith * gctx->vtcm_dst_size_per_thread); \
\
- size_t src0_spad_half_size = actx->src0_spad_half_size; \
- size_t src1_spad_half_size = actx->src1_spad_half_size; \
- size_t dst_spad_half_size = actx->dst_spad_half_size; \
+ size_t src0_spad_half_size = gctx->src0_spad_half_size; \
+ size_t src1_spad_half_size = gctx->src1_spad_half_size; \
+ size_t dst_spad_half_size = gctx->dst_spad_half_size; \
\
- const int BLOCK = actx->block; \
- if (BLOCK == 0) { \
- FARF(ERROR, "%s : VTCM reservation %zu is too small, needed %zu\n", \
- actx->op_str, actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \
- return; \
- } \
+ const int BLOCK = gctx->block; \
\
- dma_queue * dma_q = actx->octx->ctx->dma[ith]; \
- glu_compute_##SUFFIX##_fn_t compute = (glu_compute_##SUFFIX##_fn_t) actx->compute; \
+ dma_queue * dma_q = gctx->octx->ctx->dma[ith]; \
+ glu_compute_##SUFFIX##_fn_t compute = (glu_compute_##SUFFIX##_fn_t) gctx->compute; \
\
for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \
const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \
@@ -662,7 +839,7 @@ static void geglu_f16(const _Float16 * restrict src0,
TYPE * src1_spad = (TYPE *) dma_queue_pop(dma_q).dst; \
\
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \
- compute(src0_spad, src1_spad, dst_spad, block_size, actx); \
+ compute(src0_spad, src1_spad, dst_spad, block_size, gctx); \
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \
\
dma_queue_push(dma_q, dma_make_data(data_dst + (ir * dst_row_size), dst_spad), \
@@ -685,16 +862,11 @@ static void geglu_f16(const _Float16 * restrict src0,
DEFINE_GLU_PER_THREAD(float, f32)
DEFINE_GLU_PER_THREAD(_Float16, f16)
-static int execute_op_activations_f32(struct htp_ops_context * octx) {
+static int execute_op_glu_f32(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
- if ((dst->ne[0] * SIZEOF_FP32) != dst->nb[1]) {
- FARF(ERROR, "Non-contiguous dst is not supported at this time \n");
- return HTP_STATUS_NO_SUPPORT;
- }
-
glu_compute_f32_fn_t compute_fn = NULL;
const char * op_type = NULL;
@@ -729,11 +901,27 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
op_type = "geglu-erf-f32";
break;
+ case HTP_OP_GLU_REGLU:
+ compute_fn = reglu_f32;
+ op_type = "reglu-f32";
+ break;
+
+ case HTP_OP_GLU_SIGMOID:
+ compute_fn = sigglu_f32;
+ op_type = "sigglu-f32";
+ break;
+
default:
- FARF(ERROR, "Unsupported activations Op %u\n", octx->op);
+ FARF(ERROR, "Unsupported GLU Op %u\n", octx->op);
return HTP_STATUS_NO_SUPPORT;
}
+ const struct htp_glu_kernel_params * kparams = (const struct htp_glu_kernel_params *) octx->kernel_params;
+
+ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const size_t dst_row_size = dst->ne[0] * SIZEOF_FP32;
@@ -762,17 +950,16 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
const size_t src0_row_stride = src0->nb[1];
const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1];
- const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);
- const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);
- const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);
+ struct htp_glu_vtcm_layout L;
+ htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, kparams->vtcm_row_per_thread);
- struct htp_act_vtcm_layout L;
- htp_act_vtcm_layout_build(&L, src0_row_size_aligned, src1_row_size_aligned, dst_row_size_aligned, n_threads,
- octx->ctx->vtcm_size);
+ const size_t src0_row_size_aligned = L.row_size_aligned;
+ const size_t src1_row_size_aligned = L.row_size_aligned;
+ const size_t dst_row_size_aligned = L.row_size_aligned;
// Make sure the reserved vtcm size is sufficient
- if (L.vtcm_row_per_thread == 0) {
- FARF(ERROR, "act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,
+ if (kparams->vtcm_row_per_thread == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) {
+ FARF(ERROR, "glu-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,
(src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads);
return HTP_STATUS_VTCM_TOO_SMALL;
}
@@ -789,58 +976,53 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
}
// Prepare context
- struct htp_act_context actx;
- actx.octx = octx;
- actx.compute = (void *) compute_fn;
- actx.op_str = op_type;
+ struct htp_glu_context gctx;
+ gctx.octx = octx;
+ gctx.compute = (void *) compute_fn;
+ gctx.op_str = op_type;
- actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
+ gctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
- actx.src0_row_size = src0_row_size;
- actx.src1_row_size = src1_row_size;
- actx.dst_row_size = dst_row_size;
+ gctx.src0_row_size = src0_row_size;
+ gctx.src1_row_size = src1_row_size;
+ gctx.dst_row_size = dst_row_size;
- actx.src0_row_size_aligned = src0_row_size_aligned;
- actx.src1_row_size_aligned = src1_row_size_aligned;
- actx.dst_row_size_aligned = dst_row_size_aligned;
+ gctx.src0_row_size_aligned = src0_row_size_aligned;
+ gctx.src1_row_size_aligned = src1_row_size_aligned;
+ gctx.dst_row_size_aligned = dst_row_size_aligned;
- actx.src0_row_stride = src0_row_stride;
- actx.src1_row_stride = src1_row_stride;
+ gctx.src0_row_stride = src0_row_stride;
+ gctx.src1_row_stride = src1_row_stride;
uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;
- actx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);
- actx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);
- actx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);
+ gctx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);
+ gctx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);
+ gctx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);
- actx.vtcm_src0_size_per_thread = L.src0_bytes_per_thread;
- actx.vtcm_src1_size_per_thread = L.src1_bytes_per_thread;
- actx.vtcm_dst_size_per_thread = L.dst_bytes_per_thread;
+ gctx.vtcm_src0_size_per_thread = (uint32_t) L.src0_bytes_per_thread;
+ gctx.vtcm_src1_size_per_thread = (uint32_t) L.src1_bytes_per_thread;
+ gctx.vtcm_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread;
- actx.src0_spad_half_size = L.src0_bytes_per_thread / 2;
- actx.src1_spad_half_size = L.src1_bytes_per_thread / 2;
- actx.dst_spad_half_size = L.dst_bytes_per_thread / 2;
+ gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_thread >> 1;
+ gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_thread >> 1;
+ gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_thread >> 1;
- actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned;
- actx.src0_nrows = nrows;
- actx.row_start = row_start;
+ gctx.block = L.block;
+ gctx.src0_nrows = nrows;
+ gctx.row_start = row_start;
- actx.nc = dst->ne[0];
+ gctx.nc = dst->ne[0];
// Addresses and GLU logic
dma_addr_t data_src0 = src0->data;
dma_addr_t data_src1 = src1 ? src1->data : 0;
- if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU ||
- octx->op == HTP_OP_GLU_SWIGLU_OAI ||
- octx->op == HTP_OP_GLU_SWIGLU_CLAMP ||
- octx->op == HTP_OP_GLU_GEGLU ||
- octx->op == HTP_OP_GLU_GEGLU_QUICK ||
- octx->op == HTP_OP_GLU_GEGLU_ERF)) {
+ if (!src1) {
const int32_t swapped = octx->op_params[1];
data_src1 = data_src0;
- actx.src1_row_size = actx.src0_row_size;
+ gctx.src1_row_size = gctx.src0_row_size;
- size_t nc_in_bytes = actx.nc * SIZEOF_FP32;
+ size_t nc_in_bytes = gctx.nc * SIZEOF_FP32;
if (swapped) {
data_src0 += nc_in_bytes;
} else {
@@ -848,24 +1030,19 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
}
}
- actx.data_src0 = data_src0;
- actx.data_src1 = data_src1;
- actx.data_dst = dst->data;
+ gctx.data_src0 = data_src0;
+ gctx.data_src1 = data_src1;
+ gctx.data_dst = dst->data;
- work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f32_per_thread, &actx, n_threads);
+ work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f32_per_thread, &gctx, n_threads);
return HTP_STATUS_OK;
}
-static int execute_op_activations_f16(struct htp_ops_context * octx) {
+static int execute_op_glu_f16(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
- if ((dst->ne[0] * SIZEOF_FP16) != dst->nb[1]) {
- FARF(ERROR, "Non-contiguous dst is not supported at this time \n");
- return HTP_STATUS_NO_SUPPORT;
- }
-
glu_compute_f16_fn_t compute_fn = NULL;
const char * op_type = NULL;
@@ -890,11 +1067,27 @@ static int execute_op_activations_f16(struct htp_ops_context * octx) {
op_type = "geglu-f16";
break;
+ case HTP_OP_GLU_REGLU:
+ compute_fn = reglu_f16;
+ op_type = "reglu-f16";
+ break;
+
+ case HTP_OP_GLU_SIGMOID:
+ compute_fn = sigglu_f16;
+ op_type = "sigglu-f16";
+ break;
+
default:
- FARF(ERROR, "Unsupported activations Op %u\n", octx->op);
+ FARF(ERROR, "Unsupported GLU Op %u\n", octx->op);
return HTP_STATUS_NO_SUPPORT;
}
+ const struct htp_glu_kernel_params * kparams = (const struct htp_glu_kernel_params *) octx->kernel_params;
+
+ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const size_t dst_row_size = dst->ne[0] * SIZEOF_FP16;
@@ -923,17 +1116,16 @@ static int execute_op_activations_f16(struct htp_ops_context * octx) {
const size_t src0_row_stride = src0->nb[1];
const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1];
- const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);
- const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);
- const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);
+ struct htp_glu_vtcm_layout L;
+ htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, kparams->vtcm_row_per_thread);
- struct htp_act_vtcm_layout L;
- htp_act_vtcm_layout_build(&L, src0_row_size_aligned, src1_row_size_aligned, dst_row_size_aligned, n_threads,
- octx->ctx->vtcm_size);
+ const size_t src0_row_size_aligned = L.row_size_aligned;
+ const size_t src1_row_size_aligned = L.row_size_aligned;
+ const size_t dst_row_size_aligned = L.row_size_aligned;
// Make sure the reserved vtcm size is sufficient
- if (L.vtcm_row_per_thread == 0) {
- FARF(ERROR, "act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,
+ if (kparams->vtcm_row_per_thread == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) {
+ FARF(ERROR, "glu-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,
(src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads);
return HTP_STATUS_VTCM_TOO_SMALL;
}
@@ -950,56 +1142,53 @@ static int execute_op_activations_f16(struct htp_ops_context * octx) {
}
// Prepare context
- struct htp_act_context actx;
- actx.octx = octx;
- actx.compute = (void *) compute_fn;
- actx.op_str = op_type;
+ struct htp_glu_context gctx;
+ gctx.octx = octx;
+ gctx.compute = (void *) compute_fn;
+ gctx.op_str = op_type;
- actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
+ gctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
- actx.src0_row_size = src0_row_size;
- actx.src1_row_size = src1_row_size;
- actx.dst_row_size = dst_row_size;
+ gctx.src0_row_size = src0_row_size;
+ gctx.src1_row_size = src1_row_size;
+ gctx.dst_row_size = dst_row_size;
- actx.src0_row_size_aligned = src0_row_size_aligned;
- actx.src1_row_size_aligned = src1_row_size_aligned;
- actx.dst_row_size_aligned = dst_row_size_aligned;
+ gctx.src0_row_size_aligned = src0_row_size_aligned;
+ gctx.src1_row_size_aligned = src1_row_size_aligned;
+ gctx.dst_row_size_aligned = dst_row_size_aligned;
- actx.src0_row_stride = src0_row_stride;
- actx.src1_row_stride = src1_row_stride;
+ gctx.src0_row_stride = src0_row_stride;
+ gctx.src1_row_stride = src1_row_stride;
uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;
- actx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);
- actx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);
- actx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);
+ gctx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);
+ gctx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);
+ gctx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);
- actx.vtcm_src0_size_per_thread = L.src0_bytes_per_thread;
- actx.vtcm_src1_size_per_thread = L.src1_bytes_per_thread;
- actx.vtcm_dst_size_per_thread = L.dst_bytes_per_thread;
+ gctx.vtcm_src0_size_per_thread = (uint32_t) L.src0_bytes_per_thread;
+ gctx.vtcm_src1_size_per_thread = (uint32_t) L.src1_bytes_per_thread;
+ gctx.vtcm_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread;
- actx.src0_spad_half_size = L.src0_bytes_per_thread / 2;
- actx.src1_spad_half_size = L.src1_bytes_per_thread / 2;
- actx.dst_spad_half_size = L.dst_bytes_per_thread / 2;
+ gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_thread >> 1;
+ gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_thread >> 1;
+ gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_thread >> 1;
- actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned;
- actx.src0_nrows = nrows;
- actx.row_start = row_start;
+ gctx.block = L.block;
+ gctx.src0_nrows = nrows;
+ gctx.row_start = row_start;
- actx.nc = dst->ne[0];
+ gctx.nc = dst->ne[0];
// Addresses and GLU logic
dma_addr_t data_src0 = src0->data;
dma_addr_t data_src1 = src1 ? src1->data : 0;
- if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU ||
- octx->op == HTP_OP_GLU_SWIGLU_OAI ||
- octx->op == HTP_OP_GLU_SWIGLU_CLAMP ||
- octx->op == HTP_OP_GLU_GEGLU)) {
+ if (!src1) {
const int32_t swapped = octx->op_params[1];
data_src1 = data_src0;
- actx.src1_row_size = actx.src0_row_size;
+ gctx.src1_row_size = gctx.src0_row_size;
- size_t nc_in_bytes = actx.nc * SIZEOF_FP16;
+ size_t nc_in_bytes = gctx.nc * SIZEOF_FP16;
if (swapped) {
data_src0 += nc_in_bytes;
} else {
@@ -1007,22 +1196,22 @@ static int execute_op_activations_f16(struct htp_ops_context * octx) {
}
}
- actx.data_src0 = data_src0;
- actx.data_src1 = data_src1;
- actx.data_dst = dst->data;
+ gctx.data_src0 = data_src0;
+ gctx.data_src1 = data_src1;
+ gctx.data_dst = dst->data;
- work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f16_per_thread, &actx, n_threads);
+ work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f16_per_thread, &gctx, n_threads);
return HTP_STATUS_OK;
}
-int op_activations(struct htp_ops_context * octx) {
+int op_glu(struct htp_ops_context * octx) {
switch (octx->src[0]->type) {
case HTP_TYPE_F32:
- return execute_op_activations_f32(octx);
+ return execute_op_glu_f32(octx);
case HTP_TYPE_F16:
- return execute_op_activations_f16(octx);
+ return execute_op_glu_f16(octx);
default:
return HTP_STATUS_NO_SUPPORT;
diff --git a/ggml/src/ggml-hexagon/htp/glu-ops.h b/ggml/src/ggml-hexagon/htp/glu-ops.h
new file mode 100644
index 000000000..41b3857b1
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/glu-ops.h
@@ -0,0 +1,76 @@
+#ifndef HTP_GLU_OPS_H
+#define HTP_GLU_OPS_H
+
+#include "hex-common.h"
+#include "htp-ops.h"
+#include "htp-vtcm.h"
+#include <stdint.h>
+
+struct htp_glu_kernel_params {
+ uint32_t n_threads;
+ uint32_t vtcm_row_per_thread;
+ uint32_t vtcm_size;
+};
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_glu_kernel_params) <= 128, "htp_glu_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_glu_kernel_params) <= 128, "htp_glu_kernel_params is too large for kernel_params blob");
+#endif
+
+static inline bool htp_op_is_glu(uint32_t opcode) {
+ switch (opcode) {
+ case HTP_OP_GLU_SWIGLU:
+ case HTP_OP_GLU_SWIGLU_OAI:
+ case HTP_OP_GLU_SWIGLU_CLAMP:
+ case HTP_OP_GLU_GEGLU:
+ case HTP_OP_GLU_GEGLU_QUICK:
+ case HTP_OP_GLU_GEGLU_ERF:
+ case HTP_OP_GLU_REGLU:
+ case HTP_OP_GLU_SIGMOID:
+ return true;
+ default:
+ return false;
+ }
+}
+
+struct htp_glu_vtcm_layout {
+ size_t total_bytes;
+ size_t off_src0;
+ size_t off_src1;
+ size_t off_dst;
+
+ size_t row_size_aligned;
+ size_t src0_bytes_per_thread;
+ size_t src1_bytes_per_thread;
+ size_t dst_bytes_per_thread;
+
+ uint32_t vtcm_row_per_thread;
+ uint32_t block;
+};
+
+static inline void htp_glu_vtcm_layout_build(
+ struct htp_glu_vtcm_layout * L,
+ size_t dst_row_size,
+ uint32_t n_threads,
+ uint32_t vtcm_row_per_thread
+) {
+ const size_t row_size_aligned = hex_round_up(dst_row_size, 128);
+
+ L->row_size_aligned = row_size_aligned;
+ L->vtcm_row_per_thread = vtcm_row_per_thread;
+ L->block = vtcm_row_per_thread >> 1;
+
+ L->src0_bytes_per_thread = row_size_aligned * vtcm_row_per_thread;
+ L->src1_bytes_per_thread = row_size_aligned * vtcm_row_per_thread;
+ L->dst_bytes_per_thread = row_size_aligned * vtcm_row_per_thread;
+
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+
+ L->total_bytes = off;
+}
+
+#endif // HTP_GLU_OPS_H
diff --git a/ggml/src/ggml-hexagon/htp/hex-common.h b/ggml/src/ggml-hexagon/htp/hex-common.h
index e6a52540d..b943453a3 100644
--- a/ggml/src/ggml-hexagon/htp/hex-common.h
+++ b/ggml/src/ggml-hexagon/htp/hex-common.h
@@ -17,6 +17,10 @@
#define MIN(a, b) ((a) < (b) ? (a) : (b))
#endif
+#ifndef HEX_L2_LINE_SIZE
+#define HEX_L2_LINE_SIZE 128
+#endif
+
static inline uint32_t hex_ceil_pow2(uint32_t x) {
if (x <= 1) { return 1; }
int p = 2;
diff --git a/ggml/src/ggml-hexagon/htp/hex-profile.h b/ggml/src/ggml-hexagon/htp/hex-profile.h
index a26961fc9..5c7a1a488 100644
--- a/ggml/src/ggml-hexagon/htp/hex-profile.h
+++ b/ggml/src/ggml-hexagon/htp/hex-profile.h
@@ -44,12 +44,14 @@ struct htp_thread_trace {
};
static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) {
- if (tr->count < tr->max_events) {
- uint32_t i = tr->count;
+ if (!tr->max_events) {
+ return;
+ }
+ uint32_t i = tr->count++;
+ if (i < tr->max_events) {
tr->events[i].id = id;
tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0);
tr->events[i].cycles = (uint32_t) hex_get_cycles();
- tr->count++;
}
}
diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h
index 22a05b917..5d14a8504 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ctx.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h
@@ -12,8 +12,10 @@
#include <assert.h>
#include <dspqueue.h>
#include <stdatomic.h>
+#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>
+#include <string.h>
#ifndef HTP_MAX_NTHREADS
#define HTP_MAX_NTHREADS 10
@@ -66,14 +68,18 @@ struct htp_ops_context {
// TODO convert these to an array
struct htp_spad src0_spad;
struct htp_spad src1_spad;
- struct htp_spad src2_spad;
- struct htp_spad src3_spad;
struct htp_spad dst_spad;
uint32_t flags;
uint32_t n_threads;
struct fastdiv_values n_threads_div;
int status;
+
+ // Must stay last: not cleared by htp_ops_context_reset(), ops init the caches they use
+ union {
+ dma_cache_dm dma_cache_dm[HTP_MAX_NTHREADS];
+ dma_cache_fa dma_cache_fa[HTP_MAX_NTHREADS];
+ } __attribute__((aligned(128)));
};
// Main context for htp DSP backend
@@ -118,6 +124,13 @@ struct htp_context {
size_t footprint;
};
+static inline void htp_ops_context_reset(struct htp_ops_context * octx, struct htp_context * ctx) {
+ memset(octx, 0, offsetof(struct htp_ops_context, dma_cache_dm));
+ octx->ctx = ctx;
+ octx->n_threads = ctx->n_threads;
+ octx->n_threads_div = ctx->n_threads_div;
+}
+
static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) {
if (n_threads == 0 || n_threads > octx->ctx->n_threads) {
return false;
@@ -148,7 +161,7 @@ int op_unary(struct htp_ops_context * octx);
int op_sum(struct htp_ops_context * octx);
int op_sum_rows(struct htp_ops_context * octx);
int op_argmax(struct htp_ops_context * octx);
-int op_activations(struct htp_ops_context * octx);
+int op_glu(struct htp_ops_context * octx);
int op_softmax(struct htp_ops_context * octx);
int op_add_id(struct htp_ops_context * octx);
int op_rope(struct htp_ops_context * octx);
diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h
index 9b4b86d3b..63cb309ff 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ops.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h
@@ -38,11 +38,6 @@ enum htp_data_type {
HTP_TYPE_INVALID
};
-// Constants for internal types
-#define QK_Q8_0_TILED 128 // 32x32 Q8_0 tiled layout
-
-
-
// Do not reorder first 4 (used as an index)
enum htp_op_code {
HTP_OP_MUL = 0,
@@ -115,6 +110,8 @@ enum htp_op_code {
HTP_OP_UNARY_GELU_QUICK,
HTP_OP_IM2COL_3D,
HTP_OP_HADAMARD,
+ HTP_OP_GLU_REGLU,
+ HTP_OP_GLU_SIGMOID,
HTP_OP_INVALID
};
@@ -125,8 +122,8 @@ enum htp_op_code {
#define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS
#define HTP_OP_MAX_KERN_PARAMS 32
-#define HTP_OP_MAX_BUFS 16
-#define HTP_OP_MAX_TENSORS 8192 // must stay under 64K (uint16)
+#define HTP_OP_MAX_BUFS 64
+#define HTP_OP_MAX_TENSORS 65534 // must stay under 64K (uint16)
#define HTP_FENCE_TIMEOUT (1000000000ULL)
#define HTP_FENCE_SLOT_SIZE 128
@@ -136,9 +133,10 @@ enum htp_op_code {
#define HTP_MMAP_MAX_VMEM (2147483648u)
enum htp_tensor_flags {
- HTP_TENSOR_WEIGHT = (1U << 0), // Tensor buffer model weight data (not compute)
- HTP_TENSOR_TILED = (1U << 1), // Tensor is in tiled format
- HTP_TENSOR_FENCE = (1U << 2) // Tensor is synchronization fence (explicitly managed)
+ HTP_TENSOR_WEIGHT = (1U << 0), // Tensor buffer model weight data (not compute)
+ HTP_TENSOR_TILED = (1U << 1), // Tensor is in tiled format
+ HTP_TENSOR_FENCE = (1U << 2), // Tensor is synchronization fence (explicitly managed)
+ HTP_TENSOR_FUSEABLE = (1U << 3), // Tensor can be fused (host-only)
};
enum htp_buf_flags {
diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h
index 9b2e13133..01c671679 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h
@@ -233,7 +233,7 @@ static inline void quantize_block_f32_q8_0_tiled(float * restrict x, uint8_t * r
static void quantize_row_f32_q8_0_tiled(float * restrict x, uint8_t * restrict y, uint32_t k) {
assert(k % 32 == 0);
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (k + qk - 1) / qk;
for (uint32_t i = 0; i < nb; i++) {
@@ -244,7 +244,7 @@ static void quantize_row_f32_q8_0_tiled(float * restrict x, uint8_t * restrict y
static void quantize_row_f32_q8_1_tiled(float * restrict x, uint8_t * restrict y, uint32_t k) {
assert(k % 32 == 0);
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (k + qk - 1) / qk;
for (uint32_t i = 0; i < nb; i++) {
@@ -255,7 +255,7 @@ static void quantize_row_f32_q8_1_tiled(float * restrict x, uint8_t * restrict y
static void quantize_row_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restrict y, uint32_t k) {
assert(k % 32 == 0);
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (k + qk - 1) / qk;
for (uint32_t i = 0; i < nb; i++) {
@@ -310,7 +310,7 @@ static inline void quantize_block_f16_f16_tiled(const __fp16 * restrict x, uint8
static void quantize_row_f32_f16_tiled(const float * restrict x, uint8_t * restrict y, uint32_t k) {
assert(k % 32 == 0);
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (k + qk - 1) / qk;
for (uint32_t i = 0; i < nb; i++) {
@@ -321,7 +321,7 @@ static void quantize_row_f32_f16_tiled(const float * restrict x, uint8_t * restr
static void quantize_row_f16_f16_tiled(const __fp16 * restrict x, uint8_t * restrict y, uint32_t k) {
assert(k % 32 == 0);
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (k + qk - 1) / qk;
for (uint32_t i = 0; i < nb; i++) {
@@ -1963,7 +1963,7 @@ static inline void quantize_f32_q8_0_tiled_block_kernel(
uint32_t c
) {
(void) tmp_data;
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (ne0 + qk - 1) / qk;
for (uint32_t ib = ib_first; ib < ib_last; ++ib) {
@@ -1993,7 +1993,7 @@ static inline void quantize_f32_q8_1_tiled_block_kernel(
uint32_t c
) {
(void) tmp_data;
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (ne0 + qk - 1) / qk;
for (uint32_t ib = ib_first; ib < ib_last; ++ib) {
@@ -2023,7 +2023,7 @@ static inline void quantize_f32_q8_1_s16_tiled_block_kernel(
uint32_t c
) {
(void) tmp_data;
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (ne0 + qk - 1) / qk;
for (uint32_t ib = ib_first; ib < ib_last; ++ib) {
@@ -2053,7 +2053,7 @@ static inline void quantize_f32_f16_tiled_block_kernel(
uint32_t c
) {
(void) tmp_data;
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (ne0 + qk - 1) / qk;
for (uint32_t ib = ib_first; ib < ib_last; ++ib) {
@@ -2083,7 +2083,7 @@ static inline void quantize_f16_f16_tiled_block_kernel(
uint32_t c
) {
(void) tmp_data;
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (ne0 + qk - 1) / qk;
for (uint32_t ib = ib_first; ib < ib_last; ++ib) {
diff --git a/ggml/src/ggml-hexagon/htp/hvx-norm.h b/ggml/src/ggml-hexagon/htp/hvx-norm.h
index 7ea945a33..145c3086e 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-norm.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-norm.h
@@ -11,7 +11,8 @@
static inline void hvx_fast_rms_norm_f32(const uint8_t * restrict src,
uint8_t * restrict dst,
const int num_elems,
- float epsilon) {
+ float epsilon,
+ float scale) {
const HVX_Vector * restrict v_src = (HVX_Vector *) src;
HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
@@ -48,6 +49,9 @@ static inline void hvx_fast_rms_norm_f32(const uint8_t * restrict src,
// Scale full vectors
HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v));
+ if (scale != 0.0f) {
+ scale_v = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(scale_v, hvx_vec_splat_f32(scale)));
+ }
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
@@ -72,7 +76,8 @@ static inline void hvx_fast_rms_norm_mul_f32(const uint8_t * restrict src,
const uint8_t * restrict weight,
uint8_t * restrict dst,
const int num_elems,
- float epsilon) {
+ float epsilon,
+ float scale) {
const HVX_Vector * restrict v_src = (const HVX_Vector *) src;
const HVX_Vector * restrict v_weight = (const HVX_Vector *) weight;
HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
@@ -109,6 +114,9 @@ static inline void hvx_fast_rms_norm_mul_f32(const uint8_t * restrict src,
// Scale and multiply
HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v));
+ if (scale != 0.0f) {
+ scale_v = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(scale_v, hvx_vec_splat_f32(scale)));
+ }
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
diff --git a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h
index b7c2d7409..d4d67ff2f 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h
@@ -68,6 +68,14 @@ static inline HVX_Vector hvx_vec_tanh_f32(HVX_Vector x) {
return Q6_Vsf_equals_Vqf32(res);
}
+static inline HVX_Vector hvx_vec_silu_f32(HVX_Vector x) {
+ HVX_Vector one = hvx_vec_splat_f32(1.0f);
+ HVX_Vector max_exp = hvx_vec_splat_f32(87.f);
+ HVX_Vector min_exp = hvx_vec_splat_f32(-87.f);
+ HVX_Vector sig = hvx_vec_fast_sigmoid_f32_guard(x, one, max_exp, min_exp);
+ return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(x, sig));
+}
+
#define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.c b/ggml/src/ggml-hexagon/htp/im2col-ops.c
index ee3c7da11..fc49c403b 100644
--- a/ggml/src/ggml-hexagon/htp/im2col-ops.c
+++ b/ggml/src/ggml-hexagon/htp/im2col-ops.c
@@ -768,19 +768,13 @@ int op_im2col(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
- return HTP_STATUS_INVAL_PARAMS;
- }
-
const struct htp_tensor * dst = octx->dst;
uint32_t row_base = 0;
uint32_t nrows = kparams->total_rows;
if (octx->ctx->mdev.count > 1) {
- const uint32_t row_size = dst->nb[2];
- const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;
const struct htp_tensor_mdev_range range =
- htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0,
+ htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0,
octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_base = range.start;
nrows = range.count;
@@ -790,20 +784,27 @@ int op_im2col(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
+ if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nrows))) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
+ struct htp_im2col_vtcm_layout layout;
+ htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads);
+
struct htp_im2col_context ictx = { 0 };
uint8_t * const base = octx->ctx->vtcm_base;
- const uint32_t pth = MIN(octx->n_threads, nrows);
+ const uint32_t pth = octx->n_threads;
ictx.octx = octx;
ictx.pe_row_base = row_base;
ictx.pe_nrows = nrows;
- ictx.pe_rows_per_thread = (nrows + pth - 1) / pth;
+ ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div);
ictx.pe_src_row_bytes = kparams->src_row_bytes;
ictx.pe_dst_row_bytes = kparams->dst_row_bytes;
- ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, kparams->off_src);
- ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, kparams->off_dst);
- ictx.pe_src_size_per_thread = kparams->src_bytes_per_thread;
- ictx.pe_dst_size_per_thread = kparams->dst_bytes_per_thread;
+ ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src);
+ ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst);
+ ictx.pe_src_size_per_thread = layout.src_bytes_per_thread;
+ ictx.pe_dst_size_per_thread = layout.dst_bytes_per_thread;
ictx.pe_owb = kparams->owb;
ictx.pe_wb = kparams->wb;
ictx.pe_nbpr = kparams->nbpr;
@@ -821,9 +822,10 @@ int op_im2col(struct htp_ops_context * octx) {
ictx.patch_stride = kparams->patch_stride;
for (uint32_t ith = 0; ith < pth; ith++) {
- uint32_t r = row_base + ictx.pe_rows_per_thread * ith;
- ictx.thread_ioh[ith] = r % kparams->OH;
- ictx.thread_in[ith] = r / kparams->OH;
+ const uint32_t r = row_base + ictx.pe_rows_per_thread * ith;
+ const uint32_t in = fastdiv(r, &kparams->div_OH);
+ ictx.thread_ioh[ith] = r - in * kparams->OH;
+ ictx.thread_in[ith] = in;
}
ictx.dst_elem = dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float);
@@ -850,19 +852,13 @@ int op_im2col_3d(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
- return HTP_STATUS_INVAL_PARAMS;
- }
-
const struct htp_tensor * dst = octx->dst;
uint32_t row_base = 0;
uint32_t nrows = kparams->total_rows;
if (octx->ctx->mdev.count > 1) {
- const uint32_t row_size = dst->nb[2];
- const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;
const struct htp_tensor_mdev_range range =
- htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0,
+ htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0,
octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_base = range.start;
nrows = range.count;
@@ -872,20 +868,27 @@ int op_im2col_3d(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
+ if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nrows))) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
+ struct htp_im2col_vtcm_layout layout;
+ htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads);
+
struct htp_im2col_context ictx = { 0 };
uint8_t * const base = octx->ctx->vtcm_base;
- const uint32_t pth = MIN(octx->n_threads, nrows);
+ const uint32_t pth = octx->n_threads;
ictx.octx = octx;
ictx.pe_row_base = row_base;
ictx.pe_nrows = nrows;
- ictx.pe_rows_per_thread = (nrows + pth - 1) / pth;
+ ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div);
ictx.pe_src_row_bytes = kparams->src_row_bytes;
ictx.pe_dst_row_bytes = kparams->dst_row_bytes;
- ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, kparams->off_src);
- ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, kparams->off_dst);
- ictx.pe_src_size_per_thread = kparams->src_bytes_per_thread;
- ictx.pe_dst_size_per_thread = kparams->dst_bytes_per_thread;
+ ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src);
+ ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst);
+ ictx.pe_src_size_per_thread = layout.src_bytes_per_thread;
+ ictx.pe_dst_size_per_thread = layout.dst_bytes_per_thread;
ictx.pe_owb = kparams->owb;
ictx.pe_wb = kparams->wb;
ictx.pe_nbpr = kparams->nbpr;
@@ -903,11 +906,12 @@ int op_im2col_3d(struct htp_ops_context * octx) {
ictx.patch_stride = kparams->patch_stride;
for (uint32_t ith = 0; ith < pth; ith++) {
- uint32_t rem = row_base + ictx.pe_rows_per_thread * ith;
- ictx.thread_ioh[ith] = rem % kparams->OH;
- rem /= kparams->OH;
- ictx.thread_iod[ith] = rem % kparams->OD;
- ictx.thread_in[ith] = rem / kparams->OD;
+ const uint32_t rem = row_base + ictx.pe_rows_per_thread * ith;
+ const uint32_t q_oh = fastdiv(rem, &kparams->div_OH);
+ ictx.thread_ioh[ith] = rem - q_oh * kparams->OH;
+ const uint32_t in = fastdiv(q_oh, &kparams->div_OD);
+ ictx.thread_iod[ith] = q_oh - in * kparams->OD;
+ ictx.thread_in[ith] = in;
}
ictx.dst_elem = dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float);
diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.h b/ggml/src/ggml-hexagon/htp/im2col-ops.h
index 9ccc21fd7..85aa3ca35 100644
--- a/ggml/src/ggml-hexagon/htp/im2col-ops.h
+++ b/ggml/src/ggml-hexagon/htp/im2col-ops.h
@@ -2,6 +2,8 @@
#define HTP_IM2COL_OPS_H
#include "hex-common.h"
+#include "hex-fastdiv.h"
+#include "htp-vtcm.h"
#include <stdint.h>
enum htp_im2col_kernel_type {
@@ -11,38 +13,33 @@ enum htp_im2col_kernel_type {
};
struct htp_im2col_vtcm_layout {
- uint32_t off_src;
- uint32_t off_dst;
+ size_t total_bytes;
+ size_t off_src;
+ size_t off_dst;
uint32_t src_bytes_per_thread;
uint32_t dst_bytes_per_thread;
- uint32_t total_bytes;
};
static inline void htp_im2col_vtcm_layout_build(
- struct htp_im2col_vtcm_layout * layout,
+ struct htp_im2col_vtcm_layout * L,
uint32_t src_row_bytes,
uint32_t dst_row_bytes,
uint32_t n_threads) {
- layout->src_bytes_per_thread = 2 * src_row_bytes + 256;
- layout->dst_bytes_per_thread = 2 * dst_row_bytes;
+ L->src_bytes_per_thread = 2 * src_row_bytes + 256;
+ L->dst_bytes_per_thread = 2 * dst_row_bytes;
- layout->off_src = 0;
- layout->off_dst = layout->off_src + layout->src_bytes_per_thread * n_threads;
- layout->total_bytes = layout->off_dst + layout->dst_bytes_per_thread * n_threads;
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_src, (size_t) L->src_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) L->dst_bytes_per_thread * n_threads);
+ L->total_bytes = off;
}
struct htp_im2col_kernel_params {
uint8_t kernel_type;
- uint8_t is_2d;
- uint8_t is_3d;
uint8_t n_threads;
uint32_t vtcm_size;
- uint32_t off_src;
- uint32_t off_dst;
- uint32_t src_bytes_per_thread;
- uint32_t dst_bytes_per_thread;
uint32_t src_row_bytes;
uint32_t dst_row_bytes;
@@ -57,6 +54,10 @@ struct htp_im2col_kernel_params {
uint32_t OD, OH, OW;
uint32_t patch_stride;
uint32_t total_rows;
+ uint32_t rows_per_chunk;
+
+ struct fastdiv_values div_OH;
+ struct fastdiv_values div_OD;
};
#if defined(__cplusplus)
diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c
index 5ac6c3c52..01128057b 100644
--- a/ggml/src/ggml-hexagon/htp/main.c
+++ b/ggml/src/ggml-hexagon/htp/main.c
@@ -955,7 +955,9 @@ static int execute_op(struct htp_ops_context * octx) {
case HTP_OP_GLU_GEGLU:
case HTP_OP_GLU_GEGLU_QUICK:
case HTP_OP_GLU_GEGLU_ERF:
- return op_activations(octx);
+ case HTP_OP_GLU_REGLU:
+ case HTP_OP_GLU_SIGMOID:
+ return op_glu(octx);
case HTP_OP_SOFTMAX:
return op_softmax(octx);
@@ -1244,8 +1246,6 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs
octx->src0_spad.src = NULL;
octx->src1_spad.src = NULL;
- octx->src2_spad.src = NULL;
- octx->src3_spad.src = NULL;
octx->dst_spad.src = NULL;
return octx->status;
@@ -1306,10 +1306,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
prep_tensors(ctx, bufs, tens, n_tens);
struct htp_ops_context *octx = &ctx->octx;
- memset(octx, 0, sizeof(*octx));
- octx->n_threads = ctx->n_threads;
- octx->n_threads_div = ctx->n_threads_div;
- octx->ctx = ctx;
+ htp_ops_context_reset(octx, ctx);
mdev_group_init(ctx, req);
diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c
index 9af312925..e290835fc 100644
--- a/ggml/src/ggml-hexagon/htp/matmul-ops.c
+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c
@@ -1816,7 +1816,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (ne10 + qk - 1) / qk;
const uint32_t total_nb = act_nrows * nb;
@@ -1886,7 +1886,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
mmctx->vtcm_src0_stride = src0_row_size_padded;
mmctx->vtcm_act_stride = act_row_size;
if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW) {
- mmctx->vtcm_act_raw_stride = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));
+ mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(ne10);
} else if (kparams->kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM) {
mmctx->vtcm_act_raw_stride = hex_round_up(ne10 * sizeof(float), 128);
} else {
@@ -1904,7 +1904,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
if (need_quant) {
hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, m_start, cur_m_rows);
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (ne10 + qk - 1) / qk;
const uint32_t total_nb = cur_m_rows * nb;
uint32_t quant_tasks;
@@ -3914,7 +3914,7 @@ static int hvx_mm_matmul_id(
const struct htp_tensor * restrict ids = octx->src[2];
const size_t src0_row_size = nb01;
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (ne10 + qk - 1) / qk;
const uint32_t total_nb = act_nrows * nb;
@@ -3966,7 +3966,7 @@ static int hvx_mm_matmul_id(
mmctx->vtcm_src0_stride = src0_row_size_padded;
mmctx->vtcm_act_stride = act_row_size;
- mmctx->vtcm_act_raw_stride = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));
+ mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(ne10);
mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div);
mmctx->vtcm_act_size_per_thread = L.act_bytes;
@@ -4081,7 +4081,7 @@ static int hvx_mm_matmul_id_nx(
const struct htp_tensor * restrict ids = octx->src[n_weights + 1];
const size_t src0_row_size = src0->nb[1];
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (act->ne[0] + qk - 1) / qk;
const uint32_t total_nb = act_nrows * nb;
@@ -4124,7 +4124,7 @@ static int hvx_mm_matmul_id_nx(
mmctx->vtcm_src0_stride = 0;
mmctx->vtcm_act_stride = act_row_size;
- mmctx->vtcm_act_raw_stride = hex_round_up(act->ne[0] * sizeof(float), QK_Q8_0_TILED * sizeof(float));
+ mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(act->ne[0]);
mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div);
mmctx->vtcm_act_size_per_thread = L.act_bytes;
@@ -4474,7 +4474,7 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
- const uint32_t qk = QK_Q8_0_TILED;
+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;
const uint32_t nb = (act->ne[0] + qk - 1) / qk;
const uint32_t total_nb = act_nrows * nb;
@@ -4518,7 +4518,7 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) {
mmctx->vtcm_src0_stride = 0;
mmctx->vtcm_act_stride = act_row_size;
- mmctx->vtcm_act_raw_stride = hex_round_up(act->ne[0] * sizeof(float), QK_Q8_0_TILED * sizeof(float));
+ mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(act->ne[0]);
mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div);
mmctx->vtcm_act_size_per_thread = L.act_bytes;
diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h
index 09875a603..474c2dbc0 100644
--- a/ggml/src/ggml-hexagon/htp/matmul-ops.h
+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h
@@ -25,6 +25,7 @@ extern "C" {
#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0 640
// --- Activation Tiled Block Sizes (including padding) ---
+#define HTP_MM_ACT_BLOCK_K 128
#define HTP_MM_ACT_TILE_SIZE_Q8_0 1152
#define HTP_MM_ACT_TILE_SIZE_Q8_1 1280
#define HTP_MM_ACT_TILE_SIZE_F16 2048
@@ -74,10 +75,7 @@ struct htp_mm_kernel_params {
int32_t aligned_tile_size; // Aligned weight tile size (padded to 128)
int32_t act_row_size; // Row size for activation scratchpad
int32_t vtcm_size; // Total required scratchpad size in VTCM
- int32_t vtcm_src0_size; // src0 scratchpad size in VTCM
- int32_t vtcm_act_size; // activation scratchpad size in VTCM
int32_t vtcm_bias_size; // bias scratchpad size in VTCM (fused only)
- int32_t vtcm_dst_size; // dst scratchpad size in VTCM
// Precomputed division values
struct fastdiv_values div_ne12_ne1;
@@ -192,19 +190,19 @@ static inline bool htp_mm_weight_has_offset(int weight_type) {
// --- Activation/Row Size Helpers ---
static inline size_t htp_mm_q8_0_tiled_row_size(uint32_t ne) {
- const uint32_t ne_padded = ((ne + 127) / 128) * 128;
+ const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K);
const uint32_t nb_32 = ne_padded / 32;
return nb_32 * HTP_MM_ACT_TILE_SIZE_Q8_0;
}
static inline size_t htp_mm_q8_1_tiled_row_size(uint32_t ne) {
- const uint32_t ne_padded = ((ne + 127) / 128) * 128;
+ const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K);
const uint32_t nb_32 = ne_padded / 32;
return nb_32 * HTP_MM_ACT_TILE_SIZE_Q8_1;
}
static inline size_t htp_mm_f16_tiled_row_size(uint32_t ne) {
- const uint32_t ne_padded = ((ne + 127) / 128) * 128;
+ const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K);
const uint32_t nb_32 = ne_padded / 32;
return nb_32 * HTP_MM_ACT_TILE_SIZE_F16;
}
@@ -216,6 +214,10 @@ static inline size_t htp_mm_tiled_act_row_size(int weight_type, uint32_t ne) {
return htp_mm_weight_has_offset(weight_type) ? htp_mm_q8_1_tiled_row_size(ne) : htp_mm_q8_0_tiled_row_size(ne);
}
+static inline size_t htp_mm_act_raw_row_size(uint32_t ne) {
+ return hex_round_up(ne, HTP_MM_ACT_BLOCK_K) * sizeof(float);
+}
+
static inline size_t htp_mm_round_up(size_t n, size_t m) {
return ((n + m - 1) / m) * m;
}
@@ -445,7 +447,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(
size_t tiled_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
size_t q_act_sz = hex_round_up(tiled_act_row_size * act_nrows, 128);
- size_t raw_row_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));
+ size_t raw_row_size = htp_mm_act_raw_row_size(ne10);
src0_sz = weight_sz_per_thread * n_threads; // shared single-weight prefetch buffer
act_sz = q_act_sz; // quantized activation buffer
@@ -461,7 +463,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(
act_sz = htp_mm_round_up(act_row_size_tiled * act_nrows, 256);
- size_t raw_row_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));
+ size_t raw_row_size = htp_mm_act_raw_row_size(ne10);
src0_sz = src0_sz_per_thread * n_threads;
dst_sz = 0;
@@ -502,7 +504,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(
size_t dst_slice_per_thread = (dst_nrows > 0 && act_nrows == 1) ? htp_mm_round_up((dst_row_size + n_threads - 1) / n_threads, 128) : 0;
dst_sz = dst_slice_per_thread * n_threads;
- size_t raw_row_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));
+ size_t raw_row_size = htp_mm_act_raw_row_size(ne10);
act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128);
break;
}
@@ -582,7 +584,7 @@ static inline bool htp_mm_hvx_solve_vtcm_params(
size_t eff_row_size = row_size;
if (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {
- eff_row_size += hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));
+ eff_row_size += htp_mm_act_raw_row_size(ne10);
}
uint32_t m_chunk = (uint32_t) (avail_act / eff_row_size);
diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.c b/ggml/src/ggml-hexagon/htp/pad-ops.c
index 85f25a8eb..cda6df318 100644
--- a/ggml/src/ggml-hexagon/htp/pad-ops.c
+++ b/ggml/src/ggml-hexagon/htp/pad-ops.c
@@ -17,18 +17,28 @@
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
+#include "htp-vtcm.h"
+#include "pad-ops.h"
/* Circular wrap: maps any integer x into [0, n) */
-static inline uint32_t wrap_around(int32_t x, uint32_t n) {
- return (uint32_t)(((x % (int32_t)n) + (int32_t)n) % (int32_t)n);
+static inline uint32_t wrap_around(int32_t x, uint32_t n, const struct fastdiv_values * div_n) {
+ if (x >= 0) {
+ const uint32_t ux = (uint32_t) x;
+ return ux < n ? ux : fastmodulo(ux, n, div_n);
+ }
+ const uint32_t ux = (uint32_t) (-x);
+ const uint32_t rem = ux < n ? ux : fastmodulo(ux, n, div_n);
+ return rem ? (n - rem) : 0;
}
/* Decompose a flat dst row index into (i1, i2, i3) */
static inline void pad_decompose_row(uint32_t ir, uint32_t ne1, uint32_t ne2,
+ const struct htp_pad_kernel_params * kparams,
uint32_t *i1, uint32_t *i2, uint32_t *i3) {
- *i1 = ir % ne1;
- *i2 = (ir / ne1) % ne2;
- *i3 = ir / (ne1 * ne2);
+ *i3 = fastdiv(ir, &kparams->div_ne1_ne2);
+ const uint32_t rem = ir - (*i3) * (ne1 * ne2);
+ *i2 = fastdiv(rem, &kparams->div_ne1);
+ *i1 = rem - (*i2) * ne1;
}
/* Return non-zero if row (i1,i2,i3) falls in the non-padded interior */
@@ -62,25 +72,28 @@ static inline dma_addr_t pad_src_row_data(const struct htp_tensor * src,
/* Compute the DDR src row pointer for a circular row (wrap-around indexing) */
static inline const uint8_t * pad_circ_src_row_ptr(const struct htp_tensor * src,
+ const struct htp_pad_kernel_params * kparams,
uint32_t i1, uint32_t i2, uint32_t i3,
int32_t lp1, int32_t lp2, int32_t lp3) {
return (const uint8_t *) src->data
- + wrap_around((int32_t)i1 - lp1, src->ne[1]) * src->nb[1]
- + wrap_around((int32_t)i2 - lp2, src->ne[2]) * src->nb[2]
- + wrap_around((int32_t)i3 - lp3, src->ne[3]) * src->nb[3];
+ + wrap_around((int32_t)i1 - lp1, src->ne[1], &kparams->div_ne01) * src->nb[1]
+ + wrap_around((int32_t)i2 - lp2, src->ne[2], &kparams->div_ne02) * src->nb[2]
+ + wrap_around((int32_t)i3 - lp3, src->ne[3], &kparams->div_ne03) * src->nb[3];
}
static inline dma_addr_t pad_circ_src_row_data(const struct htp_tensor * src,
+ const struct htp_pad_kernel_params * kparams,
uint32_t i1, uint32_t i2, uint32_t i3,
int32_t lp1, int32_t lp2, int32_t lp3) {
return src->data
- + wrap_around((int32_t)i1 - lp1, src->ne[1]) * src->nb[1]
- + wrap_around((int32_t)i2 - lp2, src->ne[2]) * src->nb[2]
- + wrap_around((int32_t)i3 - lp3, src->ne[3]) * src->nb[3];
+ + wrap_around((int32_t)i1 - lp1, src->ne[1], &kparams->div_ne01) * src->nb[1]
+ + wrap_around((int32_t)i2 - lp2, src->ne[2], &kparams->div_ne02) * src->nb[2]
+ + wrap_around((int32_t)i3 - lp3, src->ne[3], &kparams->div_ne03) * src->nb[3];
}
struct htp_pad_context {
struct htp_ops_context * octx;
+ const struct htp_pad_kernel_params * kparams;
int32_t lp0, rp0;
int32_t lp1, rp1;
@@ -152,7 +165,7 @@ static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * da
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
uint32_t i1, i2, i3;
- pad_decompose_row(dst_row, ne1, ne2, &i1, &i2, &i3);
+ pad_decompose_row(dst_row, ne1, ne2, pctx->kparams, &i1, &i2, &i3);
uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
@@ -219,7 +232,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void
dst_row_size, dst_row_size_aligned, dst_row_size, 0);
uint32_t i1, i2, i3;
- pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3);
+ pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3);
const int interior = pad_is_interior(i1, i2, i3,
lp1, rp1, ne1,
lp2, rp2, ne2,
@@ -246,7 +259,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void
uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma_q).dst;
uint32_t i1, i2, i3;
- pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3);
+ pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3);
const dma_addr_t dst_data = dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
@@ -278,7 +291,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void
const uint32_t next_row = ir + 2;
if (next_row < row_end) {
uint32_t ni1, ni2, ni3;
- pad_decompose_row(next_row, ne1, ne2, &ni1, &ni2, &ni3);
+ pad_decompose_row(next_row, ne1, ne2, pctx->kparams, &ni1, &ni2, &ni3);
const int next_interior = pad_is_interior(ni1, ni2, ni3,
lp1, rp1, ne1,
lp2, rp2, ne2,
@@ -315,10 +328,10 @@ static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith,
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
uint32_t i1, i2, i3;
- pad_decompose_row(dst_row, ne1, ne2, &i1, &i2, &i3);
+ pad_decompose_row(dst_row, ne1, ne2, pctx->kparams, &i1, &i2, &i3);
uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
- const uint8_t * src_row = pad_circ_src_row_ptr(src, i1, i2, i3, lp1, lp2, lp3);
+ const uint8_t * src_row = pad_circ_src_row_ptr(src, pctx->kparams, i1, i2, i3, lp1, lp2, lp3);
if (nb00 == type_size) {
@@ -393,8 +406,8 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i
dst_row_size, dst_row_size_aligned, dst_row_size, 0);
uint32_t pi1, pi2, pi3;
- pad_decompose_row(ir, ne1, ne2, &pi1, &pi2, &pi3);
- const dma_addr_t src_data = pad_circ_src_row_data(src, pi1, pi2, pi3, lp1, lp2, lp3);
+ pad_decompose_row(ir, ne1, ne2, pctx->kparams, &pi1, &pi2, &pi3);
+ const dma_addr_t src_data = pad_circ_src_row_data(src, pctx->kparams, pi1, pi2, pi3, lp1, lp2, lp3);
dma_queue_push(dma_q,
dma_make_data(src_spad_cur, src_data),
src_row_size_aligned, src_row_size, src_row_size, 1);
@@ -411,7 +424,7 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i
uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma_q).dst;
uint32_t i1, i2, i3;
- pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3);
+ pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3);
const dma_addr_t dst_data = dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
@@ -455,8 +468,8 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i
const uint32_t next_row = ir + 2;
if (next_row < row_end) {
uint32_t nri1, nri2, nri3;
- pad_decompose_row(next_row, ne1, ne2, &nri1, &nri2, &nri3);
- const dma_addr_t next_src_data = pad_circ_src_row_data(src, nri1, nri2, nri3, lp1, lp2, lp3);
+ pad_decompose_row(next_row, ne1, ne2, pctx->kparams, &nri1, &nri2, &nri3);
+ const dma_addr_t next_src_data = pad_circ_src_row_data(src, pctx->kparams, nri1, nri2, nri3, lp1, lp2, lp3);
dma_queue_push(dma_q,
dma_make_data(src_spad_cur, next_src_data),
src_row_size_aligned, src_row_size, src_row_size, 1);
@@ -475,15 +488,8 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i
int op_pad(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
-
- // Only F32 supported
- size_t type_size;
- switch (src0->type) {
- case HTP_TYPE_F32: type_size = 4; break;
- default:
- FARF(ERROR, "pad-hvx: unsupported type %u\n", src0->type);
- return HTP_STATUS_NO_SUPPORT;
- }
+ const struct htp_pad_kernel_params * kparams =
+ (const struct htp_pad_kernel_params *) octx->kernel_params;
const int32_t lp0 = octx->op_params[0];
const int32_t rp0 = octx->op_params[1];
@@ -495,11 +501,10 @@ int op_pad(struct htp_ops_context * octx) {
const int32_t rp3 = octx->op_params[7];
const int32_t circular = octx->op_params[8];
- const uint32_t ne0 = dst->ne[0];
- const uint32_t ne00 = src0->ne[0];
-
- const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3];
- const size_t dst_row_size = (size_t)ne0 * type_size;
+ const uint32_t total_dst_rows = kparams->total_rows;
+ const size_t type_size = kparams->elem_size;
+ const size_t src_row_size = src0->ne[0] * type_size;
+ const size_t dst_row_size = dst->ne[0] * type_size;
uint32_t row_start = 0;
uint32_t nrows = total_dst_rows;
@@ -516,60 +521,68 @@ int op_pad(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
-
- const size_t src_row_size = (size_t)ne00 * type_size;
- const size_t src_row_size_aligned = hex_round_up(src_row_size, VLEN);
- const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);
-
- // Total VTCM needed: 2 buffers (ping+pong) for src and dst, per thread
- const size_t vtcm_needed = (size_t)n_threads * 2 * (src_row_size_aligned + dst_row_size_aligned);
+ if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
- const int use_dma = (src0->nb[0] == (uint32_t)type_size) && (ne00 >= 512) &&
- (octx->ctx->vtcm_size >= vtcm_needed);
+ const uint32_t n_threads = octx->n_threads;
- if (!use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {
+ if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {
return HTP_STATUS_NO_SUPPORT;
}
- if (use_dma) {
- octx->src0_spad.size_per_thread = 2 * src_row_size_aligned;
- octx->dst_spad.size_per_thread = 2 * dst_row_size_aligned;
- octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread;
- octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread;
- octx->src0_spad.data = octx->ctx->vtcm_base;
- octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size;
+ struct htp_pad_vtcm_layout layout = { 0 };
+ if (kparams->use_dma) {
+ htp_pad_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_threads);
+
+ octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread;
+ octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread;
+ octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread;
+ octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread;
+ octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0);
+ octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst);
octx->src0_spad.src = NULL;
octx->dst_spad.src = NULL;
}
struct htp_pad_context pctx = {
- .octx = octx,
+ .octx = octx,
+ .kparams = kparams,
.lp0 = lp0, .rp0 = rp0,
.lp1 = lp1, .rp1 = rp1,
.lp2 = lp2, .rp2 = rp2,
.lp3 = lp3, .rp3 = rp3,
- .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
- .total_dst_rows = nrows,
- .row_start = row_start,
- .type_size = type_size,
+ .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
+ .total_dst_rows = nrows,
+ .row_start = row_start,
+ .type_size = type_size,
.src_row_size = src_row_size,
- .src_row_size_aligned = src_row_size_aligned,
+ .src_row_size_aligned = layout.src_row_size_aligned,
.dst_row_size = dst_row_size,
- .dst_row_size_aligned = dst_row_size_aligned,
+ .dst_row_size_aligned = layout.dst_row_size_aligned,
};
FARF(HIGH, "pad-hvx%s%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) pads=(%d,%d,%d,%d,%d,%d,%d,%d)\n",
- circular ? "-circ" : "",
- use_dma ? "-dma" : "",
+ circular ? "-circ" : "",
+ kparams->use_dma ? "-dma" : "",
src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3);
- if (circular && use_dma) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); }
- else if (circular) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular, &pctx, n_threads); }
- else if (use_dma) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_dma, &pctx, n_threads); }
- else { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx, &pctx, n_threads); }
+ switch (kparams->kernel_type) {
+ case HTP_PAD_KERNEL_HVX_CIRCULAR_DMA:
+ work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads);
+ break;
+ case HTP_PAD_KERNEL_HVX_CIRCULAR:
+ work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular, &pctx, n_threads);
+ break;
+ case HTP_PAD_KERNEL_HVX_DMA:
+ work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_dma, &pctx, n_threads);
+ break;
+ default:
+ work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx, &pctx, n_threads);
+ break;
+ }
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.h b/ggml/src/ggml-hexagon/htp/pad-ops.h
new file mode 100644
index 000000000..5b8557538
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/pad-ops.h
@@ -0,0 +1,65 @@
+#ifndef HTP_PAD_OPS_H
+#define HTP_PAD_OPS_H
+
+#include "hex-common.h"
+#include "hex-fastdiv.h"
+#include "htp-vtcm.h"
+#include <stdint.h>
+
+enum htp_pad_kernel_type {
+ HTP_PAD_KERNEL_HVX = 0,
+ HTP_PAD_KERNEL_HVX_DMA = 1,
+ HTP_PAD_KERNEL_HVX_CIRCULAR = 2,
+ HTP_PAD_KERNEL_HVX_CIRCULAR_DMA = 3,
+};
+
+struct htp_pad_vtcm_layout {
+ size_t total_bytes;
+ size_t off_src0;
+ size_t off_dst;
+ size_t src_row_size_aligned;
+ size_t dst_row_size_aligned;
+ size_t src0_bytes_per_thread;
+ size_t dst_bytes_per_thread;
+};
+
+static inline void htp_pad_vtcm_layout_build(
+ struct htp_pad_vtcm_layout * L,
+ size_t src_row_size,
+ size_t dst_row_size,
+ uint32_t n_threads
+) {
+ L->src_row_size_aligned = hex_round_up(src_row_size, 128);
+ L->dst_row_size_aligned = hex_round_up(dst_row_size, 128);
+ L->src0_bytes_per_thread = 2 * L->src_row_size_aligned;
+ L->dst_bytes_per_thread = 2 * L->dst_row_size_aligned;
+
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+
+ L->total_bytes = off;
+}
+
+struct htp_pad_kernel_params {
+ uint8_t kernel_type;
+ uint32_t n_threads;
+ uint32_t total_rows;
+ uint32_t use_dma;
+ uint32_t elem_size;
+ uint32_t vtcm_size;
+
+ struct fastdiv_values div_ne1;
+ struct fastdiv_values div_ne1_ne2;
+ struct fastdiv_values div_ne01;
+ struct fastdiv_values div_ne02;
+ struct fastdiv_values div_ne03;
+};
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_pad_kernel_params) <= 128, "htp_pad_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_pad_kernel_params) <= 128, "htp_pad_kernel_params is too large for kernel_params blob");
+#endif
+
+#endif // HTP_PAD_OPS_H
diff --git a/ggml/src/ggml-hexagon/htp/pool-ops.c b/ggml/src/ggml-hexagon/htp/pool-ops.c
index ea1f1a597..b41c7bed2 100644
--- a/ggml/src/ggml-hexagon/htp/pool-ops.c
+++ b/ggml/src/ggml-hexagon/htp/pool-ops.c
@@ -247,6 +247,7 @@ typedef void (*pool_plane_fn_t)(const float * src, float * dst, const struct htp
struct pool_2d_context {
struct htp_ops_context * octx;
const struct htp_pool_2d_kernel_params * kparams;
+ struct htp_pool_vtcm_layout layout;
pool_plane_fn_t pool_plane;
uint32_t n_threads;
uint32_t plane_start;
@@ -257,6 +258,7 @@ struct pool_2d_context {
static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) {
struct pool_2d_context * ctx = (struct pool_2d_context *) data;
const struct htp_pool_2d_kernel_params * p = ctx->kparams;
+ const struct htp_pool_vtcm_layout * L = &ctx->layout;
const struct htp_tensor * src0 = ctx->octx->src[0];
const struct htp_tensor * dst = ctx->octx->dst;
pool_plane_fn_t pool_plane = ctx->pool_plane;
@@ -271,15 +273,14 @@ static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) {
struct htp_thread_trace * tr = &ctx->octx->ctx->trace[ith];
dma_queue * dma_queue = ctx->octx->ctx->dma[ith];
- const uint32_t src_spad_half = p->src_plane_bytes_aligned;
- const uint32_t dst_spad_half = p->dst_plane_bytes_aligned;
- const uint32_t src_bytes_per_thread = 2 * src_spad_half;
- const uint32_t dst_bytes_per_thread = 2 * dst_spad_half;
- const size_t off_dst = (size_t) ctx->n_threads * src_bytes_per_thread;
+ const uint32_t src_plane_bytes = L->src_plane_bytes;
+ const uint32_t dst_plane_bytes = L->dst_plane_bytes;
+ const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size;
+ const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size;
uint8_t * vtcm_base = (uint8_t *) ctx->octx->ctx->vtcm_base;
- uint8_t * src_spad = vtcm_base + ith * src_bytes_per_thread;
- uint8_t * dst_spad = vtcm_base + off_dst + ith * dst_bytes_per_thread;
+ uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread;
+ uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread;
float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) };
float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) };
@@ -290,14 +291,14 @@ static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) {
for (uint32_t i = 0; i < total && i < 2; ++i) {
dma_queue_push(dma_queue,
dma_make_data(dst->data, dstb2[i]),
- p->dst_plane_bytes, dst_spad_half,
- p->dst_plane_bytes, 0);
+ dst_plane_bytes, dst_spad_half,
+ dst_plane_bytes, 0);
- const dma_addr_t src_addr = src0->data + (first + i) * p->src_plane_bytes;
+ const dma_addr_t src_addr = src0->data + (first + i) * src_plane_bytes;
dma_queue_push(dma_queue,
dma_make_data(srcb2[i], src_addr),
- src_spad_half, p->src_plane_bytes,
- p->src_plane_bytes, 1);
+ src_spad_half, src_plane_bytes,
+ src_plane_bytes, 1);
}
for (uint32_t i = 0; i < total; ++i) {
@@ -313,18 +314,18 @@ static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) {
pool_plane(srcb, dstb, p);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) plane);
- const dma_addr_t dst_addr = dst->data + plane * p->dst_plane_bytes;
+ const dma_addr_t dst_addr = dst->data + plane * dst_plane_bytes;
dma_queue_push(dma_queue,
dma_make_data(dst_addr, dstb),
- p->dst_plane_bytes, dst_spad_half,
- p->dst_plane_bytes, 1);
+ dst_plane_bytes, dst_spad_half,
+ dst_plane_bytes, 1);
if (i + 2 < total) {
- const dma_addr_t next_src_addr = src0->data + (plane + 2) * p->src_plane_bytes;
+ const dma_addr_t next_src_addr = src0->data + (plane + 2) * src_plane_bytes;
dma_queue_push(dma_queue,
dma_make_data(srcb, next_src_addr),
- src_spad_half, p->src_plane_bytes,
- p->src_plane_bytes, 1);
+ src_spad_half, src_plane_bytes,
+ src_plane_bytes, 1);
}
}
@@ -337,22 +338,15 @@ static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) {
}
int op_pool_2d(struct htp_ops_context * octx) {
- const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
const struct htp_pool_2d_kernel_params * p =
(const struct htp_pool_2d_kernel_params *) octx->kernel_params;
- if (src0->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32 ||
- (p->pool_op != HTP_POOL_AVG && p->pool_op != HTP_POOL_MAX)) {
- return HTP_STATUS_NO_SUPPORT;
- }
-
uint32_t plane_start = 0;
uint32_t plane_count = p->planes;
if (octx->ctx->mdev.count > 1) {
- const uint32_t planes_per_chunk = (p->dst_plane_bytes > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(p->dst_plane_bytes, HEX_L2_LINE_SIZE)) : 1;
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
- plane_count, htp_tensor_mdev_data_aligned(dst) ? planes_per_chunk : 0,
+ plane_count, htp_tensor_mdev_data_aligned(dst) ? p->planes_per_chunk : 0,
octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
plane_start = range.start;
plane_count = range.count;
@@ -369,16 +363,12 @@ int op_pool_2d(struct htp_ops_context * octx) {
const uint32_t planes_per_thread = fastdiv(plane_count + n_threads - 1, &octx->n_threads_div);
pool_plane_fn_t pool_plane;
- if (p->global_path) {
- pool_plane = pool_plane_global;
- } else if (p->block_path) {
- pool_plane = pool_plane_block;
- } else if (p->narrow_path) {
- pool_plane = pool_plane_hvx_narrow;
- } else if (p->fast_path) {
- pool_plane = pool_plane_hvx;
- } else {
- pool_plane = pool_plane_general;
+ switch (p->kernel_type) {
+ case HTP_POOL_KERNEL_GLOBAL: pool_plane = pool_plane_global; break;
+ case HTP_POOL_KERNEL_BLOCK: pool_plane = pool_plane_block; break;
+ case HTP_POOL_KERNEL_HVX_NARROW: pool_plane = pool_plane_hvx_narrow; break;
+ case HTP_POOL_KERNEL_HVX: pool_plane = pool_plane_hvx; break;
+ default: pool_plane = pool_plane_general; break;
}
struct pool_2d_context ctx = {
@@ -390,6 +380,7 @@ int op_pool_2d(struct htp_ops_context * octx) {
.plane_count = plane_count,
.planes_per_thread = planes_per_thread,
};
+ htp_pool_vtcm_layout_build(&ctx.layout, p->src_x, p->src_y, p->dst_x, p->dst_y, n_threads);
work_queue_run(octx->ctx->work_queue, pool_2d_thread, &ctx, n_threads);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/pool-ops.h b/ggml/src/ggml-hexagon/htp/pool-ops.h
index 66b405a8b..178e5d00f 100644
--- a/ggml/src/ggml-hexagon/htp/pool-ops.h
+++ b/ggml/src/ggml-hexagon/htp/pool-ops.h
@@ -7,8 +7,18 @@
#include <string.h>
#include "hex-common.h"
+#include "htp-vtcm.h"
+
+enum htp_pool_kernel_type {
+ HTP_POOL_KERNEL_GENERAL = 0,
+ HTP_POOL_KERNEL_HVX,
+ HTP_POOL_KERNEL_HVX_NARROW,
+ HTP_POOL_KERNEL_GLOBAL,
+ HTP_POOL_KERNEL_BLOCK,
+};
struct htp_pool_2d_kernel_params {
+ uint8_t kernel_type;
uint32_t src_x;
uint32_t src_y;
uint32_t dst_x;
@@ -19,17 +29,10 @@ struct htp_pool_2d_kernel_params {
uint32_t stride_y;
int32_t pad_x;
int32_t pad_y;
- uint32_t src_plane_bytes;
- uint32_t dst_plane_bytes;
- uint32_t src_plane_bytes_aligned;
- uint32_t dst_plane_bytes_aligned;
uint32_t n_threads;
uint32_t planes;
+ uint32_t planes_per_chunk;
uint32_t pool_op;
- uint32_t fast_path;
- uint32_t narrow_path;
- uint32_t global_path;
- uint32_t block_path;
uint32_t avg_divide_count;
uint32_t ox_lo;
uint32_t ox_hi;
@@ -46,12 +49,36 @@ struct htp_pool_vtcm_layout {
size_t total_bytes;
size_t off_src;
size_t off_dst;
+ uint32_t src_plane_bytes;
+ uint32_t dst_plane_bytes;
size_t src_bytes_per_thread;
size_t dst_bytes_per_thread;
size_t src_spad_half_size;
size_t dst_spad_half_size;
};
+static inline void htp_pool_vtcm_layout_build(
+ struct htp_pool_vtcm_layout * L,
+ uint32_t src_x,
+ uint32_t src_y,
+ uint32_t dst_x,
+ uint32_t dst_y,
+ uint32_t n_threads
+) {
+ // Full-plane double buffering (256 bytes guard space for vector loads)
+ L->src_plane_bytes = src_x * src_y * sizeof(float);
+ L->dst_plane_bytes = dst_x * dst_y * sizeof(float);
+ L->src_spad_half_size = hex_round_up(L->src_plane_bytes + 256, 128);
+ L->dst_spad_half_size = hex_round_up(L->dst_plane_bytes, 128);
+ L->src_bytes_per_thread = 2 * L->src_spad_half_size;
+ L->dst_bytes_per_thread = 2 * L->dst_spad_half_size;
+
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+ L->total_bytes = off;
+}
+
static inline bool htp_pool_solve_layout(
struct htp_pool_vtcm_layout * layout,
uint32_t src_x,
@@ -61,25 +88,8 @@ static inline bool htp_pool_solve_layout(
uint32_t n_threads,
size_t vtcm_budget
) {
- // Full-plane double buffering (256 bytes guard space for vector loads)
- const size_t src_plane_bytes = (size_t) src_x * src_y * sizeof(float);
- const size_t dst_plane_bytes = (size_t) dst_x * dst_y * sizeof(float);
- const size_t src_plane_aligned = hex_round_up((uint32_t) src_plane_bytes + 256, 128);
- const size_t dst_plane_aligned = hex_round_up((uint32_t) dst_plane_bytes, 128);
-
- const size_t spad_per_thread = 2 * (src_plane_aligned + dst_plane_aligned);
- if (spad_per_thread * n_threads > vtcm_budget) {
- return false;
- }
-
- layout->src_spad_half_size = src_plane_aligned;
- layout->dst_spad_half_size = dst_plane_aligned;
- layout->src_bytes_per_thread = 2 * src_plane_aligned;
- layout->dst_bytes_per_thread = 2 * dst_plane_aligned;
- layout->off_src = 0;
- layout->off_dst = layout->src_bytes_per_thread * n_threads;
- layout->total_bytes = layout->off_dst + layout->dst_bytes_per_thread * n_threads;
- return true;
+ htp_pool_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, n_threads);
+ return layout->total_bytes <= vtcm_budget;
}
#endif // HTP_POOL_OPS_H
diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.c b/ggml/src/ggml-hexagon/htp/repeat-ops.c
index 2551be225..827ef24c5 100644
--- a/ggml/src/ggml-hexagon/htp/repeat-ops.c
+++ b/ggml/src/ggml-hexagon/htp/repeat-ops.c
@@ -16,14 +16,11 @@
#include "hex-profile.h"
#include "htp-ops.h"
#include "htp-tensor.h"
+#include "repeat-ops.h"
struct htp_repeat_context {
- struct htp_ops_context * octx;
-
- uint32_t nr0;
- uint32_t nr1;
- uint32_t nr2;
- uint32_t nr3;
+ struct htp_ops_context * octx;
+ const struct htp_repeat_kernel_params * kparams;
uint32_t nrows_per_thread;
uint32_t total_dst_rows; // ne1 * ne2 * ne3
@@ -35,6 +32,7 @@ struct htp_repeat_context {
static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * data) {
const struct htp_repeat_context * rctx = (const struct htp_repeat_context *) data;
struct htp_ops_context * octx = rctx->octx;
+ const struct htp_repeat_kernel_params * kparams = rctx->kparams;
const struct htp_tensor * src = octx->src[0];
const struct htp_tensor * dst = octx->dst;
@@ -58,29 +56,30 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat
const uint32_t nb2 = dst->nb[2];
const uint32_t nb3 = dst->nb[3];
- const uint32_t nr0 = rctx->nr0;
- const uint32_t nr1 = rctx->nr1;
- const uint32_t nr2 = rctx->nr2;
- const uint32_t nr3 = rctx->nr3;
+ const uint32_t nr0 = kparams->nr0;
const size_t row_bytes = ne00 * rctx->type_size;
const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith;
const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->total_dst_rows);
+ if (row_start >= row_end) {
+ return;
+ }
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
// Decompose flat dst row index into (i1, i2, i3)
- const uint32_t i1 = dst_row % ne1;
- const uint32_t i2 = (dst_row / ne1) % ne2;
- const uint32_t i3 = dst_row / (ne1 * ne2);
+ const uint32_t dst_row_div_ne1 = fastdiv(dst_row, &kparams->div_ne1);
+ const uint32_t i1 = dst_row - dst_row_div_ne1 * ne1;
+ const uint32_t i3 = fastdiv(dst_row, &kparams->div_ne12);
+ const uint32_t i2 = dst_row_div_ne1 - i3 * ne2;
// Map to source indices (tiling)
- const uint32_t k1 = i1 % ne01;
- const uint32_t k2 = i2 % ne02;
- const uint32_t k3 = i3 % ne03;
+ const uint32_t k1 = fastmodulo(i1, ne01, &kparams->div_ne01);
+ const uint32_t k2 = fastmodulo(i2, ne02, &kparams->div_ne02);
+ const uint32_t k3 = fastmodulo(i3, ne03, &kparams->div_ne03);
const uint8_t * src_row = (const uint8_t *) src->data + k1 * nb01 + k2 * nb02 + k3 * nb03;
uint8_t * dst_base = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
@@ -104,15 +103,6 @@ int op_repeat(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
- // Validate that dst dims are multiples of src dims
- if (dst->ne[0] % src0->ne[0] != 0 ||
- dst->ne[1] % src0->ne[1] != 0 ||
- dst->ne[2] % src0->ne[2] != 0 ||
- dst->ne[3] % src0->ne[3] != 0) {
- FARF(ERROR, "repeat: dst dims must be multiples of src dims\n");
- return HTP_STATUS_INVAL_PARAMS;
- }
-
size_t type_size;
switch (src0->type) {
case HTP_TYPE_F32: type_size = 4; break;
@@ -126,6 +116,13 @@ int op_repeat(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
+ const struct htp_repeat_kernel_params * kparams =
+ (const struct htp_repeat_kernel_params *) octx->kernel_params;
+
+ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3];
const size_t dst_row_size = dst->ne[0] * type_size;
@@ -148,10 +145,7 @@ int op_repeat(struct htp_ops_context * octx) {
struct htp_repeat_context rctx = {
.octx = octx,
- .nr0 = dst->ne[0] / src0->ne[0],
- .nr1 = dst->ne[1] / src0->ne[1],
- .nr2 = dst->ne[2] / src0->ne[2],
- .nr3 = dst->ne[3] / src0->ne[3],
+ .kparams = kparams,
.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
.total_dst_rows = nrows,
.row_start = row_start,
@@ -161,7 +155,7 @@ int op_repeat(struct htp_ops_context * octx) {
FARF(HIGH, "repeat: (%ux%ux%ux%u) -> (%ux%ux%ux%u) nr=(%u,%u,%u,%u)\n",
src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
- rctx.nr0, rctx.nr1, rctx.nr2, rctx.nr3);
+ kparams->nr0, kparams->nr1, kparams->nr2, kparams->nr3);
work_queue_run(octx->ctx->work_queue, repeat_job_per_thread, &rctx, n_threads);
diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.h b/ggml/src/ggml-hexagon/htp/repeat-ops.h
new file mode 100644
index 000000000..47759e037
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/repeat-ops.h
@@ -0,0 +1,27 @@
+#ifndef HTP_REPEAT_OPS_H
+#define HTP_REPEAT_OPS_H
+
+#include "hex-fastdiv.h"
+#include <stdint.h>
+
+struct htp_repeat_kernel_params {
+ uint32_t n_threads;
+ uint32_t nr0;
+ uint32_t nr1;
+ uint32_t nr2;
+ uint32_t nr3;
+
+ struct fastdiv_values div_ne1;
+ struct fastdiv_values div_ne12;
+ struct fastdiv_values div_ne01;
+ struct fastdiv_values div_ne02;
+ struct fastdiv_values div_ne03;
+};
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_repeat_kernel_params) <= 128, "htp_repeat_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_repeat_kernel_params) <= 128, "htp_repeat_kernel_params is too large for kernel_params blob");
+#endif
+
+#endif // HTP_REPEAT_OPS_H
diff --git a/ggml/src/ggml-hexagon/htp/roll-ops.c b/ggml/src/ggml-hexagon/htp/roll-ops.c
index 9c373f56d..77ded0fe9 100644
--- a/ggml/src/ggml-hexagon/htp/roll-ops.c
+++ b/ggml/src/ggml-hexagon/htp/roll-ops.c
@@ -17,16 +17,15 @@
#include "hex-profile.h"
#include "htp-ops.h"
#include "htp-tensor.h"
+#include "roll-ops.h"
struct htp_roll_context {
struct htp_ops_context * octx;
+ const struct htp_roll_kernel_params * kparams;
uint32_t row_start;
uint32_t nrows;
uint32_t nrows_per_thread;
-
- struct fastdiv_values div_ne1;
- struct fastdiv_values div_ne2_ne1;
};
static inline uint32_t htp_roll_wrap(int32_t i, uint32_t ne) {
@@ -39,30 +38,32 @@ static inline uint32_t htp_roll_wrap(int32_t i, uint32_t ne) {
return (uint32_t) i;
}
-#define htp_roll_preamble \
- const struct htp_tensor * src0 = octx->src[0]; \
- const struct htp_tensor * dst = octx->dst; \
- \
- const uint32_t ne0 = dst->ne[0]; \
- const uint32_t ne1 = dst->ne[1]; \
- const uint32_t ne2 = dst->ne[2]; \
- const uint32_t ne3 = dst->ne[3]; \
- \
- const uint32_t nb01 = src0->nb[1]; \
- const uint32_t nb02 = src0->nb[2]; \
- const uint32_t nb03 = src0->nb[3]; \
- \
- const uint32_t nb1 = dst->nb[1]; \
- const uint32_t nb2 = dst->nb[2]; \
- const uint32_t nb3 = dst->nb[3]; \
- \
- const int32_t s0 = octx->op_params[0]; \
- const int32_t s1 = octx->op_params[1]; \
- const int32_t s2 = octx->op_params[2]; \
- const int32_t s3 = octx->op_params[3]; \
- \
- const uint32_t i0_src0 = htp_roll_wrap(-s0, ne0); \
- const uint32_t n0 = ne0 - i0_src0;
+#define htp_roll_preamble \
+ const struct htp_tensor * src0 = octx->src[0]; \
+ const struct htp_tensor * dst = octx->dst; \
+ const struct htp_roll_kernel_params * kparams = \
+ (const struct htp_roll_kernel_params *) octx->kernel_params; \
+ \
+ const uint32_t ne0 = dst->ne[0]; \
+ const uint32_t ne1 = dst->ne[1]; \
+ const uint32_t ne2 = dst->ne[2]; \
+ const uint32_t ne3 = dst->ne[3]; \
+ \
+ const uint32_t nb01 = src0->nb[1]; \
+ const uint32_t nb02 = src0->nb[2]; \
+ const uint32_t nb03 = src0->nb[3]; \
+ \
+ const uint32_t nb1 = dst->nb[1]; \
+ const uint32_t nb2 = dst->nb[2]; \
+ const uint32_t nb3 = dst->nb[3]; \
+ \
+ const int32_t s0 = octx->op_params[0]; \
+ const int32_t s1 = octx->op_params[1]; \
+ const int32_t s2 = octx->op_params[2]; \
+ const int32_t s3 = octx->op_params[3]; \
+ \
+ const uint32_t i0_src0 = kparams->i0_src0; \
+ const uint32_t n0 = kparams->n0;
#define htp_roll_dma_preamble dma_queue * q = octx->ctx->dma[0];
@@ -91,10 +92,10 @@ static inline void roll_dma_push_rows(dma_queue * q,
uint32_t src_row,
uint32_t nrows,
uint32_t row_size,
- uint32_t i0_src0) {
+ uint32_t i0_src0,
+ uint32_t n0) {
const dma_addr_t dst_base = dst->data + (size_t) dst_row * row_size;
const dma_addr_t src_base = src0->data + (size_t) src_row * row_size;
- const uint32_t n0 = src0->ne[0] - i0_src0;
roll_dma_push(q, dst_base, src_base + (size_t) i0_src0 * sizeof(float),
row_size, row_size, n0 * sizeof(float), nrows);
@@ -125,7 +126,7 @@ static int roll_dma_f32_contiguous(struct htp_ops_context * octx) {
const uint32_t row_size = ne0 * sizeof(float);
if (s1 == 0 && s2 == 0 && s3 == 0) {
- roll_dma_push_rows(q, dst, src0, 0, 0, ne1 * ne2 * ne3, row_size, i0_src0);
+ roll_dma_push_rows(q, dst, src0, 0, 0, ne1 * ne2 * ne3, row_size, i0_src0, n0);
dma_queue_flush(q);
return HTP_STATUS_OK;
}
@@ -139,9 +140,9 @@ static int roll_dma_f32_contiguous(struct htp_ops_context * octx) {
const uint32_t n2_first = ne2 - i2_src0;
roll_dma_push_rows(q, dst, src0, dst_row0, src_row0, n2_first * ne1,
- row_size, i0_src0);
+ row_size, i0_src0, n0);
roll_dma_push_rows(q, dst, src0, dst_row0 + n2_first * ne1, i03 * ne2 * ne1,
- i2_src0 * ne1, row_size, i0_src0);
+ i2_src0 * ne1, row_size, i0_src0, n0);
}
dma_queue_flush(q);
@@ -159,9 +160,9 @@ static int roll_dma_f32_contiguous(struct htp_ops_context * octx) {
const uint32_t src_row0 = (i03 * ne2 + i02) * ne1;
roll_dma_push_rows(q, dst, src0, dst_row0, src_row0 + i1_src0,
- n1_first, row_size, i0_src0);
+ n1_first, row_size, i0_src0, n0);
roll_dma_push_rows(q, dst, src0, dst_row0 + n1_first, src_row0,
- i1_src0, row_size, i0_src0);
+ i1_src0, row_size, i0_src0, n0);
}
}
@@ -214,9 +215,9 @@ static void roll_thread_f32(unsigned int nth, unsigned int ith, void * data) {
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
for (uint32_t row = row_start; row < row_end; row++) {
- const uint32_t i3 = fastdiv(row, &rctx->div_ne2_ne1);
+ const uint32_t i3 = fastdiv(row, &kparams->div_ne2_ne1);
const uint32_t rem = row - i3 * ne2 * ne1;
- const uint32_t i2 = fastdiv(rem, &rctx->div_ne1);
+ const uint32_t i2 = fastdiv(rem, &kparams->div_ne1);
const uint32_t i1 = rem - i2 * ne1;
const uint32_t i01 = htp_roll_wrap((int32_t) i1 - s1, ne1);
@@ -245,23 +246,14 @@ static void roll_thread_f32(unsigned int nth, unsigned int ith, void * data) {
int execute_op_roll_f32(struct htp_ops_context * octx) {
htp_roll_preamble;
- if (src0->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) {
- FARF(ERROR, "roll: unsupported type %u -> %u\n", src0->type, dst->type);
- return HTP_STATUS_NO_SUPPORT;
- }
-
- if (src0->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) {
- FARF(ERROR, "roll: unsupported nb0 %u -> %u\n", src0->nb[0], dst->nb[0]);
- return HTP_STATUS_NO_SUPPORT;
+ if (kparams->kernel_type == HTP_ROLL_KERNEL_DMA_CONTIGUOUS) {
+ return roll_dma_f32_contiguous(octx);
}
-
- if (src0->ne[0] != ne0 || src0->ne[1] != ne1 ||
- src0->ne[2] != ne2 || src0->ne[3] != ne3) {
- FARF(ERROR, "roll: shape mismatch\n");
- return HTP_STATUS_INVAL_PARAMS;
+ if (kparams->kernel_type == HTP_ROLL_KERNEL_DMA_STRIDED) {
+ return roll_dma_f32_strided(octx);
}
- const uint32_t total_rows = ne1 * ne2 * ne3;
+ const uint32_t total_rows = kparams->total_rows;
const size_t dst_row_size = ne0 * sizeof(float);
uint32_t row_start = 0;
@@ -279,25 +271,21 @@ int execute_op_roll_f32(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- if (octx->ctx->mdev.count <= 1) {
- if (htp_tensor_is_contiguous(src0, sizeof(float)) && htp_tensor_is_contiguous(dst, sizeof(float))) {
- return roll_dma_f32_contiguous(octx);
- }
- return roll_dma_f32_strided(octx);
- }
-
if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {
return HTP_STATUS_NO_SUPPORT;
}
+ if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
const uint32_t n_threads = octx->n_threads;
struct htp_roll_context rctx = {
.octx = octx,
+ .kparams = kparams,
.row_start = row_start,
.nrows = nrows,
.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
- .div_ne1 = init_fastdiv_values(dst->ne[1]),
- .div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]),
};
work_queue_run(octx->ctx->work_queue, roll_thread_f32, &rctx, n_threads);
diff --git a/ggml/src/ggml-hexagon/htp/roll-ops.h b/ggml/src/ggml-hexagon/htp/roll-ops.h
new file mode 100644
index 000000000..2646aaa4c
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/roll-ops.h
@@ -0,0 +1,30 @@
+#ifndef HTP_ROLL_OPS_H
+#define HTP_ROLL_OPS_H
+
+#include "hex-fastdiv.h"
+#include <stdint.h>
+
+enum htp_roll_kernel_type {
+ HTP_ROLL_KERNEL_DMA_CONTIGUOUS = 1,
+ HTP_ROLL_KERNEL_DMA_STRIDED = 2,
+ HTP_ROLL_KERNEL_HVX = 3,
+};
+
+struct htp_roll_kernel_params {
+ uint8_t kernel_type;
+ uint32_t n_threads;
+ uint32_t total_rows;
+ uint32_t i0_src0;
+ uint32_t n0;
+
+ struct fastdiv_values div_ne1;
+ struct fastdiv_values div_ne2_ne1;
+};
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_roll_kernel_params) <= 128, "htp_roll_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_roll_kernel_params) <= 128, "htp_roll_kernel_params is too large for kernel_params blob");
+#endif
+
+#endif // HTP_ROLL_OPS_H
diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c
index f6b4d383c..8c55ac3af 100644
--- a/ggml/src/ggml-hexagon/htp/rope-ops.c
+++ b/ggml/src/ggml-hexagon/htp/rope-ops.c
@@ -700,15 +700,6 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
const struct htp_tensor * src2 = octx->src[2];
const struct htp_tensor * dst = octx->dst;
- switch (octx->op) {
- case HTP_OP_ROPE:
- break;
-
- default:
- FARF(ERROR, "Unsupported Op %u\n", octx->op);
- return HTP_STATUS_NO_SUPPORT;
- }
-
const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params;
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
@@ -746,20 +737,23 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
const size_t dst_row_size = dst->ne[0] * sizeof(float);
const size_t dst_row_stride = dst->nb[1];
+ struct htp_rope_vtcm_layout layout;
+ htp_rope_vtcm_layout_build(&layout, src0->ne[0], kparams->n_threads, src2 ? src2->ne[0] : 0);
+
struct htp_rope_context rctx;
memset(&rctx, 0, sizeof(struct htp_rope_context));
rctx.octx = octx;
- rctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
- rctx.spad_per_thread = kparams->spad_per_thread;
- rctx.theta_cache_offset = kparams->theta_cache_offset;
+ rctx.vtcm_base = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_spad);
+ rctx.spad_per_thread = layout.bytes_per_thread;
+ rctx.theta_cache_offset = layout.theta_cache_size_aligned;
if (src2) {
dma_queue * dma_q = octx->ctx->dma[0];
const size_t ff_size = src2->ne[0] * sizeof(float);
- float * vtcm_freq_factors = (float *) (rctx.vtcm_base + kparams->freq_factors_offset);
+ float * vtcm_freq_factors = VTCM_LAYOUT_PTR(float, octx->ctx->vtcm_base, layout.off_freq_factors);
dma_queue_push(dma_q, dma_make_data(vtcm_freq_factors, src2->data),
- kparams->freq_factors_size, 0, ff_size, 1);
+ layout.freq_factors_size_aligned, 0, ff_size, 1);
dma_queue_pop(dma_q);
rctx.freq_factors = vtcm_freq_factors;
}
@@ -791,7 +785,7 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
rctx.src0_row_stride = src0_row_stride;
rctx.dst_row_size = dst_row_size;
rctx.dst_row_stride = dst_row_stride;
- rctx.src0_row_size_aligned = kparams->src0_row_size_aligned;
+ rctx.src0_row_size_aligned = layout.src0_row_size_aligned;
rctx.src0_nrows = nrows;
rctx.nrows = nrows;
diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.h b/ggml/src/ggml-hexagon/htp/rope-ops.h
index ee055ccbc..0c28ef826 100644
--- a/ggml/src/ggml-hexagon/htp/rope-ops.h
+++ b/ggml/src/ggml-hexagon/htp/rope-ops.h
@@ -3,6 +3,7 @@
#include "hex-common.h"
#include "hex-fastdiv.h"
+#include "htp-vtcm.h"
#define HTP_ROPE_SPAD_BLOCK 8
#define HTP_ROPE_SPAD_NSLOTS 4
@@ -10,14 +11,7 @@
struct htp_rope_kernel_params {
uint32_t n_threads;
- uint32_t src0_nrows;
- uint32_t src0_nrows_per_thread;
uint32_t vtcm_size;
- uint32_t spad_per_thread;
- uint32_t theta_cache_offset;
- uint32_t src0_row_size_aligned;
- uint32_t freq_factors_offset;
- uint32_t freq_factors_size;
struct fastdiv_values div_ne2_ne1;
struct fastdiv_values div_ne1;
@@ -31,6 +25,8 @@ _Static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_pa
struct htp_rope_vtcm_layout {
size_t total_bytes;
+ size_t off_spad;
+ size_t off_freq_factors;
size_t bytes_per_thread;
size_t theta_cache_size_aligned;
size_t src0_row_size_aligned;
@@ -38,7 +34,7 @@ struct htp_rope_vtcm_layout {
};
static inline void htp_rope_vtcm_layout_build(
- struct htp_rope_vtcm_layout * layout,
+ struct htp_rope_vtcm_layout * L,
uint32_t ne00,
uint32_t n_threads,
uint32_t n_freq_factors
@@ -48,11 +44,15 @@ static inline void htp_rope_vtcm_layout_build(
const size_t theta_cache_size_aligned = hex_round_up((uint32_t) src0_row_size, 256);
const size_t freq_factors_size_aligned = hex_round_up(n_freq_factors * sizeof(float), 256);
- layout->src0_row_size_aligned = src0_row_size_aligned;
- layout->theta_cache_size_aligned = theta_cache_size_aligned;
- layout->freq_factors_size_aligned = freq_factors_size_aligned;
- layout->bytes_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned;
- layout->total_bytes = layout->bytes_per_thread * n_threads + freq_factors_size_aligned;
+ L->src0_row_size_aligned = src0_row_size_aligned;
+ L->theta_cache_size_aligned = theta_cache_size_aligned;
+ L->freq_factors_size_aligned = freq_factors_size_aligned;
+ L->bytes_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned;
+
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_spad, L->bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_freq_factors, freq_factors_size_aligned, n_freq_factors > 0);
+ L->total_bytes = off;
}
static inline uint8_t * rope_spad_slot(uint8_t * base, uint32_t slot, size_t row_size_aligned) {
diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.c b/ggml/src/ggml-hexagon/htp/set-rows-ops.c
index 1d72538f1..191cf0c6a 100644
--- a/ggml/src/ggml-hexagon/htp/set-rows-ops.c
+++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.c
@@ -79,8 +79,10 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig
const uint32_t ir1 = MIN(ir0 + dr, srctx->task_start + srctx->tasks); \
dma_queue * dma_q = octx->ctx->dma[ith]; \
const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \
- uint8_t * vtcm_src0 = srctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \
- uint8_t * vtcm_dst = srctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \
+ uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, srctx->vtcm_base, vtcm_layout->off_src0) + \
+ ith * vtcm_layout->src0_bytes_per_thread; \
+ uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, srctx->vtcm_base, vtcm_layout->off_dst) + \
+ ith * vtcm_layout->dst_bytes_per_thread; \
const uint32_t src0_row_size = ne00 * sizeof(float); \
const uint32_t dst_row_size = htp_tensor_get_row_size(octx->dst->type, ne00); \
const uint32_t nrows_per_thread = ir1 - ir0; \
@@ -188,14 +190,6 @@ int op_set_rows(struct htp_ops_context * octx) {
const struct htp_set_rows_kernel_params * kparams = (const struct htp_set_rows_kernel_params *)octx->kernel_params;
set_rows_preamble;
- if (octx->src[0]->type != HTP_TYPE_F32) {
- return HTP_STATUS_NO_SUPPORT;
- }
-
- if (octx->dst->type != HTP_TYPE_F32 && octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_Q8_0) {
- return HTP_STATUS_NO_SUPPORT;
- }
-
if (htp_tensor_is_extended(octx->src[1])) {
return HTP_STATUS_NO_SUPPORT;
}
diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.h b/ggml/src/ggml-hexagon/htp/set-rows-ops.h
index 5e98d2cb5..b9cb6cc90 100644
--- a/ggml/src/ggml-hexagon/htp/set-rows-ops.h
+++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.h
@@ -2,18 +2,13 @@
#define HTP_SET_ROWS_OPS_H
#include "hex-fastdiv.h"
+#include "htp-tensor.h"
+#include "htp-vtcm.h"
struct htp_set_rows_kernel_params {
- int32_t n_threads;
- int32_t total_tasks;
- int32_t tasks_per_thread;
- int32_t vtcm_size;
-
- // Fastdiv helpers
- struct fastdiv_values div_ne11;
- struct fastdiv_values div_ne12;
- struct fastdiv_values div_tasks_per_thread;
- struct fastdiv_values div_ne02;
+ uint32_t n_threads;
+ uint32_t total_tasks;
+ uint32_t vtcm_size;
};
struct htp_set_rows_vtcm_layout {
@@ -29,40 +24,27 @@ struct htp_set_rows_vtcm_layout {
};
static inline void htp_set_rows_vtcm_layout_build(
- struct htp_set_rows_vtcm_layout * vtcm_layout,
+ struct htp_set_rows_vtcm_layout * L,
int dst_type,
uint32_t ne00,
uint32_t n_threads) {
- size_t src0_row_size = ne00 * 4;
- size_t dst_row_size = 0;
- switch (dst_type) {
- case 0: // HTP_TYPE_F32
- dst_row_size = ne00 * 4;
- break;
- case 1: // HTP_TYPE_F16
- dst_row_size = ne00 * 2;
- break;
- case 8: // HTP_TYPE_Q8_0
- dst_row_size = (ne00 / 32) * 34;
- break;
- default:
- dst_row_size = 0;
- break;
- }
+ size_t src0_row_size = ne00 * sizeof(float);
+ size_t dst_row_size = htp_tensor_get_row_size(dst_type, ne00);
size_t src0_row_size_aligned = (src0_row_size + 255) & ~255;
size_t dst_row_size_aligned = (dst_row_size + 255) & ~255;
- vtcm_layout->src0_spad_half_size = src0_row_size_aligned;
- vtcm_layout->dst_spad_half_size = dst_row_size_aligned;
+ L->src0_spad_half_size = src0_row_size_aligned;
+ L->dst_spad_half_size = dst_row_size_aligned;
- vtcm_layout->src0_bytes_per_thread = src0_row_size_aligned * 2;
- vtcm_layout->dst_bytes_per_thread = dst_row_size_aligned * 2;
+ L->src0_bytes_per_thread = src0_row_size_aligned * 2;
+ L->dst_bytes_per_thread = dst_row_size_aligned * 2;
- vtcm_layout->off_src0 = 0;
- vtcm_layout->off_dst = vtcm_layout->off_src0 + vtcm_layout->src0_bytes_per_thread * n_threads;
- vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads;
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+ L->total_bytes = off;
}
#if defined(__cplusplus)
diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.c b/ggml/src/ggml-hexagon/htp/softmax-ops.c
index 48be5d725..d65c3fae3 100644
--- a/ggml/src/ggml-hexagon/htp/softmax-ops.c
+++ b/ggml/src/ggml-hexagon/htp/softmax-ops.c
@@ -496,7 +496,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) {
return HTP_STATUS_INVAL_PARAMS;
}
- if (kparams->kernel_id >= HTP_SOFTMAX_KERNEL_COUNT) {
+ if (kparams->kernel_type >= HTP_SOFTMAX_KERNEL_COUNT) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -532,34 +532,38 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) {
const uint32_t n_threads = octx->n_threads;
uint8_t * const vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
- const uint32_t off_src0 = 0;
- const uint32_t off_dst = off_src0 + kparams->vtcm_src0_size_per_thread * kparams->n_threads;
- const uint32_t off_src1 = off_dst + kparams->vtcm_dst_size_per_thread * kparams->n_threads;
+ struct htp_softmax_vtcm_layout layout;
+ htp_softmax_vtcm_layout_build(&layout,
+ src0->ne[0],
+ kparams->use_src1 ? octx->src[1]->ne[0] : 0,
+ kparams->use_src1 != 0,
+ kparams->use_f16 != 0,
+ kparams->n_threads);
struct htp_softmax_context smctx = {
.octx = octx,
.kparams = kparams,
- .compute = (void *) softmax_kernels[kparams->kernel_id],
+ .compute = (void *) softmax_kernels[kparams->kernel_type],
.data_src0 = src0->data,
.data_src1 = kparams->use_src1 ? octx->src[1]->data : 0,
.data_dst = dst->data,
- .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, off_src0),
- .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, off_dst),
- .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, vtcm_base, off_src1, kparams->use_src1),
+ .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_src0),
+ .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_dst),
+ .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, vtcm_base, layout.off_src1, kparams->use_src1),
- .vtcm_src0_size_per_thread = kparams->vtcm_src0_size_per_thread,
- .vtcm_src1_size_per_thread = kparams->vtcm_src1_size_per_thread,
- .vtcm_dst_size_per_thread = kparams->vtcm_dst_size_per_thread,
+ .vtcm_src0_size_per_thread = (uint32_t) layout.src0_bytes_per_thread,
+ .vtcm_src1_size_per_thread = (uint32_t) layout.src1_bytes_per_thread,
+ .vtcm_dst_size_per_thread = (uint32_t) layout.dst_bytes_per_thread,
- .src0_spad_half_size = kparams->src0_spad_half_size,
- .src1_spad_half_size = kparams->src1_spad_half_size,
- .dst_spad_half_size = kparams->dst_spad_half_size,
+ .src0_spad_half_size = (uint32_t) layout.src0_spad_half_size,
+ .src1_spad_half_size = (uint32_t) layout.src1_spad_half_size,
+ .dst_spad_half_size = (uint32_t) layout.dst_spad_half_size,
- .src0_row_size_aligned = kparams->src0_row_size_aligned,
- .src1_row_size_aligned = kparams->src1_row_size_aligned,
- .dst_row_size_aligned = kparams->dst_row_size_aligned,
+ .src0_row_size_aligned = (uint32_t) layout.src0_spad_half_size,
+ .src1_row_size_aligned = (uint32_t) layout.src1_spad_half_size,
+ .dst_row_size_aligned = (uint32_t) layout.dst_spad_half_size,
.use_f16 = kparams->use_f16 != 0,
diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.h b/ggml/src/ggml-hexagon/htp/softmax-ops.h
index 8d976adb8..ef6567306 100644
--- a/ggml/src/ggml-hexagon/htp/softmax-ops.h
+++ b/ggml/src/ggml-hexagon/htp/softmax-ops.h
@@ -7,8 +7,9 @@
#include <math.h>
#include "hex-fastdiv.h"
#include "hex-common.h"
+#include "htp-vtcm.h"
-enum htp_softmax_kernel_id {
+enum htp_softmax_kernel_type {
HTP_SOFTMAX_KERNEL_NOMASK = 0,
HTP_SOFTMAX_KERNEL_MASK_F32,
HTP_SOFTMAX_KERNEL_MASK_F16,
@@ -16,28 +17,14 @@ enum htp_softmax_kernel_id {
};
struct htp_softmax_kernel_params {
+ uint8_t kernel_type;
uint32_t n_threads;
- uint32_t src0_nrows;
- uint32_t src0_nrows_per_thread;
uint32_t vtcm_size;
- uint32_t vtcm_src0_size_per_thread;
- uint32_t vtcm_src1_size_per_thread;
- uint32_t vtcm_dst_size_per_thread;
-
- uint32_t src0_row_size_aligned;
- uint32_t src1_row_size_aligned;
- uint32_t dst_row_size_aligned;
-
- uint32_t src0_spad_half_size;
- uint32_t src1_spad_half_size;
- uint32_t dst_spad_half_size;
-
uint32_t n_head;
uint32_t n_head_log2;
uint32_t use_src1;
uint32_t use_f16;
- uint32_t kernel_id;
float scale;
float max_bias;
@@ -72,7 +59,7 @@ struct htp_softmax_vtcm_layout {
};
static inline void htp_softmax_vtcm_layout_build(
- struct htp_softmax_vtcm_layout * layout,
+ struct htp_softmax_vtcm_layout * L,
uint32_t ne00,
uint32_t ne10,
bool use_src1,
@@ -87,20 +74,21 @@ static inline void htp_softmax_vtcm_layout_build(
size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128);
size_t src1_row_size_aligned = use_src1 ? hex_round_up(src1_row_size, 128) : 0;
- layout->src0_spad_half_size = src0_row_size_aligned;
- layout->dst_spad_half_size = dst_row_size_aligned;
- layout->src1_spad_half_size = src1_row_size_aligned;
+ L->src0_spad_half_size = src0_row_size_aligned;
+ L->dst_spad_half_size = dst_row_size_aligned;
+ L->src1_spad_half_size = src1_row_size_aligned;
// Double buffering: 2 half-buffers per thread
- layout->src0_bytes_per_thread = src0_row_size_aligned * 2;
- layout->dst_bytes_per_thread = dst_row_size_aligned * 2;
- layout->src1_bytes_per_thread = src1_row_size_aligned * 2;
+ L->src0_bytes_per_thread = src0_row_size_aligned * 2;
+ L->dst_bytes_per_thread = dst_row_size_aligned * 2;
+ L->src1_bytes_per_thread = src1_row_size_aligned * 2;
- layout->off_src0 = 0;
- layout->off_dst = layout->off_src0 + layout->src0_bytes_per_thread * n_threads;
- layout->off_src1 = layout->off_dst + layout->dst_bytes_per_thread * n_threads;
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes_per_thread * n_threads, use_src1);
- layout->total_bytes = layout->off_src1 + layout->src1_bytes_per_thread * n_threads;
+ L->total_bytes = off;
}
#endif // HTP_SOFTMAX_OPS_H
diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c
index 182982fcd..628ca65e8 100644
--- a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c
+++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c
@@ -13,14 +13,16 @@
#include "htp-tensor.h"
#include "hvx-types.h"
#include "hvx-utils.h"
+#include "solve-tri-ops.h"
struct htp_solve_tri_context {
- struct htp_ops_context * octx;
- uint32_t jobs_per_thread;
- uint32_t total_jobs;
- uint32_t job_start;
- uint32_t k_chunks;
- uint32_t col_block;
+ struct htp_ops_context * octx;
+ const struct htp_solve_tri_kernel_params * kparams;
+ uint32_t jobs_per_thread;
+ uint32_t total_jobs;
+ uint32_t job_start;
+ uint32_t k_chunks;
+ uint32_t col_block;
};
static inline void solve_tri_row_scalar(const float * A_row,
@@ -80,6 +82,7 @@ static inline void solve_tri_row_hvx(const float * A_row,
static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void * data) {
struct htp_solve_tri_context * sctx = (struct htp_solve_tri_context *) data;
struct htp_ops_context * octx = sctx->octx;
+ const struct htp_solve_tri_kernel_params * kparams = sctx->kparams;
const struct htp_tensor * src0 = octx->src[0]; // A
const struct htp_tensor * src1 = octx->src[1]; // B
@@ -91,7 +94,7 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void
const uint32_t ne02 = src0->ne[2];
const uint32_t col_block = VLEN_FP32;
- const uint32_t k_full = (k / col_block) * col_block;
+ const uint32_t k_full = k & ~(col_block - 1);
const uint32_t start_batch = sctx->job_start + sctx->jobs_per_thread * ith;
const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs);
@@ -100,7 +103,7 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_batch);
for (uint32_t batch = start_batch; batch < end_batch; ++batch) {
- const uint32_t i03 = batch / ne02;
+ const uint32_t i03 = fastdiv(batch, &kparams->div_ne02);
const uint32_t i02 = batch - i03 * ne02;
const float * A_batch =
@@ -141,6 +144,7 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void
static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void * data) {
struct htp_solve_tri_context * sctx = (struct htp_solve_tri_context *) data;
struct htp_ops_context * octx = sctx->octx;
+ const struct htp_solve_tri_kernel_params * kparams = sctx->kparams;
const struct htp_tensor * src0 = octx->src[0]; // A
const struct htp_tensor * src1 = octx->src[1]; // B
@@ -158,10 +162,10 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_job);
for (uint32_t job = start_job; job < end_job; ++job) {
- const uint32_t batch = job / sctx->k_chunks;
+ const uint32_t batch = fastdiv(job, &kparams->div_k_chunks);
const uint32_t chunk = job - batch * sctx->k_chunks;
- const uint32_t i03 = batch / ne02;
+ const uint32_t i03 = fastdiv(batch, &kparams->div_ne02);
const uint32_t i02 = batch - i03 * ne02;
const float * A_batch =
@@ -198,36 +202,17 @@ int op_solve_tri(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0]; // A
const struct htp_tensor * src1 = octx->src[1]; // B
const struct htp_tensor * dst = octx->dst; // X
-
- if (src0->type != HTP_TYPE_F32 || src1->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) {
- return HTP_STATUS_NO_SUPPORT;
- }
-
- // left=true, lower=true, uni=false only
- if (src0->ne[0] != src0->ne[1]) {
- return HTP_STATUS_INVAL_PARAMS;
- }
- if (src0->ne[1] != src1->ne[1]) {
- return HTP_STATUS_INVAL_PARAMS;
- }
- if (src0->ne[2] != src1->ne[2] || src0->ne[3] != src1->ne[3]) {
- return HTP_STATUS_INVAL_PARAMS;
- }
- if (dst->ne[0] != src1->ne[0] || dst->ne[1] != src1->ne[1] || dst->ne[2] != src1->ne[2] ||
- dst->ne[3] != src1->ne[3]) {
- return HTP_STATUS_INVAL_PARAMS;
- }
+ const struct htp_solve_tri_kernel_params * kparams =
+ (const struct htp_solve_tri_kernel_params *) octx->kernel_params;
if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(src1) || htp_tensor_is_extended(dst)) {
return HTP_STATUS_NO_SUPPORT;
}
- const uint32_t k = src1->ne[0];
-
- const uint32_t col_block = VLEN_FP32;
- const uint32_t k_chunks = (k + col_block - 1) / col_block;
- const uint32_t total_batches = src0->ne[2] * src0->ne[3];
- const bool batched = total_batches >= (uint32_t) octx->n_threads;
+ const uint32_t col_block = kparams->col_block;
+ const uint32_t k_chunks = kparams->k_chunks;
+ const uint32_t total_batches = kparams->total_batches;
+ const bool batched = (kparams->batched != 0);
FARF(HIGH, "solve-tri: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : batched %d\n",
src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
@@ -239,9 +224,7 @@ int op_solve_tri(struct htp_ops_context * octx) {
uint32_t njobs = total_batches;
if (octx->ctx->mdev.count > 1) {
- const uint32_t batch_size = dst->nb[2];
- const uint32_t batches_per_chunk = (batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(batch_size, HEX_L2_LINE_SIZE)) : 1;
- const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, htp_tensor_mdev_data_aligned(dst) ? batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
+ const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, htp_tensor_mdev_data_aligned(dst) ? kparams->batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
job_start = range.start;
njobs = range.count;
}
@@ -250,11 +233,15 @@ int op_solve_tri(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- // Batch-level parallelism
+ if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, njobs))) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
const uint32_t n_threads = octx->n_threads;
struct htp_solve_tri_context sctx = {
.octx = octx,
+ .kparams = kparams,
.jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div),
.total_jobs = njobs,
.job_start = job_start,
@@ -264,8 +251,7 @@ int op_solve_tri(struct htp_ops_context * octx) {
work_queue_run(octx->ctx->work_queue, solve_tri_batch_thread_f32, &sctx, n_threads);
} else {
- // Chunk-level parallelism
- const uint32_t total_jobs = total_batches * k_chunks;
+ const uint32_t total_jobs = kparams->total_jobs;
uint32_t job_start = 0;
uint32_t njobs = total_jobs;
@@ -281,10 +267,15 @@ int op_solve_tri(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
+ if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, njobs))) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
const uint32_t n_threads = octx->n_threads;
struct htp_solve_tri_context sctx = {
.octx = octx,
+ .kparams = kparams,
.jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div),
.total_jobs = njobs,
.job_start = job_start,
diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.h b/ggml/src/ggml-hexagon/htp/solve-tri-ops.h
new file mode 100644
index 000000000..327415814
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.h
@@ -0,0 +1,26 @@
+#ifndef HTP_SOLVE_TRI_OPS_H
+#define HTP_SOLVE_TRI_OPS_H
+
+#include "hex-fastdiv.h"
+#include <stdint.h>
+
+struct htp_solve_tri_kernel_params {
+ uint32_t n_threads;
+ uint32_t total_batches;
+ uint32_t total_jobs;
+ uint32_t k_chunks;
+ uint32_t col_block;
+ uint32_t batched;
+ uint32_t batches_per_chunk;
+
+ struct fastdiv_values div_ne02;
+ struct fastdiv_values div_k_chunks;
+};
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_solve_tri_kernel_params) <= 128, "htp_solve_tri_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_solve_tri_kernel_params) <= 128, "htp_solve_tri_kernel_params is too large for kernel_params blob");
+#endif
+
+#endif // HTP_SOLVE_TRI_OPS_H
diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.c b/ggml/src/ggml-hexagon/htp/ssm-conv.c
index 0d28c8990..dae339e4d 100644
--- a/ggml/src/ggml-hexagon/htp/ssm-conv.c
+++ b/ggml/src/ggml-hexagon/htp/ssm-conv.c
@@ -135,6 +135,13 @@ static inline void hvx_ssm_conv_unpack_to_T(const float * raw, float * T, uint32
}
}
+static __attribute__((noinline)) void hvx_ssm_conv_silu_vtcm(float * restrict buf, uint32_t n_vecs) {
+ HVX_Vector * restrict vbuf = (HVX_Vector *) buf;
+ for (uint32_t i = 0; i < n_vecs; ++i) {
+ vbuf[i] = hvx_vec_silu_f32(vbuf[i]);
+ }
+}
+
// Decode dot product specialization for d_conv == 4: multiply in the raw channel-major layout,
// then deinterleave the products so each vector holds one tap of 32 channels, and sum.
// Keeps both operands in DMA layout - no transpose, no scratch.
@@ -189,9 +196,10 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void
dma_queue * dma_q = octx->ctx->dma[ith];
- const uint32_t d_conv = kparams->d_conv;
- const uint32_t d_inner = kparams->d_inner;
- const uint32_t n_s = kparams->n_s;
+ const uint32_t d_conv = kparams->d_conv;
+ const uint32_t d_inner = kparams->d_inner;
+ const uint32_t n_s = kparams->n_s;
+ const bool apply_silu = kparams->silu != 0;
const uint32_t dr = scctx->nrows_per_thread;
const uint32_t ir0 = scctx->row_start + dr * ith;
@@ -232,7 +240,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void
// raw_dot keeps both operands in the DMA layout, so src1 needs no prep pass
const bool raw_dot = (d_conv == 4) && (d_inner_per_thread % VLEN_FP32 == 0);
- const uint32_t n_tiles = (d_inner_per_thread + d_inner_tile - 1) / d_inner_tile;
+ const uint32_t n_tiles = fastdiv(d_inner_per_thread + d_inner_tile - 1, &kparams->div_d_inner_tile);
const uint32_t n_chunks = n_s * n_tiles;
const size_t row_bytes = d_conv * sizeof(float);
@@ -295,21 +303,18 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void
hvx_ssm_conv_unpack_to_T(src0_tile_raw[c & 1], src0_T, tile_n, tile_stride, d_conv);
for (uint32_t cb = 0; cb < tile_n; cb += VLEN_FP32) {
- const uint32_t cb_n = MIN(VLEN_FP32, tile_n - cb);
HVX_Vector acc = hvx_vec_splat_f32(0.0f);
for (uint32_t j = 0; j < d_conv; ++j) {
HVX_Vector x = *(const HVX_Vector *)(src0_T + j * tile_stride + cb);
HVX_Vector w = *(const HVX_Vector *)(src1_T + j * d_inner_stride + tile_off + cb);
acc = Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(x, w));
}
- HVX_Vector y = Q6_Vsf_equals_Vqf32(acc);
- if (cb_n == VLEN_FP32) {
- *(HVX_Vector *)(out + cb) = y;
- } else {
- hvx_vec_store_u(out + cb, cb_n * sizeof(float), y);
- }
+ *(HVX_Vector *)(out + cb) = Q6_Vsf_equals_Vqf32(acc);
}
}
+ if (apply_silu) {
+ hvx_ssm_conv_silu_vtcm(out, hex_round_up(tile_n, VLEN_FP32) / VLEN_FP32);
+ }
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i3);
const dma_addr_t dst_ddr = dst->data + i3 * dst_stride_seq_bytes + (ir0 + tile_off) * sizeof(float);
@@ -352,11 +357,12 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
dma_queue * dma_q = octx->ctx->dma[ith];
- const uint32_t d_conv = kparams->d_conv;
- const uint32_t d_inner = kparams->d_inner;
- const uint32_t n_t = kparams->n_t;
- const uint32_t n_s = kparams->n_s;
- const uint32_t ncs = src0->ne[0];
+ const uint32_t d_conv = kparams->d_conv;
+ const uint32_t d_inner = kparams->d_inner;
+ const uint32_t n_t = kparams->n_t;
+ const uint32_t n_s = kparams->n_s;
+ const uint32_t ncs = src0->ne[0];
+ const bool apply_silu = kparams->silu != 0;
const uint32_t dr = scctx->nrows_per_thread;
const uint32_t ir0 = scctx->row_start + dr * ith;
@@ -397,7 +403,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
- const uint32_t n_tiles = (d_inner_per_thread + d_inner_tile - 1) / d_inner_tile;
+ const uint32_t n_tiles = fastdiv(d_inner_per_thread + d_inner_tile - 1, &kparams->div_d_inner_tile);
const uint32_t n_chunks = n_s * n_tiles;
const size_t row_bytes = ncs * sizeof(float);
@@ -513,10 +519,14 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off);
}
+ if (apply_silu) {
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off);
+ hvx_ssm_conv_silu_vtcm(out, n_t * (dst_tile_stride / C_TILE));
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off);
+ }
+
// Writeback dst_tile VTCM -> DDR via 2D DMA
- dma_queue_push(dma_q,
- dma_make_data(dst->data + i3 * dst_stride_seq_bytes + (ir0 + tile_off) * sizeof(float),
- (uint8_t *) out),
+ dma_queue_push(dma_q, dma_make_data(dst->data + i3 * dst_stride_seq_bytes + (ir0 + tile_off) * sizeof(float), (uint8_t *) out),
dst_stride_token_bytes, dst_tile_stride * sizeof(float), tile_n * sizeof(float), n_t);
if (c + 2 < n_chunks) {
@@ -547,13 +557,8 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) {
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
- if (src0->type != HTP_TYPE_F32 || src1->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) {
- return HTP_STATUS_NO_SUPPORT;
- }
-
const struct htp_ssm_conv_kernel_params * kparams = (const struct htp_ssm_conv_kernel_params *) octx->kernel_params;
-
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -582,26 +587,30 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) {
return HTP_STATUS_VTCM_TOO_SMALL;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_threads = octx->n_threads;
+ const uint32_t raw_rpt = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
+ const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, VLEN_FP32);
+
+ struct htp_ssm_conv_vtcm_layout L;
+ htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_thread, kparams->d_conv, kparams->n_t, kparams->d_inner_tile, n_threads);
- octx->src0_spad.size_per_thread = kparams->vtcm_src0_size_per_thread;
- octx->src1_spad.size_per_thread = kparams->vtcm_src1_size_per_thread;
- octx->dst_spad.size_per_thread = kparams->vtcm_dst_size_per_thread;
+ uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;
- octx->src0_spad.size = kparams->vtcm_src0_size;
- octx->src1_spad.size = kparams->vtcm_src1_size;
- octx->dst_spad.size = kparams->vtcm_dst_size;
+ octx->src0_spad.size_per_thread = L.src0_bytes_per_thread;
+ octx->src1_spad.size_per_thread = L.src1_bytes_per_thread;
+ octx->dst_spad.size_per_thread = L.dst_bytes_per_thread;
- octx->src0_spad.data = octx->ctx->vtcm_base;
- octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;
- octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size;
+ octx->src0_spad.size = L.src0_bytes;
+ octx->src1_spad.size = L.src1_bytes;
+ octx->dst_spad.size = L.dst_bytes;
+
+ octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);
+ octx->src1_spad.data = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);
+ octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);
octx->src0_spad.src = NULL;
octx->src1_spad.src = NULL;
octx->dst_spad.src = NULL;
- const uint32_t raw_rpt = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
- const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, VLEN_FP32);
-
struct htp_ssm_conv_context scctx = {
.octx = octx,
.kparams = kparams,
diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.h b/ggml/src/ggml-hexagon/htp/ssm-conv.h
index a071d74b0..43b2575a2 100644
--- a/ggml/src/ggml-hexagon/htp/ssm-conv.h
+++ b/ggml/src/ggml-hexagon/htp/ssm-conv.h
@@ -3,8 +3,10 @@
#include <stdint.h>
+#include "hex-common.h"
#include "hex-fastdiv.h"
#include "htp-ops.h"
+#include "htp-vtcm.h"
struct htp_ssm_conv_kernel_params {
uint32_t n_threads;
@@ -14,16 +16,62 @@ struct htp_ssm_conv_kernel_params {
uint32_t n_s;
uint32_t d_inner_tile;
- uint32_t vtcm_src0_size_per_thread;
- uint32_t vtcm_src1_size_per_thread;
- uint32_t vtcm_dst_size_per_thread;
-
- uint32_t vtcm_src0_size;
- uint32_t vtcm_src1_size;
- uint32_t vtcm_dst_size;
uint32_t vtcm_size;
+ uint32_t silu;
+
+ struct fastdiv_values div_d_inner_tile;
};
+struct htp_ssm_conv_vtcm_layout {
+ size_t total_bytes;
+ size_t off_src0;
+ size_t off_src1;
+ size_t off_dst;
+ size_t src0_bytes_per_thread;
+ size_t src1_bytes_per_thread;
+ size_t dst_bytes_per_thread;
+ size_t src0_bytes;
+ size_t src1_bytes;
+ size_t dst_bytes;
+};
+
+static inline void htp_ssm_conv_vtcm_layout_build(
+ struct htp_ssm_conv_vtcm_layout * L,
+ uint32_t d_inner_per_thread,
+ uint32_t d_conv,
+ uint32_t n_t,
+ uint32_t d_inner_tile,
+ uint32_t n_threads
+) {
+ const size_t src1_raw_bytes = hex_round_up((size_t) d_inner_per_thread * d_conv * sizeof(float), 128) + 128;
+ const size_t src1_T_bytes = hex_round_up((size_t) d_conv * d_inner_per_thread * sizeof(float), 128);
+
+ L->src1_bytes_per_thread = src1_raw_bytes + src1_T_bytes;
+
+ if (n_t == 1) {
+ const size_t src0_tile_raw_bytes = hex_round_up((size_t) d_inner_tile * d_conv * sizeof(float), 128);
+ const size_t src0_T_bytes = hex_round_up((size_t) d_conv * d_inner_tile * sizeof(float), 128);
+ L->src0_bytes_per_thread = 2 * src0_tile_raw_bytes + src0_T_bytes;
+ L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * sizeof(float), 128);
+ } else {
+ const uint32_t ncs = d_conv - 1 + n_t;
+ const size_t src0_block_T = hex_round_up((size_t) ncs * 32 * sizeof(float), 128);
+ const size_t src0_tile_raw = hex_round_up((size_t) d_inner_tile * ncs * sizeof(float), 128);
+ L->src0_bytes_per_thread = 2 * src0_tile_raw + src0_block_T;
+ L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * n_t * sizeof(float), 128);
+ }
+
+ L->src0_bytes = L->src0_bytes_per_thread * n_threads;
+ L->src1_bytes = L->src1_bytes_per_thread * n_threads;
+ L->dst_bytes = L->dst_bytes_per_thread * n_threads;
+
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes);
+ VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes);
+ L->total_bytes = off;
+}
+
#if defined(__cplusplus)
static_assert(sizeof(struct htp_ssm_conv_kernel_params) <= 128, "htp_ssm_conv_kernel_params is too large for kernel_params blob");
#else
diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c
index 177c74b33..3d931ed19 100644
--- a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c
+++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c
@@ -18,6 +18,7 @@
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
+#include "sum-rows-ops.h"
#define sum_rows_preamble \
const struct htp_tensor *src0 = octx->src[0]; \
@@ -52,7 +53,6 @@ struct sum_rows_context {
size_t dst_stride;
uint32_t rows_per_thread;
uint32_t total_rows;
- bool opt_path;
};
static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) {
@@ -71,7 +71,6 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data)
const size_t src_stride = smctx->src_stride;
const size_t dst_stride = smctx->dst_stride;
const uint32_t ne00 = smctx->ne00;
- const bool opt_path = smctx->opt_path;
const float * restrict src_th = (const float *) (smctx->src_data + (start_row * src_stride));
float * restrict dst_th = (float *) (smctx->dst_data + (start_row * dst_stride));
@@ -90,11 +89,7 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data)
hex_l2fetch((const uint8_t *) src_local + src_stride, src_stride, src_stride, 1);
}
- if (opt_path) {
- *dst_local = hvx_reduce_sum_f32_a((const uint8_t *) src_local, ne00);
- } else {
- *dst_local = hvx_reduce_sum_f32((const uint8_t *) src_local, ne00);
- }
+ *dst_local = hvx_reduce_sum_f32((const uint8_t *) src_local, ne00);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row);
@@ -111,6 +106,13 @@ int op_sum_rows(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
+ const struct htp_sum_rows_kernel_params * kparams =
+ (const struct htp_sum_rows_kernel_params *) octx->kernel_params;
+
+ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
const uint32_t src0_nrows = ne01 * ne02 * ne03;
const size_t dst_data_row_size = dst->ne[0] * sizeof(float);
@@ -129,14 +131,9 @@ int op_sum_rows(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = octx->n_threads;
+ const uint32_t n_threads = octx->n_threads;
const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
- bool opt_path = false;
- if ((0 == hex_is_aligned((void *) src0->data, VLEN)) && !(nb01 & (VLEN - 1))) {
- opt_path = true;
- }
-
struct sum_rows_context smctx = {
.octx = octx,
.src_data = (const uint8_t *) src0->data + row_start * nb01,
@@ -146,7 +143,6 @@ int op_sum_rows(struct htp_ops_context * octx) {
.dst_stride = nb1,
.rows_per_thread = rows_per_thread,
.total_rows = nrows,
- .opt_path = opt_path,
};
work_queue_run(octx->ctx->work_queue, sum_rows_thread_f32, &smctx, n_threads);
@@ -197,6 +193,13 @@ int op_sum(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
+ const struct htp_sum_rows_kernel_params * kparams =
+ (const struct htp_sum_rows_kernel_params *) octx->kernel_params;
+
+ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) {
return HTP_STATUS_OK;
}
@@ -207,9 +210,8 @@ int op_sum(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = (total_elems >= 1024) ? MIN(octx->n_threads, HTP_MAX_NTHREADS) : 1;
- const uint32_t raw_chunk = (total_elems + n_threads - 1) / n_threads;
- const uint32_t elems_per_thread = hex_round_up(raw_chunk, 32);
+ const uint32_t n_threads = octx->n_threads;
+ const uint32_t elems_per_thread = kparams->elems_per_thread;
struct sum_context sctx = {
.octx = octx,
@@ -324,6 +326,13 @@ int op_argmax(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
+ const struct htp_sum_rows_kernel_params * kparams =
+ (const struct htp_sum_rows_kernel_params *) octx->kernel_params;
+
+ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
const uint32_t ne00 = src0->ne[0];
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
@@ -341,9 +350,8 @@ int op_argmax(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = (ne00 >= 1024) ? MIN(octx->n_threads, HTP_MAX_NTHREADS) : 1;
- const uint32_t raw_chunk = (ne00 + n_threads - 1) / n_threads;
- const uint32_t elems_per_thread = hex_round_up(raw_chunk, 32);
+ const uint32_t n_threads = octx->n_threads;
+ const uint32_t elems_per_thread = kparams->elems_per_thread;
struct argmax_context actx = {
.octx = octx,
@@ -389,8 +397,8 @@ int op_argmax(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t n_threads = MIN(octx->n_threads, nrows);
- const uint32_t rows_per_thread = (nrows + n_threads - 1) / n_threads;
+ const uint32_t n_threads = octx->n_threads;
+ const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
struct argmax_context actx = {
.octx = octx,
diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.h b/ggml/src/ggml-hexagon/htp/sum-rows-ops.h
new file mode 100644
index 000000000..cc9ddd483
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.h
@@ -0,0 +1,17 @@
+#ifndef HTP_SUM_ROWS_OPS_H
+#define HTP_SUM_ROWS_OPS_H
+
+#include <stdint.h>
+
+struct htp_sum_rows_kernel_params {
+ uint32_t n_threads;
+ uint32_t elems_per_thread;
+};
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_sum_rows_kernel_params) <= 128, "htp_sum_rows_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_sum_rows_kernel_params) <= 128, "htp_sum_rows_kernel_params is too large for kernel_params blob");
+#endif
+
+#endif // HTP_SUM_ROWS_OPS_H
diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c
index ea3d92ad0..0d7440cc1 100644
--- a/ggml/src/ggml-hexagon/htp/unary-ops.c
+++ b/ggml/src/ggml-hexagon/htp/unary-ops.c
@@ -109,18 +109,17 @@ static inline size_t unary_row_offset(uint32_t ir,
return i1 * nb1 + i2 * nb2 + i3 * nb3;
}
-// Safe DMA block size from row `ir`: clamp to the tighter dim-1 slice
-// boundary of src and dst so the nb1 stride stays valid for all rows.
+// Safe DMA block size from row `ir`: clamp to the dim-1 slice boundary
+// when non-contiguous so the nb1 stride stays valid for all rows.
static inline uint32_t unary_block_size(uint32_t ir,
uint32_t end_row,
uint32_t block,
- bool src_contig,
- bool dst_contig,
+ bool contig,
uint32_t ne1,
const struct fastdiv_values * div_ne1) {
uint32_t limit = MIN(block, end_row - ir);
- if (!src_contig || !dst_contig) {
+ if (!contig) {
const uint32_t slice_end = (fastdiv(ir, div_ne1) + 1) * ne1;
limit = MIN(limit, slice_end - ir);
}
@@ -213,13 +212,15 @@ static void rms_norm_f32(const void * restrict src,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
float epsilon = 0.f;
- memcpy(&epsilon, op_params, sizeof(float));
+ float scale = 0.f;
+ memcpy(&epsilon, &op_params[0], sizeof(float));
+ memcpy(&scale, &op_params[1], sizeof(float));
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
- hvx_fast_rms_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon);
+ hvx_fast_rms_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon, scale);
}
}
@@ -230,14 +231,16 @@ static void rms_norm_mul_f32(const void * restrict src,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
float epsilon = 0.f;
- memcpy(&epsilon, op_params, sizeof(float));
+ float scale = 0.f;
+ memcpy(&epsilon, &op_params[0], sizeof(float));
+ memcpy(&scale, &op_params[1], sizeof(float));
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
const uint8_t * restrict w_local = (const uint8_t *)weight + (uctx->broadcast_weight ? 0 : ir * uctx->src1_row_size_aligned);
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
- hvx_fast_rms_norm_mul_f32(src_local, w_local, dst_local, ne0, epsilon);
+ hvx_fast_rms_norm_mul_f32(src_local, w_local, dst_local, ne0, epsilon, scale);
}
}
@@ -617,10 +620,11 @@ static void tri_f32(const void * restrict src,
const uint32_t nloe = ne0 % VLEN_FP32;
const uint32_t ne01 = uctx->octx->src[0]->ne[1];
+ const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01;
for (uint32_t b = 0; b < num_rows; b++) {
const uint32_t abs_row = ir + b;
- const uint32_t i01 = abs_row % ne01;
+ const uint32_t i01 = fastmodulo(abs_row, ne01, div_ne01);
const HVX_Vector * restrict v_src = (const HVX_Vector *) ((const uint8_t *) src + b * src0_row_size_aligned);
HVX_Vector * restrict v_dst = (HVX_Vector *) ((uint8_t *) dst + b * dst_row_size_aligned);
@@ -1163,35 +1167,23 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da
const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size;
const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size;
- const bool src0_contig = (nb02 == (size_t)ne01 * nb01) &&
- (nb03 == (size_t)ne02 * nb02);
- const bool dst_contig = (nb2 == (size_t)ne1 * nb1) &&
- (nb3 == (size_t)ne2 * nb2);
+ const bool src0_contig = uctx->kparams->src0_contig;
+ const uint32_t BLOCK = uctx->block;
const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01;
const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02;
const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;
- const uint32_t src0_max_block = src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01);
- const uint32_t dst_max_block = dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1);
- const uint32_t BLOCK = MIN(src0_max_block, dst_max_block);
- if (BLOCK == 0) {
- FARF(ERROR, "unary-row-block : current VTCM reservation %zu is too small, needed at least %zu\n",
- uctx->vtcm_src0_size_per_thread, src0_row_size_aligned);
- return;
- }
-
dma_queue * dma_q = octx->ctx->dma[ith];
for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) {
- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig,
- ne01, div_ne01);
+ const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01);
dma_queue_push(dma_q,
dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)),
nb1, dst_row_size_aligned, dst_data_row_size, 0);
- const size_t src0_off = src0_contig ? (ir * nb01) :
+ const size_t src0_off = src0_contig ? ((size_t) ir * nb01) :
unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03);
dma_queue_push(dma_q,
dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off),
@@ -1203,8 +1195,7 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da
unary_compute_fn_t compute = (unary_compute_fn_t) uctx->compute;
for (uint32_t ir = src0_start_row; ir < src0_end_row; ) {
- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig,
- ne01, div_ne01);
+ const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01);
void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src;
void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst;
@@ -1213,8 +1204,7 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da
compute(src0_vtcm, dst_vtcm, block_size, uctx);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir);
- const size_t dst_off = dst_contig ? (ir * nb1) :
- unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3);
+ const size_t dst_off = (size_t) ir * nb1;
dma_queue_push(dma_q,
dma_make_data(data_dst + dst_off, dst_vtcm),
nb1, dst_row_size_aligned, dst_data_row_size, block_size);
@@ -1222,12 +1212,12 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da
const uint32_t next_ir = ir + block_size;
if (next_ir < src0_end_row) {
const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, src0_contig,
- dst_contig, ne01, div_ne01);
+ ne01, div_ne01);
const uint32_t pref_ir = next_ir + next_block_size;
if (pref_ir < src0_end_row) {
const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, src0_contig,
- dst_contig, ne01, div_ne01);
- const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) :
+ ne01, div_ne01);
+ const size_t src0_pref_off = src0_contig ? ((size_t) pref_ir * nb01) :
unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03);
dma_queue_push(dma_q,
dma_make_data(src0_vtcm, data_src + src0_pref_off),
@@ -1273,10 +1263,6 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo
const uint32_t nb11 = src1->nb[1];
const uint32_t nb12 = src1->nb[2];
const uint32_t nb13 = src1->nb[3];
- const uint32_t nb11_bc = (src1->ne[1] > 1) ? nb11 : 0;
- const uint32_t nb12_bc = (src1->ne[2] > 1) ? nb12 : 0;
- const uint32_t nb13_bc = (src1->ne[3] > 1) ? nb13 : 0;
- const bool src1_contig = ((nb12 == (size_t)ne01 * nb11) && (nb13 == (size_t)ne02 * nb12));
uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread);
uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_thread)) : NULL;
@@ -1286,28 +1272,15 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo
const size_t src1_vtcm_half_size = uctx->src1_vtcm_half_size;
const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size;
- const bool src0_contig = (nb02 == (size_t)ne01 * nb01) &&
- (nb03 == (size_t)ne02 * nb02);
- const bool dst_contig = (nb2 == (size_t)ne1 * nb1) &&
- (nb3 == (size_t)ne2 * nb2);
+ const bool src0_contig = uctx->kparams->src0_contig;
+ const bool src1_contig = uctx->kparams->src1_contig;
+ const bool block_contig = src0_contig && src1_contig;
+ const uint32_t BLOCK = uctx->block;
const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01;
const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02;
const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;
- const bool src1_needs_row_clip = !uctx->broadcast_weight && !src1_contig;
- const bool block_src0_contig = src0_contig && !src1_needs_row_clip;
- const bool block_dst_contig = dst_contig && !src1_needs_row_clip;
-
- const uint32_t src0_max_block = block_src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01);
- const uint32_t dst_max_block = block_dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1);
- const uint32_t BLOCK = MIN(src0_max_block, dst_max_block);
- if (BLOCK == 0) {
- FARF(ERROR, "unary-rms-norm-mul : current VTCM reservation %zu is too small, needed at least %zu\n",
- uctx->vtcm_src0_size_per_thread, src0_row_size_aligned);
- return;
- }
-
dma_queue * dma_q = octx->ctx->dma[ith];
if (uctx->broadcast_weight) {
@@ -1317,22 +1290,21 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo
}
for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) {
- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig,
- ne01, div_ne01);
+ const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_contig, ne01, div_ne01);
dma_queue_push(dma_q,
dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)),
nb1, dst_row_size_aligned, dst_data_row_size, 0);
- const size_t src0_off = src0_contig ? (ir * nb01) :
+ const size_t src0_off = src0_contig ? ((size_t) ir * nb01) :
unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03);
dma_queue_push(dma_q,
dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off),
src0_row_size_aligned, nb01, src0_data_row_size, block_size);
if (!uctx->broadcast_weight) {
- const size_t src1_off = src1_contig ? (ir * nb11) :
- unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc, nb13_bc);
+ const size_t src1_off = src1_contig ? ((size_t) ir * nb11) :
+ unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11, nb12, nb13);
dma_queue_push(dma_q,
dma_make_data(src1_vtcm_data + (vtcm_idx * src1_vtcm_half_size), data_src1 + src1_off),
uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, block_size);
@@ -1344,8 +1316,7 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo
unary_rms_norm_mul_compute_fn_t compute = (unary_rms_norm_mul_compute_fn_t) uctx->compute;
for (uint32_t ir = src0_start_row; ir < src0_end_row; ) {
- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig,
- ne01, div_ne01);
+ const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_contig, ne01, div_ne01);
void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src;
void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst;
@@ -1359,30 +1330,28 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo
compute(src0_vtcm, w, dst_vtcm, block_size, uctx);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir);
- const size_t dst_off = dst_contig ? (ir * nb1) :
- unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3);
+ const size_t dst_off = (size_t) ir * nb1;
dma_queue_push(dma_q,
dma_make_data(data_dst + dst_off, dst_vtcm),
nb1, dst_row_size_aligned, dst_data_row_size, block_size);
const uint32_t next_ir = ir + block_size;
if (next_ir < src0_end_row) {
- const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, block_src0_contig,
- block_dst_contig, ne01, div_ne01);
+ const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, block_contig,
+ ne01, div_ne01);
const uint32_t pref_ir = next_ir + next_block_size;
if (pref_ir < src0_end_row) {
- const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, block_src0_contig,
- block_dst_contig, ne01, div_ne01);
- const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) :
+ const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, block_contig,
+ ne01, div_ne01);
+ const size_t src0_pref_off = src0_contig ? ((size_t) pref_ir * nb01) :
unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03);
dma_queue_push(dma_q,
dma_make_data(src0_vtcm, data_src + src0_pref_off),
src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size);
if (!uctx->broadcast_weight) {
- const size_t src1_pref_off = src1_contig ? (pref_ir * nb11) :
- unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc,
- nb13_bc);
+ const size_t src1_pref_off = src1_contig ? ((size_t) pref_ir * nb11) :
+ unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11, nb12, nb13);
dma_queue_push(dma_q,
dma_make_data(src1_vtcm, data_src1 + src1_pref_off),
uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, pref_block_size);
@@ -1428,37 +1397,18 @@ static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data
const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size;
const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size;
-
- const bool src0_contig = (nb02 == (size_t)ne01 * nb01) &&
- (nb03 == (size_t)ne02 * nb02);
- const bool dst_contig = (nb2 == (size_t)ne1 * nb1) &&
- (nb3 == (size_t)ne2 * nb2);
-
- const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01;
- const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02;
- const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;
-
- const uint32_t src0_max_block = src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01);
- const uint32_t dst_max_block = dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1);
- const uint32_t BLOCK = MIN(src0_max_block, dst_max_block);
- if (BLOCK == 0) {
- FARF(ERROR, "unary-tri : current VTCM reservation %zu is too small, needed at least %zu\n",
- uctx->vtcm_src0_size_per_thread, src0_row_size_aligned);
- return;
- }
+ const uint32_t BLOCK = uctx->block;
dma_queue * dma_q = octx->ctx->dma[ith];
for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) {
- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig,
- ne01, div_ne01);
+ const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
dma_queue_push(dma_q,
dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)),
nb1, dst_row_size_aligned, dst_data_row_size, 0);
- const size_t src0_off = src0_contig ? (ir * nb01) :
- unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03);
+ const size_t src0_off = (size_t) ir * nb01;
dma_queue_push(dma_q,
dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off),
src0_row_size_aligned, nb01, src0_data_row_size, block_size);
@@ -1469,8 +1419,7 @@ static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data
unary_tri_compute_fn_t compute = (unary_tri_compute_fn_t) uctx->compute;
for (uint32_t ir = src0_start_row; ir < src0_end_row; ) {
- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig,
- ne01, div_ne01);
+ const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src;
void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst;
@@ -1479,22 +1428,18 @@ static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data
compute(src0_vtcm, dst_vtcm, block_size, ir, uctx);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir);
- const size_t dst_off = dst_contig ? (ir * nb1) :
- unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3);
+ const size_t dst_off = (size_t) ir * nb1;
dma_queue_push(dma_q,
dma_make_data(data_dst + dst_off, dst_vtcm),
nb1, dst_row_size_aligned, dst_data_row_size, block_size);
const uint32_t next_ir = ir + block_size;
if (next_ir < src0_end_row) {
- const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, src0_contig,
- dst_contig, ne01, div_ne01);
+ const uint32_t next_block_size = MIN(BLOCK, src0_end_row - next_ir);
const uint32_t pref_ir = next_ir + next_block_size;
if (pref_ir < src0_end_row) {
- const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, src0_contig,
- dst_contig, ne01, div_ne01);
- const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) :
- unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03);
+ const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_ir);
+ const size_t src0_pref_off = (size_t) pref_ir * nb01;
dma_queue_push(dma_q,
dma_make_data(src0_vtcm, data_src + src0_pref_off),
src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size);
@@ -1544,21 +1489,17 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data)
const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;
const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr;
- const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile;
-
- const bool src0_contig = (nb02 == (size_t)ne01 * nb01) &&
- (nb03 == (size_t)ne02 * nb02);
- const bool dst_contig = (nb2 == (size_t)ne1 * nb1) &&
- (nb3 == (size_t)ne2 * nb2);
+ const uint32_t tiles_per_row = uctx->kparams->tiles_per_row;
+ const bool src0_contig = uctx->kparams->src0_contig;
const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row;
for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) {
- const uint32_t row = src0_start_row + t / tiles_per_row;
- const uint32_t col = (t % tiles_per_row) * col_tile;
+ const uint32_t row = src0_start_row + fastdiv(t, div_tpr);
+ const uint32_t col = fastmodulo(t, tiles_per_row, div_tpr) * col_tile;
const uint32_t tw = MIN(col_tile, ne0 - col);
const size_t tb = (size_t) tw * sizeof(float);
- const size_t soff = (src0_contig ? (row * nb01) :
+ const size_t soff = (src0_contig ? ((size_t) row * nb01) :
unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) +
(size_t) col * sizeof(float);
@@ -1586,9 +1527,7 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data)
compute(dst_vtcm, src_vtcm, tw, uctx);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t);
- const size_t doff = (dst_contig ? (row * nb1) :
- unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) +
- (size_t) col * sizeof(float);
+ const size_t doff = (size_t) row * nb1 + (size_t) col * sizeof(float);
const size_t tb = (size_t) tw * sizeof(float);
dma_queue_push(dma_q, dma_make_data(data_dst + doff, dst_vtcm), tb, tb, tb, 1);
@@ -1596,7 +1535,7 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data)
if (pt < total_tiles) {
const uint32_t ptw = MIN(col_tile, ne0 - pcol);
const size_t ptb = (size_t) ptw * sizeof(float);
- const size_t psoff = (src0_contig ? (prow * nb01) :
+ const size_t psoff = (src0_contig ? ((size_t) prow * nb01) :
unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02,
nb03)) +
(size_t) pcol * sizeof(float);
@@ -1657,29 +1596,20 @@ static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, void
dma_queue * dma_q = octx->ctx->dma[ith];
- const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01;
- const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02;
- const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;
- const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr;
+ const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01;
+ const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr;
- const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile;
+ const uint32_t tiles_per_row = uctx->kparams->tiles_per_row;
const int32_t tri_ttype = op_params[0];
- const bool src0_contig = (nb02 == (size_t)ne01 * nb01) &&
- (nb03 == (size_t)ne02 * nb02);
- const bool dst_contig = (nb2 == (size_t)ne1 * nb1) &&
- (nb3 == (size_t)ne2 * nb2);
-
const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row;
for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) {
- const uint32_t row = src0_start_row + t / tiles_per_row;
- const uint32_t col = (t % tiles_per_row) * col_tile;
+ const uint32_t row = src0_start_row + fastdiv(t, div_tpr);
+ const uint32_t col = fastmodulo(t, tiles_per_row, div_tpr) * col_tile;
const uint32_t tw = MIN(col_tile, ne0 - col);
const size_t tb = (size_t) tw * sizeof(float);
- const size_t soff = (src0_contig ? (row * nb01) :
- unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) +
- (size_t) col * sizeof(float);
+ const size_t soff = (size_t) row * nb01 + (size_t) col * sizeof(float);
dma_queue_push(dma_q, dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0);
dma_queue_push(dma_q, dma_make_data(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1);
@@ -1706,20 +1636,15 @@ static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, void
compute(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t);
- const size_t doff = (dst_contig ? (row * nb1) :
- unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) +
- (size_t) col * sizeof(float);
+ const size_t doff = (size_t) row * nb1 + (size_t) col * sizeof(float);
const size_t tb = (size_t) tw * sizeof(float);
dma_queue_push(dma_q, dma_make_data(data_dst + doff, dst_vtcm), tb, tb, tb, 1);
const uint32_t pt = t + 2;
if (pt < total_tiles) {
- const uint32_t ptw = MIN(col_tile, ne0 - pcol);
- const size_t ptb = (size_t) ptw * sizeof(float);
- const size_t psoff = (src0_contig ? (prow * nb01) :
- unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02,
- nb03)) +
- (size_t) pcol * sizeof(float);
+ const uint32_t ptw = MIN(col_tile, ne0 - pcol);
+ const size_t ptb = (size_t) ptw * sizeof(float);
+ const size_t psoff = (size_t) prow * nb01 + (size_t) pcol * sizeof(float);
dma_queue_push(dma_q, dma_make_data(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1);
}
@@ -1787,28 +1712,6 @@ static int execute_op_unary(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
- // F16 only has row-block kernels for this subset of ops (see the dispatch switch
- // below) - reject everything else up front, before touching kparams/VTCM.
- if (is_f16) {
- switch (octx->op) {
- case HTP_OP_NORM:
- case HTP_OP_RMS_NORM:
- case HTP_OP_SCALE:
- case HTP_OP_CLAMP:
- case HTP_OP_SQR:
- case HTP_OP_SQRT:
- case HTP_OP_L2_NORM:
- case HTP_OP_UNARY_ABS:
- case HTP_OP_UNARY_LOG:
- case HTP_OP_UNARY_STEP:
- case HTP_OP_HADAMARD:
- break;
- default:
- FARF(ERROR, "unary-%s: not supported for F16\n", op_type);
- return HTP_STATUS_NO_SUPPORT;
- }
- }
-
const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params;
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
@@ -1838,15 +1741,11 @@ static int execute_op_unary(struct htp_ops_context * octx) {
const uint32_t n_threads = octx->n_threads;
- const size_t src0_row_size_aligned = kparams->src0_row_size_aligned;
- const size_t dst_row_size_aligned = kparams->dst_row_size_aligned;
-
- // Always 0 for F16 - htp_unary_vtcm_layout_build() keeps F16 on the row-block path,
+ // Always 0 for F16 - htp_unary_solve_layout() keeps F16 on the row-block path,
// since only F32 has unary_task_f32_tiled_* kernels.
const uint32_t col_tile = kparams->col_tile;
size_t src1_data_row_size = 0;
- size_t src1_row_size_aligned = kparams->src1_row_size_aligned;
bool broadcast_weight = kparams->broadcast_weight;
const struct htp_tensor * src1 = NULL;
@@ -1857,8 +1756,8 @@ static int execute_op_unary(struct htp_ops_context * octx) {
src1_data_row_size = src1->ne[0] * sizeof(float);
}
- if (octx->ctx->vtcm_size < (size_t)kparams->vtcm_size) {
- FARF(ERROR, "unary-%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, (size_t)kparams->vtcm_size);
+ if ((!col_tile && kparams->block == 0) || octx->ctx->vtcm_size < (size_t) kparams->vtcm_size) {
+ FARF(ERROR, "unary-%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, (size_t) kparams->vtcm_size);
return HTP_STATUS_VTCM_TOO_SMALL;
}
@@ -1866,9 +1765,13 @@ static int execute_op_unary(struct htp_ops_context * octx) {
octx->src1_spad.src = NULL;
octx->dst_spad.src = NULL;
- FARF(HIGH, "%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %u src1-vtcm-size %u dst-vtcm-size %u\n", op_type,
+ struct htp_unary_vtcm_layout L;
+ htp_unary_vtcm_layout_build(&L, octx->op, src0_data_row_size, dst_data_row_size, src1_data_row_size,
+ src0_elem_size, col_tile, kparams->block, broadcast_weight, kparams->n_threads);
+
+ FARF(HIGH, "%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type,
src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
- kparams->vtcm_src0_size, kparams->vtcm_src1_size, kparams->vtcm_dst_size);
+ L.src0_bytes * n_threads, L.src1_bytes * n_threads, L.dst_bytes * n_threads);
uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;
struct htp_unary_context uctx = {
@@ -1886,26 +1789,26 @@ static int execute_op_unary(struct htp_ops_context * octx) {
.src1_data_row_size = src1_data_row_size,
.dst_data_row_size = dst_data_row_size,
- .src0_row_size_aligned = src0_row_size_aligned,
- .src1_row_size_aligned = src1_row_size_aligned,
- .dst_row_size_aligned = dst_row_size_aligned,
+ .src0_row_size_aligned = L.src0_row_size_aligned,
+ .src1_row_size_aligned = L.src1_row_size_aligned,
+ .dst_row_size_aligned = L.dst_row_size_aligned,
- .src0_vtcm_half_size = kparams->vtcm_src0_size_per_thread / 2,
- .src1_vtcm_half_size = (octx->op == HTP_OP_RMS_NORM_MUL) ? (kparams->vtcm_src1_size_per_thread / (broadcast_weight ? 1 : 2)) : 0,
- .dst_vtcm_half_size = kparams->vtcm_dst_size_per_thread / 2,
+ .src0_vtcm_half_size = L.src0_bytes >> 1,
+ .src1_vtcm_half_size = (octx->op == HTP_OP_RMS_NORM_MUL) ? (L.src1_bytes >> (broadcast_weight ? 0 : 1)) : 0,
+ .dst_vtcm_half_size = L.dst_bytes >> 1,
.block = kparams->block,
.nc = src0->ne[0],
.col_tile = col_tile,
.broadcast_weight = broadcast_weight,
- .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, 0),
- .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, kparams->vtcm_src0_size, kparams->vtcm_src1_size > 0),
- .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, kparams->vtcm_src0_size + kparams->vtcm_src1_size),
+ .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0),
+ .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_src1, L.src1_bytes > 0),
+ .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst),
- .vtcm_src0_size_per_thread = kparams->vtcm_src0_size_per_thread,
- .vtcm_src1_size_per_thread = kparams->vtcm_src1_size_per_thread,
- .vtcm_dst_size_per_thread = kparams->vtcm_dst_size_per_thread,
+ .vtcm_src0_size_per_thread = L.src0_bytes,
+ .vtcm_src1_size_per_thread = L.src1_bytes,
+ .vtcm_dst_size_per_thread = L.dst_bytes,
};
FARF(HIGH, "%s: %s mode (col_tile %u)\n", op_type, col_tile ? "tiled" : "row-block", col_tile);
@@ -2007,8 +1910,25 @@ static int execute_op_unary(struct htp_ops_context * octx) {
int op_unary(struct htp_ops_context * octx) {
switch (octx->src[0]->type) {
case HTP_TYPE_F32:
- case HTP_TYPE_F16:
- return execute_op_unary(octx);
+ case HTP_TYPE_F16: {
+ if (!htp_unary_is_elementwise(octx->op)) {
+ return execute_op_unary(octx);
+ }
+
+ struct htp_tensor src0_c = *octx->src[0];
+ struct htp_tensor dst_c = *octx->dst;
+ const size_t elem_size = (src0_c.type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float);
+ HTP_UNARY_COLLAPSE_DIMS(src0_c.ne, src0_c.nb, dst_c.ne, dst_c.nb, elem_size);
+
+ const struct htp_tensor * orig_src0 = octx->src[0];
+ const struct htp_tensor * orig_dst = octx->dst;
+ octx->src[0] = &src0_c;
+ octx->dst = &dst_c;
+ const int status = execute_op_unary(octx);
+ octx->src[0] = orig_src0;
+ octx->dst = orig_dst;
+ return status;
+ }
default:
return HTP_STATUS_NO_SUPPORT;
diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h
index 902028635..ec67612b9 100644
--- a/ggml/src/ggml-hexagon/htp/unary-ops.h
+++ b/ggml/src/ggml-hexagon/htp/unary-ops.h
@@ -2,29 +2,22 @@
#define HTP_UNARY_OPS_H
#include "hex-common.h"
+#include "hex-fastdiv.h"
#include "htp-ops.h"
+#include "htp-vtcm.h"
// Op-specific struct for precomputed unary params
struct htp_unary_kernel_params {
uint32_t n_threads;
uint32_t col_tile;
- uint32_t vtcm_row_per_thread;
uint32_t block;
uint32_t broadcast_weight;
- uint32_t vtcm_src0_size_per_thread;
- uint32_t vtcm_src1_size_per_thread;
- uint32_t vtcm_dst_size_per_thread;
-
- uint32_t vtcm_src0_size;
- uint32_t vtcm_src1_size;
- uint32_t vtcm_dst_size;
-
- uint32_t src0_row_size_aligned;
- uint32_t src1_row_size_aligned;
- uint32_t dst_row_size_aligned;
+ uint32_t src0_contig;
+ uint32_t src1_contig;
uint32_t vtcm_size;
+ uint32_t tiles_per_row;
// Fastdiv helpers
struct fastdiv_values div_ne01;
@@ -71,23 +64,105 @@ static inline bool htp_op_is_unary(uint32_t opcode) {
}
}
+static inline bool htp_unary_is_elementwise(uint32_t opcode) {
+ switch (opcode) {
+ case HTP_OP_NORM:
+ case HTP_OP_RMS_NORM:
+ case HTP_OP_RMS_NORM_MUL:
+ case HTP_OP_L2_NORM:
+ case HTP_OP_HADAMARD:
+ case HTP_OP_TRI:
+ return false;
+ default:
+ return true;
+ }
+}
+
+#define HTP_UNARY_COLLAPSE_DIMS(src_ne, src_nb, dst_ne, dst_nb, elem_size) \
+ do { \
+ while (((src_ne)[1] > 1 || (src_ne)[2] > 1 || (src_ne)[3] > 1) && \
+ (size_t) (src_nb)[0] == (elem_size) && \
+ (size_t) (dst_nb)[0] == (elem_size) && \
+ (((src_ne)[1] == 1 && (dst_ne)[1] == 1) || \
+ ((size_t) (src_nb)[1] == (size_t) (src_ne)[0] * (size_t) (src_nb)[0] && \
+ (size_t) (dst_nb)[1] == (size_t) (dst_ne)[0] * (size_t) (dst_nb)[0] && \
+ ((src_ne)[0] == 1 || \
+ (uint64_t) (src_ne)[0] * (uint64_t) (src_ne)[1] * (elem_size) <= 128)))) { \
+ (src_ne)[0] *= (src_ne)[1]; \
+ (src_ne)[1] = (src_ne)[2]; \
+ (src_ne)[2] = (src_ne)[3]; \
+ (src_ne)[3] = 1; \
+ (dst_ne)[0] *= (dst_ne)[1]; \
+ (dst_ne)[1] = (dst_ne)[2]; \
+ (dst_ne)[2] = (dst_ne)[3]; \
+ (dst_ne)[3] = 1; \
+ (src_nb)[1] = (src_nb)[2]; \
+ (src_nb)[2] = (src_nb)[3]; \
+ (dst_nb)[1] = (dst_nb)[2]; \
+ (dst_nb)[2] = (dst_nb)[3]; \
+ } \
+ } while (0)
+
struct htp_unary_vtcm_layout {
size_t total_bytes;
size_t off_src0;
size_t off_src1;
size_t off_dst;
+ size_t src0_row_size_aligned;
+ size_t src1_row_size_aligned;
+ size_t dst_row_size_aligned;
+
size_t src0_bytes;
size_t src1_bytes;
size_t dst_bytes;
};
static inline void htp_unary_vtcm_layout_build(
+ struct htp_unary_vtcm_layout * L,
+ uint32_t op,
+ size_t src0_data_row_size,
+ size_t dst_data_row_size,
+ size_t src1_data_row_size,
+ size_t elem_size,
+ uint32_t col_tile,
+ uint32_t block,
+ bool broadcast_weight,
+ uint32_t n_threads
+) {
+ L->src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);
+ L->dst_row_size_aligned = hex_round_up(dst_data_row_size, 128);
+ L->src1_row_size_aligned = (op == HTP_OP_RMS_NORM_MUL) ? hex_round_up(src1_data_row_size, 128) : 0;
+
+ if (col_tile > 0) {
+ const size_t col_tile_bytes = col_tile * elem_size;
+ L->src0_bytes = col_tile_bytes * 2;
+ L->dst_bytes = col_tile_bytes * 2;
+ L->src1_bytes = 0;
+ } else {
+ L->src0_bytes = L->src0_row_size_aligned * block * 2;
+ L->dst_bytes = L->dst_row_size_aligned * block * 2;
+ if (op == HTP_OP_RMS_NORM_MUL) {
+ L->src1_bytes = broadcast_weight ? L->src1_row_size_aligned : (L->src1_row_size_aligned * block * 2);
+ } else {
+ L->src1_bytes = 0;
+ }
+ }
+
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes * n_threads);
+ VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes * n_threads, L->src1_bytes > 0);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes * n_threads);
+ L->total_bytes = off;
+}
+
+static inline void htp_unary_solve_layout(
struct htp_unary_vtcm_layout * L,
uint32_t op,
uint32_t ne00,
uint32_t ne10,
uint32_t ne11,
+ uint32_t max_block,
bool broadcast_weight,
uint32_t n_threads,
size_t vtcm_size,
@@ -98,16 +173,11 @@ static inline void htp_unary_vtcm_layout_build(
const size_t dst_elem_size = (op == HTP_OP_HADAMARD) ? sizeof(float) : elem_size;
const size_t src0_data_row_size = ne00 * elem_size;
const size_t dst_data_row_size = ne10 * dst_elem_size;
+ const size_t src1_data_row_size = (op == HTP_OP_RMS_NORM_MUL) ? (ne11 * sizeof(float)) : 0;
const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);
const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128);
-
- size_t src1_row_size_aligned = 0;
- if (op == HTP_OP_RMS_NORM_MUL) {
- // RMS_NORM_MUL fusion is F32-only; its weight tensor is always F32.
- const size_t src1_data_row_size = ne11 * sizeof(float);
- src1_row_size_aligned = hex_round_up(src1_data_row_size, 128);
- }
+ const size_t src1_row_size_aligned = (op == HTP_OP_RMS_NORM_MUL) ? hex_round_up(src1_data_row_size, 128) : 0;
size_t vtcm_size_per_row = 0;
size_t vtcm_row_per_thread = 0;
@@ -132,6 +202,10 @@ static inline void htp_unary_vtcm_layout_build(
vtcm_row_per_thread = vtcm_size / (n_threads * vtcm_size_per_row);
}
+ if (max_block > 0 && vtcm_row_per_thread > max_block) {
+ vtcm_row_per_thread = max_block;
+ }
+
const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM ||
op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM ||
op == HTP_OP_HADAMARD);
@@ -148,37 +222,14 @@ static inline void htp_unary_vtcm_layout_build(
const size_t per_thread_budget = vtcm_size / n_threads;
const size_t col_tile_bytes = hex_align_down(per_thread_budget / 4, 128);
col_tile = (uint32_t) (col_tile_bytes / elem_size);
-
- L->src0_bytes = col_tile_bytes * 2;
- L->dst_bytes = col_tile_bytes * 2;
- L->src1_bytes = 0;
- } else {
- L->src0_bytes = src0_row_size_aligned * vtcm_row_per_thread * 2;
- L->dst_bytes = dst_row_size_aligned * vtcm_row_per_thread * 2;
- if (op == HTP_OP_RMS_NORM_MUL) {
- if (broadcast_weight) {
- L->src1_bytes = src1_row_size_aligned;
- } else {
- L->src1_bytes = src1_row_size_aligned * vtcm_row_per_thread * 2;
- }
- } else {
- L->src1_bytes = 0;
- }
- }
-
- L->off_src0 = 0;
- if (op == HTP_OP_RMS_NORM_MUL) {
- L->off_src1 = L->off_src0 + L->src0_bytes * n_threads;
- L->off_dst = L->off_src1 + L->src1_bytes * n_threads;
- } else {
- L->off_src1 = 0;
- L->off_dst = L->off_src0 + L->src0_bytes * n_threads;
}
- L->total_bytes = L->off_dst + L->dst_bytes * n_threads;
+ const uint32_t block = col_tile ? 0 : (uint32_t) vtcm_row_per_thread;
+ htp_unary_vtcm_layout_build(L, op, src0_data_row_size, dst_data_row_size, src1_data_row_size,
+ elem_size, col_tile, block, broadcast_weight, n_threads);
*out_col_tile = col_tile;
- *out_vtcm_row_per_thread = vtcm_row_per_thread;
+ *out_vtcm_row_per_thread = (uint32_t) vtcm_row_per_thread;
}
#endif /* HTP_UNARY_OPS_H */
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index f490d201c..0fc5c0b5f 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -9971,7 +9971,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
// fused unary + mul (gated activations that are not expressed as GGML_OP_GLU)
- for (ggml_unary_op op : { GGML_UNARY_OP_GELU, GGML_UNARY_OP_SILU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) {
+ for (ggml_unary_op op : { GGML_UNARY_OP_GELU, GGML_UNARY_OP_SILU, GGML_UNARY_OP_RELU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) {
for (ggml_type type : { GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16 }) {
for (bool swap : { false, true }) {
test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, swap));