Commit 7aca0d2ea for llama.cpp
commit 7aca0d2ea8d92b56911233b822db01f9fbb1ae50
Author: Ehsan Bateni <ebateni@qti.qualcomm.com>
Date: Mon Oct 26 19:56:01 2026 -0400
hexagon: add GGML_HEXAGON_VTCM_SIZE (#31221)
* hexagon: add GGML_HEXAGON_VTCM_MB and GGML_HEXAGON_NTHREADS caps
Change-Id: Idffe1613c8db6f96aa842d43d1b2efaed45462a8
* hexagon: rename VTCM cap to GGML_HEXAGON_VTCM_SIZE, drop NTHREADS cap
Change-Id: If4c5b9ce9fd6b725c3561421c2295d40599d012d
* hexagon: make GGML_HEXAGON_VTCM_SIZE an override, accept K/M/G suffixes
Change-Id: Ib8eea84c22f24e6d986b37f215e504b046478c2c
diff --git a/docs/backend/snapdragon/README.md b/docs/backend/snapdragon/README.md
index f0682ce7a..ea4e3f386 100644
--- a/docs/backend/snapdragon/README.md
+++ b/docs/backend/snapdragon/README.md
@@ -324,6 +324,9 @@ on 4 physical NPUs, or `--devices 'HTP0[0-1:0],HTP1[0-1:1]'` on 2 physical NPUs
- `GGML_HEXAGON_NHVX=0`
Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version).
+- `GGML_HEXAGON_VTCM_SIZE=0`
+ Overrides the VTCM size used by each session. Accepts `K`, `M` and `G` suffixes; a plain number is in MB. The default (0) uses the size the hardware reports. Values above the hardware size will fail to start the session.
+
- `GGML_HEXAGON_HOSTBUF=1` (default: 0, disabled)
Enables allocating host buffers for debugging. By default, host buffers are disabled.
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index 45e6ae902..cf2e37aa6 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -85,6 +85,7 @@ int opt_arch = 0; // autodetect
size_t opt_ndev = 1;
size_t opt_nhvx = 0; // use all
int opt_nhmx = 1; // when set, enable HMX; when 0, use HVX only
+size_t opt_vtcm_size = 0; // VTCM size override in bytes; 0 = auto-detect
size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for buffer mappings
int opt_etm = 0;
int opt_verbose = 0;
@@ -2152,6 +2153,12 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
this->n_hvx = this->n_threads;
this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0;
this->vtcm_size = (uint64_t)hw_vtcm_size;
+ if (opt_vtcm_size > 0) {
+ if (opt_vtcm_size > hw_vtcm_size) {
+ GGML_LOG_WARN("ggml-hex: %s VTCM override %zu exceeds HW size %llu\n", this->c_name(), opt_vtcm_size, hw_vtcm_size);
+ }
+ this->vtcm_size = opt_vtcm_size;
+ }
GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n",
this->c_name(), this->n_threads, this->n_hvx, this->n_hmx,
(unsigned long long)(this->vtcm_size / (1024 * 1024)));
@@ -2161,7 +2168,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads;
this->n_hvx = this->n_threads;
this->n_hmx = (opt_nhmx != 0) ? 1 : 0;
- this->vtcm_size = 8 * 1024 * 1024;
+ this->vtcm_size = opt_vtcm_size > 0 ? opt_vtcm_size : 8 * 1024 * 1024;
}
}
@@ -2230,7 +2237,8 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem);
// Start dspqueue/opbatch processing
- err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem, (uint32_t) opt_power_mode);
+ err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, opt_vtcm_size, this->max_vmem,
+ (uint32_t) opt_power_mode);
if (err != 0) {
GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err);
throw std::runtime_error("ggml-hex: iface start failed (see log for details)");
@@ -3391,6 +3399,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
const char * str_etm = getenv("GGML_HEXAGON_ETM");
const char * str_nhvx = getenv("GGML_HEXAGON_NHVX");
const char * str_nhmx = getenv("GGML_HEXAGON_NHMX");
+ const char * str_vtcm_size = getenv("GGML_HEXAGON_VTCM_SIZE");
const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT");
const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT");
const char * str_fa_split = getenv("GGML_HEXAGON_FA_SPLIT");
@@ -3456,6 +3465,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
}
opt_nhvx = str_nhvx ? strtoul(str_nhvx, NULL, 0) : opt_nhvx;
opt_nhmx = str_nhmx ? atoi(str_nhmx) : opt_nhmx;
+ opt_vtcm_size = str_vtcm_size ? parse_size(str_vtcm_size) : opt_vtcm_size;
opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select;
opt_fa_select = str_fa_select ? atoi(str_fa_select) : opt_fa_select;
opt_fa_split = str_fa_split ? atoi(str_fa_split) : opt_fa_split;
diff --git a/ggml/src/ggml-hexagon/htp/htp_iface.idl b/ggml/src/ggml-hexagon/htp/htp_iface.idl
index 303dbfb0a..6e3390ba1 100644
--- a/ggml/src/ggml-hexagon/htp/htp_iface.idl
+++ b/ggml/src/ggml-hexagon/htp/htp_iface.idl
@@ -11,7 +11,7 @@ struct htp_iface_pmu_conf {
};
interface htp_iface : remote_handle64 {
- AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 max_vmem, in uint32 power_mode);
+ AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 vtcm_size, in uint64 max_vmem, in uint32 power_mode);
AEEResult stop();
AEEResult mmap(in uint32 fd, in uint64 size);
AEEResult munmap(in uint32 fd);
diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c
index d1a4afcd6..414c6f661 100644
--- a/ggml/src/ggml-hexagon/htp/main.c
+++ b/ggml/src/ggml-hexagon/htp/main.c
@@ -325,8 +325,11 @@ static unsigned int vtcm_query_size(void) {
return vtcm_size;
}
-static int vtcm_alloc(struct htp_context * ctx) {
+static int vtcm_alloc(struct htp_context * ctx, uint64_t size_override) {
unsigned int vtcm_size = vtcm_query_size();
+ if (size_override) {
+ vtcm_size = (unsigned int) size_override;
+ }
compute_res_attr_t attr;
HAP_compute_res_attr_init(&attr);
@@ -438,7 +441,7 @@ static const struct htp_power_profile htp_power_profiles[HTP_POWER_MODE_COUNT] =
#undef HTP_POWER_PROFILE_ENTRY
-AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t max_vmem, uint32_t power_mode) {
+AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t vtcm_size, uint64_t max_vmem, uint32_t power_mode) {
struct htp_handle * h = (struct htp_handle *) handle;
if (!h) {
return AEE_EBADPARM;
@@ -557,7 +560,7 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp
ctx->max_vmem = max_vmem;
ctx->dsp_queue = dsp_queue;
- err = vtcm_alloc(ctx);
+ err = vtcm_alloc(ctx, vtcm_size);
if (err != AEE_SUCCESS) {
FARF(ERROR, "Unable to allocate VTCM");
htp_iface_stop(handle);