Commit 9daa0aa22 for llama.cpp
commit 9daa0aa22b8aa0143344d7549dee1f0e32f048bb
Author: Max Krasnyansky <maxk@qti.qualcomm.com>
Date: Fri Nov 13 07:56:20 2026 -0800
hexagon: improve buffer allocations and mappings (mmap stacking) (#32144)
* hex-bufs: combine queue and fence buffers
* hex-bufs: use n_threads for queue sizing
* hex-mmap: optimize buffer stacking at the host to avoid iova fragmentation
* hex-mmap: add simply retry for spurios mmap failures
* hex-mmap: use eager mode for extended mmap retries
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index 03da443c8..89c9d9a61 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -316,6 +316,7 @@ struct ggml_hexagon_session {
ggml_hexagon_opbatch* op_batch;
ggml_hexagon_opqueue* op_queue;
+ std::unordered_set<ggml_hexagon_shared_buffer *> buffers;
std::unordered_map<int, std::unique_ptr<ggml_hexagon_shared_buffer>> cloned_buffers;
std::unordered_set<ggml_hexagon_session *> virt_peers;
std::unordered_set<ggml_hexagon_session *> phys_peers;
@@ -337,6 +338,7 @@ struct ggml_hexagon_session {
ggml_hexagon_mdev_group mdev;
ggml_backend_dev_t dev = nullptr;
ggml_backend_hexagon_device_context * dev_ctx = nullptr;
+ ggml_hexagon_shared_buffer * shm_buf = nullptr;
ggml_hexagon_fence_buffer * fence_buf = nullptr;
ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev = nullptr, uint32_t mdev_idx = 0, uint32_t mdev_count = 0) noexcept(false);
@@ -373,8 +375,8 @@ struct ggml_hexagon_session {
void flush_peers();
void flush_pending(bool all = true);
- ggml_hexagon_shared_buffer * mmap_tensor(const ggml_tensor * t);
- bool clone_buffer(const ggml_hexagon_shared_buffer*);
+ void mmap_buffers();
+ void clone_buffer(const ggml_hexagon_shared_buffer*);
void release_buffer(const ggml_hexagon_shared_buffer*);
void unclone_buffer(const ggml_hexagon_shared_buffer*);
@@ -434,7 +436,7 @@ struct ggml_hexagon_rpcmem_block {
int fd = -1;
size_t size = 0;
- std::unordered_set<ggml_hexagon_session *> mapped_clones;
+ std::unordered_set<ggml_hexagon_session *> clones;
ggml_hexagon_rpcmem_block(size_t size) {
base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, size);
@@ -468,16 +470,24 @@ struct ggml_hexagon_shared_buffer {
size_t size() const { return mem ? mem->size : 0; }
int fd() const { return mem ? mem->fd : -1; }
- void mmap(bool extended = false) {
+ void mmap() {
if (!this->mem) return;
if (this->mapped) return;
- GGML_ASSERT(!this->pinned || !extended);
+ GGML_ASSERT(!this->pinned || !this->extended);
- this->extended = extended;
- fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : (extended ? FASTRPC_MAP_FD_DELAYED_EXTENDED : FASTRPC_MAP_FD_DELAYED);
+ fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : (this->extended ? FASTRPC_MAP_FD_DELAYED_EXTENDED : FASTRPC_MAP_FD_DELAYED);
int err = fastrpc_mmap(sess->domain_id, fd(), (void *) base(), 0, size(), flags);
+ if (err != 0) {
+ if (this->extended) {
+ flags = FASTRPC_MAP_FD_EXTENDED;
+ }
+ HEX_VERBOSE("ggml-hex: %s mmap failed (err 0x%08x), retrying in 100ms...\n", sess->c_name(), (unsigned) err);
+ std::this_thread::sleep_for(std::chrono::milliseconds(100));
+ err = fastrpc_mmap(sess->domain_id, fd(), (void *) base(), 0, size(), flags);
+ }
+
if (err != 0) {
GGML_LOG_ERROR("ggml-hex: %s buffer mapping failed : domain_id %d size %zu fd %d error 0x%08x\n", sess->c_name(),
sess->domain_id, size(), fd(), (unsigned) err);
@@ -550,10 +560,10 @@ struct ggml_hexagon_shared_buffer {
// Clone constructor for cross-session mapping
ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, const ggml_hexagon_shared_buffer & other) {
- this->sess = sess;
- this->mem = other.mem;
- this->mapped = false;
- this->pinned = other.pinned;
+ this->sess = sess;
+ this->mem = other.mem;
+ this->mapped = false;
+ this->pinned = false;
this->extended = other.extended;
}
@@ -563,19 +573,30 @@ struct ggml_hexagon_shared_buffer {
};
struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer {
+ uint8_t * data = nullptr;
uint32_t slot_count = 0;
uint32_t slot_head = 0;
std::vector<uint32_t> free_slots;
ggml_backend_buffer backend_buffer{};
- ggml_hexagon_fence_buffer(ggml_hexagon_session * sess, ggml_backend_buffer_type_t buft, size_t size)
- : ggml_hexagon_shared_buffer(sess, size, false /* pinned */),
- slot_count(size / HTP_FENCE_SLOT_SIZE),
+ static size_t size_of(size_t n_slots) {
+ return n_slots * HTP_FENCE_SLOT_SIZE;
+ }
+
+ ggml_hexagon_fence_buffer(ggml_hexagon_shared_buffer * shm_buf, ggml_backend_buffer_type_t buft, size_t offset, size_t n_slots)
+ : ggml_hexagon_shared_buffer(shm_buf->sess, *shm_buf),
+ data(shm_buf->base() + offset),
+ slot_count(n_slots),
slot_head(0) {
+ GGML_ASSERT((uintptr_t) data % HEX_L2_LINE_SIZE == 0);
+ this->mapped = shm_buf->mapped;
backend_buffer.buft = buft;
backend_buffer.context = static_cast<ggml_hexagon_shared_buffer *>(this);
- backend_buffer.size = size;
- mmap(false);
+ backend_buffer.size = size_of(n_slots);
+ }
+
+ ~ggml_hexagon_fence_buffer() {
+ this->mapped = false;
}
uint8_t * alloc_slot(uint32_t n_slots = 1) {
@@ -583,11 +604,11 @@ struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer {
if (n_slots == 1 && !free_slots.empty()) {
uint32_t slot = free_slots.back();
free_slots.pop_back();
- ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE;
+ ptr = data + (size_t) slot * HTP_FENCE_SLOT_SIZE;
} else if (slot_head + n_slots <= slot_count) {
uint32_t slot = slot_head;
slot_head += n_slots;
- ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE;
+ ptr = data + (size_t) slot * HTP_FENCE_SLOT_SIZE;
}
if (ptr) {
memset(ptr, 0, (size_t) n_slots * HTP_FENCE_SLOT_SIZE);
@@ -597,7 +618,7 @@ struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer {
void free_slot(void * ptr, uint32_t n_slots = 1) {
if (!ptr) return;
- uint32_t slot = ((uint8_t *) ptr - base()) / HTP_FENCE_SLOT_SIZE;
+ uint32_t slot = ((uint8_t *) ptr - data) / HTP_FENCE_SLOT_SIZE;
for (uint32_t i = 0; i < n_slots; i++) {
free_slots.push_back(slot + i);
}
@@ -636,6 +657,7 @@ inline ggml_tensor ggml_hexagon_session::make_fence_tensor(const volatile void *
static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) {
auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(buffer->context);
+ sbuf->sess->buffers.erase(sbuf);
sbuf->sess->unclone_buffer(sbuf);
delete sbuf;
}
@@ -975,31 +997,44 @@ static bool ggml_backend_hexagon_host_buffer_type_is_host(ggml_backend_buffer_ty
GGML_UNUSED(buft);
}
-struct ggml_backend_hexagon_alloc_buffer_n_plan_item {
+struct ggml_backend_hexagon_alloc_unit {
size_t size;
int first;
int last;
};
+struct ggml_backend_hexagon_alloc_buffer_n_plan_item {
+ size_t size;
+ std::vector<ggml_backend_hexagon_alloc_unit> units;
+};
+
using ggml_backend_hexagon_alloc_buffer_n_plan_t = std::vector<ggml_backend_hexagon_alloc_buffer_n_plan_item>;
-static const char * ggml_hexagon_kv_layer_suffix(const struct ggml_tensor * t) {
- if (strncmp(t->name, "cache_", 6) != 0) {
- return NULL;
- }
- const char * p = strstr(t->name, "_l");
- if (!p || !isdigit((unsigned char)p[2])) {
- return NULL;
+static bool ggml_hexagon_tensor_layer(const struct ggml_tensor * t, int & kind, int & layer) {
+ if (strncmp(t->name, "blk.", 4) == 0 && isdigit((unsigned char) t->name[4])) {
+ char * end = NULL;
+ long l = strtol(t->name + 4, &end, 10);
+ if (end && *end == '.') {
+ kind = 0;
+ layer = (int) l;
+ return true;
+ }
+ }
+ if (strncmp(t->name, "cache_", 6) == 0) {
+ const char * p = strstr(t->name, "_l");
+ if (p && isdigit((unsigned char) p[2])) {
+ char * end = NULL;
+ long l = strtol(p + 2, &end, 10);
+ if (end && *end == '\0') {
+ kind = 1;
+ layer = (int) l;
+ return true;
+ }
+ }
}
- return p;
+ return false;
}
-struct ggml_backend_hexagon_alloc_unit {
- size_t size;
- int first;
- int last;
-};
-
static ggml_backend_hexagon_alloc_buffer_n_plan_t ggml_backend_hexagon_alloc_buffer_n_plan(
ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) {
ggml_backend_hexagon_alloc_buffer_n_plan_t plan;
@@ -1012,61 +1047,72 @@ static ggml_backend_hexagon_alloc_buffer_n_plan_t ggml_backend_hexagon_alloc_buf
int i = 0;
while (i < n_tensors) {
struct ggml_tensor * t = tensors[i];
- size_t unit_size = 0;
- int unit_first = i;
- int unit_last = i + 1;
-
- if (t->data == NULL && t->view_src == NULL) {
- unit_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);
+ if (t->data != NULL || t->view_src != NULL) {
+ i++;
+ continue;
}
- const char * layer_suffix = ggml_hexagon_kv_layer_suffix(t);
+ size_t unit_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);
+ int unit_first = i;
+ int unit_last = i + 1;
+
+ int kind = -1, layer = -1;
+ const bool has_layer = ggml_hexagon_tensor_layer(t, kind, layer);
- while (unit_last < n_tensors) {
+ while (has_layer && unit_last < n_tensors) {
struct ggml_tensor * next = tensors[unit_last];
-
- if (next->view_src != NULL) {
+ if (next->data != NULL || next->view_src != NULL) {
unit_last++;
continue;
}
- if (layer_suffix != NULL) {
- const char * next_suffix = ggml_hexagon_kv_layer_suffix(next);
- if (next_suffix != NULL && strcmp(layer_suffix, next_suffix) == 0) {
- if (next->data == NULL) {
- unit_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, next), alignment);
- }
- unit_last++;
- continue;
- }
+ int next_kind = -1, next_layer = -1;
+ if (!ggml_hexagon_tensor_layer(next, next_kind, next_layer) ||
+ next_kind != kind || next_layer != layer) {
+ break;
}
- break;
+ size_t next_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, next), alignment);
+ if (unit_size + next_size > max_size) {
+ break;
+ }
+
+ unit_size += next_size;
+ unit_last++;
}
units.push_back({ unit_size, unit_first, unit_last });
i = unit_last;
}
- size_t cur_buf_size = 0;
- int cur_buf_first = 0;
-
for (const auto & unit : units) {
if (unit.size == 0) {
continue;
}
-
- if (cur_buf_size > 0 && (cur_buf_size + unit.size) > max_size) {
- plan.push_back({ cur_buf_size, cur_buf_first, unit.first });
- cur_buf_size = 0;
- cur_buf_first = unit.first;
+ bool placed = false;
+ for (auto & item : plan) {
+ if (item.size + unit.size <= max_size) {
+ item.size += unit.size;
+ item.units.push_back(unit);
+ placed = true;
+ break;
+ }
+ }
+ if (!placed) {
+ plan.push_back({ unit.size, { unit } });
}
-
- cur_buf_size += unit.size;
}
- if (cur_buf_size > 0) {
- plan.push_back({ cur_buf_size, cur_buf_first, n_tensors });
+ std::stable_sort(plan.begin(), plan.end(), [](const auto & a, const auto & b) {
+ return a.size > b.size;
+ });
+
+ if (!plan.empty()) {
+ for (const auto & unit : units) {
+ if (unit.size == 0) {
+ plan[0].units.push_back(unit);
+ }
+ }
}
return plan;
@@ -1092,28 +1138,19 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer_n(
struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);
struct ggml_tensor * t_failed = NULL;
- for (int j = item.first; j < item.last; j++) {
- struct ggml_tensor * t = tensors[j];
- if (t->data == NULL) {
- if (t->view_src == NULL) {
+ for (const auto & unit : item.units) {
+ for (int j = unit.first; j < unit.last; j++) {
+ struct ggml_tensor * t = tensors[j];
+ if (t->data == NULL && t->view_src == NULL) {
if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) {
t_failed = t;
break;
}
- } else if (t->buffer == NULL) {
- if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
- t_failed = t;
- break;
- }
- }
- } else {
- if (t->view_src != NULL && t->buffer == NULL) {
- if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
- t_failed = t;
- break;
- }
}
}
+ if (t_failed != NULL) {
+ break;
+ }
}
if (t_failed != NULL) {
GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t_failed->name);
@@ -1131,6 +1168,19 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer_n(
return NULL;
}
+ for (int j = 0; j < n_tensors; j++) {
+ struct ggml_tensor * t = tensors[j];
+ if (t->view_src != NULL && t->buffer == NULL) {
+ if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
+ GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t->name);
+ for (ggml_backend_buffer_t b : buffers) {
+ ggml_backend_buffer_free(b);
+ }
+ return NULL;
+ }
+ }
+ }
+
if (buffers.size() == 1) {
return buffers[0];
}
@@ -1200,7 +1250,6 @@ struct ggml_hexagon_opbatch {
ggml_hexagon_session* sess;
std::vector<htp_opnode> ops; // htp_opnode of ops
-
std::vector<htp_buf_desc> h_bufs; // htp buffer descriptors
std::vector<htp_tensor> h_tens; // htp tensor descriptors
std::vector<htp_op_desc> h_ops; // htp op descriptors
@@ -1529,45 +1578,45 @@ struct ggml_hexagon_registry {
struct ggml_hexagon_opqueue {
// Shared buffer for storing batches
ggml_hexagon_shared_buffer *shm_buf;
- size_t shm_blk_size;
+ uint8_t * data;
+ size_t slot_size;
size_t depth;
using opvec = std::vector<htp_opnode>;
std::vector<opvec> op_cache; // per batch op cache
- ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) : depth(depth) {
+ static size_t size_of(size_t batch_size, size_t n_threads, size_t depth = 1) {
size_t n_bufs = HTP_OP_MAX_BUFS;
size_t n_ops = batch_size;
size_t n_tensors = std::min<size_t>(n_ops * 4 + HTP_OP_MAX_INPUTS + HTP_OP_MAX_OUTPUTS, HTP_OP_MAX_TENSORS);
size_t tr_size = 0;
if (opt_profile == 3) {
- tr_size = sess->n_threads * opt_optrace * sizeof(htp_trace_desc);
+ tr_size = n_threads * opt_optrace * sizeof(htp_trace_desc);
}
- shm_blk_size = sizeof(htp_buf_desc) * n_bufs +
- sizeof(htp_tensor) * n_tensors +
- sizeof(htp_op_desc) * n_ops +
- sizeof(htp_prof_desc) * n_ops +
- tr_size;
-
- shm_buf = new ggml_hexagon_shared_buffer(sess, shm_blk_size * depth, true /* pinned */);
+ size_t slot_size = sizeof(htp_buf_desc) * n_bufs +
+ sizeof(htp_tensor) * n_tensors +
+ sizeof(htp_op_desc) * n_ops +
+ sizeof(htp_prof_desc) * n_ops +
+ tr_size;
+ return slot_size * depth;
+ }
+ ggml_hexagon_opqueue(ggml_hexagon_shared_buffer *shm_buf, size_t offset, size_t batch_size, size_t depth)
+ : shm_buf(shm_buf),
+ data(shm_buf->base() + offset),
+ slot_size(size_of(batch_size, shm_buf->sess->n_threads)),
+ depth(depth) {
op_cache.resize(depth);
if (opt_verbose) {
- GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n",
- sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size);
+ GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu slot-size %zu\n",
+ shm_buf->c_name(), batch_size, depth, size_of(batch_size, shm_buf->sess->n_threads, depth), slot_size);
}
}
- ~ggml_hexagon_opqueue() {
- delete shm_buf;
- }
-
- size_t shm_size() const { return shm_buf ? shm_buf->size() : 0; }
-
// push new batch
bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, const ggml_hexagon_opbatch* op_batch, uint64_t seq) {
static_assert(sizeof(htp_opbatch_req) % 8 == 0, "sizeof(htp_opbatch_req) must be multiple of 8");
@@ -1603,13 +1652,13 @@ struct ggml_hexagon_opqueue {
req.n_traces = 0;
}
- dbuf.ptr = shm_buf->base() + ((size_t) slot * shm_blk_size);
+ dbuf.ptr = data + ((size_t) slot * slot_size);
dbuf.fd = shm_buf->fd();
dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;
dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base();
dbuf.size = b_size + t_size + o_size + p_size + tr_size;
- GGML_ASSERT(dbuf.size <= shm_blk_size);
+ GGML_ASSERT(dbuf.size <= slot_size);
uint8_t * m_ptr = (uint8_t*) dbuf.ptr;
uint8_t * b_ptr = m_ptr; m_ptr += b_size;
@@ -1657,7 +1706,7 @@ struct ggml_hexagon_opqueue {
}
const size_t m_size = b_size + t_size + o_size + p_size + tr_size;
- GGML_ASSERT(m_size <= shm_blk_size);
+ GGML_ASSERT(m_size <= slot_size);
HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%llu : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n",
shm_buf->sess->c_name(), (unsigned long long) rsp.seq, rsp.n_bufs, rsp.n_tensors, rsp.n_ops,
@@ -1775,6 +1824,7 @@ void ggml_hexagon_session::flush_batch(size_t min_ops) {
if (op_batch->n_ops < min_ops) { return; }
op_batch->sort_buffers();
+ this->mmap_buffers();
htp_opbatch_req req {};
dspqueue_buffer dbuf{};
@@ -1819,10 +1869,17 @@ void ggml_hexagon_session::flush_batch(size_t min_ops) {
}
void ggml_hexagon_session::enqueue_op(const htp_opnode & node) {
- auto clone_tensor_buffer = [this](const ggml_tensor * t) {
- auto sbuf = this->mmap_tensor(t);
- if (!sbuf) return;
- if (sbuf->sess != this) {
+ auto register_tensor = [&](const ggml_tensor * t) {
+ if (!t) return;
+ auto * sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
+ if (ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
+ sbuf->extended = opt_dma64;
+ }
+ if (sbuf->sess == this) {
+ if (!sbuf->mapped) {
+ this->buffers.insert(sbuf);
+ }
+ } else {
this->clone_buffer(sbuf);
}
for (auto & sub : this->mdev.sessions) {
@@ -1830,11 +1887,11 @@ void ggml_hexagon_session::enqueue_op(const htp_opnode & node) {
}
};
- for (auto t : node.get_inputs()) {
- clone_tensor_buffer(t);
+ for (const auto * src : node.get_inputs()) {
+ register_tensor(src);
}
- for (auto t : node.get_outputs()) {
- clone_tensor_buffer(t);
+ for (const auto * dst : node.get_outputs()) {
+ register_tensor(dst);
}
const bool fits_tensors = op_batch->fit_tensors(node);
@@ -1945,46 +2002,61 @@ void ggml_hexagon_session::enqueue_allreduce(
this->enqueue_op(ar_node);
}
-ggml_hexagon_shared_buffer * ggml_hexagon_session::mmap_tensor(const ggml_tensor * t) {
- if (!t) return nullptr;
+void ggml_hexagon_session::mmap_buffers() {
+ auto collect_session = [](ggml_hexagon_session * sess, std::unordered_set<ggml_hexagon_shared_buffer *> & unmapped) {
+ for (auto * sbuf : sess->buffers) {
+ if (!sbuf->mapped) {
+ unmapped.insert(sbuf);
+ }
+ }
+ for (auto & [fd, clone] : sess->cloned_buffers) {
+ if (!clone->mapped) {
+ unmapped.insert(clone.get());
+ }
+ }
+ };
+
+ std::unordered_set<ggml_hexagon_shared_buffer *> unmapped;
+ collect_session(this, unmapped);
+ for (auto & sub : this->mdev.sessions) {
+ collect_session(sub.get(), unmapped);
+ }
- auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
- if (!sbuf->mapped) {
- const bool is_weight = ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS;
- const bool extended = opt_dma64 && is_weight;
- sbuf->mmap(extended);
+ if (!unmapped.empty()) {
+ std::vector<ggml_hexagon_shared_buffer *> sorted(unmapped.begin(), unmapped.end());
+ std::sort(sorted.begin(), sorted.end(), [](const auto * a, const auto * b) {
+ if (a->size() != b->size()) {
+ return a->size() > b->size();
+ }
+ if (a->fd() != b->fd()) {
+ return a->fd() < b->fd();
+ }
+ return a->sess->domain_id < b->sess->domain_id;
+ });
+ for (auto * sbuf : sorted) {
+ sbuf->mmap();
+ }
}
- return sbuf;
}
-bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf)
+void ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf)
{
GGML_ASSERT(sbuf && sbuf->mem);
- if (sbuf->sess == this) return true;
+ if (sbuf->sess == this) return;
auto mem = sbuf->mem;
int fd = mem->fd;
GGML_ASSERT(fd >= 0);
- if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return true;
-
- GGML_ASSERT(sbuf->mapped);
+ if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return;
HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(),
sbuf->c_name(), sbuf->base(), sbuf->size(), fd);
auto clone = std::make_unique<ggml_hexagon_shared_buffer>(this, *sbuf);
- try {
- clone->mmap(sbuf->extended);
- } catch (const std::exception & exc) {
- GGML_LOG_ERROR("ggml-hex: %s lazy mapping of buffer context failed: %s\n", this->c_name(), exc.what());
- return false;
- }
-
this->cloned_buffers[fd] = std::move(clone);
- mem->mapped_clones.insert(this);
- return true;
+ mem->clones.insert(this);
}
void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbuf) {
@@ -2000,14 +2072,14 @@ void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbu
auto clone = std::move(it->second);
this->cloned_buffers.erase(it);
}
- mem->mapped_clones.erase(this);
+ mem->clones.erase(this);
}
void ggml_hexagon_session::unclone_buffer(const ggml_hexagon_shared_buffer * sbuf) {
GGML_ASSERT(sbuf && sbuf->mem);
auto mem = sbuf->mem;
- std::vector<ggml_hexagon_session *> sessions(mem->mapped_clones.begin(), mem->mapped_clones.end());
+ std::vector<ggml_hexagon_session *> sessions(mem->clones.begin(), mem->clones.end());
for (auto * sess : sessions) {
sess->release_buffer(sbuf);
@@ -2276,9 +2348,14 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
}
// Allocate buffers and state for op batching
- this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue);
-
- this->fence_buf = new ggml_hexagon_fence_buffer(this, &dev_ctx->fence_buffer_type, 64 * 1024);
+ const size_t n_fences = 512;
+ const size_t opq_size = ggml_hexagon_opqueue::size_of(opt_opbatch, this->n_threads, opt_opqueue);
+ const size_t fence_offset = GGML_PAD(opq_size, HEX_L2_LINE_SIZE);
+ const size_t fence_size = ggml_hexagon_fence_buffer::size_of(n_fences);
+
+ this->shm_buf = new ggml_hexagon_shared_buffer(this, fence_offset + fence_size, true /* pinned */);
+ this->op_queue = new ggml_hexagon_opqueue(this->shm_buf, 0, opt_opbatch, opt_opqueue);
+ this->fence_buf = new ggml_hexagon_fence_buffer(this->shm_buf, &dev_ctx->fence_buffer_type, fence_offset, n_fences);
if (this->mdev.count > 1) {
this->mdev_fence_slot = this->alloc_fence(this->mdev.count);
}
@@ -2287,7 +2364,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
opt_vmem = ggml_hexagon_measure_max_vmem(this);
GGML_LOG_INFO("ggml-hex: %s measured max vmem %zu\n", this->c_name(), opt_vmem);
}
- const size_t shm_size = this->op_queue->shm_size();
+ const size_t shm_size = this->shm_buf->size();
this->max_vmem = (opt_vmem > shm_size) ? (opt_vmem - shm_size) : opt_vmem;
this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem);
@@ -2328,8 +2405,6 @@ void ggml_hexagon_session::release() noexcept(true) {
}
}
- delete this->op_batch;
- delete this->op_queue;
for (auto & it : this->cpy_fence_slots) {
free_fence((void *) it.second, 1);
}
@@ -2340,6 +2415,14 @@ void ggml_hexagon_session::release() noexcept(true) {
delete this->fence_buf;
this->fence_buf = nullptr;
}
+
+ delete this->op_batch;
+ delete this->op_queue;
+ delete this->shm_buf;
+ this->op_batch = nullptr;
+ this->op_queue = nullptr;
+ this->shm_buf = nullptr;
+
while (!this->cloned_buffers.empty()) {
release_buffer(this->cloned_buffers.begin()->second.get());
}
@@ -2734,19 +2817,12 @@ static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_
auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context);
auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context);
- sess_src->mmap_tensor(src);
- auto sbuf_dst = sess_dst->mmap_tensor(dst);
-
- if (!sess_src->clone_buffer(sbuf_dst)) { return false; }
-
const uint64_t src_key = ggml_hexagon_session_key(sess_src);
auto & fence_slot = sess_dst->cpy_fence_slots[src_key];
if (!fence_slot) {
fence_slot = (volatile uint32_t *) sess_dst->alloc_fence(1);
}
- if (!sess_src->clone_buffer(sess_dst->fence_buf)) { return false; }
-
uint32_t fence_seq = sess_dst->next_fence_seq();
HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq 0x%x\n",
@@ -2766,11 +2842,6 @@ static bool ggml_hexagon_cpy_tensor_async_virt(ggml_backend_t backend_src, ggml_
auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context);
auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context);
- auto sbuf_src = sess_src->mmap_tensor(src);
- sess_dst->mmap_tensor(dst);
-
- if (!sess_dst->clone_buffer(sbuf_src)) { return false; }
-
HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n",
sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src));
@@ -2893,7 +2964,6 @@ static void ggml_backend_hexagon_event_wait(ggml_backend_t backend, ggml_backend
return;
}
- sess->clone_buffer(hex_event->fence_sess->fence_buf);
sess->add_peer(hex_event->sess);
sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ true);
}
diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h
index 721fa2162..b6b608631 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ctx.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h
@@ -82,6 +82,7 @@ struct htp_ops_context {
struct htp_context {
dspqueue_t dsp_queue;
+ struct htp_mmap shm;
struct htp_mmap mmap[HTP_MAX_MMAPS];
dma_queue_t dma[HTP_MAX_NTHREADS];
struct htp_thread_trace trace[HTP_MAX_NTHREADS];
diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c
index ad457d1a6..c1df5a164 100644
--- a/ggml/src/ggml-hexagon/htp/main.c
+++ b/ggml/src/ggml-hexagon/htp/main.c
@@ -1089,7 +1089,12 @@ static int execute_op(struct htp_ops_context * octx) {
}
static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct htp_buf_desc *b) {
- b->base = NULL;
+ if (b->fd == ctx->shm.fd) {
+ b->base = ctx->shm.base;
+ return true;
+ }
+
+ b->base = 0;
for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {
struct htp_mmap *m = ctx->mmap + i;
@@ -1203,7 +1208,7 @@ static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uin
for (uint32_t i=0; i < n_bufs; i++) {
struct htp_buf_desc *b = bufs + i;
- if (!(b->flags & HTP_BUF_EXTENDED)) {
+ if (!(b->flags & HTP_BUF_EXTENDED) && b->fd != ctx->shm.fd) {
b->base = 0;
}
if (!mmap_buf(ctx, b)) {
@@ -1316,6 +1321,9 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
n_bufs, n_tens, n_ops, req->n_traces, dbuf->size, b_size, t_size, o_size);
// Setup descriptor pointers
+ ctx->shm.fd = dbuf->fd;
+ ctx->shm.base = (uintptr_t) dbuf->ptr - dbuf->offset;
+
uint8_t * m_ptr = dbuf->ptr;
struct htp_buf_desc* bufs = (struct htp_buf_desc*) m_ptr; m_ptr += b_size;
struct htp_tensor* tens = (struct htp_tensor*) m_ptr; m_ptr += t_size;