Commit e6ab7c1a4 for llama.cpp
commit e6ab7c1a41054a888ada952eab4c886444c2f5ad
Author: Max Krasnyansky <maxk@qti.qualcomm.com>
Date: Tue Sep 22 15:20:08 2026 -0700
hex-dma: introduce direct-mapped DMA cache that is better suited for HVX FA mask handling (#29282)
diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.h
index d256e6bef..c77359f89 100644
--- a/ggml/src/ggml-hexagon/htp/dma-queue.h
+++ b/ggml/src/ggml-hexagon/htp/dma-queue.h
@@ -428,15 +428,16 @@ static inline bool dma_queue_push(dma_queue *q, dma_data ddata, size_t dst_strid
#define DMA_CACHE_MAX_SIZE 256U
+// Fully assoc LRU cache
typedef struct {
uint8_t *base;
uint32_t line_size;
uint32_t capacity;
dma_addr_t src[DMA_CACHE_MAX_SIZE];
uint16_t age[DMA_CACHE_MAX_SIZE];
-} dma_cache;
+} dma_cache_fa;
-static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_size, uint32_t capacity)
+static inline void dma_cache_fa_init(dma_cache_fa *c, uint8_t *base, uint32_t line_size, uint32_t capacity)
{
c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity;
c->base = base;
@@ -448,7 +449,7 @@ static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_siz
}
}
-static inline bool dma_cache_push(dma_queue *q, dma_cache *c, dma_addr_t src_addr, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows)
+static inline bool dma_cache_fa_push(dma_queue *q, dma_cache_fa *c, dma_addr_t src_addr, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows)
{
uint32_t o_idx = 0;
uint16_t o_age = 0;
@@ -473,6 +474,40 @@ static inline bool dma_cache_push(dma_queue *q, dma_cache *c, dma_addr_t src_add
return dma_queue_push_single_1d(q, dma_make_data(dst, src_addr), 0);
}
+// Direct mapped cache
+typedef struct {
+ uint8_t *base;
+ uint32_t line_size;
+ uint32_t capacity;
+ uint32_t idx_mask;
+ dma_addr_t src[DMA_CACHE_MAX_SIZE];
+} dma_cache_dm;
+
+static inline void dma_cache_dm_init(dma_cache_dm *c, uint8_t *base, uint32_t line_size, uint32_t capacity)
+{
+ c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity;
+ c->idx_mask = c->capacity - 1;
+ c->base = base;
+ c->line_size = line_size;
+
+ for (unsigned i=0; i < c->capacity; i++) {
+ c->src[i] = 0;
+ }
+}
+
+static inline bool dma_cache_dm_push(dma_queue *q, dma_cache_dm *c, uint32_t slot, dma_addr_t src_addr, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows)
+{
+ const uint32_t i = slot & c->idx_mask;
+ uint8_t * dst = c->base + (i * c->line_size);
+
+ if (c->src[i] == src_addr) {
+ return dma_queue_push_single_1d(q, dma_make_data(dst, src_addr), 0); // dummy dma
+ }
+
+ c->src[i] = src_addr;
+ return dma_queue_push(q, dma_make_data(dst, src_addr), dst_stride, src_stride, row_size, nrows);
+}
+
#ifdef __cplusplus
} // extern "C"
#endif
diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c
index bfcf7cb0c..f079f7389 100644
--- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c
+++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c
@@ -160,7 +160,7 @@ struct hmx_fa_context {
size_t col_vec_bytes;
size_t d_tile_bytes;
bool mask_broadcast; // true when mask->ne[2] == 1 (head-independent, single 2D DMA)
- dma_cache m_cache;
+ dma_cache_fa m_cache;
};
static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * data) {
@@ -233,8 +233,8 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
uint8_t * spad_m = factx->spad_m + (mask ? factx->size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * ith;
uint8_t * spad_a = factx->spad_a + factx->size_vkq_acc * ith;
- dma_cache m_cache;
- dma_cache_init(&m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE);
+ dma_cache_dm m_cache;
+ dma_cache_dm_init(&m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE);
const size_t size_vkq_acc_single = hex_round_up(DV * sizeof(float), 128);
@@ -328,7 +328,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
// Mask
if (mask) {
const dma_addr_t m_src = mp_base + ic_start * sizeof(__fp16);
- dma_cache_push(dma_q, &m_cache, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1);
+ dma_cache_dm_push(dma_q, &m_cache, ib, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1);
}
}
@@ -537,7 +537,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
// Mask
if (mask) {
const dma_addr_t m_src = mp_base + next_ic_start * sizeof(__fp16);
- dma_cache_push(dma_q, &m_cache, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1);
+ dma_cache_dm_push(dma_q, &m_cache, next_ib, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1);
}
}
} // end for g
@@ -1796,7 +1796,7 @@ static inline void fa_prefetch_block(dma_queue * dma_q, const struct htp_tensor
if (mask) {
if (__builtin_expect(factx->mask_broadcast, true)) {
const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + prefetch_start * sizeof(__fp16);
- dma_cache_push(dma_q, &factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q);
+ dma_cache_fa_push(dma_q, &factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q);
} else {
fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, prefetch_start, kv_head, G, m_line_bytes, prefetch_rows, n_rows_q, factx);
}
@@ -1975,7 +1975,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
const size_t m_line_bytes = L.m_line_bytes; // used by the mask DMAs in the KV loop
- dma_cache_init(&factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE);
+ dma_cache_fa_init(&factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE);
// Head-dim padding: the K/V DMA staging buffers and the flat-Q buffer are laid out
// with padded row strides (size_{k,v,q}_row_padded, covering D_pad columns) but the
@@ -2057,7 +2057,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
if (factx.pipeline && mask) {
if (__builtin_expect(factx.mask_broadcast, true)) {
const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + 0;
- dma_cache_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), n_rows_q);
+ dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), n_rows_q);
} else {
fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, 0, kv_head, G, m_line_bytes, kv_rows0, n_rows_q, &factx);
}
@@ -2254,7 +2254,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
if (mask) {
if (__builtin_expect(factx.mask_broadcast, true)) {
const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + kv_start * sizeof(__fp16);
- dma_cache_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q);
+ dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q);
} else {
fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, kv_start, kv_head, G, m_line_bytes, kv_rows, n_rows_q, &factx);
}
@@ -2398,7 +2398,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
}
if (__builtin_expect(factx.mask_broadcast, true)) {
const dma_addr_t ms_src = mask->data + next_q_start * mask->nb[1] + next_im3 * mask->nb[3] + 0;
- dma_cache_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), next_n_rows_q);
+ dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), next_n_rows_q);
} else {
fa_push_mask_dma_gqa(dma_q, mask, next_q_start, next_im3, 0, next_kv_head, G, m_line_bytes, kv_rows0, next_n_rows_q, &factx);
}