Commit 8bb3cf0d2 for llama.cpp
commit 8bb3cf0d216a5ae48360179f2931daf284be4d01
Author: Max Krasnyansky <maxk@qti.qualcomm.com>
Date: Mon Oct 12 07:39:55 2026 -0700
hexagon: optimize CPY/CONCAT — round 3, short-rows edition (#30341)
* hexagon: move the converting CPY in blocks of rows per DMA descriptor so short rows are not DMA-latency bound
* hex-cpy: cleanup cpy cursor handling
* hex-cpy-concat: introduce threaded dma/hvx gather-row helper
Co-authored-by: Ruslan Karymov <67752306+karusrus@users.noreply.github.com>
* hex-cpy: remove leftover rejections for extended buffers
* hex-cpy: relax the gate for fast-path
* hex-cpy: remove scalar divs and vtcm access and use explicit gather_sync
* hex-cpy: add proper support for mdev work splitting to fastpath
* hex-cpy: optimize dense gather-row tasks
* hex-cpy: unroll gather-rows loops
* hex-cpy: add fastpath for f16
* hex-cpy-concat: support for colapsing 3/4D dims
* hex-cpy: double-buffer support
* hex-cpy: factor out gather-rows into more of an op by itself
* hex-cpy-concat: dedup kernel-params
* hex-cpy-concat: step-down nr to fit instead of hard fail
* hex-gather: replace div with a mask
* hex-gather-rows: cleanup dma pipeline and remove the stalls
* hex-gather: specialize compute funcs
* hex-gather: reduce scalar reg.spills and func calls
* hex-gather: do not recompute rows-per-thread on the npu
* hex-build: add support for building/installing just hexagon
* hex-cpy-concat: remove redundant barrier in concat and move cpy barrier under fence
* hex-gather: annotations for dma-out
* hex-mdev: make all key vtcm_acquire sync state atomic (fixes random hangs in mdev setups)
---------
Co-authored-by: Yiwei Shao <yiwei@aizip.ai>
Co-authored-by: Ruslan Karymov <67752306+karusrus@users.noreply.github.com>
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index 533f51a0e..a32ed5919 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -6593,6 +6593,37 @@ static bool ggml_hexagon_precompute_concat_params(
}
}
+ const uint32_t max_gather_ne0 = 128 / type_size;
+ const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);
+ const bool b_dense = (src1->nb[0] == type_size);
+ const bool gather_strides_ok =
+ src0->nb[1] >= (size_t) src0->ne[0] * type_size &&
+ (b_dense ? src1->nb[1] >= (size_t) src1->ne[0] * type_size
+ : src1->nb[0] >= (size_t) src1->ne[1] * type_size);
+
+ if (dim == 0 &&
+ (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16) &&
+ dst->ne[0] <= max_gather_ne0 &&
+ dst->nb[0] == type_size && dst->nb[1] == (size_t) dst->ne[0] * type_size &&
+ src0->nb[0] == type_size &&
+ (src1->nb[0] == type_size || src1->nb[1] == type_size) &&
+ gather_strides_ok &&
+ htp_gather_rows_tensor_outer_contiguous(dst) &&
+ htp_gather_rows_tensor_outer_contiguous(src0) &&
+ htp_gather_rows_tensor_outer_contiguous(src1)) {
+
+ if (htp_gather_rows_solve_layout(&kparams->u.gather,
+ (uint32_t) src0->ne[0], (uint32_t) src0->nb[1],
+ (uint32_t) src1->ne[0], (uint32_t) src1->nb[1], b_dense,
+ type_size, total_rows,
+ sess->n_threads, sess->mdev.count, sess->vtcm_size)) {
+ kparams->kernel_type = HTP_CONCAT_KERNEL_GATHER_ROWS;
+ kparams->n_threads = (uint8_t) sess->n_threads;
+ kparams->vtcm_size = kparams->u.gather.slice * sess->n_threads;
+ return true;
+ }
+ }
+
const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size);
if (dma_strides_ok) {
@@ -6616,11 +6647,11 @@ static bool ggml_hexagon_precompute_concat_params(
return false;
}
- kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED;
- kparams->n_threads = n_threads;
- kparams->vtcm_size = layout.total_bytes;
- kparams->spad0_size_per_thread = layout.src0_spad_size_per_thread;
- kparams->spad1_size_per_thread = layout.src1_spad_size_per_thread;
+ kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED;
+ kparams->n_threads = n_threads;
+ kparams->vtcm_size = layout.total_bytes;
+ kparams->u.transposed.spad0_size_per_thread = layout.src0_spad_size_per_thread;
+ kparams->u.transposed.spad1_size_per_thread = layout.src1_spad_size_per_thread;
return true;
}
@@ -6706,6 +6737,27 @@ static bool ggml_hexagon_precompute_cpy_params(
return true;
}
+ const uint32_t max_gather_ne0 = 128 / src_type_size;
+ const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
+ if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) &&
+ dst_is_contiguous && !src_is_contiguous &&
+ src0->nb[0] == src_type_size && src0->ne[0] <= max_gather_ne0 &&
+ src0->nb[1] >= (size_t) src0->ne[0] * src_type_size &&
+ total_rows > 1 &&
+ htp_gather_rows_tensor_outer_contiguous(src0)) {
+
+ if (htp_gather_rows_solve_layout(&kparams->u.gather,
+ (uint32_t) src0->ne[0], (uint32_t) src0->nb[1],
+ 0, 0, false,
+ src_type_size, total_rows,
+ sess->n_threads, sess->mdev.count, sess->vtcm_size)) {
+ kparams->kernel_type = HTP_COPY_KERNEL_GATHER_ROWS;
+ kparams->n_threads = (uint8_t) sess->n_threads;
+ kparams->vtcm_size = kparams->u.gather.slice * sess->n_threads;
+ return true;
+ }
+ }
+
if (sameshape) {
kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE;
kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
@@ -6737,7 +6789,7 @@ static bool ggml_hexagon_precompute_cpy_params(
const uint32_t n_threads = sess->n_threads > 0 ? (uint32_t) sess->n_threads : 4;
struct htp_copy_convert_vtcm_layout layout;
- htp_copy_convert_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], src_type_size, dst_type_size, n_threads);
+ htp_copy_convert_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1], src_type_size, dst_type_size, n_threads);
if (sess->vtcm_size > 0 && layout.total_bytes > sess->vtcm_size) {
return false;
@@ -6751,6 +6803,9 @@ static bool ggml_hexagon_precompute_cpy_params(
kparams->u.convert.dst_buf_size = layout.dst_buf_size;
kparams->u.convert.spad0_size_per_thread = layout.spad0_size_per_thread;
kparams->u.convert.spad1_size_per_thread = layout.spad1_size_per_thread;
+ kparams->u.convert.src0_row_stride = layout.src0_row_stride;
+ kparams->u.convert.dst_row_stride = layout.dst_row_stride;
+ kparams->u.convert.blk_rows = layout.blk_rows;
kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);
kparams->u.convert.div_ne02_ne01 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1]));
diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt
index 787badf07..bc1474ae8 100644
--- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt
+++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt
@@ -33,6 +33,7 @@ add_library(${HTP_LIB} SHARED
set-rows-ops.c
get-rows-ops.c
cpy-ops.c
+ gather-rows.c
repeat-ops.c
ssm-conv.c
cumsum-ops.c
diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c
index f7dda1408..40fa09151 100644
--- a/ggml/src/ggml-hexagon/htp/concat-ops.c
+++ b/ggml/src/ggml-hexagon/htp/concat-ops.c
@@ -13,6 +13,7 @@
#include "htp-vtcm.h"
#include "hvx-utils.h"
#include "hvx_hexagon_protos.h"
+#include "gather-rows.h"
#include <string.h>
@@ -302,9 +303,9 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con
struct htp_concat_context cctx;
cctx.octx = octx;
cctx.spad0_base = vtcm_base;
- cctx.spad1_base = vtcm_base + n_threads * kparams->spad0_size_per_thread;
- cctx.spad0_size_per_thread = kparams->spad0_size_per_thread;
- cctx.spad1_size_per_thread = kparams->spad1_size_per_thread;
+ cctx.spad1_base = vtcm_base + n_threads * kparams->u.transposed.spad0_size_per_thread;
+ cctx.spad0_size_per_thread = kparams->u.transposed.spad0_size_per_thread;
+ cctx.spad1_size_per_thread = kparams->u.transposed.spad1_size_per_thread;
cctx.row_start = row_start;
cctx.nrows = nrows;
cctx.nplanes = dst->ne[2] * dst->ne[3];
@@ -318,10 +319,13 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con
int op_concat(struct htp_ops_context * octx) {
const struct htp_concat_kernel_params * kparams = (const struct htp_concat_kernel_params *) octx->kernel_params;
- const struct htp_tensor * dst = octx->dst;
- const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;
+ const struct htp_tensor * src0 = octx->src[0];
+ const struct htp_tensor * src1 = octx->src[1];
+ const struct htp_tensor * dst = octx->dst;
int status = HTP_STATUS_OK;
+ const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;
+
switch (kparams->kernel_type) {
case HTP_CONCAT_KERNEL_REGULAR:
status = concat_regular(octx, kparams->dim, type_size);
@@ -331,6 +335,10 @@ int op_concat(struct htp_ops_context * octx) {
status = concat_transposed(octx, kparams, type_size);
break;
+ case HTP_CONCAT_KERNEL_GATHER_ROWS:
+ status = htp_gather_rows(octx, src0, src1, &kparams->u.gather, kparams->n_threads);
+ break;
+
default:
status = HTP_STATUS_NO_SUPPORT;
break;
@@ -338,9 +346,5 @@ int op_concat(struct htp_ops_context * octx) {
htp_ops_context_set_status(octx, status);
- if (octx->ctx->mdev.count > 1) {
- htp_mdev_group_barrier(octx);
- }
-
return octx->status;
}
diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.h b/ggml/src/ggml-hexagon/htp/concat-ops.h
index 1d7218441..4bfba9894 100644
--- a/ggml/src/ggml-hexagon/htp/concat-ops.h
+++ b/ggml/src/ggml-hexagon/htp/concat-ops.h
@@ -2,12 +2,19 @@
#define HTP_CONCAT_OPS_H
#include "hex-common.h"
+#include "gather-rows.h"
#include <stdint.h>
enum htp_concat_kernel_type {
HTP_CONCAT_KERNEL_UNSUPPORTED = 0,
HTP_CONCAT_KERNEL_REGULAR = 1,
HTP_CONCAT_KERNEL_TRANSPOSED = 2,
+ HTP_CONCAT_KERNEL_GATHER_ROWS = 3,
+};
+
+struct htp_concat_transposed_params {
+ uint32_t spad0_size_per_thread;
+ uint32_t spad1_size_per_thread;
};
struct htp_concat_kernel_params {
@@ -17,8 +24,11 @@ struct htp_concat_kernel_params {
uint8_t pad;
uint32_t vtcm_size;
- uint32_t spad0_size_per_thread;
- uint32_t spad1_size_per_thread;
+
+ union {
+ struct htp_concat_transposed_params transposed;
+ struct htp_gather_rows_params gather;
+ } u;
};
#if defined(__cplusplus)
diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c
index d366acb75..63ebb38e6 100644
--- a/ggml/src/ggml-hexagon/htp/cpy-ops.c
+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c
@@ -18,6 +18,7 @@
#include "htp-ops.h"
#include "htp-tensor.h"
#include "hvx-utils.h"
+#include "gather-rows.h"
struct htp_copy_context {
struct htp_ops_context * octx;
@@ -141,6 +142,25 @@ DEFINE_CPY_RESHAPE(f32, float, 4)
DEFINE_CPY_RESHAPE(f16, __fp16, 2)
DEFINE_CPY_RESHAPE(i32, int32_t, 4)
+struct cpy_row_cursor {
+ uint32_t r;
+ uint32_t i01;
+ uint32_t i02;
+ uint32_t i03;
+};
+
+static inline void cpy_cursor_advance(struct cpy_row_cursor * c, uint32_t n, uint32_t ne01, uint32_t ne02) {
+ c->r += n;
+ c->i01 += n;
+ if (c->i01 == ne01) {
+ c->i01 = 0;
+ if (++c->i02 == ne02) {
+ c->i02 = 0;
+ c->i03++;
+ }
+ }
+}
+
#define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC) \
static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
@@ -151,7 +171,6 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
const uint32_t ir0 = ct->row_start + dr * ith; \
const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \
if (ir0 >= ir1) return; \
- const uint32_t nrows_thread = ir1 - ir0; \
\
dma_queue * dma_q = octx->ctx->dma[ith]; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
@@ -159,6 +178,9 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
const struct htp_copy_convert_params * cvt = &ct->kparams->u.convert; \
const uint32_t src0_buf_size = cvt->src0_buf_size; \
const uint32_t dst_buf_size = cvt->dst_buf_size; \
+ const uint32_t src0_row_stride = cvt->src0_row_stride; \
+ const uint32_t dst_row_stride = cvt->dst_row_stride; \
+ const uint32_t blk_rows = cvt->blk_rows; \
uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * cvt->spad0_size_per_thread; \
uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * cvt->spad1_size_per_thread; \
const uint32_t src0_row_size = ne00 * ct->kparams->src0_type_size; \
@@ -170,62 +192,43 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
uint32_t i02 = fastdiv(rem, &cvt->div_ne01); \
uint32_t i01 = rem - i02 * ne01; \
\
- uint32_t f_i01 = i01, f_i02 = i02, f_i03 = i03; \
- dma_addr_t f_src0_addr = src0->data + f_i01*nb01 + f_i02*nb02 + f_i03*nb03; \
+ struct cpy_row_cursor f = { ir0, i01, i02, i03 }; \
+ struct cpy_row_cursor c = f; \
\
- uint32_t c_i01 = i01, c_i02 = i02, c_i03 = i03; \
- dma_addr_t c_dst_addr = dst->data + c_i01*nb1 + c_i02*nb2 + c_i03*nb3; \
- \
- for (uint32_t r = 0; r < nrows_thread && r < 2; ++r) { \
- uint8_t * src_spad = vtcm_src0_base + r * src0_buf_size; \
- uint8_t * dst_spad = vtcm_dst_base + r * dst_buf_size; \
+ for (uint32_t b = 0; b < 2 && f.r < ir1; ++b) { \
+ uint8_t * src_spad = vtcm_src0_base + b * src0_buf_size; \
+ uint8_t * dst_spad = vtcm_dst_base + b * dst_buf_size; \
+ const uint32_t nr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \
+ dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \
dma_queue_push(dma_q, dma_make_data(dst->data, dst_spad), \
dst_row_size, dst_buf_size, dst_row_size, 0); \
dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \
- src0_buf_size, src0_row_size, src0_row_size, 1); \
- f_src0_addr += nb01; \
- if (++f_i01 == ne01) { \
- f_i01 = 0; \
- if (++f_i02 == ne02) { \
- f_i02 = 0; \
- f_i03++; \
- } \
- f_src0_addr = src0->data + f_i02*nb02 + f_i03*nb03; \
- } \
+ src0_row_stride, nb01, src0_row_size, nr); \
+ cpy_cursor_advance(&f, nr, ne01, ne02); \
} \
\
- for (uint32_t r = 0; r < nrows_thread; ++r) { \
+ for (uint32_t blk = 0; c.r < ir1; ++blk) { \
+ const uint32_t nr = MIN(blk_rows, MIN(ne01 - c.i01, ir1 - c.r)); \
uint8_t * dst_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).src; \
uint8_t * src_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; \
\
- htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
- CONV_FUNC(dst_spad, src_spad, ne00); \
- htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \
+ for (uint32_t r = 0; r < nr; ++r) { \
+ CONV_FUNC(dst_spad + r * dst_row_stride, src_spad + r * src0_row_stride, ne00); \
+ } \
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \
\
+ dma_addr_t c_dst_addr = dst->data + c.i01*nb1 + c.i02*nb2 + c.i03*nb3; \
dma_queue_push(dma_q, dma_make_data(c_dst_addr, dst_spad), \
- dst_row_size, dst_buf_size, dst_row_size, 1); \
- c_dst_addr += nb1; \
- if (++c_i01 == ne01) { \
- c_i01 = 0; \
- if (++c_i02 == ne02) { \
- c_i02 = 0; \
- c_i03++; \
- } \
- c_dst_addr = dst->data + c_i02*nb2 + c_i03*nb3; \
- } \
+ nb1, dst_row_stride, dst_row_size, nr); \
+ cpy_cursor_advance(&c, nr, ne01, ne02); \
\
- if (r + 2 < nrows_thread) { \
+ if (f.r < ir1) { \
+ const uint32_t fnr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \
+ dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \
dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \
- src0_buf_size, src0_row_size, src0_row_size, 1); \
- f_src0_addr += nb01; \
- if (++f_i01 == ne01) { \
- f_i01 = 0; \
- if (++f_i02 == ne02) { \
- f_i02 = 0; \
- f_i03++; \
- } \
- f_src0_addr = src0->data + f_i02*nb02 + f_i03*nb03; \
- } \
+ src0_row_stride, nb01, src0_row_size, fnr); \
+ cpy_cursor_advance(&f, fnr, ne01, ne02); \
} \
} \
dma_queue_flush(dma_q); \
@@ -337,10 +340,6 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
- if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {
- return HTP_STATUS_NO_SUPPORT;
- }
-
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -398,10 +397,6 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
- if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {
- return HTP_STATUS_NO_SUPPORT;
- }
-
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -461,6 +456,9 @@ int op_cpy(struct htp_ops_context * octx) {
case HTP_COPY_KERNEL_RESHAPE:
status = cpy_reshape(octx, kparams);
break;
+ case HTP_COPY_KERNEL_GATHER_ROWS:
+ status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_threads);
+ break;
default:
status = HTP_STATUS_NO_SUPPORT;
break;
@@ -468,11 +466,11 @@ int op_cpy(struct htp_ops_context * octx) {
htp_ops_context_set_status(octx, status);
- if (octx->ctx->mdev.count > 1) {
- htp_mdev_group_barrier(octx);
- }
-
if (octx->op == HTP_OP_CPY_FENCE) {
+ if (octx->ctx->mdev.count > 1) {
+ htp_mdev_group_barrier(octx);
+ }
+
if (octx->ctx->mdev.idx == 0) {
const struct htp_tensor * sync = octx->src[1];
const uint32_t seq = (uint32_t) octx->op_params[0];
diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.h b/ggml/src/ggml-hexagon/htp/cpy-ops.h
index 4cf7dc13b..16695c2ab 100644
--- a/ggml/src/ggml-hexagon/htp/cpy-ops.h
+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.h
@@ -3,6 +3,7 @@
#include "hex-common.h"
#include "hex-fastdiv.h"
+#include "gather-rows.h"
#include <stdint.h>
enum htp_copy_kernel_type {
@@ -12,6 +13,7 @@ enum htp_copy_kernel_type {
HTP_COPY_KERNEL_SAMESHAPE_CONVERT = 3,
HTP_COPY_KERNEL_RESHAPE = 4,
HTP_COPY_KERNEL_SCALAR = 5,
+ HTP_COPY_KERNEL_GATHER_ROWS = 6,
};
struct htp_copy_convert_params {
@@ -19,6 +21,9 @@ struct htp_copy_convert_params {
uint32_t dst_buf_size;
uint32_t spad0_size_per_thread;
uint32_t spad1_size_per_thread;
+ uint32_t src0_row_stride; // row pitch in the VTCM buffers
+ uint32_t dst_row_stride;
+ uint32_t blk_rows; // rows moved per DMA descriptor
struct fastdiv_values div_ne01;
struct fastdiv_values div_ne02_ne01;
};
@@ -45,10 +50,19 @@ struct htp_copy_kernel_params {
union {
struct htp_copy_convert_params convert;
struct htp_copy_reshape_params reshape;
+ struct htp_gather_rows_params gather;
} u;
};
+// Size of one VTCM staging buffer: big enough that a DMA descriptor moves many short
+// rows at once. The copy kernels keep four per thread (two in flight each way), about
+// 1 MB of VTCM with 8 threads.
+#define HTP_COPY_VTCM_BLK_BYTES (32 * 1024)
+
struct htp_copy_convert_vtcm_layout {
+ uint32_t src0_row_stride;
+ uint32_t dst_row_stride;
+ uint32_t blk_rows;
uint32_t src0_buf_size;
uint32_t dst_buf_size;
uint32_t spad0_size_per_thread;
@@ -56,15 +70,30 @@ struct htp_copy_convert_vtcm_layout {
uint32_t total_bytes;
};
+// ne01 bounds a block: rows of one block share the dim-1 stride.
static inline void htp_copy_convert_vtcm_layout_build(
struct htp_copy_convert_vtcm_layout * layout,
uint32_t ne00,
+ uint32_t ne01,
uint32_t src_type_size,
uint32_t dst_type_size,
uint32_t n_threads) {
- layout->src0_buf_size = hex_round_up(ne00 * src_type_size, 256);
- layout->dst_buf_size = hex_round_up(ne00 * dst_type_size, 256);
+ layout->src0_row_stride = hex_round_up(ne00 * src_type_size, 256);
+ layout->dst_row_stride = hex_round_up(ne00 * dst_type_size, 256);
+
+ const uint32_t max_row = layout->src0_row_stride > layout->dst_row_stride ? layout->src0_row_stride : layout->dst_row_stride;
+ uint32_t blk_rows = HTP_COPY_VTCM_BLK_BYTES / max_row;
+ if (blk_rows > ne01) {
+ blk_rows = ne01;
+ }
+ if (blk_rows == 0) {
+ blk_rows = 1;
+ }
+ layout->blk_rows = blk_rows;
+
+ layout->src0_buf_size = blk_rows * layout->src0_row_stride;
+ layout->dst_buf_size = blk_rows * layout->dst_row_stride;
layout->spad0_size_per_thread = 2 * layout->src0_buf_size;
layout->spad1_size_per_thread = 2 * layout->dst_buf_size;
layout->total_bytes = n_threads * (layout->spad0_size_per_thread + layout->spad1_size_per_thread);
diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.c b/ggml/src/ggml-hexagon/htp/gather-rows.c
new file mode 100644
index 000000000..1db48e59c
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/gather-rows.c
@@ -0,0 +1,387 @@
+#include "gather-rows.h"
+
+#include <assert.h>
+
+#include "hex-common.h"
+#include "hex-profile.h"
+#include "hvx-utils.h"
+#include "dma-queue.h"
+#include "htp-ctx.h"
+#include "htp-ops.h"
+#include "htp-tensor.h"
+#include "work-queue.h"
+
+typedef void (*gather_rows_compute_fn_t)(
+ uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
+ const struct htp_gather_rows_params * restrict kparams,
+ const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur);
+
+struct gather_rows_task {
+ struct htp_ops_context * octx;
+ const struct htp_tensor * a;
+ const struct htp_tensor * b;
+ dma_addr_t dst;
+ uint32_t r_start;
+ uint32_t rows;
+ uint32_t rows_per_thread;
+ const struct htp_gather_rows_params * kparams;
+ gather_rows_compute_fn_t compute;
+};
+
+static inline __attribute__((always_inline)) void gather_rows_dma_in(
+ dma_queue * q, const struct gather_rows_task * task,
+ uint8_t * region, uint32_t r, uint32_t nr, uint32_t elem_size) {
+ const struct htp_tensor * a = task->a;
+ const struct htp_tensor * b = task->b;
+ const struct htp_gather_rows_params * kparams = task->kparams;
+
+ const size_t a_size = (nr - 1) * a->nb[1] + kparams->na * elem_size;
+ dma_queue_push(q, dma_make_data(region, a->data + r * a->nb[1]), a_size, a_size, a_size, 1);
+ if (b) {
+ if (kparams->b_dense) {
+ const size_t b_size = (nr - 1) * b->nb[1] + kparams->nb * elem_size;
+ dma_queue_push(q, dma_make_data(region + kparams->v1, b->data + r * b->nb[1]), b_size, b_size, b_size, 1);
+ } else {
+ const size_t b_size = (nr - 1) * b->nb[1] + elem_size;
+ for (uint32_t k = 0; k < kparams->nb; k++) {
+ dma_queue_push(q, dma_make_data(region + kparams->v1 + k * kparams->span1,
+ b->data + k * b->nb[0] + r * b->nb[1]), b_size, b_size, b_size, 1);
+ }
+ }
+ }
+}
+
+static inline void gather_rows_dma_out(dma_queue * q, const struct gather_rows_task * task,
+ uint8_t * out, uint32_t r, uint32_t nr, uint32_t elem_size) {
+ const struct htp_gather_rows_params * kparams = task->kparams;
+ const size_t out_size = nr * kparams->ne * elem_size;
+ dma_queue_push(q, dma_make_data(task->dst + r * kparams->ne * elem_size, out), out_size, out_size, out_size, 1);
+}
+
+static __attribute__((noinline)) void gather_rows_init_tables_f32(
+ HVX_Vector * tab, HVX_Vector * inc,
+ const struct htp_gather_rows_params * kparams,
+ const struct htp_tensor * a, const struct htp_tensor * b) {
+
+ const int32_t s_a = kparams->rg * a->nb[1];
+ const int32_t s_b = b ? kparams->rg * b->nb[1] : 0;
+ const uint32_t b_nb1 = b ? b->nb[1] : 0;
+ uint32_t row = 0;
+ uint32_t col = 0;
+ int32_t t_buf[32] __attribute__((aligned(128)));
+ int32_t s_buf[32] __attribute__((aligned(128)));
+ for (uint32_t k = 0; k < kparams->vg; k++) {
+ for (uint32_t lane = 0; lane < 32; lane++) {
+ if (col < kparams->na) {
+ t_buf[lane] = row * a->nb[1] + col * 4;
+ s_buf[lane] = s_a;
+ } else {
+ t_buf[lane] = kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1;
+ s_buf[lane] = s_b;
+ }
+ if (++col == kparams->ne) {
+ col = 0;
+ row++;
+ }
+ }
+ tab[k] = *(const HVX_Vector *) t_buf;
+ inc[k] = *(const HVX_Vector *) s_buf;
+ }
+}
+
+static __attribute__((noinline)) void gather_rows_init_tables_f16(
+ HVX_Vector * tab, HVX_Vector * inc,
+ const struct htp_gather_rows_params * kparams,
+ const struct htp_tensor * a, const struct htp_tensor * b) {
+
+ const int16_t s_a = (int16_t) (kparams->rg * a->nb[1]);
+ const int16_t s_b = (int16_t) (b ? kparams->rg * b->nb[1] : 0);
+ const uint32_t b_nb1 = b ? b->nb[1] : 0;
+ uint32_t row = 0;
+ uint32_t col = 0;
+ int16_t t_buf[64] __attribute__((aligned(128)));
+ int16_t s_buf[64] __attribute__((aligned(128)));
+ for (uint32_t k = 0; k < kparams->vg; k++) {
+ for (uint32_t lane = 0; lane < 64; lane++) {
+ if (col < kparams->na) {
+ t_buf[lane] = (int16_t) (row * a->nb[1] + col * 2);
+ s_buf[lane] = s_a;
+ } else {
+ t_buf[lane] = (int16_t) (kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1);
+ s_buf[lane] = s_b;
+ }
+ if (++col == kparams->ne) {
+ col = 0;
+ row++;
+ }
+ }
+ tab[k] = *(const HVX_Vector *) t_buf;
+ inc[k] = *(const HVX_Vector *) s_buf;
+ }
+}
+
+static void gather_rows_compute_f32_vg1(
+ uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
+ const struct htp_gather_rows_params * restrict kparams,
+ const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
+ (void) cur;
+ const uint32_t region_size = kparams->region_size;
+ HVX_Vector vcur = tab[0];
+ const HVX_Vector vinc = inc[0];
+ #pragma unroll(4)
+ for (uint32_t v = 0; v < nvec; v++) {
+ Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur);
+ vcur = Q6_Vw_vadd_VwVw(vcur, vinc);
+ }
+}
+
+static void gather_rows_compute_f32_vgn(
+ uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
+ const struct htp_gather_rows_params * restrict kparams,
+ const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
+ const uint32_t vg = kparams->vg;
+ const uint32_t region_size = kparams->region_size;
+ for (uint32_t k = 0; k < vg; k++) {
+ cur[k] = tab[k];
+ }
+ uint32_t k = 0;
+ #pragma unroll(4)
+ for (uint32_t v = 0; v < nvec; v++) {
+ Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]);
+ cur[k] = Q6_Vw_vadd_VwVw(cur[k], inc[k]);
+ if (++k == vg) {
+ k = 0;
+ }
+ }
+}
+
+static void gather_rows_compute_f16_vg1(
+ uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
+ const struct htp_gather_rows_params * restrict kparams,
+ const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
+ (void) cur;
+ const uint32_t region_size = kparams->region_size;
+ HVX_Vector vcur = tab[0];
+ const HVX_Vector vinc = inc[0];
+ #pragma unroll(4)
+ for (uint32_t v = 0; v < nvec; v++) {
+ Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur);
+ vcur = Q6_Vh_vadd_VhVh(vcur, vinc);
+ }
+}
+
+static void gather_rows_compute_f16_vgn(
+ uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
+ const struct htp_gather_rows_params * restrict kparams,
+ const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
+ const uint32_t vg = kparams->vg;
+ const uint32_t region_size = kparams->region_size;
+ for (uint32_t k = 0; k < vg; k++) {
+ cur[k] = tab[k];
+ }
+ uint32_t k = 0;
+ #pragma unroll(4)
+ for (uint32_t v = 0; v < nvec; v++) {
+ Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]);
+ cur[k] = Q6_Vh_vadd_VhVh(cur[k], inc[k]);
+ if (++k == vg) {
+ k = 0;
+ }
+ }
+}
+
+static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * data) {
+ (void) nth;
+ const struct gather_rows_task * task = (const struct gather_rows_task *) data;
+ const struct htp_gather_rows_params * kparams = task->kparams;
+ const struct htp_tensor * a = task->a;
+ const struct htp_tensor * b = task->b;
+
+ const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows);
+ const uint32_t r_beg = task->r_start + r_offset;
+ const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows);
+ if (r_beg >= r_end) {
+ return;
+ }
+
+ struct htp_thread_trace * tr = &task->octx->ctx->trace[ith];
+
+ uint8_t * base = task->octx->ctx->vtcm_base + ith * kparams->slice;
+ HVX_Vector * tab = (HVX_Vector *) base;
+ HVX_Vector * inc = tab + kparams->vg;
+ HVX_Vector * cur = inc + kparams->vg;
+ uint8_t * region_base = (uint8_t *) (cur + kparams->vg);
+
+ uint8_t * region[2] = { region_base, region_base + kparams->buf_stride };
+ uint8_t * out[2] = { region[0] + kparams->out_off, region[1] + kparams->out_off };
+
+ htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
+ gather_rows_init_tables_f32(tab, inc, kparams, a, b);
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
+
+ dma_queue * q = task->octx->ctx->dma[ith];
+ const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0);
+ const uint32_t nr_max = kparams->nr_max;
+ const uint32_t nr0 = MIN(nr_max, r_end - r_beg);
+
+ dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out
+ gather_rows_dma_in(q, task, region[0], r_beg, nr0, 4);
+
+ const uint32_t r1 = r_beg + nr_max;
+ if (r1 < r_end) {
+ const uint32_t nr1 = MIN(nr_max, r_end - r1);
+ dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out
+ gather_rows_dma_in(q, task, region[1], r1, nr1, 4);
+ }
+
+ uint32_t buf = 0;
+ for (uint32_t r = r_beg; r < r_end; r += nr_max) {
+ const uint32_t nr = MIN(nr_max, r_end - r);
+ const uint32_t next_buf = buf ^ 1;
+
+ dma_queue_pop(q); // complete out
+ for (uint32_t i = 0; i < n_in; i++) {
+ dma_queue_pop(q);
+ }
+
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
+ const uint32_t nvec = (nr * kparams->ne + 31) >> 5;
+ task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur);
+ hvx_gather_sync(out[buf]);
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
+
+ gather_rows_dma_out(q, task, out[buf], r, nr, 4);
+
+ const uint32_t r_prefetch = r + 2 * nr_max;
+ if (r_prefetch < r_end) {
+ const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch);
+ gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 4);
+ }
+
+ buf = next_buf;
+ }
+ dma_queue_flush(q);
+}
+
+static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * data) {
+ (void) nth;
+ const struct gather_rows_task * task = (const struct gather_rows_task *) data;
+ const struct htp_gather_rows_params * kparams = task->kparams;
+ const struct htp_tensor * a = task->a;
+ const struct htp_tensor * b = task->b;
+
+ const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows);
+ const uint32_t r_beg = task->r_start + r_offset;
+ const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows);
+ if (r_beg >= r_end) {
+ return;
+ }
+
+ struct htp_thread_trace * tr = &task->octx->ctx->trace[ith];
+
+ uint8_t * base = task->octx->ctx->vtcm_base + ith * kparams->slice;
+ HVX_Vector * tab = (HVX_Vector *) base;
+ HVX_Vector * inc = tab + kparams->vg;
+ HVX_Vector * cur = inc + kparams->vg;
+ uint8_t * region_base = (uint8_t *) (cur + kparams->vg);
+
+ uint8_t * region[2] = { region_base, region_base + kparams->buf_stride };
+ uint8_t * out[2] = { region[0] + kparams->out_off, region[1] + kparams->out_off };
+
+ htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
+ gather_rows_init_tables_f16(tab, inc, kparams, a, b);
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
+
+ dma_queue * q = task->octx->ctx->dma[ith];
+ const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0);
+
+ const uint32_t nr_max = kparams->nr_max;
+ const uint32_t nr0 = MIN(nr_max, r_end - r_beg);
+ dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out
+ gather_rows_dma_in(q, task, region[0], r_beg, nr0, 2);
+
+ const uint32_t r1 = r_beg + nr_max;
+ if (r1 < r_end) {
+ const uint32_t nr1 = MIN(nr_max, r_end - r1);
+ dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out
+ gather_rows_dma_in(q, task, region[1], r1, nr1, 2);
+ }
+
+ uint32_t buf = 0;
+ for (uint32_t r = r_beg; r < r_end; r += nr_max) {
+ const uint32_t nr = MIN(nr_max, r_end - r);
+ const uint32_t next_buf = buf ^ 1;
+
+ dma_queue_pop(q); // complete out
+ for (uint32_t i = 0; i < n_in; i++) {
+ dma_queue_pop(q);
+ }
+
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
+ const uint32_t nvec = (nr * kparams->ne + 63) >> 6;
+ task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur);
+ hvx_gather_sync(out[buf]);
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
+
+ gather_rows_dma_out(q, task, out[buf], r, nr, 2);
+
+ const uint32_t r_prefetch = r + 2 * nr_max;
+ if (r_prefetch < r_end) {
+ const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch);
+ gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 2);
+ }
+
+ buf = next_buf;
+ }
+ dma_queue_flush(q);
+}
+
+int htp_gather_rows(
+ struct htp_ops_context * octx,
+ const struct htp_tensor * a,
+ const struct htp_tensor * b,
+ const struct htp_gather_rows_params * kparams,
+ uint32_t n_threads) {
+
+ if (!htp_ops_context_set_n_threads(octx, n_threads)) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
+ assert(kparams->slice * octx->n_threads <= octx->ctx->vtcm_size);
+
+ const uint32_t total_rows = kparams->total_rows;
+ uint32_t r_start = 0;
+ uint32_t r_count = total_rows;
+ if (octx->ctx->mdev.count > 1) {
+ const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
+ total_rows, kparams->rg, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
+ r_start = range.start;
+ r_count = range.count;
+ }
+
+ if (r_count == 0) {
+ return HTP_STATUS_OK;
+ }
+
+ const uint32_t elem_size = (a->type == HTP_TYPE_F16) ? 2 : 4;
+
+ struct gather_rows_task task;
+ task.octx = octx;
+ task.a = a;
+ task.b = b;
+ task.dst = octx->dst->data;
+ task.r_start = r_start;
+ task.rows = r_count;
+ task.kparams = kparams;
+ task.rows_per_thread = kparams->rows_per_thread;
+
+ if (elem_size == 2) {
+ task.compute = (kparams->vg == 1) ? gather_rows_compute_f16_vg1 : gather_rows_compute_f16_vgn;
+ } else {
+ task.compute = (kparams->vg == 1) ? gather_rows_compute_f32_vg1 : gather_rows_compute_f32_vgn;
+ }
+
+ work_queue_func_t worker_func = (elem_size == 2) ? gather_rows_thread_f16 : gather_rows_thread_f32;
+ work_queue_run(octx->ctx->work_queue, worker_func, &task, octx->n_threads);
+ return HTP_STATUS_OK;
+}
diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.h b/ggml/src/ggml-hexagon/htp/gather-rows.h
new file mode 100644
index 000000000..3bbb1c635
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/gather-rows.h
@@ -0,0 +1,160 @@
+#ifndef HTP_GATHER_ROWS_H
+#define HTP_GATHER_ROWS_H
+
+#include "hex-common.h"
+#include <stdint.h>
+#include <stdbool.h>
+
+#ifndef GGML_MAX_DIMS
+#define GGML_MAX_DIMS 4
+#endif
+
+struct htp_gather_rows_params {
+ uint32_t na;
+ uint32_t nb;
+ uint32_t ne;
+ uint32_t rg;
+ uint32_t vg;
+ uint32_t nr_max;
+ uint32_t v1;
+ uint32_t span1;
+ uint32_t region_size;
+ uint32_t out_off;
+ uint32_t buf_stride;
+ uint32_t slice;
+ uint32_t total_rows;
+ uint32_t rows_per_thread;
+ uint8_t b_dense;
+ uint8_t pad[3];
+};
+
+static inline bool htp_gather_rows_solve_layout(
+ struct htp_gather_rows_params * kparams,
+ uint32_t na, uint32_t a_nb1,
+ uint32_t nb, uint32_t b_nb1, bool b_dense,
+ uint32_t elem_size,
+ uint32_t total_rows,
+ uint32_t n_threads,
+ uint32_t mdev_count,
+ uint32_t vtcm_size) {
+
+ const uint32_t max_ne0 = 128 / elem_size;
+ const uint32_t ne = na + nb;
+ if (ne == 0 || ne > max_ne0) {
+ return false;
+ }
+
+ if (elem_size != 4 && elem_size != 2) {
+ return false;
+ }
+
+ const uint32_t v_lanes = 128 / elem_size;
+ const uint32_t g = hex_gcd_u32(ne, v_lanes);
+ const uint32_t rg = v_lanes / g;
+ const uint32_t vg = ne / g;
+
+ const uint32_t slice = (vtcm_size / n_threads) & ~127u;
+ const uint32_t total_chunks = total_rows / rg;
+ const uint32_t chunks_per_dev = (total_chunks + mdev_count - 1) / mdev_count;
+ const uint32_t max_rows_per_dev = (mdev_count > 1 && total_chunks >= mdev_count)
+ ? (chunks_per_dev * rg + (total_rows % rg))
+ : total_rows;
+ const uint32_t rows_per_thread = hex_round_up((max_rows_per_dev + n_threads - 1) / n_threads, rg);
+
+ const uint32_t fixed = 3 * vg * 128;
+ const uint32_t per_row = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0) + ne * elem_size;
+ if (slice <= fixed + 2 * per_row * rg) {
+ return false;
+ }
+
+ const uint32_t avail_per_buf = (slice - fixed) / 2;
+ uint32_t nr_max = MIN(avail_per_buf / per_row, rows_per_thread);
+ if (elem_size == 2) {
+ const uint32_t per_row_region = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0);
+ if (per_row_region > 0) {
+ nr_max = MIN(nr_max, (32768u - 256u) / per_row_region);
+ }
+ }
+ nr_max = (nr_max / rg) * rg;
+
+ uint32_t v1 = 0;
+ uint32_t span1 = 0;
+ uint32_t region_size = 0;
+ uint32_t out_off = 0;
+ uint32_t buf_stride = 0;
+
+ while (nr_max > 0) {
+ v1 = hex_round_up((nr_max - 1) * a_nb1 + na * elem_size, 128);
+ if (nb > 0) {
+ if (b_dense) {
+ span1 = elem_size;
+ region_size = v1 + (nr_max - 1) * b_nb1 + nb * elem_size;
+ } else {
+ span1 = (nr_max - 1) * b_nb1 + elem_size;
+ region_size = v1 + nb * span1;
+ }
+ } else {
+ span1 = 0;
+ region_size = v1;
+ }
+
+ out_off = hex_round_up(region_size, 128);
+ buf_stride = hex_round_up(out_off + nr_max * ne * elem_size, 128);
+
+ if ((elem_size != 2 || region_size <= 32768) && fixed + 2 * buf_stride <= slice) {
+ break;
+ }
+
+ nr_max -= rg;
+ }
+
+ if (nr_max == 0) {
+ return false;
+ }
+
+ kparams->na = na;
+ kparams->nb = nb;
+ kparams->ne = ne;
+ kparams->rg = rg;
+ kparams->vg = vg;
+ kparams->nr_max = nr_max;
+ kparams->v1 = v1;
+ kparams->span1 = span1;
+ kparams->region_size = region_size;
+ kparams->out_off = out_off;
+ kparams->buf_stride = buf_stride;
+ kparams->slice = slice;
+ kparams->total_rows = total_rows;
+ kparams->rows_per_thread = rows_per_thread;
+ kparams->b_dense = b_dense ? 1 : 0;
+ kparams->pad[0] = 0;
+ kparams->pad[1] = 0;
+ kparams->pad[2] = 0;
+ return true;
+}
+
+#ifdef __cplusplus
+struct ggml_tensor;
+static inline bool htp_gather_rows_tensor_outer_contiguous(const struct ggml_tensor * t) {
+ size_t next_nb = ((const size_t *) t->nb)[1] * ((const int64_t *) t->ne)[1];
+ for (int i = 2; i < GGML_MAX_DIMS; i++) {
+ if (((const int64_t *) t->ne)[i] != 1 && ((const size_t *) t->nb)[i] != next_nb) {
+ return false;
+ }
+ next_nb *= ((const int64_t *) t->ne)[i];
+ }
+ return true;
+}
+#else
+struct htp_ops_context;
+struct htp_tensor;
+
+int htp_gather_rows(
+ struct htp_ops_context * octx,
+ const struct htp_tensor * a,
+ const struct htp_tensor * b,
+ const struct htp_gather_rows_params * kparams,
+ uint32_t n_threads);
+#endif
+
+#endif // HTP_GATHER_ROWS_H
diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c
index 6e3482b91..aff1c6c0f 100644
--- a/ggml/src/ggml-hexagon/htp/main.c
+++ b/ggml/src/ggml-hexagon/htp/main.c
@@ -287,15 +287,15 @@ AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) {
}
static void vtcm_acquire(struct htp_context * ctx) {
- if (!ctx->vtcm_valid) {
+ if (!atomic_load(&ctx->vtcm_valid)) {
int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 10000000u);
if (err != 0) {
FARF(ERROR, "ggml-hex: failed to acquire VTCM: 0x%08x", (unsigned)err);
abort();
}
- ctx->vtcm_needs_release = false;
- ctx->vtcm_valid = true;
+ atomic_store(&ctx->vtcm_needs_release, false);
+ atomic_store(&ctx->vtcm_valid, true);
// Drop the priority to make sure we get the release callback from other GGML-HTP and QNN-HTP sessions
HAP_compute_res_update_priority(ctx->vtcm_rctx, ctx->thread_prio + 10);
@@ -303,16 +303,16 @@ static void vtcm_acquire(struct htp_context * ctx) {
}
static void vtcm_release(struct htp_context * ctx) {
- if (ctx->vtcm_valid) {
- ctx->vtcm_valid = false;
- ctx->vtcm_needs_release = false;
+ if (atomic_load(&ctx->vtcm_valid)) {
+ atomic_store(&ctx->vtcm_valid, false);
+ atomic_store(&ctx->vtcm_needs_release, false);
HAP_compute_res_release_cached(ctx->vtcm_rctx);
}
}
static int vtcm_release_callback(unsigned int rctx, void * state) {
struct htp_context * ctx = (struct htp_context *) state;
- ctx->vtcm_needs_release = true;
+ atomic_store(&ctx->vtcm_needs_release, true);
return 0;
}
@@ -345,8 +345,8 @@ static int vtcm_alloc(struct htp_context * ctx) {
ctx->vtcm_base = (uint8_t *) vtcm_ptr;
ctx->vtcm_size = vtcm_size;
ctx->vtcm_rctx = rctx;
- ctx->vtcm_valid = false;
- ctx->vtcm_needs_release = false;
+ atomic_store(&ctx->vtcm_valid, false);
+ atomic_store(&ctx->vtcm_needs_release, false);
return 0;
}
@@ -1309,7 +1309,7 @@ static void process_ops(struct htp_context * ctx) {
vtcm_acquire(ctx);
- while (!ctx->vtcm_needs_release && !atomic_load(&ctx->killed)) {
+ while (!atomic_load(&ctx->vtcm_needs_release) && !atomic_load(&ctx->killed)) {
struct htp_opbatch_req req;
uint32_t r_size = sizeof(req);
diff --git a/scripts/snapdragon/build.py b/scripts/snapdragon/build.py
index b36e6361b..62a114e04 100755
--- a/scripts/snapdragon/build.py
+++ b/scripts/snapdragon/build.py
@@ -10,6 +10,7 @@ import subprocess
import platform
import shutil
import logging
+import glob
from sdk import validate_windows_sdks
@@ -51,6 +52,8 @@ def main():
parser.add_argument("--no-docker", action="store_true", help="Build natively on the host instead of in a docker container")
parser.add_argument("--preset", help="Override the CMake preset to use")
parser.add_argument("--debug", action="store_true", help="Build in debug mode (uses -debug presets instead of -release)")
+ parser.add_argument("--hex-only", "--hexagon-only", action="store_true", help="Only build and push ggml-hexagon and libggml-htp libraries")
+ parser.add_argument("--hex-arch", choices=["v73", "v75", "v79", "v81", "73", "75", "79", "81"], help="Specific Hexagon DSP architecture to build with --hex-only (default: all)")
# Push options
parser.add_argument("--push", action="store_true", help="Push built package to the target device via ADB or SSH/SCP")
@@ -129,6 +132,16 @@ def main():
jobs = args.jobs if args.jobs else os.cpu_count() or 4
+ hex_arch = None
+ if args.hex_arch:
+ hex_arch = args.hex_arch if args.hex_arch.startswith("v") else f"v{args.hex_arch}"
+
+ if args.hex_only:
+ if hex_arch:
+ hex_targets = ["ggml-hexagon", f"htp-{hex_arch}"]
+ else:
+ hex_targets = ["ggml-hexagon", "htp-v73", "htp-v75", "htp-v79", "htp-v81"]
+
if args.no_docker:
# Native/local host build
logger.info("Running native/local CMake build...")
@@ -143,7 +156,10 @@ def main():
sys.exit(res.returncode)
# Build
- build_cmd = ["cmake", "--build", build_dir, "-j", str(jobs)]
+ build_cmd = ["cmake", "--build", build_dir]
+ if args.hex_only:
+ build_cmd += ["--target"] + hex_targets
+ build_cmd += ["-j", str(jobs)]
logger.info(f"+ {' '.join(build_cmd)}")
res = subprocess.run(build_cmd, cwd=repo_root)
if res.returncode != 0:
@@ -165,9 +181,10 @@ def main():
install_prefix_container = f"/workspace/{install_dir}/llama.cpp"
+ target_args = f"--target {' '.join(hex_targets)} " if args.hex_only else ""
build_sh_cmd = (
f"cmake --preset {preset} -B /workspace/{build_dir} && "
- f"cmake --build /workspace/{build_dir} -j {jobs} && "
+ f"cmake --build /workspace/{build_dir} {target_args}-j {jobs} && "
f"cmake --install /workspace/{build_dir} --prefix {install_prefix_container}"
)
@@ -206,58 +223,91 @@ def main():
sub_items = [item for item in os.listdir(src_path) if not item.startswith(".")]
+ if args.hex_only:
+ lib_dir = os.path.join(src_path, "lib")
+ hex_libs = glob.glob(os.path.join(lib_dir, "libggml-hexagon*"))
+ if hex_arch:
+ hex_libs += glob.glob(os.path.join(lib_dir, f"libggml-htp-{hex_arch}*"))
+ else:
+ hex_libs += glob.glob(os.path.join(lib_dir, "libggml-htp*"))
+
+ if not hex_libs:
+ logger.error(f"Error: no hexagon libraries found in {lib_dir} to push.")
+ sys.exit(1)
+
if target_type == "android":
- logger.info("\nPushing built artifacts to Android device via ADB...")
adb_cmd = ["adb"]
if target_val: # serial
adb_cmd += ["-s", target_val]
- # Clean stale package files on device
- if sub_items:
- clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)
- clean_cmd = adb_cmd + ["shell", f"rm -rf {clean_paths}"]
- logger.info(f"+ {' '.join(clean_cmd)}")
- subprocess.run(clean_cmd)
-
- # Android destination directory is target_dir
- push_cmd = adb_cmd + ["push", os.path.join(src_path, "."), target_dir]
- logger.info(f"+ {' '.join(push_cmd)}")
- res = subprocess.run(push_cmd)
- if res.returncode != 0:
- logger.error("ADB push failed.")
- sys.exit(res.returncode)
-
- chmod_cmd = adb_cmd + ["shell", f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]
- logger.info(f"+ {' '.join(chmod_cmd)}")
- subprocess.run(chmod_cmd)
- logger.info("ADB push completed successfully!")
+ if args.hex_only:
+ logger.info("\nPushing Hexagon libraries to Android device via ADB...")
+ target_lib_dir = f"{target_dir}/lib"
+ for lib in hex_libs:
+ push_cmd = adb_cmd + ["push", lib, f"{target_lib_dir}/"]
+ logger.info(f"+ {' '.join(push_cmd)}")
+ res = subprocess.run(push_cmd)
+ if res.returncode != 0:
+ logger.error("ADB push failed.")
+ sys.exit(res.returncode)
+ logger.info("Hexagon libraries deployed successfully!")
+ else:
+ # Clean stale package files on device
+ if sub_items:
+ clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)
+ clean_cmd = adb_cmd + ["shell", f"rm -rf {clean_paths}"]
+ logger.info(f"+ {' '.join(clean_cmd)}")
+ subprocess.run(clean_cmd)
+
+ # Android destination directory is target_dir
+ push_cmd = adb_cmd + ["push", os.path.join(src_path, "."), target_dir]
+ logger.info(f"+ {' '.join(push_cmd)}")
+ res = subprocess.run(push_cmd)
+ if res.returncode != 0:
+ logger.error("ADB push failed.")
+ sys.exit(res.returncode)
+
+ chmod_cmd = adb_cmd + ["shell", f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]
+ logger.info(f"+ {' '.join(chmod_cmd)}")
+ subprocess.run(chmod_cmd)
+ logger.info("ADB push completed successfully!")
elif target_type == "linux":
ssh_host = target_val
if not ssh_host:
logger.error("Error: SSH host not specified in target (e.g. use linux:user@host, lnx:user@host, or ubuntu:user@host). Cannot deploy.")
sys.exit(1)
- logger.info(f"\nDeploying built artifacts to Linux device {ssh_host} via SSH/SCP...")
-
- # Clean stale package files on remote host
- if sub_items:
- clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)
- clean_cmd = ["ssh", ssh_host, f"rm -rf {clean_paths}"]
- logger.info(f"+ {' '.join(clean_cmd)}")
- subprocess.run(clean_cmd)
-
- # Deploy to target_dir
- deploy_cmd = ["scp", "-r", os.path.join(src_path, "."), f"{ssh_host}:{target_dir}"]
- logger.info(f"+ {' '.join(deploy_cmd)}")
- res = subprocess.run(deploy_cmd)
- if res.returncode != 0:
- logger.error("SSH/SCP deploy failed.")
- sys.exit(res.returncode)
-
- chmod_cmd = ["ssh", ssh_host, f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]
- logger.info(f"+ {' '.join(chmod_cmd)}")
- subprocess.run(chmod_cmd)
- logger.info("SSH/SCP deploy completed successfully!")
+
+ if args.hex_only:
+ logger.info(f"\nDeploying Hexagon libraries to Linux device {ssh_host} via SCP...")
+ target_lib_dir = f"{target_dir}/lib"
+ deploy_cmd = ["scp"] + hex_libs + [f"{ssh_host}:{target_lib_dir}/"]
+ logger.info(f"+ {' '.join(deploy_cmd)}")
+ res = subprocess.run(deploy_cmd)
+ if res.returncode != 0:
+ logger.error("SCP deploy failed.")
+ sys.exit(res.returncode)
+ logger.info("Hexagon libraries deployed successfully!")
+ else:
+ # Clean stale package files on remote host
+ if sub_items:
+ clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)
+ clean_cmd = ["ssh", ssh_host, f"rm -rf {clean_paths}"]
+ logger.info(f"+ {' '.join(clean_cmd)}")
+ subprocess.run(clean_cmd)
+
+ # Deploy to target_dir
+ deploy_cmd = ["scp", "-r", os.path.join(src_path, "."), f"{ssh_host}:{target_dir}"]
+ logger.info(f"+ {' '.join(deploy_cmd)}")
+ res = subprocess.run(deploy_cmd)
+ if res.returncode != 0:
+ logger.error("SSH/SCP deploy failed.")
+ sys.exit(res.returncode)
+
+ chmod_cmd = ["ssh", ssh_host, f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]
+ logger.info(f"+ {' '.join(chmod_cmd)}")
+ subprocess.run(chmod_cmd)
+ logger.info("SSH/SCP deploy completed successfully!")
elif target_type == "windows":
logger.info("\nPush for Windows on Snapdragon (windows) target is currently a stub.")