Commit 51fe42c5d for llama.cpp
commit 51fe42c5d14bfb9adf0612ff98a9ab79e0e7275c
Author: Mark Todorovich <mtodorov@qti.qualcomm.com>
Date: Mon Nov 2 21:13:38 2026 -0800
hexagon: tiled hvx permute (#31446)
* hexagon: add tiled HVX transpose path for permuted CONT/CPY
Permuted copies where dst dim 0 is strided in src, but some other src dim
is contiguous, used to fall back to HTP_COPY_KERNEL_RESHAPE. That issues
one DMA descriptor per element. The channel-first <-> channel-last
permutes around per-channel norms (e.g. [W,H,T,C] <-> [C,W,H,T] in video
VAEs) ht this path, and these copies ran at about 45 MB/s.
Add HTP_COPY_KERNEL_TRANSPOSE for same-type, same-shape f32/f16 copies
into a contiguous dst.
Host:
- drop extent-1 dims and merge dims that stay adjacent in src into a
canonical view, stored in the kernel params. There are no minimum
size limits.
- the copy is treated as a 2D transpose between dst dim 0 and the
contiguous src dim, batched over the remaining dims.
- tiles are sized to 64 KB per buffer and halved until they fit in
session VTCM. blkk is capped so the halfword vgather offsets fit in
16 bits.
Kernel, per tile:
- 2D DMA of blk0 strided src rows (blkk contiguous elements each)
into VTCM
- vgather transpose in 32x32 (f32) or 64x64 (f16) blocks
- 2D DMA of blkk contiguous rows out to dst
Staged and transposed buffers are double-buffered, so the next tile's
input DMA overlaps the current transpose and output DMA. Tiles are split
across threads and mdev sessions.
Small dims:
- packed mode: when dim k follows dim 0 and dim 0 divides the HVX
lanes (f32: 2/4/8/16, f16: 2/4/.../32), one vgather writes several
whole dst rows, and each tile goes out as one contiguous DMA.
- when the contiguous src dim fits in one vector, it is staged exactly
and the tile is made long along dim 0.
All other copies select the same kernels as before.
Tests: add test_cont cases for perms {1,2,3,0} and {3,0,1,2}, f32 and
f16, at various sizes.
* hexagon: cpy transpose cleanups
- fix vgather halfword offset comments: offsets are unsigned 16-bit, caps are kept lower for speed
- set packed directly in the packed case
- add dim_k == 3 CONT tests (full reversal)
Assisted-by: Claude Opus 5.5
* hexagon: use cpy transpose path for ne[0] == 1 permutes
- drop the nb[0] check in the transpose solver: size-1 dims are skipped, and the canonical tp.nb[0] check covers packed dim 0
- try the transpose path before SAMESHAPE_SAMETYPE, which did one DMA per element for these
- add ne[0] == 1 permute CPY tests
Assisted-by: Claude Opus 5.5
* hex-cpy: further improve cpy
Bring in improvements from aizip branch/PR.
Cleanup kparams and replace divs with fastdiv.
Co-authored-by: Yiwei Shao <yiwei@aizip.ai>
* hex-cpy: avoid vtcm bank conflicts
* hex-cpy: trim down the comments
* cpy-test: trim down new tests that cover permute scenarios
---------
Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
Co-authored-by: Yiwei Shao <yiwei@aizip.ai>
diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp
index 03b5a3b50..e1d3385e2 100644
--- a/ggml/src/ggml-hexagon/htp-kparams.cpp
+++ b/ggml/src/ggml-hexagon/htp-kparams.cpp
@@ -2660,6 +2660,129 @@ static bool htp_kparams_solve_concat(
return false;
}
+// Permuted copy with strided dim 0 (e.g. channel-first <-> channel-last)
+static bool htp_kparams_solve_cpy_transpose(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * dst,
+ uint32_t type_size,
+ struct htp_copy_kernel_params * kparams) {
+
+ // View contiguous dst with src shape to support CPY with reshape
+ const bool dst_cont = ggml_is_contiguous(dst);
+ if (!dst_cont && !ggml_are_same_shape(src0, dst)) {
+ return false;
+ }
+
+ // Drop size-1 dims and merge dims adjacent in both tensors
+ size_t ne[GGML_MAX_DIMS];
+ size_t nbs[GGML_MAX_DIMS];
+ size_t nbd[GGML_MAX_DIMS];
+ size_t dst_stride = type_size;
+ int n = 0;
+ for (int d = 0; d < GGML_MAX_DIMS; d++) {
+ const size_t sd = dst_cont ? dst_stride : dst->nb[d];
+ dst_stride *= (size_t) src0->ne[d];
+ if (src0->ne[d] == 1) {
+ continue;
+ }
+ if (n > 0 && src0->nb[d] == nbs[n - 1] * ne[n - 1] && sd == nbd[n - 1] * ne[n - 1]) {
+ ne[n - 1] *= (size_t) src0->ne[d];
+ continue;
+ }
+ ne[n] = (size_t) src0->ne[d];
+ nbs[n] = src0->nb[d];
+ nbd[n] = sd;
+ n++;
+ }
+
+ // Fold leading dense dim into element block
+ size_t es = type_size;
+ for (int i = 0; i < n; i++) {
+ if (nbs[i] == type_size && nbd[i] == type_size) {
+ es *= ne[i];
+ for (int j = i; j + 1 < n; j++) {
+ ne[j] = ne[j + 1];
+ nbs[j] = nbs[j + 1];
+ nbd[j] = nbd[j + 1];
+ }
+ n--;
+ break;
+ }
+ }
+ if (es > 128 || (es & (es - 1)) != 0) {
+ return false;
+ }
+
+ // Transpose axes: dim a dense in dst, dim k dense in src
+ int a = -1;
+ int k = -1;
+ for (int i = 0; i < n; i++) {
+ if (a < 0 && nbd[i] == es) {
+ a = i;
+ }
+ if (k < 0 && nbs[i] == es) {
+ k = i;
+ }
+ }
+ if (a < 0 || k < 0 || a == k) {
+ return false;
+ }
+
+ struct htp_copy_transpose_params tp = {};
+ tp.elem_size = (uint8_t) es;
+ tp.ne[0] = (uint32_t) ne[a];
+ tp.ne[1] = (uint32_t) ne[k];
+ tp.ne[2] = 1;
+ tp.src_nb[0] = (uint32_t) nbs[a];
+ tp.dst_nb[0] = (uint32_t) nbd[k];
+ tp.packed = nbd[k] == ne[a] * es && ne[a] * es < 128 && 128 % (ne[a] * es) == 0;
+
+ size_t ne3 = 1;
+ int m = 0;
+ for (int i = 0; i < n; i++) {
+ if (i == a || i == k) {
+ continue;
+ }
+ if (m == 0) {
+ tp.ne[2] = (uint32_t) ne[i];
+ } else {
+ ne3 = ne[i];
+ }
+ tp.src_nb[1 + m] = (uint32_t) nbs[i];
+ tp.dst_nb[1 + m] = (uint32_t) nbd[i];
+ m++;
+ }
+
+ const uint32_t n_threads = htp_session_n_threads(sess);
+ const size_t vtcm_size = htp_session_vtcm_size(sess);
+ uint64_t vtcm = 0;
+ for (uint32_t blk_bytes = HTP_COPY_TRANSPOSE_BLK_BYTES; ; blk_bytes /= 2) {
+ if (!htp_copy_transpose_layout_build(&tp, blk_bytes)) {
+ return false;
+ }
+ vtcm = (uint64_t) n_threads * 2 * (tp.in_buf_size + tp.out_buf_size);
+ if (vtcm <= vtcm_size) {
+ break;
+ }
+ }
+
+ const uint64_t total_tiles = (uint64_t) tp.n0b * tp.nkb * tp.ne[2] * ne3;
+ if (total_tiles > UINT32_MAX) {
+ return false;
+ }
+ tp.total_tiles = (uint32_t) total_tiles;
+ tp.div_nkb = init_fastdiv_values(tp.nkb);
+ tp.div_n0b = init_fastdiv_values(tp.n0b);
+ tp.div_ne2 = init_fastdiv_values(tp.ne[2]);
+
+ kparams->kernel_type = HTP_COPY_KERNEL_TRANSPOSE;
+ kparams->n_threads = (uint8_t) n_threads;
+ kparams->vtcm_size = (uint32_t) vtcm;
+ kparams->u.transpose = tp;
+ return true;
+}
+
static bool htp_kparams_solve_cpy(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * op,
@@ -2757,6 +2880,10 @@ static bool htp_kparams_solve_cpy(
}
}
+ if (htp_kparams_solve_cpy_transpose(sess, src0, dst, src_type_size, kparams)) {
+ return true;
+ }
+
if (sameshape) {
kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE;
kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c
index fce45c2d4..59f406c7a 100644
--- a/ggml/src/ggml-hexagon/htp/cpy-ops.c
+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c
@@ -35,6 +35,10 @@ struct htp_copy_context {
struct htp_copy_convert_vtcm_layout layout;
uint8_t * vtcm_src0;
uint8_t * vtcm_dst;
+
+ uint32_t tile_start;
+ uint32_t ntiles;
+ uint32_t tiles_per_thread;
};
#define cpy_preamble \
@@ -162,6 +166,171 @@ static inline void cpy_cursor_advance(struct cpy_row_cursor * c, uint32_t n, uin
}
}
+struct cpy_tile_cursor {
+ uint32_t ibk;
+ uint32_t ib0;
+ uint32_t i2;
+ uint32_t i3;
+};
+
+static inline void cpy_tile_cursor_advance(struct cpy_tile_cursor * c, const struct htp_copy_transpose_params * tp) {
+ if (++c->ibk == tp->nkb) {
+ c->ibk = 0;
+ if (++c->ib0 == tp->n0b) {
+ c->ib0 = 0;
+ if (++c->i2 == tp->ne[2]) {
+ c->i2 = 0;
+ c->i3++;
+ }
+ }
+ }
+}
+
+// Gather offsets for element e = l >> log2_m at (e % r, e / r)
+static inline HVX_Vector cpy_transpose_offsets(const struct htp_copy_transpose_params * tp) {
+ const uint32_t r_mask = (1u << tp->log2_r) - 1;
+ if (tp->elem_size != 2) {
+ const uint32_t m_mask = (1u << tp->log2_m) - 1;
+ int32_t offsets[32] __attribute__((aligned(128)));
+ for (uint32_t l = 0; l < 32; l++) {
+ const uint32_t e = l >> tp->log2_m;
+ offsets[l] = (e & r_mask) * tp->in_stride + (e >> tp->log2_r) * tp->elem_size + (l & m_mask) * 4;
+ }
+ return *(HVX_Vector *) offsets;
+ }
+ int16_t offsets[64] __attribute__((aligned(128)));
+ for (uint32_t l = 0; l < 64; l++) {
+ offsets[l] = (l & r_mask) * tp->in_stride + (l >> tp->log2_r) * 2;
+ }
+ return *(HVX_Vector *) offsets;
+}
+
+static inline void cpy_transpose_push_in(dma_queue * q, const struct htp_copy_transpose_params * tp, dma_addr_t src,
+ const struct cpy_tile_cursor * c, uint8_t * buf) {
+ const uint32_t a0 = c->ib0 * tp->blk0;
+ const uint32_t ak = c->ibk * tp->blkk;
+ const uint32_t off = a0 * tp->src_nb[0] + ak * tp->elem_size + c->i2 * tp->src_nb[1] + c->i3 * tp->src_nb[2];
+ dma_queue_push(q, dma_make_data(buf, src + off), tp->in_stride, tp->src_nb[0],
+ MIN(tp->blkk, tp->ne[1] - ak) * tp->elem_size, MIN(tp->blk0, tp->ne[0] - a0));
+}
+
+// Pipelined tile worker: DMA in -> vgather transpose -> DMA out
+static void cpy_thread_transpose(unsigned int nth, unsigned int ith, void * data) {
+ struct htp_copy_context * ct = (struct htp_copy_context *) data;
+ struct htp_ops_context * octx = ct->octx;
+ const struct htp_copy_transpose_params * tp = &ct->kparams->u.transpose;
+
+ const uint32_t t0 = ct->tile_start + ith * ct->tiles_per_thread;
+ const uint32_t t1 = MIN(t0 + ct->tiles_per_thread, ct->tile_start + ct->ntiles);
+ if (t0 >= t1) {
+ return;
+ }
+
+ dma_queue * dma_q = octx->ctx->dma[ith];
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+
+ const dma_addr_t src = octx->src[0]->data;
+ const dma_addr_t dst = octx->dst->data;
+
+ const uint32_t es = tp->elem_size;
+ const uint32_t ne0 = tp->ne[0];
+ const uint32_t nek = tp->ne[1];
+ const uint32_t in_stride = tp->in_stride;
+ const uint32_t out_stride = tp->out_stride;
+
+ const HVX_Vector offsets = cpy_transpose_offsets(tp);
+ const uint32_t packed_mu = (ne0 - 1) * in_stride + 127;
+ const uint32_t packed_step = 128 >> tp->log2_r;
+ const uint32_t epv = 1u << tp->log2_r;
+
+ uint8_t * in_buf[2];
+ uint8_t * out_buf[2];
+ in_buf[0] = ct->vtcm_src0 + ith * 2 * tp->in_buf_size;
+ in_buf[1] = in_buf[0] + tp->in_buf_size;
+ out_buf[0] = ct->vtcm_dst + ith * 2 * tp->out_buf_size;
+ out_buf[1] = out_buf[0] + tp->out_buf_size;
+
+ struct cpy_tile_cursor cur;
+ {
+ const uint32_t q0 = fastdiv(t0, &tp->div_nkb);
+ const uint32_t q1 = fastdiv(q0, &tp->div_n0b);
+ cur.ibk = t0 - q0 * tp->nkb;
+ cur.ib0 = q0 - q1 * tp->n0b;
+ cur.i3 = fastdiv(q1, &tp->div_ne2);
+ cur.i2 = q1 - cur.i3 * tp->ne[2];
+ }
+
+ struct cpy_tile_cursor nxt = cur;
+
+ cpy_transpose_push_in(dma_q, tp, src, &cur, in_buf[0]);
+
+ for (uint32_t t = t0, i = 0; t < t1; t++, i++) {
+ while (dma_queue_depth(dma_q) > (i > 0 ? 1u : 0u)) {
+ dma_queue_pop(dma_q);
+ }
+
+ uint8_t * in = in_buf[i & 1];
+ uint8_t * out = out_buf[i & 1];
+
+ const uint32_t a0 = cur.ib0 * tp->blk0;
+ const uint32_t ak = cur.ibk * tp->blkk;
+ const uint32_t n0 = MIN(tp->blk0, ne0 - a0);
+ const uint32_t nk = MIN(tp->blkk, nek - ak);
+
+ if (t + 1 < t1) {
+ cpy_tile_cursor_advance(&nxt, tp);
+ cpy_transpose_push_in(dma_q, tp, src, &nxt, in_buf[(i + 1) & 1]);
+ }
+
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) t);
+ if (tp->packed) {
+ const uint32_t nvec = (nk * ne0 * es + 127) >> 7;
+ if (es != 2) {
+ #pragma unroll(4)
+ for (uint32_t v = 0; v < nvec; v++) {
+ Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) (in + v * packed_step), packed_mu, offsets);
+ }
+ } else {
+ #pragma unroll(4)
+ for (uint32_t v = 0; v < nvec; v++) {
+ Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) (in + v * packed_step), packed_mu, offsets);
+ }
+ }
+ } else {
+ for (uint32_t rb = 0; rb < n0; rb += epv) {
+ const uint32_t mu = (MIN(epv, n0 - rb) - 1) * in_stride + es - 1;
+ uint8_t * d = out + rb * es;
+ const uint8_t * s = in + rb * in_stride;
+ if (es != 2) {
+ #pragma unroll(4)
+ for (uint32_t ik = 0; ik < nk; ik++) {
+ Q6_vgather_ARMVw((HVX_Vector *) (d + ik * out_stride), (size_t) (s + ik * es), mu, offsets);
+ }
+ } else {
+ #pragma unroll(4)
+ for (uint32_t ik = 0; ik < nk; ik++) {
+ Q6_vgather_ARMVh((HVX_Vector *) (d + ik * out_stride), (size_t) (s + ik * 2), mu, offsets);
+ }
+ }
+ }
+ }
+ hvx_gather_sync(out);
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) t);
+
+ const uint32_t doff = a0 * es + ak * tp->dst_nb[0] + cur.i2 * tp->dst_nb[1] + cur.i3 * tp->dst_nb[2];
+ if (tp->packed) {
+ const uint32_t n = nk * ne0 * es;
+ dma_queue_push(dma_q, dma_make_data(dst + doff, out), n, n, n, 1);
+ } else {
+ dma_queue_push(dma_q, dma_make_data(dst + doff, out), tp->dst_nb[0], out_stride, n0 * es, nk);
+ }
+
+ cur = nxt;
+ }
+
+ dma_queue_flush(dma_q);
+}
+
#define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC) \
static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
@@ -439,6 +608,48 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern
return HTP_STATUS_OK;
}
+static int cpy_transpose(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) {
+ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
+ const struct htp_copy_transpose_params * tp = &kparams->u.transpose;
+
+ uint32_t tile_start = 0;
+ uint32_t ntiles = tp->total_tiles;
+
+ if (octx->ctx->mdev.count > 1) {
+ const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
+ tp->total_tiles, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
+ tile_start = range.start;
+ ntiles = range.count;
+ }
+
+ if (ntiles == 0) {
+ return HTP_STATUS_OK;
+ }
+
+ if (kparams->vtcm_size > octx->ctx->vtcm_size) {
+ return HTP_STATUS_VTCM_TOO_SMALL;
+ }
+
+ const uint32_t n_threads = octx->n_threads;
+
+ struct htp_copy_context ct;
+ ct.octx = octx;
+ ct.kparams = kparams;
+ ct.tile_start = tile_start;
+ ct.ntiles = ntiles;
+ ct.tiles_per_thread = fastdiv(ntiles + n_threads - 1, &octx->n_threads_div);
+
+ uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
+ ct.vtcm_src0 = vtcm_base;
+ ct.vtcm_dst = vtcm_base + (size_t) n_threads * 2 * tp->in_buf_size;
+
+ work_queue_run(octx->ctx->work_queue, cpy_thread_transpose, &ct, n_threads);
+ return HTP_STATUS_OK;
+}
+
int op_cpy(struct htp_ops_context * octx) {
const struct htp_copy_kernel_params * kparams = (const struct htp_copy_kernel_params *) octx->kernel_params;
int status = HTP_STATUS_OK;
@@ -459,6 +670,9 @@ int op_cpy(struct htp_ops_context * octx) {
case HTP_COPY_KERNEL_RESHAPE:
status = cpy_reshape(octx, kparams);
break;
+ case HTP_COPY_KERNEL_TRANSPOSE:
+ status = cpy_transpose(octx, kparams);
+ break;
case HTP_COPY_KERNEL_GATHER_ROWS:
status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_threads);
break;
diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.h b/ggml/src/ggml-hexagon/htp/cpy-ops.h
index c3b97c0ca..6bb96ca44 100644
--- a/ggml/src/ggml-hexagon/htp/cpy-ops.h
+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.h
@@ -15,6 +15,7 @@ enum htp_copy_kernel_type {
HTP_COPY_KERNEL_RESHAPE = 4,
HTP_COPY_KERNEL_SCALAR = 5,
HTP_COPY_KERNEL_GATHER_ROWS = 6,
+ HTP_COPY_KERNEL_TRANSPOSE = 7,
};
struct htp_copy_convert_params {
@@ -32,6 +33,29 @@ struct htp_copy_reshape_params {
struct fastdiv_values div_ne02_ne01_ne00;
};
+// Batched 2D transpose of elem_size elements (dim 0 dense in dst, dim 1 dense in src)
+struct htp_copy_transpose_params {
+ uint32_t ne[3]; // dims 0..2
+ uint32_t src_nb[3]; // src strides for dims 0, 2, 3
+ uint32_t dst_nb[3]; // dst strides for dims 1, 2, 3
+ struct fastdiv_values div_nkb;
+ struct fastdiv_values div_n0b;
+ struct fastdiv_values div_ne2;
+ uint32_t blk0; // dim-0 elements per tile
+ uint32_t blkk; // dim-1 elements per tile
+ uint32_t n0b; // tiles along dim 0
+ uint32_t nkb; // tiles along dim 1
+ uint32_t in_stride; // VTCM row pitch of staged src tile
+ uint32_t out_stride; // VTCM row pitch of transposed tile
+ uint32_t in_buf_size;
+ uint32_t out_buf_size;
+ uint32_t total_tiles;
+ uint8_t elem_size;
+ uint8_t packed; // gather straight to dst order
+ uint8_t log2_m; // log2(gather lanes per element)
+ uint8_t log2_r; // log2(dim-0 elements per vector)
+};
+
struct htp_copy_kernel_params {
uint8_t kernel_type;
uint8_t src0_type_size;
@@ -46,6 +70,7 @@ struct htp_copy_kernel_params {
struct htp_copy_convert_params convert;
struct htp_copy_reshape_params reshape;
struct htp_gather_rows_params gather;
+ struct htp_copy_transpose_params transpose;
} u;
};
@@ -116,6 +141,76 @@ static inline void htp_copy_convert_solve_layout(
htp_copy_convert_vtcm_layout_build(layout, ne00, src_type_size, dst_type_size, blk_rows, n_threads);
}
+#define HTP_COPY_TRANSPOSE_BLK_BYTES (64 * 1024)
+
+static inline bool htp_copy_transpose_layout_build(struct htp_copy_transpose_params * tp, uint32_t blk_bytes) {
+ const uint32_t ne0 = tp->ne[0];
+ const uint32_t nek = tp->ne[1];
+ const uint32_t es = tp->elem_size;
+ const uint32_t epv = 128 / es;
+
+ uint32_t blk0;
+ uint32_t blkk;
+ if (tp->packed) {
+ // Packed: vector holds epv / ne0 whole dst rows
+ blk0 = ne0;
+ blkk = blk_bytes / (ne0 * es);
+ if (es == 2 && blkk > 16384 / ne0 - 64) {
+ blkk = 16384 / ne0 - 64;
+ }
+ if (blkk > nek) {
+ blkk = nek;
+ }
+ if (blkk == 0) {
+ return false;
+ }
+ } else if (nek <= epv) {
+ // Stage short contiguous dim and expand dim 0
+ blkk = nek;
+ blk0 = blk_bytes / (nek * es);
+ blk0 -= blk0 % epv;
+ if (blk0 > hex_round_up(ne0, epv)) {
+ blk0 = hex_round_up(ne0, epv);
+ }
+ if (blk0 < epv) {
+ return false;
+ }
+ } else {
+ blk0 = hex_round_up(ne0, epv);
+ if (blk0 > 256) {
+ blk0 = 256;
+ }
+ blkk = blk_bytes / (blk0 * es);
+ blkk -= blkk % epv;
+ // Cap f16 blkk to 256 for 16-bit gather offsets
+ const uint32_t max_blkk = (es == 2 ? 512 : 4096) / es;
+ if (blkk > max_blkk) {
+ blkk = max_blkk;
+ }
+ if (blkk > hex_round_up(nek, epv)) {
+ blkk = hex_round_up(nek, epv);
+ }
+ if (blkk < epv) {
+ return false;
+ }
+ }
+
+ tp->blk0 = blk0;
+ tp->blkk = blkk;
+ tp->n0b = (ne0 + blk0 - 1) / blk0;
+ tp->nkb = (nek + blkk - 1) / blkk;
+ // Pad row pitch by 1 element to avoid VTCM bank conflicts
+ const uint32_t pad = (es == 2) ? 2 : 4;
+ tp->in_stride = blkk * es + pad;
+ tp->out_stride = blk0 * es;
+ // Extra vector for packed gather overhang
+ tp->in_buf_size = hex_round_up(blk0 * tp->in_stride + (tp->packed ? 128 : 0), 128);
+ tp->out_buf_size = hex_round_up(blkk * tp->out_stride, 128);
+ tp->log2_m = es == 2 ? 0 : __builtin_ctz(es) - 2;
+ tp->log2_r = __builtin_ctz(tp->packed ? ne0 : epv);
+ return true;
+}
+
#if defined(__cplusplus)
static_assert(sizeof(struct htp_copy_kernel_params) <= 128, "htp_copy_kernel_params is too large for kernel_params blob");
#else
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index eec0423cb..64fdb8259 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -10865,6 +10865,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_I32, {256, 1, 4, 1}, {-1,-1,-1,-1}, {1, 2, 0, 3}, {0, 0, 0, 0}));
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 1, 4, 1}, {-1,-1,-1,-1}, {1, 2, 0, 3}, {0, 0, 0, 0}));
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {2, 2097121, 1, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3}));
+ test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {1, 64, 48, 1}, {-1,-1,-1,-1}, {0, 2, 1, 3})); // ne[0] == 1 permute
+ test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {1, 64, 48, 1}, {-1,-1,-1,-1}, {0, 2, 1, 3})); // ne[0] == 1 permute
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {2, 2, 524281, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3}));
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {128, 2, 3, 1}, {128, 2, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {128, 4, 3, 1})); // strided dst
test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {128, 2, 3, 1}, {128, 2, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {128, 4, 3, 1})); // strided dst
@@ -10931,6 +10933,31 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {512, 8193, 2, 1}, false, {2, 1, 0, 3}));
test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {256, 16385, 2, 1}, false, {0, 2, 1, 3}));
+ // channel swap with sizes not multiples of 32/64 tile
+ for (ggml_type type_dst : { GGML_TYPE_F32, GGML_TYPE_F16 }) {
+ test_cases.emplace_back(new test_cont(type_dst, {33, 17, 5, 19}, false, {1, 2, 3, 0}));
+ }
+
+ // small channel dims and extent-1 dims
+ test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {8, 104, 60, 13}, false, {1, 2, 3, 0}));
+ test_cases.emplace_back(new test_cont(GGML_TYPE_F16, {8, 104, 60, 13}, false, {1, 2, 3, 0}));
+ test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {16, 1, 50, 7}, false, {1, 2, 3, 0}));
+
+ // full reversal: contiguous src dim lands on dst dim 3
+ test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {33, 5, 7, 9}, false, {3, 2, 1, 0}));
+ test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {33, 64, 32, 16}, false, {3, 2, 1, 0}));
+
+ // permutes that move dense blocks (ne[0] stays dim 0)
+ for (ggml_type type_dst : { GGML_TYPE_F32, GGML_TYPE_F16 }) {
+ test_cases.emplace_back(new test_cont(type_dst, {4, 65, 9, 2}, false, {0, 2, 1, 3}));
+ }
+ test_cases.emplace_back(new test_cont(GGML_TYPE_F16, {64, 9, 7, 1}, false, {0, 2, 1, 3}));
+
+ // transpose with i32 and CPY with reshape
+ test_cases.emplace_back(new test_cont(GGML_TYPE_I32, {33, 17, 5, 19}, false, {1, 2, 3, 0}));
+ test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {33, 17, 5, 1}, {17, 33, 5, 1}, {0, 0, 0, 0}, {1, 0, 2, 3}));
+ test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {8, 52, 39, 1}, {8, 39, 52, 1}, {0, 0, 0, 0}, {0, 2, 1, 3}));
+
auto add_test_bin_bcast = [&](ggml_type type, std::array<int64_t, 4> ne, std::array<int, 4> nr, bool perm1 = false, bool src_overlap = false) {
for (auto op : {ggml_add, ggml_sub, ggml_mul, ggml_div}) {
test_cases.emplace_back(new test_bin_bcast(op, type, ne, nr, 1, perm1, src_overlap));