Commit 2e0d86beb for llama.cpp
commit 2e0d86bebc210947680e8f0d67daa79f3260c106
Author: Marco Colombo <ing.marco.colombo@gmail.com>
Date: Thu Oct 29 03:04:18 2026 +0100
Hexagon: IM2COL improvement - block based HVX parallelization (#31050)
* hexagon: IM2COL over multiple threads with output-col blocks
Assisted-by: Claude Opus 5.5
* hex-im2col: fix padding handling
* hex-im2col: loop unroll common cases
---------
Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp
index 44b42c3f8..e5b2cd05d 100644
--- a/ggml/src/ggml-hexagon/htp-kparams.cpp
+++ b/ggml/src/ggml-hexagon/htp-kparams.cpp
@@ -2726,7 +2726,8 @@ static bool htp_kparams_solve_im2col(
const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads,
+ htp_im2col_scratch_bytes(HTP_IM2COL_KERNEL_PATCHEMBED_DMA, KH, patch_stride));
if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;
@@ -2775,18 +2776,24 @@ static bool htp_kparams_solve_im2col(
const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);
const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
+ // threads work on output-col blocks, so a single output row (1D) can use them all
+ const uint32_t nbpr = (OW + owb - 1) / owb;
+ const uint64_t nblocks = (uint64_t) total_rows * nbpr;
+ const uint32_t blk_threads = (uint32_t) (nblocks < max_threads ? nblocks : max_threads);
+
struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, blk_threads,
+ htp_im2col_scratch_bytes(HTP_IM2COL_KERNEL_BLOCKED_DMA, KH, patch_stride));
if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA;
- kparams->n_threads = (uint8_t) n_threads;
+ kparams->n_threads = (uint8_t) blk_threads;
kparams->vtcm_size = (uint32_t) layout.total_bytes;
kparams->src_row_bytes = src_row_bytes;
kparams->dst_row_bytes = dst_row_bytes;
kparams->owb = owb;
kparams->wb = (uint32_t) Wb;
- kparams->nbpr = (OW + owb - 1) / owb;
+ kparams->nbpr = nbpr;
kparams->N = N;
kparams->IC = IC;
kparams->ID = 1;
@@ -2803,6 +2810,7 @@ static bool htp_kparams_solve_im2col(
kparams->rows_per_chunk = rows_per_chunk;
kparams->div_OH = init_fastdiv_values(OH);
kparams->div_OD = init_fastdiv_values(1);
+ kparams->div_nbpr = init_fastdiv_values(nbpr);
return true;
}
if (owb == 1) break;
@@ -2904,7 +2912,7 @@ static bool htp_kparams_solve_im2col_3d(
const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads, 256);
if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;
@@ -2954,7 +2962,7 @@ static bool htp_kparams_solve_im2col_3d(
const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads, 256);
if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA;
diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.c b/ggml/src/ggml-hexagon/htp/im2col-ops.c
index fc49c403b..9e662694b 100644
--- a/ggml/src/ggml-hexagon/htp/im2col-ops.c
+++ b/ggml/src/ggml-hexagon/htp/im2col-ops.c
@@ -26,6 +26,7 @@ struct htp_im2col_context {
uint32_t pe_row_base; // first output row index assigned to this dev (DMA path)
uint32_t pe_nrows; // number of output rows assigned to this dev (DMA path)
uint32_t pe_rows_per_thread; // output rows per worker
+ uint32_t pe_blocks_per_thread; // output-col blocks per worker (blocked DMA path)
uint32_t pe_src_row_bytes; // one output row's source, rounded 256
uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256
@@ -38,6 +39,7 @@ struct htp_im2col_context {
uint32_t pe_owb; // output-col block size
uint32_t pe_wb; // staged source window width
uint32_t pe_nbpr; // blocks per row
+ uint32_t pe_gather_nvec; // vectors per gathered patch (KH == 1 blocked path)
// Precomputed dimensions for worker threads
uint32_t N;
@@ -51,6 +53,7 @@ struct htp_im2col_context {
uint32_t thread_in[HTP_MAX_NTHREADS];
uint32_t thread_iod[HTP_MAX_NTHREADS];
uint32_t thread_ioh[HTP_MAX_NTHREADS];
+ uint32_t thread_c0[HTP_MAX_NTHREADS];
void * compute;
uint32_t dst_elem;
@@ -215,6 +218,109 @@ static void FNAME(
IM2COL_BLOCKED_COMPUTE_BODY(im2col_blocked_compute_f16, uint16_t, hvx_copy_f16_f32_uu, hvx_splat_u16_u)
IM2COL_BLOCKED_COMPUTE_BODY(im2col_blocked_compute_f32, float, hvx_copy_f32_uu, hvx_splat_f32_u)
+// Offsets of one output patch in the staged window, for KH == 1: lane k reads channel k / KW at tap k % KW.
+// Lanes past the patch read the first element and are not stored.
+// The first vector is built lane by lane; each next one is 32 lanes further: the tap advances by 32 % KW
+// and the channel by 32 / KW, plus one more when the tap wraps past KW.
+static __attribute__((noinline)) void im2col_gather_table_build(const struct htp_im2col_context * ictx,
+ HVX_Vector * restrict table) {
+ const int32_t d0 = ictx->octx->op_params[4];
+ const uint32_t KW = ictx->KW, Wb = ictx->pe_wb;
+
+ int32_t tap0[32] __attribute__((aligned(128)));
+ int32_t off0[32] __attribute__((aligned(128)));
+ uint32_t iic = 0, ikw = 0;
+ for (uint32_t k = 0; k < 32; k++) {
+ tap0[k] = (int32_t) ikw;
+ off0[k] = (int32_t) ((iic * Wb + ikw * d0) * sizeof(float));
+ if (++ikw == KW) {
+ ikw = 0;
+ iic++;
+ }
+ }
+ // (iic, ikw) is now (32 / KW, 32 % KW)
+ const HVX_Vector v_kw = Q6_V_vsplat_R(KW);
+ const HVX_Vector v_kw_m1 = Q6_V_vsplat_R(KW - 1);
+ const HVX_Vector v_tap_step = Q6_V_vsplat_R(ikw);
+ const HVX_Vector v_off_step = Q6_V_vsplat_R((iic * Wb + ikw * d0) * sizeof(float));
+ const HVX_Vector v_off_wrap = Q6_V_vsplat_R((Wb - KW * d0) * sizeof(float));
+ const HVX_Vector v_32 = Q6_V_vsplat_R(32);
+ const HVX_Vector v_n = Q6_V_vsplat_R(ictx->patch_stride);
+
+ HVX_Vector lane = hvx_vec_gather_offsets_w(1);
+ HVX_Vector tap = *(HVX_Vector *) tap0;
+ HVX_Vector off = *(HVX_Vector *) off0;
+ for (uint32_t g = 0; g < ictx->pe_gather_nvec; g++) {
+ table[g] = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(v_n, lane), off, Q6_V_vzero());
+
+ lane = Q6_Vw_vadd_VwVw(lane, v_32);
+ tap = Q6_Vw_vadd_VwVw(tap, v_tap_step);
+ off = Q6_Vw_vadd_VwVw(off, v_off_step);
+
+ const HVX_VectorPred wrap = Q6_Q_vcmp_gt_VwVw(tap, v_kw_m1);
+ tap = Q6_V_vmux_QVV(wrap, Q6_Vw_vsub_VwVw(tap, v_kw), tap);
+ off = Q6_V_vmux_QVV(wrap, Q6_Vw_vadd_VwVw(off, v_off_wrap), off);
+ }
+}
+
+// KH == 1: each output row is one patch, gathered from the staged [IC][Wb] window with a fixed offset
+// table shifted by s0 per row, then converted with a single copy.
+// scratch holds the offset table followed by the gathered row (both VTCM).
+#define IM2COL_GATHER_COMPUTE_BODY(FNAME, DST_CTYPE, COPY_FN) \
+static void FNAME( \
+ const struct htp_im2col_context * ictx, \
+ void * restrict dstb_raw, const float * restrict srcb_c, \
+ float * restrict scratch, \
+ uint32_t nb, uint32_t c0, uint32_t ioh) { \
+ DST_CTYPE * restrict dstb = (DST_CTYPE *) dstb_raw; \
+ float * restrict srcb = (float *) srcb_c; \
+ const int32_t s0 = ictx->octx->op_params[0], p0 = ictx->octx->op_params[2]; \
+ const int32_t s1 = ictx->octx->op_params[1], p1 = ictx->octx->op_params[3]; \
+ const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW, Wb = ictx->pe_wb; \
+ const uint32_t patch_stride = ictx->patch_stride; \
+ const uint32_t ng = ictx->pe_gather_nvec; \
+ const int32_t win0 = (int32_t) c0 * s0 - p0; \
+ const int32_t iih = (int32_t) ioh * s1 - p1; \
+ \
+ /* the DMA stage fills only the columns inside [0, IW): zero the rest of the window */ \
+ const int32_t lo = win0 < 0 ? -win0 : 0; \
+ const int32_t hi = MIN((int32_t) Wb, (int32_t) IW - win0); \
+ if (iih < 0 || iih >= (int32_t) IH || hi <= lo) { \
+ for (uint32_t iic = 0; iic < IC; iic++) { \
+ hvx_splat_f32_u(srcb + (size_t) iic * Wb, 0, Wb); \
+ } \
+ } else if (lo > 0 || hi < (int32_t) Wb) { \
+ for (uint32_t iic = 0; iic < IC; iic++) { \
+ float * row = srcb + (size_t) iic * Wb; \
+ if (lo > 0) { \
+ hvx_splat_f32_u(row, 0, (uint32_t) lo); \
+ } \
+ if (hi < (int32_t) Wb) { \
+ hvx_splat_f32_u(row + hi, 0, Wb - (uint32_t) hi); \
+ } \
+ } \
+ } \
+ \
+ const HVX_Vector * table = (const HVX_Vector *) scratch; \
+ float * row = scratch + (size_t) ng * 32; \
+ const uint32_t region = IC * Wb * sizeof(float); \
+ \
+ for (uint32_t j = 0; j < nb; j++) { \
+ const uint32_t shift = j * (uint32_t) s0 * sizeof(float); \
+ const size_t base = (size_t) ((const uint8_t *) srcb + shift); \
+ const uint32_t mu = region - shift - 1; \
+ for (uint32_t g = 0; g < ng; g++) { \
+ Q6_vgather_ARMVw((HVX_Vector *) (row + g * 32), base, mu, table[g]); \
+ } \
+ hvx_gather_sync(row + (ng - 1) * 32); \
+ DST_CTYPE * out = dstb + (size_t) j * patch_stride; \
+ COPY_FN((uint8_t *) out, (const uint8_t *) row, patch_stride); \
+ } \
+}
+
+IM2COL_GATHER_COMPUTE_BODY(im2col_gather_compute_f16, uint16_t, hvx_copy_f16_f32_uu)
+IM2COL_GATHER_COMPUTE_BODY(im2col_gather_compute_f32, float, hvx_copy_f32_uu)
+
static void im2col_blocked_dma_thread(unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct htp_im2col_context * ictx = (struct htp_im2col_context *) data;
@@ -234,24 +340,24 @@ static void im2col_blocked_dma_thread(unsigned int nth, unsigned int ith, void *
float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) };
void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes };
float * scratch = (float *) (srcb_base + 2 * ictx->pe_src_row_bytes);
- const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows;
- const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith;
- const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max);
- if (row_start >= row_end) {
- return;
- }
- const uint32_t nrows_local = row_end - row_start;
- const uint32_t total_blocks = nrows_local * nbpr;
- if (total_blocks == 0) {
+ // work is split over output-col blocks, so a single output row (1D) still uses every thread
+ const uint32_t blk_end_max = ictx->pe_nrows * nbpr;
+ const uint32_t blk_start = ictx->pe_blocks_per_thread * ith;
+ const uint32_t blk_end = MIN(blk_start + ictx->pe_blocks_per_thread, blk_end_max);
+ if (blk_start >= blk_end) {
return;
}
+ const uint32_t total_blocks = blk_end - blk_start;
struct im2col_cursor cur = {
- .r = row_start,
+ .r = ictx->thread_in[ith] * ictx->OH + ictx->thread_ioh[ith],
.in = ictx->thread_in[ith],
.iod = 0,
.ioh = ictx->thread_ioh[ith],
- .c0 = 0,
+ .c0 = ictx->thread_c0[ith],
};
+ if (KH == 1) {
+ im2col_gather_table_build(ictx, (HVX_Vector *) scratch);
+ }
struct im2col_cursor stage = cur;
for (uint32_t i = 0; i < total_blocks && i < 2; i++) {
dma_queue_push(dma_q, dma_make_data(dst_data, dstb2[i]), 0, 0, 0, 0);
@@ -307,6 +413,37 @@ static void im2col_patchembed_compute_f16(
const uint32_t KH = ictx->KH, KW = ictx->KW;
const uint32_t OW = ictx->OW, patch_stride = ictx->patch_stride;
+ if (KW == 16 && (KH % 4 == 0)) {
+ const HVX_VectorPred q_lo = Q6_Q_vsetq_R(64);
+ for (uint32_t iow = 0; iow < OW; iow++) {
+ uint16_t * dst_patch = dstb + (size_t) iow * patch_stride;
+ for (uint32_t iic = 0; iic < IC; iic++) {
+ for (uint32_t ikh = 0; ikh < KH; ikh += 4) {
+ const int32_t iih0 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 0);
+ const int32_t iih1 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 1);
+ const int32_t iih2 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 2);
+ const int32_t iih3 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 3);
+
+ const float * s0 = srcb + (size_t) (iic * KH + ikh + 0) * IW + (size_t) iow * 16;
+ const float * s1 = srcb + (size_t) (iic * KH + ikh + 1) * IW + (size_t) iow * 16;
+ const float * s2 = srcb + (size_t) (iic * KH + ikh + 2) * IW + (size_t) iow * 16;
+ const float * s3 = srcb + (size_t) (iic * KH + ikh + 3) * IW + (size_t) iow * 16;
+
+ const HVX_Vector v0 = (iih0 >= 0 && iih0 < (int32_t) IH) ? *((const HVX_UVector *) s0) : Q6_V_vzero();
+ const HVX_Vector v1 = (iih1 >= 0 && iih1 < (int32_t) IH) ? *((const HVX_UVector *) s1) : Q6_V_vzero();
+ const HVX_Vector v2 = (iih2 >= 0 && iih2 < (int32_t) IH) ? *((const HVX_UVector *) s2) : Q6_V_vzero();
+ const HVX_Vector v3 = (iih3 >= 0 && iih3 < (int32_t) IH) ? *((const HVX_UVector *) s3) : Q6_V_vzero();
+
+ const HVX_Vector w01 = Q6_V_vmux_QVV(q_lo, v0, Q6_V_vror_VR(v1, 64));
+ const HVX_Vector w23 = Q6_V_vmux_QVV(q_lo, v2, Q6_V_vror_VR(v3, 64));
+
+ *((HVX_Vector *) (dst_patch + (size_t) (iic * KH + ikh) * 16)) = hvx_vec_f32_to_f16(w01, w23);
+ }
+ }
+ }
+ return;
+ }
+
for (uint32_t iow = 0; iow < OW; iow++) {
uint16_t * dst_patch = dstb + (size_t) iow * patch_stride;
for (uint32_t ikh = 0; ikh < KH; ikh++) {
@@ -332,6 +469,30 @@ static void im2col_patchembed_compute_f32(
const uint32_t KH = ictx->KH, KW = ictx->KW;
const uint32_t OW = ictx->OW, patch_stride = ictx->patch_stride;
+ if (KW == 16 && (KH % 2 == 0)) {
+ const HVX_VectorPred q_lo = Q6_Q_vsetq_R(64);
+ for (uint32_t iow = 0; iow < OW; iow++) {
+ float * dst_patch = dstb + (size_t) iow * patch_stride;
+ for (uint32_t iic = 0; iic < IC; iic++) {
+ for (uint32_t ikh = 0; ikh < KH; ikh += 2) {
+ const int32_t iih0 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 0);
+ const int32_t iih1 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 1);
+
+ const float * s0 = srcb + (size_t) (iic * KH + ikh + 0) * IW + (size_t) iow * 16;
+ const float * s1 = srcb + (size_t) (iic * KH + ikh + 1) * IW + (size_t) iow * 16;
+
+ const HVX_Vector v0 = (iih0 >= 0 && iih0 < (int32_t) IH) ? *((const HVX_UVector *) s0) : Q6_V_vzero();
+ const HVX_Vector v1 = (iih1 >= 0 && iih1 < (int32_t) IH) ? *((const HVX_UVector *) s1) : Q6_V_vzero();
+
+ const HVX_Vector w01 = Q6_V_vmux_QVV(q_lo, v0, Q6_V_vror_VR(v1, 64));
+
+ *((HVX_Vector *) (dst_patch + (size_t) (iic * KH + ikh) * 16)) = w01;
+ }
+ }
+ }
+ return;
+ }
+
for (uint32_t iow = 0; iow < OW; iow++) {
float * dst_patch = dstb + (size_t) iow * patch_stride;
for (uint32_t ikh = 0; ikh < KH; ikh++) {
@@ -784,12 +945,16 @@ int op_im2col(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nrows))) {
+ // the blocked path splits work over output-col blocks, so a single output row (1D) can use every thread
+ const bool blocked = kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA;
+ const uint32_t nwork = blocked ? nrows * kparams->nbpr : nrows;
+ if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nwork))) {
return HTP_STATUS_INVAL_PARAMS;
}
struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads);
+ htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads,
+ htp_im2col_scratch_bytes(kparams->kernel_type, kparams->KH, kparams->patch_stride));
struct htp_im2col_context ictx = { 0 };
uint8_t * const base = octx->ctx->vtcm_base;
@@ -799,6 +964,7 @@ int op_im2col(struct htp_ops_context * octx) {
ictx.pe_row_base = row_base;
ictx.pe_nrows = nrows;
ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div);
+ ictx.pe_blocks_per_thread = fastdiv(nwork + pth - 1, &octx->n_threads_div);
ictx.pe_src_row_bytes = kparams->src_row_bytes;
ictx.pe_dst_row_bytes = kparams->dst_row_bytes;
ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src);
@@ -820,12 +986,22 @@ int op_im2col(struct htp_ops_context * octx) {
ictx.OH = kparams->OH;
ictx.OW = kparams->OW;
ictx.patch_stride = kparams->patch_stride;
+ ictx.pe_gather_nvec = hex_round_up(kparams->patch_stride, 32) / 32;
+ // a thread starts at a row, or at an output-col block (possibly mid-row) on the blocked path
for (uint32_t ith = 0; ith < pth; ith++) {
- const uint32_t r = row_base + ictx.pe_rows_per_thread * ith;
+ uint32_t r = row_base + ictx.pe_rows_per_thread * ith;
+ uint32_t c0 = 0;
+ if (blocked) {
+ const uint32_t blk = ictx.pe_blocks_per_thread * ith;
+ const uint32_t q = fastdiv(blk, &kparams->div_nbpr);
+ r = row_base + q;
+ c0 = (blk - q * kparams->nbpr) * kparams->owb;
+ }
const uint32_t in = fastdiv(r, &kparams->div_OH);
ictx.thread_ioh[ith] = r - in * kparams->OH;
ictx.thread_in[ith] = in;
+ ictx.thread_c0[ith] = c0;
}
ictx.dst_elem = dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float);
@@ -835,9 +1011,14 @@ int op_im2col(struct htp_ops_context * octx) {
: (void *) im2col_patchembed_compute_f32;
work_queue_run(octx->ctx->work_queue, im2col_patchembed_dma_thread, &ictx, pth);
return HTP_STATUS_OK;
- } else if (kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA) {
- ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_blocked_compute_f16
- : (void *) im2col_blocked_compute_f32;
+ } else if (blocked) {
+ if (kparams->KH == 1) {
+ ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_gather_compute_f16
+ : (void *) im2col_gather_compute_f32;
+ } else {
+ ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_blocked_compute_f16
+ : (void *) im2col_blocked_compute_f32;
+ }
work_queue_run(octx->ctx->work_queue, im2col_blocked_dma_thread, &ictx, pth);
return HTP_STATUS_OK;
}
@@ -873,7 +1054,7 @@ int op_im2col_3d(struct htp_ops_context * octx) {
}
struct htp_im2col_vtcm_layout layout;
- htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads);
+ htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads, 256);
struct htp_im2col_context ictx = { 0 };
uint8_t * const base = octx->ctx->vtcm_base;
diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.h b/ggml/src/ggml-hexagon/htp/im2col-ops.h
index 85aa3ca35..b34e6a09d 100644
--- a/ggml/src/ggml-hexagon/htp/im2col-ops.h
+++ b/ggml/src/ggml-hexagon/htp/im2col-ops.h
@@ -20,13 +20,15 @@ struct htp_im2col_vtcm_layout {
uint32_t dst_bytes_per_thread;
};
+// scratch_bytes: per-thread VTCM after the two src buffers (at least 256)
static inline void htp_im2col_vtcm_layout_build(
struct htp_im2col_vtcm_layout * L,
uint32_t src_row_bytes,
uint32_t dst_row_bytes,
- uint32_t n_threads) {
+ uint32_t n_threads,
+ uint32_t scratch_bytes) {
- L->src_bytes_per_thread = 2 * src_row_bytes + 256;
+ L->src_bytes_per_thread = 2 * src_row_bytes + (scratch_bytes > 256 ? scratch_bytes : 256);
L->dst_bytes_per_thread = 2 * dst_row_bytes;
size_t off = 0;
@@ -35,6 +37,15 @@ static inline void htp_im2col_vtcm_layout_build(
L->total_bytes = off;
}
+// per-thread scratch: the KH == 1 blocked path keeps its offset table and one gathered row there,
+// both padded to whole vectors
+static inline uint32_t htp_im2col_scratch_bytes(uint32_t kernel_type, uint32_t KH, uint32_t patch_stride) {
+ if (kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA && KH == 1) {
+ return 2 * hex_round_up(patch_stride, 32) * (uint32_t) sizeof(float);
+ }
+ return 256;
+}
+
struct htp_im2col_kernel_params {
uint8_t kernel_type;
uint8_t n_threads;
@@ -58,6 +69,7 @@ struct htp_im2col_kernel_params {
struct fastdiv_values div_OH;
struct fastdiv_values div_OD;
+ struct fastdiv_values div_nbpr;
};
#if defined(__cplusplus)
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index e2c3daf25..f33fe177e 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -10317,6 +10317,20 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false));
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false));
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {3000, 384, 1, 1}, {3, 384, 384, 1}, 1, 0, 1, 0, 1, 0, false));
+ // 1D convs of an audio decoder: long sequences, dilated taps, pointwise kernels
+ for (int d0 : {1, 3, 9}) {
+ test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {2000 + 6 * d0, 96, 1, 1}, {7, 96, 96, 1}, 1, 0, 0, 0, d0, 0, false));
+ }
+ test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {1500, 192, 1, 1}, {1, 192, 192, 1}, 1, 0, 0, 0, 1, 0, false));
+ test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {1001, 64, 1, 1}, {5, 64, 64, 1}, 2, 0, 2, 0, 1, 0, false));
+ test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {777, 48, 2, 1}, {3, 48, 48, 1}, 1, 0, 4, 0, 3, 0, false));
+ // several rows of a few output-col blocks each: a thread's range of blocks can start mid-row
+ for (int64_t n : {5, 7}) {
+ test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {600, 8, n, 1}, {3, 8, 8, 1}, 1, 0, 1, 0, 1, 0, false));
+ }
+ // 2D with a kernel one row tall: with p1 > 0 some output rows fall outside the input height
+ test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {40, 6, 8, 1}, {3, 1, 8, 4}, 1, 1, 1, 1, 1, 1, true));
+ test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {40, 6, 8, 1}, {3, 1, 8, 4}, 1, 1, 1, 0, 1, 1, true));
for (int s0 : {1, 3}) {
for (int p0 : {0, 3}) {
for (int d0 : {1, 3}) {