Commit 4413a1924 for llama.cpp
commit 4413a192410fad1f054e9216114087a3e01527a0
Author: kurquhar <kurquhar@qti.qualcomm.com>
Date: Mon Oct 12 20:33:05 2026 -0700
hexagon: extend scalar DMA broadcasting support for binary ops (#30245)
* tests: cover weighted scalar broadcasts
Assisted-by: OpenCode
* ggml : support extended scalar DMA binary ops
Assisted-by: OpenCode
* hex-binary: remove all non-dma paths
* hex-binary: add support for non-contig strides and cleanup dma preable a bit
* hex-binary: cleanup the gating and simplify dma retry
* hex-geglu-erf: add a gate for F32 only, we dont support F16 for this op
* tests: cover weighted repeat broadcast
Assisted-by: OpenCode
* hex-binary: update REPEAT and COMPLEX kernels to use dma as well (missed them last time)
* hex-add-id: add dma support for src2 (idx)
---------
Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index a32ed5919..1c1c92da2 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -344,7 +344,8 @@ static bool ggml_hexagon_precompute_binary_params(
const struct ggml_tensor * src0,
const struct ggml_tensor * src1,
const struct ggml_tensor * dst,
- struct htp_binary_kernel_params * kparams
+ struct htp_binary_kernel_params * kparams,
+ const struct ggml_tensor * src2 = nullptr
);
static void ggml_hexagon_precompute_get_rows_params(
@@ -5864,7 +5865,8 @@ static bool ggml_hexagon_precompute_binary_params(
const struct ggml_tensor * src0,
const struct ggml_tensor * src1,
const struct ggml_tensor * dst,
- struct htp_binary_kernel_params * kparams
+ struct htp_binary_kernel_params * kparams,
+ const struct ggml_tensor * src2
) {
memset(kparams, 0, sizeof(*kparams));
@@ -5877,14 +5879,17 @@ static bool ggml_hexagon_precompute_binary_params(
const size_t src1_row_size_aligned = hex_round_up(src1_row_size, 128);
const size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128);
+ if (!htp_binary_strides_safe(src0, src1, dst, elem_size)) {
+ return false;
+ }
+
const bool is_add_id = op == HTP_OP_ADD_ID;
const bool is_scalar = !is_add_id && src1->ne[0] == 1;
- const bool is_transposed = src0->nb[1] < src0_row_size || src1->nb[1] < src1_row_size || dst->nb[1] < dst_row_size;
- const bool is_row_bcast = !is_add_id && !is_scalar && !is_transposed &&
+ const bool is_row_bcast = !is_add_id && !is_scalar &&
src1->ne[0] == src0->ne[0] &&
(src0->ne[1] > 1 || src0->ne[2] > 1 || src0->ne[3] > 1) &&
src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1;
- const bool is_same_shape = !is_add_id && !is_scalar && !is_transposed &&
+ const bool is_same_shape = !is_add_id && !is_scalar &&
src1->ne[0] == src0->ne[0] &&
src1->ne[1] == src0->ne[1] &&
(src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) &&
@@ -5925,28 +5930,42 @@ static bool ggml_hexagon_precompute_binary_params(
enum htp_binary_kernel_type kernel_type;
size_t src1_size = 0;
+ size_t src2_size = 0;
if (is_add_id) {
+ GGML_ASSERT(src2 != nullptr);
kernel_type = HTP_BINARY_KERNEL_ADD_ID;
src1_size = hex_round_up(src1->ne[1] * src1_row_size_aligned, 128);
+ src2_size = hex_round_up(ggml_nelements(src2) * sizeof(int32_t), 128);
+ if (src1_size + src2_size > sess->vtcm_size) {
+ return false;
+ }
} else if (is_row_bcast) {
kernel_type = HTP_BINARY_KERNEL_ROW_BCAST;
src1_size = src1_row_size_aligned;
} else if (is_scalar) {
- const bool is_scalar_static = (src1->ne[2] == 1 && src1->ne[3] == 1) &&
- (src1->ne[1] == 1 || src1->nb[1] == elem_size);
- if (is_scalar_static) {
- kernel_type = HTP_BINARY_KERNEL_SCALAR_DMA;
- src1_size = hex_round_up(src1->ne[1] * elem_size, 128);
- } else {
- kernel_type = HTP_BINARY_KERNEL_SCALAR;
+ src1_size = hex_round_up(ggml_nelements(src1) * elem_size, 128);
+ if (src1_size > sess->vtcm_size) {
+ return false;
}
+ kernel_type = HTP_BINARY_KERNEL_SCALAR;
+ kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;
} else if (is_same_shape) {
kernel_type = HTP_BINARY_KERNEL_SAME_SHAPE;
} else if (is_complex) {
+ src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128);
+ if (src1_size > sess->vtcm_size) {
+ return false;
+ }
kernel_type = HTP_BINARY_KERNEL_COMPLEX;
+ kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;
} else {
+ src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128);
+ if (src1_size > sess->vtcm_size) {
+ return false;
+ }
kernel_type = HTP_BINARY_KERNEL_REPEAT;
+ kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;
}
kparams->kernel_type = kernel_type;
@@ -5955,6 +5974,39 @@ static bool ggml_hexagon_precompute_binary_params(
kparams->src1_row_size_aligned = src1_row_size_aligned;
kparams->dst_row_size_aligned = dst_row_size_aligned;
kparams->src1_size = src1_size;
+ kparams->src2_size = src2_size;
+
+ const bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]);
+ const bool src1_contig_dim1 = (src1->nb[2] == src1->ne[1] * src1->nb[1]);
+ const bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]);
+
+ const bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]);
+ const bool src1_contig_dim2 = (src1->nb[3] == src1->ne[2] * src1->nb[2]);
+ const bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]);
+
+ bool split_at_ne01 = false;
+ if (src0->ne[2] > 1) {
+ if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {
+ split_at_ne01 = (src1->ne[2] != src0->ne[2]) || !src0_contig_dim1 || !src1_contig_dim1 || !dst_contig_dim1;
+ } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {
+ split_at_ne01 = !src0_contig_dim1 || !dst_contig_dim1;
+ } else {
+ split_at_ne01 = true;
+ }
+ }
+ kparams->split_at_ne01 = split_at_ne01 ? 1 : 0;
+
+ bool split_at_ne02 = false;
+ if (src0->ne[3] > 1) {
+ if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {
+ split_at_ne02 = (src1->ne[3] != src0->ne[3]) || !src0_contig_dim2 || !src1_contig_dim2 || !dst_contig_dim2;
+ } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {
+ split_at_ne02 = !src0_contig_dim2 || !dst_contig_dim2;
+ } else {
+ split_at_ne02 = true;
+ }
+ }
+ kparams->split_at_ne02 = split_at_ne02 ? 1 : 0;
struct htp_binary_vtcm_layout L;
htp_binary_vtcm_layout_build(&L, kparams, sess->vtcm_size);
@@ -7139,9 +7191,12 @@ static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * se
if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {
return false;
}
+ if (src2->nb[1] < src2->ne[0] * sizeof(int32_t)) {
+ return false;
+ }
struct htp_binary_kernel_params kparams;
- return ggml_hexagon_precompute_binary_params(sess, HTP_OP_ADD_ID, src0, src1, dst, &kparams);
+ return ggml_hexagon_precompute_binary_params(sess, HTP_OP_ADD_ID, src0, src1, dst, &kparams, src2);
}
static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -8504,9 +8559,10 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
);
} else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {
const ggml_tensor * src1 = node.node->src[1];
+ const ggml_tensor * src2 = (node.opcode == HTP_OP_ADD_ID) ? node.node->src[2] : nullptr;
GGML_ASSERT(ggml_hexagon_precompute_binary_params(sess,
node.opcode, node.node->src[0], src1, node.node,
- (struct htp_binary_kernel_params *) node.kernel_params));
+ (struct htp_binary_kernel_params *) node.kernel_params, src2));
} else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) {
ggml_hexagon_precompute_flash_attn_params(sess,
node.node,
@@ -9391,9 +9447,11 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
case GGML_GLU_OP_SWIGLU_OAI:
case GGML_GLU_OP_SWIGLU_CLAMP:
case GGML_GLU_OP_GEGLU:
+ supp = ggml_hexagon_supported_activations(sess, op);
+ break;
case GGML_GLU_OP_GEGLU_QUICK:
case GGML_GLU_OP_GEGLU_ERF:
- supp = ggml_hexagon_supported_activations(sess, op);
+ supp = (op->src[0]->type == GGML_TYPE_F32) && ggml_hexagon_supported_activations(sess, op);
break;
default:
supp = false;
diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.c
index 1286176af..78ce4bb63 100644
--- a/ggml/src/ggml-hexagon/htp/binary-ops.c
+++ b/ggml/src/ggml-hexagon/htp/binary-ops.c
@@ -103,13 +103,13 @@ static inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_
// Out-of-line compute micro-kernels
-typedef void (*compute_scalar_dma_t)(
+typedef void (*compute_scalar_t)(
uint8_t * dst, const uint8_t * src0, const void * s1_table,
uint32_t cur_i11, uint32_t ne11, uint32_t n_rows,
size_t dst_stride, size_t src0_stride, uint32_t ne00);
-#define DEFINE_COMPUTE_SCALAR_DMA(NAME, TYPE, HVX_STMT) \
-static void compute_scalar_dma_##NAME( \
+#define DEFINE_COMPUTE_SCALAR(NAME, TYPE, HVX_STMT) \
+static void compute_scalar_##NAME( \
uint8_t * dst, const uint8_t * src0, const void * s1_table, \
uint32_t cur_i11, uint32_t ne11, uint32_t n_rows, \
size_t dst_stride, size_t src0_stride, uint32_t ne00) { \
@@ -125,31 +125,6 @@ static void compute_scalar_dma_##NAME( \
} \
}
-DEFINE_COMPUTE_SCALAR_DMA(add_f32, float, hvx_add_scalar_f32_aa(r_dst, r_src0, val, ne00))
-DEFINE_COMPUTE_SCALAR_DMA(add_f16, _Float16, hvx_add_scalar_f16_aa(r_dst, r_src0, val, ne00))
-DEFINE_COMPUTE_SCALAR_DMA(sub_f32, float, hvx_sub_scalar_f32_aa(r_dst, r_src0, val, ne00))
-DEFINE_COMPUTE_SCALAR_DMA(sub_f16, _Float16, hvx_sub_scalar_f16_aa(r_dst, r_src0, val, ne00))
-DEFINE_COMPUTE_SCALAR_DMA(mul_f32, float, hvx_mul_scalar_f32_aa(r_dst, r_src0, val, ne00))
-DEFINE_COMPUTE_SCALAR_DMA(mul_f16, _Float16, hvx_mul_scalar_f16_aa(r_dst, r_src0, val, ne00))
-DEFINE_COMPUTE_SCALAR_DMA(div_f32, float, hvx_mul_scalar_f32_aa(r_dst, r_src0, 1.0f / (val), ne00))
-DEFINE_COMPUTE_SCALAR_DMA(div_f16, _Float16, hvx_div_scalar_f16_aa(r_dst, r_src0, val, ne00))
-
-typedef void (*compute_scalar_t)(
- uint8_t * dst, const uint8_t * src0, const uint8_t * src1_ptr, uint32_t s1_stride,
- uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00);
-
-#define DEFINE_COMPUTE_SCALAR(NAME, TYPE, HVX_STMT) \
-static void compute_scalar_##NAME( \
- uint8_t * dst, const uint8_t * src0, const uint8_t * src1_ptr, uint32_t s1_stride, \
- uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { \
- for (uint32_t r = 0; r < n_rows; r++) { \
- uint8_t * r_dst = dst + r * dst_stride; \
- const uint8_t * r_src0 = src0 + r * src0_stride; \
- TYPE val = *(const TYPE *)(src1_ptr + r * s1_stride); \
- HVX_STMT; \
- } \
-}
-
DEFINE_COMPUTE_SCALAR(add_f32, float, hvx_add_scalar_f32_aa(r_dst, r_src0, val, ne00))
DEFINE_COMPUTE_SCALAR(add_f16, _Float16, hvx_add_scalar_f16_aa(r_dst, r_src0, val, ne00))
DEFINE_COMPUTE_SCALAR(sub_f32, float, hvx_sub_scalar_f32_aa(r_dst, r_src0, val, ne00))
@@ -205,17 +180,17 @@ DEFINE_COMPUTE_ROW_BCAST(div_f16, hvx_div_f16_aaa)
typedef void (*compute_complex_t)(
uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane,
- uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11,
+ uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride,
uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00);
#define DEFINE_COMPUTE_COMPLEX(NAME, HVX_FN) \
static void compute_complex_##NAME( \
uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, \
- uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, \
+ uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, \
uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { \
for (uint32_t r = 0; r < n_rows; r++) { \
uint32_t i11 = fastmodulo(i01 + r, ne11, div11); \
- const uint8_t * r_src1 = src1_plane + i11 * nb11; \
+ const uint8_t * r_src1 = src1_plane + i11 * src1_stride; \
HVX_FN(dst + r * dst_stride, src0 + r * src0_stride, r_src1, ne00); \
} \
}
@@ -231,17 +206,17 @@ DEFINE_COMPUTE_COMPLEX(div_f16, hvx_div_f16_aau)
typedef void (*compute_repeat_t)(
uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane,
- uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11,
+ uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride,
uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00, uint32_t ne10);
#define DEFINE_COMPUTE_REPEAT(NAME, TYPE, HVX_FN) \
static void compute_repeat_##NAME( \
uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, \
- uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, \
+ uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, \
uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00, uint32_t ne10) { \
for (uint32_t r = 0; r < n_rows; r++) { \
uint32_t i11 = fastmodulo(i01 + r, ne11, div11); \
- const uint8_t * r_src1_row = src1_plane + i11 * nb11; \
+ const uint8_t * r_src1_row = src1_plane + i11 * src1_stride; \
uint8_t * r_dst = dst + r * dst_stride; \
const uint8_t * r_src0 = src0 + r * src0_stride; \
for (uint32_t c = 0; c < ne00; c += ne10) { \
@@ -261,17 +236,17 @@ DEFINE_COMPUTE_REPEAT(div_f32, float, hvx_div_f32_uuu)
DEFINE_COMPUTE_REPEAT(div_f16, _Float16, hvx_div_f16_uuu)
typedef void (*compute_add_id_t)(
- uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const char * src2_data,
- uint32_t i01, uint32_t i02, uint32_t nb20, uint32_t nb21, uint32_t src1_stride,
+ uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const int32_t * src2_ids,
+ uint32_t i01, uint32_t i02, uint32_t ne01, uint32_t src1_stride,
uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00);
static void compute_add_id_f32(
- uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const char * src2_data,
- uint32_t i01, uint32_t i02, uint32_t nb20, uint32_t nb21, uint32_t src1_stride,
+ uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const int32_t * src2_ids,
+ uint32_t i01, uint32_t i02, uint32_t ne01, uint32_t src1_stride,
uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) {
for (uint32_t r = 0; r < n_rows; r++) {
uint32_t r_i01 = i01 + r;
- const int32_t idx = *(const int32_t *)(src2_data + r_i01 * nb20 + i02 * nb21);
+ const int32_t idx = src2_ids[i02 * ne01 + r_i01];
if (idx < 0) {
memcpy(dst + r * dst_stride, src0 + r * src0_stride, ne00 * sizeof(float));
continue;
@@ -283,18 +258,22 @@ static void compute_add_id_f32(
}
}
-// 1a. Scalar src1 in VTCM via DMA (ne10 == 1, ne12 == 1, ne13 == 1)
-static void binary_thread_scalar_dma(unsigned int nth, unsigned int ith, void * data) {
+// 1. Scalar src1 in VTCM via DMA (ne10 == 1)
+static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data) {
struct htp_binary_context * bctx = (struct htp_binary_context *) data;
struct htp_ops_context * octx = bctx->octx;
htp_binary_preamble;
+ const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16);
+ const size_t plane_stride = (size_t) ne12 * ne11 * elem_size;
+ const size_t row_stride = (size_t) ne11 * elem_size;
+
const uint32_t row_size_bytes = bctx->row_size_bytes;
const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;
const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);
if (start_row >= end_row) return;
- FARF(HIGH, "binary-scalar-dma: %d/%d (%u:%u) row-size %u (%u)",
+ FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)",
ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);
const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout;
@@ -329,94 +308,6 @@ static void binary_thread_scalar_dma(unsigned int nth, unsigned int ith, void *
spad_idx ^= 1;
}
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
- compute_scalar_dma_t compute = (compute_scalar_dma_t) bctx->compute;
-
- for (uint32_t ir = start_row; ir < end_row; ) {
- uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);
-
- uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src;
- uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst;
-
- uint32_t i03, i02, i01, rem;
- i03 = fastdiv(ir, &bctx->src0_dim12_div);
- rem = ir - i03 * (ne02 * ne01);
- i02 = fastdiv(rem, &bctx->src0_dim1_div);
- i01 = rem - i02 * ne01;
-
- uint32_t cur_i11 = fastmodulo(i01, ne11, &bctx->src1_dim1_div);
-
- htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
- compute(d_spad, s0_spad, s1_table, cur_i11, ne11, current_block_size,
- bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00);
- htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
-
- dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;
- dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes,
- current_block_size);
-
- if (ir_prefetch < end_row) {
- uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);
- uint32_t p03, p02, p01, prem;
- p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div);
- prem = ir_prefetch - p03 * (ne02 * ne01);
- p02 = fastdiv(prem, &bctx->src0_dim1_div);
- p01 = prem - p02 * ne01;
- dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01;
- dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes,
- next_block_size);
- ir_prefetch += next_block_size;
- }
- ir += current_block_size;
- }
-
- dma_queue_flush(dma_q);
-}
-
-// 1b. Scalar src1 dynamic / pointer (ne10 == 1)
-static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data) {
- struct htp_binary_context * bctx = (struct htp_binary_context *) data;
- struct htp_ops_context * octx = bctx->octx;
- htp_binary_preamble;
-
- const uint32_t row_size_bytes = bctx->row_size_bytes;
- const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;
- const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);
- if (start_row >= end_row) return;
-
- FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)",
- ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);
-
- const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout;
- uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread);
- uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);
- size_t src0_spad_half = layout->src0_spad_half_size;
- size_t dst_spad_half = layout->dst_spad_half_size;
-
- dma_queue * dma_q = octx->ctx->dma[ith];
- uint32_t ir_prefetch = start_row;
- int spad_idx = 0;
-
- for (int k = 0; k < 2 && ir_prefetch < end_row; k++) {
- uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);
- uint32_t i03, i02, i01, rem;
- i03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div);
- rem = ir_prefetch - i03 * (ne02 * ne01);
- i02 = fastdiv(rem, &bctx->src0_dim1_div);
- i01 = rem - i02 * ne01;
-
- dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01;
- dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;
-
- uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half;
- uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half;
-
- dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0);
- dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size);
- ir_prefetch += current_block_size;
- spad_idx ^= 1;
- }
-
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
compute_scalar_t compute = (compute_scalar_t) bctx->compute;
@@ -435,17 +326,17 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data
uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div);
uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div);
uint32_t i11 = fastmodulo(i01, ne11, &bctx->src1_dim1_div);
-
- const uint8_t * src1_ptr = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11;
- uint32_t s1_stride = (ne11 == 1) ? 0 : nb11;
+ const uint8_t * scalar_table = (const uint8_t *) s1_table +
+ i13 * plane_stride + i12 * row_stride;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
- compute(d_spad, s0_spad, src1_ptr, s1_stride, current_block_size,
+ compute(d_spad, s0_spad, scalar_table, i11, ne11, current_block_size,
bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;
- dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size);
+ dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes,
+ current_block_size);
if (ir_prefetch < end_row) {
uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);
@@ -455,7 +346,8 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data
p02 = fastdiv(prem, &bctx->src0_dim1_div);
p01 = prem - p02 * ne01;
dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01;
- dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size);
+ dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes,
+ next_block_size);
ir_prefetch += next_block_size;
}
ir += current_block_size;
@@ -668,6 +560,12 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi
uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);
size_t src0_spad_half = layout->src0_spad_half_size;
size_t dst_spad_half = layout->dst_spad_half_size;
+ const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1);
+
+ const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16);
+ const size_t vtcm_row_stride = (size_t) ne10 * elem_size;
+ const size_t vtcm_plane_stride = (size_t) ne11 * vtcm_row_stride;
+ const size_t vtcm_vol_stride = (size_t) ne12 * vtcm_plane_stride;
dma_queue * dma_q = octx->ctx->dma[ith];
uint32_t ir_prefetch = start_row;
@@ -707,10 +605,10 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi
uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div);
uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div);
- const uint8_t * src1_plane = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12;
+ const uint8_t * src1_plane = vtcm_src1 + i13 * vtcm_vol_stride + i12 * vtcm_plane_stride;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
- compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, nb11,
+ compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, vtcm_row_stride,
current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
@@ -750,6 +648,12 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi
uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);
size_t src0_spad_half = layout->src0_spad_half_size;
size_t dst_spad_half = layout->dst_spad_half_size;
+ const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1);
+
+ const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16);
+ const size_t vtcm_row_stride = (size_t) ne10 * elem_size;
+ const size_t vtcm_plane_stride = (size_t) ne11 * vtcm_row_stride;
+ const size_t vtcm_vol_stride = (size_t) ne12 * vtcm_plane_stride;
FARF(HIGH, "binary-repeat: %d/%d (%u:%u) row-size %u (%u)",
ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);
@@ -792,10 +696,10 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi
uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div);
uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div);
- const uint8_t * src1_plane = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12;
+ const uint8_t * src1_plane = vtcm_src1 + i13 * vtcm_vol_stride + i12 * vtcm_plane_stride;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
- compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, nb11,
+ compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, vtcm_row_stride,
current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00, ne10);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
@@ -826,7 +730,6 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void *
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
- const struct htp_tensor * src2 = octx->src[2];
const struct htp_tensor * dst = octx->dst;
const uint32_t ne00 = src0->ne[0];
@@ -852,6 +755,7 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void *
uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread);
uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);
const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1);
+ const int32_t * vtcm_src2 = VTCM_LAYOUT_PTR(const int32_t, bctx->vtcm_base, layout->off_src2);
size_t src0_spad_half = layout->src0_spad_half_size;
size_t dst_spad_half = layout->dst_spad_half_size;
@@ -892,8 +796,8 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void *
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
compute_add_id_t compute = (compute_add_id_t) bctx->compute;
- compute(d_spad, s0_spad, vtcm_src1, (const char *)(uintptr_t)src2->data,
- i01, i02, src2->nb[0], src2->nb[1], src1_stride,
+ compute(d_spad, s0_spad, vtcm_src1, vtcm_src2,
+ i01, i02, ne01, src1_stride,
current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
@@ -1234,19 +1138,6 @@ static int execute_op_binary(struct htp_ops_context * octx) {
const size_t src1_row_size_aligned = kparams->src1_row_size_aligned;
const size_t dst_row_size_aligned = kparams->dst_row_size_aligned;
- if (htp_tensor_is_extended(src1)) {
- if (kparams->kernel_type != HTP_BINARY_KERNEL_SAME_SHAPE &&
- kparams->kernel_type != HTP_BINARY_KERNEL_ROW_BCAST &&
- kparams->kernel_type != HTP_BINARY_KERNEL_SCALAR_DMA &&
- kparams->kernel_type != HTP_BINARY_KERNEL_ADD_ID) {
- return HTP_STATUS_NO_SUPPORT;
- }
- }
-
- if (octx->op == HTP_OP_ADD_ID && htp_tensor_is_extended(octx->src[2])) {
- return HTP_STATUS_NO_SUPPORT;
- }
-
struct htp_binary_context bctx;
bctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
htp_binary_vtcm_layout_build(&bctx.vtcm_layout, kparams, octx->ctx->vtcm_size);
@@ -1259,12 +1150,38 @@ static int execute_op_binary(struct htp_ops_context * octx) {
uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src1);
if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {
dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[0] * elem_size, 1);
- } else if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR_DMA) {
- dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[1] * elem_size, 1);
+ } else if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR ||
+ kparams->kernel_type == HTP_BINARY_KERNEL_REPEAT ||
+ kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX) {
+ if (!kparams->src1_strided) {
+ const size_t src1_bytes = (size_t) src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3] * elem_size;
+ dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1_bytes, 1);
+ } else {
+ const uint32_t ne11 = src1->ne[1];
+ const uint32_t ne12 = src1->ne[2];
+ const uint32_t ne13 = src1->ne[3];
+ const size_t row_bytes = src1->ne[0] * elem_size;
+ for (uint32_t i13 = 0; i13 < ne13; i13++) {
+ for (uint32_t i12 = 0; i12 < ne12; i12++) {
+ dma_addr_t src_addr = src1->data + i13 * src1->nb[3] + i12 * src1->nb[2];
+ uint8_t * dst_ptr = vtcm_src1 + (i13 * ne12 * ne11 + i12 * ne11) * row_bytes;
+ while (!dma_queue_push(dma_q, dma_make_data(dst_ptr, src_addr),
+ row_bytes, src1->nb[1], row_bytes, ne11)) {
+ dma_queue_pop(dma_q);
+ }
+ }
+ }
+ }
} else if (kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) {
+ const struct htp_tensor * src2 = octx->src[2];
+ uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2);
dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data),
kparams->src1_row_size_aligned, src1->nb[1],
src1->ne[0] * elem_size, src1->ne[1]);
+ const size_t src2_row_bytes = src2->ne[0] * sizeof(int32_t);
+ dma_queue_push(dma_q, dma_make_data(vtcm_src2, src2->data),
+ src2_row_bytes, src2->nb[1],
+ src2_row_bytes, src2->ne[1]);
}
bctx.octx = octx;
@@ -1285,15 +1202,8 @@ static int execute_op_binary(struct htp_ops_context * octx) {
bctx.src1_dim2_div = init_fastdiv_values(src1->ne[2]);
bctx.src1_dim3_div = init_fastdiv_values(src1->ne[3]);
- bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]);
- bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]);
-
- bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]);
- bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]);
-
- bctx.split_at_ne01 = (octx->op == HTP_OP_ADD_ID) ||
- ((src0->ne[2] > 1) && ((src1->ne[1] > 1) || (src1->ne[2] > 1) || !src0_contig_dim1 || !dst_contig_dim1));
- bctx.split_at_ne02 = (src0->ne[3] > 1) && ((src1->ne[2] > 1) || (src1->ne[3] > 1) || !src0_contig_dim2 || !dst_contig_dim2);
+ bctx.split_at_ne01 = kparams->split_at_ne01;
+ bctx.split_at_ne02 = kparams->split_at_ne02;
worker_callback_t worker_func = NULL;
void * compute_func = NULL;
@@ -1339,26 +1249,6 @@ static int execute_op_binary(struct htp_ops_context * octx) {
}
}
break;
- case HTP_BINARY_KERNEL_SCALAR_DMA:
- worker_func = binary_thread_scalar_dma;
- if (src0_type == HTP_TYPE_F32) {
- switch (octx->op) {
- case HTP_OP_ADD: compute_func = compute_scalar_dma_add_f32; break;
- case HTP_OP_SUB: compute_func = compute_scalar_dma_sub_f32; break;
- case HTP_OP_MUL: compute_func = compute_scalar_dma_mul_f32; break;
- case HTP_OP_DIV: compute_func = compute_scalar_dma_div_f32; break;
- default: break;
- }
- } else if (src0_type == HTP_TYPE_F16) {
- switch (octx->op) {
- case HTP_OP_ADD: compute_func = compute_scalar_dma_add_f16; break;
- case HTP_OP_SUB: compute_func = compute_scalar_dma_sub_f16; break;
- case HTP_OP_MUL: compute_func = compute_scalar_dma_mul_f16; break;
- case HTP_OP_DIV: compute_func = compute_scalar_dma_div_f16; break;
- default: break;
- }
- }
- break;
case HTP_BINARY_KERNEL_SCALAR:
worker_func = binary_thread_scalar;
if (src0_type == HTP_TYPE_F32) {
@@ -1435,9 +1325,11 @@ static int execute_op_binary(struct htp_ops_context * octx) {
bctx.compute = compute_func;
if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST ||
- kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR_DMA ||
+ kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR ||
+ kparams->kernel_type == HTP_BINARY_KERNEL_REPEAT ||
+ kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX ||
kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) {
- dma_queue_pop(dma_q);
+ dma_queue_flush(dma_q);
}
work_queue_run(octx->ctx->work_queue, worker_func, &bctx, n_threads);
@@ -1446,13 +1338,6 @@ static int execute_op_binary(struct htp_ops_context * octx) {
}
int op_binary(struct htp_ops_context * octx) {
-
- // Does not support permutations of src1
- const struct htp_tensor * src1 = octx->src[1];
- if (src1->nb[1] < src1->nb[0]) {
- return HTP_STATUS_NO_SUPPORT;
- }
-
const uint32_t src0_type = octx->src[0]->type;
if ((src0_type == HTP_TYPE_F32) || (src0_type == HTP_TYPE_F16)) {
return execute_op_binary(octx);
diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.h b/ggml/src/ggml-hexagon/htp/binary-ops.h
index aa8ff4959..db3d51489 100644
--- a/ggml/src/ggml-hexagon/htp/binary-ops.h
+++ b/ggml/src/ggml-hexagon/htp/binary-ops.h
@@ -12,7 +12,6 @@
enum htp_binary_kernel_type {
HTP_BINARY_KERNEL_SAME_SHAPE = 0,
HTP_BINARY_KERNEL_ROW_BCAST,
- HTP_BINARY_KERNEL_SCALAR_DMA,
HTP_BINARY_KERNEL_SCALAR,
HTP_BINARY_KERNEL_ADD_ID,
HTP_BINARY_KERNEL_COMPLEX,
@@ -30,15 +29,41 @@ struct htp_binary_kernel_params {
uint32_t dst_row_size_aligned;
uint32_t src1_size;
+ uint32_t src2_size;
uint32_t vtcm_size;
uint32_t chunk_size;
uint32_t chunk_bytes;
uint32_t is_scalar;
+
+ uint8_t split_at_ne01;
+ uint8_t split_at_ne02;
+ uint8_t src1_strided;
+ uint8_t pad;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_binary_kernel_params) <= 128, "htp_binary_kernel_params is too large for kernel_params blob");
+
+struct ggml_tensor;
+static inline bool htp_binary_strides_safe(
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * dst,
+ size_t elem_size
+) {
+ if ((src0->ne[0] != 1 && src0->nb[0] != elem_size) ||
+ (src1->ne[0] != 1 && src1->nb[0] != elem_size) ||
+ (dst->ne[0] != 1 && dst->nb[0] != elem_size)) {
+ return false;
+ }
+ if ((src0->ne[1] > 1 && src0->nb[1] < src0->ne[0] * elem_size) ||
+ (src1->ne[1] > 1 && src1->nb[1] < src1->ne[0] * elem_size) ||
+ (dst->ne[1] > 1 && dst->nb[1] < dst->ne[0] * elem_size)) {
+ return false;
+ }
+ return true;
+}
#else
_Static_assert(sizeof(struct htp_binary_kernel_params) <= 128, "htp_binary_kernel_params is too large for kernel_params blob");
#endif
@@ -47,6 +72,7 @@ struct htp_binary_vtcm_layout {
size_t total_bytes;
size_t off_src0;
size_t off_src1;
+ size_t off_src2;
size_t off_dst;
size_t src0_bytes_per_thread;
@@ -58,6 +84,7 @@ struct htp_binary_vtcm_layout {
size_t dst_spad_half_size;
size_t src1_size;
+ size_t src2_size;
uint32_t rows_per_buffer;
};
@@ -111,17 +138,19 @@ static inline void htp_binary_vtcm_layout_build(
? 2 * (kparams->src0_row_size_aligned + kparams->src1_row_size_aligned + kparams->dst_row_size_aligned)
: 2 * (kparams->src0_row_size_aligned + kparams->dst_row_size_aligned);
- if (spad_row_total == 0 || vtcm_size < kparams->src1_size) {
+ const size_t static_vtcm_size = kparams->src1_size + kparams->src2_size;
+ if (spad_row_total == 0 || vtcm_size < static_vtcm_size) {
return;
}
- const size_t rows_per_buffer = (vtcm_size - kparams->src1_size) / (n_threads * spad_row_total);
+ const size_t rows_per_buffer = (vtcm_size - static_vtcm_size) / (n_threads * spad_row_total);
if (rows_per_buffer == 0) {
return;
}
L->rows_per_buffer = (uint32_t) rows_per_buffer;
L->src1_size = kparams->src1_size;
+ L->src2_size = kparams->src2_size;
L->src0_bytes_per_thread = rows_per_buffer * 2 * kparams->src0_row_size_aligned;
L->dst_bytes_per_thread = rows_per_buffer * 2 * kparams->dst_row_size_aligned;
@@ -137,11 +166,13 @@ static inline void htp_binary_vtcm_layout_build(
const size_t src1_total = (kparams->src1_size > 0)
? kparams->src1_size
: n_threads * L->src1_bytes_per_thread;
+ const size_t src2_total = kparams->src2_size;
const size_t dst_total = n_threads * L->dst_bytes_per_thread;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src0, src0_total);
VTCM_LAYOUT_ALLOC(off, off_src1, src1_total);
+ VTCM_LAYOUT_ALLOC(off, off_src2, src2_total);
VTCM_LAYOUT_ALLOC(off, off_dst, dst_total);
L->total_bytes = off;
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index bd599e99e..b54abbea8 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -3303,6 +3303,7 @@ struct test_bin_bcast : public test_case {
bool perm1; // permute src1?
bool src_overlap; // src0 and src1 are overlapping views of the same buffer
const ggml_type type_b; // type of src1, GGML_TYPE_COUNT -> same as src0
+ bool weights;
bool run_whole_graph() override { return nf > 1; }
@@ -3311,6 +3312,9 @@ struct test_bin_bcast : public test_case {
if (type_b != GGML_TYPE_COUNT) {
s += "," + VAR_TO_STR(type_b);
}
+ if (weights) {
+ s += ",weights=1";
+ }
return s;
}
@@ -3323,28 +3327,38 @@ struct test_bin_bcast : public test_case {
std::array<int, 4> nr = {1, 2, 1, 1},
int nf = 1,
bool perm1 = false, bool src_overlap = false,
- ggml_type type_b = GGML_TYPE_COUNT)
- : op(op), type(type), ne(ne), nr(nr), nf(nf), perm1(perm1), src_overlap(src_overlap), type_b(type_b) {}
+ ggml_type type_b = GGML_TYPE_COUNT, bool weights = false)
+ : op(op), type(type), ne(ne), nr(nr), nf(nf), perm1(perm1), src_overlap(src_overlap), type_b(type_b), weights(weights) {}
+
+ bool use_weight_context() override {
+ return weights;
+ }
ggml_tensor * build_graph(ggml_context * ctx) override {
+ return build_graph(ctx, ctx);
+ }
+
+ ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override {
GGML_ASSERT(nf <= 16);
ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0]*nr[0], ne[1]*nr[1], ne[2]*nr[2], ne[3]*nr[3]);
ggml_set_name(a, "a");
const ggml_type tb = type_b == GGML_TYPE_COUNT ? type : type_b;
+ ggml_context * ctx_b = weights ? ctx_weights : ctx;
+ GGML_ASSERT(ctx_b);
ggml_tensor * b[16];
for (int i = 0; i < nf; ++i) {
if (perm1) {
const int p[4] = { 1, 2, 0, 3 }; // hardcoded for now
- b[i] = ggml_new_tensor_4d(ctx, tb, ne[p[0]], ne[p[1]], ne[p[2]], ne[p[3]]);
- b[i] = ggml_permute(ctx, b[i], p[0], p[1], p[2], p[3]);
+ b[i] = ggml_new_tensor_4d(ctx_b, tb, ne[p[0]], ne[p[1]], ne[p[2]], ne[p[3]]);
+ b[i] = ggml_permute(ctx_b, b[i], p[0], p[1], p[2], p[3]);
} else if (src_overlap) {
b[i] = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], 2 * (ne[3] / 3), a->nb[1], a->nb[2], a->nb[3], (ne[3] / 3) * a->nb[3]);
} else {
- b[i] = ggml_new_tensor(ctx, tb, 4, ne.data());
+ b[i] = ggml_new_tensor(ctx_b, tb, 4, ne.data());
}
ggml_set_name(b[i], (std::string("b") + std::to_string(i)).c_str());
}
@@ -10387,6 +10401,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
//add_test_bin_bcast(type, {3, 3, 2560, 1280}, {2, 1, 1, 1});
}
+ for (auto op : {ggml_add, ggml_mul}) {
+ test_cases.emplace_back(new test_bin_bcast(op, GGML_TYPE_F32, {1, 2, 2, 2}, {17, 1, 1, 1}, 1, false, false, GGML_TYPE_COUNT, true));
+ }
+ test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {2, 2, 2, 2}, {17, 1, 1, 1}, 1, false, false, GGML_TYPE_COUNT, true));
+
// single inplace tests, especially important for WebGPU backend since kernels for inplace vs. not are different
test_cases.emplace_back(new test_bin_bcast(ggml_add_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16));
test_cases.emplace_back(new test_bin_bcast(ggml_mul_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16));