Commit 38645c2ad for llama.cpp
commit 38645c2ad972964be37ff65310dfc15ef121ab8d
Author: Titaniumtown <titaniumtown@proton.me>
Date: Mon Oct 26 07:12:56 2026 -0700
sycl: template the per-quant concat impls (#31008)
diff --git a/ggml/src/ggml-sycl/concat.cpp b/ggml/src/ggml-sycl/concat.cpp
index d4c257cf7..77a17412f 100644
--- a/ggml/src/ggml-sycl/concat.cpp
+++ b/ggml/src/ggml-sycl/concat.cpp
@@ -196,7 +196,8 @@ void concat_impl_sycl(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
}
}
-static void concat_impl_q4_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
+template <typename block_t, int qk, ggml_type qtype>
+static void concat_impl_q_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
@@ -204,25 +205,25 @@ static void concat_impl_q4_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor *
const int32_t dim = ((int32_t *) dst->op_params)[0];
- GGML_ASSERT(src0->type == GGML_TYPE_Q4_0);
- GGML_ASSERT(src1->type == GGML_TYPE_Q4_0);
- GGML_ASSERT(dst->type == GGML_TYPE_Q4_0);
- GGML_ASSERT(src0->ne[0] % QK4_0 == 0);
- GGML_ASSERT(src1->ne[0] % QK4_0 == 0);
- GGML_ASSERT(dst->ne[0] % QK4_0 == 0);
+ GGML_ASSERT(src0->type == qtype);
+ GGML_ASSERT(src1->type == qtype);
+ GGML_ASSERT(dst->type == qtype);
+ GGML_ASSERT(src0->ne[0] % qk == 0);
+ GGML_ASSERT(src1->ne[0] % qk == 0);
+ GGML_ASSERT(dst->ne[0] % qk == 0);
- const int ne00_blk = src0->ne[0] / QK4_0;
- const int ne0_blk = dst->ne[0] / QK4_0;
+ const int ne00_blk = src0->ne[0] / qk;
+ const int ne0_blk = dst->ne[0] / qk;
if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
- const block_q4_0 * src0_d = (const block_q4_0 *) src0->data;
- const block_q4_0 * src1_d = (const block_q4_0 *) src1->data;
- block_q4_0 * dst_d = (block_q4_0 *) dst->data;
- const size_t type_size = sizeof(block_q4_0);
+ const block_t * src0_d = (const block_t *) src0->data;
+ const block_t * src1_d = (const block_t *) src1->data;
+ block_t * dst_d = (block_t *) dst->data;
+ const size_t type_size = sizeof(block_t);
if (dim != 3) {
for (int i3 = 0; i3 < dst->ne[3]; i3++) {
- concat_T_sycl<block_q4_0>(
+ concat_T_sycl<block_t>(
src0_d + i3 * (src0->nb[3] / type_size),
src1_d + i3 * (src1->nb[3] / type_size),
dst_d + i3 * (dst->nb[3] / type_size),
@@ -237,223 +238,12 @@ static void concat_impl_q4_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor *
SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
}
} else {
- concat_T_sycl_non_cont<block_q4_0>(
+ concat_T_sycl_non_cont<block_t>(
stream, (const char *) src0->data, (const char *) src1->data,
(char *) dst->data,
ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
- src1->ne[0] / QK4_0, src1->ne[1], src1->ne[2], src1->ne[3],
- src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
- ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
- dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
- }
-}
-
-static void concat_impl_q4_1_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
- scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
- const ggml_tensor * src0 = dst->src[0];
- const ggml_tensor * src1 = dst->src[1];
- queue_ptr stream = ctx.stream();
-
- const int32_t dim = ((int32_t *) dst->op_params)[0];
-
- GGML_ASSERT(src0->type == GGML_TYPE_Q4_1);
- GGML_ASSERT(src1->type == GGML_TYPE_Q4_1);
- GGML_ASSERT(dst->type == GGML_TYPE_Q4_1);
- GGML_ASSERT(src0->ne[0] % QK4_1 == 0);
- GGML_ASSERT(src1->ne[0] % QK4_1 == 0);
- GGML_ASSERT(dst->ne[0] % QK4_1 == 0);
-
- const int ne00_blk = src0->ne[0] / QK4_1;
- const int ne0_blk = dst->ne[0] / QK4_1;
-
- if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
- const block_q4_1 * src0_d = (const block_q4_1 *) src0->data;
- const block_q4_1 * src1_d = (const block_q4_1 *) src1->data;
- block_q4_1 * dst_d = (block_q4_1 *) dst->data;
- const size_t type_size = sizeof(block_q4_1);
-
- if (dim != 3) {
- for (int i3 = 0; i3 < dst->ne[3]; i3++) {
- concat_T_sycl<block_q4_1>(
- src0_d + i3 * (src0->nb[3] / type_size),
- src1_d + i3 * (src1->nb[3] / type_size),
- dst_d + i3 * (dst->nb[3] / type_size),
- ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
- dst->ne[1], dst->ne[2], dim, stream);
- }
- } else {
- const size_t size0 = ggml_nbytes(src0);
- const size_t size1 = ggml_nbytes(src1);
-
- SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
- SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
- }
- } else {
- concat_T_sycl_non_cont<block_q4_1>(
- stream, (const char *) src0->data, (const char *) src1->data,
- (char *) dst->data,
- ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
- src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
- src1->ne[0] / QK4_1, src1->ne[1], src1->ne[2], src1->ne[3],
- src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
- ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
- dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
- }
-}
-
-static void concat_impl_q5_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
- scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
- const ggml_tensor * src0 = dst->src[0];
- const ggml_tensor * src1 = dst->src[1];
- queue_ptr stream = ctx.stream();
-
- const int32_t dim = ((int32_t *) dst->op_params)[0];
-
- GGML_ASSERT(src0->type == GGML_TYPE_Q5_0);
- GGML_ASSERT(src1->type == GGML_TYPE_Q5_0);
- GGML_ASSERT(dst->type == GGML_TYPE_Q5_0);
- GGML_ASSERT(src0->ne[0] % QK5_0 == 0);
- GGML_ASSERT(src1->ne[0] % QK5_0 == 0);
- GGML_ASSERT(dst->ne[0] % QK5_0 == 0);
-
- const int ne00_blk = src0->ne[0] / QK5_0;
- const int ne0_blk = dst->ne[0] / QK5_0;
-
- if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
- const block_q5_0 * src0_d = (const block_q5_0 *) src0->data;
- const block_q5_0 * src1_d = (const block_q5_0 *) src1->data;
- block_q5_0 * dst_d = (block_q5_0 *) dst->data;
- const size_t type_size = sizeof(block_q5_0);
-
- if (dim != 3) {
- for (int i3 = 0; i3 < dst->ne[3]; i3++) {
- concat_T_sycl<block_q5_0>(
- src0_d + i3 * (src0->nb[3] / type_size),
- src1_d + i3 * (src1->nb[3] / type_size),
- dst_d + i3 * (dst->nb[3] / type_size),
- ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
- dst->ne[1], dst->ne[2], dim, stream);
- }
- } else {
- const size_t size0 = ggml_nbytes(src0);
- const size_t size1 = ggml_nbytes(src1);
-
- SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
- SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
- }
- } else {
- concat_T_sycl_non_cont<block_q5_0>(
- stream, (const char *) src0->data, (const char *) src1->data,
- (char *) dst->data,
- ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
- src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
- src1->ne[0] / QK5_0, src1->ne[1], src1->ne[2], src1->ne[3],
- src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
- ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
- dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
- }
-}
-
-static void concat_impl_q5_1_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
- scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
- const ggml_tensor * src0 = dst->src[0];
- const ggml_tensor * src1 = dst->src[1];
- queue_ptr stream = ctx.stream();
-
- const int32_t dim = ((int32_t *) dst->op_params)[0];
-
- GGML_ASSERT(src0->type == GGML_TYPE_Q5_1);
- GGML_ASSERT(src1->type == GGML_TYPE_Q5_1);
- GGML_ASSERT(dst->type == GGML_TYPE_Q5_1);
- GGML_ASSERT(src0->ne[0] % QK5_1 == 0);
- GGML_ASSERT(src1->ne[0] % QK5_1 == 0);
- GGML_ASSERT(dst->ne[0] % QK5_1 == 0);
-
- const int ne00_blk = src0->ne[0] / QK5_1;
- const int ne0_blk = dst->ne[0] / QK5_1;
-
- if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
- const block_q5_1 * src0_d = (const block_q5_1 *) src0->data;
- const block_q5_1 * src1_d = (const block_q5_1 *) src1->data;
- block_q5_1 * dst_d = (block_q5_1 *) dst->data;
- const size_t type_size = sizeof(block_q5_1);
-
- if (dim != 3) {
- for (int i3 = 0; i3 < dst->ne[3]; i3++) {
- concat_T_sycl<block_q5_1>(
- src0_d + i3 * (src0->nb[3] / type_size),
- src1_d + i3 * (src1->nb[3] / type_size),
- dst_d + i3 * (dst->nb[3] / type_size),
- ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
- dst->ne[1], dst->ne[2], dim, stream);
- }
- } else {
- const size_t size0 = ggml_nbytes(src0);
- const size_t size1 = ggml_nbytes(src1);
-
- SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
- SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
- }
- } else {
- concat_T_sycl_non_cont<block_q5_1>(
- stream, (const char *) src0->data, (const char *) src1->data,
- (char *) dst->data,
- ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
- src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
- src1->ne[0] / QK5_1, src1->ne[1], src1->ne[2], src1->ne[3],
- src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
- ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
- dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
- }
-}
-
-static void concat_impl_q8_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
- scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
- const ggml_tensor * src0 = dst->src[0];
- const ggml_tensor * src1 = dst->src[1];
- queue_ptr stream = ctx.stream();
-
- const int32_t dim = ((int32_t *) dst->op_params)[0];
-
- GGML_ASSERT(src0->type == GGML_TYPE_Q8_0);
- GGML_ASSERT(src1->type == GGML_TYPE_Q8_0);
- GGML_ASSERT(dst->type == GGML_TYPE_Q8_0);
- GGML_ASSERT(src0->ne[0] % QK8_0 == 0);
- GGML_ASSERT(src1->ne[0] % QK8_0 == 0);
- GGML_ASSERT(dst->ne[0] % QK8_0 == 0);
-
- const int ne00_blk = src0->ne[0] / QK8_0;
- const int ne0_blk = dst->ne[0] / QK8_0;
-
- if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
- const block_q8_0 * src0_d = (const block_q8_0 *) src0->data;
- const block_q8_0 * src1_d = (const block_q8_0 *) src1->data;
- block_q8_0 * dst_d = (block_q8_0 *) dst->data;
- const size_t type_size = sizeof(block_q8_0);
-
- if (dim != 3) {
- for (int i3 = 0; i3 < dst->ne[3]; i3++) {
- concat_T_sycl<block_q8_0>(
- src0_d + i3 * (src0->nb[3] / type_size),
- src1_d + i3 * (src1->nb[3] / type_size),
- dst_d + i3 * (dst->nb[3] / type_size),
- ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
- dst->ne[1], dst->ne[2], dim, stream);
- }
- } else {
- const size_t size0 = ggml_nbytes(src0);
- const size_t size1 = ggml_nbytes(src1);
- SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
- SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
- }
- } else {
- concat_T_sycl_non_cont<block_q8_0>(
- stream, (const char *) src0->data, (const char *) src1->data,
- (char *) dst->data,
- ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
- src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
- src1->ne[0] / QK8_0, src1->ne[1], src1->ne[2], src1->ne[3],
+ src1->ne[0] / qk, src1->ne[1], src1->ne[2], src1->ne[3],
src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
@@ -487,19 +277,19 @@ void ggml_sycl_op_concat(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
concat_impl_sycl<int8_t>(ctx, dst);
break;
case GGML_TYPE_Q4_0:
- concat_impl_q4_0_sycl(ctx, dst);
+ concat_impl_q_sycl<block_q4_0, QK4_0, GGML_TYPE_Q4_0>(ctx, dst);
break;
case GGML_TYPE_Q4_1:
- concat_impl_q4_1_sycl(ctx, dst);
+ concat_impl_q_sycl<block_q4_1, QK4_1, GGML_TYPE_Q4_1>(ctx, dst);
break;
case GGML_TYPE_Q5_0:
- concat_impl_q5_0_sycl(ctx, dst);
+ concat_impl_q_sycl<block_q5_0, QK5_0, GGML_TYPE_Q5_0>(ctx, dst);
break;
case GGML_TYPE_Q5_1:
- concat_impl_q5_1_sycl(ctx, dst);
+ concat_impl_q_sycl<block_q5_1, QK5_1, GGML_TYPE_Q5_1>(ctx, dst);
break;
case GGML_TYPE_Q8_0:
- concat_impl_q8_0_sycl(ctx, dst);
+ concat_impl_q_sycl<block_q8_0, QK8_0, GGML_TYPE_Q8_0>(ctx, dst);
break;
default:
fprintf(stderr, "%s: unsupported types: dst: %s\n", __func__, ggml_type_name(dst->type));