Commit a4d4f2fd4 for llama.cpp
commit a4d4f2fd4ec85ec6a0a9ece4870416c1b6cc24f5
Author: Aaron Teo <aaron.teo1@ibm.com>
Date: Mon Nov 9 21:48:32 2026 +0800
ggml-zdnn: accept non-zdnn tensors, invalidate compute tensors after every op (#31824)
Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>
diff --git a/ggml/src/ggml-zdnn/fattn.cpp b/ggml/src/ggml-zdnn/fattn.cpp
index e877b7a75..e6a0c24ca 100644
--- a/ggml/src/ggml-zdnn/fattn.cpp
+++ b/ggml/src/ggml-zdnn/fattn.cpp
@@ -246,10 +246,4 @@ void ggml_zdnn_flash_attn_ext(ggml_backend_zdnn_context * ctx, ggml_tensor * dst
memcpy((char *) dst->data + h*dst->nb[1] + iq*dst->nb[2], o_host + (h*n_q + iq)*d_v, d_v*sizeof(float));
}
}
-
- // the result is written to dst->data directly, so the dst ztensor is stale
- ggml_backend_zdnn_buffer * dst_extra = (ggml_backend_zdnn_buffer *) dst->extra;
- if (dst_extra != nullptr) {
- dst_extra->ztensor.is_transformed = false;
- }
}
diff --git a/ggml/src/ggml-zdnn/ggml-zdnn.cpp b/ggml/src/ggml-zdnn/ggml-zdnn.cpp
index 27415935b..533aefd9c 100644
--- a/ggml/src/ggml-zdnn/ggml-zdnn.cpp
+++ b/ggml/src/ggml-zdnn/ggml-zdnn.cpp
@@ -95,6 +95,10 @@ static int64_t ggml_zdnn_mul_mat_min_batch(ggml_type type, int32_t n_threads) {
}
}
+static bool ggml_zdnn_is_tensor_own(const ggml_tensor * tensor) {
+ return tensor->buffer != nullptr && tensor->buffer->buft == ggml_backend_zdnn_buffer_type();
+}
+
static bool ggml_zdnn_supports_op(const ggml_backend_zdnn_device_context * ctx_dev, const ggml_tensor * op) {
switch (op->op) {
case GGML_OP_NONE:
@@ -127,14 +131,24 @@ static bool ggml_zdnn_supports_op(const ggml_backend_zdnn_device_context * ctx_d
return false;
}
+ // tensors with no buffer yet get a zDNN buffer, computed or graph input weights and tensors in other buffers have no ztensor
+ if (weights->op != GGML_OP_NONE || (weights->flags & GGML_TENSOR_FLAG_INPUT) ||
+ (weights->buffer != nullptr && !ggml_zdnn_is_tensor_own(weights)) ||
+ (op->buffer != nullptr && !ggml_zdnn_is_tensor_own(op))) {
+ return false;
+ }
+
+ // training writes allocated weights through ->data, test-backend-ops sets PARAM before it allocates and uses set_tensor
+ if ((weights->flags & GGML_TENSOR_FLAG_PARAM) && weights->buffer != nullptr) {
+ return false;
+ }
+
switch (weights->type) {
case GGML_TYPE_F32:
case GGML_TYPE_F16:
case GGML_TYPE_BF16:
- // the inputs ztensor is used, and views have none
- return inputs->view_src == nullptr;
+ return true;
case GGML_TYPE_Q8_0:
- // the inputs are quantized from their data, so views work as well
return ctx_dev->has_parmblkformat_1 && inputs->type == GGML_TYPE_F32;
default:
return false;
@@ -274,6 +288,11 @@ static enum ggml_status ggml_backend_zdnn_buffer_init_tensor(ggml_backend_buffer
return GGML_STATUS_SUCCESS;
}
+ // ops read other compute tensors through ->data
+ if (tensor->op != GGML_OP_NONE && tensor->op != GGML_OP_MUL_MAT) {
+ return GGML_STATUS_SUCCESS;
+ }
+
ggml_backend_zdnn_buffer_context * ctx = (ggml_backend_zdnn_buffer_context *)buffer->context;
const int64_t tsize = ggml_nbytes(tensor);
@@ -324,23 +343,27 @@ static enum ggml_status ggml_backend_zdnn_buffer_init_tensor(ggml_backend_buffer
GGML_UNUSED(buffer_idx);
}
+static void ggml_zdnn_reset_tensor(const ggml_tensor * tensor) {
+ const ggml_tensor * src = tensor->view_src != nullptr ? tensor->view_src : tensor;
+
+ ggml_backend_zdnn_buffer * extra = (ggml_backend_zdnn_buffer *)src->extra;
+ if (extra != nullptr) {
+ zdnn_reset_ztensor(&extra->ztensor);
+ }
+}
+
static void ggml_backend_zdnn_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
memset((char *)tensor->data + offset, value, size);
+ ggml_zdnn_reset_tensor(tensor);
+
GGML_UNUSED(buffer);
}
static void ggml_backend_zdnn_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
memcpy((char *)tensor->data + offset, data, size);
- // views share the ztensor of their source, which is now stale, and empty tensors have none
- if (tensor->view_src != nullptr) {
- ggml_backend_zdnn_buffer * src_extra = (ggml_backend_zdnn_buffer *)tensor->view_src->extra;
- if (src_extra != nullptr && src_extra->ztensor.is_transformed) {
- zdnn_reset_ztensor(&src_extra->ztensor);
- }
- return;
- }
+ ggml_zdnn_reset_tensor(tensor);
ggml_backend_zdnn_buffer * extra = (ggml_backend_zdnn_buffer *)tensor->extra;
if (extra == nullptr) {
@@ -355,12 +378,10 @@ static void ggml_backend_zdnn_buffer_set_tensor(ggml_backend_buffer_t buffer, gg
// Fixes the LLAMA_SET_ROWS bug
// see: https://github.com/ggml-org/llama.cpp/issues/15414
- // only weights are transformed here, others on first use, as not every op reads the ztensor and some data (masks) is not finite
- if (tensor->buffer->usage != GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
- zdnn_reset_ztensor(&extra->ztensor);
- return;
+ // only complete writes to weights are transformed here, others on first use, as not every op reads the ztensor and some data (masks) is not finite
+ if (tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && size == ggml_nbytes(tensor)) {
+ ggml_zdnn_load_tensor(extra->ztensor, tensor->data);
}
- if (extra->ztensor.is_transformed == false) ggml_zdnn_load_tensor(extra->ztensor, tensor->data);
GGML_UNUSED(buffer);
}
@@ -375,6 +396,17 @@ static void ggml_backend_zdnn_buffer_clear(ggml_backend_buffer_t buffer, uint8_t
ggml_backend_zdnn_buffer_context * ctx = (ggml_backend_zdnn_buffer_context *)buffer->context;
memset(ctx->all_data, value, ctx->all_size);
+
+ for (const auto & buf_ptr : ctx->buffers) {
+ zdnn_reset_ztensor(&buf_ptr->ztensor);
+ }
+
+ // the bias has its own data, and no op transforms it on first use
+ for (const auto & buf_ptr : ctx->buffers) {
+ if (buf_ptr->extra != nullptr) {
+ ggml_zdnn_load_tensor(buf_ptr->extra->ztensor, buf_ptr->extra->data);
+ }
+ }
}
static void ggml_backend_zdnn_buffer_reset(ggml_backend_buffer_t buffer) {
@@ -620,8 +652,7 @@ static bool ggml_backend_zdnn_device_supports_op(ggml_backend_dev_t dev, const g
}
static bool ggml_backend_zdnn_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
- return
- buft->iface.get_name == ggml_backend_zdnn_buffer_type_get_name;
+ return ggml_backend_buft_is_host(buft);
GGML_UNUSED(dev);
}
diff --git a/ggml/src/ggml-zdnn/mmf.cpp b/ggml/src/ggml-zdnn/mmf.cpp
index 0551fb88b..acd443e5b 100644
--- a/ggml/src/ggml-zdnn/mmf.cpp
+++ b/ggml/src/ggml-zdnn/mmf.cpp
@@ -3,7 +3,7 @@
#include "utils.hpp"
void ggml_zdnn_mul_mat_f(
- const ggml_backend_zdnn_context * ctx,
+ ggml_backend_zdnn_context * ctx,
const ggml_tensor * src0,
const ggml_tensor * src1,
ggml_tensor * dst) {
@@ -31,10 +31,14 @@ void ggml_zdnn_mul_mat_f(
ggml_tensor * output = dst;
ggml_backend_zdnn_buffer * weights_extra = (ggml_backend_zdnn_buffer *)weights->extra;
- ggml_backend_zdnn_buffer * inputs_extra = (ggml_backend_zdnn_buffer *)inputs->extra;
ggml_backend_zdnn_buffer * output_extra = (ggml_backend_zdnn_buffer *)output->extra;
ggml_backend_zdnn_buffer * bias_extra = (ggml_backend_zdnn_buffer *)output_extra->extra;
+ zdnn_tensor_desc inputs_pre_tfm_desc;
+ zdnn_tensor_desc inputs_tfm_desc;
+ zdnn_ztensor inputs_ztensor;
+ ggml_zdnn_pool_alloc<uint8_t> inputs_ztensor_alloc;
+
const int64_t weights_rows = ne01;
const int64_t weights_cols = ne00;
const int64_t inputs_rows = ne11;
@@ -53,37 +57,29 @@ void ggml_zdnn_mul_mat_f(
// weights_extra->pre_tfm_desc.dim3,
// weights_extra->pre_tfm_desc.dim4);
- // GGML_LOG_INFO("%s: tensor '%s' tensor dimensions: [%ld, %ld, %ld, %ld] pre_tfm_desc dimensions: [%ld, %ld, %ld, %ld]\n",
- // __func__, inputs_extra->name,
- // inputs->ne[3], inputs->ne[2], inputs->ne[1], inputs->ne[0],
- // inputs_extra->pre_tfm_desc.dim1,
- // inputs_extra->pre_tfm_desc.dim2,
- // inputs_extra->pre_tfm_desc.dim3,
- // inputs_extra->pre_tfm_desc.dim4);
-
GGML_ASSERT(weights_extra->pre_tfm_desc.dim1 == weights->ne[0] && "weights_extra->pre_tfm_desc.dim1 must match weights->ne[0]");
GGML_ASSERT(weights_extra->pre_tfm_desc.dim2 == weights->ne[1] && "weights_extra->pre_tfm_desc.dim2 must match weights->ne[1]");
- GGML_ASSERT(inputs_extra->pre_tfm_desc.dim1 == inputs->ne[0] && "inputs_extra->pre_tfm_desc.dim1 must match inputs->ne[0]");
- GGML_ASSERT(inputs_extra->pre_tfm_desc.dim2 == inputs->ne[1] && "inputs_extra->pre_tfm_desc.dim2 must match inputs->ne[1]");
- // compute tensors are transformed on first use
+ // weights read straight into a host buffer skip set_tensor, so transform them on first use
if (!weights_extra->ztensor.is_transformed) {
ggml_zdnn_load_tensor(weights_extra->ztensor, weights->data);
}
- if (!inputs_extra->ztensor.is_transformed) {
- ggml_zdnn_load_tensor(inputs_extra->ztensor, inputs->data);
+
+ // the inputs can be in any host buffer or be a view, so transform them from their data on every call
+ {
+ zdnn_init_pre_transformed_desc(ZDNN_2D, ggml_zdnn_type_mapping(inputs->type), &inputs_pre_tfm_desc, inputs_rows, inputs_cols);
+ ZDNN_CHECK(zdnn_generate_transformed_desc(&inputs_pre_tfm_desc, &inputs_tfm_desc));
+ ggml_zdnn_init_scratch_ztensor_dlf16(ctx, &inputs_pre_tfm_desc, &inputs_tfm_desc, &inputs_ztensor, inputs_ztensor_alloc);
+ ggml_zdnn_load_tensor(inputs_ztensor, inputs->data);
}
- ZDNN_CHECK(zdnn_matmul_transpose_op(&inputs_extra->ztensor, &weights_extra->ztensor, &bias_extra->ztensor,
+ ZDNN_CHECK(zdnn_matmul_transpose_op(&inputs_ztensor, &weights_extra->ztensor, &bias_extra->ztensor,
false, true, MATMUL_OP_ADDITION, &output_extra->ztensor));
// TODO: Remove in the future as we are currently DLF16 -> FP32 then in the next op, FP32 -> DLF16 again. Inefficient.
ZDNN_CHECK(zdnn_transform_origtensor(&output_extra->ztensor, output->data));
- GGML_UNUSED(ctx);
GGML_UNUSED(weights_rows);
GGML_UNUSED(weights_cols);
- GGML_UNUSED(inputs_rows);
- GGML_UNUSED(inputs_cols);
GGML_UNUSED(output_rows);
GGML_UNUSED(output_cols);
}
diff --git a/ggml/src/ggml-zdnn/mmf.hpp b/ggml/src/ggml-zdnn/mmf.hpp
index a12f1b8f8..afc02bc0e 100644
--- a/ggml/src/ggml-zdnn/mmf.hpp
+++ b/ggml/src/ggml-zdnn/mmf.hpp
@@ -4,7 +4,7 @@
#include "common.hpp"
void ggml_zdnn_mul_mat_f(
- const ggml_backend_zdnn_context * ctx,
+ ggml_backend_zdnn_context * ctx,
const ggml_tensor * src0,
const ggml_tensor * src1,
ggml_tensor * dst);
diff --git a/ggml/src/ggml-zdnn/mmq.cpp b/ggml/src/ggml-zdnn/mmq.cpp
index 0c4472845..285479b41 100644
--- a/ggml/src/ggml-zdnn/mmq.cpp
+++ b/ggml/src/ggml-zdnn/mmq.cpp
@@ -188,7 +188,6 @@ void ggml_zdnn_mul_mat_q(
GGML_ASSERT(inputs->type == GGML_TYPE_F32);
ggml_backend_zdnn_buffer * weights_extra = (ggml_backend_zdnn_buffer *)weights->extra;
- ggml_backend_zdnn_buffer * output_extra = (ggml_backend_zdnn_buffer *)output->extra;
// weights read straight into a host buffer skip set_tensor, so transform them on first use
if (!weights_extra->ztensor.is_transformed) {
@@ -202,13 +201,11 @@ void ggml_zdnn_mul_mat_q(
GGML_ASSERT(weights_extra->pre_tfm_desc.dim1 == weights->ne[1] && "weights_extra->pre_tfm_desc.dim1 must match weights->ne[1]");
ggml_zdnn_mul_mat_q_grouped(ctx, n_threads, inputs, weights_extra, output);
-
- // the result is written to output->data directly, so the output ztensor is stale
- output_extra->ztensor.is_transformed = false;
return;
}
- ggml_backend_zdnn_buffer * bias_extra = (ggml_backend_zdnn_buffer *)output_extra->extra;
+ ggml_backend_zdnn_buffer * output_extra = (ggml_backend_zdnn_buffer *)output->extra;
+ ggml_backend_zdnn_buffer * bias_extra = (ggml_backend_zdnn_buffer *)output_extra->extra;
GGML_ASSERT(weights_extra->pre_tfm_desc.dim2 == weights->ne[0] && "weights_extra->pre_tfm_desc.dim2 must match weights->ne[0]");
GGML_ASSERT(weights_extra->pre_tfm_desc.dim1 == weights->ne[1] && "weights_extra->pre_tfm_desc.dim1 must match weights->ne[1]");
@@ -271,9 +268,6 @@ void ggml_zdnn_mul_mat_q(
}
});
- // the result is written to output->data directly, so the output ztensor is stale
- output_extra->ztensor.is_transformed = false;
-
const float * weights_scales = weights_extra->scales.data();
ggml_zdnn_parallel_for(n_threads, ne1, [&](int64_t i1_start, int64_t i1_end) {
for (int64_t i1 = i1_start; i1 < i1_end; i1++) {