Commit 055c3c26e for llama.cpp
commit 055c3c26efe6995c99344417ddedb2f54d1aeeb4
Author: Ruben Ortlam <rortlam@redhat.com>
Date: Wed Nov 11 20:56:39 2026 +0100
vulkan: support batched tensor copy API (#31418)
* vulkan: support batched tensor copy API
* allocate staging buffer only for copies that actually need it
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp
index 3e04eef61..9c6edb569 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp
@@ -404,6 +404,33 @@ void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t si
}
}
+static vk_context ggml_vk_begin_sync_ctx(vk_device & device, vk_command_pool & pool) {
+ vk_context subctx = ggml_vk_create_temporary_context(pool);
+ ggml_vk_ctx_begin(device, subctx);
+ return subctx;
+}
+
+static void ggml_vk_submit_sync(vk_device & device, vk_context & subctx) {
+ ggml_vk_ctx_end(subctx);
+
+ for (auto & cpy : subctx->in_memcpys) {
+ memcpy(cpy.dst, cpy.src, cpy.n);
+ }
+
+ for (auto & mset : subctx->memsets) {
+ memset(mset.dst, mset.val, mset.n);
+ }
+
+ ggml_vk_submit(subctx, device->fence);
+ VK_CHECK(device->device.waitForFences({ device->fence }, true, UINT64_MAX), "vk_submit_sync waitForFences", device);
+ device->device.resetFences({ device->fence });
+ ggml_vk_queue_command_pools_cleanup(device);
+
+ for (auto & cpy : subctx->out_memcpys) {
+ memcpy(cpy.dst, cpy.src, cpy.n);
+ }
+}
+
static void ggml_vk_buffer_write_nc_async(ggml_backend_vk_context * ctx, vk_context& subctx, vk_buffer& dst, size_t offset, const ggml_tensor * tensor, bool sync_staging = false) {
VK_LOG_DEBUG("ggml_vk_buffer_write_nc_async(" << tensor << ")");
GGML_ASSERT(!ggml_is_contiguous(tensor));
@@ -590,24 +617,10 @@ void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void * src, si
} else {
std::lock_guard<std::recursive_mutex> guard(dst->device->mutex);
- vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool);
- ggml_vk_ctx_begin(dst->device, subctx);
+ vk_context subctx = ggml_vk_begin_sync_ctx(dst->device, dst->device->transfer_queue->cmd_pool);
bool ret = ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, spitch, dpitch, width, height, true);
GGML_ASSERT(ret);
- ggml_vk_ctx_end(subctx);
-
- for (auto& cpy : subctx->in_memcpys) {
- memcpy(cpy.dst, cpy.src, cpy.n);
- }
-
- for (auto& mset : subctx->memsets) {
- memset(mset.dst, mset.val, mset.n);
- }
-
- ggml_vk_submit(subctx, dst->device->fence);
- VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences", dst->device);
- dst->device->device.resetFences({ dst->device->fence });
- ggml_vk_queue_command_pools_cleanup(dst->device);
+ ggml_vk_submit_sync(dst->device, subctx);
}
}
@@ -695,21 +708,24 @@ static bool ggml_vk_buffer_read_async(vk_context subctx, vk_buffer& src, size_t
return ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, size, size, size, 1, sync_staging);
}
-void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height) {
- VK_LOG_DEBUG("ggml_vk_buffer_read_2d(" << src->buffer << ", " << offset << ", " << width << ", " << height << ")");
-
+static bool ggml_vk_buffer_host_read_direct(const vk_buffer & buf) {
// If the device is not an UMA device the memory is host-accessible through rebar. While writing
// through PCIe is sufficient fast reading back data from PCIe is slower than going through
// the HW device to host copy path.
// AMD UMA: uncached host-visible memory is write-combined, CPU reads are slow
- const bool slow_host_read = src->device->vendor_id == VK_VENDOR_ID_AMD &&
- !(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached);
- if(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible && src->device->uma && !slow_host_read) {
+ const bool slow_host_read = buf->device->vendor_id == VK_VENDOR_ID_AMD &&
+ !(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached);
+ return (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) && buf->device->uma && !slow_host_read;
+}
+
+void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height) {
+ VK_LOG_DEBUG("ggml_vk_buffer_read_2d(" << src->buffer << ", " << offset << ", " << width << ", " << height << ")");
+
+ if (ggml_vk_buffer_host_read_direct(src)) {
GGML_ASSERT(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent);
std::lock_guard<std::recursive_mutex> guard(src->device->mutex);
- vk_context subctx = ggml_vk_create_temporary_context(src->device->compute_queue->cmd_pool);
- ggml_vk_ctx_begin(src->device, subctx);
+ vk_context subctx = ggml_vk_begin_sync_ctx(src->device, src->device->compute_queue->cmd_pool);
subctx->s->buffer->buf.pipelineBarrier(
vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost,
@@ -717,12 +733,7 @@ void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t sp
{ { vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite,
vk::AccessFlagBits::eHostRead } },
{}, {});
- ggml_vk_ctx_end(subctx);
- ggml_vk_submit(subctx, src->device->fence);
- VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX),
- "vk_buffer_read_2d uma waitForFences", src->device);
- src->device->device.resetFences({ src->device->fence });
- ggml_vk_queue_command_pools_cleanup(src->device);
+ ggml_vk_submit_sync(src->device, subctx);
if (width == spitch && width == dpitch) {
memcpy(dst, (const uint8_t *) src->ptr + offset, width * height);
@@ -734,20 +745,10 @@ void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t sp
} else {
std::lock_guard<std::recursive_mutex> guard(src->device->mutex);
- vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool);
- ggml_vk_ctx_begin(src->device, subctx);
+ vk_context subctx = ggml_vk_begin_sync_ctx(src->device, src->device->transfer_queue->cmd_pool);
bool ret = ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, spitch, dpitch, width, height, true);
GGML_ASSERT(ret);
- ggml_vk_ctx_end(subctx);
-
- ggml_vk_submit(subctx, src->device->fence);
- VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences", src->device);
- src->device->device.resetFences({ src->device->fence });
- ggml_vk_queue_command_pools_cleanup(src->device);
-
- for (auto& cpy : subctx->out_memcpys) {
- memcpy(cpy.dst, cpy.src, cpy.n);
- }
+ ggml_vk_submit_sync(src->device, subctx);
}
}
@@ -771,14 +772,9 @@ void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size
std::lock_guard<std::recursive_mutex> guard(src->device->mutex);
VK_LOG_DEBUG("ggml_vk_buffer_copy(SINGLE_DEVICE, " << size << ")");
// Copy within the device
- vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool);
- ggml_vk_ctx_begin(src->device, subctx);
+ vk_context subctx = ggml_vk_begin_sync_ctx(src->device, src->device->transfer_queue->cmd_pool);
ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size);
- ggml_vk_ctx_end(subctx);
- ggml_vk_submit(subctx, src->device->fence);
- VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences", src->device);
- src->device->device.resetFences({ src->device->fence });
- ggml_vk_queue_command_pools_cleanup(src->device);
+ ggml_vk_submit_sync(src->device, subctx);
} else {
VK_LOG_DEBUG("ggml_vk_buffer_copy(MULTI_DEVICE, " << size << ")");
// Copy device to device
@@ -814,15 +810,222 @@ void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t siz
}
std::lock_guard<std::recursive_mutex> guard(dst->device->mutex);
- vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool);
- ggml_vk_ctx_begin(dst->device, subctx);
+ vk_context subctx = ggml_vk_begin_sync_ctx(dst->device, dst->device->transfer_queue->cmd_pool);
subctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c);
- ggml_vk_ctx_end(subctx);
+ ggml_vk_submit_sync(dst->device, subctx);
+}
+
+static size_t ggml_vk_staging_size_n(const vk_device & device, const std::vector<vk_buffer_region> & regions) {
+ size_t total = 0;
+ size_t max = 0;
+ for (const auto & r : regions) {
+ vk_buffer pinned;
+ size_t pinned_offset;
+ ggml_vk_host_get(device, r.host, pinned, pinned_offset);
+ if (pinned != nullptr) {
+ continue;
+ }
+ total += r.size;
+ max = std::max(max, r.size);
+ }
+ return std::max(max, std::min(total, (size_t) device->suballocation_block_size));
+}
+
+// returns number of regions recorded, stops when staging is full
+size_t ggml_vk_buffer_write_n_async(vk_context& subctx, const vk_buffer_region * regions, size_t n, vk_buffer * staging, size_t * staging_offset) {
+ VK_LOG_DEBUG("ggml_vk_buffer_write_n_async(" << n << ")");
+ ggml_vk_sync_buffers(nullptr, subctx);
+
+ size_t i = 0;
+ for (; i < n; i++) {
+ const vk_buffer_region & r = regions[i];
+
+ vk_buffer pinned = nullptr;
+ size_t pinned_offset = 0;
+ ggml_vk_host_get(r.buf->device, r.host, pinned, pinned_offset);
+
+ if (pinned != nullptr) {
+ subctx->s->buffer->buf.copyBuffer(pinned->buffer, r.buf->buffer, { vk::BufferCopy{ pinned_offset, r.offset, r.size } });
+ continue;
+ }
+
+ if (staging == nullptr || *staging_offset + r.size > (*staging)->size) {
+ break;
+ }
+
+ subctx->s->buffer->buf.copyBuffer((*staging)->buffer, r.buf->buffer, { vk::BufferCopy{ *staging_offset, r.offset, r.size } });
+ deferred_memcpy((uint8_t *) (*staging)->ptr + *staging_offset, r.host, r.size, &subctx->in_memcpys);
+ *staging_offset += r.size;
+ }
+ return i;
+}
+
+// returns number of regions recorded, stops when staging is full
+size_t ggml_vk_buffer_read_n_async(vk_context& subctx, const vk_buffer_region * regions, size_t n, vk_buffer * staging, size_t * staging_offset) {
+ VK_LOG_DEBUG("ggml_vk_buffer_read_n_async(" << n << ")");
+ ggml_vk_sync_buffers(nullptr, subctx);
+
+ size_t i = 0;
+ for (; i < n; i++) {
+ const vk_buffer_region & r = regions[i];
+
+ vk_buffer pinned = nullptr;
+ size_t pinned_offset = 0;
+ ggml_vk_host_get(r.buf->device, r.host, pinned, pinned_offset);
+
+ if (pinned != nullptr) {
+ subctx->s->buffer->buf.copyBuffer(r.buf->buffer, pinned->buffer, { vk::BufferCopy{ r.offset, pinned_offset, r.size } });
+ continue;
+ }
+
+ if (staging == nullptr || *staging_offset + r.size > (*staging)->size) {
+ break;
+ }
+
+ subctx->s->buffer->buf.copyBuffer(r.buf->buffer, (*staging)->buffer, { vk::BufferCopy{ r.offset, *staging_offset, r.size } });
+ deferred_memcpy(r.host, (const uint8_t *) (*staging)->ptr + *staging_offset, r.size, &subctx->out_memcpys);
+ *staging_offset += r.size;
+ }
+ return i;
+}
+
+void ggml_vk_buffer_copy_n_async(vk_context& subctx, const vk_buffer_copy_region * regions, size_t n) {
+ VK_LOG_DEBUG("ggml_vk_buffer_copy_n_async(" << n << ")");
+ ggml_vk_sync_buffers(nullptr, subctx);
+
+ for (size_t i = 0; i < n; i++) {
+ const vk_buffer_copy_region & r = regions[i];
+ GGML_ASSERT(r.src->device == r.dst->device);
+ subctx->s->buffer->buf.copyBuffer(r.src->buffer, r.dst->buffer, { vk::BufferCopy{ r.src_offset, r.dst_offset, r.size } });
+ }
+}
+
+void ggml_vk_buffer_write_n(const vk_buffer_region * regions, size_t n) {
+ VK_LOG_DEBUG("ggml_vk_buffer_write_n(" << n << ")");
+ std::vector<vk_buffer_region> pending;
+
+ for (size_t i = 0; i < n; i++) {
+ const vk_buffer_region & r = regions[i];
+ if (r.size == 0) {
+ continue;
+ }
+ if (r.buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {
+ GGML_ASSERT(r.buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent);
+ memcpy((uint8_t *) r.buf->ptr + r.offset, r.host, r.size);
+ continue;
+ }
+ GGML_ASSERT(pending.empty() || r.buf->device == pending[0].buf->device);
+ pending.push_back(r);
+ }
+
+ if (pending.empty()) {
+ return;
+ }
+
+ vk_device device = pending[0].buf->device;
+ std::lock_guard<std::recursive_mutex> guard(device->mutex);
+ const size_t staging_size = ggml_vk_staging_size_n(device, pending);
+ if (staging_size > 0) {
+ ggml_vk_ensure_sync_staging_buffer(device, staging_size);
+ }
+
+ for (size_t i = 0; i < pending.size();) {
+ vk_context subctx = ggml_vk_begin_sync_ctx(device, device->transfer_queue->cmd_pool);
+ size_t staging_offset = 0;
+ const size_t recorded = ggml_vk_buffer_write_n_async(subctx, pending.data() + i, pending.size() - i, &device->sync_staging, &staging_offset);
+ GGML_ASSERT(recorded > 0);
+ ggml_vk_submit_sync(device, subctx);
+ i += recorded;
+ }
+}
+
+void ggml_vk_buffer_read_n(const vk_buffer_region * regions, size_t n) {
+ VK_LOG_DEBUG("ggml_vk_buffer_read_n(" << n << ")");
+ std::vector<vk_buffer_region> direct;
+ std::vector<vk_buffer_region> pending;
+
+ for (size_t i = 0; i < n; i++) {
+ const vk_buffer_region & r = regions[i];
+ if (r.size == 0) {
+ continue;
+ }
+ GGML_ASSERT((direct.empty() && pending.empty()) || r.buf->device == (direct.empty() ? pending[0] : direct[0]).buf->device);
+ if (ggml_vk_buffer_host_read_direct(r.buf)) {
+ GGML_ASSERT(r.buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent);
+ direct.push_back(r);
+ } else {
+ pending.push_back(r);
+ }
+ }
+
+ if (direct.empty() && pending.empty()) {
+ return;
+ }
+
+ vk_device device = (direct.empty() ? pending[0] : direct[0]).buf->device;
+ std::lock_guard<std::recursive_mutex> guard(device->mutex);
+
+ if (!direct.empty()) {
+ vk_context subctx = ggml_vk_begin_sync_ctx(device, device->compute_queue->cmd_pool);
+ subctx->s->buffer->buf.pipelineBarrier(
+ vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer,
+ vk::PipelineStageFlagBits::eHost,
+ {},
+ { { vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite,
+ vk::AccessFlagBits::eHostRead } },
+ {}, {});
+ ggml_vk_submit_sync(device, subctx);
+
+ for (const auto & r : direct) {
+ memcpy(r.host, (const uint8_t *) r.buf->ptr + r.offset, r.size);
+ }
+ }
+
+ if (pending.empty()) {
+ return;
+ }
+
+ const size_t staging_size = ggml_vk_staging_size_n(device, pending);
+ if (staging_size > 0) {
+ ggml_vk_ensure_sync_staging_buffer(device, staging_size);
+ }
+
+ for (size_t i = 0; i < pending.size();) {
+ vk_context subctx = ggml_vk_begin_sync_ctx(device, device->transfer_queue->cmd_pool);
+ size_t staging_offset = 0;
+ const size_t recorded = ggml_vk_buffer_read_n_async(subctx, pending.data() + i, pending.size() - i, &device->sync_staging, &staging_offset);
+ GGML_ASSERT(recorded > 0);
+ ggml_vk_submit_sync(device, subctx);
+ i += recorded;
+ }
+}
+
+void ggml_vk_buffer_copy_n(const vk_buffer_copy_region * regions, size_t n) {
+ VK_LOG_DEBUG("ggml_vk_buffer_copy_n(" << n << ")");
+ std::vector<vk_buffer_copy_region> same_device;
+
+ for (size_t i = 0; i < n; i++) {
+ vk_buffer_copy_region r = regions[i];
+ if (r.size == 0) {
+ continue;
+ }
+ if (r.src->device != r.dst->device) {
+ ggml_vk_buffer_copy(r.dst, r.dst_offset, r.src, r.src_offset, r.size);
+ continue;
+ }
+ GGML_ASSERT(same_device.empty() || r.dst->device == same_device[0].dst->device);
+ same_device.push_back(std::move(r));
+ }
+
+ if (same_device.empty()) {
+ return;
+ }
- ggml_vk_submit(subctx, dst->device->fence);
- VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences", dst->device);
- dst->device->device.resetFences({ dst->device->fence });
- ggml_vk_queue_command_pools_cleanup(dst->device);
+ vk_device device = same_device[0].dst->device;
+ std::lock_guard<std::recursive_mutex> guard(device->mutex);
+ vk_context subctx = ggml_vk_begin_sync_ctx(device, device->transfer_queue->cmd_pool);
+ ggml_vk_buffer_copy_n_async(subctx, same_device.data(), same_device.size());
+ ggml_vk_submit_sync(device, subctx);
}
ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {
@@ -837,9 +1040,9 @@ ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {
/* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor,
/* .clear = */ ggml_backend_vk_buffer_clear,
/* .reset = */ NULL,
- /* .set_tensor_n = */ NULL,
- /* .get_tensor_n = */ NULL,
- /* .cpy_tensor_n = */ NULL,
+ /* .set_tensor_n = */ ggml_backend_vk_buffer_set_tensor_n,
+ /* .get_tensor_n = */ ggml_backend_vk_buffer_get_tensor_n,
+ /* .cpy_tensor_n = */ ggml_backend_vk_buffer_cpy_tensor_n,
};
vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size) {
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-common.h b/ggml/src/ggml-vulkan/ggml-vulkan-common.h
index b5491fc4b..2c39a525f 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-common.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-common.h
@@ -59,6 +59,12 @@ void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t sp
void ggml_vk_buffer_read(vk_buffer& src, size_t offset, void * dst, size_t size);
void ggml_vk_buffer_copy_async(vk_context& ctx, vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size);
void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size);
+size_t ggml_vk_buffer_write_n_async(vk_context& subctx, const vk_buffer_region * regions, size_t n, vk_buffer * staging = nullptr, size_t * staging_offset = nullptr);
+size_t ggml_vk_buffer_read_n_async(vk_context& subctx, const vk_buffer_region * regions, size_t n, vk_buffer * staging = nullptr, size_t * staging_offset = nullptr);
+void ggml_vk_buffer_copy_n_async(vk_context& subctx, const vk_buffer_copy_region * regions, size_t n);
+void ggml_vk_buffer_write_n(const vk_buffer_region * regions, size_t n);
+void ggml_vk_buffer_read_n(const vk_buffer_region * regions, size_t n);
+void ggml_vk_buffer_copy_n(const vk_buffer_copy_region * regions, size_t n);
void ggml_vk_buffer_memset_async(vk_context& ctx, vk_buffer& dst, size_t offset, uint32_t c, size_t size);
void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t size);
vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size);
@@ -227,6 +233,9 @@ void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_ten
void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size);
void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst);
+void ggml_backend_vk_buffer_set_tensor_n(ggml_backend_buffer_t buffer, ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors);
+void ggml_backend_vk_buffer_get_tensor_n(ggml_backend_buffer_t buffer, const ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors);
+bool ggml_backend_vk_buffer_cpy_tensor_n(ggml_backend_buffer_t buffer, const ggml_tensor * const * srcs, ggml_tensor * const * dsts, size_t n_tensors);
void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value);
const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft);
ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
index 786f8f019..bf2c02d4d 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
@@ -1135,6 +1135,21 @@ struct vk_staging_memset {
size_t n;
};
+struct vk_buffer_region {
+ vk_buffer buf;
+ size_t offset;
+ void * host;
+ size_t size;
+};
+
+struct vk_buffer_copy_region {
+ vk_buffer dst;
+ size_t dst_offset;
+ vk_buffer src;
+ size_t src_offset;
+ size_t size;
+};
+
struct vk_context_struct {
vk_submission * s;
std::vector<vk_sequence> seqs;
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index 502f73344..e1de84191 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -13345,6 +13345,54 @@ bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_
UNUSED(buffer);
}
+static vk_buffer & ggml_vk_tensor_dev_buffer(const ggml_tensor * tensor) {
+ ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
+ return ((ggml_backend_vk_buffer_context *) buffer->context)->dev_buffer;
+}
+
+void ggml_backend_vk_buffer_set_tensor_n(ggml_backend_buffer_t buffer, ggml_tensor * const * tensors, const void * const * data,
+ const size_t * offsets, const size_t * sizes, size_t n_tensors) {
+ VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor_n(" << buffer << ", " << n_tensors << ")");
+ std::vector<vk_buffer_region> regions(n_tensors);
+ for (size_t i = 0; i < n_tensors; i++) {
+ regions[i] = { ggml_vk_tensor_dev_buffer(tensors[i]), vk_tensor_offset(tensors[i]) + tensors[i]->view_offs + offsets[i], const_cast<void *>(data[i]), sizes[i] };
+ }
+ ggml_vk_buffer_write_n(regions.data(), n_tensors);
+
+ UNUSED(buffer);
+}
+
+void ggml_backend_vk_buffer_get_tensor_n(ggml_backend_buffer_t buffer, const ggml_tensor * const * tensors, void * const * data,
+ const size_t * offsets, const size_t * sizes, size_t n_tensors) {
+ VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor_n(" << buffer << ", " << n_tensors << ")");
+ std::vector<vk_buffer_region> regions(n_tensors);
+ for (size_t i = 0; i < n_tensors; i++) {
+ regions[i] = { ggml_vk_tensor_dev_buffer(tensors[i]), vk_tensor_offset(tensors[i]) + tensors[i]->view_offs + offsets[i], data[i], sizes[i] };
+ }
+ ggml_vk_buffer_read_n(regions.data(), n_tensors);
+
+ UNUSED(buffer);
+}
+
+bool ggml_backend_vk_buffer_cpy_tensor_n(ggml_backend_buffer_t buffer, const ggml_tensor * const * srcs, ggml_tensor * const * dsts, size_t n_tensors) {
+ VK_LOG_DEBUG("ggml_backend_vk_buffer_cpy_tensor_n(" << buffer << ", " << n_tensors << ")");
+ for (size_t i = 0; i < n_tensors; i++) {
+ if (!ggml_backend_buffer_is_vk(srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer)) {
+ return false;
+ }
+ }
+
+ std::vector<vk_buffer_copy_region> regions(n_tensors);
+ for (size_t i = 0; i < n_tensors; i++) {
+ regions[i] = { ggml_vk_tensor_dev_buffer(dsts[i]), vk_tensor_offset(dsts[i]) + dsts[i]->view_offs,
+ ggml_vk_tensor_dev_buffer(srcs[i]), vk_tensor_offset(srcs[i]) + srcs[i]->view_offs, ggml_nbytes(srcs[i]) };
+ }
+ ggml_vk_buffer_copy_n(regions.data(), n_tensors);
+ return true;
+
+ UNUSED(buffer);
+}
+
void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context;