Commit f172be756 for llama.cpp
commit f172be756ae6e12a6aa294ecaeb2a387649ac6f8
Author: Ruben Ortlam <rortlam@redhat.com>
Date: Thu Sep 17 10:16:35 2026 +0200
vulkan: split buffers and debug code into separate files, add shared headers (#28732)
diff --git a/ggml/src/ggml-vulkan/CMakeLists.txt b/ggml/src/ggml-vulkan/CMakeLists.txt
index e733ad5cc..af951a2bd 100644
--- a/ggml/src/ggml-vulkan/CMakeLists.txt
+++ b/ggml/src/ggml-vulkan/CMakeLists.txt
@@ -62,6 +62,11 @@ if (Vulkan_FOUND)
ggml_add_backend_library(ggml-vulkan
ggml-vulkan.cpp
../../include/ggml-vulkan.h
+ ggml-vulkan-types.h
+ ggml-vulkan-push-constants.h
+ ggml-vulkan-common.h
+ ggml-vulkan-buffers.cpp
+ ggml-vulkan-debug.cpp
)
set(VULKAN_SHADER_GEN_CMAKE_ARGS "")
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp
new file mode 100644
index 000000000..4d4c84951
--- /dev/null
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp
@@ -0,0 +1,783 @@
+#include "ggml-vulkan-common.h"
+
+ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = {
+ /* .get_name = */ ggml_backend_vk_buffer_type_name,
+ /* .alloc_buffer = */ ggml_backend_vk_buffer_type_alloc_buffer,
+ /* .get_alignment = */ ggml_backend_vk_buffer_type_get_alignment,
+ /* .get_max_size = */ ggml_backend_vk_buffer_type_get_max_size,
+ /* .get_alloc_size = */ ggml_backend_vk_buffer_type_get_alloc_size,
+ /* .is_host = */ NULL,
+};
+
+static std::vector<uint32_t> ggml_vk_find_memory_properties(const vk::PhysicalDeviceMemoryProperties* mem_props, vk::MemoryRequirements* mem_req, vk::MemoryPropertyFlags flags) {
+ std::vector<uint32_t> indices;
+
+ for (uint32_t i = 0; i < mem_props->memoryTypeCount; ++i) {
+ vk::MemoryType memory_type = mem_props->memoryTypes[i];
+ if ((mem_req->memoryTypeBits & ((uint64_t)1 << i)) &&
+ (flags & memory_type.propertyFlags) == flags &&
+ mem_props->memoryHeaps[memory_type.heapIndex].size >= mem_req->size) {
+ indices.push_back(i);
+ }
+ }
+ return indices;
+}
+
+static vk_buffer ggml_vk_create_buffer(vk_device& device, size_t size, const std::initializer_list<vk::MemoryPropertyFlags> & req_flags_list,
+ void *import_ptr = nullptr) {
+ VK_LOG_DEBUG("ggml_vk_create_buffer(" << device->name << ", " << size << ", " << to_string(req_flags_list.begin()[0]) << ", " << to_string(req_flags_list.begin()[req_flags_list.size()-1]) << ")");
+ if (size > device->max_buffer_size) {
+ throw vk::OutOfDeviceMemoryError("Requested buffer size exceeds device buffer size limit");
+ }
+
+ vk_buffer buf = std::make_shared<vk_buffer_struct>();
+
+ if (size == 0) {
+ buf->size = 0;
+ return buf;
+ }
+
+ vk::BufferUsageFlags usage_flags = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst;
+ vk::MemoryAllocateFlags mem_flags {};
+ if (device->buffer_device_address) {
+ usage_flags |= vk::BufferUsageFlagBits::eShaderDeviceAddress;
+ mem_flags |= vk::MemoryAllocateFlagBits::eDeviceAddress;
+ }
+
+ vk::BufferCreateInfo buffer_create_info{
+ vk::BufferCreateFlags(),
+ size,
+ usage_flags,
+ vk::SharingMode::eExclusive,
+ 0,
+ nullptr,
+ };
+
+ vk::ExternalMemoryBufferCreateInfo external_memory_bci;
+ if (import_ptr) {
+ external_memory_bci.handleTypes = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT;
+ buffer_create_info.setPNext(&external_memory_bci);
+ }
+
+ buf->buffer = device->device.createBuffer(buffer_create_info);
+
+ vk::MemoryRequirements mem_req = device->device.getBufferMemoryRequirements(buf->buffer);
+
+ vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties();
+
+ const vk::MemoryPriorityAllocateInfoEXT mem_priority_info { 1.0f };
+
+ vk::MemoryAllocateFlagsInfo mem_flags_info { mem_flags };
+
+ if (device->memory_priority) {
+ mem_flags_info.setPNext(&mem_priority_info);
+ }
+
+ if (import_ptr) {
+ vk::MemoryHostPointerPropertiesEXT host_pointer_props;
+ try {
+ host_pointer_props = device->device.getMemoryHostPointerPropertiesEXT(vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT, import_ptr);
+ } catch (vk::SystemError& e) {
+ GGML_LOG_WARN("ggml_vulkan: Failed getMemoryHostPointerPropertiesEXT (%s)\n", e.what());
+ device->device.destroyBuffer(buf->buffer);
+ return {};
+ }
+ vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties();
+
+ uint32_t memory_type_idx;
+ vk::MemoryPropertyFlags property_flags = *req_flags_list.begin();
+ for (memory_type_idx = 0; memory_type_idx < 32; ++memory_type_idx) {
+ if (!(host_pointer_props.memoryTypeBits & (1u << memory_type_idx))) {
+ continue;
+ }
+ if (!(mem_req.memoryTypeBits & (1u << memory_type_idx))) {
+ continue;
+ }
+
+ vk::MemoryType memory_type = mem_props.memoryTypes[memory_type_idx];
+ // check for visible+coherent+cached. Other flags (e.g. devicelocal) are allowed
+ if ((memory_type.propertyFlags & property_flags) == property_flags) {
+ property_flags = memory_type.propertyFlags;
+ break;
+ }
+ }
+ if (memory_type_idx == 32) {
+ GGML_LOG_WARN("ggml_vulkan: Memory type for host allocation not found\n");
+ device->device.destroyBuffer(buf->buffer);
+ return {};
+ }
+
+ buf->memory_property_flags = mem_props.memoryTypes[memory_type_idx].propertyFlags;
+ try {
+ vk::ImportMemoryHostPointerInfoEXT import_info;
+ import_info.handleType = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT;
+ import_info.pHostPointer = import_ptr;
+ import_info.setPNext(&mem_flags_info);
+ buf->device_memory = device->device.allocateMemory({ size, memory_type_idx, &import_info });
+ } catch (const vk::SystemError& e) {
+ }
+ } else {
+ for (auto it = req_flags_list.begin(); it != req_flags_list.end(); it++) {
+ const auto & req_flags = *it;
+
+ const std::vector<uint32_t> memory_type_indices = ggml_vk_find_memory_properties(&mem_props, &mem_req, req_flags);
+
+ if (memory_type_indices.empty()) {
+ continue;
+ }
+
+ bool done = false;
+
+ for (auto mtype_it = memory_type_indices.begin(); mtype_it != memory_type_indices.end(); mtype_it++) {
+ try {
+ buf->device_memory = device->device.allocateMemory({ mem_req.size, *mtype_it, &mem_flags_info });
+ buf->memory_property_flags = mem_props.memoryTypes[*mtype_it].propertyFlags;
+ done = true;
+ break;
+ } catch (const vk::SystemError& e) {
+ // loop and retry
+ // during last attempt throw the exception
+ if (it + 1 == req_flags_list.end() && mtype_it + 1 == memory_type_indices.end()) {
+ device->device.destroyBuffer(buf->buffer);
+ throw e;
+ }
+ }
+ }
+
+ if (done) {
+ break;
+ }
+ }
+ }
+
+ if (!buf->device_memory) {
+ device->device.destroyBuffer(buf->buffer);
+ throw vk::OutOfDeviceMemoryError("No suitable memory type found");
+ }
+
+ buf->ptr = nullptr;
+
+ if (import_ptr) {
+ buf->ptr = import_ptr;
+ } else {
+ if (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {
+ buf->ptr = device->device.mapMemory(buf->device_memory, 0, VK_WHOLE_SIZE);
+ }
+ }
+
+ device->device.bindBufferMemory(buf->buffer, buf->device_memory, 0);
+
+ buf->device = device;
+ buf->size = size;
+
+ if (device->buffer_device_address) {
+ const vk::BufferDeviceAddressInfo addressInfo(buf->buffer);
+ buf->bda_addr = device->device.getBufferAddress(addressInfo);
+ }
+
+ device->memory_logger->log_allocation(buf, size);
+
+ return buf;
+}
+
+vk_buffer ggml_vk_create_buffer_check(vk_device& device, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags) {
+ try {
+ return ggml_vk_create_buffer(device, size, {req_flags, fallback_flags});
+ } catch (const vk::SystemError& e) {
+ std::cerr << "ggml_vulkan: Memory allocation of size " << size << " failed." << std::endl;
+ std::cerr << "ggml_vulkan: " << e.what() << std::endl;
+ throw e;
+ }
+}
+
+vk_buffer ggml_vk_create_buffer_device(vk_device& device, size_t size) {
+ vk_buffer buf;
+ try {
+ if (device->prefer_host_memory) {
+ buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
+ vk::MemoryPropertyFlagBits::eDeviceLocal});
+ } else if (device->uma) {
+ // On UMA, prefer host-visible memory so direct tensor borrowing works.
+ // If unavailable, fall back to device-local memory.
+ buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
+ vk::MemoryPropertyFlagBits::eDeviceLocal,
+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
+ } else if (device->disable_host_visible_vidmem) {
+ if (device->allow_sysmem_fallback) {
+ buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal,
+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
+ } else {
+ buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal});
+ }
+ } else {
+ // use rebar if available, otherwise fallback to device only visible memory
+ if (device->allow_sysmem_fallback) {
+ buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
+ vk::MemoryPropertyFlagBits::eDeviceLocal,
+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
+ } else {
+ buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
+ vk::MemoryPropertyFlagBits::eDeviceLocal});
+ }
+ }
+ } catch (const vk::SystemError& e) {
+ std::cerr << "ggml_vulkan: Device memory allocation of size " << size << " failed." << std::endl;
+ std::cerr << "ggml_vulkan: " << e.what() << std::endl;
+ throw e;
+ }
+
+ return buf;
+}
+
+void ggml_vk_destroy_buffer(vk_buffer& buf) {
+ if (buf == nullptr) {
+ return;
+ }
+
+ if (buf->device != nullptr) {
+ buf->device->memory_logger->log_deallocation(buf);
+ }
+
+ buf.reset();
+}
+
+void * ggml_vk_host_malloc(vk_device& device, size_t size) {
+ VK_LOG_MEMORY("ggml_vk_host_malloc(" << size << ")");
+ vk_buffer buf = ggml_vk_create_buffer(device, size,
+ {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,
+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
+
+ if(!(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible)) {
+ fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory\n",
+ size/1024.0/1024.0);
+ device->device.freeMemory(buf->device_memory);
+ device->device.destroyBuffer(buf->buffer);
+ return nullptr;
+ }
+
+ std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex);
+ device->pinned_memory.push_back(std::make_tuple(buf->ptr, size, buf));
+
+ return buf->ptr;
+}
+
+void ggml_vk_host_free(vk_device& device, void* ptr) {
+ if (ptr == nullptr) {
+ return;
+ }
+ VK_LOG_MEMORY("ggml_vk_host_free(" << ptr << ")");
+ std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex);
+
+ vk_buffer buf;
+ size_t index;
+ for (size_t i = 0; i < device->pinned_memory.size(); i++) {
+ const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]);
+ const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]);
+ if (ptr >= addr && ptr < endr) {
+ buf = std::get<2>(device->pinned_memory[i]);
+ index = i;
+ break;
+ }
+ }
+ if (buf == nullptr) {
+ fprintf(stderr, "WARNING: failed to free pinned memory: memory not in map\n");
+ return;
+ }
+
+ ggml_vk_destroy_buffer(buf);
+
+ device->pinned_memory.erase(device->pinned_memory.begin() + index);
+}
+
+void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset) {
+ std::shared_lock<std::shared_mutex> guard(device->pinned_memory_mutex);
+ buf = nullptr;
+ buf_offset = 0;
+ for (size_t i = 0; i < device->pinned_memory.size(); i++) {
+ const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]);
+ const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]);
+ if (ptr >= addr && ptr < endr) {
+ buf = std::get<2>(device->pinned_memory[i]);
+ buf_offset = ((const uint8_t *)ptr) - addr;
+ break;
+ }
+ }
+}
+
+void ggml_vk_ensure_sync_staging_buffer(vk_device& device, size_t size) {
+ if (device->sync_staging == nullptr || device->sync_staging->size < size) {
+ VK_LOG_MEMORY("ggml_vk_ensure_sync_staging_buffer(" << size << ")");
+ ggml_vk_destroy_buffer(device->sync_staging);
+ device->sync_staging = ggml_vk_create_buffer_check(device, size,
+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,
+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent);
+ }
+}
+
+void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t size) {
+ if (ctx->sync_staging == nullptr || ctx->sync_staging->size < size) {
+ VK_LOG_MEMORY("ggml_vk_ensure_sync_staging_buffer(" << size << ")");
+ ggml_vk_destroy_buffer(ctx->sync_staging);
+ ctx->sync_staging = ggml_vk_create_buffer_check(ctx->device, size,
+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,
+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent);
+ }
+}
+
+static void ggml_vk_buffer_write_nc_async(ggml_backend_vk_context * ctx, vk_context& subctx, vk_buffer& dst, size_t offset, const ggml_tensor * tensor, bool sync_staging = false) {
+ VK_LOG_DEBUG("ggml_vk_buffer_write_nc_async(" << tensor << ")");
+ GGML_ASSERT(!ggml_is_contiguous(tensor));
+ // Buffer is already mapped
+ if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {
+ std::cerr << "ggml_vulkan: buffer_write_nc_async dst buffer is host_visible. Use synchronous write." << std::endl;
+ GGML_ABORT("fatal error");
+ }
+ // Check if src is pinned memory
+ vk_buffer buf = nullptr;
+ size_t buf_offset = 0;
+ ggml_vk_host_get(ctx->device, tensor->data, buf, buf_offset);
+
+ const uint64_t ne0 = tensor->ne[0];
+ const uint64_t ne1 = tensor->ne[1];
+ const uint64_t ne2 = tensor->ne[2];
+ const uint64_t ne3 = tensor->ne[3];
+ const uint64_t nb0 = tensor->nb[0];
+ const uint64_t nb1 = tensor->nb[1];
+ const uint64_t nb2 = tensor->nb[2];
+ const uint64_t nb3 = tensor->nb[3];
+ const ggml_type type = tensor->type;
+ const uint64_t ts = ggml_type_size(type);
+ const uint64_t bs = ggml_blck_size(type);
+
+ const uint64_t dstnb0 = ts;
+ const uint64_t dstnb1 = dstnb0*(ne0/bs);
+ const uint64_t dstnb2 = dstnb1*ne1;
+ const uint64_t dstnb3 = dstnb2*ne2;
+
+ const uint64_t ne = ggml_nelements(tensor);
+
+ if (buf != nullptr) {
+ // Memory is pinned, use as staging buffer
+ std::vector<vk::BufferCopy> slices;
+
+ for (uint64_t i3 = 0; i3 < ne3; i3++) {
+ for (uint64_t i2 = 0; i2 < ne2; i2++) {
+ // Find longest contiguous slice
+ if (ne1*nb1 == dstnb2) {
+ slices.push_back({ buf_offset + i3*nb3 + i2*nb2, offset + i3*dstnb3 + i2*dstnb2, dstnb2 });
+ } else {
+ for (uint64_t i1 = 0; i1 < ne1; i1++) {
+ if (ne0*nb0/bs == dstnb1) {
+ slices.push_back({ buf_offset + i3*nb3 + i2*nb2 + i1*nb1, offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, dstnb1 });
+ } else {
+ const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1;
+ const uint64_t d_off = offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1;
+ for (uint64_t i0 = 0; i0 < ne0; i0++) {
+ slices.push_back({ s_off + i0*nb0, d_off + i0*dstnb0, dstnb0 });
+ }
+ }
+ }
+ }
+ }
+ }
+
+ ggml_vk_sync_buffers(ctx, subctx);
+ subctx->s->buffer->buf.copyBuffer(buf->buffer, dst->buffer, slices);
+ return;
+ }
+
+ if (!sync_staging) {
+ GGML_ABORT("Asynchronous write to non-pinned memory not supported");
+ }
+
+ // Staging buffer required
+ vk_buffer& staging = ctx->device->sync_staging;
+ const uint64_t copy_size = ts*ne/bs;
+ ggml_vk_ensure_sync_staging_buffer(ctx->device, copy_size);
+ VkBufferCopy buf_copy{ 0, offset, copy_size };
+
+ ggml_vk_sync_buffers(ctx, subctx);
+ vkCmdCopyBuffer(subctx->s->buffer->buf, (VkBuffer)staging->buffer, (VkBuffer)dst->buffer, 1, &buf_copy);
+
+ for (uint64_t i3 = 0; i3 < ne3; i3++) {
+ for (uint64_t i2 = 0; i2 < ne2; i2++) {
+ // Find longest contiguous slice
+ if (ne1*nb1 == dstnb2) {
+ deferred_memcpy((uint8_t *)staging->ptr + i3*dstnb3 + i2*dstnb2, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2, dstnb2, &subctx->in_memcpys);
+ } else {
+ for (uint64_t i1 = 0; i1 < ne1; i1++) {
+ if (ne0*nb0/bs == dstnb1) {
+ deferred_memcpy((uint8_t *)staging->ptr + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2 + i1*nb1, dstnb1, &subctx->in_memcpys);
+ } else {
+ const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1;
+ const uint64_t d_off = i3*dstnb3 + i2*dstnb2 + i1*dstnb1;
+ for (uint64_t i0 = 0; i0 < ne0; i0++) {
+ deferred_memcpy((uint8_t *)staging->ptr + d_off + i0*dstnb0, (const uint8_t *) tensor->data + s_off + i0*nb0, dstnb0, &subctx->in_memcpys);
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+bool ggml_vk_buffer_write_2d_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging) {
+ VK_LOG_DEBUG("ggml_vk_buffer_write_2d_async(" << width << ", " << height << ")");
+ // Check if src is pinned memory
+ vk_buffer buf = nullptr;
+ size_t buf_offset = 0;
+ ggml_vk_host_get(dst->device, src, buf, buf_offset);
+
+ if (buf != nullptr) {
+ // Memory is pinned, use as staging buffer
+ std::vector<vk::BufferCopy> slices(1);
+ if (width == spitch && width == dpitch) {
+ // Only do single write if stride is equal
+ slices[0].srcOffset = buf_offset;
+ slices[0].dstOffset = offset;
+ slices[0].size = width * height;
+ } else {
+ slices.resize(height);
+ for (size_t i = 0; i < height; i++) {
+ slices[i].srcOffset = buf_offset + i * spitch;
+ slices[i].dstOffset = offset + i * dpitch;
+ slices[i].size = width;
+ }
+ }
+
+ ggml_vk_sync_buffers(nullptr, subctx);
+ subctx->s->buffer->buf.copyBuffer(buf->buffer, dst->buffer, slices);
+ return true;
+ }
+ VK_LOG_DEBUG("STAGING");
+
+ if (!sync_staging) {
+ // copy was not handled caller needs to fall back
+ return false;
+ }
+
+ // Staging buffer required
+ const size_t staging_size = width * height;
+ ggml_vk_ensure_sync_staging_buffer(dst->device, staging_size);
+
+ vk_buffer& staging_buffer = dst->device->sync_staging;
+
+ std::vector<vk::BufferCopy> slices(1);
+ if (width == dpitch) {
+ slices[0].srcOffset = 0;
+ slices[0].dstOffset = offset;
+ slices[0].size = staging_size;
+ } else {
+ slices.resize(height);
+ for (size_t i = 0; i < height; i++) {
+ slices[i].srcOffset = i * width;
+ slices[i].dstOffset = offset + i * dpitch;
+ slices[i].size = width;
+ }
+ }
+
+ ggml_vk_sync_buffers(nullptr, subctx);
+ subctx->s->buffer->buf.copyBuffer(staging_buffer->buffer, dst->buffer, slices);
+
+ if (width == spitch) {
+ deferred_memcpy((uint8_t *)staging_buffer->ptr, src, staging_size, &subctx->in_memcpys);
+ } else {
+ for (size_t i = 0; i < height; i++) {
+ deferred_memcpy((uint8_t *)staging_buffer->ptr + i * width, (const uint8_t *) src + i * spitch, width, &subctx->in_memcpys);
+ }
+ }
+ return true;
+}
+
+bool ggml_vk_buffer_write_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t size, bool sync_staging) {
+ VK_LOG_DEBUG("ggml_vk_buffer_write_async(" << size << ")");
+ return ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, size, size, size, 1, sync_staging);
+}
+
+void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height) {
+ VK_LOG_DEBUG("ggml_vk_buffer_write_2d(" << width << ", " << height << ")");
+ // Buffer is already mapped
+ if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {
+ GGML_ASSERT(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent);
+
+ if (width == spitch && width == dpitch) {
+ memcpy((uint8_t *)dst->ptr + offset, src, width * height);
+ } else {
+ for (size_t i = 0; i < height; i++) {
+ memcpy((uint8_t *)dst->ptr + offset + i * dpitch, (const uint8_t *) src + i * spitch, width);
+ }
+ }
+ } else {
+ std::lock_guard<std::recursive_mutex> guard(dst->device->mutex);
+
+ vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool);
+ ggml_vk_ctx_begin(dst->device, subctx);
+ bool ret = ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, spitch, dpitch, width, height, true);
+ GGML_ASSERT(ret);
+ ggml_vk_ctx_end(subctx);
+
+ for (auto& cpy : subctx->in_memcpys) {
+ memcpy(cpy.dst, cpy.src, cpy.n);
+ }
+
+ for (auto& mset : subctx->memsets) {
+ memset(mset.dst, mset.val, mset.n);
+ }
+
+ ggml_vk_submit(subctx, dst->device->fence);
+ VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences", dst->device);
+ dst->device->device.resetFences({ dst->device->fence });
+ ggml_vk_queue_command_pools_cleanup(dst->device);
+ }
+}
+
+void ggml_vk_buffer_write(vk_buffer& dst, size_t offset, const void * src, size_t size) {
+ VK_LOG_DEBUG("ggml_vk_buffer_write(" << size << ")");
+ ggml_vk_buffer_write_2d(dst, offset, src, size, size, size, 1);
+}
+
+bool ggml_vk_buffer_read_2d_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging) {
+ VK_LOG_DEBUG("ggml_vk_buffer_read_2d_async(offset=" << offset << ", width=" << width << ", height=" << height << ")");
+ GGML_ASSERT(width > 0);
+ GGML_ASSERT(height > 0);
+ GGML_ASSERT(src != nullptr);
+
+ // TODO: staging_offset is not used
+
+ // Check if dst is pinned memory
+ vk_buffer buf = nullptr;
+ size_t buf_offset = 0;
+ ggml_vk_host_get(src->device, dst, buf, buf_offset);
+
+ std::vector<vk::BufferCopy> slices(1);
+ if (width == spitch && width == dpitch) {
+ // Only do single write if stride is equal
+ slices[0].srcOffset = offset;
+ slices[0].dstOffset = buf_offset;
+ slices[0].size = width * height;
+ } else {
+ slices.resize(height);
+ for (size_t i = 0; i < height; i++) {
+ slices[i].srcOffset = offset + i * spitch;
+ slices[i].dstOffset = buf_offset + i * dpitch;
+ slices[i].size = width;
+ }
+ }
+
+ if (buf != nullptr) {
+ // Memory is pinned, use as staging buffer
+ ggml_vk_sync_buffers(nullptr, subctx);
+ subctx->s->buffer->buf.copyBuffer(src->buffer, buf->buffer, slices);
+
+ return true;
+ }
+ VK_LOG_DEBUG("STAGING");
+
+ if (!sync_staging) {
+ // copy was not handled caller needs to fall back
+ return false;
+ }
+
+ // Fall back to staging buffer
+ const size_t staging_size = width * height;
+ ggml_vk_ensure_sync_staging_buffer(src->device, staging_size);
+
+ vk_buffer& staging_buffer = src->device->sync_staging;
+
+ std::vector<vk::BufferCopy> staging_slices(1);
+ if (width == spitch) {
+ staging_slices[0].srcOffset = offset;
+ staging_slices[0].dstOffset = 0;
+ staging_slices[0].size = staging_size;
+ } else {
+ staging_slices.resize(height);
+ for (size_t i = 0; i < height; i++) {
+ staging_slices[i].srcOffset = offset + i * spitch;
+ staging_slices[i].dstOffset = i * width;
+ staging_slices[i].size = width;
+ }
+ }
+
+ ggml_vk_sync_buffers(nullptr, subctx);
+ subctx->s->buffer->buf.copyBuffer(src->buffer, staging_buffer->buffer, staging_slices);
+
+ if (width == dpitch) {
+ deferred_memcpy(dst, staging_buffer->ptr, staging_size, &subctx->out_memcpys);
+ } else {
+ for (size_t i = 0; i < height; i++) {
+ deferred_memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) staging_buffer->ptr + i * width, width, &subctx->out_memcpys);
+ }
+ }
+ return true;
+}
+
+static bool ggml_vk_buffer_read_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t size, bool sync_staging = false) {
+ return ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, size, size, size, 1, sync_staging);
+}
+
+void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height) {
+ VK_LOG_DEBUG("ggml_vk_buffer_read_2d(" << src->buffer << ", " << offset << ", " << width << ", " << height << ")");
+
+ // If the device is not an UMA device the memory is host-accessible through rebar. While writing
+ // through PCIe is sufficient fast reading back data from PCIe is slower than going through
+ // the HW device to host copy path.
+ if(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible && src->device->uma) {
+ GGML_ASSERT(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent);
+
+ std::lock_guard<std::recursive_mutex> guard(src->device->mutex);
+ vk_context subctx = ggml_vk_create_temporary_context(src->device->compute_queue->cmd_pool);
+ ggml_vk_ctx_begin(src->device, subctx);
+ subctx->s->buffer->buf.pipelineBarrier(
+ vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer,
+ vk::PipelineStageFlagBits::eHost,
+ {},
+ { { vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite,
+ vk::AccessFlagBits::eHostRead } },
+ {}, {});
+ ggml_vk_ctx_end(subctx);
+ ggml_vk_submit(subctx, src->device->fence);
+ VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX),
+ "vk_buffer_read_2d uma waitForFences", src->device);
+ src->device->device.resetFences({ src->device->fence });
+ ggml_vk_queue_command_pools_cleanup(src->device);
+
+ if (width == spitch && width == dpitch) {
+ memcpy(dst, (const uint8_t *) src->ptr + offset, width * height);
+ } else {
+ for (size_t i = 0; i < height; i++) {
+ memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) src->ptr + offset + i * spitch, width);
+ }
+ }
+ } else {
+ std::lock_guard<std::recursive_mutex> guard(src->device->mutex);
+
+ vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool);
+ ggml_vk_ctx_begin(src->device, subctx);
+ bool ret = ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, spitch, dpitch, width, height, true);
+ GGML_ASSERT(ret);
+ ggml_vk_ctx_end(subctx);
+
+ ggml_vk_submit(subctx, src->device->fence);
+ VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences", src->device);
+ src->device->device.resetFences({ src->device->fence });
+ ggml_vk_queue_command_pools_cleanup(src->device);
+
+ for (auto& cpy : subctx->out_memcpys) {
+ memcpy(cpy.dst, cpy.src, cpy.n);
+ }
+ }
+}
+
+void ggml_vk_buffer_read(vk_buffer& src, size_t offset, void * dst, size_t size) {
+ VK_LOG_DEBUG("ggml_vk_buffer_read(" << src->buffer << ", " << offset << ", " << size << ")");
+ ggml_vk_buffer_read_2d(src, offset, dst, size, size, size, 1);
+}
+
+void ggml_vk_buffer_copy_async(vk_context& ctx, vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) {
+ VK_LOG_DEBUG("ggml_vk_buffer_copy_async(" << size << ")");
+ // Make sure both buffers are on same device
+ GGML_ASSERT(src->device == dst->device);
+
+ VkBufferCopy bc{ src_offset, dst_offset, size };
+
+ vkCmdCopyBuffer(ctx->s->buffer->buf, (VkBuffer)src->buffer, (VkBuffer)dst->buffer, 1, &bc);
+}
+
+void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) {
+ if (src->device == dst->device) {
+ std::lock_guard<std::recursive_mutex> guard(src->device->mutex);
+ VK_LOG_DEBUG("ggml_vk_buffer_copy(SINGLE_DEVICE, " << size << ")");
+ // Copy within the device
+ vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool);
+ ggml_vk_ctx_begin(src->device, subctx);
+ ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size);
+ ggml_vk_ctx_end(subctx);
+ ggml_vk_submit(subctx, src->device->fence);
+ VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences", src->device);
+ src->device->device.resetFences({ src->device->fence });
+ ggml_vk_queue_command_pools_cleanup(src->device);
+ } else {
+ VK_LOG_DEBUG("ggml_vk_buffer_copy(MULTI_DEVICE, " << size << ")");
+ // Copy device to device
+ ggml_vk_ensure_sync_staging_buffer(src->device, size);
+
+ // Copy to src staging buffer
+ ggml_vk_buffer_copy(src->device->sync_staging, 0, src, src_offset, size);
+ // Copy to dst buffer
+ ggml_vk_buffer_write(dst, dst_offset, src->device->sync_staging->ptr, size);
+ }
+}
+
+void ggml_vk_buffer_memset_async(vk_context& ctx, vk_buffer& dst, size_t offset, uint32_t c, size_t size) {
+ VK_LOG_DEBUG("ggml_vk_buffer_memset_async(" << offset << ", " << c << ", " << size << ")");
+
+ if (dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible &&
+ dst->device->uma) {
+ deferred_memset((uint8_t*)dst->ptr + offset, c, size, &ctx->memsets);
+ return;
+ }
+
+ // Fall back to GPU fillBuffer for non-UMA or non-host-visible buffers
+ ctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c);
+}
+
+void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t size) {
+ VK_LOG_DEBUG("ggml_vk_buffer_memset(" << offset << ", " << c << ", " << size << ")");
+
+ if (dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible &&
+ dst->device->uma) {
+ memset((uint8_t*)dst->ptr + offset, c, size);
+ return;
+ }
+
+ std::lock_guard<std::recursive_mutex> guard(dst->device->mutex);
+ vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool);
+ ggml_vk_ctx_begin(dst->device, subctx);
+ subctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c);
+ ggml_vk_ctx_end(subctx);
+
+ ggml_vk_submit(subctx, dst->device->fence);
+ VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences", dst->device);
+ dst->device->device.resetFences({ dst->device->fence });
+ ggml_vk_queue_command_pools_cleanup(dst->device);
+}
+
+ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {
+ /* .free_buffer = */ ggml_backend_vk_buffer_free_buffer,
+ /* .get_base = */ ggml_backend_vk_buffer_get_base,
+ /* .init_tensor = */ ggml_backend_vk_buffer_init_tensor,
+ /* .memset_tensor = */ ggml_backend_vk_buffer_memset_tensor,
+ /* .set_tensor = */ ggml_backend_vk_buffer_set_tensor,
+ /* .get_tensor = */ ggml_backend_vk_buffer_get_tensor,
+ /* .set_tensor_2d = */ ggml_backend_vk_buffer_set_tensor_2d,
+ /* .get_tensor_2d = */ ggml_backend_vk_buffer_get_tensor_2d,
+ /* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor,
+ /* .clear = */ ggml_backend_vk_buffer_clear,
+ /* .reset = */ NULL,
+};
+
+vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size) {
+ if (!device->external_memory_host) {
+ return {};
+ }
+
+ uintptr_t uptr = reinterpret_cast<uintptr_t>(ptr);
+ if (uptr & (device->min_imported_host_pointer_alignment - 1)) {
+ return {};
+ }
+ if (size & (device->min_imported_host_pointer_alignment - 1)) {
+ return {};
+ }
+
+ const vk::MemoryPropertyFlags property_flags = vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached;
+
+ vk_buffer buf {};
+ try {
+ buf = ggml_vk_create_buffer(device, size, { property_flags }, ptr);
+ } catch (vk::SystemError& e) {
+ GGML_LOG_WARN("ggml_vulkan: Failed ggml_vk_create_buffer (%s)\n", e.what());
+ }
+
+ return buf;
+}
+
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-common.h b/ggml/src/ggml-vulkan/ggml-vulkan-common.h
new file mode 100644
index 000000000..4ae5fea7a
--- /dev/null
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-common.h
@@ -0,0 +1,282 @@
+#pragma once
+#include "ggml-vulkan-push-constants.h"
+
+// shared globals
+extern ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface;
+extern bool vk_memory_logger_enabled;
+extern bool vk_perf_logger_enabled;
+extern bool vk_perf_logger_concurrent;
+extern bool vk_enable_sync_logger;
+extern uint32_t vk_perf_logger_frequency;
+extern std::string vk_pipeline_stats_filter;
+extern void * const vk_ptr_base;
+extern vk_instance_t vk_instance;
+extern ggml_backend_buffer_i ggml_backend_vk_buffer_interface;
+
+// instance
+vk_device ggml_vk_get_device(size_t idx);
+DispatchLoaderDynamic & ggml_vk_default_dispatcher();
+void ggml_vk_instance_init();
+void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx);
+int ggml_vk_get_device_count();
+void ggml_vk_get_device_description(int device, char * description, size_t description_size);
+bool ggml_vk_instance_layer_settings_available();
+bool ggml_vk_instance_portability_enumeration_ext_available(const std::vector<vk::ExtensionProperties>& instance_extensions);
+bool ggml_vk_instance_debug_utils_ext_available(const std::vector<vk::ExtensionProperties> & instance_extensions);
+bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev);
+bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch);
+uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev);
+bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint32_t lower_major, uint32_t lower_minor, uint32_t upper_major, uint32_t upper_minor);
+
+// shaders
+void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline);
+vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc);
+vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc, bool use_mask, bool use_mask_opt, bool use_logit_softcap, bool use_sparse, ggml_type k_type, ggml_type v_type);
+uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_device_architecture &arch);
+void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested = nullptr);
+bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type);
+bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type = GGML_TYPE_F16, ggml_type v_type = GGML_TYPE_F16);
+
+// buffers
+vk_buffer ggml_vk_create_buffer_check(vk_device& device, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags = vk::MemoryPropertyFlags(0));
+vk_buffer ggml_vk_create_buffer_device(vk_device& device, size_t size);
+void ggml_vk_destroy_buffer(vk_buffer& buf);
+void * ggml_vk_host_malloc(vk_device& device, size_t size);
+void ggml_vk_host_free(vk_device& device, void* ptr);
+void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset);
+void ggml_vk_ensure_sync_staging_buffer(vk_device& device, size_t size);
+void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t size);
+bool ggml_vk_buffer_write_2d_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false);
+bool ggml_vk_buffer_write_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t size, bool sync_staging = false);
+void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height);
+void ggml_vk_buffer_write(vk_buffer& dst, size_t offset, const void * src, size_t size);
+bool ggml_vk_buffer_read_2d_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false);
+void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height);
+void ggml_vk_buffer_read(vk_buffer& src, size_t offset, void * dst, size_t size);
+void ggml_vk_buffer_copy_async(vk_context& ctx, vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size);
+void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size);
+void ggml_vk_buffer_memset_async(vk_context& ctx, vk_buffer& dst, size_t offset, uint32_t c, size_t size);
+void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t size);
+vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size);
+
+// pipelines
+uint64_t vk_tensor_offset(const ggml_tensor * tensor);
+uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t);
+void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx);
+void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx, vk_pipeline& pipeline, uint32_t n);
+void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx);
+void ggml_vk_submit(vk_context& ctx, vk::Fence fence);
+uint32_t ggml_vk_find_queue_family_index(std::vector<vk::QueueFamilyProperties>& queue_family_props, const vk::QueueFlags& required, const vk::QueueFlags& avoid, int32_t compute_index, uint32_t min_num_queues);
+std::unique_ptr<vk_queue> ggml_vk_create_queue(vk_device& device, uint32_t queue_family_index, uint32_t queue_index, vk::PipelineStageFlags&& stage_flags, bool transfer_only);
+std::unique_ptr<vk_queue> ggml_vk_create_aliased_queue(vk_device& device, const std::unique_ptr<vk_queue>& source);
+vk_context ggml_vk_create_context(ggml_backend_vk_context * ctx, vk_command_pool& p);
+vk_context ggml_vk_create_temporary_context(vk_command_pool& p);
+void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p);
+void ggml_vk_queue_command_pools_cleanup(vk_device& device);
+vk_subbuffer ggml_vk_subbuffer(const ggml_backend_vk_context* ctx, const vk_buffer& buf, size_t offset = 0);
+void ggml_vk_sync_buffers(ggml_backend_vk_context* ctx, vk_context& subctx);
+void ggml_vk_set_event(vk_context& ctx, vk::Event& event);
+void ggml_vk_wait_events(vk_context& ctx, std::vector<vk::Event>&& events);
+vk_subbuffer ggml_vk_tensor_subbuffer(const ggml_backend_vk_context * ctx, const ggml_tensor * tensor, bool allow_misalign = false);
+void ggml_vk_cmd_label_begin(vk::CommandBuffer buf, const char * name);
+void ggml_vk_ctx_end(vk_context& ctx);
+void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx);
+vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx);
+vk_context ggml_vk_get_transfer_ctx(ggml_backend_vk_context * ctx);
+bool ggml_vk_submit_transfer_ctx(ggml_backend_vk_context * ctx);
+size_t ggml_vk_align_size(size_t width, size_t align);
+void deferred_memcpy(void * dst, const void * src, size_t size, std::vector<vk_staging_memcpy>* memcpys = nullptr);
+void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<vk_staging_memset>* memsets = nullptr);
+
+// matmul
+vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type);
+void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t padded_n);
+bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor);
+vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src, const ggml_tensor * dst, ggml_type to);
+vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx, ggml_type type);
+void ggml_vk_quantize_q8_1(ggml_backend_vk_context * ctx, vk_context& subctx, const vk_subbuffer & in, const vk_subbuffer & out, uint32_t ne);
+void ggml_vk_dsv4_hc_comb(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * mixes, const ggml_tensor * scale, const ggml_tensor * base, ggml_tensor * dst);
+void ggml_vk_dsv4_hc_pre(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * weights, ggml_tensor * dst);
+void ggml_vk_dsv4_hc_post(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * residual, const ggml_tensor * post, const ggml_tensor * comb, ggml_tensor * dst);
+void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx);
+bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx);
+void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx);
+
+// flash-attn
+void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * v, const ggml_tensor * mask, const ggml_tensor * sinks, ggml_tensor * dst);
+
+// operators
+void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline pipeline, const ggml_tensor * tensor, const vk_subbuffer & in, const vk_subbuffer & out);
+bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor * src1, const ggml_tensor * dst);
+void ggml_vk_fwht(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src, ggml_tensor * dst);
+void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_get_rows_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx);
+void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_out_prod(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_sub(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+int ggml_vk_unary_mul_op_index(ggml_unary_op op);
+void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx);
+void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_add_id(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst);
+void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);
+void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);
+void ggml_vk_gated_linear_attn(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);
+void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);
+void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);
+void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);
+void ggml_vk_ssm_conv(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx);
+void ggml_vk_opt_step_adamw(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);
+void ggml_vk_opt_step_sgd(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst);
+void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_upscale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_sqr(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_sqrt(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_add1(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);
+void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);
+void ggml_vk_sin(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_cos(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_log(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_diag(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_pad(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_pad_reflect_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_roll(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_repeat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_repeat_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_set_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_silu_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+uint32_t ggml_vk_rms_partials_size(ggml_backend_vk_context * ctx, const ggml_tensor *node);
+void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, float * op_params);
+void ggml_vk_rms_norm_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst);
+void ggml_vk_soft_max_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx);
+void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx, bool backprop);
+void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_topk(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_topk_qsa(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx);
+void ggml_vk_sum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_sum_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_mean(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_cumsum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_cross_entropy_loss(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);
+void ggml_vk_cross_entropy_loss_back(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);
+void ggml_vk_argmax(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_count_equal(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_solve_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_im2col(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_im2col_3d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_timestep_embedding(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_conv_transpose_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_col2im_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_snake_dispatch_fused(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx);
+void ggml_vk_pool_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_pool_2d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+void ggml_vk_conv_2d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_conv_3d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_conv_2d_dw(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+void ggml_vk_leaky_relu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);
+
+// graph
+void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx);
+bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int node_idx, ggml_tensor *node_begin, int node_idx_begin, bool last_node, bool almost_ready, bool submit);
+void ggml_vk_compute_forward(ggml_backend_vk_context* ctx, ggml_cgraph * cgraph, ggml_tensor* tensor, int tensor_idx, bool almost_ready);
+void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx);
+void ggml_vk_cleanup(ggml_backend_vk_context * ctx);
+void ggml_vk_synchronize(ggml_backend_vk_context * ctx);
+bool ggml_vk_is_empty(ggml_tensor * node);
+bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops);
+bool ggml_vk_can_fuse_ssm_conv(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, int num_extra);
+bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, topk_moe_mode mode);
+bool ggml_vk_can_fuse_topk_qsa(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);
+bool ggml_vk_can_fuse_rope_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);
+bool ggml_vk_can_fuse_rms_norm_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);
+bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);
+bool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b, bool elementwise);
+bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);
+uint32_t ggml_vk_fuse_multi_add(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);
+void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, struct ggml_backend_graph_optimize_params * params);
+
+// backend
+bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer);
+void ggml_backend_vk_buffer_free_buffer(ggml_backend_buffer_t buffer);
+void * ggml_backend_vk_buffer_get_base(ggml_backend_buffer_t buffer);
+enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor);
+void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);
+void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
+void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size);
+void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
+bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst);
+void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value);
+const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft);
+ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);
+size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft);
+size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft);
+size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor);
+void ggml_backend_vk_free(ggml_backend_t backend);
+ggml_backend_reg_t ggml_backend_vk_reg();
+
+// debug
+int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op);
+
+// ggml-vulkan.cpp (residual)
+bool ggml_vk_lightning_indexer_k_type_supported(ggml_type type);
+void ggml_vk_print_device_fault_info(const vk_device& device);
+uint64_t ggml_vk_get_node_flops(const ggml_tensor * node);
+void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int end);
+void ggml_vk_print_device_lost_info(const vk_device& device);
+size_t ggml_vk_tensor_buffer_offset(const ggml_backend_vk_context * ctx, const ggml_tensor * t);
+size_t ggml_vk_descriptor_offset(size_t tensor_offset, size_t alignment, size_t type_size);
+uint32_t ggml_vk_concat_unit_size(ggml_type type);
+bool ggml_vk_concat_supported(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst);
+
+template <typename T>
+inline void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& subctx, vk_pipeline& pipeline, std::initializer_list<vk::DescriptorBufferInfo> const& descriptor_buffer_infos, const T &push_constants, std::array<uint32_t, 3> elements) {
+ const uint32_t wg0 = CEIL_DIV(elements[0], pipeline->wg_denoms[0]);
+ const uint32_t wg1 = CEIL_DIV(elements[1], pipeline->wg_denoms[1]);
+ const uint32_t wg2 = CEIL_DIV(elements[2], pipeline->wg_denoms[2]);
+ VK_LOG_DEBUG("ggml_vk_dispatch_pipeline(" << pipeline->name << ", {";
+ for (auto& buffer : descriptor_buffer_infos) {
+ std::cerr << "(" << buffer.buffer << ", " << buffer.offset << ", " << buffer.range << "), ";
+ }
+ std::cerr << "}, (" << wg0 << "," << wg1 << "," << wg2 << "))");
+ GGML_ASSERT(wg0 <= ctx->device->properties.limits.maxComputeWorkGroupCount[0] &&
+ wg1 <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] &&
+ wg2 <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]);
+ GGML_ASSERT(ctx->descriptor_set_idx < ctx->descriptor_sets.size());
+ GGML_ASSERT(descriptor_buffer_infos.size() <= MAX_PARAMETER_COUNT);
+ GGML_ASSERT(pipeline->parameter_count == descriptor_buffer_infos.size());
+ GGML_ASSERT(pipeline->push_constant_size == push_constant_size(push_constants));
+
+ vk::DescriptorSet& descriptor_set = ctx->descriptor_sets[ctx->descriptor_set_idx++];
+ vk::WriteDescriptorSet write_descriptor_set{ descriptor_set, 0, 0, pipeline->parameter_count, vk::DescriptorType::eStorageBuffer, nullptr, descriptor_buffer_infos.begin() };
+ ctx->device->device.updateDescriptorSets({ write_descriptor_set }, {});
+
+ subctx->s->buffer->buf.pushConstants(pipeline->layout, vk::ShaderStageFlagBits::eCompute, 0, push_constant_size(push_constants), push_constant_data(push_constants));
+ subctx->s->buffer->buf.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline->pipeline);
+ subctx->s->buffer->buf.bindDescriptorSets(vk::PipelineBindPoint::eCompute,
+ pipeline->layout,
+ 0,
+ { descriptor_set },
+ {});
+ {
+ ggml_vk_debug_label dbg(subctx, pipeline->name, wg0, wg1, wg2);
+ subctx->s->buffer->buf.dispatch(wg0, wg1, wg2);
+ }
+}
+
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp
new file mode 100644
index 000000000..15abd5468
--- /dev/null
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp
@@ -0,0 +1,1561 @@
+#include "ggml-vulkan-common.h"
+
+bool vk_memory_logger_enabled = false;
+
+bool vk_perf_logger_enabled = false;
+
+bool vk_perf_logger_concurrent = false;
+
+bool vk_enable_sync_logger = false;
+
+uint32_t vk_perf_logger_frequency = 1;
+
+std::string vk_pipeline_stats_filter;
+
+void vk_memory_logger::log_allocation(vk_buffer_ref buf_ref, size_t size) {
+ if (!vk_memory_logger_enabled) {
+ return;
+ }
+ std::lock_guard<std::mutex> guard(log_mutex);
+ vk_buffer buf = buf_ref.lock();
+ const bool device = bool(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eDeviceLocal);
+ const std::string type = device ? "device" : "host";
+ allocations[buf->buffer] = size;
+ total_device += device ? size : 0;
+ total_host += device ? 0 : size;
+ VK_LOG_MEMORY(buf->device->name << ": +" << format_size(size) << " " << type << " at " << buf->buffer << ". Total device: " << format_size(total_device) << ", total host: " << format_size(total_host));
+}
+
+void vk_memory_logger::log_deallocation(vk_buffer_ref buf_ref) {
+ if (buf_ref.expired() || buf_ref.lock()->size == 0 || !vk_memory_logger_enabled) {
+ return;
+ }
+
+ std::lock_guard<std::mutex> guard(log_mutex);
+ vk_buffer buf = buf_ref.lock();
+ const bool device = bool(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eDeviceLocal);
+ std::string type = device ? "device" : "host";
+ auto it = allocations.find(buf->buffer);
+ if (it != allocations.end()) {
+ total_device -= device ? it->second : 0;
+ total_host -= device ? 0 : it->second;
+ VK_LOG_MEMORY(buf->device->name << ": -" << format_size(it->second) << " " << type << " at " << buf->buffer << ". Total device: " << format_size(total_device) << ", total host: " << format_size(total_host));
+ allocations.erase(it);
+ } else {
+ VK_LOG_MEMORY("ERROR " << buf->device->name << ": Attempted to deallocate unknown " << type << " memory at " << buf->buffer);
+ }
+}
+
+#ifdef GGML_VULKAN_CHECK_RESULTS
+static size_t vk_skip_checks;
+static size_t vk_output_tensor;
+
+static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name);
+static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx);
+static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx);
+#endif
+
+#ifdef GGML_VULKAN_RUN_TESTS
+static void ggml_vk_print_matrix_area(const void * data, ggml_type type, int ne0, int ne1, int i0, int i1, int i2) {
+ if (type != GGML_TYPE_F32 && type != GGML_TYPE_F16) {
+ return;
+ }
+ i0 = std::max(i0, 5);
+ i1 = std::max(i1, 5);
+ i2 = std::max(i2, 0);
+ fprintf(stderr, " ");
+ for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {
+ fprintf(stderr, "%7d ", idx1);
+ }
+ fprintf(stderr, "\n");
+ for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) {
+ fprintf(stderr, "%7d: ", idx0);
+ for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {
+ if (idx0 >= 0 && idx0 < ne0 && idx1 >= 0 && idx1 < ne1) {
+ float val;
+ if (type == GGML_TYPE_F32) {
+ val = *((const float *) data + i2*ne1*ne0 + idx1*ne0 + idx0);
+ } else if (type == GGML_TYPE_F16) {
+ val = ggml_fp16_to_fp32(*((const ggml_fp16_t *) data + i2*ne1*ne0 + idx1*ne0 + idx0));
+ } else {
+ GGML_ABORT("fatal error");
+ }
+ fprintf(stderr, "% 7.2f ", val);
+ } else {
+ fprintf(stderr, " ");
+ }
+ }
+ fprintf(stderr, "\n");
+ }
+}
+
+template <typename X_TYPE, typename Y_TYPE>
+static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size_t k, size_t batch, size_t num_it, int split_k, int shader_size) {
+ VK_LOG_DEBUG("ggml_vk_test_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << shader_size << ")");
+ const size_t x_ne = m * k * batch;
+ const size_t y_ne = k * n * batch;
+ const size_t d_ne = m * n * batch;
+
+ ggml_type x_type = std::is_same<float, X_TYPE>() ? GGML_TYPE_F32 : GGML_TYPE_F16;
+ ggml_type y_type = std::is_same<float, Y_TYPE>() ? GGML_TYPE_F32 : GGML_TYPE_F16;
+ vk_matmul_pipeline_key mm_test_key{x_type, y_type, false, false};
+ auto mm_test_it = ctx->device->pipeline_matmul.find(mm_test_key);
+ GGML_ASSERT(mm_test_it != ctx->device->pipeline_matmul.end() && !mm_test_it->second.empty());
+ auto& mm_test_configs = mm_test_it->second;
+ GGML_ASSERT(shader_size >= 0 && shader_size < (int)mm_test_configs.size());
+
+ std::string shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_ALIGNED_" + std::to_string(shader_size);
+ vk_pipeline p = mm_test_configs[shader_size].aligned ? mm_test_configs[shader_size].aligned : mm_test_configs[shader_size].unaligned;
+
+ const size_t kpad = ggml_vk_align_size(k, mm_test_configs[shader_size].align);
+
+ if (k != kpad) {
+ p = mm_test_configs[shader_size].unaligned;
+ shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_" + std::to_string(shader_size);
+ }
+
+ if (split_k > 1) {
+ ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, num_it);
+
+ if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) {
+ // Resize buffer
+ if (ctx->prealloc_split_k != nullptr) {
+ ggml_vk_destroy_buffer(ctx->prealloc_split_k);
+ }
+ ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, {vk::MemoryPropertyFlagBits::eDeviceLocal});
+ }
+ }
+
+ ggml_pipeline_allocate_descriptor_sets(ctx);
+
+ vk_buffer d_X = ggml_vk_create_buffer_check(ctx->device, sizeof(X_TYPE) * x_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});
+ vk_buffer d_Y = ggml_vk_create_buffer_check(ctx->device, sizeof(Y_TYPE) * y_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});
+ vk_buffer d_D = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});
+
+ X_TYPE* x = (X_TYPE *) malloc(sizeof(X_TYPE) * x_ne);
+ Y_TYPE* y = (Y_TYPE *) malloc(sizeof(Y_TYPE) * y_ne);
+ float* d = (float *) malloc(sizeof(float) * d_ne);
+
+ for (size_t i = 0; i < x_ne; i++) {
+ if (std::is_same<float, X_TYPE>()) {
+ x[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;
+ // x[i] = 1.0f;
+ // x[i] = i + 1;
+ // x[i] = (i % k == i / k) ? 1.0f : 0.0f;
+ } else if (std::is_same<ggml_fp16_t, X_TYPE>()) {
+ x[i] = ggml_fp32_to_fp16((rand() / (float)RAND_MAX) * 2.0f - 1.0f);
+ // x[i] = ggml_fp32_to_fp16(1.0f);
+ // x[i] = ggml_fp32_to_fp16(i + 1);
+ // x[i] = ggml_fp32_to_fp16((i % k == i / k) ? 1.0f : 0.0f);
+ } else {
+ GGML_ABORT("fatal error");
+ }
+ }
+ for (size_t i = 0; i < y_ne; i++) {
+ if (std::is_same<float, Y_TYPE>()) {
+ y[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;
+ // y[i] = (i % k == i / k) ? 1.0f : 0.0f;
+ // y[i] = i + 1;
+ } else if (std::is_same<ggml_fp16_t, Y_TYPE>()) {
+ y[i] = ggml_fp32_to_fp16((rand() / (float)RAND_MAX) * 2.0f - 1.0f);
+ // y[i] = ggml_fp32_to_fp16((i % k == i / k) ? 1.0f : 0.0f);
+ // y[i] = ggml_fp32_to_fp16(i + 1);
+ } else {
+ GGML_ABORT("fatal error");
+ }
+ }
+
+ ggml_vk_buffer_write(d_X, 0, x, sizeof(X_TYPE) * k * m * batch);
+ ggml_vk_buffer_write(d_Y, 0, y, sizeof(Y_TYPE) * k * n * batch);
+
+ vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);
+ ggml_vk_ctx_begin(ctx->device, subctx);
+ for (size_t i = 0; i < num_it; i++) {
+ ggml_vk_matmul(
+ ctx, subctx, p, ggml_vk_subbuffer(ctx, d_X), ggml_vk_subbuffer(ctx, d_Y), ggml_vk_subbuffer(ctx, d_D), ggml_vk_subbuffer(ctx, ctx->prealloc_split_k),
+ m, n, k,
+ k, k, m, k*m, k*n, m*n,
+ split_k, batch, batch, batch, 1, 1, n
+ );
+ }
+ ggml_vk_ctx_end(subctx);
+
+ auto begin = std::chrono::high_resolution_clock::now();
+ ggml_vk_submit(subctx, ctx->fence);
+ VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences", ctx->device);
+ ctx->device->device.resetFences({ ctx->fence });
+ ggml_vk_queue_command_pools_cleanup(ctx->device);
+
+ auto end = std::chrono::high_resolution_clock::now();
+ double time = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;
+
+ // copy dst to host
+ ggml_vk_buffer_read(d_D, 0, d, sizeof(float) * d_ne);
+
+ float * d_chk = (float *) malloc(sizeof(float) * d_ne);
+
+ ggml_init_params iparams = {
+ /*.mem_size =*/ 1024*1024*1024,
+ /*.mem_buffer =*/ NULL,
+ /*.no_alloc =*/ true,
+ };
+
+ ggml_context * ggml_ctx = ggml_init(iparams);
+
+ ggml_type src0_type;
+ ggml_type src1_type;
+
+ if (std::is_same<float, X_TYPE>()) {
+ src0_type = GGML_TYPE_F32;
+ } else if (std::is_same<ggml_fp16_t, X_TYPE>()) {
+ src0_type = GGML_TYPE_F16;
+ } else {
+ GGML_ABORT("fatal error");
+ }
+ if (std::is_same<float, Y_TYPE>()) {
+ src1_type = GGML_TYPE_F32;
+ } else if (std::is_same<ggml_fp16_t, Y_TYPE>()) {
+ src1_type = GGML_TYPE_F16;
+ } else {
+ GGML_ABORT("fatal error");
+ }
+
+ ggml_tensor * src0_ggml = ggml_new_tensor_3d(ggml_ctx, src0_type, k, m, batch);
+ ggml_tensor * src1_ggml = ggml_new_tensor_3d(ggml_ctx, src1_type, k, n, batch);
+ ggml_tensor * tensor_ggml = ggml_mul_mat(ggml_ctx, src0_ggml, src1_ggml);
+
+ src0_ggml->data = x;
+ src1_ggml->data = y;
+ tensor_ggml->data = d_chk;
+
+ ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx);
+ ggml_build_forward_expand(cgraph, tensor_ggml);
+
+ ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1);
+
+ ggml_free(ggml_ctx);
+
+ double avg_err = 0.0;
+ int first_err_n = -1;
+ int first_err_m = -1;
+ int first_err_b = -1;
+
+ for (size_t i = 0; i < m*n*batch; i++) {
+ double err = std::fabs(d[i] - d_chk[i]);
+ avg_err += err;
+
+ if ((err > 0.05f || std::isnan(err)) && first_err_n == -1) {
+ first_err_b = i / (m * n);
+ first_err_n = (i % (m * n)) / m;
+ first_err_m = (i % (m * n)) % m;
+ }
+ }
+
+ avg_err /= m * n;
+
+ double tflops = 2.0*m*n*k*batch*num_it / (time / 1000.0) / (1000.0*1000.0*1000.0*1000.0);
+
+ std::cerr << "TEST " << shname << " m=" << m << " n=" << n << " k=" << k << " batch=" << batch << " split_k=" << split_k << " matmul " << time / num_it << "ms " << tflops << " TFLOPS avg_err=" << avg_err << std::endl;
+
+ if (avg_err > 0.1 || std::isnan(avg_err)) {
+ std::cerr << "m = " << first_err_m << " n = " << first_err_n << " b = " << first_err_b << std::endl;
+ std::cerr << "Actual result: " << std::endl << std::endl;
+ ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+ std::cerr << "Expected result: " << std::endl << std::endl;
+ ggml_vk_print_matrix_area(d_chk, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+
+ if (split_k > 1) {
+ float * split_k_buf = (float *) malloc(sizeof(float) * d_ne * split_k);
+ ggml_vk_buffer_read(ctx->prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k);
+
+ std::cerr << "d_buf0: " << std::endl << std::endl;
+ ggml_vk_print_matrix_area(split_k_buf, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+
+ std::cerr << "d_buf1: " << std::endl << std::endl;
+ ggml_vk_print_matrix_area(split_k_buf + d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+
+ std::cerr << "d_buf2: " << std::endl << std::endl;
+ ggml_vk_print_matrix_area(split_k_buf + 2 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+
+ std::cerr << "d_buf3: " << std::endl << std::endl;
+ ggml_vk_print_matrix_area(split_k_buf + 3 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+
+ free(split_k_buf);
+ }
+ }
+
+ free(d_chk);
+
+ ggml_vk_command_pool_cleanup(ctx->device, ctx->compute_cmd_pool);
+
+ ggml_vk_destroy_buffer(d_X);
+ ggml_vk_destroy_buffer(d_Y);
+ ggml_vk_destroy_buffer(d_D);
+
+ free(x);
+ free(y);
+ free(d);
+}
+
+static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, int i0, int i1, int i2, int i3) {
+ if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16) {
+ return;
+ }
+ i0 = std::max(i0, 5);
+ i1 = std::max(i1, 5);
+ i2 = std::max(i2, 0);
+ i3 = std::max(i3, 0);
+ fprintf(stderr, " ");
+ for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {
+ fprintf(stderr, "%7d ", idx1);
+ }
+ fprintf(stderr, "\n");
+ for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) {
+ fprintf(stderr, "%7d: ", idx0);
+ for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {
+ if (idx0 >= 0 && idx0 < tensor->ne[0] && idx1 >= 0 && idx1 < tensor->ne[1] && i2 >= 0 && i2 < tensor->ne[2] && i3 >= 0 && i3 < tensor->ne[3]) {
+ float val;
+ if (tensor->type == GGML_TYPE_F32) {
+ val = *(float *) ((char *) tensor->data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]);
+ } else if (tensor->type == GGML_TYPE_F16) {
+ val = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) tensor->data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]));
+ } else {
+ GGML_ABORT("fatal error");
+ }
+ fprintf(stderr, "% 7.2f ", val);
+ } else {
+ fprintf(stderr, " ");
+ }
+ }
+ fprintf(stderr, "\n");
+ }
+}
+
+static void ggml_vk_quantize_data(const float * from, void * to, size_t ne, ggml_type quant) {
+ ggml_quantize_chunk(quant, from, to, 0, 1, ne, nullptr);
+}
+
+static void ggml_vk_dequantize_data(const void * from, float * to, size_t ne, ggml_type quant) {
+ if (quant == GGML_TYPE_F32) {
+ memcpy(to, from, sizeof(float) * ne);
+ return;
+ }
+
+ const auto * tt = ggml_get_type_traits(quant);
+
+ ggml_to_float_t dequant_fn = tt->to_float;
+
+ dequant_fn(from, to, ne);
+}
+
+static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) {
+ VK_LOG_DEBUG("ggml_vk_test_dequant(" << ne << ")");
+ const size_t x_sz = sizeof(float) * ne;
+ const size_t x_sz_f16 = sizeof(ggml_fp16_t) * ne;
+ const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant);
+ float * x = (float *) malloc(x_sz);
+ void * qx = malloc(qx_sz);
+ vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
+ vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz_f16, {vk::MemoryPropertyFlagBits::eDeviceLocal});
+ float * x_ref = (float *) malloc(x_sz);
+ ggml_fp16_t * x_chk = (ggml_fp16_t *) malloc(x_sz_f16);
+
+ for (size_t i = 0; i < ne; i++) {
+ x[i] = rand() / (float)RAND_MAX;
+ }
+
+ vk_pipeline p = ggml_vk_get_to_fp16(ctx, quant);
+
+ ggml_vk_quantize_data(x, qx, ne, quant);
+ ggml_vk_dequantize_data(qx, x_ref, ne, quant);
+
+ ggml_pipeline_request_descriptor_sets(ctx, p, 1);
+
+ ggml_pipeline_allocate_descriptor_sets(ctx);
+
+ ggml_vk_buffer_write(qx_buf, 0, qx, qx_sz);
+
+ vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);
+ ggml_vk_ctx_begin(ctx->device, subctx);
+ const std::vector<uint32_t> pc = { 1, (uint32_t)ne, (uint32_t)ne, (uint32_t)ne, (uint32_t)ne };
+ ggml_vk_dispatch_pipeline(ctx, subctx, p, { vk_subbuffer{ qx_buf, 0, qx_sz }, vk_subbuffer{ x_buf, 0, x_sz_f16 } }, pc, { (uint32_t)ne, 1, 1});
+ ggml_vk_ctx_end(subctx);
+
+ auto begin = std::chrono::high_resolution_clock::now();
+
+ ggml_vk_submit(subctx, ctx->fence);
+ VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);
+ ctx->device->device.resetFences({ ctx->fence });
+ ggml_vk_queue_command_pools_cleanup(ctx->device);
+
+ auto end = std::chrono::high_resolution_clock::now();
+
+ double ms_dequant = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;
+ ggml_vk_buffer_read(x_buf, 0, x_chk, x_sz_f16);
+
+ int first_err = -1;
+
+ double avg_err = 0.0;
+ for (size_t i = 0; i < ne; i++) {
+ double error = std::fabs(x_ref[i] - ggml_fp16_to_fp32(x_chk[i]));
+ avg_err += error;
+
+ if (first_err < 0 && error > 0.05) {
+ first_err = i;
+ }
+ }
+
+ avg_err /= ne;
+
+ std::cerr << "TEST DEQUANT " << ggml_type_name(quant) << " time=" << ms_dequant << "ms avg_err=" << avg_err << std::endl;
+
+ if (avg_err > 0.1) {
+ std::cerr << "first_error = " << first_err << std::endl;
+ std::cerr << "Actual result: " << std::endl << std::endl;
+ for (int i = std::max(0, first_err - 5); i < std::min((int)ne, first_err + 5); i++) {
+ std::cerr << ggml_fp16_to_fp32(x_chk[i]) << ", ";
+ }
+ std::cerr << std::endl << "Expected result: " << std::endl << std::endl;
+ for (int i = std::max(0, first_err - 5); i < std::min((int)ne, first_err + 5); i++) {
+ std::cerr << x_ref[i] << ", ";
+ }
+ std::cerr << std::endl;
+ }
+
+ ggml_vk_destroy_buffer(x_buf);
+ ggml_vk_destroy_buffer(qx_buf);
+
+ free(x);
+ free(qx);
+ free(x_ref);
+ free(x_chk);
+}
+
+// This does not work without ggml q8_1 quantization support
+//
+// typedef uint16_t ggml_half;
+// typedef uint32_t ggml_half2;
+//
+// #define QK8_1 32
+// typedef struct {
+// union {
+// struct {
+// ggml_half d; // delta
+// ggml_half s; // d * sum(qs[i])
+// } GGML_COMMON_AGGR_S;
+// ggml_half2 ds;
+// } GGML_COMMON_AGGR_U;
+// int8_t qs[QK8_1]; // quants
+// } block_q8_1;
+//
+// static void ggml_vk_test_quantize(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) {
+// VK_LOG_DEBUG("ggml_vk_test_quantize(" << ne << ")");
+// GGML_ASSERT(quant == GGML_TYPE_Q8_1);
+//
+// const size_t x_sz = sizeof(float) * ne;
+// const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant);
+// float * x = (float *) malloc(x_sz);
+// block_q8_1 * qx = (block_q8_1 *)malloc(qx_sz);
+// block_q8_1 * qx_res = (block_q8_1 *)malloc(qx_sz);
+// vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
+// vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
+//
+// for (size_t i = 0; i < ne; i++) {
+// x[i] = rand() / (float)RAND_MAX;
+// }
+//
+// vk_pipeline p = ggml_vk_get_quantize_pipeline(ctx, quant);
+//
+// ggml_pipeline_request_descriptor_sets(ctx, p, 1);
+//
+// ggml_pipeline_allocate_descriptor_sets(ctx);
+//
+// ggml_vk_buffer_write(x_buf, 0, x, x_sz);
+//
+// vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);
+// ggml_vk_ctx_begin(ctx->device, subctx);
+// ggml_vk_quantize_q8_1(ctx, subctx, ggml_vk_subbuffer(ctx, x_buf), ggml_vk_subbuffer(ctx, qx_buf), ne);
+// ggml_vk_ctx_end(subctx);
+//
+// auto begin = std::chrono::high_resolution_clock::now();
+//
+// ggml_vk_submit(subctx, ctx->fence);
+// VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_quantize waitForFences");
+// ctx->device->device.resetFences({ ctx->fence });
+// ggml_vk_queue_command_pools_cleanup(ctx->device);
+//
+// auto end = std::chrono::high_resolution_clock::now();
+//
+// double ms_quant = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;
+// ggml_vk_buffer_read(qx_buf, 0, qx, qx_sz);
+//
+// ggml_vk_quantize_data(x, qx_res, ne, quant);
+//
+// int first_err = -1;
+//
+// for (size_t i = 0; i < ne / 32; i++) {
+// double error = std::fabs(ggml_fp16_to_fp32(qx_res[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) - ggml_fp16_to_fp32(qx[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d));
+//
+// if (first_err < 0 && error > 0.1) {
+// first_err = i;
+// }
+//
+// error = std::fabs(ggml_fp16_to_fp32(qx_res[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) - ggml_fp16_to_fp32(qx[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s));
+//
+// if (first_err < 0 && error > 0.1) {
+// first_err = i;
+// }
+//
+// for (size_t j = 0; j < 32; j++) {
+// uint64_t error = std::abs(qx_res[i].qs[j] - qx[i].qs[j]);
+//
+// if (first_err < 0 && error > 1) {
+// first_err = i;
+// }
+// }
+// }
+//
+// std::cerr << "TEST QUANTIZE " << ggml_type_name(quant) << " time=" << ms_quant << "ms " << (first_err == -1 ? "CORRECT" : "INCORRECT") << std::endl;
+//
+// if (first_err != -1) {
+// std::cerr << "first_error = " << first_err << std::endl;
+// std::cerr << "Actual result: " << std::endl << std::endl;
+// std::cout << "d=" << ggml_fp16_to_fp32(qx[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) << " s=" << ggml_fp16_to_fp32(qx[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) << " ";
+// for (size_t j = 0; j < 32; j++) {
+// std::cout << " qs" << j << "=" << (uint32_t)qx[first_err].qs[j] << " ";
+// }
+// std::cerr << std::endl << std::endl << "Expected result: " << std::endl << std::endl;
+// std::cout << "d=" << ggml_fp16_to_fp32(qx_res[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) << " s=" << ggml_fp16_to_fp32(qx_res[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) << " ";
+// for (size_t j = 0; j < 32; j++) {
+// std::cout << " qs" << j << "=" << (uint32_t)qx_res[first_err].qs[j] << " ";
+// }
+// std::cerr << std::endl;
+// }
+//
+// ggml_vk_destroy_buffer(x_buf);
+// ggml_vk_destroy_buffer(qx_buf);
+//
+// free(x);
+// free(qx);
+// free(qx_res);
+// }
+
+static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size_t k, size_t batch, size_t num_it, size_t split_k, size_t shader_size, ggml_type quant, bool mmq = false) {
+ VK_LOG_DEBUG("ggml_vk_test_dequant_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << ggml_type_name(quant) << ")");
+ const size_t x_ne = m * k * batch;
+ const size_t y_ne = k * n * batch;
+ const size_t d_ne = m * n * batch;
+
+ ggml_type b_type = mmq ? GGML_TYPE_Q8_1 : GGML_TYPE_F32;
+ bool f16acc = ctx->device->fp16 && !mmq;
+ vk_matmul_pipeline_key dq_key{quant, b_type, false, f16acc};
+ auto dq_it = ctx->device->pipeline_matmul.find(dq_key);
+ if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) {
+ if (f16acc) {
+ dq_key.f16acc = false;
+ dq_it = ctx->device->pipeline_matmul.find(dq_key);
+ }
+ }
+ if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) {
+ std::cerr << "error: no pipeline for ggml_vk_test_dequant_matmul " << ggml_type_name(quant) << std::endl;
+ return;
+ }
+ auto& dq_configs = dq_it->second;
+ if (shader_size >= (int)dq_configs.size()) {
+ std::cerr << "error: shader_size " << shader_size << " >= configs.size() " << dq_configs.size() << " for " << ggml_type_name(quant) << std::endl;
+ return;
+ }
+
+ std::string shname = std::string(ggml_type_name(quant)) + "_ALIGNED_" + std::to_string(shader_size);
+ vk_pipeline p = dq_configs[shader_size].aligned ? dq_configs[shader_size].aligned : dq_configs[shader_size].unaligned;
+
+ const size_t kpad = mmq ? 0 : ggml_vk_align_size(k, dq_configs[shader_size].align);
+
+ if (mmq || k != kpad) {
+ p = dq_configs[shader_size].unaligned;
+ shname = std::string(ggml_type_name(quant)) + "_" + std::to_string(shader_size);
+ }
+
+ if (p == nullptr) {
+ std::cerr << "error: no pipeline for ggml_vk_test_dequant_matmul " << ggml_type_name(quant) << std::endl;
+ return;
+ }
+
+ const size_t x_sz = sizeof(float) * x_ne;
+ const size_t y_sz = sizeof(float) * y_ne;
+ const size_t qx_sz = x_ne * ggml_type_size(quant)/ggml_blck_size(quant);
+ const size_t qy_sz = mmq ? y_ne * ggml_type_size(GGML_TYPE_Q8_1)/ggml_blck_size(GGML_TYPE_Q8_1) : y_sz;
+ const size_t d_sz = sizeof(float) * d_ne;
+ float * x = (float *) malloc(x_sz);
+ float * y = (float *) malloc(y_sz);
+ void * qx = malloc(qx_sz);
+ vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
+ vk_buffer y_buf = ggml_vk_create_buffer_check(ctx->device, y_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
+ vk_buffer qy_buf = ggml_vk_create_buffer_check(ctx->device, qy_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
+ vk_buffer d_buf = ggml_vk_create_buffer_check(ctx->device, d_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
+ float * d = (float *) malloc(d_sz);
+ float * d_chk = (float *) malloc(d_sz);
+
+ for (size_t i = 0; i < x_ne; i++) {
+ x[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;
+ // x[i] = (i % k == i / k) ? 1.0f : 0.0f;
+ // x[i] = i % k;
+ }
+
+ ggml_vk_quantize_data(x, qx, x_ne, quant);
+
+ for (size_t i = 0; i < y_ne; i++) {
+ y[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;
+ // y[i] = (i % k == i / k) ? 1.0f : 0.0f;
+ // y[i] = i % k;
+ }
+
+ if (split_k > 1) {
+ ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, num_it);
+
+ if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) {
+ // Resize buffer
+ if (ctx->prealloc_split_k != nullptr) {
+ ggml_vk_destroy_buffer(ctx->prealloc_split_k);
+ }
+ ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, {vk::MemoryPropertyFlagBits::eDeviceLocal});
+ }
+ }
+ if (mmq) {
+ vk_pipeline pipeline_quantize_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1);
+ ggml_pipeline_request_descriptor_sets(ctx, pipeline_quantize_q8_1, num_it);
+ }
+
+ ggml_pipeline_allocate_descriptor_sets(ctx);
+
+ ggml_vk_buffer_write(qx_buf, 0, qx, qx_sz);
+ ggml_vk_buffer_write(y_buf, 0, y, y_sz);
+
+ vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);
+ ggml_vk_ctx_begin(ctx->device, subctx);
+ if (mmq) {
+ for (size_t i = 0; i < num_it; i++) {
+ ggml_vk_quantize_q8_1(ctx, subctx, { y_buf, 0, y_sz }, { qy_buf, 0, qy_sz }, y_ne);
+ ggml_vk_matmul(
+ ctx, subctx, p, { qx_buf, 0, qx_sz }, { qy_buf, 0, qy_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k },
+ m, n, k,
+ k, k, m, k*m, k*n, m*n,
+ split_k, batch, batch, batch, 1, 1, n
+ );
+ }
+ } else {
+ for (size_t i = 0; i < num_it; i++) {
+ ggml_vk_matmul(
+ ctx, subctx, p, { qx_buf, 0, qx_sz }, { y_buf, 0, y_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k },
+ m, n, k,
+ k, k, m, k*m, k*n, m*n,
+ split_k, batch, batch, batch, 1, 1, n
+ );
+ }
+ }
+ ggml_vk_ctx_end(subctx);
+
+ auto begin = std::chrono::high_resolution_clock::now();
+
+ ggml_vk_submit(subctx, ctx->fence);
+ VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);
+ ctx->device->device.resetFences({ ctx->fence });
+ ggml_vk_queue_command_pools_cleanup(ctx->device);
+
+ auto end = std::chrono::high_resolution_clock::now();
+
+ double time_ms = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;
+ ggml_vk_buffer_read(d_buf, 0, d, d_sz);
+
+ ggml_init_params iparams = {
+ /*.mem_size =*/ 1024*1024*1024,
+ /*.mem_buffer =*/ NULL,
+ /*.no_alloc =*/ true,
+ };
+
+ ggml_context * ggml_ctx = ggml_init(iparams);
+
+ ggml_tensor * src0_ggml = ggml_new_tensor_3d(ggml_ctx, quant, k, m, batch);
+ ggml_tensor * src1_ggml = ggml_new_tensor_3d(ggml_ctx, GGML_TYPE_F32, k, n, batch);
+ ggml_tensor * tensor_ggml = ggml_mul_mat(ggml_ctx, src0_ggml, src1_ggml);
+
+ src0_ggml->data = qx;
+ src1_ggml->data = y;
+ tensor_ggml->data = d_chk;
+
+ ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx);
+ ggml_build_forward_expand(cgraph, tensor_ggml);
+
+ ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1);
+
+ ggml_free(ggml_ctx);
+
+ double avg_err = 0.0;
+ int first_err_n = -1;
+ int first_err_m = -1;
+ int first_err_b = -1;
+
+ for (size_t i = 0; i < m*n*batch; i++) {
+ double err = std::fabs(d[i] - d_chk[i]);
+ avg_err += err;
+
+ if ((err > 0.05f || std::isnan(err)) && first_err_n == -1) {
+ first_err_b = i / (m * n);
+ first_err_n = (i % (m * n)) / m;
+ first_err_m = (i % (m * n)) % m;
+ }
+ }
+
+ avg_err /= m * n;
+
+ double tflops = 2.0*m*n*k*batch*num_it / (time_ms / 1000.0) / (1000.0*1000.0*1000.0*1000.0);
+
+ std::cerr << "TEST dequant matmul " << shname;
+ if (mmq) {
+ std::cerr << " mmq";
+ }
+ std::cerr << " m=" << m << " n=" << n << " k=" << k << " batch=" << batch << " split_k=" << split_k << " matmul " << time_ms / num_it << "ms " << tflops << " TFLOPS avg_err=" << avg_err << std::endl;
+
+ if (avg_err > 0.01 || std::isnan(avg_err)) {
+ std::cerr << "m = " << first_err_m << " n = " << first_err_n << " b = " << first_err_b << std::endl;
+ std::cerr << "Actual result: " << std::endl << std::endl;
+ ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+ std::cerr << std::endl;
+ std::cerr << "Expected result: " << std::endl << std::endl;
+ ggml_vk_print_matrix_area(d_chk, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+
+ std::cerr << "src0: " << std::endl << std::endl;
+ ggml_vk_print_matrix_area(x, GGML_TYPE_F32, k, m, first_err_m, first_err_n, first_err_b);
+ std::cerr << std::endl;
+ std::cerr << "src1: " << std::endl << std::endl;
+ ggml_vk_print_matrix_area(y, GGML_TYPE_F32, k, n, first_err_m, first_err_n, first_err_b);
+
+ if (split_k > 1) {
+ float * split_k_buf = (float *) malloc(sizeof(float) * d_ne * split_k);
+ ggml_vk_buffer_read(ctx->prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k);
+
+ std::cerr << "d_buf0: " << std::endl << std::endl;
+ ggml_vk_print_matrix_area(split_k_buf, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+
+ std::cerr << "d_buf1: " << std::endl << std::endl;
+ ggml_vk_print_matrix_area(split_k_buf + d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+
+ std::cerr << "d_buf2: " << std::endl << std::endl;
+ ggml_vk_print_matrix_area(split_k_buf + 2 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+
+ std::cerr << "d_buf3: " << std::endl << std::endl;
+ ggml_vk_print_matrix_area(split_k_buf + 3 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+
+ free(split_k_buf);
+ }
+ }
+
+ ggml_vk_destroy_buffer(qx_buf);
+ ggml_vk_destroy_buffer(y_buf);
+ ggml_vk_destroy_buffer(qy_buf);
+ ggml_vk_destroy_buffer(d_buf);
+
+ free(x);
+ free(qx);
+ free(y);
+ free(d);
+ free(d_chk);
+}
+#endif
+
+int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op) {
+ switch (op->op) {
+ case GGML_OP_GET_ROWS:
+ return 0;
+ case GGML_OP_MUL_MAT:
+ return op->ne[1];
+ case GGML_OP_MUL_MAT_ID:
+ case GGML_OP_ROPE:
+ case GGML_OP_ROPE_BACK:
+ return op->ne[2];
+ default:
+ return ggml_nrows(op);
+ }
+}
+
+#ifdef GGML_VULKAN_CHECK_RESULTS
+static void ggml_vk_print_graph_origin(const ggml_tensor * tensor, std::vector<const ggml_tensor *>& done, int level = 0) {
+ if (std::find(done.begin(), done.end(), tensor) != done.end() || level > 10) {
+ return;
+ }
+ for (int j = 0; j < level; j++) {
+ std::cerr << " ";
+ }
+ std::cerr << ggml_op_name(tensor->op) << " gpu=" << (tensor->extra != nullptr) << std::endl;
+
+ done.push_back(tensor);
+
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
+ if (tensor->src[i] != nullptr) {
+ ggml_vk_print_graph_origin(tensor->src[i], done, level + 1);
+ }
+ }
+}
+
+static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, const void * data, int i0, int i1, int i2, int i3) {
+ if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16 && tensor->type != GGML_TYPE_I32) {
+ return;
+ }
+ i0 = std::max(i0, 5);
+ i1 = std::max(i1, 5);
+ i2 = std::max(i2, 0);
+ i3 = std::max(i3, 0);
+ fprintf(stderr, " ");
+ for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {
+ fprintf(stderr, "%7d ", idx1);
+ }
+ fprintf(stderr, "\n");
+ for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) {
+ fprintf(stderr, "%7d: ", idx0);
+ for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {
+ if (idx0 >= 0 && idx0 < tensor->ne[0] && idx1 >= 0 && idx1 < tensor->ne[1] && i2 >= 0 && i2 < tensor->ne[2] && i3 >= 0 && i3 < tensor->ne[3]) {
+ float val;
+ if (tensor->type == GGML_TYPE_F32) {
+ val = *(const float *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]);
+ } else if (tensor->type == GGML_TYPE_F16) {
+ val = ggml_fp16_to_fp32(*(const ggml_fp16_t *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]));
+ } else if (tensor->type == GGML_TYPE_I32) {
+ val = *(const int32_t *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]);
+ } else {
+ GGML_ABORT("fatal error");
+ }
+ fprintf(stderr, "% 7.2f ", val);
+ } else {
+ fprintf(stderr, " ");
+ }
+ }
+ fprintf(stderr, "\n");
+ }
+}
+
+static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name) {
+ void * tensor_data = tensor->data;
+
+ const bool is_gpu = tensor->buffer != nullptr && ggml_backend_buffer_is_vk(tensor->buffer);
+
+ if (is_gpu) {
+ const size_t tensor_size = ggml_nbytes(tensor);
+ tensor_data = malloc(tensor_size);
+
+ ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context;
+
+ vk_buffer buffer_gpu = buf_ctx->dev_buffer;
+ ggml_vk_buffer_read(buffer_gpu, vk_tensor_offset(tensor) + tensor->view_offs, tensor_data, tensor_size);
+ }
+
+ std::cerr << "TENSOR CHECK " << name << " (" << tensor->name << "): " << ggml_op_name(tensor->op) << std::endl;
+ std::cerr << "tensor=" << tensor << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << std::endl;
+ if (tensor->src[0] != nullptr) {
+ std::cerr << "tensor->src[0]=" << tensor->src[0] << " name=" << tensor->src[0]->name << " op=" << ggml_op_name(tensor->src[0]->op) << " type=" << ggml_type_name(tensor->src[0]->type) << " ne0=" << tensor->src[0]->ne[0] << " nb0=" << tensor->src[0]->nb[0] << " ne1=" << tensor->src[0]->ne[1] << " nb1=" << tensor->src[0]->nb[1] << " ne2=" << tensor->src[0]->ne[2] << " nb2=" << tensor->src[0]->nb[2] << " ne3=" << tensor->src[0]->ne[3] << " nb3=" << tensor->src[0]->nb[3] << std::endl;
+ }
+ if (tensor->src[1] != nullptr) {
+ std::cerr << "tensor->src[1]=" << tensor->src[1] << " name=" << tensor->src[1]->name << " op=" << ggml_op_name(tensor->src[1]->op) << " type=" << ggml_type_name(tensor->src[1]->type) << " ne0=" << tensor->src[1]->ne[0] << " nb0=" << tensor->src[1]->nb[0] << " ne1=" << tensor->src[1]->ne[1] << " nb1=" << tensor->src[1]->nb[1] << " ne2=" << tensor->src[1]->ne[2] << " nb2=" << tensor->src[1]->nb[2] << " ne3=" << tensor->src[1]->ne[3] << " nb3=" << tensor->src[1]->nb[3] << std::endl;
+ }
+ std::cerr << std::endl << "Result:" << std::endl;
+ ggml_vk_print_tensor_area(tensor, tensor_data, 5, 5, 0, 0);
+ std::cerr << std::endl;
+ std::vector<const ggml_tensor *> done;
+ ggml_vk_print_graph_origin(tensor, done);
+
+ if (is_gpu) {
+ free(tensor_data);
+ }
+}
+
+void * comp_result;
+size_t comp_size;
+size_t comp_nb[GGML_MAX_DIMS];
+size_t check_counter = 0;
+static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx) {
+ ggml_tensor * tensor = cgraph->nodes[tensor_idx + ctx->num_additional_fused_ops];
+ if (tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_SET_ROWS) {
+ return;
+ }
+
+ check_counter++;
+ if (!(vk_output_tensor > 0 && vk_output_tensor == check_counter) && check_counter <= vk_skip_checks) {
+ return;
+ }
+
+ VK_LOG_DEBUG("ggml_vk_check_results_0(" << tensor->name << ")");
+
+ struct ggml_init_params iparams = {
+ /*.mem_size =*/ 2ul*1024ul*1024ul*1024ul,
+ /*.mem_buffer =*/ NULL,
+ /*.no_alloc =*/ false,
+ };
+
+ struct ggml_context * ggml_ctx = ggml_init(iparams);
+
+ std::array<struct ggml_tensor *, GGML_MAX_SRC> src_clone = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr};
+ const char * srci_name[GGML_MAX_SRC] = {"src0", "src1", "src2", "src3", "src4", "src5", "src6", "src7", "src8", "src9"};
+
+ std::map<ggml_tensor *, ggml_tensor *> cloned_tensors;
+ std::vector<void *> cloned_mallocs;
+
+ struct ggml_tensor * tensor_clone = nullptr;
+
+ for (int f = 0; f < ctx->num_additional_fused_ops + 1; ++f) {
+ tensor = cgraph->nodes[tensor_idx + f];
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
+ ggml_tensor * srci = tensor->src[i];
+ if (srci == nullptr) {
+ continue;
+ }
+ // If a src tensor has been cloned, use that one
+ auto it = cloned_tensors.find(srci);
+ if (it != cloned_tensors.end()) {
+ src_clone[i] = it->second;
+ continue;
+ }
+ ggml_tensor * srci_clone = ggml_dup_tensor(ggml_ctx, srci);
+ size_t srci_size = ggml_nbytes(srci);
+
+ src_clone[i] = srci_clone;
+ void *src_buffer = malloc(srci_size);
+ cloned_mallocs.push_back(src_buffer);
+
+ srci_clone->data = src_buffer;
+ if (ggml_backend_buffer_is_host(srci->buffer)) {
+ memcpy(srci_clone->data, srci->data, srci_size);
+ memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS);
+ } else if (ggml_backend_buffer_is_vk(srci->buffer)) {
+ ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)srci->buffer->context;
+ vk_buffer& buffer_gpu = buf_ctx->dev_buffer;
+ uint64_t offset = vk_tensor_offset(srci) + srci->view_offs;
+ if (!ggml_is_contiguous(srci) && ggml_vk_dim01_contiguous(srci)) {
+ for (int i3 = 0; i3 < srci->ne[3]; i3++) {
+ for (int i2 = 0; i2 < srci->ne[2]; i2++) {
+ const int idx = i3*srci->ne[2] + i2;
+ ggml_vk_buffer_read(buffer_gpu, offset + idx * srci->nb[2], ((char *)srci_clone->data + idx * srci_clone->nb[2]), srci->ne[1] * srci->nb[1]);
+ }
+ }
+
+ srci_clone->nb[0] = srci->nb[0];
+ srci_clone->nb[1] = srci->nb[1];
+ for (int i = 2; i < GGML_MAX_DIMS; i++) {
+ srci_clone->nb[i] = srci_clone->nb[i - 1]*srci_clone->ne[i - 1];
+ }
+ } else {
+ if (offset + srci_size >= buffer_gpu->size) {
+ srci_size = buffer_gpu->size - offset;
+ }
+ ggml_vk_buffer_read(buffer_gpu, offset, srci_clone->data, srci_size);
+ memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS);
+ }
+ } else {
+ GGML_ABORT("fatal error");
+ }
+
+ if (vk_output_tensor > 0 && vk_output_tensor == check_counter) {
+ ggml_vk_print_tensor(srci, srci_name[i]);
+ }
+ }
+
+ if (tensor->op == GGML_OP_FLASH_ATTN_EXT) {
+ const float * params = (const float *)tensor->op_params;
+ tensor_clone = ggml_flash_attn_ext(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3], params[0], params[1], params[2]);
+ if (src_clone[4]) {
+ ggml_flash_attn_ext_add_sinks(tensor_clone, src_clone[4]);
+ }
+ } else if (tensor->op == GGML_OP_MUL_MAT) {
+ tensor_clone = ggml_mul_mat(ggml_ctx, src_clone[0], src_clone[1]);
+ } else if (tensor->op == GGML_OP_MUL_MAT_ID) {
+ tensor_clone = ggml_mul_mat_id(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]);
+ } else if (tensor->op == GGML_OP_SUB) {
+ tensor_clone = ggml_sub(ggml_ctx, src_clone[0], src_clone[1]);
+ } else if (tensor->op == GGML_OP_MUL) {
+ tensor_clone = ggml_mul(ggml_ctx, src_clone[0], src_clone[1]);
+ } else if (tensor->op == GGML_OP_DIV) {
+ tensor_clone = ggml_div(ggml_ctx, src_clone[0], src_clone[1]);
+ } else if (tensor->op == GGML_OP_CONCAT) {
+ tensor_clone = ggml_concat(ggml_ctx, src_clone[0], src_clone[1], *(int *)tensor->op_params);
+ } else if (tensor->op == GGML_OP_UPSCALE) {
+ tensor_clone = ggml_interpolate(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], (ggml_scale_mode) tensor->op_params[0]);
+ } else if (tensor->op == GGML_OP_SCALE) {
+ const float * params = (const float *)tensor->op_params;
+ tensor_clone = ggml_scale_bias(ggml_ctx, src_clone[0], params[0], params[1]);
+ } else if (tensor->op == GGML_OP_ADD1) {
+ tensor_clone = ggml_add1(ggml_ctx, src_clone[0], src_clone[1]);
+ } else if (tensor->op == GGML_OP_ARANGE) {
+ const float start = ggml_get_op_params_f32(tensor, 0);
+ const float stop = ggml_get_op_params_f32(tensor, 1);
+ const float step = ggml_get_op_params_f32(tensor, 2);
+ tensor_clone = ggml_arange(ggml_ctx, start, stop, step);
+ } else if (tensor->op == GGML_OP_FILL) {
+ const float value = ggml_get_op_params_f32(tensor, 0);
+ tensor_clone = ggml_fill(ggml_ctx, src_clone[0], value);
+ } else if (tensor->op == GGML_OP_SQR) {
+ tensor_clone = ggml_sqr(ggml_ctx, src_clone[0]);
+ } else if (tensor->op == GGML_OP_SQRT) {
+ tensor_clone = ggml_sqrt(ggml_ctx, src_clone[0]);
+ } else if (tensor->op == GGML_OP_SIN) {
+ tensor_clone = ggml_sin(ggml_ctx, src_clone[0]);
+ } else if (tensor->op == GGML_OP_COS) {
+ tensor_clone = ggml_cos(ggml_ctx, src_clone[0]);
+ } else if (tensor->op == GGML_OP_LOG) {
+ tensor_clone = ggml_log(ggml_ctx, src_clone[0]);
+ } else if (tensor->op == GGML_OP_TRI) {
+ tensor_clone = ggml_tri(ggml_ctx, src_clone[0], (ggml_tri_type)ggml_get_op_params_i32(tensor, 0));
+ } else if (tensor->op == GGML_OP_DIAG) {
+ tensor_clone = ggml_diag(ggml_ctx, src_clone[0]);
+ } else if (tensor->op == GGML_OP_CLAMP) {
+ const float * params = (const float *)tensor->op_params;
+ tensor_clone = ggml_clamp(ggml_ctx, src_clone[0], params[0], params[1]);
+ } else if (tensor->op == GGML_OP_PAD) {
+ tensor_clone = ggml_pad_ext(ggml_ctx, src_clone[0], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3],
+ tensor->op_params[4], tensor->op_params[5], tensor->op_params[6], tensor->op_params[7]);
+ } else if (tensor->op == GGML_OP_PAD_REFLECT_1D) {
+ tensor_clone = ggml_pad_reflect_1d(ggml_ctx, src_clone[0], tensor->op_params[0], tensor->op_params[1]);
+ } else if (tensor->op == GGML_OP_REPEAT) {
+ tensor_clone = ggml_repeat(ggml_ctx, src_clone[0], tensor);
+ } else if (tensor->op == GGML_OP_REPEAT_BACK) {
+ tensor_clone = ggml_repeat_back(ggml_ctx, src_clone[0], tensor);
+ } else if (tensor->op == GGML_OP_ADD) {
+ tensor_clone = ggml_add(ggml_ctx, src_clone[0], src_clone[1]);
+ } else if (tensor->op == GGML_OP_ACC) {
+ tensor_clone = ggml_acc(ggml_ctx, src_clone[0], src_clone[1], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3]);
+ } else if (tensor->op == GGML_OP_SET) {
+ tensor_clone = ggml_set(ggml_ctx, src_clone[0], src_clone[1], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3]);
+ } else if (tensor->op == GGML_OP_NORM) {
+ tensor_clone = ggml_norm(ggml_ctx, src_clone[0], *(float *)tensor->op_params);
+ } else if (tensor->op == GGML_OP_GROUP_NORM) {
+ const float * float_params = (const float *)tensor->op_params;
+ tensor_clone = ggml_group_norm(ggml_ctx, src_clone[0], tensor->op_params[0], float_params[1]);
+ } else if (tensor->op == GGML_OP_RMS_NORM) {
+ tensor_clone = ggml_rms_norm(ggml_ctx, src_clone[0], *(float *)tensor->op_params);
+ } else if (tensor->op == GGML_OP_RMS_NORM_BACK) {
+ const float eps = ((float *) tensor->op_params)[0];
+ tensor_clone = ggml_rms_norm_back(ggml_ctx, src_clone[0], src_clone[1], eps);
+ } else if (tensor->op == GGML_OP_SILU_BACK) {
+ tensor_clone = ggml_silu_back(ggml_ctx, src_clone[0], src_clone[1]);
+ } else if (tensor->op == GGML_OP_L2_NORM) {
+ const float eps = ((float *) tensor->op_params)[0];
+ tensor_clone = ggml_l2_norm(ggml_ctx, src_clone[0], eps);
+ } else if (tensor->op == GGML_OP_SOFT_MAX) {
+ if (tensor->src[1] != nullptr) {
+ const float * params = (const float *)tensor->op_params;
+ tensor_clone = ggml_soft_max_ext(ggml_ctx, src_clone[0], src_clone[1], params[0], params[1]);
+ } else {
+ tensor_clone = ggml_soft_max(ggml_ctx, src_clone[0]);
+ }
+ } else if (tensor->op == GGML_OP_SOFT_MAX_BACK) {
+ tensor_clone = ggml_soft_max_ext_back(ggml_ctx, src_clone[0], src_clone[1], ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]);
+ } else if (tensor->op == GGML_OP_DIAG_MASK_INF) {
+ tensor_clone = ggml_diag_mask_inf(ggml_ctx, src_clone[0], tensor->op_params[0]);
+ } else if (tensor->op == GGML_OP_ROPE || tensor->op == GGML_OP_ROPE_BACK) {
+ const int n_dims = ((int32_t *) tensor->op_params)[1];
+ const int mode = ((int32_t *) tensor->op_params)[2];
+ //const int n_ctx_ggml = ((int32_t *) tensor->op_params)[3];
+ const int n_ctx_orig_ggml = ((int32_t *) tensor->op_params)[4];
+ const float freq_base = ((float *) tensor->op_params)[5];
+ const float freq_scale = ((float *) tensor->op_params)[6];
+ const float ext_factor = ((float *) tensor->op_params)[7];
+ const float attn_factor = ((float *) tensor->op_params)[8];
+ const float beta_fast = ((float *) tensor->op_params)[9];
+ const float beta_slow = ((float *) tensor->op_params)[10];
+ if (mode & GGML_ROPE_TYPE_MROPE) {
+ int32_t *sections = ((int32_t *) tensor->op_params) + 11;
+ if (tensor->op == GGML_OP_ROPE) {
+ tensor_clone = ggml_rope_multi(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+ } else {
+ tensor_clone = ggml_rope_multi_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+ }
+ } else {
+ if (tensor->op == GGML_OP_ROPE) {
+ tensor_clone = ggml_rope_ext(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+ } else {
+ tensor_clone = ggml_rope_ext_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+ }
+ }
+ const int n_offs = ((int32_t *) tensor->op_params)[15];
+ if (n_offs != 0) {
+ tensor_clone = ggml_rope_set_offset(tensor_clone, n_offs);
+ }
+ } else if (tensor->op == GGML_OP_UNARY) {
+ switch (ggml_get_unary_op(tensor)) {
+ case GGML_UNARY_OP_EXP:
+ tensor_clone = ggml_exp(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_EXPM1:
+ tensor_clone = ggml_expm1(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_ELU:
+ tensor_clone = ggml_elu(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_SILU:
+ tensor_clone = ggml_silu(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_GELU:
+ tensor_clone = ggml_gelu(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_GELU_ERF:
+ tensor_clone = ggml_gelu_erf(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_GELU_QUICK:
+ tensor_clone = ggml_gelu_quick(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_RELU:
+ tensor_clone = ggml_relu(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_XIELU:
+ tensor_clone = ggml_xielu(ggml_ctx, src_clone[0], 0, 0, 0, 0);
+ ggml_set_op_params_f32(tensor_clone, 1, ggml_get_op_params_f32(tensor, 1));
+ ggml_set_op_params_f32(tensor_clone, 2, ggml_get_op_params_f32(tensor, 2));
+ ggml_set_op_params_f32(tensor_clone, 3, ggml_get_op_params_f32(tensor, 3));
+ ggml_set_op_params_f32(tensor_clone, 4, ggml_get_op_params_f32(tensor, 4));
+ break;
+ case GGML_UNARY_OP_NEG:
+ tensor_clone = ggml_neg(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_TANH:
+ tensor_clone = ggml_tanh(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_SIGMOID:
+ tensor_clone = ggml_sigmoid(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_HARDSIGMOID:
+ tensor_clone = ggml_hardsigmoid(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_HARDSWISH:
+ tensor_clone = ggml_hardswish(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_ABS:
+ tensor_clone = ggml_abs(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_SOFTPLUS:
+ tensor_clone = ggml_softplus(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_STEP:
+ tensor_clone = ggml_step(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_ROUND:
+ tensor_clone = ggml_round(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_CEIL:
+ tensor_clone = ggml_ceil(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_FLOOR:
+ tensor_clone = ggml_floor(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_TRUNC:
+ tensor_clone = ggml_trunc(ggml_ctx, src_clone[0]);
+ break;
+ case GGML_UNARY_OP_SGN:
+ tensor_clone = ggml_sgn(ggml_ctx, src_clone[0]);
+ break;
+ default:
+ std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl;
+ GGML_ABORT("fatal error");
+ }
+ } else if (tensor->op == GGML_OP_GLU) {
+ if (src_clone[1] == nullptr) {
+ tensor_clone = ggml_glu(ggml_ctx, src_clone[0], (ggml_glu_op) tensor->op_params[0], tensor->op_params[1]);
+ } else {
+ tensor_clone = ggml_glu_split(ggml_ctx, src_clone[0], src_clone[1], (ggml_glu_op) tensor->op_params[0]);
+ }
+ ggml_set_op_params_i32(tensor_clone, 2, ggml_get_op_params_i32(tensor, 2));
+ ggml_set_op_params_i32(tensor_clone, 3, ggml_get_op_params_i32(tensor, 3));
+ } else if (tensor->op == GGML_OP_CPY || tensor->op == GGML_OP_DUP) {
+ if (tensor->src[1] == nullptr) {
+ tensor_clone = ggml_dup(ggml_ctx, src_clone[0]);
+ tensor_clone->type = tensor->type;
+ } else {
+ tensor_clone = ggml_cpy(ggml_ctx, src_clone[0], src_clone[1]);
+ }
+ } else if (tensor->op == GGML_OP_CONT) {
+ tensor_clone = ggml_cont_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]);
+ } else if (tensor->op == GGML_OP_RESHAPE) {
+ tensor_clone = ggml_reshape_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]);
+ } else if (tensor->op == GGML_OP_VIEW) {
+ tensor_clone = ggml_view_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], tensor->nb[1], tensor->nb[2], tensor->nb[3], ((int32_t *) tensor->op_params)[0]);
+ } else if (tensor->op == GGML_OP_PERMUTE) {
+ int32_t * params = (int32_t *)tensor->op_params;
+ tensor_clone = ggml_permute(ggml_ctx, src_clone[0], params[0], params[1], params[2], params[3]);
+ } else if (tensor->op == GGML_OP_TRANSPOSE) {
+ tensor_clone = ggml_transpose(ggml_ctx, src_clone[0]);
+ } else if (tensor->op == GGML_OP_GET_ROWS) {
+ tensor_clone = ggml_get_rows(ggml_ctx, src_clone[0], src_clone[1]);
+ } else if (tensor->op == GGML_OP_ARGSORT) {
+ tensor_clone = ggml_argsort(ggml_ctx, src_clone[0], (ggml_sort_order) *(int *)tensor->op_params);
+ } else if (tensor->op == GGML_OP_TOP_K) {
+ tensor_clone = ggml_top_k(ggml_ctx, src_clone[0], tensor->ne[0]);
+ } else if (tensor->op == GGML_OP_SUM) {
+ tensor_clone = ggml_sum(ggml_ctx, src_clone[0]);
+ } else if (tensor->op == GGML_OP_SUM_ROWS) {
+ tensor_clone = ggml_sum_rows(ggml_ctx, src_clone[0]);
+ } else if (tensor->op == GGML_OP_CUMSUM) {
+ tensor_clone = ggml_cumsum(ggml_ctx, src_clone[0]);
+ } else if (tensor->op == GGML_OP_DSV4_HC_COMB) {
+ tensor_clone = ggml_dsv4_hc_comb(ggml_ctx, src_clone[0], src_clone[1], src_clone[2],
+ ggml_get_op_params_f32(tensor, 0), ggml_get_op_params_i32(tensor, 1));
+ } else if (tensor->op == GGML_OP_DSV4_HC_PRE) {
+ if (ggml_get_op_params_i32(tensor, 1) != 0) {
+ tensor_clone = ggml_dsv4_hc_pre_gated(ggml_ctx, src_clone[0], src_clone[1], ggml_get_op_params_f32(tensor, 0));
+ } else {
+ tensor_clone = ggml_dsv4_hc_pre(ggml_ctx, src_clone[0], src_clone[1]);
+ }
+ } else if (tensor->op == GGML_OP_DSV4_HC_POST) {
+ tensor_clone = ggml_dsv4_hc_post(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3]);
+ } else if (tensor->op == GGML_OP_MEAN) {
+ tensor_clone = ggml_mean(ggml_ctx, src_clone[0]);
+ } else if (tensor->op == GGML_OP_ARGMAX) {
+ tensor_clone = ggml_argmax(ggml_ctx, src_clone[0]);
+ } else if (tensor->op == GGML_OP_CROSS_ENTROPY_LOSS) {
+ tensor_clone = ggml_cross_entropy_loss(ggml_ctx, src_clone[0], src_clone[1]);
+ } else if (tensor->op == GGML_OP_CROSS_ENTROPY_LOSS_BACK) {
+ tensor_clone = ggml_cross_entropy_loss_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]);
+ } else if (tensor->op == GGML_OP_COUNT_EQUAL) {
+ tensor_clone = ggml_count_equal(ggml_ctx, src_clone[0], src_clone[1]);
+ } else if (tensor->op == GGML_OP_SOLVE_TRI) {
+ tensor_clone = ggml_solve_tri(ggml_ctx, src_clone[0], src_clone[1], true, true, false);
+ } else if (tensor->op == GGML_OP_IM2COL) {
+ const int32_t s0 = tensor->op_params[0];
+ const int32_t s1 = tensor->op_params[1];
+ const int32_t p0 = tensor->op_params[2];
+ const int32_t p1 = tensor->op_params[3];
+ const int32_t d0 = tensor->op_params[4];
+ const int32_t d1 = tensor->op_params[5];
+
+ const bool is_2D = tensor->op_params[6] == 1;
+ tensor_clone = ggml_im2col(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1, is_2D, tensor->type);
+ } else if (tensor->op == GGML_OP_IM2COL_3D) {
+ const int32_t s0 = tensor->op_params[0];
+ const int32_t s1 = tensor->op_params[1];
+ const int32_t s2 = tensor->op_params[2];
+ const int32_t p0 = tensor->op_params[3];
+ const int32_t p1 = tensor->op_params[4];
+ const int32_t p2 = tensor->op_params[5];
+ const int32_t d0 = tensor->op_params[6];
+ const int32_t d1 = tensor->op_params[7];
+ const int32_t d2 = tensor->op_params[8];
+ const int32_t IC = tensor->op_params[9];
+
+ tensor_clone = ggml_im2col_3d(ggml_ctx, src_clone[0], src_clone[1], IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, tensor->type);
+ } else if (tensor->op == GGML_OP_TIMESTEP_EMBEDDING) {
+ const int32_t dim = tensor->op_params[0];
+ const int32_t max_period = tensor->op_params[1];
+ tensor_clone = ggml_timestep_embedding(ggml_ctx, src_clone[0], dim, max_period);
+ } else if (tensor->op == GGML_OP_CONV_TRANSPOSE_1D){
+ const int32_t s0 = tensor->op_params[0];
+ const int32_t p0 = tensor->op_params[1];
+ const int32_t d0 = tensor->op_params[2];
+ tensor_clone = ggml_conv_transpose_1d(ggml_ctx, src_clone[0], src_clone[1], s0, p0, d0);
+ } else if (tensor->op == GGML_OP_COL2IM_1D) {
+ const int32_t stride = tensor->op_params[0];
+ const int32_t oc = tensor->op_params[1];
+ const int32_t p0 = tensor->op_params[2];
+ tensor_clone = ggml_col2im_1d(ggml_ctx, src_clone[0], stride, oc, p0);
+ } else if (tensor->op == GGML_OP_POOL_1D) {
+ enum ggml_op_pool op = static_cast<ggml_op_pool>(tensor->op_params[0]);
+ const int32_t k0 = tensor->op_params[1];
+ const int32_t s0 = tensor->op_params[2];
+ const int32_t p0 = tensor->op_params[3];
+
+ tensor_clone = ggml_pool_1d(ggml_ctx, src_clone[0], op, k0, s0, p0);
+ } else if (tensor->op == GGML_OP_POOL_2D) {
+ enum ggml_op_pool op = static_cast<ggml_op_pool>(tensor->op_params[0]);
+ const int32_t k0 = tensor->op_params[1];
+ const int32_t k1 = tensor->op_params[2];
+ const int32_t s0 = tensor->op_params[3];
+ const int32_t s1 = tensor->op_params[4];
+ const int32_t p0 = tensor->op_params[5];
+ const int32_t p1 = tensor->op_params[6];
+
+ tensor_clone = ggml_pool_2d(ggml_ctx, src_clone[0], op, k0, k1, s0, s1, p0, p1);
+ } else if (tensor->op == GGML_OP_CONV_2D) {
+ const int32_t s0 = tensor->op_params[0];
+ const int32_t s1 = tensor->op_params[1];
+ const int32_t p0 = tensor->op_params[2];
+ const int32_t p1 = tensor->op_params[3];
+ const int32_t d0 = tensor->op_params[4];
+ const int32_t d1 = tensor->op_params[5];
+ tensor_clone = ggml_conv_2d(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1);
+ } else if (tensor->op == GGML_OP_CONV_3D) {
+ const int32_t s0 = tensor->op_params[0];
+ const int32_t s1 = tensor->op_params[1];
+ const int32_t s2 = tensor->op_params[2];
+ const int32_t p0 = tensor->op_params[3];
+ const int32_t p1 = tensor->op_params[4];
+ const int32_t p2 = tensor->op_params[5];
+ const int32_t d0 = tensor->op_params[6];
+ const int32_t d1 = tensor->op_params[7];
+ const int32_t d2 = tensor->op_params[8];
+ const int32_t IC = tensor->op_params[9];
+ const int32_t N = tensor->op_params[10];
+ const int32_t OC = tensor->op_params[11];
+ tensor_clone = ggml_conv_3d_direct(ggml_ctx, src_clone[0], src_clone[1], s0, s1, s2, p0, p1, p2, d0, d1, d2, IC, N, OC);
+ } else if (tensor->op == GGML_OP_CONV_2D_DW) {
+ const int32_t s0 = tensor->op_params[0];
+ const int32_t s1 = tensor->op_params[1];
+ const int32_t p0 = tensor->op_params[2];
+ const int32_t p1 = tensor->op_params[3];
+ const int32_t d0 = tensor->op_params[4];
+ const int32_t d1 = tensor->op_params[5];
+ tensor_clone = ggml_conv_2d_dw_direct(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1);
+ } else if (tensor->op == GGML_OP_CONV_TRANSPOSE_2D) {
+ const int32_t s = tensor->op_params[0];
+ tensor_clone = ggml_conv_transpose_2d_p0(ggml_ctx, src_clone[0], src_clone[1], s);
+ } else if (tensor->op == GGML_OP_LEAKY_RELU) {
+ const float * op_params = (const float *)tensor->op_params;
+ tensor_clone = ggml_leaky_relu(ggml_ctx, src_clone[0], op_params[0], false);
+ } else if (tensor->op == GGML_OP_RWKV_WKV6) {
+ tensor_clone = ggml_rwkv_wkv6(ggml_ctx, src_clone[0], src_clone[1],
+ src_clone[2], src_clone[3], src_clone[4], src_clone[5]);
+ } else if (tensor->op == GGML_OP_RWKV_WKV7) {
+ tensor_clone = ggml_rwkv_wkv7(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3],
+ src_clone[4], src_clone[5], src_clone[6]);
+ } else if (tensor->op == GGML_OP_GATED_LINEAR_ATTN) {
+ const float * op_params = (const float *)tensor->op_params;
+ tensor_clone = ggml_gated_linear_attn(ggml_ctx, src_clone[0], src_clone[1],
+ src_clone[2], src_clone[3], src_clone[4], op_params[0]);
+ } else if (tensor->op == GGML_OP_LIGHTNING_INDEXER) {
+ tensor_clone = ggml_lightning_indexer(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3]);
+ } else if (tensor->op == GGML_OP_GATED_DELTA_NET) {
+ tensor_clone = ggml_gated_delta_net(ggml_ctx, src_clone[0], src_clone[1],
+ src_clone[2], src_clone[3], src_clone[4], src_clone[5],
+ ggml_get_op_params_i32(tensor, 0));
+ } else if (tensor->op == GGML_OP_OPT_STEP_ADAMW) {
+ src_clone[0]->flags = tensor->src[0]->flags;
+ tensor_clone = ggml_opt_step_adamw(ggml_ctx, src_clone[0], src_clone[1],
+ src_clone[2], src_clone[3], src_clone[4]);
+ } else if (tensor->op == GGML_OP_OPT_STEP_SGD) {
+ src_clone[0]->flags = tensor->src[0]->flags;
+ tensor_clone = ggml_opt_step_sgd(ggml_ctx, src_clone[0], src_clone[1],
+ src_clone[2]);
+ } else if (tensor->op == GGML_OP_ADD_ID) {
+ tensor_clone = ggml_add_id(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]);
+ } else if (tensor->op == GGML_OP_SSM_SCAN) {
+ const int32_t K = ggml_get_op_params_i32(tensor, 0);
+ tensor_clone = ggml_ssm_scan(ggml_ctx, src_clone[0], src_clone[1], src_clone[2],
+ src_clone[3], src_clone[4], src_clone[5], src_clone[6], K);
+ } else if (tensor->op == GGML_OP_SSM_CONV) {
+ tensor_clone = ggml_ssm_conv(ggml_ctx, src_clone[0], src_clone[1]);
+ } else if (tensor->op == GGML_OP_ROLL) {
+ const int32_t s0 = tensor->op_params[0];
+ const int32_t s1 = tensor->op_params[1];
+ const int32_t s2 = tensor->op_params[2];
+ const int32_t s3 = tensor->op_params[3];
+ tensor_clone = ggml_roll(ggml_ctx, src_clone[0], s0, s1, s2, s3);
+ }
+ else {
+ std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl;
+ GGML_ABORT("fatal error");
+ }
+ cloned_tensors[tensor] = tensor_clone;
+ }
+
+ ggml_cgraph * cgraph_cpu = ggml_new_graph(ggml_ctx);
+ ggml_build_forward_expand(cgraph_cpu, tensor_clone);
+
+ ggml_graph_compute_with_ctx(ggml_ctx, cgraph_cpu, 8);
+
+ if (vk_output_tensor > 0 && vk_output_tensor == check_counter) {
+ ggml_vk_print_tensor(tensor_clone, "tensor_clone");
+ }
+
+ comp_size = ggml_nbytes(tensor_clone);
+
+ comp_result = malloc(comp_size);
+ memcpy(comp_result, tensor_clone->data, comp_size);
+ memcpy(comp_nb, tensor_clone->nb, sizeof(size_t) * GGML_MAX_DIMS);
+
+ for (auto m : cloned_mallocs) {
+ free(m);
+ }
+
+ ggml_free(ggml_ctx);
+
+ VK_LOG_DEBUG("END ggml_vk_check_results_0(" << tensor->name << ")");
+}
+
+static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx) {
+ ggml_tensor * tensor = cgraph->nodes[tensor_idx + ctx->num_additional_fused_ops];
+ if (tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_SET_ROWS) {
+ return;
+ }
+
+ if (!(vk_output_tensor > 0 && vk_output_tensor == check_counter) && check_counter <= vk_skip_checks) {
+ return;
+ }
+
+ VK_LOG_DEBUG("ggml_vk_check_results_1(" << tensor->name << ")");
+
+ ggml_tensor * src0 = tensor->src[0];
+ ggml_tensor * src1 = tensor->src[1];
+ ggml_tensor * src2 = tensor->src[2];
+ ggml_tensor * src3 = tensor->src[3];
+
+ void * tensor_data = tensor->data;
+
+ if (ggml_backend_buffer_is_vk(tensor->buffer)) {
+ size_t tensor_size = ggml_nbytes(tensor);
+ tensor_data = malloc(tensor_size);
+
+ ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context;
+
+ vk_buffer& buffer_gpu = buf_ctx->dev_buffer;
+ uint64_t offset = vk_tensor_offset(tensor) + tensor->view_offs;
+ if (offset + tensor_size >= buffer_gpu->size) {
+ tensor_size = buffer_gpu->size - offset;
+ }
+
+ ggml_vk_buffer_read(buffer_gpu, offset, tensor_data, tensor_size);
+ }
+
+ float first_error_result = -1.0f;
+ float first_error_correct = -1.0f;
+ std::array<int, 4> first_error = { -1, -1, -1, -1 };
+ double avg_err = 0.0;
+ size_t counter = 0;
+
+ for (int i3 = 0; i3 < tensor->ne[3]; i3++) {
+ for (int i2 = 0; i2 < tensor->ne[2]; i2++) {
+ for (int i1 = 0; i1 < tensor->ne[1]; i1++) {
+ for (int i0 = 0; i0 < tensor->ne[0]; i0++) {
+ const bool buffer_size_fit = i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0] < comp_size;
+ float correct = 0.0f;
+ float result = 0.0f;
+
+ if (buffer_size_fit) {
+ if (tensor->type == GGML_TYPE_F32) {
+ correct = *(float *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]);
+ result = *(float *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]);
+ } else if (tensor->type == GGML_TYPE_F16) {
+ correct = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]));
+ result = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]));
+ } else if (tensor->type == GGML_TYPE_BF16) {
+ correct = ggml_bf16_to_fp32(*(ggml_bf16_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]));
+ result = ggml_bf16_to_fp32(*(ggml_bf16_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]));
+ } else if (tensor->type == GGML_TYPE_I32) {
+ correct = *(int32_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]);
+ result = *(int32_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]);
+ } else if (tensor->type == GGML_TYPE_I64) {
+ correct = *(int64_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]);
+ result = *(int64_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]);
+ } else {
+ std::cerr << "Results check not implemented for type " << ggml_type_name(tensor->type) << std::endl;
+ }
+ } else {
+ std::cerr << "Missing debug code for type " << ggml_type_name(tensor->type) << std::endl;
+ GGML_ABORT("fatal error");
+ }
+
+ if ((std::isnan(correct) != std::isnan(result)) || (std::isinf(correct) != std::isinf(result)) || !buffer_size_fit) {
+ std::cerr << "ERROR: Invalid value in " << ggml_op_name(tensor->op) << " i3=" << i3 << " i2=" << i2 << " i1=" << i1 << " i0=" << i0 << " result=" << result << " correct=" << correct << " avg_err=" << (avg_err / counter) << std::endl;
+ std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl;
+ if (src0 != nullptr) {
+ std::cerr << "src0=" << src0 << " src0->name=" << src0->name << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl;
+ }
+ if (src1 != nullptr) {
+ std::cerr << "src1=" << src1 << " src1->name=" << src1->name << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl;
+ }
+ if (src2 != nullptr) {
+ std::cerr << "src2=" << src2 << " src2->name=" << src2->name << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl;
+ }
+ if (src3 != nullptr) {
+ std::cerr << "src3=" << src3 << " src3->name=" << src3->name << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl;
+ }
+ std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl;
+ std::cerr << std::endl << "Result:" << std::endl;
+ ggml_vk_print_tensor_area(tensor, tensor_data, i0, i1, i2, i3);
+ std::cerr << std::endl << "Correct:" << std::endl;
+ ggml_vk_print_tensor_area(tensor, comp_result, i0, i1, i2, i3);
+ std::cerr << std::endl;
+ std::vector<const ggml_tensor *> done;
+ ggml_vk_print_graph_origin(tensor, done);
+ GGML_ABORT("fatal error");
+ }
+ const double denom = std::fabs(correct) > 1.0f ? (std::fabs(correct) > 1e-8 ? std::fabs(correct) : 1e-8) : 1.0f;
+ if (first_error[0] == -1 && std::fabs(correct - result) / denom > 0.5) {
+ first_error[0] = i0;
+ first_error[1] = i1;
+ first_error[2] = i2;
+ first_error[3] = i3;
+ first_error_result = result;
+ first_error_correct = correct;
+ }
+
+ // Special case, value is infinite, avoid NaN result in avg_err
+ // NaN also appears in results, if both are nan error is 0
+ if (!std::isinf(correct) && !std::isinf(result) && !std::isnan(correct) && !std::isnan(result)) {
+ avg_err += std::fabs(correct - result) / denom;
+ }
+ counter++;
+ }
+ }
+ }
+ }
+
+ avg_err /= counter;
+
+ if (vk_output_tensor > 0 && vk_output_tensor == check_counter) {
+ std::cerr << "TENSOR CHECK: avg_err=" << avg_err << " in " << ggml_op_name(tensor->op) << " (check " << check_counter << ")" << std::endl;
+ std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl;
+ if (src0 != nullptr) {
+ std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl;
+ }
+ if (src1 != nullptr) {
+ std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl;
+ }
+ if (src2 != nullptr) {
+ std::cerr << "src2=" << src2 << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl;
+ }
+ if (src3 != nullptr) {
+ std::cerr << "src3=" << src3 << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl;
+ }
+ std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl;
+ std::cerr << std::endl << "Result:" << std::endl;
+ ggml_vk_print_tensor_area(tensor, tensor_data, 5, 5, 0, 0);
+ std::cerr << std::endl << "Correct:" << std::endl;
+ ggml_vk_print_tensor_area(tensor, comp_result, 5, 5, 0, 0);
+ std::cerr << std::endl;
+ std::vector<const ggml_tensor *> done;
+ ggml_vk_print_graph_origin(tensor, done);
+ }
+
+ if (avg_err > 0.01 || std::isnan(avg_err)) {
+ std::cerr << "ERROR: avg_err=" << avg_err << " in " << ggml_op_name(tensor->op) << " (check " << check_counter << ")" << std::endl;
+ std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl;
+ if (src0 != nullptr) {
+ std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl;
+ }
+ if (src1 != nullptr) {
+ std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl;
+ }
+ if (src2 != nullptr) {
+ std::cerr << "src2=" << src2 << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl;
+ }
+ if (src3 != nullptr) {
+ std::cerr << "src3=" << src3 << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl;
+ }
+ std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl;
+ std::cerr << std::endl << "Result:" << std::endl;
+ ggml_vk_print_tensor_area(tensor, tensor_data, first_error[0], first_error[1], first_error[2], first_error[3]);
+ std::cerr << std::endl << "Correct:" << std::endl;
+ ggml_vk_print_tensor_area(tensor, comp_result, first_error[0], first_error[1], first_error[2], first_error[3]);
+ std::cerr << std::endl;
+ std::vector<const ggml_tensor *> done;
+ ggml_vk_print_graph_origin(tensor, done);
+ GGML_ABORT("fatal error");
+ } else {
+ std::cerr << check_counter << " " << tensor->name << " op=" << ggml_op_name(tensor->op) << " avg_err=" << avg_err << std::endl;
+ }
+
+ free(comp_result);
+ comp_result = nullptr;
+ comp_size = 0;
+
+ if (ggml_backend_buffer_is_vk(tensor->buffer)) {
+ free(tensor_data);
+ }
+
+ VK_LOG_DEBUG("END ggml_vk_check_results_1(" << tensor->name << ")");
+}
+#endif
+
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h
new file mode 100644
index 000000000..68b3200b3
--- /dev/null
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h
@@ -0,0 +1,1092 @@
+#pragma once
+#include "ggml-vulkan-types.h"
+
+uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t);
+
+uint32_t ggml_vk_concat_unit_size(ggml_type type);
+
+struct vk_mat_mat_push_constants {
+ uint32_t M; uint32_t N; uint32_t K;
+ uint32_t stride_a; uint32_t stride_b; uint32_t stride_d;
+ uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;
+ uint32_t base_work_group_z; uint32_t num_batches;
+ uint32_t k_split;
+ uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3;
+ uint32_t padded_N;
+};
+
+struct vk_mat_vec_push_constants {
+ uint32_t ncols;
+ uint32_t stride_a;
+ uint32_t stride_b;
+ uint32_t stride_d;
+ uint32_t batch_stride_a;
+ uint32_t batch_stride_b;
+ uint32_t batch_stride_d;
+ uint32_t fusion_flags;
+ uint32_t base_work_group_y;
+ uint32_t ne02;
+ uint32_t ne12;
+ uint32_t broadcast2;
+ uint32_t broadcast3;
+};
+
+struct vk_mat_vec_p021_push_constants {
+ uint32_t ncols_x;
+ uint32_t nrows_x;
+ uint32_t nchannels_x;
+ uint32_t nchannels_y;
+ uint32_t b_offset;
+ uint32_t d_offset;
+ uint32_t fusion_flags;
+};
+
+struct vk_mat_vec_nc_push_constants {
+ uint32_t ncols_x;
+ uint32_t nrows_x;
+ uint32_t row_stride_x;
+ uint32_t channel_stride_x;
+ uint32_t channel_stride_y;
+ uint32_t channel_x_divisor;
+ uint32_t ne12;
+ uint32_t b_offset;
+ uint32_t d_offset;
+ uint32_t nb03;
+ uint32_t nb13;
+ uint32_t nb23;
+ uint32_t fusion_flags;
+};
+
+struct vk_mat_mat_id_push_constants {
+ uint32_t M; uint32_t N; uint32_t K;
+ uint32_t stride_a; uint32_t stride_b; uint32_t stride_d;
+ uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;
+ uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11;
+ uint32_t n_experts;
+ uint32_t hoist_row_ids;
+};
+
+struct vk_mat_vec_id_push_constants {
+ uint32_t ncols;
+ uint32_t stride_a;
+ uint32_t stride_b;
+ uint32_t stride_d;
+ uint32_t batch_stride_a;
+ uint32_t batch_stride_b;
+ uint32_t batch_stride_d;
+ uint32_t fusion_flags;
+ uint32_t nei0;
+ uint32_t ne11;
+ uint32_t expert_i1;
+ uint32_t nbi1;
+};
+
+struct vk_flash_attn_push_constants {
+ uint32_t N;
+ uint32_t KV;
+
+ uint32_t ne1;
+ uint32_t ne2;
+ uint32_t ne3;
+
+ uint32_t neq2;
+ uint32_t neq3;
+ uint32_t nek2;
+ uint32_t nek3;
+ uint32_t nev2;
+ uint32_t nev3;
+ uint32_t nem1;
+ uint32_t nem2;
+ uint32_t nem3;
+
+ uint32_t nb01;
+ uint32_t nb02;
+ uint32_t nb03;
+ uint32_t nb11;
+ uint32_t nb12;
+ uint32_t nb13;
+ uint32_t nb21;
+ uint32_t nb22;
+ uint32_t nb23;
+
+ float scale;
+ float max_bias;
+ float logit_softcap;
+
+ uint32_t mask_n_head_log2;
+ float m0;
+ float m1;
+
+ uint32_t gqa_ratio;
+ uint32_t split_kv;
+ uint32_t k_num;
+};
+
+static_assert(sizeof(vk_flash_attn_push_constants) <= 128, "sizeof(vk_flash_attn_push_constants) must be <= 128");
+
+struct vk_op_push_constants {
+ uint32_t KX;
+ uint32_t KY;
+ float param1;
+ float param2;
+ float param3;
+ float param4;
+};
+
+struct vk_op_fwht_push_constants {
+ uint32_t n_rows;
+ uint32_t src_offset;
+ uint32_t dst_offset;
+ float scale;
+};
+
+struct vk_op_dsv4_hc_comb_push_constants {
+ uint32_t n_tokens;
+
+ uint32_t nbm0; uint32_t nbm1;
+ uint32_t nbs0;
+ uint32_t nbb0;
+ uint32_t nbd0; uint32_t nbd1; uint32_t nbd2;
+
+ uint32_t m_offset;
+ uint32_t s_offset;
+ uint32_t b_offset;
+ uint32_t d_offset;
+
+ float eps;
+ uint32_t n_iter;
+};
+
+struct vk_op_dsv4_hc_pre_push_constants {
+ uint32_t n_embd;
+ uint32_t n_tokens;
+
+ uint32_t nbx0; uint32_t nbx1; uint32_t nbx2;
+ uint32_t nbw0; uint32_t nbw1; uint32_t nbw2;
+ uint32_t nbd0; uint32_t nbd1;
+
+ uint32_t x_offset;
+ uint32_t w_offset;
+ uint32_t d_offset;
+
+ float scale;
+};
+
+struct vk_op_dsv4_hc_post_push_constants {
+ uint32_t n_embd;
+ uint32_t n_tokens;
+
+ uint32_t nbx0; uint32_t nbx1;
+ uint32_t nbr0; uint32_t nbr1; uint32_t nbr2;
+ uint32_t nbp0; uint32_t nbp1;
+ uint32_t nbc0; uint32_t nbc1; uint32_t nbc2;
+ uint32_t nbd0; uint32_t nbd1; uint32_t nbd2;
+
+ uint32_t x_offset;
+ uint32_t r_offset;
+ uint32_t p_offset;
+ uint32_t c_offset;
+ uint32_t d_offset;
+};
+
+struct vk_op_count_experts_push_constants {
+ uint32_t ne00;
+ uint32_t ne01;
+ uint32_t nb00;
+ uint32_t nb01;
+ uint32_t a_offset;
+ uint32_t n_experts;
+ uint32_t hoist_row_ids;
+ uint32_t ne00mp;
+ uint32_t ne00L;
+};
+
+struct vk_op_glu_push_constants {
+ uint32_t N;
+ uint32_t ne00;
+ uint32_t ne20;
+ uint32_t mode; // 0: default, 1: swapped, 2: split
+ float alpha; // for swiglu_oai
+ float limit;
+ uint32_t nb00;
+ uint32_t nb01;
+ uint32_t nb02;
+ uint32_t nb03;
+ uint32_t nb10;
+ uint32_t nb11;
+ uint32_t nb12;
+ uint32_t nb13;
+ uint32_t nb20;
+ uint32_t nb21;
+ uint32_t nb22;
+ uint32_t nb23;
+ uint32_t ne21;
+ uint32_t ne22;
+ uint32_t misalign_offsets;
+ uint32_t ne2_012mp; uint32_t ne2_012L;
+ uint32_t ne2_01mp; uint32_t ne2_01L;
+ uint32_t ne2_0mp; uint32_t ne2_0L;
+};
+
+static_assert(sizeof(vk_op_glu_push_constants) <= 128, "sizeof(vk_op_glu_push_constants) must be <= 128");
+
+struct vk_op_unary_push_constants {
+ uint32_t ne;
+ uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;
+ uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;
+ uint32_t misalign_offsets;
+ float param1; float param2; float param3; float param4;
+ uint32_t ne0_012mp; uint32_t ne0_01mp; uint32_t ne0_0mp; uint32_t ne0_Ls;
+ uint32_t ne1_012mp; uint32_t ne1_01mp; uint32_t ne1_0mp; uint32_t ne1_Ls;
+};
+
+static_assert(sizeof(vk_op_unary_push_constants) <= 128, "sizeof(vk_op_unary_push_constants) must be <= 128");
+
+static vk_op_unary_push_constants vk_op_unary_push_constants_init(const ggml_tensor * src0, const ggml_tensor * dst, int64_t ne = 0) {
+ GGML_ASSERT(ne != 0 || (ggml_nelements(src0) == ggml_nelements(dst)));
+ ne = ne != 0 ? ne : ggml_nelements(dst);
+ GGML_ASSERT(ne <= (int64_t)std::numeric_limits<uint32_t>::max());
+
+ vk_op_unary_push_constants p{};
+ p.ne = (uint32_t)ne;
+
+ size_t src0_tsize = ggml_type_size(src0->type);
+ p.ne00 = (uint32_t)src0->ne[0];
+ p.ne01 = (uint32_t)src0->ne[1];
+ p.ne02 = (uint32_t)src0->ne[2];
+ p.ne03 = (uint32_t)src0->ne[3];
+ p.nb00 = (uint32_t)(src0->nb[0] / src0_tsize);
+ p.nb01 = (uint32_t)(src0->nb[1] / src0_tsize);
+ p.nb02 = (uint32_t)(src0->nb[2] / src0_tsize);
+ p.nb03 = (uint32_t)(src0->nb[3] / src0_tsize);
+
+ size_t dst_tsize = ggml_type_size(dst->type);
+ p.ne10 = (uint32_t)dst->ne[0];
+ p.ne11 = (uint32_t)dst->ne[1];
+ p.ne12 = (uint32_t)dst->ne[2];
+ p.ne13 = (uint32_t)dst->ne[3];
+ p.nb10 = (uint32_t)(dst->nb[0] / dst_tsize);
+ p.nb11 = (uint32_t)(dst->nb[1] / dst_tsize);
+ p.nb12 = (uint32_t)(dst->nb[2] / dst_tsize);
+ p.nb13 = (uint32_t)(dst->nb[3] / dst_tsize);
+
+ return p; // offsets are initialized later in ggml_vk_op
+}
+
+struct vk_op_pad_push_constants {
+ uint32_t ne;
+ uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;
+ uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;
+ uint32_t misalign_offsets;
+ uint32_t circular;
+
+ uint32_t lp0; uint32_t rp0;
+ uint32_t lp1; uint32_t rp1;
+ uint32_t lp2; uint32_t rp2;
+ uint32_t lp3; uint32_t rp3;
+};
+
+static vk_op_pad_push_constants vk_op_pad_push_constants_init(const ggml_tensor * src0, const ggml_tensor * dst) {
+ int64_t ne = ggml_nelements(dst);
+ GGML_ASSERT(ne <= (int64_t)std::numeric_limits<uint32_t>::max());
+
+ vk_op_pad_push_constants p{};
+ p.ne = (uint32_t)ne;
+
+ size_t src0_tsize = ggml_type_size(src0->type);
+ p.ne00 = (uint32_t)src0->ne[0];
+ p.ne01 = (uint32_t)src0->ne[1];
+ p.ne02 = (uint32_t)src0->ne[2];
+ p.ne03 = (uint32_t)src0->ne[3];
+ p.nb00 = (uint32_t)(src0->nb[0] / src0_tsize);
+ p.nb01 = (uint32_t)(src0->nb[1] / src0_tsize);
+ p.nb02 = (uint32_t)(src0->nb[2] / src0_tsize);
+ p.nb03 = (uint32_t)(src0->nb[3] / src0_tsize);
+
+ size_t dst_tsize = ggml_type_size(dst->type);
+ p.ne10 = (uint32_t)dst->ne[0];
+ p.ne11 = (uint32_t)dst->ne[1];
+ p.ne12 = (uint32_t)dst->ne[2];
+ p.ne13 = (uint32_t)dst->ne[3];
+ p.nb10 = (uint32_t)(dst->nb[0] / dst_tsize);
+ p.nb11 = (uint32_t)(dst->nb[1] / dst_tsize);
+ p.nb12 = (uint32_t)(dst->nb[2] / dst_tsize);
+ p.nb13 = (uint32_t)(dst->nb[3] / dst_tsize);
+
+ p.lp0 = dst->op_params[0];
+ p.rp0 = dst->op_params[1];
+ p.lp1 = dst->op_params[2];
+ p.rp1 = dst->op_params[3];
+ p.lp2 = dst->op_params[4];
+ p.rp2 = dst->op_params[5];
+ p.lp3 = dst->op_params[6];
+ p.rp3 = dst->op_params[7];
+ p.circular = dst->op_params[8];
+
+ return p; // fastdiv values and offsets are initialized later in ggml_vk_op
+}
+
+static void init_fastdiv_values(uint32_t d, uint32_t &mp, uint32_t &L)
+{
+ // compute L = ceil(log2(d));
+ L = 0;
+ while (L < 32 && (uint32_t{1} << L) < d) {
+ L++;
+ }
+
+ mp = (uint32_t)((uint64_t{1} << 32) * ((uint64_t{1} << L) - d) / d + 1);
+}
+
+static uint32_t pack_fastdiv_L(uint32_t L0, uint32_t L1, uint32_t L2) {
+ return L0 | (L1 << 8) | (L2 << 16);
+}
+
+template <typename T> void init_pushconst_fastdiv(T &p) {
+ GGML_UNUSED(p);
+ static_assert(!std::is_const<T>::value, "unexpected type");
+}
+
+template <> inline void init_pushconst_fastdiv(vk_op_unary_push_constants &p) {
+ // Compute magic values to divide by these six numbers.
+ uint32_t ne0_012L;
+ uint32_t ne0_01L;
+ uint32_t ne0_0L;
+ uint32_t ne1_012L;
+ uint32_t ne1_01L;
+ uint32_t ne1_0L;
+
+ init_fastdiv_values(p.ne02*p.ne01*p.ne00, p.ne0_012mp, ne0_012L);
+ init_fastdiv_values(p.ne01*p.ne00, p.ne0_01mp, ne0_01L);
+ init_fastdiv_values(p.ne00, p.ne0_0mp, ne0_0L);
+ init_fastdiv_values(p.ne12*p.ne11*p.ne10, p.ne1_012mp, ne1_012L);
+ init_fastdiv_values(p.ne11*p.ne10, p.ne1_01mp, ne1_01L);
+ init_fastdiv_values(p.ne10, p.ne1_0mp, ne1_0L);
+
+ p.ne0_Ls = pack_fastdiv_L(ne0_012L, ne0_01L, ne0_0L);
+ p.ne1_Ls = pack_fastdiv_L(ne1_012L, ne1_01L, ne1_0L);
+}
+
+template <> inline void init_pushconst_fastdiv(vk_op_glu_push_constants &p) {
+ // GLU linearizes over dst, then uses dst coordinates for src0/src1.
+ init_fastdiv_values(p.ne22*p.ne21*p.ne20, p.ne2_012mp, p.ne2_012L);
+ init_fastdiv_values(p.ne21*p.ne20, p.ne2_01mp, p.ne2_01L);
+ init_fastdiv_values(p.ne20, p.ne2_0mp, p.ne2_0L);
+}
+
+template <> inline void init_pushconst_fastdiv(vk_op_count_experts_push_constants &p) {
+ init_fastdiv_values(p.ne00, p.ne00mp, p.ne00L);
+}
+
+struct vk_op_binary_push_constants {
+ uint32_t ne;
+ uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;
+ uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;
+ uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23; uint32_t nb20; uint32_t nb21; uint32_t nb22; uint32_t nb23;
+ uint32_t misalign_offsets;
+ float param1; float param2; int32_t param3;
+};
+
+struct vk_op_concat_push_constants : vk_op_binary_push_constants {};
+
+static_assert(sizeof(vk_op_concat_push_constants) == sizeof(vk_op_binary_push_constants));
+
+static_assert(std::is_standard_layout_v<vk_op_concat_push_constants>);
+
+struct vk_op_multi_add_push_constants {
+ // shape for dst
+ uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23;
+
+ // strides for srcs+dst
+ uint32_t nb[MAX_PARAMETER_COUNT][4];
+
+ uint32_t rms_partials;
+};
+
+static_assert(MAX_PARAMETER_COUNT == 12);
+
+static_assert(sizeof(vk_op_multi_add_push_constants) <= 256);
+
+struct vk_op_topk_moe_push_constants {
+ uint32_t n_rows;
+ uint32_t n_experts_push;
+ uint32_t n_expert_used;
+ float clamp_min;
+ float clamp_max;
+ uint32_t gating_func;
+ uint32_t has_bias;
+ uint32_t with_norm;
+ float output_scale;
+ float output_bias;
+};
+
+struct vk_op_add_id_push_constants {
+ uint32_t ne0;
+ uint32_t ne1;
+ uint32_t s01;
+ uint32_t s02;
+ uint32_t s11;
+ uint32_t s21;
+};
+
+struct vk_op_diag_mask_push_constants {
+ uint32_t ncols;
+ uint32_t rows_per_channel;
+ int32_t n_past;
+};
+
+struct vk_op_rope_push_constants {
+ uint32_t rope_mode;
+ uint32_t nrows;
+ uint32_t n_dims;
+ uint32_t n_offs;
+ float freq_scale;
+ float freq_base;
+ float ext_factor;
+ float attn_factor;
+ float corr_dims[2];
+ float theta_scale;
+ uint32_t has_ff;
+ int32_t sections[4];
+ uint32_t is_imrope;
+ uint32_t is_back;
+ uint32_t set_rows_stride;
+ uint32_t ne00;
+ uint32_t ne01;
+ uint32_t ne02;
+ uint32_t nb01;
+ uint32_t nb02;
+ uint32_t nb03;
+ uint32_t nb11;
+ uint32_t nb12;
+ uint32_t nb13;
+ uint32_t a_offset;
+ uint32_t d_offset;
+};
+
+static_assert(sizeof(vk_op_rope_push_constants) <= 128, "sizeof(vk_op_rope_push_constants) must be <= 128");
+
+struct vk_op_rms_norm_mul_rope_push_constants {
+ vk_op_binary_push_constants bin;
+ vk_op_rope_push_constants rope;
+};
+
+struct vk_op_soft_max_push_constants {
+ uint32_t KX;
+ uint32_t KY;
+ uint32_t ne00;
+ uint32_t ne01;
+ uint32_t ne02;
+ uint32_t ne12;
+ uint32_t ne13;
+ uint32_t nb11;
+ uint32_t nb12;
+ uint32_t nb13;
+ float scale;
+ float max_bias;
+ float m0;
+ float m1;
+ uint32_t n_head_log2;
+ uint32_t nrows_x;
+ uint32_t has_sinks;
+};
+
+struct vk_op_argsort_push_constants {
+ uint32_t ncols;
+ uint32_t ncols_padded;
+ uint32_t ncols_padded_log2;
+ uint32_t nrows;
+ uint32_t order;
+ uint32_t outer_start;
+ uint32_t outer_end;
+ uint32_t inner_start;
+ uint32_t inner_end;
+};
+
+struct vk_op_topk_push_constants {
+ uint32_t orig_ncols;
+ uint32_t ncols_input;
+ uint32_t ncols_output;
+ uint32_t k;
+ uint32_t nrows;
+ uint32_t first_pass;
+ uint32_t last_pass;
+};
+
+struct vk_op_topk_radix_push_constants {
+ uint32_t ncols;
+ uint32_t k;
+ uint32_t nrows;
+ uint32_t n_tps; // QSA only
+ uint32_t n_blocks; // QSA only
+ uint32_t n_stream; // QSA only
+};
+
+struct vk_op_im2col_push_constants {
+ uint64_t dst_addr;
+ uint32_t batch_offset; uint32_t offset_delta;
+ uint32_t IC;
+ uint32_t IW; uint32_t IH;
+ uint32_t OW; uint32_t OH;
+ uint32_t KW; uint32_t KH;
+ uint32_t OH_batch;
+ uint32_t CHW;
+ int32_t s0; int32_t s1;
+ int32_t p0; int32_t p1;
+ int32_t d0; int32_t d1;
+ uint32_t batch_IC;
+};
+
+struct vk_op_im2col_3d_push_constants {
+ uint64_t dst_addr;
+ uint32_t nb10;
+ uint32_t nb11;
+ uint32_t nb12;
+ uint32_t nb13;
+ uint32_t s0;
+ uint32_t s1;
+ uint32_t s2;
+ uint32_t p0;
+ uint32_t p1;
+ uint32_t p2;
+ uint32_t d0;
+ uint32_t d1;
+ uint32_t d2;
+ uint32_t IW;
+ uint32_t IH;
+ uint32_t ID;
+ uint32_t IC;
+ uint32_t KW;
+ uint32_t OH;
+ uint32_t KD_KH_KW;
+ uint32_t KH_KW;
+ uint32_t IC_KD_KH_KW;
+ uint32_t N_OD_OH;
+ uint32_t OD_OH;
+ uint32_t OD_OH_OW_IC_KD_KH_KW;
+ uint32_t OH_OW_IC_KD_KH_KW;
+ uint32_t OW_IC_KD_KH_KW;
+ uint32_t misalign_offsets;
+};
+
+struct vk_op_timestep_embedding_push_constants {
+ uint32_t nb1;
+ uint32_t dim;
+ uint32_t max_period;
+};
+
+struct vk_op_col2im_1d_push_constants {
+ uint32_t T_out;
+ uint32_t OC;
+ uint32_t K_OC;
+ uint32_t T_in;
+ uint32_t K;
+ int32_t stride;
+ int32_t p0;
+};
+
+struct vk_op_conv_transpose_1d_push_constants {
+ uint32_t Cout;
+ uint32_t Cin;
+ uint32_t K;
+ uint32_t L;
+ uint32_t KL;
+
+ uint32_t nb01;
+ uint32_t nb02;
+ uint32_t nb11;
+ uint32_t nb1;
+
+ int32_t s0;
+};
+
+struct vk_op_snake_push_constants {
+ uint32_t ne0;
+ uint32_t ne1;
+};
+
+struct vk_op_pool1d_push_constants {
+ uint32_t IL;
+ uint32_t OL;
+ uint32_t OC;
+ uint32_t pelements;
+ uint32_t op;
+ int32_t k0;
+ int32_t s0;
+ int32_t p0;
+};
+
+struct vk_op_pool2d_push_constants {
+ uint32_t IW; uint32_t IH;
+ uint32_t OW; uint32_t OH;
+ uint32_t OC;
+ uint32_t pelements;
+ uint32_t op;
+ int32_t k0; int32_t k1;
+ int32_t s0; int32_t s1;
+ int32_t p0; int32_t p1;
+};
+
+struct vk_op_rwkv_wkv6_push_constants {
+ uint32_t B;
+ uint32_t T;
+ uint32_t C;
+ uint32_t H;
+};
+
+struct vk_op_rwkv_wkv7_push_constants {
+ uint32_t B;
+ uint32_t T;
+ uint32_t C;
+ uint32_t H;
+};
+
+struct vk_op_gated_linear_attn_push_constants {
+ uint32_t B;
+ uint32_t T;
+ uint32_t C;
+ uint32_t H;
+ float scale;
+};
+
+struct vk_op_lightning_indexer_push_constants {
+ uint32_t n_kv;
+ uint32_t n_heads;
+ uint32_t n_tokens;
+ uint32_t n_streams;
+ uint32_t n_masks;
+ uint32_t dispatch_x;
+ uint32_t q_nb1;
+ uint32_t q_nb2;
+ uint32_t q_nb3;
+ uint32_t k_nb2;
+ uint32_t k_nb3;
+ uint32_t w_nb1;
+ uint32_t w_nb3;
+ uint32_t m_nb1;
+ uint32_t m_nb3;
+ uint32_t d_nb1;
+ uint32_t d_nb3;
+};
+
+static_assert(sizeof(vk_op_lightning_indexer_push_constants) <= 128);
+
+struct vk_op_gated_delta_net_push_constants {
+ uint32_t H;
+ uint32_t n_tokens;
+ uint32_t n_seqs;
+ uint32_t s_off;
+ uint32_t sq1, sq2, sq3;
+ uint32_t sv1, sv2, sv3;
+ uint32_t sb1, sb2, sb3;
+ uint32_t neq1, rq3;
+ float scale;
+ uint32_t K;
+};
+
+struct vk_op_ssm_scan_push_constants {
+ uint32_t nb02, nb03, nb12, nb13;
+ uint32_t nb21, nb22, nb31;
+ uint32_t nb42, nb43, nb52, nb53;
+ uint32_t s_off;
+ uint32_t n_head, d_head, n_group, n_tok;
+ uint32_t n_seq, K;
+};
+
+struct vk_op_ssm_conv_push_constants {
+ uint32_t nb01, nb02;
+ uint32_t nb11;
+ uint32_t dst_nb0, dst_nb1, dst_nb2;
+ uint32_t nc, ncs, nr, n_t, n_s;
+};
+
+struct vk_op_conv2d_push_constants {
+ uint32_t Cout;
+ uint32_t Cin;
+ uint32_t N;
+
+ uint32_t W;
+ uint32_t H;
+ uint32_t OW;
+ uint32_t OH;
+
+ uint32_t nb01;
+ uint32_t nb02;
+ uint32_t nb03;
+
+ uint32_t nb11;
+ uint32_t nb12;
+ uint32_t nb13;
+
+ uint32_t nb1;
+ uint32_t nb2;
+ uint32_t nb3;
+
+ // init_fastdiv_values constants for dividing by OW, OW*OH
+ uint32_t OWmp; uint32_t OWL;
+ uint32_t OWOHmp; uint32_t OWOHL;
+};
+
+template <> inline void init_pushconst_fastdiv(vk_op_conv2d_push_constants &p) {
+ // Compute magic values to divide by OW, OW*OH
+ init_fastdiv_values(p.OW, p.OWmp, p.OWL);
+ init_fastdiv_values(p.OW*p.OH, p.OWOHmp, p.OWOHL);
+}
+
+struct vk_op_conv3d_push_constants {
+ uint32_t OC;
+ uint32_t IC;
+ uint32_t N;
+
+ uint32_t IW;
+ uint32_t IH;
+ uint32_t ID;
+ uint32_t OW;
+ uint32_t OH;
+ uint32_t OD;
+
+ uint32_t nb01;
+ uint32_t nb02;
+ uint32_t nb03;
+
+ uint32_t nb11;
+ uint32_t nb12;
+ uint32_t nb13;
+
+ uint32_t nb1;
+ uint32_t nb2;
+ uint32_t nb3;
+
+ uint32_t OWmp; uint32_t OWL;
+ uint32_t OWOHmp; uint32_t OWOHL;
+ uint32_t OWOHODmp; uint32_t OWOHODL;
+};
+
+template <> inline void init_pushconst_fastdiv(vk_op_conv3d_push_constants &p) {
+ init_fastdiv_values(p.OW, p.OWmp, p.OWL);
+ init_fastdiv_values(p.OW*p.OH, p.OWOHmp, p.OWOHL);
+ init_fastdiv_values(p.OW*p.OH*p.OD, p.OWOHODmp, p.OWOHODL);
+}
+
+struct vk_op_conv2d_dw_push_constants {
+ uint32_t ne;
+ uint32_t batches;
+ uint32_t channels;
+ uint32_t dst_w;
+ uint32_t dst_h;
+ uint32_t src_w;
+ uint32_t src_h;
+ uint32_t knl_w;
+ uint32_t knl_h;
+ int32_t stride_x;
+ int32_t stride_y;
+ int32_t pad_x;
+ int32_t pad_y;
+ int32_t dilation_x;
+ int32_t dilation_y;
+};
+
+struct vk_op_upscale_push_constants {
+ uint32_t ne; uint32_t a_offset; uint32_t d_offset;
+ uint32_t ne00; uint32_t ne01;
+ uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;
+ uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13;
+ float sf0; float sf1; float sf2; float sf3;
+ float pixel_offset;
+};
+
+struct vk_op_sum_rows_push_constants
+{
+ uint32_t n_cols;
+ uint32_t ne01, ne02;
+ uint32_t nb01, nb02, nb03;
+ uint32_t nb11, nb12, nb13;
+ float weight;
+ uint32_t misalign_offsets;
+ uint32_t ne0_12mp, ne0_12L;
+ uint32_t ne0_1mp, ne0_1L;
+};
+
+static vk_op_sum_rows_push_constants vk_op_sum_rows_push_constants_init(const ggml_tensor * src, const ggml_tensor * dst, int64_t n_cols) {
+ uint32_t type_size = (uint32_t)ggml_type_size(src->type);
+ vk_op_sum_rows_push_constants p = {};
+ p.n_cols = (uint32_t)n_cols;
+ p.ne01 = (uint32_t)src->ne[1];
+ p.ne02 = (uint32_t)src->ne[2];
+ p.nb01 = (uint32_t)src->nb[1] / type_size;
+ p.nb02 = (uint32_t)src->nb[2] / type_size;
+ p.nb03 = (uint32_t)src->nb[3] / type_size;
+ p.nb11 = (uint32_t)dst->nb[1] / type_size;
+ p.nb12 = (uint32_t)dst->nb[2] / type_size;
+ p.nb13 = (uint32_t)dst->nb[3] / type_size;
+ p.weight = 1.0f;
+ return p;
+}
+
+template <> inline void init_pushconst_fastdiv(vk_op_sum_rows_push_constants &p) {
+ init_fastdiv_values(p.ne01*p.ne02, p.ne0_12mp, p.ne0_12L);
+ init_fastdiv_values(p.ne01, p.ne0_1mp, p.ne0_1L);
+}
+
+struct vk_quantize_q8_1_push_constants {
+ uint32_t ne;
+ uint32_t num_blocks;
+};
+
+struct vk_op_flash_attn_split_k_reduce_push_constants {
+ uint32_t D;
+ uint32_t ne1;
+ uint32_t ne2;
+ uint32_t ne3;
+ uint32_t k_num;
+ uint32_t sinks;
+};
+
+struct vk_op_flash_attn_mask_opt_push_constants {
+ uint32_t nem0;
+ uint32_t nem1;
+ uint32_t nem2;
+ uint32_t nbm1;
+ uint32_t nbm2;
+ uint32_t nbm3;
+ uint32_t nbd1;
+ uint32_t nbd2;
+ uint32_t nbd3;
+};
+
+struct vk_op_flash_attn_sparse_compact_push_constants {
+ uint32_t KV;
+ uint32_t nem1;
+ uint32_t nem2;
+ uint32_t nbm1;
+ uint32_t nbm2;
+ uint32_t nbm3;
+ uint32_t n_kv_max;
+};
+
+template <typename T> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, T &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ GGML_UNUSED(p);
+ GGML_UNUSED(src0);
+ GGML_UNUSED(src1);
+ GGML_UNUSED(src2);
+ GGML_UNUSED(src3);
+ GGML_UNUSED(dst);
+ static_assert(!std::is_const<T>::value, "unexpected type");
+ GGML_ASSERT(!src0 || get_misalign_bytes(ctx, src0) == 0);
+ GGML_ASSERT(!src1 || get_misalign_bytes(ctx, src1) == 0);
+ GGML_ASSERT(!src2 || get_misalign_bytes(ctx, src2) == 0);
+ GGML_ASSERT(!src3 || get_misalign_bytes(ctx, src3) == 0);
+ GGML_ASSERT(!dst || get_misalign_bytes(ctx, dst) == 0);
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_mat_vec_p021_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);
+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+
+ p.b_offset = b_offset;
+ p.d_offset = d_offset;
+
+ GGML_UNUSED(src0);
+ GGML_UNUSED(src2);
+ GGML_UNUSED(src3);
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_mat_vec_nc_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);
+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+
+ p.b_offset = b_offset;
+ p.d_offset = d_offset;
+
+ GGML_UNUSED(src0);
+ GGML_UNUSED(src2);
+ GGML_UNUSED(src3);
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_fwht_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ p.src_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
+ p.dst_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+
+ GGML_UNUSED(src1);
+ GGML_UNUSED(src2);
+ GGML_UNUSED(src3);
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_comb_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ p.m_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
+ p.s_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);
+ p.b_offset = get_misalign_bytes(ctx, src2) / ggml_type_size(src2->type);
+ p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+
+ GGML_UNUSED(src3);
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_pre_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ p.x_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
+ p.w_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);
+ p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+
+ GGML_UNUSED(src2);
+ GGML_UNUSED(src3);
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_post_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ p.x_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
+ p.r_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);
+ p.p_offset = get_misalign_bytes(ctx, src2) / ggml_type_size(src2->type);
+ p.c_offset = src3 ? get_misalign_bytes(ctx, src3) / ggml_type_size(src3->type) : 0;
+ p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+}
+
+template <typename T> size_t push_constant_size(const T &t) {
+ static_assert(std::is_class<T>::value, "T must be a struct/class");
+ GGML_UNUSED(t);
+ return sizeof(T);
+}
+
+template <typename T> size_t push_constant_size(const std::vector<T> &t) {
+ GGML_UNUSED(t);
+ return sizeof(T) * t.size();
+}
+
+template <typename T, uint32_t N> size_t push_constant_size(const std::array<T, N> &t) {
+ GGML_UNUSED(t);
+ return sizeof(T) * N;
+}
+
+template <typename T> const T *push_constant_data(const T &t) {
+ static_assert(std::is_class<T>::value, "T must be a struct/class");
+ return &t;
+}
+
+template <typename T> const T *push_constant_data(const std::vector<T> &t) {
+ return t.data();
+}
+
+template <typename T, uint32_t N> const T *push_constant_data(const std::array<T, N> &t) {
+ return t.data();
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_unary_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+
+ p.misalign_offsets = (a_offset << 16) | d_offset;
+
+ GGML_UNUSED(src1);
+ GGML_UNUSED(src2);
+ GGML_UNUSED(src3);
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_glu_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
+ const uint32_t b_offset = src1 ? get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type) : a_offset;
+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+
+ GGML_ASSERT(a_offset < (1u << 8));
+ GGML_ASSERT(b_offset < (1u << 8));
+ GGML_ASSERT(d_offset < (1u << 8));
+
+ p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;
+
+ GGML_UNUSED(src2);
+ GGML_UNUSED(src3);
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_sum_rows_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+
+ p.misalign_offsets = (a_offset << 16) | d_offset;
+
+ GGML_UNUSED(src1);
+ GGML_UNUSED(src2);
+ GGML_UNUSED(src3);
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_pad_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+
+ p.misalign_offsets = (a_offset << 16) | d_offset;
+
+ GGML_UNUSED(src1);
+ GGML_UNUSED(src2);
+ GGML_UNUSED(src3);
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_im2col_3d_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ const uint32_t a_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);
+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+
+ p.misalign_offsets = (a_offset << 16) | d_offset;
+
+ GGML_UNUSED(src0);
+ GGML_UNUSED(src2);
+ GGML_UNUSED(src3);
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_binary_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
+ const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);
+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+
+ GGML_ASSERT(a_offset <= 0xFFFF);
+ GGML_ASSERT(b_offset <= 0xFF);
+ GGML_ASSERT(d_offset <= 0xFF);
+
+ p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;
+
+ GGML_UNUSED(src2);
+ GGML_UNUSED(src3);
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_concat_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type);
+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / unit_size;
+ const uint32_t b_offset = get_misalign_bytes(ctx, src1) / unit_size;
+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / unit_size;
+
+ p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;
+
+ GGML_UNUSED(src2);
+ GGML_UNUSED(src3);
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_upscale_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+
+ p.a_offset = a_offset;
+ p.d_offset = d_offset;
+
+ GGML_UNUSED(src1);
+ GGML_UNUSED(src2);
+ GGML_UNUSED(src3);
+}
+
+template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_rope_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
+ p.a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
+ p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+
+ GGML_UNUSED(src1);
+ GGML_UNUSED(src2);
+ GGML_UNUSED(src3);
+}
+
+static vk_op_binary_push_constants ggml_vk_rms_norm_push_constants(
+ const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst,
+ float eps, uint32_t num_partials) {
+ const uint32_t src0_type_size = ggml_type_size(src0->type);
+ const uint32_t src1_type_size = ggml_type_size(src1->type);
+ const uint32_t dst_type_size = ggml_type_size(dst->type);
+
+ return {
+ (uint32_t)ggml_nelements(src0),
+ (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size,
+ (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size,
+ (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size,
+ 0,
+ eps, 0.0f, (int32_t)num_partials,
+ };
+}
+
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
new file mode 100644
index 000000000..67e3361ed
--- /dev/null
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
@@ -0,0 +1,1432 @@
+#pragma once
+
+#include "ggml-vulkan.h"
+
+#include <vulkan/vulkan_core.h>
+
+#if defined(GGML_VULKAN_RUN_TESTS) || defined(GGML_VULKAN_CHECK_RESULTS)
+#include <chrono>
+#include "ggml-cpu.h"
+#endif
+
+#define VULKAN_HPP_DISPATCH_LOADER_DYNAMIC 1
+
+#if VK_HEADER_VERSION >= 301
+namespace vk::detail { class DispatchLoaderDynamic; }
+using vk::detail::DispatchLoaderDynamic;
+#else
+namespace vk { class DispatchLoaderDynamic; }
+using vk::DispatchLoaderDynamic;
+#endif
+
+DispatchLoaderDynamic & ggml_vk_default_dispatcher();
+
+#define VULKAN_HPP_DEFAULT_DISPATCHER ggml_vk_default_dispatcher()
+
+#include <vulkan/vulkan.hpp>
+
+#ifndef VK_NV_cooperative_matrix_decode_vector
+#define VK_NV_cooperative_matrix_decode_vector 1
+#define VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME "VK_NV_cooperative_matrix_decode_vector"
+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV ((VkStructureType)1000689000)
+typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV {
+ VkStructureType sType;
+ void* pNext;
+ VkBool32 cooperativeMatrixDecodeVector;
+} VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV;
+#endif
+
+#if __has_include(<spirv/unified1/spirv.hpp>)
+# include <spirv/unified1/spirv.hpp>
+#elif __has_include(<spirv-headers/spirv.hpp>)
+# include <spirv-headers/spirv.hpp>
+#elif __has_include(<spirv.hpp>)
+# include <spirv.hpp>
+#else
+ // Fallback to let the compiler throw a standard "file not found" error
+# include <spirv/unified1/spirv.hpp>
+#endif
+
+#include <algorithm>
+
+#include <cmath>
+
+#include <iomanip>
+
+#include <iostream>
+
+#include <tuple>
+
+#include <vector>
+
+#include <deque>
+
+#include <sstream>
+
+#include <utility>
+
+#include <memory>
+
+#include <limits>
+
+#include <map>
+
+#include <set>
+
+#include <unordered_map>
+
+#include <shared_mutex>
+
+#include <mutex>
+
+#include <future>
+
+#include <condition_variable>
+
+#include <thread>
+
+#if defined(_MSC_VER)
+# define NOMINMAX 1
+# include <windows.h>
+# define YIELD() YieldProcessor()
+#elif defined(__clang__) || defined(__GNUC__)
+# if defined(__x86_64__) ||defined(__i386__)
+# include <immintrin.h>
+# define YIELD() _mm_pause()
+# elif defined(__arm__) || defined(__aarch64__)
+# if defined(__clang__)
+# include <arm_acle.h>
+# define YIELD() __yield()
+# else
+# define YIELD() asm volatile("yield")
+# endif
+# endif
+#endif
+
+#if !defined(YIELD)
+#define YIELD()
+#endif
+
+#include "ggml-impl.h"
+
+#include "ggml-backend-impl.h"
+
+#include "ggml-vulkan-shaders.hpp"
+
+#if !defined(VK_KHR_shader_bfloat16)
+
+#define VK_KHR_shader_bfloat16 1
+#define VK_KHR_SHADER_BFLOAT16_SPEC_VERSION 1
+#define VK_KHR_SHADER_BFLOAT16_EXTENSION_NAME "VK_KHR_shader_bfloat16"
+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_BFLOAT16_FEATURES_KHR ((VkStructureType)1000141000)
+#define VK_COMPONENT_TYPE_BFLOAT16_KHR ((VkComponentTypeKHR)1000141000)
+
+typedef struct VkPhysicalDeviceShaderBfloat16FeaturesKHR {
+ VkStructureType sType;
+ void* pNext;
+ VkBool32 shaderBFloat16Type;
+ VkBool32 shaderBFloat16DotProduct;
+ VkBool32 shaderBFloat16CooperativeMatrix;
+} VkPhysicalDeviceShaderBfloat16FeaturesKHR;
+#endif
+
+#if !defined(VK_VALVE_shader_mixed_float_dot_product)
+#define VK_VALVE_shader_mixed_float_dot_product 1
+#define VK_VALVE_SHADER_MIXED_FLOAT_DOT_PRODUCT_SPEC_VERSION 1
+#define VK_VALVE_SHADER_MIXED_FLOAT_DOT_PRODUCT_EXTENSION_NAME "VK_VALVE_shader_mixed_float_dot_product"
+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MIXED_FLOAT_DOT_PRODUCT_FEATURES_VALVE ((VkStructureType)1000673000)
+typedef struct VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE {
+ VkStructureType sType;
+ void* pNext;
+ VkBool32 shaderMixedFloatDotProductFloat16AccFloat32;
+ VkBool32 shaderMixedFloatDotProductFloat16AccFloat16;
+ VkBool32 shaderMixedFloatDotProductBFloat16Acc;
+ VkBool32 shaderMixedFloatDotProductFloat8AccFloat32;
+} VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE;
+#endif
+
+#if !defined(VK_EXT_shader_ocp_microscaling_types)
+#define VK_EXT_shader_ocp_microscaling_types 1
+#define VK_EXT_SHADER_OCP_MICROSCALING_TYPES_SPEC_VERSION 1
+#define VK_EXT_SHADER_OCP_MICROSCALING_TYPES_EXTENSION_NAME "VK_EXT_shader_ocp_microscaling_types"
+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OCP_MICROSCALING_TYPES_FEATURES_EXT ((VkStructureType)1000672000)
+typedef struct VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT {
+ VkStructureType sType;
+ void* pNext;
+ VkBool32 shaderFloat4;
+ VkBool32 shaderFloat6;
+ VkBool32 shaderFloat8UnsignedE8M0;
+ VkBool32 shaderMXInt8;
+} VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT;
+#endif
+
+#if !defined(VK_EXT_shader_float8)
+#define VK_EXT_shader_float8 1
+#define VK_EXT_SHADER_FLOAT8_SPEC_VERSION 1
+#define VK_EXT_SHADER_FLOAT8_EXTENSION_NAME "VK_EXT_shader_float8"
+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT8_FEATURES_EXT ((VkStructureType)1000567000)
+typedef struct VkPhysicalDeviceShaderFloat8FeaturesEXT {
+ VkStructureType sType;
+ void* pNext;
+ VkBool32 shaderFloat8;
+ VkBool32 shaderFloat8CooperativeMatrix;
+} VkPhysicalDeviceShaderFloat8FeaturesEXT;
+#endif
+
+#ifndef VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME
+#define VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME "VK_KHR_internally_synchronized_queues"
+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INTERNALLY_SYNCHRONIZED_QUEUES_FEATURES_KHR ((VkStructureType)1000504000)
+#define VK_DEVICE_QUEUE_CREATE_INTERNALLY_SYNCHRONIZED_BIT_KHR ((VkDeviceQueueCreateFlagBits)0x00000004)
+
+// Compile-time constant guaranteed; no runtime initialization overhead
+static constexpr vk::DeviceQueueCreateFlagBits eInternallySynchronizedKHR =
+ static_cast<vk::DeviceQueueCreateFlagBits>(0x00000004);
+
+typedef struct VkPhysicalDeviceInternallySynchronizedQueuesFeaturesKHR {
+ VkStructureType sType;
+ void* pNext;
+ VkBool32 internallySynchronizedQueues;
+} VkPhysicalDeviceInternallySynchronizedQueuesFeaturesKHR;
+#else
+static constexpr vk::DeviceQueueCreateFlagBits eInternallySynchronizedKHR = vk::DeviceQueueCreateFlagBits::eInternallySynchronizedKHR;
+#endif
+
+#define ROUNDUP_POW2(M, N) (((M) + (N) - 1) & ~((N) - 1))
+
+#define CEIL_DIV(M, N) (((M) / (N)) + (((M) % (N)) != 0))
+
+static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; }
+
+#define VK_VENDOR_ID_AMD 0x1002
+
+#define VK_VENDOR_ID_APPLE 0x106b
+
+#define VK_VENDOR_ID_INTEL 0x8086
+
+#define VK_VENDOR_ID_NVIDIA 0x10de
+
+#define VK_VENDOR_ID_QUALCOMM 0x5143
+
+#define VK_DEVICE_DESCRIPTOR_POOL_SIZE 256
+
+#define VK_CHECK(err, msg, dev) \
+ do { \
+ vk::Result err_; \
+ try { \
+ err_ = (err); \
+ } catch (vk::DeviceLostError &) { \
+ ggml_vk_print_device_lost_info(dev); \
+ GGML_LOG_ERROR("ggml_vulkan: %s at %s:%d\n", \
+ #err, __FILE__, __LINE__); \
+ throw; \
+ } \
+ if (err_ != vk::Result::eSuccess) { \
+ GGML_LOG_ERROR("ggml_vulkan: %s error %s at %s:%d\n", \
+ #err, to_string(err_).c_str(), __FILE__, __LINE__); \
+ throw vk::SystemError(vk::make_error_code(err_), \
+ "ggml_vulkan: " msg); \
+ } \
+ } while (0)
+
+#ifdef GGML_VULKAN_DEBUG
+#define VK_LOG_DEBUG(msg) std::cerr << msg << std::endl
+#else
+#define VK_LOG_DEBUG(msg) ((void) 0)
+#endif // GGML_VULKAN_DEBUG
+
+#define MAX_PARAMETER_COUNT 12
+
+#define MAX_FUSED_ADDS (MAX_PARAMETER_COUNT - 3)
+
+struct vk_pipeline_struct;
+
+typedef std::shared_ptr<struct vk_pipeline_struct> vk_pipeline;
+
+struct vk_pipeline_struct {
+ std::string name;
+ vk::ShaderModule shader_module;
+ vk::PipelineLayout layout;
+ vk::Pipeline pipeline;
+ uint32_t push_constant_size;
+ uint32_t parameter_count;
+ std::array<uint32_t, 3> wg_denoms;
+ uint32_t align;
+ // true if fields have been set by ggml_vk_create_pipeline
+ bool initialized {};
+ // true while a compile is in flight, used to dedupe concurrent claims.
+ // Protected by device->compile_mutex.
+ bool compile_pending {};
+ // set to true when the shader has been compiled
+ std::atomic<bool> compiled {};
+ // number of registers used, extracted from pipeline executable properties
+ uint32_t register_count {};
+
+#if defined(VK_EXT_shader_64bit_indexing)
+ bool is_64b_indexing {};
+#endif
+ // linked list of pipelines for multiple compilation variants.
+ // currently only used to compile a 64-bit indexing variant.
+ vk_pipeline next;
+};
+
+typedef std::weak_ptr<vk_pipeline_struct> vk_pipeline_ref;
+
+struct vk_matmul_pipeline_key {
+ ggml_type type_a;
+ ggml_type type_b;
+ bool mul_mat_id;
+ bool f16acc;
+
+ bool operator<(const vk_matmul_pipeline_key & o) const {
+ return std::tie(type_a, type_b, mul_mat_id, f16acc)
+ < std::tie(o.type_a, o.type_b, o.mul_mat_id, o.f16acc);
+ }
+};
+
+struct vk_matmul_pipeline_pair {
+ vk_pipeline unaligned;
+ vk_pipeline aligned;
+ uint32_t align;
+};
+
+struct vk_tile_config {
+ std::vector<uint32_t> warptile;
+ std::array<uint32_t, 3> wg_denoms;
+ uint32_t align;
+};
+
+using matmul_tile_selector_t = std::function<uint32_t(
+ uint32_t m, uint32_t n, uint32_t k, uint32_t shader_core_count,
+ const std::vector<vk_matmul_pipeline_pair>& configs)>;
+
+struct vk_device_struct;
+
+typedef std::shared_ptr<vk_device_struct> vk_device;
+
+typedef std::weak_ptr<vk_device_struct> vk_device_ref;
+
+struct vk_buffer_struct;
+
+typedef std::shared_ptr<vk_buffer_struct> vk_buffer;
+
+typedef std::weak_ptr<vk_buffer_struct> vk_buffer_ref;
+
+struct ggml_backend_vk_buffer_type_context {
+ std::string name;
+ vk_device device;
+};
+
+struct vk_command_buffer {
+ vk::CommandBuffer buf;
+ uint64_t use_counter = 0;
+ bool in_use = false;
+};
+
+struct vk_queue;
+
+struct vk_command_pool {
+ void init(vk_device& device, vk_queue *q_);
+ void destroy(vk::Device& device);
+
+ vk::CommandPool pool;
+ // Using deque so the pointers to command buffers
+ // remain valid even if we add more
+ std::deque<vk_command_buffer> cmd_buffers;
+
+ vk_queue *q;
+
+ size_t buffers_in_use() const {
+ return std::count_if(cmd_buffers.begin(), cmd_buffers.end(),
+ [](const auto& cb) { return cb.in_use; });
+ }
+};
+
+struct vk_queue_handle {
+ vk::Queue queue;
+ vk_device_ref device;
+ std::mutex * device_submit_mutex = nullptr;
+ virtual void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) = 0;
+ virtual void lock() {} // no-op by default (internally synchronized case)
+ virtual void unlock() {}
+ virtual ~vk_queue_handle() = default;
+};
+
+struct vk_queue_handle_synchronized : vk_queue_handle {
+ std::mutex mutex;
+ void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override;
+
+ void lock() override { mutex.lock(); }
+ void unlock() override { mutex.unlock(); }
+};
+
+struct vk_queue_handle_unsynchronized : vk_queue_handle {
+ void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override;
+
+ // lock()/unlock() inherited no-ops
+};
+
+struct vk_queue {
+ uint32_t queue_family_index;
+ std::shared_ptr<vk_queue_handle> handle;
+
+ vk_command_pool cmd_pool;
+
+ vk::PipelineStageFlags stage_flags;
+
+ bool transfer_only;
+};
+
+static constexpr uint32_t mul_mat_vec_max_cols = 8;
+
+static constexpr uint32_t p021_max_gqa_ratio = 8;
+
+enum vk_device_architecture {
+ OTHER,
+ AMD_GCN,
+ AMD_RDNA1,
+ AMD_RDNA2,
+ AMD_RDNA3,
+ INTEL_XE1,
+ INTEL_XE2,
+ NVIDIA_PRE_TURING,
+ NVIDIA_TURING,
+};
+
+enum vk_conv_shapes {
+ CONV_SHAPE_128x128,
+ CONV_SHAPE_64x32,
+ CONV_SHAPE_32x256,
+ CONV_SHAPE_64x128,
+ CONV_SHAPE_COUNT,
+};
+
+struct vk_conv_block_size {
+ uint32_t K;
+ uint32_t NPQ;
+ uint32_t CRS;
+};
+
+inline vk_conv_block_size vk_conv_block_sizes[CONV_SHAPE_COUNT] = {
+ // K NPQ CRS
+ { 128, 128, 16 }, // CONV_SHAPE_128x128
+ { 64, 32, 32 }, // CONV_SHAPE_64x32
+ { 32, 256, 16 }, // CONV_SHAPE_32x256
+ { 64, 128, 16 }, // CONV_SHAPE_64x128
+};
+
+enum dmmv_wg_sizes {
+ DMMV_WG_SIZE_SUBGROUP,
+ DMMV_WG_SIZE_LARGE,
+ DMMV_WG_SIZE_COUNT,
+};
+
+enum FaCodePath {
+ FA_SCALAR,
+ FA_COOPMAT1,
+ FA_COOPMAT2,
+};
+
+struct vk_fa_pipeline_state {
+ uint32_t HSK, HSV;
+ uint32_t Br, Bc;
+ uint32_t D_split, row_split;
+ bool shmem_staging;
+ FaCodePath path;
+ uint32_t workgroup_size, subgroup_size;
+ bool aligned;
+ bool f32acc;
+ uint32_t flags;
+ uint32_t limit_occupancy_shmem;
+ ggml_type k_type;
+ ggml_type v_type;
+
+ bool operator<(const vk_fa_pipeline_state &b) const {
+ return std::tie(HSK, HSV, Br, Bc, D_split, row_split, shmem_staging, path, workgroup_size, subgroup_size, aligned, f32acc, flags, limit_occupancy_shmem, k_type, v_type) <
+ std::tie(b.HSK, b.HSV, b.Br, b.Bc, b.D_split, b.row_split, b.shmem_staging, b.path, b.workgroup_size, b.subgroup_size, b.aligned, b.f32acc, b.flags, b.limit_occupancy_shmem, b.k_type, b.v_type);
+ }
+};
+
+struct vk_conv2d_pipeline_state {
+ vk_conv2d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t p0, uint32_t p1, uint32_t d0, uint32_t d1, uint32_t KW, uint32_t KH, uint32_t aligned)
+ : s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), KW(KW), KH(KH), aligned(aligned) {}
+
+ uint32_t s0, s1, p0, p1, d0, d1, KW, KH;
+ // when set, shader can skip K/CRS/NPQ bounds checks and address clamps
+ uint32_t aligned;
+
+ bool operator<(const vk_conv2d_pipeline_state &b) const {
+ return std::tie(s0, s1, p0, p1, d0, d1, KW, KH, aligned) <
+ std::tie(b.s0, b.s1, b.p0, b.p1, b.d0, b.d1, b.KW, b.KH, b.aligned);
+ }
+};
+
+struct vk_conv3d_pipeline_state {
+ vk_conv3d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t s2, uint32_t p0, uint32_t p1, uint32_t p2,
+ uint32_t d0, uint32_t d1, uint32_t d2, uint32_t KW, uint32_t KH, uint32_t KD, uint32_t aligned)
+ : s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), KW(KW), KH(KH), KD(KD), aligned(aligned) {}
+
+ uint32_t s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD;
+ uint32_t aligned;
+
+ bool operator<(const vk_conv3d_pipeline_state &b) const {
+ return std::tie(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned) <
+ std::tie(b.s0, b.s1, b.s2, b.p0, b.p1, b.p2, b.d0, b.d1, b.d2, b.KW, b.KH, b.KD, b.aligned);
+ }
+};
+
+struct vk_solve_tri_pipeline_state {
+ vk_solve_tri_pipeline_state(uint32_t N, uint32_t K)
+ : N(N), K(K) {}
+
+ uint32_t N, K;
+
+ bool operator<(const vk_solve_tri_pipeline_state &b) const {
+ return std::tie(N, K) <
+ std::tie(b.N, b.K);
+ }
+};
+
+enum shader_reduction_mode {
+ SHADER_REDUCTION_MODE_SHMEM,
+ SHADER_REDUCTION_MODE_HYBRID,
+ SHADER_REDUCTION_MODE_SUBGROUP,
+ SHADER_REDUCTION_MODE_COUNT,
+};
+
+static constexpr uint32_t num_argsort_pipelines = 11;
+
+static constexpr uint32_t num_topk_moe_pipelines = 10;
+
+static constexpr uint32_t num_topk_pipelines = 11;
+
+static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax_norm{ GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT,
+ GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_RESHAPE,
+ GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV,
+ GGML_OP_RESHAPE };
+
+static constexpr std::initializer_list<ggml_op> topk_moe_sigmoid_norm_bias{ GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD,
+ GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS,
+ GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP,
+ GGML_OP_DIV, GGML_OP_RESHAPE };
+
+static constexpr std::initializer_list<ggml_op> topk_moe_sqrt_softplus_norm_bias{ GGML_OP_UNARY, GGML_OP_SQRT,
+ GGML_OP_RESHAPE, GGML_OP_ADD,
+ GGML_OP_ARGSORT, GGML_OP_VIEW,
+ GGML_OP_GET_ROWS, GGML_OP_RESHAPE,
+ GGML_OP_SUM_ROWS, GGML_OP_CLAMP,
+ GGML_OP_DIV, GGML_OP_RESHAPE };
+
+static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax { GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT,
+ GGML_OP_VIEW, GGML_OP_GET_ROWS };
+
+static constexpr std::initializer_list<ggml_op> topk_moe_late_softmax { GGML_OP_ARGSORT, GGML_OP_VIEW,
+ GGML_OP_GET_ROWS, GGML_OP_RESHAPE,
+ GGML_OP_SOFT_MAX, GGML_OP_RESHAPE };
+
+static constexpr std::initializer_list<ggml_op> snake_pattern { GGML_OP_MUL, GGML_OP_SIN,
+ GGML_OP_SQR, GGML_OP_MUL,
+ GGML_OP_ADD };
+
+static constexpr std::initializer_list<ggml_op> topk_qsa_pattern { GGML_OP_GET_ROWS, GGML_OP_PERMUTE,
+ GGML_OP_CONT, GGML_OP_CPY,
+ GGML_OP_RESHAPE, GGML_OP_ADD,
+ GGML_OP_TOP_K };
+
+static constexpr std::initializer_list<std::array<int, 3>> topk_qsa_edges {
+ { 1, 0, 0 }, // permute->src[0] == get_rows
+ { 2, 0, 1 }, // cont->src[0] == permute
+ { 4, 0, 3 }, // reshape->src[0] == cpy (mask cast)
+ { 5, 0, 2 }, // add->src[0] == cont
+ { 5, 1, 4 }, // add->src[1] == reshape
+ { 6, 0, 5 }, // top_k->src[0] == add
+};
+
+static constexpr std::initializer_list<ggml_op> rms_norm_mul_add_mul_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD, GGML_OP_MUL };
+
+static constexpr std::initializer_list<ggml_op> rms_norm_mul_add_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD };
+
+static constexpr std::initializer_list<ggml_op> rms_norm_mul_rope_view_set_rows_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS };
+
+static constexpr std::initializer_list<ggml_op> rms_norm_view_set_rows_pattern { GGML_OP_RMS_NORM, GGML_OP_VIEW, GGML_OP_SET_ROWS };
+
+static constexpr std::initializer_list<ggml_op> rope_view_set_rows_pattern { GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS };
+
+static constexpr std::initializer_list<std::array<int, 3>> topk_moe_early_softmax_norm_edges {
+ { 1, 0, 0 }, // reshape->src[0] == softmax
+ { 2, 0, 0 }, // argsort->src[0] == softmax
+ { 3, 0, 2 }, // view->src[0] == argsort
+ { 4, 0, 1 }, // get_rows->src[0] == reshape
+ { 4, 1, 3 }, // get_rows->src[1] == view
+ { 5, 0, 4 }, // reshape->src[0] == get_rows
+ { 6, 0, 5 }, // sum_rows->src[0] == reshape
+ { 7, 0, 6 }, // clamp->src[0] == sum_rows
+ { 8, 0, 5 }, // div->src[0] == reshape
+ { 8, 1, 7 }, // div->src[1] == clamp
+ { 9, 0, 8 }, // reshape->src[0] == div
+};
+
+static constexpr std::initializer_list<std::array<int, 3>> topk_moe_sigmoid_norm_bias_edges {
+ { 1, 0, 0 }, // reshape->src[0] == sigmoid
+ { 2, 0, 0 }, // add->src[0] == sigmoid
+ { 3, 0, 2 }, // argsort->src[0] == add
+ { 4, 0, 3 }, // view->src[0] == argsort
+ { 5, 0, 1 }, // get_rows->src[0] == reshape
+ { 5, 1, 4 }, // get_rows->src[1] == view
+ { 6, 0, 5 }, // reshape->src[0] == get_rows
+ { 7, 0, 6 }, // sum_rows->src[0] == reshape
+ { 8, 0, 7 }, // clamp->src[0] == sum_rows
+ { 9, 0, 6 }, // div->src[0] == reshape
+ { 9, 1, 8 }, // div->src[1] == clamp
+ {10, 0, 9 }, // reshape->src[0] == div
+};
+
+static constexpr std::initializer_list<std::array<int, 3>> topk_moe_sqrt_softplus_norm_bias_edges {
+ { 1, 0, 0 }, // sqrt->src[0] == softplus
+ { 2, 0, 1 }, // reshape->src[0] == sqrt
+ { 3, 0, 1 }, // add->src[0] == sqrt
+ { 4, 0, 3 }, // argsort->src[0] == add
+ { 5, 0, 4 }, // view->src[0] == argsort
+ { 6, 0, 2 }, // get_rows->src[0] == reshape
+ { 6, 1, 5 }, // get_rows->src[1] == view
+ { 7, 0, 6 }, // reshape->src[0] == get_rows
+ { 8, 0, 7 }, // sum_rows->src[0] == reshape
+ { 9, 0, 8 }, // clamp->src[0] == sum_rows
+ {10, 0, 7 }, // div->src[0] == reshape
+ {10, 1, 9 }, // div->src[1] == clamp
+ {11, 0,10 }, // reshape->src[0] == div
+};
+
+static constexpr std::initializer_list<std::array<int, 3>> topk_moe_early_softmax_edges {
+ { 1, 0, 0 }, // reshape->src[0] == softmax
+ { 2, 0, 0 }, // argsort->src[0] == softmax
+ { 3, 0, 2 }, // view->src[0] == argsort
+ { 4, 0, 1 }, // get_rows->src[0] == reshape
+ { 4, 1, 3 }, // get_rows->src[1] == view
+};
+
+static constexpr std::initializer_list<std::array<int, 3>> topk_moe_late_softmax_edges {
+ { 1, 0, 0 }, // view->src[0] == argsort
+ { 2, 1, 1 }, // get_rows->src[1] == view
+ { 3, 0, 2 }, // reshape->src[0] == get_rows
+ { 4, 0, 3 }, // soft_max->src[0] == reshape
+ { 5, 0, 4 }, // reshape->src[0] == soft_max
+};
+
+enum topk_moe_mode {
+ TOPK_MOE_EARLY_SOFTMAX,
+ TOPK_MOE_EARLY_SOFTMAX_NORM,
+ TOPK_MOE_LATE_SOFTMAX,
+ TOPK_MOE_SIGMOID_NORM_BIAS,
+ TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS,
+ TOPK_MOE_COUNT,
+};
+
+enum rms_norm_mode {
+ RMS_NORM_MUL,
+ RMS_NORM_MUL_ADD,
+ RMS_NORM_MUL_ADD_MUL,
+ RMS_NORM_MUL_ROPE,
+ RMS_NORM_MUL_ROPE_VIEW_SET_ROWS,
+ RMS_NORM_VIEW_SET_ROWS,
+ RMS_NORM_COUNT,
+};
+
+static constexpr std::initializer_list<std::array<int, 3>> rope_view_set_rows_edges {
+ { 1, 0, 0 }, // view->src[0] == rope
+ { 2, 0, 1 }, // set_rows->src[0] == view
+};
+
+static constexpr std::initializer_list<std::array<int, 3>> rms_norm_mul_rope_view_set_rows_edges {
+ { 1, 0, 0 }, // mul->src[0] == rms
+ { 2, 0, 1 }, // rope->src[0] == mul
+ { 3, 0, 2 }, // view->src[0] == rope
+ { 4, 0, 3 }, // set_rows->src[0] == view
+};
+
+static constexpr std::initializer_list<std::array<int, 3>> rms_norm_view_set_rows_edges {
+ { 1, 0, 0 }, // view->src[0] == rms_norm
+ { 2, 0, 1 }, // set_rows->src[0] == view
+};
+
+static constexpr std::array<ggml_type, 9> lightning_indexer_k_types = {
+ GGML_TYPE_F32,
+ GGML_TYPE_F16,
+ GGML_TYPE_BF16,
+ GGML_TYPE_Q8_0,
+ GGML_TYPE_Q5_1,
+ GGML_TYPE_Q5_0,
+ GGML_TYPE_Q4_1,
+ GGML_TYPE_Q4_0,
+ GGML_TYPE_IQ4_NL,
+};
+
+class vk_memory_logger;
+
+struct vk_device_struct {
+ std::recursive_mutex mutex;
+ std::mutex queue_submit_mutex;
+ mutable std::shared_mutex pinned_memory_mutex;
+
+ // Guards compile_pending, all_pipelines, and the dynamic pipeline maps
+ // (flash_attn, fa_mask_opt, solve_tri, conv2d, etc). The actual compile
+ // runs with no lock held, so different pipelines can compile in parallel.
+ // Lock order is device->mutex -> compile_mutex, never the reverse.
+ std::mutex compile_mutex;
+ std::condition_variable compile_cv;
+
+ uint32_t debug_cmdbuf_idx {};
+
+ vk::PhysicalDevice physical_device;
+ vk::PhysicalDeviceProperties properties;
+ std::string name;
+ uint64_t max_memory_allocation_size;
+ uint64_t max_buffer_size;
+ uint64_t suballocation_block_size;
+ uint64_t min_imported_host_pointer_alignment;
+ bool external_memory_host {};
+ bool fp16;
+ bool bf16;
+ bool pipeline_robustness;
+ bool memory_priority;
+ vk::Device device;
+ uint32_t vendor_id;
+ vk::DriverId driver_id;
+ vk_device_architecture architecture;
+ std::unique_ptr<vk_queue> compute_queue;
+ std::unique_ptr<vk_queue> transfer_queue;
+ bool single_queue;
+ bool support_async;
+ bool async_use_transfer_queue;
+ bool has_internally_synchronized_queues = false;
+ uint32_t subgroup_size;
+ uint32_t subgroup_size_log2;
+ uint32_t shader_core_count;
+ bool uma;
+ bool prefer_host_memory;
+ bool float_controls_rte_fp16;
+ bool float_controls_denorm_preserve_fp16;
+ bool subgroup_basic;
+ bool subgroup_arithmetic;
+ bool subgroup_shuffle;
+ bool subgroup_ballot;
+ bool subgroup_clustered;
+ bool subgroup_vote;
+ bool multi_add;
+ bool shader_int64;
+ bool buffer_device_address;
+ bool vulkan_memory_model;
+
+ bool add_rms_fusion;
+ uint32_t partials_binding_alignment;
+ uint32_t max_nodes_per_submit;
+
+ bool shader_64b_indexing;
+
+ bool integer_dot_product;
+ // 0: default, 1: force mmvq, -1: disable mmvq
+ int32_t mmvq_mode;
+
+ bool subgroup_size_control;
+ uint32_t subgroup_min_size;
+ uint32_t subgroup_max_size;
+ bool subgroup_require_full_support;
+
+ // floor(log2(maxComputeWorkGroupInvocations))
+ uint32_t max_workgroup_size_log2 {};
+
+ bool coopmat_support;
+ bool coopmat_acc_f32_support {};
+ bool coopmat_acc_f16_support {};
+ bool coopmat_bf16_support {};
+ bool coopmat_support_16x16x16_f16acc {};
+ bool coopmat_support_16x16x16_f32acc {};
+ bool coopmat1_fa_support {};
+ uint32_t coopmat_m;
+ uint32_t coopmat_n;
+ uint32_t coopmat_k;
+
+ bool coopmat_int_support;
+ uint32_t coopmat_int_m;
+ uint32_t coopmat_int_n;
+ uint32_t coopmat_int_k;
+
+ bool coopmat2;
+ bool coopmat2_bf16_support {};
+ bool coopmat2_decode_vector;
+
+ bool dot2_f16 {};
+ bool ocp_fp4 {};
+
+ bool pipeline_executable_properties_support {};
+
+ bool device_fault {};
+ PFN_vkGetDeviceFaultInfoEXT pfn_vkGetDeviceFaultInfoEXT {};
+
+ bool serialize_submissions {};
+
+ const ggml_cgraph * diag_cgraph {};
+ int diag_prev_start = -1;
+ int diag_prev_end = -1;
+
+ size_t idx;
+
+ bool mul_mat_l[GGML_TYPE_COUNT];
+ bool mul_mat_m[GGML_TYPE_COUNT];
+ bool mul_mat_s[GGML_TYPE_COUNT];
+ bool mul_mat_id_l[GGML_TYPE_COUNT];
+ bool mul_mat_id_m[GGML_TYPE_COUNT];
+ bool mul_mat_id_s[GGML_TYPE_COUNT];
+
+ // Separate flags for the q8_1 (integer dot) mmq path, whose shader uses
+ // a different shared-memory layout than the float matmul shaders.
+ bool mul_mat_l_int[GGML_TYPE_COUNT];
+ bool mul_mat_m_int[GGML_TYPE_COUNT];
+ bool mul_mat_s_int[GGML_TYPE_COUNT];
+ bool mul_mat_id_l_int[GGML_TYPE_COUNT];
+ bool mul_mat_id_m_int[GGML_TYPE_COUNT];
+ bool mul_mat_id_s_int[GGML_TYPE_COUNT];
+
+ vk::DescriptorSetLayout dsl;
+
+ std::map<vk_matmul_pipeline_key, std::vector<vk_matmul_pipeline_pair>> pipeline_matmul;
+ matmul_tile_selector_t matmul_tile_selector;
+ matmul_tile_selector_t matmul_id_tile_selector;
+
+ vk_pipeline pipeline_matmul_split_k_reduce;
+ vk_pipeline pipeline_quantize_q8_1_x4;
+
+ vk_pipeline pipeline_dequant[GGML_TYPE_COUNT];
+ vk_pipeline pipeline_dequant_transpose[GGML_TYPE_COUNT]; // fused dequant+transpose for FA quant-KV
+ vk_pipeline pipeline_dequant_mul_mat_vec_f32_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols];
+ vk_pipeline pipeline_dequant_mul_mat_vec_f16_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols];
+ vk_pipeline pipeline_dequant_mul_mat_vec_id_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT];
+
+ vk_pipeline pipeline_dequant_mul_mat_vec_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols];
+ vk_pipeline pipeline_dequant_mul_mat_vec_id_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT];
+
+ vk_pipeline pipeline_mul_mat_vec_p021_f16_f32[p021_max_gqa_ratio];
+ vk_pipeline pipeline_mul_mat_vec_nc_f16_f32;
+ vk_pipeline pipeline_get_rows[GGML_TYPE_COUNT];
+ vk_pipeline pipeline_get_rows_f32[GGML_TYPE_COUNT];
+ vk_pipeline pipeline_get_rows_back_f32;
+ vk_pipeline pipeline_acc_f32;
+ vk_pipeline pipeline_set_f32;
+
+ // [src0 0=fp32,1=fp16][src1 0=fp32,1=fp16][dst 0=fp32,1=fp16]
+ vk_pipeline pipeline_add[2][2][2];
+ vk_pipeline pipeline_add_norepeat[2][2][2];
+ vk_pipeline pipeline_sub[2][2][2];
+ vk_pipeline pipeline_sub_norepeat[2][2][2];
+ vk_pipeline pipeline_mul[2][2][2];
+ vk_pipeline pipeline_mul_norepeat[2][2][2];
+ vk_pipeline pipeline_div[2][2][2];
+ vk_pipeline pipeline_div_norepeat[2][2][2];
+ vk_pipeline pipeline_add_rms[2][2][2];
+ vk_pipeline pipeline_add_rms_norepeat[2][2][2];
+
+ // indexed by num_additional_fused_ops == num_adds - 1
+ vk_pipeline pipeline_multi_add[MAX_FUSED_ADDS];
+ vk_pipeline pipeline_multi_add_rms[MAX_FUSED_ADDS];
+
+ vk_pipeline pipeline_add_id_f32;
+
+ vk_pipeline pipeline_concat_i8, pipeline_concat_i16, pipeline_concat_i32, pipeline_concat_i64;
+ vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32;
+ vk_pipeline pipeline_scale_f32;
+ vk_pipeline pipeline_log[2];
+ vk_pipeline pipeline_tri[2];
+ vk_pipeline pipeline_diag[2];
+ vk_pipeline pipeline_clamp[2];
+ vk_pipeline pipeline_pad_f32;
+ vk_pipeline pipeline_pad_reflect_1d_f32;
+ vk_pipeline pipeline_roll_f32;
+ vk_pipeline pipeline_repeat_i32, pipeline_repeat_back_f32;
+ vk_pipeline pipeline_repeat_i16;
+ vk_pipeline pipeline_cpy_f32_f32, pipeline_cpy_f32_f16, pipeline_cpy_f16_f16, pipeline_cpy_f16_f32, pipeline_cpy_f32_bf16, pipeline_cpy_bf16_f32, pipeline_cpy_f32_i32, pipeline_cpy_i32_f32;
+ vk_pipeline pipeline_contig_cpy_f32_f32, pipeline_contig_cpy_f32_f16, pipeline_contig_cpy_f16_f16, pipeline_contig_cpy_f16_f32, pipeline_contig_cpy_f32_bf16, pipeline_contig_cpy_bf16_f32, pipeline_contig_cpy_f32_i32, pipeline_contig_cpy_i32_f32;
+ vk_pipeline pipeline_cpy_f32_quant[GGML_TYPE_COUNT];
+ vk_pipeline pipeline_cpy_quant_f32[GGML_TYPE_COUNT];
+ vk_pipeline pipeline_cpy_transpose_16, pipeline_cpy_transpose_32;
+ vk_pipeline pipeline_cpy_transpose_02_16, pipeline_cpy_transpose_02_32;
+ // [src0 0=fp32,1=fp16][dst]
+ vk_pipeline pipeline_set_rows_i32[2][GGML_TYPE_COUNT];
+ vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT];
+ vk_pipeline pipeline_norm_f32;
+ vk_pipeline pipeline_group_norm_f32;
+ vk_pipeline pipeline_rms_norm_f32;
+ vk_pipeline pipeline_rms_norm_mul_f32;
+ vk_pipeline pipeline_rms_norm_mul_add_f32;
+ vk_pipeline pipeline_rms_norm_mul_add_mul_f32;
+ vk_pipeline pipeline_rms_norm_mul_add_partials_f32;
+ vk_pipeline pipeline_rms_norm_mul_add_mul_partials_f32;
+ vk_pipeline pipeline_rms_norm_set_rows_f32_f32;
+ vk_pipeline pipeline_rms_norm_set_rows_f32_f16;
+ vk_pipeline pipeline_rms_norm_partials_f32;
+ vk_pipeline pipeline_rms_norm_mul_partials_f32;
+ vk_pipeline pipeline_rms_norm_mul_rope_f32_f32;
+ vk_pipeline pipeline_rms_norm_mul_rope_f32_f16;
+ vk_pipeline pipeline_rms_norm_back_f32;
+ vk_pipeline pipeline_l2_norm_f32;
+
+ // [src/dst 0=fp32,1=fp16]
+ vk_pipeline pipeline_exp[2];
+ vk_pipeline pipeline_expm1[2];
+ vk_pipeline pipeline_elu[2];
+ vk_pipeline pipeline_gelu[2];
+ vk_pipeline pipeline_gelu_erf[2];
+ vk_pipeline pipeline_gelu_quick[2];
+ vk_pipeline pipeline_silu[2];
+ vk_pipeline pipeline_relu[2];
+ vk_pipeline pipeline_sqr[2];
+ vk_pipeline pipeline_sqrt[2];
+ vk_pipeline pipeline_sin[2];
+ vk_pipeline pipeline_cos[2];
+ vk_pipeline pipeline_xielu[2];
+ vk_pipeline pipeline_neg[2];
+ vk_pipeline pipeline_tanh[2];
+ vk_pipeline pipeline_sigmoid[2];
+ vk_pipeline pipeline_hardsigmoid[2];
+ vk_pipeline pipeline_hardswish[2];
+ vk_pipeline pipeline_abs[2];
+ vk_pipeline pipeline_softplus[2];
+ vk_pipeline pipeline_step[2];
+ vk_pipeline pipeline_round[2];
+ vk_pipeline pipeline_ceil[2];
+ vk_pipeline pipeline_floor[2];
+ vk_pipeline pipeline_trunc[2];
+ vk_pipeline pipeline_sgn[2];
+
+ // fused UNARY+MUL pipelines: [op][f16][norepeat][op_on_b]
+ vk_pipeline pipeline_unary_mul[4][2][2][2];
+
+ vk_pipeline pipeline_add1_f16_f16;
+ vk_pipeline pipeline_add1_f16_f32;
+ vk_pipeline pipeline_add1_f32_f32;
+
+ vk_pipeline pipeline_arange_f32;
+
+ vk_pipeline pipeline_fill_f32;
+ vk_pipeline pipeline_fill_f16;
+
+ vk_pipeline pipeline_geglu[2];
+ vk_pipeline pipeline_reglu[2];
+ vk_pipeline pipeline_swiglu[2];
+ vk_pipeline pipeline_swiglu_oai[2];
+ vk_pipeline pipeline_swiglu_clamp[2];
+ vk_pipeline pipeline_geglu_erf[2];
+ vk_pipeline pipeline_geglu_quick[2];
+
+ vk_pipeline pipeline_leaky_relu[2];
+ vk_pipeline pipeline_silu_back_f32;
+ vk_pipeline pipeline_diag_mask_inf_f32;
+ vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16;
+ vk_pipeline pipeline_soft_max_f32_wg512, pipeline_soft_max_f32_f16_wg512;
+ vk_pipeline pipeline_soft_max_back_f32;
+
+ vk_pipeline pipeline_soft_max_large1_f32, pipeline_soft_max_large1_f32_f16;
+ vk_pipeline pipeline_soft_max_large2_f32, pipeline_soft_max_large2_f32_f16;
+ vk_pipeline pipeline_soft_max_large3_f32, pipeline_soft_max_large3_f32_f16;
+
+ vk_pipeline pipeline_rope_norm_f32, pipeline_rope_norm_f16, pipeline_rope_norm_f32_f16;
+ vk_pipeline pipeline_rope_neox_f32, pipeline_rope_neox_f16, pipeline_rope_neox_f32_f16;
+ vk_pipeline pipeline_rope_multi_f32, pipeline_rope_multi_f16, pipeline_rope_multi_f32_f16;
+ vk_pipeline pipeline_rope_vision_f32, pipeline_rope_vision_f16;
+ vk_pipeline pipeline_argsort_f32[num_argsort_pipelines];
+ vk_pipeline pipeline_argsort_large_f32[num_argsort_pipelines];
+ vk_pipeline pipeline_topk_f32[num_topk_pipelines];
+ vk_pipeline pipeline_topk_radix_f32;
+ vk_pipeline pipeline_topk_radix_qsa; // qwen4 QSA indexer fusion (f16 mask)
+ vk_pipeline pipeline_sum_rows_f32;
+ vk_pipeline pipeline_cross_entropy_loss_f32, pipeline_cross_entropy_loss_f32_wg512;
+ vk_pipeline pipeline_cross_entropy_loss_back_f32, pipeline_cross_entropy_loss_back_f32_wg512;
+ vk_pipeline pipeline_fwht_f32[4];
+ vk_pipeline pipeline_cumsum_f32;
+ vk_pipeline pipeline_cumsum_small_f32;
+ vk_pipeline pipeline_cumsum_multipass1_f32;
+ vk_pipeline pipeline_cumsum_multipass2_f32;
+ vk_pipeline pipeline_argmax_f32;
+ vk_pipeline pipeline_count_equal_i32;
+ vk_pipeline pipeline_dsv4_hc_comb_f32;
+ vk_pipeline pipeline_dsv4_hc_pre_f32;
+ vk_pipeline pipeline_dsv4_hc_pre_gated_f32;
+ vk_pipeline pipeline_dsv4_hc_post_f32;
+ vk_pipeline pipeline_dsv4_hc_post_nocomb_f32;
+ std::map<vk_solve_tri_pipeline_state, vk_pipeline> pipeline_solve_tri_f32;
+ vk_pipeline pipeline_im2col_f32, pipeline_im2col_f32_f16;
+ vk_pipeline pipeline_im2col_3d_f32, pipeline_im2col_3d_f32_f16;
+ vk_pipeline pipeline_timestep_embedding_f32;
+ vk_pipeline pipeline_conv_transpose_1d_f32;
+ vk_pipeline pipeline_col2im_1d_f32;
+ vk_pipeline pipeline_col2im_1d_f16;
+ vk_pipeline pipeline_col2im_1d_bf16;
+ vk_pipeline pipeline_out_prod_f32;
+ vk_pipeline pipeline_snake_f32;
+ vk_pipeline pipeline_snake_f16;
+ vk_pipeline pipeline_snake_bf16;
+ vk_pipeline pipeline_pool1d_f32;
+ vk_pipeline pipeline_pool2d_f32;
+ vk_pipeline pipeline_rwkv_wkv6_f32;
+ vk_pipeline pipeline_rwkv_wkv7_f32;
+ vk_pipeline pipeline_gated_linear_attn_f32;
+ vk_pipeline pipeline_lightning_indexer_f32[GGML_TYPE_COUNT];
+ // [size_idx][kda] where size_idx: 0=d16, 1=d32, 2=d64, 3=d128
+ vk_pipeline pipeline_gated_delta_net[4][2];
+ vk_pipeline pipeline_ssm_scan_f32_d128;
+ vk_pipeline pipeline_ssm_scan_f32_d256;
+ vk_pipeline pipeline_ssm_conv_f32;
+ vk_pipeline pipeline_ssm_conv_silu_f32;
+ vk_pipeline pipeline_ssm_conv_bias_silu_f32;
+ vk_pipeline pipeline_opt_step_adamw_f32;
+ vk_pipeline pipeline_opt_step_sgd_f32;
+ std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv2d_f32[CONV_SHAPE_COUNT];
+ std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv2d_f16_f32[CONV_SHAPE_COUNT];
+ std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv_transpose_2d_f32[CONV_SHAPE_COUNT];
+ std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv_transpose_2d_f16_f32[CONV_SHAPE_COUNT];
+ std::map<vk_conv3d_pipeline_state, vk_pipeline> pipeline_conv3d_f32[CONV_SHAPE_COUNT];
+ std::map<vk_conv3d_pipeline_state, vk_pipeline> pipeline_conv3d_f16_f32[CONV_SHAPE_COUNT];
+ vk_pipeline pipeline_conv2d_dw_whcn_f32, pipeline_conv2d_dw_whcn_f16_f32;
+ vk_pipeline pipeline_conv2d_dw_cwhn_f32, pipeline_conv2d_dw_cwhn_f16_f32;
+
+ std::map<vk_fa_pipeline_state, vk_pipeline> pipeline_flash_attn_f32_f16;
+
+ std::map<std::pair<uint32_t, uint32_t>, vk_pipeline> pipeline_fa_mask_opt;
+
+ vk_pipeline pipeline_fa_sparse_compact;
+ vk_pipeline pipeline_fa_sparse_compact_subgroup;
+ bool fa_sparse_compact_use_subgroups;
+
+ vk_pipeline pipeline_flash_attn_split_k_reduce;
+ vk_pipeline pipeline_count_experts;
+
+ // [2] is for whether to take n_experts from spec constant (0) or push constant (1)
+ vk_pipeline pipeline_topk_moe[num_topk_moe_pipelines][2];
+
+ std::vector<vk_pipeline_ref> all_pipelines;
+
+ std::vector<std::tuple<void*, size_t, vk_buffer>> pinned_memory;
+
+ vk::Fence fence;
+ vk_buffer sync_staging;
+
+ ggml_backend_buffer_type buffer_type;
+
+ bool disable_fusion;
+ bool disable_host_visible_vidmem;
+ bool allow_sysmem_fallback;
+ bool disable_graph_optimize;
+
+ std::unique_ptr<vk_memory_logger> memory_logger;
+
+ ~vk_device_struct();
+
+};
+
+inline void vk_command_pool::init(vk_device& device, vk_queue *q_) {
+ cmd_buffers.clear();
+ q = q_;
+
+ vk::CommandPoolCreateInfo command_pool_create_info(
+ vk::CommandPoolCreateFlags(VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT),
+ q->queue_family_index);
+ pool = device->device.createCommandPool(command_pool_create_info);
+}
+
+inline void vk_command_pool::destroy(vk::Device& device) {
+ device.destroyCommandPool(pool);
+ pool = nullptr;
+ cmd_buffers.clear();
+}
+
+struct vk_buffer_struct {
+ vk::Buffer buffer = VK_NULL_HANDLE;
+ vk::DeviceMemory device_memory = VK_NULL_HANDLE;
+ vk::MemoryPropertyFlags memory_property_flags;
+ void * ptr;
+ size_t size = 0;
+ vk::DeviceAddress bda_addr {};
+
+ vk_device device;
+
+ ~vk_buffer_struct() {
+ if (size == 0) {
+ return;
+ }
+ VK_LOG_DEBUG("~vk_buffer_struct(" << buffer << ", " << size << ")");
+
+ device->device.freeMemory(device_memory);
+ device->device.destroyBuffer(buffer);
+ }
+};
+
+struct vk_subbuffer {
+ vk_buffer buffer;
+ uint64_t offset;
+ uint64_t size;
+
+ operator vk::DescriptorBufferInfo() const {
+ return { buffer->buffer, offset, size };
+ }
+};
+
+struct vk_semaphore {
+ vk::Semaphore s;
+ uint64_t value;
+};
+
+struct vk_event {
+ std::vector<vk::Event> events_free; // Events available for reuse
+ std::vector<vk::Event> events_submitted; // Events that are fully submitted and can be reused on next synchronize
+ vk::Event event;
+ bool has_event;
+
+ vk_semaphore tl_semaphore;
+ vk_command_buffer* cmd_buffer = nullptr;
+ uint64_t cmd_buffer_use_counter = 0;
+};
+
+struct vk_submission {
+ vk_command_buffer* buffer = nullptr;
+ std::vector<vk_semaphore> wait_semaphores;
+ std::vector<vk_semaphore> signal_semaphores;
+};
+
+typedef std::vector<vk_submission> vk_sequence;
+
+#define MAT_VEC_FUSION_FLAGS_BIAS0 0x1
+
+#define MAT_VEC_FUSION_FLAGS_BIAS1 0x2
+
+#define MAT_VEC_FUSION_FLAGS_SCALE0 0x4
+
+#define MAT_VEC_FUSION_FLAGS_SCALE1 0x8
+
+struct vk_staging_memcpy {
+ vk_staging_memcpy(void * _dst, const void * _src, size_t _n) : dst(_dst), src(_src), n(_n) {}
+
+ void * dst;
+ const void * src;
+ size_t n;
+};
+
+struct vk_staging_memset {
+ vk_staging_memset(void * _dst, uint32_t _val, size_t _n) : dst(_dst), val(_val), n(_n) {}
+
+ void * dst;
+ uint32_t val;
+ size_t n;
+};
+
+struct vk_context_struct {
+ vk_submission * s;
+ std::vector<vk_sequence> seqs;
+
+ int exit_tensor_idx;
+
+ std::vector<vk_staging_memcpy> in_memcpys;
+ std::vector<vk_staging_memcpy> out_memcpys;
+ std::vector<vk_staging_memset> memsets;
+
+ std::vector<std::string> debug_labels;
+
+ vk_command_pool * p {};
+};
+
+typedef std::shared_ptr<vk_context_struct> vk_context;
+
+typedef std::weak_ptr<vk_context_struct> vk_context_ref;
+
+struct ggml_vk_garbage_collector {
+ std::vector<vk_semaphore> tl_semaphores;
+ std::vector<vk_semaphore> semaphores;
+ std::vector<vk::Event> events;
+ std::vector<vk_context> contexts;
+};
+
+#define VK_LOG_MEMORY(msg) if (vk_memory_logger_enabled) { std::cerr << "ggml_vulkan memory: " << msg << std::endl; }
+
+static std::string format_size(size_t size) {
+ const size_t kib = 1024;
+ const size_t mib = kib * 1024;
+ const size_t gib = mib * 1024;
+
+ std::ostringstream oss;
+ oss << std::fixed << std::setprecision(2);
+
+ if (size >= gib) {
+ oss << static_cast<double>(size) / gib << " GiB";
+ } else if (size >= mib) {
+ oss << static_cast<double>(size) / mib << " MiB";
+ } else if (size >= kib) {
+ oss << static_cast<double>(size) / kib << " KiB";
+ } else {
+ oss << size << " B";
+ }
+
+ return oss.str();
+}
+
+class vk_memory_logger {
+public:
+ vk_memory_logger(): total_device(0), total_host(0) {}
+ void log_allocation(vk_buffer_ref buf_ref, size_t size);
+ void log_deallocation(vk_buffer_ref buf_ref);
+
+private:
+ std::map<vk::Buffer, size_t> allocations; // Track allocations
+ size_t total_device;
+ size_t total_host;
+ static std::mutex log_mutex;
+};
+
+inline std::mutex vk_memory_logger::log_mutex;
+
+class vk_perf_logger {
+ public:
+ void print_timings(bool force = false);
+
+
+ std::string get_node_fusion_name(const ggml_tensor * node, const char *fusion_name, uint64_t *n_flops);
+
+
+ void log_timing(const ggml_tensor * node, const char *fusion_name, uint64_t time) {
+ uint64_t n_flops;
+ std::string name = get_node_fusion_name(node, fusion_name, &n_flops);
+ if (n_flops) {
+ flops[name].push_back(n_flops);
+ }
+ timings[name].push_back(time);
+ }
+
+ void log_timing(const std::vector<ggml_tensor *> &nodes, const std::vector<const char *> &names, uint64_t time) {
+ uint64_t total_flops = 0;
+ std::string name;
+ for (size_t n = 0; n < nodes.size(); ++n) {
+ uint64_t n_flops = 0;
+ name += get_node_fusion_name(nodes[n], names[n], &n_flops);
+ total_flops += n_flops;
+
+ if (n != nodes.size() - 1) {
+ name += ", ";
+ }
+ }
+ if (total_flops) {
+ flops[name].push_back(total_flops);
+ }
+ timings[name].push_back(time);
+ }
+
+ private:
+ std::map<std::string, std::vector<uint64_t>> timings;
+ std::map<std::string, std::vector<uint64_t>> flops;
+ uint32_t print_count {};
+};
+
+struct ggml_backend_vk_context {
+ std::string name;
+
+ vk_device device;
+
+ size_t semaphore_idx, event_idx;
+ ggml_vk_garbage_collector gc;
+ size_t prealloc_size_x, prealloc_size_y, prealloc_size_split_k, prealloc_size_add_rms_partials, prealloc_size_add_rms_partials_offset;
+ vk_buffer prealloc_x, prealloc_y, prealloc_split_k, prealloc_add_rms_partials, sync_staging;
+ vk::Fence fence, almost_ready_fence;
+ bool submit_pending {};
+ bool almost_ready_fence_pending {};
+ // Set before op_add and unset after op_rms_norm to indicate that the add should
+ // write partial sums to accumulate the square of the vector components
+ bool do_add_rms_partials_offset_calculation;
+ bool do_add_rms_partials;
+
+ uint64_t last_total_flops {UINT64_MAX};
+
+ // Cache most recent tensor that was converted into prealloc_y, and what pipeline it used to convert.
+ vk_pipeline_struct * prealloc_y_last_pipeline_used {};
+ const ggml_tensor * prealloc_y_last_tensor_used {};
+ // True when the K dimension in prealloc_y is padded.
+ bool prealloc_y_last_k_padded {};
+
+ // Track which nodes have been used since the last sync, and whether they were written to
+ std::vector<const ggml_tensor *> unsynced_nodes_written;
+ std::vector<const ggml_tensor *> unsynced_nodes_read;
+ // Track which prealloc buffers have pending reads that need to be synchronized.
+ // These are checked before writing to the buffer (and call ggml_vk_sync_buffers if set),
+ // and set to true after the buffer contents are consumed.
+ bool prealloc_x_need_sync, prealloc_y_need_sync, prealloc_split_k_need_sync;
+
+ vk_context_ref compute_ctx;
+
+ vk_context_ref transfer_ctx;
+ vk_semaphore transfer_semaphore;
+ uint64_t transfer_semaphore_last_submitted {};
+
+ std::vector<vk_context_ref> tensor_ctxs;
+
+ std::vector<vk::DescriptorPool> descriptor_pools;
+ std::vector<vk::DescriptorSet> descriptor_sets;
+ uint32_t descriptor_set_idx {};
+ uint32_t pipeline_descriptor_set_requirements {};
+
+ vk_command_pool compute_cmd_pool;
+ vk_command_pool transfer_cmd_pool;
+
+ // number of additional consecutive nodes that are being fused with the
+ // node currently being processed
+ int num_additional_fused_ops {};
+ // Bitmask of which fused ops need to write an intermediate value to memory.
+ // Bit 'i' means nodes[start_of_fusion + i] writes to memory.
+ // If there's no fusion, bit 0 is still set.
+ int fused_ops_write_mask {};
+ topk_moe_mode fused_topk_moe_mode {};
+ bool fused_topk_moe_scale {};
+ // QSA indexer gather+add+top_k fused into one radix-select
+ bool fused_topk_qsa {};
+ rms_norm_mode fused_rms_norm_mode {RMS_NORM_COUNT};
+
+ // for GGML_VK_PERF_LOGGER
+ std::unique_ptr<vk_perf_logger> perf_logger;
+ vk::QueryPool query_pool;
+ std::vector<const char *> query_fusion_names;
+ std::vector<int> query_fusion_node_count;
+ std::vector<ggml_tensor *> query_nodes;
+ std::vector<int> query_node_idx;
+ int32_t num_queries {};
+ int32_t query_idx {};
+};
+
+struct ggml_backend_vk_buffer_context {
+ vk_device_ref device;
+ vk_buffer dev_buffer;
+ std::string name;
+
+ ggml_backend_vk_buffer_context(vk_device_ref device, vk_buffer&& dev_buffer, std::string& name) :
+ device(device),
+ dev_buffer(dev_buffer),
+ name(name) {
+ }
+
+ ~ggml_backend_vk_buffer_context();
+
+};
+
+struct vk_instance_t {
+ vk::Instance instance;
+
+ bool debug_utils_support = false; // VK_EXT_debug_utils enabled
+ PFN_vkSetDebugUtilsObjectNameEXT pfn_vkSetDebugUtilsObjectNameEXT = {};
+ PFN_vkQueueBeginDebugUtilsLabelEXT pfn_vkQueueBeginDebugUtilsLabelEXT = {};
+ PFN_vkQueueEndDebugUtilsLabelEXT pfn_vkQueueEndDebugUtilsLabelEXT = {};
+ PFN_vkCmdBeginDebugUtilsLabelEXT pfn_vkCmdBeginDebugUtilsLabelEXT = {};
+ PFN_vkCmdEndDebugUtilsLabelEXT pfn_vkCmdEndDebugUtilsLabelEXT = {};
+ PFN_vkCmdInsertDebugUtilsLabelEXT pfn_vkCmdInsertDebugUtilsLabelEXT = {};
+
+ std::vector<size_t> device_indices;
+ std::vector<bool> device_supports_membudget;
+ vk_device devices[GGML_VK_MAX_DEVICES];
+};
+
+typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
+
+static constexpr uint32_t kSpvOpCooperativeMatrixLoadTensorNV = 5367;
+
+static constexpr uint32_t kSpvCapabilityCooperativeMatrixDecodeVectorNV = 5447;
+
+static constexpr uint32_t kSpvTensorAddressingDecodeVectorFuncBit = 0x4;
+
+struct vk_fa_tuning_params {
+ FaCodePath path;
+ uint32_t workgroup_size;
+ uint32_t subgroup_size;
+ uint32_t block_rows;
+ uint32_t block_cols;
+ uint32_t d_split;
+ uint32_t row_split;
+ bool shmem_staging;
+ bool disable_subgroups;
+ uint32_t limit_occupancy_shmem;
+
+ void print() const {
+ std::cerr << "path=" << path << " workgroup_size=" << workgroup_size << " subgroup_size=" << subgroup_size <<
+ " block_rows=" << block_rows << " block_cols=" << block_cols << " d_split=" << d_split <<
+ " row_split=" << row_split << " shmem_staging=" << shmem_staging << " disable_subgroups=" << disable_subgroups <<
+ " limit_occupancy_shmem=" << limit_occupancy_shmem << std::endl;
+ }
+};
+
+struct GpuPipelineConfig {
+ // GPU architecture identifier.
+ // Example: vk_device_architecture::AMD_GCN
+ vk_device_architecture arch;
+
+ // Mapping of pipeline names to their specific subgroup sizes.
+ // Example: {"soft_max_f32", 64}
+ std::unordered_map<std::string, uint32_t> pipelines;
+
+ // Default subgroup size for this GPU.
+ // Defaults to 0 if not explicitly provided.
+ uint32_t default_subgroup_size = 0;
+};
+
+static constexpr uint32_t RDNA_DEFAULT_SUBGROUP_SIZE = 32;
+
+struct CompileTask {
+ vk_pipeline pipeline;
+ size_t spv_size;
+ const void * spv_data;
+ std::string entrypoint;
+ uint32_t parameter_count;
+ std::array<uint32_t, 3> wg_denoms;
+ std::vector<uint32_t> specialization_constants;
+ bool disable_robustness;
+ bool require_full_subgroups;
+ uint32_t required_subgroup_size;
+};
+
+struct ggml_vk_debug_label {
+ // at most one of these is set, depending on the scope the label was opened in
+ vk_context_struct * subctx {};
+ vk_queue_handle * qhandle {};
+
+ // one region per dispatch, e.g. "matmul_q4_k_f32_f16acc_aligned_m (192,8,1)".
+ // RGP cannot recover the pipeline name on its own, it only has the hash
+ ggml_vk_debug_label(vk_context & ctx, const std::string & pipeline_name, uint32_t wg0, uint32_t wg1, uint32_t wg2);
+
+
+ // one region per graph node
+ // fused nodes are joined with '+', e.g. "RMS_NORM+MUL+ROPE Qcur-19"
+ ggml_vk_debug_label(vk_context & ctx, const ggml_cgraph * cgraph, int node_idx, int n_fused);
+
+
+ // one region per graph evaluation, opened on the queue instead of a command buffer
+ // so it spans every submit the evaluation makes
+ ggml_vk_debug_label(vk_queue_handle * handle, const char * name);
+
+
+ // call before the command buffer can end, the destructor covers the rest
+ void close();
+
+
+ ~ggml_vk_debug_label() {
+ close();
+ }
+
+ ggml_vk_debug_label(const ggml_vk_debug_label &) = delete;
+ ggml_vk_debug_label & operator=(const ggml_vk_debug_label &) = delete;
+
+private:
+ // the constructors check this too, so the name is not built when markers are off
+ void begin(vk_context & ctx, const std::string & name);
+
+};
+
+#define UNUSED GGML_UNUSED
+
+struct ggml_backend_vk_device_context {
+ size_t device;
+ std::string name;
+ std::string description;
+ bool is_integrated_gpu;
+ std::string pci_bus_id;
+ int op_offload_min_batch_size;
+};
+
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index 917b03dea..7b53d1b5c 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -1,432 +1,10 @@
-#include "ggml-vulkan.h"
-#include <vulkan/vulkan_core.h>
-#if defined(GGML_VULKAN_RUN_TESTS) || defined(GGML_VULKAN_CHECK_RESULTS)
-#include <chrono>
-#include "ggml-cpu.h"
-#endif
-
-// See https://github.com/KhronosGroup/Vulkan-Hpp?tab=readme-ov-file#extensions--per-device-function-pointers-
-#define VULKAN_HPP_DISPATCH_LOADER_DYNAMIC 1
-// We use VULKAN_HPP_DEFAULT_DISPATCHER, but not VULKAN_HPP_DEFAULT_DISPATCH_LOADER_DYNAMIC_STORAGE
-// to avoid conflicts with applications or other libraries who might use it.
-#if VK_HEADER_VERSION >= 301
-namespace vk::detail { class DispatchLoaderDynamic; }
-using vk::detail::DispatchLoaderDynamic;
-#else
-namespace vk { class DispatchLoaderDynamic; }
-using vk::DispatchLoaderDynamic;
-#endif
-DispatchLoaderDynamic & ggml_vk_default_dispatcher();
-#define VULKAN_HPP_DEFAULT_DISPATCHER ggml_vk_default_dispatcher()
-
-#include <vulkan/vulkan.hpp>
-
-// Fallback definitions for VK_NV_cooperative_matrix_decode_vector in case the
-// installed Vulkan headers predate the extension.
-#ifndef VK_NV_cooperative_matrix_decode_vector
-#define VK_NV_cooperative_matrix_decode_vector 1
-#define VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME "VK_NV_cooperative_matrix_decode_vector"
-#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV ((VkStructureType)1000689000)
-typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV {
- VkStructureType sType;
- void* pNext;
- VkBool32 cooperativeMatrixDecodeVector;
-} VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV;
-#endif
-
-// SPIR-V Headers: different SDK installations expose different include paths.
-// LunarG Vulkan SDK on Windows typically provides <spirv-headers/spirv.hpp>.
-// Linux packages, MSYS2 and MinGW often use the Khronos layout <spirv/unified1/spirv.hpp>.
-#if __has_include(<spirv/unified1/spirv.hpp>)
-# include <spirv/unified1/spirv.hpp>
-#elif __has_include(<spirv-headers/spirv.hpp>)
-# include <spirv-headers/spirv.hpp>
-#elif __has_include(<spirv.hpp>)
-# include <spirv.hpp>
-#else
- // Fallback to let the compiler throw a standard "file not found" error
-# include <spirv/unified1/spirv.hpp>
-#endif
-
-#include <algorithm>
-#include <cmath>
-#include <iomanip>
-#include <iostream>
-#include <tuple>
-#include <vector>
-#include <deque>
-#include <sstream>
-#include <utility>
-#include <memory>
-#include <limits>
-#include <map>
-#include <set>
-#include <unordered_map>
-#include <shared_mutex>
-#include <mutex>
-#include <future>
-#include <condition_variable>
-#include <thread>
-
-#if defined(_MSC_VER)
-# define NOMINMAX 1
-# include <windows.h>
-# define YIELD() YieldProcessor()
-#elif defined(__clang__) || defined(__GNUC__)
-# if defined(__x86_64__) ||defined(__i386__)
-# include <immintrin.h>
-# define YIELD() _mm_pause()
-# elif defined(__arm__) || defined(__aarch64__)
-# if defined(__clang__)
-# include <arm_acle.h>
-# define YIELD() __yield()
-# else
-# define YIELD() asm volatile("yield")
-# endif
-# endif
-#endif
-
-#if !defined(YIELD)
-#define YIELD()
-#endif
+#include "ggml-vulkan-common.h"
-#include "ggml-impl.h"
-#include "ggml-backend-impl.h"
-
-#include "ggml-vulkan-shaders.hpp"
-
-// On 32-bit platforms, Vulkan non-dispatchable handles such as VkBuffer are represented as uint64_t,
-// and Vulkan-Hpp disables implicit conversions for type safety.
namespace {
inline std::ostream & operator<<(std::ostream & os, vk::Buffer buffer) {
return os << static_cast<VkBuffer>(buffer);
}
}
-
-// remove this once it's more widely available in the SDK
-#if !defined(VK_KHR_shader_bfloat16)
-
-#define VK_KHR_shader_bfloat16 1
-#define VK_KHR_SHADER_BFLOAT16_SPEC_VERSION 1
-#define VK_KHR_SHADER_BFLOAT16_EXTENSION_NAME "VK_KHR_shader_bfloat16"
-#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_BFLOAT16_FEATURES_KHR ((VkStructureType)1000141000)
-#define VK_COMPONENT_TYPE_BFLOAT16_KHR ((VkComponentTypeKHR)1000141000)
-
-typedef struct VkPhysicalDeviceShaderBfloat16FeaturesKHR {
- VkStructureType sType;
- void* pNext;
- VkBool32 shaderBFloat16Type;
- VkBool32 shaderBFloat16DotProduct;
- VkBool32 shaderBFloat16CooperativeMatrix;
-} VkPhysicalDeviceShaderBfloat16FeaturesKHR;
-#endif
-
-#if !defined(VK_VALVE_shader_mixed_float_dot_product)
-#define VK_VALVE_shader_mixed_float_dot_product 1
-#define VK_VALVE_SHADER_MIXED_FLOAT_DOT_PRODUCT_SPEC_VERSION 1
-#define VK_VALVE_SHADER_MIXED_FLOAT_DOT_PRODUCT_EXTENSION_NAME "VK_VALVE_shader_mixed_float_dot_product"
-#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MIXED_FLOAT_DOT_PRODUCT_FEATURES_VALVE ((VkStructureType)1000673000)
-typedef struct VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE {
- VkStructureType sType;
- void* pNext;
- VkBool32 shaderMixedFloatDotProductFloat16AccFloat32;
- VkBool32 shaderMixedFloatDotProductFloat16AccFloat16;
- VkBool32 shaderMixedFloatDotProductBFloat16Acc;
- VkBool32 shaderMixedFloatDotProductFloat8AccFloat32;
-} VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE;
-#endif
-
-#if !defined(VK_EXT_shader_ocp_microscaling_types)
-#define VK_EXT_shader_ocp_microscaling_types 1
-#define VK_EXT_SHADER_OCP_MICROSCALING_TYPES_SPEC_VERSION 1
-#define VK_EXT_SHADER_OCP_MICROSCALING_TYPES_EXTENSION_NAME "VK_EXT_shader_ocp_microscaling_types"
-#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OCP_MICROSCALING_TYPES_FEATURES_EXT ((VkStructureType)1000672000)
-typedef struct VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT {
- VkStructureType sType;
- void* pNext;
- VkBool32 shaderFloat4;
- VkBool32 shaderFloat6;
- VkBool32 shaderFloat8UnsignedE8M0;
- VkBool32 shaderMXInt8;
-} VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT;
-#endif
-
-#if !defined(VK_EXT_shader_float8)
-#define VK_EXT_shader_float8 1
-#define VK_EXT_SHADER_FLOAT8_SPEC_VERSION 1
-#define VK_EXT_SHADER_FLOAT8_EXTENSION_NAME "VK_EXT_shader_float8"
-#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT8_FEATURES_EXT ((VkStructureType)1000567000)
-typedef struct VkPhysicalDeviceShaderFloat8FeaturesEXT {
- VkStructureType sType;
- void* pNext;
- VkBool32 shaderFloat8;
- VkBool32 shaderFloat8CooperativeMatrix;
-} VkPhysicalDeviceShaderFloat8FeaturesEXT;
-#endif
-
-#ifndef VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME
-#define VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME "VK_KHR_internally_synchronized_queues"
-#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INTERNALLY_SYNCHRONIZED_QUEUES_FEATURES_KHR ((VkStructureType)1000504000)
-#define VK_DEVICE_QUEUE_CREATE_INTERNALLY_SYNCHRONIZED_BIT_KHR ((VkDeviceQueueCreateFlagBits)0x00000004)
-
-// Compile-time constant guaranteed; no runtime initialization overhead
-static constexpr vk::DeviceQueueCreateFlagBits eInternallySynchronizedKHR =
- static_cast<vk::DeviceQueueCreateFlagBits>(0x00000004);
-
-typedef struct VkPhysicalDeviceInternallySynchronizedQueuesFeaturesKHR {
- VkStructureType sType;
- void* pNext;
- VkBool32 internallySynchronizedQueues;
-} VkPhysicalDeviceInternallySynchronizedQueuesFeaturesKHR;
-#else
-static constexpr vk::DeviceQueueCreateFlagBits eInternallySynchronizedKHR = vk::DeviceQueueCreateFlagBits::eInternallySynchronizedKHR;
-#endif
-
-#define ROUNDUP_POW2(M, N) (((M) + (N) - 1) & ~((N) - 1))
-#define CEIL_DIV(M, N) (((M) / (N)) + (((M) % (N)) != 0))
-static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; }
-
-#define VK_VENDOR_ID_AMD 0x1002
-#define VK_VENDOR_ID_APPLE 0x106b
-#define VK_VENDOR_ID_INTEL 0x8086
-#define VK_VENDOR_ID_NVIDIA 0x10de
-#define VK_VENDOR_ID_QUALCOMM 0x5143
-
-#define VK_DEVICE_DESCRIPTOR_POOL_SIZE 256
-
-#define VK_CHECK(err, msg, dev) \
- do { \
- vk::Result err_; \
- try { \
- err_ = (err); \
- } catch (vk::DeviceLostError &) { \
- ggml_vk_print_device_lost_info(dev); \
- GGML_LOG_ERROR("ggml_vulkan: %s at %s:%d\n", \
- #err, __FILE__, __LINE__); \
- throw; \
- } \
- if (err_ != vk::Result::eSuccess) { \
- GGML_LOG_ERROR("ggml_vulkan: %s error %s at %s:%d\n", \
- #err, to_string(err_).c_str(), __FILE__, __LINE__); \
- throw vk::SystemError(vk::make_error_code(err_), \
- "ggml_vulkan: " msg); \
- } \
- } while (0)
-
-#ifdef GGML_VULKAN_DEBUG
-#define VK_LOG_DEBUG(msg) std::cerr << msg << std::endl
-#else
-#define VK_LOG_DEBUG(msg) ((void) 0)
-#endif // GGML_VULKAN_DEBUG
-
-struct ggml_backend_vk_context;
-
-#define MAX_PARAMETER_COUNT 12
-// Max number of adds that can be fused without exceeding MAX_PARAMETER_COUNT.
-#define MAX_FUSED_ADDS (MAX_PARAMETER_COUNT - 3)
-
-typedef std::shared_ptr<struct vk_pipeline_struct> vk_pipeline;
-
-struct vk_pipeline_struct {
- std::string name;
- vk::ShaderModule shader_module;
- vk::PipelineLayout layout;
- vk::Pipeline pipeline;
- uint32_t push_constant_size;
- uint32_t parameter_count;
- std::array<uint32_t, 3> wg_denoms;
- uint32_t align;
- // true if fields have been set by ggml_vk_create_pipeline
- bool initialized {};
- // true while a compile is in flight, used to dedupe concurrent claims.
- // Protected by device->compile_mutex.
- bool compile_pending {};
- // set to true when the shader has been compiled
- std::atomic<bool> compiled {};
- // number of registers used, extracted from pipeline executable properties
- uint32_t register_count {};
-
-#if defined(VK_EXT_shader_64bit_indexing)
- bool is_64b_indexing {};
-#endif
- // linked list of pipelines for multiple compilation variants.
- // currently only used to compile a 64-bit indexing variant.
- vk_pipeline next;
-};
-
-typedef std::weak_ptr<vk_pipeline_struct> vk_pipeline_ref;
-
-static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline);
-
-struct vk_matmul_pipeline_key {
- ggml_type type_a;
- ggml_type type_b;
- bool mul_mat_id;
- bool f16acc;
-
- bool operator<(const vk_matmul_pipeline_key & o) const {
- return std::tie(type_a, type_b, mul_mat_id, f16acc)
- < std::tie(o.type_a, o.type_b, o.mul_mat_id, o.f16acc);
- }
-};
-
-struct vk_matmul_pipeline_pair {
- vk_pipeline unaligned;
- vk_pipeline aligned;
- uint32_t align;
-};
-
-struct vk_tile_config {
- std::vector<uint32_t> warptile;
- std::array<uint32_t, 3> wg_denoms;
- uint32_t align;
-};
-
-using matmul_tile_selector_t = std::function<uint32_t(
- uint32_t m, uint32_t n, uint32_t k, uint32_t shader_core_count,
- const std::vector<vk_matmul_pipeline_pair>& configs)>;
-
-struct vk_device_struct;
-typedef std::shared_ptr<vk_device_struct> vk_device;
-typedef std::weak_ptr<vk_device_struct> vk_device_ref;
-
-struct vk_buffer_struct;
-typedef std::shared_ptr<vk_buffer_struct> vk_buffer;
-typedef std::weak_ptr<vk_buffer_struct> vk_buffer_ref;
-
-struct ggml_backend_vk_buffer_type_context {
- std::string name;
- vk_device device;
-};
-
-struct vk_queue;
-
-struct vk_command_buffer {
- vk::CommandBuffer buf;
- uint64_t use_counter = 0;
- bool in_use = false;
-};
-
-// Stores command pool/buffers. There's an instance of this
-// for each (context,queue) pair and for each (device,queue) pair.
-struct vk_command_pool {
- void init(vk_device& device, vk_queue *q_);
- void destroy(vk::Device& device);
-
- vk::CommandPool pool;
- // Using deque so the pointers to command buffers
- // remain valid even if we add more
- std::deque<vk_command_buffer> cmd_buffers;
-
- vk_queue *q;
-
- size_t buffers_in_use() const {
- return std::count_if(cmd_buffers.begin(), cmd_buffers.end(),
- [](const auto& cb) { return cb.in_use; });
- }
-};
-
-static void ggml_vk_print_device_fault_info(const vk_device& device);
-static void ggml_vk_print_device_lost_info(const vk_device& device);
-
-// Prevent simultaneous submissions to the same queue.
-struct vk_queue_handle {
- vk::Queue queue;
- vk_device_ref device;
- std::mutex * device_submit_mutex = nullptr;
- virtual void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) = 0;
- virtual void lock() {} // no-op by default (internally synchronized case)
- virtual void unlock() {}
- virtual ~vk_queue_handle() = default;
-};
-
-struct vk_queue_handle_synchronized : vk_queue_handle {
- std::mutex mutex;
- void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override {
- // Workaround for NVIDIA driver bug
- std::unique_lock<std::mutex> device_guard;
- if (device_submit_mutex) {
- device_guard = std::unique_lock<std::mutex>(*device_submit_mutex);
- }
- std::lock_guard<std::mutex> guard(mutex);
- try {
- queue.submit(submits, fence);
- } catch (vk::DeviceLostError &) {
- if (auto dev = device.lock()) {
- ggml_vk_print_device_lost_info(dev);
- }
- throw;
- }
- }
- void lock() override { mutex.lock(); }
- void unlock() override { mutex.unlock(); }
-};
-
-// Driver guarantees internal synchronization via VK_KHR_internally_synchronized_queues
-struct vk_queue_handle_unsynchronized : vk_queue_handle {
- void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override {
- // Workaround for NVIDIA driver bug
- std::unique_lock<std::mutex> device_guard;
- if (device_submit_mutex) {
- device_guard = std::unique_lock<std::mutex>(*device_submit_mutex);
- }
- try {
- queue.submit(submits, fence);
- } catch (vk::DeviceLostError &) {
- if (auto dev = device.lock()) {
- ggml_vk_print_device_lost_info(dev);
- }
- throw;
- }
- }
- // lock()/unlock() inherited no-ops
-};
-
-struct vk_queue {
- uint32_t queue_family_index;
- std::shared_ptr<vk_queue_handle> handle;
-
- vk_command_pool cmd_pool;
-
- vk::PipelineStageFlags stage_flags;
-
- bool transfer_only;
-};
-
-static const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft);
-static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);
-static size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft);
-static size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft);
-static size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor);
-static ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = {
- /* .get_name = */ ggml_backend_vk_buffer_type_name,
- /* .alloc_buffer = */ ggml_backend_vk_buffer_type_alloc_buffer,
- /* .get_alignment = */ ggml_backend_vk_buffer_type_get_alignment,
- /* .get_max_size = */ ggml_backend_vk_buffer_type_get_max_size,
- /* .get_alloc_size = */ ggml_backend_vk_buffer_type_get_alloc_size,
- /* .is_host = */ NULL,
-};
-
-class vk_memory_logger;
-class vk_perf_logger;
-static void ggml_vk_destroy_buffer(vk_buffer& buf);
-static void ggml_vk_synchronize(ggml_backend_vk_context * ctx);
-
-static constexpr uint32_t mul_mat_vec_max_cols = 8;
-static constexpr uint32_t p021_max_gqa_ratio = 8;
-
-enum vk_device_architecture {
- OTHER,
- AMD_GCN,
- AMD_RDNA1,
- AMD_RDNA2,
- AMD_RDNA3,
- INTEL_XE1,
- INTEL_XE2,
- NVIDIA_PRE_TURING,
- NVIDIA_TURING,
-};
-
static vk_device_architecture get_device_architecture(const vk::PhysicalDevice& device) {
vk::PhysicalDeviceProperties props = device.getProperties();
@@ -547,706 +125,10 @@ static vk_device_architecture get_device_architecture(const vk::PhysicalDevice&
return vk_device_architecture::OTHER;
}
-enum vk_conv_shapes {
- CONV_SHAPE_128x128,
- CONV_SHAPE_64x32,
- CONV_SHAPE_32x256,
- CONV_SHAPE_64x128,
- CONV_SHAPE_COUNT,
-};
-
-struct vk_conv_block_size {
- uint32_t K;
- uint32_t NPQ;
- uint32_t CRS;
-};
-
-vk_conv_block_size vk_conv_block_sizes[CONV_SHAPE_COUNT] = {
- // K NPQ CRS
- { 128, 128, 16 }, // CONV_SHAPE_128x128
- { 64, 32, 32 }, // CONV_SHAPE_64x32
- { 32, 256, 16 }, // CONV_SHAPE_32x256
- { 64, 128, 16 }, // CONV_SHAPE_64x128
-};
-
-enum dmmv_wg_sizes {
- DMMV_WG_SIZE_SUBGROUP,
- DMMV_WG_SIZE_LARGE,
- DMMV_WG_SIZE_COUNT,
-};
-
-enum FaCodePath {
- FA_SCALAR,
- FA_COOPMAT1,
- FA_COOPMAT2,
-};
-
-struct vk_fa_pipeline_state {
- uint32_t HSK, HSV;
- uint32_t Br, Bc;
- uint32_t D_split, row_split;
- bool shmem_staging;
- FaCodePath path;
- uint32_t workgroup_size, subgroup_size;
- bool aligned;
- bool f32acc;
- uint32_t flags;
- uint32_t limit_occupancy_shmem;
- ggml_type k_type;
- ggml_type v_type;
-
- bool operator<(const vk_fa_pipeline_state &b) const {
- return std::tie(HSK, HSV, Br, Bc, D_split, row_split, shmem_staging, path, workgroup_size, subgroup_size, aligned, f32acc, flags, limit_occupancy_shmem, k_type, v_type) <
- std::tie(b.HSK, b.HSV, b.Br, b.Bc, b.D_split, b.row_split, b.shmem_staging, b.path, b.workgroup_size, b.subgroup_size, b.aligned, b.f32acc, b.flags, b.limit_occupancy_shmem, b.k_type, b.v_type);
- }
-};
-
-struct vk_conv2d_pipeline_state {
- vk_conv2d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t p0, uint32_t p1, uint32_t d0, uint32_t d1, uint32_t KW, uint32_t KH, uint32_t aligned)
- : s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), KW(KW), KH(KH), aligned(aligned) {}
-
- uint32_t s0, s1, p0, p1, d0, d1, KW, KH;
- // when set, shader can skip K/CRS/NPQ bounds checks and address clamps
- uint32_t aligned;
-
- bool operator<(const vk_conv2d_pipeline_state &b) const {
- return std::tie(s0, s1, p0, p1, d0, d1, KW, KH, aligned) <
- std::tie(b.s0, b.s1, b.p0, b.p1, b.d0, b.d1, b.KW, b.KH, b.aligned);
- }
-};
-
-struct vk_conv3d_pipeline_state {
- vk_conv3d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t s2, uint32_t p0, uint32_t p1, uint32_t p2,
- uint32_t d0, uint32_t d1, uint32_t d2, uint32_t KW, uint32_t KH, uint32_t KD, uint32_t aligned)
- : s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), KW(KW), KH(KH), KD(KD), aligned(aligned) {}
-
- uint32_t s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD;
- uint32_t aligned;
-
- bool operator<(const vk_conv3d_pipeline_state &b) const {
- return std::tie(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned) <
- std::tie(b.s0, b.s1, b.s2, b.p0, b.p1, b.p2, b.d0, b.d1, b.d2, b.KW, b.KH, b.KD, b.aligned);
- }
-};
-
-struct vk_solve_tri_pipeline_state {
- vk_solve_tri_pipeline_state(uint32_t N, uint32_t K)
- : N(N), K(K) {}
-
- uint32_t N, K;
-
- bool operator<(const vk_solve_tri_pipeline_state &b) const {
- return std::tie(N, K) <
- std::tie(b.N, b.K);
- }
-};
-
-enum shader_reduction_mode {
- SHADER_REDUCTION_MODE_SHMEM,
- SHADER_REDUCTION_MODE_HYBRID,
- SHADER_REDUCTION_MODE_SUBGROUP,
- SHADER_REDUCTION_MODE_COUNT,
-};
-
-// argsort pipelines for up to 1<<10 invocations per workgroup
-static constexpr uint32_t num_argsort_pipelines = 11;
-static constexpr uint32_t num_topk_moe_pipelines = 10;
-static constexpr uint32_t num_topk_pipelines = 11;
-
-static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax_norm{ GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT,
- GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_RESHAPE,
- GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV,
- GGML_OP_RESHAPE };
-
-static constexpr std::initializer_list<ggml_op> topk_moe_sigmoid_norm_bias{ GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD,
- GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS,
- GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP,
- GGML_OP_DIV, GGML_OP_RESHAPE };
-
-static constexpr std::initializer_list<ggml_op> topk_moe_sqrt_softplus_norm_bias{ GGML_OP_UNARY, GGML_OP_SQRT,
- GGML_OP_RESHAPE, GGML_OP_ADD,
- GGML_OP_ARGSORT, GGML_OP_VIEW,
- GGML_OP_GET_ROWS, GGML_OP_RESHAPE,
- GGML_OP_SUM_ROWS, GGML_OP_CLAMP,
- GGML_OP_DIV, GGML_OP_RESHAPE };
-
-static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax { GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT,
- GGML_OP_VIEW, GGML_OP_GET_ROWS };
-
-static constexpr std::initializer_list<ggml_op> topk_moe_late_softmax { GGML_OP_ARGSORT, GGML_OP_VIEW,
- GGML_OP_GET_ROWS, GGML_OP_RESHAPE,
- GGML_OP_SOFT_MAX, GGML_OP_RESHAPE };
-
-// Snake activation: y = x + sin(a*x)^2 * inv_b. Used by the optimize_graph reorder
-// pass so it keeps the chain contiguous and by the dispatcher to detect the fusion.
-static constexpr std::initializer_list<ggml_op> snake_pattern { GGML_OP_MUL, GGML_OP_SIN,
- GGML_OP_SQR, GGML_OP_MUL,
- GGML_OP_ADD };
-
-// qwen4 QSA indexer: gather per-block scores to cells + add f16 mask (cast+reshape) + top-k,
-// fused into one radix-select. The cast/reshape are elided; the raw f16 mask is read in-shader.
-static constexpr std::initializer_list<ggml_op> topk_qsa_pattern { GGML_OP_GET_ROWS, GGML_OP_PERMUTE,
- GGML_OP_CONT, GGML_OP_CPY,
- GGML_OP_RESHAPE, GGML_OP_ADD,
- GGML_OP_TOP_K };
-static constexpr std::initializer_list<std::array<int, 3>> topk_qsa_edges {
- { 1, 0, 0 }, // permute->src[0] == get_rows
- { 2, 0, 1 }, // cont->src[0] == permute
- { 4, 0, 3 }, // reshape->src[0] == cpy (mask cast)
- { 5, 0, 2 }, // add->src[0] == cont
- { 5, 1, 4 }, // add->src[1] == reshape
- { 6, 0, 5 }, // top_k->src[0] == add
-};
-static constexpr std::initializer_list<ggml_op> rms_norm_mul_add_mul_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD, GGML_OP_MUL };
-static constexpr std::initializer_list<ggml_op> rms_norm_mul_add_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD };
-static constexpr std::initializer_list<ggml_op> rms_norm_mul_rope_view_set_rows_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS };
-static constexpr std::initializer_list<ggml_op> rms_norm_view_set_rows_pattern { GGML_OP_RMS_NORM, GGML_OP_VIEW, GGML_OP_SET_ROWS };
-static constexpr std::initializer_list<ggml_op> rope_view_set_rows_pattern { GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS };
-
-//node #978 ( SOFT_MAX): ffn_moe_probs-15 ( 0K) [Vulka ] use=2: ffn_moe_logits-15 ( 0K) [Vulka ]
-//node #979 ( RESHAPE): ffn_moe_probs-15 (re ( 0K) [Vulka ] use=1: ffn_moe_probs-15 ( 0K) [Vulka ]
-//node #980 ( ARGSORT): ffn_moe_argsort-15 ( 0K) [Vulka ] use=1: ffn_moe_probs-15 ( 0K) [Vulka ]
-//node #981 ( VIEW): ffn_moe_topk-15 ( 0K) [Vulka ] use=4: ffn_moe_argsort-15 ( 0K) [Vulka ]
-//node #982 ( GET_ROWS): ffn_moe_weights-15 ( 0K) [Vulka ] use=1: ffn_moe_probs-15 (re ( 0K) [Vulka ] ffn_moe_topk-15 ( 0K) [Vulka ]
-//node #983 ( RESHAPE): ffn_moe_weights-15 ( ( 0K) [Vulka ] use=2: ffn_moe_weights-15 ( 0K) [Vulka ]
-//node #984 ( SUM_ROWS): ffn_moe_weights_sum- ( 0K) [Vulka ] use=1: ffn_moe_weights-15 ( ( 0K) [Vulka ]
-//node #985 ( CLAMP): ffn_moe_weights_sum_ ( 0K) [Vulka ] use=1: ffn_moe_weights_sum- ( 0K) [Vulka ]
-//node #986 ( DIV): ffn_moe_weights_norm ( 0K) [Vulka ] use=1: ffn_moe_weights-15 ( ( 0K) [Vulka ] ffn_moe_weights_sum_ ( 0K) [Vulka ]
-//node #987 ( RESHAPE): ffn_moe_weights_norm ( 0K) [Vulka ] use=1: ffn_moe_weights_norm ( 0K) [Vulka ]
-static constexpr std::initializer_list<std::array<int, 3>> topk_moe_early_softmax_norm_edges {
- { 1, 0, 0 }, // reshape->src[0] == softmax
- { 2, 0, 0 }, // argsort->src[0] == softmax
- { 3, 0, 2 }, // view->src[0] == argsort
- { 4, 0, 1 }, // get_rows->src[0] == reshape
- { 4, 1, 3 }, // get_rows->src[1] == view
- { 5, 0, 4 }, // reshape->src[0] == get_rows
- { 6, 0, 5 }, // sum_rows->src[0] == reshape
- { 7, 0, 6 }, // clamp->src[0] == sum_rows
- { 8, 0, 5 }, // div->src[0] == reshape
- { 8, 1, 7 }, // div->src[1] == clamp
- { 9, 0, 8 }, // reshape->src[0] == div
-};
-
-//node #436 ( UNARY): ffn_moe_probs-10 ( 256K) [Vulka ] use=2: ffn_moe_logits-10 ( 256K) [Vulka ]
-//node #437 ( RESHAPE): ffn_moe_probs-10 (re ( 256K) [Vulka ] use=1: ffn_moe_probs-10 ( 256K) [Vulka ]
-//node #438 ( ADD): ffn_moe_probs_biased ( 256K) [Vulka ] use=1: ffn_moe_probs-10 ( 256K) [Vulka ] blk.10.exp_probs_b.b ( 0K) [Vulka ]
-//node #439 ( ARGSORT): ffn_moe_argsort-10 ( 256K) [Vulka ] use=1: ffn_moe_probs_biased ( 256K) [Vulka ]
-//node #440 ( VIEW): ffn_moe_topk-10 ( 255K) [Vulka ] use=3: ffn_moe_argsort-10 ( 256K) [Vulka ]
-//node #441 ( GET_ROWS): ffn_moe_weights-10 ( 12K) [Vulka ] use=1: ffn_moe_probs-10 (re ( 256K) [Vulka ] ffn_moe_topk-10 ( 255K) [Vulka ]
-//node #442 ( RESHAPE): ffn_moe_weights-10 ( ( 12K) [Vulka ] use=2: ffn_moe_weights-10 ( 12K) [Vulka ]
-//node #443 ( SUM_ROWS): ffn_moe_weights_sum- ( 2K) [Vulka ] use=1: ffn_moe_weights-10 ( ( 12K) [Vulka ]
-//node #444 ( CLAMP): ffn_moe_weights_sum_ ( 2K) [Vulka ] use=1: ffn_moe_weights_sum- ( 2K) [Vulka ]
-//node #445 ( DIV): ffn_moe_weights_norm ( 12K) [Vulka ] use=1: ffn_moe_weights-10 ( ( 12K) [Vulka ] ffn_moe_weights_sum_ ( 2K) [Vulka ]
-//node #446 ( RESHAPE): ffn_moe_weights_norm ( 12K) [Vulka ] use=1: ffn_moe_weights_norm ( 12K) [Vulka ]
-static constexpr std::initializer_list<std::array<int, 3>> topk_moe_sigmoid_norm_bias_edges {
- { 1, 0, 0 }, // reshape->src[0] == sigmoid
- { 2, 0, 0 }, // add->src[0] == sigmoid
- { 3, 0, 2 }, // argsort->src[0] == add
- { 4, 0, 3 }, // view->src[0] == argsort
- { 5, 0, 1 }, // get_rows->src[0] == reshape
- { 5, 1, 4 }, // get_rows->src[1] == view
- { 6, 0, 5 }, // reshape->src[0] == get_rows
- { 7, 0, 6 }, // sum_rows->src[0] == reshape
- { 8, 0, 7 }, // clamp->src[0] == sum_rows
- { 9, 0, 6 }, // div->src[0] == reshape
- { 9, 1, 8 }, // div->src[1] == clamp
- {10, 0, 9 }, // reshape->src[0] == div
-};
-
-static constexpr std::initializer_list<std::array<int, 3>> topk_moe_sqrt_softplus_norm_bias_edges {
- { 1, 0, 0 }, // sqrt->src[0] == softplus
- { 2, 0, 1 }, // reshape->src[0] == sqrt
- { 3, 0, 1 }, // add->src[0] == sqrt
- { 4, 0, 3 }, // argsort->src[0] == add
- { 5, 0, 4 }, // view->src[0] == argsort
- { 6, 0, 2 }, // get_rows->src[0] == reshape
- { 6, 1, 5 }, // get_rows->src[1] == view
- { 7, 0, 6 }, // reshape->src[0] == get_rows
- { 8, 0, 7 }, // sum_rows->src[0] == reshape
- { 9, 0, 8 }, // clamp->src[0] == sum_rows
- {10, 0, 7 }, // div->src[0] == reshape
- {10, 1, 9 }, // div->src[1] == clamp
- {11, 0,10 }, // reshape->src[0] == div
-};
-
-// same as early_softmax_norm but ending after the get_rows
-static constexpr std::initializer_list<std::array<int, 3>> topk_moe_early_softmax_edges {
- { 1, 0, 0 }, // reshape->src[0] == softmax
- { 2, 0, 0 }, // argsort->src[0] == softmax
- { 3, 0, 2 }, // view->src[0] == argsort
- { 4, 0, 1 }, // get_rows->src[0] == reshape
- { 4, 1, 3 }, // get_rows->src[1] == view
-};
-
-//node #652 ( ARGSORT): ffn_moe_argsort-11 ( 0K) [Vulka ] use=1: ffn_moe_probs-11 ( 0K) [Vulka ]
-//node #653 ( VIEW): ffn_moe_topk-11 ( 0K) [Vulka ] use=7: ffn_moe_argsort-11 ( 0K) [Vulka ]
-//node #654 ( GET_ROWS): ffn_moe_weights-11 ( 0K) [Vulka ] use=1: ffn_moe_probs-11 (re ( 0K) [Vulka ] ffn_moe_topk-11 ( 0K) [Vulka ]
-//node #655 ( RESHAPE): ffn_moe_weights-11 ( ( 0K) [Vulka ] use=1: ffn_moe_weights-11 ( 0K) [Vulka ]
-//node #656 ( SOFT_MAX): node_656 ( 0K) [Vulka ] use=1: ffn_moe_weights-11 ( ( 0K) [Vulka ]
-//node #657 ( RESHAPE): ffn_moe_weights_soft ( 0K) [Vulka ] use=1: node_656 ( 0K) [Vulka ]
-static constexpr std::initializer_list<std::array<int, 3>> topk_moe_late_softmax_edges {
- { 1, 0, 0 }, // view->src[0] == argsort
- { 2, 1, 1 }, // get_rows->src[1] == view
- { 3, 0, 2 }, // reshape->src[0] == get_rows
- { 4, 0, 3 }, // soft_max->src[0] == reshape
- { 5, 0, 4 }, // reshape->src[0] == soft_max
-};
-
-enum topk_moe_mode {
- TOPK_MOE_EARLY_SOFTMAX,
- TOPK_MOE_EARLY_SOFTMAX_NORM,
- TOPK_MOE_LATE_SOFTMAX,
- TOPK_MOE_SIGMOID_NORM_BIAS,
- TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS,
- TOPK_MOE_COUNT,
-};
-
-enum rms_norm_mode {
- RMS_NORM_MUL,
- RMS_NORM_MUL_ADD,
- RMS_NORM_MUL_ADD_MUL,
- RMS_NORM_MUL_ROPE,
- RMS_NORM_MUL_ROPE_VIEW_SET_ROWS,
- RMS_NORM_VIEW_SET_ROWS,
- RMS_NORM_COUNT,
-};
-
-static constexpr std::initializer_list<std::array<int, 3>> rope_view_set_rows_edges {
- { 1, 0, 0 }, // view->src[0] == rope
- { 2, 0, 1 }, // set_rows->src[0] == view
-};
-
-static constexpr std::initializer_list<std::array<int, 3>> rms_norm_mul_rope_view_set_rows_edges {
- { 1, 0, 0 }, // mul->src[0] == rms
- { 2, 0, 1 }, // rope->src[0] == mul
- { 3, 0, 2 }, // view->src[0] == rope
- { 4, 0, 3 }, // set_rows->src[0] == view
-};
-
-static constexpr std::initializer_list<std::array<int, 3>> rms_norm_view_set_rows_edges {
- { 1, 0, 0 }, // view->src[0] == rms_norm
- { 2, 0, 1 }, // set_rows->src[0] == view
-};
-
-static constexpr std::array<ggml_type, 9> lightning_indexer_k_types = {
- GGML_TYPE_F32,
- GGML_TYPE_F16,
- GGML_TYPE_BF16,
- GGML_TYPE_Q8_0,
- GGML_TYPE_Q5_1,
- GGML_TYPE_Q5_0,
- GGML_TYPE_Q4_1,
- GGML_TYPE_Q4_0,
- GGML_TYPE_IQ4_NL,
-};
-
-static bool ggml_vk_lightning_indexer_k_type_supported(ggml_type type) {
+bool ggml_vk_lightning_indexer_k_type_supported(ggml_type type) {
return std::find(lightning_indexer_k_types.begin(), lightning_indexer_k_types.end(), type) != lightning_indexer_k_types.end();
}
-
-struct vk_device_struct {
- std::recursive_mutex mutex;
- std::mutex queue_submit_mutex;
- mutable std::shared_mutex pinned_memory_mutex;
-
- // Guards compile_pending, all_pipelines, and the dynamic pipeline maps
- // (flash_attn, fa_mask_opt, solve_tri, conv2d, etc). The actual compile
- // runs with no lock held, so different pipelines can compile in parallel.
- // Lock order is device->mutex -> compile_mutex, never the reverse.
- std::mutex compile_mutex;
- std::condition_variable compile_cv;
-
- uint32_t debug_cmdbuf_idx {};
-
- vk::PhysicalDevice physical_device;
- vk::PhysicalDeviceProperties properties;
- std::string name;
- uint64_t max_memory_allocation_size;
- uint64_t max_buffer_size;
- uint64_t suballocation_block_size;
- uint64_t min_imported_host_pointer_alignment;
- bool external_memory_host {};
- bool fp16;
- bool bf16;
- bool pipeline_robustness;
- bool memory_priority;
- vk::Device device;
- uint32_t vendor_id;
- vk::DriverId driver_id;
- vk_device_architecture architecture;
- std::unique_ptr<vk_queue> compute_queue;
- std::unique_ptr<vk_queue> transfer_queue;
- bool single_queue;
- bool support_async;
- bool async_use_transfer_queue;
- bool has_internally_synchronized_queues = false;
- uint32_t subgroup_size;
- uint32_t subgroup_size_log2;
- uint32_t shader_core_count;
- bool uma;
- bool prefer_host_memory;
- bool float_controls_rte_fp16;
- bool float_controls_denorm_preserve_fp16;
- bool subgroup_basic;
- bool subgroup_arithmetic;
- bool subgroup_shuffle;
- bool subgroup_ballot;
- bool subgroup_clustered;
- bool subgroup_vote;
- bool multi_add;
- bool shader_int64;
- bool buffer_device_address;
- bool vulkan_memory_model;
-
- bool add_rms_fusion;
- uint32_t partials_binding_alignment;
- uint32_t max_nodes_per_submit;
-
- bool shader_64b_indexing;
-
- bool integer_dot_product;
- // 0: default, 1: force mmvq, -1: disable mmvq
- int32_t mmvq_mode;
-
- bool subgroup_size_control;
- uint32_t subgroup_min_size;
- uint32_t subgroup_max_size;
- bool subgroup_require_full_support;
-
- // floor(log2(maxComputeWorkGroupInvocations))
- uint32_t max_workgroup_size_log2 {};
-
- bool coopmat_support;
- bool coopmat_acc_f32_support {};
- bool coopmat_acc_f16_support {};
- bool coopmat_bf16_support {};
- bool coopmat_support_16x16x16_f16acc {};
- bool coopmat_support_16x16x16_f32acc {};
- bool coopmat1_fa_support {};
- uint32_t coopmat_m;
- uint32_t coopmat_n;
- uint32_t coopmat_k;
-
- bool coopmat_int_support;
- uint32_t coopmat_int_m;
- uint32_t coopmat_int_n;
- uint32_t coopmat_int_k;
-
- bool coopmat2;
- bool coopmat2_bf16_support {};
- bool coopmat2_decode_vector;
-
- bool dot2_f16 {};
- bool ocp_fp4 {};
-
- bool pipeline_executable_properties_support {};
-
- bool device_fault {};
- PFN_vkGetDeviceFaultInfoEXT pfn_vkGetDeviceFaultInfoEXT {};
-
- bool serialize_submissions {};
-
- const ggml_cgraph * diag_cgraph {};
- int diag_prev_start = -1;
- int diag_prev_end = -1;
-
- size_t idx;
-
- bool mul_mat_l[GGML_TYPE_COUNT];
- bool mul_mat_m[GGML_TYPE_COUNT];
- bool mul_mat_s[GGML_TYPE_COUNT];
- bool mul_mat_id_l[GGML_TYPE_COUNT];
- bool mul_mat_id_m[GGML_TYPE_COUNT];
- bool mul_mat_id_s[GGML_TYPE_COUNT];
-
- // Separate flags for the q8_1 (integer dot) mmq path, whose shader uses
- // a different shared-memory layout than the float matmul shaders.
- bool mul_mat_l_int[GGML_TYPE_COUNT];
- bool mul_mat_m_int[GGML_TYPE_COUNT];
- bool mul_mat_s_int[GGML_TYPE_COUNT];
- bool mul_mat_id_l_int[GGML_TYPE_COUNT];
- bool mul_mat_id_m_int[GGML_TYPE_COUNT];
- bool mul_mat_id_s_int[GGML_TYPE_COUNT];
-
- vk::DescriptorSetLayout dsl;
-
- std::map<vk_matmul_pipeline_key, std::vector<vk_matmul_pipeline_pair>> pipeline_matmul;
- matmul_tile_selector_t matmul_tile_selector;
- matmul_tile_selector_t matmul_id_tile_selector;
-
- vk_pipeline pipeline_matmul_split_k_reduce;
- vk_pipeline pipeline_quantize_q8_1_x4;
-
- vk_pipeline pipeline_dequant[GGML_TYPE_COUNT];
- vk_pipeline pipeline_dequant_transpose[GGML_TYPE_COUNT]; // fused dequant+transpose for FA quant-KV
- vk_pipeline pipeline_dequant_mul_mat_vec_f32_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols];
- vk_pipeline pipeline_dequant_mul_mat_vec_f16_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols];
- vk_pipeline pipeline_dequant_mul_mat_vec_id_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT];
-
- vk_pipeline pipeline_dequant_mul_mat_vec_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols];
- vk_pipeline pipeline_dequant_mul_mat_vec_id_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT];
-
- vk_pipeline pipeline_mul_mat_vec_p021_f16_f32[p021_max_gqa_ratio];
- vk_pipeline pipeline_mul_mat_vec_nc_f16_f32;
- vk_pipeline pipeline_get_rows[GGML_TYPE_COUNT];
- vk_pipeline pipeline_get_rows_f32[GGML_TYPE_COUNT];
- vk_pipeline pipeline_get_rows_back_f32;
- vk_pipeline pipeline_acc_f32;
- vk_pipeline pipeline_set_f32;
-
- // [src0 0=fp32,1=fp16][src1 0=fp32,1=fp16][dst 0=fp32,1=fp16]
- vk_pipeline pipeline_add[2][2][2];
- vk_pipeline pipeline_add_norepeat[2][2][2];
- vk_pipeline pipeline_sub[2][2][2];
- vk_pipeline pipeline_sub_norepeat[2][2][2];
- vk_pipeline pipeline_mul[2][2][2];
- vk_pipeline pipeline_mul_norepeat[2][2][2];
- vk_pipeline pipeline_div[2][2][2];
- vk_pipeline pipeline_div_norepeat[2][2][2];
- vk_pipeline pipeline_add_rms[2][2][2];
- vk_pipeline pipeline_add_rms_norepeat[2][2][2];
-
- // indexed by num_additional_fused_ops == num_adds - 1
- vk_pipeline pipeline_multi_add[MAX_FUSED_ADDS];
- vk_pipeline pipeline_multi_add_rms[MAX_FUSED_ADDS];
-
- vk_pipeline pipeline_add_id_f32;
-
- vk_pipeline pipeline_concat_i8, pipeline_concat_i16, pipeline_concat_i32, pipeline_concat_i64;
- vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32;
- vk_pipeline pipeline_scale_f32;
- vk_pipeline pipeline_log[2];
- vk_pipeline pipeline_tri[2];
- vk_pipeline pipeline_diag[2];
- vk_pipeline pipeline_clamp[2];
- vk_pipeline pipeline_pad_f32;
- vk_pipeline pipeline_pad_reflect_1d_f32;
- vk_pipeline pipeline_roll_f32;
- vk_pipeline pipeline_repeat_i32, pipeline_repeat_back_f32;
- vk_pipeline pipeline_repeat_i16;
- vk_pipeline pipeline_cpy_f32_f32, pipeline_cpy_f32_f16, pipeline_cpy_f16_f16, pipeline_cpy_f16_f32, pipeline_cpy_f32_bf16, pipeline_cpy_bf16_f32, pipeline_cpy_f32_i32, pipeline_cpy_i32_f32;
- vk_pipeline pipeline_contig_cpy_f32_f32, pipeline_contig_cpy_f32_f16, pipeline_contig_cpy_f16_f16, pipeline_contig_cpy_f16_f32, pipeline_contig_cpy_f32_bf16, pipeline_contig_cpy_bf16_f32, pipeline_contig_cpy_f32_i32, pipeline_contig_cpy_i32_f32;
- vk_pipeline pipeline_cpy_f32_quant[GGML_TYPE_COUNT];
- vk_pipeline pipeline_cpy_quant_f32[GGML_TYPE_COUNT];
- vk_pipeline pipeline_cpy_transpose_16, pipeline_cpy_transpose_32;
- vk_pipeline pipeline_cpy_transpose_02_16, pipeline_cpy_transpose_02_32;
- // [src0 0=fp32,1=fp16][dst]
- vk_pipeline pipeline_set_rows_i32[2][GGML_TYPE_COUNT];
- vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT];
- vk_pipeline pipeline_norm_f32;
- vk_pipeline pipeline_group_norm_f32;
- vk_pipeline pipeline_rms_norm_f32;
- vk_pipeline pipeline_rms_norm_mul_f32;
- vk_pipeline pipeline_rms_norm_mul_add_f32;
- vk_pipeline pipeline_rms_norm_mul_add_mul_f32;
- vk_pipeline pipeline_rms_norm_mul_add_partials_f32;
- vk_pipeline pipeline_rms_norm_mul_add_mul_partials_f32;
- vk_pipeline pipeline_rms_norm_set_rows_f32_f32;
- vk_pipeline pipeline_rms_norm_set_rows_f32_f16;
- vk_pipeline pipeline_rms_norm_partials_f32;
- vk_pipeline pipeline_rms_norm_mul_partials_f32;
- vk_pipeline pipeline_rms_norm_mul_rope_f32_f32;
- vk_pipeline pipeline_rms_norm_mul_rope_f32_f16;
- vk_pipeline pipeline_rms_norm_back_f32;
- vk_pipeline pipeline_l2_norm_f32;
-
- // [src/dst 0=fp32,1=fp16]
- vk_pipeline pipeline_exp[2];
- vk_pipeline pipeline_expm1[2];
- vk_pipeline pipeline_elu[2];
- vk_pipeline pipeline_gelu[2];
- vk_pipeline pipeline_gelu_erf[2];
- vk_pipeline pipeline_gelu_quick[2];
- vk_pipeline pipeline_silu[2];
- vk_pipeline pipeline_relu[2];
- vk_pipeline pipeline_sqr[2];
- vk_pipeline pipeline_sqrt[2];
- vk_pipeline pipeline_sin[2];
- vk_pipeline pipeline_cos[2];
- vk_pipeline pipeline_xielu[2];
- vk_pipeline pipeline_neg[2];
- vk_pipeline pipeline_tanh[2];
- vk_pipeline pipeline_sigmoid[2];
- vk_pipeline pipeline_hardsigmoid[2];
- vk_pipeline pipeline_hardswish[2];
- vk_pipeline pipeline_abs[2];
- vk_pipeline pipeline_softplus[2];
- vk_pipeline pipeline_step[2];
- vk_pipeline pipeline_round[2];
- vk_pipeline pipeline_ceil[2];
- vk_pipeline pipeline_floor[2];
- vk_pipeline pipeline_trunc[2];
- vk_pipeline pipeline_sgn[2];
-
- // fused UNARY+MUL pipelines: [op][f16][norepeat][op_on_b]
- vk_pipeline pipeline_unary_mul[4][2][2][2];
-
- vk_pipeline pipeline_add1_f16_f16;
- vk_pipeline pipeline_add1_f16_f32;
- vk_pipeline pipeline_add1_f32_f32;
-
- vk_pipeline pipeline_arange_f32;
-
- vk_pipeline pipeline_fill_f32;
- vk_pipeline pipeline_fill_f16;
-
- vk_pipeline pipeline_geglu[2];
- vk_pipeline pipeline_reglu[2];
- vk_pipeline pipeline_swiglu[2];
- vk_pipeline pipeline_swiglu_oai[2];
- vk_pipeline pipeline_swiglu_clamp[2];
- vk_pipeline pipeline_geglu_erf[2];
- vk_pipeline pipeline_geglu_quick[2];
-
- vk_pipeline pipeline_leaky_relu[2];
- vk_pipeline pipeline_silu_back_f32;
- vk_pipeline pipeline_diag_mask_inf_f32;
- vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16;
- vk_pipeline pipeline_soft_max_f32_wg512, pipeline_soft_max_f32_f16_wg512;
- vk_pipeline pipeline_soft_max_back_f32;
-
- vk_pipeline pipeline_soft_max_large1_f32, pipeline_soft_max_large1_f32_f16;
- vk_pipeline pipeline_soft_max_large2_f32, pipeline_soft_max_large2_f32_f16;
- vk_pipeline pipeline_soft_max_large3_f32, pipeline_soft_max_large3_f32_f16;
-
- vk_pipeline pipeline_rope_norm_f32, pipeline_rope_norm_f16, pipeline_rope_norm_f32_f16;
- vk_pipeline pipeline_rope_neox_f32, pipeline_rope_neox_f16, pipeline_rope_neox_f32_f16;
- vk_pipeline pipeline_rope_multi_f32, pipeline_rope_multi_f16, pipeline_rope_multi_f32_f16;
- vk_pipeline pipeline_rope_vision_f32, pipeline_rope_vision_f16;
- vk_pipeline pipeline_argsort_f32[num_argsort_pipelines];
- vk_pipeline pipeline_argsort_large_f32[num_argsort_pipelines];
- vk_pipeline pipeline_topk_f32[num_topk_pipelines];
- vk_pipeline pipeline_topk_radix_f32;
- vk_pipeline pipeline_topk_radix_qsa; // qwen4 QSA indexer fusion (f16 mask)
- vk_pipeline pipeline_sum_rows_f32;
- vk_pipeline pipeline_cross_entropy_loss_f32, pipeline_cross_entropy_loss_f32_wg512;
- vk_pipeline pipeline_cross_entropy_loss_back_f32, pipeline_cross_entropy_loss_back_f32_wg512;
- vk_pipeline pipeline_fwht_f32[4];
- vk_pipeline pipeline_cumsum_f32;
- vk_pipeline pipeline_cumsum_small_f32;
- vk_pipeline pipeline_cumsum_multipass1_f32;
- vk_pipeline pipeline_cumsum_multipass2_f32;
- vk_pipeline pipeline_argmax_f32;
- vk_pipeline pipeline_count_equal_i32;
- vk_pipeline pipeline_dsv4_hc_comb_f32;
- vk_pipeline pipeline_dsv4_hc_pre_f32;
- vk_pipeline pipeline_dsv4_hc_pre_gated_f32;
- vk_pipeline pipeline_dsv4_hc_post_f32;
- vk_pipeline pipeline_dsv4_hc_post_nocomb_f32;
- std::map<vk_solve_tri_pipeline_state, vk_pipeline> pipeline_solve_tri_f32;
- vk_pipeline pipeline_im2col_f32, pipeline_im2col_f32_f16;
- vk_pipeline pipeline_im2col_3d_f32, pipeline_im2col_3d_f32_f16;
- vk_pipeline pipeline_timestep_embedding_f32;
- vk_pipeline pipeline_conv_transpose_1d_f32;
- vk_pipeline pipeline_col2im_1d_f32;
- vk_pipeline pipeline_col2im_1d_f16;
- vk_pipeline pipeline_col2im_1d_bf16;
- vk_pipeline pipeline_out_prod_f32;
- vk_pipeline pipeline_snake_f32;
- vk_pipeline pipeline_snake_f16;
- vk_pipeline pipeline_snake_bf16;
- vk_pipeline pipeline_pool1d_f32;
- vk_pipeline pipeline_pool2d_f32;
- vk_pipeline pipeline_rwkv_wkv6_f32;
- vk_pipeline pipeline_rwkv_wkv7_f32;
- vk_pipeline pipeline_gated_linear_attn_f32;
- vk_pipeline pipeline_lightning_indexer_f32[GGML_TYPE_COUNT];
- // [size_idx][kda] where size_idx: 0=d16, 1=d32, 2=d64, 3=d128
- vk_pipeline pipeline_gated_delta_net[4][2];
- vk_pipeline pipeline_ssm_scan_f32_d128;
- vk_pipeline pipeline_ssm_scan_f32_d256;
- vk_pipeline pipeline_ssm_conv_f32;
- vk_pipeline pipeline_ssm_conv_silu_f32;
- vk_pipeline pipeline_ssm_conv_bias_silu_f32;
- vk_pipeline pipeline_opt_step_adamw_f32;
- vk_pipeline pipeline_opt_step_sgd_f32;
- std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv2d_f32[CONV_SHAPE_COUNT];
- std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv2d_f16_f32[CONV_SHAPE_COUNT];
- std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv_transpose_2d_f32[CONV_SHAPE_COUNT];
- std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv_transpose_2d_f16_f32[CONV_SHAPE_COUNT];
- std::map<vk_conv3d_pipeline_state, vk_pipeline> pipeline_conv3d_f32[CONV_SHAPE_COUNT];
- std::map<vk_conv3d_pipeline_state, vk_pipeline> pipeline_conv3d_f16_f32[CONV_SHAPE_COUNT];
- vk_pipeline pipeline_conv2d_dw_whcn_f32, pipeline_conv2d_dw_whcn_f16_f32;
- vk_pipeline pipeline_conv2d_dw_cwhn_f32, pipeline_conv2d_dw_cwhn_f16_f32;
-
- std::map<vk_fa_pipeline_state, vk_pipeline> pipeline_flash_attn_f32_f16;
-
- std::map<std::pair<uint32_t, uint32_t>, vk_pipeline> pipeline_fa_mask_opt;
-
- vk_pipeline pipeline_fa_sparse_compact;
- vk_pipeline pipeline_fa_sparse_compact_subgroup;
- bool fa_sparse_compact_use_subgroups;
-
- vk_pipeline pipeline_flash_attn_split_k_reduce;
- vk_pipeline pipeline_count_experts;
-
- // [2] is for whether to take n_experts from spec constant (0) or push constant (1)
- vk_pipeline pipeline_topk_moe[num_topk_moe_pipelines][2];
-
- std::vector<vk_pipeline_ref> all_pipelines;
-
- std::vector<std::tuple<void*, size_t, vk_buffer>> pinned_memory;
-
- vk::Fence fence;
- vk_buffer sync_staging;
-
- ggml_backend_buffer_type buffer_type;
-
- bool disable_fusion;
- bool disable_host_visible_vidmem;
- bool allow_sysmem_fallback;
- bool disable_graph_optimize;
-
- std::unique_ptr<vk_memory_logger> memory_logger;
-
- ~vk_device_struct() {
- VK_LOG_DEBUG("destroy device " << name);
-
- device.destroyFence(fence);
-
- ggml_vk_destroy_buffer(sync_staging);
-
- if (compute_queue) compute_queue->cmd_pool.destroy(device);
- if (transfer_queue) transfer_queue->cmd_pool.destroy(device);
-
- // Explicitly clear to ensure queues drop their shared_ptrs to handles
- // before the Vulkan logical device instance is destroyed
- compute_queue.reset();
- transfer_queue.reset();
-
- for (auto& pipeline : all_pipelines) {
- if (pipeline.expired()) {
- continue;
- }
-
- vk_pipeline pl = pipeline.lock();
- ggml_vk_destroy_pipeline(device, pl);
- }
- all_pipelines.clear();
-
- device.destroyDescriptorSetLayout(dsl);
-
- device.destroy();
- }
-};
-
-void vk_command_pool::init(vk_device& device, vk_queue *q_) {
- cmd_buffers.clear();
- q = q_;
-
- vk::CommandPoolCreateInfo command_pool_create_info(
- vk::CommandPoolCreateFlags(VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT),
- q->queue_family_index);
- pool = device->device.createCommandPool(command_pool_create_info);
-}
-
-void vk_command_pool::destroy(vk::Device& device) {
- device.destroyCommandPool(pool);
- pool = nullptr;
- cmd_buffers.clear();
-}
-
-static void ggml_vk_print_device_fault_info(const vk_device& device) {
+void ggml_vk_print_device_fault_info(const vk_device& device) {
if (!device->device_fault || !device->pfn_vkGetDeviceFaultInfoEXT) {
return;
}
@@ -1271,1043 +153,32 @@ static void ggml_vk_print_device_fault_info(const vk_device& device) {
if (res != VK_SUCCESS) {
GGML_LOG_ERROR("ggml_vulkan: vkGetDeviceFaultInfoEXT (info) failed: %d\n", res);
return;
- }
-
- if (fault_counts.addressInfoCount == 0 && fault_counts.vendorInfoCount == 0 && fault_info.description[0] == '\0') {
- return;
- }
-
- if (fault_info.description[0] != '\0') {
- GGML_LOG_ERROR("ggml_vulkan: device fault on %s: %s\n", device->name.c_str(), fault_info.description);
- }
-
- for (uint32_t i = 0; i < fault_counts.addressInfoCount; i++) {
- const auto& info = address_infos[i];
- GGML_LOG_CONT(" address fault %u: type=%d address=0x%llx precision=0x%llx\n",
- i, (int)info.addressType,
- (unsigned long long)info.reportedAddress,
- (unsigned long long)info.addressPrecision);
- }
- for (uint32_t i = 0; i < fault_counts.vendorInfoCount; i++) {
- const auto& info = vendor_infos[i];
- GGML_LOG_CONT(" vendor fault %u: %s (code=0x%llx data=0x%llx)\n",
- i, info.description,
- (unsigned long long)info.vendorFaultCode,
- (unsigned long long)info.vendorFaultData);
- }
-}
-
-struct vk_buffer_struct {
- vk::Buffer buffer = VK_NULL_HANDLE;
- vk::DeviceMemory device_memory = VK_NULL_HANDLE;
- vk::MemoryPropertyFlags memory_property_flags;
- void * ptr;
- size_t size = 0;
- vk::DeviceAddress bda_addr {};
-
- vk_device device;
-
- ~vk_buffer_struct() {
- if (size == 0) {
- return;
- }
- VK_LOG_DEBUG("~vk_buffer_struct(" << buffer << ", " << size << ")");
-
- device->device.freeMemory(device_memory);
- device->device.destroyBuffer(buffer);
- }
-};
-
-struct vk_subbuffer {
- vk_buffer buffer;
- uint64_t offset;
- uint64_t size;
-
- operator vk::DescriptorBufferInfo() const {
- return { buffer->buffer, offset, size };
- }
-};
-
-struct vk_semaphore {
- vk::Semaphore s;
- uint64_t value;
-};
-
-// vk_event is used for the event-related backend interfaces. It uses vk::Events for
-// event_wait and a timeline semaphore for event_synchronize. Polling on an event for
-// event_synchronize wouldn't be sufficient to wait for command buffers to complete,
-// and would lead to validation errors.
-struct vk_event {
- std::vector<vk::Event> events_free; // Events available for reuse
- std::vector<vk::Event> events_submitted; // Events that are fully submitted and can be reused on next synchronize
- vk::Event event;
- bool has_event;
-
- vk_semaphore tl_semaphore;
- vk_command_buffer* cmd_buffer = nullptr;
- uint64_t cmd_buffer_use_counter = 0;
-};
-
-struct vk_submission {
- vk_command_buffer* buffer = nullptr;
- std::vector<vk_semaphore> wait_semaphores;
- std::vector<vk_semaphore> signal_semaphores;
-};
-
-typedef std::vector<vk_submission> vk_sequence;
-
-struct vk_mat_mat_push_constants {
- uint32_t M; uint32_t N; uint32_t K;
- uint32_t stride_a; uint32_t stride_b; uint32_t stride_d;
- uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;
- uint32_t base_work_group_z; uint32_t num_batches;
- uint32_t k_split;
- uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3;
- uint32_t padded_N;
-};
-
-#define MAT_VEC_FUSION_FLAGS_BIAS0 0x1
-#define MAT_VEC_FUSION_FLAGS_BIAS1 0x2
-#define MAT_VEC_FUSION_FLAGS_SCALE0 0x4
-#define MAT_VEC_FUSION_FLAGS_SCALE1 0x8
-
-struct vk_mat_vec_push_constants {
- uint32_t ncols;
- uint32_t stride_a;
- uint32_t stride_b;
- uint32_t stride_d;
- uint32_t batch_stride_a;
- uint32_t batch_stride_b;
- uint32_t batch_stride_d;
- uint32_t fusion_flags;
- uint32_t base_work_group_y;
- uint32_t ne02;
- uint32_t ne12;
- uint32_t broadcast2;
- uint32_t broadcast3;
-};
-
-struct vk_mat_vec_p021_push_constants {
- uint32_t ncols_x;
- uint32_t nrows_x;
- uint32_t nchannels_x;
- uint32_t nchannels_y;
- uint32_t b_offset;
- uint32_t d_offset;
- uint32_t fusion_flags;
-};
-
-struct vk_mat_vec_nc_push_constants {
- uint32_t ncols_x;
- uint32_t nrows_x;
- uint32_t row_stride_x;
- uint32_t channel_stride_x;
- uint32_t channel_stride_y;
- uint32_t channel_x_divisor;
- uint32_t ne12;
- uint32_t b_offset;
- uint32_t d_offset;
- uint32_t nb03;
- uint32_t nb13;
- uint32_t nb23;
- uint32_t fusion_flags;
-};
-
-struct vk_mat_mat_id_push_constants {
- uint32_t M; uint32_t N; uint32_t K;
- uint32_t stride_a; uint32_t stride_b; uint32_t stride_d;
- uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;
- uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11;
- uint32_t n_experts;
- uint32_t hoist_row_ids;
-};
-struct vk_mat_vec_id_push_constants {
- uint32_t ncols;
- uint32_t stride_a;
- uint32_t stride_b;
- uint32_t stride_d;
- uint32_t batch_stride_a;
- uint32_t batch_stride_b;
- uint32_t batch_stride_d;
- uint32_t fusion_flags;
- uint32_t nei0;
- uint32_t ne11;
- uint32_t expert_i1;
- uint32_t nbi1;
-};
-
-struct vk_flash_attn_push_constants {
- uint32_t N;
- uint32_t KV;
-
- uint32_t ne1;
- uint32_t ne2;
- uint32_t ne3;
-
- uint32_t neq2;
- uint32_t neq3;
- uint32_t nek2;
- uint32_t nek3;
- uint32_t nev2;
- uint32_t nev3;
- uint32_t nem1;
- uint32_t nem2;
- uint32_t nem3;
-
- uint32_t nb01;
- uint32_t nb02;
- uint32_t nb03;
- uint32_t nb11;
- uint32_t nb12;
- uint32_t nb13;
- uint32_t nb21;
- uint32_t nb22;
- uint32_t nb23;
-
- float scale;
- float max_bias;
- float logit_softcap;
-
- uint32_t mask_n_head_log2;
- float m0;
- float m1;
-
- uint32_t gqa_ratio;
- uint32_t split_kv;
- uint32_t k_num;
-};
-static_assert(sizeof(vk_flash_attn_push_constants) <= 128, "sizeof(vk_flash_attn_push_constants) must be <= 128");
-
-struct vk_op_push_constants {
- uint32_t KX;
- uint32_t KY;
- float param1;
- float param2;
- float param3;
- float param4;
-};
-
-struct vk_op_fwht_push_constants {
- uint32_t n_rows;
- uint32_t src_offset;
- uint32_t dst_offset;
- float scale;
-};
-
-struct vk_op_dsv4_hc_comb_push_constants {
- uint32_t n_tokens;
-
- uint32_t nbm0; uint32_t nbm1;
- uint32_t nbs0;
- uint32_t nbb0;
- uint32_t nbd0; uint32_t nbd1; uint32_t nbd2;
-
- uint32_t m_offset;
- uint32_t s_offset;
- uint32_t b_offset;
- uint32_t d_offset;
-
- float eps;
- uint32_t n_iter;
-};
-
-struct vk_op_dsv4_hc_pre_push_constants {
- uint32_t n_embd;
- uint32_t n_tokens;
-
- uint32_t nbx0; uint32_t nbx1; uint32_t nbx2;
- uint32_t nbw0; uint32_t nbw1; uint32_t nbw2;
- uint32_t nbd0; uint32_t nbd1;
-
- uint32_t x_offset;
- uint32_t w_offset;
- uint32_t d_offset;
-
- float scale;
-};
-
-struct vk_op_dsv4_hc_post_push_constants {
- uint32_t n_embd;
- uint32_t n_tokens;
-
- uint32_t nbx0; uint32_t nbx1;
- uint32_t nbr0; uint32_t nbr1; uint32_t nbr2;
- uint32_t nbp0; uint32_t nbp1;
- uint32_t nbc0; uint32_t nbc1; uint32_t nbc2;
- uint32_t nbd0; uint32_t nbd1; uint32_t nbd2;
-
- uint32_t x_offset;
- uint32_t r_offset;
- uint32_t p_offset;
- uint32_t c_offset;
- uint32_t d_offset;
-};
-
-struct vk_op_count_experts_push_constants {
- uint32_t ne00;
- uint32_t ne01;
- uint32_t nb00;
- uint32_t nb01;
- uint32_t a_offset;
- uint32_t n_experts;
- uint32_t hoist_row_ids;
- uint32_t ne00mp;
- uint32_t ne00L;
-};
-
-struct vk_op_glu_push_constants {
- uint32_t N;
- uint32_t ne00;
- uint32_t ne20;
- uint32_t mode; // 0: default, 1: swapped, 2: split
- float alpha; // for swiglu_oai
- float limit;
- uint32_t nb00;
- uint32_t nb01;
- uint32_t nb02;
- uint32_t nb03;
- uint32_t nb10;
- uint32_t nb11;
- uint32_t nb12;
- uint32_t nb13;
- uint32_t nb20;
- uint32_t nb21;
- uint32_t nb22;
- uint32_t nb23;
- uint32_t ne21;
- uint32_t ne22;
- uint32_t misalign_offsets;
- uint32_t ne2_012mp; uint32_t ne2_012L;
- uint32_t ne2_01mp; uint32_t ne2_01L;
- uint32_t ne2_0mp; uint32_t ne2_0L;
-};
-static_assert(sizeof(vk_op_glu_push_constants) <= 128, "sizeof(vk_op_glu_push_constants) must be <= 128");
-
-struct vk_op_unary_push_constants {
- uint32_t ne;
- uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;
- uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;
- uint32_t misalign_offsets;
- float param1; float param2; float param3; float param4;
- uint32_t ne0_012mp; uint32_t ne0_01mp; uint32_t ne0_0mp; uint32_t ne0_Ls;
- uint32_t ne1_012mp; uint32_t ne1_01mp; uint32_t ne1_0mp; uint32_t ne1_Ls;
-};
-static_assert(sizeof(vk_op_unary_push_constants) <= 128, "sizeof(vk_op_unary_push_constants) must be <= 128");
-
-static vk_op_unary_push_constants vk_op_unary_push_constants_init(const ggml_tensor * src0, const ggml_tensor * dst, int64_t ne = 0) {
- GGML_ASSERT(ne != 0 || (ggml_nelements(src0) == ggml_nelements(dst)));
- ne = ne != 0 ? ne : ggml_nelements(dst);
- GGML_ASSERT(ne <= (int64_t)std::numeric_limits<uint32_t>::max());
-
- vk_op_unary_push_constants p{};
- p.ne = (uint32_t)ne;
-
- size_t src0_tsize = ggml_type_size(src0->type);
- p.ne00 = (uint32_t)src0->ne[0];
- p.ne01 = (uint32_t)src0->ne[1];
- p.ne02 = (uint32_t)src0->ne[2];
- p.ne03 = (uint32_t)src0->ne[3];
- p.nb00 = (uint32_t)(src0->nb[0] / src0_tsize);
- p.nb01 = (uint32_t)(src0->nb[1] / src0_tsize);
- p.nb02 = (uint32_t)(src0->nb[2] / src0_tsize);
- p.nb03 = (uint32_t)(src0->nb[3] / src0_tsize);
-
- size_t dst_tsize = ggml_type_size(dst->type);
- p.ne10 = (uint32_t)dst->ne[0];
- p.ne11 = (uint32_t)dst->ne[1];
- p.ne12 = (uint32_t)dst->ne[2];
- p.ne13 = (uint32_t)dst->ne[3];
- p.nb10 = (uint32_t)(dst->nb[0] / dst_tsize);
- p.nb11 = (uint32_t)(dst->nb[1] / dst_tsize);
- p.nb12 = (uint32_t)(dst->nb[2] / dst_tsize);
- p.nb13 = (uint32_t)(dst->nb[3] / dst_tsize);
-
- return p; // offsets are initialized later in ggml_vk_op
-}
-
-struct vk_op_pad_push_constants {
- uint32_t ne;
- uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;
- uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;
- uint32_t misalign_offsets;
- uint32_t circular;
-
- uint32_t lp0; uint32_t rp0;
- uint32_t lp1; uint32_t rp1;
- uint32_t lp2; uint32_t rp2;
- uint32_t lp3; uint32_t rp3;
-};
-
-static vk_op_pad_push_constants vk_op_pad_push_constants_init(const ggml_tensor * src0, const ggml_tensor * dst) {
- int64_t ne = ggml_nelements(dst);
- GGML_ASSERT(ne <= (int64_t)std::numeric_limits<uint32_t>::max());
-
- vk_op_pad_push_constants p{};
- p.ne = (uint32_t)ne;
-
- size_t src0_tsize = ggml_type_size(src0->type);
- p.ne00 = (uint32_t)src0->ne[0];
- p.ne01 = (uint32_t)src0->ne[1];
- p.ne02 = (uint32_t)src0->ne[2];
- p.ne03 = (uint32_t)src0->ne[3];
- p.nb00 = (uint32_t)(src0->nb[0] / src0_tsize);
- p.nb01 = (uint32_t)(src0->nb[1] / src0_tsize);
- p.nb02 = (uint32_t)(src0->nb[2] / src0_tsize);
- p.nb03 = (uint32_t)(src0->nb[3] / src0_tsize);
-
- size_t dst_tsize = ggml_type_size(dst->type);
- p.ne10 = (uint32_t)dst->ne[0];
- p.ne11 = (uint32_t)dst->ne[1];
- p.ne12 = (uint32_t)dst->ne[2];
- p.ne13 = (uint32_t)dst->ne[3];
- p.nb10 = (uint32_t)(dst->nb[0] / dst_tsize);
- p.nb11 = (uint32_t)(dst->nb[1] / dst_tsize);
- p.nb12 = (uint32_t)(dst->nb[2] / dst_tsize);
- p.nb13 = (uint32_t)(dst->nb[3] / dst_tsize);
-
- p.lp0 = dst->op_params[0];
- p.rp0 = dst->op_params[1];
- p.lp1 = dst->op_params[2];
- p.rp1 = dst->op_params[3];
- p.lp2 = dst->op_params[4];
- p.rp2 = dst->op_params[5];
- p.lp3 = dst->op_params[6];
- p.rp3 = dst->op_params[7];
- p.circular = dst->op_params[8];
-
- return p; // fastdiv values and offsets are initialized later in ggml_vk_op
-}
-
-// See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1.
-// Precompute mp (m' in the paper) and L such that division
-// can be computed using a multiply (high 32b of 64b result)
-// and a shift:
-//
-// n/d = (mulhi(n, mp) + n) >> L;
-static void init_fastdiv_values(uint32_t d, uint32_t &mp, uint32_t &L)
-{
- // compute L = ceil(log2(d));
- L = 0;
- while (L < 32 && (uint32_t{1} << L) < d) {
- L++;
- }
-
- mp = (uint32_t)((uint64_t{1} << 32) * ((uint64_t{1} << L) - d) / d + 1);
-}
-
-static uint32_t pack_fastdiv_L(uint32_t L0, uint32_t L1, uint32_t L2) {
- return L0 | (L1 << 8) | (L2 << 16);
-}
-
-template <typename T> void init_pushconst_fastdiv(T &p) {
- GGML_UNUSED(p);
- static_assert(!std::is_const<T>::value, "unexpected type");
-}
-
-template <> void init_pushconst_fastdiv(vk_op_unary_push_constants &p) {
- // Compute magic values to divide by these six numbers.
- uint32_t ne0_012L;
- uint32_t ne0_01L;
- uint32_t ne0_0L;
- uint32_t ne1_012L;
- uint32_t ne1_01L;
- uint32_t ne1_0L;
-
- init_fastdiv_values(p.ne02*p.ne01*p.ne00, p.ne0_012mp, ne0_012L);
- init_fastdiv_values(p.ne01*p.ne00, p.ne0_01mp, ne0_01L);
- init_fastdiv_values(p.ne00, p.ne0_0mp, ne0_0L);
- init_fastdiv_values(p.ne12*p.ne11*p.ne10, p.ne1_012mp, ne1_012L);
- init_fastdiv_values(p.ne11*p.ne10, p.ne1_01mp, ne1_01L);
- init_fastdiv_values(p.ne10, p.ne1_0mp, ne1_0L);
-
- p.ne0_Ls = pack_fastdiv_L(ne0_012L, ne0_01L, ne0_0L);
- p.ne1_Ls = pack_fastdiv_L(ne1_012L, ne1_01L, ne1_0L);
-}
-
-template <> void init_pushconst_fastdiv(vk_op_glu_push_constants &p) {
- // GLU linearizes over dst, then uses dst coordinates for src0/src1.
- init_fastdiv_values(p.ne22*p.ne21*p.ne20, p.ne2_012mp, p.ne2_012L);
- init_fastdiv_values(p.ne21*p.ne20, p.ne2_01mp, p.ne2_01L);
- init_fastdiv_values(p.ne20, p.ne2_0mp, p.ne2_0L);
-}
-
-template <> void init_pushconst_fastdiv(vk_op_count_experts_push_constants &p) {
- init_fastdiv_values(p.ne00, p.ne00mp, p.ne00L);
-}
-
-struct vk_op_binary_push_constants {
- uint32_t ne;
- uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;
- uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;
- uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23; uint32_t nb20; uint32_t nb21; uint32_t nb22; uint32_t nb23;
- uint32_t misalign_offsets;
- float param1; float param2; int32_t param3;
-};
-
-// Distinct type with the same layout so concat can overload tensor offset initialization.
-struct vk_op_concat_push_constants : vk_op_binary_push_constants {};
-static_assert(sizeof(vk_op_concat_push_constants) == sizeof(vk_op_binary_push_constants));
-static_assert(std::is_standard_layout_v<vk_op_concat_push_constants>);
-
-struct vk_op_multi_add_push_constants {
- // shape for dst
- uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23;
-
- // strides for srcs+dst
- uint32_t nb[MAX_PARAMETER_COUNT][4];
-
- uint32_t rms_partials;
-};
-// update multi_add.comp if this changes
-static_assert(MAX_PARAMETER_COUNT == 12);
-static_assert(sizeof(vk_op_multi_add_push_constants) <= 256);
-
-struct vk_op_topk_moe_push_constants {
- uint32_t n_rows;
- uint32_t n_experts_push;
- uint32_t n_expert_used;
- float clamp_min;
- float clamp_max;
- uint32_t gating_func;
- uint32_t has_bias;
- uint32_t with_norm;
- float output_scale;
- float output_bias;
-};
-
-struct vk_op_add_id_push_constants {
- uint32_t ne0;
- uint32_t ne1;
- uint32_t s01;
- uint32_t s02;
- uint32_t s11;
- uint32_t s21;
-};
-
-struct vk_op_diag_mask_push_constants {
- uint32_t ncols;
- uint32_t rows_per_channel;
- int32_t n_past;
-};
-
-struct vk_op_rope_push_constants {
- uint32_t rope_mode;
- uint32_t nrows;
- uint32_t n_dims;
- uint32_t n_offs;
- float freq_scale;
- float freq_base;
- float ext_factor;
- float attn_factor;
- float corr_dims[2];
- float theta_scale;
- uint32_t has_ff;
- int32_t sections[4];
- uint32_t is_imrope;
- uint32_t is_back;
- uint32_t set_rows_stride;
- uint32_t ne00;
- uint32_t ne01;
- uint32_t ne02;
- uint32_t nb01;
- uint32_t nb02;
- uint32_t nb03;
- uint32_t nb11;
- uint32_t nb12;
- uint32_t nb13;
- uint32_t a_offset;
- uint32_t d_offset;
-};
-static_assert(sizeof(vk_op_rope_push_constants) <= 128, "sizeof(vk_op_rope_push_constants) must be <= 128");
-
-// For fused rms_norm+mul+rope(+view+set_rows)
-struct vk_op_rms_norm_mul_rope_push_constants {
- vk_op_binary_push_constants bin;
- vk_op_rope_push_constants rope;
-};
-
-struct vk_op_soft_max_push_constants {
- uint32_t KX;
- uint32_t KY;
- uint32_t ne00;
- uint32_t ne01;
- uint32_t ne02;
- uint32_t ne12;
- uint32_t ne13;
- uint32_t nb11;
- uint32_t nb12;
- uint32_t nb13;
- float scale;
- float max_bias;
- float m0;
- float m1;
- uint32_t n_head_log2;
- uint32_t nrows_x;
- uint32_t has_sinks;
-};
-
-struct vk_op_argsort_push_constants {
- uint32_t ncols;
- uint32_t ncols_padded;
- uint32_t ncols_padded_log2;
- uint32_t nrows;
- uint32_t order;
- uint32_t outer_start;
- uint32_t outer_end;
- uint32_t inner_start;
- uint32_t inner_end;
-};
-
-struct vk_op_topk_push_constants {
- uint32_t orig_ncols;
- uint32_t ncols_input;
- uint32_t ncols_output;
- uint32_t k;
- uint32_t nrows;
- uint32_t first_pass;
- uint32_t last_pass;
-};
-
-struct vk_op_topk_radix_push_constants {
- uint32_t ncols;
- uint32_t k;
- uint32_t nrows;
- uint32_t n_tps; // QSA only
- uint32_t n_blocks; // QSA only
- uint32_t n_stream; // QSA only
-};
-
-struct vk_op_im2col_push_constants {
- uint64_t dst_addr;
- uint32_t batch_offset; uint32_t offset_delta;
- uint32_t IC;
- uint32_t IW; uint32_t IH;
- uint32_t OW; uint32_t OH;
- uint32_t KW; uint32_t KH;
- uint32_t OH_batch;
- uint32_t CHW;
- int32_t s0; int32_t s1;
- int32_t p0; int32_t p1;
- int32_t d0; int32_t d1;
- uint32_t batch_IC;
-};
-
-struct vk_op_im2col_3d_push_constants {
- uint64_t dst_addr;
- uint32_t nb10;
- uint32_t nb11;
- uint32_t nb12;
- uint32_t nb13;
- uint32_t s0;
- uint32_t s1;
- uint32_t s2;
- uint32_t p0;
- uint32_t p1;
- uint32_t p2;
- uint32_t d0;
- uint32_t d1;
- uint32_t d2;
- uint32_t IW;
- uint32_t IH;
- uint32_t ID;
- uint32_t IC;
- uint32_t KW;
- uint32_t OH;
- uint32_t KD_KH_KW;
- uint32_t KH_KW;
- uint32_t IC_KD_KH_KW;
- uint32_t N_OD_OH;
- uint32_t OD_OH;
- uint32_t OD_OH_OW_IC_KD_KH_KW;
- uint32_t OH_OW_IC_KD_KH_KW;
- uint32_t OW_IC_KD_KH_KW;
- uint32_t misalign_offsets;
-};
-
-struct vk_op_timestep_embedding_push_constants {
- uint32_t nb1;
- uint32_t dim;
- uint32_t max_period;
-};
-
-struct vk_op_col2im_1d_push_constants {
- uint32_t T_out;
- uint32_t OC;
- uint32_t K_OC;
- uint32_t T_in;
- uint32_t K;
- int32_t stride;
- int32_t p0;
-};
-
-struct vk_op_conv_transpose_1d_push_constants {
- uint32_t Cout;
- uint32_t Cin;
- uint32_t K;
- uint32_t L;
- uint32_t KL;
-
- uint32_t nb01;
- uint32_t nb02;
- uint32_t nb11;
- uint32_t nb1;
-
- int32_t s0;
-};
-
-struct vk_op_snake_push_constants {
- uint32_t ne0;
- uint32_t ne1;
-};
-
-struct vk_op_pool1d_push_constants {
- uint32_t IL;
- uint32_t OL;
- uint32_t OC;
- uint32_t pelements;
- uint32_t op;
- int32_t k0;
- int32_t s0;
- int32_t p0;
-};
-
-struct vk_op_pool2d_push_constants {
- uint32_t IW; uint32_t IH;
- uint32_t OW; uint32_t OH;
- uint32_t OC;
- uint32_t pelements;
- uint32_t op;
- int32_t k0; int32_t k1;
- int32_t s0; int32_t s1;
- int32_t p0; int32_t p1;
-};
-
-struct vk_op_rwkv_wkv6_push_constants {
- uint32_t B;
- uint32_t T;
- uint32_t C;
- uint32_t H;
-};
-
-struct vk_op_rwkv_wkv7_push_constants {
- uint32_t B;
- uint32_t T;
- uint32_t C;
- uint32_t H;
-};
-struct vk_op_gated_linear_attn_push_constants {
- uint32_t B;
- uint32_t T;
- uint32_t C;
- uint32_t H;
- float scale;
-};
-struct vk_op_lightning_indexer_push_constants {
- uint32_t n_kv;
- uint32_t n_heads;
- uint32_t n_tokens;
- uint32_t n_streams;
- uint32_t n_masks;
- uint32_t dispatch_x;
- uint32_t q_nb1;
- uint32_t q_nb2;
- uint32_t q_nb3;
- uint32_t k_nb2;
- uint32_t k_nb3;
- uint32_t w_nb1;
- uint32_t w_nb3;
- uint32_t m_nb1;
- uint32_t m_nb3;
- uint32_t d_nb1;
- uint32_t d_nb3;
-};
-static_assert(sizeof(vk_op_lightning_indexer_push_constants) <= 128);
-struct vk_op_gated_delta_net_push_constants {
- uint32_t H;
- uint32_t n_tokens;
- uint32_t n_seqs;
- uint32_t s_off;
- uint32_t sq1, sq2, sq3;
- uint32_t sv1, sv2, sv3;
- uint32_t sb1, sb2, sb3;
- uint32_t neq1, rq3;
- float scale;
- uint32_t K;
-};
-
-struct vk_op_ssm_scan_push_constants {
- uint32_t nb02, nb03, nb12, nb13;
- uint32_t nb21, nb22, nb31;
- uint32_t nb42, nb43, nb52, nb53;
- uint32_t s_off;
- uint32_t n_head, d_head, n_group, n_tok;
- uint32_t n_seq, K;
-};
-struct vk_op_ssm_conv_push_constants {
- uint32_t nb01, nb02;
- uint32_t nb11;
- uint32_t dst_nb0, dst_nb1, dst_nb2;
- uint32_t nc, ncs, nr, n_t, n_s;
-};
-
-struct vk_op_conv2d_push_constants {
- uint32_t Cout;
- uint32_t Cin;
- uint32_t N;
-
- uint32_t W;
- uint32_t H;
- uint32_t OW;
- uint32_t OH;
-
- uint32_t nb01;
- uint32_t nb02;
- uint32_t nb03;
-
- uint32_t nb11;
- uint32_t nb12;
- uint32_t nb13;
-
- uint32_t nb1;
- uint32_t nb2;
- uint32_t nb3;
-
- // init_fastdiv_values constants for dividing by OW, OW*OH
- uint32_t OWmp; uint32_t OWL;
- uint32_t OWOHmp; uint32_t OWOHL;
-};
-
-template <> void init_pushconst_fastdiv(vk_op_conv2d_push_constants &p) {
- // Compute magic values to divide by OW, OW*OH
- init_fastdiv_values(p.OW, p.OWmp, p.OWL);
- init_fastdiv_values(p.OW*p.OH, p.OWOHmp, p.OWOHL);
-}
-
-struct vk_op_conv3d_push_constants {
- uint32_t OC;
- uint32_t IC;
- uint32_t N;
-
- uint32_t IW;
- uint32_t IH;
- uint32_t ID;
- uint32_t OW;
- uint32_t OH;
- uint32_t OD;
-
- uint32_t nb01;
- uint32_t nb02;
- uint32_t nb03;
-
- uint32_t nb11;
- uint32_t nb12;
- uint32_t nb13;
-
- uint32_t nb1;
- uint32_t nb2;
- uint32_t nb3;
-
- uint32_t OWmp; uint32_t OWL;
- uint32_t OWOHmp; uint32_t OWOHL;
- uint32_t OWOHODmp; uint32_t OWOHODL;
-};
-
-template <> void init_pushconst_fastdiv(vk_op_conv3d_push_constants &p) {
- init_fastdiv_values(p.OW, p.OWmp, p.OWL);
- init_fastdiv_values(p.OW*p.OH, p.OWOHmp, p.OWOHL);
- init_fastdiv_values(p.OW*p.OH*p.OD, p.OWOHODmp, p.OWOHODL);
-}
-
-struct vk_op_conv2d_dw_push_constants {
- uint32_t ne;
- uint32_t batches;
- uint32_t channels;
- uint32_t dst_w;
- uint32_t dst_h;
- uint32_t src_w;
- uint32_t src_h;
- uint32_t knl_w;
- uint32_t knl_h;
- int32_t stride_x;
- int32_t stride_y;
- int32_t pad_x;
- int32_t pad_y;
- int32_t dilation_x;
- int32_t dilation_y;
-};
-
-struct vk_op_upscale_push_constants {
- uint32_t ne; uint32_t a_offset; uint32_t d_offset;
- uint32_t ne00; uint32_t ne01;
- uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;
- uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13;
- float sf0; float sf1; float sf2; float sf3;
- float pixel_offset;
-};
-
-struct vk_op_sum_rows_push_constants
-{
- uint32_t n_cols;
- uint32_t ne01, ne02;
- uint32_t nb01, nb02, nb03;
- uint32_t nb11, nb12, nb13;
- float weight;
- uint32_t misalign_offsets;
- uint32_t ne0_12mp, ne0_12L;
- uint32_t ne0_1mp, ne0_1L;
-};
-
-static vk_op_sum_rows_push_constants vk_op_sum_rows_push_constants_init(const ggml_tensor * src, const ggml_tensor * dst, int64_t n_cols) {
- uint32_t type_size = (uint32_t)ggml_type_size(src->type);
- vk_op_sum_rows_push_constants p = {};
- p.n_cols = (uint32_t)n_cols;
- p.ne01 = (uint32_t)src->ne[1];
- p.ne02 = (uint32_t)src->ne[2];
- p.nb01 = (uint32_t)src->nb[1] / type_size;
- p.nb02 = (uint32_t)src->nb[2] / type_size;
- p.nb03 = (uint32_t)src->nb[3] / type_size;
- p.nb11 = (uint32_t)dst->nb[1] / type_size;
- p.nb12 = (uint32_t)dst->nb[2] / type_size;
- p.nb13 = (uint32_t)dst->nb[3] / type_size;
- p.weight = 1.0f;
- return p;
-}
-
-template <> void init_pushconst_fastdiv(vk_op_sum_rows_push_constants &p) {
- init_fastdiv_values(p.ne01*p.ne02, p.ne0_12mp, p.ne0_12L);
- init_fastdiv_values(p.ne01, p.ne0_1mp, p.ne0_1L);
-}
-
-struct vk_quantize_q8_1_push_constants {
- uint32_t ne;
- uint32_t num_blocks;
-};
-
-struct vk_op_flash_attn_split_k_reduce_push_constants {
- uint32_t D;
- uint32_t ne1;
- uint32_t ne2;
- uint32_t ne3;
- uint32_t k_num;
- uint32_t sinks;
-};
-
-struct vk_op_flash_attn_mask_opt_push_constants {
- uint32_t nem0;
- uint32_t nem1;
- uint32_t nem2;
- uint32_t nbm1;
- uint32_t nbm2;
- uint32_t nbm3;
- uint32_t nbd1;
- uint32_t nbd2;
- uint32_t nbd3;
-};
-
-struct vk_op_flash_attn_sparse_compact_push_constants {
- uint32_t KV;
- uint32_t nem1;
- uint32_t nem2;
- uint32_t nbm1;
- uint32_t nbm2;
- uint32_t nbm3;
- uint32_t n_kv_max;
-};
-
-// Allow pre-recording command buffers
-struct vk_staging_memcpy {
- vk_staging_memcpy(void * _dst, const void * _src, size_t _n) : dst(_dst), src(_src), n(_n) {}
-
- void * dst;
- const void * src;
- size_t n;
-};
-
-struct vk_staging_memset {
- vk_staging_memset(void * _dst, uint32_t _val, size_t _n) : dst(_dst), val(_val), n(_n) {}
-
- void * dst;
- uint32_t val;
- size_t n;
-};
-
-struct vk_context_struct {
- vk_submission * s;
- std::vector<vk_sequence> seqs;
-
- int exit_tensor_idx;
-
- std::vector<vk_staging_memcpy> in_memcpys;
- std::vector<vk_staging_memcpy> out_memcpys;
- std::vector<vk_staging_memset> memsets;
-
- std::vector<std::string> debug_labels;
-
- vk_command_pool * p {};
-};
-typedef std::shared_ptr<vk_context_struct> vk_context;
-typedef std::weak_ptr<vk_context_struct> vk_context_ref;
-
-struct ggml_vk_garbage_collector {
- std::vector<vk_semaphore> tl_semaphores;
- std::vector<vk_semaphore> semaphores;
- std::vector<vk::Event> events;
- std::vector<vk_context> contexts;
-};
-
-static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx);
-static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested = nullptr);
-static void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx);
-static bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint32_t lower_major, uint32_t lower_minor, uint32_t upper_major, uint32_t upper_minor);
-
-static bool vk_memory_logger_enabled = false;
-
-#define VK_LOG_MEMORY(msg) if (vk_memory_logger_enabled) { std::cerr << "ggml_vulkan memory: " << msg << std::endl; }
-
-static std::string format_size(size_t size) {
- const size_t kib = 1024;
- const size_t mib = kib * 1024;
- const size_t gib = mib * 1024;
+ }
- std::ostringstream oss;
- oss << std::fixed << std::setprecision(2);
+ if (fault_counts.addressInfoCount == 0 && fault_counts.vendorInfoCount == 0 && fault_info.description[0] == '\0') {
+ return;
+ }
- if (size >= gib) {
- oss << static_cast<double>(size) / gib << " GiB";
- } else if (size >= mib) {
- oss << static_cast<double>(size) / mib << " MiB";
- } else if (size >= kib) {
- oss << static_cast<double>(size) / kib << " KiB";
- } else {
- oss << size << " B";
+ if (fault_info.description[0] != '\0') {
+ GGML_LOG_ERROR("ggml_vulkan: device fault on %s: %s\n", device->name.c_str(), fault_info.description);
}
- return oss.str();
+ for (uint32_t i = 0; i < fault_counts.addressInfoCount; i++) {
+ const auto& info = address_infos[i];
+ GGML_LOG_CONT(" address fault %u: type=%d address=0x%llx precision=0x%llx\n",
+ i, (int)info.addressType,
+ (unsigned long long)info.reportedAddress,
+ (unsigned long long)info.addressPrecision);
+ }
+ for (uint32_t i = 0; i < fault_counts.vendorInfoCount; i++) {
+ const auto& info = vendor_infos[i];
+ GGML_LOG_CONT(" vendor fault %u: %s (code=0x%llx data=0x%llx)\n",
+ i, info.description,
+ (unsigned long long)info.vendorFaultCode,
+ (unsigned long long)info.vendorFaultData);
+ }
}
-
-class vk_memory_logger {
-public:
- vk_memory_logger(): total_device(0), total_host(0) {}
- void log_allocation(vk_buffer_ref buf_ref, size_t size);
- void log_deallocation(vk_buffer_ref buf_ref);
-
-private:
- std::map<vk::Buffer, size_t> allocations; // Track allocations
- size_t total_device;
- size_t total_host;
- static std::mutex log_mutex;
-};
-
-std::mutex vk_memory_logger::log_mutex;
-
-static bool vk_perf_logger_enabled = false;
-static bool vk_perf_logger_concurrent = false;
-static bool vk_enable_sync_logger = false;
-// number of calls between perf logger prints
-static uint32_t vk_perf_logger_frequency = 1;
-static std::string vk_pipeline_stats_filter;
-
-static uint64_t ggml_vk_get_node_flops(const ggml_tensor * node) {
+uint64_t ggml_vk_get_node_flops(const ggml_tensor * node) {
if (node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) {
const uint64_t m = node->ne[0];
const uint64_t n = node->ne[1];
@@ -2338,8 +209,7 @@ static uint64_t ggml_vk_get_node_flops(const ggml_tensor * node) {
}
return 0;
}
-
-static void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int end) {
+void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int end) {
uint64_t total_flops = 0;
int n_ops = 0;
for (int j = start; j <= end && j < cgraph->n_nodes; j++) {
@@ -2357,8 +227,7 @@ static void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int e
}
GGML_LOG_CONT(" total: %d ops, %.2f GFLOP\n", n_ops, total_flops / 1e9);
}
-
-static void ggml_vk_print_device_lost_info(const vk_device& device) {
+void ggml_vk_print_device_lost_info(const vk_device& device) {
ggml_vk_print_device_fault_info(device);
if (device->serialize_submissions && device->diag_cgraph != nullptr && device->diag_prev_start >= 0) {
GGML_LOG_ERROR("ggml_vulkan: device lost on %s, likely caused by previous submission (nodes %d to %d):\n",
@@ -2368,244 +237,16 @@ static void ggml_vk_print_device_lost_info(const vk_device& device) {
GGML_LOG_ERROR("ggml_vulkan: device lost on %s\n", device->name.c_str());
}
}
+void * const vk_ptr_base = (void *)(uintptr_t) 0x1000; // NOLINT
-class vk_perf_logger {
- public:
- void print_timings(bool force = false) {
- if (timings.empty()) {
- return;
- }
- print_count++;
- if ((print_count % vk_perf_logger_frequency) != 0 && !force) {
- return;
- }
- print_count = 0;
- uint64_t total_all_op_times = 0;
- std::cerr << "----------------\nVulkan Timings:" << std::endl;
- for (const auto & t : timings) {
- uint64_t total_op_times = 0;
- for (const auto & time : t.second) {
- total_op_times += time;
- }
- std::cerr << t.first << ": " << t.second.size() << " x " << (total_op_times / t.second.size() / 1000.0)
- << " us = " << (total_op_times / 1000.0) << " us";
-
- // If we have as many flops entries as timing entries for the op, then compute and log the flops/S.
- auto it = flops.find(t.first);
- if (it != flops.end() && (it->second).size() == t.second.size()) {
- uint64_t total_op_flops = 0;
- for (const auto & elem : it->second) {
- total_op_flops += elem;
- }
- std::cerr << " ("
- << (double(total_op_flops) / (1000.0 * 1000.0 * 1000.0)) /
- (double(total_op_times) / (1000.0 * 1000.0 * 1000.0))
- << " GFLOPS/s)";
- }
-
- total_all_op_times += total_op_times;
-
- std::cerr << std::endl;
- }
-
- if (timings.size() > 0) {
- std::cerr << "Total time: " << total_all_op_times / 1000.0 << " us." << std::endl;
- }
-
- timings.clear();
- flops.clear();
- }
-
- std::string get_node_fusion_name(const ggml_tensor * node, const char *fusion_name, uint64_t *n_flops) {
- *n_flops = ggml_vk_get_node_flops(node);
- std::string fusion_str;
- if (fusion_name) {
- fusion_str = fusion_name + std::string(" ");
- }
- if (node->op == GGML_OP_UNARY) {
- return fusion_str + ggml_unary_op_name(ggml_get_unary_op(node));
- }
- if (node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) {
- const uint64_t m = node->ne[0];
- const uint64_t n = node->ne[1];
- const uint64_t k = node->src[1]->ne[0];
- const uint64_t batch = node->ne[2] * node->ne[3];
- std::string name = ggml_op_name(node->op);
- if ((node->op == GGML_OP_MUL_MAT && n <= mul_mat_vec_max_cols) ||
- (node->op == GGML_OP_MUL_MAT_ID && node->src[2]->ne[1] == 1)) {
- name += "_VEC";
- }
- name += " ";
- name += ggml_type_name(node->src[0]->type);
- name += " m=" + std::to_string(m) + " n=" + std::to_string(n) + " k=" + std::to_string(k);
- if (node->op == GGML_OP_MUL_MAT_ID) {
- name += " n_expert=" + std::to_string(node->src[0]->ne[2]);
- }
- if (batch > 1) {
- name += " batch=" + std::to_string(batch);
- }
- return fusion_str + name;
- }
- if (node->op == GGML_OP_CONV_2D || node->op == GGML_OP_CONV_TRANSPOSE_2D) {
- std::string name = ggml_op_name(node->op);
- const ggml_tensor * knl = node->src[0];
- uint64_t Cout = node->ne[2];
- uint64_t size_K = node->src[1]->ne[2] * knl->ne[0] * knl->ne[1];
- uint64_t size_N = node->ne[3] * node->ne[0] * node->ne[1];
- name += " M=Cout=" + std::to_string(Cout) + ", K=Cin*KW*KH=" + std::to_string(size_K) +
- ", N=N*OW*OH=" + std::to_string(size_N);
- return fusion_str + name;
- }
- if (node->op == GGML_OP_RMS_NORM) {
- std::string name = ggml_op_name(node->op);
- name += "(" + std::to_string(node->ne[0]) + "," + std::to_string(node->ne[1]) + "," + std::to_string(node->ne[2]) + "," + std::to_string(node->ne[3]) + ")";
- return fusion_str + name;
- }
- if (node->op == GGML_OP_FLASH_ATTN_EXT) {
- const ggml_tensor * dst = node;
- const ggml_tensor * q = node->src[0];
- const ggml_tensor * k = node->src[1];
- const ggml_tensor * v = node->src[2];
- const ggml_tensor * m = node->src[3];
- std::stringstream name;
- name << fusion_str;
- name << ggml_op_name(node->op) <<
- " dst(" << dst->ne[0] << "," << dst->ne[1] << "," << dst->ne[2] << "," << dst->ne[3] << "), " <<
- " q(" << q->ne[0] << "," << q->ne[1] << "," << q->ne[2] << "," << q->ne[3] << "), " <<
- " k(" << k->ne[0] << "," << k->ne[1] << "," << k->ne[2] << "," << k->ne[3] << "), " <<
- " v(" << v->ne[0] << "," << v->ne[1] << "," << v->ne[2] << "," << v->ne[3] << "), " <<
- " m(" << (m?m->ne[0]:0) << "," << (m?m->ne[1]:0) << "," << (m?m->ne[2]:0) << "," << (m?m->ne[3]:0) << ")";
- return name.str();
- }
- if (node->op == GGML_OP_TOP_K) {
- std::stringstream name;
- name << fusion_str;
- name << ggml_op_name(node->op) <<
- " K=" << node->ne[0] <<
- " (" << node->src[0]->ne[0] << "," << node->src[0]->ne[1] << "," << node->src[0]->ne[2] << "," << node->src[0]->ne[3] << ")";
- return name.str();
- }
- return fusion_str + ggml_op_name(node->op);
- }
-
- void log_timing(const ggml_tensor * node, const char *fusion_name, uint64_t time) {
- uint64_t n_flops;
- std::string name = get_node_fusion_name(node, fusion_name, &n_flops);
- if (n_flops) {
- flops[name].push_back(n_flops);
- }
- timings[name].push_back(time);
- }
-
- void log_timing(const std::vector<ggml_tensor *> &nodes, const std::vector<const char *> &names, uint64_t time) {
- uint64_t total_flops = 0;
- std::string name;
- for (size_t n = 0; n < nodes.size(); ++n) {
- uint64_t n_flops = 0;
- name += get_node_fusion_name(nodes[n], names[n], &n_flops);
- total_flops += n_flops;
-
- if (n != nodes.size() - 1) {
- name += ", ";
- }
- }
- if (total_flops) {
- flops[name].push_back(total_flops);
- }
- timings[name].push_back(time);
- }
-
- private:
- std::map<std::string, std::vector<uint64_t>> timings;
- std::map<std::string, std::vector<uint64_t>> flops;
- uint32_t print_count {};
-};
-
-struct ggml_backend_vk_context {
- std::string name;
-
- vk_device device;
-
- size_t semaphore_idx, event_idx;
- ggml_vk_garbage_collector gc;
- size_t prealloc_size_x, prealloc_size_y, prealloc_size_split_k, prealloc_size_add_rms_partials, prealloc_size_add_rms_partials_offset;
- vk_buffer prealloc_x, prealloc_y, prealloc_split_k, prealloc_add_rms_partials, sync_staging;
- vk::Fence fence, almost_ready_fence;
- bool submit_pending {};
- bool almost_ready_fence_pending {};
- // Set before op_add and unset after op_rms_norm to indicate that the add should
- // write partial sums to accumulate the square of the vector components
- bool do_add_rms_partials_offset_calculation;
- bool do_add_rms_partials;
-
- uint64_t last_total_flops {UINT64_MAX};
-
- // Cache most recent tensor that was converted into prealloc_y, and what pipeline it used to convert.
- vk_pipeline_struct * prealloc_y_last_pipeline_used {};
- const ggml_tensor * prealloc_y_last_tensor_used {};
- // True when the K dimension in prealloc_y is padded.
- bool prealloc_y_last_k_padded {};
-
- // Track which nodes have been used since the last sync, and whether they were written to
- std::vector<const ggml_tensor *> unsynced_nodes_written;
- std::vector<const ggml_tensor *> unsynced_nodes_read;
- // Track which prealloc buffers have pending reads that need to be synchronized.
- // These are checked before writing to the buffer (and call ggml_vk_sync_buffers if set),
- // and set to true after the buffer contents are consumed.
- bool prealloc_x_need_sync, prealloc_y_need_sync, prealloc_split_k_need_sync;
-
- vk_context_ref compute_ctx;
-
- vk_context_ref transfer_ctx;
- vk_semaphore transfer_semaphore;
- uint64_t transfer_semaphore_last_submitted {};
-
- std::vector<vk_context_ref> tensor_ctxs;
-
- std::vector<vk::DescriptorPool> descriptor_pools;
- std::vector<vk::DescriptorSet> descriptor_sets;
- uint32_t descriptor_set_idx {};
- uint32_t pipeline_descriptor_set_requirements {};
-
- vk_command_pool compute_cmd_pool;
- vk_command_pool transfer_cmd_pool;
-
- // number of additional consecutive nodes that are being fused with the
- // node currently being processed
- int num_additional_fused_ops {};
- // Bitmask of which fused ops need to write an intermediate value to memory.
- // Bit 'i' means nodes[start_of_fusion + i] writes to memory.
- // If there's no fusion, bit 0 is still set.
- int fused_ops_write_mask {};
- topk_moe_mode fused_topk_moe_mode {};
- bool fused_topk_moe_scale {};
- // QSA indexer gather+add+top_k fused into one radix-select
- bool fused_topk_qsa {};
- rms_norm_mode fused_rms_norm_mode {RMS_NORM_COUNT};
-
- // for GGML_VK_PERF_LOGGER
- std::unique_ptr<vk_perf_logger> perf_logger;
- vk::QueryPool query_pool;
- std::vector<const char *> query_fusion_names;
- std::vector<int> query_fusion_node_count;
- std::vector<ggml_tensor *> query_nodes;
- std::vector<int> query_node_idx;
- int32_t num_queries {};
- int32_t query_idx {};
-};
-
-static void * const vk_ptr_base = (void *)(uintptr_t) 0x1000; // NOLINT
-
-static uint64_t vk_tensor_offset(const ggml_tensor * tensor) {
+uint64_t vk_tensor_offset(const ggml_tensor * tensor) {
if (tensor->view_src) {
return (uint8_t *) tensor->view_src->data - (uint8_t *) vk_ptr_base;
}
return (uint8_t *) tensor->data - (uint8_t *) vk_ptr_base;
}
-static void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset);
-
-static size_t ggml_vk_tensor_buffer_offset(const ggml_backend_vk_context * ctx, const ggml_tensor * t) {
+size_t ggml_vk_tensor_buffer_offset(const ggml_backend_vk_context * ctx, const ggml_tensor * t) {
// vk_tensor_offset() is relative to vk_ptr_base, but mapped host tensors need an offset relative to their Vulkan buffer.
if (ctx->device->uma) {
vk_buffer buf = nullptr;
@@ -2617,8 +258,7 @@ static size_t ggml_vk_tensor_buffer_offset(const ggml_backend_vk_context * ctx,
}
return (size_t)(vk_tensor_offset(t) + t->view_offs);
}
-
-static size_t ggml_vk_descriptor_offset(size_t tensor_offset, size_t alignment, size_t type_size) {
+size_t ggml_vk_descriptor_offset(size_t tensor_offset, size_t alignment, size_t type_size) {
// Move the descriptor back until its distance to the tensor is divisible by the tensor type size.
size_t descriptor_offset = tensor_offset & ~(alignment - 1);
while ((tensor_offset - descriptor_offset) % type_size != 0) {
@@ -2628,8 +268,7 @@ static size_t ggml_vk_descriptor_offset(size_t tensor_offset, size_t alignment,
return descriptor_offset;
}
-
-static uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t) {
+uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t) {
const size_t tensor_offset = ggml_vk_tensor_buffer_offset(ctx, t);
const size_t descriptor_offset = ggml_vk_descriptor_offset(
tensor_offset, ctx->device->properties.limits.minStorageBufferOffsetAlignment, ggml_type_size(t->type));
@@ -2637,7 +276,7 @@ static uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const gg
return tensor_offset - descriptor_offset;
}
-static uint32_t ggml_vk_concat_unit_size(ggml_type type) {
+uint32_t ggml_vk_concat_unit_size(ggml_type type) {
const uint32_t type_size = ggml_type_size(type);
if (!ggml_is_quantized(type)) {
@@ -2656,8 +295,7 @@ static uint32_t ggml_vk_concat_unit_size(ggml_type type) {
}
return 1;
}
-
-static bool ggml_vk_concat_supported(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) {
+bool ggml_vk_concat_supported(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) {
if (src0->type != src1->type || src0->type != dst->type) {
return false;
}
@@ -2670,162 +308,9 @@ static bool ggml_vk_concat_supported(const ggml_tensor * src0, const ggml_tensor
// Quantized tensor rows are block-aligned when created.
return ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(src1) && ggml_is_contiguous_rows(dst);
}
-
-template <typename T> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, T &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- GGML_UNUSED(p);
- GGML_UNUSED(src0);
- GGML_UNUSED(src1);
- GGML_UNUSED(src2);
- GGML_UNUSED(src3);
- GGML_UNUSED(dst);
- static_assert(!std::is_const<T>::value, "unexpected type");
- GGML_ASSERT(!src0 || get_misalign_bytes(ctx, src0) == 0);
- GGML_ASSERT(!src1 || get_misalign_bytes(ctx, src1) == 0);
- GGML_ASSERT(!src2 || get_misalign_bytes(ctx, src2) == 0);
- GGML_ASSERT(!src3 || get_misalign_bytes(ctx, src3) == 0);
- GGML_ASSERT(!dst || get_misalign_bytes(ctx, dst) == 0);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_mat_vec_p021_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);
- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
-
- p.b_offset = b_offset;
- p.d_offset = d_offset;
-
- GGML_UNUSED(src0);
- GGML_UNUSED(src2);
- GGML_UNUSED(src3);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_mat_vec_nc_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);
- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
-
- p.b_offset = b_offset;
- p.d_offset = d_offset;
-
- GGML_UNUSED(src0);
- GGML_UNUSED(src2);
- GGML_UNUSED(src3);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_fwht_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- p.src_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
- p.dst_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
-
- GGML_UNUSED(src1);
- GGML_UNUSED(src2);
- GGML_UNUSED(src3);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_comb_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- p.m_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
- p.s_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);
- p.b_offset = get_misalign_bytes(ctx, src2) / ggml_type_size(src2->type);
- p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
-
- GGML_UNUSED(src3);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_pre_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- p.x_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
- p.w_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);
- p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
-
- GGML_UNUSED(src2);
- GGML_UNUSED(src3);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_post_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- p.x_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
- p.r_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);
- p.p_offset = get_misalign_bytes(ctx, src2) / ggml_type_size(src2->type);
- p.c_offset = src3 ? get_misalign_bytes(ctx, src3) / ggml_type_size(src3->type) : 0;
- p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
-}
-
-struct ggml_backend_vk_buffer_context {
- vk_device_ref device;
- vk_buffer dev_buffer;
- std::string name;
-
- ggml_backend_vk_buffer_context(vk_device_ref device, vk_buffer&& dev_buffer, std::string& name) :
- device(device),
- dev_buffer(dev_buffer),
- name(name) {
- }
-
- ~ggml_backend_vk_buffer_context() {
- ggml_vk_destroy_buffer(dev_buffer);
- }
-};
-
-void vk_memory_logger::log_allocation(vk_buffer_ref buf_ref, size_t size) {
- if (!vk_memory_logger_enabled) {
- return;
- }
- std::lock_guard<std::mutex> guard(log_mutex);
- vk_buffer buf = buf_ref.lock();
- const bool device = bool(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eDeviceLocal);
- const std::string type = device ? "device" : "host";
- allocations[buf->buffer] = size;
- total_device += device ? size : 0;
- total_host += device ? 0 : size;
- VK_LOG_MEMORY(buf->device->name << ": +" << format_size(size) << " " << type << " at " << buf->buffer << ". Total device: " << format_size(total_device) << ", total host: " << format_size(total_host));
-}
-
-void vk_memory_logger::log_deallocation(vk_buffer_ref buf_ref) {
- if (buf_ref.expired() || buf_ref.lock()->size == 0 || !vk_memory_logger_enabled) {
- return;
- }
-
- std::lock_guard<std::mutex> guard(log_mutex);
- vk_buffer buf = buf_ref.lock();
- const bool device = bool(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eDeviceLocal);
- std::string type = device ? "device" : "host";
- auto it = allocations.find(buf->buffer);
- if (it != allocations.end()) {
- total_device -= device ? it->second : 0;
- total_host -= device ? 0 : it->second;
- VK_LOG_MEMORY(buf->device->name << ": -" << format_size(it->second) << " " << type << " at " << buf->buffer << ". Total device: " << format_size(total_device) << ", total host: " << format_size(total_host));
- allocations.erase(it);
- } else {
- VK_LOG_MEMORY("ERROR " << buf->device->name << ": Attempted to deallocate unknown " << type << " memory at " << buf->buffer);
- }
-}
-
-struct vk_instance_t {
- vk::Instance instance;
-
- bool debug_utils_support = false; // VK_EXT_debug_utils enabled
- PFN_vkSetDebugUtilsObjectNameEXT pfn_vkSetDebugUtilsObjectNameEXT = {};
- PFN_vkQueueBeginDebugUtilsLabelEXT pfn_vkQueueBeginDebugUtilsLabelEXT = {};
- PFN_vkQueueEndDebugUtilsLabelEXT pfn_vkQueueEndDebugUtilsLabelEXT = {};
- PFN_vkCmdBeginDebugUtilsLabelEXT pfn_vkCmdBeginDebugUtilsLabelEXT = {};
- PFN_vkCmdEndDebugUtilsLabelEXT pfn_vkCmdEndDebugUtilsLabelEXT = {};
- PFN_vkCmdInsertDebugUtilsLabelEXT pfn_vkCmdInsertDebugUtilsLabelEXT = {};
-
- std::vector<size_t> device_indices;
- std::vector<bool> device_supports_membudget;
- vk_device devices[GGML_VK_MAX_DEVICES];
-};
-
static bool vk_instance_initialized = false;
-static vk_instance_t vk_instance;
-#ifdef GGML_VULKAN_CHECK_RESULTS
-static size_t vk_skip_checks;
-static size_t vk_output_tensor;
-
-static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name);
-static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx);
-static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx);
-#endif
-
-typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
-
-static void ggml_backend_vk_free(ggml_backend_t backend);
+vk_instance_t vk_instance;
static VkDeviceSize ggml_vk_get_max_buffer_range(const ggml_backend_vk_context * ctx, const vk_buffer &buf, const VkDeviceSize offset) {
const VkDeviceSize range = std::min(VkDeviceSize{buf->size - offset},
@@ -2833,8 +318,7 @@ static VkDeviceSize ggml_vk_get_max_buffer_range(const ggml_backend_vk_context *
return range;
}
-// Wait for ctx->fence to be signaled.
-static void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) {
+void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) {
// Use waitForFences while most of the graph executes. Hopefully the CPU can sleep
// during this wait.
if (ctx->almost_ready_fence_pending) {
@@ -2876,16 +360,6 @@ static void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) {
ctx->device->device.resetFences({ ctx->fence });
}
-static constexpr uint32_t kSpvOpCooperativeMatrixLoadTensorNV = 5367;
-static constexpr uint32_t kSpvCapabilityCooperativeMatrixDecodeVectorNV = 5447;
-static constexpr uint32_t kSpvTensorAddressingDecodeVectorFuncBit = 0x4;
-
-// Remove SPV_NV_cooperative_matrix_decode_vector usage from a SPIR-V module so it
-// can be loaded on drivers that only support SPV_NV_cooperative_matrix2. Drops the
-// OpExtension declaration, the CooperativeMatrixDecodeVectorNV OpCapability, and the
-// DecodeVectorFunc operand from any OpCooperativeMatrixLoadTensorNV instruction.
-// Returns true when the input used the extension (and `out` was populated with a
-// stripped copy); returns false otherwise without touching `out`.
static bool ggml_vk_strip_decode_vector(const uint32_t * code, size_t word_count, std::vector<uint32_t> & out) {
static const char kDecodeVectorExt[] = "SPV_NV_cooperative_matrix_decode_vector";
@@ -3006,14 +480,6 @@ static bool ggml_vk_strip_decode_vector(const uint32_t * code, size_t word_count
return true;
}
-// Remove the loop unrolling hint of the matmul shader's BK loop
-// and replace it with the dont_unroll hint for better performance on
-// hardware like Apple M1/M2.
-// Assumes 1. code comes from mul_mm.comp 2. the K-tile loop has no loop
-// control hint and 3. the BK loop is the last loop nested directly inside
-// the K-tile loop.
-// Returns true when the input was modified; returns false otherwise
-// without touching `out`.
static bool ggml_vk_roll_bk_loop(const uint32_t * code, size_t word_count, std::vector<uint32_t> & out) {
if (word_count < 5) {
return false;
@@ -3338,7 +804,7 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin
device->compile_cv.notify_all();
}
-static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline) {
+void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline) {
VK_LOG_DEBUG("ggml_pipeline_destroy_pipeline(" << pipeline->name << ")");
device.destroyPipelineLayout(pipeline->layout);
@@ -3347,7 +813,7 @@ static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline)
device.destroyPipeline(pipeline->pipeline);
}
-static void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx, vk_pipeline& pipeline, uint32_t n) {
+void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx, vk_pipeline& pipeline, uint32_t n) {
VK_LOG_DEBUG("ggml_pipeline_request_descriptor_sets(" << pipeline->name << ", " << n << ")");
ctx->pipeline_descriptor_set_requirements += n;
if (!pipeline->compiled) {
@@ -3356,7 +822,7 @@ static void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx,
ggml_pipeline_allocate_descriptor_sets(ctx);
}
-static void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx) {
+void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx) {
if (ctx->descriptor_sets.size() >= ctx->pipeline_descriptor_set_requirements) {
// Enough descriptors are available
@@ -3405,7 +871,7 @@ static vk_command_buffer* ggml_vk_create_cmd_buffer(vk_device& device, vk_comman
return &p.cmd_buffers[p.cmd_buffers.size()-1];
}
-static void ggml_vk_submit(vk_context& ctx, vk::Fence fence) {
+void ggml_vk_submit(vk_context& ctx, vk::Fence fence) {
if (ctx->seqs.empty()) {
if (fence) {
ctx->p->q->handle->submit({}, fence);
@@ -3482,7 +948,7 @@ static void ggml_vk_submit(vk_context& ctx, vk::Fence fence) {
ctx->seqs.clear();
}
-static uint32_t ggml_vk_find_queue_family_index(std::vector<vk::QueueFamilyProperties>& queue_family_props, const vk::QueueFlags& required, const vk::QueueFlags& avoid, int32_t compute_index, uint32_t min_num_queues) {
+uint32_t ggml_vk_find_queue_family_index(std::vector<vk::QueueFamilyProperties>& queue_family_props, const vk::QueueFlags& required, const vk::QueueFlags& avoid, int32_t compute_index, uint32_t min_num_queues) {
VK_LOG_DEBUG("ggml_vk_find_queue_family_index()");
const uint32_t qfsize = queue_family_props.size();
@@ -3528,7 +994,7 @@ static uint32_t ggml_vk_find_queue_family_index(std::vector<vk::QueueFamilyPrope
abort();
}
-static std::unique_ptr<vk_queue> ggml_vk_create_queue(vk_device& device, uint32_t queue_family_index, uint32_t queue_index, vk::PipelineStageFlags&& stage_flags, bool transfer_only) {
+std::unique_ptr<vk_queue> ggml_vk_create_queue(vk_device& device, uint32_t queue_family_index, uint32_t queue_index, vk::PipelineStageFlags&& stage_flags, bool transfer_only) {
VK_LOG_DEBUG("ggml_vk_create_queue()");
std::lock_guard<std::recursive_mutex> guard(device->mutex);
@@ -3562,7 +1028,7 @@ static std::unique_ptr<vk_queue> ggml_vk_create_queue(vk_device& device, uint32_
return q;
}
-static std::unique_ptr<vk_queue> ggml_vk_create_aliased_queue(vk_device& device, const std::unique_ptr<vk_queue>& source) {
+std::unique_ptr<vk_queue> ggml_vk_create_aliased_queue(vk_device& device, const std::unique_ptr<vk_queue>& source) {
std::lock_guard<std::recursive_mutex> guard(device->mutex);
auto q = std::make_unique<vk_queue>();
q->handle = source->handle;
@@ -3573,7 +1039,7 @@ static std::unique_ptr<vk_queue> ggml_vk_create_aliased_queue(vk_device& device,
return q;
}
-static vk_context ggml_vk_create_context(ggml_backend_vk_context * ctx, vk_command_pool& p) {
+vk_context ggml_vk_create_context(ggml_backend_vk_context * ctx, vk_command_pool& p) {
vk_context result = std::make_shared<vk_context_struct>();
VK_LOG_DEBUG("ggml_vk_create_context(" << result << ")");
ctx->gc.contexts.emplace_back(result);
@@ -3581,7 +1047,7 @@ static vk_context ggml_vk_create_context(ggml_backend_vk_context * ctx, vk_comma
return result;
}
-static vk_context ggml_vk_create_temporary_context(vk_command_pool& p) {
+vk_context ggml_vk_create_temporary_context(vk_command_pool& p) {
vk_context result = std::make_shared<vk_context_struct>();
VK_LOG_DEBUG("ggml_vk_create_temporary_context(" << result << ")");
result->p = &p;
@@ -3617,7 +1083,7 @@ static vk::Event ggml_vk_create_event(ggml_backend_vk_context * ctx) {
return ctx->gc.events[ctx->event_idx++];
}
-static void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p) {
+void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p) {
VK_LOG_DEBUG("ggml_vk_command_pool_cleanup()");
// Requires command buffers to be done
@@ -3629,7 +1095,7 @@ static void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p)
}
}
-static void ggml_vk_queue_command_pools_cleanup(vk_device& device) {
+void ggml_vk_queue_command_pools_cleanup(vk_device& device) {
VK_LOG_DEBUG("ggml_vk_queue_command_pools_cleanup()");
// Arbitrary frequency to cleanup/reuse command buffers
@@ -3643,243 +1109,11 @@ static void ggml_vk_queue_command_pools_cleanup(vk_device& device) {
}
}
-static std::vector<uint32_t> ggml_vk_find_memory_properties(const vk::PhysicalDeviceMemoryProperties* mem_props, vk::MemoryRequirements* mem_req, vk::MemoryPropertyFlags flags) {
- std::vector<uint32_t> indices;
-
- for (uint32_t i = 0; i < mem_props->memoryTypeCount; ++i) {
- vk::MemoryType memory_type = mem_props->memoryTypes[i];
- if ((mem_req->memoryTypeBits & ((uint64_t)1 << i)) &&
- (flags & memory_type.propertyFlags) == flags &&
- mem_props->memoryHeaps[memory_type.heapIndex].size >= mem_req->size) {
- indices.push_back(i);
- }
- }
- return indices;
-}
-
-static vk_buffer ggml_vk_create_buffer(vk_device& device, size_t size, const std::initializer_list<vk::MemoryPropertyFlags> & req_flags_list,
- void *import_ptr = nullptr) {
- VK_LOG_DEBUG("ggml_vk_create_buffer(" << device->name << ", " << size << ", " << to_string(req_flags_list.begin()[0]) << ", " << to_string(req_flags_list.begin()[req_flags_list.size()-1]) << ")");
- if (size > device->max_buffer_size) {
- throw vk::OutOfDeviceMemoryError("Requested buffer size exceeds device buffer size limit");
- }
-
- vk_buffer buf = std::make_shared<vk_buffer_struct>();
-
- if (size == 0) {
- buf->size = 0;
- return buf;
- }
-
- vk::BufferUsageFlags usage_flags = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst;
- vk::MemoryAllocateFlags mem_flags {};
- if (device->buffer_device_address) {
- usage_flags |= vk::BufferUsageFlagBits::eShaderDeviceAddress;
- mem_flags |= vk::MemoryAllocateFlagBits::eDeviceAddress;
- }
-
- vk::BufferCreateInfo buffer_create_info{
- vk::BufferCreateFlags(),
- size,
- usage_flags,
- vk::SharingMode::eExclusive,
- 0,
- nullptr,
- };
-
- vk::ExternalMemoryBufferCreateInfo external_memory_bci;
- if (import_ptr) {
- external_memory_bci.handleTypes = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT;
- buffer_create_info.setPNext(&external_memory_bci);
- }
-
- buf->buffer = device->device.createBuffer(buffer_create_info);
-
- vk::MemoryRequirements mem_req = device->device.getBufferMemoryRequirements(buf->buffer);
-
- vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties();
-
- const vk::MemoryPriorityAllocateInfoEXT mem_priority_info { 1.0f };
-
- vk::MemoryAllocateFlagsInfo mem_flags_info { mem_flags };
-
- if (device->memory_priority) {
- mem_flags_info.setPNext(&mem_priority_info);
- }
-
- if (import_ptr) {
- vk::MemoryHostPointerPropertiesEXT host_pointer_props;
- try {
- host_pointer_props = device->device.getMemoryHostPointerPropertiesEXT(vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT, import_ptr);
- } catch (vk::SystemError& e) {
- GGML_LOG_WARN("ggml_vulkan: Failed getMemoryHostPointerPropertiesEXT (%s)\n", e.what());
- device->device.destroyBuffer(buf->buffer);
- return {};
- }
- vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties();
-
- uint32_t memory_type_idx;
- vk::MemoryPropertyFlags property_flags = *req_flags_list.begin();
- for (memory_type_idx = 0; memory_type_idx < 32; ++memory_type_idx) {
- if (!(host_pointer_props.memoryTypeBits & (1u << memory_type_idx))) {
- continue;
- }
- if (!(mem_req.memoryTypeBits & (1u << memory_type_idx))) {
- continue;
- }
-
- vk::MemoryType memory_type = mem_props.memoryTypes[memory_type_idx];
- // check for visible+coherent+cached. Other flags (e.g. devicelocal) are allowed
- if ((memory_type.propertyFlags & property_flags) == property_flags) {
- property_flags = memory_type.propertyFlags;
- break;
- }
- }
- if (memory_type_idx == 32) {
- GGML_LOG_WARN("ggml_vulkan: Memory type for host allocation not found\n");
- device->device.destroyBuffer(buf->buffer);
- return {};
- }
-
- buf->memory_property_flags = mem_props.memoryTypes[memory_type_idx].propertyFlags;
- try {
- vk::ImportMemoryHostPointerInfoEXT import_info;
- import_info.handleType = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT;
- import_info.pHostPointer = import_ptr;
- import_info.setPNext(&mem_flags_info);
- buf->device_memory = device->device.allocateMemory({ size, memory_type_idx, &import_info });
- } catch (const vk::SystemError& e) {
- }
- } else {
- for (auto it = req_flags_list.begin(); it != req_flags_list.end(); it++) {
- const auto & req_flags = *it;
-
- const std::vector<uint32_t> memory_type_indices = ggml_vk_find_memory_properties(&mem_props, &mem_req, req_flags);
-
- if (memory_type_indices.empty()) {
- continue;
- }
-
- bool done = false;
-
- for (auto mtype_it = memory_type_indices.begin(); mtype_it != memory_type_indices.end(); mtype_it++) {
- try {
- buf->device_memory = device->device.allocateMemory({ mem_req.size, *mtype_it, &mem_flags_info });
- buf->memory_property_flags = mem_props.memoryTypes[*mtype_it].propertyFlags;
- done = true;
- break;
- } catch (const vk::SystemError& e) {
- // loop and retry
- // during last attempt throw the exception
- if (it + 1 == req_flags_list.end() && mtype_it + 1 == memory_type_indices.end()) {
- device->device.destroyBuffer(buf->buffer);
- throw e;
- }
- }
- }
-
- if (done) {
- break;
- }
- }
- }
-
- if (!buf->device_memory) {
- device->device.destroyBuffer(buf->buffer);
- throw vk::OutOfDeviceMemoryError("No suitable memory type found");
- }
-
- buf->ptr = nullptr;
-
- if (import_ptr) {
- buf->ptr = import_ptr;
- } else {
- if (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {
- buf->ptr = device->device.mapMemory(buf->device_memory, 0, VK_WHOLE_SIZE);
- }
- }
-
- device->device.bindBufferMemory(buf->buffer, buf->device_memory, 0);
-
- buf->device = device;
- buf->size = size;
-
- if (device->buffer_device_address) {
- const vk::BufferDeviceAddressInfo addressInfo(buf->buffer);
- buf->bda_addr = device->device.getBufferAddress(addressInfo);
- }
-
- device->memory_logger->log_allocation(buf, size);
-
- return buf;
-}
-
-static vk_buffer ggml_vk_create_buffer_check(vk_device& device, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags = vk::MemoryPropertyFlags(0)) {
- try {
- return ggml_vk_create_buffer(device, size, {req_flags, fallback_flags});
- } catch (const vk::SystemError& e) {
- std::cerr << "ggml_vulkan: Memory allocation of size " << size << " failed." << std::endl;
- std::cerr << "ggml_vulkan: " << e.what() << std::endl;
- throw e;
- }
-}
-
-static vk_buffer ggml_vk_create_buffer_device(vk_device& device, size_t size) {
- vk_buffer buf;
- try {
- if (device->prefer_host_memory) {
- buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
- vk::MemoryPropertyFlagBits::eDeviceLocal});
- } else if (device->uma) {
- // On UMA, prefer host-visible memory so direct tensor borrowing works.
- // If unavailable, fall back to device-local memory.
- buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
- vk::MemoryPropertyFlagBits::eDeviceLocal,
- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
- } else if (device->disable_host_visible_vidmem) {
- if (device->allow_sysmem_fallback) {
- buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal,
- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
- } else {
- buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal});
- }
- } else {
- // use rebar if available, otherwise fallback to device only visible memory
- if (device->allow_sysmem_fallback) {
- buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
- vk::MemoryPropertyFlagBits::eDeviceLocal,
- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
- } else {
- buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
- vk::MemoryPropertyFlagBits::eDeviceLocal});
- }
- }
- } catch (const vk::SystemError& e) {
- std::cerr << "ggml_vulkan: Device memory allocation of size " << size << " failed." << std::endl;
- std::cerr << "ggml_vulkan: " << e.what() << std::endl;
- throw e;
- }
-
- return buf;
-}
-
-static void ggml_vk_destroy_buffer(vk_buffer& buf) {
- if (buf == nullptr) {
- return;
- }
-
- if (buf->device != nullptr) {
- buf->device->memory_logger->log_deallocation(buf);
- }
-
- buf.reset();
-}
-
-static vk_subbuffer ggml_vk_subbuffer(const ggml_backend_vk_context* ctx, const vk_buffer& buf, size_t offset = 0) {
+vk_subbuffer ggml_vk_subbuffer(const ggml_backend_vk_context* ctx, const vk_buffer& buf, size_t offset) {
return { buf, offset, ggml_vk_get_max_buffer_range(ctx, buf, offset) };
}
-static void ggml_vk_sync_buffers(ggml_backend_vk_context* ctx, vk_context& subctx) {
+void ggml_vk_sync_buffers(ggml_backend_vk_context* ctx, vk_context& subctx) {
VK_LOG_DEBUG("ggml_vk_sync_buffers()");
const bool transfer_queue = subctx->p->q->transfer_only;
@@ -3910,7 +1144,7 @@ static void ggml_vk_reset_event(vk_context& ctx, vk::Event& event) {
);
}
-static void ggml_vk_set_event(vk_context& ctx, vk::Event& event) {
+void ggml_vk_set_event(vk_context& ctx, vk::Event& event) {
VK_LOG_DEBUG("ggml_vk_set_event()");
ctx->s->buffer->buf.setEvent(
@@ -3919,7 +1153,7 @@ static void ggml_vk_set_event(vk_context& ctx, vk::Event& event) {
);
}
-static void ggml_vk_wait_events(vk_context& ctx, std::vector<vk::Event>&& events) {
+void ggml_vk_wait_events(vk_context& ctx, std::vector<vk::Event>&& events) {
VK_LOG_DEBUG("ggml_vk_wait_events()");
if (events.empty()) {
return;
@@ -3935,29 +1169,6 @@ static void ggml_vk_wait_events(vk_context& ctx, std::vector<vk::Event>&& events
);
}
-struct vk_fa_tuning_params {
- FaCodePath path;
- uint32_t workgroup_size;
- uint32_t subgroup_size;
- uint32_t block_rows;
- uint32_t block_cols;
- uint32_t d_split;
- uint32_t row_split;
- bool shmem_staging;
- bool disable_subgroups;
- uint32_t limit_occupancy_shmem;
-
- void print() const {
- std::cerr << "path=" << path << " workgroup_size=" << workgroup_size << " subgroup_size=" << subgroup_size <<
- " block_rows=" << block_rows << " block_cols=" << block_cols << " d_split=" << d_split <<
- " row_split=" << row_split << " shmem_staging=" << shmem_staging << " disable_subgroups=" << disable_subgroups <<
- " limit_occupancy_shmem=" << limit_occupancy_shmem << std::endl;
- }
-};
-
-static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type);
-static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type = GGML_TYPE_F16, ggml_type v_type = GGML_TYPE_F16);
-
static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) {
vk_fa_tuning_params result{};
@@ -4092,7 +1303,7 @@ static vk_fa_tuning_params get_fa_tuning_params_coopmat2(const vk_device& device
return result;
}
-static vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) {
+vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) {
FaCodePath path = device->coopmat2 ? FA_COOPMAT2 :
device->coopmat1_fa_support ? FA_COOPMAT1 : FA_SCALAR;
@@ -4136,7 +1347,7 @@ static vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_
}
}
-static vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc,
+vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc,
bool use_mask, bool use_mask_opt, bool use_logit_softcap, bool use_sparse, ggml_type k_type, ggml_type v_type) {
const bool old_amd_windows = device->vendor_id == VK_VENDOR_ID_AMD && device->driver_id == vk::DriverId::eAmdProprietary &&
(device->architecture == AMD_GCN || device->architecture == AMD_RDNA1 || device->architecture == AMD_RDNA2);
@@ -4152,8 +1363,6 @@ static vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const
return vk_fa_pipeline_state{hsk, hsv, params.block_rows, params.block_cols, params.d_split, params.row_split, params.shmem_staging, params.path, params.workgroup_size, subgroup_size, aligned, f32acc, flags, params.limit_occupancy_shmem, k_type, v_type};
}
-// Bytes per buffer block for the FaBlockBytesK/V spec constants. F32 is fed as
-// a vec4 "block" of 4 floats, everything else uses its ggml block size.
static uint32_t fa_block_bytes(ggml_type t) {
if (t == GGML_TYPE_F32) {
return 16u;
@@ -4242,10 +1451,6 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec
return supported;
}
-// Shmem usage for the q8_1 mmq shader (mul_mmq.comp), which uses
-// block_a_cache / block_b_cache layouts (see mul_mmq_shmem_types.glsl) rather
-// than the float load buffers checked by ggml_vk_matmul_shmem_support.
-// Sizes follow std430 rules. Returns false for types without a q8_1 pipeline.
static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std::vector<uint32_t>& warptile, bool mul_mat_id, ggml_type src0_type) {
// FLOAT_TYPE in the shader is float16_t with fp16 support, otherwise float.
@@ -4307,35 +1512,16 @@ static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std:
return supported;
}
-struct GpuPipelineConfig {
- // GPU architecture identifier.
- // Example: vk_device_architecture::AMD_GCN
- vk_device_architecture arch;
-
- // Mapping of pipeline names to their specific subgroup sizes.
- // Example: {"soft_max_f32", 64}
- std::unordered_map<std::string, uint32_t> pipelines;
-
- // Default subgroup size for this GPU.
- // Defaults to 0 if not explicitly provided.
- uint32_t default_subgroup_size = 0;
-};
-
-// Pipeline configuration for RDNA1 GPUs.
static const std::unordered_map<std::string, uint32_t> rdna1_pipelines = {
{"soft_max", 64}, {"im2col", 64},
{"argmax", 64}, {"mul_mat_vec", 64},
{"mul_mat_vec_f16", 32}, {"mul_mat_vec_f32_f16", 32}
};
-// Pipeline configuration for RDNA2 GPUs.
static const std::unordered_map<std::string, uint32_t> rdna2_pipelines = {
{"soft_max", 64}, {"im2col", 64},
};
-static constexpr uint32_t RDNA_DEFAULT_SUBGROUP_SIZE = 32;
-
-// Define configurations for different GPUs.
static std::vector<GpuPipelineConfig> gpu_pipeline_configs = {
{
vk_device_architecture::AMD_RDNA1,
@@ -4353,7 +1539,7 @@ static std::vector<GpuPipelineConfig> gpu_pipeline_configs = {
},
};
-static uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_device_architecture &arch) {
+uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_device_architecture &arch) {
for (const auto &config : gpu_pipeline_configs) {
if (config.arch == arch) {
auto pipIt = config.pipelines.find(pipeline_name);
@@ -4374,7 +1560,6 @@ static uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_dev
return 0; // If no matching configuration is found
}
-// Whether scalar flash attention will use the MMQ path for the given K/V types.
static bool ggml_vk_fa_type_needs_shmem(ggml_type type) {
switch (type) {
case GGML_TYPE_IQ4_NL:
@@ -4399,24 +1584,7 @@ static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type
#endif
}
-// load_shaders walks the pipeline list under compile_mutex and either claims
-// the requested pipeline for compilation or, if another thread is already
-// compiling it, drops the lock and waits on compile_cv. Compiles themselves
-// run unlocked.
-struct CompileTask {
- vk_pipeline pipeline;
- size_t spv_size;
- const void * spv_data;
- std::string entrypoint;
- uint32_t parameter_count;
- std::array<uint32_t, 3> wg_denoms;
- std::vector<uint32_t> specialization_constants;
- bool disable_robustness;
- bool require_full_subgroups;
- uint32_t required_subgroup_size;
-};
-
-static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
+void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
VK_LOG_DEBUG("ggml_vk_load_shaders(" << device->name << ")");
// some shaders have a minimum subgroup size
@@ -6560,10 +3728,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
}
}
-static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch);
-static uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev);
-
-static vk_device ggml_vk_get_device(size_t idx) {
+vk_device ggml_vk_get_device(size_t idx) {
VK_LOG_DEBUG("ggml_vk_get_device(" << idx << ")");
if (vk_instance.devices[idx] == nullptr) {
@@ -7738,17 +4903,13 @@ static void ggml_vk_print_gpu_info(size_t idx) {
}
}
-static bool ggml_vk_instance_layer_settings_available();
-static bool ggml_vk_instance_portability_enumeration_ext_available(const std::vector<vk::ExtensionProperties>& instance_extensions);
-static bool ggml_vk_instance_debug_utils_ext_available(const std::vector<vk::ExtensionProperties> & instance_extensions);
-static bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev);
-
static DispatchLoaderDynamic ggml_vk_default_dispatcher_instance;
+
DispatchLoaderDynamic & ggml_vk_default_dispatcher() {
return ggml_vk_default_dispatcher_instance;
}
-static void ggml_vk_instance_init() {
+void ggml_vk_instance_init() {
if (vk_instance_initialized) {
return;
}
@@ -7998,7 +5159,7 @@ static void ggml_vk_instance_init() {
}
}
-static void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) {
+void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) {
VK_LOG_DEBUG("ggml_vk_init(" << ctx->name << ", " << idx << ")");
ggml_vk_instance_init();
GGML_ASSERT(idx < vk_instance.device_indices.size());
@@ -8042,7 +5203,7 @@ static void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) {
#endif
}
-static vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type) {
+vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type) {
VK_LOG_DEBUG("ggml_vk_get_to_fp16()");
switch (type) {
case GGML_TYPE_F32:
@@ -8248,77 +5409,14 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec_id(ggml_backend_vk_context
if (ctx->device->vendor_id == VK_VENDOR_ID_INTEL) {
dmmv_wg = DMMV_WG_SIZE_SUBGROUP;
}
- return ctx->device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[dmmv_wg][a_type];
- }
-
- return ctx->device->pipeline_dequant_mul_mat_vec_id_f32[dmmv_wg][a_type];
-}
-
-static void * ggml_vk_host_malloc(vk_device& device, size_t size) {
- VK_LOG_MEMORY("ggml_vk_host_malloc(" << size << ")");
- vk_buffer buf = ggml_vk_create_buffer(device, size,
- {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,
- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
-
- if(!(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible)) {
- fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory\n",
- size/1024.0/1024.0);
- device->device.freeMemory(buf->device_memory);
- device->device.destroyBuffer(buf->buffer);
- return nullptr;
- }
-
- std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex);
- device->pinned_memory.push_back(std::make_tuple(buf->ptr, size, buf));
-
- return buf->ptr;
-}
-
-static void ggml_vk_host_free(vk_device& device, void* ptr) {
- if (ptr == nullptr) {
- return;
- }
- VK_LOG_MEMORY("ggml_vk_host_free(" << ptr << ")");
- std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex);
-
- vk_buffer buf;
- size_t index;
- for (size_t i = 0; i < device->pinned_memory.size(); i++) {
- const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]);
- const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]);
- if (ptr >= addr && ptr < endr) {
- buf = std::get<2>(device->pinned_memory[i]);
- index = i;
- break;
- }
- }
- if (buf == nullptr) {
- fprintf(stderr, "WARNING: failed to free pinned memory: memory not in map\n");
- return;
- }
-
- ggml_vk_destroy_buffer(buf);
-
- device->pinned_memory.erase(device->pinned_memory.begin() + index);
-}
-
-static void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset) {
- std::shared_lock<std::shared_mutex> guard(device->pinned_memory_mutex);
- buf = nullptr;
- buf_offset = 0;
- for (size_t i = 0; i < device->pinned_memory.size(); i++) {
- const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]);
- const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]);
- if (ptr >= addr && ptr < endr) {
- buf = std::get<2>(device->pinned_memory[i]);
- buf_offset = ((const uint8_t *)ptr) - addr;
- break;
- }
+ return ctx->device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[dmmv_wg][a_type];
}
+
+ return ctx->device->pipeline_dequant_mul_mat_vec_id_f32[dmmv_wg][a_type];
}
-static vk_subbuffer ggml_vk_tensor_subbuffer(
- const ggml_backend_vk_context * ctx, const ggml_tensor * tensor, bool allow_misalign = false) {
+vk_subbuffer ggml_vk_tensor_subbuffer(
+ const ggml_backend_vk_context * ctx, const ggml_tensor * tensor, bool allow_misalign) {
vk_buffer buffer = nullptr;
size_t offset = 0;
@@ -8345,7 +5443,6 @@ static vk_subbuffer ggml_vk_tensor_subbuffer(
return vk_subbuffer{buffer, offset, size};
}
-// Get a command buffer from pool. Create a new one if no reusable buffer is available
static vk_command_buffer* ggml_vk_get_or_create_cmd_buffer(vk_device& device, vk_command_pool& pool) {
for (auto& cmd_buffer : pool.cmd_buffers) {
if (!cmd_buffer.in_use) {
@@ -8369,156 +5466,14 @@ static vk_submission ggml_vk_begin_submission(vk_device& device, vk_command_pool
return s;
}
-template <typename T> size_t push_constant_size(const T &t) {
- static_assert(std::is_class<T>::value, "T must be a struct/class");
- GGML_UNUSED(t);
- return sizeof(T);
-}
-template <typename T> size_t push_constant_size(const std::vector<T> &t) {
- GGML_UNUSED(t);
- return sizeof(T) * t.size();
-}
-template <typename T, uint32_t N> size_t push_constant_size(const std::array<T, N> &t) {
- GGML_UNUSED(t);
- return sizeof(T) * N;
-}
-
-template <typename T> const T *push_constant_data(const T &t) {
- static_assert(std::is_class<T>::value, "T must be a struct/class");
- return &t;
-}
-template <typename T> const T *push_constant_data(const std::vector<T> &t) {
- return t.data();
-}
-template <typename T, uint32_t N> const T *push_constant_data(const std::array<T, N> &t) {
- return t.data();
-}
-
-static void ggml_vk_cmd_label_begin(vk::CommandBuffer buf, const char * name) {
+void ggml_vk_cmd_label_begin(vk::CommandBuffer buf, const char * name) {
vk::DebugUtilsLabelEXT label = {};
label.pLabelName = name;
label.color = std::array<float, 4>{1.0f, 1.0f, 1.0f, 1.0f};
vk_instance.pfn_vkCmdBeginDebugUtilsLabelEXT(buf, reinterpret_cast<VkDebugUtilsLabelEXT *>(&label));
}
-// no-op unless GGML_VK_DEBUG_MARKERS is set
-struct ggml_vk_debug_label {
- // at most one of these is set, depending on the scope the label was opened in
- vk_context_struct * subctx {};
- vk_queue_handle * qhandle {};
-
- // one region per dispatch, e.g. "matmul_q4_k_f32_f16acc_aligned_m (192,8,1)".
- // RGP cannot recover the pipeline name on its own, it only has the hash
- ggml_vk_debug_label(vk_context & ctx, const std::string & pipeline_name, uint32_t wg0, uint32_t wg1, uint32_t wg2) {
- if (!vk_instance.debug_utils_support || ctx->s == nullptr) {
- return;
- }
- begin(ctx, pipeline_name + " (" + std::to_string(wg0) + "," + std::to_string(wg1) + "," + std::to_string(wg2) + ")");
- }
-
- // one region per graph node
- // fused nodes are joined with '+', e.g. "RMS_NORM+MUL+ROPE Qcur-19"
- ggml_vk_debug_label(vk_context & ctx, const ggml_cgraph * cgraph, int node_idx, int n_fused) {
- if (!vk_instance.debug_utils_support || ctx->s == nullptr) {
- return;
- }
- std::string name = ggml_op_name(cgraph->nodes[node_idx]->op);
- for (int i = 1; i <= n_fused; i++) {
- name += "+";
- name += ggml_op_name(cgraph->nodes[node_idx + i]->op);
- }
- name += " ";
- name += cgraph->nodes[node_idx]->name;
- begin(ctx, name);
- }
-
- // one region per graph evaluation, opened on the queue instead of a command buffer
- // so it spans every submit the evaluation makes
- ggml_vk_debug_label(vk_queue_handle * handle, const char * name) {
- if (!vk_instance.debug_utils_support || handle == nullptr) {
- return;
- }
- vk::DebugUtilsLabelEXT label = {};
- label.pLabelName = name;
- label.color = std::array<float, 4>{1.0f, 1.0f, 1.0f, 1.0f};
-
- qhandle = handle;
- std::lock_guard<vk_queue_handle> guard(*qhandle);
- vk_instance.pfn_vkQueueBeginDebugUtilsLabelEXT(qhandle->queue, reinterpret_cast<VkDebugUtilsLabelEXT *>(&label));
- }
-
- // call before the command buffer can end, the destructor covers the rest
- void close() {
- if (subctx != nullptr) {
- // close on the current command buffer, which may differ from the one begin used
- if (subctx->s != nullptr) {
- vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(subctx->s->buffer->buf);
- }
- subctx->debug_labels.pop_back();
- subctx = nullptr;
- }
- if (qhandle != nullptr) {
- std::lock_guard<vk_queue_handle> guard(*qhandle);
- vk_instance.pfn_vkQueueEndDebugUtilsLabelEXT(qhandle->queue);
- qhandle = nullptr;
- }
- }
-
- ~ggml_vk_debug_label() {
- close();
- }
-
- ggml_vk_debug_label(const ggml_vk_debug_label &) = delete;
- ggml_vk_debug_label & operator=(const ggml_vk_debug_label &) = delete;
-
-private:
- // the constructors check this too, so the name is not built when markers are off
- void begin(vk_context & ctx, const std::string & name) {
- if (!vk_instance.debug_utils_support || ctx->s == nullptr) {
- return;
- }
- subctx = ctx.get();
- subctx->debug_labels.push_back(name);
- ggml_vk_cmd_label_begin(subctx->s->buffer->buf, subctx->debug_labels.back().c_str());
- }
-};
-
-template <typename T>
-static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& subctx, vk_pipeline& pipeline, std::initializer_list<vk::DescriptorBufferInfo> const& descriptor_buffer_infos, const T &push_constants, std::array<uint32_t, 3> elements) {
- const uint32_t wg0 = CEIL_DIV(elements[0], pipeline->wg_denoms[0]);
- const uint32_t wg1 = CEIL_DIV(elements[1], pipeline->wg_denoms[1]);
- const uint32_t wg2 = CEIL_DIV(elements[2], pipeline->wg_denoms[2]);
- VK_LOG_DEBUG("ggml_vk_dispatch_pipeline(" << pipeline->name << ", {";
- for (auto& buffer : descriptor_buffer_infos) {
- std::cerr << "(" << buffer.buffer << ", " << buffer.offset << ", " << buffer.range << "), ";
- }
- std::cerr << "}, (" << wg0 << "," << wg1 << "," << wg2 << "))");
- GGML_ASSERT(wg0 <= ctx->device->properties.limits.maxComputeWorkGroupCount[0] &&
- wg1 <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] &&
- wg2 <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]);
- GGML_ASSERT(ctx->descriptor_set_idx < ctx->descriptor_sets.size());
- GGML_ASSERT(descriptor_buffer_infos.size() <= MAX_PARAMETER_COUNT);
- GGML_ASSERT(pipeline->parameter_count == descriptor_buffer_infos.size());
- GGML_ASSERT(pipeline->push_constant_size == push_constant_size(push_constants));
-
- vk::DescriptorSet& descriptor_set = ctx->descriptor_sets[ctx->descriptor_set_idx++];
- vk::WriteDescriptorSet write_descriptor_set{ descriptor_set, 0, 0, pipeline->parameter_count, vk::DescriptorType::eStorageBuffer, nullptr, descriptor_buffer_infos.begin() };
- ctx->device->device.updateDescriptorSets({ write_descriptor_set }, {});
-
- subctx->s->buffer->buf.pushConstants(pipeline->layout, vk::ShaderStageFlagBits::eCompute, 0, push_constant_size(push_constants), push_constant_data(push_constants));
- subctx->s->buffer->buf.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline->pipeline);
- subctx->s->buffer->buf.bindDescriptorSets(vk::PipelineBindPoint::eCompute,
- pipeline->layout,
- 0,
- { descriptor_set },
- {});
- {
- ggml_vk_debug_label dbg(subctx, pipeline->name, wg0, wg1, wg2);
- subctx->s->buffer->buf.dispatch(wg0, wg1, wg2);
- }
-}
-
-static void ggml_vk_ctx_end(vk_context& ctx) {
+void ggml_vk_ctx_end(vk_context& ctx) {
VK_LOG_DEBUG("ggml_vk_ctx_end(" << ctx << ", " << ctx->seqs.size() << ")");
if (ctx->s == nullptr) {
return;
@@ -8537,7 +5492,7 @@ static void ggml_vk_ctx_end(vk_context& ctx) {
ctx->s = nullptr;
}
-static void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx) {
+void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx) {
VK_LOG_DEBUG("ggml_vk_ctx_begin(" << device->name << ")");
if (subctx->s != nullptr) {
ggml_vk_ctx_end(subctx);
@@ -8558,7 +5513,7 @@ static void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx) {
}
}
-static vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx) {
+vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx) {
vk_context result;
if (!ctx->compute_ctx.expired()) {
result = ctx->compute_ctx.lock();
@@ -8577,7 +5532,7 @@ static vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx) {
return result;
}
-static vk_context ggml_vk_get_transfer_ctx(ggml_backend_vk_context * ctx) {
+vk_context ggml_vk_get_transfer_ctx(ggml_backend_vk_context * ctx) {
vk_context result;
if (!ctx->transfer_ctx.expired()) {
result = ctx->transfer_ctx.lock();
@@ -8591,10 +5546,7 @@ static vk_context ggml_vk_get_transfer_ctx(ggml_backend_vk_context * ctx) {
return result;
}
-// Submit any pending transfer queue work and signal the transfer semaphore.
-// The next compute context created via ggml_vk_get_compute_ctx will wait on this semaphore.
-// Returns true if work was submitted.
-static bool ggml_vk_submit_transfer_ctx(ggml_backend_vk_context * ctx) {
+bool ggml_vk_submit_transfer_ctx(ggml_backend_vk_context * ctx) {
if (!ctx->device->async_use_transfer_queue || ctx->transfer_ctx.expired()) {
return false;
}
@@ -8614,12 +5566,12 @@ static bool ggml_vk_submit_transfer_ctx(ggml_backend_vk_context * ctx) {
return true;
}
-static size_t ggml_vk_align_size(size_t width, size_t align) {
+size_t ggml_vk_align_size(size_t width, size_t align) {
VK_LOG_DEBUG("ggml_vk_align_size(" << width << ", " << align << ")");
return CEIL_DIV(width, align) * align;
}
-static void deferred_memcpy(void * dst, const void * src, size_t size, std::vector<vk_staging_memcpy>* memcpys = nullptr) {
+void deferred_memcpy(void * dst, const void * src, size_t size, std::vector<vk_staging_memcpy>* memcpys) {
if (memcpys == nullptr) {
memcpy(dst, src, size);
} else {
@@ -8627,7 +5579,7 @@ static void deferred_memcpy(void * dst, const void * src, size_t size, std::vect
}
}
-static void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<vk_staging_memset>* memsets = nullptr) {
+void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<vk_staging_memset>* memsets) {
if (memsets == nullptr) {
memset(dst, val, size);
} else {
@@ -8635,444 +5587,6 @@ static void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<v
}
}
-static void ggml_vk_ensure_sync_staging_buffer(vk_device& device, size_t size) {
- if (device->sync_staging == nullptr || device->sync_staging->size < size) {
- VK_LOG_MEMORY("ggml_vk_ensure_sync_staging_buffer(" << size << ")");
- ggml_vk_destroy_buffer(device->sync_staging);
- device->sync_staging = ggml_vk_create_buffer_check(device, size,
- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,
- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent);
- }
-}
-
-static void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t size) {
- if (ctx->sync_staging == nullptr || ctx->sync_staging->size < size) {
- VK_LOG_MEMORY("ggml_vk_ensure_sync_staging_buffer(" << size << ")");
- ggml_vk_destroy_buffer(ctx->sync_staging);
- ctx->sync_staging = ggml_vk_create_buffer_check(ctx->device, size,
- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,
- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent);
- }
-}
-
-static void ggml_vk_buffer_write_nc_async(ggml_backend_vk_context * ctx, vk_context& subctx, vk_buffer& dst, size_t offset, const ggml_tensor * tensor, bool sync_staging = false) {
- VK_LOG_DEBUG("ggml_vk_buffer_write_nc_async(" << tensor << ")");
- GGML_ASSERT(!ggml_is_contiguous(tensor));
- // Buffer is already mapped
- if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {
- std::cerr << "ggml_vulkan: buffer_write_nc_async dst buffer is host_visible. Use synchronous write." << std::endl;
- GGML_ABORT("fatal error");
- }
- // Check if src is pinned memory
- vk_buffer buf = nullptr;
- size_t buf_offset = 0;
- ggml_vk_host_get(ctx->device, tensor->data, buf, buf_offset);
-
- const uint64_t ne0 = tensor->ne[0];
- const uint64_t ne1 = tensor->ne[1];
- const uint64_t ne2 = tensor->ne[2];
- const uint64_t ne3 = tensor->ne[3];
- const uint64_t nb0 = tensor->nb[0];
- const uint64_t nb1 = tensor->nb[1];
- const uint64_t nb2 = tensor->nb[2];
- const uint64_t nb3 = tensor->nb[3];
- const ggml_type type = tensor->type;
- const uint64_t ts = ggml_type_size(type);
- const uint64_t bs = ggml_blck_size(type);
-
- const uint64_t dstnb0 = ts;
- const uint64_t dstnb1 = dstnb0*(ne0/bs);
- const uint64_t dstnb2 = dstnb1*ne1;
- const uint64_t dstnb3 = dstnb2*ne2;
-
- const uint64_t ne = ggml_nelements(tensor);
-
- if (buf != nullptr) {
- // Memory is pinned, use as staging buffer
- std::vector<vk::BufferCopy> slices;
-
- for (uint64_t i3 = 0; i3 < ne3; i3++) {
- for (uint64_t i2 = 0; i2 < ne2; i2++) {
- // Find longest contiguous slice
- if (ne1*nb1 == dstnb2) {
- slices.push_back({ buf_offset + i3*nb3 + i2*nb2, offset + i3*dstnb3 + i2*dstnb2, dstnb2 });
- } else {
- for (uint64_t i1 = 0; i1 < ne1; i1++) {
- if (ne0*nb0/bs == dstnb1) {
- slices.push_back({ buf_offset + i3*nb3 + i2*nb2 + i1*nb1, offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, dstnb1 });
- } else {
- const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1;
- const uint64_t d_off = offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1;
- for (uint64_t i0 = 0; i0 < ne0; i0++) {
- slices.push_back({ s_off + i0*nb0, d_off + i0*dstnb0, dstnb0 });
- }
- }
- }
- }
- }
- }
-
- ggml_vk_sync_buffers(ctx, subctx);
- subctx->s->buffer->buf.copyBuffer(buf->buffer, dst->buffer, slices);
- return;
- }
-
- if (!sync_staging) {
- GGML_ABORT("Asynchronous write to non-pinned memory not supported");
- }
-
- // Staging buffer required
- vk_buffer& staging = ctx->device->sync_staging;
- const uint64_t copy_size = ts*ne/bs;
- ggml_vk_ensure_sync_staging_buffer(ctx->device, copy_size);
- VkBufferCopy buf_copy{ 0, offset, copy_size };
-
- ggml_vk_sync_buffers(ctx, subctx);
- vkCmdCopyBuffer(subctx->s->buffer->buf, (VkBuffer)staging->buffer, (VkBuffer)dst->buffer, 1, &buf_copy);
-
- for (uint64_t i3 = 0; i3 < ne3; i3++) {
- for (uint64_t i2 = 0; i2 < ne2; i2++) {
- // Find longest contiguous slice
- if (ne1*nb1 == dstnb2) {
- deferred_memcpy((uint8_t *)staging->ptr + i3*dstnb3 + i2*dstnb2, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2, dstnb2, &subctx->in_memcpys);
- } else {
- for (uint64_t i1 = 0; i1 < ne1; i1++) {
- if (ne0*nb0/bs == dstnb1) {
- deferred_memcpy((uint8_t *)staging->ptr + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2 + i1*nb1, dstnb1, &subctx->in_memcpys);
- } else {
- const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1;
- const uint64_t d_off = i3*dstnb3 + i2*dstnb2 + i1*dstnb1;
- for (uint64_t i0 = 0; i0 < ne0; i0++) {
- deferred_memcpy((uint8_t *)staging->ptr + d_off + i0*dstnb0, (const uint8_t *) tensor->data + s_off + i0*nb0, dstnb0, &subctx->in_memcpys);
- }
- }
- }
- }
- }
- }
-}
-
-static bool ggml_vk_buffer_write_2d_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false) {
- VK_LOG_DEBUG("ggml_vk_buffer_write_2d_async(" << width << ", " << height << ")");
- // Check if src is pinned memory
- vk_buffer buf = nullptr;
- size_t buf_offset = 0;
- ggml_vk_host_get(dst->device, src, buf, buf_offset);
-
- if (buf != nullptr) {
- // Memory is pinned, use as staging buffer
- std::vector<vk::BufferCopy> slices(1);
- if (width == spitch && width == dpitch) {
- // Only do single write if stride is equal
- slices[0].srcOffset = buf_offset;
- slices[0].dstOffset = offset;
- slices[0].size = width * height;
- } else {
- slices.resize(height);
- for (size_t i = 0; i < height; i++) {
- slices[i].srcOffset = buf_offset + i * spitch;
- slices[i].dstOffset = offset + i * dpitch;
- slices[i].size = width;
- }
- }
-
- ggml_vk_sync_buffers(nullptr, subctx);
- subctx->s->buffer->buf.copyBuffer(buf->buffer, dst->buffer, slices);
- return true;
- }
- VK_LOG_DEBUG("STAGING");
-
- if (!sync_staging) {
- // copy was not handled caller needs to fall back
- return false;
- }
-
- // Staging buffer required
- const size_t staging_size = width * height;
- ggml_vk_ensure_sync_staging_buffer(dst->device, staging_size);
-
- vk_buffer& staging_buffer = dst->device->sync_staging;
-
- std::vector<vk::BufferCopy> slices(1);
- if (width == dpitch) {
- slices[0].srcOffset = 0;
- slices[0].dstOffset = offset;
- slices[0].size = staging_size;
- } else {
- slices.resize(height);
- for (size_t i = 0; i < height; i++) {
- slices[i].srcOffset = i * width;
- slices[i].dstOffset = offset + i * dpitch;
- slices[i].size = width;
- }
- }
-
- ggml_vk_sync_buffers(nullptr, subctx);
- subctx->s->buffer->buf.copyBuffer(staging_buffer->buffer, dst->buffer, slices);
-
- if (width == spitch) {
- deferred_memcpy((uint8_t *)staging_buffer->ptr, src, staging_size, &subctx->in_memcpys);
- } else {
- for (size_t i = 0; i < height; i++) {
- deferred_memcpy((uint8_t *)staging_buffer->ptr + i * width, (const uint8_t *) src + i * spitch, width, &subctx->in_memcpys);
- }
- }
- return true;
-}
-
-static bool ggml_vk_buffer_write_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t size, bool sync_staging = false) {
- VK_LOG_DEBUG("ggml_vk_buffer_write_async(" << size << ")");
- return ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, size, size, size, 1, sync_staging);
-}
-
-static void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height) {
- VK_LOG_DEBUG("ggml_vk_buffer_write_2d(" << width << ", " << height << ")");
- // Buffer is already mapped
- if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {
- GGML_ASSERT(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent);
-
- if (width == spitch && width == dpitch) {
- memcpy((uint8_t *)dst->ptr + offset, src, width * height);
- } else {
- for (size_t i = 0; i < height; i++) {
- memcpy((uint8_t *)dst->ptr + offset + i * dpitch, (const uint8_t *) src + i * spitch, width);
- }
- }
- } else {
- std::lock_guard<std::recursive_mutex> guard(dst->device->mutex);
-
- vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool);
- ggml_vk_ctx_begin(dst->device, subctx);
- bool ret = ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, spitch, dpitch, width, height, true);
- GGML_ASSERT(ret);
- ggml_vk_ctx_end(subctx);
-
- for (auto& cpy : subctx->in_memcpys) {
- memcpy(cpy.dst, cpy.src, cpy.n);
- }
-
- for (auto& mset : subctx->memsets) {
- memset(mset.dst, mset.val, mset.n);
- }
-
- ggml_vk_submit(subctx, dst->device->fence);
- VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences", dst->device);
- dst->device->device.resetFences({ dst->device->fence });
- ggml_vk_queue_command_pools_cleanup(dst->device);
- }
-}
-
-static void ggml_vk_buffer_write(vk_buffer& dst, size_t offset, const void * src, size_t size) {
- VK_LOG_DEBUG("ggml_vk_buffer_write(" << size << ")");
- ggml_vk_buffer_write_2d(dst, offset, src, size, size, size, 1);
-}
-
-static bool ggml_vk_buffer_read_2d_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false) {
- VK_LOG_DEBUG("ggml_vk_buffer_read_2d_async(offset=" << offset << ", width=" << width << ", height=" << height << ")");
- GGML_ASSERT(width > 0);
- GGML_ASSERT(height > 0);
- GGML_ASSERT(src != nullptr);
-
- // TODO: staging_offset is not used
-
- // Check if dst is pinned memory
- vk_buffer buf = nullptr;
- size_t buf_offset = 0;
- ggml_vk_host_get(src->device, dst, buf, buf_offset);
-
- std::vector<vk::BufferCopy> slices(1);
- if (width == spitch && width == dpitch) {
- // Only do single write if stride is equal
- slices[0].srcOffset = offset;
- slices[0].dstOffset = buf_offset;
- slices[0].size = width * height;
- } else {
- slices.resize(height);
- for (size_t i = 0; i < height; i++) {
- slices[i].srcOffset = offset + i * spitch;
- slices[i].dstOffset = buf_offset + i * dpitch;
- slices[i].size = width;
- }
- }
-
- if (buf != nullptr) {
- // Memory is pinned, use as staging buffer
- ggml_vk_sync_buffers(nullptr, subctx);
- subctx->s->buffer->buf.copyBuffer(src->buffer, buf->buffer, slices);
-
- return true;
- }
- VK_LOG_DEBUG("STAGING");
-
- if (!sync_staging) {
- // copy was not handled caller needs to fall back
- return false;
- }
-
- // Fall back to staging buffer
- const size_t staging_size = width * height;
- ggml_vk_ensure_sync_staging_buffer(src->device, staging_size);
-
- vk_buffer& staging_buffer = src->device->sync_staging;
-
- std::vector<vk::BufferCopy> staging_slices(1);
- if (width == spitch) {
- staging_slices[0].srcOffset = offset;
- staging_slices[0].dstOffset = 0;
- staging_slices[0].size = staging_size;
- } else {
- staging_slices.resize(height);
- for (size_t i = 0; i < height; i++) {
- staging_slices[i].srcOffset = offset + i * spitch;
- staging_slices[i].dstOffset = i * width;
- staging_slices[i].size = width;
- }
- }
-
- ggml_vk_sync_buffers(nullptr, subctx);
- subctx->s->buffer->buf.copyBuffer(src->buffer, staging_buffer->buffer, staging_slices);
-
- if (width == dpitch) {
- deferred_memcpy(dst, staging_buffer->ptr, staging_size, &subctx->out_memcpys);
- } else {
- for (size_t i = 0; i < height; i++) {
- deferred_memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) staging_buffer->ptr + i * width, width, &subctx->out_memcpys);
- }
- }
- return true;
-}
-
-static bool ggml_vk_buffer_read_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t size, bool sync_staging = false) {
- return ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, size, size, size, 1, sync_staging);
-}
-
-static void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height) {
- VK_LOG_DEBUG("ggml_vk_buffer_read_2d(" << src->buffer << ", " << offset << ", " << width << ", " << height << ")");
-
- // If the device is not an UMA device the memory is host-accessible through rebar. While writing
- // through PCIe is sufficient fast reading back data from PCIe is slower than going through
- // the HW device to host copy path.
- if(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible && src->device->uma) {
- GGML_ASSERT(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent);
-
- std::lock_guard<std::recursive_mutex> guard(src->device->mutex);
- vk_context subctx = ggml_vk_create_temporary_context(src->device->compute_queue->cmd_pool);
- ggml_vk_ctx_begin(src->device, subctx);
- subctx->s->buffer->buf.pipelineBarrier(
- vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer,
- vk::PipelineStageFlagBits::eHost,
- {},
- { { vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite,
- vk::AccessFlagBits::eHostRead } },
- {}, {});
- ggml_vk_ctx_end(subctx);
- ggml_vk_submit(subctx, src->device->fence);
- VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX),
- "vk_buffer_read_2d uma waitForFences", src->device);
- src->device->device.resetFences({ src->device->fence });
- ggml_vk_queue_command_pools_cleanup(src->device);
-
- if (width == spitch && width == dpitch) {
- memcpy(dst, (const uint8_t *) src->ptr + offset, width * height);
- } else {
- for (size_t i = 0; i < height; i++) {
- memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) src->ptr + offset + i * spitch, width);
- }
- }
- } else {
- std::lock_guard<std::recursive_mutex> guard(src->device->mutex);
-
- vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool);
- ggml_vk_ctx_begin(src->device, subctx);
- bool ret = ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, spitch, dpitch, width, height, true);
- GGML_ASSERT(ret);
- ggml_vk_ctx_end(subctx);
-
- ggml_vk_submit(subctx, src->device->fence);
- VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences", src->device);
- src->device->device.resetFences({ src->device->fence });
- ggml_vk_queue_command_pools_cleanup(src->device);
-
- for (auto& cpy : subctx->out_memcpys) {
- memcpy(cpy.dst, cpy.src, cpy.n);
- }
- }
-}
-
-static void ggml_vk_buffer_read(vk_buffer& src, size_t offset, void * dst, size_t size) {
- VK_LOG_DEBUG("ggml_vk_buffer_read(" << src->buffer << ", " << offset << ", " << size << ")");
- ggml_vk_buffer_read_2d(src, offset, dst, size, size, size, 1);
-}
-
-static void ggml_vk_buffer_copy_async(vk_context& ctx, vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) {
- VK_LOG_DEBUG("ggml_vk_buffer_copy_async(" << size << ")");
- // Make sure both buffers are on same device
- GGML_ASSERT(src->device == dst->device);
-
- VkBufferCopy bc{ src_offset, dst_offset, size };
-
- vkCmdCopyBuffer(ctx->s->buffer->buf, (VkBuffer)src->buffer, (VkBuffer)dst->buffer, 1, &bc);
-}
-
-static void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) {
- if (src->device == dst->device) {
- std::lock_guard<std::recursive_mutex> guard(src->device->mutex);
- VK_LOG_DEBUG("ggml_vk_buffer_copy(SINGLE_DEVICE, " << size << ")");
- // Copy within the device
- vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool);
- ggml_vk_ctx_begin(src->device, subctx);
- ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size);
- ggml_vk_ctx_end(subctx);
- ggml_vk_submit(subctx, src->device->fence);
- VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences", src->device);
- src->device->device.resetFences({ src->device->fence });
- ggml_vk_queue_command_pools_cleanup(src->device);
- } else {
- VK_LOG_DEBUG("ggml_vk_buffer_copy(MULTI_DEVICE, " << size << ")");
- // Copy device to device
- ggml_vk_ensure_sync_staging_buffer(src->device, size);
-
- // Copy to src staging buffer
- ggml_vk_buffer_copy(src->device->sync_staging, 0, src, src_offset, size);
- // Copy to dst buffer
- ggml_vk_buffer_write(dst, dst_offset, src->device->sync_staging->ptr, size);
- }
-}
-
-static void ggml_vk_buffer_memset_async(vk_context& ctx, vk_buffer& dst, size_t offset, uint32_t c, size_t size) {
- VK_LOG_DEBUG("ggml_vk_buffer_memset_async(" << offset << ", " << c << ", " << size << ")");
-
- if (dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible &&
- dst->device->uma) {
- deferred_memset((uint8_t*)dst->ptr + offset, c, size, &ctx->memsets);
- return;
- }
-
- // Fall back to GPU fillBuffer for non-UMA or non-host-visible buffers
- ctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c);
-}
-
-static void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t size) {
- VK_LOG_DEBUG("ggml_vk_buffer_memset(" << offset << ", " << c << ", " << size << ")");
-
- if (dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible &&
- dst->device->uma) {
- memset((uint8_t*)dst->ptr + offset, c, size);
- return;
- }
-
- std::lock_guard<std::recursive_mutex> guard(dst->device->mutex);
- vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool);
- ggml_vk_ctx_begin(dst->device, subctx);
- subctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c);
- ggml_vk_ctx_end(subctx);
-
- ggml_vk_submit(subctx, dst->device->fence);
- VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences", dst->device);
- dst->device->device.resetFences({ dst->device->fence });
- ggml_vk_queue_command_pools_cleanup(dst->device);
-}
-
static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, uint32_t n, uint32_t k, bool disable_split_k, const vk_pipeline& pipeline) {
VK_LOG_DEBUG("ggml_vk_guess_split_k(" << m << ", " << n << ", " << k << ", " << disable_split_k << ")");
@@ -9115,9 +5629,7 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m,
return split_k;
}
-
-
-static void ggml_vk_matmul(
+void ggml_vk_matmul(
ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline,
vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer,
uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d,
@@ -9166,7 +5678,6 @@ static void ggml_vk_matmul(
ctx->prealloc_split_k_need_sync = true;
}
-
static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) {
if (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16) return false;
if (src1_type == GGML_TYPE_Q8_1) return false;
@@ -9234,14 +5745,14 @@ static void ggml_vk_matmul_id(
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, ids, expert_count_buf }, pc, { m, nei1, n_as });
}
-static bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) {
+bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) {
return
tensor->nb[0] == ggml_type_size(tensor->type) &&
tensor->nb[1] == (tensor->nb[0]*tensor->ne[0])/ggml_blck_size(tensor->type) &&
(tensor->ne[3] == 1 || tensor->nb[3] == tensor->nb[2]*tensor->ne[2]);
}
-static vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src, const ggml_tensor * dst, ggml_type to) {
+vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src, const ggml_tensor * dst, ggml_type to) {
// Choose "contiguous copy" shader if src/dst are contiguous
bool contig = ggml_is_contiguous(src) && (!dst || ggml_is_contiguous(dst));
@@ -9382,7 +5893,7 @@ static vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const
GGML_ABORT("fatal error");
}
-static void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline pipeline, const ggml_tensor * tensor, const vk_subbuffer & in, const vk_subbuffer & out) {
+void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline pipeline, const ggml_tensor * tensor, const vk_subbuffer & in, const vk_subbuffer & out) {
VK_LOG_DEBUG("ggml_vk_cpy_to_contiguous((" << tensor << ", type=" << tensor->type << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << "), ";
std::cerr << "buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ")");
@@ -9407,8 +5918,6 @@ static void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context&
ggml_vk_sync_buffers(ctx, subctx);
}
-// Copy/convert tensor into a caller-defined dense layout. Destination strides
-// are in output elements, not bytes.
static void ggml_vk_cpy_to_strided(
ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline pipeline, const ggml_tensor * tensor,
const vk_subbuffer & in, const vk_subbuffer & out,
@@ -9437,7 +5946,7 @@ static void ggml_vk_cpy_to_strided(
ggml_vk_sync_buffers(ctx, subctx);
}
-static vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx, ggml_type type) {
+vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx, ggml_type type) {
switch(type) {
case GGML_TYPE_Q8_1:
return ctx->device->pipeline_quantize_q8_1_x4;
@@ -9447,7 +5956,7 @@ static vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx,
}
}
-static void ggml_vk_quantize_q8_1(ggml_backend_vk_context * ctx, vk_context& subctx, const vk_subbuffer & in, const vk_subbuffer & out, uint32_t ne) {
+void ggml_vk_quantize_q8_1(ggml_backend_vk_context * ctx, vk_context& subctx, const vk_subbuffer & in, const vk_subbuffer & out, uint32_t ne) {
VK_LOG_DEBUG("ggml_vk_quantize_q8_1(" << "buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ", " << ne << ")");
vk_pipeline pipeline = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1);
@@ -9752,7 +6261,6 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
}
}
-// Device tuning
static bool ggml_vk_should_use_mmvq(const vk_device& device, uint32_t m, uint32_t n, uint32_t k, ggml_type src0_type) {
if (device->mmvq_mode == 1) {
return true;
@@ -10273,7 +6781,7 @@ static int ggml_vk_fwht_pipeline_idx(int64_t n) {
}
}
-static bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor * src1, const ggml_tensor * dst) {
+bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor * src1, const ggml_tensor * dst) {
if (ctx->num_additional_fused_ops != 0) {
return false;
}
@@ -10299,7 +6807,7 @@ static bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml
return true;
}
-static void ggml_vk_fwht(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src, ggml_tensor * dst) {
+void ggml_vk_fwht(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src, ggml_tensor * dst) {
const int idx = ggml_vk_fwht_pipeline_idx(src->ne[0]);
vk_pipeline pipeline = ctx->device->pipeline_fwht_f32[idx];
@@ -10329,7 +6837,7 @@ static uint32_t ggml_vk_nb_elem(const ggml_tensor * t, int i) {
return (uint32_t)(t->nb[i] / ggml_type_size(t->type));
}
-static void ggml_vk_dsv4_hc_comb(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * mixes, const ggml_tensor * scale, const ggml_tensor * base, ggml_tensor * dst) {
+void ggml_vk_dsv4_hc_comb(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * mixes, const ggml_tensor * scale, const ggml_tensor * base, ggml_tensor * dst) {
VK_LOG_DEBUG("ggml_vk_dsv4_hc_comb(" << mixes << ", " << scale << ", " << base << ", " << dst << ")");
vk_pipeline pipeline = ctx->device->pipeline_dsv4_hc_comb_f32;
@@ -10359,7 +6867,7 @@ static void ggml_vk_dsv4_hc_comb(ggml_backend_vk_context * ctx, vk_context& subc
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { mixes_buf, scale_buf, base_buf, dst_buf }, pc, { n_tokens, 1, 1 });
}
-static void ggml_vk_dsv4_hc_pre(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * weights, ggml_tensor * dst) {
+void ggml_vk_dsv4_hc_pre(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * weights, ggml_tensor * dst) {
VK_LOG_DEBUG("ggml_vk_dsv4_hc_pre(" << x << ", " << weights << ", " << dst << ")");
const float scale = ggml_get_op_params_f32(dst, 0);
@@ -10390,7 +6898,7 @@ static void ggml_vk_dsv4_hc_pre(ggml_backend_vk_context * ctx, vk_context& subct
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { x_buf, w_buf, d_buf }, pc, { n_embd, n_tokens, 1 });
}
-static void ggml_vk_dsv4_hc_post(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * residual, const ggml_tensor * post, const ggml_tensor * comb, ggml_tensor * dst) {
+void ggml_vk_dsv4_hc_post(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * residual, const ggml_tensor * post, const ggml_tensor * comb, ggml_tensor * dst) {
VK_LOG_DEBUG("ggml_vk_dsv4_hc_post(" << x << ", " << residual << ", " << post << ", " << comb << ", " << dst << ")");
vk_pipeline pipeline = comb ? ctx->device->pipeline_dsv4_hc_post_f32 : ctx->device->pipeline_dsv4_hc_post_nocomb_f32;
@@ -10421,7 +6929,7 @@ static void ggml_vk_dsv4_hc_post(ggml_backend_vk_context * ctx, vk_context& subc
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { x_buf, r_buf, p_buf, c_buf, d_buf }, pc, { n_embd, n_tokens, 1 });
}
-static void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {
+void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {
ggml_tensor * dst = cgraph->nodes[node_idx];
ggml_tensor * src0 = dst->src[0];
ggml_tensor * src1 = dst->src[1];
@@ -11099,14 +7607,14 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte
}
}
-static bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx) {
+bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx) {
ggml_tensor * dst = cgraph->nodes[node_idx];
ggml_tensor * src0 = dst->src[0];
ggml_tensor * src2 = dst->src[2];
return (src2->ne[1] <= 8) && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type));
}
-static void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {
+void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {
ggml_tensor * dst = cgraph->nodes[node_idx];
ggml_tensor * src0 = dst->src[0];
ggml_tensor * src1 = dst->src[1];
@@ -11119,7 +7627,7 @@ static void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx
}
}
-static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) {
+bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) {
GGML_UNUSED(f32acc);
// Needs to be kept up to date on shader changes
const uint32_t wg_size = params.workgroup_size;
@@ -11170,7 +7678,7 @@ static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, con
return supported;
}
-static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) {
+bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) {
GGML_UNUSED(v_type);
// Needs to be kept up to date on shader changes
const uint32_t Br = params.block_rows;
@@ -11218,7 +7726,7 @@ static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, co
return supported;
}
-static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * v, const ggml_tensor * mask, const ggml_tensor * sinks, ggml_tensor * dst) {
+void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * v, const ggml_tensor * mask, const ggml_tensor * sinks, ggml_tensor * dst) {
VK_LOG_DEBUG("ggml_vk_flash_attn((" << q << ", name=" << q->name << ", type=" << q->type << ", ne0=" << q->ne[0] << ", ne1=" << q->ne[1] << ", ne2=" << q->ne[2] << ", ne3=" << q->ne[3] << ", nb0=" << q->nb[0] << ", nb1=" << q->nb[1] << ", nb2=" << q->nb[2] << ", nb3=" << q->nb[3];
std::cerr << "), (" << k << ", name=" << k->name << ", type=" << k->type << ", ne0=" << k->ne[0] << ", ne1=" << k->ne[1] << ", ne2=" << k->ne[2] << ", ne3=" << k->ne[3] << ", nb0=" << k->nb[0] << ", nb1=" << k->nb[1] << ", nb2=" << k->nb[2] << ", nb3=" << k->nb[3];
std::cerr << "), (" << v << ", name=" << v->name << ", type=" << v->type << ", ne0=" << v->ne[0] << ", ne1=" << v->ne[1] << ", ne2=" << v->ne[2] << ", ne3=" << v->ne[3] << ", nb0=" << v->nb[0] << ", nb1=" << v->nb[1] << ", nb2=" << v->nb[2] << ", nb3=" << v->nb[3];
@@ -12405,122 +8913,15 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
if (dst->type == GGML_TYPE_F32) {
return ctx->device->pipeline_fill_f32;
}
- if (dst->type == GGML_TYPE_F16) {
- return ctx->device->pipeline_fill_f16;
- }
- return nullptr;
- default:
- return nullptr;
- }
-
- GGML_UNUSED(src2);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_unary_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
-
- p.misalign_offsets = (a_offset << 16) | d_offset;
-
- GGML_UNUSED(src1);
- GGML_UNUSED(src2);
- GGML_UNUSED(src3);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_glu_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
- const uint32_t b_offset = src1 ? get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type) : a_offset;
- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
-
- GGML_ASSERT(a_offset < (1u << 8));
- GGML_ASSERT(b_offset < (1u << 8));
- GGML_ASSERT(d_offset < (1u << 8));
-
- p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;
-
- GGML_UNUSED(src2);
- GGML_UNUSED(src3);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_sum_rows_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
-
- p.misalign_offsets = (a_offset << 16) | d_offset;
-
- GGML_UNUSED(src1);
- GGML_UNUSED(src2);
- GGML_UNUSED(src3);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_pad_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
-
- p.misalign_offsets = (a_offset << 16) | d_offset;
-
- GGML_UNUSED(src1);
- GGML_UNUSED(src2);
- GGML_UNUSED(src3);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_im2col_3d_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- const uint32_t a_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);
- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
-
- p.misalign_offsets = (a_offset << 16) | d_offset;
-
- GGML_UNUSED(src0);
- GGML_UNUSED(src2);
- GGML_UNUSED(src3);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_binary_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
- const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);
- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
-
- GGML_ASSERT(a_offset <= 0xFFFF);
- GGML_ASSERT(b_offset <= 0xFF);
- GGML_ASSERT(d_offset <= 0xFF);
-
- p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;
-
- GGML_UNUSED(src2);
- GGML_UNUSED(src3);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_concat_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type);
- const uint32_t a_offset = get_misalign_bytes(ctx, src0) / unit_size;
- const uint32_t b_offset = get_misalign_bytes(ctx, src1) / unit_size;
- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / unit_size;
-
- p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;
-
- GGML_UNUSED(src2);
- GGML_UNUSED(src3);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_upscale_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
-
- p.a_offset = a_offset;
- p.d_offset = d_offset;
-
- GGML_UNUSED(src1);
- GGML_UNUSED(src2);
- GGML_UNUSED(src3);
-}
-
-template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_rope_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
- p.a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
- p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
+ if (dst->type == GGML_TYPE_F16) {
+ return ctx->device->pipeline_fill_f16;
+ }
+ return nullptr;
+ default:
+ return nullptr;
+ }
- GGML_UNUSED(src1);
GGML_UNUSED(src2);
- GGML_UNUSED(src3);
}
template<typename PC>
@@ -12925,7 +9326,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
}
}
-static void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = ggml_type_size(src1->type);
const uint32_t dst_type_size = ggml_type_size(dst->type);
@@ -12940,7 +9341,7 @@ static void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context& subctx,
});
}
-static void ggml_vk_get_rows_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_get_rows_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = ggml_type_size(src1->type);
const uint32_t dst_type_size = ggml_type_size(dst->type);
@@ -12955,7 +9356,7 @@ static void ggml_vk_get_rows_back(ggml_backend_vk_context * ctx, vk_context& sub
});
}
-static void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = ggml_type_size(src1->type);
const uint32_t dst_type_size = ggml_type_size(dst->type);
@@ -12975,7 +9376,7 @@ static void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const
});
}
-static void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) {
+void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) {
const ggml_tensor *first_node = cgraph->nodes[node_idx];
const ggml_tensor *dst = cgraph->nodes[node_idx + ctx->num_additional_fused_ops];
@@ -13082,7 +9483,7 @@ static void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx,
}, pc, elements);
}
-static void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = ggml_type_size(src1->type);
const uint32_t dst_type_size = ggml_type_size(dst->type);
@@ -13097,7 +9498,7 @@ static void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context& subctx, const
});
}
-static void ggml_vk_out_prod(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_out_prod(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = ggml_type_size(src1->type);
const uint32_t dst_type_size = ggml_type_size(dst->type);
@@ -13115,7 +9516,7 @@ static void ggml_vk_out_prod(ggml_backend_vk_context * ctx, vk_context& subctx,
});
}
-static void ggml_vk_sub(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_sub(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = ggml_type_size(src1->type);
const uint32_t dst_type_size = ggml_type_size(dst->type);
@@ -13130,7 +9531,7 @@ static void ggml_vk_sub(ggml_backend_vk_context * ctx, vk_context& subctx, const
});
}
-static void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = ggml_type_size(src1->type);
const uint32_t dst_type_size = ggml_type_size(dst->type);
@@ -13145,8 +9546,7 @@ static void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const
});
}
-// index into device->pipeline_unary_mul for the supported unary ops, or -1
-static int ggml_vk_unary_mul_op_index(ggml_unary_op op) {
+int ggml_vk_unary_mul_op_index(ggml_unary_op op) {
switch (op) {
case GGML_UNARY_OP_GELU: return 0;
case GGML_UNARY_OP_SIGMOID: return 1;
@@ -13156,7 +9556,7 @@ static int ggml_vk_unary_mul_op_index(ggml_unary_op op) {
}
}
-static void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {
+void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {
const ggml_tensor * unary = cgraph->nodes[node_idx];
ggml_tensor * mul = cgraph->nodes[node_idx + 1];
@@ -13191,7 +9591,7 @@ static void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx,
}, pipeline);
}
-static void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = ggml_type_size(src1->type);
const uint32_t dst_type_size = ggml_type_size(dst->type);
@@ -13206,7 +9606,7 @@ static void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const
});
}
-static void ggml_vk_add_id(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) {
+void ggml_vk_add_id(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) {
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = ggml_type_size(src1->type);
const uint32_t src2_type_size = ggml_type_size(src2->type);
@@ -13262,7 +9662,7 @@ static void ggml_vk_op_f32_wkv(ggml_backend_vk_context * ctx, vk_context& subctx
}
}
-static void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
const size_t seq_length = dst->src[0]->ne[2];
const size_t n_embed = dst->ne[0];
const size_t n_heads = dst->src[0]->ne[1];
@@ -13280,7 +9680,7 @@ static void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx,
);
}
-static void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
const size_t seq_length = dst->src[0]->ne[2];
const size_t n_embed = dst->ne[0];
const size_t n_heads = dst->src[0]->ne[1];
@@ -13298,7 +9698,7 @@ static void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx,
);
}
-static void ggml_vk_gated_linear_attn(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+void ggml_vk_gated_linear_attn(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
const size_t seq_length = dst->src[0]->ne[2];
const size_t n_embed = dst->ne[0];
const size_t n_heads = dst->src[0]->ne[1];
@@ -13333,7 +9733,7 @@ static void ggml_vk_gated_linear_attn(ggml_backend_vk_context * ctx, vk_context&
pc, { (uint32_t)(n_seqs * n_heads), 1, 1 });
}
-static void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
const ggml_tensor * q = dst->src[0];
const ggml_tensor * k = dst->src[1];
const ggml_tensor * w = dst->src[2];
@@ -13382,7 +9782,7 @@ static void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context&
pc, {dispatch_x, dispatch_y, 1});
}
-static void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
const ggml_tensor * src_q = dst->src[0];
const ggml_tensor * src_v = dst->src[2];
const ggml_tensor * src_beta = dst->src[4];
@@ -13439,7 +9839,7 @@ static void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& s
pc, { H, n_seqs, S_v });
}
-static void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
const ggml_tensor * src2 = dst->src[2];
@@ -13496,7 +9896,7 @@ static void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx,
pc, elements);
}
-static void ggml_vk_ssm_conv(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {
+void ggml_vk_ssm_conv(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {
ggml_tensor * conv = cgraph->nodes[node_idx];
const ggml_tensor * src0 = conv->src[0];
const ggml_tensor * src1 = conv->src[1];
@@ -13570,7 +9970,7 @@ static void ggml_vk_op_f32_opt_step_adamw(ggml_backend_vk_context * ctx, vk_cont
pc, elements);
}
-static void ggml_vk_opt_step_adamw(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+void ggml_vk_opt_step_adamw(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
const size_t n = ggml_nelements(dst->src[0]);
ggml_vk_op_f32_opt_step_adamw(
@@ -13579,13 +9979,13 @@ static void ggml_vk_opt_step_adamw(ggml_backend_vk_context * ctx, vk_context& su
);
}
-static void ggml_vk_opt_step_sgd(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) {
+void ggml_vk_opt_step_sgd(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) {
const size_t n = ggml_nelements(dst->src[0]);
ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_OPT_STEP_SGD, { (uint32_t)n, 0, 0.0f, 0.0f, 0.0f, 0.0f });
}
-static void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
int * op_params = (int *)dst->op_params;
const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type);
@@ -13612,7 +10012,7 @@ static void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, co
ggml_vk_op_f32<vk_op_concat_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONCAT, std::move(pc));
}
-static void ggml_vk_upscale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_upscale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t mode = (uint32_t)ggml_get_op_params_i32(dst, 0);
@@ -13639,7 +10039,7 @@ static void ggml_vk_upscale(ggml_backend_vk_context * ctx, vk_context& subctx, c
});
}
-static void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);
p.param1 = ggml_get_op_params_f32(dst, 0);
p.param2 = ggml_get_op_params_f32(dst, 1);
@@ -13647,15 +10047,15 @@ static void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context& subctx, con
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SCALE, std::move(p));
}
-static void ggml_vk_sqr(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_sqr(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SQR, vk_op_unary_push_constants_init(src0, dst));
}
-static void ggml_vk_sqrt(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_sqrt(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SQRT, vk_op_unary_push_constants_init(src0, dst));
}
-static void ggml_vk_add1(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_add1(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = ggml_type_size(src1->type);
const uint32_t dst_type_size = ggml_type_size(dst->type);
@@ -13670,7 +10070,7 @@ static void ggml_vk_add1(ggml_backend_vk_context * ctx, vk_context& subctx, cons
});
}
-static void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
VK_LOG_DEBUG("ggml_vk_arange(dst=" << dst << ", ne=" << ggml_nelements(dst) << ")");
vk_op_push_constants pc = {
@@ -13692,7 +10092,7 @@ static void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, gg
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements);
}
-static void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
VK_LOG_DEBUG("ggml_vk_fill(dst=" << dst << ", ne=" << ggml_nelements(dst) << ")");
const uint64_t n = ggml_nelements(dst);
GGML_ASSERT(n > 0);
@@ -13721,32 +10121,32 @@ static void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements);
}
-static void ggml_vk_sin(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_sin(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SIN, vk_op_unary_push_constants_init(src0, dst));
}
-static void ggml_vk_cos(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_cos(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_COS, vk_op_unary_push_constants_init(src0, dst));
}
-static void ggml_vk_log(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_log(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_LOG, vk_op_unary_push_constants_init(src0, dst));
}
-static void ggml_vk_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);
p.param1 = ggml_get_op_params_f32(dst, 0);
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_TRI, std::move(p));
}
-static void ggml_vk_diag(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_diag(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst));
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_DIAG, std::move(p));
}
-static void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);
p.param1 = ggml_get_op_params_f32(dst, 0);
p.param2 = ggml_get_op_params_f32(dst, 1);
@@ -13754,12 +10154,12 @@ static void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context& subctx, con
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_CLAMP, std::move(p));
}
-static void ggml_vk_pad(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_pad(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
vk_op_pad_push_constants p = vk_op_pad_push_constants_init(src0, dst);
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_PAD, std::move(p));
}
-static void ggml_vk_pad_reflect_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_pad_reflect_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
const uint32_t p0 = (uint32_t)dst->op_params[0];
const uint32_t p1 = (uint32_t)dst->op_params[1];
@@ -13770,7 +10170,7 @@ static void ggml_vk_pad_reflect_1d(ggml_backend_vk_context * ctx, vk_context& su
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_PAD_REFLECT_1D, std::move(p));
}
-static void ggml_vk_roll(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_roll(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
const int32_t s0 = ggml_get_op_params_i32(dst, 0);
const int32_t s1 = ggml_get_op_params_i32(dst, 1);
const int32_t s2 = ggml_get_op_params_i32(dst, 2);
@@ -13785,17 +10185,17 @@ static void ggml_vk_roll(ggml_backend_vk_context * ctx, vk_context& subctx, cons
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_ROLL, std::move(p));
}
-static void ggml_vk_repeat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_repeat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst));
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_REPEAT, std::move(p));
}
-static void ggml_vk_repeat_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_repeat_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst));
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_REPEAT_BACK, std::move(p));
}
-static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
uint32_t ne = (uint32_t)ggml_nelements(src0);
if (ggml_is_quantized(src0->type) && ggml_is_quantized(dst->type)) {
// Convert from number of logical elements to 2- or 4-byte units.
@@ -13811,7 +10211,7 @@ static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context& subctx, const
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_CPY, std::move(p));
}
-static void ggml_vk_set_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_set_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = ggml_type_size(src1->type);
const uint32_t dst_type_size = ggml_type_size(dst->type);
@@ -13833,11 +10233,11 @@ static void ggml_vk_set_rows(ggml_backend_vk_context * ctx, vk_context& subctx,
});
}
-static void ggml_vk_silu_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_silu_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_SILU_BACK, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f, 0.0f, 0.0f });
}
-static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
float * op_params = (float *)dst->op_params;
vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);
p.param1 = op_params[0];
@@ -13845,7 +10245,7 @@ static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, cons
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_NORM, std::move(p));
}
-static void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
const int * int_op_params = (const int *)dst->op_params;
const float * float_op_params = (const float *)dst->op_params;
@@ -13863,7 +10263,7 @@ static uint32_t ggml_vk_rms_num_partials(ggml_backend_vk_context * ctx, const gg
return num_partials;
}
-static uint32_t ggml_vk_rms_partials_size(ggml_backend_vk_context * ctx, const ggml_tensor *node) {
+uint32_t ggml_vk_rms_partials_size(ggml_backend_vk_context * ctx, const ggml_tensor *node) {
const uint32_t num_partials = ggml_vk_rms_num_partials(ctx, node);
const uint32_t num_bytes = ROUNDUP_POW2(num_partials * sizeof(uint32_t), ctx->device->partials_binding_alignment);
return num_bytes;
@@ -13917,23 +10317,6 @@ static vk_op_rope_push_constants ggml_vk_make_rope_constants(const ggml_tensor *
return rope;
}
-static vk_op_binary_push_constants ggml_vk_rms_norm_push_constants(
- const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst,
- float eps, uint32_t num_partials) {
- const uint32_t src0_type_size = ggml_type_size(src0->type);
- const uint32_t src1_type_size = ggml_type_size(src1->type);
- const uint32_t dst_type_size = ggml_type_size(dst->type);
-
- return {
- (uint32_t)ggml_nelements(src0),
- (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size,
- (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size,
- (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size,
- 0,
- eps, 0.0f, (int32_t)num_partials,
- };
-}
-
static void ggml_vk_rms_norm_finish(ggml_backend_vk_context * ctx, const ggml_tensor * src0) {
if (ctx->do_add_rms_partials_offset_calculation) {
ctx->prealloc_size_add_rms_partials_offset += ggml_vk_rms_partials_size(ctx, src0);
@@ -13942,7 +10325,7 @@ static void ggml_vk_rms_norm_finish(ggml_backend_vk_context * ctx, const ggml_te
}
}
-static void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, float * op_params) {
+void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, float * op_params) {
ggml_tensor * rms = cgraph->nodes[node_idx];
const ggml_tensor * src0 = rms->src[0];
@@ -14116,23 +10499,23 @@ static void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx,
ggml_vk_rms_norm_finish(ctx, src0);
}
-static void ggml_vk_rms_norm_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_rms_norm_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
float * op_params = (float *)dst->op_params;
ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_RMS_NORM_BACK, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f, 0.0f, 0.0f });
}
-static void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
const float * op_params = (const float *)dst->op_params;
vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);
p.param1 = op_params[0];
ggml_vk_op_f32<vk_op_unary_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_L2_NORM, std::move(p));
}
-static void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, vk_op_unary_push_constants_init(src0, dst));
}
-static void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
float * op_params = (float *)dst->op_params;
vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);
p.param1 = op_params[1];
@@ -14142,7 +10525,7 @@ static void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, con
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, std::move(p));
}
-static void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
const float * op_params_f = (const float *)dst->op_params;
const bool swapped = (bool)dst->op_params[1];
@@ -14190,12 +10573,12 @@ static void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const
});
}
-static void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
int32_t * op_params = (int32_t *)dst->op_params;
ggml_vk_op_f32<vk_op_diag_mask_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] });
}
-static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) {
+void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) {
float * op_params = (float *)dst->op_params;
float scale = op_params[0];
@@ -14279,12 +10662,12 @@ static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx,
}
}
-static void ggml_vk_soft_max_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_soft_max_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
float * op_params = (float *)dst->op_params;
ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_SOFT_MAX_BACK, { (uint32_t)src0->ne[0], (uint32_t)ggml_nrows(src0), op_params[0], op_params[1], 0.0f, 0.0f });
}
-static void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) {
+void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) {
topk_moe_mode mode = ctx->fused_topk_moe_mode;
const bool has_bias = mode == TOPK_MOE_SIGMOID_NORM_BIAS || mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS;
ggml_tensor * logits = cgraph->nodes[node_idx + 0]->src[0];
@@ -14370,7 +10753,7 @@ static void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx,
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, {logits_buf, bias_buf, weights_buf, ids_buf}, pc, elements);
}
-static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx, bool backprop) {
+void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx, bool backprop) {
ggml_tensor * dst = cgraph->nodes[node_idx];
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
@@ -14404,7 +10787,7 @@ static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, cons
ggml_vk_make_rope_constants(cgraph->nodes[node_idx], src0, src2 != nullptr, backprop, set_rows_stride));
}
-static void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
const uint32_t * op_params = (const uint32_t *)dst->op_params;
uint32_t ncols = src0->ne[0];
@@ -14490,7 +10873,7 @@ static void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context& subctx, c
}
}
-static void ggml_vk_topk(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_topk(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
uint32_t ncols = src0->ne[0];
uint32_t nrows = ggml_nrows(src0);
uint32_t k = dst->ne[0];
@@ -14623,7 +11006,7 @@ static void ggml_vk_topk(ggml_backend_vk_context * ctx, vk_context& subctx, cons
ctx->prealloc_x_need_sync = true;
}
-static void ggml_vk_topk_qsa(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx) {
+void ggml_vk_topk_qsa(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx) {
const ggml_tensor * get_rows = cgraph->nodes[node_idx + 0];
const ggml_tensor * add = cgraph->nodes[node_idx + ctx->num_additional_fused_ops - 1];
ggml_tensor * top_k = cgraph->nodes[node_idx + ctx->num_additional_fused_ops];
@@ -14672,23 +11055,23 @@ static void ggml_vk_topk_qsa(ggml_backend_vk_context * ctx, vk_context& subctx,
ctx->prealloc_x_need_sync = true;
}
-static void ggml_vk_sum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_sum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
vk_op_sum_rows_push_constants p = vk_op_sum_rows_push_constants_init(src0, dst, ggml_nelements(src0));
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SUM, p);
}
-static void ggml_vk_sum_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_sum_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
vk_op_sum_rows_push_constants p = vk_op_sum_rows_push_constants_init(src0, dst, src0->ne[0]);
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SUM_ROWS, p);
}
-static void ggml_vk_mean(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_mean(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
vk_op_sum_rows_push_constants p = vk_op_sum_rows_push_constants_init(src0, dst, src0->ne[0]);
p.weight = 1.0f / (float)src0->ne[0];
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_MEAN, p);
}
-static void ggml_vk_cumsum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_cumsum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
vk_op_sum_rows_push_constants pc = vk_op_sum_rows_push_constants_init(src0, dst, src0->ne[0]);
// Use the single pass shader when the rows are small or there are enough rows to fill the GPU.
// For fewer, larger rows, use the multipass shader to spread each row across SMs.
@@ -14745,7 +11128,7 @@ static std::array<uint32_t, 3> ggml_vk_nrows_elements(uint32_t nr) {
return { nr, 1, 1 };
}
-static void ggml_vk_cross_entropy_loss(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+void ggml_vk_cross_entropy_loss(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
@@ -14798,7 +11181,7 @@ static void ggml_vk_cross_entropy_loss(ggml_backend_vk_context * ctx, vk_context
ctx->prealloc_x_need_sync = true;
}
-static void ggml_vk_cross_entropy_loss_back(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+void ggml_vk_cross_entropy_loss_back(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
const ggml_tensor * grad = dst->src[0];
const ggml_tensor * logits = dst->src[1];
const ggml_tensor * labels = dst->src[2];
@@ -14832,15 +11215,15 @@ static void ggml_vk_cross_entropy_loss_back(ggml_backend_vk_context * ctx, vk_co
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { grad_buf, logits_buf, labels_buf, dst_buf }, pc, ggml_vk_nrows_elements(nrows));
}
-static void ggml_vk_argmax(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_argmax(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_ARGMAX, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], 0.0f, 0.0f, 0.0f, 0.0f });
}
-static void ggml_vk_count_equal(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_count_equal(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_COUNT_EQUAL, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f, 0.0f, 0.0f });
}
-static void ggml_vk_solve_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_solve_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = ggml_type_size(src1->type);
const uint32_t dst_type_size = ggml_type_size(dst->type);
@@ -14855,7 +11238,7 @@ static void ggml_vk_solve_tri(ggml_backend_vk_context * ctx, vk_context& subctx,
});
}
-static void ggml_vk_im2col(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_im2col(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
const int32_t s0 = dst->op_params[0];
const int32_t s1 = dst->op_params[1];
const int32_t p0 = dst->op_params[2];
@@ -14895,7 +11278,7 @@ static void ggml_vk_im2col(ggml_backend_vk_context * ctx, vk_context& subctx, co
});
}
-static void ggml_vk_im2col_3d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_im2col_3d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
GGML_TENSOR_BINARY_OP_LOCALS
const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
@@ -14961,7 +11344,7 @@ static void ggml_vk_im2col_3d(ggml_backend_vk_context * ctx, vk_context& subctx,
ggml_vk_op_f32<vk_op_im2col_3d_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_IM2COL_3D, std::move(pc));
}
-static void ggml_vk_timestep_embedding(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_timestep_embedding(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
const uint32_t dim = dst->op_params[0];
const uint32_t max_period = dst->op_params[1];
const uint32_t nb1 = dst->nb[1] / ggml_type_size(dst->type);
@@ -14971,7 +11354,7 @@ static void ggml_vk_timestep_embedding(ggml_backend_vk_context * ctx, vk_context
});
}
-static void ggml_vk_conv_transpose_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+void ggml_vk_conv_transpose_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
// src0: (K, Cout, Cin, 1) -- kernel
// src1: (L, Cin, 1, 1) -- input
// dst: (*, Cout, 1, 1)
@@ -15002,7 +11385,7 @@ static void ggml_vk_conv_transpose_1d(ggml_backend_vk_context * ctx, vk_context&
ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_TRANSPOSE_1D, std::move(p));
}
-static void ggml_vk_col2im_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_col2im_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
// src0: [K_OC, T_in] columns from matmul
// dst: [T_out, OC]
@@ -15028,11 +11411,7 @@ static void ggml_vk_col2im_1d(ggml_backend_vk_context * ctx, vk_context& subctx,
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_COL2IM_1D, std::move(p));
}
-// Dispatch the fused snake activation: y = x + sin^2(a * x) * inv_b.
-// Match the naive mul -> sin -> sqr -> mul -> add chain and run the
-// dedicated kernel directly. The pattern is validated by
-// ggml_vk_can_fuse_snake before this call.
-static void ggml_vk_snake_dispatch_fused(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) {
+void ggml_vk_snake_dispatch_fused(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) {
const ggml_tensor * mul0 = cgraph->nodes[node_idx + 0];
const ggml_tensor * sqr = cgraph->nodes[node_idx + 2];
const ggml_tensor * mul1 = cgraph->nodes[node_idx + 3];
@@ -15067,7 +11446,7 @@ static void ggml_vk_snake_dispatch_fused(ggml_backend_vk_context * ctx, vk_conte
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { x_buf, a_buf, inv_b_buf, dst_buf }, pc, elements);
}
-static void ggml_vk_pool_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_pool_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
uint32_t op = static_cast<uint32_t>(dst->op_params[0]);
const int32_t k0 = dst->op_params[1];
const int32_t s0 = dst->op_params[2];
@@ -15090,7 +11469,7 @@ static void ggml_vk_pool_1d(ggml_backend_vk_context * ctx, vk_context& subctx, c
});
}
-static void ggml_vk_pool_2d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+void ggml_vk_pool_2d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
uint32_t op = static_cast<uint32_t>(dst->op_params[0]);
const int32_t k1 = dst->op_params[1];
const int32_t k0 = dst->op_params[2];
@@ -15118,7 +11497,7 @@ static void ggml_vk_pool_2d(ggml_backend_vk_context * ctx, vk_context& subctx, c
});
}
-static void ggml_vk_conv_2d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0,
+void ggml_vk_conv_2d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0,
const ggml_tensor * src1, ggml_tensor * dst) {
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
GGML_ASSERT(src1->type == GGML_TYPE_F32);
@@ -15158,7 +11537,7 @@ static void ggml_vk_conv_2d(ggml_backend_vk_context * ctx, vk_context & subctx,
ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, dst->op, std::move(p));
}
-static void ggml_vk_conv_3d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0,
+void ggml_vk_conv_3d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0,
const ggml_tensor * src1, ggml_tensor * dst) {
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
GGML_ASSERT(src1->type == GGML_TYPE_F32);
@@ -15188,762 +11567,54 @@ static void ggml_vk_conv_3d(ggml_backend_vk_context * ctx, vk_context & subctx,
// total input element count must fit in a uint32.
GGML_ASSERT((uint64_t)p.IC * p.N * p.IW * p.IH * p.ID <= 0xFFFFFFFFull);
- p.nb01 = static_cast<uint32_t>(nb01 / nb00);
- p.nb02 = static_cast<uint32_t>(nb02 / nb00);
- p.nb03 = static_cast<uint32_t>(nb03 / nb00);
-
- p.nb11 = static_cast<uint32_t>(nb11 / nb10);
- p.nb12 = static_cast<uint32_t>(nb12 / nb10);
- p.nb13 = static_cast<uint32_t>(nb13 / nb10);
-
- p.nb1 = static_cast<uint32_t>(nb1 / nb0);
- p.nb2 = static_cast<uint32_t>(nb2 / nb0);
- p.nb3 = static_cast<uint32_t>(nb3 / nb0);
-
- ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_3D, std::move(p));
-}
-
-static void ggml_vk_conv_2d_dw(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
- vk_op_conv2d_dw_push_constants p{};
- p.ne = ggml_nelements(dst);
- p.channels = dst->ne[2];
- p.batches = dst->ne[3];
- p.dst_w = dst->ne[0];
- p.dst_h = dst->ne[1];
- p.src_w = src1->ne[0];
- p.src_h = src1->ne[1];
- p.knl_w = src0->ne[0];
- p.knl_h = src0->ne[1];
- p.stride_x = dst->op_params[0];
- p.stride_y = dst->op_params[1];
- p.pad_x = dst->op_params[2];
- p.pad_y = dst->op_params[3];
- p.dilation_x = dst->op_params[4];
- p.dilation_y = dst->op_params[5];
-
- GGML_ASSERT(src0->ne[3] == p.channels);
- GGML_ASSERT(src1->ne[3] == p.batches);
-
- ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_2D_DW, std::move(p));
-}
-
-static void ggml_vk_leaky_relu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
- const float * op_params = (const float *)dst->op_params;
- vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);
- p.param1 = op_params[0];
-
- ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_LEAKY_RELU, std::move(p));
-}
-
-#ifdef GGML_VULKAN_RUN_TESTS
-static void ggml_vk_print_matrix_area(const void * data, ggml_type type, int ne0, int ne1, int i0, int i1, int i2) {
- if (type != GGML_TYPE_F32 && type != GGML_TYPE_F16) {
- return;
- }
- i0 = std::max(i0, 5);
- i1 = std::max(i1, 5);
- i2 = std::max(i2, 0);
- fprintf(stderr, " ");
- for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {
- fprintf(stderr, "%7d ", idx1);
- }
- fprintf(stderr, "\n");
- for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) {
- fprintf(stderr, "%7d: ", idx0);
- for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {
- if (idx0 >= 0 && idx0 < ne0 && idx1 >= 0 && idx1 < ne1) {
- float val;
- if (type == GGML_TYPE_F32) {
- val = *((const float *) data + i2*ne1*ne0 + idx1*ne0 + idx0);
- } else if (type == GGML_TYPE_F16) {
- val = ggml_fp16_to_fp32(*((const ggml_fp16_t *) data + i2*ne1*ne0 + idx1*ne0 + idx0));
- } else {
- GGML_ABORT("fatal error");
- }
- fprintf(stderr, "% 7.2f ", val);
- } else {
- fprintf(stderr, " ");
- }
- }
- fprintf(stderr, "\n");
- }
-}
-
-template <typename X_TYPE, typename Y_TYPE>
-static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size_t k, size_t batch, size_t num_it, int split_k, int shader_size) {
- VK_LOG_DEBUG("ggml_vk_test_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << shader_size << ")");
- const size_t x_ne = m * k * batch;
- const size_t y_ne = k * n * batch;
- const size_t d_ne = m * n * batch;
-
- ggml_type x_type = std::is_same<float, X_TYPE>() ? GGML_TYPE_F32 : GGML_TYPE_F16;
- ggml_type y_type = std::is_same<float, Y_TYPE>() ? GGML_TYPE_F32 : GGML_TYPE_F16;
- vk_matmul_pipeline_key mm_test_key{x_type, y_type, false, false};
- auto mm_test_it = ctx->device->pipeline_matmul.find(mm_test_key);
- GGML_ASSERT(mm_test_it != ctx->device->pipeline_matmul.end() && !mm_test_it->second.empty());
- auto& mm_test_configs = mm_test_it->second;
- GGML_ASSERT(shader_size >= 0 && shader_size < (int)mm_test_configs.size());
-
- std::string shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_ALIGNED_" + std::to_string(shader_size);
- vk_pipeline p = mm_test_configs[shader_size].aligned ? mm_test_configs[shader_size].aligned : mm_test_configs[shader_size].unaligned;
-
- const size_t kpad = ggml_vk_align_size(k, mm_test_configs[shader_size].align);
-
- if (k != kpad) {
- p = mm_test_configs[shader_size].unaligned;
- shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_" + std::to_string(shader_size);
- }
-
- if (split_k > 1) {
- ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, num_it);
-
- if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) {
- // Resize buffer
- if (ctx->prealloc_split_k != nullptr) {
- ggml_vk_destroy_buffer(ctx->prealloc_split_k);
- }
- ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, {vk::MemoryPropertyFlagBits::eDeviceLocal});
- }
- }
-
- ggml_pipeline_allocate_descriptor_sets(ctx);
-
- vk_buffer d_X = ggml_vk_create_buffer_check(ctx->device, sizeof(X_TYPE) * x_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});
- vk_buffer d_Y = ggml_vk_create_buffer_check(ctx->device, sizeof(Y_TYPE) * y_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});
- vk_buffer d_D = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});
-
- X_TYPE* x = (X_TYPE *) malloc(sizeof(X_TYPE) * x_ne);
- Y_TYPE* y = (Y_TYPE *) malloc(sizeof(Y_TYPE) * y_ne);
- float* d = (float *) malloc(sizeof(float) * d_ne);
-
- for (size_t i = 0; i < x_ne; i++) {
- if (std::is_same<float, X_TYPE>()) {
- x[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;
- // x[i] = 1.0f;
- // x[i] = i + 1;
- // x[i] = (i % k == i / k) ? 1.0f : 0.0f;
- } else if (std::is_same<ggml_fp16_t, X_TYPE>()) {
- x[i] = ggml_fp32_to_fp16((rand() / (float)RAND_MAX) * 2.0f - 1.0f);
- // x[i] = ggml_fp32_to_fp16(1.0f);
- // x[i] = ggml_fp32_to_fp16(i + 1);
- // x[i] = ggml_fp32_to_fp16((i % k == i / k) ? 1.0f : 0.0f);
- } else {
- GGML_ABORT("fatal error");
- }
- }
- for (size_t i = 0; i < y_ne; i++) {
- if (std::is_same<float, Y_TYPE>()) {
- y[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;
- // y[i] = (i % k == i / k) ? 1.0f : 0.0f;
- // y[i] = i + 1;
- } else if (std::is_same<ggml_fp16_t, Y_TYPE>()) {
- y[i] = ggml_fp32_to_fp16((rand() / (float)RAND_MAX) * 2.0f - 1.0f);
- // y[i] = ggml_fp32_to_fp16((i % k == i / k) ? 1.0f : 0.0f);
- // y[i] = ggml_fp32_to_fp16(i + 1);
- } else {
- GGML_ABORT("fatal error");
- }
- }
-
- ggml_vk_buffer_write(d_X, 0, x, sizeof(X_TYPE) * k * m * batch);
- ggml_vk_buffer_write(d_Y, 0, y, sizeof(Y_TYPE) * k * n * batch);
-
- vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);
- ggml_vk_ctx_begin(ctx->device, subctx);
- for (size_t i = 0; i < num_it; i++) {
- ggml_vk_matmul(
- ctx, subctx, p, ggml_vk_subbuffer(ctx, d_X), ggml_vk_subbuffer(ctx, d_Y), ggml_vk_subbuffer(ctx, d_D), ggml_vk_subbuffer(ctx, ctx->prealloc_split_k),
- m, n, k,
- k, k, m, k*m, k*n, m*n,
- split_k, batch, batch, batch, 1, 1, n
- );
- }
- ggml_vk_ctx_end(subctx);
-
- auto begin = std::chrono::high_resolution_clock::now();
- ggml_vk_submit(subctx, ctx->fence);
- VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences", ctx->device);
- ctx->device->device.resetFences({ ctx->fence });
- ggml_vk_queue_command_pools_cleanup(ctx->device);
-
- auto end = std::chrono::high_resolution_clock::now();
- double time = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;
-
- // copy dst to host
- ggml_vk_buffer_read(d_D, 0, d, sizeof(float) * d_ne);
-
- float * d_chk = (float *) malloc(sizeof(float) * d_ne);
-
- ggml_init_params iparams = {
- /*.mem_size =*/ 1024*1024*1024,
- /*.mem_buffer =*/ NULL,
- /*.no_alloc =*/ true,
- };
-
- ggml_context * ggml_ctx = ggml_init(iparams);
-
- ggml_type src0_type;
- ggml_type src1_type;
-
- if (std::is_same<float, X_TYPE>()) {
- src0_type = GGML_TYPE_F32;
- } else if (std::is_same<ggml_fp16_t, X_TYPE>()) {
- src0_type = GGML_TYPE_F16;
- } else {
- GGML_ABORT("fatal error");
- }
- if (std::is_same<float, Y_TYPE>()) {
- src1_type = GGML_TYPE_F32;
- } else if (std::is_same<ggml_fp16_t, Y_TYPE>()) {
- src1_type = GGML_TYPE_F16;
- } else {
- GGML_ABORT("fatal error");
- }
-
- ggml_tensor * src0_ggml = ggml_new_tensor_3d(ggml_ctx, src0_type, k, m, batch);
- ggml_tensor * src1_ggml = ggml_new_tensor_3d(ggml_ctx, src1_type, k, n, batch);
- ggml_tensor * tensor_ggml = ggml_mul_mat(ggml_ctx, src0_ggml, src1_ggml);
-
- src0_ggml->data = x;
- src1_ggml->data = y;
- tensor_ggml->data = d_chk;
-
- ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx);
- ggml_build_forward_expand(cgraph, tensor_ggml);
-
- ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1);
-
- ggml_free(ggml_ctx);
-
- double avg_err = 0.0;
- int first_err_n = -1;
- int first_err_m = -1;
- int first_err_b = -1;
-
- for (size_t i = 0; i < m*n*batch; i++) {
- double err = std::fabs(d[i] - d_chk[i]);
- avg_err += err;
-
- if ((err > 0.05f || std::isnan(err)) && first_err_n == -1) {
- first_err_b = i / (m * n);
- first_err_n = (i % (m * n)) / m;
- first_err_m = (i % (m * n)) % m;
- }
- }
-
- avg_err /= m * n;
-
- double tflops = 2.0*m*n*k*batch*num_it / (time / 1000.0) / (1000.0*1000.0*1000.0*1000.0);
-
- std::cerr << "TEST " << shname << " m=" << m << " n=" << n << " k=" << k << " batch=" << batch << " split_k=" << split_k << " matmul " << time / num_it << "ms " << tflops << " TFLOPS avg_err=" << avg_err << std::endl;
-
- if (avg_err > 0.1 || std::isnan(avg_err)) {
- std::cerr << "m = " << first_err_m << " n = " << first_err_n << " b = " << first_err_b << std::endl;
- std::cerr << "Actual result: " << std::endl << std::endl;
- ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
- std::cerr << "Expected result: " << std::endl << std::endl;
- ggml_vk_print_matrix_area(d_chk, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
-
- if (split_k > 1) {
- float * split_k_buf = (float *) malloc(sizeof(float) * d_ne * split_k);
- ggml_vk_buffer_read(ctx->prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k);
-
- std::cerr << "d_buf0: " << std::endl << std::endl;
- ggml_vk_print_matrix_area(split_k_buf, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
-
- std::cerr << "d_buf1: " << std::endl << std::endl;
- ggml_vk_print_matrix_area(split_k_buf + d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
-
- std::cerr << "d_buf2: " << std::endl << std::endl;
- ggml_vk_print_matrix_area(split_k_buf + 2 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
-
- std::cerr << "d_buf3: " << std::endl << std::endl;
- ggml_vk_print_matrix_area(split_k_buf + 3 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
-
- free(split_k_buf);
- }
- }
-
- free(d_chk);
-
- ggml_vk_command_pool_cleanup(ctx->device, ctx->compute_cmd_pool);
-
- ggml_vk_destroy_buffer(d_X);
- ggml_vk_destroy_buffer(d_Y);
- ggml_vk_destroy_buffer(d_D);
-
- free(x);
- free(y);
- free(d);
-}
-
-static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, int i0, int i1, int i2, int i3) {
- if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16) {
- return;
- }
- i0 = std::max(i0, 5);
- i1 = std::max(i1, 5);
- i2 = std::max(i2, 0);
- i3 = std::max(i3, 0);
- fprintf(stderr, " ");
- for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {
- fprintf(stderr, "%7d ", idx1);
- }
- fprintf(stderr, "\n");
- for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) {
- fprintf(stderr, "%7d: ", idx0);
- for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {
- if (idx0 >= 0 && idx0 < tensor->ne[0] && idx1 >= 0 && idx1 < tensor->ne[1] && i2 >= 0 && i2 < tensor->ne[2] && i3 >= 0 && i3 < tensor->ne[3]) {
- float val;
- if (tensor->type == GGML_TYPE_F32) {
- val = *(float *) ((char *) tensor->data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]);
- } else if (tensor->type == GGML_TYPE_F16) {
- val = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) tensor->data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]));
- } else {
- GGML_ABORT("fatal error");
- }
- fprintf(stderr, "% 7.2f ", val);
- } else {
- fprintf(stderr, " ");
- }
- }
- fprintf(stderr, "\n");
- }
-}
-
-static void ggml_vk_quantize_data(const float * from, void * to, size_t ne, ggml_type quant) {
- ggml_quantize_chunk(quant, from, to, 0, 1, ne, nullptr);
-}
-
-static void ggml_vk_dequantize_data(const void * from, float * to, size_t ne, ggml_type quant) {
- if (quant == GGML_TYPE_F32) {
- memcpy(to, from, sizeof(float) * ne);
- return;
- }
-
- const auto * tt = ggml_get_type_traits(quant);
-
- ggml_to_float_t dequant_fn = tt->to_float;
-
- dequant_fn(from, to, ne);
-}
-
-static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) {
- VK_LOG_DEBUG("ggml_vk_test_dequant(" << ne << ")");
- const size_t x_sz = sizeof(float) * ne;
- const size_t x_sz_f16 = sizeof(ggml_fp16_t) * ne;
- const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant);
- float * x = (float *) malloc(x_sz);
- void * qx = malloc(qx_sz);
- vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
- vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz_f16, {vk::MemoryPropertyFlagBits::eDeviceLocal});
- float * x_ref = (float *) malloc(x_sz);
- ggml_fp16_t * x_chk = (ggml_fp16_t *) malloc(x_sz_f16);
-
- for (size_t i = 0; i < ne; i++) {
- x[i] = rand() / (float)RAND_MAX;
- }
-
- vk_pipeline p = ggml_vk_get_to_fp16(ctx, quant);
-
- ggml_vk_quantize_data(x, qx, ne, quant);
- ggml_vk_dequantize_data(qx, x_ref, ne, quant);
-
- ggml_pipeline_request_descriptor_sets(ctx, p, 1);
-
- ggml_pipeline_allocate_descriptor_sets(ctx);
-
- ggml_vk_buffer_write(qx_buf, 0, qx, qx_sz);
-
- vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);
- ggml_vk_ctx_begin(ctx->device, subctx);
- const std::vector<uint32_t> pc = { 1, (uint32_t)ne, (uint32_t)ne, (uint32_t)ne, (uint32_t)ne };
- ggml_vk_dispatch_pipeline(ctx, subctx, p, { vk_subbuffer{ qx_buf, 0, qx_sz }, vk_subbuffer{ x_buf, 0, x_sz_f16 } }, pc, { (uint32_t)ne, 1, 1});
- ggml_vk_ctx_end(subctx);
-
- auto begin = std::chrono::high_resolution_clock::now();
-
- ggml_vk_submit(subctx, ctx->fence);
- VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);
- ctx->device->device.resetFences({ ctx->fence });
- ggml_vk_queue_command_pools_cleanup(ctx->device);
-
- auto end = std::chrono::high_resolution_clock::now();
-
- double ms_dequant = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;
- ggml_vk_buffer_read(x_buf, 0, x_chk, x_sz_f16);
-
- int first_err = -1;
-
- double avg_err = 0.0;
- for (size_t i = 0; i < ne; i++) {
- double error = std::fabs(x_ref[i] - ggml_fp16_to_fp32(x_chk[i]));
- avg_err += error;
-
- if (first_err < 0 && error > 0.05) {
- first_err = i;
- }
- }
-
- avg_err /= ne;
-
- std::cerr << "TEST DEQUANT " << ggml_type_name(quant) << " time=" << ms_dequant << "ms avg_err=" << avg_err << std::endl;
-
- if (avg_err > 0.1) {
- std::cerr << "first_error = " << first_err << std::endl;
- std::cerr << "Actual result: " << std::endl << std::endl;
- for (int i = std::max(0, first_err - 5); i < std::min((int)ne, first_err + 5); i++) {
- std::cerr << ggml_fp16_to_fp32(x_chk[i]) << ", ";
- }
- std::cerr << std::endl << "Expected result: " << std::endl << std::endl;
- for (int i = std::max(0, first_err - 5); i < std::min((int)ne, first_err + 5); i++) {
- std::cerr << x_ref[i] << ", ";
- }
- std::cerr << std::endl;
- }
-
- ggml_vk_destroy_buffer(x_buf);
- ggml_vk_destroy_buffer(qx_buf);
-
- free(x);
- free(qx);
- free(x_ref);
- free(x_chk);
-}
-
-// This does not work without ggml q8_1 quantization support
-//
-// typedef uint16_t ggml_half;
-// typedef uint32_t ggml_half2;
-//
-// #define QK8_1 32
-// typedef struct {
-// union {
-// struct {
-// ggml_half d; // delta
-// ggml_half s; // d * sum(qs[i])
-// } GGML_COMMON_AGGR_S;
-// ggml_half2 ds;
-// } GGML_COMMON_AGGR_U;
-// int8_t qs[QK8_1]; // quants
-// } block_q8_1;
-//
-// static void ggml_vk_test_quantize(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) {
-// VK_LOG_DEBUG("ggml_vk_test_quantize(" << ne << ")");
-// GGML_ASSERT(quant == GGML_TYPE_Q8_1);
-//
-// const size_t x_sz = sizeof(float) * ne;
-// const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant);
-// float * x = (float *) malloc(x_sz);
-// block_q8_1 * qx = (block_q8_1 *)malloc(qx_sz);
-// block_q8_1 * qx_res = (block_q8_1 *)malloc(qx_sz);
-// vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
-// vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
-//
-// for (size_t i = 0; i < ne; i++) {
-// x[i] = rand() / (float)RAND_MAX;
-// }
-//
-// vk_pipeline p = ggml_vk_get_quantize_pipeline(ctx, quant);
-//
-// ggml_pipeline_request_descriptor_sets(ctx, p, 1);
-//
-// ggml_pipeline_allocate_descriptor_sets(ctx);
-//
-// ggml_vk_buffer_write(x_buf, 0, x, x_sz);
-//
-// vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);
-// ggml_vk_ctx_begin(ctx->device, subctx);
-// ggml_vk_quantize_q8_1(ctx, subctx, ggml_vk_subbuffer(ctx, x_buf), ggml_vk_subbuffer(ctx, qx_buf), ne);
-// ggml_vk_ctx_end(subctx);
-//
-// auto begin = std::chrono::high_resolution_clock::now();
-//
-// ggml_vk_submit(subctx, ctx->fence);
-// VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_quantize waitForFences");
-// ctx->device->device.resetFences({ ctx->fence });
-// ggml_vk_queue_command_pools_cleanup(ctx->device);
-//
-// auto end = std::chrono::high_resolution_clock::now();
-//
-// double ms_quant = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;
-// ggml_vk_buffer_read(qx_buf, 0, qx, qx_sz);
-//
-// ggml_vk_quantize_data(x, qx_res, ne, quant);
-//
-// int first_err = -1;
-//
-// for (size_t i = 0; i < ne / 32; i++) {
-// double error = std::fabs(ggml_fp16_to_fp32(qx_res[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) - ggml_fp16_to_fp32(qx[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d));
-//
-// if (first_err < 0 && error > 0.1) {
-// first_err = i;
-// }
-//
-// error = std::fabs(ggml_fp16_to_fp32(qx_res[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) - ggml_fp16_to_fp32(qx[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s));
-//
-// if (first_err < 0 && error > 0.1) {
-// first_err = i;
-// }
-//
-// for (size_t j = 0; j < 32; j++) {
-// uint64_t error = std::abs(qx_res[i].qs[j] - qx[i].qs[j]);
-//
-// if (first_err < 0 && error > 1) {
-// first_err = i;
-// }
-// }
-// }
-//
-// std::cerr << "TEST QUANTIZE " << ggml_type_name(quant) << " time=" << ms_quant << "ms " << (first_err == -1 ? "CORRECT" : "INCORRECT") << std::endl;
-//
-// if (first_err != -1) {
-// std::cerr << "first_error = " << first_err << std::endl;
-// std::cerr << "Actual result: " << std::endl << std::endl;
-// std::cout << "d=" << ggml_fp16_to_fp32(qx[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) << " s=" << ggml_fp16_to_fp32(qx[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) << " ";
-// for (size_t j = 0; j < 32; j++) {
-// std::cout << " qs" << j << "=" << (uint32_t)qx[first_err].qs[j] << " ";
-// }
-// std::cerr << std::endl << std::endl << "Expected result: " << std::endl << std::endl;
-// std::cout << "d=" << ggml_fp16_to_fp32(qx_res[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) << " s=" << ggml_fp16_to_fp32(qx_res[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) << " ";
-// for (size_t j = 0; j < 32; j++) {
-// std::cout << " qs" << j << "=" << (uint32_t)qx_res[first_err].qs[j] << " ";
-// }
-// std::cerr << std::endl;
-// }
-//
-// ggml_vk_destroy_buffer(x_buf);
-// ggml_vk_destroy_buffer(qx_buf);
-//
-// free(x);
-// free(qx);
-// free(qx_res);
-// }
-
-static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size_t k, size_t batch, size_t num_it, size_t split_k, size_t shader_size, ggml_type quant, bool mmq = false) {
- VK_LOG_DEBUG("ggml_vk_test_dequant_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << ggml_type_name(quant) << ")");
- const size_t x_ne = m * k * batch;
- const size_t y_ne = k * n * batch;
- const size_t d_ne = m * n * batch;
-
- ggml_type b_type = mmq ? GGML_TYPE_Q8_1 : GGML_TYPE_F32;
- bool f16acc = ctx->device->fp16 && !mmq;
- vk_matmul_pipeline_key dq_key{quant, b_type, false, f16acc};
- auto dq_it = ctx->device->pipeline_matmul.find(dq_key);
- if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) {
- if (f16acc) {
- dq_key.f16acc = false;
- dq_it = ctx->device->pipeline_matmul.find(dq_key);
- }
- }
- if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) {
- std::cerr << "error: no pipeline for ggml_vk_test_dequant_matmul " << ggml_type_name(quant) << std::endl;
- return;
- }
- auto& dq_configs = dq_it->second;
- if (shader_size >= (int)dq_configs.size()) {
- std::cerr << "error: shader_size " << shader_size << " >= configs.size() " << dq_configs.size() << " for " << ggml_type_name(quant) << std::endl;
- return;
- }
-
- std::string shname = std::string(ggml_type_name(quant)) + "_ALIGNED_" + std::to_string(shader_size);
- vk_pipeline p = dq_configs[shader_size].aligned ? dq_configs[shader_size].aligned : dq_configs[shader_size].unaligned;
-
- const size_t kpad = mmq ? 0 : ggml_vk_align_size(k, dq_configs[shader_size].align);
-
- if (mmq || k != kpad) {
- p = dq_configs[shader_size].unaligned;
- shname = std::string(ggml_type_name(quant)) + "_" + std::to_string(shader_size);
- }
-
- if (p == nullptr) {
- std::cerr << "error: no pipeline for ggml_vk_test_dequant_matmul " << ggml_type_name(quant) << std::endl;
- return;
- }
-
- const size_t x_sz = sizeof(float) * x_ne;
- const size_t y_sz = sizeof(float) * y_ne;
- const size_t qx_sz = x_ne * ggml_type_size(quant)/ggml_blck_size(quant);
- const size_t qy_sz = mmq ? y_ne * ggml_type_size(GGML_TYPE_Q8_1)/ggml_blck_size(GGML_TYPE_Q8_1) : y_sz;
- const size_t d_sz = sizeof(float) * d_ne;
- float * x = (float *) malloc(x_sz);
- float * y = (float *) malloc(y_sz);
- void * qx = malloc(qx_sz);
- vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
- vk_buffer y_buf = ggml_vk_create_buffer_check(ctx->device, y_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
- vk_buffer qy_buf = ggml_vk_create_buffer_check(ctx->device, qy_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
- vk_buffer d_buf = ggml_vk_create_buffer_check(ctx->device, d_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
- float * d = (float *) malloc(d_sz);
- float * d_chk = (float *) malloc(d_sz);
-
- for (size_t i = 0; i < x_ne; i++) {
- x[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;
- // x[i] = (i % k == i / k) ? 1.0f : 0.0f;
- // x[i] = i % k;
- }
-
- ggml_vk_quantize_data(x, qx, x_ne, quant);
-
- for (size_t i = 0; i < y_ne; i++) {
- y[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;
- // y[i] = (i % k == i / k) ? 1.0f : 0.0f;
- // y[i] = i % k;
- }
-
- if (split_k > 1) {
- ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, num_it);
-
- if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) {
- // Resize buffer
- if (ctx->prealloc_split_k != nullptr) {
- ggml_vk_destroy_buffer(ctx->prealloc_split_k);
- }
- ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, {vk::MemoryPropertyFlagBits::eDeviceLocal});
- }
- }
- if (mmq) {
- vk_pipeline pipeline_quantize_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1);
- ggml_pipeline_request_descriptor_sets(ctx, pipeline_quantize_q8_1, num_it);
- }
-
- ggml_pipeline_allocate_descriptor_sets(ctx);
-
- ggml_vk_buffer_write(qx_buf, 0, qx, qx_sz);
- ggml_vk_buffer_write(y_buf, 0, y, y_sz);
-
- vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);
- ggml_vk_ctx_begin(ctx->device, subctx);
- if (mmq) {
- for (size_t i = 0; i < num_it; i++) {
- ggml_vk_quantize_q8_1(ctx, subctx, { y_buf, 0, y_sz }, { qy_buf, 0, qy_sz }, y_ne);
- ggml_vk_matmul(
- ctx, subctx, p, { qx_buf, 0, qx_sz }, { qy_buf, 0, qy_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k },
- m, n, k,
- k, k, m, k*m, k*n, m*n,
- split_k, batch, batch, batch, 1, 1, n
- );
- }
- } else {
- for (size_t i = 0; i < num_it; i++) {
- ggml_vk_matmul(
- ctx, subctx, p, { qx_buf, 0, qx_sz }, { y_buf, 0, y_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k },
- m, n, k,
- k, k, m, k*m, k*n, m*n,
- split_k, batch, batch, batch, 1, 1, n
- );
- }
- }
- ggml_vk_ctx_end(subctx);
-
- auto begin = std::chrono::high_resolution_clock::now();
-
- ggml_vk_submit(subctx, ctx->fence);
- VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);
- ctx->device->device.resetFences({ ctx->fence });
- ggml_vk_queue_command_pools_cleanup(ctx->device);
-
- auto end = std::chrono::high_resolution_clock::now();
-
- double time_ms = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;
- ggml_vk_buffer_read(d_buf, 0, d, d_sz);
-
- ggml_init_params iparams = {
- /*.mem_size =*/ 1024*1024*1024,
- /*.mem_buffer =*/ NULL,
- /*.no_alloc =*/ true,
- };
-
- ggml_context * ggml_ctx = ggml_init(iparams);
-
- ggml_tensor * src0_ggml = ggml_new_tensor_3d(ggml_ctx, quant, k, m, batch);
- ggml_tensor * src1_ggml = ggml_new_tensor_3d(ggml_ctx, GGML_TYPE_F32, k, n, batch);
- ggml_tensor * tensor_ggml = ggml_mul_mat(ggml_ctx, src0_ggml, src1_ggml);
-
- src0_ggml->data = qx;
- src1_ggml->data = y;
- tensor_ggml->data = d_chk;
-
- ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx);
- ggml_build_forward_expand(cgraph, tensor_ggml);
-
- ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1);
-
- ggml_free(ggml_ctx);
-
- double avg_err = 0.0;
- int first_err_n = -1;
- int first_err_m = -1;
- int first_err_b = -1;
-
- for (size_t i = 0; i < m*n*batch; i++) {
- double err = std::fabs(d[i] - d_chk[i]);
- avg_err += err;
-
- if ((err > 0.05f || std::isnan(err)) && first_err_n == -1) {
- first_err_b = i / (m * n);
- first_err_n = (i % (m * n)) / m;
- first_err_m = (i % (m * n)) % m;
- }
- }
-
- avg_err /= m * n;
-
- double tflops = 2.0*m*n*k*batch*num_it / (time_ms / 1000.0) / (1000.0*1000.0*1000.0*1000.0);
-
- std::cerr << "TEST dequant matmul " << shname;
- if (mmq) {
- std::cerr << " mmq";
- }
- std::cerr << " m=" << m << " n=" << n << " k=" << k << " batch=" << batch << " split_k=" << split_k << " matmul " << time_ms / num_it << "ms " << tflops << " TFLOPS avg_err=" << avg_err << std::endl;
-
- if (avg_err > 0.01 || std::isnan(avg_err)) {
- std::cerr << "m = " << first_err_m << " n = " << first_err_n << " b = " << first_err_b << std::endl;
- std::cerr << "Actual result: " << std::endl << std::endl;
- ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
- std::cerr << std::endl;
- std::cerr << "Expected result: " << std::endl << std::endl;
- ggml_vk_print_matrix_area(d_chk, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
-
- std::cerr << "src0: " << std::endl << std::endl;
- ggml_vk_print_matrix_area(x, GGML_TYPE_F32, k, m, first_err_m, first_err_n, first_err_b);
- std::cerr << std::endl;
- std::cerr << "src1: " << std::endl << std::endl;
- ggml_vk_print_matrix_area(y, GGML_TYPE_F32, k, n, first_err_m, first_err_n, first_err_b);
+ p.nb01 = static_cast<uint32_t>(nb01 / nb00);
+ p.nb02 = static_cast<uint32_t>(nb02 / nb00);
+ p.nb03 = static_cast<uint32_t>(nb03 / nb00);
- if (split_k > 1) {
- float * split_k_buf = (float *) malloc(sizeof(float) * d_ne * split_k);
- ggml_vk_buffer_read(ctx->prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k);
+ p.nb11 = static_cast<uint32_t>(nb11 / nb10);
+ p.nb12 = static_cast<uint32_t>(nb12 / nb10);
+ p.nb13 = static_cast<uint32_t>(nb13 / nb10);
- std::cerr << "d_buf0: " << std::endl << std::endl;
- ggml_vk_print_matrix_area(split_k_buf, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+ p.nb1 = static_cast<uint32_t>(nb1 / nb0);
+ p.nb2 = static_cast<uint32_t>(nb2 / nb0);
+ p.nb3 = static_cast<uint32_t>(nb3 / nb0);
- std::cerr << "d_buf1: " << std::endl << std::endl;
- ggml_vk_print_matrix_area(split_k_buf + d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+ ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_3D, std::move(p));
+}
- std::cerr << "d_buf2: " << std::endl << std::endl;
- ggml_vk_print_matrix_area(split_k_buf + 2 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+void ggml_vk_conv_2d_dw(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ vk_op_conv2d_dw_push_constants p{};
+ p.ne = ggml_nelements(dst);
+ p.channels = dst->ne[2];
+ p.batches = dst->ne[3];
+ p.dst_w = dst->ne[0];
+ p.dst_h = dst->ne[1];
+ p.src_w = src1->ne[0];
+ p.src_h = src1->ne[1];
+ p.knl_w = src0->ne[0];
+ p.knl_h = src0->ne[1];
+ p.stride_x = dst->op_params[0];
+ p.stride_y = dst->op_params[1];
+ p.pad_x = dst->op_params[2];
+ p.pad_y = dst->op_params[3];
+ p.dilation_x = dst->op_params[4];
+ p.dilation_y = dst->op_params[5];
- std::cerr << "d_buf3: " << std::endl << std::endl;
- ggml_vk_print_matrix_area(split_k_buf + 3 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);
+ GGML_ASSERT(src0->ne[3] == p.channels);
+ GGML_ASSERT(src1->ne[3] == p.batches);
- free(split_k_buf);
- }
- }
+ ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_2D_DW, std::move(p));
+}
- ggml_vk_destroy_buffer(qx_buf);
- ggml_vk_destroy_buffer(y_buf);
- ggml_vk_destroy_buffer(qy_buf);
- ggml_vk_destroy_buffer(d_buf);
+void ggml_vk_leaky_relu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+ const float * op_params = (const float *)dst->op_params;
+ vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);
+ p.param1 = op_params[0];
- free(x);
- free(qx);
- free(y);
- free(d);
- free(d_chk);
+ ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_LEAKY_RELU, std::move(p));
}
-#endif
-static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx) {
+void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx) {
#if defined(GGML_VULKAN_RUN_TESTS)
const std::vector<size_t> vals {
512, 512, 128,
@@ -16081,11 +11752,7 @@ static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_contex
}
}
-static void ggml_vk_compute_forward(ggml_backend_vk_context* ctx, ggml_cgraph * cgraph, ggml_tensor* tensor, int tensor_idx, bool almost_ready);
-
-// Returns true if node has enqueued work into the queue, false otherwise
-// If submit is true the current all operations queued so far are being submitted to Vulkan to overlap cmdlist creation and GPU execution.
-static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int node_idx, ggml_tensor *node_begin, int node_idx_begin, bool last_node, bool almost_ready, bool submit){
+bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int node_idx, ggml_tensor *node_begin, int node_idx_begin, bool last_node, bool almost_ready, bool submit){
ggml_tensor * node = cgraph->nodes[node_idx];
if (ggml_is_empty(node) || ggml_op_is_empty(node->op) || !node->buffer) {
return false;
@@ -16658,7 +12325,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr
return true;
}
-static void ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, ggml_tensor * tensor, int tensor_idx, bool almost_ready = false) {
+void ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, ggml_tensor * tensor, int tensor_idx, bool almost_ready) {
GGML_UNUSED(cgraph);
GGML_UNUSED(tensor);
@@ -16708,8 +12375,7 @@ static void ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_cgraph *
}
}
-// Clean up after graph processing is done
-static void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) {
+void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) {
VK_LOG_DEBUG("ggml_vk_graph_cleanup()");
ctx->prealloc_y_last_pipeline_used = {};
ctx->prealloc_y_last_tensor_used = nullptr;
@@ -16747,8 +12413,7 @@ static void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) {
ctx->descriptor_set_idx = 0;
}
-// Clean up on backend free
-static void ggml_vk_cleanup(ggml_backend_vk_context * ctx) {
+void ggml_vk_cleanup(ggml_backend_vk_context * ctx) {
VK_LOG_DEBUG("ggml_vk_cleanup(" << ctx->name << ")");
// discard any unsubmitted command buffers
ctx->compute_ctx.reset();
@@ -16796,13 +12461,13 @@ static void ggml_vk_cleanup(ggml_backend_vk_context * ctx) {
}
}
-static int ggml_vk_get_device_count() {
+int ggml_vk_get_device_count() {
ggml_vk_instance_init();
return vk_instance.device_indices.size();
}
-static void ggml_vk_get_device_description(int device, char * description, size_t description_size) {
+void ggml_vk_get_device_description(int device, char * description, size_t description_size) {
ggml_vk_instance_init();
std::vector<vk::PhysicalDevice> devices = vk_instance.instance.enumeratePhysicalDevices();
@@ -16813,30 +12478,24 @@ static void ggml_vk_get_device_description(int device, char * description, size_
snprintf(description, description_size, "%s", props.deviceName.data());
}
-// backend interface
-
-#define UNUSED GGML_UNUSED
-
-// device backend
-
-static bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {
+bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {
return buffer->buft->iface.get_name == ggml_backend_vk_buffer_type_name;
}
-static void ggml_backend_vk_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+void ggml_backend_vk_buffer_free_buffer(ggml_backend_buffer_t buffer) {
VK_LOG_MEMORY("ggml_backend_vk_buffer_free_buffer()");
ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context;
ggml_vk_destroy_buffer(ctx->dev_buffer);
delete ctx;
}
-static void * ggml_backend_vk_buffer_get_base(ggml_backend_buffer_t buffer) {
+void * ggml_backend_vk_buffer_get_base(ggml_backend_buffer_t buffer) {
return vk_ptr_base;
UNUSED(buffer);
}
-static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
+enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
VK_LOG_DEBUG("ggml_backend_vk_buffer_init_tensor(" << buffer << " (" << buffer->context << "), " << tensor << ")");
if (tensor->view_src != nullptr) {
GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);
@@ -16844,7 +12503,7 @@ static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t
return GGML_STATUS_SUCCESS;
}
-static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
+void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
VK_LOG_DEBUG("ggml_backend_vk_buffer_memset_tensor(" << buffer << ", " << tensor << ", " << value << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
vk_buffer buf = buf_ctx->dev_buffer;
@@ -16857,7 +12516,7 @@ static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, g
ggml_vk_buffer_memset(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, val32, size);
}
-static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
vk_buffer buf = buf_ctx->dev_buffer;
@@ -16869,7 +12528,7 @@ static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml
ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);
}
-static void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset,
+void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset,
size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor_2d(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ", " <<
n_copies << ", " << stride_tensor << ", " << stride_data << ")");
@@ -16883,7 +12542,7 @@ static void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, g
ggml_vk_buffer_write_2d(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, stride_data, stride_tensor, size, n_copies);
}
-static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
@@ -16896,7 +12555,7 @@ static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, cons
ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);
}
-static void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset,
+void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset,
size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor_2d(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ", " <<
n_copies << ", " << stride_tensor << ", " << stride_data << ")");
@@ -16911,7 +12570,7 @@ static void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, c
ggml_vk_buffer_read_2d(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, stride_tensor, stride_data, size, n_copies);
}
-static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
+bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
if (ggml_nbytes(src) == 0) {
return true;
}
@@ -16932,34 +12591,19 @@ static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, cons
UNUSED(buffer);
}
-static void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
+void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context;
ggml_vk_buffer_memset(ctx->dev_buffer, 0, value, buffer->size);
}
-static ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {
- /* .free_buffer = */ ggml_backend_vk_buffer_free_buffer,
- /* .get_base = */ ggml_backend_vk_buffer_get_base,
- /* .init_tensor = */ ggml_backend_vk_buffer_init_tensor,
- /* .memset_tensor = */ ggml_backend_vk_buffer_memset_tensor,
- /* .set_tensor = */ ggml_backend_vk_buffer_set_tensor,
- /* .get_tensor = */ ggml_backend_vk_buffer_get_tensor,
- /* .set_tensor_2d = */ ggml_backend_vk_buffer_set_tensor_2d,
- /* .get_tensor_2d = */ ggml_backend_vk_buffer_get_tensor_2d,
- /* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor,
- /* .clear = */ ggml_backend_vk_buffer_clear,
- /* .reset = */ NULL,
-};
-
-// vk buffer type
-static const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft) {
+const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft) {
ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *)buft->context;
return ctx->name.c_str();
}
-static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
VK_LOG_MEMORY("ggml_backend_vk_buffer_type_alloc_buffer(" << size << ")");
ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context;
@@ -16975,17 +12619,17 @@ static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backe
return ggml_backend_buffer_init(buft, ggml_backend_vk_buffer_interface, bufctx, size);
}
-static size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
+size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context;
return ctx->device->properties.limits.minStorageBufferOffsetAlignment;
}
-static size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
+size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context;
return ctx->device->suballocation_block_size;
}
-static size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
+size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
return ggml_nbytes(tensor);
UNUSED(buft);
@@ -17001,8 +12645,6 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) {
return &dev->buffer_type;
}
-// host buffer type
-
static const char * ggml_backend_vk_host_buffer_type_name(ggml_backend_buffer_type_t buft) {
return GGML_VK_NAME "_Host";
@@ -17048,8 +12690,6 @@ static size_t ggml_backend_vk_host_buffer_type_get_max_size(ggml_backend_buffer_
UNUSED(buft);
}
-// Should be changed to return device-specific host buffer type
-// but that probably requires changes in llama.cpp
ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {
static struct ggml_backend_buffer_type ggml_backend_vk_buffer_type_host = {
/* .iface = */ {
@@ -17071,16 +12711,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {
return &ggml_backend_vk_buffer_type_host;
}
-
-// backend
-
static const char * ggml_backend_vk_name(ggml_backend_t backend) {
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
return ctx->name.c_str();
}
-static void ggml_backend_vk_free(ggml_backend_t backend) {
+void ggml_backend_vk_free(ggml_backend_t backend) {
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
VK_LOG_DEBUG("ggml_backend_vk_free(" << ctx->name << ")");
@@ -17286,7 +12923,7 @@ static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_ba
return false;
}
-static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {
+void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {
VK_LOG_DEBUG("ggml_vk_synchronize()");
bool do_transfer = !ctx->compute_ctx.expired();
@@ -17366,7 +13003,7 @@ static void ggml_backend_vk_synchronize(ggml_backend_t backend) {
ggml_vk_graph_cleanup(ctx);
}
-static bool ggml_vk_is_empty(ggml_tensor * node) {
+bool ggml_vk_is_empty(ggml_tensor * node) {
return ggml_is_empty(node) || node->op == GGML_OP_NONE || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE;
}
@@ -17407,7 +13044,7 @@ static bool ggml_vk_can_fuse_unary_mul_pair(const struct ggml_cgraph * cgraph, i
ggml_vk_can_fuse_unary_mul(cgraph, node_idx, node_idx + 1);
}
-static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) {
+bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) {
if (ops.size() == 2 && ops.begin()[0] == GGML_OP_UNARY && ops.begin()[1] == GGML_OP_MUL) {
return ggml_vk_can_fuse_unary_mul_pair(cgraph, node_idx);
}
@@ -17603,8 +13240,7 @@ static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct g
return true;
}
-// Match SSM_CONV + UNARY(SILU) or SSM_CONV + ADD + UNARY(SILU). num_extra is 1 or 2.
-static bool ggml_vk_can_fuse_ssm_conv(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,
+bool ggml_vk_can_fuse_ssm_conv(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,
int node_idx, int num_extra) {
const ggml_tensor * conv = cgraph->nodes[node_idx];
if (conv->op != GGML_OP_SSM_CONV) {
@@ -17659,7 +13295,7 @@ static bool ggml_vk_can_fuse_ssm_conv(const ggml_backend_vk_context * ctx, const
return true;
}
-static bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,
+bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,
int node_idx, topk_moe_mode mode) {
const ggml_tensor * softmax;
@@ -17770,7 +13406,6 @@ static bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struc
return true;
}
-// Manual op-sequence match (ggml_can_fuse_subgraph rejects the mask's external reshape/cpy).
static bool ggml_vk_match_ops(const struct ggml_cgraph * cgraph, int node_idx,
const std::initializer_list<ggml_op> & ops) {
if (node_idx + (int) ops.size() > cgraph->n_nodes) {
@@ -17787,8 +13422,7 @@ static bool ggml_vk_match_ops(const struct ggml_cgraph * cgraph, int node_idx,
return true;
}
-// True if the qwen4 QSA indexer top-k can be fused at node_idx (the get_rows).
-static bool ggml_vk_can_fuse_topk_qsa(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) {
+bool ggml_vk_can_fuse_topk_qsa(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) {
if (ctx->device->disable_fusion || !ctx->device->pipeline_topk_radix_qsa) {
return false;
}
@@ -17856,7 +13490,7 @@ static bool ggml_vk_can_fuse_topk_qsa(ggml_backend_vk_context * ctx, const struc
return true;
}
-static bool ggml_vk_can_fuse_rope_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,
+bool ggml_vk_can_fuse_rope_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,
int node_idx) {
const ggml_tensor *rope = cgraph->nodes[node_idx + 0];
const ggml_tensor *view = cgraph->nodes[node_idx + 1];
@@ -17895,7 +13529,7 @@ static bool ggml_vk_can_fuse_rope_set_rows(ggml_backend_vk_context * ctx, const
return true;
}
-static bool ggml_vk_can_fuse_rms_norm_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,
+bool ggml_vk_can_fuse_rms_norm_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,
int node_idx) {
const ggml_tensor * rms = cgraph->nodes[node_idx];
const ggml_tensor * view = cgraph->nodes[node_idx + 1];
@@ -17921,10 +13555,7 @@ static bool ggml_vk_can_fuse_rms_norm_set_rows(ggml_backend_vk_context * ctx, co
return true;
}
-// Pattern check for the 5-op Snake fusion: mul -> sin -> sqr -> mul -> add.
-// Verifies the chain shape, the closure x_in_add == x_in_mul0, and that
-// the broadcast operands a and inv_b share a [1, C] layout.
-static bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) {
+bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) {
GGML_UNUSED(ctx);
if (!ggml_can_fuse(cgraph, node_idx, snake_pattern)) {
return false;
@@ -17980,11 +13611,7 @@ static bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct g
return true;
}
-// Check whether the tensors overlap in memory.
-// Fusions can potentially overwrite src tensors in ways that are not prevented
-// by ggml-alloc. If the fusion src is being applied in a way that's elementwise
-// with the destination, then it's OK for them to overlap if they are exactly equal.
-static bool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b, bool elementwise) {
+bool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b, bool elementwise) {
ggml_backend_vk_buffer_context * a_buf_ctx = (ggml_backend_vk_buffer_context *)a->buffer->context;
vk_buffer a_buf = a_buf_ctx->dev_buffer;
ggml_backend_vk_buffer_context * b_buf_ctx = (ggml_backend_vk_buffer_context *)b->buffer->context;
@@ -18007,7 +13634,7 @@ static bool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b
return false;
}
-static bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,
+bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,
int node_idx) {
const ggml_tensor *rms = cgraph->nodes[node_idx + 0];
const ggml_tensor *mul = cgraph->nodes[node_idx + 1];
@@ -18040,7 +13667,7 @@ static bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, co
return true;
}
-static uint32_t ggml_vk_fuse_multi_add(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) {
+uint32_t ggml_vk_fuse_multi_add(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) {
const ggml_tensor *first_node = cgraph->nodes[node_idx];
if (first_node->op != GGML_OP_ADD) {
@@ -18599,8 +14226,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
UNUSED(backend);
}
-// Sort the graph for improved parallelism.
-static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, struct ggml_backend_graph_optimize_params * params)
+void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, struct ggml_backend_graph_optimize_params * params)
{
VK_LOG_DEBUG("ggml_vk_graph_optimize(" << graph->n_nodes << " nodes)");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
@@ -19037,7 +14663,6 @@ static void ggml_backend_vk_event_wait(ggml_backend_t backend, ggml_backend_even
}
}
-// TODO: enable async and synchronize
static ggml_backend_i ggml_backend_vk_interface = {
/* .get_name = */ ggml_backend_vk_name,
/* .free = */ ggml_backend_vk_free,
@@ -19178,17 +14803,6 @@ static std::string ggml_backend_vk_get_device_pci_id(int device_idx) {
return std::string(pci_bus_id);
}
-//////////////////////////
-
-struct ggml_backend_vk_device_context {
- size_t device;
- std::string name;
- std::string description;
- bool is_integrated_gpu;
- std::string pci_bus_id;
- int op_offload_min_batch_size;
-};
-
static const char * ggml_backend_vk_device_get_name(ggml_backend_dev_t dev) {
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
return ctx->name.c_str();
@@ -19909,21 +15523,6 @@ static bool ggml_backend_vk_device_supports_buft(ggml_backend_dev_t dev, ggml_ba
return buft_ctx->device->idx == ctx->device;
}
-static int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op) {
- switch (op->op) {
- case GGML_OP_GET_ROWS:
- return 0;
- case GGML_OP_MUL_MAT:
- return op->ne[1];
- case GGML_OP_MUL_MAT_ID:
- case GGML_OP_ROPE:
- case GGML_OP_ROPE_BACK:
- return op->ne[2];
- default:
- return ggml_nrows(op);
- }
-}
-
static bool ggml_backend_vk_device_offload_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
ggml_backend_vk_device_context * dev_ctx = (ggml_backend_vk_device_context *)dev->context;
@@ -20005,31 +15604,6 @@ static void ggml_backend_vk_device_event_synchronize(ggml_backend_dev_t dev, ggm
}
}
-static vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size) {
- if (!device->external_memory_host) {
- return {};
- }
-
- uintptr_t uptr = reinterpret_cast<uintptr_t>(ptr);
- if (uptr & (device->min_imported_host_pointer_alignment - 1)) {
- return {};
- }
- if (size & (device->min_imported_host_pointer_alignment - 1)) {
- return {};
- }
-
- const vk::MemoryPropertyFlags property_flags = vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached;
-
- vk_buffer buf {};
- try {
- buf = ggml_vk_create_buffer(device, size, { property_flags }, ptr);
- } catch (vk::SystemError& e) {
- GGML_LOG_WARN("ggml_vulkan: Failed ggml_vk_create_buffer (%s)\n", e.what());
- }
-
- return buf;
-}
-
static ggml_backend_buffer_t ggml_backend_vk_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {
VK_LOG_DEBUG("ggml_backend_vk_device_buffer_from_host_ptr(backend=" << dev << ", ptr=" << ptr << ", size=" << size << ")");
GGML_UNUSED(max_tensor_size);
@@ -20140,8 +15714,7 @@ ggml_backend_reg_t ggml_backend_vk_reg() {
}
}
-// Extension availability
-static bool ggml_vk_instance_layer_settings_available() {
+bool ggml_vk_instance_layer_settings_available() {
#ifdef GGML_VULKAN_VALIDATE
// Check if validation layer provides the extension
const std::string layer_name = "VK_LAYER_KHRONOS_validation";
@@ -20159,7 +15732,8 @@ static bool ggml_vk_instance_layer_settings_available() {
#endif
return false;
}
-static bool ggml_vk_instance_portability_enumeration_ext_available(const std::vector<vk::ExtensionProperties>& instance_extensions) {
+
+bool ggml_vk_instance_portability_enumeration_ext_available(const std::vector<vk::ExtensionProperties>& instance_extensions) {
#ifdef __APPLE__
// Check for portability enumeration extension for MoltenVK support
for (const auto& properties : instance_extensions) {
@@ -20174,8 +15748,7 @@ static bool ggml_vk_instance_portability_enumeration_ext_available(const std::ve
UNUSED(instance_extensions);
}
-// Extension availability
-static bool ggml_vk_instance_debug_utils_ext_available(
+bool ggml_vk_instance_debug_utils_ext_available(
const std::vector<vk::ExtensionProperties> & instance_extensions) {
// Check for portability enumeration extension for MoltenVK support
for (const auto & properties : instance_extensions) {
@@ -20190,7 +15763,7 @@ static bool ggml_vk_instance_debug_utils_ext_available(
UNUSED(instance_extensions);
}
-static bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev) {
+bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev) {
VkPhysicalDeviceFeatures2 device_features2;
device_features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2;
@@ -20204,7 +15777,7 @@ static bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev) {
return vk11_features.storageBuffer16BitAccess;
}
-static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch) {
+bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch) {
switch (props.vendorID) {
case VK_VENDOR_ID_INTEL:
// Only allowing Xe2/Xe3 GPU and integrated Xe GPUs at the moment since older hardware (ex. Arc A770) has performance regressions.
@@ -20221,7 +15794,7 @@ static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDevicePrope
}
}
-static uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev) {
+uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev) {
VkPhysicalDeviceProperties2 props = vkdev.getProperties2();
if (props.properties.vendorID != VK_VENDOR_ID_INTEL) {
@@ -20264,8 +15837,7 @@ static uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev)
}
}
-// checks whether lower <= driver_version < upper, with each bound given as xxx.yyyy
-static bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint32_t lower_major, uint32_t lower_minor, uint32_t upper_major, uint32_t upper_minor) {
+bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint32_t lower_major, uint32_t lower_minor, uint32_t upper_major, uint32_t upper_minor) {
#if defined(_WIN32)
// Intel Windows encodes xxx.yyyy as [31:14].[13:0].
const uint32_t major = driver_version >> 14;
@@ -20285,788 +15857,251 @@ static bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint3
#endif
}
+GGML_BACKEND_DL_IMPL(ggml_backend_vk_reg)
-// checks
-
-#ifdef GGML_VULKAN_CHECK_RESULTS
-static void ggml_vk_print_graph_origin(const ggml_tensor * tensor, std::vector<const ggml_tensor *>& done, int level = 0) {
- if (std::find(done.begin(), done.end(), tensor) != done.end() || level > 10) {
- return;
- }
- for (int j = 0; j < level; j++) {
- std::cerr << " ";
- }
- std::cerr << ggml_op_name(tensor->op) << " gpu=" << (tensor->extra != nullptr) << std::endl;
- done.push_back(tensor);
+// out-of-lined header method definitions
- for (int i = 0; i < GGML_MAX_SRC; i++) {
- if (tensor->src[i] != nullptr) {
- ggml_vk_print_graph_origin(tensor->src[i], done, level + 1);
+void vk_queue_handle_synchronized::submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) {
+ // Workaround for NVIDIA driver bug
+ std::unique_lock<std::mutex> device_guard;
+ if (device_submit_mutex) {
+ device_guard = std::unique_lock<std::mutex>(*device_submit_mutex);
+ }
+ std::lock_guard<std::mutex> guard(mutex);
+ try {
+ queue.submit(submits, fence);
+ } catch (vk::DeviceLostError &) {
+ if (auto dev = device.lock()) {
+ ggml_vk_print_device_lost_info(dev);
}
+ throw;
}
}
-static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, const void * data, int i0, int i1, int i2, int i3) {
- if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16 && tensor->type != GGML_TYPE_I32) {
- return;
+void vk_queue_handle_unsynchronized::submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) {
+ // Workaround for NVIDIA driver bug
+ std::unique_lock<std::mutex> device_guard;
+ if (device_submit_mutex) {
+ device_guard = std::unique_lock<std::mutex>(*device_submit_mutex);
}
- i0 = std::max(i0, 5);
- i1 = std::max(i1, 5);
- i2 = std::max(i2, 0);
- i3 = std::max(i3, 0);
- fprintf(stderr, " ");
- for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {
- fprintf(stderr, "%7d ", idx1);
- }
- fprintf(stderr, "\n");
- for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) {
- fprintf(stderr, "%7d: ", idx0);
- for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {
- if (idx0 >= 0 && idx0 < tensor->ne[0] && idx1 >= 0 && idx1 < tensor->ne[1] && i2 >= 0 && i2 < tensor->ne[2] && i3 >= 0 && i3 < tensor->ne[3]) {
- float val;
- if (tensor->type == GGML_TYPE_F32) {
- val = *(const float *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]);
- } else if (tensor->type == GGML_TYPE_F16) {
- val = ggml_fp16_to_fp32(*(const ggml_fp16_t *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]));
- } else if (tensor->type == GGML_TYPE_I32) {
- val = *(const int32_t *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]);
- } else {
- GGML_ABORT("fatal error");
- }
- fprintf(stderr, "% 7.2f ", val);
- } else {
- fprintf(stderr, " ");
- }
+ try {
+ queue.submit(submits, fence);
+ } catch (vk::DeviceLostError &) {
+ if (auto dev = device.lock()) {
+ ggml_vk_print_device_lost_info(dev);
}
- fprintf(stderr, "\n");
+ throw;
}
}
-static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name) {
- void * tensor_data = tensor->data;
+vk_device_struct::~vk_device_struct() {
+ VK_LOG_DEBUG("destroy device " << name);
- const bool is_gpu = tensor->buffer != nullptr && ggml_backend_buffer_is_vk(tensor->buffer);
+ device.destroyFence(fence);
- if (is_gpu) {
- const size_t tensor_size = ggml_nbytes(tensor);
- tensor_data = malloc(tensor_size);
+ ggml_vk_destroy_buffer(sync_staging);
- ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context;
+ if (compute_queue) compute_queue->cmd_pool.destroy(device);
+ if (transfer_queue) transfer_queue->cmd_pool.destroy(device);
- vk_buffer buffer_gpu = buf_ctx->dev_buffer;
- ggml_vk_buffer_read(buffer_gpu, vk_tensor_offset(tensor) + tensor->view_offs, tensor_data, tensor_size);
- }
+ // Explicitly clear to ensure queues drop their shared_ptrs to handles
+ // before the Vulkan logical device instance is destroyed
+ compute_queue.reset();
+ transfer_queue.reset();
- std::cerr << "TENSOR CHECK " << name << " (" << tensor->name << "): " << ggml_op_name(tensor->op) << std::endl;
- std::cerr << "tensor=" << tensor << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << std::endl;
- if (tensor->src[0] != nullptr) {
- std::cerr << "tensor->src[0]=" << tensor->src[0] << " name=" << tensor->src[0]->name << " op=" << ggml_op_name(tensor->src[0]->op) << " type=" << ggml_type_name(tensor->src[0]->type) << " ne0=" << tensor->src[0]->ne[0] << " nb0=" << tensor->src[0]->nb[0] << " ne1=" << tensor->src[0]->ne[1] << " nb1=" << tensor->src[0]->nb[1] << " ne2=" << tensor->src[0]->ne[2] << " nb2=" << tensor->src[0]->nb[2] << " ne3=" << tensor->src[0]->ne[3] << " nb3=" << tensor->src[0]->nb[3] << std::endl;
- }
- if (tensor->src[1] != nullptr) {
- std::cerr << "tensor->src[1]=" << tensor->src[1] << " name=" << tensor->src[1]->name << " op=" << ggml_op_name(tensor->src[1]->op) << " type=" << ggml_type_name(tensor->src[1]->type) << " ne0=" << tensor->src[1]->ne[0] << " nb0=" << tensor->src[1]->nb[0] << " ne1=" << tensor->src[1]->ne[1] << " nb1=" << tensor->src[1]->nb[1] << " ne2=" << tensor->src[1]->ne[2] << " nb2=" << tensor->src[1]->nb[2] << " ne3=" << tensor->src[1]->ne[3] << " nb3=" << tensor->src[1]->nb[3] << std::endl;
- }
- std::cerr << std::endl << "Result:" << std::endl;
- ggml_vk_print_tensor_area(tensor, tensor_data, 5, 5, 0, 0);
- std::cerr << std::endl;
- std::vector<const ggml_tensor *> done;
- ggml_vk_print_graph_origin(tensor, done);
+ for (auto& pipeline : all_pipelines) {
+ if (pipeline.expired()) {
+ continue;
+ }
- if (is_gpu) {
- free(tensor_data);
+ vk_pipeline pl = pipeline.lock();
+ ggml_vk_destroy_pipeline(device, pl);
}
+ all_pipelines.clear();
+
+ device.destroyDescriptorSetLayout(dsl);
+
+ device.destroy();
}
-void * comp_result;
-size_t comp_size;
-size_t comp_nb[GGML_MAX_DIMS];
-size_t check_counter = 0;
-static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx) {
- ggml_tensor * tensor = cgraph->nodes[tensor_idx + ctx->num_additional_fused_ops];
- if (tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_SET_ROWS) {
+void vk_perf_logger::print_timings(bool force) {
+ if (timings.empty()) {
return;
}
-
- check_counter++;
- if (!(vk_output_tensor > 0 && vk_output_tensor == check_counter) && check_counter <= vk_skip_checks) {
+ print_count++;
+ if ((print_count % vk_perf_logger_frequency) != 0 && !force) {
return;
}
-
- VK_LOG_DEBUG("ggml_vk_check_results_0(" << tensor->name << ")");
-
- struct ggml_init_params iparams = {
- /*.mem_size =*/ 2ul*1024ul*1024ul*1024ul,
- /*.mem_buffer =*/ NULL,
- /*.no_alloc =*/ false,
- };
-
- struct ggml_context * ggml_ctx = ggml_init(iparams);
-
- std::array<struct ggml_tensor *, GGML_MAX_SRC> src_clone = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr};
- const char * srci_name[GGML_MAX_SRC] = {"src0", "src1", "src2", "src3", "src4", "src5", "src6", "src7", "src8", "src9"};
-
- std::map<ggml_tensor *, ggml_tensor *> cloned_tensors;
- std::vector<void *> cloned_mallocs;
-
- struct ggml_tensor * tensor_clone = nullptr;
-
- for (int f = 0; f < ctx->num_additional_fused_ops + 1; ++f) {
- tensor = cgraph->nodes[tensor_idx + f];
- for (int i = 0; i < GGML_MAX_SRC; i++) {
- ggml_tensor * srci = tensor->src[i];
- if (srci == nullptr) {
- continue;
- }
- // If a src tensor has been cloned, use that one
- auto it = cloned_tensors.find(srci);
- if (it != cloned_tensors.end()) {
- src_clone[i] = it->second;
- continue;
- }
- ggml_tensor * srci_clone = ggml_dup_tensor(ggml_ctx, srci);
- size_t srci_size = ggml_nbytes(srci);
-
- src_clone[i] = srci_clone;
- void *src_buffer = malloc(srci_size);
- cloned_mallocs.push_back(src_buffer);
-
- srci_clone->data = src_buffer;
- if (ggml_backend_buffer_is_host(srci->buffer)) {
- memcpy(srci_clone->data, srci->data, srci_size);
- memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS);
- } else if (ggml_backend_buffer_is_vk(srci->buffer)) {
- ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)srci->buffer->context;
- vk_buffer& buffer_gpu = buf_ctx->dev_buffer;
- uint64_t offset = vk_tensor_offset(srci) + srci->view_offs;
- if (!ggml_is_contiguous(srci) && ggml_vk_dim01_contiguous(srci)) {
- for (int i3 = 0; i3 < srci->ne[3]; i3++) {
- for (int i2 = 0; i2 < srci->ne[2]; i2++) {
- const int idx = i3*srci->ne[2] + i2;
- ggml_vk_buffer_read(buffer_gpu, offset + idx * srci->nb[2], ((char *)srci_clone->data + idx * srci_clone->nb[2]), srci->ne[1] * srci->nb[1]);
- }
- }
-
- srci_clone->nb[0] = srci->nb[0];
- srci_clone->nb[1] = srci->nb[1];
- for (int i = 2; i < GGML_MAX_DIMS; i++) {
- srci_clone->nb[i] = srci_clone->nb[i - 1]*srci_clone->ne[i - 1];
- }
- } else {
- if (offset + srci_size >= buffer_gpu->size) {
- srci_size = buffer_gpu->size - offset;
- }
- ggml_vk_buffer_read(buffer_gpu, offset, srci_clone->data, srci_size);
- memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS);
- }
- } else {
- GGML_ABORT("fatal error");
- }
-
- if (vk_output_tensor > 0 && vk_output_tensor == check_counter) {
- ggml_vk_print_tensor(srci, srci_name[i]);
- }
+ print_count = 0;
+ uint64_t total_all_op_times = 0;
+ std::cerr << "----------------\nVulkan Timings:" << std::endl;
+ for (const auto & t : timings) {
+ uint64_t total_op_times = 0;
+ for (const auto & time : t.second) {
+ total_op_times += time;
}
+ std::cerr << t.first << ": " << t.second.size() << " x " << (total_op_times / t.second.size() / 1000.0)
+ << " us = " << (total_op_times / 1000.0) << " us";
- if (tensor->op == GGML_OP_FLASH_ATTN_EXT) {
- const float * params = (const float *)tensor->op_params;
- tensor_clone = ggml_flash_attn_ext(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3], params[0], params[1], params[2]);
- if (src_clone[4]) {
- ggml_flash_attn_ext_add_sinks(tensor_clone, src_clone[4]);
- }
- } else if (tensor->op == GGML_OP_MUL_MAT) {
- tensor_clone = ggml_mul_mat(ggml_ctx, src_clone[0], src_clone[1]);
- } else if (tensor->op == GGML_OP_MUL_MAT_ID) {
- tensor_clone = ggml_mul_mat_id(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]);
- } else if (tensor->op == GGML_OP_SUB) {
- tensor_clone = ggml_sub(ggml_ctx, src_clone[0], src_clone[1]);
- } else if (tensor->op == GGML_OP_MUL) {
- tensor_clone = ggml_mul(ggml_ctx, src_clone[0], src_clone[1]);
- } else if (tensor->op == GGML_OP_DIV) {
- tensor_clone = ggml_div(ggml_ctx, src_clone[0], src_clone[1]);
- } else if (tensor->op == GGML_OP_CONCAT) {
- tensor_clone = ggml_concat(ggml_ctx, src_clone[0], src_clone[1], *(int *)tensor->op_params);
- } else if (tensor->op == GGML_OP_UPSCALE) {
- tensor_clone = ggml_interpolate(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], (ggml_scale_mode) tensor->op_params[0]);
- } else if (tensor->op == GGML_OP_SCALE) {
- const float * params = (const float *)tensor->op_params;
- tensor_clone = ggml_scale_bias(ggml_ctx, src_clone[0], params[0], params[1]);
- } else if (tensor->op == GGML_OP_ADD1) {
- tensor_clone = ggml_add1(ggml_ctx, src_clone[0], src_clone[1]);
- } else if (tensor->op == GGML_OP_ARANGE) {
- const float start = ggml_get_op_params_f32(tensor, 0);
- const float stop = ggml_get_op_params_f32(tensor, 1);
- const float step = ggml_get_op_params_f32(tensor, 2);
- tensor_clone = ggml_arange(ggml_ctx, start, stop, step);
- } else if (tensor->op == GGML_OP_FILL) {
- const float value = ggml_get_op_params_f32(tensor, 0);
- tensor_clone = ggml_fill(ggml_ctx, src_clone[0], value);
- } else if (tensor->op == GGML_OP_SQR) {
- tensor_clone = ggml_sqr(ggml_ctx, src_clone[0]);
- } else if (tensor->op == GGML_OP_SQRT) {
- tensor_clone = ggml_sqrt(ggml_ctx, src_clone[0]);
- } else if (tensor->op == GGML_OP_SIN) {
- tensor_clone = ggml_sin(ggml_ctx, src_clone[0]);
- } else if (tensor->op == GGML_OP_COS) {
- tensor_clone = ggml_cos(ggml_ctx, src_clone[0]);
- } else if (tensor->op == GGML_OP_LOG) {
- tensor_clone = ggml_log(ggml_ctx, src_clone[0]);
- } else if (tensor->op == GGML_OP_TRI) {
- tensor_clone = ggml_tri(ggml_ctx, src_clone[0], (ggml_tri_type)ggml_get_op_params_i32(tensor, 0));
- } else if (tensor->op == GGML_OP_DIAG) {
- tensor_clone = ggml_diag(ggml_ctx, src_clone[0]);
- } else if (tensor->op == GGML_OP_CLAMP) {
- const float * params = (const float *)tensor->op_params;
- tensor_clone = ggml_clamp(ggml_ctx, src_clone[0], params[0], params[1]);
- } else if (tensor->op == GGML_OP_PAD) {
- tensor_clone = ggml_pad_ext(ggml_ctx, src_clone[0], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3],
- tensor->op_params[4], tensor->op_params[5], tensor->op_params[6], tensor->op_params[7]);
- } else if (tensor->op == GGML_OP_PAD_REFLECT_1D) {
- tensor_clone = ggml_pad_reflect_1d(ggml_ctx, src_clone[0], tensor->op_params[0], tensor->op_params[1]);
- } else if (tensor->op == GGML_OP_REPEAT) {
- tensor_clone = ggml_repeat(ggml_ctx, src_clone[0], tensor);
- } else if (tensor->op == GGML_OP_REPEAT_BACK) {
- tensor_clone = ggml_repeat_back(ggml_ctx, src_clone[0], tensor);
- } else if (tensor->op == GGML_OP_ADD) {
- tensor_clone = ggml_add(ggml_ctx, src_clone[0], src_clone[1]);
- } else if (tensor->op == GGML_OP_ACC) {
- tensor_clone = ggml_acc(ggml_ctx, src_clone[0], src_clone[1], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3]);
- } else if (tensor->op == GGML_OP_SET) {
- tensor_clone = ggml_set(ggml_ctx, src_clone[0], src_clone[1], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3]);
- } else if (tensor->op == GGML_OP_NORM) {
- tensor_clone = ggml_norm(ggml_ctx, src_clone[0], *(float *)tensor->op_params);
- } else if (tensor->op == GGML_OP_GROUP_NORM) {
- const float * float_params = (const float *)tensor->op_params;
- tensor_clone = ggml_group_norm(ggml_ctx, src_clone[0], tensor->op_params[0], float_params[1]);
- } else if (tensor->op == GGML_OP_RMS_NORM) {
- tensor_clone = ggml_rms_norm(ggml_ctx, src_clone[0], *(float *)tensor->op_params);
- } else if (tensor->op == GGML_OP_RMS_NORM_BACK) {
- const float eps = ((float *) tensor->op_params)[0];
- tensor_clone = ggml_rms_norm_back(ggml_ctx, src_clone[0], src_clone[1], eps);
- } else if (tensor->op == GGML_OP_SILU_BACK) {
- tensor_clone = ggml_silu_back(ggml_ctx, src_clone[0], src_clone[1]);
- } else if (tensor->op == GGML_OP_L2_NORM) {
- const float eps = ((float *) tensor->op_params)[0];
- tensor_clone = ggml_l2_norm(ggml_ctx, src_clone[0], eps);
- } else if (tensor->op == GGML_OP_SOFT_MAX) {
- if (tensor->src[1] != nullptr) {
- const float * params = (const float *)tensor->op_params;
- tensor_clone = ggml_soft_max_ext(ggml_ctx, src_clone[0], src_clone[1], params[0], params[1]);
- } else {
- tensor_clone = ggml_soft_max(ggml_ctx, src_clone[0]);
- }
- } else if (tensor->op == GGML_OP_SOFT_MAX_BACK) {
- tensor_clone = ggml_soft_max_ext_back(ggml_ctx, src_clone[0], src_clone[1], ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]);
- } else if (tensor->op == GGML_OP_DIAG_MASK_INF) {
- tensor_clone = ggml_diag_mask_inf(ggml_ctx, src_clone[0], tensor->op_params[0]);
- } else if (tensor->op == GGML_OP_ROPE || tensor->op == GGML_OP_ROPE_BACK) {
- const int n_dims = ((int32_t *) tensor->op_params)[1];
- const int mode = ((int32_t *) tensor->op_params)[2];
- //const int n_ctx_ggml = ((int32_t *) tensor->op_params)[3];
- const int n_ctx_orig_ggml = ((int32_t *) tensor->op_params)[4];
- const float freq_base = ((float *) tensor->op_params)[5];
- const float freq_scale = ((float *) tensor->op_params)[6];
- const float ext_factor = ((float *) tensor->op_params)[7];
- const float attn_factor = ((float *) tensor->op_params)[8];
- const float beta_fast = ((float *) tensor->op_params)[9];
- const float beta_slow = ((float *) tensor->op_params)[10];
- if (mode & GGML_ROPE_TYPE_MROPE) {
- int32_t *sections = ((int32_t *) tensor->op_params) + 11;
- if (tensor->op == GGML_OP_ROPE) {
- tensor_clone = ggml_rope_multi(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- } else {
- tensor_clone = ggml_rope_multi_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- }
- } else {
- if (tensor->op == GGML_OP_ROPE) {
- tensor_clone = ggml_rope_ext(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- } else {
- tensor_clone = ggml_rope_ext_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- }
- }
- const int n_offs = ((int32_t *) tensor->op_params)[15];
- if (n_offs != 0) {
- tensor_clone = ggml_rope_set_offset(tensor_clone, n_offs);
- }
- } else if (tensor->op == GGML_OP_UNARY) {
- switch (ggml_get_unary_op(tensor)) {
- case GGML_UNARY_OP_EXP:
- tensor_clone = ggml_exp(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_EXPM1:
- tensor_clone = ggml_expm1(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_ELU:
- tensor_clone = ggml_elu(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_SILU:
- tensor_clone = ggml_silu(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_GELU:
- tensor_clone = ggml_gelu(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_GELU_ERF:
- tensor_clone = ggml_gelu_erf(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_GELU_QUICK:
- tensor_clone = ggml_gelu_quick(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_RELU:
- tensor_clone = ggml_relu(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_XIELU:
- tensor_clone = ggml_xielu(ggml_ctx, src_clone[0], 0, 0, 0, 0);
- ggml_set_op_params_f32(tensor_clone, 1, ggml_get_op_params_f32(tensor, 1));
- ggml_set_op_params_f32(tensor_clone, 2, ggml_get_op_params_f32(tensor, 2));
- ggml_set_op_params_f32(tensor_clone, 3, ggml_get_op_params_f32(tensor, 3));
- ggml_set_op_params_f32(tensor_clone, 4, ggml_get_op_params_f32(tensor, 4));
- break;
- case GGML_UNARY_OP_NEG:
- tensor_clone = ggml_neg(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_TANH:
- tensor_clone = ggml_tanh(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_SIGMOID:
- tensor_clone = ggml_sigmoid(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_HARDSIGMOID:
- tensor_clone = ggml_hardsigmoid(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_HARDSWISH:
- tensor_clone = ggml_hardswish(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_ABS:
- tensor_clone = ggml_abs(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_SOFTPLUS:
- tensor_clone = ggml_softplus(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_STEP:
- tensor_clone = ggml_step(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_ROUND:
- tensor_clone = ggml_round(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_CEIL:
- tensor_clone = ggml_ceil(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_FLOOR:
- tensor_clone = ggml_floor(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_TRUNC:
- tensor_clone = ggml_trunc(ggml_ctx, src_clone[0]);
- break;
- case GGML_UNARY_OP_SGN:
- tensor_clone = ggml_sgn(ggml_ctx, src_clone[0]);
- break;
- default:
- std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl;
- GGML_ABORT("fatal error");
- }
- } else if (tensor->op == GGML_OP_GLU) {
- if (src_clone[1] == nullptr) {
- tensor_clone = ggml_glu(ggml_ctx, src_clone[0], (ggml_glu_op) tensor->op_params[0], tensor->op_params[1]);
- } else {
- tensor_clone = ggml_glu_split(ggml_ctx, src_clone[0], src_clone[1], (ggml_glu_op) tensor->op_params[0]);
- }
- ggml_set_op_params_i32(tensor_clone, 2, ggml_get_op_params_i32(tensor, 2));
- ggml_set_op_params_i32(tensor_clone, 3, ggml_get_op_params_i32(tensor, 3));
- } else if (tensor->op == GGML_OP_CPY || tensor->op == GGML_OP_DUP) {
- if (tensor->src[1] == nullptr) {
- tensor_clone = ggml_dup(ggml_ctx, src_clone[0]);
- tensor_clone->type = tensor->type;
- } else {
- tensor_clone = ggml_cpy(ggml_ctx, src_clone[0], src_clone[1]);
- }
- } else if (tensor->op == GGML_OP_CONT) {
- tensor_clone = ggml_cont_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]);
- } else if (tensor->op == GGML_OP_RESHAPE) {
- tensor_clone = ggml_reshape_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]);
- } else if (tensor->op == GGML_OP_VIEW) {
- tensor_clone = ggml_view_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], tensor->nb[1], tensor->nb[2], tensor->nb[3], ((int32_t *) tensor->op_params)[0]);
- } else if (tensor->op == GGML_OP_PERMUTE) {
- int32_t * params = (int32_t *)tensor->op_params;
- tensor_clone = ggml_permute(ggml_ctx, src_clone[0], params[0], params[1], params[2], params[3]);
- } else if (tensor->op == GGML_OP_TRANSPOSE) {
- tensor_clone = ggml_transpose(ggml_ctx, src_clone[0]);
- } else if (tensor->op == GGML_OP_GET_ROWS) {
- tensor_clone = ggml_get_rows(ggml_ctx, src_clone[0], src_clone[1]);
- } else if (tensor->op == GGML_OP_ARGSORT) {
- tensor_clone = ggml_argsort(ggml_ctx, src_clone[0], (ggml_sort_order) *(int *)tensor->op_params);
- } else if (tensor->op == GGML_OP_TOP_K) {
- tensor_clone = ggml_top_k(ggml_ctx, src_clone[0], tensor->ne[0]);
- } else if (tensor->op == GGML_OP_SUM) {
- tensor_clone = ggml_sum(ggml_ctx, src_clone[0]);
- } else if (tensor->op == GGML_OP_SUM_ROWS) {
- tensor_clone = ggml_sum_rows(ggml_ctx, src_clone[0]);
- } else if (tensor->op == GGML_OP_CUMSUM) {
- tensor_clone = ggml_cumsum(ggml_ctx, src_clone[0]);
- } else if (tensor->op == GGML_OP_DSV4_HC_COMB) {
- tensor_clone = ggml_dsv4_hc_comb(ggml_ctx, src_clone[0], src_clone[1], src_clone[2],
- ggml_get_op_params_f32(tensor, 0), ggml_get_op_params_i32(tensor, 1));
- } else if (tensor->op == GGML_OP_DSV4_HC_PRE) {
- if (ggml_get_op_params_i32(tensor, 1) != 0) {
- tensor_clone = ggml_dsv4_hc_pre_gated(ggml_ctx, src_clone[0], src_clone[1], ggml_get_op_params_f32(tensor, 0));
- } else {
- tensor_clone = ggml_dsv4_hc_pre(ggml_ctx, src_clone[0], src_clone[1]);
+ // If we have as many flops entries as timing entries for the op, then compute and log the flops/S.
+ auto it = flops.find(t.first);
+ if (it != flops.end() && (it->second).size() == t.second.size()) {
+ uint64_t total_op_flops = 0;
+ for (const auto & elem : it->second) {
+ total_op_flops += elem;
}
- } else if (tensor->op == GGML_OP_DSV4_HC_POST) {
- tensor_clone = ggml_dsv4_hc_post(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3]);
- } else if (tensor->op == GGML_OP_MEAN) {
- tensor_clone = ggml_mean(ggml_ctx, src_clone[0]);
- } else if (tensor->op == GGML_OP_ARGMAX) {
- tensor_clone = ggml_argmax(ggml_ctx, src_clone[0]);
- } else if (tensor->op == GGML_OP_CROSS_ENTROPY_LOSS) {
- tensor_clone = ggml_cross_entropy_loss(ggml_ctx, src_clone[0], src_clone[1]);
- } else if (tensor->op == GGML_OP_CROSS_ENTROPY_LOSS_BACK) {
- tensor_clone = ggml_cross_entropy_loss_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]);
- } else if (tensor->op == GGML_OP_COUNT_EQUAL) {
- tensor_clone = ggml_count_equal(ggml_ctx, src_clone[0], src_clone[1]);
- } else if (tensor->op == GGML_OP_SOLVE_TRI) {
- tensor_clone = ggml_solve_tri(ggml_ctx, src_clone[0], src_clone[1], true, true, false);
- } else if (tensor->op == GGML_OP_IM2COL) {
- const int32_t s0 = tensor->op_params[0];
- const int32_t s1 = tensor->op_params[1];
- const int32_t p0 = tensor->op_params[2];
- const int32_t p1 = tensor->op_params[3];
- const int32_t d0 = tensor->op_params[4];
- const int32_t d1 = tensor->op_params[5];
-
- const bool is_2D = tensor->op_params[6] == 1;
- tensor_clone = ggml_im2col(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1, is_2D, tensor->type);
- } else if (tensor->op == GGML_OP_IM2COL_3D) {
- const int32_t s0 = tensor->op_params[0];
- const int32_t s1 = tensor->op_params[1];
- const int32_t s2 = tensor->op_params[2];
- const int32_t p0 = tensor->op_params[3];
- const int32_t p1 = tensor->op_params[4];
- const int32_t p2 = tensor->op_params[5];
- const int32_t d0 = tensor->op_params[6];
- const int32_t d1 = tensor->op_params[7];
- const int32_t d2 = tensor->op_params[8];
- const int32_t IC = tensor->op_params[9];
-
- tensor_clone = ggml_im2col_3d(ggml_ctx, src_clone[0], src_clone[1], IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, tensor->type);
- } else if (tensor->op == GGML_OP_TIMESTEP_EMBEDDING) {
- const int32_t dim = tensor->op_params[0];
- const int32_t max_period = tensor->op_params[1];
- tensor_clone = ggml_timestep_embedding(ggml_ctx, src_clone[0], dim, max_period);
- } else if (tensor->op == GGML_OP_CONV_TRANSPOSE_1D){
- const int32_t s0 = tensor->op_params[0];
- const int32_t p0 = tensor->op_params[1];
- const int32_t d0 = tensor->op_params[2];
- tensor_clone = ggml_conv_transpose_1d(ggml_ctx, src_clone[0], src_clone[1], s0, p0, d0);
- } else if (tensor->op == GGML_OP_COL2IM_1D) {
- const int32_t stride = tensor->op_params[0];
- const int32_t oc = tensor->op_params[1];
- const int32_t p0 = tensor->op_params[2];
- tensor_clone = ggml_col2im_1d(ggml_ctx, src_clone[0], stride, oc, p0);
- } else if (tensor->op == GGML_OP_POOL_1D) {
- enum ggml_op_pool op = static_cast<ggml_op_pool>(tensor->op_params[0]);
- const int32_t k0 = tensor->op_params[1];
- const int32_t s0 = tensor->op_params[2];
- const int32_t p0 = tensor->op_params[3];
-
- tensor_clone = ggml_pool_1d(ggml_ctx, src_clone[0], op, k0, s0, p0);
- } else if (tensor->op == GGML_OP_POOL_2D) {
- enum ggml_op_pool op = static_cast<ggml_op_pool>(tensor->op_params[0]);
- const int32_t k0 = tensor->op_params[1];
- const int32_t k1 = tensor->op_params[2];
- const int32_t s0 = tensor->op_params[3];
- const int32_t s1 = tensor->op_params[4];
- const int32_t p0 = tensor->op_params[5];
- const int32_t p1 = tensor->op_params[6];
-
- tensor_clone = ggml_pool_2d(ggml_ctx, src_clone[0], op, k0, k1, s0, s1, p0, p1);
- } else if (tensor->op == GGML_OP_CONV_2D) {
- const int32_t s0 = tensor->op_params[0];
- const int32_t s1 = tensor->op_params[1];
- const int32_t p0 = tensor->op_params[2];
- const int32_t p1 = tensor->op_params[3];
- const int32_t d0 = tensor->op_params[4];
- const int32_t d1 = tensor->op_params[5];
- tensor_clone = ggml_conv_2d(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1);
- } else if (tensor->op == GGML_OP_CONV_3D) {
- const int32_t s0 = tensor->op_params[0];
- const int32_t s1 = tensor->op_params[1];
- const int32_t s2 = tensor->op_params[2];
- const int32_t p0 = tensor->op_params[3];
- const int32_t p1 = tensor->op_params[4];
- const int32_t p2 = tensor->op_params[5];
- const int32_t d0 = tensor->op_params[6];
- const int32_t d1 = tensor->op_params[7];
- const int32_t d2 = tensor->op_params[8];
- const int32_t IC = tensor->op_params[9];
- const int32_t N = tensor->op_params[10];
- const int32_t OC = tensor->op_params[11];
- tensor_clone = ggml_conv_3d_direct(ggml_ctx, src_clone[0], src_clone[1], s0, s1, s2, p0, p1, p2, d0, d1, d2, IC, N, OC);
- } else if (tensor->op == GGML_OP_CONV_2D_DW) {
- const int32_t s0 = tensor->op_params[0];
- const int32_t s1 = tensor->op_params[1];
- const int32_t p0 = tensor->op_params[2];
- const int32_t p1 = tensor->op_params[3];
- const int32_t d0 = tensor->op_params[4];
- const int32_t d1 = tensor->op_params[5];
- tensor_clone = ggml_conv_2d_dw_direct(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1);
- } else if (tensor->op == GGML_OP_CONV_TRANSPOSE_2D) {
- const int32_t s = tensor->op_params[0];
- tensor_clone = ggml_conv_transpose_2d_p0(ggml_ctx, src_clone[0], src_clone[1], s);
- } else if (tensor->op == GGML_OP_LEAKY_RELU) {
- const float * op_params = (const float *)tensor->op_params;
- tensor_clone = ggml_leaky_relu(ggml_ctx, src_clone[0], op_params[0], false);
- } else if (tensor->op == GGML_OP_RWKV_WKV6) {
- tensor_clone = ggml_rwkv_wkv6(ggml_ctx, src_clone[0], src_clone[1],
- src_clone[2], src_clone[3], src_clone[4], src_clone[5]);
- } else if (tensor->op == GGML_OP_RWKV_WKV7) {
- tensor_clone = ggml_rwkv_wkv7(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3],
- src_clone[4], src_clone[5], src_clone[6]);
- } else if (tensor->op == GGML_OP_GATED_LINEAR_ATTN) {
- const float * op_params = (const float *)tensor->op_params;
- tensor_clone = ggml_gated_linear_attn(ggml_ctx, src_clone[0], src_clone[1],
- src_clone[2], src_clone[3], src_clone[4], op_params[0]);
- } else if (tensor->op == GGML_OP_LIGHTNING_INDEXER) {
- tensor_clone = ggml_lightning_indexer(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3]);
- } else if (tensor->op == GGML_OP_GATED_DELTA_NET) {
- tensor_clone = ggml_gated_delta_net(ggml_ctx, src_clone[0], src_clone[1],
- src_clone[2], src_clone[3], src_clone[4], src_clone[5],
- ggml_get_op_params_i32(tensor, 0));
- } else if (tensor->op == GGML_OP_OPT_STEP_ADAMW) {
- src_clone[0]->flags = tensor->src[0]->flags;
- tensor_clone = ggml_opt_step_adamw(ggml_ctx, src_clone[0], src_clone[1],
- src_clone[2], src_clone[3], src_clone[4]);
- } else if (tensor->op == GGML_OP_OPT_STEP_SGD) {
- src_clone[0]->flags = tensor->src[0]->flags;
- tensor_clone = ggml_opt_step_sgd(ggml_ctx, src_clone[0], src_clone[1],
- src_clone[2]);
- } else if (tensor->op == GGML_OP_ADD_ID) {
- tensor_clone = ggml_add_id(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]);
- } else if (tensor->op == GGML_OP_SSM_SCAN) {
- const int32_t K = ggml_get_op_params_i32(tensor, 0);
- tensor_clone = ggml_ssm_scan(ggml_ctx, src_clone[0], src_clone[1], src_clone[2],
- src_clone[3], src_clone[4], src_clone[5], src_clone[6], K);
- } else if (tensor->op == GGML_OP_SSM_CONV) {
- tensor_clone = ggml_ssm_conv(ggml_ctx, src_clone[0], src_clone[1]);
- } else if (tensor->op == GGML_OP_ROLL) {
- const int32_t s0 = tensor->op_params[0];
- const int32_t s1 = tensor->op_params[1];
- const int32_t s2 = tensor->op_params[2];
- const int32_t s3 = tensor->op_params[3];
- tensor_clone = ggml_roll(ggml_ctx, src_clone[0], s0, s1, s2, s3);
- }
- else {
- std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl;
- GGML_ABORT("fatal error");
+ std::cerr << " ("
+ << (double(total_op_flops) / (1000.0 * 1000.0 * 1000.0)) /
+ (double(total_op_times) / (1000.0 * 1000.0 * 1000.0))
+ << " GFLOPS/s)";
}
- cloned_tensors[tensor] = tensor_clone;
- }
-
- ggml_cgraph * cgraph_cpu = ggml_new_graph(ggml_ctx);
- ggml_build_forward_expand(cgraph_cpu, tensor_clone);
- ggml_graph_compute_with_ctx(ggml_ctx, cgraph_cpu, 8);
+ total_all_op_times += total_op_times;
- if (vk_output_tensor > 0 && vk_output_tensor == check_counter) {
- ggml_vk_print_tensor(tensor_clone, "tensor_clone");
+ std::cerr << std::endl;
}
- comp_size = ggml_nbytes(tensor_clone);
-
- comp_result = malloc(comp_size);
- memcpy(comp_result, tensor_clone->data, comp_size);
- memcpy(comp_nb, tensor_clone->nb, sizeof(size_t) * GGML_MAX_DIMS);
-
- for (auto m : cloned_mallocs) {
- free(m);
+ if (timings.size() > 0) {
+ std::cerr << "Total time: " << total_all_op_times / 1000.0 << " us." << std::endl;
}
- ggml_free(ggml_ctx);
-
- VK_LOG_DEBUG("END ggml_vk_check_results_0(" << tensor->name << ")");
+ timings.clear();
+ flops.clear();
}
-static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx) {
- ggml_tensor * tensor = cgraph->nodes[tensor_idx + ctx->num_additional_fused_ops];
- if (tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_SET_ROWS) {
+std::string vk_perf_logger::get_node_fusion_name(const ggml_tensor * node, const char *fusion_name, uint64_t *n_flops) {
+ *n_flops = ggml_vk_get_node_flops(node);
+ std::string fusion_str;
+ if (fusion_name) {
+ fusion_str = fusion_name + std::string(" ");
+ }
+ if (node->op == GGML_OP_UNARY) {
+ return fusion_str + ggml_unary_op_name(ggml_get_unary_op(node));
+ }
+ if (node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) {
+ const uint64_t m = node->ne[0];
+ const uint64_t n = node->ne[1];
+ const uint64_t k = node->src[1]->ne[0];
+ const uint64_t batch = node->ne[2] * node->ne[3];
+ std::string name = ggml_op_name(node->op);
+ if ((node->op == GGML_OP_MUL_MAT && n <= mul_mat_vec_max_cols) ||
+ (node->op == GGML_OP_MUL_MAT_ID && node->src[2]->ne[1] == 1)) {
+ name += "_VEC";
+ }
+ name += " ";
+ name += ggml_type_name(node->src[0]->type);
+ name += " m=" + std::to_string(m) + " n=" + std::to_string(n) + " k=" + std::to_string(k);
+ if (node->op == GGML_OP_MUL_MAT_ID) {
+ name += " n_expert=" + std::to_string(node->src[0]->ne[2]);
+ }
+ if (batch > 1) {
+ name += " batch=" + std::to_string(batch);
+ }
+ return fusion_str + name;
+ }
+ if (node->op == GGML_OP_CONV_2D || node->op == GGML_OP_CONV_TRANSPOSE_2D) {
+ std::string name = ggml_op_name(node->op);
+ const ggml_tensor * knl = node->src[0];
+ uint64_t Cout = node->ne[2];
+ uint64_t size_K = node->src[1]->ne[2] * knl->ne[0] * knl->ne[1];
+ uint64_t size_N = node->ne[3] * node->ne[0] * node->ne[1];
+ name += " M=Cout=" + std::to_string(Cout) + ", K=Cin*KW*KH=" + std::to_string(size_K) +
+ ", N=N*OW*OH=" + std::to_string(size_N);
+ return fusion_str + name;
+ }
+ if (node->op == GGML_OP_RMS_NORM) {
+ std::string name = ggml_op_name(node->op);
+ name += "(" + std::to_string(node->ne[0]) + "," + std::to_string(node->ne[1]) + "," + std::to_string(node->ne[2]) + "," + std::to_string(node->ne[3]) + ")";
+ return fusion_str + name;
+ }
+ if (node->op == GGML_OP_FLASH_ATTN_EXT) {
+ const ggml_tensor * dst = node;
+ const ggml_tensor * q = node->src[0];
+ const ggml_tensor * k = node->src[1];
+ const ggml_tensor * v = node->src[2];
+ const ggml_tensor * m = node->src[3];
+ std::stringstream name;
+ name << fusion_str;
+ name << ggml_op_name(node->op) <<
+ " dst(" << dst->ne[0] << "," << dst->ne[1] << "," << dst->ne[2] << "," << dst->ne[3] << "), " <<
+ " q(" << q->ne[0] << "," << q->ne[1] << "," << q->ne[2] << "," << q->ne[3] << "), " <<
+ " k(" << k->ne[0] << "," << k->ne[1] << "," << k->ne[2] << "," << k->ne[3] << "), " <<
+ " v(" << v->ne[0] << "," << v->ne[1] << "," << v->ne[2] << "," << v->ne[3] << "), " <<
+ " m(" << (m?m->ne[0]:0) << "," << (m?m->ne[1]:0) << "," << (m?m->ne[2]:0) << "," << (m?m->ne[3]:0) << ")";
+ return name.str();
+ }
+ if (node->op == GGML_OP_TOP_K) {
+ std::stringstream name;
+ name << fusion_str;
+ name << ggml_op_name(node->op) <<
+ " K=" << node->ne[0] <<
+ " (" << node->src[0]->ne[0] << "," << node->src[0]->ne[1] << "," << node->src[0]->ne[2] << "," << node->src[0]->ne[3] << ")";
+ return name.str();
+ }
+ return fusion_str + ggml_op_name(node->op);
+}
+
+ggml_backend_vk_buffer_context::~ggml_backend_vk_buffer_context() {
+ ggml_vk_destroy_buffer(dev_buffer);
+}
+
+ggml_vk_debug_label::ggml_vk_debug_label(vk_context & ctx, const std::string & pipeline_name, uint32_t wg0, uint32_t wg1, uint32_t wg2) {
+ if (!vk_instance.debug_utils_support || ctx->s == nullptr) {
return;
}
+ begin(ctx, pipeline_name + " (" + std::to_string(wg0) + "," + std::to_string(wg1) + "," + std::to_string(wg2) + ")");
+}
- if (!(vk_output_tensor > 0 && vk_output_tensor == check_counter) && check_counter <= vk_skip_checks) {
+ggml_vk_debug_label::ggml_vk_debug_label(vk_context & ctx, const ggml_cgraph * cgraph, int node_idx, int n_fused) {
+ if (!vk_instance.debug_utils_support || ctx->s == nullptr) {
return;
}
+ std::string name = ggml_op_name(cgraph->nodes[node_idx]->op);
+ for (int i = 1; i <= n_fused; i++) {
+ name += "+";
+ name += ggml_op_name(cgraph->nodes[node_idx + i]->op);
+ }
+ name += " ";
+ name += cgraph->nodes[node_idx]->name;
+ begin(ctx, name);
+}
- VK_LOG_DEBUG("ggml_vk_check_results_1(" << tensor->name << ")");
-
- ggml_tensor * src0 = tensor->src[0];
- ggml_tensor * src1 = tensor->src[1];
- ggml_tensor * src2 = tensor->src[2];
- ggml_tensor * src3 = tensor->src[3];
-
- void * tensor_data = tensor->data;
-
- if (ggml_backend_buffer_is_vk(tensor->buffer)) {
- size_t tensor_size = ggml_nbytes(tensor);
- tensor_data = malloc(tensor_size);
-
- ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context;
-
- vk_buffer& buffer_gpu = buf_ctx->dev_buffer;
- uint64_t offset = vk_tensor_offset(tensor) + tensor->view_offs;
- if (offset + tensor_size >= buffer_gpu->size) {
- tensor_size = buffer_gpu->size - offset;
- }
-
- ggml_vk_buffer_read(buffer_gpu, offset, tensor_data, tensor_size);
- }
-
- float first_error_result = -1.0f;
- float first_error_correct = -1.0f;
- std::array<int, 4> first_error = { -1, -1, -1, -1 };
- double avg_err = 0.0;
- size_t counter = 0;
-
- for (int i3 = 0; i3 < tensor->ne[3]; i3++) {
- for (int i2 = 0; i2 < tensor->ne[2]; i2++) {
- for (int i1 = 0; i1 < tensor->ne[1]; i1++) {
- for (int i0 = 0; i0 < tensor->ne[0]; i0++) {
- const bool buffer_size_fit = i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0] < comp_size;
- float correct = 0.0f;
- float result = 0.0f;
-
- if (buffer_size_fit) {
- if (tensor->type == GGML_TYPE_F32) {
- correct = *(float *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]);
- result = *(float *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]);
- } else if (tensor->type == GGML_TYPE_F16) {
- correct = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]));
- result = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]));
- } else if (tensor->type == GGML_TYPE_BF16) {
- correct = ggml_bf16_to_fp32(*(ggml_bf16_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]));
- result = ggml_bf16_to_fp32(*(ggml_bf16_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]));
- } else if (tensor->type == GGML_TYPE_I32) {
- correct = *(int32_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]);
- result = *(int32_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]);
- } else if (tensor->type == GGML_TYPE_I64) {
- correct = *(int64_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]);
- result = *(int64_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]);
- } else {
- std::cerr << "Results check not implemented for type " << ggml_type_name(tensor->type) << std::endl;
- }
- } else {
- std::cerr << "Missing debug code for type " << ggml_type_name(tensor->type) << std::endl;
- GGML_ABORT("fatal error");
- }
-
- if ((std::isnan(correct) != std::isnan(result)) || (std::isinf(correct) != std::isinf(result)) || !buffer_size_fit) {
- std::cerr << "ERROR: Invalid value in " << ggml_op_name(tensor->op) << " i3=" << i3 << " i2=" << i2 << " i1=" << i1 << " i0=" << i0 << " result=" << result << " correct=" << correct << " avg_err=" << (avg_err / counter) << std::endl;
- std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl;
- if (src0 != nullptr) {
- std::cerr << "src0=" << src0 << " src0->name=" << src0->name << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl;
- }
- if (src1 != nullptr) {
- std::cerr << "src1=" << src1 << " src1->name=" << src1->name << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl;
- }
- if (src2 != nullptr) {
- std::cerr << "src2=" << src2 << " src2->name=" << src2->name << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl;
- }
- if (src3 != nullptr) {
- std::cerr << "src3=" << src3 << " src3->name=" << src3->name << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl;
- }
- std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl;
- std::cerr << std::endl << "Result:" << std::endl;
- ggml_vk_print_tensor_area(tensor, tensor_data, i0, i1, i2, i3);
- std::cerr << std::endl << "Correct:" << std::endl;
- ggml_vk_print_tensor_area(tensor, comp_result, i0, i1, i2, i3);
- std::cerr << std::endl;
- std::vector<const ggml_tensor *> done;
- ggml_vk_print_graph_origin(tensor, done);
- GGML_ABORT("fatal error");
- }
- const double denom = std::fabs(correct) > 1.0f ? (std::fabs(correct) > 1e-8 ? std::fabs(correct) : 1e-8) : 1.0f;
- if (first_error[0] == -1 && std::fabs(correct - result) / denom > 0.5) {
- first_error[0] = i0;
- first_error[1] = i1;
- first_error[2] = i2;
- first_error[3] = i3;
- first_error_result = result;
- first_error_correct = correct;
- }
-
- // Special case, value is infinite, avoid NaN result in avg_err
- // NaN also appears in results, if both are nan error is 0
- if (!std::isinf(correct) && !std::isinf(result) && !std::isnan(correct) && !std::isnan(result)) {
- avg_err += std::fabs(correct - result) / denom;
- }
- counter++;
- }
- }
- }
+ggml_vk_debug_label::ggml_vk_debug_label(vk_queue_handle * handle, const char * name) {
+ if (!vk_instance.debug_utils_support || handle == nullptr) {
+ return;
}
+ vk::DebugUtilsLabelEXT label = {};
+ label.pLabelName = name;
+ label.color = std::array<float, 4>{1.0f, 1.0f, 1.0f, 1.0f};
- avg_err /= counter;
+ qhandle = handle;
+ std::lock_guard<vk_queue_handle> guard(*qhandle);
+ vk_instance.pfn_vkQueueBeginDebugUtilsLabelEXT(qhandle->queue, reinterpret_cast<VkDebugUtilsLabelEXT *>(&label));
+}
- if (vk_output_tensor > 0 && vk_output_tensor == check_counter) {
- std::cerr << "TENSOR CHECK: avg_err=" << avg_err << " in " << ggml_op_name(tensor->op) << " (check " << check_counter << ")" << std::endl;
- std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl;
- if (src0 != nullptr) {
- std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl;
- }
- if (src1 != nullptr) {
- std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl;
- }
- if (src2 != nullptr) {
- std::cerr << "src2=" << src2 << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl;
+void ggml_vk_debug_label::close() {
+ if (subctx != nullptr) {
+ // close on the current command buffer, which may differ from the one begin used
+ if (subctx->s != nullptr) {
+ vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(subctx->s->buffer->buf);
}
- if (src3 != nullptr) {
- std::cerr << "src3=" << src3 << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl;
- }
- std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl;
- std::cerr << std::endl << "Result:" << std::endl;
- ggml_vk_print_tensor_area(tensor, tensor_data, 5, 5, 0, 0);
- std::cerr << std::endl << "Correct:" << std::endl;
- ggml_vk_print_tensor_area(tensor, comp_result, 5, 5, 0, 0);
- std::cerr << std::endl;
- std::vector<const ggml_tensor *> done;
- ggml_vk_print_graph_origin(tensor, done);
+ subctx->debug_labels.pop_back();
+ subctx = nullptr;
}
-
- if (avg_err > 0.01 || std::isnan(avg_err)) {
- std::cerr << "ERROR: avg_err=" << avg_err << " in " << ggml_op_name(tensor->op) << " (check " << check_counter << ")" << std::endl;
- std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl;
- if (src0 != nullptr) {
- std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl;
- }
- if (src1 != nullptr) {
- std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl;
- }
- if (src2 != nullptr) {
- std::cerr << "src2=" << src2 << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl;
- }
- if (src3 != nullptr) {
- std::cerr << "src3=" << src3 << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl;
- }
- std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl;
- std::cerr << std::endl << "Result:" << std::endl;
- ggml_vk_print_tensor_area(tensor, tensor_data, first_error[0], first_error[1], first_error[2], first_error[3]);
- std::cerr << std::endl << "Correct:" << std::endl;
- ggml_vk_print_tensor_area(tensor, comp_result, first_error[0], first_error[1], first_error[2], first_error[3]);
- std::cerr << std::endl;
- std::vector<const ggml_tensor *> done;
- ggml_vk_print_graph_origin(tensor, done);
- GGML_ABORT("fatal error");
- } else {
- std::cerr << check_counter << " " << tensor->name << " op=" << ggml_op_name(tensor->op) << " avg_err=" << avg_err << std::endl;
+ if (qhandle != nullptr) {
+ std::lock_guard<vk_queue_handle> guard(*qhandle);
+ vk_instance.pfn_vkQueueEndDebugUtilsLabelEXT(qhandle->queue);
+ qhandle = nullptr;
}
+}
- free(comp_result);
- comp_result = nullptr;
- comp_size = 0;
-
- if (ggml_backend_buffer_is_vk(tensor->buffer)) {
- free(tensor_data);
+void ggml_vk_debug_label::begin(vk_context & ctx, const std::string & name) {
+ if (!vk_instance.debug_utils_support || ctx->s == nullptr) {
+ return;
}
-
- VK_LOG_DEBUG("END ggml_vk_check_results_1(" << tensor->name << ")");
+ subctx = ctx.get();
+ subctx->debug_labels.push_back(name);
+ ggml_vk_cmd_label_begin(subctx->s->buffer->buf, subctx->debug_labels.back().c_str());
}
-#endif
-GGML_BACKEND_DL_IMPL(ggml_backend_vk_reg)