Commit a8d9a7e1e for llama.cpp
commit a8d9a7e1e55f5b613f297504d9e159807ec3a99b
Author: Ruben Ortlam <rortlam@redhat.com>
Date: Fri Oct 16 08:29:23 2026 +0200
vulkan: fix gpu-visible host memory issues (#30468)
* vulkan: fix log spam if host pointer import fails
* exclude MoltenVK from shared memory import
* remove unreachable code, assert instead. Shorten warnings
* disable host memory buffer sharing on AMD due to amdgpu perf issue
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp
index eabd8974a..b687b1390 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp
@@ -1,6 +1,5 @@
#include "ggml-vulkan-common.h"
-#include <atomic>
#include <cstdint>
#include <cstdlib>
#include <mutex>
@@ -363,21 +362,16 @@ void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf,
vk_host_allocation & entry = vk_instance.host_allocations[index];
if (entry.views[device->idx] == nullptr) {
- if (!vk_instance.multi_device) {
- return;
- }
- if (!device->external_memory_host) {
- static std::atomic<uint32_t> warned_mask{0};
- const uint32_t bit = 1u << device->idx;
- if (!(warned_mask.fetch_or(bit) & bit)) {
- GGML_LOG_WARN("ggml_vulkan: %s does not support VK_EXT_external_memory_host; host memory "
- "will be staged, not pinned, on this device\n", device->name.c_str());
- }
+ const uint32_t bit = 1u << device->idx;
+ if (!vk_instance.multi_device || !vk_instance.device_host_buffer_import[device->idx] || (vk_instance.host_import_failed & bit)) {
return;
}
+ GGML_ASSERT(device->external_memory_host);
// const only propagates from the const context; the pointee is mutable
entry.views[device->idx] = ggml_vk_buffer_from_host_ptr(const_cast<vk_device&>(device), entry.base, entry.size);
if (entry.views[device->idx] == nullptr) {
+ vk_instance.host_import_failed |= bit;
+ GGML_LOG_WARN("ggml_vulkan: %s: host memory import failed, using staging\n", device->name.c_str());
return;
}
}
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
index a41ce5966..00d20e843 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
@@ -1329,12 +1329,14 @@ struct vk_instance_t {
std::vector<size_t> device_indices;
std::vector<bool> device_supports_membudget;
+ std::vector<bool> device_host_buffer_import;
vk_device devices[GGML_VK_MAX_DEVICES];
std::vector<vk_host_allocation> host_allocations;
std::shared_mutex host_allocations_mutex;
size_t host_import_alignment = 0;
+ uint32_t host_import_failed = 0; // bitmask by device->idx, guarded by host_allocations_mutex
bool multi_device = false;
};
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index 436bfe3bb..fa038bb5b 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -3987,6 +3987,17 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
}
}
+static bool ggml_vk_external_memory_host_usable(vk::DriverId driver_id) {
+ // Disable external_memory_host until https://github.com/KhronosGroup/MoltenVK/pull/2622
+ // is available in the Vulkan SDK.
+ return driver_id != vk::DriverId::eMoltenvk;
+}
+
+static bool ggml_vk_host_buffer_import_usable(uint32_t vendor_id) {
+ // amdgpu re-faults all pages of every imported host allocation on each submission
+ return vendor_id != VK_VENDOR_ID_AMD;
+}
+
vk_device ggml_vk_get_device(size_t idx) {
VK_LOG_DEBUG("ggml_vk_get_device(" << idx << ")");
@@ -4174,11 +4185,7 @@ vk_device ggml_vk_get_device(size_t idx) {
device->vendor_id = device->properties.vendorID;
device->driver_id = driver_props.driverID;
- if (device->driver_id == vk::DriverId::eMoltenvk) {
- // Disable external_memory_host until https://github.com/KhronosGroup/MoltenVK/pull/2622
- // is available in the Vulkan SDK.
- device->external_memory_host = false;
- }
+ device->external_memory_host = device->external_memory_host && ggml_vk_external_memory_host_usable(device->driver_id);
// Implementing the async backend interfaces seems broken on older Intel HW,
// see https://github.com/ggml-org/llama.cpp/issues/17302.
@@ -5421,7 +5428,7 @@ void ggml_vk_instance_init() {
}
GGML_LOG_DEBUG("ggml_vulkan: Found %zu Vulkan devices:\n", vk_instance.device_indices.size());
- bool all_external_memory_host = vk_instance.device_indices.size() > 1;
+ size_t host_buffer_import_count = 0;
for (size_t i = 0; i < vk_instance.device_indices.size(); i++) {
vk::PhysicalDevice vkdev = devices[vk_instance.device_indices[i]];
std::vector<vk::ExtensionProperties> extensionprops = vkdev.enumerateDeviceExtensionProperties();
@@ -5438,20 +5445,27 @@ void ggml_vk_instance_init() {
vk_instance.device_supports_membudget.push_back(membudget_supported);
+ bool host_buffer_import = false;
if (external_memory_host_supported) {
vk::PhysicalDeviceProperties2 props2;
+ vk::PhysicalDeviceDriverProperties driver_props;
vk::PhysicalDeviceExternalMemoryHostPropertiesEXT ext_props;
- props2.pNext = &ext_props;
+ props2.pNext = &driver_props;
+ driver_props.pNext = &ext_props;
vkdev.getProperties2(&props2);
- vk_instance.host_import_alignment = std::max<size_t>(vk_instance.host_import_alignment, ext_props.minImportedHostPointerAlignment);
- } else {
- all_external_memory_host = false;
+ host_buffer_import = ggml_vk_external_memory_host_usable(driver_props.driverID) &&
+ ggml_vk_host_buffer_import_usable(props2.properties.vendorID);
+ if (host_buffer_import) {
+ vk_instance.host_import_alignment = std::max<size_t>(vk_instance.host_import_alignment, ext_props.minImportedHostPointerAlignment);
+ host_buffer_import_count++;
+ }
}
+ vk_instance.device_host_buffer_import.push_back(host_buffer_import);
ggml_vk_print_gpu_info(i);
}
- vk_instance.multi_device = all_external_memory_host;
+ vk_instance.multi_device = host_buffer_import_count > 1 && vk_instance.device_host_buffer_import[0];
}
void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) {