Commit 818ac6b07 for llama.cpp
commit 818ac6b078e345e59ffa4b1ed3b1ce1b0ae83827
Author: KyleHagy <59183061+KyleHagy@users.noreply.github.com>
Date: Sat Nov 7 15:13:04 2026 -0800
sycl: refuse device buffers that would exceed VRAM (#31285)
diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md
index fb800025d..7d86d8bd2 100644
--- a/docs/backend/SYCL.md
+++ b/docs/backend/SYCL.md
@@ -817,7 +817,7 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
| GGML_SYCL_ENABLE_VMM | 0 or 1 (default) | Enable the virtual-memory device pool. |
| GGML_SYCL_ENABLE_MKL_FA | 1 (default) or 0 | Enable oneMKL GEMM flash attention for XMX-accelerated prompt processing with quantized KV cache. Automatically activates during prefill (prompt processing) when all conditions are met: (1) flash-attn enabled (`-fa` or `--flash-attn on`), (2) KV cache quantized (`--cache-type-k q8_0 --cache-type-v q8_0` or other `*_0/*_1` types), (3) batch size ≥ 1024 (`--batch-size 1024`), (4) prompt length ≥ 1024 tokens. Set to 0 to force the TILE kernel for A/B testing. Example minimum command: `llama-cli -m model.gguf -fa -ngl 99 --cache-type-k q8_0 --cache-type-v q8_0 --batch-size 1024 -p "your prompt"` |
| GGML_SYCL_MKL_FA_DEBUG | 0 (default) or 1 | Enable per-call diagnostic logging for MKL flash attention: GEMM/softmax timings, interleaved-head detection, and buffer memory usage. |
-| GGML_SYCL_MEMTRACE | 0 (default), 1, 2 | Enable record and output memory allocation diagnostics. Requires `-lv 4`. <br>0 - Disable<br>1 - Basic memory info, including current and peak allocations, as well allocations from other sources, around 50 lines per model load.<br>2 - More verbose, logging around 900 specific allocations and deallocations. |
+| GGML_SYCL_MEMTRACE | 0 (default), 1, 2 | Enable record and output memory allocation diagnostics. Requires `-lv 4`. <br>0 - Disable<br>1 - Basic memory info, including current and peak allocations, as well allocations from other sources, and the device free memory before each model buffer allocation, around 50 lines per model load.<br>2 - More verbose, logging around 900 specific allocations and deallocations. |
| GGML_SYCL_MEMTRACE_STEP | 64 (default) or positive integer | With GGML_SYCL_MEMTRACE=1, the minimum growth in memory usage to trigger another log record. |
| GGML_SYCL_MKL_FA_DIAG | 0 (default) or 1 | Enable output fingerprinting for MKL flash attention. Dumps the first 64 float output values for the first 6 FA calls with n_kv ≥ 1024, labeled with kernel type (MKL/TILE/VEC) for cross-kernel comparison. |
| GGML_SYCL_ENABLE_FUSION | 0 or 1 (default) | Enable fused-kernel dispatch in graph compute. Unsupported types and layouts fall back to the standalone op kernels. See `ggml_sycl_can_fuse()`. |
@@ -836,6 +836,7 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
| ZES_ENABLE_SYSMAN | 0 (default) or 1 | Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.<br>Recommended to use when --split-mode = layer |
| UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS | 0 (default) or 1 | Allow SYCL/Unified Runtime Level Zero device allocations larger than 4 GiB. llama.cpp's direct Level Zero allocation path requests the relaxed maximum-size limit itself when GGML_SYCL_ENABLE_LEVEL_ZERO=1. |
| GGML_SYCL_USM_SYSTEM | 0 (default) or 1 | Enable experimental support for [USM system allocations](https://github.khronos.org/SYCL_Reference/iface/usm_basic_concept.html#system-allocations) for large GPU buffers. This requires enough host memory for model weights and caches, an Intel Xe2+ GPU such as BMG or newer and supported on Linux only, with CONFIG_DRM_XE_GPUSVM enabled. |
+| GGML_SYCL_BMG_VRAM_CHECK | 1 (default) or 0 | Workaround for BMG (Arc B-series) GPUs, where the Level Zero driver accepts device allocations beyond VRAM and moves the overflow to system RAM, which can exhaust host memory when a model does not fit (e.g. `-ngl 999`, see https://github.com/intel/compute-runtime/issues/1047). When enabled, a model or split buffer that does not fit in the device's free memory is refused and the model load fails with an error. Set to 0 to disable. |
1. <a id="sycl-graph-footnote"></a>SYCL Graph support (`GGML_SYCL_ENABLE_GRAPH=1`) is limited to single-GPU workloads and may also depend on the model architecture and kernels in use.
Performance depends on how often a recorded graph can be reused, which will vary by workload.
diff --git a/ggml/src/ggml-sycl/common.cpp b/ggml/src/ggml-sycl/common.cpp
index f01d0ff95..9ddabdf33 100644
--- a/ggml/src/ggml-sycl/common.cpp
+++ b/ggml/src/ggml-sycl/common.cpp
@@ -11,6 +11,8 @@
//
#include "common.hpp"
+#include <mutex>
+#include <unordered_map>
#include <sycl/backend.hpp>
#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API
#include <level_zero/ze_api.h>
@@ -92,9 +94,16 @@ static bool ggml_sycl_use_level_zero_device_alloc(sycl::queue &q) {
}
#endif
+// Device bytes allocated through ggml_sycl_malloc_device, per device and per pointer
+static std::mutex g_ggml_sycl_dev_alloc_mutex;
+static std::unordered_map<void *, std::pair<int, size_t>> g_ggml_sycl_dev_allocs;
+static size_t g_ggml_sycl_dev_alloc_bytes[GGML_SYCL_MAX_DEVICES] = {};
+// free memory reported when we had nothing allocated (0 = not taken yet)
+static size_t g_ggml_sycl_dev_free_baseline[GGML_SYCL_MAX_DEVICES] = {};
+
// Use Level Zero zeMemAllocDevice to avoid sycl::malloc_device triggering
// DMA-buf/TTM system RAM staging in the xe kernel driver during multi-GPU inference.
-void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type type) {
+static void * ggml_sycl_malloc_device_impl(size_t size, sycl::queue &q, ggml_sycl_mem_type type) {
#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API
if (ggml_sycl_use_level_zero_device_alloc(q)) {
void *ptr = nullptr;
@@ -134,8 +143,78 @@ void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type t
return ptr;
}
+// caller holds g_ggml_sycl_dev_alloc_mutex
+static bool ggml_sycl_free_mem_enough(int device, size_t size, ggml_sycl_mem_type type) {
+ const size_t allocated = g_ggml_sycl_dev_alloc_bytes[device];
+
+ // TODO: remove once zeMemAllocDevice refuses allocations beyond VRAM on BMG
+ // (https://github.com/intel/compute-runtime/issues/1047). Only model/split buffers are checked.
+ const gpu_arch arch = ggml_sycl_info().devices[device].hw_info.arch;
+ const bool is_bmg = arch == gpu_arch::intel_gpu_bmg_g21 || arch == gpu_arch::intel_gpu_bmg_g31;
+ const bool check = g_ggml_sycl_bmg_vram_check && is_bmg && type == GGML_SYCL_MEM_BUFFER;
+ const bool trace = type == GGML_SYCL_MEM_BUFFER && ggml_sycl_memtrace_enabled();
+ size_t free_mem = 0, total_mem = 0;
+ if (check || trace) {
+ sycl_get_mem_info(device, &free_mem, &total_mem);
+ }
+ if (trace) {
+ ggml_sycl_memtrace_report_device("before buffer alloc", device, free_mem, total_mem);
+ }
+ if (check) {
+ size_t & baseline = g_ggml_sycl_dev_free_baseline[device];
+ if (allocated == 0) {
+ baseline = free_mem;
+ }
+ // the drop in free memory since the baseline is ours once written, the rest of ours is not counted yet
+ const size_t written = baseline > free_mem ? baseline - free_mem : 0;
+ const size_t unwritten = baseline && allocated > written ? allocated - written : 0;
+ const size_t usable = free_mem > unwritten ? free_mem - unwritten : 0;
+ if (allocated + size > total_mem || size > usable) {
+ GGML_LOG_ERROR("%s: SYCL%d: cannot allocate %zu MiB: %zu MiB already allocated, %zu MiB usable "
+ "(%zu MiB free of %zu MiB), set GGML_SYCL_BMG_VRAM_CHECK=0 to disable this check\n",
+ __func__, device, size >> 20, allocated >> 20, usable >> 20, free_mem >> 20, total_mem >> 20);
+ return false;
+ }
+ }
+ return true;
+}
+
+static void ggml_sycl_dev_alloc_record(void * ptr, int device, size_t size) {
+ g_ggml_sycl_dev_allocs[ptr] = { device, size };
+ g_ggml_sycl_dev_alloc_bytes[device] += size;
+}
+
+void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type type) {
+ const int device = (int) dpct::dev_mgr::instance().get_device_id(q.get_device());
+ GGML_ASSERT(device >= 0 && device < GGML_SYCL_MAX_DEVICES);
+
+ std::lock_guard<std::mutex> lock(g_ggml_sycl_dev_alloc_mutex);
+ if (!ggml_sycl_free_mem_enough(device, size, type)) {
+ ggml_sycl_memtrace_fail(type, size);
+ return nullptr;
+ }
+
+ void * ptr = ggml_sycl_malloc_device_impl(size, q, type);
+ if (ptr) {
+ ggml_sycl_dev_alloc_record(ptr, device, size);
+ }
+ return ptr;
+}
+
void ggml_sycl_free_device(void *ptr, sycl::queue &q) {
if (!ptr) return;
+ {
+ std::lock_guard<std::mutex> lock(g_ggml_sycl_dev_alloc_mutex);
+ auto it = g_ggml_sycl_dev_allocs.find(ptr);
+ if (it != g_ggml_sycl_dev_allocs.end()) {
+ const int device = it->second.first;
+ g_ggml_sycl_dev_alloc_bytes[device] -= it->second.second;
+ if (g_ggml_sycl_dev_alloc_bytes[device] == 0) {
+ g_ggml_sycl_dev_free_baseline[device] = 0;
+ }
+ g_ggml_sycl_dev_allocs.erase(it);
+ }
+ }
ggml_sycl_memtrace_del(ptr);
#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API
if (ggml_sycl_use_level_zero_device_alloc(q)) {
diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp
index 668e9befb..22ad7fa20 100644
--- a/ggml/src/ggml-sycl/common.hpp
+++ b/ggml/src/ggml-sycl/common.hpp
@@ -428,6 +428,8 @@ struct ggml_tensor_extra_gpu {
};
extern int g_ggml_sycl_use_level_zero_api;
+extern int g_ggml_sycl_bmg_vram_check;
+bool sycl_get_mem_info(int device, size_t * free, size_t * total);
void * ggml_sycl_malloc_device(size_t size, sycl::queue &q,
ggml_sycl_mem_type type = GGML_SYCL_MEM_DIRECT);
void ggml_sycl_free_device(void *ptr, sycl::queue &q);
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index ac752f9b8..9ddca829b 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -146,6 +146,7 @@ int g_ggml_sycl_use_level_zero_api = 0;
int g_ggml_sycl_enable_flash_attention = 1;
int g_ggml_sycl_dev2dev_memcpy = DEV2DEV_MEMCPY_SYCL;
int g_ggml_sycl_usm_system = 0;
+int g_ggml_sycl_bmg_vram_check = 1;
int g_ggml_sycl_enable_host_pinned_mem = 1;
int g_ggml_sycl_host_pinned_mem_2g = 0;
int g_ggml_sycl_upload_staging_slots = 4;
@@ -475,6 +476,7 @@ static void ggml_check_sycl() try {
#endif
g_ggml_sycl_usm_system = ggml_sycl_get_env("GGML_SYCL_USM_SYSTEM", 0);
+ g_ggml_sycl_bmg_vram_check = ggml_sycl_get_env("GGML_SYCL_BMG_VRAM_CHECK", 1);
g_ggml_sycl_enable_host_pinned_mem =
ggml_sycl_get_env("GGML_SYCL_ENABLE_HOST_PINNED_MEM", 1);
@@ -612,6 +614,7 @@ static void ggml_check_sycl() try {
#endif
GGML_LOG_INFO(" GGML_SYCL_USM_SYSTEM: %d\n", g_ggml_sycl_usm_system);
+ GGML_LOG_INFO(" GGML_SYCL_BMG_VRAM_CHECK: %d\n", g_ggml_sycl_bmg_vram_check);
GGML_LOG_INFO(" GGML_SYCL_ENABLE_HOST_PINNED_MEM: %d\n", g_ggml_sycl_enable_host_pinned_mem);
GGML_LOG_INFO(" GGML_SYCL_HOST_PINNED_MEM_2G: %d\n", g_ggml_sycl_host_pinned_mem_2g);
@@ -6478,7 +6481,7 @@ catch (sycl::exception const &exc) {
std::exit(1);
}
-static bool sycl_get_mem_info(int device, size_t * free, size_t * total) {
+bool sycl_get_mem_info(int device, size_t * free, size_t * total) {
GGML_SYCL_DEBUG("[SYCL] [%s] g_ggml_sycl_get_mem_api=%d\n",
__func__, g_ggml_sycl_get_mem_api);