Commit 09338b85e for llama.cpp
commit 09338b85e8b230742fa56f93a1d9f0bf006ca2d3
Author: Titaniumtown <titaniumtown@proton.me>
Date: Wed Nov 11 22:58:26 2026 -0800
sycl: make oneDNN FA cache thread-safe and memoize runtime failures (#31138)
diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp
index f46e79784..f19b0631d 100644
--- a/ggml/src/ggml-sycl/common.hpp
+++ b/ggml/src/ggml-sycl/common.hpp
@@ -47,6 +47,7 @@ namespace syclexp = sycl::ext::oneapi::experimental;
#if GGML_SYCL_DNNL
#include "dnnl.hpp"
#include "dnnl_sycl.hpp"
+#include "oneapi/dnnl/dnnl_graph.hpp"
#endif
#define GGML_COMMON_DECL_SYCL
@@ -484,6 +485,16 @@ struct ggml_sycl_graph {
static_assert(std::is_trivial<ggml_sycl_graph::node_properties>::value, "node_properties must be trivial");
#endif
+#if GGML_SYCL_DNNL
+struct sdpa_partition {
+ dnnl::graph::compiled_partition cp;
+ std::vector<dnnl::graph::logical_tensor> ins;
+ dnnl::graph::logical_tensor out;
+ size_t id_q = 0, id_k = 0, id_v = 0, id_scale = 0, id_mask = 0;
+ bool ok = false;
+};
+#endif
+
struct ggml_backend_sycl_context {
int device;
std::string name;
@@ -552,6 +563,8 @@ struct ggml_backend_sycl_context {
dnnl::stream stream_dnnl() {
return stream_dnnl(device, 0);
}
+
+ std::unordered_map<std::string, sdpa_partition> sdpa_partitions;
#endif
// pool
diff --git a/ggml/src/ggml-sycl/fattn-onednn.cpp b/ggml/src/ggml-sycl/fattn-onednn.cpp
index bd7ce8936..66ff32f15 100644
--- a/ggml/src/ggml-sycl/fattn-onednn.cpp
+++ b/ggml/src/ggml-sycl/fattn-onednn.cpp
@@ -1,4 +1,5 @@
#include <array>
+#include <atomic>
#include <cstdint>
#include <cstdio>
#include <cstring>
@@ -25,6 +26,20 @@ bool ggml_sycl_fattn_onednn_binds_kv(const ggml_tensor * K, const ggml_tensor *
return bindable(K) && bindable(V);
}
+#if GGML_SYCL_DNNL
+// per device: set once a oneDNN SDPA execute throws, so dispatch stops picking oneDNN there
+static std::atomic<bool> g_ggml_sycl_fa_onednn_failed[GGML_SYCL_MAX_DEVICES] = {};
+#endif
+
+bool ggml_sycl_flash_attn_ext_onednn_failed(int device) {
+#if GGML_SYCL_DNNL
+ return g_ggml_sycl_fa_onednn_failed[device].load(std::memory_order_relaxed);
+#else
+ GGML_UNUSED(device);
+ return false;
+#endif
+}
+
bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use_shape_limit) {
#if !GGML_SYCL_DNNL
GGML_UNUSED(dst);
@@ -146,14 +161,6 @@ static void permute_sdpa_out_sycl(const sycl::half * out, float * dst,
});
}
-struct sdpa_partition {
- compiled_partition cp;
- std::vector<logical_tensor> ins;
- logical_tensor out;
- size_t id_q = 0, id_k = 0, id_v = 0, id_scale = 0, id_mask = 0;
- bool ok = false;
-};
-
// Build + compile the contiguous-input GQA SDPA graph (MatMul->Divide->Add->SoftMax->MatMul), f32 out.
// Mirrors the hardware-verified scratch/onednn_sdpa_probe.cpp build_gqa (partitions=1, sdp_primitive_kernel_t).
static sdpa_partition build_sdpa(const engine & eng, int H, int Hkv, int q, int seq, int d,
@@ -361,17 +368,16 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso
outf_ptr = outf_pool->get();
}
- // compile once per (device, shape, KV strides), reuse across layers/calls. Stride 2 always
- // repeats stride 1 and stride 4 is always 1, so the key covers every entry that can differ.
- static std::unordered_map<std::string, sdpa_partition> cache;
+ // compile once per (shape, KV strides) in this device's context, reuse across layers/calls. Stride 2
+ // always repeats stride 1 and stride 4 is always 1, so the key covers every entry that can differ.
char keyb[256];
- snprintf(keyb, sizeof(keyb), "%d:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld", ggml_sycl_get_device(),
+ snprintf(keyb, sizeof(keyb), "%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld",
(long long) H, (long long) Hkv, (long long) q, (long long) seq, (long long) d,
(long long) k_str[0], (long long) k_str[1], (long long) k_str[3],
(long long) v_str[0], (long long) v_str[1], (long long) v_str[3]);
- auto it = cache.find(keyb);
- if (it == cache.end()) {
- it = cache.emplace(keyb, build_sdpa(eng, (int) H, (int) Hkv, (int) q, (int) seq, (int) d, k_str, v_str)).first;
+ auto it = ctx.sdpa_partitions.find(keyb);
+ if (it == ctx.sdpa_partitions.end()) {
+ it = ctx.sdpa_partitions.emplace(keyb, build_sdpa(eng, (int) H, (int) Hkv, (int) q, (int) seq, (int) d, k_str, v_str)).first;
}
sdpa_partition & E = it->second;
if (!E.ok) {
@@ -407,8 +413,10 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso
}
}
catch (const std::exception & e) {
- // any oneDNN/SYCL failure is non-fatal: fall back to the existing kernel (strictly additive).
- GGML_LOG_WARN("%s: oneDNN SDPA failed (%s); falling back to TILE kernel\n", __func__, e.what());
+ // any oneDNN/SYCL failure is non-fatal: warn once, stop picking oneDNN on this device, run TILE instead
+ if (!g_ggml_sycl_fa_onednn_failed[ctx.device].exchange(true)) {
+ GGML_LOG_WARN("%s: oneDNN SDPA failed (%s); falling back to TILE kernel\n", __func__, e.what());
+ }
ggml_sycl_flash_attn_ext_tile(ctx, dst);
}
diff --git a/ggml/src/ggml-sycl/fattn-onednn.hpp b/ggml/src/ggml-sycl/fattn-onednn.hpp
index 9669d1bd2..4756b240a 100644
--- a/ggml/src/ggml-sycl/fattn-onednn.hpp
+++ b/ggml/src/ggml-sycl/fattn-onednn.hpp
@@ -7,6 +7,10 @@
// (f16 KV, no softcap/ALiBi, single stream, tuned head_dim, prefill-sized q.)
bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use_shape_limit = true);
+// True once a oneDNN SDPA execute has failed on this device. Not part of _supported(): that also sizes the
+// FA scratch buffer, and a size that depends on runtime state can come out smaller than a later call needs.
+bool ggml_sycl_flash_attn_ext_onednn_failed(int device);
+
// True when the oneDNN path binds an F16 KV cache in place instead of staging a dense copy of
// it. Depends only on the types and strides of K and V, so the answer holds for every call.
bool ggml_sycl_fattn_onednn_binds_kv(const ggml_tensor * K, const ggml_tensor * V);
diff --git a/ggml/src/ggml-sycl/fattn.cpp b/ggml/src/ggml-sycl/fattn.cpp
index 6f5f8a832..525e6fd8c 100644
--- a/ggml/src/ggml-sycl/fattn.cpp
+++ b/ggml/src/ggml-sycl/fattn.cpp
@@ -134,7 +134,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const
// XMX-accelerated path: oneDNN SDPA (native F16 and dequant+non-F16).
// ONEDNN requires min 32 query tokens — short-circuit decode to avoid
// calling _supported() on every decode FA call.
- if (Q->ne[1] >= 32
+ if (Q->ne[1] >= 32 && !ggml_sycl_flash_attn_ext_onednn_failed(device)
&& ggml_sycl_flash_attn_ext_onednn_supported(dst)) {
return BEST_FATTN_KERNEL_ONEDNN;
}
@@ -259,7 +259,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const
// Fused-XMX path: oneDNN Graph SDPA (flash attention). Strictly
// additive -- taken only when statically supported, otherwise falls through to VEC/TILE below.
- if (ggml_sycl_flash_attn_ext_onednn_supported(dst)) {
+ if (!ggml_sycl_flash_attn_ext_onednn_failed(device) && ggml_sycl_flash_attn_ext_onednn_supported(dst)) {
return BEST_FATTN_KERNEL_ONEDNN;
}