Commit dde9de82c for llama.cpp
commit dde9de82cbb9b1050ae75c95ab9a9f955340647f
Author: Neo Zhang <zhang.jianyu@outlook.com>
Date: Tue Nov 17 16:50:32 2026 +0800
sycl : Fix FA mkl issue: a mask broadcast over heads or batches (#31536)
* fix the SYCL fa with mkl error of a mask broadcast over heads or batches; add argument cache-ram to start-svr.sh
* add argument cache-ram to win-start-svr.bat
* fix format issue, remove --cache-ram in script: move to another PR
diff --git a/ggml/src/ggml-sycl/fattn-mkl.cpp b/ggml/src/ggml-sycl/fattn-mkl.cpp
index e3b3553de..09b184cbb 100644
--- a/ggml/src/ggml-sycl/fattn-mkl.cpp
+++ b/ggml/src/ggml-sycl/fattn-mkl.cpp
@@ -160,7 +160,7 @@ static void mkl_fa_online_softmax_chunk(
int64_t m_stride = 0;
if (mask_data) {
int m_head = (mask_n_heads > 1)
- ? (kvh_head + gqa_group) : 0;
+ ? (kvh_head + gqa_group) % mask_n_heads : 0;
mask_h = mask_data + (int64_t)m_head * mask_head_stride;
m_stride = mask_row_stride;
}
@@ -537,7 +537,7 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor *
const sycl::half * mask_batch = nullptr;
if (mask) {
- int m_batch = (mask->ne[3] > 1) ? ib : 0;
+ int m_batch = ib % (int)mask->ne[3];
mask_batch = (const sycl::half *)mask->data
+ m_batch * (mask->nb[3] / 2); // 2 = actual fp16 device size
}
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 5df6d14e8..a45d9bad5 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -9174,6 +9174,30 @@ struct test_flash_attn_ext_large_logits : public test_flash_attn_ext {
}
};
+// mask with 2 heads and 2 batches, broadcast over the Q heads and batches
+struct test_flash_attn_ext_mask_bcast : public test_flash_attn_ext {
+ using test_flash_attn_ext::test_flash_attn_ext;
+
+ std::string vars() override {
+ return test_flash_attn_ext::vars() + ",mask_ne23=[2,2]";
+ }
+
+ ggml_tensor * build_graph(ggml_context * ctx) override {
+ ggml_tensor * ref = test_flash_attn_ext::build_graph(ctx);
+ ggml_set_name(ref, "ref");
+
+ ggml_tensor * m = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, kv, nb, 2, 2);
+ ggml_set_name(m, "m");
+
+ ggml_tensor * out = ggml_flash_attn_ext(ctx, ref->src[0], ref->src[1], ref->src[2], m, 1.0f/sqrtf(hsk), max_bias, logit_softcap);
+ ggml_flash_attn_ext_add_sinks(out, ref->src[4]);
+ ggml_prec_set_acc(out, prec);
+ ggml_set_name(out, "out");
+
+ return out;
+ }
+};
+
// GGML_OP_FLASH_ATTN_EXT with kv_rows: each q slice attends to its own list of K/V cache rows
struct test_flash_attn_ext_kv_rows : public test_case {
const int64_t hs;
@@ -13182,6 +13206,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
}
+ // mask broadcast over Q heads and batches
+ for (bool sinks : { false, true }) {
+ for (ggml_type type : { GGML_TYPE_F16, GGML_TYPE_Q8_0 }) {
+ test_cases.emplace_back(new test_flash_attn_ext_mask_bcast(128, 128, 4, {8, 4}, 1024, 32, true, sinks, 0, 0, GGML_PREC_F32, type, type));
+ }
+ }
+
test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 512, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 4096, 16, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));