Commit d261e49e7 for llama.cpp
commit d261e49e7ee461d3c54559d8dbae44d859f71b49
Author: Hongqiang Wang <wangh@qti.qualcomm.com>
Date: Mon Nov 16 10:57:48 2026 -0800
opencl: fix three defects of the bin flash-attention prefill path (#32110)
* opencl: release the WMM flash-attention mask fallback image
The fallback image is created on every call of the bin flash-attention
kernel over a sub-buffer of the K cache and was never released, so each
call kept a K-cache sub-buffer alive. Every call with more than one query
row takes this path, including each speculative-decoding verify step, so
the objects pile up and the server slows with every request.
* opencl: keep q8_0 K/V at head size 512 off the WMM flash-attention kernel
With q8_0 K/V at DK=512 the bin flash-attention kernel gives wrong results
whenever V holds different values from K (test-backend-ops FLASH_ATTN_EXT
hsk=512 q8_0, max error 2.04 on X2-90); f16 K/V at DK=512 is correct. The
backend's own flash-attention kernels already decline q8_0 at DK=512, so the
shape is now handled the same way with or without the kernel library.
* opencl: decline WMM flash-attention prefill beyond the device's 3D image limits
The WMM path repacks Q/K/V into 3D images; V is n_kv wide, so past the device's
maximum 3D image width (about 16k tokens of context on X2-90) clCreateImage
failed and the call asserted. Such shapes now take the regular flash-attention
path.
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 295ec496d..c1dd3f75e 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -712,6 +712,9 @@ struct ggml_backend_opencl_context {
size_t image_max_buffer_size;
size_t image2d_max_width;
size_t image2d_max_height;
+ size_t image3d_max_width;
+ size_t image3d_max_height;
+ size_t image3d_max_depth;
cl_device_svm_capabilities svm_caps;
@@ -9258,6 +9261,9 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE_MAX_BUFFER_SIZE, sizeof(size_t), &backend_ctx->image_max_buffer_size, NULL));
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_WIDTH, sizeof(size_t), &backend_ctx->image2d_max_width, NULL));
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_HEIGHT, sizeof(size_t), &backend_ctx->image2d_max_height, NULL));
+ CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE3D_MAX_WIDTH, sizeof(size_t), &backend_ctx->image3d_max_width, NULL));
+ CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE3D_MAX_HEIGHT, sizeof(size_t), &backend_ctx->image3d_max_height, NULL));
+ CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE3D_MAX_DEPTH, sizeof(size_t), &backend_ctx->image3d_max_depth, NULL));
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(size_t), &backend_ctx->max_workgroup_size, NULL));
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_LOCAL_MEM_SIZE, sizeof(cl_ulong), &backend_ctx->local_mem_size, NULL));
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_SVM_CAPABILITIES, sizeof(cl_device_svm_capabilities), &backend_ctx->svm_caps, 0));
@@ -11660,6 +11666,26 @@ static bool use_fa_bin_kernels_prefill(const ggml_backend_opencl_context * backe
constexpr bool prefill_only = true;
+ // The repacked Q/K/V are 3D images: Q {n_q, n_batch*n_head, dk/4}, K {dk, n_kv/4, n_batch*n_head_kv},
+ // V {n_kv, dv/4, n_batch*n_head_kv}. Past the device's 3D image limits clCreateImage fails and the
+ // call asserts (X2-90: V is n_kv wide, so any prefill past ~16k tokens of context), so decline those shapes.
+ const size_t n_kv = (size_t) k->ne[1];
+ const size_t n_bh = (size_t) q->ne[3] * (size_t) q->ne[2];
+ const size_t n_bhkv = (size_t) q->ne[3] * (size_t) k->ne[2];
+ if ((size_t) n_q > backend_ctx->image3d_max_width || n_bh > backend_ctx->image3d_max_height ||
+ (size_t) dk / 4 > backend_ctx->image3d_max_depth ||
+ (size_t) dk > backend_ctx->image3d_max_width || (n_kv + 3) / 4 > backend_ctx->image3d_max_height ||
+ n_kv > backend_ctx->image3d_max_width || (size_t) dv / 4 > backend_ctx->image3d_max_height ||
+ n_bhkv > backend_ctx->image3d_max_depth) {
+ return false;
+ }
+
+ // q8_0 K/V at DK=512 gives wrong results here when V differs from K (f16 K/V is correct).
+ // Decline it, as this backend does without the kernel library.
+ if (is_q8_0 && dk == 512) {
+ return false;
+ }
+
return (backend_ctx->gpu_family == GPU_FAMILY::ADRENO &&
(is_mixed || is_q8_0) && (dk == dv)
&& (dk == 64 || dk == 128 || dk == 256 || dk == 512)
@@ -22794,6 +22820,7 @@ static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_te
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
CL_CHECK(clReleaseMemObject(mem_tex_matrixO_1dbuf));
+ CL_CHECK(clReleaseMemObject(mem_tex_mask_fallback_1dbuf));
CL_CHECK(clReleaseMemObject(img_q_wmm));
CL_CHECK(clReleaseMemObject(img_k_wmm));
CL_CHECK(clReleaseMemObject(img_v_wmm));