Commit d4f5c2604 for llama.cpp
commit d4f5c26049774bb786276eb517f1e6b070d28ff4
Author: Aman Gupta <amangupta052@gmail.com>
Date: Thu Nov 19 14:11:44 2026 +0800
CUDA: use 64-bit dst offsets in mul_mat_q (#32374)
The MMQ kernels computed the dst tile offset (jt*J*stride_col_dst) in
32-bit int. For a dst with more than 2^31 elements, column tiles that
start past INT_MAX/nrows wrap around and write out of bounds.
This happens with an all-logits output head, e.g. llama-perplexity
-c 32768 -b 32768 -ub 16384 on a 248320-vocab model: the q8_0 output
matmul has a 248320 x 16384 dst (4.07e9 elements) and aborts with an
illegal memory access.
Widen the offsets in mul_mat_q and the stream-k fixup to int64_t, and
add a (disabled by default, ~9 GB) test-backend-ops case.
Assisted-by: Claude Opus 5.5
diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh
index 7fb09a353..be2f11430 100644
--- a/ggml/src/ggml-cuda/mmq.cuh
+++ b/ggml/src/ggml-cuda/mmq.cuh
@@ -1094,7 +1094,7 @@ static __global__ void mul_mat_q(
int col_high = min(col_low + J, ncols_dst);
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
- int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
+ int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
if (ids_dst) {
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
@@ -1176,7 +1176,7 @@ static __global__ void mul_mat_q(
int col_high = min(col_low + J, ncols_dst);
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
- int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
+ int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
if (ids_dst) {
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
@@ -1257,7 +1257,7 @@ static __global__ void mul_mat_q(
int col_high = min(col_low + J, ncols_dst);
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
- int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
+ int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
if (ids_dst) {
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
@@ -1403,7 +1403,7 @@ static __global__ void mul_mat_q_stream_k_fixup(
const int it = tmp2.x;
if (!ids_dst) {
- const int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst + it*I;
+ const int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst + it*I;
dst += offset_dst;
const int i_max = nrows_x - it*I - 1;
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index b20e0d3fe..b001c79d1 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -11917,6 +11917,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
// > 2^31-element dst (8.9 GB), about 45 GB peak memory. Too large to be enabled by default.
// kernel_mul_mm dst offset of the last batch is past INT32_MAX: rows 0-31 use full tiles, rows 32-39 use edge tiles.
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 1048576, 40, 64, {1, 1}, {53, 1}));
+ // CUDA mul_mat_q: > 2^31-element dst (8.9 GB), the dst offset of column tiles starting past column 8648 does not fit in 32 bits
+ // (same shape class as a 248k-vocab output head with a 16k ubatch of outputs)
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 248320, 8960, 32, {1, 1}, {1, 1}));
// kernel_mul_mm_id: the index of tokens 32768-33791 does not fit in 16 bits (about 0.3 GB peak memory).
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 4, 2, false, 64, 33792, 64));