Commit f477c1b90 for llama.cpp
commit f477c1b90adb1f27e80276c0ebeed4d04812e605
Author: liminfei-amd <91481003+liminfei-amd@users.noreply.github.com>
Date: Mon Nov 16 17:53:06 2026 +0800
cuda : fix MUL_MAT_ID abort with odd src0 columns on AMD (#32385)
On AMD devices MUL_MAT_ID with an F32, F16 or BF16 src0 and at most 8 tokens always used MMVF.
MMVF reads src0 in pairs of values and asserts an even ne[0], so a rank 1 LoRA on MoE expert weights (lora_b has ne[0] == 1) aborted.
Use MMVF there only for an even ne[0]; other shapes take the existing fallback.
Fixes #30576
Assisted-by: GitHub Copilot
Signed-off-by: AMD ROCm Contributor <91481003+liminfei-amd@users.noreply.github.com>
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 4950b9ea7..cdd889caa 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -2526,7 +2526,7 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int c
} else if (src0->type == GGML_TYPE_F8_E4M3 &&
ggml_cuda_should_use_mmvf(src0, cc, warp_size, dst->ne[2])) {
return false;
- } else if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc)) {
+ } else if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc) && src0->ne[0] % 2 == 0) {
return false;
}
}
@@ -2570,7 +2570,8 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst);
return;
} else {
- if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc)) {
+ // MMVF reads src0 in pairs of values, so it needs an even number of columns
+ if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc) && ne00 % 2 == 0) {
ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst);
return;
}
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 7f1f1ff5e..4db0d6843 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -12279,6 +12279,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
for (int k : {1, 63, 65}) {
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 1, 1, false, 8, 16, k));
}
+ // odd k with few tokens, e.g. lora_b of a rank 1 LoRA on expert weights
+ for (int n : {1, 2}) {
+ test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 4, 2, false, 32, n, 1));
+ }
test_cases.emplace_back(new test_mul_mat_id_fusion(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, false, 32, 32, 32, 3));
for (ggml_type type : { GGML_TYPE_Q4_K, GGML_TYPE_Q8_0 }) {
for (int64_t n : { 1, 2, 3, 4, 8, 9 }) {