Commit ff6b8d176 for llama.cpp
commit ff6b8d176186ce1472a71800fcf3e6b221fcbb7b
Author: Aman Gupta <amangupta052@gmail.com>
Date: Wed Nov 18 23:39:30 2026 +0800
ggml-cpu : use all threads for large GET_ROWS (#32514)
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
index a76af8487..30edcdbac 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -2378,10 +2378,8 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
} break;
case GGML_OP_GET_ROWS:
{
- // FIXME: get_rows can use additional threads, but the cost of launching additional threads
- // decreases performance with GPU offloading
- //n_tasks = n_threads;
- n_tasks = 1;
+ // waking the other threads costs more than a small lookup, e.g. when this is the only CPU node of an offloaded model
+ n_tasks = ggml_nelements(node) >= 262144 ? n_threads : 1;
} break;
case GGML_OP_SCALE:
case GGML_OP_SET:
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index fa1ffb853..b20e0d3fe 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -10813,6 +10813,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
for (int n : {1, 3, 257}) {
test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, n, 5, 3, 1, 1, false));
}
+ // many rows of a wide table, as a per-layer embedding lookup does
+ test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, 160, 100000, 32768, 1, 1, false));
for (ggml_type type : all_types) {
for (int b : {1, 7}) {
for (bool v : {false, true}) {
@@ -13609,6 +13611,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
test_cases.emplace_back(new test_arange(GGML_TYPE_F32, 0.0f, n, 1.0f));
}
+ // embedding lookups: one token (decode) and one batch of 2048 tokens (prefill)
+ for (int r : {16, 32768}) {
+ test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, 160, 2000000, r, 1, 1, false)); // Qwen4-Exp n-gram table, 16 rows per token
+ }
+ for (int r : {1, 2048}) {
+ test_cases.emplace_back(new test_get_rows(GGML_TYPE_F16, 2560, 65536, r, 1, 1, false));
+ }
+
// LoRA at prefill: B is [rank, n_out], so K = rank
for (int64_t k : {16, 32, 48, 64}) {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 2048, 512, k, {1, 1}, {1, 1}));