Commit 1d72b05d3 for llama.cpp
commit 1d72b05d3899f2bd961cb30a29f54d7e15aacb2c
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Mon Sep 21 13:57:19 2026 +0300
tests/test-backend-ops : allow regex entries in the -o filter (#29204)
* tests/test-backend-ops : allow regex entries in the -o filter
so far -o only accepted a comma separated list of exact op names or
full test case strings. entries that are not plain op names are now
treated as regexes matched against the op name (e.g. "MUL_MAT.*"),
while plain names keep their exact-matching behavior so that
"-o ADD" does not match ADD_EX etc.
Assisted-by: pi:llama.cpp/Qwen3.8-27B
* cont : don't print the FA vec slice log when not needed
* tests/test-backend-ops : reformat the help text
use the same style as the other tools, with separate sections for
modes, options, and examples
Assisted-by: pi:llama.cpp/Qwen3.8-27B
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 887add245..cc68e9ca7 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -1145,6 +1145,47 @@ static void print_test_result_locked(printer * output_printer, const test_result
output_printer->print_test_result(result);
}
+// Splits the -o filter into comma separated entries. Commas inside parentheses
+// (i.e. inside a full test case string) are not treated as separators.
+static std::vector<std::string_view> op_filter_entries(const char * op_names_filter) {
+ std::vector<std::string_view> entries;
+ if (op_names_filter == nullptr) {
+ return entries;
+ }
+ std::string_view filter(op_names_filter);
+ while (!filter.empty()) {
+ auto comma_pos = filter.find_first_of(',');
+ const auto lparen_pos = filter.find_first_of('(');
+ if (lparen_pos < comma_pos) {
+ const auto rparen_pos = filter.find_first_of(')');
+ comma_pos = filter.find_first_of(',', rparen_pos);
+ }
+ entries.push_back(filter.substr(0, comma_pos));
+ filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : "";
+ }
+ return entries;
+}
+
+// An entry from the -o filter matches an op if it is either
+// * an exact op name as given by ggml_op_desc() (e.g. "ADD"), or
+// * a regex that matches the op name (e.g. "DSV4.*")
+static bool op_filter_entry_matches(std::string_view entry, std::string_view op_name) {
+ if (entry == op_name) {
+ return true;
+ }
+ // plain op names are matched exactly, anything else is treated as a regex
+ if (std::regex_match(std::string(entry), std::regex("[A-Z0-9_]+"))) {
+ return false;
+ }
+ std::regex re;
+ try {
+ re = std::regex(std::string(entry));
+ } catch (const std::regex_error &) {
+ return false;
+ }
+ return std::regex_search(op_name.data(), op_name.data() + op_name.size(), re);
+}
+
struct test_case {
virtual ~test_case() {}
@@ -1308,34 +1349,24 @@ struct test_case {
return t;
}
- // Checks an op against the test filter, which is a comma separated list of OP names or specific variations
+ // Checks an op against the test filter, which is a comma separated list of OP names, regexes, or specific variations
bool matches_filter(ggml_tensor * op, const char * op_names_filter) {
- if (op_names_filter) {
- const auto op_name = op_desc(op);
- const auto op_full_name = op_name + "(" + vars() + ")";
- std::string_view filter(op_names_filter);
- while (!filter.empty()) {
- auto comma_pos = filter.find_first_of(',');
- const auto lparen_pos = filter.find_first_of('(');
- if (lparen_pos < comma_pos) {
- auto rparen_pos = filter.find_first_of(')');
- comma_pos = filter.find_first_of(',', rparen_pos);
- const auto op_filter = filter.substr(0, comma_pos);
- if (op_filter == op_full_name) {
- return true;
- }
- } else {
- const auto op_filter = filter.substr(0, comma_pos);
- if (op_filter == op_name) {
- return true;
- }
+ if (op_names_filter == nullptr) {
+ return true;
+ }
+ const auto op_name = op_desc(op);
+ const auto op_full_name = op_name + "(" + vars() + ")";
+ for (const auto & entry : op_filter_entries(op_names_filter)) {
+ if (entry.find_first_of('(') != std::string_view::npos) {
+ // a full test case string, matched exactly
+ if (entry == op_full_name) {
+ return true;
}
- filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : "";
+ } else if (op_filter_entry_matches(entry, op_name)) {
+ return true;
}
- return false;
- } else {
- return true;
}
+ return false;
}
test_status_t eval(ggml_backend_t backend1,
@@ -11597,25 +11628,16 @@ static std::vector<int> fa_vec_legal_ne(int dk, int dv) {
}
static bool op_names_filter_selects(const char * op_names_filter, const char * op_name) {
- if (!op_names_filter) {
+ if (op_names_filter == nullptr) {
return true;
}
- std::string_view filter(op_names_filter);
- while (!filter.empty()) {
- auto comma_pos = filter.find_first_of(',');
- const auto lparen_pos = filter.find_first_of('(');
- std::string_view entry;
- if (lparen_pos < comma_pos) {
- const auto rparen_pos = filter.find_first_of(')');
- comma_pos = filter.find_first_of(',', rparen_pos);
- entry = filter.substr(0, lparen_pos);
- } else {
- entry = filter.substr(0, comma_pos);
- }
- if (entry == op_name) {
+ for (const auto & entry : op_filter_entries(op_names_filter)) {
+ // a full test case string is matched by its op name prefix
+ const auto lparen_pos = entry.find_first_of('(');
+ const auto op_entry = lparen_pos != std::string_view::npos ? entry.substr(0, lparen_pos) : entry;
+ if (op_filter_entry_matches(op_entry, op_name)) {
return true;
}
- filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : "";
}
return false;
}
@@ -11632,8 +11654,6 @@ static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu,
return true;
}
- printf("Running FA vec slice tests (env LLAMA_TEST_FA_VEC_DISABLE=1 to skip)\n");
-
auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend));
auto set_ov = (set_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override");
@@ -11642,13 +11662,16 @@ static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu,
return true; // not the Metal backend: nothing to force
}
+ printf("Running FA vec slice tests (env LLAMA_TEST_FA_VEC_DISABLE=1 to skip)\n");
+
struct shape_t { int dk, dv; };
const shape_t shapes[] = { { 128, 128 }, { 576, 512 } }; // mainstream head size + MLA shared K/V view
const int ne01_pts[] = { 1, 3 }; // decode, and padded rows for Q=2 and Q=4
const int ne11_pts[] = { 512, 4097 }; // nsg=1, and nsg>=2 together with kvpad
const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0 };
- int n_run = 0, n_fail = 0;
+ int n_run = 0;
+ int n_fail = 0;
for (auto s : shapes) {
for (int ne : fa_vec_legal_ne(s.dk, s.dv)) {
for (int Q : { 1, 2, 4 }) {
@@ -11968,20 +11991,29 @@ static void show_test_coverage() {
}
static void usage(char ** argv) {
- printf("Usage: %s [mode] [-o <op,..>] [-b <backend>] [-p <params regex>] [--output <console|sql|csv>] [--list-ops]", argv[0]);
- printf(" [--show-coverage] [--test-file <path>] [-j <n>]\n");
- printf(" valid modes:\n");
- printf(" - test (default, compare with CPU backend for correctness)\n");
- printf(" - grad (compare gradients from backpropagation with method of finite differences)\n");
- printf(" - perf (performance evaluation)\n");
- printf(" - support (probe backend operation support)\n");
- printf(" op names for -o are as given by ggml_op_desc() (e.g. ADD, MUL_MAT, etc),\n");
- printf(" optionally including the full test case string (e.g. \"ADD(type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1)\")\n");
- printf(" --output specifies output format (default: console, options: console, sql, csv)\n");
- printf(" --list-ops lists all available GGML operations\n");
- printf(" --show-coverage shows test coverage\n");
- printf(" --test-file reads test operators from a test file generated by test-export-graph-ops\n");
- printf(" -j <n> runs tests using <n> parallel worker threads (default: 1, test mode only)\n");
+ printf("Usage: %s [mode] [options]\n\n", argv[0]);
+ printf("Valid modes:\n");
+ printf(" test (default) compare with CPU backend for correctness\n");
+ printf(" grad compare gradients from backpropagation with method of finite differences\n");
+ printf(" perf performance evaluation\n");
+ printf(" support probe backend operation support\n\n");
+ printf("Options:\n");
+ printf(" -o <op|regex,..> comma separated list of exact op names (as given by ggml_op_desc()),\n");
+ printf(" full test case strings, and/or regexes matched against the op name\n");
+ printf(" -b <backend> run tests on the given backend (e.g. CPU, MTL0, CUDA0)\n");
+ printf(" -p <params regex> filter test cases by a regex matched against their params\n");
+ printf(" --output <console|sql|csv> output format (default: console)\n");
+ printf(" --list-ops list all available GGML operations\n");
+ printf(" --show-coverage show test coverage\n");
+ printf(" --test-file <path> read test operators from a test file generated by test-export-graph-ops\n");
+ printf(" -j <n> run tests using <n> parallel worker threads (default: 1, test mode only)\n\n");
+ printf("Examples:\n");
+ printf(" %s -j 8\n", argv[0]);
+ printf(" %s -o ADD,MUL_MAT\n", argv[0]);
+ printf(" %s -o ADD -p 'type=f16.*perm1=0'\n", argv[0]);
+ printf(" %s -b MTL0 -o 'DSV4.*'\n", argv[0]);
+ printf(" %s -b CUDA0 -o 'ADD(type=f16,ne=[1,1,1,1],nr=[32,1,1,1],nf=1,perm1=0,src_overlap=0)'\n", argv[0]);
+ printf(" %s perf -o 'MUL_MAT.*'\n", argv[0]);
}
int main(int argc, char ** argv) {