Commit 108f73adf for llama.cpp
commit 108f73adfe607a035ffa9d5996545c669a5decdf
Author: o7si <32285332+o7si@users.noreply.github.com>
Date: Fri Nov 13 19:52:21 2026 +0800
grammar : optimize matching with left factoring (#29557)
* grammar : limit the number of alternatives
* grammar : introduce an AST for grammar parsing
* grammar : revert AST refactor and alternatives limit
This reverts commits:
- 84f3b57c4d6c482e7187a30735d16b02de3784fa
- 393a707ebb7e6f798abdb8af34cb1bc102f4aaa6
* grammar: optimize big grammar matching by using left-factorization
* grammar : preserve character class boundaries during left factoring
* grammar : prevent infinite expansion during left factoring
* grammar : use ASCII characters in comments
* grammar : test empty suffixes after left factoring
* grammar : deduplicate left factoring logic
* grammar : add comments
* grammar : use number parameters and clarify RSS output
* grammar : clarify character class test names
* grammar : remove performance benchmark
* grammar : share immutable rules
---------
Co-authored-by: Piotr Wilkin <piotr.wilkin@syndatis.com>
diff --git a/src/llama-grammar.cpp b/src/llama-grammar.cpp
index 4e096a368..0ef01dbca 100644
--- a/src/llama-grammar.cpp
+++ b/src/llama-grammar.cpp
@@ -948,6 +948,160 @@ static void llama_grammar_advance_stack(
}
}
+// Left-factor grammar rules to reduce redundant parallel stacks.
+//
+// When a rule has many alternates (e.g. tool-choice ::= tc0 | tc1 | ... | tc119)
+// and the referenced rules share a common element-content prefix, extract that
+// prefix so the parser processes it once instead of N times.
+//
+// Handles two cases:
+// 1. Inline common prefix: rule = "ab1" | "ab2" -> rule = "ab" rule', rule' = "1" | "2"
+// 2. Cross-rule prefix: rule = A | B | C where A,B,C start with same elements
+// -> rule = common_prefix rule', rule' = A_suffix | B_suffix | C_suffix
+static void llama_grammar_left_factor(llama_grammar_rules & rules) {
+ auto elem_eq = [](const llama_grammar_element & a, const llama_grammar_element & b) {
+ return a.type == b.type && a.value == b.value;
+ };
+
+ // Keep the prefix boundary outside character classes.
+ auto trim_prefix = [](const llama_grammar_rule & rule, size_t start, size_t length) {
+ while (length > 0) {
+ const auto type = rule[start + length].type;
+ if (type != LLAMA_GRETYPE_CHAR_ALT && type != LLAMA_GRETYPE_CHAR_RNG_UPPER) {
+ break;
+ }
+ length--;
+ }
+ return length;
+ };
+
+ struct alt_info {
+ size_t rule_idx;
+ size_t start;
+ size_t end;
+ };
+
+ const size_t n_rules_original = rules.size();
+ // Original rules factor at most once; generated rules do not resolve references.
+ // Each new suffix removes a non-empty prefix.
+ bool changed = true;
+ while (changed) {
+ changed = false;
+ const size_t n_rules = rules.size();
+
+ for (size_t ri = 0; ri < n_rules; ri++) {
+ const auto & rule = rules[ri];
+
+ // Split rule into alternates
+ std::vector<std::pair<size_t, size_t>> alt_ranges;
+ size_t alt_start = 0;
+ for (size_t j = 0; j < rule.size(); j++) {
+ if (rule[j].type == LLAMA_GRETYPE_ALT || rule[j].type == LLAMA_GRETYPE_END) {
+ alt_ranges.push_back({alt_start, j});
+ alt_start = j + 1;
+ }
+ }
+
+ if (alt_ranges.size() < 4) {
+ continue;
+ }
+
+ // Resolve each alternate: if it's a single RULE_REF, follow it to get the
+ // actual element sequence from the referenced rule.
+ std::vector<alt_info> resolved;
+ resolved.reserve(alt_ranges.size());
+ bool all_single_refs = true;
+
+ for (const auto & [astart, aend] : alt_ranges) {
+ size_t len = aend - astart;
+ // Resolving references in generated suffixes can expand recursive rules forever.
+ if (ri < n_rules_original && len == 1 && rule[astart].type == LLAMA_GRETYPE_RULE_REF) {
+ size_t ref_id = static_cast<size_t>(rule[astart].value);
+ if (ref_id < rules.size()) {
+ size_t ref_end = 0;
+ for (size_t k = 0; k < rules[ref_id].size(); k++) {
+ if (rules[ref_id][k].type == LLAMA_GRETYPE_ALT ||
+ rules[ref_id][k].type == LLAMA_GRETYPE_END) {
+ ref_end = k;
+ break;
+ }
+ }
+ bool single_alt = (rules[ref_id][ref_end].type == LLAMA_GRETYPE_END);
+ if (single_alt) {
+ resolved.push_back({ref_id, 0, ref_end});
+ continue;
+ }
+ }
+ }
+ all_single_refs = false;
+ resolved.push_back({ri, astart, aend});
+ }
+
+ if (!all_single_refs) {
+ resolved.clear();
+ for (const auto & [astart, aend] : alt_ranges) {
+ resolved.push_back({ri, astart, aend});
+ }
+ }
+
+ size_t prefix_len = 0;
+ bool prefix_match = true;
+ while (prefix_match) {
+ const auto & first = resolved[0];
+ if (first.start + prefix_len >= first.end) {
+ break;
+ }
+ const auto & ref_elem = rules[first.rule_idx][first.start + prefix_len];
+
+ for (size_t ai = 1; ai < resolved.size(); ai++) {
+ const auto & alt = resolved[ai];
+ if (alt.start + prefix_len >= alt.end ||
+ !elem_eq(rules[alt.rule_idx][alt.start + prefix_len], ref_elem)) {
+ prefix_match = false;
+ break;
+ }
+ }
+ if (prefix_match) {
+ prefix_len++;
+ }
+ }
+
+ for (const auto & alt : resolved) {
+ prefix_len = trim_prefix(rules[alt.rule_idx], alt.start, prefix_len);
+ }
+ if (prefix_len == 0) {
+ continue;
+ }
+
+ uint32_t suffix_choice_id = static_cast<uint32_t>(rules.size());
+ llama_grammar_rule suffix_choice;
+
+ for (size_t ai = 0; ai < resolved.size(); ai++) {
+ if (ai > 0) {
+ suffix_choice.push_back({LLAMA_GRETYPE_ALT, 0});
+ }
+ const auto & alt = resolved[ai];
+ for (size_t j = alt.start + prefix_len; j < alt.end; j++) {
+ suffix_choice.push_back(rules[alt.rule_idx][j]);
+ }
+ }
+ suffix_choice.push_back({LLAMA_GRETYPE_END, 0});
+
+ llama_grammar_rule new_rule;
+ const auto & first = resolved[0];
+ for (size_t j = 0; j < prefix_len; j++) {
+ new_rule.push_back(rules[first.rule_idx][first.start + j]);
+ }
+ new_rule.push_back({LLAMA_GRETYPE_RULE_REF, suffix_choice_id});
+ new_rule.push_back({LLAMA_GRETYPE_END, 0});
+
+ rules.push_back(std::move(suffix_choice));
+ rules[ri] = std::move(new_rule);
+ changed = true;
+ }
+ }
+}
+
static bool llama_grammar_detect_left_recursion(
const llama_grammar_rules & rules,
size_t rule_index,
@@ -1006,7 +1160,7 @@ static bool llama_grammar_detect_left_recursion(
}
const llama_grammar_rules & llama_grammar_get_rules(const struct llama_grammar * grammar) {
- return grammar->rules;
+ return *grammar->rules;
}
llama_grammar_stacks & llama_grammar_get_stacks(struct llama_grammar * grammar) {
@@ -1035,7 +1189,7 @@ static void llama_grammar_accept_chr(
if (!llama_grammar_is_end_of_sequence(match.second)) {
new_stack.push_back(match.second);
}
- llama_grammar_advance_stack(grammar.rules, new_stack, new_stacks);
+ llama_grammar_advance_stack(*grammar.rules, new_stack, new_stacks);
}
}
@@ -1221,12 +1375,16 @@ struct llama_grammar * llama_grammar_init_impl(
}
}
- // Check for left recursion
- std::vector<bool> rules_visited(n_rules);
- std::vector<bool> rules_in_progress(n_rules);
- std::vector<bool> rules_may_be_empty(n_rules);
+ // Left-factor rules to reduce redundant parallel stacks
+ llama_grammar_left_factor(vec_rules);
+
+ // Check for left recursion (use post-factoring rule count)
+ const size_t n_rules_total = vec_rules.size();
+ std::vector<bool> rules_visited(n_rules_total);
+ std::vector<bool> rules_in_progress(n_rules_total);
+ std::vector<bool> rules_may_be_empty(n_rules_total);
try {
- for (size_t i = 0; i < n_rules; i++) {
+ for (size_t i = 0; i < n_rules_total; i++) {
if (rules_visited[i]) {
continue;
}
@@ -1267,7 +1425,7 @@ struct llama_grammar * llama_grammar_init_impl(
// then the pointers would be invalidated when the local vec_rules goes out of scope.
return new llama_grammar {
vocab,
- std::move(vec_rules),
+ std::make_shared<const llama_grammar_rules>(std::move(vec_rules)),
std::move(stacks),
/* .partial_utf8 = */ {},
/* .lazy = */ false,
@@ -1319,12 +1477,16 @@ struct llama_grammar * llama_grammar_init_impl(
vec_rules[i].push_back({LLAMA_GRETYPE_END, 0});
}
- // Check for left recursion
- std::vector<bool> rules_visited(n_rules);
- std::vector<bool> rules_in_progress(n_rules);
- std::vector<bool> rules_may_be_empty(n_rules);
+ // Left-factor rules to reduce redundant parallel stacks
+ llama_grammar_left_factor(vec_rules);
+
+ // Check for left recursion (use post-factoring rule count)
+ const size_t n_rules_total = vec_rules.size();
+ std::vector<bool> rules_visited(n_rules_total);
+ std::vector<bool> rules_in_progress(n_rules_total);
+ std::vector<bool> rules_may_be_empty(n_rules_total);
try {
- for (size_t i = 0; i < n_rules; i++) {
+ for (size_t i = 0; i < n_rules_total; i++) {
if (rules_visited[i]) {
continue;
}
@@ -1378,7 +1540,7 @@ struct llama_grammar * llama_grammar_init_impl(
// then the pointers would be invalidated when the local vec_rules goes out of scope.
return new llama_grammar {
vocab,
- std::move(vec_rules),
+ std::make_shared<const llama_grammar_rules>(std::move(vec_rules)),
std::move(stacks),
/* .partial_utf8 = */ {},
/* .lazy = */ lazy,
@@ -1399,33 +1561,7 @@ void llama_grammar_free_impl(struct llama_grammar * grammar) {
}
struct llama_grammar * llama_grammar_clone_impl(const struct llama_grammar & grammar) {
- auto * result = new llama_grammar {
- grammar.vocab,
- grammar.rules,
- grammar.stacks,
- grammar.partial_utf8,
- grammar.lazy,
- grammar.awaiting_trigger,
- grammar.trigger_buffer,
- grammar.trigger_buffer_positions,
- grammar.trigger_tokens,
- grammar.trigger_patterns,
- };
-
- // redirect elements in stacks to point to new rules
- for (size_t is = 0; is < result->stacks.size(); is++) {
- for (size_t ie = 0; ie < result->stacks[is].size(); ie++) {
- for (size_t ir0 = 0; ir0 < grammar.rules.size(); ir0++) {
- for (size_t ir1 = 0; ir1 < grammar.rules[ir0].size(); ir1++) {
- if (grammar.stacks[is][ie] == &grammar.rules[ir0][ir1]) {
- result->stacks[is][ie] = &result->rules[ir0][ir1];
- }
- }
- }
- }
- }
-
- return result;
+ return new llama_grammar(grammar);
}
void llama_grammar_apply_impl(const struct llama_grammar & grammar, llama_token_data_array * cur_p) {
@@ -1465,7 +1601,7 @@ void llama_grammar_apply_impl(const struct llama_grammar & grammar, llama_token_
}
}
- const auto rejects = llama_grammar_reject_candidates(grammar.rules, grammar.stacks, candidates_grammar);
+ const auto rejects = llama_grammar_reject_candidates(*grammar.rules, grammar.stacks, candidates_grammar);
for (const auto & reject : rejects) {
cur_p->data[reject.index].logit = -INFINITY;
}
@@ -1566,7 +1702,7 @@ void llama_grammar_accept_token(struct llama_grammar & grammar, llama_token toke
if (!llama_grammar_is_end_of_sequence(pos + 1)) {
new_stack.push_back(pos + 1);
}
- llama_grammar_advance_stack(grammar.rules, new_stack, stacks_new);
+ llama_grammar_advance_stack(*grammar.rules, new_stack, stacks_new);
}
} else {
llama_grammar_stacks current_stacks = {stack};
diff --git a/src/llama-grammar.h b/src/llama-grammar.h
index 2885f7705..5829d8edc 100644
--- a/src/llama-grammar.h
+++ b/src/llama-grammar.h
@@ -3,6 +3,7 @@
#include "llama.h"
#include <map>
+#include <memory>
#include <regex>
#include <string>
#include <vector>
@@ -132,8 +133,8 @@ struct llama_grammar {
// note: allow null vocab for testing (not great)
const llama_vocab * vocab;
- const llama_grammar_rules rules; // TODO: shared ptr
- llama_grammar_stacks stacks;
+ const std::shared_ptr<const llama_grammar_rules> rules;
+ llama_grammar_stacks stacks;
// buffer for partially generated UTF-8 sequence from accepted tokens
llama_partial_utf8 partial_utf8;
diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp
index 75bfca645..a7a7e968e 100644
--- a/tests/test-grammar-integration.cpp
+++ b/tests/test-grammar-integration.cpp
@@ -193,6 +193,19 @@ static void test_schema(const std::string & test_desc, const std::string & schem
}
static void test_simple_grammar() {
+ test_grammar(
+ "left factor leading empty suffix",
+ "root ::= \"ab\" | \"abc\" | \"abd\" | \"abe\"",
+ {"ab", "abc", "abd", "abe"}, {"", "a", "abb", "abcc"});
+ test_grammar(
+ "left factor middle empty suffix",
+ "root ::= \"abc\" | \"ab\" | \"abd\" | \"abe\"",
+ {"ab", "abc", "abd", "abe"}, {"", "a", "abb", "abcc"});
+ test_grammar(
+ "left factor recursive references",
+ "root ::= \"ok\" | choice\nchoice ::= a | b | c | d\na ::= \"x\" a\nb ::= \"x\" b\nc ::= \"x\" c\nd ::= \"x\" d",
+ {"ok"}, {"", "x", "xx", "okx"});
+
test_schema(
"min 0",
R"""({
@@ -623,6 +636,19 @@ static void test_special_chars() {
"šµš abcš šµ"
}
);
+
+ test_grammar(
+ "skip left factoring inside character classes",
+ "root ::= [ab] \"x\" | [ac] \"x\" | [ad] \"x\" | [ae] \"x\"",
+ {"ax", "bx", "cx", "dx", "ex"}, {"", "a", "fx", "axx"});
+ test_grammar(
+ "skip left factoring inside character ranges",
+ "root ::= [a-b] \"x\" | [a-c] \"x\" | [a-d] \"x\" | [a-e] \"x\"",
+ {"ax", "bx", "cx", "dx", "ex"}, {"", "a", "fx", "axx"});
+ test_grammar(
+ "left factor referenced character alternatives",
+ "root ::= a | b | c | d\na ::= \"p\" [a] \"x\"\nb ::= \"p\" [ab] \"x\"\nc ::= \"p\" [ac] \"x\"\nd ::= \"p\" [ad] \"x\"",
+ {"pax", "pbx", "pcx", "pdx"}, {"", "px", "pex", "paxx"});
}
static void test_quantifiers() {