Commit 93cf736a8 for llama.cpp
commit 93cf736a8d1d46aa922e27eb64aa712bcd5ebc30
Author: Aldehir Rojas <hello@alde.dev>
Date: Thu Nov 12 01:44:42 2026 -0600
chat : refactor gemma4 to build loose schema rules and clean up (#31915)
* chat : build a schema-shaped PEG parser for Gemma 4 tool arguments
* chat : enforce only the key structure of Gemma 4 tool arguments
* chat : remove old gemma4 workarounds
* cont : add node.as<>() to simplify static casts
* cont : add depth_guard and use numerical rule names
* cont : clean up
* cont : extract gemma 4 parsers
* cont : clean up
* cont : clean up
* cont : remove dynamic cast
diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp
index 4440b2ebd..968cbfa98 100644
--- a/common/chat-peg-parser.cpp
+++ b/common/chat-peg-parser.cpp
@@ -973,6 +973,21 @@ void common_chat_peg_gemma4_mapper::from_ast(const common_peg_ast_arena & arena,
}
}
+using gemma4 = common_chat_peg_gemma4_mapper;
+
+// Collects the nearest descendants carrying tag, looking through the rules the parser
+// wraps around the tagged nodes.
+static void gemma4_collect(const common_peg_ast_arena & arena, const common_peg_ast_node & node, const std::string & tag, std::vector<common_peg_ast_id> & out) {
+ for (auto child_id : node.children) {
+ const auto & child = arena.get(child_id);
+ if (child.tag == tag) {
+ out.push_back(child_id);
+ } else {
+ gemma4_collect(arena, child, tag, out);
+ }
+ }
+}
+
static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id) {
const auto & node = arena.get(id);
@@ -980,51 +995,45 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg
return "";
}
- if (node.rule == "gemma4-number" || node.rule == "gemma4-bool" || node.rule == "gemma4-null") {
+ if (node.tag == gemma4::NUMBER || node.tag == gemma4::BOOL || node.tag == gemma4::NULL_) {
return std::string(node.text);
}
- if (node.rule == "gemma4-string-content") {
+ if (node.tag == gemma4::STRING_CONTENT) {
return escape_json_string_inner(node.sanitized_text());
}
- if (node.rule == "gemma4-string") {
+ if (node.tag == gemma4::STRING) {
std::string result = "\"";
- if (!node.children.empty()) {
- result += gemma4_to_json(arena, node.children[0]);
- if (!node.is_partial) {
- result += "\"";
- }
+ for (auto child_id : node.children) {
+ result += gemma4_to_json(arena, child_id);
+ }
+ if (!node.is_partial) {
+ result += "\"";
}
return result;
}
- if (node.rule == "gemma4-array") {
+ if (node.tag == gemma4::ARRAY) {
std::string result = "[";
-
- bool add_comma = false;
- for (auto child_id : node.children) {
- if (add_comma) {
+ for (size_t i = 0; i < node.children.size(); i++) {
+ if (i > 0) {
result += ',';
}
- add_comma = true;
- result += gemma4_to_json(arena, child_id);
+ result += gemma4_to_json(arena, node.children[i]);
}
-
if (!node.is_partial) {
result += ']';
}
return result;
}
- if (node.rule == "gemma4-dict-key-name") {
- return std::string(node.text);
- }
+ if (node.tag == gemma4::DICT_KEY) {
+ auto name_id = arena.find_by_tag(node, gemma4::DICT_KEY_NAME);
- if (node.rule == "gemma4-dict-key") {
std::string result = "\"";
- if (!node.children.empty()) {
- result += escape_json_string_inner(gemma4_to_json(arena, node.children[0]));
+ if (name_id != COMMON_PEG_INVALID_AST_ID) {
+ result += escape_json_string_inner(std::string(arena.get(name_id).text));
}
if (!node.is_partial) {
result += "\":";
@@ -1032,40 +1041,29 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg
return result;
}
- if (node.rule == "gemma4-dict-kv") {
- std::string result;
- for (auto child_id : node.children) {
- result += gemma4_to_json(arena, child_id);
- }
- return result;
- }
+ if (node.tag == gemma4::DICT) {
+ std::vector<common_peg_ast_id> members;
+ gemma4_collect(arena, node, gemma4::DICT_KV, members);
- if (node.rule == "gemma4-dict") {
std::string result = "{";
-
- bool add_comma = false;
- for (auto child_id : node.children) {
- if (add_comma) {
+ for (size_t i = 0; i < members.size(); i++) {
+ if (i > 0) {
result += ',';
}
- add_comma = true;
- result += gemma4_to_json(arena, child_id);
+ result += gemma4_to_json(arena, members[i]);
}
-
if (!node.is_partial) {
result += '}';
}
return result;
}
- if (node.rule == "gemma4-value") {
- if (!node.children.empty()) {
- return gemma4_to_json(arena, node.children[0]);
- }
- return "";
+ // members, rules and other wrappers
+ std::string result;
+ for (auto child_id : node.children) {
+ result += gemma4_to_json(arena, child_id);
}
-
- return "";
+ return result;
}
void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, common_peg_ast_id id) {
diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h
index 114fa049f..b827b60b0 100644
--- a/common/chat-peg-parser.h
+++ b/common/chat-peg-parser.h
@@ -34,6 +34,18 @@ class common_chat_peg_mapper {
class common_chat_peg_gemma4_mapper : public common_chat_peg_mapper {
public:
+ // Tags of the Gemma 4 argument syntax. The parser may wrap them in any rules.
+ static constexpr const char * STRING = "gemma4-string";
+ static constexpr const char * STRING_CONTENT = "gemma4-string-content";
+ static constexpr const char * NUMBER = "gemma4-number";
+ static constexpr const char * BOOL = "gemma4-bool";
+ static constexpr const char * NULL_ = "gemma4-null";
+ static constexpr const char * ARRAY = "gemma4-array";
+ static constexpr const char * DICT = "gemma4-dict";
+ static constexpr const char * DICT_KV = "gemma4-dict-kv";
+ static constexpr const char * DICT_KEY = "gemma4-dict-key";
+ static constexpr const char * DICT_KEY_NAME = "gemma4-dict-key-name";
+
common_chat_peg_gemma4_mapper(common_chat_msg & msg) : common_chat_peg_mapper(msg) {}
virtual void from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result);
private:
diff --git a/common/chat.cpp b/common/chat.cpp
index 0e27f8197..da29cd671 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -1277,13 +1277,6 @@ static common_chat_params common_chat_params_init_lfm2_5(const common_chat_templ
return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ false);
}
-// Older gemma4 templates need their tool responses rewritten before rendering
-static common_chat_params common_chat_params_init_gemma4_legacy(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
- auto adjusted = inputs;
- workaround::convert_tool_responses_gemma4(adjusted.messages);
- return common_chat_params_init_gemma4(tmpl, adjusted);
-}
-
// Pick the dedicated handler for a template from its source, or null for the autoparser.
// Order matters: the first match wins, and later checks assume the earlier ones did not match.
static common_chat_params_init_fn common_chat_template_detect_params_init(const std::string & src) {
@@ -1418,11 +1411,6 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
// Gemma4 format detection
if (src.find("'<|tool_call>call:'") != std::string::npos) {
LOG_DBG("Using specialized template: Gemma4\n");
- if (src.find("{#- OpenAI Chat Completions:") == std::string::npos) {
- LOG_WRN("%s: detected an outdated gemma4 chat template, applying compatibility workarounds. "
- "Consider updating to the official template.\n", __func__);
- return common_chat_params_init_gemma4_legacy;
- }
return common_chat_params_init_gemma4;
}
diff --git a/common/json-schema.h b/common/json-schema.h
index 084208c96..b203e7777 100644
--- a/common/json-schema.h
+++ b/common/json-schema.h
@@ -85,92 +85,124 @@ struct common_chat_schema {
static const char * kind_name(node_kind kind);
static const char * type_name(value_type type);
+
+ template <typename T>
+ const T & as() const {
+ GGML_ASSERT(kind() == T::KIND);
+ return static_cast<const T &>(*this);
+ }
};
using common_chat_schema_ptr = std::unique_ptr<common_chat_schema>;
struct common_chat_schema_any : common_chat_schema {
- node_kind kind() const override { return KIND_ANY; }
+ static constexpr node_kind KIND = KIND_ANY;
+
+ node_kind kind() const override { return KIND; }
};
// {"$ref": "#/..."}, only references into the same document are supported
struct common_chat_schema_ref : common_chat_schema {
+ static constexpr node_kind KIND = KIND_REF;
+
std::string ref;
const common_chat_schema * target = nullptr; // owned by common_chat_schema_document::refs
explicit common_chat_schema_ref(std::string ref) : ref(std::move(ref)) {}
- node_kind kind() const override { return KIND_REF; }
+ node_kind kind() const override { return KIND; }
};
// oneOf / anyOf, or a "type" array expanded to one alternative per type
struct common_chat_schema_any_of : common_chat_schema {
+ static constexpr node_kind KIND = KIND_ANY_OF;
+
std::vector<common_chat_schema_ptr> children;
- node_kind kind() const override { return KIND_ANY_OF; }
+ node_kind kind() const override { return KIND; }
};
struct common_chat_schema_all_of : common_chat_schema {
+ static constexpr node_kind KIND = KIND_ALL_OF;
+
std::vector<common_chat_schema_ptr> children;
- node_kind kind() const override { return KIND_ALL_OF; }
+ node_kind kind() const override { return KIND; }
};
struct common_chat_schema_const : common_chat_schema {
+ static constexpr node_kind KIND = KIND_CONST;
+
common_json value;
explicit common_chat_schema_const(common_json value) : value(std::move(value)) {}
- node_kind kind() const override { return KIND_CONST; }
+ node_kind kind() const override { return KIND; }
};
struct common_chat_schema_enum : common_chat_schema {
+ static constexpr node_kind KIND = KIND_ENUM;
+
std::vector<common_json> values;
- node_kind kind() const override { return KIND_ENUM; }
+ node_kind kind() const override { return KIND; }
};
struct common_chat_schema_null : common_chat_schema {
- node_kind kind() const override { return KIND_NULL; }
+ static constexpr node_kind KIND = KIND_NULL;
+
+ node_kind kind() const override { return KIND; }
};
struct common_chat_schema_boolean : common_chat_schema {
- node_kind kind() const override { return KIND_BOOLEAN; }
+ static constexpr node_kind KIND = KIND_BOOLEAN;
+
+ node_kind kind() const override { return KIND; }
};
struct common_chat_schema_number : common_chat_schema {
- node_kind kind() const override { return KIND_NUMBER; }
+ static constexpr node_kind KIND = KIND_NUMBER;
+
+ node_kind kind() const override { return KIND; }
};
// bounds are inclusive, exclusiveMinimum / exclusiveMaximum are folded in
struct common_chat_schema_integer : common_chat_schema {
+ static constexpr node_kind KIND = KIND_INTEGER;
+
int64_t minimum = INT64_MIN; // INT64_MIN for unbounded
int64_t maximum = INT64_MAX; // INT64_MAX for unbounded
- node_kind kind() const override { return KIND_INTEGER; }
+ node_kind kind() const override { return KIND; }
};
struct common_chat_schema_string : common_chat_schema {
+ static constexpr node_kind KIND = KIND_STRING;
+
std::string pattern; // empty when absent
string_format format = FORMAT_NONE;
int min_length = 0;
int max_length = -1; // -1 for unbounded
- node_kind kind() const override { return KIND_STRING; }
+ node_kind kind() const override { return KIND; }
};
struct common_chat_schema_array : common_chat_schema {
+ static constexpr node_kind KIND = KIND_ARRAY;
+
common_chat_schema_ptr items; // a common_chat_schema_any when "items" is absent
int min_items = 0;
int max_items = -1; // -1 for unbounded
- node_kind kind() const override { return KIND_ARRAY; }
+ node_kind kind() const override { return KIND; }
};
struct common_chat_schema_tuple : common_chat_schema {
+ static constexpr node_kind KIND = KIND_TUPLE;
+
std::vector<common_chat_schema_ptr> items;
- node_kind kind() const override { return KIND_TUPLE; }
+ node_kind kind() const override { return KIND; }
};
struct common_chat_schema_property {
@@ -180,10 +212,12 @@ struct common_chat_schema_property {
};
struct common_chat_schema_object : common_chat_schema {
+ static constexpr node_kind KIND = KIND_OBJECT;
+
std::vector<common_chat_schema_property> properties; // in schema order
common_chat_schema_ptr additional_properties; // null when not allowed
- node_kind kind() const override { return KIND_OBJECT; }
+ node_kind kind() const override { return KIND; }
};
struct common_chat_schema_document {
diff --git a/common/parsers/gemma4.cpp b/common/parsers/gemma4.cpp
index 1a63fa927..5902e290b 100644
--- a/common/parsers/gemma4.cpp
+++ b/common/parsers/gemma4.cpp
@@ -1,152 +1,284 @@
#include "parsers.h"
-namespace workaround {
-
-// Gemma4 uses a custom tool_responses field instead of role:tool messages.
-//
-// This will transform a sequence of messages:
-// assistant(tool_call+) -> tool+ -> assistant(content)
-//
-// Into a single assistant message containing a tool_responses field:
-// assistant(content + tool_call + tool_responses)
-//
-// This is necessary for the Gemma4 chat template to properly format the prompt.
-// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4
-struct gemma4_model_turn_builder {
- json & messages;
- size_t pos;
- json tool_calls = json::array();
- json tool_responses = json::array();
- json content;
- json reasoning_content;
-
- gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {}
-
- void collect() {
- // Collect the first assistant message
- auto & msg = messages[pos];
- if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
- // According to the prompt formatting guide, we need to preserve reasoning_content
- // between function calls. The current chat templates do not support this, but we will do it anyway.
- reasoning_content = msg.at("reasoning_content");
- }
- for (auto & tc : msg.at("tool_calls")) {
- tool_calls.push_back(tc);
- }
- pos++;
+#include "log.h"
+
+#include <algorithm>
+#include <cctype>
+#include <map>
+#include <optional>
+
+namespace {
+
+using gemma4 = common_chat_peg_gemma4_mapper;
- // Collect tool call results
- while (pos < messages.size() && messages[pos].value("role", "") == "tool") {
- collect_result(messages[pos]);
- pos++;
+struct depth_guard {
+ int & depth;
+ depth_guard(int & depth) : depth(depth) {
+ if (++depth > COMMON_RECURSIVE_LIMIT) {
+ throw std::invalid_argument("JSON schema is nested more than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " levels deep");
}
+ }
+ ~depth_guard() { depth--; }
+};
- // Check if the next assistant message is the final message
- if (pos < messages.size() && messages[pos].value("role", "") == "assistant") {
- auto & next = messages[pos];
- if (!has_tool_calls(next) && has_content(next)) {
- content = next.at("content");
- pos++;
- }
+// Gemma 4 value parsers
+void gemma4_value_rules(common_chat_peg_builder & p) {
+ // The tags are what gemma4_to_json() in the mapper reads, the rules may wrap them freely.
+ p.rule("gemma4-string", p.tag(gemma4::STRING, p.literal("<|\"|>") + p.tag(gemma4::STRING_CONTENT, p.until("<|\"|>")) + p.literal("<|\"|>")));
+
+ p.rule("gemma4-bool", p.tag(gemma4::BOOL, p.json_bool()));
+
+ p.rule("gemma4-null", p.tag(gemma4::NULL_, p.json_null()));
+
+ p.rule("gemma4-number", p.tag(gemma4::NUMBER, p.json_number()));
+
+ // Keys are bare in the template, but Gemma 4 occasionally wraps them in quotes
+ p.rule("gemma4-dict-key", p.tag(gemma4::DICT_KEY, p.choice({
+ p.literal("<|\"|>") + p.tag(gemma4::DICT_KEY_NAME, p.until("<|\"|>")) + p.literal("<|\"|>"),
+ p.literal("\"") + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^\"]", 1, -1)) + p.literal("\""),
+ p.literal("'") + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^']", 1, -1)) + p.literal("'"),
+ p.tag(gemma4::DICT_KEY_NAME, p.chars("[^:}]", 1, -1)),
+ }) + p.literal(":")));
+
+ p.rule("gemma4-dict-kv", p.tag(gemma4::DICT_KV, p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value")));
+
+ p.rule("gemma4-dict", [&]() {
+ auto ws = p.space();
+ auto member = p.ref("gemma4-dict-kv");
+ auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))});
+ return p.tag(gemma4::DICT, p.sequence({
+ p.literal("{"), ws,
+ p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})})
+ }));
+ });
+
+ p.rule("gemma4-array", [&]() {
+ auto ws = p.space();
+ auto value = p.ref("gemma4-value");
+ auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))});
+ return p.tag(gemma4::ARRAY, p.sequence({
+ p.literal("["), ws,
+ p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})})
+ }));
+ });
+
+ p.rule("gemma4-value", [&]() {
+ return p.choice({
+ p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"),
+ p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null")
+ });
+ });
+}
+
+// Gemma 4 uses different delimiters and cannot leverage the json-schema-to-grammar implementation.
+// This implements a subset to enforce properties.
+class gemma4_schema_parser {
+ common_chat_peg_builder & p_;
+ std::string root_;
+ int depth_ = 0;
+
+ std::map<const common_chat_schema *, std::string> refs_;
+
+ public:
+ gemma4_schema_parser(common_chat_peg_builder & p, std::string root) : p_(p), root_(std::move(root)) {}
+
+ common_peg_parser visit(const common_chat_schema & node) { return visit(node, root_); }
+
+ private:
+ common_peg_parser visit(const common_chat_schema & node, const std::string & name) {
+ depth_guard guard(depth_);
+ switch (node.kind()) {
+ case common_chat_schema::KIND_REF: return visit_ref(node.as<common_chat_schema_ref>());
+ case common_chat_schema::KIND_ANY_OF: return visit_any_of(node.as<common_chat_schema_any_of>(), name);
+ case common_chat_schema::KIND_ALL_OF: return visit_all_of(node.as<common_chat_schema_all_of>(), name);
+ case common_chat_schema::KIND_ARRAY: return visit_array(node.as<common_chat_schema_array>(), name);
+ case common_chat_schema::KIND_TUPLE: return visit_tuple(node.as<common_chat_schema_tuple>(), name);
+ case common_chat_schema::KIND_OBJECT: return visit_object(node.as<common_chat_schema_object>(), name);
+ default: return p_.ref("gemma4-value");
}
}
- void collect_result(const json & curr) {
- json response;
- if (curr.contains("content")) {
- const auto & content = curr.at("content");
- if (content.is_string()) {
- // Try to parse the content as JSON; fall back to raw string
- try {
- response = json::parse(content.get<std::string>());
- } catch (...) {
- response = content;
+ // Whether a schema contributes a shape of its own, as opposed to accepting any value.
+ static bool shaped(const common_chat_schema & node) {
+ switch (node.kind()) {
+ case common_chat_schema::KIND_REF:
+ return node.as<common_chat_schema_ref>().target != nullptr;
+ case common_chat_schema::KIND_ANY_OF:
+ for (const auto & child : node.as<common_chat_schema_any_of>().children) {
+ if (shaped(*child)) {
+ return true;
+ }
}
- } else {
- response = content;
+ return false;
+ case common_chat_schema::KIND_ALL_OF: {
+ const auto & children = node.as<common_chat_schema_all_of>().children;
+ return children.size() == 1 && shaped(*children[0]);
}
+ case common_chat_schema::KIND_ARRAY:
+ case common_chat_schema::KIND_TUPLE:
+ case common_chat_schema::KIND_OBJECT:
+ return true;
+ default:
+ return false;
}
+ }
- std::string name;
+ // The template writes keys bare. Models keep that for identifiers and quote anything else,
+ // with single or double quotes or the string delimiter, so those are accepted as well.
+ common_peg_parser key(const std::string & key) {
+ auto name = [&]() { return p_.tag(gemma4::DICT_KEY_NAME, p_.literal(key)); };
- // Match name with corresponding tool call
- size_t idx = tool_responses.size();
- if (idx < tool_calls.size()) {
- auto & tc = tool_calls[idx];
- if (tc.contains("function")) {
- name = tc.at("function").value("name", "");
+ bool identifier = !key.empty() && !std::isdigit(static_cast<unsigned char>(key[0]));
+ for (char c : key) {
+ if (!std::isalnum(static_cast<unsigned char>(c)) && c != '_') {
+ identifier = false;
}
}
-
- // Fallback to the tool call id
- if (name.empty()) {
- name = curr.value("tool_call_id", "");
+ if (identifier) {
+ return p_.tag(gemma4::DICT_KEY, name() + p_.literal(":"));
}
- tool_responses.push_back({{"name", name}, {"response", response}});
+ auto alts = p_.choice();
+ if (!key.empty() && key.find_first_of(":}") == std::string::npos && std::string("'\"<").find(key[0]) == std::string::npos) {
+ // Check if key is unambiguous
+ alts |= name();
+ }
+ // Gemma 4 may emit keys in its string delimiters or quotes
+ alts |= p_.literal("<|\"|>") + name() + p_.literal("<|\"|>");
+ if (key.find('\'') == std::string::npos) {
+ alts |= p_.literal("'") + name() + p_.literal("'");
+ }
+ if (key.find('"') == std::string::npos) {
+ alts |= p_.literal("\"") + name() + p_.literal("\"");
+ }
+ return p_.tag(gemma4::DICT_KEY, alts + p_.literal(":"));
}
- json build() {
- collect();
-
- json msg = {
- {"role", "assistant"},
- {"tool_calls", tool_calls},
- };
- if (!tool_responses.empty()) {
- msg["tool_responses"] = tool_responses;
+ common_peg_parser visit_ref(const common_chat_schema_ref & ref) {
+ if (!ref.target) {
+ return p_.ref("gemma4-value");
}
- if (!content.is_null()) {
- msg["content"] = content;
+ auto it = refs_.find(ref.target);
+ if (it != refs_.end()) {
+ return p_.ref(it->second);
}
- if (!reasoning_content.is_null()) {
- msg["reasoning_content"] = reasoning_content;
- }
- return msg;
+ // A target reached from several places gets one rule, named by the order it was first seen.
+ auto rule = root_ + "-ref-" + std::to_string(refs_.size());
+ refs_[ref.target] = rule;
+ return p_.rule(rule, [&]() { return visit(*ref.target, rule + "-value"); });
}
- static bool has_content(const json & msg) {
- if (!msg.contains("content") || msg.at("content").is_null()) {
- return false;
+ // Shaped alternatives are tried first, then anything else collapses into one generic value.
+ common_peg_parser visit_any_of(const common_chat_schema_any_of & node, const std::string & name) {
+ if (!shaped(node)) {
+ return p_.ref("gemma4-value");
+ }
+ if (node.children.size() == 1) {
+ return visit(*node.children[0], name);
}
- const auto & content = msg.at("content");
- if (content.is_string() && !content.get<std::string>().empty()) {
- return true;
+ auto alts = p_.choice();
+ bool loose = false;
+ for (size_t i = 0; i < node.children.size(); i++) {
+ if (shaped(*node.children[i])) {
+ alts |= visit(*node.children[i], name + "-" + std::to_string(i));
+ } else {
+ loose = true;
+ }
}
- if (content.is_array() && !content.empty()) {
- return true;
+ if (loose) {
+ alts |= p_.ref("gemma4-value");
}
- return false;
+ return p_.rule(name, alts);
}
- static bool has_tool_calls(const json & msg) {
- return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty();
+ common_peg_parser visit_all_of(const common_chat_schema_all_of & node, const std::string & name) {
+ if (node.children.size() == 1) {
+ return visit(*node.children[0], name);
+ }
+ return p_.ref("gemma4-value");
}
-};
-void convert_tool_responses_gemma4(json & messages) {
- json result = json::array();
- size_t i = 0;
+ common_peg_parser visit_array(const common_chat_schema_array & node, const std::string & name) {
+ auto ws = p_.space();
+ auto item = node.items && shaped(*node.items) ? visit(*node.items, name + "-item") : p_.ref("gemma4-value");
+ auto elements = item + p_.zero_or_more(p_.literal(",") + ws + item);
+ return p_.rule(name, p_.tag(gemma4::ARRAY,
+ p_.literal("[") + ws + (p_.literal("]") | (elements + ws + p_.literal("]")))));
+ }
- while (i < messages.size()) {
- auto & msg = messages[i];
+ common_peg_parser visit_tuple(const common_chat_schema_tuple & node, const std::string & name) {
+ auto ws = p_.space();
+ auto items = p_.sequence();
+ for (size_t i = 0; i < node.items.size(); i++) {
+ if (i > 0) {
+ items = items + p_.literal(",") + ws;
+ }
+ items = items + visit(*node.items[i], name + "-" + std::to_string(i));
+ }
+ return p_.rule(name, p_.tag(gemma4::ARRAY, p_.literal("[") + ws + items + ws + p_.literal("]")));
+ }
+
+ common_peg_parser visit_object(const common_chat_schema_object & node, const std::string & name) {
+ auto ws = p_.space();
- if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") ||
- !msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) {
- result.push_back(msg);
- i++;
- continue;
+ std::vector<const common_chat_schema_property *> props;
+ for (const auto & prop : node.properties) {
+ props.push_back(&prop);
}
- gemma4_model_turn_builder builder(messages, i);
- result.push_back(builder.build());
- i = builder.pos;
- }
+ // The Gemma 4 jinja template sorts properties, so our rules need to match.
+ auto lower = [](const std::string & s) {
+ std::string out = s;
+ for (auto & c : out) {
+ c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
+ }
+ return out;
+ };
+ std::stable_sort(props.begin(), props.end(), [&](const auto * a, const auto * b) {
+ return lower(a->name) < lower(b->name);
+ });
- messages = result;
-}
+ if (props.empty()) {
+ if (node.additional_properties) {
+ return p_.ref("gemma4-dict");
+ }
+ return p_.rule(name, p_.tag(gemma4::DICT, p_.literal("{") + ws + p_.literal("}")));
+ }
-}
+ std::optional<common_peg_parser> rest;
+ std::optional<common_peg_parser> first;
+ if (node.additional_properties) {
+ auto kv = p_.ref("gemma4-dict-kv");
+ rest = p_.zero_or_more(p_.literal(",") + ws + kv);
+ first = p_.optional(kv + *rest);
+ }
+
+ auto then_rest = [&](const common_peg_parser & parser) {
+ return rest ? parser + *rest : parser;
+ };
+
+ for (size_t i = props.size(); i-- > 0;) {
+ const auto & prop = *props[i];
+ auto kv = p_.tag(gemma4::DICT_KV, key(prop.name) + ws + visit(*prop.schema, name + "-" + std::to_string(i)));
+
+ common_peg_parser rest_i = then_rest(p_.literal(",") + ws + kv);
+ common_peg_parser first_i = then_rest(kv);
+ if (!prop.required) {
+ rest_i = then_rest(p_.optional(p_.literal(",") + ws + kv));
+ first_i = first ? (first_i | *first) : p_.optional(first_i);
+ }
+
+ if (i > 0) {
+ rest = p_.rule(name + "-rest-" + std::to_string(i), rest_i);
+ }
+ first = first_i;
+ }
+
+ return p_.rule(name, p_.tag(gemma4::DICT, p_.literal("{") + ws + *first + ws + p_.literal("}")));
+ }
+};
+
+} // namespace
common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl,
const autoparser::generation_params & inputs) {
@@ -218,51 +350,29 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &
}
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
- // Gemma4 tool calling syntax
- // Rules should match traversal logic in gemma4_to_json()
- p.rule("gemma4-string-content", p.until("<|\"|>"));
- p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>"));
- p.rule("gemma4-bool", p.json_bool());
- p.rule("gemma4-null", p.json_null());
- p.rule("gemma4-number", p.json_number());
- p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":"));
- p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value"));
- p.rule("gemma4-dict", [&]() {
- auto ws = p.space();
- auto member = p.ref("gemma4-dict-kv");
- auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))});
- return p.sequence({
- p.literal("{"), ws,
- p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})})
- });
- });
- p.rule("gemma4-array", [&]() {
- auto ws = p.space();
- auto value = p.ref("gemma4-value");
- auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))});
- return p.sequence({
- p.literal("["), ws,
- p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})})
- });
- });
- p.rule("gemma4-value", [&]() {
- return p.choice({
- p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"),
- p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null")
- });
- });
+ gemma4_value_rules(p);
auto tool_choice = p.choice();
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
const auto & function = tool.at("function");
std::string name = function.at("name");
- // TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules
- // const auto & params = function.at("parameters");
+
+ // Arguments follow the shape of the parameter schema when it is an object,
+ // see gemma4_schema_parser, and are a generic dict otherwise.
+ auto args = p.ref("gemma4-dict");
+ try {
+ auto doc = common_chat_schema_from_json(common_chat_tool_parameters(function));
+ if (doc.root->kind() == common_chat_schema::KIND_OBJECT) {
+ args = gemma4_schema_parser(p, "tool-" + std::to_string(tool_index) + "-args").visit(*doc.root);
+ }
+ } catch (const std::exception & e) {
+ LOG_WRN("%s: tool %s uses the generic argument parser: %s\n", __func__, name.c_str(), e.what());
+ }
tool_choice |= p.rule("tool-" + std::to_string(tool_index), p.tool(p.sequence({
p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))),
- p.tool_args(p.ref("gemma4-dict")),
+ p.tool_args(args),
})));
});
diff --git a/common/parsers/parsers.h b/common/parsers/parsers.h
index 532a44f01..b1b6d166a 100644
--- a/common/parsers/parsers.h
+++ b/common/parsers/parsers.h
@@ -41,12 +41,6 @@ std::string common_chat_template_generation_prompt_impl(
bool is_lfm2_template(const std::string & src);
-namespace workaround {
-
-void convert_tool_responses_gemma4(json & messages);
-
-}
-
common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
diff --git a/models/templates/google-gemma-4-31B-it-interleaved.jinja b/models/templates/google-gemma-4-31B-it-interleaved.jinja
deleted file mode 100644
index 85791c4fe..000000000
--- a/models/templates/google-gemma-4-31B-it-interleaved.jinja
+++ /dev/null
@@ -1,282 +0,0 @@
-{%- macro format_parameters(properties, required) -%}
- {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}
- {%- set ns = namespace(found_first=false) -%}
- {%- for key, value in properties | dictsort -%}
- {%- set add_comma = false -%}
- {%- if key not in standard_keys -%}
- {%- if ns.found_first %},{% endif -%}
- {%- set ns.found_first = true -%}
- {{ key }}:{
- {%- if value['description'] -%}
- description:<|"|>{{ value['description'] }}<|"|>
- {%- set add_comma = true -%}
- {%- endif -%}
- {%- if value['nullable'] %}
- {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
- nullable:true
- {%- endif -%}
- {%- if value['type'] | upper == 'STRING' -%}
- {%- if value['enum'] -%}
- {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
- enum:{{ format_argument(value['enum']) }}
- {%- endif -%}
- {%- elif value['type'] | upper == 'OBJECT' -%}
- ,properties:{
- {%- if value['properties'] is defined and value['properties'] is mapping -%}
- {{- format_parameters(value['properties'], value['required'] | default([])) -}}
- {%- elif value is mapping -%}
- {{- format_parameters(value, value['required'] | default([])) -}}
- {%- endif -%}
- }
- {%- if value['required'] -%}
- ,required:[
- {%- for item in value['required'] | default([]) -%}
- <|"|>{{- item -}}<|"|>
- {%- if not loop.last %},{% endif -%}
- {%- endfor -%}
- ]
- {%- endif -%}
- {%- elif value['type'] | upper == 'ARRAY' -%}
- {%- if value['items'] is mapping and value['items'] -%}
- ,items:{
- {%- set ns_items = namespace(found_first=false) -%}
- {%- for item_key, item_value in value['items'] | dictsort -%}
- {%- if item_value is not none -%}
- {%- if ns_items.found_first %},{% endif -%}
- {%- set ns_items.found_first = true -%}
- {%- if item_key == 'properties' -%}
- properties:{
- {%- if item_value is mapping -%}
- {{- format_parameters(item_value, value['items']['required'] | default([])) -}}
- {%- endif -%}
- }
- {%- elif item_key == 'required' -%}
- required:[
- {%- for req_item in item_value -%}
- <|"|>{{- req_item -}}<|"|>
- {%- if not loop.last %},{% endif -%}
- {%- endfor -%}
- ]
- {%- elif item_key == 'type' -%}
- {%- if item_value is string -%}
- type:{{ format_argument(item_value | upper) }}
- {%- else -%}
- type:{{ format_argument(item_value | map('upper') | list) }}
- {%- endif -%}
- {%- else -%}
- {{ item_key }}:{{ format_argument(item_value) }}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- }
- {%- endif -%}
- {%- endif -%}
- {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
- type:<|"|>{{ value['type'] | upper }}<|"|>}
- {%- endif -%}
- {%- endfor -%}
-{%- endmacro -%}
-{%- macro format_function_declaration(tool_data) -%}
- declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|>
- {%- set params = tool_data['function']['parameters'] -%}
- {%- if params -%}
- ,parameters:{
- {%- if params['properties'] -%}
- properties:{ {{- format_parameters(params['properties'], params['required']) -}} },
- {%- endif -%}
- {%- if params['required'] -%}
- required:[
- {%- for item in params['required'] -%}
- <|"|>{{- item -}}<|"|>
- {{- ',' if not loop.last -}}
- {%- endfor -%}
- ],
- {%- endif -%}
- {%- if params['type'] -%}
- type:<|"|>{{- params['type'] | upper -}}<|"|>}
- {%- endif -%}
- {%- endif -%}
- {%- if 'response' in tool_data['function'] -%}
- {%- set response_declaration = tool_data['function']['response'] -%}
- ,response:{
- {%- if response_declaration['description'] -%}
- description:<|"|>{{- response_declaration['description'] -}}<|"|>,
- {%- endif -%}
- {%- if response_declaration['type'] | upper == 'OBJECT' -%}
- type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>}
- {%- endif -%}
- {%- endif -%}
- }
-{%- endmacro -%}
-{%- macro format_argument(argument, escape_keys=True) -%}
- {%- if argument is string -%}
- {{- '<|"|>' + argument + '<|"|>' -}}
- {%- elif argument is boolean -%}
- {{- 'true' if argument else 'false' -}}
- {%- elif argument is mapping -%}
- {{- '{' -}}
- {%- set ns = namespace(found_first=false) -%}
- {%- for key, value in argument | dictsort -%}
- {%- if ns.found_first %},{% endif -%}
- {%- set ns.found_first = true -%}
- {%- if escape_keys -%}
- {{- '<|"|>' + key + '<|"|>' -}}
- {%- else -%}
- {{- key -}}
- {%- endif -%}
- :{{- format_argument(value, escape_keys=escape_keys) -}}
- {%- endfor -%}
- {{- '}' -}}
- {%- elif argument is sequence -%}
- {{- '[' -}}
- {%- for item in argument -%}
- {{- format_argument(item, escape_keys=escape_keys) -}}
- {%- if not loop.last %},{% endif -%}
- {%- endfor -%}
- {{- ']' -}}
- {%- else -%}
- {{- argument -}}
- {%- endif -%}
-{%- endmacro -%}
-{%- macro strip_thinking(text) -%}
- {%- set ns = namespace(result='') -%}
- {%- for part in text.split('<channel|>') -%}
- {%- if '<|channel>' in part -%}
- {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}
- {%- else -%}
- {%- set ns.result = ns.result + part -%}
- {%- endif -%}
- {%- endfor -%}
- {{- ns.result | trim -}}
-{%- endmacro -%}
-
-{%- set ns = namespace(prev_message_type=None, last_user_message=-1) -%}
-{%- set loop_messages = messages -%}
-{{- bos_token -}}
-{#- Handle System/Tool Definitions Block -#}
-{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}
- {{- '<|turn>system\n' -}}
-
- {#- Inject Thinking token at the very top of the FIRST system turn -#}
- {%- if enable_thinking is defined and enable_thinking -%}
- {{- '<|think|>\n' -}}
- {%- set ns.prev_message_type = 'think' -%}
- {%- endif -%}
-
- {%- if messages[0]['role'] in ['system', 'developer'] -%}
- {{- messages[0]['content'] | trim -}}
- {%- set loop_messages = messages[1:] -%}
- {%- endif -%}
-
- {%- if tools -%}
- {%- for tool in tools %}
- {{- '<|tool>' -}}
- {{- format_function_declaration(tool) | trim -}}
- {{- '<tool|>' -}}
- {%- endfor %}
- {%- set ns.prev_message_type = 'tool' -%}
- {%- endif -%}
-
- {{- '<turn|>\n' -}}
-{%- endif %}
-
-{#- Find last user message -#}
-{%- for message in loop_messages -%}
- {%- if message['role'] == 'user' -%}
- {%- set ns.last_user_message = loop.index0 -%}
- {%- endif -%}
-{%- endfor -%}
-
-{#- Loop through messages -#}
-{%- for message in loop_messages -%}
- {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}
- {%- if not (ns.prev_message_type == 'tool_response' and message['tool_calls']) -%}
- {{- '<|turn>' + role + '\n' }}
- {%- endif -%}
-
- {%- set ns.prev_message_type = None -%}
-
- {%- if message['tool_calls'] -%}
- {#- Preserve reasoning between tool calls for model turns that come after the last user turn -#}
- {%- if message['reasoning_content'] and loop.index0 > ns.last_user_message -%}
- {{- '<|channel>thought\n' -}}
- {{- message['reasoning_content'] -}}
- {{- '<channel|>' -}}
- {%- endif -%}
- {%- for tool_call in message['tool_calls'] -%}
- {%- set function = tool_call['function'] -%}
- {{- '<|tool_call>call:' + function['name'] + '{' -}}
- {%- if function['arguments'] is mapping -%}
- {%- set ns_args = namespace(found_first=false) -%}
- {%- for key, value in function['arguments'] | dictsort -%}
- {%- if ns_args.found_first %},{% endif -%}
- {%- set ns_args.found_first = true -%}
- {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
- {%- endfor -%}
- {%- elif function['arguments'] is string -%}
- {{- function['arguments'] -}}
- {%- endif -%}
- {{- '}<tool_call|>' -}}
- {%- endfor -%}
- {%- set ns.prev_message_type = 'tool_call' -%}
- {%- endif -%}
-
- {%- if message['tool_responses'] -%}
- {#- Tool Response handling -#}
- {%- for tool_response in message['tool_responses'] -%}
- {{- '<|tool_response>' -}}
- {%- if tool_response['response'] is mapping -%}
- {{- 'response:' + tool_response['name'] | default('unknown') + '{' -}}
- {%- for key, value in tool_response['response'] | dictsort -%}
- {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
- {%- if not loop.last %},{% endif -%}
- {%- endfor -%}
- {{- '}' -}}
- {%- else -%}
- {{- 'response:' + tool_response['name'] | default('unknown') + '{value:' + format_argument(tool_response['response'], escape_keys=False) + '}' -}}
- {%- endif -%}
- {{- '<tool_response|>' -}}
- {%- endfor -%}
- {%- set ns.prev_message_type = 'tool_response' -%}
- {%- endif -%}
-
- {%- if message['content'] is string -%}
- {%- if role == 'model' -%}
- {{- strip_thinking(message['content']) -}}
- {%- else -%}
- {{- message['content'] | trim -}}
- {%- endif -%}
- {%- elif message['content'] is sequence -%}
- {%- for item in message['content'] -%}
- {%- if item['type'] == 'text' -%}
- {%- if role == 'model' -%}
- {{- strip_thinking(item['text']) -}}
- {%- else -%}
- {{- item['text'] | trim -}}
- {%- endif -%}
- {%- elif item['type'] == 'image' -%}
- {{- '<|image|>' -}}
- {%- set ns.prev_message_type = 'image' -%}
- {%- elif item['type'] == 'audio' -%}
- {{- '<|audio|>' -}}
- {%- set ns.prev_message_type = 'audio' -%}
- {%- elif item['type'] == 'video' -%}
- {{- '<|video|>' -}}
- {%- set ns.prev_message_type = 'video' -%}
- {%- endif -%}
- {%- endfor -%}
- {%- endif -%}
-
- {%- if not (message['tool_responses'] and not message['content']) -%}
- {{- '<turn|>\n' -}}
- {%- endif -%}
-{%- endfor -%}
-
-{%- if add_generation_prompt -%}
- {%- if ns.prev_message_type != 'tool_response' -%}
- {{- '<|turn>model\n' -}}
- {%- endif -%}
- {%- if not enable_thinking | default(false) -%}
- {{- '<|channel>thought\n<channel|>' -}}
- {%- endif -%}
-{%- endif -%}
diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp
index b565775bf..2c59d1222 100644
--- a/tests/test-chat.cpp
+++ b/tests/test-chat.cpp
@@ -3267,6 +3267,61 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
.expect(message_with_tool_calls("amount", R"({"orig": 1.5e10})"))
.run();
+ // The argument grammar follows the shape of the parameter schema: keys in
+ // alphabetical order, optional keys skipped in place, nested objects and arrays
+ // shaped the same way, any other value accepted as written.
+ common_chat_tool shape_tool{
+ /* .name = */ "shape",
+ /* .description = */ "Schema-shaped arguments",
+ /* .parameters = */ R"({
+ "type": "object",
+ "properties": {
+ "path": {"type": "string"},
+ "edits": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "oldText": {"type": "string"},
+ "newText": {"type": "string"}
+ },
+ "required": ["oldText", "newText"]
+ }
+ },
+ "mode": {"type": "string", "enum": ["fast", "safe"]},
+ "count": {"type": "integer"},
+ "headers": {
+ "type": "object",
+ "properties": {
+ "content-type": {"type": "string"},
+ "accept:lang": {"type": "string"}
+ }
+ },
+ "options": {"type": "object"}
+ },
+ "required": ["path", "edits"]
+ })",
+ };
+
+ tst.test(
+ "<|tool_call>call:shape{edits:[{newText:<|\"|>b<|\"|>,oldText:<|\"|>a<|\"|>}],path:<|\"|>f.c<|\"|>}<tool_call|>")
+ .tools({ shape_tool })
+ .expect(message_with_tool_calls("shape", R"({"edits":[{"newText":"b","oldText":"a"}],"path":"f.c"})"))
+ .run();
+
+ // Optional keys in place, quoted keys that are not identifiers
+ tst.test(
+ "<|tool_call>call:shape{count:3,edits:[],headers:{<|\"|>accept:lang<|\"|>:<|\"|>en<|\"|>,'content-type':<|\"|>text<|\"|>},mode:<|\"|>fast<|\"|>,options:{b:1,a:[true]},path:<|\"|>f<|\"|>}<tool_call|>")
+ .tools({ shape_tool })
+ .expect(message_with_tool_calls("shape", R"({"count":3,"edits":[],"headers":{"accept:lang":"en","content-type":"text"},"mode":"fast","options":{"b":1,"a":[true]},"path":"f"})"))
+ .run();
+
+ tst.test(
+ "<|tool_call>call:shape{edits:[],headers:{\"content-type\":<|\"|>text<|\"|>},path:<|\"|>f<|\"|>}<tool_call|>")
+ .tools({ shape_tool })
+ .expect(message_with_tool_calls("shape", R"({"edits":[],"headers":{"content-type":"text"},"path":"f"})"))
+ .run();
+
// Edge cases
tst.test(
"<|channel>thought\n<channel|>Hello, world!\nWhat's up?<channel|>")