Commit 59695669f for llama.cpp
commit 59695669f1924c81f421f2044c690ed6373bec79
Author: MuoDoo <36472129+MuoDoo@users.noreply.github.com>
Date: Mon Nov 2 18:45:40 2026 +0800
jinja : implement map('filter') (#24033)
* jinja: implement map('filter')
* jinja : validate filter keyword arguments
Assisted-by: Codex
diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp
index 7ec6606d7..d32c79fa5 100644
--- a/common/jinja/runtime.cpp
+++ b/common/jinja/runtime.cpp
@@ -307,58 +307,13 @@ value binary_expression::execute_impl(context & ctx) const {
throw std::runtime_error("Unknown operator \"" + op.value + "\" between " + left_val->type() + " and " + right_val->type());
}
-static value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false) {
- JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str());
- if (ctx.is_get_stats) {
- value_t::stats_t::mark_used(input);
- input->stats.ops.insert(name);
- }
- auto builtins = input->get_builtins();
- auto it = builtins.find(name);
- if (it != builtins.end()) {
- JJ_DEBUG("Binding built-in '%s'", name.c_str());
- return mk_val<value_func>(name, it->second, input);
- }
- if (undef_on_missing) {
- return mk_val<value_undefined>(name);
- }
- throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type());
-}
-
static value apply_filter(context & ctx, const statement_ptr & filter, value input) {
JJ_DEBUG("Applying filter to %s", input->type().c_str());
- auto set_filter_alias = [](auto & filter_id) {
- if (filter_id == "count") {
- filter_id = "length";
- } else if (filter_id == "d") {
- filter_id = "default";
- } else if (filter_id == "e") {
- filter_id = "escape";
- } else if (filter_id == "trim") {
- filter_id = "strip";
- }
- };
-
if (is_stmt<identifier>(filter)) {
auto filter_id = cast_stmt<identifier>(filter)->val;
-
- set_filter_alias(filter_id);
JJ_DEBUG("Applying filter '%s' to %s", filter_id.c_str(), input->type().c_str());
- // TODO: Refactor filters so this coercion can be done automatically
- if (!input->is_undefined() && !is_val<value_string>(input) && (
- filter_id == "capitalize" ||
- filter_id == "lower" ||
- filter_id == "replace" ||
- filter_id == "strip" ||
- filter_id == "title" ||
- filter_id == "upper" ||
- filter_id == "wordcount"
- )) {
- JJ_DEBUG("Coercing %s to String for '%s' filter", input->type().c_str(), filter_id.c_str());
- input = mk_val<value_string>(input->as_string());
- }
- return try_builtin_func(ctx, filter_id, input)->invoke(func_args(ctx));
+ return apply_filter_by_name(ctx, filter_id, input, func_args(ctx));
} else if (is_stmt<call_expression>(filter)) {
auto call = cast_stmt<call_expression>(filter);
@@ -367,14 +322,13 @@ static value apply_filter(context & ctx, const statement_ptr & filter, value inp
}
auto filter_id = cast_stmt<identifier>(call->callee)->val;
- set_filter_alias(filter_id);
JJ_DEBUG("Applying filter '%s' with arguments to %s", filter_id.c_str(), input->type().c_str());
func_args args(ctx);
for (const auto & arg_expr : call->args) {
args.push_back(arg_expr->execute(ctx));
}
- return try_builtin_func(ctx, filter_id, input)->invoke(args);
+ return apply_filter_by_name(ctx, filter_id, input, args);
} else {
throw std::runtime_error("Invalid filter expression");
diff --git a/common/jinja/value.cpp b/common/jinja/value.cpp
index 3aceca1e0..333241d7a 100644
--- a/common/jinja/value.cpp
+++ b/common/jinja/value.cpp
@@ -72,6 +72,17 @@ const std::vector<value> & func_args::get_args() const {
return args;
}
+void func_args::ensure_kwargs(std::initializer_list<const char *> names) const {
+ for (const auto & arg : args) {
+ if (is_val<value_kwarg>(arg)) {
+ auto * kwarg = cast_val<value_kwarg>(arg);
+ if (std::find(names.begin(), names.end(), kwarg->key) == names.end()) {
+ throw raised_exception("Function '" + func_name + "' got an unexpected keyword argument '" + kwarg->key + "'");
+ }
+ }
+ }
+}
+
/**
* Function that mimics Python's array slicing.
*/
@@ -134,6 +145,14 @@ static value empty_value_fn(const func_args &) {
return mk_val<T>();
}
}
+
+template<typename T>
+static value empty_value_no_args_fn(const func_args & args) {
+ args.ensure_kwargs();
+ args.ensure_count(1, 1);
+ return empty_value_fn<T>(args);
+}
+
template<typename T>
static value test_type_fn(const func_args & args) {
args.ensure_count(1);
@@ -594,6 +613,52 @@ const func_builtins & global_builtins() {
return builtins;
}
+value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing) {
+ JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str());
+ if (ctx.is_get_stats) {
+ value_t::stats_t::mark_used(input);
+ input->stats.ops.insert(name);
+ }
+ auto builtins = input->get_builtins();
+ auto it = builtins.find(name);
+ if (it != builtins.end()) {
+ JJ_DEBUG("Binding built-in '%s'", name.c_str());
+ return mk_val<value_func>(name, it->second, input);
+ }
+ if (undef_on_missing) {
+ return mk_val<value_undefined>(name);
+ }
+ throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type());
+}
+
+value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args) {
+ if (filter_id == "count") {
+ filter_id = "length";
+ } else if (filter_id == "d") {
+ filter_id = "default";
+ } else if (filter_id == "e") {
+ filter_id = "escape";
+ } else if (filter_id == "trim") {
+ filter_id = "strip";
+ }
+
+ // TODO: Refactor filters so this coercion can be done automatically
+ if (!input->is_undefined() && !is_val<value_string>(input) && (
+ filter_id == "capitalize" ||
+ filter_id == "lower" ||
+ filter_id == "replace" ||
+ filter_id == "strip" ||
+ filter_id == "title" ||
+ filter_id == "upper" ||
+ filter_id == "wordcount"
+ )) {
+ JJ_DEBUG("Coercing %s to String for '%s' filter", input->type().c_str(), filter_id.c_str());
+ input = mk_val<value_string>(input->as_string());
+ }
+
+ return try_builtin_func(ctx, filter_id, input)->invoke(args);
+}
+
const func_builtins & value_int_t::get_builtins() const {
static const func_builtins builtins = {
@@ -667,16 +732,22 @@ const func_builtins & value_string_t::get_builtins() const {
static const func_builtins builtins = {
{"default", default_value},
{"upper", [](const func_args & args) -> value {
+ args.ensure_kwargs();
+ args.ensure_count(1, 1);
args.ensure_vals<value_string>();
jinja::string str = args.get_pos(0)->as_string().uppercase();
return mk_val<value_string>(str);
}},
{"lower", [](const func_args & args) -> value {
+ args.ensure_kwargs();
+ args.ensure_count(1, 1);
args.ensure_vals<value_string>();
jinja::string str = args.get_pos(0)->as_string().lowercase();
return mk_val<value_string>(str);
}},
{"strip", [](const func_args & args) -> value {
+ args.ensure_kwargs({"chars"});
+ args.ensure_count(1, 2);
value val_input = args.get_pos(0);
if (!is_val<value_string>(val_input)) {
throw raised_exception("strip() first argument must be a string");
@@ -707,11 +778,15 @@ const func_builtins & value_string_t::get_builtins() const {
}
}},
{"title", [](const func_args & args) -> value {
+ args.ensure_kwargs();
+ args.ensure_count(1, 1);
args.ensure_vals<value_string>();
jinja::string str = args.get_pos(0)->as_string().titlecase();
return mk_val<value_string>(str);
}},
{"capitalize", [](const func_args & args) -> value {
+ args.ensure_kwargs();
+ args.ensure_count(1, 1);
args.ensure_vals<value_string>();
jinja::string str = args.get_pos(0)->as_string().capitalize();
return mk_val<value_string>(str);
@@ -1013,6 +1088,10 @@ const func_builtins & value_bool_t::get_builtins() const {
return builtins;
}
+[[noreturn]] static value array_sum_not_implemented(const func_args &) {
+ throw not_implemented_exception("Array sum builtin not implemented");
+}
+
const func_builtins & value_array_t::get_builtins() const {
static const func_builtins builtins = {
{"default", default_value},
@@ -1079,7 +1158,9 @@ const func_builtins & value_array_t::get_builtins() const {
{"select", select_or_reject<false, false>},
{"rejectattr", select_or_reject<true, true>},
{"reject", select_or_reject<true, false>},
+ {"sum", array_sum_not_implemented},
{"join", [](const func_args & args) -> value {
+ args.ensure_kwargs({"d", "attribute"});
args.ensure_count(1, 3);
if (!is_val<value_array>(args.get_pos(0))) {
throw raised_exception("join() first argument must be an array");
@@ -1117,13 +1198,28 @@ const func_builtins & value_array_t::get_builtins() const {
{"tojson", tojson},
{"map", [](const func_args & args) -> value {
args.ensure_count(2);
- if (!is_val<value_array>(args.get_pos(0))) {
+ value val = args.get_pos(0);
+ if (!is_val<value_array>(val)) {
throw raised_exception("map: first argument must be an array");
}
if (!is_val<value_kwarg>(args.get_args().at(1))) {
- throw not_implemented_exception("map: filter-mapping not implemented");
+ value filter = args.get_pos(1);
+ if (!is_val<value_string>(filter)) {
+ throw raised_exception("map: filter name must be a string");
+ }
+ auto out = mk_val<value_array>();
+ func_args filter_args(args.ctx);
+ const auto & all_args = args.get_args();
+ for (size_t i = 2; i < all_args.size(); ++i) {
+ filter_args.push_back(all_args[i]);
+ }
+ const std::string filter_id = filter->as_string().str();
+ auto arr = val->as_array();
+ for (const auto & item : arr) {
+ out->push_back(apply_filter_by_name(args.ctx, filter_id, item, filter_args));
+ }
+ return is_val<value_tuple>(val) ? mk_val<value_tuple>(out->as_array()) : out;
}
- value val = args.get_pos(0);
value attribute = args.get_kwarg_or_pos("attribute", 1);
value default_val = args.get_kwarg("default", mk_val<value_undefined>());
auto out = mk_val<value_array>();
@@ -1132,7 +1228,7 @@ const func_builtins & value_array_t::get_builtins() const {
value attr_val = get_attribute(item, attribute, default_val);
out->push_back(attr_val);
}
- return is_val<value_tuple>(val) ? mk_val<value_tuple>(std::move(out->as_array())) : out;
+ return is_val<value_tuple>(val) ? mk_val<value_tuple>(out->as_array()) : out;
}},
{"append", [](const func_args & args) -> value {
args.ensure_count(2);
@@ -1405,14 +1501,18 @@ const func_builtins & value_none_t::get_builtins() const {
const func_builtins & value_undefined_t::get_builtins() const {
static const func_builtins builtins = {
{"default", default_value},
- {"capitalize", empty_value_fn<value_string>},
+ {"capitalize", empty_value_no_args_fn<value_string>},
{"first", empty_value_fn<value_undefined>},
{"items", empty_value_fn<value_array>},
- {"join", empty_value_fn<value_string>},
+ {"join", [](const func_args & args) -> value {
+ args.ensure_kwargs({"d", "attribute"});
+ args.ensure_count(1, 3);
+ return empty_value_fn<value_string>(args);
+ }},
{"last", empty_value_fn<value_undefined>},
{"length", empty_value_fn<value_int>},
{"list", empty_value_fn<value_array>},
- {"lower", empty_value_fn<value_string>},
+ {"lower", empty_value_no_args_fn<value_string>},
{"map", empty_value_fn<value_array>},
{"max", empty_value_fn<value_undefined>},
{"min", empty_value_fn<value_undefined>},
@@ -1425,12 +1525,16 @@ const func_builtins & value_undefined_t::get_builtins() const {
{"selectattr", empty_value_fn<value_array>},
{"sort", empty_value_fn<value_array>},
{"string", empty_value_fn<value_string>},
- {"strip", empty_value_fn<value_string>},
+ {"strip", [](const func_args & args) -> value {
+ args.ensure_kwargs({"chars"});
+ args.ensure_count(1, 2);
+ return empty_value_fn<value_string>(args);
+ }},
{"sum", empty_value_fn<value_int>},
- {"title", empty_value_fn<value_string>},
+ {"title", empty_value_no_args_fn<value_string>},
{"truncate", empty_value_fn<value_string>},
{"unique", empty_value_fn<value_array>},
- {"upper", empty_value_fn<value_string>},
+ {"upper", empty_value_no_args_fn<value_string>},
{"wordcount", empty_value_fn<value_int>},
};
return builtins;
diff --git a/common/jinja/value.h b/common/jinja/value.h
index 9bb1a87bd..8fdff884e 100644
--- a/common/jinja/value.h
+++ b/common/jinja/value.h
@@ -7,6 +7,7 @@
#include <cmath>
#include <cstdint>
#include <functional>
+#include <initializer_list>
#include <map>
#include <memory>
#include <set>
@@ -176,6 +177,8 @@ protected:
//
const func_builtins & global_builtins();
+value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false);
+value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args);
std::string value_to_json(const value & val, int indent = -1, const std::string_view item_sep = ", ", const std::string_view key_sep = ": ");
@@ -658,6 +661,7 @@ public:
size_t count() const { return args.size(); }
void push_back(const value & val);
void push_front(const value & val);
+ void ensure_kwargs(std::initializer_list<const char *> names = {}) const;
void ensure_count(size_t min, size_t max = 999) const {
size_t n = args.size();
if (n < min || n > max) {
diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp
index 444d57fbe..ce5a9749b 100644
--- a/tests/test-jinja.cpp
+++ b/tests/test-jinja.cpp
@@ -677,6 +677,24 @@ static void test_set_statement(testing & t) {
}
static void test_filters(testing & t) {
+ for (const std::string filter : {"upper", "lower", "title", "capitalize"}) {
+ for (const std::string input : {"'hello'", "missing"}) {
+ test_template_error(t, filter + " rejects unknown keyword on " + input,
+ "{{ " + input + "|" + filter + "(attribute='name') }}",
+ "unexpected keyword argument 'attribute'"
+ );
+ test_template_error(t, filter + " rejects extra positional argument on " + input,
+ "{{ " + input + "|" + filter + "('extra') }}",
+ "expected between 1 and 1 arguments, got 2"
+ );
+ }
+ }
+
+ test_template_error(t, "string method rejects unknown keyword",
+ "{{ 'hello'.upper(attribute='name') }}",
+ "unexpected keyword argument 'attribute'"
+ );
+
test_template(t, "upper",
"{{ 'hello'|upper }}",
json::object(),
@@ -725,6 +743,23 @@ static void test_filters(testing & t) {
"hello"
);
+ test_template(t, "trim chars keyword",
+ "{{ 'xyxhelloxyx'|trim(chars='xy') }}|{{ missing|trim(chars='xy') }}",
+ json::object(),
+ "hello|"
+ );
+
+ for (const std::string input : {"'hello'", "missing"}) {
+ test_template_error(t, "trim rejects unknown keyword on " + input,
+ "{{ " + input + "|trim(attribute='name') }}",
+ "unexpected keyword argument 'attribute'"
+ );
+ test_template_error(t, "trim rejects extra positional argument on " + input,
+ "{{ " + input + "|trim('x', 'y') }}",
+ "expected between 1 and 2 arguments, got 3"
+ );
+ }
+
test_template(t, "length string",
"{{ 'hello'|length }}",
json::object(),
@@ -2064,6 +2099,79 @@ static void test_array_methods(testing & t) {
"{\"x\": 2}"
);
+ test_template(t, "array|map with filter int materialized",
+ "{{ arr|map('int')|list|join(',') }}",
+ {{"arr", json::array({"1", "2", "3"})}},
+ "1,2,3"
+ );
+
+ test_template(t, "array|map with filter upper for schema union type",
+ "{{ item_value|map('upper')|list|join(',') }}",
+ {{"item_value", json::array({"string", "null"})}},
+ "STRING,NULL"
+ );
+
+ test_template_error(t, "array|map rejects unsupported filter attribute",
+ "{{ [{'name': 'alice'}]|map('upper', attribute='name')|join(',') }}",
+ "unexpected keyword argument 'attribute'"
+ );
+
+ test_template_error(t, "array|map rejects unknown keyword on undefined item",
+ "{{ [missing]|map('lower', attribute='name')|join(',') }}",
+ "unexpected keyword argument 'attribute'"
+ );
+
+ test_template(t, "array|map attribute then filter upper",
+ "{{ arr|map(attribute='name')|map('upper')|join(',') }}",
+ {{"arr", json::array({{{"name", "alice"}}, {{"name", "bob"}}})}},
+ "ALICE,BOB"
+ );
+
+ test_template(t, "array|map with filter arguments",
+ "{{ arr|map('replace', 'a', 'o')|join(',') }}",
+ {{"arr", json::array({"cat", "bat"})}},
+ "cot,bot"
+ );
+
+ test_template(t, "array|map with filter keyword arguments",
+ "{{ arr|map('join', '-', attribute='name')|join(';') }}",
+ {{"arr", json::array({
+ json::array({
+ json({{"name", "a"}}),
+ json({{"name", "b"}}),
+ }),
+ json::array({
+ json({{"name", "c"}}),
+ }),
+ })}},
+ "a-b;c"
+ );
+
+ test_template(t, "array|map with count alias",
+ "{{ arr|map('count')|join(',') }}",
+ {{"arr", json::array({"cat", "hello"})}},
+ "3,5"
+ );
+
+ test_template(t, "array|map with join keyword arguments",
+ "{{ [[{'name': 'a'}, {'name': 'b'}]]|map('join', d='-', attribute='name')|join(';') }}",
+ json::object(),
+ "a-b"
+ );
+
+ for (const std::string input : {"['a', 'b']", "[]", "missing"}) {
+ test_template_error(t, "join rejects unknown keyword on " + input,
+ "{{ " + input + "|join(typo='name') }}",
+ "unexpected keyword argument 'typo'"
+ );
+ }
+
+ test_template(t, "array|map with default alias and arguments",
+ "{{ arr|map('d', 'fallback', true)|join(',') }}",
+ {{"arr", json::array({"", "value"})}},
+ "fallback,value"
+ );
+
// not used by any chat templates
// test_template(t, "array.insert()",
// "{% set _ = arr.insert(1, 'x') %}{{ arr|join(',') }}",
@@ -2131,6 +2239,12 @@ static void test_array_methods(testing & t) {
""
);
+ test_template(t, "undefined|join with keyword arguments",
+ "{{ arr|join(d='-', attribute='name') }}",
+ json::object(),
+ ""
+ );
+
test_template(t, "undefined|sort",
"{{ arr|sort|string }}",
json::object(),