Commit cd2db800d for llama.cpp
commit cd2db800db489f9bdd231998172ced624742e9be
Author: Aldehir Rojas <hello@alde.dev>
Date: Wed Nov 11 11:53:04 2026 -0600
jinja : support chained comparisons and argument unpacking (#32072)
diff --git a/common/jinja/lexer.h b/common/jinja/lexer.h
index 439c85764..3f4ebb62a 100644
--- a/common/jinja/lexer.h
+++ b/common/jinja/lexer.h
@@ -37,6 +37,7 @@ struct token {
call_operator, // ()
additive_binary_operator, // + - ~
multiplicative_binary_operator, // * / %
+ power_operator, // **
comparison_binary_operator, // < > <= >= == !=
unary_operator, // ! - +
comment, // {# ... #}
@@ -71,6 +72,7 @@ static std::string type_to_string(token::type t) {
case token::call_operator: return "call_operator";
case token::additive_binary_operator: return "additive_binary_operator";
case token::multiplicative_binary_operator: return "multiplicative_binary_operator";
+ case token::power_operator: return "power_operator";
case token::comparison_binary_operator: return "comparison_binary_operator";
case token::unary_operator: return "unary_operator";
case token::comment: return "comment";
@@ -137,6 +139,7 @@ struct lexer {
{"+", token::additive_binary_operator},
{"-", token::additive_binary_operator},
{"~", token::additive_binary_operator},
+ {"**", token::power_operator},
{"*", token::multiplicative_binary_operator},
{"/", token::multiplicative_binary_operator},
{"%", token::multiplicative_binary_operator},
diff --git a/common/jinja/parser.cpp b/common/jinja/parser.cpp
index 7e99a1ac8..0841828c4 100644
--- a/common/jinja/parser.cpp
+++ b/common/jinja/parser.cpp
@@ -178,7 +178,7 @@ private:
// bool has_caller_args = false;
if (is(token::open_paren)) {
// Optional caller arguments, e.g. {% call(user) dump_users(...) %}
- caller_args = parse_args();
+ caller_args = parse_args(false);
// has_caller_args = true;
}
auto callee = parse_primary_expression();
@@ -283,7 +283,7 @@ private:
statement_ptr parse_macro_statement(size_t start_pos) {
auto name = parse_primary_expression();
- auto args = parse_args();
+ auto args = parse_args(false);
expect(token::close_statement, "Expected %}");
statements body;
// Keep going until we hit {% endmacro
@@ -397,11 +397,13 @@ private:
statement_ptr parse_comparison_expression() {
// NOTE: membership has same precedence as comparison
- // e.g., ('a' in 'apple' == 'b' in 'banana') evaluates as ('a' in ('apple' == ('b' in 'banana')))
- auto left = parse_additive_expression();
+ // e.g., ('a' in 'apple' == 'b' in 'banana') evaluates as ('a' in 'apple' and 'apple' == 'b' and 'b' in 'banana')
+ statements operands;
+ std::vector<token> ops;
+ operands.push_back(parse_additive_expression());
+ size_t pos = current;
while (true) {
token op;
- size_t start_pos = current;
if (is_identifier("not") && peek(1).t == token::identifier && peek(1).value == "in") {
op = {token::identifier, "not in", tokens[current].pos};
++current; // consume 'not'
@@ -411,9 +413,17 @@ private:
} else if (is(token::comparison_binary_operator)) {
op = next();
} else break;
- left = mk_stmt<binary_expression>(start_pos, op, std::move(left), parse_additive_expression());
+ ops.push_back(std::move(op));
+ operands.push_back(parse_additive_expression());
}
- return left;
+ if (ops.empty()) {
+ return std::move(operands[0]);
+ }
+ if (ops.size() == 1) {
+ return mk_stmt<binary_expression>(pos, ops[0], std::move(operands[0]), std::move(operands[1]));
+ }
+ // chained comparison, e.g. 1 <= x <= 100
+ return mk_stmt<comparison_chain_expression>(pos, std::move(operands), std::move(ops));
}
statement_ptr parse_additive_expression() {
@@ -500,17 +510,22 @@ private:
return parse_call_expression(std::move(member)); // foo.x()()
}
- statements parse_args() {
+ statements parse_args(bool allow_spread = true) {
// comma-separated arguments list
expect(token::open_paren, "Expected (");
statements args;
while (!is(token::close_paren)) {
statement_ptr arg;
- // unpacking: *expr
- if (peek().t == token::multiplicative_binary_operator && peek().value == "*") {
+ // unpacking: *expr or **expr
+ const auto & t = peek();
+ if (t.t == token::power_operator || (t.t == token::multiplicative_binary_operator && t.value == "*")) {
+ if (!allow_spread) {
+ throw std::runtime_error("Unexpected token: " + t.value + " of type " + std::to_string(t.t));
+ }
size_t start_pos = current;
- ++current; // consume *
- arg = mk_stmt<spread_expression>(start_pos, parse_expression());
+ ++current; // consume * or **
+ bool is_kwargs = t.t == token::power_operator;
+ arg = mk_stmt<spread_expression>(start_pos, parse_expression(), is_kwargs);
} else {
arg = parse_expression();
if (is(token::equals)) {
diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp
index d32c79fa5..d60a99d25 100644
--- a/common/jinja/runtime.cpp
+++ b/common/jinja/runtime.cpp
@@ -117,20 +117,8 @@ value object_literal::execute_impl(context & ctx) const {
return obj;
}
-value binary_expression::execute_impl(context & ctx) const {
- value left_val = left->execute(ctx);
-
- // Logical operators
- if (op.value == "and") {
- JJ_DEBUG("Executing logical test: %s AND %s", left->type().c_str(), right->type().c_str());
- return left_val->as_bool() ? right->execute(ctx) : std::move(left_val);
- } else if (op.value == "or") {
- JJ_DEBUG("Executing logical test: %s OR %s", left->type().c_str(), right->type().c_str());
- return left_val->as_bool() ? std::move(left_val) : right->execute(ctx);
- }
-
+static value exec_binary_op(context & ctx, const token & op, value left_val, value right_val) {
// Equality operators
- value right_val = right->execute(ctx);
JJ_DEBUG("Executing binary expression %s '%s' %s", left_val->type().c_str(), op.value.c_str(), right_val->type().c_str());
if (op.value == "==") {
return mk_val<value_bool>(*left_val == *right_val);
@@ -307,6 +295,67 @@ value binary_expression::execute_impl(context & ctx) const {
throw std::runtime_error("Unknown operator \"" + op.value + "\" between " + left_val->type() + " and " + right_val->type());
}
+value binary_expression::execute_impl(context & ctx) const {
+ value left_val = left->execute(ctx);
+
+ // Logical operators
+ if (op.value == "and") {
+ JJ_DEBUG("Executing logical test: %s AND %s", left->type().c_str(), right->type().c_str());
+ return left_val->as_bool() ? right->execute(ctx) : std::move(left_val);
+ } else if (op.value == "or") {
+ JJ_DEBUG("Executing logical test: %s OR %s", left->type().c_str(), right->type().c_str());
+ return left_val->as_bool() ? std::move(left_val) : right->execute(ctx);
+ }
+
+ return exec_binary_op(ctx, op, std::move(left_val), right->execute(ctx));
+}
+
+value comparison_chain_expression::execute_impl(context & ctx) const {
+ value left_val = operands[0]->execute(ctx);
+ value res;
+ for (size_t i = 0; i < ops.size(); ++i) {
+ value right_val = operands[i + 1]->execute(ctx);
+ res = exec_binary_op(ctx, ops[i], left_val, right_val);
+ if (!res->as_bool()) {
+ break; // same as "and", the next operands are not executed
+ }
+ left_val = std::move(right_val);
+ }
+ return res;
+}
+
+static void exec_call_args(context & ctx, const statements & arg_stmts, func_args & args) {
+ for (const auto & arg_stmt : arg_stmts) {
+ if (!is_stmt<spread_expression>(arg_stmt)) {
+ auto arg_val = arg_stmt->execute(ctx);
+ JJ_DEBUG(" Argument type: %s", arg_val->type().c_str());
+ args.push_back(arg_val);
+ continue;
+ }
+ auto spread = cast_stmt<spread_expression>(arg_stmt);
+ value val = spread->argument->execute(ctx);
+ JJ_DEBUG(" Unpacked argument type: %s", val->type().c_str());
+ if (spread->is_kwargs) {
+ if (!is_val<value_object>(val)) {
+ throw std::runtime_error("Argument after ** must be an object, got " + val->type());
+ }
+ for (const auto & pair : val->as_ordered_object()) {
+ if (!is_val<value_string>(pair.first)) {
+ throw std::runtime_error("Keyword argument key must be a string, got " + pair.first->type());
+ }
+ args.push_back(mk_val<value_kwarg>(pair.first->as_string().str(), pair.second));
+ }
+ } else {
+ if (!is_val<value_array>(val)) {
+ throw std::runtime_error("Argument after * must be an array, got " + val->type());
+ }
+ for (const auto & item : val->as_array()) {
+ args.push_back(item);
+ }
+ }
+ }
+}
+
static value apply_filter(context & ctx, const statement_ptr & filter, value input) {
JJ_DEBUG("Applying filter to %s", input->type().c_str());
@@ -324,9 +373,7 @@ static value apply_filter(context & ctx, const statement_ptr & filter, value inp
JJ_DEBUG("Applying filter '%s' with arguments to %s", filter_id.c_str(), input->type().c_str());
func_args args(ctx);
- for (const auto & arg_expr : call->args) {
- args.push_back(arg_expr->execute(ctx));
- }
+ exec_call_args(ctx, call->args, args);
return apply_filter_by_name(ctx, filter_id, input, args);
@@ -369,9 +416,7 @@ value test_expression::execute_impl(context & ctx) const {
test_id = cast_stmt<identifier>(call->callee)->val;
JJ_DEBUG("Applying test '%s' with arguments to %s", test_id.c_str(), input->type().c_str());
- for (const auto & arg_expr : call->args) {
- args.push_back(arg_expr->execute(ctx));
- }
+ exec_call_args(ctx, call->args, args);
} else {
throw std::runtime_error("Invalid test expression");
@@ -762,12 +807,7 @@ value call_statement::execute_impl(context & ctx) const {
call_ctx.set_val("caller", mk_val<value_func>("caller", func));
func_args args(call_ctx);
-
- for (const auto & arg_expr : call_expr->args) {
- auto arg_val = arg_expr->execute(ctx);
- JJ_DEBUG(" Argument type: %s", arg_val->type().c_str());
- args.push_back(arg_val);
- }
+ exec_call_args(ctx, call_expr->args, args);
JJ_DEBUG("Calling macro '%s' with %zu arguments", callee_func->name.c_str(), args.count());
return callee_func->invoke(args);
@@ -907,11 +947,7 @@ value member_expression::execute_impl(context & ctx) const {
value call_expression::execute_impl(context & ctx) const {
// gather arguments
func_args args(ctx);
- for (auto & arg_stmt : this->args) {
- auto arg_val = arg_stmt->execute(ctx);
- JJ_DEBUG(" Argument type: %s", arg_val->type().c_str());
- args.push_back(arg_val);
- }
+ exec_call_args(ctx, this->args, args);
// execute callee
value callee_val = callee->execute(ctx);
if (!is_val<value_func>(callee_val)) {
diff --git a/common/jinja/runtime.h b/common/jinja/runtime.h
index 101e927c4..43e108551 100644
--- a/common/jinja/runtime.h
+++ b/common/jinja/runtime.h
@@ -471,6 +471,28 @@ struct binary_expression : public expression {
}
};
+/**
+ * Two or more comparisons in a row, e.g. `a < b <= c`
+ * It is the same as `a < b and b <= c`, but each operand is executed only one time.
+ */
+struct comparison_chain_expression : public expression {
+ statements operands;
+ std::vector<token> ops;
+
+ comparison_chain_expression(statements && operands, std::vector<token> && ops)
+ : operands(std::move(operands)), ops(std::move(ops)) {
+ assert(this->operands.size() == this->ops.size() + 1);
+ for (const auto & operand : this->operands) chk_type<expression>(operand);
+ }
+ std::string type() const override { return "ComparisonChainExpression"; }
+ value execute_impl(context & ctx) const override;
+ void visit(context & ctx) const override {
+ ctx.visitor(false, this, {
+ {"operands", stmts_to_ptr(operands)}
+ });
+ }
+};
+
/**
* An operation with two sides, separated by the | operator.
* Operator precedence: https://github.com/pallets/jinja/issues/379#issuecomment-168076202
@@ -633,7 +655,9 @@ struct keyword_argument_expression : public expression {
struct spread_expression : public expression {
statement_ptr argument;
- explicit spread_expression(statement_ptr && argument) : argument(std::move(argument)) {
+ bool is_kwargs;
+ spread_expression(statement_ptr && argument, bool is_kwargs)
+ : argument(std::move(argument)), is_kwargs(is_kwargs) {
chk_type<expression>(this->argument);
}
std::string type() const override { return "SpreadExpression"; }
diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp
index 29de01ea9..c79b0d24b 100644
--- a/tests/test-jinja.cpp
+++ b/tests/test-jinja.cpp
@@ -172,6 +172,18 @@ static void test_conditionals(testing & t) {
"big"
);
+ test_template(t, "chained comparison",
+ "{{ 1 <= x <= 100 }},{{ 1 <= y <= 100 }},{{ 1 < 2 < 3 < 2 }},{{ 1 == 1 != 2 }},{{ 'a' in 'apple' == 'apple' }}",
+ {{"x", 42}, {"y", 500}},
+ "True,False,False,True,True"
+ );
+
+ test_template(t, "chained comparison short circuit",
+ "{{ 2 < 1 < y }}",
+ json::object(),
+ "False"
+ );
+
test_template(t, "object comparison",
"{% if {0: 1, none: 2, 1.0: 3, '0': 4, true: 5} == {false: 1, none: 2, 1: 5, '0': 4} %}equal{% endif %}",
json::object(),
@@ -1321,6 +1333,18 @@ static void test_macros(testing & t) {
"Hello, John Smith,Hi, Jane Doe"
);
+ test_template(t, "macro with unpacked args",
+ "{% macro add(a, b, c=0) %}{{ a + b + c }}{% endmacro %}{{ add(*[1, 2]) }},{{ add(1, **{'b': 2, 'c': 3}) }},{{ add(*xs, **kw) }}",
+ {{"xs", json::array({1, 2})}, {"kw", {{"c", 5}}}},
+ "3,6,8"
+ );
+
+ test_template(t, "filter with unpacked args",
+ "{{ 'a-b' | replace(*['-', '+']) }}",
+ json::object(),
+ "a+b"
+ );
+
test_template(t, "macro with caller",
"\
{%- macro nest_dict(o, i, ff='') %}\n\
@@ -2429,6 +2453,12 @@ static void test_object_methods(testing & t) {
{{"obj", {{"a", 3}, {"b", 1}}}},
"{\"a\": 3, \"b\": 1, \"c\": 2}"
);
+
+ test_template(t, "dict from dict and unpacked kwargs",
+ "{% set o = dict(obj, **{k: 2}) %}{{ o|tojson }}",
+ {{"obj", {{"a", 3}, {"b", 1}}}, {"k", "c"}},
+ "{\"a\": 3, \"b\": 1, \"c\": 2}"
+ );
}
static void test_hasher(testing & t) {