Commit f7b886fe6 for llama.cpp
commit f7b886fe6fe5d4cdafaac0d60d7cfa84ae43961d
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Wed Nov 11 17:34:55 2026 +0100
chat: allow TTS model to provide custom text normalizer via jinja (#32053)
* chat: allow TTS model to provide custom text normalizer via jinja
* rm tts_norm_is_reserved_name
diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt
index 8e3c88dac..e38485ca0 100644
--- a/common/CMakeLists.txt
+++ b/common/CMakeLists.txt
@@ -66,6 +66,8 @@ add_library(${TARGET}
chat-diff-analyzer.cpp
chat-peg-parser.cpp
chat-peg-parser.h
+ chat-tts-norm.cpp
+ chat-tts-norm.h
chat.cpp
chat.h
common.cpp
diff --git a/common/chat-tts-norm.cpp b/common/chat-tts-norm.cpp
new file mode 100644
index 000000000..f84540ff9
--- /dev/null
+++ b/common/chat-tts-norm.cpp
@@ -0,0 +1,864 @@
+#include "chat-tts-norm.h"
+
+#include "jinja/runtime.h"
+#include "jinja/value.h"
+
+#include <algorithm>
+#include <climits>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+// TTS normalization patterns, see docs/tts-norm-patterns.md for the specification
+// keep this file in sync with the python reference: scripts/chat_tts_norm.py
+
+using namespace jinja;
+
+//
+// character classes
+// they are defined with explicit codepoint ranges, independent of any unicode database version,
+// so that the C++ and python implementations agree on every codepoint
+//
+
+struct tts_norm_range {
+ uint32_t lo;
+ uint32_t hi;
+};
+
+static const tts_norm_range TTS_NORM_SPACE[] = {
+ {0x0009, 0x000D}, {0x0020, 0x0020}, {0x0085, 0x0085}, {0x00A0, 0x00A0}, {0x1680, 0x1680},
+ {0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000},
+};
+
+static const tts_norm_range TTS_NORM_PUNCT[] = {
+ {0x0021, 0x002F}, {0x003A, 0x0040}, {0x005B, 0x0060}, {0x007B, 0x007E},
+ {0x00A1, 0x00A9}, {0x00AB, 0x00B4}, {0x00B6, 0x00B9}, {0x00BB, 0x00BF}, {0x00D7, 0x00D7}, {0x00F7, 0x00F7},
+ {0x2010, 0x2027}, {0x2030, 0x205E}, {0x20A0, 0x20CF}, {0x2100, 0x218F}, {0x2190, 0x2BFF},
+ {0x3001, 0x3003}, {0x3008, 0x3020}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30FB, 0x30FB},
+ {0xFE10, 0xFE19}, {0xFE30, 0xFE4F}, {0xFE50, 0xFE6B},
+ {0xFF01, 0xFF0F}, {0xFF1A, 0xFF20}, {0xFF3B, 0xFF40}, {0xFF5B, 0xFF65},
+ {0x1F000, 0x1FAFF},
+};
+
+static const tts_norm_range TTS_NORM_CONTROL[] = {
+ {0x0000, 0x001F}, {0x007F, 0x009F}, {0x200B, 0x200F}, {0x2060, 0x206F}, {0xFEFF, 0xFEFF},
+};
+
+static const tts_norm_range TTS_NORM_HAN[] = {
+ {0x2E80, 0x2FDF}, {0x3005, 0x3005}, {0x3007, 0x3007}, {0x3021, 0x3029}, {0x3038, 0x303B},
+ {0x3400, 0x4DBF}, {0x4E00, 0x9FFF}, {0xF900, 0xFAFF}, {0x20000, 0x323AF},
+};
+
+template<size_t N>
+static bool tts_norm_in(const tts_norm_range (&ranges)[N], uint32_t c) {
+ for (const auto & r : ranges) {
+ if (c >= r.lo && c <= r.hi) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static bool tts_norm_is_alpha(uint32_t c) {
+ if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
+ return true;
+ }
+ if (c == 0xAA || c == 0xB5 || c == 0xBA) {
+ return true;
+ }
+ return c >= 0xC0 && !tts_norm_in(TTS_NORM_SPACE, c) && !tts_norm_in(TTS_NORM_PUNCT, c) && !tts_norm_in(TTS_NORM_CONTROL, c);
+}
+
+// kind is one of "dspawulhc"
+static bool tts_norm_class_match(char kind, uint32_t c) {
+ switch (kind) {
+ case 'd': return c >= '0' && c <= '9';
+ case 's': return tts_norm_in(TTS_NORM_SPACE, c);
+ case 'p': return tts_norm_in(TTS_NORM_PUNCT, c);
+ case 'a': return tts_norm_is_alpha(c);
+ case 'w': return tts_norm_is_alpha(c) || (c >= '0' && c <= '9');
+ case 'u': return c >= 'A' && c <= 'Z';
+ case 'l': return c >= 'a' && c <= 'z';
+ case 'h': return tts_norm_in(TTS_NORM_HAN, c);
+ case 'c': return tts_norm_in(TTS_NORM_CONTROL, c);
+ }
+ return false;
+}
+
+static bool tts_norm_is_class_letter(uint32_t c) {
+ if (c >= 'A' && c <= 'Z') {
+ c += 'a' - 'A';
+ }
+ return c < 0x80 && std::string("dspawulhc").find(static_cast<char>(c)) != std::string::npos;
+}
+
+static uint32_t tts_norm_ascii_lower(uint32_t c) {
+ return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c;
+}
+
+static uint32_t tts_norm_ascii_swapcase(uint32_t c) {
+ if (c >= 'A' && c <= 'Z') return c + ('a' - 'A');
+ if (c >= 'a' && c <= 'z') return c - ('a' - 'A');
+ return c;
+}
+
+static bool tts_norm_is_ascii_alnum(uint32_t c) {
+ return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
+}
+
+// invalid bytes are decoded as U+FFFD, one per byte; offs[i] is the byte offset of codepoint i, offs[n] = text size
+static void tts_norm_decode_utf8(const std::string & s, std::vector<uint32_t> & cpts, std::vector<size_t> & offs) {
+ cpts.clear();
+ offs.clear();
+ size_t i = 0;
+ const size_t n = s.size();
+ while (i < n) {
+ offs.push_back(i);
+ const unsigned char c = static_cast<unsigned char>(s[i]);
+ uint32_t cp = 0;
+ size_t len = 0;
+ if (c < 0x80) {
+ cp = c; len = 1;
+ } else if ((c & 0xE0) == 0xC0) {
+ cp = c & 0x1F; len = 2;
+ } else if ((c & 0xF0) == 0xE0) {
+ cp = c & 0x0F; len = 3;
+ } else if ((c & 0xF8) == 0xF0) {
+ cp = c & 0x07; len = 4;
+ }
+ bool ok = len > 0 && i + len <= n;
+ for (size_t k = 1; ok && k < len; k++) {
+ const unsigned char cc = static_cast<unsigned char>(s[i + k]);
+ if ((cc & 0xC0) != 0x80) {
+ ok = false;
+ } else {
+ cp = (cp << 6) | (cc & 0x3F);
+ }
+ }
+ if (!ok) {
+ cpts.push_back(0xFFFD);
+ i += 1;
+ } else {
+ cpts.push_back(cp);
+ i += len;
+ }
+ }
+ offs.push_back(n);
+}
+
+//
+// AST
+//
+
+enum class tts_norm_op {
+ LITERAL, // one codepoint
+ SET, // one codepoint from a set; class escapes like %d are sets with a single class
+ ANY, // any codepoint
+ START, // ^
+ END, // $
+ BOUNDARY, // %b
+ GROUP, // (...), (?:...), (?<name>...)
+ LOOKAHEAD, // (?=...), (?!...)
+ LOOKBEHIND, // (?<=c), (?<!c) - exactly one codepoint
+};
+
+struct tts_norm_class {
+ char kind;
+ bool neg;
+};
+
+struct tts_norm_set {
+ bool neg = false;
+ std::vector<std::pair<uint32_t, uint32_t>> ranges;
+ std::vector<tts_norm_class> classes;
+};
+
+struct tts_norm_item;
+using tts_norm_seq = std::vector<tts_norm_item>;
+
+struct tts_norm_pattern_node {
+ tts_norm_op op = tts_norm_op::GROUP;
+ uint32_t cp = 0; // LITERAL
+ tts_norm_set set; // SET
+ std::vector<tts_norm_seq> alts; // GROUP, LOOKAHEAD; LOOKBEHIND holds a single item
+ int capture = -1; // GROUP: capture index, -1 for non-capturing
+ bool neg = false; // LOOKAHEAD, LOOKBEHIND
+};
+
+static constexpr int TTS_NORM_INF = INT_MAX;
+static constexpr int TTS_NORM_MAX_REPEAT = 1000;
+static constexpr size_t TTS_NORM_MAX_PATTERN_LEN = 4096;
+
+struct tts_norm_item {
+ tts_norm_pattern_node atom;
+ int min = 1;
+ int max = 1;
+};
+
+static bool tts_norm_is_zero_width(tts_norm_op op) {
+ return op == tts_norm_op::START || op == tts_norm_op::END || op == tts_norm_op::BOUNDARY
+ || op == tts_norm_op::LOOKAHEAD || op == tts_norm_op::LOOKBEHIND;
+}
+
+//
+// parser
+//
+
+struct tts_norm_parser {
+ std::vector<uint32_t> s;
+ size_t p = 0;
+ size_t & n_caps;
+
+ tts_norm_parser(std::vector<uint32_t> src, size_t & n_caps) : s(std::move(src)), n_caps(n_caps) {}
+
+ [[noreturn]] void fail(const std::string & msg) const {
+ throw std::runtime_error("invalid TTS norm pattern: " + msg + " at position " + std::to_string(p));
+ }
+
+ bool eof() const { return p >= s.size(); }
+ uint32_t peek() const { return s[p]; }
+ bool peek_is(uint32_t c) const { return !eof() && s[p] == c; }
+
+ void expect(uint32_t c, const char * what) {
+ if (!peek_is(c)) {
+ fail(std::string("expected ") + what);
+ }
+ p++;
+ }
+
+ std::vector<tts_norm_seq> parse_alts() {
+ std::vector<tts_norm_seq> alts;
+ alts.push_back(parse_seq());
+ while (peek_is('|')) {
+ p++;
+ alts.push_back(parse_seq());
+ }
+ return alts;
+ }
+
+ tts_norm_seq parse_seq() {
+ tts_norm_seq seq;
+ while (!eof() && peek() != '|' && peek() != ')') {
+ tts_norm_item item;
+ item.atom = parse_atom();
+ parse_quantifier(item);
+ seq.push_back(std::move(item));
+ }
+ return seq;
+ }
+
+ int parse_int() {
+ if (eof() || peek() < '0' || peek() > '9') {
+ fail("expected number in quantifier");
+ }
+ int v = 0;
+ while (!eof() && peek() >= '0' && peek() <= '9') {
+ v = v * 10 + static_cast<int>(peek() - '0');
+ if (v > TTS_NORM_MAX_REPEAT) {
+ fail("repetition count exceeds " + std::to_string(TTS_NORM_MAX_REPEAT));
+ }
+ p++;
+ }
+ return v;
+ }
+
+ void parse_quantifier(tts_norm_item & item) {
+ if (eof()) {
+ return;
+ }
+ const uint32_t c = peek();
+ if (c != '?' && c != '*' && c != '+' && c != '{') {
+ return;
+ }
+ if (tts_norm_is_zero_width(item.atom.op)) {
+ fail("zero-width assertion cannot be repeated");
+ }
+ p++;
+ if (c == '?') {
+ item.min = 0; item.max = 1;
+ } else if (c == '*') {
+ item.min = 0; item.max = TTS_NORM_INF;
+ } else if (c == '+') {
+ item.min = 1; item.max = TTS_NORM_INF;
+ } else {
+ item.min = parse_int();
+ item.max = item.min;
+ if (peek_is(',')) {
+ p++;
+ item.max = peek_is('}') ? TTS_NORM_INF : parse_int();
+ }
+ expect('}', "'}' to close quantifier");
+ if (item.max < item.min) {
+ fail("quantifier max is smaller than min");
+ }
+ }
+ if (!eof() && (peek() == '?' || peek() == '*' || peek() == '+' || peek() == '{')) {
+ fail("multiple quantifiers");
+ }
+ }
+
+ tts_norm_pattern_node parse_group() {
+ // '(' already consumed
+ tts_norm_pattern_node node;
+ node.op = tts_norm_op::GROUP;
+ if (peek_is('?')) {
+ p++;
+ if (eof()) {
+ fail("unterminated group");
+ }
+ const uint32_t c = peek();
+ if (c == ':') {
+ p++;
+ } else if (c == '=' || c == '!') {
+ p++;
+ node.op = tts_norm_op::LOOKAHEAD;
+ node.neg = c == '!';
+ } else if (c == '<' && p + 1 < s.size() && (s[p + 1] == '=' || s[p + 1] == '!')) {
+ node.op = tts_norm_op::LOOKBEHIND;
+ node.neg = s[p + 1] == '!';
+ p += 2;
+ tts_norm_item item;
+ if (eof() || peek() == ')') {
+ fail("lookbehind must contain exactly one codepoint matcher");
+ }
+ item.atom = parse_atom();
+ if (item.atom.op != tts_norm_op::LITERAL && item.atom.op != tts_norm_op::SET && item.atom.op != tts_norm_op::ANY) {
+ fail("lookbehind must contain exactly one codepoint matcher");
+ }
+ if (!peek_is(')')) {
+ fail("lookbehind must contain exactly one codepoint matcher");
+ }
+ p++;
+ node.alts.push_back(tts_norm_seq{});
+ node.alts[0].push_back(std::move(item));
+ return node;
+ } else if (c == 'i') {
+ fail("(?i) is only allowed at the start of the pattern");
+ } else {
+ fail("unknown group type");
+ }
+ } else {
+ node.capture = static_cast<int>(n_caps++);
+ }
+ node.alts = parse_alts();
+ expect(')', "')' to close group");
+ return node;
+ }
+
+ // after '%' inside or outside of a set
+ // returns true if it is a class (stored in cls), false if it is a literal (stored in lit)
+ bool parse_escape(bool in_set, tts_norm_class & cls, uint32_t & lit, bool & boundary) {
+ boundary = false;
+ if (eof()) {
+ fail("dangling '%'");
+ }
+ const uint32_t e = s[p++];
+ if (tts_norm_is_class_letter(e)) {
+ cls.kind = static_cast<char>(tts_norm_ascii_lower(e));
+ cls.neg = e >= 'A' && e <= 'Z';
+ return true;
+ }
+ if (e == 'b' && !in_set) {
+ boundary = true;
+ return false;
+ }
+ if (tts_norm_is_ascii_alnum(e)) {
+ p--;
+ fail(std::string("unknown escape '%") + static_cast<char>(e) + "'");
+ }
+ lit = e;
+ return false;
+ }
+
+ tts_norm_pattern_node parse_set() {
+ // '[' already consumed
+ tts_norm_pattern_node node;
+ node.op = tts_norm_op::SET;
+ if (peek_is('^')) {
+ node.set.neg = true;
+ p++;
+ }
+ bool empty = true;
+ while (true) {
+ if (eof()) {
+ fail("unterminated set");
+ }
+ if (peek() == ']') {
+ p++;
+ break;
+ }
+ empty = false;
+ uint32_t lo = 0;
+ if (peek() == '%') {
+ p++;
+ tts_norm_class cls{};
+ bool boundary = false;
+ if (parse_escape(true, cls, lo, boundary)) {
+ node.set.classes.push_back(cls);
+ continue;
+ }
+ } else {
+ lo = s[p++];
+ }
+ uint32_t hi = lo;
+ if (peek_is('-') && p + 1 < s.size() && s[p + 1] != ']') {
+ p++;
+ if (peek() == '%') {
+ p++;
+ tts_norm_class cls{};
+ bool boundary = false;
+ if (parse_escape(true, cls, hi, boundary)) {
+ fail("character class cannot be used in a range");
+ }
+ } else {
+ hi = s[p++];
+ }
+ if (hi < lo) {
+ fail("invalid range in set");
+ }
+ }
+ node.set.ranges.emplace_back(lo, hi);
+ }
+ if (empty) {
+ fail("empty set");
+ }
+ return node;
+ }
+
+ tts_norm_pattern_node parse_atom() {
+ const uint32_t c = s[p++];
+ tts_norm_pattern_node node;
+ switch (c) {
+ case '(':
+ return parse_group();
+ case '[':
+ return parse_set();
+ case '.':
+ node.op = tts_norm_op::ANY;
+ return node;
+ case '^':
+ node.op = tts_norm_op::START;
+ return node;
+ case '$':
+ node.op = tts_norm_op::END;
+ return node;
+ case '%': {
+ tts_norm_class cls{};
+ uint32_t lit = 0;
+ bool boundary = false;
+ if (parse_escape(false, cls, lit, boundary)) {
+ node.op = tts_norm_op::SET;
+ node.set.classes.push_back(cls);
+ } else if (boundary) {
+ node.op = tts_norm_op::BOUNDARY;
+ } else {
+ node.op = tts_norm_op::LITERAL;
+ node.cp = lit;
+ }
+ return node;
+ }
+ case '?':
+ case '*':
+ case '+':
+ case '{':
+ p--;
+ fail("nothing to repeat");
+ case ']':
+ case '}':
+ p--;
+ fail(std::string("unescaped '") + static_cast<char>(c) + "', use '%" + static_cast<char>(c) + "'");
+ default:
+ node.op = tts_norm_op::LITERAL;
+ node.cp = c;
+ return node;
+ }
+ }
+};
+
+//
+// matcher
+// PEG-style: alternatives are ordered (the first one that matches wins) and quantifiers are greedy and possessive
+// (they never give back what they consumed), so there is no backtracking
+//
+
+struct tts_norm_matcher {
+ const std::vector<uint32_t> & t;
+ bool icase;
+ std::vector<std::pair<int64_t, int64_t>> caps;
+
+ bool is_w(size_t i) const {
+ return i < t.size() && tts_norm_class_match('w', t[i]);
+ }
+
+ bool set_match(const tts_norm_set & st, uint32_t c) const {
+ auto test = [&](uint32_t x) {
+ for (const auto & r : st.ranges) {
+ if (x >= r.first && x <= r.second) {
+ return true;
+ }
+ }
+ for (const auto & k : st.classes) {
+ if (tts_norm_class_match(k.kind, x) != k.neg) {
+ return true;
+ }
+ }
+ return false;
+ };
+ bool m = test(c);
+ if (!m && icase) {
+ const uint32_t o = tts_norm_ascii_swapcase(c);
+ if (o != c) {
+ m = test(o);
+ }
+ }
+ return m != st.neg;
+ }
+
+ bool match_alts(const std::vector<tts_norm_seq> & alts, size_t i, size_t & end) {
+ for (const auto & seq : alts) {
+ auto saved = caps;
+ if (match_seq(seq, i, end)) {
+ return true;
+ }
+ caps = std::move(saved);
+ }
+ return false;
+ }
+
+ bool match_seq(const tts_norm_seq & seq, size_t i, size_t & end) {
+ size_t pos = i;
+ for (const auto & it : seq) {
+ int count = 0;
+ while (count < it.max) {
+ const bool is_group = it.atom.op == tts_norm_op::GROUP;
+ std::vector<std::pair<int64_t, int64_t>> saved;
+ if (is_group) {
+ saved = caps;
+ }
+ size_t e = pos;
+ if (!match_once(it.atom, pos, e)) {
+ if (is_group) {
+ caps = std::move(saved);
+ }
+ break;
+ }
+ if (e == pos) {
+ // an empty iteration would repeat forever, it satisfies any remaining minimum
+ count = std::max(count + 1, it.min);
+ break;
+ }
+ pos = e;
+ count++;
+ }
+ if (count < it.min) {
+ return false;
+ }
+ }
+ end = pos;
+ return true;
+ }
+
+ bool match_once(const tts_norm_pattern_node & node, size_t i, size_t & end) {
+ const size_t n = t.size();
+ switch (node.op) {
+ case tts_norm_op::LITERAL:
+ if (i < n && (t[i] == node.cp || (icase && tts_norm_ascii_lower(t[i]) == tts_norm_ascii_lower(node.cp)))) {
+ end = i + 1;
+ return true;
+ }
+ return false;
+ case tts_norm_op::SET:
+ if (i < n && set_match(node.set, t[i])) {
+ end = i + 1;
+ return true;
+ }
+ return false;
+ case tts_norm_op::ANY:
+ if (i < n) {
+ end = i + 1;
+ return true;
+ }
+ return false;
+ case tts_norm_op::START:
+ end = i;
+ return i == 0;
+ case tts_norm_op::END:
+ end = i;
+ return i == n;
+ case tts_norm_op::BOUNDARY:
+ end = i;
+ return (i > 0 && is_w(i - 1)) != is_w(i);
+ case tts_norm_op::GROUP: {
+ size_t e = i;
+ if (!match_alts(node.alts, i, e)) {
+ return false;
+ }
+ if (node.capture >= 0) {
+ caps[node.capture] = {static_cast<int64_t>(i), static_cast<int64_t>(e)};
+ }
+ end = e;
+ return true;
+ }
+ case tts_norm_op::LOOKAHEAD: {
+ // captures made inside a lookahead are discarded
+ auto saved = caps;
+ size_t e = i;
+ const bool ok = match_alts(node.alts, i, e);
+ caps = std::move(saved);
+ end = i;
+ return ok != node.neg;
+ }
+ case tts_norm_op::LOOKBEHIND: {
+ size_t e = 0;
+ const bool ok = i > 0 && match_once(node.alts[0][0].atom, i - 1, e) && e == i;
+ end = i;
+ return ok != node.neg;
+ }
+ }
+ return false;
+ }
+};
+
+//
+// common_tts_norm_pattern
+//
+
+common_tts_norm_pattern::common_tts_norm_pattern(const std::string & src) {
+ if (src.size() > TTS_NORM_MAX_PATTERN_LEN) {
+ throw std::runtime_error("invalid TTS norm pattern: pattern is longer than " + std::to_string(TTS_NORM_MAX_PATTERN_LEN) + " bytes");
+ }
+ std::vector<uint32_t> cpts;
+ std::vector<size_t> offs;
+ tts_norm_decode_utf8(src, cpts, offs);
+
+ tts_norm_parser parser(std::move(cpts), n_caps);
+ static const std::vector<uint32_t> icase_prefix = {'(', '?', 'i', ')'};
+ if (parser.s.size() >= 4 && std::equal(icase_prefix.begin(), icase_prefix.end(), parser.s.begin())) {
+ icase = true;
+ parser.p = 4;
+ }
+
+ root = std::make_unique<tts_norm_pattern_node>();
+ root->op = tts_norm_op::GROUP;
+ root->alts = parser.parse_alts();
+ if (!parser.eof()) {
+ parser.fail("unbalanced ')'");
+ }
+}
+
+common_tts_norm_pattern::~common_tts_norm_pattern() = default;
+common_tts_norm_pattern::common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept = default;
+common_tts_norm_pattern & common_tts_norm_pattern::operator=(common_tts_norm_pattern &&) noexcept = default;
+
+bool common_tts_norm_pattern::match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const {
+ tts_norm_matcher m{cpts, icase, std::vector<std::pair<int64_t, int64_t>>(n_caps, {-1, -1})};
+ size_t e = pos;
+ if (!m.match_alts(root->alts, pos, e)) {
+ return false;
+ }
+ end = e;
+ groups = std::move(m.caps);
+ return true;
+}
+
+size_t common_tts_norm_pattern::n_groups() const {
+ return n_caps;
+}
+
+//
+// scan / sub
+//
+
+std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules) {
+ std::vector<uint32_t> cpts;
+ std::vector<size_t> offs;
+ tts_norm_decode_utf8(text, cpts, offs);
+
+ std::vector<common_tts_norm_segment> out;
+ const size_t n = cpts.size();
+ size_t i = 0;
+ size_t pending = 0; // start of the current unmatched run
+ std::vector<std::pair<int64_t, int64_t>> groups;
+
+ while (i < n) {
+ bool matched = false;
+ for (size_t r = 0; r < rules.size(); r++) {
+ size_t end = i;
+ if (rules[r]->match_at(cpts, i, end, groups) && end > i) {
+ if (pending < i) {
+ out.push_back({-1, offs[pending], offs[i], {}});
+ }
+ common_tts_norm_segment seg;
+ seg.rule = static_cast<int>(r);
+ seg.start = offs[i];
+ seg.end = offs[end];
+ for (const auto & g : groups) {
+ if (g.first < 0) {
+ seg.groups.emplace_back(std::string::npos, std::string::npos);
+ } else {
+ seg.groups.emplace_back(offs[g.first], offs[g.second]);
+ }
+ }
+ out.push_back(std::move(seg));
+ i = end;
+ pending = i;
+ matched = true;
+ break;
+ }
+ }
+ if (!matched) {
+ i++;
+ }
+ }
+ if (pending < n) {
+ out.push_back({-1, offs[pending], offs[n], {}});
+ }
+ return out;
+}
+
+struct tts_norm_repl_piece {
+ std::string literal;
+ int group = -1; // 0 = whole match, 1.. = groups
+};
+
+static std::vector<tts_norm_repl_piece> tts_norm_parse_repl(const std::string & repl, const common_tts_norm_pattern & pattern) {
+ std::vector<tts_norm_repl_piece> pieces;
+ std::string lit;
+ auto flush = [&]() {
+ if (!lit.empty()) {
+ pieces.push_back({lit, -1});
+ lit.clear();
+ }
+ };
+ for (size_t i = 0; i < repl.size(); i++) {
+ if (repl[i] != '%') {
+ lit += repl[i];
+ continue;
+ }
+ if (i + 1 >= repl.size()) {
+ throw std::runtime_error("invalid TTS norm replacement: dangling '%'");
+ }
+ const char c = repl[++i];
+ if (c == '%') {
+ lit += '%';
+ } else if (c >= '0' && c <= '9') {
+ const int g = c - '0';
+ if (static_cast<size_t>(g) > pattern.n_groups()) {
+ throw std::runtime_error("invalid TTS norm replacement: group %" + std::string(1, c) + " does not exist");
+ }
+ flush();
+ pieces.push_back({"", g});
+ } else {
+ throw std::runtime_error("invalid TTS norm replacement: unknown escape '%" + std::string(1, c) + "'");
+ }
+ }
+ flush();
+ return pieces;
+}
+
+std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl) {
+ const auto pieces = tts_norm_parse_repl(repl, pattern);
+ std::string out;
+ for (const auto & seg : common_tts_norm_scan(text, {&pattern})) {
+ if (seg.rule < 0) {
+ out.append(text, seg.start, seg.end - seg.start);
+ continue;
+ }
+ for (const auto & piece : pieces) {
+ if (piece.group < 0) {
+ out += piece.literal;
+ } else if (piece.group == 0) {
+ out.append(text, seg.start, seg.end - seg.start);
+ } else {
+ const auto & g = seg.groups[piece.group - 1];
+ if (g.first != std::string::npos) {
+ out.append(text, g.first, g.second - g.first);
+ }
+ }
+ }
+ }
+ return out;
+}
+
+//
+// template builtins
+//
+
+// rules can be: a pattern string (rule name "match"), a dict {name: pattern}, or a list of [name, pattern] pairs
+static std::vector<std::pair<std::string, std::string>> tts_norm_rules_from_value(const value & v) {
+ std::vector<std::pair<std::string, std::string>> rules;
+ if (is_val<value_string>(v)) {
+ rules.emplace_back("match", v->as_string().str());
+ } else if (is_val<value_object>(v)) {
+ for (const auto & kv : v->as_ordered_object()) {
+ if (!is_val<value_string>(kv.second)) {
+ throw std::runtime_error("tts_norm_scan: rule pattern must be a string");
+ }
+ rules.emplace_back(kv.first->as_string().str(), kv.second->as_string().str());
+ }
+ } else if (is_val<value_array>(v)) {
+ for (const auto & item : v->as_array()) {
+ if (!is_val<value_array>(item) || item->as_array().size() != 2
+ || !is_val<value_string>(item->as_array()[0]) || !is_val<value_string>(item->as_array()[1])) {
+ throw std::runtime_error("tts_norm_scan: each rule must be a [name, pattern] pair of strings");
+ }
+ rules.emplace_back(item->as_array()[0]->as_string().str(), item->as_array()[1]->as_string().str());
+ }
+ } else {
+ throw std::runtime_error("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs, got " + v->type());
+ }
+ return rules;
+}
+
+static value tts_norm_builtin_scan(const func_args & args) {
+ args.ensure_count(2, 2);
+ args.ensure_val<value_string>(args.get_pos(0));
+ const string text_val = args.get_pos(0)->as_string();
+ const std::string text = text_val.str();
+ const bool is_input = text_val.all_parts_are_input();
+
+ const auto rule_srcs = tts_norm_rules_from_value(args.get_pos(1));
+ std::vector<common_tts_norm_pattern> patterns;
+ patterns.reserve(rule_srcs.size());
+ std::vector<const common_tts_norm_pattern *> ptrs;
+ for (const auto & r : rule_srcs) {
+ patterns.emplace_back(r.second);
+ }
+ for (const auto & p : patterns) {
+ ptrs.push_back(&p);
+ }
+
+ auto mk_str = [&](size_t start, size_t end) -> value {
+ return mk_val<value_string>(string(text.substr(start, end - start), is_input));
+ };
+
+ auto out = mk_val<value_array>();
+ for (const auto & seg : common_tts_norm_scan(text, ptrs)) {
+ auto groups = mk_val<value_array>();
+ for (const auto & g : seg.groups) {
+ // groups that did not participate are empty strings, so they render as nothing and are falsy
+ groups->push_back(g.first == std::string::npos ? mk_str(0, 0) : mk_str(g.first, g.second));
+ }
+ auto obj = mk_val<value_object>();
+ obj->insert("rule", seg.rule < 0 ? value(mk_val<value_none>()) : value(mk_val<value_string>(rule_srcs[seg.rule].first)));
+ obj->insert("text", mk_str(seg.start, seg.end));
+ obj->insert("groups", groups);
+ out->push_back(obj);
+ }
+ return out;
+}
+
+static value tts_norm_builtin_sub(const func_args & args) {
+ args.ensure_count(3, 3);
+ args.ensure_vals<value_string, value_string, value_string>();
+ const string text_val = args.get_pos(0)->as_string();
+ const common_tts_norm_pattern pattern(args.get_pos(1)->as_string().str());
+ const std::string res = common_tts_norm_sub(text_val.str(), pattern, args.get_pos(2)->as_string().str());
+ return mk_val<value_string>(string(res, text_val.all_parts_are_input()));
+}
+
+void common_chat_tts_norm_add_builtins(context & ctx) {
+ ctx.set_val("tts_norm_scan", mk_val<value_func>("tts_norm_scan", tts_norm_builtin_scan));
+ ctx.set_val("tts_norm_sub", mk_val<value_func>("tts_norm_sub", tts_norm_builtin_sub));
+}
+
diff --git a/common/chat-tts-norm.h b/common/chat-tts-norm.h
new file mode 100644
index 000000000..14695bb52
--- /dev/null
+++ b/common/chat-tts-norm.h
@@ -0,0 +1,62 @@
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+// TTS normalization patterns: a small, fully specified matching language for TTS text normalization templates
+//
+// it is NOT a regex engine: the semantics are defined by llama.cpp (see docs/tts-norm-patterns.md), so that this
+// implementation and the python reference (scripts/chat_tts_norm.py) give the same result for every input
+//
+// two functions are exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
+// tts_norm_scan(text, rules) -> list of segments {rule, text, groups}
+// tts_norm_sub(text, pattern, repl) -> string
+
+namespace jinja {
+struct context;
+}
+
+struct tts_norm_pattern_node;
+
+class common_tts_norm_pattern {
+public:
+ // throws std::runtime_error on invalid pattern
+ explicit common_tts_norm_pattern(const std::string & src);
+ ~common_tts_norm_pattern();
+ common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept;
+ common_tts_norm_pattern & operator=(common_tts_norm_pattern &&) noexcept;
+
+ // match anchored at codepoint index pos; on success, end is the codepoint index after the match
+ // groups[i] holds [start, end) codepoint indices of capture i+1, or {-1, -1} if the group did not participate
+ bool match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const;
+
+ size_t n_groups() const;
+
+private:
+ std::unique_ptr<tts_norm_pattern_node> root;
+ size_t n_caps = 0;
+ bool icase = false;
+};
+
+// segment produced by common_tts_norm_scan(); byte offsets into the input text
+struct common_tts_norm_segment {
+ int rule = -1; // index of the matched rule, -1 for unmatched text
+ size_t start = 0;
+ size_t end = 0;
+ std::vector<std::pair<size_t, size_t>> groups; // byte offsets; {npos, npos} for groups that did not participate
+};
+
+// split text into matched and unmatched segments
+// at each position, rules are tried in order and the first non-empty match wins; otherwise one codepoint is skipped
+std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules);
+
+// replace every non-overlapping match of pattern with repl
+// repl references: %0 (whole match), %1..%9 (groups), %% (literal percent)
+std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl);
+
+// add tts_norm_scan() and tts_norm_sub() to the context of a task template
+// they are not jinja global builtins on purpose: chat templates are not affected
+void common_chat_tts_norm_add_builtins(jinja::context & ctx);
diff --git a/common/chat.cpp b/common/chat.cpp
index ef9b853f0..0e27f8197 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -3,6 +3,7 @@
#include "chat-auto-parser-helpers.h"
#include "chat-auto-parser.h"
#include "chat-peg-parser.h"
+#include "chat-tts-norm.h"
#include "common.h"
#include "ggml.h"
#include "json-schema-to-grammar.h"
@@ -807,6 +808,8 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
// renders a task template (e.g. "tts") with the given variables
static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) {
jinja::context ctx(tmpl.source());
+ // text normalization helpers: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md
+ common_chat_tts_norm_add_builtins(ctx);
jinja::global_from_json(ctx, vars, false);
ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
diff --git a/docs/tts-norm-patterns.md b/docs/tts-norm-patterns.md
new file mode 100644
index 000000000..802dde5ce
--- /dev/null
+++ b/docs/tts-norm-patterns.md
@@ -0,0 +1,125 @@
+# TTS normalization patterns
+
+Text-to-speech models often need their input normalized before tokenization: numbers, currencies, dates and symbols spelled out, punctuation mapped, spacing fixed. The exact rules are model-specific (they must match what the model was trained on), so they live in the model's `tts` task template rather than in C++.
+
+Jinja has no regex support, and exposing `std::regex` / Python `re` would not help: the two differ in many details (`\d`, `\w`, `\b`, lookbehind, case folding, backtracking limits), so a template could give different results in llama.cpp and in Python. Instead, task templates get two functions with a small matching language whose semantics are defined here:
+
+- `tts_norm_scan(text, rules)`: split text into matched and unmatched segments
+- `tts_norm_sub(text, pattern, repl)`: replace every match
+
+Implementations:
+- C++: [common/chat-tts-norm.cpp](../common/chat-tts-norm.cpp), injected into task templates by `common_chat_template_render()` in [common/chat.cpp](../common/chat.cpp). They are not available to chat templates.
+- Python reference: [scripts/chat_tts_norm.py](../scripts/chat_tts_norm.py), for use with `jinja2` (`env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan`)
+- Both are checked against each other by `tests/test-jinja.cpp` (`test-jinja -py "tts norm patterns.*"`)
+
+## Example
+
+```jinja
+{%- for s in tts_norm_scan(text, [
+ ['money', '%$(%d+)(?:%.(%d%d))?'],
+ ['percent', '(%d+)%%'],
+ ['time', '(%d%d?):(%d%d)(?!%d)'],
+ ['neg', '(?<!%w)-(%d+)'],
+ ['number', '%d+'],
+]) -%}
+ {%- if s.rule == 'money' -%}{%- set whole, cents = s.groups -%}{{ num(whole) }} dollars{{ ' and ' ~ num(cents) ~ ' cents' if cents else '' }}
+ {%- elif s.rule == 'percent' -%}{{ num(s.groups[0]) }} percent
+ {%- elif s.rule == 'time' -%}{%- set h, m = s.groups -%}{{ num(h) }} {{ num(m) }}
+ {%- elif s.rule == 'neg' -%}minus {{ num(s.groups[0]) }}
+ {%- elif s.rule == 'number' -%}{{ num(s.text) }}
+ {%- else -%}{{ s.text }}
+ {%- endif -%}
+{%- endfor -%}
+```
+
+`It costs $3.50, 20% off, at 10:45, -5 degrees, 7 cats.` becomes `It costs three dollars and fifty cents, twenty percent off, at ten forty-five, minus five degrees, seven cats.` (with a `num` macro that spells numbers, see the "mini text normalizer" test in `tests/test-jinja.cpp`).
+
+## Functions
+
+### `tts_norm_scan(text, rules)`
+
+`rules` is one of:
+- a list of `[name, pattern]` pairs (recommended, the order is explicit)
+- a dict `{name: pattern}`, in insertion order
+- a single pattern string, the rule is named `match`
+
+Scanning starts at the first codepoint. At each position, the rules are tried in order and the first one that produces a **non-empty** match wins; the scan continues after the match. If no rule matches, one codepoint is skipped. Matches never overlap.
+
+The result is a list of segments covering the whole text, in order. Consecutive unmatched codepoints are merged into one segment. Each segment is a dict:
+
+| key | matched segment | unmatched segment |
+|---|---|---|
+| `rule` | rule name | `none` |
+| `text` | matched text | unmatched text |
+| `groups` | list of capture groups, in order of their `(` | `[]` |
+
+There are no named groups, unpack `groups` instead: `{% set whole, cents = s.groups %}`.
+
+A group that did not participate in the match (e.g. inside a `?` that matched nothing) is an empty string, so it renders as nothing and is falsy.
+
+### `tts_norm_sub(text, pattern, repl)`
+
+Replaces every match of `pattern`, with the same scanning rules as `tts_norm_scan`. `repl` is a literal string with references:
+
+| reference | value |
+|---|---|
+| `%0` | whole match |
+| `%1` .. `%9` | group |
+| `%%` | a literal `%` |
+
+Any other `%` sequence in `repl`, or a reference to a group that does not exist, is an error.
+
+## Pattern syntax
+
+Patterns operate on Unicode codepoints. Invalid UTF-8 bytes (C++ only) are seen as U+FFFD, one per byte, and preserved in the output.
+
+| syntax | meaning |
+|---|---|
+| `x` | the literal codepoint `x` |
+| `%x` | escape: `%` followed by any non-alphanumeric codepoint is that literal codepoint (`%.`, `%$`, `%%`, `%(`, ...) |
+| `.` | any codepoint, including newlines |
+| `%d` `%s` `%p` `%a` `%w` `%u` `%l` `%h` `%c` | character classes, see below; the uppercase form (`%D`, ...) is the negation |
+| `[...]` | set: literal codepoints, ranges `a-z`, classes `%d`; `[^...]` is the negation |
+| `(...)` | capture group |
+| `(?:...)` | group without capture |
+| `a\|b` | ordered alternation |
+| `?` `*` `+` `{m}` `{m,}` `{m,n}` | quantifiers, greedy and possessive; `m`, `n` up to 1000 |
+| `^` `$` | start / end of the whole text |
+| `%b` | boundary: between a `%w` codepoint and a non-`%w` codepoint (the edges of the text count as non-`%w`) |
+| `(?=...)` `(?!...)` | lookahead, positive / negative |
+| `(?<=c)` `(?<!c)` | lookbehind of exactly one codepoint: `c` is a literal, a set, a class or `.` |
+| `(?i)` | at the very start of the pattern only: ignore ASCII case |
+
+These characters must be escaped with `%` to be matched literally: `( ) [ ] { } . | ? * + ^ $ %`. Inside a set, only `]`, `%`, `-` (between two codepoints) and `^` (first position) are special.
+
+### Matching semantics
+
+The matcher is a PEG, not a backtracking regex:
+
+- **Ordered alternation**: alternatives are tried left to right and the first one that matches is committed to. If what follows fails, the other alternatives are not retried. `(?:a|ab)c` never matches `abc`; write `(?:ab|a)c`.
+- **Possessive quantifiers**: quantifiers match as many repetitions as possible and never give any back. `%d+0` never matches; `.*x` never matches. Write patterns that describe exactly what to consume.
+- A repetition that matches the empty string ends the loop.
+- Captures inside lookaheads are discarded. A capture inside a repeated group holds its last repetition.
+
+This gives a simple and linear-time implementation with identical results in C++ and Python.
+
+### Character classes
+
+Classes are defined by explicit codepoint ranges, not by a Unicode database, so the result does not depend on the Unicode version of the platform.
+
+| class | definition |
+|---|---|
+| `%d` | ASCII digits `0-9` only |
+| `%s` | U+0009..U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000..U+200A, U+2028, U+2029, U+202F, U+205F, U+3000 |
+| `%p` | punctuation and symbols: ASCII punctuation, U+00A1..U+00BF (except U+00AA, U+00B5, U+00BA), U+00D7, U+00F7, U+2010..U+2027, U+2030..U+205E, U+20A0..U+20CF (currency), U+2100..U+2BFF (letterlike, number forms, arrows, math, technical, box drawing, shapes, misc symbols, dingbats), CJK punctuation (U+3001..U+3003, U+3008..U+3020, U+3030, U+303D, U+30FB), U+FE10..U+FE19, U+FE30..U+FE6B, fullwidth punctuation (U+FF01..U+FF0F, U+FF1A..U+FF20, U+FF3B..U+FF40, U+FF5B..U+FF65), U+1F000..U+1FAFF (emoji) |
+| `%c` | control and invisible format characters: U+0000..U+001F, U+007F..U+009F, U+200B..U+200F, U+2060..U+206F, U+FEFF |
+| `%a` | ASCII letters, U+00AA, U+00B5, U+00BA, and every codepoint from U+00C0 that is not in `%s`, `%p` or `%c` |
+| `%w` | `%a` or `%d` |
+| `%u` / `%l` | ASCII uppercase / lowercase letters only |
+| `%h` | Han: U+2E80..U+2FDF, U+3005, U+3007, U+3021..U+3029, U+3038..U+303B, U+3400..U+4DBF, U+4E00..U+9FFF, U+F900..U+FAFF, U+20000..U+323AF |
+
+Classes may overlap (e.g. U+0085 is both `%s` and `%c`). `%a` intentionally includes all scripts, combining marks and CJK, so `(?<!%w)` and `%b` work as "not inside a word" for any language.
+
+## Errors
+
+Invalid patterns are rejected when the function is called, with a message and a position, for example: unbalanced parentheses, a quantifier with nothing to repeat, multiple quantifiers (`a+?`), repeating a zero-width assertion (`%b+`), unknown escapes (`%q`), unescaped `]` or `}`, an empty set, a lookbehind wider than one codepoint and `(?i)` not at the start.
diff --git a/scripts/chat_tts_norm.py b/scripts/chat_tts_norm.py
new file mode 100644
index 000000000..a5fbd55b6
--- /dev/null
+++ b/scripts/chat_tts_norm.py
@@ -0,0 +1,556 @@
+#!/usr/bin/env python3
+"""Reference implementation of TTS normalization patterns, the tts_norm_scan() and tts_norm_sub() template functions.
+
+The specification is in docs/tts-norm-patterns.md. This file must give the same result as the C++ implementation
+(common/chat-tts-norm.cpp) for every input; tests/test-jinja.cpp checks this in python mode (test-jinja -py).
+
+Usage with jinja2:
+
+ env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan
+ env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+# character classes, explicit codepoint ranges (keep in sync with common/chat-tts-norm.cpp)
+
+TTS_NORM_SPACE = [
+ (0x0009, 0x000D), (0x0020, 0x0020), (0x0085, 0x0085), (0x00A0, 0x00A0), (0x1680, 0x1680),
+ (0x2000, 0x200A), (0x2028, 0x2029), (0x202F, 0x202F), (0x205F, 0x205F), (0x3000, 0x3000),
+]
+
+TTS_NORM_PUNCT = [
+ (0x0021, 0x002F), (0x003A, 0x0040), (0x005B, 0x0060), (0x007B, 0x007E),
+ (0x00A1, 0x00A9), (0x00AB, 0x00B4), (0x00B6, 0x00B9), (0x00BB, 0x00BF), (0x00D7, 0x00D7), (0x00F7, 0x00F7),
+ (0x2010, 0x2027), (0x2030, 0x205E), (0x20A0, 0x20CF), (0x2100, 0x218F), (0x2190, 0x2BFF),
+ (0x3001, 0x3003), (0x3008, 0x3020), (0x3030, 0x3030), (0x303D, 0x303D), (0x30FB, 0x30FB),
+ (0xFE10, 0xFE19), (0xFE30, 0xFE4F), (0xFE50, 0xFE6B),
+ (0xFF01, 0xFF0F), (0xFF1A, 0xFF20), (0xFF3B, 0xFF40), (0xFF5B, 0xFF65),
+ (0x1F000, 0x1FAFF),
+]
+
+TTS_NORM_CONTROL = [
+ (0x0000, 0x001F), (0x007F, 0x009F), (0x200B, 0x200F), (0x2060, 0x206F), (0xFEFF, 0xFEFF),
+]
+
+TTS_NORM_HAN = [
+ (0x2E80, 0x2FDF), (0x3005, 0x3005), (0x3007, 0x3007), (0x3021, 0x3029), (0x3038, 0x303B),
+ (0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x20000, 0x323AF),
+]
+
+CLASS_LETTERS = "dspawulhc"
+INF = 1 << 31
+MAX_REPEAT = 1000
+MAX_PATTERN_LEN = 4096
+
+
+def _in(ranges: list[tuple[int, int]], c: int) -> bool:
+ return any(lo <= c <= hi for lo, hi in ranges)
+
+
+def _is_alpha(c: int) -> bool:
+ if ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z"):
+ return True
+ if c in (0xAA, 0xB5, 0xBA):
+ return True
+ return c >= 0xC0 and not _in(TTS_NORM_SPACE, c) and not _in(TTS_NORM_PUNCT, c) and not _in(TTS_NORM_CONTROL, c)
+
+
+def _class_match(kind: str, c: int) -> bool:
+ if kind == "d":
+ return ord("0") <= c <= ord("9")
+ if kind == "s":
+ return _in(TTS_NORM_SPACE, c)
+ if kind == "p":
+ return _in(TTS_NORM_PUNCT, c)
+ if kind == "a":
+ return _is_alpha(c)
+ if kind == "w":
+ return _is_alpha(c) or ord("0") <= c <= ord("9")
+ if kind == "u":
+ return ord("A") <= c <= ord("Z")
+ if kind == "l":
+ return ord("a") <= c <= ord("z")
+ if kind == "h":
+ return _in(TTS_NORM_HAN, c)
+ if kind == "c":
+ return _in(TTS_NORM_CONTROL, c)
+ return False
+
+
+def _ascii_lower(c: int) -> int:
+ return c + 32 if ord("A") <= c <= ord("Z") else c
+
+
+def _ascii_swapcase(c: int) -> int:
+ if ord("A") <= c <= ord("Z"):
+ return c + 32
+ if ord("a") <= c <= ord("z"):
+ return c - 32
+ return c
+
+
+def _is_ascii_alnum(c: int) -> bool:
+ return ord("0") <= c <= ord("9") or ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z")
+
+
+# AST
+
+LITERAL, SET, ANY, START, END, BOUNDARY, GROUP, LOOKAHEAD, LOOKBEHIND = range(9)
+ZERO_WIDTH = (START, END, BOUNDARY, LOOKAHEAD, LOOKBEHIND)
+
+
+@dataclass
+class _Set:
+ neg: bool = False
+ ranges: list[tuple[int, int]] = field(default_factory=list)
+ classes: list[tuple[str, bool]] = field(default_factory=list)
+
+
+@dataclass
+class _Node:
+ op: int = GROUP
+ cp: int = 0
+ set: _Set = field(default_factory=_Set)
+ alts: list[list[_Item]] = field(default_factory=list)
+ capture: int = -1
+ neg: bool = False
+
+
+@dataclass
+class _Item:
+ atom: _Node
+ min: int = 1
+ max: int = 1
+
+
+class PatternError(ValueError):
+ pass
+
+
+class _Parser:
+ def __init__(self, s: list[int]):
+ self.s = s
+ self.p = 0
+ self.n_caps = 0
+
+ def fail(self, msg: str):
+ raise PatternError(f"invalid TTS norm pattern: {msg} at position {self.p}")
+
+ def eof(self) -> bool:
+ return self.p >= len(self.s)
+
+ def peek_is(self, c: str) -> bool:
+ return not self.eof() and self.s[self.p] == ord(c)
+
+ def expect(self, c: str, what: str):
+ if not self.peek_is(c):
+ self.fail(f"expected {what}")
+ self.p += 1
+
+ def parse_alts(self) -> list[list[_Item]]:
+ alts = [self.parse_seq()]
+ while self.peek_is("|"):
+ self.p += 1
+ alts.append(self.parse_seq())
+ return alts
+
+ def parse_seq(self) -> list[_Item]:
+ seq: list[_Item] = []
+ while not self.eof() and self.s[self.p] not in (ord("|"), ord(")")):
+ item = _Item(self.parse_atom())
+ self.parse_quantifier(item)
+ seq.append(item)
+ return seq
+
+ def parse_int(self) -> int:
+ if self.eof() or not (ord("0") <= self.s[self.p] <= ord("9")):
+ self.fail("expected number in quantifier")
+ v = 0
+ while not self.eof() and ord("0") <= self.s[self.p] <= ord("9"):
+ v = v * 10 + self.s[self.p] - ord("0")
+ if v > MAX_REPEAT:
+ self.fail(f"repetition count exceeds {MAX_REPEAT}")
+ self.p += 1
+ return v
+
+ def parse_quantifier(self, item: _Item):
+ if self.eof():
+ return
+ c = chr(self.s[self.p])
+ if c not in "?*+{":
+ return
+ if item.atom.op in ZERO_WIDTH:
+ self.fail("zero-width assertion cannot be repeated")
+ self.p += 1
+ if c == "?":
+ item.min, item.max = 0, 1
+ elif c == "*":
+ item.min, item.max = 0, INF
+ elif c == "+":
+ item.min, item.max = 1, INF
+ else:
+ item.min = self.parse_int()
+ item.max = item.min
+ if self.peek_is(","):
+ self.p += 1
+ item.max = INF if self.peek_is("}") else self.parse_int()
+ self.expect("}", "'}' to close quantifier")
+ if item.max < item.min:
+ self.fail("quantifier max is smaller than min")
+ if not self.eof() and chr(self.s[self.p]) in "?*+{":
+ self.fail("multiple quantifiers")
+
+ def parse_group(self) -> _Node:
+ node = _Node(op=GROUP)
+ if self.peek_is("?"):
+ self.p += 1
+ if self.eof():
+ self.fail("unterminated group")
+ c = chr(self.s[self.p])
+ if c == ":":
+ self.p += 1
+ elif c in "=!":
+ self.p += 1
+ node.op = LOOKAHEAD
+ node.neg = c == "!"
+ elif c == "<" and self.p + 1 < len(self.s) and self.s[self.p + 1] in (ord("="), ord("!")):
+ node.op = LOOKBEHIND
+ node.neg = self.s[self.p + 1] == ord("!")
+ self.p += 2
+ if self.eof() or self.s[self.p] == ord(")"):
+ self.fail("lookbehind must contain exactly one codepoint matcher")
+ atom = self.parse_atom()
+ if atom.op not in (LITERAL, SET, ANY):
+ self.fail("lookbehind must contain exactly one codepoint matcher")
+ if not self.peek_is(")"):
+ self.fail("lookbehind must contain exactly one codepoint matcher")
+ self.p += 1
+ node.alts = [[_Item(atom)]]
+ return node
+ elif c == "i":
+ self.fail("(?i) is only allowed at the start of the pattern")
+ else:
+ self.fail("unknown group type")
+ else:
+ node.capture = self.n_caps
+ self.n_caps += 1
+ node.alts = self.parse_alts()
+ self.expect(")", "')' to close group")
+ return node
+
+ # returns ("class", (kind, neg)) | ("boundary", None) | ("literal", cp)
+ def parse_escape(self, in_set: bool) -> tuple[str, Any]:
+ if self.eof():
+ self.fail("dangling '%'")
+ e = self.s[self.p]
+ self.p += 1
+ if e < 0x80 and chr(e).lower() in CLASS_LETTERS:
+ return "class", (chr(e).lower(), ord("A") <= e <= ord("Z"))
+ if e == ord("b") and not in_set:
+ return "boundary", None
+ if _is_ascii_alnum(e):
+ self.p -= 1
+ self.fail(f"unknown escape '%{chr(e)}'")
+ return "literal", e
+
+ def parse_set(self) -> _Node:
+ node = _Node(op=SET)
+ if self.peek_is("^"):
+ node.set.neg = True
+ self.p += 1
+ empty = True
+ while True:
+ if self.eof():
+ self.fail("unterminated set")
+ if self.s[self.p] == ord("]"):
+ self.p += 1
+ break
+ empty = False
+ if self.s[self.p] == ord("%"):
+ self.p += 1
+ kind, val = self.parse_escape(True)
+ if kind == "class":
+ node.set.classes.append(val)
+ continue
+ lo = val
+ else:
+ lo = self.s[self.p]
+ self.p += 1
+ hi = lo
+ if self.peek_is("-") and self.p + 1 < len(self.s) and self.s[self.p + 1] != ord("]"):
+ self.p += 1
+ if self.s[self.p] == ord("%"):
+ self.p += 1
+ kind, val = self.parse_escape(True)
+ if kind == "class":
+ self.fail("character class cannot be used in a range")
+ hi = val
+ else:
+ hi = self.s[self.p]
+ self.p += 1
+ if hi < lo:
+ self.fail("invalid range in set")
+ node.set.ranges.append((lo, hi))
+ if empty:
+ self.fail("empty set")
+ return node
+
+ def parse_atom(self) -> _Node:
+ c = chr(self.s[self.p])
+ self.p += 1
+ if c == "(":
+ return self.parse_group()
+ if c == "[":
+ return self.parse_set()
+ if c == ".":
+ return _Node(op=ANY)
+ if c == "^":
+ return _Node(op=START)
+ if c == "$":
+ return _Node(op=END)
+ if c == "%":
+ kind, val = self.parse_escape(False)
+ if kind == "class":
+ node = _Node(op=SET)
+ node.set.classes.append(val)
+ return node
+ if kind == "boundary":
+ return _Node(op=BOUNDARY)
+ return _Node(op=LITERAL, cp=val)
+ if c in "?*+{":
+ self.p -= 1
+ self.fail("nothing to repeat")
+ if c in "]}":
+ self.p -= 1
+ self.fail(f"unescaped '{c}', use '%{c}'")
+ return _Node(op=LITERAL, cp=ord(c))
+
+
+class _Matcher:
+ def __init__(self, t: list[int], icase: bool, n_groups: int):
+ self.t = t
+ self.icase = icase
+ self.caps: list[tuple[int, int]] = [(-1, -1)] * n_groups
+
+ def is_w(self, i: int) -> bool:
+ return i < len(self.t) and _class_match("w", self.t[i])
+
+ def set_match(self, st: _Set, c: int) -> bool:
+ def test(x: int) -> bool:
+ if any(lo <= x <= hi for lo, hi in st.ranges):
+ return True
+ return any(_class_match(kind, x) != neg for kind, neg in st.classes)
+
+ m = test(c)
+ if not m and self.icase:
+ o = _ascii_swapcase(c)
+ if o != c:
+ m = test(o)
+ return m != st.neg
+
+ def match_alts(self, alts: list[list[_Item]], i: int) -> int:
+ """returns the end position, or -1 if no alternative matches"""
+ for seq in alts:
+ saved = list(self.caps)
+ end = self.match_seq(seq, i)
+ if end >= 0:
+ return end
+ self.caps = saved
+ return -1
+
+ def match_seq(self, seq: list[_Item], i: int) -> int:
+ pos = i
+ for it in seq:
+ count = 0
+ while count < it.max:
+ is_group = it.atom.op == GROUP
+ saved = list(self.caps) if is_group else None
+ e = self.match_once(it.atom, pos)
+ if e < 0:
+ if saved is not None:
+ self.caps = saved
+ break
+ if e == pos:
+ # an empty iteration would repeat forever, it satisfies any remaining minimum
+ count = max(count + 1, it.min)
+ break
+ pos = e
+ count += 1
+ if count < it.min:
+ return -1
+ return pos
+
+ def match_once(self, node: _Node, i: int) -> int:
+ """returns the end position, or -1 on failure"""
+ n = len(self.t)
+ op = node.op
+ if op == LITERAL:
+ if i < n and (self.t[i] == node.cp or (self.icase and _ascii_lower(self.t[i]) == _ascii_lower(node.cp))):
+ return i + 1
+ return -1
+ if op == SET:
+ return i + 1 if i < n and self.set_match(node.set, self.t[i]) else -1
+ if op == ANY:
+ return i + 1 if i < n else -1
+ if op == START:
+ return i if i == 0 else -1
+ if op == END:
+ return i if i == n else -1
+ if op == BOUNDARY:
+ return i if (i > 0 and self.is_w(i - 1)) != self.is_w(i) else -1
+ if op == GROUP:
+ e = self.match_alts(node.alts, i)
+ if e < 0:
+ return -1
+ if node.capture >= 0:
+ self.caps[node.capture] = (i, e)
+ return e
+ if op == LOOKAHEAD:
+ # captures made inside a lookahead are discarded
+ saved = list(self.caps)
+ ok = self.match_alts(node.alts, i) >= 0
+ self.caps = saved
+ return i if ok != node.neg else -1
+ if op == LOOKBEHIND:
+ ok = i > 0 and self.match_once(node.alts[0][0].atom, i - 1) == i
+ return i if ok != node.neg else -1
+ return -1
+
+
+class Pattern:
+ def __init__(self, src: str):
+ if len(src.encode("utf-8")) > MAX_PATTERN_LEN:
+ raise PatternError(f"invalid TTS norm pattern: pattern is longer than {MAX_PATTERN_LEN} bytes")
+ self.icase = False
+ parser = _Parser([ord(c) for c in src])
+ if src.startswith("(?i)"):
+ self.icase = True
+ parser.p = 4
+ self.root = _Node(op=GROUP, alts=parser.parse_alts())
+ if not parser.eof():
+ parser.fail("unbalanced ')'")
+ self.n_caps = parser.n_caps
+
+ def match_at(self, t: list[int], pos: int) -> tuple[int, list[tuple[int, int]]] | None:
+ m = _Matcher(t, self.icase, self.n_caps)
+ end = m.match_alts(self.root.alts, pos)
+ if end < 0:
+ return None
+ return end, m.caps
+
+
+# (rule index or -1, start, end, groups) with codepoint offsets
+Segment = tuple[int, int, int, list[tuple[int, int]]]
+
+
+def scan_segments(text: str, rules: list[Pattern]) -> list[Segment]:
+ t = [ord(c) for c in text]
+ n = len(t)
+ out: list[Segment] = []
+ i = 0
+ pending = 0
+ while i < n:
+ matched = False
+ for r, rule in enumerate(rules):
+ res = rule.match_at(t, i)
+ if res is not None and res[0] > i:
+ end, groups = res
+ if pending < i:
+ out.append((-1, pending, i, []))
+ out.append((r, i, end, groups))
+ i = end
+ pending = i
+ matched = True
+ break
+ if not matched:
+ i += 1
+ if pending < n:
+ out.append((-1, pending, n, []))
+ return out
+
+
+def _rules_from_value(rules: Any) -> list[tuple[str, str]]:
+ if isinstance(rules, str):
+ return [("match", rules)]
+ if isinstance(rules, dict):
+ out = []
+ for k, v in rules.items():
+ if not isinstance(v, str):
+ raise TypeError("tts_norm_scan: rule pattern must be a string")
+ out.append((str(k), v))
+ return out
+ if isinstance(rules, (list, tuple)):
+ out = []
+ for item in rules:
+ if not isinstance(item, (list, tuple)) or len(item) != 2 or not isinstance(item[0], str) or not isinstance(item[1], str):
+ raise TypeError("tts_norm_scan: each rule must be a [name, pattern] pair of strings")
+ out.append((item[0], item[1]))
+ return out
+ raise TypeError("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs")
+
+
+def tts_norm_scan(text: str, rules: Any) -> list[dict[str, Any]]:
+ rule_srcs = _rules_from_value(rules)
+ patterns = [Pattern(src) for _, src in rule_srcs]
+ out: list[dict[str, Any]] = []
+ for r, start, end, groups in scan_segments(text, patterns):
+ # groups that did not participate are empty strings, so they render as nothing and are falsy
+ vals = ["" if gs < 0 else text[gs:ge] for gs, ge in groups]
+ out.append({"rule": rule_srcs[r][0] if r >= 0 else None, "text": text[start:end], "groups": vals})
+ return out
+
+
+def _parse_repl(repl: str, pattern: Pattern) -> list[str | int]:
+ pieces: list[str | int] = []
+ lit = ""
+ i = 0
+ while i < len(repl):
+ c = repl[i]
+ if c != "%":
+ lit += c
+ i += 1
+ continue
+ if i + 1 >= len(repl):
+ raise PatternError("invalid TTS norm replacement: dangling '%'")
+ i += 1
+ c = repl[i]
+ if c == "%":
+ lit += "%"
+ elif "0" <= c <= "9":
+ g = ord(c) - ord("0")
+ if g > pattern.n_caps:
+ raise PatternError(f"invalid TTS norm replacement: group %{c} does not exist")
+ if lit:
+ pieces.append(lit)
+ lit = ""
+ pieces.append(g)
+ else:
+ raise PatternError(f"invalid TTS norm replacement: unknown escape '%{c}'")
+ i += 1
+ if lit:
+ pieces.append(lit)
+ return pieces
+
+
+def tts_norm_sub(text: str, pattern: str, repl: str) -> str:
+ pat = Pattern(pattern)
+ pieces = _parse_repl(repl, pat)
+ out = []
+ for r, start, end, groups in scan_segments(text, [pat]):
+ if r < 0:
+ out.append(text[start:end])
+ continue
+ for piece in pieces:
+ if isinstance(piece, str):
+ out.append(piece)
+ elif piece == 0:
+ out.append(text[start:end])
+ else:
+ gs, ge = groups[piece - 1]
+ if gs >= 0:
+ out.append(text[gs:ge])
+ return "".join(out)
diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp
index ce5a9749b..29de01ea9 100644
--- a/tests/test-jinja.cpp
+++ b/tests/test-jinja.cpp
@@ -3,6 +3,7 @@
#include <random>
#include <cstdlib>
+#include "chat-tts-norm.h"
#include "json.h"
#include "subproc.h"
@@ -16,8 +17,8 @@
using json = common_json;
-static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect);
-static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect);
+static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm = false);
+static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm = false);
static void test_whitespace_control(testing & t);
static void test_conditionals(testing & t);
@@ -39,6 +40,7 @@ static void test_stats(testing & t);
static void test_caps(testing & t);
static void test_string_parts(testing & t);
static void test_fuzzing(testing & t);
+static void test_tts_norm_patterns(testing & t);
static bool g_python_mode = false;
@@ -75,6 +77,7 @@ int main(int argc, char *argv[]) {
t.test("string methods", test_string_methods);
t.test("array methods", test_array_methods);
t.test("object methods", test_object_methods);
+ t.test("tts norm patterns", test_tts_norm_patterns);
if (!g_python_mode) {
t.test("hasher", test_hasher);
t.test("stats", test_stats);
@@ -2713,13 +2716,134 @@ static void test_string_parts(testing & t) {
}
-static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect) {
- t.test(name, [&tmpl, &expect](testing & t) {
+// TTS normalization patterns: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md
+// these functions are only available to task templates (e.g. "tts"), so every test here passes tts_norm = true
+static void test_tts_norm_patterns(testing & t) {
+ test_template(t, "not available without tts norm builtins",
+ "{{ tts_norm_scan is defined }}",
+ json::object(),
+ "False");
+
+ test_template(t, "tts_norm_scan splits text into matched and unmatched segments",
+ "{%- for s in tts_norm_scan(text, [['num', '%d+'], ['word', '%a+']]) -%}[{{ s.rule or '-' }}:{{ s.text }}]{%- endfor -%}",
+ {{"text", "abc 123, x9"}},
+ "[word:abc][-: ][num:123][-:, ][word:x][num:9]", true);
+
+ test_template(t, "tts_norm_scan groups, optional group that did not match is empty",
+ "{%- for s in tts_norm_scan(text, [['money', '%$(%d+)(?:%.(%d%d))?']]) -%}"
+ "{%- if s.rule -%}{%- set whole, cents = s.groups -%}{{ whole }}|{{ cents }}|{{ s.groups | length }};{%- endif -%}"
+ "{%- endfor -%}",
+ {{"text", "$3.50 and $7"}},
+ "3|50|2;7||2;", true);
+
+ test_template(t, "tts_norm_scan rules as dict, the first rule that matches wins",
+ "{%- for s in tts_norm_scan(text, {'ordinal': '%d+(?:st|nd|rd|th)%b', 'number': '%d+'}) -%}"
+ "{%- if s.rule -%}<{{ s.rule }}:{{ s.text }}>{%- else -%}{{ s.text }}{%- endif -%}"
+ "{%- endfor -%}",
+ {{"text", "the 3rd of 10 or 4three"}},
+ "the <ordinal:3rd> of <number:10> or <number:4>three", true);
+
+ // unlike regex, a failed sequence does not retry the other branches: (?:a|ab)c never matches "abc"
+ test_template(t, "ordered alternation commits to the first branch that matches",
+ "{%- for s in tts_norm_scan(text, '(?:a|ab)c') -%}[{{ s.rule or '-' }}:{{ s.text }}]{%- endfor -%}",
+ {{"text", "abc ac"}},
+ "[-:abc ][match:ac]", true);
+
+ // unlike regex, quantifiers never give back what they consumed: %d+0 never matches
+ test_template(t, "quantifiers are possessive",
+ "{{ tts_norm_sub(text, '%d+0', 'X') }}|{{ tts_norm_sub(text, '%d+', 'X') }}",
+ {{"text", "100 20"}},
+ "100 20|X X", true);
+
+ test_template(t, "quantifier ranges",
+ "{{ tts_norm_sub(text, '%d{3}-%d{2,}', '#') }}",
+ {{"text", "123-4 123-45 123-456789"}},
+ "123-4 # #", true);
+
+ test_template(t, "lookbehind: a minus sign is not part of a word",
+ "{{ tts_norm_sub(text, '(?<!%w)-(%d+)', 'minus %1') }}",
+ {{"text", "GPT-5 is -5, (-3)"}},
+ "GPT-5 is minus 5, (minus 3)", true);
+
+ test_template(t, "lookahead: clock time not followed by a digit",
+ "{{ tts_norm_sub(text, '(%d%d?):(%d%d)(?!%d)', '%1h%2') }}",
+ {{"text", "at 10:30 or 1:234"}},
+ "at 10h30 or 1:234", true);
+
+ test_template(t, "tts_norm_sub references: group, whole match and literal percent",
+ "{{ tts_norm_sub(text, '(%d+)%%', '%1 percent (%0) 100%%') }}",
+ {{"text", "up 50%"}},
+ "up 50 percent (50%) 100%", true);
+
+ test_template(t, "split letters and digits, non-ASCII letters included",
+ "{{ tts_norm_sub(tts_norm_sub(text, '(%a)(%d)', '%1 %2'), '(%d)(%a)', '%1 %2') }}",
+ {{"text", "R2D2 café2 4K"}},
+ "R 2 D 2 café 2 4 K", true);
+
+ test_template(t, "case-insensitive pattern and word boundary",
+ "{{ tts_norm_sub(text, '(?i)%bdr%.', 'doctor') }}",
+ {{"text", "Dr. Who and dr. No, not Mdr."}},
+ "doctor Who and doctor No, not Mdr.", true);
+
+ test_template(t, "sets with multi-byte codepoints, negated sets with classes",
+ "{{ tts_norm_sub(text, '[$€£]%s*(%d+)', '<%1>') }}|{{ tts_norm_sub(text, '[^%w%s]', '') }}",
+ {{"text", "€ 5 & $6!"}},
+ "<5> & <6>!| 5 6", true);
+
+ test_template(t, "remove spaces between Han characters",
+ "{{ tts_norm_sub(text, '(%h) +(%h)', '%1%2') }}",
+ {{"text", "你 好 world 世界"}},
+ "你好 world 世界", true);
+
+ test_template(t, "mini text normalizer",
+ "{%- set ones = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten',"
+ " 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'] -%}\n"
+ "{%- set tens = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'] -%}\n"
+ "{%- macro num(n) -%}{{ ones[n] if n < 20 else tens[(n / 10) | int] ~ ('-' ~ ones[n % 10] if n % 10 else '') }}{%- endmacro -%}\n"
+ "{%- for s in tts_norm_scan(text, [\n"
+ " ['money', '%$(%d+)(?:%.(%d%d))?'],\n"
+ " ['percent', '(%d+)%%'],\n"
+ " ['time', '(%d%d?):(%d%d)(?!%d)'],\n"
+ " ['neg', '(?<!%w)-(%d+)'],\n"
+ " ['number', '%d+'],\n"
+ "]) -%}\n"
+ "{%- if s.rule == 'money' -%}{%- set whole, cents = s.groups -%}{{ num(whole | int) }} dollars{{ ' and ' ~ num(cents | int) ~ ' cents' if cents else '' }}\n"
+ "{%- elif s.rule == 'percent' -%}{{ num(s.groups[0] | int) }} percent\n"
+ "{%- elif s.rule == 'time' -%}{%- set h, m = s.groups -%}{{ num(h | int) }} {{ num(m | int) }}\n"
+ "{%- elif s.rule == 'neg' -%}minus {{ num(s.groups[0] | int) }}\n"
+ "{%- elif s.rule == 'number' -%}{{ num(s.text | int) }}\n"
+ "{%- else -%}{{ s.text }}\n"
+ "{%- endif -%}\n"
+ "{%- endfor -%}",
+ {{"text", "It costs $3.50, 20% off, at 10:45, -5 degrees, 7 cats."}},
+ "It costs three dollars and fifty cents, twenty percent off, at ten forty-five, minus five degrees, seven cats.", true);
+
+ if (!g_python_mode) {
+ test_template_error(t, "error: unclosed group", "{{ tts_norm_scan('x', '(a') }}", "expected ')' to close group", true);
+ test_template_error(t, "error: unbalanced parenthesis", "{{ tts_norm_scan('x', 'a)') }}", "unbalanced ')'", true);
+ test_template_error(t, "error: nothing to repeat", "{{ tts_norm_scan('x', '*a') }}", "nothing to repeat", true);
+ test_template_error(t, "error: multiple quantifiers", "{{ tts_norm_scan('x', 'a+?') }}", "multiple quantifiers", true);
+ test_template_error(t, "error: bad quantifier range", "{{ tts_norm_scan('x', 'a{2,1}') }}", "quantifier max is smaller than min", true);
+ test_template_error(t, "error: repeated assertion", "{{ tts_norm_scan('x', '%b+') }}", "zero-width assertion cannot be repeated", true);
+ test_template_error(t, "error: unknown class", "{{ tts_norm_scan('x', '%q') }}", "unknown escape '%q'", true);
+ test_template_error(t, "error: unescaped bracket", "{{ tts_norm_scan('x', 'a]') }}", "unescaped ']'", true);
+ test_template_error(t, "error: no named groups", "{{ tts_norm_scan('x', '(?<n>a)') }}", "unknown group type", true);
+ test_template_error(t, "error: wide lookbehind", "{{ tts_norm_scan('x', '(?<=ab)c') }}", "lookbehind must contain exactly one codepoint matcher", true);
+ test_template_error(t, "error: missing group in replacement", "{{ tts_norm_sub('x', '(a)', '%2') }}", "group %2 does not exist", true);
+ test_template_error(t, "error: bad rules", "{{ tts_norm_scan('x', 1) }}", "rules must be a string, a dict or a list", true);
+ }
+}
+
+static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm) {
+ t.test(name, [&tmpl, &expect, tts_norm](testing & t) {
std::string error;
try {
jinja::lexer lexer;
jinja::program ast = jinja::parse_from_tokens(lexer.tokenize(tmpl));
jinja::context ctx(tmpl);
+ if (tts_norm) {
+ common_chat_tts_norm_add_builtins(ctx);
+ }
jinja::runtime runtime(ctx);
runtime.gather_string_parts(runtime.execute(ast));
} catch (const std::exception & e) {
@@ -2729,14 +2853,17 @@ static void test_template_error(testing & t, const std::string & name, const std
});
}
-static void test_template_cpp(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) {
- t.test(name, [&tmpl, &vars, &expect](testing & t) {
+static void test_template_cpp(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) {
+ t.test(name, [&tmpl, &vars, &expect, tts_norm](testing & t) {
jinja::lexer lexer;
auto lexer_res = lexer.tokenize(tmpl);
jinja::program ast = jinja::parse_from_tokens(lexer_res);
jinja::context ctx(tmpl);
+ if (tts_norm) {
+ common_chat_tts_norm_add_builtins(ctx);
+ }
jinja::global_from_json(ctx, vars, true);
jinja::runtime runtime(ctx);
@@ -2788,17 +2915,36 @@ env.filters["tojson"] = lambda x, ensure_ascii=False, indent=None, separators=No
env.globals["strftime_now"] = lambda format: datetime.now().strftime(format)
env.globals["raise_exception"] = raise_exception
+if merged_input.get("tts_norm_ref"):
+ import importlib.util
+ spec = importlib.util.spec_from_file_location("chat_tts_norm", merged_input["tts_norm_ref"])
+ chat_tts_norm = importlib.util.module_from_spec(spec)
+ sys.modules["chat_tts_norm"] = chat_tts_norm # required by dataclasses
+ spec.loader.exec_module(chat_tts_norm)
+ env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan
+ env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub
+
template = env.from_string(tmpl)
result = template.render(**vars_json)
sys.stdout.buffer.write(result.encode())
)";
-static void test_template_py(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) {
- t.test(name, [&tmpl, &vars, &expect](testing & t) {
+// python reference of the TTS normalization patterns, relative to this file
+static std::string tts_norm_ref_path() {
+ std::string path = __FILE__;
+ const size_t pos = path.find_last_of("/\\");
+ return (pos == std::string::npos ? std::string(".") : path.substr(0, pos)) + "/../scripts/chat_tts_norm.py";
+}
+
+static void test_template_py(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) {
+ t.test(name, [&tmpl, &vars, &expect, tts_norm](testing & t) {
// Prepare arguments
json merged;
merged["tmpl"] = json(tmpl);
merged["vars"] = vars;
+ if (tts_norm) {
+ merged["tts_norm_ref"] = tts_norm_ref_path();
+ }
#ifdef _WIN32
const char * python_executable = "python.exe";
@@ -2859,11 +3005,11 @@ static void test_template_py(testing & t, const std::string & name, const std::s
});
}
-static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) {
+static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) {
if (g_python_mode) {
- test_template_py(t, name, tmpl, vars, expect);
+ test_template_py(t, name, tmpl, vars, expect, tts_norm);
} else {
- test_template_cpp(t, name, tmpl, vars, expect);
+ test_template_cpp(t, name, tmpl, vars, expect, tts_norm);
}
}