Commit 8d1599fca for llama.cpp
commit 8d1599fca1f78a8d6f9187e0f578db9a818b4f1c
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Tue Nov 17 12:52:50 2026 +0100
chat: add tts_dict for kokoro G2P template (#32397)
* chat: add tts_dict for kokoro G2P template
* more testing
* value_float_t range check
* no json-in-meta, add llama_model_meta_arr_str/_n
diff --git a/common/chat-tts-norm.cpp b/common/chat-tts-norm.cpp
index f84540ff9..2ec93a204 100644
--- a/common/chat-tts-norm.cpp
+++ b/common/chat-tts-norm.cpp
@@ -2,6 +2,7 @@
#include "jinja/runtime.h"
#include "jinja/value.h"
+#include "json.h"
#include <algorithm>
#include <climits>
@@ -857,8 +858,35 @@ static value tts_norm_builtin_sub(const func_args & args) {
return mk_val<value_string>(string(res, text_val.all_parts_are_input()));
}
-void common_chat_tts_norm_add_builtins(context & ctx) {
+bool common_tts_dict_find(const common_tts_dict & dict, const std::string & key, std::string & value) {
+ const auto it = std::lower_bound(dict.keys.begin(), dict.keys.end(), key);
+ if (it == dict.keys.end() || *it != key) {
+ return false;
+ }
+ const size_t idx = it - dict.keys.begin();
+ if (idx >= dict.values.size()) {
+ throw std::runtime_error("tts_dict: malformed dictionary, a key has no value");
+ }
+ value = dict.values[idx];
+ return true;
+}
+
+void common_chat_tts_norm_add_builtins(context & ctx, std::shared_ptr<const common_tts_dicts> dicts) {
ctx.set_val("tts_norm_scan", mk_val<value_func>("tts_norm_scan", tts_norm_builtin_scan));
ctx.set_val("tts_norm_sub", mk_val<value_func>("tts_norm_sub", tts_norm_builtin_sub));
+ ctx.set_val("tts_dict", mk_val<value_func>("tts_dict", [dicts](const func_args & args) -> value {
+ args.ensure_count(2, 2);
+ args.ensure_vals<value_string, value_string>();
+ const std::string name = args.get_pos(0)->as_string().str();
+ const auto it = dicts ? dicts->find(name) : common_tts_dicts::const_iterator();
+ if (!dicts || it == dicts->end()) {
+ throw std::runtime_error("tts_dict: unknown dictionary '" + name + "'");
+ }
+ std::string doc;
+ if (!common_tts_dict_find(it->second, args.get_pos(1)->as_string().str(), doc)) {
+ return mk_val<value_none>();
+ }
+ return value_from_json(common_json::parse(doc), false);
+ }));
}
diff --git a/common/chat-tts-norm.h b/common/chat-tts-norm.h
index 14695bb52..84b16018c 100644
--- a/common/chat-tts-norm.h
+++ b/common/chat-tts-norm.h
@@ -1,6 +1,7 @@
#pragma once
#include <cstdint>
+#include <map>
#include <memory>
#include <string>
#include <utility>
@@ -11,9 +12,10 @@
// it is NOT a regex engine: the semantics are defined by llama.cpp (see docs/tts-norm-patterns.md), so that this
// implementation and the python reference (scripts/chat_tts_norm.py) give the same result for every input
//
-// two functions are exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
+// functions exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
// tts_norm_scan(text, rules) -> list of segments {rule, text, groups}
// tts_norm_sub(text, pattern, repl) -> string
+// tts_dict(name, key) -> value of key in a dictionary of the model, or none
namespace jinja {
struct context;
@@ -57,6 +59,17 @@ std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & te
// repl references: %0 (whole match), %1..%9 (groups), %% (literal percent)
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl);
-// add tts_norm_scan() and tts_norm_sub() to the context of a task template
+// a dictionary of the model, e.g. a pronunciation lexicon
+struct common_tts_dict {
+ std::vector<std::string> keys; // sorted by their bytes
+ std::vector<std::string> values; // the JSON document of each key
+};
+
+using common_tts_dicts = std::map<std::string, common_tts_dict>;
+
+// binary search of key in a dictionary; on success, value is set to the JSON document
+bool common_tts_dict_find(const common_tts_dict & dict, const std::string & key, std::string & value);
+
+// add tts_norm_scan(), tts_norm_sub() and tts_dict() to the context of a task template
// they are not jinja global builtins on purpose: chat templates are not affected
-void common_chat_tts_norm_add_builtins(jinja::context & ctx);
+void common_chat_tts_norm_add_builtins(jinja::context & ctx, std::shared_ptr<const common_tts_dicts> dicts = nullptr);
diff --git a/common/chat.cpp b/common/chat.cpp
index 863f31ebe..4816973c4 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -318,6 +318,7 @@ struct common_chat_templates {
std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)
std::unique_ptr<common_chat_template> template_tool_use;
std::unique_ptr<common_chat_template> template_tts; // the prompt of a text to speak
+ std::shared_ptr<common_tts_dicts> tts_dicts; // dictionaries of the model for tts_dict(), by name
};
common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {
@@ -806,10 +807,10 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
}
// renders a task template (e.g. "tts") with the given variables
-static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) {
+static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars, const std::shared_ptr<common_tts_dicts> & tts_dicts) {
jinja::context ctx(tmpl.source());
- // text normalization helpers: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md
- common_chat_tts_norm_add_builtins(ctx);
+ // text normalization helpers: tts_norm_scan(), tts_norm_sub() and tts_dict(), see docs/tts-norm-patterns.md
+ common_chat_tts_norm_add_builtins(ctx, tts_dicts);
jinja::global_from_json(ctx, vars, false);
ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
@@ -893,7 +894,7 @@ std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templa
{"messages", json::array({{{"role", "user"}, {"content", text}}})},
{"language", language},
{"voice", voice},
- });
+ }, chat_templates->tts_dicts);
}
std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
@@ -1024,6 +1025,25 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
LOG_ERR("%s: failed to parse the tts chat template (ignoring it): %s\n", __func__, e.what());
}
}
+ // dictionaries for tts_dict(), listed by name in tokenizer.tts_dicts
+ tmpls->tts_dicts = std::make_shared<common_tts_dicts>();
+ for (int32_t i = 0; i < llama_model_meta_arr_n(model, "tokenizer.tts_dicts"); i++) {
+ const std::string name = llama_model_meta_arr_str(model, "tokenizer.tts_dicts", i);
+ const std::string key = "tokenizer.tts_dict." + name + ".keys";
+ const std::string val_key = "tokenizer.tts_dict." + name + ".values";
+ const int32_t n = llama_model_meta_arr_n(model, key.c_str());
+ if (n < 0 || llama_model_meta_arr_n(model, val_key.c_str()) != n) {
+ LOG_ERR("%s: the tts dictionary '%s' has no value for each of its keys (ignoring it)\n", __func__, name.c_str());
+ continue;
+ }
+ auto & dict = (*tmpls->tts_dicts)[name];
+ dict.keys.reserve(n);
+ dict.values.reserve(n);
+ for (int32_t j = 0; j < n; j++) {
+ dict.keys.emplace_back(llama_model_meta_arr_str(model, key.c_str(), j));
+ dict.values.emplace_back(llama_model_meta_arr_str(model, val_key.c_str(), j));
+ }
+ }
}
return tmpls;
}
diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp
index 1eae8d923..433707d71 100644
--- a/common/jinja/runtime.cpp
+++ b/common/jinja/runtime.cpp
@@ -191,7 +191,7 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
if (is_float) {
return mk_val<value_float>(res);
} else {
- return mk_val<value_int>(static_cast<int64_t>(res));
+ return mk_val<value_int>(float_to_int(res));
}
} else if (op.value == "/") {
JJ_DEBUG("Division operation: %f / %f", a, b);
@@ -222,7 +222,7 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
if (is_float) {
return mk_val<value_float>(rem);
} else {
- return mk_val<value_int>(static_cast<int64_t>(rem));
+ return mk_val<value_int>(float_to_int(rem));
}
} else if (op.value == "<") {
JJ_DEBUG("Comparison operation: %f < %f is %d", a, b, a < b);
diff --git a/common/jinja/value.cpp b/common/jinja/value.cpp
index b3295edbe..9692d7cab 100644
--- a/common/jinja/value.cpp
+++ b/common/jinja/value.cpp
@@ -711,7 +711,7 @@ const func_builtins & value_float_t::get_builtins() const {
}},
{"int", [](const func_args & args) -> value {
args.ensure_vals<value_float>();
- int64_t val = static_cast<int64_t>(args.get_pos(0)->as_float());
+ int64_t val = float_to_int(args.get_pos(0)->as_float());
return mk_val<value_int>(val);
}},
{"float", [](const func_args & args) -> value {
@@ -1120,7 +1120,7 @@ static value array_sum(const func_args & args) {
is_float = is_float || is_val<value_float>(v);
sum += v->as_float();
}
- return is_float ? value(mk_val<value_float>(sum)) : value(mk_val<value_int>(static_cast<int64_t>(sum)));
+ return is_float ? value(mk_val<value_float>(sum)) : value(mk_val<value_int>(float_to_int(sum)));
}
const func_builtins & value_array_t::get_builtins() const {
@@ -1674,6 +1674,11 @@ bool value_compare(const value & a, const value & b, value_compare_op op) {
return result;
}
+template<>
+value value_from_json(const common_json & json_val, bool mark_input) {
+ return from_json(json_val, mark_input);
+}
+
template<>
void global_from_json(context & ctx, const common_json & json_obj, bool mark_input) {
// printf("global_from_json: %s\n" , json_obj.dump(2).c_str());
diff --git a/common/jinja/value.h b/common/jinja/value.h
index d39b41eac..6843bf57a 100644
--- a/common/jinja/value.h
+++ b/common/jinja/value.h
@@ -105,6 +105,10 @@ struct context; // forward declaration
template<typename T_JSON>
void global_from_json(context & ctx, const T_JSON & json_obj, bool mark_input);
+// convert a single JSON value, same rules as global_from_json()
+template<typename T_JSON>
+value value_from_json(const T_JSON & json_val, bool mark_input);
+
//
// base value type
//
@@ -233,6 +237,20 @@ struct value_equality {
// primitive value types
//
+// double to int64, saturating and with nan as 0; a plain cast is undefined outside the int64 range
+inline int64_t float_to_int(double v) {
+ if (std::isnan(v)) {
+ return 0;
+ }
+ if (v >= 9223372036854775808.0) {
+ return INT64_MAX;
+ }
+ if (v <= -9223372036854775808.0) {
+ return INT64_MIN;
+ }
+ return static_cast<int64_t>(v);
+}
+
struct value_int_t : public value_t {
static constexpr auto kind_id = value_kind::INTEGER;
int64_t val_int;
@@ -240,7 +258,7 @@ struct value_int_t : public value_t {
value_int_t(int64_t v) : value_t(kind_id) {
val_int = v;
val_flt = static_cast<double>(v);
- if (static_cast<int64_t>(val_flt) != v) {
+ if (float_to_int(val_flt) != v) {
val_flt = v < 0 ? -INFINITY : INFINITY;
}
}
@@ -276,7 +294,7 @@ struct value_float_t : public value_t {
double val_flt;
value_float_t(double v) : value_t(kind_id) {
val_flt = v;
- val_int = std::isfinite(v) ? static_cast<int64_t>(v) : 0;
+ val_int = std::isfinite(v) ? float_to_int(v) : 0;
}
virtual std::string type() const override { return "Float"; }
virtual double as_float() const override { return val_flt; }
diff --git a/docs/tts-norm-patterns.md b/docs/tts-norm-patterns.md
index 802dde5ce..563e8509a 100644
--- a/docs/tts-norm-patterns.md
+++ b/docs/tts-norm-patterns.md
@@ -7,6 +7,8 @@ Jinja has no regex support, and exposing `std::regex` / Python `re` would not he
- `tts_norm_scan(text, rules)`: split text into matched and unmatched segments
- `tts_norm_sub(text, pattern, repl)`: replace every match
+A third function, `tts_dict(name, key)`, gives the template access to dictionaries stored in the model (e.g. a pronunciation lexicon).
+
Implementations:
- C++: [common/chat-tts-norm.cpp](../common/chat-tts-norm.cpp), injected into task templates by `common_chat_template_render()` in [common/chat.cpp](../common/chat.cpp). They are not available to chat templates.
- Python reference: [scripts/chat_tts_norm.py](../scripts/chat_tts_norm.py), for use with `jinja2` (`env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan`)
@@ -69,6 +71,22 @@ Replaces every match of `pattern`, with the same scanning rules as `tts_norm_sca
Any other `%` sequence in `repl`, or a reference to a group that does not exist, is an error.
+### `tts_dict(name, key)`
+
+Looks up `key` in the dictionary `name` of the model and returns its value, or `none` if the key is missing. An unknown dictionary name is an error. A model can ship any number of dictionaries, for example a pronunciation lexicon per language:
+
+```jinja
+{%- set e = tts_dict('lexicon_en', word) -%}
+{%- if e is none -%}{{ spell(word) }}
+{%- elif e is mapping -%}{{ e[tag] if tag in e else e.DEFAULT }}
+{%- else -%}{{ e }}
+{%- endif -%}
+```
+
+The lookup is exact: no case folding and no normalization, do them in the template.
+
+The GGUF lists the names of the dictionaries in `tokenizer.tts_dicts`. Each one is stored as two arrays of strings of the same length: `tokenizer.tts_dict.<name>.keys`, sorted by the UTF-8 bytes of the key, and `tokenizer.tts_dict.<name>.values`. A value is a JSON document (string, number, list, object, ...) and is returned decoded. `GGUFWriter.add_tts_dicts(dicts)` writes all of this.
+
## Pattern syntax
Patterns operate on Unicode codepoints. Invalid UTF-8 bytes (C++ only) are seen as U+FFFD, one per byte, and preserved in the output.
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 3f13b83b2..395cad9b3 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -415,6 +415,9 @@ class Keys:
CHAT_TEMPLATE = "tokenizer.chat_template"
CHAT_TEMPLATE_N = "tokenizer.chat_template.{name}"
CHAT_TEMPLATES = "tokenizer.chat_templates"
+ TTS_DICTS = "tokenizer.tts_dicts"
+ TTS_DICT_KEYS_N = "tokenizer.tts_dict.{name}.keys"
+ TTS_DICT_VALUES_N = "tokenizer.tts_dict.{name}.values"
# Normalizer constants
NORMALIZER_NFC = "tokenizer.ggml.normalizer.nfc"
NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase"
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index edbe8cd09..edf2caf88 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import json
import logging
import os
import shutil
@@ -1458,6 +1459,15 @@ class GGUFWriter:
self.add_string(Keys.Tokenizer.CHAT_TEMPLATE, value)
+ def add_tts_dicts(self, dicts: Mapping[str, Mapping[str, Any]]) -> None:
+ # dictionaries for tts_dict(): keys sorted by their bytes, values as JSON
+ self.add_array(Keys.Tokenizer.TTS_DICTS, list(dicts))
+ for name, value in dicts.items():
+ keys = sorted(value, key=lambda k: k.encode("utf-8"))
+ self.add_array(Keys.Tokenizer.TTS_DICT_KEYS_N.format(name=name), keys)
+ self.add_array(Keys.Tokenizer.TTS_DICT_VALUES_N.format(name=name),
+ [json.dumps(value[k], ensure_ascii=False, separators=(",", ":")) for k in keys])
+
def add_suppress_tokens(self, tokens: Sequence[int]) -> None:
self.add_array(Keys.Tokenizer.SUPPRESS_TOKENS, tokens)
diff --git a/include/llama.h b/include/llama.h
index 3b2baa3bc..7122143e5 100644
--- a/include/llama.h
+++ b/include/llama.h
@@ -632,6 +632,17 @@ extern "C" {
// Get metadata value as a string by index
LLAMA_API int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);
+ // GGUF metadata arrays of strings
+ // currently, only these keys are support:
+ // - tokenizer.tts_dicts
+ // - tokenizer.tts_dict.*
+
+ // Get the number of elements, or -1 if there is no such array
+ LLAMA_API int32_t llama_model_meta_arr_n(const struct llama_model * model, const char * key);
+
+ // Get element i, owned by the model; nullptr if not found
+ LLAMA_API const char * llama_model_meta_arr_str(const struct llama_model * model, const char * key, int32_t i);
+
// Get a string describing the model type
LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);
diff --git a/scripts/chat_tts_norm.py b/scripts/chat_tts_norm.py
index a5fbd55b6..754232009 100644
--- a/scripts/chat_tts_norm.py
+++ b/scripts/chat_tts_norm.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""Reference implementation of TTS normalization patterns, the tts_norm_scan() and tts_norm_sub() template functions.
+"""Reference implementation of the template functions tts_norm_scan(), tts_norm_sub() (TTS normalization patterns) and tts_dict().
The specification is in docs/tts-norm-patterns.md. This file must give the same result as the C++ implementation
(common/chat-tts-norm.cpp) for every input; tests/test-jinja.cpp checks this in python mode (test-jinja -py).
@@ -8,12 +8,14 @@ Usage with jinja2:
env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan
env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub
+ env.globals["tts_dict"] = chat_tts_norm.make_tts_dict(dicts) # dicts: name -> {"keys", "values"} of tokenizer.tts_dict.<name>
"""
from __future__ import annotations
+import json
from dataclasses import dataclass, field
-from typing import Any
+from typing import Any, Callable
# character classes, explicit codepoint ranges (keep in sync with common/chat-tts-norm.cpp)
@@ -554,3 +556,23 @@ def tts_norm_sub(text: str, pattern: str, repl: str) -> str:
if gs >= 0:
out.append(text[gs:ge])
return "".join(out)
+
+
+def make_tts_dict(dicts: dict[str, dict[str, list[str]]]) -> Callable[[str, str], Any]:
+ # a dictionary is {"keys": [...], "values": [...]}, the values are JSON documents
+ parsed: dict[str, dict[str, str]] = {}
+
+ def tts_dict(name: str, key: str) -> Any:
+ if not isinstance(name, str) or not isinstance(key, str):
+ raise TypeError("tts_dict: name and key must be strings")
+ if name not in dicts:
+ raise KeyError(f"tts_dict: unknown dictionary '{name}'")
+ if name not in parsed:
+ keys, values = dicts[name]["keys"], dicts[name]["values"]
+ if len(keys) != len(values):
+ raise ValueError("tts_dict: malformed dictionary, a key has no value")
+ parsed[name] = dict(zip(keys, values))
+ doc = parsed[name].get(key)
+ return None if doc is None else json.loads(doc)
+
+ return tts_dict
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 9801b4b0c..66c51adde 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1471,6 +1471,20 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
for (int i = 0; i < gguf_get_n_kv(ctx); i++) {
gguf_type type = gguf_get_kv_type(ctx, i);
if (type == GGUF_TYPE_ARRAY) {
+ // only the string arrays listed here are kept, see llama_model_meta_arr_str()
+ static const char * const kept_prefixes[] = { "tokenizer.tts_dict" };
+ const std::string arr_name = gguf_get_key(ctx, i);
+ const bool is_kept = std::any_of(std::begin(kept_prefixes), std::end(kept_prefixes), [&](const char * prefix) {
+ return arr_name.rfind(prefix, 0) == 0;
+ });
+ if (is_kept && gguf_get_arr_type(ctx, i) == GGUF_TYPE_STRING) {
+ auto & arr = gguf_kv_arr[arr_name];
+ const size_t n = gguf_get_arr_n(ctx, i);
+ arr.reserve(n);
+ for (size_t j = 0; j < n; j++) {
+ arr.emplace_back(gguf_get_arr_str(ctx, i, j));
+ }
+ }
continue;
}
const char * name = gguf_get_key(ctx, i);
@@ -3944,6 +3958,22 @@ int32_t llama_model_meta_val_str_by_index(const llama_model * model, int32_t i,
return snprintf(buf, buf_size, "%s", it->second.c_str());
}
+int32_t llama_model_meta_arr_n(const llama_model * model, const char * key) {
+ const auto it = model->gguf_kv_arr.find(key);
+ if (it == model->gguf_kv_arr.end() || it->second.size() > (size_t) INT32_MAX) {
+ return -1;
+ }
+ return (int32_t) it->second.size();
+}
+
+const char * llama_model_meta_arr_str(const llama_model * model, const char * key, int32_t i) {
+ const auto it = model->gguf_kv_arr.find(key);
+ if (it == model->gguf_kv_arr.end() || i < 0 || (size_t) i >= it->second.size()) {
+ return nullptr;
+ }
+ return it->second[i].c_str();
+}
+
int32_t llama_model_desc(const llama_model * model, char * buf, size_t buf_size) {
return snprintf(buf, buf_size, "%s", model->desc().c_str());
}
diff --git a/src/llama-model.h b/src/llama-model.h
index 67250b789..19b4c4288 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -956,6 +956,9 @@ struct llama_model {
// gguf metadata
std::unordered_map<std::string, std::string> gguf_kv;
+ // the gguf metadata string arrays kept after load, see load_hparams()
+ std::unordered_map<std::string, std::vector<std::string>> gguf_kv_arr;
+
// list of devices used in this model
std::vector<llama_device> devices;
diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp
index 2fd33beeb..c9b416b58 100644
--- a/tests/test-jinja.cpp
+++ b/tests/test-jinja.cpp
@@ -45,12 +45,29 @@ static void test_caps(testing & t);
static void test_string_parts(testing & t);
static void test_fuzzing(testing & t);
static void test_tts_norm_patterns(testing & t);
+static void test_tts_dict(testing & t);
static void test_tts_template_zonos3(testing & t);
static void test_shared_render(testing & t);
static bool g_python_mode = false;
+// dictionaries given to tts_dict() in tests
+static std::shared_ptr<const common_tts_dicts> test_tts_dicts() {
+ static const auto dicts = std::make_shared<const common_tts_dicts>(common_tts_dicts{
+ {"lex", {
+ {"", "Zed", "big", "hello", "list", "null", "read", "w1", "w2", "zoo", "\xc3\xa9t\xc3\xa9", "\xe6\x97\xa5"},
+ {"\"empty\"", "\"z3d\"", "123456789012345678901234567890", "\"h@loU\"", "[1,2,null]", "null",
+ "{\"DEFAULT\":\"rid\",\"VBD\":\"rEd\"}", "{\"a\":1.5,\"b\":-2}", "{\"a\":0.25}", "\"zu\"", "\"ete\"", "\"ni\""},
+ }},
+ {"other", {{"hello"}, {"\"bonjour\""}}},
+ {"empty", {}},
+ // values that must be rejected: nested deeper than the engine allows, and not JSON
+ {"bad", {{"deep", "json"}, {std::string(200, '[') + std::string(200, ']'), "{not json"}}},
+ });
+ return dicts;
+}
+
int main(int argc, char *argv[]) {
testing t(std::cout);
t.verbose = true;
@@ -85,6 +102,7 @@ int main(int argc, char *argv[]) {
t.test("array methods", test_array_methods);
t.test("object methods", test_object_methods);
t.test("tts norm patterns", test_tts_norm_patterns);
+ t.test("tts dict", test_tts_dict);
t.test("tts template zonos3", test_tts_template_zonos3);
if (!g_python_mode) {
t.test("shared render", test_shared_render);
@@ -3041,6 +3059,113 @@ static void test_tts_norm_patterns(testing & t) {
}
}
+// tts_dict(name, key): dictionaries of the model, see docs/tts-norm-patterns.md
+// only available to task templates, like tts_norm_scan(); the dictionaries of test_tts_dicts() stand for the ones of a model
+static void test_tts_dict(testing & t) {
+ test_template(t, "tts_dict is defined for task templates",
+ "{{ tts_dict is defined }}",
+ json::object(), "True", true);
+
+ test_template(t, "tts_dict returns the decoded JSON value",
+ "{{ tts_dict('lex', 'hello') }}|{{ tts_dict('lex', 'read').VBD }}|{{ tts_dict('lex', 'read') is mapping }}|{{ tts_dict('lex', 'list')[1] + 1 }}",
+ json::object(), "h@loU|rEd|True|3", true);
+
+ test_template(t, "tts_dict returns none for a missing key",
+ "{{ tts_dict('lex', 'nope') is none }}|{{ tts_dict('lex', 'hell') is none }}|{{ tts_dict('lex', 'hello!') is none }}|{{ tts_dict('lex', 'Hello') is none }}",
+ json::object(), "True|True|True|True", true);
+
+ test_template(t, "tts_dict first, last, empty and non-ASCII keys",
+ "{{ tts_dict('lex', '') }}|{{ tts_dict('lex', 'Zed') }}|{{ tts_dict('lex', 'zoo') }}|{{ tts_dict('lex', '\xc3\xa9t\xc3\xa9') }}|{{ tts_dict('lex', '\xe6\x97\xa5') }}",
+ json::object(), "empty|z3d|zu|ete|ni", true);
+
+ test_template(t, "tts_dict null value is none, like a missing key",
+ "{{ tts_dict('lex', 'null') is none }}",
+ json::object(), "True", true);
+
+ test_template(t, "tts_dict number outside the int64 range",
+ "{{ tts_dict('lex', 'big') > 9000000000000000000 }}|{{ tts_dict('lex', 'big') is number }}",
+ json::object(), "True|True", true);
+
+ test_template(t, "tts_dict selects the dictionary by name",
+ "{{ tts_dict('other', 'hello') }}|{{ tts_dict('other', 'zoo') is none }}|{{ tts_dict('empty', 'hello') is none }}",
+ json::object(), "bonjour|True|True", true);
+
+ test_template(t, "tts_dict perceptron-style score",
+ "{%- set ws = [tts_dict('lex', 'w1'), tts_dict('lex', 'w2')] -%}"
+ "{{ ws | map(attribute='a', default=0.0) | sum }}|{{ ws | map(attribute='b', default=0.0) | sum }}",
+ json::object(), "1.75|-2.0", true);
+
+ if (!g_python_mode) {
+ test_template_error(t, "error: unknown dictionary", "{{ tts_dict('nope', 'a') }}", "unknown dictionary 'nope'", true);
+ test_template_error(t, "error: key must be a string", "{{ tts_dict('lex', 1) }}", "expected value of type", true);
+ test_template_error(t, "error: missing argument", "{{ tts_dict('lex') }}", "expected between 2 and 2 arguments", true);
+ test_template_error(t, "error: value is not JSON", "{{ tts_dict('bad', 'json') }}", "parse error", true);
+ test_template_error(t, "error: value nested too deep", "{{ tts_dict('bad', 'deep') }}", "levels deep", true);
+
+ t.test("a key without a value is an error", [](testing & t) {
+ std::string value;
+ bool thrown = false;
+ try {
+ common_tts_dict_find(common_tts_dict{{"a", "m"}, {"1"}}, "m", value);
+ } catch (const std::exception &) {
+ thrown = true;
+ }
+ t.assert_true("throws", thrown);
+ t.assert_true("the first key still has its value", common_tts_dict_find(common_tts_dict{{"a", "m"}, {"1"}}, "a", value) && value == "1");
+ });
+
+ t.test("every key of a large dictionary is found", [](testing & t) {
+ common_tts_dict dict;
+ for (int i = 0; i < 1000; i++) {
+ char key[16];
+ snprintf(key, sizeof(key), "k%04d", i);
+ dict.keys.push_back(key);
+ dict.values.push_back(std::to_string(i));
+ }
+ bool ok = true;
+ std::string value;
+ for (int i = 0; i < 1000; i++) {
+ ok = ok && common_tts_dict_find(dict, dict.keys[i], value) && value == std::to_string(i);
+ }
+ ok = ok && !common_tts_dict_find(dict, "k", value) && !common_tts_dict_find(dict, "k10000", value) && !common_tts_dict_find(dict, "a", value);
+ t.assert_true("all found, none extra", ok);
+ });
+
+ // the GGUF is not trusted: with keys in any order, a hit must be an entry of the dictionary
+ t.test("random keys in any order", [](testing & t) {
+ static const char raw[] = "ab\t\n\xff\"{";
+ const std::string alphabet(raw, sizeof(raw) - 1);
+ std::mt19937 rng(42);
+ auto random_str = [&](size_t max_len) {
+ std::string res(rng() % (max_len + 1), '\0');
+ for (auto & c : res) {
+ c = alphabet[rng() % alphabet.size()];
+ }
+ return res;
+ };
+ bool ok = true;
+ for (int i = 0; i < 20000; i++) {
+ common_tts_dict dict;
+ const size_t n = rng() % 8;
+ for (size_t j = 0; j < n; j++) {
+ dict.keys.push_back(random_str(2));
+ dict.values.push_back(std::to_string(j));
+ }
+ const std::string key = random_str(2);
+ std::string value;
+ if (common_tts_dict_find(dict, key, value)) {
+ bool found = false;
+ for (size_t j = 0; j < n; j++) {
+ found = found || (dict.keys[j] == key && dict.values[j] == value);
+ }
+ ok = ok && found;
+ }
+ }
+ t.assert_true("every hit is an entry of the dictionary", ok);
+ });
+ }
+}
+
// the ZONOS3 "tts" template spells out numbers in en, fr, de, es, it, pt, ja and zh, other languages pass through
static void test_tts_template_zonos3(testing & t) {
std::string path = __FILE__;
@@ -3115,7 +3240,7 @@ static void test_template_error(testing & t, const std::string & name, const std
jinja::program ast = jinja::parse_from_tokens(lexer.tokenize(tmpl));
jinja::context ctx(tmpl);
if (tts_norm) {
- common_chat_tts_norm_add_builtins(ctx);
+ common_chat_tts_norm_add_builtins(ctx, test_tts_dicts());
}
jinja::runtime runtime(ctx);
runtime.gather_string_parts(runtime.execute(ast));
@@ -3135,7 +3260,7 @@ static void test_template_cpp(testing & t, const std::string & name, const std::
jinja::context ctx(tmpl);
if (tts_norm) {
- common_chat_tts_norm_add_builtins(ctx);
+ common_chat_tts_norm_add_builtins(ctx, test_tts_dicts());
}
jinja::global_from_json(ctx, vars, true);
@@ -3196,6 +3321,7 @@ if merged_input.get("tts_norm_ref"):
spec.loader.exec_module(chat_tts_norm)
env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan
env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub
+ env.globals["tts_dict"] = chat_tts_norm.make_tts_dict(merged_input["tts_dicts"])
template = env.from_string(tmpl)
result = template.render(**vars_json)
@@ -3217,6 +3343,10 @@ static void test_template_py(testing & t, const std::string & name, const std::s
merged["vars"] = vars;
if (tts_norm) {
merged["tts_norm_ref"] = tts_norm_ref_path();
+ merged["tts_dicts"] = json::object();
+ for (const auto & d : *test_tts_dicts()) {
+ merged["tts_dicts"][d.first] = {{"keys", d.second.keys}, {"values", d.second.values}};
+ }
}
#ifdef _WIN32
diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md
index ce9c7a8be..5a01e425d 100644
--- a/tools/mtmd/README-dev.md
+++ b/tools/mtmd/README-dev.md
@@ -116,7 +116,7 @@ Some stages are a full LLM-sized transformer that attends to the keys/values of
- In most cases, you can hard-code model configs in the model graph class, or in `mtmd_hparams`
- If some values need to be exposed to the `mtmd_helper` layer, hard-code them in `mtmd_helper` and distinguish by pipeline and `mtmd_gen_audio_info::model_variant` if necessary
- Do NOT add new GGUF metadata or new fields to `mtmd_gen_audio_info` unless you can prove that you absolutely need them
- - If the model requires special preprocessing of the input text prompt (prompt format, text normalization), put it in `tokenizer.chat_template.tts` as a jinja program. It receives the text as `messages[0].content`, plus `language` and `voice`. See `conversion/lfm3.py` (prompt format) and `models/templates/Zyphra-ZONOS3-Mini-tts.jinja` (text normalization) as examples. For text normalization, use `tts_norm_scan()` / `tts_norm_sub()` (see `docs/tts-norm-patterns.md`) and add test cases to `tests/test-jinja.cpp`, which also checks them against the python reference with `-py`
+ - If the model requires special preprocessing of the input text prompt (prompt format, text normalization), put it in `tokenizer.chat_template.tts` as a jinja program. It receives the text as `messages[0].content`, plus `language` and `voice`. See `conversion/lfm3.py` (prompt format) and `models/templates/Zyphra-ZONOS3-Mini-tts.jinja` (text normalization) as examples. For text normalization, use `tts_norm_scan()` / `tts_norm_sub()`, and `tts_dict()` for dictionaries stored in the GGUF such as a pronunciation lexicon (see `docs/tts-norm-patterns.md`), and add test cases to `tests/test-jinja.cpp`, which also checks them against the python reference with `-py`
4. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this:
- 10-20% changes is to add new backbone (text) model and conversion
- 60% changes inside `mtmd-helper-gen.cpp`