Commit 18b5f8b18 for llama.cpp
commit 18b5f8b1862ebfe0f1c33d2355b81a54d2fec867
Author: Aldehir Rojas <hello@alde.dev>
Date: Wed Oct 7 11:49:17 2026 -0500
server : accumulate generated text and tokens as parse input (#29876)
* server : collect text and token input
* common : add a tokenize helper that aligns tokens with bytes
* common : simplify tokenize logic
diff --git a/common/chat.cpp b/common/chat.cpp
index 1c1533325..2c795c281 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -1452,14 +1452,70 @@ common_chat_params common_chat_templates_apply(const struct common_chat_template
common_chat_templates_apply_legacy(tmpls, inputs);
}
-common_chat_msg common_chat_parse(const std::string & input,
+void common_chat_input::append(const std::string & piece, llama_token token) {
+ if (piece.empty()) {
+ return;
+ }
+ tokens.push_back(token);
+ tokens.resize(tokens.size() + piece.size() - 1, LLAMA_TOKEN_NULL);
+ text += piece;
+}
+
+void common_chat_input::append(const common_chat_input & chunk) {
+ tokens.insert(tokens.end(), chunk.tokens.begin(), chunk.tokens.end());
+ text += chunk.text;
+}
+
+void common_chat_input::truncate(size_t pos) {
+ if (pos < text.size()) {
+ text.erase(pos);
+ tokens.resize(pos);
+ }
+}
+
+common_chat_input common_chat_input::substr(size_t pos, size_t n) const {
+ common_chat_input out;
+ out.text = text.substr(pos, n);
+ out.tokens.assign(tokens.begin() + pos, tokens.begin() + pos + out.size());
+ return out;
+}
+
+void common_chat_input::prepend(const std::string & prefix) {
+ tokens.insert(tokens.begin(), prefix.size(), LLAMA_TOKEN_NULL);
+ text = prefix + text;
+}
+
+void common_chat_input::prepend(const common_chat_input & prefix) {
+ tokens.insert(tokens.begin(), prefix.tokens.begin(), prefix.tokens.end());
+ text = prefix.text + text;
+}
+
+common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const std::string & text) {
+ common_chat_input input;
+ auto tokens = common_tokenize(vocab, text, false, true);
+ for (size_t i = 0; i < tokens.size(); i++) {
+ std::string piece = common_token_to_piece(vocab, tokens[i], true);
+ if (i == 0 && std::isspace(piece[0]) && !std::isspace(text[0])) {
+ // Some tokenizers will add a space before the first special token, need to exclude
+ continue;
+ }
+ input.append(piece, tokens[i]);
+ }
+ if (input.text != text) {
+ // the pieces do not give back the same text, keep the text without tokens
+ return common_chat_input(text);
+ }
+ return input;
+}
+
+common_chat_msg common_chat_parse(const common_chat_input & input,
bool is_partial,
const common_chat_parser_params & params) {
return common_chat_peg_parse(params.parser, input, is_partial, params);
}
common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser,
- const std::string & input,
+ const common_chat_input & input,
bool is_partial,
const common_chat_parser_params & params) {
const common_peg_arena & parser = src_parser.empty() ?
@@ -1470,18 +1526,17 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
LOG_DBG("No parser definition detected, assuming pure content parser.");
}
- const std::string effective_input = params.generation_prompt.empty()
- ? input
- : params.generation_prompt + input;
+ common_chat_input effective_input = input;
+ effective_input.prepend(params.generation_prompt);
- //LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.c_str());
+ //LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.text.c_str());
common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT;
if (params.debug) {
flags |= COMMON_PEG_PARSE_FLAG_DEBUG;
}
- common_peg_parse_context ctx(effective_input, flags);
+ common_peg_parse_context ctx(std::move(effective_input.text), std::move(effective_input.tokens), flags);
auto result = parser.parse(ctx);
if (result.fail()) {
@@ -1507,8 +1562,8 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
}
return msg;
}
- LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format), effective_input.substr(result.end).c_str());
- LOG_DBG("%s: full %s output triggering error:\n=== BEGIN ===\n%s\n=== END ===\n", __func__, common_chat_format_name(params.format), effective_input.c_str());
+ LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format), ctx.input.substr(result.end).c_str());
+ LOG_DBG("%s: full %s output triggering error:\n=== BEGIN ===\n%s\n=== END ===\n", __func__, common_chat_format_name(params.format), ctx.input.c_str());
throw std::runtime_error(std::string("The model produced output that does not match the expected ") + common_chat_format_name(params.format) + " format");
}
diff --git a/common/chat.h b/common/chat.h
index 0e1423a5a..4509a04cc 100644
--- a/common/chat.h
+++ b/common/chat.h
@@ -282,6 +282,31 @@ struct common_chat_params {
common_chat_msg_delimiters message_delimiters;
};
+struct common_chat_input {
+ std::string text;
+ std::vector<llama_token> tokens;
+
+ common_chat_input() = default;
+
+ // plain text, with no tokens
+ explicit common_chat_input(std::string text) : text(std::move(text)), tokens(this->text.size(), LLAMA_TOKEN_NULL) {}
+
+ size_t size() const { return text.size(); }
+ bool empty() const { return text.empty(); }
+
+ void append(const std::string & piece, llama_token token);
+ void append(const common_chat_input & chunk);
+
+ void prepend(const std::string & prefix);
+ void prepend(const common_chat_input & prefix);
+
+ void truncate(size_t pos);
+
+ common_chat_input substr(size_t pos, size_t n = std::string::npos) const;
+};
+
+common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const std::string & text);
+
// per-message parsing syntax
// should be derived from common_chat_params
struct common_chat_parser_params {
@@ -289,7 +314,7 @@ struct common_chat_parser_params {
common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool parse_reasoning"
// Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode)
bool reasoning_in_content = false;
- std::string generation_prompt;
+ common_chat_input generation_prompt;
bool parse_tool_calls = true;
bool is_continuation = false;
bool echo = false; // Include assistant prefilled msg in output
@@ -298,7 +323,7 @@ struct common_chat_parser_params {
common_chat_parser_params() = default;
common_chat_parser_params(const common_chat_params & chat_params) {
format = chat_params.format;
- generation_prompt = chat_params.generation_prompt;
+ generation_prompt = common_chat_input(chat_params.generation_prompt);
}
};
@@ -337,8 +362,8 @@ std::string common_chat_format_example(const struct common_chat_templates *
const std::map<std::string, std::string> & chat_template_kwargs);
const char * common_chat_format_name(common_chat_format format);
-common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_parser_params & params);
-common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const std::string & input, bool is_partial, const common_chat_parser_params & params);
+common_chat_msg common_chat_parse(const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
+common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
// used by arg and server
const char * common_reasoning_format_name(common_reasoning_format format);
diff --git a/common/peg-parser.h b/common/peg-parser.h
index 888325945..816d007b0 100644
--- a/common/peg-parser.h
+++ b/common/peg-parser.h
@@ -2,6 +2,7 @@
#include "json-schema.h"
#include "json.h"
+#include "llama.h"
#include <memory>
#include <set>
@@ -182,7 +183,8 @@ inline common_peg_parse_flags operator~(common_peg_parse_flags a) {
}
struct common_peg_parse_context {
- std::string input;
+ std::string input; // [h, e, l, l, o, _, w, o, r, l, d]
+ std::vector<llama_token> tokens; // [id, -1, -1, -1, -1, id, -1, -1, -1, -1, -1]
common_peg_parse_flags flags;
common_peg_ast_arena ast;
@@ -194,6 +196,11 @@ struct common_peg_parse_context {
common_peg_parse_context(const std::string & input, common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE)
: input(input), flags(flags), parse_depth(0) {}
+ common_peg_parse_context(std::string input, std::vector<llama_token> tokens, common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE)
+ : input(std::move(input)), tokens(std::move(tokens)), flags(flags), parse_depth(0) {
+ GGML_ASSERT(this->tokens.empty() || this->tokens.size() == this->input.size());
+ }
+
bool is_lenient() const { return flags & COMMON_PEG_PARSE_FLAG_LENIENT; }
bool is_debug() const { return flags & COMMON_PEG_PARSE_FLAG_DEBUG; }
};
diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp
index 9f96e1084..e31c093b0 100644
--- a/tests/test-chat.cpp
+++ b/tests/test-chat.cpp
@@ -1105,7 +1105,7 @@ struct make_peg_parser {
common_chat_msg parse(const std::string & msg, bool is_partial) const {
common_chat_parser_params parser_params(params_);
parser_params.debug = detailed_debug_;
- return common_chat_peg_parse(arena_, msg, is_partial, parser_params);
+ return common_chat_peg_parse(arena_, common_chat_input(msg), is_partial, parser_params);
}
};
@@ -4643,7 +4643,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
bool got_out_of_range = false;
std::string error_msg;
try {
- common_chat_peg_parse(arena, bad_input, /*is_partial=*/false, pp);
+ common_chat_peg_parse(arena, common_chat_input(bad_input), /*is_partial=*/false, pp);
} catch (const std::out_of_range & e) {
got_out_of_range = true;
error_msg = e.what();
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index d1af984e6..23835e185 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -267,7 +267,7 @@ struct server_slot {
bool spec_is_replay = false;
std::mt19937 spec_synth_rng;
- // TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state
+ // TODO: move members that belong to the task (such as `generated`, `has_new_line`) to task_results_state
// see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837
std::unique_ptr<const server_task> task;
std::unique_ptr<const server_task> task_prev; // used for debugging
@@ -285,7 +285,7 @@ struct server_slot {
size_t last_nl_pos = 0;
- std::string generated_text;
+ common_chat_input generated;
std::string debug_generated_text;
llama_tokens generated_tokens;
size_t n_sent_text = 0; // number of sent text character (i.e. handle partial UTF-8 on streaming)
@@ -381,7 +381,7 @@ struct server_slot {
spec_is_replay = false;
last_nl_pos = 0;
- generated_text = "";
+ generated = {};
has_new_line = false;
truncated = false;
stop = STOP_TYPE_NONE;
@@ -750,7 +750,7 @@ struct server_slot {
if (!only_metrics) {
res["prompt"] = ptask->tokens.detokenize(ctx_tgt, true);
- res["generated"] = generated_text.empty() ? debug_generated_text : generated_text;
+ res["generated"] = generated.empty() ? debug_generated_text : generated.text;
}
}
@@ -1968,28 +1968,26 @@ private:
const std::string token_str = result.text_to_send;
slot.sampled = result.tok;
- slot.generated_text += token_str;
+ slot.generated.append(token_str, result.tok);
if (slot.task->params.return_tokens) {
slot.generated_tokens.push_back(result.tok);
}
slot.has_next_token = true;
// check if there is incomplete UTF-8 character at the end
- bool incomplete = validate_utf8(slot.generated_text) < slot.generated_text.size();
+ bool incomplete = validate_utf8(slot.generated.text) < slot.generated.text.size();
// search stop word and delete it
if (!incomplete) {
- size_t pos = std::min(slot.n_sent_text, slot.generated_text.size());
+ size_t pos = std::min(slot.n_sent_text, slot.generated.text.size());
- const std::string str_test = slot.generated_text.substr(pos);
+ const std::string str_test = slot.generated.text.substr(pos);
bool send_text = true;
size_t stop_pos = slot.find_stopping_strings(str_test, token_str.size(), true);
if (stop_pos != std::string::npos) {
- slot.generated_text.erase(
- slot.generated_text.begin() + pos + stop_pos,
- slot.generated_text.end());
- pos = std::min(slot.n_sent_text, slot.generated_text.size());
+ slot.generated.truncate(pos + stop_pos);
+ pos = std::min(slot.n_sent_text, slot.generated.text.size());
} else if (slot.has_next_token && !llama_vocab_is_eog(vocab, result.tok) ) {
stop_pos = slot.find_stopping_strings(str_test, token_str.size(), false);
send_text = stop_pos == std::string::npos;
@@ -1998,7 +1996,7 @@ private:
// check if there is any token to predict
if (send_text) {
// no send the stop word in the response
- result.text_to_send = slot.generated_text.substr(pos, std::string::npos);
+ result.text_to_send = slot.generated.text.substr(pos, std::string::npos);
slot.n_sent_text += result.text_to_send.size();
// add the token to slot queue and cache
} else {
@@ -2042,17 +2040,17 @@ private:
size_t pos = slot.last_nl_pos;
int n_indent = 0;
- while (pos < slot.generated_text.size() && (slot.generated_text[pos] == ' ' || slot.generated_text[pos] == '\t')) {
+ while (pos < slot.generated.text.size() && (slot.generated.text[pos] == ' ' || slot.generated.text[pos] == '\t')) {
n_indent++;
pos++;
}
- if (pos < slot.generated_text.size() && n_indent < slot.task->params.n_indent) {
+ if (pos < slot.generated.text.size() && n_indent < slot.task->params.n_indent) {
slot.stop = STOP_TYPE_LIMIT;
slot.has_next_token = false;
// cut the last line
- slot.generated_text.erase(pos, std::string::npos);
+ slot.generated.truncate(pos);
SLT_DBG(slot, "stopped by indentation limit, n_gen = %d, n_indent = %d\n", (int) slot.stats.n_gen, n_indent);
}
@@ -2060,7 +2058,7 @@ private:
// find the next new line
{
- const size_t pos = slot.generated_text.find('\n', slot.last_nl_pos);
+ const size_t pos = slot.generated.text.find('\n', slot.last_nl_pos);
if (pos != std::string::npos) {
slot.last_nl_pos = pos + 1;
@@ -2192,7 +2190,7 @@ private:
if (is_begin) {
res->is_begin = true;
} else {
- res->content = tkn.text_to_send;
+ res->content = slot.generated.substr(slot.n_sent_text - tkn.text_to_send.size(), tkn.text_to_send.size());
res->tokens.assign(1, tkn.tok);
}
@@ -2229,15 +2227,15 @@ private:
// keep copy of last generated text for debugging purposes
if (slots_debug) {
- slot.debug_generated_text = slot.generated_text;
+ slot.debug_generated_text = slot.generated.text;
}
// in stream mode, content and tokens are already in last partial chunk
if (slot.task->params.stream) {
- res->content = "";
+ res->content = {};
res->tokens = llama_tokens{};
} else {
- res->content = std::move(slot.generated_text);
+ res->content = std::move(slot.generated);
res->tokens = std::move(slot.generated_tokens);
}
res->stats = slot.stats;
diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp
index 27ecafb7a..c7d0bc4de 100644
--- a/tools/server/server-schema.cpp
+++ b/tools/server/server-schema.cpp
@@ -311,8 +311,14 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
->set_desc("Generation prompt appended to the chat template output")
->set_handler([&](field_eval_context & ctx, const json & data) {
std::string s = data.at("generation_prompt").get<std::string>();
- ctx.params.chat_parser_params.generation_prompt = s;
ctx.params.sampling.generation_prompt = s;
+
+ if (ctx.vocab == nullptr) {
+ ctx.params.chat_parser_params.generation_prompt = common_chat_input(s);
+ return;
+ }
+
+ ctx.params.chat_parser_params.generation_prompt = common_chat_input_tokenize(ctx.vocab, s);
}));
add((new field_bool("parse_tool_calls", params.chat_parser_params.parse_tool_calls))
diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
index a5c33c056..21c1fe4e1 100644
--- a/tools/server/server-task.cpp
+++ b/tools/server/server-task.cpp
@@ -76,7 +76,7 @@ json task_params::to_json(bool only_metrics) const {
{"chat_format", common_chat_format_name(chat_parser_params.format)},
{"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},
{"reasoning_in_content", chat_parser_params.reasoning_in_content},
- {"generation_prompt", chat_parser_params.generation_prompt},
+ {"generation_prompt", chat_parser_params.generation_prompt.text},
{"samplers", samplers},
{"speculative.types", common_speculative_type_name_str(speculative.types)},
{"timings_per_token", timings_per_token},
@@ -135,7 +135,7 @@ json task_params::to_json(bool only_metrics) const {
{"chat_format", common_chat_format_name(chat_parser_params.format)},
{"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},
{"reasoning_in_content", chat_parser_params.reasoning_in_content},
- {"generation_prompt", chat_parser_params.generation_prompt},
+ {"generation_prompt", chat_parser_params.generation_prompt.text},
{"samplers", samplers},
{"speculative.types", common_speculative_type_name_str(speculative.types)},
{"timings_per_token", timings_per_token},
@@ -155,20 +155,20 @@ task_result_state::task_result_state(const common_chat_parser_params & chat_pars
, oai_resp_message_id("msg_" + random_string()) {
if (chat_parser_params.is_continuation && !chat_parser_params.echo) {
// initialize chat_msg to avoid emitting a delta containing the assistant prefill
- chat_msg = common_chat_parse("", true, chat_parser_params);
+ chat_msg = common_chat_parse(generated_input, true, chat_parser_params);
}
}
common_chat_msg task_result_state::update_chat_msg(
- const std::string & text_added,
+ const common_chat_input & added,
bool is_partial,
std::vector<common_chat_msg_diff> & diffs,
bool filter_tool_calls) {
- generated_text += text_added;
+ generated_input.append(added);
auto msg_prv_copy = chat_msg;
- //SRV_DBG("Parsing chat message: %s\n", generated_text.c_str());
+ //SRV_DBG("Parsing chat message: %s\n", generated_input.text.c_str());
auto new_msg = common_chat_parse(
- generated_text,
+ generated_input,
is_partial,
chat_parser_params);
if (!new_msg.empty()) {
@@ -340,7 +340,7 @@ json server_task_result_cmpl_final::to_json() {
json server_task_result_cmpl_final::to_json_non_oaicompat() {
json res = json {
{"index", index},
- {"content", content},
+ {"content", content.text},
{"tokens", tokens},
{"id_slot", id_slot},
{"stop", true},
@@ -386,7 +386,7 @@ json server_task_result_cmpl_final::to_json_oaicompat() {
json res = json {
{"choices", json::array({
json{
- {"text", content},
+ {"text", content.text},
{"index", index},
{"logprobs", logprobs},
{"finish_reason", finish_reason},
@@ -418,7 +418,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() {
msg = oaicompat_msg;
} else {
msg.role = "assistant";
- msg.content = content;
+ msg.content = content.text;
}
if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {
finish_reason = msg.tool_calls.empty() ? "stop" : "tool_calls";
@@ -531,7 +531,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp() {
msg = oaicompat_msg;
} else {
msg.role = "assistant";
- msg.content = content;
+ msg.content = content.text;
}
std::vector<json> output;
@@ -741,7 +741,7 @@ json server_task_result_cmpl_final::to_json_anthropic() {
msg = oaicompat_msg;
} else {
msg.role = "assistant";
- msg.content = content;
+ msg.content = content.text;
}
// thinking block comes first (Anthropic extended thinking format)
@@ -1050,7 +1050,7 @@ json server_task_result_cmpl_partial::to_json_non_oaicompat() {
// non-OAI-compat JSON
json res = json {
{"index", index},
- {"content", content},
+ {"content", content.text},
{"tokens", tokens},
{"stop", false},
{"id_slot", id_slot},
@@ -1081,7 +1081,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat() {
json res = json {
{"choices", json::array({
json{
- {"text", content},
+ {"text", content.text},
{"index", index},
{"logprobs", logprobs},
{"finish_reason", nullptr},
@@ -1316,7 +1316,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
json server_task_result_cmpl_partial::to_json_oaicompat_asr() {
json event = json {
{"type", "transcript.text.delta"},
- {"delta", content},
+ {"delta", content.text},
};
return event;
}
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index 752595b0f..552c03b07 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -108,7 +108,7 @@ struct task_result_state {
std::vector<common_chat_msg_diff> diffs;
common_chat_parser_params chat_parser_params;
common_chat_msg chat_msg;
- std::string generated_text; // append new chunks of generated text here
+ common_chat_input generated_input; // append new chunks of generated text here
std::vector<std::string> generated_tool_call_ids;
std::unordered_set<size_t> sent_tool_call_names;
@@ -128,7 +128,7 @@ struct task_result_state {
// parse partial tool calls and update the internal state
common_chat_msg update_chat_msg(
- const std::string & text_added,
+ const common_chat_input & added,
bool is_partial,
std::vector<common_chat_msg_diff> & diffs,
bool filter_tool_calls = false);
@@ -350,7 +350,7 @@ struct completion_token_output {
};
struct server_task_result_cmpl_final : server_task_result {
- std::string content;
+ common_chat_input content;
llama_tokens tokens;
bool stream;
@@ -425,8 +425,8 @@ struct server_task_result_cmpl_final : server_task_result {
};
struct server_task_result_cmpl_partial : server_task_result {
- std::string content;
- llama_tokens tokens;
+ common_chat_input content;
+ llama_tokens tokens;
int32_t n_decoded;
int32_t n_prompt_tokens;