Commit 26d52b0f9 for llama.cpp
commit 26d52b0f9543f441fc2bfa383557c992a4fbf4b9
Author: Aldehir Rojas <hello@alde.dev>
Date: Mon Nov 9 16:59:16 2026 -0600
server : use common_chat_input in realtime api (#31978)
diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp
index cabcedb78..3413eb7e8 100644
--- a/tools/server/server-realtime.cpp
+++ b/tools/server/server-realtime.cpp
@@ -772,10 +772,10 @@ struct realtime_conversation {
void on_result(const server_task_result_realtime & res) {
std::lock_guard<std::mutex> lock(mutex);
if (!transcribe_item_id.empty()) {
- if (!res.text.empty()) {
- transcript += res.text;
+ if (!res.content.empty()) {
+ transcript += res.content.text;
send_event({{"type", "conversation.item.input_audio_transcription.delta"}, {"item_id", transcribe_item_id},
- {"content_index", 0}, {"delta", res.text}});
+ {"content_index", 0}, {"delta", res.content.text}});
}
if (res.response_done) {
send_event({{"type", "conversation.item.input_audio_transcription.completed"}, {"item_id", transcribe_item_id},
@@ -788,12 +788,12 @@ struct realtime_conversation {
// with tools, the text goes through the parser: content and tool calls
std::vector<common_chat_msg_diff> diffs;
if (parse) {
- if (!res.text.empty() || res.response_done) {
- parse->update_chat_msg(common_chat_input(res.text), !res.response_done, diffs, true);
+ if (!res.content.empty() || res.response_done) {
+ parse->update_chat_msg(res.content, !res.response_done, diffs, true);
}
- } else if (!res.text.empty()) {
+ } else if (!res.content.empty()) {
diffs.emplace_back();
- diffs.back().content_delta = res.text;
+ diffs.back().content_delta = res.content.text;
}
for (const auto & diff : diffs) {
if (!diff.content_delta.empty()) {
@@ -965,11 +965,11 @@ bool server_realtime_slot::fail(const std::string & msg, enum error_type type) {
return false;
}
-void server_realtime_slot::send_result(std::string text, std::vector<float> audio, bool response_done) {
+void server_realtime_slot::send_result(common_chat_input content, std::vector<float> audio, bool response_done) {
auto res = std::make_unique<server_task_result_realtime>();
res->id = id_task;
res->index = index;
- res->text = std::move(text);
+ res->content = std::move(content);
res->audio = std::move(audio);
res->response_done = response_done;
res->cancelled = response_done && cancelled;
@@ -1092,8 +1092,8 @@ std::string server_realtime_slot::piece(llama_token tok) const {
// one text token or audio frame of the response being generated
bool server_realtime_slot::response_step() {
- bool stop = false;
- std::string text;
+ bool stop = false;
+ common_chat_input content;
std::vector<float> audio;
if (gen.valid()) {
@@ -1102,7 +1102,7 @@ bool server_realtime_slot::response_step() {
return fail("response generation failed");
}
if (sampled != LLAMA_TOKEN_NULL && !stop) {
- text = piece(sampled);
+ content.append(piece(sampled), sampled);
}
if (h_next) {
h_state.assign(h_next, h_next + llama_model_n_embd(model));
@@ -1110,7 +1110,7 @@ bool server_realtime_slot::response_step() {
} else {
stop = llama_vocab_is_eog(vocab, sampled);
if (!stop) {
- text = piece(sampled);
+ content.append(piece(sampled), sampled);
if (!decode(&sampled, nullptr, 1, true)) {
return false;
}
@@ -1137,8 +1137,8 @@ bool server_realtime_slot::response_step() {
audio.assign((const float *) data, (const float *) data + data_len / sizeof(float));
}
}
- if (!text.empty() || !audio.empty()) {
- send_result(std::move(text), std::move(audio));
+ if (!content.empty() || !audio.empty()) {
+ send_result(std::move(content), std::move(audio));
}
if (done) {
if (gen.valid()) {
diff --git a/tools/server/server-realtime.h b/tools/server/server-realtime.h
index dd871261b..f1cc57aad 100644
--- a/tools/server/server-realtime.h
+++ b/tools/server/server-realtime.h
@@ -87,7 +87,7 @@ private:
std::set<llama_token> preserved; // special tokens of the response that go out as text (tool calls)
bool fail(const std::string & msg, enum error_type type = ERROR_TYPE_SERVER);
- void send_result(std::string text, std::vector<float> audio, bool response_done = false);
+ void send_result(common_chat_input content, std::vector<float> audio, bool response_done = false);
bool decode(const llama_token * tokens, const float * rows, int32_t n, bool logits_last);
bool audio_rows();
void sample_next();
diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
index 4b2a8b2f7..be3bc539a 100644
--- a/tools/server/server-task.cpp
+++ b/tools/server/server-task.cpp
@@ -1513,7 +1513,7 @@ json server_task_result_rerank::to_json() {
//
json server_task_result_realtime::to_json() {
return json {
- {"text", text},
+ {"text", content.text},
{"n_samples", audio.size()},
{"response_done", response_done},
{"cancelled", cancelled},
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index 8faae1f18..3d2b1ad4c 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -555,7 +555,7 @@ struct server_task_result_embd : server_task_result {
// one event of a realtime conversation: text and audio of the response being generated, or its end
struct server_task_result_realtime : server_task_result {
- std::string text;
+ common_chat_input content; // the text with its token
std::vector<float> audio; // PCM F32 at the sample rate of mtmd_gen_audio_get_info()
bool response_done = false;
bool cancelled = false;