Commit 203b5f404 for llama.cpp
commit 203b5f404c913972d21ff8e42b550569907803a7
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Tue Nov 10 11:35:30 2026 +0100
server: fix output lang for realtime api (#31982)
diff --git a/common/arg.cpp b/common/arg.cpp
index 81de73cd0..3a49d86bc 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -4491,7 +4491,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.tts_lang = value;
}
- ).set_examples({LLAMA_EXAMPLE_TTS}));
+ ).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"--tts-voice"}, "VOICE",
"voice for audio generation (model-dependent)\n"
@@ -4499,7 +4499,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.tts_voice = value;
}
- ).set_examples({LLAMA_EXAMPLE_TTS}));
+ ).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"--tts-speaker-file"}, "FNAME",
"speaker file path for audio generation",
diff --git a/common/chat.cpp b/common/chat.cpp
index 5f956e64e..ef9b853f0 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -819,21 +819,29 @@ static bool is_lfm3_template(const std::string & src) {
return src.find("<additional_tools>") != std::string::npos;
}
-// LFM3-Audio: a known language code maps to its name, no language lets the model detect it
-static std::string lfm3_asr_system_prompt(const std::string & language) {
+static std::string string_lower(std::string s) {
+ std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return (char) std::tolower(c); });
+ return s;
+}
+
+// LFM3-Audio: language code -> name
+static std::string lfm3_language_name(const std::string & language) {
static const std::map<std::string, std::string> names = {
{ "ar", "Arabic" }, { "da", "Danish" }, { "de", "German" }, { "el", "Greek" }, { "en", "English" },
{ "es", "Spanish" }, { "fi", "Finnish" }, { "fr", "French" }, { "he", "Hebrew" }, { "it", "Italian" },
{ "ja", "Japanese" }, { "ko", "Korean" }, { "nl", "Dutch" }, { "pl", "Polish" }, { "pt", "Portuguese" },
{ "sv", "Swedish" }, { "tr", "Turkish" }, { "zh", "Chinese" },
};
+ const auto it = names.find(string_lower(language));
+ return it != names.end() ? it->second : language;
+}
+
+// LFM3-Audio: no language = auto detect
+static std::string lfm3_asr_system_prompt(const std::string & language) {
if (language.empty()) {
return "Perform ASR.";
}
- std::string code = language;
- std::transform(code.begin(), code.end(), code.begin(), [](unsigned char c) { return (char) std::tolower(c); });
- const auto it = names.find(code);
- return "Perform ASR in " + (it != names.end() ? it->second : language) + ".";
+ return "Perform ASR in " + lfm3_language_name(language) + ".";
}
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) {
@@ -868,6 +876,27 @@ std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templa
});
}
+std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
+ const std::string & language, const std::string & voice) {
+ if (!chat_templates || !chat_templates->template_default || !is_lfm3_template(chat_templates->template_default->source())) {
+ return "";
+ }
+ // same voices as the "tts" template
+ const std::string v = string_lower(voice);
+ const bool has_voice = v == "female" || v == "male";
+ if (language.empty() && !has_voice) {
+ return "";
+ }
+ std::string out = "Speak";
+ if (!language.empty()) {
+ out += " in " + lfm3_language_name(language);
+ }
+ if (has_voice) {
+ out += " using a " + v + " voice";
+ }
+ return out + ".";
+}
+
std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant) {
if (!variant.empty()) {
if (variant == "tool_use") {
diff --git a/common/chat.h b/common/chat.h
index ebfc80f51..304ccf9ec 100644
--- a/common/chat.h
+++ b/common/chat.h
@@ -500,3 +500,7 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates
// language and voice are optional, the template defines their defaults
std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
const std::string & language = "", const std::string & voice = "");
+
+// system prompt line for the speech language and voice (e.g. LFM3-Audio), empty if not used
+std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
+ const std::string & language, const std::string & voice);
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 5a44bb7b3..5d9726796 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -6186,7 +6186,7 @@ void server_routes::init_routes() {
// live transcription: Voxtral Realtime (no chat), or an "asr" model in a session opened with ?intent=transcription
const bool live_asr = !ctx_server.asr_prompt.empty() && req.get_param("intent") == "transcription";
if (ctx_server.mtmd_cap.audio_stream && ctx_server.mtmd_cap.chat && !live_asr) {
- server_realtime_conversation({queue_tasks, queue_results, *meta, params.sampling, ctx_server.mctx, ctx_server.mtmd_cap, ctx_server.vocab}, conn);
+ server_realtime_conversation({queue_tasks, queue_results, *meta, params, ctx_server.mctx, ctx_server.mtmd_cap, ctx_server.vocab}, conn);
return;
}
@@ -7073,8 +7073,10 @@ std::unique_ptr<server_res_generator> server_routes::handle_tts_impl(const serve
server_schema::eval_tts_schema(params, task, body);
const bool stream = task.params.stream;
// a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it
- const std::string lang = json_value(body, "lang", std::string());
- const std::string voice = json_value(body, "voice", std::string());
+ const std::string lang = json_value(body, "lang", params.tts_lang);
+ // speaker_ref in the request overrides the default voice
+ const bool has_speaker_ref = req.files.count("speaker_ref") > 0 || !json_value(body, "speaker_ref_b64", std::string()).empty();
+ const std::string voice = json_value(body, "voice", has_speaker_ref ? std::string() : params.tts_voice);
const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice);
// a model with predefined voices only speaks with one of them
mtmd::bitmap_ptr voice_ref;
diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp
index 3413eb7e8..8ed3dd535 100644
--- a/tools/server/server-realtime.cpp
+++ b/tools/server/server-realtime.cpp
@@ -130,6 +130,8 @@ struct realtime_conversation {
std::string type = "realtime";
std::string instructions;
std::string language; // transcription
+ std::string out_language;
+ std::string out_voice;
mtmd_helper_gen_audio_out_mode out_mode = MTMD_HELPER_GEN_AUDIO_OUT_AUTO;
int32_t max_output_tokens = -1; // "inf"
int input_rate = 16000;
@@ -176,7 +178,8 @@ struct realtime_conversation {
std::atomic<bool> closed{false};
- realtime_conversation(const server_realtime_deps & deps, server_http_context::ws_conn & conn) : deps(deps), conn(conn) {}
+ realtime_conversation(const server_realtime_deps & deps, server_http_context::ws_conn & conn)
+ : deps(deps), conn(conn), out_language(deps.params.tts_lang), out_voice(deps.params.tts_voice) {}
//
// events
@@ -207,7 +210,11 @@ struct realtime_conversation {
}
json audio = {{"input", audio_in}};
if (deps.cap.audio_gen) {
- audio["output"] = {{"format", {{"type", "audio/pcm"}, {"rate", output_rate}}}};
+ audio["output"] = {
+ {"format", {{"type", "audio/pcm"}, {"rate", output_rate}}},
+ {"voice", out_voice.empty() ? json(nullptr) : json(out_voice)},
+ {"language", out_language.empty() ? json(nullptr) : json(out_language)},
+ };
}
return {
{"id", id},
@@ -245,7 +252,7 @@ struct realtime_conversation {
// sampling of a response: the server defaults, with the tool call grammar of chat
common_params_sampling response_sampling(const common_chat_session * chat) const {
- common_params_sampling sampling = deps.sampling;
+ common_params_sampling sampling = deps.params.sampling;
if (chat) {
chat->apply_sampling(sampling);
}
@@ -544,6 +551,8 @@ struct realtime_conversation {
return;
}
response.out_mode = mode;
+ response.lang = out_language;
+ response.voice = out_voice;
// with tools, the output goes through the tool call parser of the chat template and its grammar
std::optional<common_chat_session> chat;
@@ -661,6 +670,27 @@ struct realtime_conversation {
if (in.contains("transcription") && in["transcription"].is_object()) {
language = json_value(in["transcription"], "language", language);
}
+ const json out = audio.value("output", json::object());
+ if (out.contains("voice") || out.contains("language")) {
+ std::string voice = out_voice;
+ if (out.contains("voice")) {
+ // string or {"id": name}
+ const json & v = out["voice"];
+ voice = v.is_object() ? json_value(v, "id", std::string()) : v.is_string() ? v.get<std::string>() : std::string();
+ }
+ const std::string lang = json_value(out, "language", out_language);
+ const auto & voices = deps.meta.tts_voices;
+ if (!voice.empty() && !voices.empty() && std::find(voices.begin(), voices.end(), voice) == voices.end()) {
+ send_error("session.update: unknown voice \"" + voice + "\", see \"voices\" in GET /props");
+ return;
+ }
+ if (started && (voice != out_voice || lang != out_language)) {
+ send_error("session.update: the output voice and language cannot change after the first audio or message");
+ return;
+ }
+ out_voice = voice;
+ out_language = lang;
+ }
if (!started && !configure()) {
send_error("the chat template does not fit the realtime turn format", "server_error");
return;
@@ -674,6 +704,11 @@ struct realtime_conversation {
if (type == "transcription") {
const auto preset = common_chat_get_asr_prompt(deps.meta.chat_params.tmpls.get(), language);
instructions = preset.system;
+ } else if (deps.cap.audio_gen) {
+ const std::string speech = common_chat_get_speech_system_prompt(deps.meta.chat_params.tmpls.get(), out_language, out_voice);
+ if (!speech.empty()) {
+ instructions = instructions.empty() ? speech : speech + "\n" + instructions;
+ }
}
const bool ok = derive_turn_text();
instructions = saved;
@@ -869,7 +904,7 @@ struct realtime_conversation {
{
server_task task(SERVER_TASK_TYPE_REALTIME);
task.id = rd.get_new_id();
- task.params.sampling = deps.sampling;
+ task.params.sampling = deps.params.sampling;
task.params.stream = true;
task.params.oaicompat_cmpl_id = id;
task.params.oaicompat_model = deps.meta.model_name;
@@ -1052,6 +1087,15 @@ bool server_realtime_slot::response_start(const server_realtime_op & op) {
inp.data.stream = true;
inp.data.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM;
inp.data.out_mode = op.response.out_mode;
+ inp.set_lang(op.response.lang);
+ // predefined voice -> speaker ref, LFM3 uses the system prompt instead
+ if (!op.response.voice.empty() && mtmd_gen_audio_get_info(mctx).n_voices > 0) {
+ mtmd::bitmap_ptr voice(mtmd_gen_audio_get_voice(mctx, op.response.voice.c_str()));
+ if (!voice) {
+ return fail("unknown voice \"" + op.response.voice + "\"", ERROR_TYPE_INVALID_REQUEST);
+ }
+ inp.set_speaker_ref(std::move(voice));
+ }
if (gen.set_input(inp.get()) != 0) {
return fail("failed to start the response");
}
diff --git a/tools/server/server-realtime.h b/tools/server/server-realtime.h
index f1cc57aad..93c864df2 100644
--- a/tools/server/server-realtime.h
+++ b/tools/server/server-realtime.h
@@ -15,7 +15,7 @@
#include <vector>
struct server_context_meta;
-struct common_params_sampling;
+struct common_params;
struct mtmd_context;
struct mtmd_helper_cap;
struct llama_vocab;
@@ -24,7 +24,7 @@ struct server_realtime_deps {
server_queue & queue_tasks;
server_response & queue_results;
const server_context_meta & meta;
- const common_params_sampling & sampling;
+ const common_params & params;
mtmd_context * mctx;
const mtmd_helper_cap & cap;
const llama_vocab * vocab;
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index 3d2b1ad4c..6d550d104 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -70,6 +70,8 @@ struct server_control_response {
mtmd_helper_gen_audio_out_mode out_mode = MTMD_HELPER_GEN_AUDIO_OUT_AUTO;
common_params_sampling sampling; // of the text tokens: tool call grammar and preserved tokens
int32_t n_predict = -1; // text tokens and audio frames, -1: until the context is full
+ std::string lang; // empty: model default
+ std::string voice; // empty: model default
};
struct task_params {