Commit 48a32194e for llama.cpp
commit 48a32194eefc4bbedd31774ead928f64ef8aede2
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Wed Nov 18 12:09:22 2026 +0100
server: fix transcription prompt for v1/realtime (#32507)
diff --git a/tools/server/README.md b/tools/server/README.md
index 4643eb7a5..292fe598e 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -1640,7 +1640,7 @@ Models that answer after the audio, like LFM3-Audio and LFM3-Omni, hold a conver
`session.type` selects the session, it can be changed with `session.update` before the first audio or message:
- `realtime` (default): a conversation, every response stays in context for the next turns
-- `transcription`: each committed audio buffer is transcribed on its own, `audio.input.transcription.language` picks the language
+- `transcription`: each committed audio buffer is transcribed on its own, `audio.input.transcription.language` picks the language and `audio.input.transcription.prompt` replaces the default instruction (`Transcribe audio to text`) as the `prompt` of `/v1/audio/transcriptions` does; the prompt can change between utterances
*Client events:*
diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp
index 201740b2e..6f08120dd 100644
--- a/tools/server/server-realtime.cpp
+++ b/tools/server/server-realtime.cpp
@@ -130,6 +130,7 @@ struct realtime_conversation {
std::string type = "realtime";
std::string instructions;
std::string language; // transcription
+ std::string prompt; // transcription
std::string out_language;
std::string out_voice;
mtmd_helper_gen_audio_out_mode out_mode = MTMD_HELPER_GEN_AUDIO_OUT_AUTO;
@@ -201,7 +202,11 @@ struct realtime_conversation {
json session_json() const {
json audio_in = {{"format", {{"type", "audio/pcm"}, {"rate", input_rate}}}, {"turn_detection", nullptr}};
if (type == "transcription") {
- audio_in["transcription"] = {{"model", deps.meta.model_name}, {"language", language.empty() ? json(nullptr) : json(language)}};
+ audio_in["transcription"] = {
+ {"model", deps.meta.model_name},
+ {"language", language.empty() ? json(nullptr) : json(language)},
+ {"prompt", prompt},
+ };
return {
{"id", id},
{"object", "realtime.transcription_session"},
@@ -342,12 +347,28 @@ struct realtime_conversation {
// conversation
//
+ // the transcription instruction, built the same way as /v1/audio/transcriptions
+ std::string asr_user_text(const common_chat_prompt_preset & preset) const {
+ std::string text = prompt.empty() || preset.has_context ? preset.user : prompt;
+ if (!language.empty() && !preset.has_language) {
+ text += string_format(" (language: %s)", language.c_str());
+ }
+ return text;
+ }
+
// the header of a user turn goes in before its first audio or text
bool ensure_turn() {
if (turn_open) {
return true;
}
- const std::string header = n_turns == 0 ? first_header : next_header + user_header;
+ std::string header = n_turns == 0 ? first_header : next_header + user_header;
+ if (type == "transcription") {
+ // most models take the instruction before the audio, the others get it in on_commit
+ const auto preset = common_chat_get_asr_prompt(deps.meta.chat_params.tmpls.get(), language, 0.0, prompt);
+ if (!preset.media_first) {
+ header += asr_user_text(preset);
+ }
+ }
if (!control(SERVER_CONTROL_ACTION_TEXT_APPEND, header)) {
return false;
}
@@ -440,11 +461,11 @@ struct realtime_conversation {
}
// some models (VibeVoice-ASR) take an instruction after the audio, with its length
const auto * tmpls = deps.meta.chat_params.tmpls.get();
- common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language);
+ common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language, 0.0, prompt);
if (preset.needs_audio_length) {
- preset = common_chat_get_asr_prompt(tmpls, language, (double) n_audio / mtmd_get_audio_sample_rate(deps.mctx));
+ preset = common_chat_get_asr_prompt(tmpls, language, (double) n_audio / mtmd_get_audio_sample_rate(deps.mctx), prompt);
}
- const std::string after_audio = preset.media_first ? preset.user : "";
+ const std::string after_audio = preset.media_first ? asr_user_text(preset) : "";
server_control_response response;
response.n_predict = max_output_tokens;
@@ -682,6 +703,7 @@ struct realtime_conversation {
}
if (in.contains("transcription") && in["transcription"].is_object()) {
language = json_value(in["transcription"], "language", language);
+ prompt = json_value(in["transcription"], "prompt", prompt);
}
const json out = audio.value("output", json::object());
if (out.contains("voice") || out.contains("language")) {