Commit 08f945ffb for llama.cpp
commit 08f945ffb5c305afad252f7328080be9b46315ad
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Mon Nov 9 23:58:52 2026 +0100
server: add oai-compat /v1/audio/speech endpoint (#31763)
* server: add oai-compat /v1/audio/speech endpoint
* add e2e pytest
* server: share the s16 rounding, type speech form fields
The WAV writer of the audio helper truncates while pcm_f32_to_s16
rounds, so wav and pcm differ by one LSB. Both now round.
pcm_f32_to_s16 takes a pointer and a count and converts the TTS
bytes without an intermediate copy.
/tts and /v1/audio/speech parse their body with parse_tts_body, so
multipart fields such as speed get their type before the speech
conversion checks them.
---------
Co-authored-by: Pascal <admin@serveurperso.com>
diff --git a/tests/test-server-component.cpp b/tests/test-server-component.cpp
index 2c76e8904..c4ed1cacc 100644
--- a/tests/test-server-component.cpp
+++ b/tests/test-server-component.cpp
@@ -1,5 +1,6 @@
#include "testing.h"
+#include "../tools/server/server-chat.h"
#include "../tools/server/server-common.h"
#include "../tools/server/server-decision.h"
#include "../tools/server/server-schema.h"
@@ -7,6 +8,7 @@
#include <algorithm>
#include <cmath>
#include <cstdlib>
+#include <cstring>
#include <regex>
#include <string>
#include <thread>
@@ -665,6 +667,79 @@ static void test_glob_match(testing & t) {
});
}
+//
+// OpenAI speech API
+//
+
+static void test_speech_convert(testing & t) {
+ const json base = {{"model", "tts"}, {"input", "Hello"}, {"voice", "alba"}};
+
+ auto with = [&](const char * key, const json & value) {
+ json body = base;
+ body[key] = value;
+ return body;
+ };
+ auto rejected = [&](const char * key, const json & value) {
+ try {
+ convert_speech_to_tts(with(key, value));
+ } catch (const std::invalid_argument &) {
+ return true;
+ }
+ return false;
+ };
+
+ // defaults
+ json out = convert_speech_to_tts(base);
+ t.assert_equal("input kept", std::string("Hello"), out.at("input").get<std::string>());
+ t.assert_equal("voice kept", std::string("alba"), out.at("voice").get<std::string>());
+ t.assert_equal("default format", std::string("wav"), out.at("response_format").get<std::string>());
+ t.assert_true ("no stream", !out.contains("stream"));
+
+ // response_format
+ out = convert_speech_to_tts(with("response_format", "wav"));
+ t.assert_equal("wav", std::string("wav"), out.at("response_format").get<std::string>());
+ out = convert_speech_to_tts(with("response_format", "pcm"));
+ t.assert_equal("pcm is 16-bit", std::string("pcm16"), out.at("response_format").get<std::string>());
+ for (const char * format : {"mp3", "opus", "aac", "flac", "pcm16"}) {
+ t.assert_true(std::string("rejects ") + format, rejected("response_format", format));
+ }
+
+ // custom voice
+ out = convert_speech_to_tts(with("voice", json{{"id", "voice_1234"}}));
+ t.assert_equal("voice object", std::string("voice_1234"), out.at("voice").get<std::string>());
+
+ // stream_format
+ out = convert_speech_to_tts(with("stream_format", "audio"));
+ t.assert_true ("audio streams", out.at("stream").get<bool>());
+ t.assert_true ("rejects sse", rejected("stream_format", "sse"));
+
+ // speed
+ t.assert_true ("speed 1.0", !rejected("speed", 1.0));
+ t.assert_true ("rejects speed 1.5", rejected("speed", 1.5));
+
+ // instructions are ignored
+ t.assert_true ("instructions", !rejected("instructions", "whisper"));
+}
+
+static void test_pcm_f32_to_s16(testing & t) {
+ auto sample = [](const std::string & bytes, size_t i) {
+ int16_t v;
+ memcpy(&v, bytes.data() + i * sizeof(v), sizeof(v));
+ return (int) v;
+ };
+
+ const std::vector<float> samples = {0.0f, 1.0f, -1.0f, 0.5f, 2.0f, -2.0f};
+ const std::string bytes = pcm_f32_to_s16(samples.data(), samples.size());
+ t.assert_equal("size", (size_t) 12, bytes.size());
+ t.assert_equal("zero", 0, sample(bytes, 0));
+ t.assert_equal("max", 32767, sample(bytes, 1));
+ t.assert_equal("min", -32767, sample(bytes, 2));
+ t.assert_equal("rounds", 16384, sample(bytes, 3));
+ t.assert_equal("clamps above", 32767, sample(bytes, 4));
+ t.assert_equal("clamps below", -32767, sample(bytes, 5));
+ t.assert_true ("empty", pcm_f32_to_s16(nullptr, 0).empty());
+}
+
int main(int argc, char ** argv) {
testing t;
@@ -690,6 +765,8 @@ int main(int argc, char ** argv) {
t.test("decision_depth", test_decision_depth);
t.test("glob_match", test_glob_match);
t.test("dry_sequence_breakers", test_dry_sequence_breakers);
+ t.test("speech_convert", test_speech_convert);
+ t.test("pcm_f32_to_s16", test_pcm_f32_to_s16);
return t.summary();
}
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 5e77eccd9..73e7e1b27 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -72,7 +72,7 @@ static void write_wav16_header(std::vector<char> & buf, uint32_t data_sz, int32_
static void append_wav16_pcm(std::vector<char> & buf, const float * pcm, size_t n) {
for (size_t i = 0; i < n; i++) {
- int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f);
+ int16_t s = (int16_t) std::lround(std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f);
put_bytes(buf, &s, 2);
}
}
diff --git a/tools/server/README.md b/tools/server/README.md
index 712205dd5..defc09c6b 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -872,7 +872,7 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info
`n_predict`: Max number of audio frames to generate. Defaults to `512`; generation normally stops earlier once the model emits an end-of-speech token.
-`response_format`: `wav` (default) or `pcm` (raw `float32` little-endian mono samples, no header; the response `Content-Type` carries the sample rate, e.g. `audio/pcm;rate=24000;encoding=float;bits=32`).
+`response_format`: `wav` (default), `pcm` (raw `float32` little-endian mono samples, no header; the response `Content-Type` carries the sample rate, e.g. `audio/pcm;rate=24000;encoding=float;bits=32`) or `pcm16` (same, as signed 16-bit samples, e.g. `audio/pcm;rate=24000;encoding=signed-int;bits=16`).
`stream`: If `true`, the response is streamed as audio becomes available instead of waiting for the full generation to finish. WAV streaming writes an RFC-noncompliant header with an unknown (`0xFFFFFFFF`) size field, since the final length isn't known up front; most players and decoders (ffmpeg, VLC, ...) handle this by reading until EOF.
@@ -1652,6 +1652,33 @@ Tools go into the system prompt with the chat template, and the tool calls are p
See [examples/realtime_client.py](../../examples/realtime_client.py) for a client with the `openai` library.
+### POST `/v1/audio/speech`: OpenAI-compatible Speech API
+
+Generates speech from text, for models that can generate audio. The request is converted to a [`/tts`](#post-tts-generate-speech-audio-from-text) request, so the `/tts` options such as `lang`, `seed` or `speaker_ref_b64` can also be used.
+
+*Options:*
+
+`input`: The text to speak.
+
+`voice`: One of the `voices` listed by `GET /props`, as a name or as `{"id": "..."}`. OpenAI voice names such as `alloy` are not mapped: a model with predefined voices rejects them.
+
+`response_format`: `wav` or `pcm` (raw signed 16-bit little-endian mono samples, no header). Defaults to `wav`, unlike the `mp3` of the OpenAI API. Other formats are rejected. The sample rate is the one of the model and is not resampled to the 24 kHz of the OpenAI API, the response `Content-Type` carries it.
+
+`stream_format`: `audio` streams the response as it becomes available, see `stream` in `/tts` for the WAV header it writes. `sse` is rejected. The response is not streamed when omitted.
+
+`speed`: Only `1.0` is accepted.
+
+`instructions`: Ignored.
+
+*Examples:*
+
+```shell
+curl http://127.0.0.1:8080/v1/audio/speech \
+ -H "Content-Type: application/json" \
+ -d '{"model": "tts", "input": "Hello, this is a test."}' \
+ -o output.wav
+```
+
### POST `/v1/responses`: OpenAI-compatible Responses API
*Options:*
diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp
index a7431800b..8cf883a3a 100644
--- a/tools/server/server-chat.cpp
+++ b/tools/server/server-chat.cpp
@@ -737,6 +737,42 @@ json convert_transcriptions_to_chatcmpl(
return chatcmpl_body;
}
+json convert_speech_to_tts(const json & inp_body) {
+ json tts_body = inp_body; // copy all fields
+
+ // a custom voice is given as {"id": "..."}
+ if (inp_body.contains("voice") && inp_body.at("voice").is_object()) {
+ tts_body["voice"] = json_value(inp_body.at("voice"), "id", std::string());
+ }
+
+ // the default of the OpenAI API is mp3, we have no encoder for it
+ const std::string response_format = json_value(inp_body, "response_format", std::string("wav"));
+ if (response_format == "pcm") {
+ tts_body["response_format"] = "pcm16"; // "pcm" is float32 in /tts
+ } else if (response_format == "wav") {
+ tts_body["response_format"] = "wav";
+ } else {
+ throw std::invalid_argument("Only 'wav' and 'pcm' response_format are supported for speech");
+ }
+
+ const std::string stream_format = json_value(inp_body, "stream_format", std::string());
+ if (stream_format == "audio") {
+ tts_body["stream"] = true;
+ } else if (!stream_format.empty()) {
+ throw std::invalid_argument("Only 'audio' stream_format is supported for speech");
+ }
+
+ if (json_value(inp_body, "speed", 1.0) != 1.0) {
+ throw std::invalid_argument("'speed' is not supported for speech");
+ }
+
+ if (!json_value(inp_body, "instructions", std::string()).empty()) {
+ SRV_WRN("%s", "'instructions' is not supported for speech and is ignored\n");
+ }
+
+ return tts_body;
+}
+
llama_tokens server_chat_get_asr_prompt(const llama_model * model) {
const char * src = llama_model_chat_template(model, "asr");
if (src == nullptr) {
diff --git a/tools/server/server-chat.h b/tools/server/server-chat.h
index b4b6f1ec6..cba4a073c 100644
--- a/tools/server/server-chat.h
+++ b/tools/server/server-chat.h
@@ -24,6 +24,9 @@ json convert_transcriptions_to_chatcmpl(
const std::map<std::string, uploaded_file> & in_files,
std::vector<raw_buffer> & out_files);
+// convert OpenAI speech API format to the /tts format
+json convert_speech_to_tts(const json & body);
+
// prompt of the "asr" chat template, live ASR models (e.g. LFM3-Audio live ASR) transcribe with it
// empty if the model has no such template, throws on a template error
llama_tokens server_chat_get_asr_prompt(const llama_model * model);
diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp
index 05fd04c29..5bf2ec12a 100644
--- a/tools/server/server-common.cpp
+++ b/tools/server/server-common.cpp
@@ -9,6 +9,8 @@
#include "server-common.h"
+#include <algorithm>
+#include <cmath>
#include <filesystem>
#include <random>
#include <sstream>
@@ -200,6 +202,19 @@ const char * get_media_marker() {
return marker.c_str();
}
+//
+// audio utils
+//
+
+std::string pcm_f32_to_s16(const float * samples, size_t n) {
+ std::string bytes(n * sizeof(int16_t), '\0');
+ for (size_t i = 0; i < n; i++) {
+ const int16_t v = (int16_t) std::lround(std::max(-1.0f, std::min(1.0f, samples[i])) * 32767.0f);
+ memcpy(&bytes[2 * i], &v, sizeof(v));
+ }
+ return bytes;
+}
+
//
// model output modalities
//
diff --git a/tools/server/server-common.h b/tools/server/server-common.h
index 01bf12e61..ca870465a 100644
--- a/tools/server/server-common.h
+++ b/tools/server/server-common.h
@@ -110,6 +110,13 @@ std::string gen_tool_call_id();
// get a random marker; note: each time the server restarts, the marker will be different
const char * get_media_marker();
+//
+// audio utils
+//
+
+// float samples to signed 16-bit PCM bytes, clamped to [-1, 1]
+std::string pcm_f32_to_s16(const float * samples, size_t n);
+
//
// model output modalities
//
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index be44c3e2e..5a44bb7b3 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -5744,6 +5744,25 @@ json server_routes::get_model_info() const {
return get_res_model_info(*meta);
}
+// body of /tts and /v1/audio/speech; multipart form fields arrive as strings, the numeric and boolean params get their types back
+static json parse_tts_body(const server_http_req & req) {
+ json body = json::parse(req.body);
+ for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) {
+ if (body.contains(k) && body[k].is_string()) {
+ body[k] = std::stoll(body[k].get<std::string>());
+ }
+ }
+ for (const char * k : { "top_p", "temperature", "repeat_penalty", "speed" }) {
+ if (body.contains(k) && body[k].is_string()) {
+ body[k] = std::stof(body[k].get<std::string>());
+ }
+ }
+ if (body.contains("stream") && body["stream"].is_string()) {
+ body["stream"] = body["stream"] == "true";
+ }
+ return body;
+}
+
void server_routes::init_routes() {
// IMPORTANT: all lambda functions must start with create_response()
// this is to ensure that the server_res_generator can handle sleeping case correctly
@@ -6504,199 +6523,14 @@ void server_routes::init_routes() {
};
this->post_tts = [this](const server_http_req & req) {
- auto res = create_response();
- res->set_req(&req); // will also set spipe if needed
-
- if (!ctx_server.has_cap_tts()) {
- res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED));
- return res;
- }
-
- const auto info = mtmd_gen_audio_get_info(ctx_server.mctx);
-
- json body = json::parse(req.body);
-
- // multipart form fields arrive as strings, correct the types of the numeric and boolean params
- for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) {
- if (body.contains(k) && body[k].is_string()) {
- body[k] = std::stoll(body[k].get<std::string>());
- }
- }
- for (const char * k : { "top_p", "temperature", "repeat_penalty" }) {
- if (body.contains(k) && body[k].is_string()) {
- body[k] = std::stof(body[k].get<std::string>());
- }
- }
- if (body.contains("stream") && body["stream"].is_string()) {
- body["stream"] = body["stream"] == "true";
- }
-
- std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string()));
- if (prompt.empty()) {
- res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST));
- return res;
- }
-
- const std::string response_format = json_value(body, "response_format", std::string("wav"));
-
- server_task task(SERVER_TASK_TYPE_TTS);
- server_schema::eval_tts_schema(params, task, body);
- const bool stream = task.params.stream;
- // a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it
- const std::string lang = json_value(body, "lang", std::string());
- const std::string voice = json_value(body, "voice", std::string());
- const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice);
- // a model with predefined voices only speaks with one of them
- mtmd::bitmap_ptr voice_ref;
- if (!voice.empty() && info.n_voices > 0) {
- voice_ref.reset(mtmd_gen_audio_get_voice(ctx_server.mctx, voice.c_str()));
- if (!voice_ref) {
- res->error(format_error_response("unknown voice \"" + voice + "\", see \"voices\" in GET /props", ERROR_TYPE_INVALID_REQUEST));
- return res;
- }
- }
- task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt);
- task.tts_inp.set_lang(lang);
- task.tts_inp.data.stream = stream;
- task.tts_inp.data.out_type = response_format == "pcm"
- ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM
- : MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
- // note: -1 is clamped to 0 by the sampler, it will disable the penalty
- task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512;
- task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG
- if (task.tts_inp.data.top_k > 0) {
- task.params.sampling.top_k = task.tts_inp.data.top_k;
- }
- if (task.tts_inp.data.top_p > 0) {
- task.params.sampling.top_p = task.tts_inp.data.top_p;
- }
- if (task.tts_inp.data.temp >= 0) {
- task.params.sampling.temp = task.tts_inp.data.temp;
- }
-
- // speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64")
- const unsigned char * speaker_ref_data = nullptr;
- size_t speaker_ref_len = 0;
- std::string speaker_ref_b64_decoded;
-
- auto speaker_ref_file = req.files.find("speaker_ref");
- if (speaker_ref_file != req.files.end()) {
- speaker_ref_data = speaker_ref_file->second.data.data();
- speaker_ref_len = speaker_ref_file->second.data.size();
- } else {
- std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string());
- if (!speaker_ref_b64.empty()) {
- try {
- speaker_ref_b64_decoded = base64::decode(speaker_ref_b64);
- } catch (const std::exception &) {
- res->error(format_error_response("\"speaker_ref_b64\" is not valid base64", ERROR_TYPE_INVALID_REQUEST));
- return res;
- }
- speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data();
- speaker_ref_len = speaker_ref_b64_decoded.size();
- }
- }
-
- if (speaker_ref_len > 0) {
- if (voice_ref) {
- res->error(format_error_response("\"voice\" and \"speaker_ref\" cannot be used together", ERROR_TYPE_INVALID_REQUEST));
- return res;
- }
- if (!mtmd_support_audio(ctx_server.mctx)) {
- res->error(format_error_response("this model has no voice encoder, \"speaker_ref\" cannot be used", ERROR_TYPE_INVALID_REQUEST));
- return res;
- }
- auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt);
- if (!wrapper.bitmap) {
- res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));
- return res;
- }
- task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap));
- } else if (voice_ref) {
- task.tts_inp.set_speaker_ref(std::move(voice_ref));
- } else if (info.n_voices > 0) {
- // without a voice prompt the model produces near silence: the first voice is the default
- const char * first = mtmd_gen_audio_voice_name(ctx_server.mctx, 0);
- SRV_WRN("no voice given, speaking with \"%s\"\n", first);
- task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(mtmd_gen_audio_get_voice(ctx_server.mctx, first)));
- } else {
- SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n");
- }
-
- auto & rd = res->rd;
- task.id = rd.get_new_id();
- rd.post_task(std::move(task));
-
- // raw float32 LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586)
- const std::string content_type = response_format == "pcm"
- ? "audio/pcm;rate=" + std::to_string(info.sample_rate) + ";encoding=float;bits=32"
- : "audio/wav";
-
- if (!stream) {
- auto result = rd.next(req.should_stop);
- if (!result) {
- GGML_ASSERT(req.should_stop());
- return res; // connection is closed
- }
- if (result->is_error()) {
- res->error(result->to_json());
- return res;
- }
- auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
- GGML_ASSERT(tts_res != nullptr);
- res->status = 200;
- res->content_type = content_type;
- res->data = std::move(tts_res->audio);
- return res;
- } else {
- auto first_result = rd.next(req.should_stop);
- if (!first_result) {
- GGML_ASSERT(req.should_stop());
- return res; // connection is closed
- }
- if (first_result->is_error()) {
- res->error(first_result->to_json());
- return res;
- }
- auto * first_tts_res = dynamic_cast<server_task_result_tts *>(first_result.get());
- GGML_ASSERT(first_tts_res != nullptr);
-
- res->status = 200;
- res->content_type = content_type;
- res->data = std::move(first_tts_res->audio);
- bool is_done = first_tts_res->final;
-
- res->set_next([res_this = res.get(), is_done](std::string & output) mutable -> bool {
- // flush buffered audio before is_done - the first result can already be final
- if (!res_this->data.empty()) {
- output = std::move(res_this->data);
- res_this->data.clear();
- return true;
- }
- if (is_done) {
- return false;
- }
- if (res_this->should_stop()) {
- return false;
- }
-
- server_response_reader & rd = res_this->rd;
- if (!rd.has_next()) {
- return false;
- }
- auto result = rd.next([&res_this]() { return res_this->should_stop(); });
- if (!result || result->is_error()) {
- return false;
- }
- auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
- GGML_ASSERT(tts_res != nullptr);
- output = std::move(tts_res->audio);
- is_done = tts_res->final;
- return true;
- });
- }
+ return handle_tts_impl(req, parse_tts_body(req));
+ };
- return res;
+ this->post_speech_oai = [this](const server_http_req & req) {
+ json body = convert_speech_to_tts(parse_tts_body(req));
+ SRV_DBG("%s\n", "Request converted: OpenAI Speech -> TTS");
+ SRV_DBG("converted request: %s\n", body.dump().c_str());
+ return handle_tts_impl(req, std::move(body));
};
this->post_systemone = [this](const server_http_req & req) {
@@ -7215,6 +7049,193 @@ std::unique_ptr<server_res_generator> server_routes::handle_rerank_impl(const se
return res;
}
+std::unique_ptr<server_res_generator> server_routes::handle_tts_impl(const server_http_req & req, json body) {
+ auto res = create_response();
+ res->set_req(&req); // will also set spipe if needed
+
+ if (!ctx_server.has_cap_tts()) {
+ res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED));
+ return res;
+ }
+
+ const auto info = mtmd_gen_audio_get_info(ctx_server.mctx);
+
+ std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string()));
+ if (prompt.empty()) {
+ res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+
+ const std::string response_format = json_value(body, "response_format", std::string("wav"));
+ const bool pcm16 = response_format == "pcm16";
+
+ server_task task(SERVER_TASK_TYPE_TTS);
+ server_schema::eval_tts_schema(params, task, body);
+ const bool stream = task.params.stream;
+ // a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it
+ const std::string lang = json_value(body, "lang", std::string());
+ const std::string voice = json_value(body, "voice", std::string());
+ const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice);
+ // a model with predefined voices only speaks with one of them
+ mtmd::bitmap_ptr voice_ref;
+ if (!voice.empty() && info.n_voices > 0) {
+ voice_ref.reset(mtmd_gen_audio_get_voice(ctx_server.mctx, voice.c_str()));
+ if (!voice_ref) {
+ res->error(format_error_response("unknown voice \"" + voice + "\", see \"voices\" in GET /props", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+ }
+ task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt);
+ task.tts_inp.set_lang(lang);
+ task.tts_inp.data.stream = stream;
+ task.tts_inp.data.out_type = response_format == "pcm" || pcm16
+ ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM
+ : MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
+ // note: -1 is clamped to 0 by the sampler, it will disable the penalty
+ task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512;
+ task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG
+ if (task.tts_inp.data.top_k > 0) {
+ task.params.sampling.top_k = task.tts_inp.data.top_k;
+ }
+ if (task.tts_inp.data.top_p > 0) {
+ task.params.sampling.top_p = task.tts_inp.data.top_p;
+ }
+ if (task.tts_inp.data.temp >= 0) {
+ task.params.sampling.temp = task.tts_inp.data.temp;
+ }
+
+ // speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64")
+ const unsigned char * speaker_ref_data = nullptr;
+ size_t speaker_ref_len = 0;
+ std::string speaker_ref_b64_decoded;
+
+ auto speaker_ref_file = req.files.find("speaker_ref");
+ if (speaker_ref_file != req.files.end()) {
+ speaker_ref_data = speaker_ref_file->second.data.data();
+ speaker_ref_len = speaker_ref_file->second.data.size();
+ } else {
+ std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string());
+ if (!speaker_ref_b64.empty()) {
+ try {
+ speaker_ref_b64_decoded = base64::decode(speaker_ref_b64);
+ } catch (const std::exception &) {
+ res->error(format_error_response("\"speaker_ref_b64\" is not valid base64", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+ speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data();
+ speaker_ref_len = speaker_ref_b64_decoded.size();
+ }
+ }
+
+ if (speaker_ref_len > 0) {
+ if (voice_ref) {
+ res->error(format_error_response("\"voice\" and \"speaker_ref\" cannot be used together", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+ if (!mtmd_support_audio(ctx_server.mctx)) {
+ res->error(format_error_response("this model has no voice encoder, \"speaker_ref\" cannot be used", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+ auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt);
+ if (!wrapper.bitmap) {
+ res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+ task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap));
+ } else if (voice_ref) {
+ task.tts_inp.set_speaker_ref(std::move(voice_ref));
+ } else if (info.n_voices > 0) {
+ // without a voice prompt the model produces near silence: the first voice is the default
+ const char * first = mtmd_gen_audio_voice_name(ctx_server.mctx, 0);
+ SRV_WRN("no voice given, speaking with \"%s\"\n", first);
+ task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(mtmd_gen_audio_get_voice(ctx_server.mctx, first)));
+ } else {
+ SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n");
+ }
+
+ auto & rd = res->rd;
+ task.id = rd.get_new_id();
+ rd.post_task(std::move(task));
+
+ // raw LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586)
+ const std::string pcm_type = "audio/pcm;rate=" + std::to_string(info.sample_rate);
+ const std::string content_type = pcm16 ? pcm_type + ";encoding=signed-int;bits=16"
+ : response_format == "pcm" ? pcm_type + ";encoding=float;bits=32"
+ : "audio/wav";
+
+ // the pipeline outputs float32 samples
+ auto take_audio = [pcm16](server_task_result_tts * tts_res) {
+ const std::string & audio = tts_res->audio;
+ return pcm16 ? pcm_f32_to_s16((const float *) audio.data(), audio.size() / sizeof(float)) : std::move(tts_res->audio);
+ };
+
+ if (!stream) {
+ auto result = rd.next(req.should_stop);
+ if (!result) {
+ GGML_ASSERT(req.should_stop());
+ return res; // connection is closed
+ }
+ if (result->is_error()) {
+ res->error(result->to_json());
+ return res;
+ }
+ auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
+ GGML_ASSERT(tts_res != nullptr);
+ res->status = 200;
+ res->content_type = content_type;
+ res->data = take_audio(tts_res);
+ return res;
+ } else {
+ auto first_result = rd.next(req.should_stop);
+ if (!first_result) {
+ GGML_ASSERT(req.should_stop());
+ return res; // connection is closed
+ }
+ if (first_result->is_error()) {
+ res->error(first_result->to_json());
+ return res;
+ }
+ auto * first_tts_res = dynamic_cast<server_task_result_tts *>(first_result.get());
+ GGML_ASSERT(first_tts_res != nullptr);
+
+ res->status = 200;
+ res->content_type = content_type;
+ res->data = take_audio(first_tts_res);
+ bool is_done = first_tts_res->final;
+
+ res->set_next([res_this = res.get(), is_done, take_audio](std::string & output) mutable -> bool {
+ // flush buffered audio before is_done - the first result can already be final
+ if (!res_this->data.empty()) {
+ output = std::move(res_this->data);
+ res_this->data.clear();
+ return true;
+ }
+ if (is_done) {
+ return false;
+ }
+ if (res_this->should_stop()) {
+ return false;
+ }
+
+ server_response_reader & rd = res_this->rd;
+ if (!rd.has_next()) {
+ return false;
+ }
+ auto result = rd.next([&res_this]() { return res_this->should_stop(); });
+ if (!result || result->is_error()) {
+ return false;
+ }
+ auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
+ GGML_ASSERT(tts_res != nullptr);
+ output = take_audio(tts_res);
+ is_done = tts_res->final;
+ return true;
+ });
+ }
+
+ return res;
+}
+
std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const server_http_req & req, task_response_type res_type) {
auto res = create_response();
std::vector<raw_buffer> files;
diff --git a/tools/server/server-context.h b/tools/server/server-context.h
index 05c538ea1..4b145cb8a 100644
--- a/tools/server/server-context.h
+++ b/tools/server/server-context.h
@@ -134,6 +134,7 @@ struct server_routes {
server_http_context::handler_t post_embeddings_oai;
server_http_context::handler_t post_rerank;
server_http_context::handler_t post_tts;
+ server_http_context::handler_t post_speech_oai;
server_http_context::handler_t post_systemone;
server_http_context::handler_t get_lora_adapters;
server_http_context::handler_t post_lora_adapters;
@@ -154,6 +155,7 @@ private:
std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
std::unique_ptr<server_res_generator> handle_rerank_impl(const server_http_req & req);
+ std::unique_ptr<server_res_generator> handle_tts_impl(const server_http_req & req, json body);
std::unique_ptr<server_res_generator> handle_count_tokens(const server_http_req & req, task_response_type res_type);
// using unique_ptr to allow late initialization of const
diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp
index b608b87ab..cabcedb78 100644
--- a/tools/server/server-realtime.cpp
+++ b/tools/server/server-realtime.cpp
@@ -714,12 +714,7 @@ struct realtime_conversation {
//
static std::string encode_pcm16(const std::vector<float> & samples) {
- std::string bytes(samples.size() * sizeof(int16_t), '\0');
- for (size_t i = 0; i < samples.size(); i++) {
- const int16_t v = (int16_t) std::lround(std::max(-1.0f, std::min(1.0f, samples[i])) * 32767.0f);
- memcpy(&bytes[2 * i], &v, sizeof(v));
- }
- return base64::encode(bytes);
+ return base64::encode(pcm_f32_to_s16(samples.data(), samples.size()));
}
// the message item of the response is added with its first delta
diff --git a/tools/server/server.cpp b/tools/server/server.cpp
index 8b1d42650..4e838cb10 100644
--- a/tools/server/server.cpp
+++ b/tools/server/server.cpp
@@ -186,6 +186,8 @@ static void server_register_routes(server_http_context & ctx_http, server_routes
ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));
ctx_http.post("/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));
ctx_http.post("/tts", ex_wrapper(routes.post_tts));
+ ctx_http.post("/v1/audio/speech", ex_wrapper(routes.post_speech_oai));
+ ctx_http.post("/audio/speech", ex_wrapper(routes.post_speech_oai));
ctx_http.ws ("/v1/realtime", routes.ws_realtime);
ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API
ctx_http.post("/infill", ex_wrapper(routes.post_infill));
@@ -504,6 +506,7 @@ static int server_run_router(common_params & params, int argc, char ** argv, con
routes.post_responses_oai = models_routes->proxy_post;
routes.post_transcriptions_oai = models_routes->proxy_post;
routes.post_tts = models_routes->proxy_post;
+ routes.post_speech_oai = models_routes->proxy_post;
routes.ws_realtime = models_routes->proxy_ws;
routes.post_anthropic_messages = models_routes->proxy_post;
routes.post_anthropic_count_tokens = models_routes->proxy_post;
diff --git a/tools/server/tests/unit/test_compat_oai_speech.py b/tools/server/tests/unit/test_compat_oai_speech.py
new file mode 100644
index 000000000..888942f02
--- /dev/null
+++ b/tools/server/tests/unit/test_compat_oai_speech.py
@@ -0,0 +1,137 @@
+import pytest
+import requests
+import struct
+from utils import *
+
+server: ServerProcess
+
+BODY = {
+ "model": "tts",
+ "input": "Hello there.",
+ "voice": "female",
+ "seed": 42,
+ "n_predict": 8,
+}
+
+
+@pytest.fixture(autouse=True)
+def create_server():
+ global server
+ server = ServerPreset.tinylfm3omni()
+
+
+def post_audio(path: str, **extra) -> requests.Response:
+ return requests.post(server.make_url(path), json={**BODY, **extra}, timeout=DEFAULT_REQUEST_TIMEOUT)
+
+
+def assert_wav(res: requests.Response, data_size: int | None = None):
+ assert res.status_code == 200
+ assert res.headers["Content-Type"] == "audio/wav"
+ assert res.content[:4] == b"RIFF" and res.content[8:12] == b"WAVE"
+ assert struct.unpack("<H", res.content[34:36])[0] == 16 # bits per sample
+ pos = res.content.index(b"data")
+ size = struct.unpack("<I", res.content[pos + 4:pos + 8])[0]
+ assert size == (len(res.content) - pos - 8 if data_size is None else data_size)
+ assert len(res.content) > pos + 8
+
+
+@pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"])
+def test_speech_default_format_is_wav(path: str):
+ server.start()
+ assert_wav(post_audio(path))
+
+
+def test_speech_wav_same_as_tts():
+ server.start()
+ res = post_audio("/v1/audio/speech", response_format="wav")
+ assert_wav(res)
+ assert res.content == post_audio("/tts").content
+
+
+def test_speech_pcm_is_signed_16_bit():
+ server.start()
+ res = post_audio("/v1/audio/speech", response_format="pcm")
+ assert res.status_code == 200
+ assert res.headers["Content-Type"] == "audio/pcm;rate=24000;encoding=signed-int;bits=16"
+
+ # /tts keeps the float samples
+ ref = post_audio("/tts", response_format="pcm")
+ assert ref.headers["Content-Type"] == "audio/pcm;rate=24000;encoding=float;bits=32"
+
+ samples = struct.unpack(f"<{len(res.content) // 2}h", res.content)
+ floats = struct.unpack(f"<{len(ref.content) // 4}f", ref.content)
+ assert len(samples) == len(floats) > 0
+ assert any(v != 0 for v in samples)
+ for s, f in zip(samples, floats):
+ assert abs(s - max(-1.0, min(1.0, f)) * 32767) <= 1
+
+
+def test_speech_stream_pcm():
+ server.start()
+ res = post_audio("/v1/audio/speech", response_format="pcm", stream_format="audio")
+ assert res.status_code == 200
+ assert res.headers.get("Transfer-Encoding") == "chunked"
+ assert res.content == post_audio("/v1/audio/speech", response_format="pcm").content
+
+
+def test_speech_stream_wav_has_unknown_size():
+ server.start()
+ res = post_audio("/v1/audio/speech", stream_format="audio")
+ assert_wav(res, data_size=0xFFFFFFFF)
+
+
+def test_speech_voice():
+ server.start()
+ female = post_audio("/v1/audio/speech", voice="female")
+ male = post_audio("/v1/audio/speech", voice="male")
+ assert_wav(female)
+ assert_wav(male)
+ assert female.content != male.content
+ assert post_audio("/v1/audio/speech", voice={"id": "male"}).content == male.content
+
+
+def test_speech_multipart():
+ server.start()
+ fields = {k: (None, str(v)) for k, v in BODY.items()}
+ res = requests.post(server.make_url("/v1/audio/speech"), files=fields, timeout=DEFAULT_REQUEST_TIMEOUT)
+ assert_wav(res)
+ assert res.content == post_audio("/v1/audio/speech").content
+
+
+@pytest.mark.parametrize("extra", [
+ {"speed": 1.0},
+ {"instructions": "Speak slowly."},
+])
+def test_speech_accepted_params(extra: dict):
+ server.start()
+ assert_wav(post_audio("/v1/audio/speech", **extra))
+
+
+@pytest.mark.parametrize("extra,message", [
+ ({"response_format": "mp3"}, "response_format"),
+ ({"response_format": "opus"}, "response_format"),
+ ({"stream_format": "sse"}, "stream_format"),
+ ({"speed": 1.5}, "speed"),
+ ({"voice": "alloy"}, "unknown voice"),
+ ({"input": ""}, "input"),
+])
+def test_speech_invalid_request(extra: dict, message: str):
+ server.start()
+ res = post_audio("/v1/audio/speech", **extra)
+ assert res.status_code == 400
+ assert message in res.json()["error"]["message"]
+
+
+def test_speech_no_tts_model():
+ global server
+ server = ServerPreset.tinyllama2()
+ server.start()
+ assert post_audio("/v1/audio/speech").status_code == 501 # ERROR_TYPE_NOT_SUPPORTED
+
+
+def test_speech_router():
+ global server
+ model = server.model_hf_repo
+ server = ServerPreset.router()
+ server.start()
+ assert_wav(post_audio("/v1/audio/speech", model=model))
diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py
index c1534fd72..af69fafe0 100644
--- a/tools/server/tests/utils.py
+++ b/tools/server/tests/utils.py
@@ -695,6 +695,20 @@ class ServerPreset:
server.seed = 42
return server
+ @staticmethod
+ def tinylfm3omni() -> ServerProcess:
+ server = ServerProcess()
+ server.offline = True # will be downloaded by load_all()
+ # mmproj is already provided by HF registry API
+ server.model_hf_file = None
+ server.model_hf_repo = "ggml-org/tinyLFM3-Omni-for-testing-gguf:Q2_K"
+ server.model_alias = "tinylfm3omni"
+ server.n_ctx = 1024
+ server.n_batch = 512
+ server.n_slots = 1
+ server.seed = 42
+ return server
+
@staticmethod
def router() -> ServerProcess:
server = ServerProcess()