Commit 3d94ffe45 for llama.cpp
commit 3d94ffe452f42dbedde25277e2e6e97905ce6c02
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Mon Nov 9 23:04:39 2026 +0100
mtmd: add mtmd_helper_resampler (#31964)
* mtmd: add mtmd_helper_resampler
* update docs
* add test
* MA_MAX_FILTER_ORDER
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp
index a0201bdde..3fd720088 100644
--- a/tests/test-mtmd-impl.cpp
+++ b/tests/test-mtmd-impl.cpp
@@ -665,6 +665,85 @@ MAKE_TEST(test_audio_log_mel_blocks) {
t.assert_true("max_diff = " + std::to_string(max_diff), max_diff < 1e-5f);
}
+//
+// mtmd_helper_resampler
+//
+
+static std::vector<float> make_tone(double freq, int rate, size_t n) {
+ const double pi = 3.14159265358979323846;
+ std::vector<float> x(n);
+ for (size_t i = 0; i < n; i++) {
+ x[i] = (float) (0.5 * std::sin(2 * pi * freq * i / rate));
+ }
+ return x;
+}
+
+// amplitude of the freq component, skipping the edges
+static double tone_amplitude(const std::vector<float> & x, double freq, int rate) {
+ const double pi = 3.14159265358979323846;
+ const size_t skip = rate / 10;
+ double re = 0.0, im = 0.0;
+ for (size_t i = skip; i < x.size() - skip; i++) {
+ re += x[i] * std::cos(2 * pi * freq * i / rate);
+ im += x[i] * std::sin(2 * pi * freq * i / rate);
+ }
+ return 2.0 * std::sqrt(re * re + im * im) / (x.size() - 2 * skip);
+}
+
+static std::vector<float> resample_chunked(mtmd_helper_resampler * rs, const std::vector<float> & in, size_t chunk) {
+ std::vector<float> res;
+ for (size_t off = 0; off < in.size(); off += chunk) {
+ const float * out = nullptr;
+ const int64_t n = mtmd_helper_resampler_process(rs, in.data() + off, std::min(chunk, in.size() - off), &out);
+ if (n < 0) {
+ return {};
+ }
+ res.insert(res.end(), out, out + n);
+ }
+ return res;
+}
+
+MAKE_TEST(test_helper_resampler) {
+ const int rate_out = 16000;
+ for (int rate_in : { 8000, 24000, 44100, 48000 }) {
+ const std::string name = std::to_string(rate_in) + " -> " + std::to_string(rate_out);
+ const std::vector<float> in = make_tone(440, rate_in, (size_t) rate_in * 2);
+
+ mtmd_helper::resampler_ptr rs_all(mtmd_helper_resampler_init(rate_in, rate_out));
+ mtmd_helper::resampler_ptr rs_chk(mtmd_helper_resampler_init(rate_in, rate_out));
+ if (!t.assert_true(name + " init", rs_all && rs_chk)) {
+ continue;
+ }
+ const std::vector<float> all = resample_chunked(rs_all.get(), in, in.size());
+ const std::vector<float> chk = resample_chunked(rs_chk.get(), in, 1234); // odd size, not a multiple of the ratio
+
+ t.assert_equal(name + " n_samples", (size_t) rate_out * 2, all.size());
+ t.assert_true(name + " chunked == one-shot", all == chk);
+
+ const double amp = tone_amplitude(all, 440, rate_out);
+ t.assert_true(name + " amplitude = " + std::to_string(amp), std::fabs(amp - 0.5) < 0.02);
+ }
+
+ // above the output Nyquist: 10 kHz at 48 kHz aliases to 6 kHz at 16 kHz, the low-pass filter must cut it
+ {
+ mtmd_helper::resampler_ptr rs(mtmd_helper_resampler_init(48000, rate_out));
+ const std::vector<float> out = resample_chunked(rs.get(), make_tone(10000, 48000, 48000 * 2), 4800);
+ const double db = 20 * std::log10(tone_amplitude(out, 6000, rate_out) / 0.5);
+ t.assert_true("aliasing = " + std::to_string(db) + " dB", db < -15); // about -20 dB with lpfOrder = 8, -10 dB with the default
+ }
+
+ // same rate: no copy
+ {
+ mtmd_helper::resampler_ptr rs(mtmd_helper_resampler_init(rate_out, rate_out));
+ const std::vector<float> in = make_tone(440, rate_out, 1000);
+ const float * out = nullptr;
+ const int64_t n = rs ? mtmd_helper_resampler_process(rs.get(), in.data(), in.size(), &out) : -1;
+ t.assert_true("passthrough", n == (int64_t) in.size() && out == in.data());
+ }
+
+ t.assert_true("invalid rate", mtmd_helper_resampler_init(0, rate_out) == nullptr);
+}
+
//
// main
//
diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp
index d86537e55..aefe504b2 100644
--- a/tools/mtmd/mtmd-helper.cpp
+++ b/tools/mtmd/mtmd-helper.cpp
@@ -360,6 +360,74 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int
} // namespace audio_helpers
+struct mtmd_helper_resampler {
+ ma_resampler rs;
+ bool passthrough = false;
+ std::vector<float> out;
+};
+
+mtmd_helper_resampler * mtmd_helper_resampler_init(int32_t rate_in, int32_t rate_out) {
+ if (rate_in <= 0 || rate_out <= 0) {
+ LOG_ERR("%s: invalid sample rate %d -> %d\n", __func__, rate_in, rate_out);
+ return nullptr;
+ }
+ auto * ctx = new mtmd_helper_resampler;
+ if (rate_in == rate_out) {
+ ctx->passthrough = true;
+ return ctx;
+ }
+ ma_resampler_config config = ma_resampler_config_init(ma_format_f32, 1, (ma_uint32) rate_in, (ma_uint32) rate_out,
+ ma_resample_algorithm_linear);
+ config.linear.lpfOrder = MA_MAX_FILTER_ORDER;
+ if (ma_resampler_init(&config, nullptr, &ctx->rs) != MA_SUCCESS) {
+ LOG_ERR("%s: failed to initialize the resampler %d -> %d\n", __func__, rate_in, rate_out);
+ delete ctx;
+ return nullptr;
+ }
+ return ctx;
+}
+
+void mtmd_helper_resampler_free(mtmd_helper_resampler * ctx) {
+ if (!ctx) {
+ return;
+ }
+ if (!ctx->passthrough) {
+ ma_resampler_uninit(&ctx->rs, nullptr);
+ }
+ delete ctx;
+}
+
+int64_t mtmd_helper_resampler_process(mtmd_helper_resampler * ctx, const float * in, size_t n_in, const float ** out) {
+ if (ctx->passthrough) {
+ *out = in;
+ return (int64_t) n_in;
+ }
+ ma_uint64 n_expected = 0;
+ if (ma_resampler_get_expected_output_frame_count(&ctx->rs, n_in, &n_expected) != MA_SUCCESS) {
+ return -1;
+ }
+ ctx->out.resize(n_expected + 16);
+ size_t n_done_in = 0;
+ size_t n_done_out = 0;
+ while (n_done_in < n_in) {
+ if (n_done_out == ctx->out.size()) {
+ ctx->out.resize(ctx->out.size() * 2);
+ }
+ ma_uint64 cnt_in = n_in - n_done_in;
+ ma_uint64 cnt_out = ctx->out.size() - n_done_out;
+ if (ma_resampler_process_pcm_frames(&ctx->rs, in + n_done_in, &cnt_in, ctx->out.data() + n_done_out, &cnt_out) != MA_SUCCESS) {
+ return -1;
+ }
+ if (cnt_in == 0 && cnt_out == 0) {
+ break;
+ }
+ n_done_in += cnt_in;
+ n_done_out += cnt_out;
+ }
+ *out = ctx->out.data();
+ return (int64_t) n_done_out;
+}
+
static bool is_webp_file(const unsigned char * buf, size_t len) {
// WEBP ref: https://developers.google.com/speed/webp/docs/riff_container
return len >= 12 && memcmp(buf, "RIFF", 4) == 0 && memcmp(buf + 8, "WEBP", 4) == 0;
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index 8a1d3aae1..7d72fdb99 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -78,6 +78,19 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(
bool placeholder,
struct mtmd_helper_init_opt opt);
+// streaming resampler for raw mono F32 PCM (e.g. realtime input), to match mtmd_get_audio_sample_rate()
+// it keeps state between calls, use one per stream
+struct mtmd_helper_resampler;
+typedef struct mtmd_helper_resampler mtmd_helper_resampler;
+
+// returns NULL on failure
+MTMD_API mtmd_helper_resampler * mtmd_helper_resampler_init(int32_t rate_in, int32_t rate_out);
+MTMD_API void mtmd_helper_resampler_free(mtmd_helper_resampler * ctx);
+
+// *out is valid until the next call
+// returns the number of output samples (can be 0), < 0 on error
+MTMD_API int64_t mtmd_helper_resampler_process(mtmd_helper_resampler * ctx, const float * in, size_t n_in, const float ** out);
+
// helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache
MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks);
@@ -334,6 +347,12 @@ struct mtmd_helper_video_deleter {
};
using video_ptr = std::unique_ptr<mtmd_helper_video, mtmd_helper_video_deleter>;
+// audio input-related C++ wrappers
+struct mtmd_helper_resampler_deleter {
+ void operator()(mtmd_helper_resampler * val) { mtmd_helper_resampler_free(val); }
+};
+using resampler_ptr = std::unique_ptr<mtmd_helper_resampler, mtmd_helper_resampler_deleter>;
+
// audio generation-related C++ wrappers
struct mtmd_helper_gen_audio_deleter {
void operator()(mtmd_helper_gen_audio * val) { mtmd_helper_gen_audio_free(val); }
diff --git a/tools/server/README.md b/tools/server/README.md
index dad14aaf1..712205dd5 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -1596,11 +1596,11 @@ Models that write the text while they listen use this endpoint: Voxtral Realtime
Live ASR models can also hold a conversation, so they transcribe live only in sessions opened with `/v1/realtime?intent=transcription`, as OpenAI transcription sessions are; other sessions are a [realtime conversation](#realtime-conversation).
-Audio must be PCM16, mono, at the model sample rate. The sample rate is returned in `session.created` as `audio.input.format.rate`. Each `input_audio_buffer.append` carries a base64-encoded chunk of PCM16 samples.
+Audio must be PCM16, mono. By default it is at the model sample rate, returned in `session.created` as `audio.input.format.rate`. A client can send another rate (8000 to 192000 Hz) by setting `audio.input.format.rate` with `session.update`, the server then resamples it. Each `input_audio_buffer.append` carries a base64-encoded chunk of PCM16 samples.
*Client events:*
-- `session.update`: accepted for compatibility, the server replies with `session.updated`
+- `session.update`: only `audio.input.format.rate` is used, the server replies with `session.updated`
- `input_audio_buffer.append`: `audio` holds base64 PCM16 samples
- `input_audio_buffer.commit`: flushes the audio received so far, so the words still held back by the model's delay are produced. The server replies with `input_audio_buffer.committed` and starts a new item. With `"final": true`, the input ends instead and the last item is completed once the stream drains.
- `input_audio_buffer.clear`: accepted for compatibility, audio already sent cannot be discarded
@@ -1619,7 +1619,7 @@ Note that words held back by the model's delay at the time of a commit are attri
#### Realtime conversation
-Models that answer after the audio, like LFM3-Audio and LFM3-Omni, hold a conversation on the same endpoint (with an `asr` template, unless the session is opened with `?intent=transcription`). The audio is prefilled as it arrives, so the response starts right after the last chunk. Audio input is PCM16 mono at the model sample rate (`audio.input.format.rate` of `session.created`), audio output is PCM16 mono at `audio.output.format.rate`.
+Models that answer after the audio, like LFM3-Audio and LFM3-Omni, hold a conversation on the same endpoint (with an `asr` template, unless the session is opened with `?intent=transcription`). The audio is prefilled as it arrives, so the response starts right after the last chunk. Audio input is PCM16 mono, by default at the model sample rate (`audio.input.format.rate` of `session.created`); another rate set with `session.update` is resampled by the server. Audio output is PCM16 mono at `audio.output.format.rate`.
`session.type` selects the session, it can be changed with `session.update` before the first audio or message:
@@ -1628,7 +1628,7 @@ Models that answer after the audio, like LFM3-Audio and LFM3-Omni, hold a conver
*Client events:*
-- `session.update`: `type`, `instructions` (the system prompt, before the first audio or message), `output_modalities`, `max_output_tokens` (an integer or `"inf"`, the default), `tools` (functions with `name`, `description` and `parameters`, before the first audio or message) and `tool_choice` (`"auto"`, the default, `"none"` or `"required"`)
+- `session.update`: `type`, `audio.input.format.rate` (8000 to 192000 Hz), `instructions` (the system prompt, before the first audio or message), `output_modalities`, `max_output_tokens` (an integer or `"inf"`, the default), `tools` (functions with `name`, `description` and `parameters`, before the first audio or message) and `tool_choice` (`"auto"`, the default, `"none"` or `"required"`)
- `input_audio_buffer.append`: `audio` holds base64 PCM16 samples
- `input_audio_buffer.commit`: ends the audio of the user turn, in a transcription session the transcript follows
- `conversation.item.create`: a user message with `input_text` content, or a `function_call_output` with the `call_id` and `output` of a tool call of the last response
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 407672d5e..be44c3e2e 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -6178,12 +6178,13 @@ void server_routes::init_routes() {
}
const std::string session_id = "sess_" + random_string();
- const json session_config = json {
+ const int model_rate = mtmd_get_audio_sample_rate(ctx_server.mctx);
+ json session_config = json {
{"id", session_id},
{"object", "realtime.transcription_session"},
{"type", "transcription"},
{"audio", {{"input", {
- {"format", {{"type", "audio/pcm"}, {"rate", mtmd_get_audio_sample_rate(ctx_server.mctx)}}},
+ {"format", {{"type", "audio/pcm"}, {"rate", model_rate}}},
{"transcription", {{"model", meta->model_name}}},
{"turn_detection", nullptr},
{"noise_reduction", nullptr},
@@ -6248,6 +6249,8 @@ void server_routes::init_routes() {
// client events are read on this thread, the transcript is sent from the thread below
std::atomic<bool> input_ended{false};
std::thread reader([&]() {
+ int input_rate = model_rate;
+ mtmd_helper::resampler_ptr resampler; // input_rate -> model_rate
std::string msg;
while (conn.read(msg) != 0) {
json ev;
@@ -6259,6 +6262,21 @@ void server_routes::init_routes() {
}
const std::string type = json_value(ev, "type", std::string());
if (type == "session.update" || type == "transcription_session.update") {
+ const json session = json_value(ev, "session", json::object());
+ const json audio = session.is_object() ? session.value("audio", json::object()) : json::object();
+ const json in = audio.is_object() ? audio.value("input", json::object()) : json::object();
+ if (in.is_object() && in.contains("format") && in["format"].is_object()) {
+ const int rate = json_value(in["format"], "rate", input_rate);
+ if (rate < 8000 || rate > 192000) {
+ send_error_event("session.update: the input audio rate must be between 8000 and 192000 Hz", "invalid_request_error");
+ continue;
+ }
+ if (rate != input_rate) {
+ input_rate = rate;
+ resampler.reset();
+ session_config["audio"]["input"]["format"]["rate"] = rate;
+ }
+ }
send_event(json {{"type", "session.updated"}, {"session", session_config}});
} else if (type == "input_audio_buffer.append") {
std::string pcm;
@@ -6274,6 +6292,25 @@ void server_routes::init_routes() {
memcpy(&v, pcm.data() + 2 * i, sizeof(v));
samples[i] = v / 32768.0f;
}
+ if (!resampler) {
+ resampler.reset(mtmd_helper_resampler_init(input_rate, model_rate));
+ if (!resampler) {
+ send_error_event("failed to initialize the audio resampler", "server_error");
+ break;
+ }
+ }
+ const float * out = nullptr;
+ const int64_t n = mtmd_helper_resampler_process(resampler.get(), samples.data(), samples.size(), &out);
+ if (n < 0) {
+ send_error_event("failed to resample the audio", "server_error");
+ break;
+ }
+ if (n == 0) {
+ continue;
+ }
+ if (out != samples.data()) {
+ samples.assign(out, out + n);
+ }
if (!post_control(SERVER_CONTROL_ACTION_AUDIO_APPEND, std::move(samples))) {
break;
}
diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp
index 5ef36a2b8..689626f5f 100644
--- a/tools/server/server-realtime.cpp
+++ b/tools/server/server-realtime.cpp
@@ -156,6 +156,7 @@ struct realtime_conversation {
bool tool_results = false; // tool results went in after the last response, the next one answers them
int n_turns = 0;
std::string next_header; // text that goes in before the next user turn
+ mtmd_helper::resampler_ptr resampler; // input_rate -> model rate
// the item or response the results belong to, shared with the result loop
std::mutex mutex;
@@ -362,7 +363,10 @@ struct realtime_conversation {
memcpy(&v, pcm.data() + 2 * i, sizeof(v));
samples[i] = v / 32768.0f;
}
- if (samples.empty() || !ensure_turn()) {
+ if (samples.empty()) {
+ return;
+ }
+ if (!resample(samples) || samples.empty() || !ensure_turn()) {
return;
}
if (control(SERVER_CONTROL_ACTION_AUDIO_APPEND, {}, std::move(samples))) {
@@ -370,6 +374,26 @@ struct realtime_conversation {
}
}
+ bool resample(std::vector<float> & samples) {
+ if (!resampler) {
+ resampler.reset(mtmd_helper_resampler_init(input_rate, mtmd_get_audio_sample_rate(deps.mctx)));
+ if (!resampler) {
+ send_error("failed to initialize the audio resampler", "server_error");
+ return false;
+ }
+ }
+ const float * out = nullptr;
+ const int64_t n = mtmd_helper_resampler_process(resampler.get(), samples.data(), samples.size(), &out);
+ if (n < 0) {
+ send_error("failed to resample the audio", "server_error");
+ return false;
+ }
+ if (out != samples.data()) {
+ samples.assign(out, out + n);
+ }
+ return true;
+ }
+
// ends the audio of the open turn, it becomes a user item
bool commit() {
if (!has_audio) {
@@ -623,8 +647,16 @@ struct realtime_conversation {
}
const json audio = session.value("audio", json::object());
const json in = audio.value("input", json::object());
- if (in.contains("format") && in["format"].is_object() && json_value(in["format"], "rate", input_rate) != input_rate) {
- send_error("session.update: the input audio must be PCM16 at " + std::to_string(input_rate) + " Hz");
+ if (in.contains("format") && in["format"].is_object()) {
+ const int rate = json_value(in["format"], "rate", input_rate);
+ if (rate < 8000 || rate > 192000) {
+ send_error("session.update: the input audio rate must be between 8000 and 192000 Hz");
+ return;
+ }
+ if (rate != input_rate) {
+ input_rate = rate;
+ resampler.reset();
+ }
}
if (in.contains("transcription") && in["transcription"].is_object()) {
language = json_value(in["transcription"], "language", language);