Commit a2654ec43 for llama.cpp
commit a2654ec43148b8ae827d956276ce2c1eacfd3853
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Sat Nov 7 23:53:11 2026 +0100
mtmd: move log-mel computation to cgraph (#31808)
* mtmd: move log-mel computation to cgraph
* rm stray file
* also include source in mtmd_audio_track
* rm stray file
* rm mtmd_audio_stft_op
* add test
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp
index b837f45da..e9c6ac856 100644
--- a/tests/test-mtmd-impl.cpp
+++ b/tests/test-mtmd-impl.cpp
@@ -1,10 +1,13 @@
#include "testing.h"
+#include "ggml-backend.h"
+#include "ggml-cpp.h"
#include "mtmd-audio.h"
#include "mtmd-helper.h"
#include "mtmd-image.h"
#include "mtmd-internal.h"
+#include <algorithm>
#include <cmath>
#include <cstdint>
#include <cstdlib>
@@ -479,15 +482,6 @@ MAKE_TEST(test_audio_preprocessor_conformer) {
}
}
- mtmd_audio_preprocessor_conformer preproc(hparams);
- preproc.initialize();
- std::vector<mtmd_audio_mel> mels;
- if (!t.assert_true("preprocess", preproc.preprocess(samples.data(), samples.size(), mels))) {
- return;
- }
- const mtmd_audio_mel & mel = mels[0];
- t.assert_equal("n_len", (int64_t) 101, mel.n_len);
-
// reference values from NeMo AudioToMelSpectrogramPreprocessor (liquid-audio 1.3.0)
// { mel bin, frame, value }
const std::vector<std::tuple<int, int, float>> cases = {
@@ -496,12 +490,180 @@ MAKE_TEST(test_audio_preprocessor_conformer) {
{ 24, 60, -0.0476f }, // silence -> quiet tone
{ 100, 20, -0.0913f }, // tones, high bin
};
- for (const auto & [bin, frame, expected] : cases) {
- const float actual = mel.data[bin * mel.n_len + frame];
- t.assert_true("mel[" + std::to_string(bin) + "][" + std::to_string(frame) + "] = " + std::to_string(actual) + ", expected " + std::to_string(expected), std::fabs(actual - expected) < 1e-3f);
+
+ ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr));
+ ggml_backend_t backends[] = { backend.get() };
+ ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true));
+
+ // the result must not depend on the number of threads
+ for (int n_threads : { 1, 4, 7 }) {
+ mtmd_audio_preprocessor_conformer preproc(hparams, sched.get());
+ preproc.mel_config.n_threads = n_threads;
+ preproc.initialize();
+ mtmd_audio_preproc_out out = preproc.preprocess(samples.data(), samples.size());
+ if (!t.assert_true("preprocess", !out.entries.empty() && preproc.compute(out))) {
+ return;
+ }
+ const clip_image_f32 & mel = out.entries[0];
+ t.assert_equal("n_len", 101, mel.nx());
+
+ for (const auto & [bin, frame, expected] : cases) {
+ const float actual = mel.get_ro_buf()[bin * mel.nx() + frame];
+ t.assert_true("mel[" + std::to_string(bin) + "][" + std::to_string(frame) + "] = " + std::to_string(actual) + ", expected " + std::to_string(expected), std::fabs(actual - expected) < 1e-3f);
+ }
}
}
+MAKE_TEST(test_audio_preprocessor_threads) {
+ clip_hparams hparams;
+ hparams.n_mel_bins = 128;
+ hparams.audio_sample_rate = 16000;
+ hparams.audio_n_fft = 512;
+ hparams.audio_window_len = 400;
+ hparams.audio_hop_len = 160;
+
+ // chirp: every frame has a different content
+ std::vector<float> samples(hparams.audio_sample_rate * 5);
+ for (size_t i = 0; i < samples.size(); i++) {
+ samples[i] = (float) (0.3 * std::sin(0.05 * i + 2e-6 * i * i));
+ }
+
+ ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr));
+ ggml_backend_t backends[] = { backend.get() };
+ ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true));
+
+ mtmd_audio_preprocessor_conformer preproc(hparams, sched.get());
+ preproc.initialize();
+ // tokenize only gets the shapes, encode must get the same ones
+ mtmd_audio_preproc_out ref = preproc.preprocess(samples.data(), samples.size());
+ if (!t.assert_true("preprocess", ref.entries.size() == 1 && ref.needs_compute())) {
+ return;
+ }
+ const clip_image_size shape = ref.entries[0].get_size();
+ if (!t.assert_true("compute", preproc.compute(ref))) {
+ return;
+ }
+ t.assert_true("shape", ref.entries.size() == 1 && ref.entries[0].get_size() == shape);
+ t.assert_true("computed", !ref.needs_compute());
+
+ // frames are independent, so the whole output must be bit-exact with the single-threaded one
+ for (int n_threads : { 4, 12 }) {
+ preproc.mel_config.n_threads = n_threads;
+ mtmd_audio_preproc_out out = preproc.preprocess(samples.data(), samples.size());
+ const bool ok = preproc.compute(out);
+ t.assert_true("n_threads = " + std::to_string(n_threads), ok && out.entries[0].get_ro_buf() == ref.entries[0].get_ro_buf());
+ }
+}
+
+MAKE_TEST(test_audio_log_mel_precision) {
+ const int n_mel = 80;
+ const int n_fft = 400;
+ const int hop = 160;
+ const int n_frames = 200;
+
+ mtmd_audio_cache cache;
+ cache.fill_sin_cos_table(n_fft);
+ cache.fill_hann_window(n_fft, true);
+ cache.fill_mel_filterbank_matrix(n_mel, n_fft, 16000);
+
+ ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr));
+ ggml_backend_t backends[] = { backend.get() };
+ ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true));
+
+ mtmd_audio_mel_config config;
+ config.sched = sched.get();
+ config.n_threads = 2;
+ config.n_mel = n_mel;
+ config.n_fft_bins = 1 + n_fft / 2;
+ config.hop_length = hop;
+ config.cache = &cache;
+
+ // a loud tone next to a 74 dB weaker one: rounding the DFT inputs to F16 or TF32 loses the weak one
+ const double pi = 3.14159265358979323846;
+ std::vector<float> samples((size_t) (n_frames - 1) * hop + n_fft);
+ for (size_t i = 0; i < samples.size(); i++) {
+ samples[i] = (float) (0.5 * std::sin(2 * pi * 200 * i / 16000) + 1e-4 * std::sin(2 * pi * 6000 * i / 16000));
+ }
+
+ std::vector<float> out((size_t) n_frames * n_mel);
+ if (!t.assert_true("log mel", clip_audio_log_mel(config, samples.data(), cache.hann_window.data(), n_frames, out.data()))) {
+ return;
+ }
+
+ const int n_bins = n_fft / 2 + 1;
+ double max_diff = 0.0;
+ std::vector<double> power(n_bins);
+ for (int i = 0; i < n_frames; i++) {
+ for (int k = 0; k < n_bins; k++) {
+ double re = 0.0, im = 0.0;
+ for (int j = 0; j < n_fft; j++) {
+ const double x = (double) cache.hann_window[j] * samples[(size_t) i * hop + j];
+ re += x * std::cos(2 * pi * j * k / n_fft);
+ im -= x * std::sin(2 * pi * j * k / n_fft);
+ }
+ power[k] = re * re + im * im;
+ }
+ for (int m = 0; m < n_mel; m++) {
+ double sum = 0.0;
+ for (int k = 0; k < n_bins; k++) {
+ sum += power[k] * cache.filters.data[(size_t) m * n_bins + k];
+ }
+ const double expected = std::log10(std::max(sum, (double) config.mel_floor));
+ max_diff = std::max(max_diff, std::fabs(expected - out[(size_t) m * n_frames + i]));
+ }
+ }
+ t.assert_true("max_diff = " + std::to_string(max_diff), max_diff < 0.05);
+}
+
+MAKE_TEST(test_audio_log_mel_blocks) {
+ const int n_mel = 80;
+ const int n_fft = 400;
+ const int hop = 160;
+ const int n_frames = 8192 + 100; // long audio is split into blocks of 8192 frames
+
+ mtmd_audio_cache cache;
+ cache.fill_sin_cos_table(n_fft);
+ cache.fill_hann_window(n_fft, true);
+ cache.fill_mel_filterbank_matrix(n_mel, n_fft, 16000);
+
+ ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr));
+ ggml_backend_t backends[] = { backend.get() };
+ ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true));
+
+ mtmd_audio_mel_config config;
+ config.sched = sched.get();
+ config.n_threads = 2;
+ config.n_mel = n_mel;
+ config.n_fft_bins = 1 + n_fft / 2;
+ config.hop_length = hop;
+ config.cache = &cache;
+
+ std::vector<float> samples((size_t) (n_frames - 1) * hop + n_fft);
+ for (size_t i = 0; i < samples.size(); i++) {
+ samples[i] = (float) (0.3 * std::sin(0.05 * i + 1e-6 * i * i));
+ }
+
+ // frames are independent: the frames around the seam must not change when computed on their own
+ // not bit-exact, matmul kernels may round differently depending on the shape
+ const int off = 8192 - 50;
+ const int n_sub = n_frames - off;
+ std::vector<float> all((size_t) n_frames * n_mel);
+ std::vector<float> sub((size_t) n_sub * n_mel);
+ const float * window = cache.hann_window.data();
+ if (!t.assert_true("all", clip_audio_log_mel(config, samples.data(), window, n_frames, all.data())) ||
+ !t.assert_true("sub", clip_audio_log_mel(config, samples.data() + (size_t) off * hop, window, n_sub, sub.data()))) {
+ return;
+ }
+
+ float max_diff = 0.0f;
+ for (int j = 0; j < n_mel; j++) {
+ for (int i = 0; i < n_sub; i++) {
+ max_diff = std::max(max_diff, std::fabs(all[(size_t) j * n_frames + off + i] - sub[(size_t) j * n_sub + i]));
+ }
+ }
+ t.assert_true("max_diff = " + std::to_string(max_diff), max_diff < 1e-5f);
+}
+
//
// main
//
@@ -510,6 +672,9 @@ int main(int argc, char ** argv) {
testing t(std::cout);
t.verbose = true;
+ // the audio preprocessors run on the CPU backend
+ ggml_backend_load_all();
+
// the video helper logs every frame at debug level
mtmd_helper_log_set([](ggml_log_level level, const char * text, void *) {
if (level != GGML_LOG_LEVEL_DEBUG) {
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 7ab36a9fc..ac99b762e 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -3,11 +3,13 @@
#include "clip-model.h"
#include "clip-graph.h"
#include "models/models.h"
+#include "mtmd-audio.h"
#include "ggml.h"
#include "ggml-cpp.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
+#include "ggml-cpu.h"
#include "gguf.h"
#include <algorithm>
@@ -17,6 +19,7 @@
#include <cstring>
#include <fstream>
#include <map>
+#include <memory>
#include <random>
#include <stdexcept>
#include <unordered_set>
@@ -144,6 +147,38 @@ static void clip_image_convert_f32_to_u8(const clip_image_f32& src, clip_image_u
#endif
+// ggml_backend_cpu_set_n_threads(backend_cpu, n_threads);
+static void clip_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) {
+ ggml_backend_dev_t dev = ggml_backend_get_device(backend_cpu);
+ ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
+ if (reg) {
+ auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
+ if (ggml_backend_set_n_threads_fn) {
+ ggml_backend_set_n_threads_fn(backend_cpu, n_threads);
+ }
+ }
+}
+
+static std::shared_ptr<ggml_threadpool> clip_threadpool_init(int n_threads) {
+ ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
+ ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
+ if (!reg || n_threads < 2) {
+ return nullptr;
+ }
+ auto * new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new");
+ auto * free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");
+ if (!new_fn || !free_fn) {
+ return nullptr;
+ }
+ ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads);
+ tpp.poll = 0; // the pool is idle most of the time, don't take CPU away from the text model
+ ggml_threadpool_t threadpool = new_fn(&tpp);
+ if (!threadpool) {
+ throw std::runtime_error("failed to create threadpool");
+ }
+ return std::shared_ptr<ggml_threadpool>(threadpool, free_fn);
+}
+
struct clip_ctx {
clip_model model;
@@ -155,6 +190,8 @@ struct clip_ctx {
std::vector<ggml_backend_t> backend_ptrs;
std::vector<ggml_backend_buffer_type_t> backend_buft;
+ std::shared_ptr<ggml_threadpool> threadpool; // shared by the contexts of a model
+
ggml_backend_t backend = nullptr;
ggml_backend_t backend_cpu = nullptr;
ggml_backend_buffer_ptr buf;
@@ -174,13 +211,20 @@ struct clip_ctx {
bool support_batch = false;
- clip_ctx(clip_context_params & ctx_params) {
+ clip_ctx(clip_context_params & ctx_params, std::shared_ptr<ggml_threadpool> threadpool) : threadpool(std::move(threadpool)) {
flash_attn_type = ctx_params.flash_attn_type;
no_alloc = ctx_params.no_alloc;
backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
if (!backend_cpu) {
throw std::runtime_error("failed to initialize CPU backend");
}
+ if (this->threadpool) {
+ ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu));
+ auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool");
+ if (set_threadpool_fn) {
+ set_threadpool_fn(backend_cpu, this->threadpool.get());
+ }
+ }
if (ctx_params.use_gpu) {
if (ctx_params.device != nullptr) {
backend = ggml_backend_dev_init(ctx_params.device, nullptr);
@@ -5130,7 +5174,12 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
clip_ctx * ctx_audio = nullptr;
clip_ctx * ctx_gen_audio = nullptr;
+ std::shared_ptr<ggml_threadpool> threadpool;
+
try {
+ // no_alloc only measures memory, nothing is computed
+ threadpool = clip_threadpool_init(ctx_params.no_alloc ? 1 : ctx_params.n_threads);
+
clip_model_loader loader(fname,
/* skip_tensors */ false,
ctx_params.progress_callback,
@@ -5138,7 +5187,7 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
bool skip_audio = false;
if (loader.has_vision) {
- ctx_vision = new clip_ctx(ctx_params);
+ ctx_vision = new clip_ctx(ctx_params, threadpool);
loader.load_hparams(ctx_vision->model, CLIP_MODALITY_VISION);
loader.load_tensors(*ctx_vision);
loader.init_ctx(*ctx_vision);
@@ -5152,7 +5201,7 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
}
if (loader.has_audio && !skip_audio) {
- ctx_audio = new clip_ctx(ctx_params);
+ ctx_audio = new clip_ctx(ctx_params, threadpool);
loader.load_hparams(ctx_audio->model, CLIP_MODALITY_AUDIO);
loader.load_tensors(*ctx_audio);
loader.init_ctx(*ctx_audio);
@@ -5162,7 +5211,7 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
}
if (loader.has_gen_audio) {
- ctx_gen_audio = new clip_ctx(ctx_params);
+ ctx_gen_audio = new clip_ctx(ctx_params, threadpool);
loader.load_hparams(ctx_gen_audio->model, CLIP_MODALITY_GEN_AUDIO);
// TODO: CUDA has no flash attention kernel for the depthformer head dim 32, enable it once it does
if (ctx_gen_audio->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN && ctx_gen_audio->flash_attn_type == CLIP_FLASH_ATTN_TYPE_AUTO) {
@@ -5641,6 +5690,142 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
return n_patches;
}
+ggml_backend_sched_t clip_get_sched(const struct clip_ctx * ctx) {
+ return ctx->sched.get();
+}
+
+// TODO: remove this and use ggml_mul_mat with GGML_PREC_F32 once all backends honor it for F32 x F32
+// ggml_mul_mat for 2D F32 tensors, always in F32 on CPU: some GPU backends round the inputs to F16 or TF32, which loses the quiet bins of a spectrum
+static void mtmd_mul_mat_prec_op(ggml_tensor * dst, int ith, int nth, void * /*userdata*/) {
+ const ggml_tensor * a = dst->src[0];
+ const ggml_tensor * b = dst->src[1];
+ const int64_t n_k = a->ne[0];
+
+ for (int64_t n = ith; n < b->ne[1]; n += nth) {
+ const float * b_row = (const float *) ((const char *) b->data + n * b->nb[1]);
+ float * dst_row = (float *) ((char *) dst->data + n * dst->nb[1]);
+ for (int64_t m = 0; m < a->ne[1]; m++) {
+ const float * a_row = (const float *) ((const char *) a->data + m * a->nb[1]);
+ // independent partial sums, so that the compiler can vectorize the loop
+ float acc[16] = {};
+ int64_t k = 0;
+ for (; k + 16 <= n_k; k += 16) {
+ for (int l = 0; l < 16; l++) {
+ acc[l] += a_row[k + l] * b_row[k + l];
+ }
+ }
+ float sum = 0.0f;
+ for (; k < n_k; k++) {
+ sum += a_row[k] * b_row[k];
+ }
+ for (int l = 0; l < 16; l++) {
+ sum += acc[l];
+ }
+ dst_row[m] = sum;
+ }
+ }
+}
+
+static ggml_tensor * mtmd_mul_mat_prec(ggml_context * ctx, ggml_tensor * a, ggml_tensor * b) {
+ GGML_ASSERT(a->type == GGML_TYPE_F32 && b->type == GGML_TYPE_F32);
+ GGML_ASSERT(ggml_n_dims(a) <= 2 && ggml_n_dims(b) <= 2 && a->ne[0] == b->ne[0]);
+ GGML_ASSERT(a->nb[0] == sizeof(float) && b->nb[0] == sizeof(float));
+ ggml_tensor * args[] = { a, b };
+ return ggml_custom_4d(ctx, GGML_TYPE_F32, a->ne[1], b->ne[1], 1, 1, args, 2, mtmd_mul_mat_prec_op, GGML_N_TASKS_MAX, nullptr);
+}
+
+bool clip_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out) {
+ ggml_backend_sched_t sched = config.sched;
+ ggml_backend_t backend_cpu = ggml_backend_sched_get_backend(sched, ggml_backend_sched_get_n_backends(sched) - 1);
+ clip_backend_cpu_set_n_threads(backend_cpu, std::max(1, config.n_threads));
+
+ const auto & cache = *config.cache;
+ const auto & filters = cache.filters.data;
+ const int64_t n_fft_bins = config.n_fft_bins;
+ const int64_t frame_size = (n_fft_bins - 1) * 2;
+ const int64_t n_mel = config.n_mel;
+ GGML_ASSERT(filters.size() == (size_t) (n_mel * n_fft_bins));
+ GGML_ASSERT(cache.cos_vals.size() == (size_t) frame_size && cache.sin_vals.size() == (size_t) frame_size);
+
+ // DFT matrix with the window folded in, the first n_fft_bins rows give the real part
+ std::vector<float> dft((size_t) 2 * n_fft_bins * frame_size);
+ for (int64_t k = 0; k < n_fft_bins; k++) {
+ for (int64_t j = 0; j < frame_size; j++) {
+ const int64_t q = (j * k) % frame_size;
+ dft[k * frame_size + j] = window[j] * cache.cos_vals[q];
+ dft[(k + n_fft_bins) * frame_size + j] = window[j] * cache.sin_vals[q];
+ }
+ }
+
+ // the spectrum is several times larger than the result, so long audio is processed in blocks
+ const int64_t n_frames_max = 8192;
+
+ for (int64_t off = 0; off < n_frames; off += n_frames_max) {
+ const int64_t n_cur = std::min(n_frames_max, n_frames - off);
+
+ ggml_init_params params = {
+ /*.mem_size =*/ ggml_tensor_overhead() * 32 + ggml_graph_overhead(),
+ /*.mem_buffer =*/ nullptr,
+ /*.no_alloc =*/ true,
+ };
+ ggml_context_ptr ctx_ptr(ggml_init(params));
+ ggml_context * ctx0 = ctx_ptr.get();
+
+ ggml_tensor * inp_samples = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, (n_cur - 1) * config.hop_length + frame_size);
+ ggml_tensor * inp_dft = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, frame_size, 2 * n_fft_bins);
+ ggml_tensor * inp_filters = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_fft_bins, n_mel);
+ ggml_set_input(inp_samples);
+ ggml_set_input(inp_dft);
+ ggml_set_input(inp_filters);
+
+ ggml_tensor * cur = ggml_cont(ctx0, ggml_view_2d(ctx0, inp_samples, frame_size, n_cur, config.hop_length * ggml_element_size(inp_samples), 0));
+ cur = mtmd_mul_mat_prec(ctx0, inp_dft, cur); // [2 * n_fft_bins, n_cur]
+
+ cur = ggml_sqr(ctx0, cur);
+ cur = ggml_add(ctx0,
+ ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], 0),
+ ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], n_fft_bins * ggml_element_size(cur)));
+ if (config.use_magnitude) {
+ cur = ggml_sqrt(ctx0, cur);
+ }
+
+ cur = mtmd_mul_mat_prec(ctx0, cur, inp_filters); // [n_cur, n_mel]
+ cur = config.mel_floor_add
+ ? ggml_scale_bias(ctx0, cur, 1.0f, config.mel_floor)
+ : ggml_clamp(ctx0, cur, config.mel_floor, FLT_MAX);
+ cur = ggml_log(ctx0, cur);
+ if (!config.use_natural_log) {
+ cur = ggml_scale(ctx0, cur, 1.0f / logf(10.0f));
+ }
+ ggml_set_output(cur);
+
+ ggml_cgraph * gf = ggml_new_graph(ctx0);
+ ggml_build_forward_expand(gf, cur);
+
+ ggml_backend_sched_reset(sched);
+ if (!ggml_backend_sched_alloc_graph(sched, gf)) {
+ LOG_ERR("%s: failed to allocate compute graph\n", __func__);
+ return false;
+ }
+
+ ggml_backend_tensor_set(inp_samples, samples + off * config.hop_length, 0, ggml_nbytes(inp_samples));
+ ggml_backend_tensor_set(inp_dft, dft.data(), 0, ggml_nbytes(inp_dft));
+ ggml_backend_tensor_set(inp_filters, filters.data(), 0, ggml_nbytes(inp_filters));
+
+ auto status = ggml_backend_sched_graph_compute(sched, gf);
+ if (status != GGML_STATUS_SUCCESS) {
+ LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status);
+ return false;
+ }
+
+ for (int64_t j = 0; j < n_mel; j++) {
+ ggml_backend_tensor_get(cur, out + j * n_frames + off, j * cur->nb[1], n_cur * ggml_element_size(cur));
+ }
+ }
+
+ return true;
+}
+
bool clip_image_encode(struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector<float> & out_vec) {
clip_image_f32_batch imgs;
clip_image_f32 img_copy = *img;
@@ -7574,15 +7759,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
GGML_ABORT("Unknown projector type");
}
- // ggml_backend_cpu_set_n_threads(ctx->backend_cpu, n_threads);
- ggml_backend_dev_t dev = ggml_backend_get_device(ctx->backend_cpu);
- ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
- if (reg) {
- auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
- if (ggml_backend_set_n_threads_fn) {
- ggml_backend_set_n_threads_fn(ctx->backend_cpu, params->n_threads);
- }
- }
+ clip_backend_cpu_set_n_threads(ctx->backend_cpu, params->n_threads);
auto status = ggml_backend_sched_graph_compute(ctx->sched.get(), gf);
if (status != GGML_STATUS_SUCCESS) {
diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h
index 6c8dcf31c..f8fd41ef5 100644
--- a/tools/mtmd/clip.h
+++ b/tools/mtmd/clip.h
@@ -60,6 +60,7 @@ struct clip_context_params {
bool no_alloc;
mtmd_progress_callback progress_callback;
void * progress_callback_user_data;
+ int n_threads;
};
struct clip_init_result {
@@ -88,6 +89,13 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img);
// this should be equal to the embedding dimension of the text model
int clip_n_mmproj_embd(const struct clip_ctx * ctx);
+ggml_backend_sched_t clip_get_sched(const struct clip_ctx * ctx);
+
+// compute the log-mel of n_frames frames, the waveform must already be padded and window must hold one whole frame
+// out has the layout [n_frames, n_mel]; like the encoders, must not run at the same time as another pass on config.sched
+struct mtmd_audio_mel_config;
+bool clip_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out);
+
// TODO: remove clip_image_encode() and always use batched version
bool clip_image_encode (struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector<float> & out_vec);
bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, std::vector<float> & out_batch_embd);
diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp
index 9e60d9697..60087b16d 100644
--- a/tools/mtmd/mtmd-audio.cpp
+++ b/tools/mtmd/mtmd-audio.cpp
@@ -4,7 +4,6 @@
#include <cmath>
#include <cstdint>
#include <cstring>
-#include <thread>
#include <vector>
#include <fstream>
#include <algorithm>
@@ -270,100 +269,44 @@ static void ifft(const mtmd_audio_cache & cache, float * in, int N, float * out)
fft_impl<true, false>(cache, in, N, out);
}
-struct filter_params {
- int64_t n_mel;
- int64_t n_fft_bins;
- int32_t hann_window_size;
- int32_t hop_length;
- int32_t sample_rate;
- bool no_padding = false;
- bool center_padding = false;
- float preemph = 0.f;
- bool use_natural_log = false;
- bool norm_per_feature = false;
- bool use_magnitude = false; // |X| instead of |X|^2
- float mel_floor = 5.960464477539063e-08f;
- bool mel_floor_add = false; // log(x + floor) instead of log(max(x, floor))
- bool std_eps_after_sqrt = false; // std + eps instead of sqrt(var + eps)
-};
-
-static void log_mel_spectrogram_worker_thread(int ith,
- const float * hann,
- const std::vector<float> & samples,
- int n_samples,
- int frame_size,
- int frame_step,
- int n_threads,
- const filter_params & params,
- const mtmd_audio_cache & cache,
- mtmd_audio_mel & out) {
- std::vector<float> fft_in(frame_size * 2, 0.0);
- std::vector<float> fft_out(frame_size * 2 * 2 * 2);
-
- int64_t n_fft_bins = params.n_fft_bins;
- int64_t i = ith;
-
- const auto & filters = cache.filters;
-
- // make sure n_fft == 1 + (WHISPER_N_FFT / 2), bin_0 to bin_nyquist
- GGML_ASSERT(n_fft_bins == 1 + (frame_size / 2));
- GGML_ASSERT(cache.sin_vals.size() == cache.cos_vals.size());
- // calculate FFT only when fft_in are not all zero
- for (; i < std::min((int64_t)(n_samples / frame_step + 1), out.n_len); i += n_threads) {
- const int64_t offset = i * frame_step;
-
- // apply Hann window (~10% faster)
- const int valid_len = std::min(frame_size, std::max(0, n_samples - (int)offset));
- for (int j = 0; j < valid_len; j++) {
- fft_in[j] = hann[j] * samples[offset + j];
- }
-
- // fill the rest with zeros
- if (valid_len < frame_size) {
- std::fill(fft_in.begin() + valid_len, fft_in.end(), 0.0);
- }
-
- // FFT
- fft(cache, fft_in.data(), frame_size, fft_out.data());
-
- // Calculate modulus^2 (power) or modulus (magnitude)
- for (int j = 0; j < n_fft_bins; j++) {
- float power = (fft_out[2 * j + 0] * fft_out[2 * j + 0] + fft_out[2 * j + 1] * fft_out[2 * j + 1]);
- fft_out[j] = params.use_magnitude ? sqrtf(power) : power;
- }
-
- // mel spectrogram
- for (int64_t j = 0; j < out.n_mel; j++) {
- double sum = 0.0;
- // unroll loop (suggested by GH user @lunixbochs)
- int k = 0;
- for (k = 0; k < n_fft_bins - 3; k += 4) {
- size_t idx = size_t(j) * size_t(n_fft_bins) + size_t(k);
- sum +=
- fft_out[k + 0] * filters.data[idx + 0] +
- fft_out[k + 1] * filters.data[idx + 1] +
- fft_out[k + 2] * filters.data[idx + 2] +
- fft_out[k + 3] * filters.data[idx + 3];
- }
- // handle n_fft remainder
- for (; k < n_fft_bins; k++) {
- sum += fft_out[k] * filters.data[(size_t)j * n_fft_bins + k];
- }
- sum = params.mel_floor_add ? sum + (double)params.mel_floor : std::max(sum, (double)params.mel_floor);
- sum = params.use_natural_log
- ? log(sum)
- : log10(sum);
- out.data[(size_t)j * out.n_len + i] = sum;
+static void mtmd_audio_fill_entries(std::vector<mtmd_audio_mel> & mels, bool shape_only, std::vector<clip_image_f32> & entries) {
+ entries.clear();
+ for (auto & mel : mels) {
+ GGML_ASSERT(mel.n_len <= INT32_MAX && mel.n_len >= 0);
+ GGML_ASSERT(mel.n_mel <= INT32_MAX && mel.n_mel >= 0);
+ clip_image_f32 entry;
+ entry.set_size({(int) mel.n_len, (int) mel.n_mel}, shape_only || mel.data.empty(), /* is_audio */ true);
+ if (!entry.is_placeholder()) {
+ entry.cpy_buf(mel.data);
}
+ entries.push_back(std::move(entry));
}
+}
- // Otherwise fft_out are all zero
- double sum = params.use_natural_log ? log(1e-10) : log10(1e-10);
- for (; i < out.n_len; i += n_threads) {
- for (int64_t j = 0; j < out.n_mel; j++) {
- out.data[(size_t)j * out.n_len + i] = sum;
+mtmd_audio_preproc_out mtmd_audio_preprocessor::preprocess(const float * samples, size_t n_samples) const {
+ mtmd_audio_preproc_out out;
+ const bool deferred = mel_config.cache != nullptr;
+ std::vector<mtmd_audio_mel> mels;
+ if (process(samples, n_samples, mels, deferred)) {
+ mtmd_audio_fill_entries(mels, deferred, out.entries);
+ if (deferred) {
+ out.samples.assign(samples, samples + n_samples);
}
}
+ return out;
+}
+
+bool mtmd_audio_preprocessor::compute(mtmd_audio_preproc_out & out) const {
+ if (!out.needs_compute()) {
+ return true;
+ }
+ std::vector<mtmd_audio_mel> mels;
+ if (!process(out.samples.data(), out.samples.size(), mels, false)) {
+ return false;
+ }
+ mtmd_audio_fill_entries(mels, false, out.entries);
+ out.samples = {};
+ return true;
}
void mtmd_audio_log_mel_frame(const mtmd_audio_cache & cache, const float * samples, int n_fft, int n_mel, float * out) {
@@ -395,15 +338,16 @@ void mtmd_audio_log_mel_frame(const mtmd_audio_cache & cache, const float * samp
static bool log_mel_spectrogram(
const float * samples,
const int n_samples_in,
- const int n_threads,
- const filter_params & params,
- const mtmd_audio_cache & cache,
+ const mtmd_audio_mel_config & params,
+ const bool shape_only,
mtmd_audio_mel & out) {
//const int64_t t_start_us = ggml_time_us();
out.n_len_org = n_samples_in;
int n_samples = n_samples_in;
+ const auto & cache = *params.cache;
+
// Hann window
const float * hann = cache.hann_window.data();
const int frame_size = (params.n_fft_bins - 1) * 2;
@@ -486,20 +430,8 @@ static bool log_mel_spectrogram(
}
out.data.resize(total_size);
- {
- std::vector<std::thread> workers(n_threads - 1);
- for (int iw = 0; iw < n_threads - 1; ++iw) {
- workers[iw] =
- std::thread(log_mel_spectrogram_worker_thread, iw + 1, hann, std::cref(samples_padded), n_samples,
- frame_size, frame_step, n_threads, std::cref(params), std::cref(cache), std::ref(out));
- }
-
- // main thread
- log_mel_spectrogram_worker_thread(0, hann, samples_padded, n_samples, frame_size, frame_step, n_threads, params,
- cache, out);
- for (int iw = 0; iw < n_threads - 1; ++iw) {
- workers[iw].join();
- }
+ if (!shape_only && !clip_audio_log_mel(params, samples_padded.data(), hann, out.n_len, out.data.data())) {
+ return false;
}
const int64_t effective_n_len = n_samples_in / frame_step;
@@ -572,11 +504,23 @@ void mtmd_audio_preprocessor_whisper::initialize() {
cache.fill_sin_cos_table(hparams.audio_n_fft);
cache.fill_hann_window(hparams.audio_window_len, true);
cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);
+
+ mel_config.n_mel = hparams.n_mel_bins;
+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
+ mel_config.hann_window_size = hparams.audio_window_len;
+ mel_config.hop_length = hparams.audio_hop_len;
+ mel_config.sample_rate = hparams.audio_sample_rate;
+ mel_config.center_padding = false;
+ mel_config.preemph = 0.0f; // disabled
+ mel_config.use_natural_log = false;
+ mel_config.norm_per_feature = false;
+ mel_config.cache = &cache;
}
-bool mtmd_audio_preprocessor_whisper::preprocess(const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_whisper::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool shape_only) const {
if (n_samples == 0) {
// empty audio
return false;
@@ -592,26 +536,13 @@ bool mtmd_audio_preprocessor_whisper::preprocess(const float * s
n_samples = smpl.size();
}
- filter_params params;
- params.n_mel = hparams.n_mel_bins;
- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
- params.hann_window_size = hparams.audio_window_len;
- params.hop_length = hparams.audio_hop_len;
- params.sample_rate = hparams.audio_sample_rate;
- params.center_padding = false;
- params.preemph = 0.0f; // disabled
- params.use_natural_log = false;
- params.norm_per_feature = false;
-
// make sure the cache is initialized
GGML_ASSERT(!cache.sin_vals.empty());
GGML_ASSERT(!cache.cos_vals.empty());
GGML_ASSERT(!cache.filters.data.empty());
mtmd_audio_mel out_full;
- bool ok = log_mel_spectrogram(samples, n_samples,
- 4, // n_threads
- params, cache, out_full);
+ bool ok = log_mel_spectrogram(samples, n_samples, mel_config, shape_only, out_full);
if (!ok) {
return false;
}
@@ -660,11 +591,21 @@ void mtmd_audio_preprocessor_qwen3a::initialize() {
cache.fill_sin_cos_table(hparams.audio_n_fft);
cache.fill_hann_window(hparams.audio_window_len, true);
cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);
+
+ mel_config.n_mel = hparams.n_mel_bins;
+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
+ mel_config.hann_window_size = hparams.audio_window_len;
+ mel_config.hop_length = hparams.audio_hop_len;
+ mel_config.sample_rate = hparams.audio_sample_rate;
+ mel_config.no_padding = true; // reflection padding applied in preprocess()
+ mel_config.use_natural_log = false; // log10
+ mel_config.cache = &cache;
}
-bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_qwen3a::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool shape_only) const {
if (n_samples == 0) {
return false;
}
@@ -689,17 +630,8 @@ bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * sa
padded[n_samples + pad + i] = (src >= 0) ? samples[src] : 0.0f;
}
- filter_params params;
- params.n_mel = hparams.n_mel_bins;
- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
- params.hann_window_size = hparams.audio_window_len;
- params.hop_length = hparams.audio_hop_len;
- params.sample_rate = hparams.audio_sample_rate;
- params.no_padding = true; // reflection padding already applied above
- params.use_natural_log = false; // log10
-
mtmd_audio_mel mel_full;
- bool ok = log_mel_spectrogram(padded.data(), (int)padded.size(), 4, params, cache, mel_full);
+ bool ok = log_mel_spectrogram(padded.data(), (int)padded.size(), mel_config, shape_only, mel_full);
if (!ok) {
return false;
}
@@ -762,11 +694,21 @@ void mtmd_audio_preprocessor_dots3note::initialize() {
cache.fill_sin_cos_table(hparams.audio_n_fft);
cache.fill_hann_window(hparams.audio_window_len, true);
cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);
+
+ mel_config.n_mel = hparams.n_mel_bins;
+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
+ mel_config.hann_window_size = hparams.audio_window_len;
+ mel_config.hop_length = hparams.audio_hop_len;
+ mel_config.sample_rate = hparams.audio_sample_rate;
+ mel_config.no_padding = true; // padding applied in preprocess()
+ mel_config.use_natural_log = false;
+ mel_config.cache = &cache;
}
-bool mtmd_audio_preprocessor_dots3note::preprocess(const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_dots3note::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool shape_only) const {
if (n_samples == 0) {
return false;
}
@@ -803,17 +745,8 @@ bool mtmd_audio_preprocessor_dots3note::preprocess(const float *
}
}
- filter_params params;
- params.n_mel = hparams.n_mel_bins;
- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
- params.hann_window_size = hparams.audio_window_len;
- params.hop_length = hop;
- params.sample_rate = hparams.audio_sample_rate;
- params.no_padding = true; // padding already applied above
- params.use_natural_log = false;
-
mtmd_audio_mel mel_full;
- if (!log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, mel_full)) {
+ if (!log_mel_spectrogram(padded.data(), (int) padded.size(), mel_config, shape_only, mel_full)) {
return false;
}
GGML_ASSERT(mel_full.n_len >= n_valid);
@@ -862,11 +795,24 @@ void mtmd_audio_preprocessor_mimo_audio::initialize() {
/*scale=*/ 1.0f,
/*use_htk=*/ true
);
+
+ mel_config.n_mel = hparams.n_mel_bins;
+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
+ mel_config.hann_window_size = hparams.audio_window_len;
+ mel_config.hop_length = hparams.audio_hop_len;
+ mel_config.sample_rate = hparams.audio_sample_rate;
+ mel_config.no_padding = true; // reflect padding applied in preprocess()
+ mel_config.use_natural_log = true;
+ mel_config.use_magnitude = true;
+ mel_config.mel_floor = 1e-7f;
+ mel_config.norm_per_feature = false;
+ mel_config.cache = &cache;
}
-bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_mimo_audio::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool shape_only) const {
if (n_samples == 0) {
return false;
}
@@ -888,20 +834,8 @@ bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float *
padded[n_samples + pad + i] = (src >= 0) ? samples[src] : 0.0f;
}
- filter_params params;
- params.n_mel = hparams.n_mel_bins;
- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
- params.hann_window_size = hparams.audio_window_len;
- params.hop_length = hparams.audio_hop_len;
- params.sample_rate = hparams.audio_sample_rate;
- params.no_padding = true; // reflect padding already applied above
- params.use_natural_log = true;
- params.use_magnitude = true;
- params.mel_floor = 1e-7f;
- params.norm_per_feature = false;
-
mtmd_audio_mel out;
- bool ok = log_mel_spectrogram(padded.data(), (int)padded.size(), 4, params, cache, out);
+ bool ok = log_mel_spectrogram(padded.data(), (int)padded.size(), mel_config, shape_only, out);
if (!ok) {
return false;
}
@@ -921,11 +855,23 @@ void mtmd_audio_preprocessor_qwen3tts_spk::initialize() {
cache.fill_sin_cos_table(hparams.audio_n_fft);
cache.fill_hann_window(hparams.audio_window_len, true);
cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);
+
+ mel_config.n_mel = hparams.n_mel_bins;
+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
+ mel_config.hann_window_size = hparams.audio_window_len;
+ mel_config.hop_length = hparams.audio_hop_len;
+ mel_config.sample_rate = hparams.audio_sample_rate;
+ mel_config.no_padding = true; // reflect padding applied in preprocess()
+ mel_config.use_natural_log = true;
+ mel_config.use_magnitude = true;
+ mel_config.mel_floor = 1e-5f;
+ mel_config.cache = &cache;
}
-bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_qwen3tts_spk::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool shape_only) const {
if (n_samples == 0) {
return false;
}
@@ -949,19 +895,8 @@ bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float *
padded[n_samples + pad + i] = samples[n_samples - 2 - i];
}
- filter_params params;
- params.n_mel = hparams.n_mel_bins;
- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
- params.hann_window_size = hparams.audio_window_len;
- params.hop_length = hparams.audio_hop_len;
- params.sample_rate = hparams.audio_sample_rate;
- params.no_padding = true; // reflect padding already applied above
- params.use_natural_log = true;
- params.use_magnitude = true;
- params.mel_floor = 1e-5f;
-
mtmd_audio_mel out;
- bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, out);
+ bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), mel_config, shape_only, out);
if (!ok) {
return false;
}
@@ -1294,9 +1229,10 @@ void mtmd_audio_preprocessor_chatterbox_ref::initialize() {
matcha_cache.fill_mel_filterbank_matrix(80, 1920, 24000, 0.0f, 8000.0f);
}
-bool mtmd_audio_preprocessor_chatterbox_ref::preprocess(const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_chatterbox_ref::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool /*shape_only*/) const {
output.clear();
const int sr = (int) hparams.audio_sample_rate;
@@ -1453,38 +1389,37 @@ void mtmd_audio_preprocessor_conformer::initialize() {
// NeMo uses a symmetric window: torch.hann_window(periodic=False)
cache.fill_hann_window(hparams.audio_window_len, false);
cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);
+
+ mel_config.n_mel = hparams.n_mel_bins;
+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
+ mel_config.hann_window_size = hparams.audio_window_len;
+ mel_config.hop_length = hparams.audio_hop_len;
+ mel_config.sample_rate = hparams.audio_sample_rate;
+ mel_config.center_padding = true;
+ mel_config.preemph = 0.97f;
+ mel_config.use_natural_log = true;
+ mel_config.norm_per_feature = true;
+ mel_config.mel_floor_add = true;
+ mel_config.std_eps_after_sqrt = true;
+ mel_config.cache = &cache;
}
-bool mtmd_audio_preprocessor_conformer::preprocess(const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_conformer::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool shape_only) const {
// empty audio
if (n_samples == 0) {
return false;
}
- filter_params params;
- params.n_mel = hparams.n_mel_bins;
- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
- params.hann_window_size = hparams.audio_window_len;
- params.hop_length = hparams.audio_hop_len;
- params.sample_rate = hparams.audio_sample_rate;
- params.center_padding = true;
- params.preemph = 0.97f;
- params.use_natural_log = true;
- params.norm_per_feature = true;
- params.mel_floor_add = true;
- params.std_eps_after_sqrt = true;
-
// make sure the cache is initialized
GGML_ASSERT(!cache.sin_vals.empty());
GGML_ASSERT(!cache.cos_vals.empty());
GGML_ASSERT(!cache.filters.data.empty());
mtmd_audio_mel out_full;
- bool ok = log_mel_spectrogram(samples, n_samples,
- 4, // n_threads
- params, cache, out_full);
+ bool ok = log_mel_spectrogram(samples, n_samples, mel_config, shape_only, out_full);
if (!ok) {
return false;
}
@@ -1497,9 +1432,10 @@ bool mtmd_audio_preprocessor_conformer::preprocess(const float *
// mtmd_audio_preprocessor_d1omni
//
-bool mtmd_audio_preprocessor_d1omni::preprocess(const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_d1omni::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool shape_only) const {
if (n_samples == 0) {
return false;
}
@@ -1509,7 +1445,7 @@ bool mtmd_audio_preprocessor_d1omni::preprocess(const float * sa
std::vector<float> buf(samples, samples + std::min(n_samples, n_max));
buf.resize(std::max(buf.size(), n_min), 0.0f);
- if (!mtmd_audio_preprocessor_conformer::preprocess(buf.data(), buf.size(), output)) {
+ if (!mtmd_audio_preprocessor_conformer::process(buf.data(), buf.size(), output, shape_only)) {
return false;
}
@@ -1537,11 +1473,21 @@ void mtmd_audio_preprocessor_lfm3a::initialize() {
cache.fill_sin_cos_table(hparams.audio_n_fft);
cache.fill_hann_window(hparams.audio_window_len, true);
cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);
+
+ mel_config.n_mel = hparams.n_mel_bins;
+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
+ mel_config.hann_window_size = hparams.audio_window_len;
+ mel_config.hop_length = hparams.audio_hop_len;
+ mel_config.sample_rate = hparams.audio_sample_rate;
+ mel_config.no_padding = true;
+ mel_config.mel_floor = 1e-12f; // log10 floor of -12
+ mel_config.cache = &cache;
}
-bool mtmd_audio_preprocessor_lfm3a::preprocess(const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_lfm3a::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool shape_only) const {
if (n_samples == 0) {
return false;
}
@@ -1557,21 +1503,12 @@ bool mtmd_audio_preprocessor_lfm3a::preprocess(const float * sam
std::vector<float> padded(samples, samples + n_samples);
padded.resize(n_padded, 0.0f);
- filter_params params;
- params.n_mel = hparams.n_mel_bins;
- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
- params.hann_window_size = hparams.audio_window_len;
- params.hop_length = hparams.audio_hop_len;
- params.sample_rate = hparams.audio_sample_rate;
- params.no_padding = true;
- params.mel_floor = 1e-12f; // log10 floor of -12
-
GGML_ASSERT(!cache.sin_vals.empty());
GGML_ASSERT(!cache.cos_vals.empty());
GGML_ASSERT(!cache.filters.data.empty());
mtmd_audio_mel out;
- bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, out);
+ bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), mel_config, shape_only, out);
if (!ok) {
return false;
}
@@ -1594,11 +1531,25 @@ void mtmd_audio_preprocessor_granite_speech::initialize() {
cache.fill_mel_filterbank_matrix(
hparams.n_mel_bins / 2, hparams.audio_n_fft, hparams.audio_sample_rate,
0.0f, -1.0f, false, 1.0f, true);
+
+ mel_config.n_mel = hparams.n_mel_bins / 2;
+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
+ mel_config.hann_window_size = hparams.audio_window_len;
+ mel_config.hop_length = hparams.audio_hop_len;
+ mel_config.sample_rate = hparams.audio_sample_rate;
+ mel_config.no_padding = true;
+ mel_config.center_padding = false;
+ mel_config.preemph = 0.0f;
+ mel_config.use_natural_log = false;
+ mel_config.norm_per_feature = false;
+ mel_config.mel_floor = 1e-10f;
+ mel_config.cache = &cache;
}
-bool mtmd_audio_preprocessor_granite_speech::preprocess(const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_granite_speech::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool shape_only) const {
if (n_samples == 0) {
return false;
}
@@ -1629,21 +1580,8 @@ bool mtmd_audio_preprocessor_granite_speech::preprocess(const float *
padded[pad + (int)n_samples + i] = samples[src];
}
- filter_params params;
- params.n_mel = hparams.n_mel_bins / 2;
- params.n_fft_bins = 1 + (n_fft / 2);
- params.hann_window_size = hparams.audio_window_len;
- params.hop_length = hparams.audio_hop_len;
- params.sample_rate = hparams.audio_sample_rate;
- params.no_padding = true;
- params.center_padding = false;
- params.preemph = 0.0f;
- params.use_natural_log = false;
- params.norm_per_feature = false;
- params.mel_floor = 1e-10f;
-
mtmd_audio_mel mel;
- if (!log_mel_spectrogram(padded.data(), n_padded, 4, params, cache, mel)) {
+ if (!log_mel_spectrogram(padded.data(), n_padded, mel_config, shape_only, mel)) {
return false;
}
@@ -1708,11 +1646,26 @@ void mtmd_audio_preprocessor_gemma4a::initialize() {
/*scale=*/ 1.0f,
/*use_htk=*/ true
);
+
+ mel_config.n_mel = hparams.n_mel_bins;
+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
+ mel_config.hann_window_size = hparams.audio_n_fft; // window is zero-padded to FFT size
+ mel_config.hop_length = hparams.audio_hop_len;
+ mel_config.sample_rate = hparams.audio_sample_rate;
+ mel_config.no_padding = true;
+ mel_config.center_padding = false;
+ mel_config.preemph = 0.0f;
+ mel_config.use_natural_log = true;
+ mel_config.use_magnitude = true;
+ mel_config.mel_floor = 0.001f;
+ mel_config.norm_per_feature = false;
+ mel_config.cache = &cache;
}
-bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_gemma4a::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool shape_only) const {
if (n_samples == 0) {
return false;
}
@@ -1721,20 +1674,6 @@ bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * s
GGML_ASSERT(!cache.cos_vals.empty());
GGML_ASSERT(!cache.filters.data.empty());
- filter_params params;
- params.n_mel = hparams.n_mel_bins;
- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
- params.hann_window_size = hparams.audio_n_fft; // window is zero-padded to FFT size
- params.hop_length = hparams.audio_hop_len;
- params.sample_rate = hparams.audio_sample_rate;
- params.no_padding = true;
- params.center_padding = false;
- params.preemph = 0.0f;
- params.use_natural_log = true;
- params.use_magnitude = true;
- params.mel_floor = 0.001f;
- params.norm_per_feature = false;
-
// Split into 30-second chunks (model context limit, ~750 tokens each)
const size_t chunk_samples = 30 * hparams.audio_sample_rate;
for (size_t off = 0; off < n_samples; off += chunk_samples) {
@@ -1754,7 +1693,7 @@ bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * s
std::copy(chunk_ptr, chunk_ptr + chunk_len, padded_samples.data() + pad_left);
mtmd_audio_mel out_chunk;
- bool ok = log_mel_spectrogram(padded_samples.data(), padded_samples.size(), 4, params, cache, out_chunk);
+ bool ok = log_mel_spectrogram(padded_samples.data(), padded_samples.size(), mel_config, shape_only, out_chunk);
if (!ok) {
return false;
}
@@ -1772,79 +1711,6 @@ bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * s
// mtmd_audio_preprocessor_parakeet implementation
//
-void mtmd_audio_preprocessor_parakeet::worker_thread(
- int ith,
- const float * window_func,
- int window_size,
- const std::vector<float> & samples,
- int n_samples,
- int frame_size,
- int frame_step,
- int n_threads,
- int n_fft_bins,
- const mtmd_audio_cache & cache,
- mtmd_audio_mel & mel) {
- std::vector<float> fft_in(frame_size * 2, 0.0);
- std::vector<float> fft_out(frame_size * 2 * 2 * 2);
-
- int n_fb = n_fft_bins;
- int i = ith;
-
- GGML_ASSERT(n_fb == 1 + (frame_size / 2));
-
- const double eps = 5.960464477539063e-08;
-
- for (; i < std::min(n_samples / frame_step + 1, (int) mel.n_len); i += n_threads) {
- const int offset = i * frame_step;
- const int window_pad_left = (frame_size - window_size) / 2;
-
- // Zero-pad left.
- std::fill(fft_in.begin(), fft_in.begin() + window_pad_left, 0.0f);
-
- // Apply windowed samples in the center.
- const int n_to_process = std::min({window_size, n_samples - offset});
- for (int j = 0; j < n_to_process; j++) {
- fft_in[window_pad_left + j] = window_func[j] * samples[offset + window_pad_left + j];
- }
-
- // Zero-pad right.
- std::fill(fft_in.begin() + window_pad_left + n_to_process, fft_in.begin() + frame_size, 0.0f);
-
- // FFT.
- fft(cache, fft_in.data(), frame_size, fft_out.data());
-
- // Calculate modulus^2 of complex numbers.
- for (int j = 0; j < n_fb; j++) {
- fft_out[j] = (fft_out[2 * j + 0] * fft_out[2 * j + 0] + fft_out[2 * j + 1] * fft_out[2 * j + 1]);
- }
-
- // mel spectrogram.
- for (int j = 0; j < mel.n_mel; j++) {
- double sum = 0.0;
- int k = 0;
- for (k = 0; k < n_fb - 3; k += 4) {
- sum +=
- fft_out[k + 0] * cache.filters.data[j * n_fb + k + 0] +
- fft_out[k + 1] * cache.filters.data[j * n_fb + k + 1] +
- fft_out[k + 2] * cache.filters.data[j * n_fb + k + 2] +
- fft_out[k + 3] * cache.filters.data[j * n_fb + k + 3];
- }
- for (; k < n_fb; k++) {
- sum += fft_out[k] * cache.filters.data[j * n_fb + k];
- }
- mel.data[j * mel.n_len + i] = std::log(sum + eps);
- }
- }
-
- // Otherwise fft_out are all zero.
- const double empty_sum = std::log(eps);
- for (; i < mel.n_len; i += n_threads) {
- for (int j = 0; j < mel.n_mel; j++) {
- mel.data[j * mel.n_len + i] = empty_sum;
- }
- }
-}
-
void mtmd_audio_preprocessor_parakeet::initialize() {
cache.fill_sin_cos_table(hparams.audio_n_fft);
@@ -1857,25 +1723,25 @@ void mtmd_audio_preprocessor_parakeet::initialize() {
GGML_ASSERT(hparams.window.size() == (size_t)hparams.audio_window_len);
GGML_ASSERT(hparams.window.size() <= (size_t) hparams.audio_n_fft);
cache.hann_window = hparams.window;
+
+ mel_config.n_mel = hparams.n_mel_bins;
+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
+ mel_config.hann_window_size = hparams.audio_window_len;
+ mel_config.hop_length = hparams.audio_hop_len;
+ mel_config.sample_rate = hparams.audio_sample_rate;
+ mel_config.no_padding = true; // preemphasis and padding already applied in log_mel()
+ mel_config.use_natural_log = true;
+ mel_config.mel_floor_add = true;
+ mel_config.cache = &cache;
}
// log-mel without normalization
-void mtmd_audio_preprocessor_parakeet::log_mel(const float * samples, size_t n_samples_in, mtmd_audio_mel & out_full) const {
- filter_params params;
- params.n_mel = hparams.n_mel_bins;
- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
- params.hann_window_size = hparams.audio_window_len;
- params.hop_length = hparams.audio_hop_len;
- params.sample_rate = hparams.audio_sample_rate;
-
+bool mtmd_audio_preprocessor_parakeet::log_mel(const float * samples, size_t n_samples_in, mtmd_audio_mel & out_full, bool shape_only) const {
GGML_ASSERT(!cache.sin_vals.empty());
GGML_ASSERT(!cache.cos_vals.empty());
GGML_ASSERT(!cache.filters.data.empty());
- const float * window_func = cache.hann_window.data();
- const int window_size = params.hann_window_size;
- const int frame_size = (params.n_fft_bins - 1) * 2;
- const int frame_step = params.hop_length;
+ const int frame_size = (mel_config.n_fft_bins - 1) * 2;
// Apply preemphasis filter (high-pass): x[i] = x[i] - 0.97 * x[i-1]
std::vector<float> samples_preprocessed(samples, samples + n_samples_in);
@@ -1891,55 +1757,25 @@ void mtmd_audio_preprocessor_parakeet::log_mel(const float * samples, size_t n_s
std::vector<float> samples_padded(n_samples_in + 2 * pad, 0.0f);
std::copy(samples_preprocessed.begin(), samples_preprocessed.end(), samples_padded.begin() + pad);
- out_full.n_mel = params.n_mel;
- out_full.n_len = (samples_padded.size() - frame_size) / frame_step + 1;
- out_full.n_len_org = out_full.n_len;
- out_full.data.resize(out_full.n_mel * out_full.n_len);
-
- const int n_threads = 4;
- std::vector<std::thread> workers(n_threads - 1);
- for (int iw = 0; iw < n_threads - 1; ++iw) {
- workers[iw] = std::thread(
- worker_thread, iw + 1,
- window_func,
- window_size,
- std::cref(samples_padded),
- samples_padded.size(),
- frame_size,
- frame_step,
- n_threads,
- params.n_fft_bins,
- std::cref(cache),
- std::ref(out_full)
- );
- }
-
- worker_thread(0,
- window_func,
- window_size,
- samples_padded,
- samples_padded.size(),
- frame_size,
- frame_step,
- n_threads,
- params.n_fft_bins,
- cache,
- out_full);
-
- for (int iw = 0; iw < n_threads - 1; ++iw) {
- workers[iw].join();
+ if (!log_mel_spectrogram(samples_padded.data(), (int) samples_padded.size(), mel_config, shape_only, out_full)) {
+ return false;
}
+ out_full.n_len_org = out_full.n_len;
+ return true;
}
-bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples,
- size_t n_samples_in,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_parakeet::process(const float * samples,
+ size_t n_samples_in,
+ std::vector<mtmd_audio_mel> & output,
+ bool shape_only) const {
if (n_samples_in == 0) {
return false;
}
mtmd_audio_mel out_full;
- log_mel(samples, n_samples_in, out_full);
+ if (!log_mel(samples, n_samples_in, out_full, shape_only)) {
+ return false;
+ }
// Per-feature normalization (only on valid frames)
{
@@ -1980,15 +1816,18 @@ bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples,
// mtmd_audio_preprocessor_nemotron_transcribe implementation
//
-bool mtmd_audio_preprocessor_nemotron_transcribe::preprocess(const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_nemotron_transcribe::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool shape_only) const {
if (n_samples < (size_t) hparams.audio_n_fft) {
return false;
}
mtmd_audio_mel mel;
- log_mel(samples, n_samples, mel);
+ if (!log_mel(samples, n_samples, mel, shape_only)) {
+ return false;
+ }
// frames past n_samples / hop are padding (NeMo get_seq_len); kept raw, encode_windows normalizes them for ASR
const int64_t n_frames = n_samples / hparams.audio_hop_len;
@@ -2010,9 +1849,10 @@ void mtmd_audio_preprocessor_gemma4ua::initialize() {
// no-op: no FFT or filterbank needed
}
-bool mtmd_audio_preprocessor_gemma4ua::preprocess(const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_gemma4ua::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool /*shape_only*/) const {
if (n_samples == 0) {
return false;
}
@@ -2151,9 +1991,10 @@ std::vector<float> mtmd_audio_streaming_istft::flush() {
// the samples are handed over as a single-row "mel", to reuse the normal chunk path
//
-bool mtmd_audio_preprocessor_pockettts::preprocess(const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_pockettts::process(const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool /*shape_only*/) const {
// the encoder needs whole frames, see pad_for_conv1d() in the reference
const int64_t frame_size = (int64_t) hparams.mimi_downsample * 120;
if (n_samples == 0 || frame_size <= 0) {
@@ -2183,10 +2024,11 @@ bool mtmd_audio_preprocessor_pockettts::preprocess(const float *
}
// Muse Spark reads the raw waveform: pass the samples through, zero padded to a whole number of embeddings
-bool mtmd_audio_preprocessor_muse_spark_audio::preprocess(
- const float * samples,
- size_t n_samples,
- std::vector<mtmd_audio_mel> & output) const {
+bool mtmd_audio_preprocessor_muse_spark_audio::process(
+ const float * samples,
+ size_t n_samples,
+ std::vector<mtmd_audio_mel> & output,
+ bool /*shape_only*/) const {
const int64_t fold = hparams.audio_samples_per_token();
const int64_t n_padded = ((int64_t) n_samples + fold - 1) / fold * fold;
diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h
index ca95b217f..c0751b5e6 100644
--- a/tools/mtmd/mtmd-audio.h
+++ b/tools/mtmd/mtmd-audio.h
@@ -1,7 +1,9 @@
#pragma once
#include "ggml.h"
+#include "ggml-backend.h"
#include "clip-model.h"
+#include "clip-impl.h"
#include <cstdint>
#include <vector>
@@ -53,21 +55,69 @@ struct mtmd_audio_cache {
// compute one log10-mel frame from n_fft windowed samples, cache must be initialized for n_fft
void mtmd_audio_log_mel_frame(const mtmd_audio_cache & cache, const float * samples, int n_fft, int n_mel, float * out);
+struct mtmd_audio_mel_config {
+ int64_t n_mel = 0;
+ int64_t n_fft_bins = 0;
+ int32_t hann_window_size = 0;
+ int32_t hop_length = 0;
+ int32_t sample_rate = 0;
+ bool no_padding = false;
+ bool center_padding = false;
+ float preemph = 0.f;
+ bool use_natural_log = false;
+ bool norm_per_feature = false;
+ bool use_magnitude = false; // |X| instead of |X|^2
+ float mel_floor = 5.960464477539063e-08f;
+ bool mel_floor_add = false; // log(x + floor) instead of log(max(x, floor))
+ bool std_eps_after_sqrt = false; // std + eps instead of sqrt(var + eps)
+ int n_threads = 1;
+
+ const mtmd_audio_cache * cache = nullptr;
+ ggml_backend_sched_t sched = nullptr;
+};
+
+struct mtmd_audio_preproc_out {
+ std::vector<clip_image_f32> entries; // one per chunk, empty on failure
+ std::vector<float> samples; // input of compute(), empty if entries already hold the data
+
+ bool needs_compute() const {
+ return !samples.empty() && (entries.empty() || entries[0].is_placeholder());
+ }
+};
+
struct mtmd_audio_preprocessor {
const clip_hparams & hparams;
+ mtmd_audio_mel_config mel_config; // filled by initialize() of the preprocessors that use the log-mel graph
+
+ mtmd_audio_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {
+ mel_config.sched = clip_get_sched(ctx);
+ }
+ mtmd_audio_preprocessor(const clip_hparams & hparams, ggml_backend_sched_t sched): hparams(hparams) {
+ mel_config.sched = sched;
+ }
- mtmd_audio_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {}
- mtmd_audio_preprocessor(const clip_hparams & hparams): hparams(hparams) {}
+ // mel_config points to the cache of this instance
+ mtmd_audio_preprocessor(const mtmd_audio_preprocessor &) = delete;
+ mtmd_audio_preprocessor & operator=(const mtmd_audio_preprocessor &) = delete;
virtual ~mtmd_audio_preprocessor() = default;
virtual void initialize() = 0; // NOT thread-safe
- virtual bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const = 0;
+
+ // entries have the final shapes, but no data if needs_compute()
+ mtmd_audio_preproc_out preprocess(const float * samples, size_t n_samples) const;
+
+ // run it on the thread that encodes, it uses mel_config.sched
+ bool compute(mtmd_audio_preproc_out & out) const;
+
+ protected:
+ // shape_only skips the log-mel graph: the shapes are right, the data is zero-filled
+ virtual bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const = 0;
};
struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_whisper(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
@@ -76,14 +126,14 @@ struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_muse_spark_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override {}
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
- mtmd_audio_preprocessor_conformer(const clip_hparams & hparams) : mtmd_audio_preprocessor(hparams) {}
+ mtmd_audio_preprocessor_conformer(const clip_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {}
void initialize() override;
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
@@ -92,13 +142,13 @@ struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
// same as conformer, the audio is cut to 30 s and padded to 0.5 s (d1-omni audio.py)
struct mtmd_audio_preprocessor_d1omni : mtmd_audio_preprocessor_conformer {
using mtmd_audio_preprocessor_conformer::mtmd_audio_preprocessor_conformer;
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
struct mtmd_audio_preprocessor_lfm3a : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_lfm3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
@@ -107,7 +157,7 @@ struct mtmd_audio_preprocessor_lfm3a : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_granite_speech(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
@@ -116,7 +166,7 @@ struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_gemma4a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
@@ -125,13 +175,13 @@ struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_gemma4ua : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_gemma4ua(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_qwen3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
@@ -140,7 +190,7 @@ struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_dots3note(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
@@ -149,7 +199,7 @@ struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
@@ -158,7 +208,7 @@ struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_qwen3tts_spk(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
@@ -168,7 +218,7 @@ struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_pockettts(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override {}
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
// chatterbox reference front-end: one clip gives five entries, used by position in mtmd.cpp
@@ -181,7 +231,7 @@ struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_chatterbox_ref : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_chatterbox_ref(const clip_ctx * ctx, bool is_mtl) : mtmd_audio_preprocessor(ctx), is_mtl(is_mtl) {}
void initialize() override;
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
bool is_mtl;
@@ -194,25 +244,19 @@ struct mtmd_audio_preprocessor_chatterbox_ref : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_parakeet(clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) { }
void initialize() override;
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
protected:
- void log_mel(const float * samples, size_t n_samples, mtmd_audio_mel & out) const;
+ bool log_mel(const float * samples, size_t n_samples, mtmd_audio_mel & out, bool shape_only) const;
private:
mtmd_audio_cache cache;
-
- static void worker_thread(int ith, const float * window_func, int window_size,
- const std::vector<float> & samples, int n_samples,
- int frame_size, int frame_step, int n_threads,
- int n_fft_bins,
- const mtmd_audio_cache & cache, mtmd_audio_mel & mel);
};
// one chunk per recording: the raw log-mel of its valid frames
struct mtmd_audio_preprocessor_nemotron_transcribe : mtmd_audio_preprocessor_parakeet {
mtmd_audio_preprocessor_nemotron_transcribe(clip_ctx * ctx) : mtmd_audio_preprocessor_parakeet(ctx) { }
- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
//
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 72c92179f..538f90013 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -348,10 +348,14 @@ struct mtmd_image_tokens {
};
using mtmd_image_tokens_ptr = std::unique_ptr<mtmd_image_tokens>;
+// the mel of a clip is computed by its first encode, its chunks share the result
+using mtmd_audio_source_ptr = std::shared_ptr<mtmd_audio_preproc_out>;
+
// an audio track split across chunks (e.g. the soundtrack of a video): it is encoded once, as one clip, and every
// chunk takes its rows of it
struct mtmd_audio_track {
clip_image_f32_batch batch_f32; // the mel of the whole track
+ mtmd_audio_source_ptr source;
uint32_t n_tokens = 0;
std::vector<float> embd; // filled by the first encode
std::shared_ptr<const clip_gen_ref> gen_ref; // gen-side reference of a predefined voice, if any
@@ -362,12 +366,18 @@ struct mtmd_audio_tokens {
clip_image_f32_batch batch_f32; // preprocessed image patches
std::string id; // optional user-defined ID, useful for KV cache tracking
+ mtmd_audio_source_ptr source; // if set, batch_f32 has the shape of the mel of chunk source_idx
+ uint32_t source_idx = 0;
+
// a slice of a track: the rows [row_begin, row_begin + n_tokens) of it, batch_f32 is unused
std::shared_ptr<mtmd_audio_track> track;
uint32_t row_begin = 0;
// true if one of entries in batch_f32 is a placeholder
bool is_placeholder() const {
+ if (source || (track && track->source)) {
+ return false;
+ }
if (track && track->batch_f32.entries.empty()) {
return track->embd.empty(); // a voice: its rows are not serialized
}
@@ -384,6 +394,8 @@ struct mtmd_audio_tokens {
n_tokens,
batch_f32.clone(),
id,
+ source,
+ source_idx,
track,
row_begin,
};
@@ -413,6 +425,7 @@ struct mtmd_audio_tokens {
track->n_tokens = ser.read<uint32_t>();
track->batch_f32.deserialize(ser);
}
+ source.reset();
}
};
using mtmd_audio_tokens_ptr = std::unique_ptr<mtmd_audio_tokens>;
@@ -682,6 +695,7 @@ struct mtmd_context {
/* no_alloc */ no_alloc,
/* progress_callback */ ctx_params.progress_callback,
/* progress_callback_user_data */ ctx_params.progress_callback_user_data,
+ /* n_threads */ ctx_params.n_threads,
};
auto res = clip_init(mmproj_fname, ctx_clip_params);
@@ -1216,6 +1230,7 @@ struct mtmd_context {
// initialize audio preprocessor
if (audio_preproc) {
+ audio_preproc->mel_config.n_threads = n_threads;
audio_preproc->initialize();
}
}
@@ -1840,7 +1855,7 @@ struct mtmd_tokenizer {
GGML_ASSERT(ctx->audio_preproc != nullptr);
// preprocess audio
- std::vector<mtmd_audio_mel> mel_spec_chunks;
+ auto source = std::make_shared<mtmd_audio_preproc_out>();
{
std::vector<float> dummy;
const float * samples = nullptr;
@@ -1864,8 +1879,8 @@ struct mtmd_tokenizer {
LOG_ERR("%s: this model takes audio input only via mtmd_audio_stream\n", __func__);
return 2;
}
- bool ok = ctx->audio_preproc->preprocess(samples, n_samples, mel_spec_chunks);
- if (!ok) {
+ *source = ctx->audio_preproc->preprocess(samples, n_samples);
+ if (source->entries.empty()) {
LOG_ERR("Unable to preprocess audio\n");
return 2;
}
@@ -1873,22 +1888,14 @@ struct mtmd_tokenizer {
// one chunk for the five entries of a chatterbox clip, one token per talker conditioning row
if (clip_get_projector_type(ctx->ctx_a) == PROJECTOR_TYPE_CHATTERBOX_SPKENC) {
- GGML_ASSERT(mel_spec_chunks.size() == 5);
+ GGML_ASSERT(source->entries.size() == 5);
// mtl: spkr, perceiver query rows, emotion; turbo: spkr, t3 speech tokens
const size_t n_query = clip_cbx_read_tensor(ctx->ctx_a, "a.cenc.perceiver.pre_attention_query", nullptr, 0) / ctx->n_embd_out();
- const size_t n_tokens = cbx_is_mtl(ctx->ctx_a) ? 2 + n_query : 1 + (size_t) mel_spec_chunks[2].n_len / 4;
+ const size_t n_tokens = cbx_is_mtl(ctx->ctx_a) ? 2 + n_query : 1 + (size_t) source->entries[2].nx() / 4;
clip_image_f32_batch batch_f32;
batch_f32.is_audio = true;
- for (auto & mel_spec : mel_spec_chunks) {
- GGML_ASSERT(mel_spec.n_len <= INT32_MAX && mel_spec.n_len >= 0);
- GGML_ASSERT(mel_spec.n_mel <= INT32_MAX && mel_spec.n_mel >= 0);
- clip_image_f32 mel_f32;
- mel_f32.set_size({(int) mel_spec.n_len, (int) mel_spec.n_mel},
- mel_spec.data.empty(), /* is_audio */ true);
- mel_f32.cpy_buf(mel_spec.data);
- batch_f32.entries.push_back(std::move(mel_f32));
- }
+ batch_f32.entries = std::move(source->entries);
mtmd_audio_tokens_ptr audio_tokens(new mtmd_audio_tokens);
audio_tokens->n_tokens = (uint32_t) n_tokens;
@@ -1905,17 +1912,8 @@ struct mtmd_tokenizer {
} else
// consider each mel_spec as a separate audio chunk
// TODO: maybe support batching, but this may come with memory cost
- for (auto & mel_spec : mel_spec_chunks) {
- const bool is_placeholder = mel_spec.data.empty();
-
- // Validate dimensions fit in clip_image_size (int)
- GGML_ASSERT(mel_spec.n_len <= INT32_MAX && mel_spec.n_len >= 0);
- GGML_ASSERT(mel_spec.n_mel <= INT32_MAX && mel_spec.n_mel >= 0);
- clip_image_f32 mel_f32;
- mel_f32.set_size(
- {(int)mel_spec.n_len, (int)mel_spec.n_mel},
- is_placeholder, /* is_audio */ true);
- mel_f32.cpy_buf(mel_spec.data);
+ for (size_t i = 0; i < source->entries.size(); i++) {
+ clip_image_f32 mel_f32 = source->entries[i];
size_t n_tokens = clip_n_output_tokens(ctx->ctx_a, &mel_f32);
@@ -1927,6 +1925,10 @@ struct mtmd_tokenizer {
audio_tokens->n_tokens = n_tokens;
audio_tokens->batch_f32 = std::move(batch_f32);
audio_tokens->id = bitmap->id; // optional
+ if (source->needs_compute()) {
+ audio_tokens->source = source;
+ audio_tokens->source_idx = (uint32_t) i;
+ }
LOG_DBG("audio_tokens->n_tokens = %d\n", audio_tokens->n_tokens);
@@ -1984,16 +1986,21 @@ struct mtmd_tokenizer {
samples.insert(samples.end(), data, data + piece->nx);
}
}
- std::vector<mtmd_audio_mel> mels;
- if (!ctx->audio_preproc || !ctx->audio_preproc->preprocess(samples.data(), samples.size(), mels) || mels.size() != 1) {
+ if (!ctx->audio_preproc) {
+ LOG_ERR("%s: unable to preprocess the soundtrack\n", __func__);
+ return 2;
+ }
+ auto source = std::make_shared<mtmd_audio_preproc_out>(ctx->audio_preproc->preprocess(samples.data(), samples.size()));
+ if (source->entries.size() != 1) {
LOG_ERR("%s: unable to preprocess the soundtrack\n", __func__);
return 2;
}
- clip_image_f32 mel_f32;
- mel_f32.set_size({(int) mels[0].n_len, (int) mels[0].n_mel}, mels[0].data.empty(), /* is_audio */ true);
- mel_f32.cpy_buf(mels[0].data);
+ clip_image_f32 mel_f32 = source->entries[0];
st.track = std::make_shared<mtmd_audio_track>();
st.track->n_tokens = clip_n_output_tokens(ctx->ctx_a, &mel_f32);
+ if (source->needs_compute()) {
+ st.track->source = source;
+ }
st.track->batch_f32.is_audio = true;
st.track->batch_f32.entries.push_back(std::move(mel_f32));
}
@@ -2250,6 +2257,19 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im
static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd);
+static bool mtmd_audio_batch_encode(mtmd_context * ctx, const clip_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector<float> & out_embd) {
+ clip_image_f32_batch batch = batch_f32.clone();
+ if (source) {
+ if (!ctx->audio_preproc->compute(*source)) {
+ LOG_ERR("%s: unable to compute the mel\n", __func__);
+ return false;
+ }
+ GGML_ASSERT(idx < source->entries.size());
+ batch.entries[0] = source->entries[idx];
+ }
+ return clip_image_batch_encode(ctx->ctx_a, ctx->n_threads, &batch, out_embd);
+}
+
static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk * chunk, std::vector<float> & out_embd) {
ctx->gen_ref = {}; // see mtmd_gen_state_set_ref()
if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
@@ -2287,7 +2307,7 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk
if (track) {
if (track->embd.empty()) {
track->embd.resize((size_t) track->n_tokens * ctx->n_embd_out());
- if (!clip_image_batch_encode(ctx->ctx_a, ctx->n_threads, &track->batch_f32, track->embd)) {
+ if (!mtmd_audio_batch_encode(ctx, track->batch_f32, track->source.get(), 0, track->embd)) {
track->embd.clear();
return 1;
}
@@ -2305,11 +2325,7 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk
return cbx_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
}
out_embd.resize((size_t)chunk->tokens_audio->n_tokens * ctx->n_embd_out());
- bool ok = clip_image_batch_encode(
- ctx->ctx_a,
- ctx->n_threads,
- &chunk->tokens_audio->batch_f32,
- out_embd);
+ bool ok = mtmd_audio_batch_encode(ctx, chunk->tokens_audio->batch_f32, chunk->tokens_audio->source.get(), chunk->tokens_audio->source_idx, out_embd);
return ok ? 0 : 1;
}
@@ -3885,21 +3901,21 @@ void mtmd_debug_preprocess_audio(mtmd_context * ctx, const std::vector<float> &
LOG_ERR("%s: model does not support audio input\n", __func__);
return;
}
- std::vector<mtmd_audio_mel> mel_spec_chunks;
- bool ok = ctx->audio_preproc->preprocess(samples.data(), samples.size(), mel_spec_chunks);
- if (!ok) {
+ mtmd_audio_preproc_out out = ctx->audio_preproc->preprocess(samples.data(), samples.size());
+ if (out.entries.empty() || !ctx->audio_preproc->compute(out)) {
LOG_ERR("%s: failed to preprocess audio\n", __func__);
return;
}
- LOG_INF("%s: preprocessed audio to %zu mel spec chunks\n", __func__, mel_spec_chunks.size());
- for (size_t i = 0; i < mel_spec_chunks.size(); i++) {
- LOG_INF("%s: mel spec chunk %zu has n_len=%d, n_mel=%d\n", __func__, i, mel_spec_chunks[i].n_len, mel_spec_chunks[i].n_mel);
+ LOG_INF("%s: preprocessed audio to %zu mel spec chunks\n", __func__, out.entries.size());
+ for (size_t i = 0; i < out.entries.size(); i++) {
+ const auto & mel = out.entries[i];
+ const auto & data = mel.get_ro_buf();
+ LOG_INF("%s: mel spec chunk %zu has n_len=%d, n_mel=%d\n", __func__, i, mel.nx(), mel.ny());
// dump mel entries: data is stored as [n_mel][n_len] (mel-major)
- const auto & mel = mel_spec_chunks[i];
- for (int m = 0; m < mel.n_mel; m++) {
- for (int t = 0; t < mel.n_len; t++) {
- LOG_INF("mel[%zu][m=%d][t=%d] = %f\n", i, m, t, mel.data[m * mel.n_len + t]);
+ for (int m = 0; m < mel.ny(); m++) {
+ for (int t = 0; t < mel.nx(); t++) {
+ LOG_INF("mel[%zu][m=%d][t=%d] = %f\n", i, m, t, data[(size_t) m * mel.nx() + t]);
}
}
}
@@ -4450,16 +4466,17 @@ static bool mtmd_audio_stream_compute_mel(mtmd_audio_stream_lfm3a * s) {
}
const int64_t start = s->n_frames * s->hop - s->buf_pos;
const size_t len = (size_t) (s->win + (n_new - 1) * s->hop);
- std::vector<mtmd_audio_mel> mel_chunks;
- if (!s->ctx->audio_preproc->preprocess(s->buf.data() + start, len, mel_chunks) || mel_chunks.size() != 1) {
+ mtmd_audio_preproc_out out = s->ctx->audio_preproc->preprocess(s->buf.data() + start, len);
+ if (out.entries.size() != 1 || !s->ctx->audio_preproc->compute(out)) {
LOG_ERR("%s: unable to preprocess audio\n", __func__);
return false;
}
- const auto & mel = mel_chunks[0];
- GGML_ASSERT(mel.n_len == n_new && mel.n_mel == s->n_mel);
+ const auto & mel = out.entries[0];
+ const auto & data = mel.get_ro_buf();
+ GGML_ASSERT(mel.nx() == n_new && mel.ny() == s->n_mel);
for (int64_t t = 0; t < n_new; t++) {
for (int m = 0; m < s->n_mel; m++) {
- s->mel.push_back(mel.data[(size_t) m * mel.n_len + t]);
+ s->mel.push_back(data[(size_t) m * mel.nx() + t]);
}
}
s->n_frames = n_total;