Commit fcc891545 for llama.cpp
commit fcc891545b0f06de346d8f67d1e6c61f9bf0e777
Author: Yuri Khrustalev <ykhrustalev@users.noreply.github.com>
Date: Fri Sep 25 12:52:44 2026 -0400
mtmd: fix mel preprocessor in LFM2 audio (#29403)
which resulted in different greedy transcripts for 4.5% of English and 6.5% of Japanese
test utterances. In Japanese, some differences changed entire words.
This change:
* uses `log(x + 2^-24)` instead of clamping to the log floor
* uses a symmetric Hann window, equivalent to `torch.hann_window(periodic=False)`
* adds the normalization epsilon to the standard deviation instead of inside the square root
Only the `lfm2a` preprocessor opts into these behaviors. Other audio preprocessors are unchanged.
Tested on top of 84e76d8 using `llama-server` with CUDA and `temperature=0`, compared against
http://github.com/Liquid4All/liquid-audio fp32.
Test set:
* 200 LibriSpeech `test-clean` utterances (EN)
* 200 Common Voice `ja` test utterances (JP)
* identical 16 kHz audio passed to both implementations
| Greedy transcript identical to `liquid-audio` | Without fix | With fix |
| --------------------------------------------- | ----------: | ----------: |
| EN F16 | 191/200 | 200/200 |
| JP F32 | 187/200 | 200/200 |
| JP F16 | 187/200 | 199/200 |
The remaining JP F16 difference is a comma and matches the reference implementation's own bf16
output.
Mel relative L2 error versus `liquid-audio`:
* EN: 3.2% -> ~2e-6 median
* JP: 3.9% -> ~2e-6 median
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp
index 2ec6b2391..5321f2ed1 100644
--- a/tests/test-mtmd-impl.cpp
+++ b/tests/test-mtmd-impl.cpp
@@ -1,8 +1,10 @@
#include "testing.h"
+#include "mtmd-audio.h"
#include "mtmd-image.h"
#include "mtmd-internal.h"
+#include <cmath>
#include <iostream>
#include <stdexcept>
#include <string>
@@ -137,6 +139,54 @@ MAKE_TEST(test_temporal_merge_grouping) {
}
}
+//
+// mtmd_audio
+//
+
+MAKE_TEST(test_audio_preprocessor_conformer) {
+ clip_hparams hparams;
+ hparams.n_mel_bins = 128;
+ hparams.audio_sample_rate = 16000;
+ hparams.audio_n_fft = 512;
+ hparams.audio_window_len = 400;
+ hparams.audio_hop_len = 160;
+
+ // 0.4 s of tones, 0.2 s of silence, 0.4 s of a quiet tone
+ const double pi = 3.14159265358979323846;
+ const int sr = hparams.audio_sample_rate;
+ std::vector<float> samples(sr, 0.0f);
+ for (int i = 0; i < sr; i++) {
+ const double ts = (double) i / sr;
+ if (i < 0.4 * sr) {
+ samples[i] = (float) (0.3 * std::sin(2 * pi * 300 * ts) + 0.2 * std::sin(2 * pi * 1200 * ts) + 0.1 * std::sin(2 * pi * 3500 * ts));
+ } else if (i >= 0.6 * sr) {
+ samples[i] = (float) (1e-3 * std::sin(2 * pi * 800 * ts));
+ }
+ }
+
+ mtmd_audio_preprocessor_conformer preproc(hparams);
+ preproc.initialize();
+ std::vector<mtmd_audio_mel> mels;
+ if (!t.assert_true("preprocess", preproc.preprocess(samples.data(), samples.size(), mels))) {
+ return;
+ }
+ const mtmd_audio_mel & mel = mels[0];
+ t.assert_equal("n_len", (int64_t) 101, mel.n_len);
+
+ // reference values from NeMo AudioToMelSpectrogramPreprocessor (liquid-audio 1.3.0)
+ // { mel bin, frame, value }
+ const std::vector<std::tuple<int, int, float>> cases = {
+ { 8, 0, 3.6104f }, // tones
+ { 24, 50, -0.6371f }, // silence
+ { 24, 60, -0.0476f }, // silence -> quiet tone
+ { 100, 20, -0.0913f }, // tones, high bin
+ };
+ for (const auto & [bin, frame, expected] : cases) {
+ const float actual = mel.data[bin * mel.n_len + frame];
+ t.assert_true("mel[" + std::to_string(bin) + "][" + std::to_string(frame) + "] = " + std::to_string(actual) + ", expected " + std::to_string(expected), std::fabs(actual - expected) < 1e-3f);
+ }
+}
+
//
// main
//
diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp
index c25bf4ec8..6c8b19441 100644
--- a/tools/mtmd/mtmd-audio.cpp
+++ b/tools/mtmd/mtmd-audio.cpp
@@ -283,6 +283,8 @@ struct filter_params {
bool norm_per_feature = false;
bool use_magnitude = false; // |X| instead of |X|^2
float mel_floor = 5.960464477539063e-08f;
+ bool mel_floor_add = false; // log(x + floor) instead of log(max(x, floor))
+ bool std_eps_after_sqrt = false; // std + eps instead of sqrt(var + eps)
};
static void log_mel_spectrogram_worker_thread(int ith,
@@ -347,7 +349,7 @@ static void log_mel_spectrogram_worker_thread(int ith,
for (; k < n_fft_bins; k++) {
sum += fft_out[k] * filters.data[(size_t)j * n_fft_bins + k];
}
- sum = std::max(sum, (double)params.mel_floor);
+ sum = params.mel_floor_add ? sum + (double)params.mel_floor : std::max(sum, (double)params.mel_floor);
sum = params.use_natural_log
? log(sum)
: log10(sum);
@@ -491,7 +493,7 @@ static bool log_mel_spectrogram(
var += value * value;
}
var /= effective_n_len - 1; // unbiased
- const double mstd = std::sqrt(var + 1e-5);
+ const double mstd = params.std_eps_after_sqrt ? std::sqrt(var) + 1e-5 : std::sqrt(var + 1e-5);
for (int64_t j = 0; j < effective_n_len; ++j) {
auto &value = out.data[(size_t)i * out.n_len + j];
@@ -949,7 +951,8 @@ bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float *
void mtmd_audio_preprocessor_conformer::initialize() {
cache.fill_sin_cos_table(hparams.audio_n_fft);
- cache.fill_hann_window(hparams.audio_window_len, true);
+ // NeMo uses a symmetric window: torch.hann_window(periodic=False)
+ cache.fill_hann_window(hparams.audio_window_len, false);
cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);
}
@@ -971,6 +974,8 @@ bool mtmd_audio_preprocessor_conformer::preprocess(const float *
params.preemph = 0.97f;
params.use_natural_log = true;
params.norm_per_feature = true;
+ params.mel_floor_add = true;
+ params.std_eps_after_sqrt = true;
// make sure the cache is initialized
GGML_ASSERT(!cache.sin_vals.empty());
diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h
index 4a15fe6d4..36db50526 100644
--- a/tools/mtmd/mtmd-audio.h
+++ b/tools/mtmd/mtmd-audio.h
@@ -54,6 +54,7 @@ struct mtmd_audio_preprocessor {
const clip_hparams & hparams;
mtmd_audio_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {}
+ mtmd_audio_preprocessor(const clip_hparams & hparams): hparams(hparams) {}
virtual ~mtmd_audio_preprocessor() = default;
virtual void initialize() = 0; // NOT thread-safe
@@ -71,6 +72,7 @@ struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
+ mtmd_audio_preprocessor_conformer(const clip_hparams & hparams) : mtmd_audio_preprocessor(hparams) {}
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;