Commit 0cce7580b for llama.cpp

commit 0cce7580b8169c044def5a01d6cbe61fff7c2b45
Author: Hera <58063257+hera2019@users.noreply.github.com>
Date:   Sat Nov 14 21:33:59 2026 +0900

    mtmd : avoid padding-only Whisper audio chunks (#30403)

    Assisted-by: Claude Opus 5

diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp
index 3fd720088..45c15f8c3 100644
--- a/tests/test-mtmd-impl.cpp
+++ b/tests/test-mtmd-impl.cpp
@@ -462,6 +462,58 @@ MAKE_TEST(test_video_sequence_lfm3_omni) {
 // mtmd_audio
 //

+MAKE_TEST(test_audio_preprocessor_whisper_chunks) {
+    mtmd_hparams hparams;
+    hparams.n_mel_bins        = 128;
+    hparams.audio_sample_rate = 16000;
+    hparams.audio_n_fft       = 400;
+    hparams.audio_window_len  = 400;
+    hparams.audio_hop_len     = 160;
+
+    ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr));
+    ggml_backend_t backends[] = { backend.get() };
+    ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true));
+
+    mtmd_audio_preprocessor_whisper preproc(hparams, sched.get());
+    preproc.initialize();
+
+    // { sample count, expected chunks }, one chunk of 3000 frames covers 480000 samples
+    const std::vector<std::pair<size_t, size_t>> cases = {
+        {       0, 0 },
+        {       1, 1 },
+        {     200, 1 },
+        {     201, 1 },
+        {   32000, 1 },
+        {  479834, 1 },
+        {  479839, 1 },
+        {  479840, 1 },
+        {  479999, 1 },
+        {  480000, 1 },
+        {  480001, 2 },
+        {  480159, 2 },
+        {  480160, 2 },
+        {  959840, 2 },
+        {  959999, 2 },
+        {  960000, 2 },
+        {  960001, 3 },
+        { 1440000, 3 },
+    };
+
+    for (const auto & [n_samples, expected] : cases) {
+        const std::string name = std::to_string(n_samples) + " samples";
+        std::vector<float> samples(n_samples, 0.0f);
+        if (n_samples > 0) {
+            samples.back() = 0.5f;
+        }
+        mtmd_audio_preproc_out out = preproc.preprocess(samples.data(), samples.size());
+        t.assert_equal("chunks for " + name, expected, out.entries.size());
+        for (const auto & mel : out.entries) {
+            t.assert_equal("frames for " + name, 3000, mel.nx());
+            t.assert_equal("mel bins for " + name, 128, mel.ny());
+        }
+    }
+}
+
 MAKE_TEST(test_audio_preprocessor_conformer) {
     mtmd_hparams hparams;
     hparams.n_mel_bins        = 128;
diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp
index aceaf5391..33c26c31f 100644
--- a/tools/mtmd/mtmd-audio.cpp
+++ b/tools/mtmd/mtmd-audio.cpp
@@ -555,7 +555,9 @@ bool mtmd_audio_preprocessor_whisper::process(const float *                 samp
     }
     const size_t frames_per_chunk = 3000;
     GGML_ASSERT((size_t) out_full.n_len > frames_per_chunk);
-    for (size_t off = 0; off < (size_t) out_full.n_len; off += frames_per_chunk) {
+    // round up, so a partial audio frame is not dropped with the padding
+    const size_t n_audio_frames = (n_samples - 1) / (size_t) mel_config.hop_length + 1;
+    for (size_t off = 0; off < n_audio_frames; off += frames_per_chunk) {
         int64_t n_len = std::min((int64_t)frames_per_chunk, out_full.n_len - (int64_t)off);
         if (n_len < (int64_t)frames_per_chunk) {
             break;  // last incomplete chunk will always be a padded chunk, safe to ignore
diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h
index 1175ad624..7251f82db 100644
--- a/tools/mtmd/mtmd-audio.h
+++ b/tools/mtmd/mtmd-audio.h
@@ -114,6 +114,7 @@ struct mtmd_audio_preprocessor {

 struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
     mtmd_audio_preprocessor_whisper(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
+    mtmd_audio_preprocessor_whisper(const mtmd_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {}
     void initialize() override;
     bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;