Commit 20c2e9665 for llama.cpp
commit 20c2e966559310abcc9a3ae8113d3f01eb885d9a
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Mon Nov 9 23:36:31 2026 +0100
mtmd: support gemma 4 input audio stream (#31965)
* mtmd: support gemma 4 input audio stream
* use 480ms step
* server: enable realtime embeddings only for audio generation
The realtime slot forced embeddings on, which marks every row of a
batch as an output. Only a model that generates audio reserves
n_batch outputs, so a chat model with an audio stream (Gemma 4)
aborted on its first batch of audio rows. The hidden state is only
read by the audio generation pipeline.
---------
Co-authored-by: Pascal <admin@serveurperso.com>
diff --git a/tools/mtmd/models/gemma4a.cpp b/tools/mtmd/models/gemma4a.cpp
index f1ac867ca..2b81f036c 100644
--- a/tools/mtmd/models/gemma4a.cpp
+++ b/tools/mtmd/models/gemma4a.cpp
@@ -7,19 +7,60 @@
#include "models.h"
#include <cmath>
+#include <map>
+
+// state of the streaming encoder, a zero state is a cold start
+std::vector<c2w_state_slot> list_gemma4a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
+ std::vector<c2w_state_slot> slots;
+
+ const int64_t n_mel = hparams.n_mel_bins;
+ const int64_t n_ch = model.sscp_conv_w[0]->ne[3];
+
+ slots.push_back({"pos", 1, 1});
+ slots.push_back({"sscp_0", n_mel, 1});
+ slots.push_back({"sscp_1", (n_mel - 1) / 2 + 1, n_ch});
+ for (size_t il = 0; il < model.layers.size(); il++) {
+ const auto & layer = model.layers[il];
+ if (layer.q_w && layer.k_w && layer.v_w && layer.o_w) {
+ slots.push_back({"k_" + std::to_string(il), layer.k_w->ne[1], mtmd_graph_gemma4a::max_past});
+ slots.push_back({"v_" + std::to_string(il), layer.v_w->ne[1], mtmd_graph_gemma4a::max_past});
+ }
+ if (layer.norm_conv_w && layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w) {
+ slots.push_back({"conv_" + std::to_string(il), layer.conv_dw_w->ne[0] - 1, layer.conv_dw_w->ne[1]});
+ }
+ }
+
+ return slots;
+}
ggml_cgraph * mtmd_graph_gemma4a::build() {
const float res_weight = 0.5f;
const float norm_eps = 1e-6f;
+ std::map<std::string, ggml_tensor *> state_in;
+ std::vector<std::pair<std::string, ggml_tensor *>> state_out;
+ for (const auto & slot : list_gemma4a_state_slots(hparams, model)) {
+ ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
+ ggml_set_name(t, ("state_in_" + slot.name).c_str());
+ ggml_set_input(t);
+ state_in[slot.name] = t;
+ }
+
// 1. Input
ggml_tensor * inp = build_inp_raw(1);
auto * cur = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
- // 2. Subsampling Conv2D (symmetric padding=1, matching PyTorch)
+ // 2. Subsampling Conv2D (padding=1, matching PyTorch)
+ // in time, the left pad comes from the state
{
for (int i = 0; i < 2; i++) {
- cur = ggml_conv_2d(ctx0, model.sscp_conv_w[i], cur, 2, 2, 1, 1, 1, 1);
+ const std::string name = "sscp_" + std::to_string(i);
+ ggml_tensor * left = ggml_reshape_3d(ctx0, state_in.at(name), cur->ne[0], 1, cur->ne[2]);
+ cur = ggml_concat(ctx0, left, cur, 1);
+ ggml_tensor * last = ggml_view_3d(ctx0, cur, cur->ne[0], 1, cur->ne[2], cur->nb[1], cur->nb[2], (cur->ne[1] - 1) * cur->nb[1]);
+ state_out.push_back({name, ggml_reshape_2d(ctx0, ggml_cont(ctx0, last), cur->ne[0], cur->ne[2])});
+ cur = ggml_pad_ext(ctx0, cur, 0, 0, 0, 1, 0, 0, 0, 0);
+ cur = ggml_conv_2d(ctx0, model.sscp_conv_w[i], cur, 2, 2, 1, 0, 1, 1);
if (model.sscp_conv_b[i]) {
cur = ggml_add(ctx0, cur, model.sscp_conv_b[i]);
}
@@ -44,10 +85,11 @@ ggml_cgraph * mtmd_graph_gemma4a::build() {
}
const int64_t n_pos = cur->ne[1];
+ state_out.push_back({"pos", ggml_scale_bias(ctx0, state_in.at("pos"), 1.0f, (float) n_pos)});
// Chunked local attention parameters
- const int64_t C = 12; // chunk_size
- const int64_t P = 12; // max_past_horizon (context_left - 1)
+ const int64_t C = chunk_size;
+ const int64_t P = max_past;
const int64_t S = C + P; // context_size = 24
const int64_t R = P + 1; // RPE positions = 13
const int64_t B = (n_pos + C - 1) / C; // num_blocks
@@ -117,12 +159,12 @@ ggml_cgraph * mtmd_graph_gemma4a::build() {
Qcur = ggml_cont(ctx0, ggml_permute(ctx0, Qcur, 0, 3, 1, 2)); // [D, C, B, H]
// K/V block context extraction via overlapping view:
- // Left pad by P and right pad to S*B elements,
- // then view with stride C in the block dimension (overlapping windows).
- auto extract_blocks = [&](ggml_tensor * t) -> ggml_tensor * {
- // [D, H, N] -> left pad by P, right pad to S*B
- const int64_t pad_kv = S * B - n_pos;
- t = ggml_pad_ext(ctx0, t, 0, 0, 0, 0, P, pad_kv - P, 0, 0); // [D, H, S*B]
+ // put the P cached positions in front, right pad to S*B, then view with stride C in the block dimension
+ auto extract_blocks = [&](ggml_tensor * t, const std::string & name) -> ggml_tensor * {
+ t = ggml_concat(ctx0, ggml_reshape_3d(ctx0, state_in.at(name), d_head, n_head, P), t, 2);
+ ggml_tensor * tail = ggml_view_3d(ctx0, t, d_head, n_head, P, t->nb[1], t->nb[2], n_pos * t->nb[2]);
+ state_out.push_back({name, ggml_reshape_2d(ctx0, ggml_cont(ctx0, tail), d_head * n_head, P)});
+ t = ggml_pad_ext(ctx0, t, 0, 0, 0, 0, 0, S * B - P - n_pos, 0, 0); // [D, H, S*B]
// Overlapping view: stride for B dim is C positions, not S
// ne = [D, H, S, B], data_size = D*H*S*B*sizeof = source_nbytes (exact fit)
// nb1=D*sizeof, nb2=D*H*sizeof, nb3=C*D*H*sizeof (overlap: C < S)
@@ -132,11 +174,11 @@ ggml_cgraph * mtmd_graph_gemma4a::build() {
return t;
};
- ggml_tensor * Kblk = extract_blocks(Kcur);
+ ggml_tensor * Kblk = extract_blocks(Kcur, "k_" + std::to_string(il));
// [D, H, S, B] -> [D, S, B, H] via permute(0,3,1,2)
Kblk = ggml_cont(ctx0, ggml_permute(ctx0, Kblk, 0, 3, 1, 2));
- ggml_tensor * Vblk = extract_blocks(Vcur);
+ ggml_tensor * Vblk = extract_blocks(Vcur, "v_" + std::to_string(il));
// [D, H, S, B] -> [S, D, B, H] via permute(1,3,0,2)
Vblk = ggml_cont(ctx0, ggml_permute(ctx0, Vblk, 1, 3, 0, 2));
@@ -217,8 +259,13 @@ ggml_cgraph * mtmd_graph_gemma4a::build() {
x = ggml_cont(ctx0, ggml_transpose(ctx0, x));
}
- // Causal depthwise Conv1D via ggml_ssm_conv, left padded only.
- x = ggml_pad_ext(ctx0, x, 4, 0, 0, 0, 0, 0, 0, 0);
+ // Causal depthwise Conv1D via ggml_ssm_conv, the left context comes from the state
+ {
+ const int64_t d_conv = layer.conv_dw_w->ne[0] - 1;
+ const std::string name = "conv_" + std::to_string(il);
+ x = ggml_concat(ctx0, state_in.at(name), x, 0);
+ state_out.push_back({name, ggml_cont(ctx0, ggml_view_2d(ctx0, x, d_conv, x->ne[1], x->nb[1], n_pos * x->nb[0]))});
+ }
x = ggml_ssm_conv(ctx0, x, layer.conv_dw_w);
if (layer.conv_dw_b) {
x = ggml_add(ctx0, x, layer.conv_dw_b);
@@ -269,6 +316,13 @@ ggml_cgraph * mtmd_graph_gemma4a::build() {
cur = build_mm(model.mm_input_proj_w, cur);
}
+ // the embeddings must stay the last node
+ for (const auto & s : state_out) {
+ ggml_set_name(s.second, ("state_out_" + s.first).c_str());
+ ggml_set_output(s.second);
+ ggml_build_forward_expand(gf, s.second);
+ }
+
ggml_build_forward_expand(gf, cur);
return gf;
}
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 17bd57d95..4f68462ba 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -274,6 +274,10 @@ struct mtmd_graph_muse_spark_audio : mtmd_graph {
};
struct mtmd_graph_gemma4a : mtmd_graph {
+ // attention_chunk_size, attention_context_left - 1
+ static constexpr int64_t chunk_size = 12;
+ static constexpr int64_t max_past = 12;
+
mtmd_graph_gemma4a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override;
@@ -574,6 +578,9 @@ std::vector<c2w_state_slot> list_lfm3a_gen_state_slots(const mtmd_hparams & hpar
// state slots of the streaming LFM3-Audio encoder (lfm3a)
std::vector<c2w_state_slot> list_lfm3a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
+// state slots of the streaming Gemma 4 audio encoder (gemma4a)
+std::vector<c2w_state_slot> list_gemma4a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
+
struct mtmd_graph_kimik25 : mtmd_graph {
mtmd_graph_kimik25(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp
index fcf94445a..aceaf5391 100644
--- a/tools/mtmd/mtmd-audio.cpp
+++ b/tools/mtmd/mtmd-audio.cpp
@@ -1676,7 +1676,7 @@ bool mtmd_audio_preprocessor_gemma4a::process(const float * samp
GGML_ASSERT(!cache.filters.data.empty());
// Split into 30-second chunks (model context limit, ~750 tokens each)
- const size_t chunk_samples = 30 * hparams.audio_sample_rate;
+ const size_t chunk_samples = (size_t) segment_seconds * hparams.audio_sample_rate;
for (size_t off = 0; off < n_samples; off += chunk_samples) {
const float * chunk_ptr = samples + off;
size_t chunk_len = std::min(chunk_samples, n_samples - off);
diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h
index ba71913e0..1175ad624 100644
--- a/tools/mtmd/mtmd-audio.h
+++ b/tools/mtmd/mtmd-audio.h
@@ -162,6 +162,9 @@ struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor {
};
struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor {
+ // each segment is padded and encoded on its own
+ static constexpr int segment_seconds = 30;
+
mtmd_audio_preprocessor_gemma4a(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index 7f30d3949..4c2b15107 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -434,6 +434,15 @@ static std::vector<c2w_state_slot> list_gen_state_slots(const mtmd_hparams & hpa
}
}
+// state slots of the streaming audio encoders
+static std::vector<c2w_state_slot> list_enc_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
+ switch (model.proj_type) {
+ case PROJECTOR_TYPE_LFM3A: return list_lfm3a_state_slots(hparams, model);
+ case PROJECTOR_TYPE_GEMMA4A: return list_gemma4a_state_slots(hparams, model);
+ default: return {};
+ }
+}
+
// state_out can have another shape than its slot, only the byte size must match
static void copy_gen_state_out(ggml_tensor * src, ggml_tensor * dst) {
GGML_ASSERT(ggml_nbytes(src) == ggml_nbytes(dst));
@@ -479,9 +488,9 @@ mtmd_subctx_gen_state * mtmd_subcontext::gen_state_init(uint32_t seed) {
mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) {
const auto & hparams = model.hparams;
- if (model.proj_type == PROJECTOR_TYPE_LFM3A) {
- // downsampler context, short-conv taps and attention window, see list_lfm3a_state_slots()
- const auto slots = list_lfm3a_state_slots(hparams, model);
+ if (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A) {
+ // see list_enc_state_slots()
+ const auto slots = list_enc_state_slots(hparams, model);
auto st = std::make_unique<mtmd_subctx_enc_state>();
ggml_init_params params = {
/*.mem_size =*/ ggml_tensor_overhead() * slots.size(),
@@ -657,6 +666,28 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
}
};
+ // returns the "pos" slot; without a stream, the state is zero
+ auto set_enc_state_in = [&]() -> int {
+ const auto state_slots = list_enc_state_slots(hparams, model);
+ GGML_ASSERT(!state_slots.empty() && state_slots[0].name == "pos");
+ if (params->enc_state == nullptr) {
+ for (const auto & slot : state_slots) {
+ ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str());
+ ggml_backend_tensor_memset(t, 0, 0, ggml_nbytes(t));
+ }
+ return 0;
+ }
+ const auto & slots = params->enc_state->slots;
+ GGML_ASSERT(slots.size() == state_slots.size());
+ float base_f = 0.0f;
+ ggml_backend_tensor_get(slots[0], &base_f, 0, sizeof(float));
+ size_t i = 0;
+ for (const auto & slot : state_slots) {
+ ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str()));
+ }
+ return (int) base_f;
+ };
+
// lfm3a encoder and codec: causal sliding window over [cached window | this call]
auto set_lfm3a_window_inputs = [&](int base, int n_pos, int window) {
std::vector<int32_t> positions(n_pos);
@@ -1934,11 +1965,14 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
for (int i = 0; i < 2; i++) { n_pos = (n_pos - 1) / 2 + 1; }
// Chunked local attention: blocked causal mask and RPE
- const int chunk_size = 12;
- const int max_past = 12;
+ const int chunk_size = mtmd_graph_gemma4a::chunk_size;
+ const int max_past = mtmd_graph_gemma4a::max_past;
const int context_size = chunk_size + max_past;
const int num_blocks = (n_pos + chunk_size - 1) / chunk_size;
+ // cached keys of a stream are at positions -n_hist .. -1
+ const int n_hist = std::min(set_enc_state_in(), max_past);
+
// Blocked causal attention mask: [context_size, chunk_size, num_blocks]
{
std::vector<float> mask(context_size * chunk_size * num_blocks, -1e9f);
@@ -1947,7 +1981,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
int gq = b * chunk_size + q;
for (int k = 0; k < context_size; k++) {
int gk = b * chunk_size - max_past + k;
- if (gq < n_pos && gk >= 0 && gk < n_pos && gk <= gq && (gq - gk) < max_past) {
+ if (gq < n_pos && gk >= -n_hist && gk < n_pos && gk <= gq && (gq - gk) < max_past) {
mask[k + q * context_size + b * context_size * chunk_size] = 0.0f;
}
}
@@ -2054,24 +2088,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
GGML_ASSERT(imgs.entries.size() == 1);
const int n_pos = model.n_output_tokens(&imgs.entries.front());
- // the "pos" slot leads the state; without a stream the state is zero, the clip is encoded from its start
- int base = 0;
- if (params->enc_state != nullptr) {
- const auto & slots = params->enc_state->slots;
- float base_f = 0.0f;
- ggml_backend_tensor_get(slots[0], &base_f, 0, sizeof(float));
- base = (int) base_f;
- size_t i = 0;
- for (const auto & slot : list_lfm3a_state_slots(hparams, model)) {
- ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str()));
- }
- } else {
- for (const auto & slot : list_lfm3a_state_slots(hparams, model)) {
- ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str());
- ggml_backend_tensor_memset(t, 0, 0, ggml_nbytes(t));
- }
- }
-
+ const int base = set_enc_state_in();
set_lfm3a_window_inputs(base, n_pos, hparams.attn_window_size);
} break;
case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
@@ -2584,10 +2601,10 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
}
}
}
- if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_LFM3A) {
+ if (params->enc_state != nullptr && (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A)) {
const auto & slots = params->enc_state->slots;
size_t i = 0;
- for (const auto & slot : list_lfm3a_state_slots(hparams, model)) {
+ for (const auto & slot : list_enc_state_slots(hparams, model)) {
ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str());
GGML_ASSERT(t != nullptr);
copy_gen_state_out(t, slots[i++]);
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 7c6bcc7be..1399b7ad4 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -1,4 +1,5 @@
#include "mtmd-subcontext.h"
+#include "models/models.h"
#include "mtmd-impl.h"
#include "mtmd.h"
#include "mtmd-internal.h"
@@ -4094,12 +4095,40 @@ struct mtmd_audio_stream_lfm3a : mtmd_audio_stream {
int32_t step_live(llama_token last_token);
};
+// Gemma 4: the encoder is causal
+struct mtmd_audio_stream_gemma4a : mtmd_audio_stream {
+ mtmd_subctx_enc_state * enc_state = nullptr;
+
+ int n_fft;
+ int win;
+ int pad_left;
+ int64_t seg_len;
+ int64_t seg_start = 0; // first sample of the current segment, n_frames counts its frames
+
+ std::vector<float> beg_rows;
+ std::vector<float> end_rows;
+ bool beg_done = false;
+ bool end_done = false;
+
+ ~mtmd_audio_stream_gemma4a() override {
+ delete enc_state;
+ }
+
+ bool init(const mtmd_audio_stream_params & params) override;
+ int32_t flush(bool end) override;
+ int32_t step(llama_token last_token) override;
+
+ int64_t n_seg_frames(int64_t n, bool complete) const;
+ bool compute_mel(int64_t n_seg, bool complete);
+ bool encode(int n);
+};
+
bool mtmd_support_audio_stream(const mtmd_context * ctx) {
if (!ctx->ctx_a || !ctx->text_model) {
return false;
}
const projector_type proj = ctx->ctx_a->model.proj_type;
- return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A;
+ return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A || proj == PROJECTOR_TYPE_GEMMA4A;
}
mtmd_audio_stream_params mtmd_audio_stream_params_default(void) {
@@ -4121,6 +4150,7 @@ mtmd_audio_stream * mtmd_audio_stream_init(mtmd_context * ctx, mtmd_audio_stream
switch (ctx->ctx_a->model.proj_type) {
case PROJECTOR_TYPE_VOXTRAL_RT: s = std::make_unique<mtmd_audio_stream_voxtral_rt>(); break;
case PROJECTOR_TYPE_LFM3A: s = std::make_unique<mtmd_audio_stream_lfm3a>(); break;
+ case PROJECTOR_TYPE_GEMMA4A: s = std::make_unique<mtmd_audio_stream_gemma4a>(); break;
default: GGML_ABORT("unsupported projector");
}
s->ctx = ctx;
@@ -4171,6 +4201,23 @@ bool mtmd_audio_stream_is_seq_start(const mtmd_audio_stream * s) {
}
+// token embeddings of aud_beg / aud_end
+static bool mtmd_audio_stream_marker_rows(const mtmd_context * ctx, const std::string & text, std::vector<float> & rows) {
+ rows.clear();
+ if (text.empty()) {
+ return true;
+ }
+ const llama_vocab * vocab = llama_model_get_vocab(ctx->text_model);
+ std::vector<llama_token> tokens(text.size() + 4);
+ const int32_t n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), tokens.data(), (int32_t) tokens.size(), false, true);
+ if (n <= 0) {
+ LOG_ERR("%s: cannot tokenize the audio marker %s\n", __func__, text.c_str());
+ return false;
+ }
+ rows.resize((size_t) n * ctx->n_embd_text);
+ return llama_model_get_tok_embd_rows(ctx->text_model, tokens.data(), n, rows.data()) != 0;
+}
+
//
// Voxtral Realtime
//
@@ -4432,20 +4479,7 @@ bool mtmd_audio_stream_lfm3a::init(const mtmd_audio_stream_params & params) {
}
// the markers the tokenizer puts around an audio clip go around the stream as well
- const llama_vocab * vocab = llama_model_get_vocab(ctx->text_model);
- auto marker_rows = [&](const std::string & text, std::vector<float> & rows) {
- if (text.empty()) {
- return true;
- }
- std::vector<llama_token> tokens(text.size() + 4);
- const int32_t n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), tokens.data(), (int32_t) tokens.size(), false, true);
- if (n <= 0) {
- return false;
- }
- rows.resize((size_t) n * n_embd);
- return llama_model_get_tok_embd_rows(ctx->text_model, tokens.data(), n, rows.data()) != 0;
- };
- return marker_rows(ctx->aud_beg, beg_rows) && marker_rows(ctx->aud_end, end_rows);
+ return mtmd_audio_stream_marker_rows(ctx, ctx->aud_beg, beg_rows) && mtmd_audio_stream_marker_rows(ctx, ctx->aud_end, end_rows);
}
// end: pad the tail like mtmd_audio_preprocessor_lfm3a, so the rows match encoding the clip at once
@@ -4667,6 +4701,192 @@ int32_t mtmd_audio_stream_lfm3a::step_live(llama_token last_token) {
return (int32_t) (out.size() / n_embd);
}
+//
+// Gemma 4
+//
+
+bool mtmd_audio_stream_gemma4a::init(const mtmd_audio_stream_params & params) {
+ const mtmd_hparams * hparams = &sctx->model.hparams;
+
+ if (params.n_prompt > 0) {
+ LOG_ERR("%s: Gemma 4 takes no prompt, its audio is a chat turn\n", __func__);
+ return false;
+ }
+ if (!ctx->audio_preproc || !ctx->audio_preproc->mel_config.cache) {
+ LOG_ERR("%s: the audio preprocessor is not initialized\n", __func__);
+ return false;
+ }
+
+ // one row per 4 mel frames (two stride-2 convs)
+ n_fft = hparams->audio_n_fft;
+ win = hparams->audio_window_len;
+ pad_left = win / 2;
+ seg_len = (int64_t) mtmd_audio_preprocessor_gemma4a::segment_seconds * hparams->audio_sample_rate;
+ frames_per_tok = 4;
+ samples_per_tok = frames_per_tok * hop;
+
+ // whole attention blocks, so the rows match the offline path
+ const int n_step = (int) ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 / hop);
+ n_step_mel = MTMD_ALIGN(std::max(n_step, frames_per_tok), (int) mtmd_graph_gemma4a::chunk_size * frames_per_tok);
+
+ enc_state = sctx->enc_state_init(n_step_mel);
+ if (!enc_state) {
+ return false;
+ }
+
+ return mtmd_audio_stream_marker_rows(ctx, ctx->aud_beg, beg_rows) && mtmd_audio_stream_marker_rows(ctx, ctx->aud_end, end_rows);
+}
+
+// mel frames of a segment with n samples
+// complete: same count as mtmd_audio_preprocessor_gemma4a, otherwise: only frames with a full window
+int64_t mtmd_audio_stream_gemma4a::n_seg_frames(int64_t n, bool complete) const {
+ if (complete) {
+ return std::max<int64_t>(0, (n + pad_left - (win + 1)) / hop + 1);
+ }
+ return n + pad_left >= win + 1 ? (n + pad_left - (win + 1)) / hop + 1 : 0;
+}
+
+// frame k reads samples [k * hop - pad_left, k * hop - pad_left + n_fft) of the segment, zero outside of it
+// the window is zero past win, so the samples after the segment end do not count
+bool mtmd_audio_stream_gemma4a::compute_mel(int64_t n_seg, bool complete) {
+ const int64_t n_total = n_seg_frames(n_seg, complete);
+ const int64_t n_new = n_total - n_frames;
+ if (n_new <= 0) {
+ return true;
+ }
+
+ const int64_t first = seg_start + n_frames * hop - pad_left;
+ std::vector<float> samples((size_t) ((n_new - 1) * hop + n_fft), 0.0f);
+ for (size_t j = 0; j < samples.size(); j++) {
+ const int64_t idx = first + (int64_t) j;
+ if (idx >= seg_start && idx < seg_start + n_seg) {
+ GGML_ASSERT(idx >= buf_pos && idx - buf_pos < (int64_t) buf.size());
+ samples[j] = buf[idx - buf_pos];
+ }
+ }
+
+ const mtmd_audio_mel_config & cfg = ctx->audio_preproc->mel_config;
+ std::vector<float> frames((size_t) n_new * n_mel);
+ if (!mtmd_audio_log_mel(cfg, samples.data(), cfg.cache->hann_window.data(), n_new, frames.data())) {
+ LOG_ERR("%s: unable to compute the mel spectrogram\n", __func__);
+ return false;
+ }
+ for (int64_t t = 0; t < n_new; t++) {
+ for (int m = 0; m < n_mel; m++) {
+ mel.push_back(frames[(size_t) m * n_new + t]);
+ }
+ }
+ n_frames = n_total;
+
+ // drop the samples no frame will need again
+ const int64_t keep_from = std::max(seg_start, seg_start + n_frames * hop - pad_left);
+ if (keep_from > buf_pos) {
+ buf.erase(buf.begin(), buf.begin() + (keep_from - buf_pos));
+ buf_pos = keep_from;
+ }
+ return true;
+}
+
+bool mtmd_audio_stream_gemma4a::encode(int n) {
+ std::vector<float> data((size_t) n_mel * n);
+ for (int m = 0; m < n_mel; m++) {
+ for (int t = 0; t < n; t++) {
+ data[(size_t) m * n + t] = mel[(size_t) t * n_mel + m];
+ }
+ }
+ mtmd_image_f32 img;
+ img.set_size({n, n_mel}, false, /* is_audio */ true);
+ img.cpy_buf(data);
+ const int n_out = sctx->model.n_output_tokens(&img);
+
+ mtmd_image_f32_batch batch;
+ batch.is_audio = true;
+ batch.entries.push_back(std::move(img));
+
+ std::vector<float> res((size_t) n_embd * n_out);
+ mtmd_subctx_encode_params params;
+ params.n_threads = ctx->n_threads;
+ params.imgs = &batch;
+ params.out_embd = &res;
+ params.enc_state = enc_state;
+ if (!sctx->encode(¶ms)) {
+ LOG_ERR("%s: failed to encode audio\n", __func__);
+ return false;
+ }
+ embd.insert(embd.end(), res.begin(), res.end());
+ mel.erase(mel.begin(), mel.begin() + (size_t) n * n_mel);
+ return true;
+}
+
+// no padding here, step() pads a segment once it is complete
+int32_t mtmd_audio_stream_gemma4a::flush(bool end) {
+ const int64_t n_full = n_samples / seg_len;
+ const int64_t n_rest = n_samples % seg_len;
+ const int64_t rows_full = (n_seg_frames(seg_len, true) + frames_per_tok - 1) / frames_per_tok;
+ const int64_t rows_rest = end ? (n_seg_frames(n_rest, true) + frames_per_tok - 1) / frames_per_tok
+ : n_seg_frames(n_rest, false) / frames_per_tok;
+ n_rows_flushed = n_full * rows_full + rows_rest;
+ pending_flush = true;
+ ended = end;
+ return 0;
+}
+
+int32_t mtmd_audio_stream_gemma4a::step(llama_token /* last_token */) {
+ const bool flush = pending_flush;
+ pending_flush = false;
+
+ // whole rows only, except at the end of a segment: a partial row puts its zero padding into the state
+ while (true) {
+ const int64_t seg_end = seg_start + seg_len;
+ const int64_t n_seg = std::min(n_samples, seg_end) - seg_start;
+ const bool complete = n_samples >= seg_end || ended;
+ if (!compute_mel(n_seg, complete)) {
+ return -2;
+ }
+ const int n_avail = (int) (mel.size() / n_mel);
+ int n = n_avail;
+ if (!complete) {
+ n -= n % frames_per_tok;
+ if (!flush) {
+ n -= n % n_step_mel;
+ }
+ }
+ if (n > 0 && !encode(n)) {
+ return -2;
+ }
+ if (n_samples < seg_end) {
+ break;
+ }
+ // next segment starts from a zero state
+ seg_start = seg_end;
+ n_frames = 0;
+ ggml_backend_buffer_clear(enc_state->buf.get(), 0);
+ }
+
+ // rows carry no text, all of them go out at once and last_token is not used
+ const int64_t n_avail = (int64_t) (embd.size() / n_embd);
+ out.clear();
+ if (n_avail > 0 || ended) {
+ if (!beg_done) {
+ beg_done = true;
+ out = beg_rows;
+ }
+ out.insert(out.end(), embd.begin(), embd.end());
+ embd.clear();
+ if (ended && !end_done) {
+ end_done = true;
+ out.insert(out.end(), end_rows.begin(), end_rows.end());
+ }
+ }
+ const int64_t n_out = (int64_t) (out.size() / n_embd);
+ if (n_out == 0) {
+ return ended ? -1 : 0;
+ }
+ n_rows += n_out;
+ n_rows_done += n_avail; // the markers are not audio rows
+ return (int32_t) n_out;
+}
+
int64_t mtmd_audio_stream_get_n_rows(const mtmd_audio_stream * s) {
return s->n_rows_done;
}
diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp
index 689626f5f..b608b87ab 100644
--- a/tools/server/server-realtime.cpp
+++ b/tools/server/server-realtime.cpp
@@ -1158,7 +1158,7 @@ bool server_realtime_slot::response_step() {
}
bool server_realtime_slot::update() {
- llama_set_embeddings(ctx, true);
+ llama_set_embeddings(ctx, gen.valid());
if (responding) {
return response_step();