Commit fa6769818 for llama.cpp
commit fa6769818708afd9807b22183ccda112fd563427
Author: Jesus Gulfo <jesdga95@gmail.com>
Date: Thu Sep 10 10:10:55 2026 -0500
spec: fix failed to decode mtmd chunk with DFlash (#28587)
* speculative: fix failed to decode mtmd chunk with DFlash
When using DFlash w/ vision models, the drafter memory fails to
allocate new tokens because images report a fixed offset. Stop copying
them to allow the drafter to continue.
* address PR feedback
limit M-RoPE skip to images only, allow audio to pass through. Clean up
comments to align to the updated implementation
diff --git a/common/speculative.cpp b/common/speculative.cpp
index 2db381d58..b7811b853 100644
--- a/common/speculative.cpp
+++ b/common/speculative.cpp
@@ -1094,8 +1094,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
// Target prefill may contain token IDs or multimodal embeddings. Both
// produce the target-layer features used to seed the draft KV cache, so
- // skipping the embedding batches leaves a hole in the draft's cache and
- // the next injection fails to initialize.
+ // embeddings are injected too, except the pinned ones skipped below.
// TODO: revisit after https://github.com/ggml-org/llama.cpp/pull/24669 is merged
const bool has_tokens = batch_in.token != nullptr;
const bool has_embeddings = batch_in.embd != nullptr;
@@ -1131,6 +1130,13 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
}
const int32_t n_rows = i_batch_end[seq_id] - i_batch_beg[seq_id] + 1;
+ // an M-RoPE image pins all its rows to one position, so a windowed draft
+ // cache cannot free cells for it - skip it, the draft can jump over the gap
+ const bool pos_pinned = batch_in.pos[i_batch_beg[seq_id]] == batch_in.pos[i_batch_end[seq_id]];
+ if (has_embeddings && n_rows > 1 && pos_pinned) {
+ continue;
+ }
+
for (int32_t offset = 0; offset < n_rows; offset += n_ubatch) {
const int32_t n_chunk = std::min(n_ubatch, n_rows - offset);