Commit fa3645feb for llama.cpp
commit fa3645feb5f47cbb30df5e6d031c22543d43d240
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Sun Oct 25 11:56:04 2026 +0200
llama : respect no_alloc in all memory caches (#31087)
* llama : respect no_alloc in memory caches
Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
* llama : TODO: move no_alloc out of llama_hparams
Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
* llama : TODO: move no_alloc out of llama_hparams
Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
diff --git a/src/llama-hparams.h b/src/llama-hparams.h
index 2b92f5d60..a3bf9e688 100644
--- a/src/llama-hparams.h
+++ b/src/llama-hparams.h
@@ -61,7 +61,7 @@ struct llama_hparams {
// for example: n_embd_out() vs n_embd_out_impl
bool vocab_only;
- bool no_alloc;
+ bool no_alloc; // TODO: no_alloc is not a model hyperparameter; move it out of llama_hparams
bool rope_finetuned;
bool use_par_res;
bool swin_norm;
diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp
index e479dc564..c18d913da 100644
--- a/src/llama-kv-cache-dsv4.cpp
+++ b/src/llama-kv-cache-dsv4.cpp
@@ -946,13 +946,12 @@ llama_dsv4_comp_state::llama_dsv4_comp_state(
uint32_t n_rs_seq,
const char * name,
const llama_memory_i::layer_filter_cb & filter) :
+ hparams(model.hparams),
ratio(ratio),
state_size(state_size),
n_embd_state(n_embd_state),
n_stream(unified ? 1 : n_seq_max),
n_rs_seq(n_rs_seq) {
- const llama_hparams & hparams = model.hparams;
-
struct ggml_backend_buft_comparator {
bool operator()(const ggml_backend_buffer_type_t & lhs, const ggml_backend_buffer_type_t & rhs) const {
return strcmp(ggml_backend_buft_name(lhs), ggml_backend_buft_name(rhs)) < 0;
@@ -1027,7 +1026,15 @@ llama_dsv4_comp_state::llama_dsv4_comp_state(
}
for (auto & [buft, ctx] : ctx_map) {
- ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft);
+ ggml_backend_buffer_t buf;
+ if (hparams.no_alloc) {
+ buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer
+ for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) {
+ t->buffer = buf; // set dummy buffer for DSV4 state so that the backend scheduler won't try to allocate it
+ }
+ } else {
+ buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer
+ }
if (!buf) {
throw std::runtime_error("failed to allocate buffer for DSV4 compressor state");
}
@@ -1119,9 +1126,15 @@ uint32_t llama_dsv4_comp_state::get_n_rows() const {
std::map<ggml_backend_buffer_type_t, size_t> llama_dsv4_comp_state::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, size_t> ret;
- for (const auto & [_, buf] : ctxs_bufs) {
+ for (const auto & [ctx, buf] : ctxs_bufs) {
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
- ret[buft] += ggml_backend_buffer_get_size(buf.get());
+
+ if (hparams.no_alloc) {
+ GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);
+ ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
+ } else {
+ ret[buft] += ggml_backend_buffer_get_size(buf.get());
+ }
}
return ret;
}
diff --git a/src/llama-kv-cache-dsv4.h b/src/llama-kv-cache-dsv4.h
index 0368ea59f..99ac73ddb 100644
--- a/src/llama-kv-cache-dsv4.h
+++ b/src/llama-kv-cache-dsv4.h
@@ -61,6 +61,8 @@ private:
std::vector<ggml_tensor *> score_stream;
};
+ const llama_hparams & hparams;
+
const uint32_t ratio;
const uint32_t state_size;
const uint32_t n_embd_state;
diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cpp
index 2893d819e..d00630d4c 100644
--- a/src/llama-memory-recurrent.cpp
+++ b/src/llama-memory-recurrent.cpp
@@ -116,7 +116,15 @@ llama_memory_recurrent::llama_memory_recurrent(
// allocate tensors and initialize the buffers to avoid NaNs in the padding
for (auto & [buft, ctx] : ctx_map) {
- ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft);
+ ggml_backend_buffer_t buf;
+ if (hparams.no_alloc) {
+ buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer
+ for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) {
+ t->buffer = buf; // set dummy buffer for RS cache so that the backend scheduler won't try to allocate it
+ }
+ } else {
+ buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer
+ }
if (!buf) {
throw std::runtime_error("failed to allocate buffer for rs cache");
}
@@ -435,8 +443,15 @@ void llama_memory_recurrent::set_rs_idx(llama_seq_id seq_id, uint32_t idx) {
std::map<ggml_backend_buffer_type_t, size_t> llama_memory_recurrent::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, size_t> ret;
- for (const auto & [_, buf] : ctxs_bufs) {
- ret[ggml_backend_buffer_get_type(buf.get())] += ggml_backend_buffer_get_size(buf.get());
+ for (const auto & [ctx, buf] : ctxs_bufs) {
+ ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
+
+ if (hparams.no_alloc) {
+ GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);
+ ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
+ } else {
+ ret[buft] += ggml_backend_buffer_get_size(buf.get());
+ }
}
return ret;
}