Commit 972b5fcc1 for llama.cpp
commit 972b5fcc19670e597f8de9bf5cacfb85752a5fff
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Mon Oct 12 11:53:58 2026 +0300
llama : report compute graph reallocations during inference (#30329)
Add an optional ggml_backend_sched_realloc_callback that user code registers with
ggml_backend_sched_set_realloc_callback to be notified when the buffers reserved for a
graph are not sufficient and have to be reserved again. The callback reports whether the
backend assignments and the graph size changed since the previous scheduling.
llama_context registers the callback when creating the scheduler (wrapped into the new
llama_context::sched_new helper) and treats the reallocations that follow a change of the
backend assignments as expected, counting the rest by graph size - unchanged or changed
from the previously scheduled graph. Both counters are reported on one line in the perf
stats.
Assisted-by: pi:llama.cpp/Qwen3.8-Flash-Next
diff --git a/common/sampling.cpp b/common/sampling.cpp
index 9e16693fb..564fa9c4d 100644
--- a/common/sampling.cpp
+++ b/common/sampling.cpp
@@ -589,6 +589,7 @@ void common_perf_print(const struct llama_context * ctx, const struct common_sam
LOG_INF("%s: total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval));
LOG_INF("%s: unaccounted time = %10.2f ms / %5.1f %% (total - sampling - prompt eval - eval) / (total)\n", __func__, t_unacc_ms, t_unacc_pc);
LOG_INF("%s: graphs reused = %10d\n", __func__, data.n_reused);
+ LOG_INF("%s: graph reallocs = %10d / %5d (same size) / (changed size)\n", __func__, data.n_reallocs_same, data.n_reallocs_diff);
common_memory_breakdown_print(ctx);
}
diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h
index 38197285c..7a65d8121 100644
--- a/ggml/include/ggml-backend.h
+++ b/ggml/include/ggml-backend.h
@@ -326,6 +326,14 @@ extern "C" {
// `graph` is the compute graph of the split
typedef bool (*ggml_backend_sched_copy_callback)(ggml_backend_t backend, const struct ggml_tensor * src, struct ggml_tensor * dst, struct ggml_cgraph * graph, void * user_data);
+ // Callback when the buffers reserved for a graph are not sufficient and have to be reserved again
+ // it happens on the graph allocation path and costs a synchronization of the backends
+ // `sched` is the scheduler
+ // `graph` is the compute graph being allocated
+ // `backend_ids_changed` is true if the backend assignment of the graph nodes changed since the last allocation
+ // `graph_size_changed` is true if the graph size changed since the last allocation
+ typedef void (*ggml_backend_sched_realloc_callback)(ggml_backend_sched_t sched, struct ggml_cgraph * graph, bool backend_ids_changed, bool graph_size_changed, void * user_data);
+
// Initialize a backend scheduler, backends with low index are given priority over backends with high index
GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload);
GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);
@@ -367,6 +375,9 @@ extern "C" {
// Set a callback to be called when the inputs weights of a split are being copied
GGML_API void ggml_backend_sched_set_copy_callback(ggml_backend_sched_t sched, ggml_backend_sched_copy_callback callback, void * user_data);
+ // Set a callback to be called when the compute buffers of a graph are being reallocated
+ GGML_API void ggml_backend_sched_set_realloc_callback(ggml_backend_sched_t sched, ggml_backend_sched_realloc_callback callback, void * user_data);
+
//
// Meta backend
//
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
index ff3f07998..ffbcba366 100644
--- a/ggml/src/ggml-backend.cpp
+++ b/ggml/src/ggml-backend.cpp
@@ -969,6 +969,9 @@ struct ggml_backend_sched {
ggml_backend_sched_copy_callback callback_copy;
void * callback_copy_user_data;
+ ggml_backend_sched_realloc_callback callback_realloc;
+ void * callback_realloc_user_data;
+
char * context_buffer;
size_t context_buffer_size;
@@ -976,11 +979,13 @@ struct ggml_backend_sched {
int debug;
- // used for debugging graph reallocations [GGML_SCHED_DEBUG_REALLOC]
+ // graph size of the current and the previous scheduling, used by the realloc callback and for debugging
+ // graph reallocations [GGML_SCHED_DEBUG_REALLOC]
// ref: https://github.com/ggml-org/llama.cpp/pull/17617
+ int graph_size;
+ int prev_graph_size;
+
int debug_realloc;
- int debug_graph_size;
- int debug_prev_graph_size;
};
#define hash_id(tensor) ggml_hash_find_or_insert(&sched->hash_set, tensor)
@@ -1633,9 +1638,9 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
}
int graph_size = std::max(graph->n_nodes, graph->n_leafs) + total_inputs * 2 * sched->n_copies + n_dep_nodes;
- // remember the actual graph_size for performing reallocation checks later [GGML_SCHED_DEBUG_REALLOC]
- sched->debug_prev_graph_size = sched->debug_graph_size;
- sched->debug_graph_size = graph_size;
+ // remember the actual graph_size for performing reallocation checks later
+ sched->prev_graph_size = sched->graph_size;
+ sched->graph_size = graph_size;
if (sched->graph.size < graph_size) {
sched->graph.size = graph_size;
@@ -1772,14 +1777,21 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) {
GGML_LOG_DEBUG("%s: failed to allocate graph, reserving (backend_ids_changed = %d)\n", __func__, backend_ids_changed);
#endif
+ // a reallocation with an unchanged graph size means that the previous reservation was not sufficient
+ const bool graph_size_changed = sched->prev_graph_size != sched->graph_size;
+
+ if (sched->callback_realloc != NULL) {
+ sched->callback_realloc(sched, &sched->graph, backend_ids_changed, graph_size_changed, sched->callback_realloc_user_data);
+ }
+
if (sched->debug_realloc > 0) {
// we are interested only in situations where the graph was reallocated even though its size remained the same [GGML_SCHED_DEBUG_REALLOC]
// example: https://github.com/ggml-org/llama.cpp/pull/17143
- const bool unexpected = !backend_ids_changed && sched->debug_prev_graph_size == sched->debug_graph_size;
+ const bool unexpected = !backend_ids_changed && !graph_size_changed;
if (unexpected || sched->debug_realloc > 1) {
GGML_ABORT("%s: unexpected graph reallocation (graph size = %d, nodes = %d, leafs = %d), debug_realloc = %d\n", __func__,
- sched->debug_graph_size, sched->graph.n_nodes, sched->graph.n_leafs, sched->debug_realloc);
+ sched->graph_size, sched->graph.n_nodes, sched->graph.n_leafs, sched->debug_realloc);
}
}
@@ -1973,8 +1985,8 @@ ggml_backend_sched_t ggml_backend_sched_new(
sched->prev_node_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_node_backend_ids[0]));
sched->prev_leaf_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_leaf_backend_ids[0]));
- sched->debug_graph_size = 0;
- sched->debug_prev_graph_size = 0;
+ sched->graph_size = 0;
+ sched->prev_graph_size = 0;
sched->context_buffer_size = ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + ggml_graph_overhead_custom(graph_size, false);
sched->context_buffer = (char *) malloc(sched->context_buffer_size);
@@ -2143,6 +2155,12 @@ void ggml_backend_sched_set_copy_callback(ggml_backend_sched_t sched, ggml_backe
sched->callback_copy_user_data = user_data;
}
+void ggml_backend_sched_set_realloc_callback(ggml_backend_sched_t sched, ggml_backend_sched_realloc_callback callback, void * user_data) {
+ GGML_ASSERT(sched);
+ sched->callback_realloc = callback;
+ sched->callback_realloc_user_data = user_data;
+}
+
int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) {
GGML_ASSERT(sched);
return sched->n_splits;
diff --git a/include/llama.h b/include/llama.h
index 0d476a299..6f76c819f 100644
--- a/include/llama.h
+++ b/include/llama.h
@@ -1686,6 +1686,8 @@ extern "C" {
int32_t n_p_eval; // number of prompt tokens
int32_t n_eval; // number of generated tokens
int32_t n_reused; // number of times a ggml compute graph had been reused
+ int32_t n_reallocs_same; // number of graph reallocations during inference with an unchanged graph size
+ int32_t n_reallocs_diff; // number of graph reallocations during inference with a changed graph size
};
struct llama_perf_sampler_data {
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 58df29876..23170d8a3 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -704,6 +704,14 @@ static int llama_graph_n_input_tensors(ggml_cgraph * gf) {
return (int) users.size();
}
+void llama_context::sched_new(size_t max_nodes, bool parallel) {
+ sched.reset(ggml_backend_sched_new(
+ backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, parallel, cparams.op_offload));
+
+ ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this);
+ ggml_backend_sched_set_realloc_callback(sched.get(), sched_graph_realloc, this);
+}
+
void llama_context::sched_reserve() {
if (!sched_need_reserve) {
return;
@@ -730,8 +738,7 @@ void llama_context::sched_reserve() {
gf_res_reserve.reset(new llm_graph_result(max_nodes));
gf_res_prev_active = nullptr;
- sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, cparams.pipeline_parallel, cparams.op_offload));
- ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this);
+ sched_new(max_nodes, cparams.pipeline_parallel);
llama_memory_context_ptr mctx;
if (memory) {
@@ -770,8 +777,7 @@ void llama_context::sched_reserve() {
if (cparams.pipeline_parallel) {
LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__);
cparams.pipeline_parallel = false;
- sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, cparams.op_offload));
- ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this);
+ sched_new(max_nodes, false);
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get());
}
if (!gf) {
@@ -2697,6 +2703,25 @@ llm_graph_params llama_context::graph_params(
};
}
+void llama_context::sched_graph_realloc(ggml_backend_sched_t sched, ggml_cgraph * gf, bool backend_ids_changed, bool graph_size_changed, void * user_data) {
+ // reallocating after the backend assignments changed is expected, trace only the unexpected ones
+ if (backend_ids_changed) {
+ return;
+ }
+
+ auto * lctx = static_cast<llama_context *>(user_data);
+
+ if (graph_size_changed) {
+ lctx->n_reallocs_diff++;
+ } else {
+ lctx->n_reallocs_same++;
+ }
+
+ LLAMA_LOG_INFO("%s: graph reallocation: nodes = %d, splits = %d, n_same = %d, n_diff = %d\n",
+ __func__, ggml_graph_n_nodes(gf), ggml_backend_sched_get_n_splits(sched),
+ lctx->n_reallocs_same, lctx->n_reallocs_diff);
+}
+
ggml_status llama_context::graph_compute(
ggml_cgraph * gf,
bool batched) {
@@ -3651,6 +3676,9 @@ llama_perf_context_data llama_context::perf_get_data() const {
data.n_eval = std::max(1, n_eval);
data.n_reused = std::max(0, n_reused);
+ data.n_reallocs_same = n_reallocs_same;
+ data.n_reallocs_diff = n_reallocs_diff;
+
return data;
}
@@ -3659,6 +3687,8 @@ void llama_context::perf_reset() {
t_eval_us = n_eval = 0;
t_p_eval_us = n_p_eval = 0;
n_reused = 0;
+
+ // note: the reallocation counters are not reset - they track the lifetime totals for this context
}
llama_memory_breakdown llama_context::memory_breakdown() const {
@@ -4624,6 +4654,7 @@ void llama_perf_context_print(const llama_context * ctx) {
__func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data.n_eval, 1e3 / data.t_eval_ms * data.n_eval);
LLAMA_LOG_INFO("%s: total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval));
LLAMA_LOG_INFO("%s: graphs reused = %10d\n", __func__, data.n_reused);
+ LLAMA_LOG_INFO("%s: graph reallocs = %10d / %5d (same size) / (changed size)\n", __func__, data.n_reallocs_same, data.n_reallocs_diff);
}
void llama_perf_context_reset(llama_context * ctx) {
diff --git a/src/llama-context.h b/src/llama-context.h
index cc749e574..277aff71d 100644
--- a/src/llama-context.h
+++ b/src/llama-context.h
@@ -273,9 +273,15 @@ private:
llm_graph_cb graph_get_cb() const;
+ // create a new backend scheduler for a graph of the specified size and register the callbacks below
+ void sched_new(size_t max_nodes, bool parallel);
+
// ggml_backend_sched copy callback, copies only the experts used by MUL_MAT_ID and updates the MoE cache
static bool sched_copy_experts(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph, void * user_data);
+ // ggml_backend_sched realloc callback, counts and traces the graph reallocations of this context
+ static void sched_graph_realloc(ggml_backend_sched_t sched, ggml_cgraph * graph, bool backend_ids_changed, bool graph_size_changed, void * user_data);
+
// disable auto fused ops (Flash Attention, Gated Delta Net) whose op lands on a device
// that differs from the layer it belongs to (usually due to missing backend support)
void resolve_fused_ops(const llama_memory_context_i * mctx, uint32_t n_seqs);
@@ -377,6 +383,11 @@ private:
copy_experts_info copy_experts;
+ // graph reallocations during inference where the backend assignments did not change, split by whether the
+ // graph size is the same as the previously scheduled one (unexpected) or different
+ int32_t n_reallocs_same = 0;
+ int32_t n_reallocs_diff = 0;
+
ggml_backend_t backend_cpu = nullptr;
std::vector<ggml_backend_ptr> backends;
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 44aaaddea..2a25b75d8 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -693,6 +693,10 @@ struct server_slot {
" graphs reused = %10d\n",
llama_perf_context(ctx_tgt).n_reused);
+ SLT_INF(*this,
+ " graph reallocs = %10d / %5d (same size) / (changed size)\n",
+ llama_perf_context(ctx_tgt).n_reallocs_same, llama_perf_context(ctx_tgt).n_reallocs_diff);
+
const int32_t n_draft_total = stats.n_draft_tokens;
const int32_t n_draft_accepted = stats.n_draft_accepted;
const int32_t n_draft_verif_steps = stats.n_draft_verif_steps;