Commit 806b3c2be for llama.cpp
commit 806b3c2bed44490233f1e8ffa403519370965fe7
Author: ArshM17-NV <amaknojia@nvidia.com>
Date: Mon Nov 9 12:40:17 2026 +0000
ggml-backend: don't upload inputs that are not read (#31419)
* stop uploading inputs nothing reads
* Address nitpicks
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
index 92a53bf38..961990f85 100644
--- a/ggml/src/ggml-backend.cpp
+++ b/ggml/src/ggml-backend.cpp
@@ -1195,8 +1195,9 @@ struct ggml_backend_sched {
// hash map of the nodes in the graph
struct ggml_hash_set hash_set;
- int * hv_tensor_backend_ids; // [hash_set.size]
- struct ggml_tensor ** hv_tensor_copies; // [hash_set.size][n_backends][n_copies]
+ int * hv_tensor_backend_ids; // [hash_set.size]
+ struct ggml_tensor ** hv_tensor_copies; // [hash_set.size][n_backends][n_copies]
+ bool * hv_tensor_requires_copy; // [hash_set.size] decides if input needs to be copied to each split
int * node_backend_ids; // [graph_size]
int * leaf_backend_ids; // [graph_size]
@@ -1808,6 +1809,11 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
}
split->inputs[n_inputs] = src;
}
+ // inputs of nodes that are not computed (e.g. the unselected branches of ggml_build_forward_select)
+ // stay in the split so that the allocation does not change, but they do not need to be copied
+ if (node->flags & GGML_TENSOR_FLAG_COMPUTE) {
+ sched->hv_tensor_requires_copy[src_id] = true;
+ }
node->src[j] = tensor_id_copy(src_id, cur_backend_id, sched->cur_copy);
}
}
@@ -2220,16 +2226,18 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
// copy the input tensors to the split backend
// the weights in host memory are copied last, so that the copy callback can read the other inputs of the split
for (int input_id = 0; input_id < split->n_inputs; input_id++) {
- if (!ggml_backend_sched_is_host_weight(split->inputs[input_id])) {
- ggml_backend_sched_copy_input(sched, split, split->inputs[input_id]);
+ struct ggml_tensor * input = split->inputs[input_id];
+ if (!ggml_backend_sched_is_host_weight(input) && sched->hv_tensor_requires_copy[hash_id(input)]) {
+ ggml_backend_sched_copy_input(sched, split, input);
}
}
// after the copies above, which may read host memory reused by the copies of the next split
// before the weights, whose copy callback may queue large uploads
ggml_backend_sched_prefetch_inputs(sched, split_id);
for (int input_id = 0; input_id < split->n_inputs; input_id++) {
- if (ggml_backend_sched_is_host_weight(split->inputs[input_id])) {
- ggml_backend_sched_copy_input(sched, split, split->inputs[input_id]);
+ struct ggml_tensor * input = split->inputs[input_id];
+ if (ggml_backend_sched_is_host_weight(input) && sched->hv_tensor_requires_copy[hash_id(input)]) {
+ ggml_backend_sched_copy_input(sched, split, input);
}
}
@@ -2312,8 +2320,9 @@ ggml_backend_sched_t ggml_backend_sched_new(
// initialize hash table
// FIXME: needs to be size*2 to account for leafs (do it in graph_split instead)
sched->hash_set = ggml_hash_set_new(graph_size);
- sched->hv_tensor_backend_ids = (int *) malloc(sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));
- sched->hv_tensor_copies = (ggml_tensor **) malloc(sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));
+ sched->hv_tensor_backend_ids = (int *) malloc(sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));
+ sched->hv_tensor_copies = (ggml_tensor **) malloc(sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));
+ sched->hv_tensor_requires_copy = (bool *) malloc(sched->hash_set.size * sizeof(sched->hv_tensor_requires_copy[0]));
const size_t ggml_sched_max_splits = graph_size; // at most there is one split for each node in the graph
const size_t nodes_size = graph_size + ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2;
@@ -2375,6 +2384,7 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) {
free(sched->graph_inputs);
free(sched->hv_tensor_backend_ids);
free(sched->hv_tensor_copies);
+ free(sched->hv_tensor_requires_copy);
free(sched->node_backend_ids);
free(sched->leaf_backend_ids);
free(sched->prev_node_backend_ids);
@@ -2390,8 +2400,9 @@ void ggml_backend_sched_reset(ggml_backend_sched_t sched) {
// reset state for the next run
if (!sched->is_reset) {
ggml_hash_set_reset(&sched->hash_set);
- memset(sched->hv_tensor_backend_ids, -1, sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));
- memset(sched->hv_tensor_copies, 0, sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));
+ memset(sched->hv_tensor_backend_ids, -1, sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));
+ memset(sched->hv_tensor_copies, 0, sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));
+ memset(sched->hv_tensor_requires_copy, 0, sched->hash_set.size * sizeof(sched->hv_tensor_requires_copy[0]));
sched->is_reset = true;
}
sched->is_alloc = false;