Commit c7468107e for llama.cpp

commit c7468107efba061fe83d32b17c91e2b5295677d4
Author: Bhargav Krish <bkrishnamurt@nvidia.com>
Date:   Thu Nov 19 06:44:47 2026 -0800

    cuda: Scatter kernel for set_n (#32489)

    * cuda: add scatter for batched tensor writes

    * update test case to mirror get_n test case

    * save restore timing logs

    * refactor code into batch-copy.cu file

diff --git a/ggml/src/ggml-cuda/batch-get.cu b/ggml/src/ggml-cuda/batch-copy.cu
similarity index 58%
rename from ggml/src/ggml-cuda/batch-get.cu
rename to ggml/src/ggml-cuda/batch-copy.cu
index 60c4747b9..920b2b15d 100644
--- a/ggml/src/ggml-cuda/batch-get.cu
+++ b/ggml/src/ggml-cuda/batch-copy.cu
@@ -1,17 +1,17 @@
-#include "batch-get.cuh"
+#include "batch-copy.cuh"
 #include "common.cuh"
 #include "ggml-cpp.h"

 #include <climits>

 #if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
-struct ggml_cuda_batch_get_descriptor {
+struct ggml_cuda_batch_copy_descriptor {
     const unsigned char * src;
     unsigned char * dst;
     size_t size;
 };

-static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_descriptor * copies, size_t count) {
+static __global__ void ggml_cuda_batch_copy_kernel(const ggml_cuda_batch_copy_descriptor * copies, size_t count) {
     // Blocks along y cooperate on each descriptor.
     const size_t index = blockIdx.x;
     if (index >= count) {
@@ -50,11 +50,11 @@ static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_desc
     }
 }

-struct ggml_cuda_batch_get_mapping {
+struct ggml_cuda_batch_copy_mapping {
     void * host = nullptr;
     unsigned char * device = nullptr;

-    ggml_cuda_batch_get_mapping(void * host, size_t size) {
+    ggml_cuda_batch_copy_mapping(void * host, size_t size) {
         const cudaError_t err = cudaHostRegister(host, size, cudaHostRegisterMapped | cudaHostRegisterPortable);
         if (err != cudaSuccess) {
             (void) cudaGetLastError();
@@ -70,10 +70,10 @@ struct ggml_cuda_batch_get_mapping {
         }
     }

-    ggml_cuda_batch_get_mapping(const ggml_cuda_batch_get_mapping &) = delete;
-    ggml_cuda_batch_get_mapping & operator=(const ggml_cuda_batch_get_mapping &) = delete;
+    ggml_cuda_batch_copy_mapping(const ggml_cuda_batch_copy_mapping &) = delete;
+    ggml_cuda_batch_copy_mapping & operator=(const ggml_cuda_batch_copy_mapping &) = delete;

-    ~ggml_cuda_batch_get_mapping() {
+    ~ggml_cuda_batch_copy_mapping() {
         if (host) {
             // Release only the registration acquired by this object.
             CUDA_CHECK(cudaHostUnregister(host));
@@ -82,11 +82,12 @@ struct ggml_cuda_batch_get_mapping {
 };
 #endif

-bool ggml_cuda_get_tensor_n(int device,
-        const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
+static bool ggml_cuda_copy_tensor_n(int device, cudaMemcpyKind direction,
+        const ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
         const size_t * sizes, size_t n_tensors) {
 #if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
     GGML_UNUSED(device);
+    GGML_UNUSED(direction);
     GGML_UNUSED(tensors);
     GGML_UNUSED(data);
     GGML_UNUSED(offsets);
@@ -94,14 +95,17 @@ bool ggml_cuda_get_tensor_n(int device,
     GGML_UNUSED(n_tensors);
     return false;
 #else
+    const bool to_host = direction == cudaMemcpyDeviceToHost;
     // Provisional dispatch values based on experiments.
     constexpr size_t min_batch_reads = 128;
-    constexpr size_t max_gather_read_bytes = 128 * 1024;
-    // Collect non-empty CPU destination ranges.
+    constexpr size_t max_kernel_read_bytes = 128 * 1024;
+    // Collect destination ranges; repeated sources are allowed.
     std::vector<std::pair<uintptr_t, uintptr_t>> ranges;
     ranges.reserve(n_tensors);
     size_t payload_bytes = 0;
-    // Number of non-empty reads whose size is at most max_gather_read_bytes.
+    uintptr_t begin = UINTPTR_MAX;
+    uintptr_t end = 0;
+    // Number of non-empty reads whose size is at most max_kernel_read_bytes.
     size_t n_reads_small = 0;
     for (size_t i = 0; i < n_tensors; ++i) {
         if (sizes[i] == 0) {
@@ -111,15 +115,19 @@ bool ggml_cuda_get_tensor_n(int device,
         GGML_ASSERT(address <= UINTPTR_MAX - sizes[i]);
         GGML_ASSERT(payload_bytes <= SIZE_MAX - sizes[i]);
         payload_bytes += sizes[i];
-        n_reads_small += sizes[i] <= max_gather_read_bytes;
-        ranges.emplace_back(address, address + sizes[i]);
+        n_reads_small += sizes[i] <= max_kernel_read_bytes;
+        begin = std::min(begin, address);
+        end = std::max(end, address + sizes[i]);
+        const uintptr_t destination = to_host ? address : (uintptr_t) tensors[i]->data + offsets[i];
+        GGML_ASSERT(destination <= UINTPTR_MAX - sizes[i]);
+        ranges.emplace_back(destination, destination + sizes[i]);
     }
     if (ranges.empty()) {
         return true;
     }
     // Total number of non-empty reads.
     const size_t n_reads = ranges.size();
-    if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_get_descriptor) || n_reads > INT_MAX) {
+    if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_copy_descriptor) || n_reads > INT_MAX) {
         return false;
     }
     if (n_reads < 2 || (n_reads < min_batch_reads && n_reads_small == n_reads)) {
@@ -127,72 +135,85 @@ bool ggml_cuda_get_tensor_n(int device,
     }
     std::sort(ranges.begin(), ranges.end());
     for (size_t i = 1; i < ranges.size(); ++i) {
-        // Preserve the ordered copy path when CPU destinations overlap.
+        // Preserve the ordered copy path when destinations overlap.
         if (ranges[i].first < ranges[i - 1].second) {
             return false;
         }
     }
-    // Find the CPU span to register, including gaps between destinations.
-    const uintptr_t begin = ranges.front().first;
-    const uintptr_t end = ranges.back().second;
-    if (end - begin - payload_bytes > payload_bytes) {
+    // Limit gaps in the CPU span without underflow when sources overlap.
+    if (end - begin > payload_bytes && end - begin - payload_bytes > payload_bytes) {
         return false;
     }
-    // Register and map the existing CPU destination so the kernel can write to it directly.
-    ggml_cuda_batch_get_mapping mapping((void *) begin, end - begin);
+    // Register and map the CPU span for direct kernel access.
+    ggml_cuda_batch_copy_mapping mapping((void *) begin, end - begin);
     if (!mapping.host) {
         return false;
     }
-    // Reads selected for gather; zero means all reads use DMA.
-    size_t n_reads_gather = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0;
+    // Reads selected for the kernel; zero means all reads use DMA.
+    size_t n_reads_kernel = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0;
     // Keep descriptor storage alive until the final stream synchronization.
     constexpr size_t max_batch_descriptors = 128 * 1024;
-    const size_t bytes = std::min(n_reads_gather, max_batch_descriptors) * sizeof(ggml_cuda_batch_get_descriptor);
-    ggml_backend_buffer_ptr descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr);
+    const size_t bytes = std::min(n_reads_kernel, max_batch_descriptors) * sizeof(ggml_cuda_batch_copy_descriptor);
+    ggml_backend_buffer_ptr descriptors(n_reads_kernel ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr);
     if (!descriptors) {
-        n_reads_gather = 0;
+        n_reads_kernel = 0;
     }
     const auto host_type = ggml_backend_cuda_host_buffer_type();
-    ggml_backend_buffer_ptr host_descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr);
+    ggml_backend_buffer_ptr host_descriptors(n_reads_kernel ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr);
     if (!host_descriptors || ggml_backend_buffer_get_type(host_descriptors.get()) != host_type) {
-        n_reads_gather = 0;
+        n_reads_kernel = 0;
     }
-    if (n_reads_gather) {
-        auto * copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(host_descriptors.get()));
-        auto * device_copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(descriptors.get()));
+    if (n_reads_kernel) {
+        auto * copies = static_cast<ggml_cuda_batch_copy_descriptor *>(ggml_backend_buffer_get_base(host_descriptors.get()));
+        auto * device_copies = static_cast<ggml_cuda_batch_copy_descriptor *>(ggml_backend_buffer_get_base(descriptors.get()));
         const int threads = 256;
         const size_t nsm = ggml_cuda_info().devices[device].nsm;
         size_t i = 0;
-        for (size_t first = 0; first < n_reads_gather; first += max_batch_descriptors) {
-            const size_t n_batch = std::min(n_reads_gather - first, max_batch_descriptors);
+        for (size_t first = 0; first < n_reads_kernel; first += max_batch_descriptors) {
+            const size_t n_batch = std::min(n_reads_kernel - first, max_batch_descriptors);
             size_t count = 0;
             for (; i < n_tensors && count < n_batch; ++i) {
-                if (sizes[i] == 0 || sizes[i] > max_gather_read_bytes) {
+                if (sizes[i] == 0 || sizes[i] > max_kernel_read_bytes) {
                     continue;
                 }
-                copies[count++] = {(const unsigned char *) tensors[i]->data + offsets[i],
-                        mapping.device + ((uintptr_t) data[i] - begin), sizes[i]};
+                auto * host = mapping.device + ((uintptr_t) data[i] - begin);
+                auto * tensor = (unsigned char *) tensors[i]->data + offsets[i];
+                copies[count++] = {to_host ? tensor : host, to_host ? host : tensor, sizes[i]};
             }
             const size_t blocks_per_copy = std::min<size_t>(UINT16_MAX, std::max<size_t>(1, (nsm + n_batch - 1) / n_batch));
             const dim3 blocks((unsigned int) n_batch, (unsigned int) blocks_per_copy);
             // Stream ordering keeps each kernel ahead of the next descriptor overwrite.
             CUDA_CHECK(cudaMemcpyAsync(device_copies, copies, n_batch * sizeof(copies[0]), cudaMemcpyHostToDevice, cudaStreamPerThread));
-            ggml_cuda_batch_get_kernel<<<blocks, threads, 0, cudaStreamPerThread>>>(device_copies, n_batch);
+            ggml_cuda_batch_copy_kernel<<<blocks, threads, 0, cudaStreamPerThread>>>(device_copies, n_batch);
             CUDA_CHECK(cudaGetLastError());
-            if (first + n_batch < n_reads_gather) {
+            if (first + n_batch < n_reads_kernel) {
                 // Wait before refilling the pinned host descriptor buffer.
                 CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
             }
         }
     }
-    // Keep the destination pinned for DMA too; overlapping destinations already declined above.
+    // Keep the CPU span pinned for DMA too; overlapping destinations already declined above.
     for (size_t i = 0; i < n_tensors; ++i) {
-        if (sizes[i] && (!n_reads_gather || sizes[i] > max_gather_read_bytes)) {
-            CUDA_CHECK(cudaMemcpyAsync(data[i], (const char *) tensors[i]->data + offsets[i], sizes[i], cudaMemcpyDeviceToHost, cudaStreamPerThread));
+        if (sizes[i] && (!n_reads_kernel || sizes[i] > max_kernel_read_bytes)) {
+            void * host = const_cast<void *>(data[i]);
+            void * tensor = (char *) tensors[i]->data + offsets[i];
+            CUDA_CHECK(cudaMemcpyAsync(to_host ? host : tensor, to_host ? tensor : host, sizes[i], direction, cudaStreamPerThread));
         }
     }
-    // Wait for GPU writes before releasing the CPU mapping and descriptor buffers.
+    // Wait for transfers before releasing the CPU mapping and descriptor buffers.
     CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
     return true;
 #endif
 }
+
+bool ggml_cuda_get_tensor_n(int device,
+        const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
+        const size_t * sizes, size_t n_tensors) {
+    return ggml_cuda_copy_tensor_n(device, cudaMemcpyDeviceToHost, tensors, data, offsets, sizes, n_tensors);
+}
+
+bool ggml_cuda_set_tensor_n(int device,
+        ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
+        const size_t * sizes, size_t n_tensors) {
+    return ggml_cuda_copy_tensor_n(device, cudaMemcpyHostToDevice, tensors, data, offsets, sizes, n_tensors);
+}
diff --git a/ggml/src/ggml-cuda/batch-get.cuh b/ggml/src/ggml-cuda/batch-copy.cuh
similarity index 55%
rename from ggml/src/ggml-cuda/batch-get.cuh
rename to ggml/src/ggml-cuda/batch-copy.cuh
index eb9bf0cc8..bb7fca4c0 100644
--- a/ggml/src/ggml-cuda/batch-get.cuh
+++ b/ggml/src/ggml-cuda/batch-copy.cuh
@@ -5,3 +5,7 @@
 bool ggml_cuda_get_tensor_n(int device,
         const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
         const size_t * sizes, size_t n_tensors);
+
+bool ggml_cuda_set_tensor_n(int device,
+        ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
+        const size_t * sizes, size_t n_tensors);
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index c6622d1b3..2ce3d0b80 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -10,7 +10,7 @@
 #include "ggml-cuda/arange.cuh"
 #include "ggml-cuda/argmax.cuh"
 #include "ggml-cuda/argsort.cuh"
-#include "ggml-cuda/batch-get.cuh"
+#include "ggml-cuda/batch-copy.cuh"
 #include "ggml-cuda/binbcast.cuh"
 #include "ggml-cuda/clamp.cuh"
 #include "ggml-cuda/col2im-1d.cuh"
@@ -1117,6 +1117,11 @@ static void ggml_backend_cuda_buffer_set_tensor_n(ggml_backend_buffer_t buffer,

     ggml_cuda_set_device(ctx->device);

+    if (!ggml_cuda_pinned_memory_disabled() &&
+            ggml_cuda_set_tensor_n(ctx->device, tensors, data, offsets, sizes, n_tensors)) {
+        return;
+    }
+
     for (size_t i = 0; i < n_tensors; i++) {
         if (sizes[i] == 0) {
             continue;
diff --git a/tests/test-backend-tensor.cpp b/tests/test-backend-tensor.cpp
index 9685bc277..ed617939d 100644
--- a/tests/test-backend-tensor.cpp
+++ b/tests/test-backend-tensor.cpp
@@ -170,6 +170,40 @@ static bool test_tensor_n(ggml_backend_buffer_type_t buft) {
         }
     }

+    {
+        std::vector<ggml_tensor *> batch;
+        ggml_backend_buffer_ptr scatter_buf = alloc_tensors(ctx.get(), buft, { 64 }, batch);
+        if (!scatter_buf) {
+            return false;
+        }
+        batch.resize(128, batch[0]);
+        uint8_t input[32], expected[256], out[256];
+        for (size_t i = 0; i < sizeof(input); i++) {
+            input[i] = (uint8_t) (i + 1);
+        }
+        std::vector<size_t> offsets(batch.size()), sizes(batch.size(), 1), data_offsets(batch.size());
+        for (bool overlap : { false, true }) {
+            memset(expected, 0, sizeof(expected));
+            ggml_backend_tensor_set(batch[0], expected, 0, sizeof(expected));
+            for (size_t i = 0; i < batch.size(); i++) {
+                offsets[i] = 2*((i*5) % batch.size()) + 1;
+                data_offsets[i] = i % sizeof(input);
+            }
+            // Repeated sources are valid; overlapping destinations must retain copy order.
+            if (overlap) {
+                offsets.back() = offsets.front();
+            }
+            for (size_t i = 0; i < batch.size(); i++) {
+                expected[offsets[i]] = input[data_offsets[i]];
+            }
+            ggml_backend_tensor_set_n(batch.data(), input, offsets.data(), sizes.data(), data_offsets.data(), batch.size());
+            ggml_backend_tensor_get(batch[0], out, 0, sizeof(out));
+            if (memcmp(out, expected, sizeof(out)) != 0) {
+                return false;
+            }
+        }
+    }
+
     return true;
 }

diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 36082df89..f821ac6cb 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -353,6 +353,8 @@ struct server_slot {
             return false;
         }

+        const int64_t t_start = ggml_time_us();
+
         const size_t cur_size_tgt =           llama_state_seq_get_size_ext(ctx_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE);
         const size_t cur_size_dft = ctx_dft ? llama_state_seq_get_size_ext(ctx_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE) : 0;

@@ -374,6 +376,8 @@ struct server_slot {
             return false;
         }

+        SLT_TRC(*this, "prompt cache save took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0);
+
         return true;
     }

diff --git a/tools/server/server-slot-sched.cpp b/tools/server/server-slot-sched.cpp
index 5f8aa5c33..9f9779348 100644
--- a/tools/server/server-slot-sched.cpp
+++ b/tools/server/server-slot-sched.cpp
@@ -208,6 +208,8 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
     if (it_best != states.end()) {
         SRV_TRC(" - found better prompt with f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);

+        const int64_t t_start = ggml_time_us();
+
         {
             auto & data = it_best->data.main;

@@ -252,6 +254,8 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
         prompt = std::move(it_best->prompt);

         states.erase(it_best);
+
+        SRV_TRC("slot %d: prompt cache restore took %.2f ms\n", id_slot, (ggml_time_us() - t_start) / 1000.0);
     }

     return true;