Commit 0a7675746 for llama.cpp

commit 0a76757469ea94ad5f88d14e18cc7370b9931f99
Author: mahounairo <24420747+mahounairo@users.noreply.github.com>
Date:   Thu Nov 19 00:08:12 2026 +0100

    llama : fix sticky NaN in recurrent state (#31913)



    Co-authored-by: Alexey Kopytko <alexey@kopytko.com>

diff --git a/src/llama-ext.h b/src/llama-ext.h
index 2b2256fb6..b8b7fba0b 100644
--- a/src/llama-ext.h
+++ b/src/llama-ext.h
@@ -116,6 +116,8 @@ LLAMA_API ggml_backend_dev_t llama_model_get_device(const struct llama_model * m

 LLAMA_API llama_memory_breakdown llama_get_memory_breakdown(const struct llama_context * ctx);

+LLAMA_API std::vector<ggml_tensor *> llama_memory_get_tensors(const llama_memory_t mem);
+
 // the backend buffers that a state write of this memory would touch - empty caches report none
 LLAMA_API std::vector<ggml_backend_buffer_t> llama_memory_get_buffers(const llama_memory_t mem);

diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index 7b01af1b9..58c5b6c9c 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -4103,7 +4103,7 @@ ggml_tensor * llm_graph_context::build_rs(
     // Clear a single state which will then be copied to the other cleared states.
     // Note that this is a no-op when the view is zero-sized.
     ggml_tensor * state_zero = ggml_view_1d(ctx0, states, state_size*(rs_zero >= 0), rs_zero*states->nb[1]*(rs_zero >= 0));
-    ggml_build_forward_expand(gf, ggml_scale_inplace(ctx0, state_zero, 0));
+    ggml_build_forward_expand(gf, ggml_fill_inplace(ctx0, state_zero, 0.0f));

     // copy states
     // NOTE: assuming the copy destinations are ALL contained between rs_head and rs_head + n_rs
diff --git a/src/llama-memory.cpp b/src/llama-memory.cpp
index 042aab608..80099e76a 100644
--- a/src/llama-memory.cpp
+++ b/src/llama-memory.cpp
@@ -112,9 +112,9 @@ std::map<ggml_backend_buffer_type_t, size_t> llama_memory_bufs::memory_breakdown
     return ret;
 }

-// collects the buffers of the tensors that a state write visits, without keeping the data
-struct llama_memory_buffers_writer : public llama_io_write_i {
-    std::set<ggml_backend_buffer_t> buffers;
+// collects the tensors that a state write visits, without keeping the data
+struct llama_memory_tensors_writer : public llama_io_write_i {
+    std::set<ggml_tensor *> tensors;
     size_t size = 0;

     void write(const void *, size_t n) override {
@@ -122,7 +122,7 @@ struct llama_memory_buffers_writer : public llama_io_write_i {
     }

     void write_tensor(ggml_tensor * tensor, size_t, size_t n) override {
-        buffers.insert(tensor->buffer);
+        tensors.insert(tensor);
         size += n;
     }

@@ -132,13 +132,22 @@ struct llama_memory_buffers_writer : public llama_io_write_i {
 };

 std::vector<ggml_backend_buffer_t> llama_memory_get_buffers(const llama_memory_t mem) {
+    std::set<ggml_backend_buffer_t> buffers;
+    for (ggml_tensor * tensor : llama_memory_get_tensors(mem)) {
+        buffers.insert(tensor->buffer);
+    }
+
+    return std::vector<ggml_backend_buffer_t>(buffers.begin(), buffers.end());
+}
+
+std::vector<ggml_tensor *> llama_memory_get_tensors(const llama_memory_t mem) {
     if (mem == nullptr) {
         return {};
     }

-    llama_memory_buffers_writer writer;
+    llama_memory_tensors_writer writer;

     mem->state_write(writer);

-    return std::vector<ggml_backend_buffer_t>(writer.buffers.begin(), writer.buffers.end());
+    return std::vector<ggml_tensor *>(writer.tensors.begin(), writer.tensors.end());
 }
diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp
index 0d5d9a469..69ebe5a60 100644
--- a/src/models/deepseek4.cpp
+++ b/src/models/deepseek4.cpp
@@ -200,7 +200,7 @@ static ggml_tensor * dsv4_view_2d(

 static ggml_tensor * dsv4_append_zero_row(ggml_context * ctx, ggml_tensor * t, bool neg_inf) {
     ggml_tensor * row = ggml_view_1d(ctx, t, t->ne[0], 0);
-    row = neg_inf ? ggml_scale_bias(ctx, row, 0.0f, -INFINITY) : ggml_scale(ctx, row, 0.0f);
+    row = ggml_fill(ctx, row, neg_inf ? -INFINITY : 0.0f);
     row = ggml_reshape_2d(ctx, row, t->ne[0], 1);

     return ggml_concat(ctx, t, row, 1);
diff --git a/tests/llama-context/test-runner.cpp b/tests/llama-context/test-runner.cpp
index 1ca3f4c03..e2d01af4a 100644
--- a/tests/llama-context/test-runner.cpp
+++ b/tests/llama-context/test-runner.cpp
@@ -72,8 +72,9 @@ std::vector<case_run> run_cases(model_run & mr, const std::vector<const ctx_test
             } else {
                 // the recurrent cases are also run against a cache pre-filled with a known
                 // pattern, so that a restore reading cells it never wrote cannot pass
+                // with 0xff repeated every float of the state becomes NaN to test that the next sequence is not affected

-                for (const uint8_t fill : { 0, 0x3e }) {
+                for (const uint8_t fill : { 0, 0x3e, 0xff }) {
                     TLOG_INF("(fill 0x%02x)\n", fill);

                     mr.fill = fill;
diff --git a/tests/llama-context/test-util.cpp b/tests/llama-context/test-util.cpp
index b67e2a03b..e95ee7feb 100644
--- a/tests/llama-context/test-util.cpp
+++ b/tests/llama-context/test-util.cpp
@@ -2,7 +2,7 @@

 #include "test-log.h"

-#include "../../src/llama-ext.h" // llama_memory_get_buffers, llama_arch_from_string
+#include "../../src/llama-ext.h" // llama_memory_get_buffers, llama_memory_get_tensors, llama_arch_from_string

 #include "common.h"

@@ -10,6 +10,7 @@
 #include <cstdio>
 #include <cstring>
 #include <limits>
+#include <regex>
 #include <stdexcept>

 const double NMSE_THRESHOLD = 1e-5;
@@ -297,6 +298,13 @@ bool corrupt_state(std::vector<uint8_t> & data) {
 // context
 //

+// the recurrent state tensors: the conv and ssm rows of llama_memory_recurrent and the DeepSeek V4 compressor state
+static bool is_state_tensor(const ggml_tensor * tensor) {
+    static const std::regex re_state("^cache_(r|s|ple_r)_l[0-9]+$|_state_");
+
+    return std::regex_search(tensor->name, re_state);
+}
+
 // fill the buffers of the cache before a restore - a restore that reads cells it never wrote
 // then produces garbage instead of silently passing
 llama_context_ptr init_ctx(llama_model * model, llama_context_params cparams, uint8_t fill) {
@@ -313,11 +321,29 @@ llama_context_ptr init_ctx(llama_model * model, llama_context_params cparams, ui
     llama_synchronize(ctx.get());

     const auto buffers = llama_memory_get_buffers(llama_get_memory(ctx.get()));
+    const auto tensors = llama_memory_get_tensors(llama_get_memory(ctx.get()));
     llama_memory_clear(llama_get_memory(ctx.get()), true);
     if (buffers.empty()) {
         TLOG_ERR("%s: no cache buffers found\n", __func__);
         return nullptr;
     }
+
+    // the KV cache only masks the free cells, so NaN there would corrupt the output
+    if (fill == 0xff) {
+        size_t n_state = 0;
+        for (ggml_tensor * tensor : tensors) {
+            if (is_state_tensor(tensor)) {
+                ggml_backend_tensor_memset(tensor, fill, 0, ggml_nbytes(tensor));
+                n_state++;
+            }
+        }
+        if (n_state == 0) {
+            TLOG_ERR("%s: no state tensors found\n", __func__);
+            return nullptr;
+        }
+        return ctx;
+    }
+
     for (auto * buffer : buffers) {
         ggml_backend_buffer_clear(buffer, fill);
     }